1// SPDX-License-Identifier: MIT
2
3// Package dnsfwd is a small DNS forwarder for job sandboxes (stdlib only). It
4// listens on the gateway address of each sandbox (UDP and TCP, port 53 in
5// production) and relays queries unchanged to the host's resolvers; the network
6// policy lets guests reach nothing else on the host. It does not parse queries
7// beyond what framing needs, so it supports every record type and EDNS.
8//
9// Abuse limits, per listener (one listener per guest, so one guest cannot
10// starve another): at most MaxInflight concurrent upstream exchanges (UDP
11// queries beyond that are dropped, never queued), at most MaxTCPConns open TCP
12// connections, an absolute lifetime of TCPLifetime per TCP connection (a client
13// cannot hold a slot by trickling bytes), a deadline on every upstream
14// exchange, and a cap on queries per TCP connection.
15//
16// One name can be overridden (Forwarder.Override, the opt-in guest forge
17// access): queries for it are answered locally with the address of the
18// listener, see override.go. Every other name is forwarded.
19package dnsfwd
20
21import (
22 "context"
23 "encoding/binary"
24 "errors"
25 "fmt"
26 "io"
27 "net"
28 "net/netip"
29 "strconv"
30 "sync"
31 "sync/atomic"
32 "syscall"
33 "time"
34
35 "github.com/bnema/zerowrap"
36 "golang.org/x/sys/unix"
37)
38
39const (
40 // TCPLifetime is the absolute lifetime of a TCP connection, set at accept.
41 TCPLifetime = 30 * time.Second
42 // UpstreamTimeout is the deadline of one upstream TCP attempt.
43 UpstreamTimeout = 3 * time.Second
44 // UDPAttemptTimeout is the deadline of one upstream UDP attempt. It is short
45 // on purpose: glibc guests use "timeout:2 attempts:2", so the reply (or its
46 // TC=1 / TCP fallback) must reach them well inside the first 2 s window.
47 UDPAttemptTimeout = time.Second
48 // MaxExchange caps one whole exchange over all upstreams and transports, so
49 // a long upstream list cannot hold an in-flight slot for long.
50 MaxExchange = 6 * time.Second
51 // TCPIdleTimeout closes TCP client connections that stay silent.
52 TCPIdleTimeout = 10 * time.Second
53
54 // MaxInflight bounds concurrent upstream exchanges per listener.
55 MaxInflight = 32
56 // MaxTCPConns bounds open TCP connections per listener.
57 MaxTCPConns = 16
58 // TCPMaxQueries caps queries per TCP connection.
59 TCPMaxQueries = 64
60
61 maxUDP = 4096 // largest datagram relayed (EDNS0 sizes up to 4096)
62 hdrLen = 12
63
64 // plainUDPSize is the largest UDP reply for a client without EDNS (RFC 1035).
65 plainUDPSize = 512
66)
67
68// Forwarder relays DNS queries. The zero value is not usable; use New.
69type Forwarder struct {
70 upstreams func() []string
71 port int
72 // Lifetime overrides TCPLifetime; set before Listen (tests).
73 Lifetime time.Duration
74 // Upstream deadlines, set to the constants above by New; tests shorten
75 // them before Listen.
76 udpTimeout, tcpTimeout, maxExchange time.Duration
77 // dial opens upstream connections; tests replace it with in-memory fakes.
78 dial func(ctx context.Context, network, addr string) (net.Conn, error)
79 // FreeBind binds with IP_FREEBIND, so that the gateway address may be
80 // configured after the listener. Off by default: with Docker the address
81 // exists when the network is created and a missing one is an error to
82 // report at once, not a listener on an address nobody owns. Set before Listen.
83 FreeBind bool
84 // Override, when set, is a host name that every listener answers itself:
85 // A with the address the listener is bound to, and NOERROR with no data for
86 // every other type, whatever the host's resolvers say. Case-insensitive; a
87 // trailing dot is ignored. Set before Listen.
88 Override string
89
90 mu sync.Mutex
91 listeners map[netip.Addr]*listener
92}
93
94type listener struct {
95 addr netip.Addr
96 udp net.PacketConn
97 tcp net.Listener
98 cancel context.CancelFunc
99 done sync.WaitGroup
100 // inflight and conns are the per-listener limits (counting semaphores).
101 inflight chan struct{}
102 conns chan struct{}
103 // Dropped counts UDP queries dropped because the listener was saturated.
104 dropped atomic.Int64
105}
106
107// New returns a Forwarder. upstreams returns the "host:port" resolvers to try
108// in order (see Resolvers.Addrs); port is the port to listen on (53, or another
109// port in tests).
110func New(upstreams func() []string, port int) *Forwarder {
111 return &Forwarder{
112 upstreams: upstreams,
113 port: port,
114 Lifetime: TCPLifetime,
115
116 udpTimeout: UDPAttemptTimeout,
117 tcpTimeout: UpstreamTimeout,
118 maxExchange: MaxExchange,
119 dial: (&net.Dialer{}).DialContext,
120 listeners: map[netip.Addr]*listener{},
121 }
122}
123
124// Listen starts serving on addr (UDP and TCP). It returns once both sockets are
125// bound. Listening twice on the same address is an error.
126func (f *Forwarder) Listen(ctx context.Context, addr netip.Addr) error {
127 f.mu.Lock()
128 defer f.mu.Unlock()
129 if _, ok := f.listeners[addr]; ok {
130 return fmt.Errorf("dnsfwd: already listening on %s", addr)
131 }
132 hp := net.JoinHostPort(addr.String(), strconv.Itoa(f.port))
133 var lc net.ListenConfig
134 if f.FreeBind {
135 // The gateway address may not exist yet (podman's netavark configures the
136 // bridge address when the first container starts, after this listener).
137 lc.Control = func(_, _ string, c syscall.RawConn) error {
138 var serr error
139 if err := c.Control(func(fd uintptr) { serr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_FREEBIND, 1) }); err != nil {
140 return err
141 }
142 return serr
143 }
144 }
145 var (
146 pc net.PacketConn
147 tl net.Listener
148 err error
149 )
150 // With port 0 (tests) UDP and TCP must share the port the kernel picked for
151 // UDP, which a TCP socket may already hold: pick again a few times.
152 for attempt := 0; ; attempt++ {
153 bind := hp
154 if pc, err = lc.ListenPacket(ctx, "udp", bind); err != nil {
155 return fmt.Errorf("dnsfwd: listen udp %s: %w", bind, err)
156 }
157 if f.port == 0 {
158 bind = pc.LocalAddr().String()
159 }
160 if tl, err = lc.Listen(ctx, "tcp", bind); err == nil {
161 break
162 }
163 _ = pc.Close()
164 if f.port != 0 || attempt == 9 || !errors.Is(err, syscall.EADDRINUSE) {
165 return fmt.Errorf("dnsfwd: listen tcp %s: %w", bind, err)
166 }
167 }
168 lctx, cancel := context.WithCancel(context.WithoutCancel(ctx))
169 l := &listener{addr: addr, udp: pc, tcp: tl, cancel: cancel, inflight: make(chan struct{}, MaxInflight), conns: make(chan struct{}, MaxTCPConns)}
170 f.listeners[addr] = l
171 log := zerowrap.FromCtx(ctx)
172 l.done.Add(2)
173 go func() { defer l.done.Done(); f.serveUDP(lctx, l, log) }()
174 go func() { defer l.done.Done(); f.serveTCP(lctx, l, log) }()
175 return nil
176}
177
178// Addr returns the bound UDP address for addr, useful when port is 0.
179func (f *Forwarder) Addr(addr netip.Addr) (net.Addr, bool) {
180 f.mu.Lock()
181 defer f.mu.Unlock()
182 l, ok := f.listeners[addr]
183 if !ok {
184 return nil, false
185 }
186 return l.udp.LocalAddr(), true
187}
188
189// Close stops listening on addr and waits for its goroutines. Unknown
190// addresses are ignored.
191func (f *Forwarder) Close(addr netip.Addr) {
192 f.mu.Lock()
193 l, ok := f.listeners[addr]
194 delete(f.listeners, addr)
195 f.mu.Unlock()
196 if !ok {
197 return
198 }
199 l.cancel()
200 _ = l.udp.Close()
201 _ = l.tcp.Close()
202 l.done.Wait()
203}
204
205// CloseAll stops every listener.
206func (f *Forwarder) CloseAll() {
207 f.mu.Lock()
208 addrs := make([]netip.Addr, 0, len(f.listeners))
209 for a := range f.listeners {
210 addrs = append(addrs, a)
211 }
212 f.mu.Unlock()
213 for _, a := range addrs {
214 f.Close(a)
215 }
216}
217
218// Dropped returns how many UDP queries the listener on addr dropped because it
219// was saturated.
220func (f *Forwarder) Dropped(addr netip.Addr) int64 {
221 f.mu.Lock()
222 defer f.mu.Unlock()
223 if l, ok := f.listeners[addr]; ok {
224 return l.dropped.Load()
225 }
226 return 0
227}
228
229func (f *Forwarder) serveUDP(ctx context.Context, l *listener, log zerowrap.Logger) {
230 pc := l.udp
231 buf := make([]byte, maxUDP)
232 for {
233 n, from, err := pc.ReadFrom(buf)
234 if err != nil {
235 if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) {
236 log.Warn().Err(err).Msg("dns forwarder: udp read")
237 }
238 return
239 }
240 if n < hdrLen {
241 continue
242 }
243 select {
244 case l.inflight <- struct{}{}:
245 default:
246 // Saturated: drop. The client retries; blocking here would let one
247 // guest's flood stall the read loop.
248 l.dropped.Add(1)
249 continue
250 }
251 q := append([]byte(nil), buf[:n]...)
252 go func() {
253 defer func() { <-l.inflight }()
254 resp := f.exchange(ctx, l, q, false)
255 // A reply larger than the client accepts would be dropped or cut
256 // by its stack: send TC=1 so that it retries over TCP at once.
257 if len(resp) > clientUDPSize(q) {
258 resp = truncated(resp)
259 }
260 if len(resp) > 0 {
261 _, _ = pc.WriteTo(resp, from)
262 }
263 }()
264 }
265}
266
267func (f *Forwarder) serveTCP(ctx context.Context, l *listener, log zerowrap.Logger) {
268 var wg sync.WaitGroup
269 defer wg.Wait()
270 for {
271 c, err := l.tcp.Accept()
272 if err != nil {
273 if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) {
274 log.Warn().Err(err).Msg("dns forwarder: tcp accept")
275 }
276 return
277 }
278 select {
279 case l.conns <- struct{}{}:
280 default:
281 _ = c.Close() // too many open connections on this listener
282 continue
283 }
284 wg.Add(1)
285 go func() {
286 defer wg.Done()
287 defer func() { <-l.conns }()
288 defer func() { _ = c.Close() }()
289 stop := context.AfterFunc(ctx, func() { _ = c.Close() })
290 defer stop()
291 // Absolute deadline from accept: neither idle nor slow-loris
292 // connections keep their slot past TCPLifetime.
293 deadline := time.Now().Add(f.Lifetime)
294 _ = c.SetDeadline(deadline)
295 f.serveConn(ctx, l, c, deadline)
296 }()
297 }
298}
299
300func (f *Forwarder) serveConn(ctx context.Context, l *listener, c net.Conn, deadline time.Time) {
301 for range TCPMaxQueries {
302 // The idle timeout can only shorten the absolute deadline.
303 dl := time.Now().Add(TCPIdleTimeout)
304 if deadline.Before(dl) {
305 dl = deadline
306 }
307 _ = c.SetReadDeadline(dl)
308 var lb [2]byte
309 if _, err := io.ReadFull(c, lb[:]); err != nil {
310 return
311 }
312 n := int(binary.BigEndian.Uint16(lb[:]))
313 if n < hdrLen {
314 return
315 }
316 q := make([]byte, n)
317 if _, err := io.ReadFull(c, q); err != nil {
318 return
319 }
320 select {
321 case l.inflight <- struct{}{}:
322 default:
323 return // saturated: close, the client retries
324 }
325 resp := f.exchange(ctx, l, q, true)
326 <-l.inflight
327 if len(resp) == 0 || len(resp) > 0xffff {
328 return
329 }
330 out := make([]byte, 2+len(resp))
331 binary.BigEndian.PutUint16(out, uint16(len(resp)))
332 copy(out[2:], resp)
333 if _, err := c.Write(out); err != nil {
334 return
335 }
336 }
337}
338
339// exchange relays q to the upstreams in order and returns the first answer,
340// SERVFAIL when none answers, or nil when ctx is cancelled. A TCP client is
341// served over TCP. A UDP client is served over UDP for at most udpTimeout; if
342// that fails, times out or returns TC=1, the same upstream is retried over TCP
343// (at most tcpTimeout) before the next upstream is tried, because some
344// resolvers or network paths lose large plain-UDP answers. Each upstream gets
345// its own budget, and the whole exchange is capped at maxExchange. The
346// overridden name, if any, is answered without asking the upstreams.
347func (f *Forwarder) exchange(ctx context.Context, l *listener, q []byte, overTCP bool) []byte {
348 if resp := f.overrideAnswer(l.addr, q); resp != nil {
349 return resp
350 }
351 cctx, cancel := context.WithTimeout(ctx, f.maxExchange)
352 defer cancel()
353 valid := func(resp []byte, err error) bool {
354 return err == nil && len(resp) >= hdrLen && resp[0] == q[0] && resp[1] == q[1]
355 }
356 for _, up := range f.upstreams() {
357 if cctx.Err() != nil {
358 break
359 }
360 if overTCP {
361 if resp, err := f.exchangeTCP(cctx, up, q, f.tcpTimeout); valid(resp, err) {
362 return resp
363 }
364 continue
365 }
366 resp, err := f.exchangeUDP(cctx, up, q, f.udpTimeout)
367 var tc []byte
368 if valid(resp, err) {
369 if resp[2]&0x02 == 0 {
370 return resp
371 }
372 tc = resp
373 }
374 if full, err := f.exchangeTCP(cctx, up, q, f.tcpTimeout); valid(full, err) {
375 return full
376 }
377 if tc != nil {
378 return tc // the client retries over TCP itself
379 }
380 }
381 if ctx.Err() != nil {
382 return nil
383 }
384 return servfail(q)
385}
386
387// clientUDPSize returns the largest UDP reply the client of q accepts: 512
388// without EDNS, else its advertised OPT size (at least 512, at most maxUDP).
389func clientUDPSize(q []byte) int {
390 size := plainUDPSize
391 if len(q) < hdrLen {
392 return size
393 }
394 off := hdrLen
395 for i := 0; i < int(binary.BigEndian.Uint16(q[4:])); i++ { // questions
396 if off = skipName(q, off); off < 0 || off+4 > len(q) {
397 return size
398 }
399 off += 4
400 }
401 rrs := int(binary.BigEndian.Uint16(q[6:])) + int(binary.BigEndian.Uint16(q[8:])) + int(binary.BigEndian.Uint16(q[10:]))
402 for i := 0; i < rrs; i++ {
403 if off = skipName(q, off); off < 0 || off+10 > len(q) {
404 return size
405 }
406 if binary.BigEndian.Uint16(q[off:]) == 41 { // OPT: class is the UDP size
407 size = max(size, int(binary.BigEndian.Uint16(q[off+2:])))
408 break
409 }
410 off += 10 + int(binary.BigEndian.Uint16(q[off+8:]))
411 }
412 return min(size, maxUDP)
413}
414
415// skipName returns the offset after the name at off, or -1 if malformed.
416func skipName(m []byte, off int) int {
417 for off < len(m) {
418 switch c := int(m[off]); {
419 case c == 0:
420 return off + 1
421 case c&0xc0 == 0xc0:
422 if off+2 > len(m) {
423 return -1
424 }
425 return off + 2
426 case c&0xc0 != 0:
427 return -1
428 default:
429 off += 1 + c
430 }
431 }
432 return -1
433}
434
435// truncated returns the header and question of resp with TC=1 and no records.
436func truncated(resp []byte) []byte {
437 if len(resp) < hdrLen {
438 return nil
439 }
440 end := hdrLen
441 for i := 0; i < int(binary.BigEndian.Uint16(resp[4:])); i++ {
442 if end = skipName(resp, end); end < 0 || end+4 > len(resp) {
443 end = hdrLen
444 break
445 }
446 end += 4
447 }
448 r := append([]byte(nil), resp[:end]...)
449 r[2] |= 0x02 // TC
450 if end == hdrLen {
451 r[4], r[5] = 0, 0
452 }
453 for i := 6; i < hdrLen; i++ { // ANCOUNT, NSCOUNT, ARCOUNT
454 r[i] = 0
455 }
456 return r
457}
458
459func (f *Forwarder) exchangeUDP(ctx context.Context, up string, q []byte, timeout time.Duration) ([]byte, error) {
460 ctx, cancel := context.WithTimeout(ctx, timeout)
461 defer cancel()
462 c, err := f.dial(ctx, "udp", up)
463 if err != nil {
464 return nil, err
465 }
466 defer func() { _ = c.Close() }()
467 dl, _ := ctx.Deadline()
468 _ = c.SetDeadline(dl)
469 if _, err := c.Write(q); err != nil {
470 return nil, err
471 }
472 buf := make([]byte, maxUDP)
473 for {
474 n, err := c.Read(buf)
475 if err != nil {
476 return nil, err
477 }
478 // Ignore datagrams that do not answer this query (ID mismatch).
479 if n >= hdrLen && buf[0] == q[0] && buf[1] == q[1] {
480 return append([]byte(nil), buf[:n]...), nil
481 }
482 }
483}
484
485func (f *Forwarder) exchangeTCP(ctx context.Context, up string, q []byte, timeout time.Duration) ([]byte, error) {
486 ctx, cancel := context.WithTimeout(ctx, timeout)
487 defer cancel()
488 c, err := f.dial(ctx, "tcp", up)
489 if err != nil {
490 return nil, err
491 }
492 defer func() { _ = c.Close() }()
493 dl, _ := ctx.Deadline()
494 _ = c.SetDeadline(dl)
495 out := make([]byte, 2+len(q))
496 binary.BigEndian.PutUint16(out, uint16(len(q)))
497 copy(out[2:], q)
498 if _, err := c.Write(out); err != nil {
499 return nil, err
500 }
501 var l [2]byte
502 if _, err := io.ReadFull(c, l[:]); err != nil {
503 return nil, err
504 }
505 resp := make([]byte, binary.BigEndian.Uint16(l[:]))
506 if _, err := io.ReadFull(c, resp); err != nil {
507 return nil, err
508 }
509 return resp, nil
510}
511
512// servfail builds a SERVFAIL reply for q: same ID and opcode, QR and RA set,
513// RD copied, no records.
514func servfail(q []byte) []byte {
515 if len(q) < hdrLen {
516 return nil
517 }
518 r := make([]byte, hdrLen)
519 r[0], r[1] = q[0], q[1]
520 r[2] = 0x80 | (q[2] & 0x79) // QR, opcode, RD kept; AA/TC cleared
521 r[3] = 0x80 | 0x02 // RA, RCODE=SERVFAIL
522 return r
523}