1// SPDX-License-Identifier: MIT
2
3// Package dnsfwd is a small DNS forwarder for job sandboxes (stdlib only). It
4// listens on the gateway address of each sandbox (UDP and TCP, port 53 in
5// production) and relays queries unchanged to the host's resolvers; the network
6// policy lets guests reach nothing else on the host. It does not parse queries
7// beyond what framing needs, so it supports every record type and EDNS.
8//
9// Abuse limits, per listener (one listener per guest, so one guest cannot
10// starve another): at most MaxInflight concurrent upstream exchanges (UDP
11// queries beyond that are dropped, never queued), at most MaxTCPConns open TCP
12// connections, an absolute lifetime of TCPLifetime per TCP connection (a client
13// cannot hold a slot by trickling bytes), a deadline on every upstream
14// exchange, and a cap on queries per TCP connection.
15//
16// One name can be overridden (Forwarder.Override, the opt-in guest forge
17// access): queries for it are answered locally with the address of the
18// listener, see override.go. Every other name is forwarded.
19package dnsfwd
20
21import (
22 "context"
23 "encoding/binary"
24 "errors"
25 "fmt"
26 "io"
27 "net"
28 "net/netip"
29 "strconv"
30 "sync"
31 "sync/atomic"
32 "syscall"
33 "time"
34
35 "github.com/bnema/zerowrap"
36 "golang.org/x/sys/unix"
37)
38
39const (
40 // TCPLifetime is the absolute lifetime of a TCP connection, set at accept.
41 TCPLifetime = 30 * time.Second
42 // UpstreamTimeout is the deadline of one upstream attempt.
43 UpstreamTimeout = 3 * time.Second
44 // TCPIdleTimeout closes TCP client connections that stay silent.
45 TCPIdleTimeout = 10 * time.Second
46
47 // MaxInflight bounds concurrent upstream exchanges per listener.
48 MaxInflight = 32
49 // MaxTCPConns bounds open TCP connections per listener.
50 MaxTCPConns = 16
51 // TCPMaxQueries caps queries per TCP connection.
52 TCPMaxQueries = 64
53
54 maxUDP = 4096 // largest datagram relayed (EDNS0 sizes up to 4096)
55 hdrLen = 12
56)
57
58// Forwarder relays DNS queries. The zero value is not usable; use New.
59type Forwarder struct {
60 upstreams func() []string
61 port int
62 // Lifetime overrides TCPLifetime; set before Listen (tests).
63 Lifetime time.Duration
64 // FreeBind binds with IP_FREEBIND, so that the gateway address may be
65 // configured after the listener. Off by default: with Docker the address
66 // exists when the network is created and a missing one is an error to
67 // report at once, not a listener on an address nobody owns. Set before Listen.
68 FreeBind bool
69 // Override, when set, is a host name that every listener answers itself:
70 // A with the address the listener is bound to, and NOERROR with no data for
71 // every other type, whatever the host's resolvers say. Case-insensitive; a
72 // trailing dot is ignored. Set before Listen.
73 Override string
74
75 mu sync.Mutex
76 listeners map[netip.Addr]*listener
77}
78
79type listener struct {
80 addr netip.Addr
81 udp net.PacketConn
82 tcp net.Listener
83 cancel context.CancelFunc
84 done sync.WaitGroup
85 // inflight and conns are the per-listener limits (counting semaphores).
86 inflight chan struct{}
87 conns chan struct{}
88 // Dropped counts UDP queries dropped because the listener was saturated.
89 dropped atomic.Int64
90}
91
92// New returns a Forwarder. upstreams returns the "host:port" resolvers to try
93// in order (see Resolvers.Addrs); port is the port to listen on (53, or another
94// port in tests).
95func New(upstreams func() []string, port int) *Forwarder {
96 return &Forwarder{
97 upstreams: upstreams,
98 port: port,
99 Lifetime: TCPLifetime,
100 listeners: map[netip.Addr]*listener{},
101 }
102}
103
104// Listen starts serving on addr (UDP and TCP). It returns once both sockets are
105// bound. Listening twice on the same address is an error.
106func (f *Forwarder) Listen(ctx context.Context, addr netip.Addr) error {
107 f.mu.Lock()
108 defer f.mu.Unlock()
109 if _, ok := f.listeners[addr]; ok {
110 return fmt.Errorf("dnsfwd: already listening on %s", addr)
111 }
112 hp := net.JoinHostPort(addr.String(), strconv.Itoa(f.port))
113 var lc net.ListenConfig
114 if f.FreeBind {
115 // The gateway address may not exist yet (podman's netavark configures the
116 // bridge address when the first container starts, after this listener).
117 lc.Control = func(_, _ string, c syscall.RawConn) error {
118 var serr error
119 if err := c.Control(func(fd uintptr) { serr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_FREEBIND, 1) }); err != nil {
120 return err
121 }
122 return serr
123 }
124 }
125 pc, err := lc.ListenPacket(ctx, "udp", hp)
126 if err != nil {
127 return fmt.Errorf("dnsfwd: listen udp %s: %w", hp, err)
128 }
129 // With port 0 (tests) UDP and TCP must share the port the kernel picked.
130 if f.port == 0 {
131 hp = pc.LocalAddr().String()
132 }
133 tl, err := lc.Listen(ctx, "tcp", hp)
134 if err != nil {
135 _ = pc.Close()
136 return fmt.Errorf("dnsfwd: listen tcp %s: %w", hp, err)
137 }
138 lctx, cancel := context.WithCancel(context.WithoutCancel(ctx))
139 l := &listener{addr: addr, udp: pc, tcp: tl, cancel: cancel, inflight: make(chan struct{}, MaxInflight), conns: make(chan struct{}, MaxTCPConns)}
140 f.listeners[addr] = l
141 log := zerowrap.FromCtx(ctx)
142 l.done.Add(2)
143 go func() { defer l.done.Done(); f.serveUDP(lctx, l, log) }()
144 go func() { defer l.done.Done(); f.serveTCP(lctx, l, log) }()
145 return nil
146}
147
148// Addr returns the bound UDP address for addr, useful when port is 0.
149func (f *Forwarder) Addr(addr netip.Addr) (net.Addr, bool) {
150 f.mu.Lock()
151 defer f.mu.Unlock()
152 l, ok := f.listeners[addr]
153 if !ok {
154 return nil, false
155 }
156 return l.udp.LocalAddr(), true
157}
158
159// Close stops listening on addr and waits for its goroutines. Unknown
160// addresses are ignored.
161func (f *Forwarder) Close(addr netip.Addr) {
162 f.mu.Lock()
163 l, ok := f.listeners[addr]
164 delete(f.listeners, addr)
165 f.mu.Unlock()
166 if !ok {
167 return
168 }
169 l.cancel()
170 _ = l.udp.Close()
171 _ = l.tcp.Close()
172 l.done.Wait()
173}
174
175// CloseAll stops every listener.
176func (f *Forwarder) CloseAll() {
177 f.mu.Lock()
178 addrs := make([]netip.Addr, 0, len(f.listeners))
179 for a := range f.listeners {
180 addrs = append(addrs, a)
181 }
182 f.mu.Unlock()
183 for _, a := range addrs {
184 f.Close(a)
185 }
186}
187
188// Dropped returns how many UDP queries the listener on addr dropped because it
189// was saturated.
190func (f *Forwarder) Dropped(addr netip.Addr) int64 {
191 f.mu.Lock()
192 defer f.mu.Unlock()
193 if l, ok := f.listeners[addr]; ok {
194 return l.dropped.Load()
195 }
196 return 0
197}
198
199func (f *Forwarder) serveUDP(ctx context.Context, l *listener, log zerowrap.Logger) {
200 pc := l.udp
201 buf := make([]byte, maxUDP)
202 for {
203 n, from, err := pc.ReadFrom(buf)
204 if err != nil {
205 if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) {
206 log.Warn().Err(err).Msg("dns forwarder: udp read")
207 }
208 return
209 }
210 if n < hdrLen {
211 continue
212 }
213 select {
214 case l.inflight <- struct{}{}:
215 default:
216 // Saturated: drop. The client retries; blocking here would let one
217 // guest's flood stall the read loop.
218 l.dropped.Add(1)
219 continue
220 }
221 q := append([]byte(nil), buf[:n]...)
222 go func() {
223 defer func() { <-l.inflight }()
224 resp := f.exchange(ctx, l, q, false)
225 if len(resp) > 0 {
226 _, _ = pc.WriteTo(resp, from)
227 }
228 }()
229 }
230}
231
232func (f *Forwarder) serveTCP(ctx context.Context, l *listener, log zerowrap.Logger) {
233 var wg sync.WaitGroup
234 defer wg.Wait()
235 for {
236 c, err := l.tcp.Accept()
237 if err != nil {
238 if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) {
239 log.Warn().Err(err).Msg("dns forwarder: tcp accept")
240 }
241 return
242 }
243 select {
244 case l.conns <- struct{}{}:
245 default:
246 _ = c.Close() // too many open connections on this listener
247 continue
248 }
249 wg.Add(1)
250 go func() {
251 defer wg.Done()
252 defer func() { <-l.conns }()
253 defer func() { _ = c.Close() }()
254 stop := context.AfterFunc(ctx, func() { _ = c.Close() })
255 defer stop()
256 // Absolute deadline from accept: neither idle nor slow-loris
257 // connections keep their slot past TCPLifetime.
258 deadline := time.Now().Add(f.Lifetime)
259 _ = c.SetDeadline(deadline)
260 f.serveConn(ctx, l, c, deadline)
261 }()
262 }
263}
264
265func (f *Forwarder) serveConn(ctx context.Context, l *listener, c net.Conn, deadline time.Time) {
266 for range TCPMaxQueries {
267 // The idle timeout can only shorten the absolute deadline.
268 dl := time.Now().Add(TCPIdleTimeout)
269 if deadline.Before(dl) {
270 dl = deadline
271 }
272 _ = c.SetReadDeadline(dl)
273 var lb [2]byte
274 if _, err := io.ReadFull(c, lb[:]); err != nil {
275 return
276 }
277 n := int(binary.BigEndian.Uint16(lb[:]))
278 if n < hdrLen {
279 return
280 }
281 q := make([]byte, n)
282 if _, err := io.ReadFull(c, q); err != nil {
283 return
284 }
285 select {
286 case l.inflight <- struct{}{}:
287 default:
288 return // saturated: close, the client retries
289 }
290 resp := f.exchange(ctx, l, q, true)
291 <-l.inflight
292 if len(resp) == 0 || len(resp) > 0xffff {
293 return
294 }
295 out := make([]byte, 2+len(resp))
296 binary.BigEndian.PutUint16(out, uint16(len(resp)))
297 copy(out[2:], resp)
298 if _, err := c.Write(out); err != nil {
299 return
300 }
301 }
302}
303
304// exchange relays q to the upstreams in order and returns the first answer, or
305// a SERVFAIL when none answers. The transport towards the upstream matches the
306// client's: a client that fell back to TCP gets a complete answer. The
307// overridden name, if any, is answered without asking the upstreams.
308func (f *Forwarder) exchange(ctx context.Context, l *listener, q []byte, overTCP bool) []byte {
309 if resp := f.overrideAnswer(l.addr, q); resp != nil {
310 return resp
311 }
312 for _, up := range f.upstreams() {
313 var resp []byte
314 var err error
315 if overTCP {
316 resp, err = exchangeTCP(ctx, up, q)
317 } else {
318 resp, err = exchangeUDP(ctx, up, q)
319 }
320 if err == nil && len(resp) >= hdrLen && resp[0] == q[0] && resp[1] == q[1] {
321 return resp
322 }
323 }
324 return servfail(q)
325}
326
327func exchangeUDP(ctx context.Context, up string, q []byte) ([]byte, error) {
328 ctx, cancel := context.WithTimeout(ctx, UpstreamTimeout)
329 defer cancel()
330 var d net.Dialer
331 c, err := d.DialContext(ctx, "udp", up)
332 if err != nil {
333 return nil, err
334 }
335 defer func() { _ = c.Close() }()
336 dl, _ := ctx.Deadline()
337 _ = c.SetDeadline(dl)
338 if _, err := c.Write(q); err != nil {
339 return nil, err
340 }
341 buf := make([]byte, maxUDP)
342 for {
343 n, err := c.Read(buf)
344 if err != nil {
345 return nil, err
346 }
347 // Ignore datagrams that do not answer this query (ID mismatch).
348 if n >= hdrLen && buf[0] == q[0] && buf[1] == q[1] {
349 return append([]byte(nil), buf[:n]...), nil
350 }
351 }
352}
353
354func exchangeTCP(ctx context.Context, up string, q []byte) ([]byte, error) {
355 ctx, cancel := context.WithTimeout(ctx, UpstreamTimeout)
356 defer cancel()
357 var d net.Dialer
358 c, err := d.DialContext(ctx, "tcp", up)
359 if err != nil {
360 return nil, err
361 }
362 defer func() { _ = c.Close() }()
363 dl, _ := ctx.Deadline()
364 _ = c.SetDeadline(dl)
365 out := make([]byte, 2+len(q))
366 binary.BigEndian.PutUint16(out, uint16(len(q)))
367 copy(out[2:], q)
368 if _, err := c.Write(out); err != nil {
369 return nil, err
370 }
371 var l [2]byte
372 if _, err := io.ReadFull(c, l[:]); err != nil {
373 return nil, err
374 }
375 resp := make([]byte, binary.BigEndian.Uint16(l[:]))
376 if _, err := io.ReadFull(c, resp); err != nil {
377 return nil, err
378 }
379 return resp, nil
380}
381
382// servfail builds a SERVFAIL reply for q: same ID and opcode, QR and RA set,
383// RD copied, no records.
384func servfail(q []byte) []byte {
385 if len(q) < hdrLen {
386 return nil
387 }
388 r := make([]byte, hdrLen)
389 r[0], r[1] = q[0], q[1]
390 r[2] = 0x80 | (q[2] & 0x79) // QR, opcode, RD kept; AA/TC cleared
391 r[3] = 0x80 | 0x02 // RA, RCODE=SERVFAIL
392 return r
393}