1// SPDX-License-Identifier: MIT
2
3// Package dnsfwd is a small DNS forwarder for job sandboxes (stdlib only). It
4// listens on the gateway address of each sandbox (UDP and TCP, port 53 in
5// production) and relays queries unchanged to the host's resolvers; the network
6// policy lets guests reach nothing else on the host. It does not parse queries
7// beyond what framing needs, so it supports every record type and EDNS.
8//
9// Abuse limits, per listener (one listener per guest, so one guest cannot
10// starve another): at most MaxInflight concurrent upstream exchanges (UDP
11// queries beyond that are dropped, never queued), at most MaxTCPConns open TCP
12// connections, an absolute lifetime of TCPLifetime per TCP connection (a client
13// cannot hold a slot by trickling bytes), a deadline on every upstream
14// exchange, and a cap on queries per TCP connection.
15package dnsfwd
16
17import (
18 "context"
19 "encoding/binary"
20 "errors"
21 "fmt"
22 "io"
23 "net"
24 "net/netip"
25 "strconv"
26 "sync"
27 "sync/atomic"
28 "syscall"
29 "time"
30
31 "github.com/bnema/zerowrap"
32 "golang.org/x/sys/unix"
33)
34
35const (
36 // TCPLifetime is the absolute lifetime of a TCP connection, set at accept.
37 TCPLifetime = 30 * time.Second
38 // UpstreamTimeout is the deadline of one upstream attempt.
39 UpstreamTimeout = 3 * time.Second
40 // TCPIdleTimeout closes TCP client connections that stay silent.
41 TCPIdleTimeout = 10 * time.Second
42
43 // MaxInflight bounds concurrent upstream exchanges per listener.
44 MaxInflight = 32
45 // MaxTCPConns bounds open TCP connections per listener.
46 MaxTCPConns = 16
47 // TCPMaxQueries caps queries per TCP connection.
48 TCPMaxQueries = 64
49
50 maxUDP = 4096 // largest datagram relayed (EDNS0 sizes up to 4096)
51 hdrLen = 12
52)
53
54// Forwarder relays DNS queries. The zero value is not usable; use New.
55type Forwarder struct {
56 upstreams func() []string
57 port int
58 // Lifetime overrides TCPLifetime; set before Listen (tests).
59 Lifetime time.Duration
60 // FreeBind binds with IP_FREEBIND, so that the gateway address may be
61 // configured after the listener. Off by default: with Docker the address
62 // exists when the network is created and a missing one is an error to
63 // report at once, not a listener on an address nobody owns. Set before Listen.
64 FreeBind bool
65
66 mu sync.Mutex
67 listeners map[netip.Addr]*listener
68}
69
70type listener struct {
71 udp net.PacketConn
72 tcp net.Listener
73 cancel context.CancelFunc
74 done sync.WaitGroup
75 // inflight and conns are the per-listener limits (counting semaphores).
76 inflight chan struct{}
77 conns chan struct{}
78 // Dropped counts UDP queries dropped because the listener was saturated.
79 dropped atomic.Int64
80}
81
82// New returns a Forwarder. upstreams returns the "host:port" resolvers to try
83// in order (see Resolvers.Addrs); port is the port to listen on (53, or another
84// port in tests).
85func New(upstreams func() []string, port int) *Forwarder {
86 return &Forwarder{
87 upstreams: upstreams,
88 port: port,
89 Lifetime: TCPLifetime,
90 listeners: map[netip.Addr]*listener{},
91 }
92}
93
94// Listen starts serving on addr (UDP and TCP). It returns once both sockets are
95// bound. Listening twice on the same address is an error.
96func (f *Forwarder) Listen(ctx context.Context, addr netip.Addr) error {
97 f.mu.Lock()
98 defer f.mu.Unlock()
99 if _, ok := f.listeners[addr]; ok {
100 return fmt.Errorf("dnsfwd: already listening on %s", addr)
101 }
102 hp := net.JoinHostPort(addr.String(), strconv.Itoa(f.port))
103 var lc net.ListenConfig
104 if f.FreeBind {
105 // The gateway address may not exist yet (podman's netavark configures the
106 // bridge address when the first container starts, after this listener).
107 lc.Control = func(_, _ string, c syscall.RawConn) error {
108 var serr error
109 if err := c.Control(func(fd uintptr) { serr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_FREEBIND, 1) }); err != nil {
110 return err
111 }
112 return serr
113 }
114 }
115 pc, err := lc.ListenPacket(ctx, "udp", hp)
116 if err != nil {
117 return fmt.Errorf("dnsfwd: listen udp %s: %w", hp, err)
118 }
119 // With port 0 (tests) UDP and TCP must share the port the kernel picked.
120 if f.port == 0 {
121 hp = pc.LocalAddr().String()
122 }
123 tl, err := lc.Listen(ctx, "tcp", hp)
124 if err != nil {
125 _ = pc.Close()
126 return fmt.Errorf("dnsfwd: listen tcp %s: %w", hp, err)
127 }
128 lctx, cancel := context.WithCancel(context.WithoutCancel(ctx))
129 l := &listener{udp: pc, tcp: tl, cancel: cancel, inflight: make(chan struct{}, MaxInflight), conns: make(chan struct{}, MaxTCPConns)}
130 f.listeners[addr] = l
131 log := zerowrap.FromCtx(ctx)
132 l.done.Add(2)
133 go func() { defer l.done.Done(); f.serveUDP(lctx, l, log) }()
134 go func() { defer l.done.Done(); f.serveTCP(lctx, l, log) }()
135 return nil
136}
137
138// Addr returns the bound UDP address for addr, useful when port is 0.
139func (f *Forwarder) Addr(addr netip.Addr) (net.Addr, bool) {
140 f.mu.Lock()
141 defer f.mu.Unlock()
142 l, ok := f.listeners[addr]
143 if !ok {
144 return nil, false
145 }
146 return l.udp.LocalAddr(), true
147}
148
149// Close stops listening on addr and waits for its goroutines. Unknown
150// addresses are ignored.
151func (f *Forwarder) Close(addr netip.Addr) {
152 f.mu.Lock()
153 l, ok := f.listeners[addr]
154 delete(f.listeners, addr)
155 f.mu.Unlock()
156 if !ok {
157 return
158 }
159 l.cancel()
160 _ = l.udp.Close()
161 _ = l.tcp.Close()
162 l.done.Wait()
163}
164
165// CloseAll stops every listener.
166func (f *Forwarder) CloseAll() {
167 f.mu.Lock()
168 addrs := make([]netip.Addr, 0, len(f.listeners))
169 for a := range f.listeners {
170 addrs = append(addrs, a)
171 }
172 f.mu.Unlock()
173 for _, a := range addrs {
174 f.Close(a)
175 }
176}
177
178// Dropped returns how many UDP queries the listener on addr dropped because it
179// was saturated.
180func (f *Forwarder) Dropped(addr netip.Addr) int64 {
181 f.mu.Lock()
182 defer f.mu.Unlock()
183 if l, ok := f.listeners[addr]; ok {
184 return l.dropped.Load()
185 }
186 return 0
187}
188
189func (f *Forwarder) serveUDP(ctx context.Context, l *listener, log zerowrap.Logger) {
190 pc := l.udp
191 buf := make([]byte, maxUDP)
192 for {
193 n, from, err := pc.ReadFrom(buf)
194 if err != nil {
195 if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) {
196 log.Warn().Err(err).Msg("dns forwarder: udp read")
197 }
198 return
199 }
200 if n < hdrLen {
201 continue
202 }
203 select {
204 case l.inflight <- struct{}{}:
205 default:
206 // Saturated: drop. The client retries; blocking here would let one
207 // guest's flood stall the read loop.
208 l.dropped.Add(1)
209 continue
210 }
211 q := append([]byte(nil), buf[:n]...)
212 go func() {
213 defer func() { <-l.inflight }()
214 resp := f.exchange(ctx, q, false)
215 if len(resp) > 0 {
216 _, _ = pc.WriteTo(resp, from)
217 }
218 }()
219 }
220}
221
222func (f *Forwarder) serveTCP(ctx context.Context, l *listener, log zerowrap.Logger) {
223 var wg sync.WaitGroup
224 defer wg.Wait()
225 for {
226 c, err := l.tcp.Accept()
227 if err != nil {
228 if ctx.Err() == nil && !errors.Is(err, net.ErrClosed) {
229 log.Warn().Err(err).Msg("dns forwarder: tcp accept")
230 }
231 return
232 }
233 select {
234 case l.conns <- struct{}{}:
235 default:
236 _ = c.Close() // too many open connections on this listener
237 continue
238 }
239 wg.Add(1)
240 go func() {
241 defer wg.Done()
242 defer func() { <-l.conns }()
243 defer func() { _ = c.Close() }()
244 stop := context.AfterFunc(ctx, func() { _ = c.Close() })
245 defer stop()
246 // Absolute deadline from accept: neither idle nor slow-loris
247 // connections keep their slot past TCPLifetime.
248 deadline := time.Now().Add(f.Lifetime)
249 _ = c.SetDeadline(deadline)
250 f.serveConn(ctx, l, c, deadline)
251 }()
252 }
253}
254
255func (f *Forwarder) serveConn(ctx context.Context, l *listener, c net.Conn, deadline time.Time) {
256 for range TCPMaxQueries {
257 // The idle timeout can only shorten the absolute deadline.
258 dl := time.Now().Add(TCPIdleTimeout)
259 if deadline.Before(dl) {
260 dl = deadline
261 }
262 _ = c.SetReadDeadline(dl)
263 var lb [2]byte
264 if _, err := io.ReadFull(c, lb[:]); err != nil {
265 return
266 }
267 n := int(binary.BigEndian.Uint16(lb[:]))
268 if n < hdrLen {
269 return
270 }
271 q := make([]byte, n)
272 if _, err := io.ReadFull(c, q); err != nil {
273 return
274 }
275 select {
276 case l.inflight <- struct{}{}:
277 default:
278 return // saturated: close, the client retries
279 }
280 resp := f.exchange(ctx, q, true)
281 <-l.inflight
282 if len(resp) == 0 || len(resp) > 0xffff {
283 return
284 }
285 out := make([]byte, 2+len(resp))
286 binary.BigEndian.PutUint16(out, uint16(len(resp)))
287 copy(out[2:], resp)
288 if _, err := c.Write(out); err != nil {
289 return
290 }
291 }
292}
293
294// exchange relays q to the upstreams in order and returns the first answer, or
295// a SERVFAIL when none answers. The transport towards the upstream matches the
296// client's: a client that fell back to TCP gets a complete answer.
297func (f *Forwarder) exchange(ctx context.Context, q []byte, overTCP bool) []byte {
298 for _, up := range f.upstreams() {
299 var resp []byte
300 var err error
301 if overTCP {
302 resp, err = exchangeTCP(ctx, up, q)
303 } else {
304 resp, err = exchangeUDP(ctx, up, q)
305 }
306 if err == nil && len(resp) >= hdrLen && resp[0] == q[0] && resp[1] == q[1] {
307 return resp
308 }
309 }
310 return servfail(q)
311}
312
313func exchangeUDP(ctx context.Context, up string, q []byte) ([]byte, error) {
314 ctx, cancel := context.WithTimeout(ctx, UpstreamTimeout)
315 defer cancel()
316 var d net.Dialer
317 c, err := d.DialContext(ctx, "udp", up)
318 if err != nil {
319 return nil, err
320 }
321 defer func() { _ = c.Close() }()
322 dl, _ := ctx.Deadline()
323 _ = c.SetDeadline(dl)
324 if _, err := c.Write(q); err != nil {
325 return nil, err
326 }
327 buf := make([]byte, maxUDP)
328 for {
329 n, err := c.Read(buf)
330 if err != nil {
331 return nil, err
332 }
333 // Ignore datagrams that do not answer this query (ID mismatch).
334 if n >= hdrLen && buf[0] == q[0] && buf[1] == q[1] {
335 return append([]byte(nil), buf[:n]...), nil
336 }
337 }
338}
339
340func exchangeTCP(ctx context.Context, up string, q []byte) ([]byte, error) {
341 ctx, cancel := context.WithTimeout(ctx, UpstreamTimeout)
342 defer cancel()
343 var d net.Dialer
344 c, err := d.DialContext(ctx, "tcp", up)
345 if err != nil {
346 return nil, err
347 }
348 defer func() { _ = c.Close() }()
349 dl, _ := ctx.Deadline()
350 _ = c.SetDeadline(dl)
351 out := make([]byte, 2+len(q))
352 binary.BigEndian.PutUint16(out, uint16(len(q)))
353 copy(out[2:], q)
354 if _, err := c.Write(out); err != nil {
355 return nil, err
356 }
357 var l [2]byte
358 if _, err := io.ReadFull(c, l[:]); err != nil {
359 return nil, err
360 }
361 resp := make([]byte, binary.BigEndian.Uint16(l[:]))
362 if _, err := io.ReadFull(c, resp); err != nil {
363 return nil, err
364 }
365 return resp, nil
366}
367
368// servfail builds a SERVFAIL reply for q: same ID and opcode, QR and RA set,
369// RD copied, no records.
370func servfail(q []byte) []byte {
371 if len(q) < hdrLen {
372 return nil
373 }
374 r := make([]byte, hdrLen)
375 r[0], r[1] = q[0], q[1]
376 r[2] = 0x80 | (q[2] & 0x79) // QR, opcode, RD kept; AA/TC cleared
377 r[3] = 0x80 | 0x02 // RA, RCODE=SERVFAIL
378 return r
379}