22211fd fix(tcprelay): free-bind the relay under podman and rate-limit saturation warnings · ermite-runner

M internal/sandbox/container/container_test.go +6 -0


@@ -1020,6 +1020,12 @@ func TestOnlyPodmanGetsAFreeBindDNSListener(t *testing.T) {
1020	1020		if !newNet(nil, 0, true, "", "").DNS.FreeBind {
1021	1021		t.Fatal("podman needs the listener before the bridge address")
1022	1022		}
	1023	+	if newNet(nil, 0, false, "forge", "forge:80").Relay.FreeBind {
	1024	+	t.Fatal("Docker must keep a normal bind for the relay")
	1025	+	}
	1026	+	if !newNet(nil, 0, true, "forge", "forge:80").Relay.FreeBind {
	1027	+	t.Fatal("podman needs the relay listener before the bridge address")
	1028	+	}
1023	1029		}
1024	1030
1025	1031		// Guest forge access: the exception reaches the policy of every guest, and the


@@ -1020,6 +1020,12 @@ func TestOnlyPodmanGetsAFreeBindDNSListener(t *testing.T) {
1020	if !newNet(nil, 0, true, "", "").DNS.FreeBind {	1020		if !newNet(nil, 0, true, "", "").DNS.FreeBind {
1021	t.Fatal("podman needs the listener before the bridge address")	1021		t.Fatal("podman needs the listener before the bridge address")
1022	}	1022		}
		1023	+	if newNet(nil, 0, false, "forge", "forge:80").Relay.FreeBind {
		1024	+	t.Fatal("Docker must keep a normal bind for the relay")
		1025	+	}
		1026	+	if !newNet(nil, 0, true, "forge", "forge:80").Relay.FreeBind {
		1027	+	t.Fatal("podman needs the relay listener before the bridge address")
		1028	+	}
1023	}	1029		}
1024		1030
1025	// Guest forge access: the exception reaches the policy of every guest, and the	1031		// Guest forge access: the exception reaches the policy of every guest, and the

M internal/sandbox/container/driver.go +3 -2


@@ -177,14 +177,15 @@ func New(ctx context.Context, cfg sandbox.Config, opt Options) (*Driver, error)
177	177		}
178	178
179	179		// newNet builds the host network plumbing. Only podman creates the bridge
180		-	// address late (see daemonInfo.Podman), so only there the DNS listener may come
181		-	// before the address; on Docker a missing gateway address stays a startup error.
	180	+	// address late (see daemonInfo.Podman), so only there the DNS and relay
	181	+	// listeners may come before the address; on Docker a missing gateway address stays a startup error.
182	182		func newNet(run cmdrun.Runner, dnsPort int, podman bool, forgeHost, relay string) *netsetup.Net {
183	183		n := netsetup.New(run, dnsPort, "")
184	184		n.DNS.FreeBind = podman
185	185		n.DNS.Override = forgeHost
186	186		if relay != "" {
187	187		n.Relay = tcprelay.New(relay)
	188	+	n.Relay.FreeBind = podman
188	189		}
189	190		return n
190	191		}

 }
 // newNet builds the host network plumbing. Only podman creates the bridge
-// address late (see daemonInfo.Podman), so only there the DNS listener may come
+// address late (see daemonInfo.Podman), so only there the DNS and relay
-// before the address; on Docker a missing gateway address stays a startup error.
+// listeners may come before the address; on Docker a missing gateway address stays a startup error.
 func newNet(run cmdrun.Runner, dnsPort int, podman bool, forgeHost, relay string) *netsetup.Net {
 	n := netsetup.New(run, dnsPort, "")
 	n.DNS.FreeBind = podman
 	n.DNS.Override = forgeHost
 	if relay != "" {
 		n.Relay = tcprelay.New(relay)
+		n.Relay.FreeBind = podman
 	}
 	return n
 }

M internal/sandbox/netsetup/netsetup_test.go +18 -7


@@ -5,6 +5,7 @@ package netsetup
5	5		import (
6	6		"context"
7	7		"errors"
	8	+	"net"
8	9		"net/netip"
9	10		"os"
10	11		"path/filepath"

@@ -16,6 +17,17 @@ import (
16	17		"bnema.dev/ermite-runner/internal/tcprelay"
17	18		)
18	19
	20	+	// freePort returns a TCP port of 127.0.0.1 that was free when probed.
	21	+	func freePort(t *testing.T) uint16 {
	22	+	t.Helper()
	23	+	ln, err := net.Listen("tcp", "127.0.0.1:0")
	24	+	if err != nil {
	25	+	t.Skip("cannot bind loopback:", err)
	26	+	}
	27	+	defer func() { _ = ln.Close() }()
	28	+	return uint16(ln.Addr().(*net.TCPAddr).Port)
	29	+	}
	30	+
19	31		func guest() netpolicy.Guest {
20	32		return netpolicy.Guest{Iface: "ertap0", Gateway: netip.MustParseAddr("127.0.0.1"), Addr: netip.MustParseAddr("10.200.0.2")}
21	33		}

@@ -100,14 +112,13 @@ func TestAttachStartsAndStopsTheForgeRelay(t *testing.T) {
100	112		if err := n.Detach(ctx, g.Iface, g.Gateway); err != nil {
101	113		t.Fatal(err)
102	114		}
103		-	// Port 0 lets the kernel pick a free port for the relay.
104		-	g.ForgePort = 0
105		-	n.Relay = tcprelay.New("127.0.0.1:1")
106		-	if err := n.Relay.Listen(ctx, g.Gateway, 0); err != nil {
107		-	t.Skip(err)
	115	+	// A port that was free a moment ago: Attach itself starts the relay.
	116	+	g.ForgePort = freePort(t)
	117	+	if err := n.Attach(ctx, g); err != nil {
	118	+	t.Fatalf("Attach with a forge port: %v", err)
108	119		}
109		-	if _, ok := n.Relay.Addr(g.Gateway); !ok {
110		-	t.Fatal("relay not listening")
	120	+	if a, ok := n.Relay.Addr(g.Gateway); !ok \|\| a.(*net.TCPAddr).Port != int(g.ForgePort) {
	121	+	t.Fatalf("relay not listening on the forge port: %v %v", a, ok)
111	122		}
112	123		if err := n.Detach(ctx, g.Iface, g.Gateway); err != nil {
113	124		t.Fatal(err)

 import (
 	"context"
 	"errors"
+	"net"
 	"net/netip"
 	"os"
 	"path/filepath"
 	"bnema.dev/ermite-runner/internal/tcprelay"
 )
+// freePort returns a TCP port of 127.0.0.1 that was free when probed.
+func freePort(t *testing.T) uint16 {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Skip("cannot bind loopback:", err)
+	}
+	defer func() { _ = ln.Close() }()
+	return uint16(ln.Addr().(*net.TCPAddr).Port)
+}
 func guest() netpolicy.Guest {
 	return netpolicy.Guest{Iface: "ertap0", Gateway: netip.MustParseAddr("127.0.0.1"), Addr: netip.MustParseAddr("10.200.0.2")}
 }
 	if err := n.Detach(ctx, g.Iface, g.Gateway); err != nil {
 		t.Fatal(err)
 	}
-	// Port 0 lets the kernel pick a free port for the relay.
+	// A port that was free a moment ago: Attach itself starts the relay.
-	g.ForgePort = 0
+	g.ForgePort = freePort(t)
-	n.Relay = tcprelay.New("127.0.0.1:1")
+	if err := n.Attach(ctx, g); err != nil {
-	if err := n.Relay.Listen(ctx, g.Gateway, 0); err != nil {
+		t.Fatalf("Attach with a forge port: %v", err)
-		t.Skip(err)
 	}
-	if _, ok := n.Relay.Addr(g.Gateway); !ok {
+	if a, ok := n.Relay.Addr(g.Gateway); !ok || a.(*net.TCPAddr).Port != int(g.ForgePort) {
-		t.Fatal("relay not listening")
+		t.Fatalf("relay not listening on the forge port: %v %v", a, ok)
 	}
 	if err := n.Detach(ctx, g.Iface, g.Gateway); err != nil {
 		t.Fatal(err)

M internal/tcprelay/tcprelay.go +28 -1


@@ -16,9 +16,11 @@ import (
16	16		"net/netip"
17	17		"strconv"
18	18		"sync"
	19	+	"syscall"
19	20		"time"
20	21
21	22		"github.com/bnema/zerowrap"
	23	+	"golang.org/x/sys/unix"
22	24		)
23	25
24	26		// MaxConns bounds the connections relayed at once for one listener: a guest

@@ -28,12 +30,20 @@ const MaxConns = 64
28	30		// dialTimeout bounds the connection to the target.
29	31		const dialTimeout = 10 * time.Second
30	32
	33	+	// saturationLogEvery bounds the warnings about refused connections: a guest
	34	+	// that floods its gateway must not flood the runner's log.
	35	+	const saturationLogEvery = 10 * time.Second
	36	+
31	37		// Relay forwards every connection accepted on its listeners to Target.
32	38		type Relay struct {
33	39		// Target is host:port, resolved again for every connection.
34	40		Target string
35	41		// Dial connects to Target; nil means a net.Dialer.
36	42		Dial func(ctx context.Context, network, addr string) (net.Conn, error)
	43	+	// FreeBind binds with IP_FREEBIND, so that the gateway address may be
	44	+	// configured after Listen (podman's netavark creates the bridge address
	45	+	// when the first container starts).
	46	+	FreeBind bool
37	47
38	48		mu sync.Mutex
39	49		listeners map[netip.Addr]*listener

@@ -63,6 +73,15 @@ func (r *Relay) Listen(ctx context.Context, addr netip.Addr, port uint16) error
63	73		return fmt.Errorf("tcprelay: already listening on %s", addr)
64	74		}
65	75		var lc net.ListenConfig
	76	+	if r.FreeBind {
	77	+	lc.Control = func(_, _ string, c syscall.RawConn) error {
	78	+	var serr error
	79	+	if err := c.Control(func(fd uintptr) { serr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_FREEBIND, 1) }); err != nil {
	80	+	return err
	81	+	}
	82	+	return serr
	83	+	}
	84	+	}
66	85		ln, err := lc.Listen(ctx, "tcp", net.JoinHostPort(addr.String(), strconv.Itoa(int(port))))
67	86		if err != nil {
68	87		return fmt.Errorf("tcprelay: listen %s:%d: %w", addr, port, err)

@@ -120,6 +139,10 @@ func (l *listener) stop() {
120	139
121	140		func (r Relay) serve(ctx context.Context, l listener) {
122	141		log := zerowrap.FromCtx(ctx)
	142	+	var (
	143	+	refused int
	144	+	lastWarn time.Time
	145	+	)
123	146		for {
124	147		c, err := l.ln.Accept()
125	148		if err != nil {

@@ -132,8 +155,12 @@ func (r Relay) serve(ctx context.Context, l listener) {
132	155		select {
133	156		case l.slots <- struct{}{}:
134	157		default:
135		-	log.Warn().Str("target", r.Target).Msg("tcprelay: too many relayed connections, refusing one")
136	158		_ = c.Close()
	159	+	refused++
	160	+	if now := time.Now(); now.Sub(lastWarn) >= saturationLogEvery {
	161	+	log.Warn().Str("target", r.Target).Int("refused", refused).Int("max", MaxConns).Msg("tcprelay: too many relayed connections, refusing")
	162	+	lastWarn, refused = now, 0
	163	+	}
137	164		continue
138	165		}
139	166		l.done.Add(1)

 	"net/netip"
 	"strconv"
 	"sync"
+	"syscall"
 	"time"
 	"github.com/bnema/zerowrap"
+	"golang.org/x/sys/unix"
 )
 // MaxConns bounds the connections relayed at once for one listener: a guest
 // dialTimeout bounds the connection to the target.
 const dialTimeout = 10 * time.Second
+// saturationLogEvery bounds the warnings about refused connections: a guest
+// that floods its gateway must not flood the runner's log.
+const saturationLogEvery = 10 * time.Second
 // Relay forwards every connection accepted on its listeners to Target.
 type Relay struct {
 	// Target is host:port, resolved again for every connection.
 	Target string
 	// Dial connects to Target; nil means a net.Dialer.
 	Dial func(ctx context.Context, network, addr string) (net.Conn, error)
+	// FreeBind binds with IP_FREEBIND, so that the gateway address may be
+	// configured after Listen (podman's netavark creates the bridge address
+	// when the first container starts).
+	FreeBind bool
 	mu        sync.Mutex
 	listeners map[netip.Addr]*listener
 		return fmt.Errorf("tcprelay: already listening on %s", addr)
 	}
 	var lc net.ListenConfig
+	if r.FreeBind {
+		lc.Control = func(_, _ string, c syscall.RawConn) error {
+			var serr error
+			if err := c.Control(func(fd uintptr) { serr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_FREEBIND, 1) }); err != nil {
+				return err
+			}
+			return serr
+		}
+	}
 	ln, err := lc.Listen(ctx, "tcp", net.JoinHostPort(addr.String(), strconv.Itoa(int(port))))
 	if err != nil {
 		return fmt.Errorf("tcprelay: listen %s:%d: %w", addr, port, err)
 func (r *Relay) serve(ctx context.Context, l *listener) {
 	log := zerowrap.FromCtx(ctx)
+	var (
+		refused  int
+		lastWarn time.Time
+	)
 	for {
 		c, err := l.ln.Accept()
 		if err != nil {
 		select {
 		case l.slots <- struct{}{}:
 		default:
-			log.Warn().Str("target", r.Target).Msg("tcprelay: too many relayed connections, refusing one")
 			_ = c.Close()
+			refused++
+			if now := time.Now(); now.Sub(lastWarn) >= saturationLogEvery {
+				log.Warn().Str("target", r.Target).Int("refused", refused).Int("max", MaxConns).Msg("tcprelay: too many relayed connections, refusing")
+				lastWarn, refused = now, 0
+			}
 			continue
 		}
 		l.done.Add(1)

M internal/tcprelay/tcprelay_test.go +59 -0


@@ -103,6 +103,65 @@ func TestRelayCloseStopsListenerAndConnections(t *testing.T) {
103	103		r.Close(lo) // idempotent
104	104		}
105	105
	106	+	// By default Listen fails on an address the host does not own; FreeBind
	107	+	// (podman: the bridge address comes when the first container starts) binds it.
	108	+	func TestRelayFreeBind(t *testing.T) {
	109	+	ip := netip.MustParseAddr("203.0.113.77") // TEST-NET-3: never configured here
	110	+	r := New("127.0.0.1:1")
	111	+	t.Cleanup(r.CloseAll)
	112	+	if err := r.Listen(context.Background(), ip, 0); err == nil {
	113	+	t.Fatal("listening on an address the host does not own must fail without FreeBind")
	114	+	}
	115	+	r.FreeBind = true
	116	+	if err := r.Listen(context.Background(), ip, 0); err != nil {
	117	+	t.Fatalf("FreeBind: %v", err)
	118	+	}
	119	+	if _, ok := r.Addr(ip); !ok {
	120	+	t.Fatal("no listener")
	121	+	}
	122	+	}
	123	+
	124	+	// At most MaxConns connections are relayed at once; the next one is closed,
	125	+	// and a slot freed by a closed connection is reused.
	126	+	func TestRelayConnectionLimit(t *testing.T) {
	127	+	r := New(echo(t))
	128	+	if err := r.Listen(context.Background(), lo, 0); err != nil {
	129	+	t.Fatal(err)
	130	+	}
	131	+	t.Cleanup(r.CloseAll)
	132	+	roundTrip := func(c net.Conn) error {
	133	+	_ = c.SetDeadline(time.Now().Add(5 * time.Second))
	134	+	if _, err := c.Write([]byte("x\n")); err != nil {
	135	+	return err
	136	+	}
	137	+	_, err := bufio.NewReader(c).ReadString('\n')
	138	+	return err
	139	+	}
	140	+	conns := make([]net.Conn, MaxConns)
	141	+	for i := range conns {
	142	+	conns[i] = dialRelay(t, r)
	143	+	if err := roundTrip(conns[i]); err != nil {
	144	+	t.Fatalf("connection %d: %v", i, err)
	145	+	}
	146	+	}
	147	+	extra := dialRelay(t, r)
	148	+	if err := roundTrip(extra); err == nil {
	149	+	t.Fatal("a connection over MaxConns was relayed")
	150	+	}
	151	+	_ = conns[0].Close()
	152	+	deadline := time.Now().Add(5 * time.Second)
	153	+	for {
	154	+	c := dialRelay(t, r)
	155	+	if roundTrip(c) == nil {
	156	+	break
	157	+	}
	158	+	if time.Now().After(deadline) {
	159	+	t.Fatal("the freed slot was not reused")
	160	+	}
	161	+	time.Sleep(20 * time.Millisecond)
	162	+	}
	163	+	}
	164	+
106	165		func TestRelayUnreachableTargetClosesTheGuestSide(t *testing.T) {
107	166		ln, err := net.Listen("tcp", "127.0.0.1:0")
108	167		if err != nil {


@@ -103,6 +103,65 @@ func TestRelayCloseStopsListenerAndConnections(t *testing.T) {
103	r.Close(lo) // idempotent	103		r.Close(lo) // idempotent
104	}	104		}
105		105
		106	+	// By default Listen fails on an address the host does not own; FreeBind
		107	+	// (podman: the bridge address comes when the first container starts) binds it.
		108	+	func TestRelayFreeBind(t *testing.T) {
		109	+	ip := netip.MustParseAddr("203.0.113.77") // TEST-NET-3: never configured here
		110	+	r := New("127.0.0.1:1")
		111	+	t.Cleanup(r.CloseAll)
		112	+	if err := r.Listen(context.Background(), ip, 0); err == nil {
		113	+	t.Fatal("listening on an address the host does not own must fail without FreeBind")
		114	+	}
		115	+	r.FreeBind = true
		116	+	if err := r.Listen(context.Background(), ip, 0); err != nil {
		117	+	t.Fatalf("FreeBind: %v", err)
		118	+	}
		119	+	if _, ok := r.Addr(ip); !ok {
		120	+	t.Fatal("no listener")
		121	+	}
		122	+	}
		123	+
		124	+	// At most MaxConns connections are relayed at once; the next one is closed,
		125	+	// and a slot freed by a closed connection is reused.
		126	+	func TestRelayConnectionLimit(t *testing.T) {
		127	+	r := New(echo(t))
		128	+	if err := r.Listen(context.Background(), lo, 0); err != nil {
		129	+	t.Fatal(err)
		130	+	}
		131	+	t.Cleanup(r.CloseAll)
		132	+	roundTrip := func(c net.Conn) error {
		133	+	_ = c.SetDeadline(time.Now().Add(5 * time.Second))
		134	+	if _, err := c.Write([]byte("x\n")); err != nil {
		135	+	return err
		136	+	}
		137	+	_, err := bufio.NewReader(c).ReadString('\n')
		138	+	return err
		139	+	}
		140	+	conns := make([]net.Conn, MaxConns)
		141	+	for i := range conns {
		142	+	conns[i] = dialRelay(t, r)
		143	+	if err := roundTrip(conns[i]); err != nil {
		144	+	t.Fatalf("connection %d: %v", i, err)
		145	+	}
		146	+	}
		147	+	extra := dialRelay(t, r)
		148	+	if err := roundTrip(extra); err == nil {
		149	+	t.Fatal("a connection over MaxConns was relayed")
		150	+	}
		151	+	_ = conns[0].Close()
		152	+	deadline := time.Now().Add(5 * time.Second)
		153	+	for {
		154	+	c := dialRelay(t, r)
		155	+	if roundTrip(c) == nil {
		156	+	break
		157	+	}
		158	+	if time.Now().After(deadline) {
		159	+	t.Fatal("the freed slot was not reused")
		160	+	}
		161	+	time.Sleep(20 * time.Millisecond)
		162	+	}
		163	+	}
		164	+
106	func TestRelayUnreachableTargetClosesTheGuestSide(t *testing.T) {	165		func TestRelayUnreachableTargetClosesTheGuestSide(t *testing.T) {
107	ln, err := net.Listen("tcp", "127.0.0.1:0")	166		ln, err := net.Listen("tcp", "127.0.0.1:0")
108	if err != nil {	167		if err != nil {