detached 21cbe0b

feat(client): add autonomous broker supervisor

commit
21cbe0bb8b93e008d9079d27463775394ca6d024
author
brice · · 2026-09-17 08:59 UTC
parent
c16729e
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
13 files · +2805 -1
Split
UnifiedSplit
M internal/adapters/brokeripc/client.go +42 -0
@@ -35,6 +35,34 @@ func Dial(ctx context.Context, path string, cfg Config) (ports.BrokerService, er
3535 return dial(ctx, path, cfg, ipc.SameUserPeerVerifier())
3636 }
3737
38+// Connector implements ports.BrokerConnector over the private broker IPC
39+// carriage. It holds only the endpoint and its bounds, so every Connect is an
40+// independent attempt and the caller owns Close on the returned service. The
41+// client process reaches the broker use case exclusively through this seam.
42+type Connector struct {
43+ path string
44+ cfg Config
45+}
46+
47+var _ ports.BrokerConnector = (*Connector)(nil)
48+
49+// NewConnector returns a connector for one broker endpoint. An empty path is
50+// refused by Connect, exactly as Dial refuses it, so a misconfigured connector
51+// fails closed instead of dialing an unspecified socket.
52+func NewConnector(path string, cfg Config) *Connector {
53+ return &Connector{path: path, cfg: cfg}
54+}
55+
56+// Connect dials the configured endpoint and returns the connection-scoped
57+// service. Cancellation is honored by the underlying Dial, which bounds the
58+// whole setup and closes the carriage when its context expires.
59+func (c *Connector) Connect(ctx context.Context) (ports.BrokerService, error) {
60+ if c == nil {
61+ return nil, ErrConfig
62+ }
63+ return Dial(ctx, c.path, c.cfg)
64+}
65+
3866 // dial is Dial with an injected peer verifier, so a test can observe a
3967 // deterministic same-user refusal without root.
4068 func dial(ctx context.Context, path string, cfg Config, verify ipc.PeerVerifier) (ports.BrokerService, error) {
@@ -351,6 +379,20 @@ func (c *client) completeOperation(operation ports.BrokerOperationID, result ope
351379 // ConnectionID returns the identity assigned to this connection at accept.
352380 func (c *client) ConnectionID() ports.BrokerConnectionID { return c.scope.Connection }
353381
382+// Done closes exactly once when this connection is terminal: the carriage
383+// failed, the broker retired it, or this side closed it. Err is stable
384+// afterwards.
385+func (c *client) Done() <-chan struct{} { return c.done }
386+
387+// Err returns this connection's terminal cause: nil for an orderly local Close,
388+// otherwise the failure that settled it. The cause is recorded exactly once and
389+// never overwritten, so it is stable once Done is closed.
390+func (c *client) Err() error {
391+ c.mu.Lock()
392+ defer c.mu.Unlock()
393+ return c.err
394+}
395+
354396 // Snapshot returns the newest fully committed publication.
355397 func (c *client) Snapshot() ports.BrokerSnapshot {
356398 c.mu.Lock()
@@ -35,6 +35,34 @@ func Dial(ctx context.Context, path string, cfg Config) (ports.BrokerService, er
35 return dial(ctx, path, cfg, ipc.SameUserPeerVerifier())35 return dial(ctx, path, cfg, ipc.SameUserPeerVerifier())
36 }36 }
37 37
38+// Connector implements ports.BrokerConnector over the private broker IPC
39+// carriage. It holds only the endpoint and its bounds, so every Connect is an
40+// independent attempt and the caller owns Close on the returned service. The
41+// client process reaches the broker use case exclusively through this seam.
42+type Connector struct {
43+ path string
44+ cfg Config
45+}
46+
47+var _ ports.BrokerConnector = (*Connector)(nil)
48+
49+// NewConnector returns a connector for one broker endpoint. An empty path is
50+// refused by Connect, exactly as Dial refuses it, so a misconfigured connector
51+// fails closed instead of dialing an unspecified socket.
52+func NewConnector(path string, cfg Config) *Connector {
53+ return &Connector{path: path, cfg: cfg}
54+}
55+
56+// Connect dials the configured endpoint and returns the connection-scoped
57+// service. Cancellation is honored by the underlying Dial, which bounds the
58+// whole setup and closes the carriage when its context expires.
59+func (c *Connector) Connect(ctx context.Context) (ports.BrokerService, error) {
60+ if c == nil {
61+ return nil, ErrConfig
62+ }
63+ return Dial(ctx, c.path, c.cfg)
64+}
65+
38 // dial is Dial with an injected peer verifier, so a test can observe a66 // dial is Dial with an injected peer verifier, so a test can observe a
39 // deterministic same-user refusal without root.67 // deterministic same-user refusal without root.
40 func dial(ctx context.Context, path string, cfg Config, verify ipc.PeerVerifier) (ports.BrokerService, error) {68 func dial(ctx context.Context, path string, cfg Config, verify ipc.PeerVerifier) (ports.BrokerService, error) {
@@ -351,6 +379,20 @@ func (c *client) completeOperation(operation ports.BrokerOperationID, result ope
351 // ConnectionID returns the identity assigned to this connection at accept.379 // ConnectionID returns the identity assigned to this connection at accept.
352 func (c *client) ConnectionID() ports.BrokerConnectionID { return c.scope.Connection }380 func (c *client) ConnectionID() ports.BrokerConnectionID { return c.scope.Connection }
353 381
382+// Done closes exactly once when this connection is terminal: the carriage
383+// failed, the broker retired it, or this side closed it. Err is stable
384+// afterwards.
385+func (c *client) Done() <-chan struct{} { return c.done }
386+
387+// Err returns this connection's terminal cause: nil for an orderly local Close,
388+// otherwise the failure that settled it. The cause is recorded exactly once and
389+// never overwritten, so it is stable once Done is closed.
390+func (c *client) Err() error {
391+ c.mu.Lock()
392+ defer c.mu.Unlock()
393+ return c.err
394+}
395+
354 // Snapshot returns the newest fully committed publication.396 // Snapshot returns the newest fully committed publication.
355 func (c *client) Snapshot() ports.BrokerSnapshot {397 func (c *client) Snapshot() ports.BrokerSnapshot {
356 c.mu.Lock()398 c.mu.Lock()
A internal/adapters/brokeripc/connector_test.go +140 -0
@@ -0,0 +1,140 @@
1+package brokeripc
2+
3+import (
4+ "context"
5+ "testing"
6+ "time"
7+
8+ "github.com/stretchr/testify/require"
9+
10+ "github.com/bnema/vev/internal/ports"
11+)
12+
13+// awaitDone waits for one broker service to become terminal.
14+func awaitDone(t *testing.T, service ports.BrokerService) {
15+ t.Helper()
16+ select {
17+ case <-service.Done():
18+ case <-time.After(5 * time.Second):
19+ t.Fatal("broker service did not become terminal")
20+ }
21+}
22+
23+// TestConnectorConnectReturnsIndependentService proves the ports.BrokerConnector
24+// implementation dials one connection per Connect and hands ownership to the
25+// caller.
26+func TestConnectorConnectReturnsIndependentService(t *testing.T) {
27+ e := startEndpoint(t, Config{})
28+ connector := NewConnector(e.path, Config{})
29+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
30+ defer cancel()
31+
32+ first, err := connector.Connect(ctx)
33+ require.NoError(t, err)
34+ require.NotNil(t, first)
35+
36+ second, err := connector.Connect(ctx)
37+ require.NoError(t, err)
38+ require.NotNil(t, second)
39+ require.NotEqual(t, first.ConnectionID(), second.ConnectionID(), "each Connect admits an independent connection")
40+
41+ // An orderly local Close reports no terminal cause and is stable.
42+ require.NoError(t, first.Close())
43+ awaitDone(t, first)
44+ require.NoError(t, first.Err())
45+ require.NoError(t, first.Err(), "Err is stable after Done")
46+ require.False(t, func() bool {
47+ select {
48+ case <-second.Done():
49+ return true
50+ default:
51+ return false
52+ }
53+ }(), "closing one connection must not close another")
54+
55+ require.NoError(t, second.Close())
56+ awaitDone(t, second)
57+}
58+
59+// TestConnectorReportsCarriageLoss proves the client service surfaces a broker
60+// loss through Done and Err while it holds no logical stream.
61+func TestConnectorReportsCarriageLoss(t *testing.T) {
62+ e := startEndpoint(t, Config{})
63+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
64+ defer cancel()
65+
66+ service, err := NewConnector(e.path, Config{}).Connect(ctx)
67+ require.NoError(t, err)
68+
69+ // The broker side is torn down: the client observes a non-nil terminal
70+ // cause instead of a silent close.
71+ require.NoError(t, e.accept().Close())
72+ awaitDone(t, service)
73+ require.Error(t, service.Err())
74+
75+ require.NoError(t, service.Close())
76+ require.Error(t, service.Err(), "the first terminal cause is never overwritten by a local Close")
77+}
78+
79+func TestConnectorRefusesInvalidConfiguration(t *testing.T) {
80+ var absent *Connector
81+ _, err := absent.Connect(context.Background())
82+ require.ErrorIs(t, err, ErrConfig)
83+
84+ _, err = NewConnector("", Config{}).Connect(context.Background())
85+ require.ErrorIs(t, err, ErrConfig)
86+}
87+
88+// TestConnectorCancellationInterruptsStalledRegistration proves the
89+// ports.BrokerConnector seam honors cancellation while a peer stalls in the
90+// registration exchange: Connect returns the caller's cancellation instead of
91+// waiting out the handshake budget. The supervisor's cancellation lifetime
92+// depends on exactly this contract.
93+func TestConnectorCancellationInterruptsStalledRegistration(t *testing.T) {
94+ path := testSocketPath(t)
95+ peer := startSilentBrokerPeer(t, path)
96+
97+ connector := NewConnector(path, Config{HandshakeTimeout: 30 * time.Second})
98+ ctx, cancel := context.WithCancel(context.Background())
99+ defer cancel()
100+ result := make(chan error, 1)
101+ go func() {
102+ _, err := connector.Connect(ctx)
103+ result <- err
104+ }()
105+ peer.awaitPreamble(t)
106+ cancel()
107+
108+ select {
109+ case err := <-result:
110+ require.ErrorIs(t, err, context.Canceled, "Connect must return the caller's cancellation, not wait out the handshake budget")
111+ case <-time.After(5 * time.Second):
112+ t.Fatal("Connect must return on cancellation instead of waiting out HandshakeTimeout")
113+ }
114+ peer.stop()
115+}
116+
117+// TestConnectorServiceOutlivesSetupContext pins the ports.BrokerConnector
118+// contract that a successful service is independent of the setup context:
119+// cancelling that context after Connect returns must not close or disturb the
120+// established connection.
121+func TestConnectorServiceOutlivesSetupContext(t *testing.T) {
122+ e := startEndpoint(t, Config{})
123+ ctx, cancel := context.WithCancel(context.Background())
124+ service, err := NewConnector(e.path, Config{}).Connect(ctx)
125+ require.NoError(t, err)
126+ cancel()
127+
128+ select {
129+ case <-service.Done():
130+ t.Fatal("cancelling the setup context must not close the established service")
131+ case <-time.After(100 * time.Millisecond):
132+ }
133+
134+ // The detached connection still serves its own scope and can be closed by
135+ // its owner, which reports an orderly end.
136+ require.NotEqual(t, ports.BrokerConnectionID{}, service.ConnectionID())
137+ require.NoError(t, service.Close())
138+ awaitDone(t, service)
139+ require.NoError(t, service.Err())
140+}
@@ -0,0 +1,140 @@
1+package brokeripc
2+
3+import (
4+ "context"
5+ "testing"
6+ "time"
7+
8+ "github.com/stretchr/testify/require"
9+
10+ "github.com/bnema/vev/internal/ports"
11+)
12+
13+// awaitDone waits for one broker service to become terminal.
14+func awaitDone(t *testing.T, service ports.BrokerService) {
15+ t.Helper()
16+ select {
17+ case <-service.Done():
18+ case <-time.After(5 * time.Second):
19+ t.Fatal("broker service did not become terminal")
20+ }
21+}
22+
23+// TestConnectorConnectReturnsIndependentService proves the ports.BrokerConnector
24+// implementation dials one connection per Connect and hands ownership to the
25+// caller.
26+func TestConnectorConnectReturnsIndependentService(t *testing.T) {
27+ e := startEndpoint(t, Config{})
28+ connector := NewConnector(e.path, Config{})
29+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
30+ defer cancel()
31+
32+ first, err := connector.Connect(ctx)
33+ require.NoError(t, err)
34+ require.NotNil(t, first)
35+
36+ second, err := connector.Connect(ctx)
37+ require.NoError(t, err)
38+ require.NotNil(t, second)
39+ require.NotEqual(t, first.ConnectionID(), second.ConnectionID(), "each Connect admits an independent connection")
40+
41+ // An orderly local Close reports no terminal cause and is stable.
42+ require.NoError(t, first.Close())
43+ awaitDone(t, first)
44+ require.NoError(t, first.Err())
45+ require.NoError(t, first.Err(), "Err is stable after Done")
46+ require.False(t, func() bool {
47+ select {
48+ case <-second.Done():
49+ return true
50+ default:
51+ return false
52+ }
53+ }(), "closing one connection must not close another")
54+
55+ require.NoError(t, second.Close())
56+ awaitDone(t, second)
57+}
58+
59+// TestConnectorReportsCarriageLoss proves the client service surfaces a broker
60+// loss through Done and Err while it holds no logical stream.
61+func TestConnectorReportsCarriageLoss(t *testing.T) {
62+ e := startEndpoint(t, Config{})
63+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
64+ defer cancel()
65+
66+ service, err := NewConnector(e.path, Config{}).Connect(ctx)
67+ require.NoError(t, err)
68+
69+ // The broker side is torn down: the client observes a non-nil terminal
70+ // cause instead of a silent close.
71+ require.NoError(t, e.accept().Close())
72+ awaitDone(t, service)
73+ require.Error(t, service.Err())
74+
75+ require.NoError(t, service.Close())
76+ require.Error(t, service.Err(), "the first terminal cause is never overwritten by a local Close")
77+}
78+
79+func TestConnectorRefusesInvalidConfiguration(t *testing.T) {
80+ var absent *Connector
81+ _, err := absent.Connect(context.Background())
82+ require.ErrorIs(t, err, ErrConfig)
83+
84+ _, err = NewConnector("", Config{}).Connect(context.Background())
85+ require.ErrorIs(t, err, ErrConfig)
86+}
87+
88+// TestConnectorCancellationInterruptsStalledRegistration proves the
89+// ports.BrokerConnector seam honors cancellation while a peer stalls in the
90+// registration exchange: Connect returns the caller's cancellation instead of
91+// waiting out the handshake budget. The supervisor's cancellation lifetime
92+// depends on exactly this contract.
93+func TestConnectorCancellationInterruptsStalledRegistration(t *testing.T) {
94+ path := testSocketPath(t)
95+ peer := startSilentBrokerPeer(t, path)
96+
97+ connector := NewConnector(path, Config{HandshakeTimeout: 30 * time.Second})
98+ ctx, cancel := context.WithCancel(context.Background())
99+ defer cancel()
100+ result := make(chan error, 1)
101+ go func() {
102+ _, err := connector.Connect(ctx)
103+ result <- err
104+ }()
105+ peer.awaitPreamble(t)
106+ cancel()
107+
108+ select {
109+ case err := <-result:
110+ require.ErrorIs(t, err, context.Canceled, "Connect must return the caller's cancellation, not wait out the handshake budget")
111+ case <-time.After(5 * time.Second):
112+ t.Fatal("Connect must return on cancellation instead of waiting out HandshakeTimeout")
113+ }
114+ peer.stop()
115+}
116+
117+// TestConnectorServiceOutlivesSetupContext pins the ports.BrokerConnector
118+// contract that a successful service is independent of the setup context:
119+// cancelling that context after Connect returns must not close or disturb the
120+// established connection.
121+func TestConnectorServiceOutlivesSetupContext(t *testing.T) {
122+ e := startEndpoint(t, Config{})
123+ ctx, cancel := context.WithCancel(context.Background())
124+ service, err := NewConnector(e.path, Config{}).Connect(ctx)
125+ require.NoError(t, err)
126+ cancel()
127+
128+ select {
129+ case <-service.Done():
130+ t.Fatal("cancelling the setup context must not close the established service")
131+ case <-time.After(100 * time.Millisecond):
132+ }
133+
134+ // The detached connection still serves its own scope and can be closed by
135+ // its owner, which reports an orderly end.
136+ require.NotEqual(t, ports.BrokerConnectionID{}, service.ConnectionID())
137+ require.NoError(t, service.Close())
138+ awaitDone(t, service)
139+ require.NoError(t, service.Err())
140+}
M internal/adapters/brokeripc/helpers_test.go +4 -0
@@ -241,6 +241,10 @@ type fakeCore struct {
241241
242242 func (c *fakeCore) ConnectionID() ports.BrokerConnectionID { return c.id }
243243
244+func (c *fakeCore) Done() <-chan struct{} { return c.done }
245+
246+func (c *fakeCore) Err() error { return nil }
247+
244248 func (c *fakeCore) Snapshot() ports.BrokerSnapshot { return c.hub.current() }
245249
246250 func (c *fakeCore) Subscribe() (ports.BrokerSubscription, error) { return c.hub.subscribe(), nil }
@@ -241,6 +241,10 @@ type fakeCore struct {
241 241
242 func (c *fakeCore) ConnectionID() ports.BrokerConnectionID { return c.id }242 func (c *fakeCore) ConnectionID() ports.BrokerConnectionID { return c.id }
243 243
244+func (c *fakeCore) Done() <-chan struct{} { return c.done }
245+
246+func (c *fakeCore) Err() error { return nil }
247+
244 func (c *fakeCore) Snapshot() ports.BrokerSnapshot { return c.hub.current() }248 func (c *fakeCore) Snapshot() ports.BrokerSnapshot { return c.hub.current() }
245 249
246 func (c *fakeCore) Subscribe() (ports.BrokerSubscription, error) { return c.hub.subscribe(), nil }250 func (c *fakeCore) Subscribe() (ports.BrokerSubscription, error) { return c.hub.subscribe(), nil }
M internal/adapters/brokeripc/server.go +18 -0
@@ -889,6 +889,24 @@ func orderlyDisconnect(err error) bool {
889889 // ConnectionID returns the identity assigned to this connection at accept.
890890 func (s *serverSession) ConnectionID() ports.BrokerConnectionID { return s.scope.Connection }
891891
892+// Done closes exactly once when this session is terminal: the carriage failed,
893+// the peer disconnected, or the owner closed it. Err is stable afterwards.
894+func (s *serverSession) Done() <-chan struct{} { return s.done }
895+
896+// Err returns the cause that settled this session, or nil for an orderly end
897+// (a peer disconnect, a registration timeout, or a local close). The underlying
898+// terminal cause is never overwritten, so the result is stable once Done is
899+// closed. A cause Close already reports as orderly is reported as nil here too,
900+// so a client that observes Done/Err never mistakes a peer's ordinary departure
901+// for a sandbox failure.
902+func (s *serverSession) Err() error {
903+ err := s.terminalErr()
904+ if orderlyDisconnect(err) {
905+ return nil
906+ }
907+ return err
908+}
909+
892910 // Snapshot returns the admitted core service's current publication.
893911 func (s *serverSession) Snapshot() ports.BrokerSnapshot { return s.core.Snapshot() }
894912
@@ -889,6 +889,24 @@ func orderlyDisconnect(err error) bool {
889 // ConnectionID returns the identity assigned to this connection at accept.889 // ConnectionID returns the identity assigned to this connection at accept.
890 func (s *serverSession) ConnectionID() ports.BrokerConnectionID { return s.scope.Connection }890 func (s *serverSession) ConnectionID() ports.BrokerConnectionID { return s.scope.Connection }
891 891
892+// Done closes exactly once when this session is terminal: the carriage failed,
893+// the peer disconnected, or the owner closed it. Err is stable afterwards.
894+func (s *serverSession) Done() <-chan struct{} { return s.done }
895+
896+// Err returns the cause that settled this session, or nil for an orderly end
897+// (a peer disconnect, a registration timeout, or a local close). The underlying
898+// terminal cause is never overwritten, so the result is stable once Done is
899+// closed. A cause Close already reports as orderly is reported as nil here too,
900+// so a client that observes Done/Err never mistakes a peer's ordinary departure
901+// for a sandbox failure.
902+func (s *serverSession) Err() error {
903+ err := s.terminalErr()
904+ if orderlyDisconnect(err) {
905+ return nil
906+ }
907+ return err
908+}
909+
892 // Snapshot returns the admitted core service's current publication.910 // Snapshot returns the admitted core service's current publication.
893 func (s *serverSession) Snapshot() ports.BrokerSnapshot { return s.core.Snapshot() }911 func (s *serverSession) Snapshot() ports.BrokerSnapshot { return s.core.Snapshot() }
894 912
A internal/adapters/brokeripc/session_terminal_test.go +79 -0
@@ -0,0 +1,79 @@
1+package brokeripc
2+
3+import (
4+ "net"
5+ "testing"
6+ "time"
7+
8+ "github.com/stretchr/testify/require"
9+
10+ "github.com/bnema/vev/internal/adapters/brokerwire"
11+ "github.com/bnema/vev/internal/adapters/ipc"
12+ "github.com/bnema/vev/internal/ports"
13+ "github.com/bnema/vev/internal/protocol/wire"
14+)
15+
16+// newRunningSession starts one registered server session over a real net.Pipe
17+// carriage so a test can drive its terminal outcome from the peer side.
18+func newRunningSession(t *testing.T, epoch ports.BrokerEpoch) (*serverSession, net.Conn) {
19+ t.Helper()
20+ serverConn, peerConn := net.Pipe()
21+ t.Cleanup(func() { _ = peerConn.Close() })
22+ transport, ok := ipc.NewTransport(serverConn).(wire.BoundedTransport)
23+ require.True(t, ok, "the IPC carriage must be a bounded transport")
24+ session, err := newServerSession(epoch, transport, brokerwire.DefaultCeilings(), newTestCore(epoch), Config{}, func() {}, time.Time{})
25+ require.NoError(t, err)
26+ require.NoError(t, session.conn.Register())
27+ go session.run()
28+ return session, peerConn
29+}
30+
31+// awaitSessionDone waits for one session to reach its terminal state.
32+func awaitSessionDone(t *testing.T, session *serverSession) {
33+ t.Helper()
34+ select {
35+ case <-session.Done():
36+ case <-time.After(5 * time.Second):
37+ t.Fatal("session did not become terminal")
38+ }
39+}
40+
41+// TestSessionErrReportsOrderlyEndAsNil pins the server session Err contract: a
42+// peer disconnect and a local Close are orderly ends and report nil, while a
43+// protocol failure reports the typed cause that settled the session. Done is
44+// closed and Err stable in every case.
45+func TestSessionErrReportsOrderlyEndAsNil(t *testing.T) {
46+ t.Run("peer EOF is orderly", func(t *testing.T) {
47+ session, peer := newRunningSession(t, 0x61)
48+ require.NoError(t, peer.Close())
49+ awaitSessionDone(t, session)
50+
51+ // The underlying cause is the peer's EOF, but Err presents it as an
52+ // orderly end so a watcher never mistakes a departure for a failure.
53+ require.Error(t, session.terminalErr(), "the peer EOF remains the recorded internal cause")
54+ require.True(t, orderlyDisconnect(session.terminalErr()))
55+ require.NoError(t, session.Err())
56+ require.NoError(t, session.Err(), "Err is stable after Done")
57+ require.NoError(t, session.Close())
58+ })
59+
60+ t.Run("local Close is orderly", func(t *testing.T) {
61+ session, _ := newRunningSession(t, 0x62)
62+ require.NoError(t, session.Close())
63+ awaitSessionDone(t, session)
64+ require.NoError(t, session.Err())
65+ })
66+
67+ t.Run("protocol failure is typed", func(t *testing.T) {
68+ session, peer := newRunningSession(t, 0x63)
69+ raw := &rawPeer{conn: peer, ceilings: brokerwire.DefaultCeilings()}
70+ // A second Register is a protocol-order violation and settles the
71+ // connection.
72+ raw.send(t, brokerwire.Register{})
73+ awaitSessionDone(t, session)
74+
75+ require.Error(t, session.Err())
76+ require.ErrorIs(t, session.Err(), ErrProtocol)
77+ require.Error(t, session.Close(), "a protocol failure is not an orderly end")
78+ })
79+}
@@ -0,0 +1,79 @@
1+package brokeripc
2+
3+import (
4+ "net"
5+ "testing"
6+ "time"
7+
8+ "github.com/stretchr/testify/require"
9+
10+ "github.com/bnema/vev/internal/adapters/brokerwire"
11+ "github.com/bnema/vev/internal/adapters/ipc"
12+ "github.com/bnema/vev/internal/ports"
13+ "github.com/bnema/vev/internal/protocol/wire"
14+)
15+
16+// newRunningSession starts one registered server session over a real net.Pipe
17+// carriage so a test can drive its terminal outcome from the peer side.
18+func newRunningSession(t *testing.T, epoch ports.BrokerEpoch) (*serverSession, net.Conn) {
19+ t.Helper()
20+ serverConn, peerConn := net.Pipe()
21+ t.Cleanup(func() { _ = peerConn.Close() })
22+ transport, ok := ipc.NewTransport(serverConn).(wire.BoundedTransport)
23+ require.True(t, ok, "the IPC carriage must be a bounded transport")
24+ session, err := newServerSession(epoch, transport, brokerwire.DefaultCeilings(), newTestCore(epoch), Config{}, func() {}, time.Time{})
25+ require.NoError(t, err)
26+ require.NoError(t, session.conn.Register())
27+ go session.run()
28+ return session, peerConn
29+}
30+
31+// awaitSessionDone waits for one session to reach its terminal state.
32+func awaitSessionDone(t *testing.T, session *serverSession) {
33+ t.Helper()
34+ select {
35+ case <-session.Done():
36+ case <-time.After(5 * time.Second):
37+ t.Fatal("session did not become terminal")
38+ }
39+}
40+
41+// TestSessionErrReportsOrderlyEndAsNil pins the server session Err contract: a
42+// peer disconnect and a local Close are orderly ends and report nil, while a
43+// protocol failure reports the typed cause that settled the session. Done is
44+// closed and Err stable in every case.
45+func TestSessionErrReportsOrderlyEndAsNil(t *testing.T) {
46+ t.Run("peer EOF is orderly", func(t *testing.T) {
47+ session, peer := newRunningSession(t, 0x61)
48+ require.NoError(t, peer.Close())
49+ awaitSessionDone(t, session)
50+
51+ // The underlying cause is the peer's EOF, but Err presents it as an
52+ // orderly end so a watcher never mistakes a departure for a failure.
53+ require.Error(t, session.terminalErr(), "the peer EOF remains the recorded internal cause")
54+ require.True(t, orderlyDisconnect(session.terminalErr()))
55+ require.NoError(t, session.Err())
56+ require.NoError(t, session.Err(), "Err is stable after Done")
57+ require.NoError(t, session.Close())
58+ })
59+
60+ t.Run("local Close is orderly", func(t *testing.T) {
61+ session, _ := newRunningSession(t, 0x62)
62+ require.NoError(t, session.Close())
63+ awaitSessionDone(t, session)
64+ require.NoError(t, session.Err())
65+ })
66+
67+ t.Run("protocol failure is typed", func(t *testing.T) {
68+ session, peer := newRunningSession(t, 0x63)
69+ raw := &rawPeer{conn: peer, ceilings: brokerwire.DefaultCeilings()}
70+ // A second Register is a protocol-order violation and settles the
71+ // connection.
72+ raw.send(t, brokerwire.Register{})
73+ awaitSessionDone(t, session)
74+
75+ require.Error(t, session.Err())
76+ require.ErrorIs(t, session.Err(), ErrProtocol)
77+ require.Error(t, session.Close(), "a protocol failure is not an orderly end")
78+ })
79+}
M internal/ports/broker.go +33 -0
@@ -611,6 +611,14 @@ type BrokerService interface {
611611 // ConnectionID returns the ID assigned when this client connection was
612612 // accepted. Clients carry it on stream operations to fence stale requests.
613613 ConnectionID() BrokerConnectionID
614+ // Done closes exactly once when this connection is terminal: the broker
615+ // retired it, the carriage failed, or the client closed it locally. Err is
616+ // stable afterwards. A client observes Done/Err to attribute a broker loss
617+ // to the connection even while it holds no logical stream.
618+ Done() <-chan struct{}
619+ // Err returns the terminal cause, stable once Done is closed. It is nil for
620+ // an orderly local Close and non-nil for a broker-side loss or failure.
621+ Err() error
614622 Snapshot() BrokerSnapshot
615623 Subscribe() (BrokerSubscription, error)
616624 OpenStream(ctx context.Context, request BrokerOpenStreamRequest) (BrokerLogicalConnection, error)
@@ -680,6 +688,31 @@ type BrokerEndpointConnector interface {
680688 Connect(ctx context.Context, endpoint BrokerResolvedEndpoint) (BrokerPhysicalConnection, error)
681689 }
682690
691+// BrokerConnector establishes the client-facing broker connection: it connects
692+// to the per-user broker endpoint and returns the connection-scoped
693+// BrokerService for exactly that accepted connection. It is the client
694+// process's admission seam, distinct from BrokerEndpointConnector (which opens
695+// a pooled physical transport for one endpoint inside the broker) and from
696+// BrokerAuthority (which admits a listener-side accepted connection).
697+//
698+// Connect must honor cancellation: a cancelled connect returns promptly and
699+// leaves no service behind, so a client that abandons an attempt never leaks a
700+// connection. Each successful Connect is independent and the caller owns Close
701+// on the returned service; a connector retains no connection state of its own.
702+// Semantic rejection is reported as a typed BrokerError and cancellation as a
703+// context error.
704+//
705+// A successful service is independent of the setup context: Connect must be
706+// given a context that bounds only the setup, and once Connect returns a
707+// service the caller may cancel or let that context expire without closing,
708+// interrupting, or otherwise disturbing the established connection. The
709+// returned service's lifetime is bounded solely by its own Close and by the
710+// broker, so a caller that abandons the setup context (a bounded attempt) still
711+// owns a live connection it must Close.
712+type BrokerConnector interface {
713+ Connect(ctx context.Context) (BrokerService, error)
714+}
715+
683716 // BrokerAuthority admits one accepted client connection to the broker core
684717 // and returns the BrokerService scoped to exactly that connection. It is the
685718 // admission seam between the broker listener adapter and the broker use case:
@@ -611,6 +611,14 @@ type BrokerService interface {
611 // ConnectionID returns the ID assigned when this client connection was611 // ConnectionID returns the ID assigned when this client connection was
612 // accepted. Clients carry it on stream operations to fence stale requests.612 // accepted. Clients carry it on stream operations to fence stale requests.
613 ConnectionID() BrokerConnectionID613 ConnectionID() BrokerConnectionID
614+ // Done closes exactly once when this connection is terminal: the broker
615+ // retired it, the carriage failed, or the client closed it locally. Err is
616+ // stable afterwards. A client observes Done/Err to attribute a broker loss
617+ // to the connection even while it holds no logical stream.
618+ Done() <-chan struct{}
619+ // Err returns the terminal cause, stable once Done is closed. It is nil for
620+ // an orderly local Close and non-nil for a broker-side loss or failure.
621+ Err() error
614 Snapshot() BrokerSnapshot622 Snapshot() BrokerSnapshot
615 Subscribe() (BrokerSubscription, error)623 Subscribe() (BrokerSubscription, error)
616 OpenStream(ctx context.Context, request BrokerOpenStreamRequest) (BrokerLogicalConnection, error)624 OpenStream(ctx context.Context, request BrokerOpenStreamRequest) (BrokerLogicalConnection, error)
@@ -680,6 +688,31 @@ type BrokerEndpointConnector interface {
680 Connect(ctx context.Context, endpoint BrokerResolvedEndpoint) (BrokerPhysicalConnection, error)688 Connect(ctx context.Context, endpoint BrokerResolvedEndpoint) (BrokerPhysicalConnection, error)
681 }689 }
682 690
691+// BrokerConnector establishes the client-facing broker connection: it connects
692+// to the per-user broker endpoint and returns the connection-scoped
693+// BrokerService for exactly that accepted connection. It is the client
694+// process's admission seam, distinct from BrokerEndpointConnector (which opens
695+// a pooled physical transport for one endpoint inside the broker) and from
696+// BrokerAuthority (which admits a listener-side accepted connection).
697+//
698+// Connect must honor cancellation: a cancelled connect returns promptly and
699+// leaves no service behind, so a client that abandons an attempt never leaks a
700+// connection. Each successful Connect is independent and the caller owns Close
701+// on the returned service; a connector retains no connection state of its own.
702+// Semantic rejection is reported as a typed BrokerError and cancellation as a
703+// context error.
704+//
705+// A successful service is independent of the setup context: Connect must be
706+// given a context that bounds only the setup, and once Connect returns a
707+// service the caller may cancel or let that context expire without closing,
708+// interrupting, or otherwise disturbing the established connection. The
709+// returned service's lifetime is bounded solely by its own Close and by the
710+// broker, so a caller that abandons the setup context (a bounded attempt) still
711+// owns a live connection it must Close.
712+type BrokerConnector interface {
713+ Connect(ctx context.Context) (BrokerService, error)
714+}
715+
683 // BrokerAuthority admits one accepted client connection to the broker core716 // BrokerAuthority admits one accepted client connection to the broker core
684 // and returns the BrokerService scoped to exactly that connection. It is the717 // and returns the BrokerService scoped to exactly that connection. It is the
685 // admission seam between the broker listener adapter and the broker use case:718 // admission seam between the broker listener adapter and the broker use case:
M internal/ports/broker_test.go +1 -0
@@ -535,6 +535,7 @@ func TestBrokerInterfacesArePorts(t *testing.T) {
535535 "BrokerHostProbe": reflect.TypeOf((*BrokerHostProbe)(nil)).Elem(),
536536 "BrokerPhysicalConnection": reflect.TypeOf((*BrokerPhysicalConnection)(nil)).Elem(),
537537 "BrokerEndpointConnector": reflect.TypeOf((*BrokerEndpointConnector)(nil)).Elem(),
538+ "BrokerConnector": reflect.TypeOf((*BrokerConnector)(nil)).Elem(),
538539 "BrokerAuthority": reflect.TypeOf((*BrokerAuthority)(nil)).Elem(),
539540 "BrokerListener": reflect.TypeOf((*BrokerListener)(nil)).Elem(),
540541 "BrokerSubscription": reflect.TypeOf((*BrokerSubscription)(nil)).Elem(),
@@ -535,6 +535,7 @@ func TestBrokerInterfacesArePorts(t *testing.T) {
535 "BrokerHostProbe": reflect.TypeOf((*BrokerHostProbe)(nil)).Elem(),535 "BrokerHostProbe": reflect.TypeOf((*BrokerHostProbe)(nil)).Elem(),
536 "BrokerPhysicalConnection": reflect.TypeOf((*BrokerPhysicalConnection)(nil)).Elem(),536 "BrokerPhysicalConnection": reflect.TypeOf((*BrokerPhysicalConnection)(nil)).Elem(),
537 "BrokerEndpointConnector": reflect.TypeOf((*BrokerEndpointConnector)(nil)).Elem(),537 "BrokerEndpointConnector": reflect.TypeOf((*BrokerEndpointConnector)(nil)).Elem(),
538+ "BrokerConnector": reflect.TypeOf((*BrokerConnector)(nil)).Elem(),
538 "BrokerAuthority": reflect.TypeOf((*BrokerAuthority)(nil)).Elem(),539 "BrokerAuthority": reflect.TypeOf((*BrokerAuthority)(nil)).Elem(),
539 "BrokerListener": reflect.TypeOf((*BrokerListener)(nil)).Elem(),540 "BrokerListener": reflect.TypeOf((*BrokerListener)(nil)).Elem(),
540 "BrokerSubscription": reflect.TypeOf((*BrokerSubscription)(nil)).Elem(),541 "BrokerSubscription": reflect.TypeOf((*BrokerSubscription)(nil)).Elem(),
M internal/ports/mocks/broker_mock.go +179 -0
@@ -349,6 +349,96 @@ func (_c *MockBrokerService_ConnectionID_Call) RunAndReturn(run func() ports.Bro
349349 return _c
350350 }
351351
352+// Done provides a mock function for the type MockBrokerService
353+func (_mock *MockBrokerService) Done() <-chan struct{} {
354+ ret := _mock.Called()
355+
356+ if len(ret) == 0 {
357+ panic("no return value specified for Done")
358+ }
359+
360+ var r0 <-chan struct{}
361+ if returnFunc, ok := ret.Get(0).(func() <-chan struct{}); ok {
362+ r0 = returnFunc()
363+ } else {
364+ if ret.Get(0) != nil {
365+ r0 = ret.Get(0).(<-chan struct{})
366+ }
367+ }
368+ return r0
369+}
370+
371+// MockBrokerService_Done_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Done'
372+type MockBrokerService_Done_Call struct {
373+ *mock.Call
374+}
375+
376+// Done is a helper method to define mock.On call
377+func (_e *MockBrokerService_Expecter) Done() *MockBrokerService_Done_Call {
378+ return &MockBrokerService_Done_Call{Call: _e.mock.On("Done")}
379+}
380+
381+func (_c *MockBrokerService_Done_Call) Run(run func()) *MockBrokerService_Done_Call {
382+ _c.Call.Run(func(args mock.Arguments) {
383+ run()
384+ })
385+ return _c
386+}
387+
388+func (_c *MockBrokerService_Done_Call) Return(valCh <-chan struct{}) *MockBrokerService_Done_Call {
389+ _c.Call.Return(valCh)
390+ return _c
391+}
392+
393+func (_c *MockBrokerService_Done_Call) RunAndReturn(run func() <-chan struct{}) *MockBrokerService_Done_Call {
394+ _c.Call.Return(run)
395+ return _c
396+}
397+
398+// Err provides a mock function for the type MockBrokerService
399+func (_mock *MockBrokerService) Err() error {
400+ ret := _mock.Called()
401+
402+ if len(ret) == 0 {
403+ panic("no return value specified for Err")
404+ }
405+
406+ var r0 error
407+ if returnFunc, ok := ret.Get(0).(func() error); ok {
408+ r0 = returnFunc()
409+ } else {
410+ r0 = ret.Error(0)
411+ }
412+ return r0
413+}
414+
415+// MockBrokerService_Err_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Err'
416+type MockBrokerService_Err_Call struct {
417+ *mock.Call
418+}
419+
420+// Err is a helper method to define mock.On call
421+func (_e *MockBrokerService_Expecter) Err() *MockBrokerService_Err_Call {
422+ return &MockBrokerService_Err_Call{Call: _e.mock.On("Err")}
423+}
424+
425+func (_c *MockBrokerService_Err_Call) Run(run func()) *MockBrokerService_Err_Call {
426+ _c.Call.Run(func(args mock.Arguments) {
427+ run()
428+ })
429+ return _c
430+}
431+
432+func (_c *MockBrokerService_Err_Call) Return(err error) *MockBrokerService_Err_Call {
433+ _c.Call.Return(err)
434+ return _c
435+}
436+
437+func (_c *MockBrokerService_Err_Call) RunAndReturn(run func() error) *MockBrokerService_Err_Call {
438+ _c.Call.Return(run)
439+ return _c
440+}
441+
352442 // OpenStream provides a mock function for the type MockBrokerService
353443 func (_mock *MockBrokerService) OpenStream(ctx context.Context, request ports.BrokerOpenStreamRequest) (ports.BrokerLogicalConnection, error) {
354444 ret := _mock.Called(ctx, request)
@@ -1348,6 +1438,95 @@ func (_c *MockBrokerEndpointConnector_Connect_Call) RunAndReturn(run func(ctx co
13481438 return _c
13491439 }
13501440
1441+// NewMockBrokerConnector creates a new instance of MockBrokerConnector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
1442+// The first argument is typically a *testing.T value.
1443+func NewMockBrokerConnector(t interface {
1444+ mock.TestingT
1445+ Cleanup(func())
1446+}) *MockBrokerConnector {
1447+ mock := &MockBrokerConnector{}
1448+ mock.Mock.Test(t)
1449+
1450+ t.Cleanup(func() { mock.AssertExpectations(t) })
1451+
1452+ return mock
1453+}
1454+
1455+// MockBrokerConnector is an autogenerated mock type for the BrokerConnector type
1456+type MockBrokerConnector struct {
1457+ mock.Mock
1458+}
1459+
1460+type MockBrokerConnector_Expecter struct {
1461+ mock *mock.Mock
1462+}
1463+
1464+func (_m *MockBrokerConnector) EXPECT() *MockBrokerConnector_Expecter {
1465+ return &MockBrokerConnector_Expecter{mock: &_m.Mock}
1466+}
1467+
1468+// Connect provides a mock function for the type MockBrokerConnector
1469+func (_mock *MockBrokerConnector) Connect(ctx context.Context) (ports.BrokerService, error) {
1470+ ret := _mock.Called(ctx)
1471+
1472+ if len(ret) == 0 {
1473+ panic("no return value specified for Connect")
1474+ }
1475+
1476+ var r0 ports.BrokerService
1477+ var r1 error
1478+ if returnFunc, ok := ret.Get(0).(func(context.Context) (ports.BrokerService, error)); ok {
1479+ return returnFunc(ctx)
1480+ }
1481+ if returnFunc, ok := ret.Get(0).(func(context.Context) ports.BrokerService); ok {
1482+ r0 = returnFunc(ctx)
1483+ } else {
1484+ if ret.Get(0) != nil {
1485+ r0 = ret.Get(0).(ports.BrokerService)
1486+ }
1487+ }
1488+ if returnFunc, ok := ret.Get(1).(func(context.Context) error); ok {
1489+ r1 = returnFunc(ctx)
1490+ } else {
1491+ r1 = ret.Error(1)
1492+ }
1493+ return r0, r1
1494+}
1495+
1496+// MockBrokerConnector_Connect_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Connect'
1497+type MockBrokerConnector_Connect_Call struct {
1498+ *mock.Call
1499+}
1500+
1501+// Connect is a helper method to define mock.On call
1502+// - ctx context.Context
1503+func (_e *MockBrokerConnector_Expecter) Connect(ctx any) *MockBrokerConnector_Connect_Call {
1504+ return &MockBrokerConnector_Connect_Call{Call: _e.mock.On("Connect", ctx)}
1505+}
1506+
1507+func (_c *MockBrokerConnector_Connect_Call) Run(run func(ctx context.Context)) *MockBrokerConnector_Connect_Call {
1508+ _c.Call.Run(func(args mock.Arguments) {
1509+ var arg0 context.Context
1510+ if args[0] != nil {
1511+ arg0 = args[0].(context.Context)
1512+ }
1513+ run(
1514+ arg0,
1515+ )
1516+ })
1517+ return _c
1518+}
1519+
1520+func (_c *MockBrokerConnector_Connect_Call) Return(brokerService ports.BrokerService, err error) *MockBrokerConnector_Connect_Call {
1521+ _c.Call.Return(brokerService, err)
1522+ return _c
1523+}
1524+
1525+func (_c *MockBrokerConnector_Connect_Call) RunAndReturn(run func(ctx context.Context) (ports.BrokerService, error)) *MockBrokerConnector_Connect_Call {
1526+ _c.Call.Return(run)
1527+ return _c
1528+}
1529+
13511530 // NewMockBrokerAuthority creates a new instance of MockBrokerAuthority. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
13521531 // The first argument is typically a *testing.T value.
13531532 func NewMockBrokerAuthority(t interface {
@@ -349,6 +349,96 @@ func (_c *MockBrokerService_ConnectionID_Call) RunAndReturn(run func() ports.Bro
349 return _c349 return _c
350 }350 }
351 351
352+// Done provides a mock function for the type MockBrokerService
353+func (_mock *MockBrokerService) Done() <-chan struct{} {
354+ ret := _mock.Called()
355+
356+ if len(ret) == 0 {
357+ panic("no return value specified for Done")
358+ }
359+
360+ var r0 <-chan struct{}
361+ if returnFunc, ok := ret.Get(0).(func() <-chan struct{}); ok {
362+ r0 = returnFunc()
363+ } else {
364+ if ret.Get(0) != nil {
365+ r0 = ret.Get(0).(<-chan struct{})
366+ }
367+ }
368+ return r0
369+}
370+
371+// MockBrokerService_Done_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Done'
372+type MockBrokerService_Done_Call struct {
373+ *mock.Call
374+}
375+
376+// Done is a helper method to define mock.On call
377+func (_e *MockBrokerService_Expecter) Done() *MockBrokerService_Done_Call {
378+ return &MockBrokerService_Done_Call{Call: _e.mock.On("Done")}
379+}
380+
381+func (_c *MockBrokerService_Done_Call) Run(run func()) *MockBrokerService_Done_Call {
382+ _c.Call.Run(func(args mock.Arguments) {
383+ run()
384+ })
385+ return _c
386+}
387+
388+func (_c *MockBrokerService_Done_Call) Return(valCh <-chan struct{}) *MockBrokerService_Done_Call {
389+ _c.Call.Return(valCh)
390+ return _c
391+}
392+
393+func (_c *MockBrokerService_Done_Call) RunAndReturn(run func() <-chan struct{}) *MockBrokerService_Done_Call {
394+ _c.Call.Return(run)
395+ return _c
396+}
397+
398+// Err provides a mock function for the type MockBrokerService
399+func (_mock *MockBrokerService) Err() error {
400+ ret := _mock.Called()
401+
402+ if len(ret) == 0 {
403+ panic("no return value specified for Err")
404+ }
405+
406+ var r0 error
407+ if returnFunc, ok := ret.Get(0).(func() error); ok {
408+ r0 = returnFunc()
409+ } else {
410+ r0 = ret.Error(0)
411+ }
412+ return r0
413+}
414+
415+// MockBrokerService_Err_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Err'
416+type MockBrokerService_Err_Call struct {
417+ *mock.Call
418+}
419+
420+// Err is a helper method to define mock.On call
421+func (_e *MockBrokerService_Expecter) Err() *MockBrokerService_Err_Call {
422+ return &MockBrokerService_Err_Call{Call: _e.mock.On("Err")}
423+}
424+
425+func (_c *MockBrokerService_Err_Call) Run(run func()) *MockBrokerService_Err_Call {
426+ _c.Call.Run(func(args mock.Arguments) {
427+ run()
428+ })
429+ return _c
430+}
431+
432+func (_c *MockBrokerService_Err_Call) Return(err error) *MockBrokerService_Err_Call {
433+ _c.Call.Return(err)
434+ return _c
435+}
436+
437+func (_c *MockBrokerService_Err_Call) RunAndReturn(run func() error) *MockBrokerService_Err_Call {
438+ _c.Call.Return(run)
439+ return _c
440+}
441+
352 // OpenStream provides a mock function for the type MockBrokerService442 // OpenStream provides a mock function for the type MockBrokerService
353 func (_mock *MockBrokerService) OpenStream(ctx context.Context, request ports.BrokerOpenStreamRequest) (ports.BrokerLogicalConnection, error) {443 func (_mock *MockBrokerService) OpenStream(ctx context.Context, request ports.BrokerOpenStreamRequest) (ports.BrokerLogicalConnection, error) {
354 ret := _mock.Called(ctx, request)444 ret := _mock.Called(ctx, request)
@@ -1348,6 +1438,95 @@ func (_c *MockBrokerEndpointConnector_Connect_Call) RunAndReturn(run func(ctx co
1348 return _c1438 return _c
1349 }1439 }
1350 1440
1441+// NewMockBrokerConnector creates a new instance of MockBrokerConnector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
1442+// The first argument is typically a *testing.T value.
1443+func NewMockBrokerConnector(t interface {
1444+ mock.TestingT
1445+ Cleanup(func())
1446+}) *MockBrokerConnector {
1447+ mock := &MockBrokerConnector{}
1448+ mock.Mock.Test(t)
1449+
1450+ t.Cleanup(func() { mock.AssertExpectations(t) })
1451+
1452+ return mock
1453+}
1454+
1455+// MockBrokerConnector is an autogenerated mock type for the BrokerConnector type
1456+type MockBrokerConnector struct {
1457+ mock.Mock
1458+}
1459+
1460+type MockBrokerConnector_Expecter struct {
1461+ mock *mock.Mock
1462+}
1463+
1464+func (_m *MockBrokerConnector) EXPECT() *MockBrokerConnector_Expecter {
1465+ return &MockBrokerConnector_Expecter{mock: &_m.Mock}
1466+}
1467+
1468+// Connect provides a mock function for the type MockBrokerConnector
1469+func (_mock *MockBrokerConnector) Connect(ctx context.Context) (ports.BrokerService, error) {
1470+ ret := _mock.Called(ctx)
1471+
1472+ if len(ret) == 0 {
1473+ panic("no return value specified for Connect")
1474+ }
1475+
1476+ var r0 ports.BrokerService
1477+ var r1 error
1478+ if returnFunc, ok := ret.Get(0).(func(context.Context) (ports.BrokerService, error)); ok {
1479+ return returnFunc(ctx)
1480+ }
1481+ if returnFunc, ok := ret.Get(0).(func(context.Context) ports.BrokerService); ok {
1482+ r0 = returnFunc(ctx)
1483+ } else {
1484+ if ret.Get(0) != nil {
1485+ r0 = ret.Get(0).(ports.BrokerService)
1486+ }
1487+ }
1488+ if returnFunc, ok := ret.Get(1).(func(context.Context) error); ok {
1489+ r1 = returnFunc(ctx)
1490+ } else {
1491+ r1 = ret.Error(1)
1492+ }
1493+ return r0, r1
1494+}
1495+
1496+// MockBrokerConnector_Connect_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Connect'
1497+type MockBrokerConnector_Connect_Call struct {
1498+ *mock.Call
1499+}
1500+
1501+// Connect is a helper method to define mock.On call
1502+// - ctx context.Context
1503+func (_e *MockBrokerConnector_Expecter) Connect(ctx any) *MockBrokerConnector_Connect_Call {
1504+ return &MockBrokerConnector_Connect_Call{Call: _e.mock.On("Connect", ctx)}
1505+}
1506+
1507+func (_c *MockBrokerConnector_Connect_Call) Run(run func(ctx context.Context)) *MockBrokerConnector_Connect_Call {
1508+ _c.Call.Run(func(args mock.Arguments) {
1509+ var arg0 context.Context
1510+ if args[0] != nil {
1511+ arg0 = args[0].(context.Context)
1512+ }
1513+ run(
1514+ arg0,
1515+ )
1516+ })
1517+ return _c
1518+}
1519+
1520+func (_c *MockBrokerConnector_Connect_Call) Return(brokerService ports.BrokerService, err error) *MockBrokerConnector_Connect_Call {
1521+ _c.Call.Return(brokerService, err)
1522+ return _c
1523+}
1524+
1525+func (_c *MockBrokerConnector_Connect_Call) RunAndReturn(run func(ctx context.Context) (ports.BrokerService, error)) *MockBrokerConnector_Connect_Call {
1526+ _c.Call.Return(run)
1527+ return _c
1528+}
1529+
1351 // NewMockBrokerAuthority creates a new instance of MockBrokerAuthority. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.1530 // NewMockBrokerAuthority creates a new instance of MockBrokerAuthority. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
1352 // The first argument is typically a *testing.T value.1531 // The first argument is typically a *testing.T value.
1353 func NewMockBrokerAuthority(t interface {1532 func NewMockBrokerAuthority(t interface {
M internal/usecase/broker/service.go +52 -1
@@ -133,22 +133,66 @@ type Service struct {
133133 wg sync.WaitGroup
134134
135135 closeOnce sync.Once
136+
137+ // done closes exactly once when this connection is terminal, either because
138+ // the broker root shut down or because Close settled it locally. Err is
139+ // stable afterwards (nil for an orderly local Close). stopRoot detaches the
140+ // root watcher once Close has committed so a closed connection never leaves
141+ // a root callback behind.
142+ done chan struct{}
143+ stopRoot func() bool
144+ termOnce sync.Once
145+ termErr error
136146 }
137147
138148 var _ ports.BrokerService = (*Service)(nil)
139149
140150 func newService(epoch ports.BrokerEpoch, id ports.BrokerConnectionID, registry *Registry, pool *Pool, supervisor *Supervisor, lease *Lease) *Service {
141151 ctx, cancel := context.WithCancel(supervisor.RootContext())
142- return &Service{
152+ s := &Service{
143153 epoch: epoch, id: id, registry: registry, pool: pool, supervisor: supervisor, lease: lease,
144154 ctx: ctx, cancel: cancel, subs: make(map[*serviceSubscription]struct{}),
155+ done: make(chan struct{}),
145156 }
157+ // Broker shutdown ends every admitted connection. The watcher records the
158+ // loss as the terminal cause unless a local Close has already settled the
159+ // connection, so a connection closed by its owner reports nil and one lost
160+ // to a broker shutdown reports the typed loss.
161+ s.stopRoot = context.AfterFunc(supervisor.RootContext(), func() {
162+ s.terminalize(ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "broker shutdown", Cause: context.Canceled})
163+ })
164+ return s
146165 }
147166
148167 // ConnectionID returns the pool identity assigned at admission. Clients carry
149168 // it on stream operations so a stale request can be fenced.
150169 func (s *Service) ConnectionID() ports.BrokerConnectionID { return s.id }
151170
171+// Done closes exactly once when this connection is terminal: the broker root
172+// shut down or Close settled it locally. Err is stable afterwards.
173+func (s *Service) Done() <-chan struct{} { return s.done }
174+
175+// Err returns this connection's terminal cause: nil for an orderly local Close,
176+// otherwise the broker-side loss that settled it. The cause is recorded exactly
177+// once and never overwritten, so it is stable once Done is closed.
178+func (s *Service) Err() error {
179+ s.mu.Lock()
180+ defer s.mu.Unlock()
181+ return s.termErr
182+}
183+
184+// terminalize records the terminal cause and closes Done exactly once. The
185+// first cause wins, so a race between a broker-root loss and a local Close
186+// yields one stable, observable outcome.
187+func (s *Service) terminalize(err error) {
188+ s.termOnce.Do(func() {
189+ s.mu.Lock()
190+ s.termErr = err
191+ s.mu.Unlock()
192+ close(s.done)
193+ })
194+}
195+
152196 // Snapshot delegates to the registry's current immutable publication.
153197 func (s *Service) Snapshot() ports.BrokerSnapshot { return s.registry.Snapshot() }
154198
@@ -252,6 +296,13 @@ func (s *Service) Close() error {
252296 }
253297 s.pool.CloseClient(s.id)
254298 s.lease.Release()
299+ // The connection is locally and orderly closed: detach the broker-root
300+ // watcher so it can neither fire later nor overwrite the clean outcome,
301+ // then publish the terminal state.
302+ if s.stopRoot != nil {
303+ s.stopRoot()
304+ }
305+ s.terminalize(nil)
255306 })
256307 return nil
257308 }
@@ -133,22 +133,66 @@ type Service struct {
133 wg sync.WaitGroup133 wg sync.WaitGroup
134 134
135 closeOnce sync.Once135 closeOnce sync.Once
136+
137+ // done closes exactly once when this connection is terminal, either because
138+ // the broker root shut down or because Close settled it locally. Err is
139+ // stable afterwards (nil for an orderly local Close). stopRoot detaches the
140+ // root watcher once Close has committed so a closed connection never leaves
141+ // a root callback behind.
142+ done chan struct{}
143+ stopRoot func() bool
144+ termOnce sync.Once
145+ termErr error
136 }146 }
137 147
138 var _ ports.BrokerService = (*Service)(nil)148 var _ ports.BrokerService = (*Service)(nil)
139 149
140 func newService(epoch ports.BrokerEpoch, id ports.BrokerConnectionID, registry *Registry, pool *Pool, supervisor *Supervisor, lease *Lease) *Service {150 func newService(epoch ports.BrokerEpoch, id ports.BrokerConnectionID, registry *Registry, pool *Pool, supervisor *Supervisor, lease *Lease) *Service {
141 ctx, cancel := context.WithCancel(supervisor.RootContext())151 ctx, cancel := context.WithCancel(supervisor.RootContext())
142- return &Service{152+ s := &Service{
143 epoch: epoch, id: id, registry: registry, pool: pool, supervisor: supervisor, lease: lease,153 epoch: epoch, id: id, registry: registry, pool: pool, supervisor: supervisor, lease: lease,
144 ctx: ctx, cancel: cancel, subs: make(map[*serviceSubscription]struct{}),154 ctx: ctx, cancel: cancel, subs: make(map[*serviceSubscription]struct{}),
155+ done: make(chan struct{}),
145 }156 }
157+ // Broker shutdown ends every admitted connection. The watcher records the
158+ // loss as the terminal cause unless a local Close has already settled the
159+ // connection, so a connection closed by its owner reports nil and one lost
160+ // to a broker shutdown reports the typed loss.
161+ s.stopRoot = context.AfterFunc(supervisor.RootContext(), func() {
162+ s.terminalize(ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "broker shutdown", Cause: context.Canceled})
163+ })
164+ return s
146 }165 }
147 166
148 // ConnectionID returns the pool identity assigned at admission. Clients carry167 // ConnectionID returns the pool identity assigned at admission. Clients carry
149 // it on stream operations so a stale request can be fenced.168 // it on stream operations so a stale request can be fenced.
150 func (s *Service) ConnectionID() ports.BrokerConnectionID { return s.id }169 func (s *Service) ConnectionID() ports.BrokerConnectionID { return s.id }
151 170
171+// Done closes exactly once when this connection is terminal: the broker root
172+// shut down or Close settled it locally. Err is stable afterwards.
173+func (s *Service) Done() <-chan struct{} { return s.done }
174+
175+// Err returns this connection's terminal cause: nil for an orderly local Close,
176+// otherwise the broker-side loss that settled it. The cause is recorded exactly
177+// once and never overwritten, so it is stable once Done is closed.
178+func (s *Service) Err() error {
179+ s.mu.Lock()
180+ defer s.mu.Unlock()
181+ return s.termErr
182+}
183+
184+// terminalize records the terminal cause and closes Done exactly once. The
185+// first cause wins, so a race between a broker-root loss and a local Close
186+// yields one stable, observable outcome.
187+func (s *Service) terminalize(err error) {
188+ s.termOnce.Do(func() {
189+ s.mu.Lock()
190+ s.termErr = err
191+ s.mu.Unlock()
192+ close(s.done)
193+ })
194+}
195+
152 // Snapshot delegates to the registry's current immutable publication.196 // Snapshot delegates to the registry's current immutable publication.
153 func (s *Service) Snapshot() ports.BrokerSnapshot { return s.registry.Snapshot() }197 func (s *Service) Snapshot() ports.BrokerSnapshot { return s.registry.Snapshot() }
154 198
@@ -252,6 +296,13 @@ func (s *Service) Close() error {
252 }296 }
253 s.pool.CloseClient(s.id)297 s.pool.CloseClient(s.id)
254 s.lease.Release()298 s.lease.Release()
299+ // The connection is locally and orderly closed: detach the broker-root
300+ // watcher so it can neither fire later nor overwrite the clean outcome,
301+ // then publish the terminal state.
302+ if s.stopRoot != nil {
303+ s.stopRoot()
304+ }
305+ s.terminalize(nil)
255 })306 })
256 return nil307 return nil
257 }308 }
M internal/usecase/broker/service_test.go +30 -0
@@ -380,6 +380,36 @@ func TestServiceCloseIsIdempotentAndConcurrent(t *testing.T) {
380380 requireBrokerClosed(t, err)
381381 }
382382
383+func TestServiceDoneErrContract(t *testing.T) {
384+ t.Run("local close is orderly", func(t *testing.T) {
385+ authority, _, _, _, _, _ := newTestAuthority(t, 1, nil, immediateConnector)
386+ service, err := authority.AdmitClient(context.Background())
387+ require.NoError(t, err)
388+
389+ select {
390+ case <-service.Done():
391+ t.Fatal("Done closed before any terminal event")
392+ default:
393+ }
394+ require.NoError(t, service.Close())
395+ await(t, service.Done())
396+ require.NoError(t, service.Err(), "an orderly local Close reports no terminal cause")
397+ require.NoError(t, service.Err(), "Err is stable after Done")
398+ })
399+
400+ t.Run("broker shutdown is a typed loss", func(t *testing.T) {
401+ authority, _, _, supervisor, _, _ := newTestAuthority(t, 1, nil, immediateConnector)
402+ service, err := authority.AdmitClient(context.Background())
403+ require.NoError(t, err)
404+
405+ require.NoError(t, supervisor.Close())
406+ await(t, service.Done())
407+ var typed ports.BrokerError
408+ require.ErrorAs(t, service.Err(), &typed)
409+ require.Equal(t, ports.BrokerErrorUnavailable, typed.Code)
410+ })
411+}
412+
383413 func TestServiceShutdownTerminatesOwnedWork(t *testing.T) {
384414 authority, _, _, supervisor, _, _ := newTestAuthority(t, 1, nil, immediateConnector)
385415 service, err := authority.AdmitClient(context.Background())
@@ -380,6 +380,36 @@ func TestServiceCloseIsIdempotentAndConcurrent(t *testing.T) {
380 requireBrokerClosed(t, err)380 requireBrokerClosed(t, err)
381 }381 }
382 382
383+func TestServiceDoneErrContract(t *testing.T) {
384+ t.Run("local close is orderly", func(t *testing.T) {
385+ authority, _, _, _, _, _ := newTestAuthority(t, 1, nil, immediateConnector)
386+ service, err := authority.AdmitClient(context.Background())
387+ require.NoError(t, err)
388+
389+ select {
390+ case <-service.Done():
391+ t.Fatal("Done closed before any terminal event")
392+ default:
393+ }
394+ require.NoError(t, service.Close())
395+ await(t, service.Done())
396+ require.NoError(t, service.Err(), "an orderly local Close reports no terminal cause")
397+ require.NoError(t, service.Err(), "Err is stable after Done")
398+ })
399+
400+ t.Run("broker shutdown is a typed loss", func(t *testing.T) {
401+ authority, _, _, supervisor, _, _ := newTestAuthority(t, 1, nil, immediateConnector)
402+ service, err := authority.AdmitClient(context.Background())
403+ require.NoError(t, err)
404+
405+ require.NoError(t, supervisor.Close())
406+ await(t, service.Done())
407+ var typed ports.BrokerError
408+ require.ErrorAs(t, service.Err(), &typed)
409+ require.Equal(t, ports.BrokerErrorUnavailable, typed.Code)
410+ })
411+}
412+
383 func TestServiceShutdownTerminatesOwnedWork(t *testing.T) {413 func TestServiceShutdownTerminatesOwnedWork(t *testing.T) {
384 authority, _, _, supervisor, _, _ := newTestAuthority(t, 1, nil, immediateConnector)414 authority, _, _, supervisor, _, _ := newTestAuthority(t, 1, nil, immediateConnector)
385 service, err := authority.AdmitClient(context.Background())415 service, err := authority.AdmitClient(context.Background())
A internal/usecase/client/supervisor.go +730 -0
@@ -0,0 +1,730 @@
1+package client
2+
3+import (
4+ "context"
5+ "errors"
6+ "fmt"
7+ "io"
8+ "math/rand/v2"
9+ "reflect"
10+ "sync"
11+ "time"
12+
13+ "github.com/bnema/vev/internal/ports"
14+)
15+
16+// Autonomous client supervisor (Plan 001 P5.1a, offline and unactivated).
17+//
18+// The supervisor is the attachment-free, terminal-owning owner of one
19+// autonomous client process. It enters raw mode once, before it ever reaches
20+// for the broker, and owns a single terminal input lifetime for the whole run.
21+// While the broker connection is down it renders the picker and keeps
22+// reconnecting; it never attaches a session and never interprets terminal
23+// input. Only the picker presentation and the broker-connectivity lifecycle are
24+// exercised by this slice: PresentConnecting and PresentAttached are defined so
25+// the attach slices can drive them later, but nothing here enters them.
26+//
27+// Connectivity is one attempt at a time. An attempt connects, subscribes, and
28+// waits for the first committed publication before it is Ready; readiness
29+// resets the equal-jitter exponential retry cadence (100ms doubling to a 2s
30+// cap). A transient connect, subscribe, or snapshot failure, and the loss of an
31+// established connection observed through BrokerService.Done/Err while no
32+// logical stream is held, both retire the old subscription and service, wait
33+// the cadence, and reconnect under a fresh generation. A failure that can never
34+// succeed by retrying (an incompatible carriage or a conflicting policy) is
35+// surfaced once and stops automatic reconnection; only the domain's terminal
36+// codes end the process.
37+//
38+// A connection attempt is joined before Run returns. Since the connector
39+// contract requires Connect to honor cancellation, a completion that arrives
40+// after the supervisor has begun terminating is still drained and its service
41+// and subscription are closed, so a late superseded success never leaks. Raw
42+// mode is restored exactly once on every exit path, terminal EOF and process
43+// cancellation included.
44+//
45+// Parent cancellation settles the run: an attempt that settles at the same
46+// instant as cancellation is retired instead of adopted, and the run ends with
47+// ctx.Err without publishing a retry cadence, rendering ConnectivityRetryWait,
48+// or notifying a connectivity failure.
49+
50+// Supervisor retry cadence bounds. The exponential cap doubles from the initial
51+// interval to the maximum; the equal-jitter interval is then half the cap plus
52+// a uniform fraction of the other half.
53+const (
54+ supervisorRetryInitial = 100 * time.Millisecond
55+ supervisorRetryMax = 2 * time.Second
56+)
57+
58+// Presentation is what the autonomous client is presenting on the terminal.
59+type Presentation uint8
60+
61+const (
62+ // PresentPicker shows the session picker. P5.1a exercises only this state.
63+ PresentPicker Presentation = iota + 1
64+ // PresentConnecting shows the attach transition. Reserved for the attach
65+ // slice; the P5.1a driver never enters it.
66+ PresentConnecting
67+ // PresentAttached shows a live attachment. Reserved for the attach slice.
68+ PresentAttached
69+ // PresentTerminating is the terminal state: raw mode is being restored and
70+ // the process is leaving.
71+ PresentTerminating
72+)
73+
74+func (p Presentation) String() string {
75+ switch p {
76+ case PresentPicker:
77+ return "picker"
78+ case PresentConnecting:
79+ return "connecting"
80+ case PresentAttached:
81+ return "attached"
82+ case PresentTerminating:
83+ return "terminating"
84+ default:
85+ return "unknown"
86+ }
87+}
88+
89+// Connectivity is the broker-connection lifecycle.
90+type Connectivity uint8
91+
92+const (
93+ // ConnectivityDisconnected holds no broker connection: either before the
94+ // first attempt, after a non-retryable failure, or while terminating.
95+ ConnectivityDisconnected Connectivity = iota + 1
96+ // ConnectivityConnectingBroker is one in-flight connect/subscribe/first
97+ // publication attempt.
98+ ConnectivityConnectingBroker
99+ // ConnectivityReady holds an established connection with its first
100+ // publication committed.
101+ ConnectivityReady
102+ // ConnectivityRetryWait waits the retry cadence before the next attempt.
103+ ConnectivityRetryWait
104+)
105+
106+func (c Connectivity) String() string {
107+ switch c {
108+ case ConnectivityDisconnected:
109+ return "disconnected"
110+ case ConnectivityConnectingBroker:
111+ return "connecting_broker"
112+ case ConnectivityReady:
113+ return "ready"
114+ case ConnectivityRetryWait:
115+ return "retry_wait"
116+ default:
117+ return "unknown"
118+ }
119+}
120+
121+// State is the immutable projection of the supervisor.
122+type State struct {
123+ Presentation Presentation
124+ Connectivity Connectivity
125+ // Attempt counts consecutive broker-connection failures since the last
126+ // Ready. It is zero before the first attempt and after Ready, and drives
127+ // the equal-jitter exponential backoff.
128+ Attempt uint64
129+ // Generation identifies the current connection attempt. It increases before
130+ // every attempt, so a settled attempt carries the generation that produced
131+ // it. Run joins each attempt before it advances, so every completion today
132+ // carries the current generation: the comparison in adoptAttempt is a
133+ // defensive invariant for a future concurrent driver, not a path this
134+ // driver can currently reach.
135+ Generation uint64
136+ // Err is the most recent visible connectivity failure, nil while healthy
137+ // and before any failure.
138+ Err error
139+}
140+
141+// supervisorEventKind is one input to the pure supervisor reducer.
142+type supervisorEventKind uint8
143+
144+const (
145+ // supervisorBeginAttempt starts one connect/subscribe attempt.
146+ supervisorBeginAttempt supervisorEventKind = iota + 1
147+ // supervisorReady marks the first committed publication of the current
148+ // attempt.
149+ supervisorReady
150+ // supervisorTransientFailure is a connect, subscribe, or first-publication
151+ // failure that is worth retrying.
152+ supervisorTransientFailure
153+ // supervisorBrokerLoss is the loss of an established connection observed
154+ // through BrokerService.Done/Err while no logical stream is held.
155+ supervisorBrokerLoss
156+ // supervisorNonRetryable is a failure that retrying can never fix.
157+ supervisorNonRetryable
158+ // supervisorTerminal ends the process: terminal EOF, process cancellation,
159+ // or a terminal broker failure.
160+ supervisorTerminal
161+)
162+
163+type supervisorEvent struct {
164+ kind supervisorEventKind
165+ err error
166+}
167+
168+// reduceSupervisor is the pure connectivity/presentation reducer. It is
169+// side-effect free so the state machine can be exhaustively table-tested; the
170+// driver applies it under a mutex and renders the result.
171+func reduceSupervisor(state State, event supervisorEvent) State {
172+ switch event.kind {
173+ case supervisorBeginAttempt:
174+ state.Presentation = PresentPicker
175+ state.Connectivity = ConnectivityConnectingBroker
176+ state.Generation++
177+ state.Err = nil
178+ case supervisorReady:
179+ state.Connectivity = ConnectivityReady
180+ state.Attempt = 0
181+ state.Err = nil
182+ case supervisorTransientFailure, supervisorBrokerLoss:
183+ state.Connectivity = ConnectivityRetryWait
184+ state.Attempt++
185+ state.Err = event.err
186+ case supervisorNonRetryable:
187+ state.Connectivity = ConnectivityDisconnected
188+ state.Err = event.err
189+ case supervisorTerminal:
190+ state.Presentation = PresentTerminating
191+ state.Connectivity = ConnectivityDisconnected
192+ state.Err = event.err
193+ }
194+ return state
195+}
196+
197+// SupervisorConfig supplies the autonomous client's dependencies. The
198+// connector, terminal, and clock are required; render, notify, and jitter are
199+// injectable seams with safe defaults. Render, notify, and jitter must not
200+// block: the supervisor drives them from its single run goroutine.
201+type SupervisorConfig struct {
202+ // Connector establishes the broker connection. Required.
203+ Connector ports.BrokerConnector
204+ // Terminal is the controlling terminal the supervisor owns. Required.
205+ Terminal ports.Terminal
206+ // Clock supplies the retry timers. Required; use ports.Clock, not the wall
207+ // clock.
208+ Clock ports.Clock
209+ // Render paints the current state. It is called once per visible
210+ // transition. Optional; the default renders nothing.
211+ Render func(State)
212+ // Notify surfaces a connectivity failure without leaving the picker.
213+ // Optional; the default notifies nothing.
214+ Notify func(State, error)
215+ // Jitter returns an equal-jitter fraction in [0,1). It is the only
216+ // randomness seam so tests can pin the retry cadence exactly. Optional; the
217+ // default is a process-local random source.
218+ Jitter func() float64
219+}
220+
221+// Supervisor owns one autonomous client process: raw mode, one terminal input
222+// lifetime, and the broker-connectivity lifecycle.
223+type Supervisor struct {
224+ cfg SupervisorConfig
225+
226+ mu sync.Mutex
227+ state State
228+}
229+
230+// NewSupervisor validates the required dependencies and returns a supervisor
231+// parked on the picker with no connection. It does not touch the terminal; Run
232+// enters raw mode.
233+func NewSupervisor(cfg SupervisorConfig) (*Supervisor, error) {
234+ if supervisorNil(cfg.Connector) {
235+ return nil, errors.New("vev: supervisor requires a broker connector")
236+ }
237+ if supervisorNil(cfg.Terminal) {
238+ return nil, errors.New("vev: supervisor requires a terminal")
239+ }
240+ if supervisorNil(cfg.Clock) {
241+ return nil, errors.New("vev: supervisor requires a clock")
242+ }
243+ if cfg.Render == nil {
244+ cfg.Render = func(State) {}
245+ }
246+ if cfg.Jitter == nil {
247+ cfg.Jitter = rand.Float64
248+ }
249+ return &Supervisor{
250+ cfg: cfg,
251+ state: State{Presentation: PresentPicker, Connectivity: ConnectivityDisconnected},
252+ }, nil
253+}
254+
255+// State returns the current immutable projection.
256+func (s *Supervisor) State() State {
257+ s.mu.Lock()
258+ defer s.mu.Unlock()
259+ return s.state
260+}
261+
262+// Run owns the terminal for exactly its lifetime. It enters raw mode before it
263+// connects to the broker, starts the single terminal input reader, and then
264+// drives connect/subscribe/retry until the terminal reaches EOF, the process
265+// context is cancelled, or a terminal broker failure settles the process. Raw
266+// mode is restored exactly once on every exit path.
267+func (s *Supervisor) Run(ctx context.Context) (retErr error) {
268+ restore, err := s.cfg.Terminal.EnterRaw()
269+ if err != nil {
270+ return fmt.Errorf("vev: entering raw mode: %w", err)
271+ }
272+ if restore == nil {
273+ restore = func() error { return nil }
274+ }
275+ restoreTerminal := sync.OnceValue(restore)
276+ defer func() {
277+ if rerr := restoreTerminal(); rerr != nil && retErr == nil {
278+ retErr = fmt.Errorf("vev: restoring terminal: %w", rerr)
279+ }
280+ }()
281+
282+ input := startTerminalInputLifetime(s.cfg.Terminal.In())
283+
284+ // The active connection is owned across loop iterations. retire is
285+ // idempotent so every exit path closes exactly the connection it adopted.
286+ var (
287+ service ports.BrokerService
288+ sub ports.BrokerSubscription
289+ )
290+ retire := func() {
291+ if sub != nil {
292+ sub.Close()
293+ sub = nil
294+ }
295+ if service != nil {
296+ _ = service.Close()
297+ service = nil
298+ }
299+ }
300+ defer retire()
301+
302+ for {
303+ // The generation is claimed before the connector runs, so the attempt is
304+ // always attributable to the state that started it and a completion can be
305+ // matched against the generation that produced it.
306+ s.transition(supervisorEvent{kind: supervisorBeginAttempt})
307+ generation := s.State().Generation
308+ attemptCtx, cancelAttempt := context.WithCancel(ctx)
309+ outcome := make(chan supervisorAttempt, 1)
310+ go func() { outcome <- s.connectAttempt(attemptCtx, generation) }()
311+
312+ result, terminated, termErr := s.awaitAttempt(ctx, input, outcome)
313+ if terminated {
314+ // Connect must honor cancellation, so joining the attempt here both
315+ // retires a late superseded success and leaves no goroutine behind.
316+ cancelAttempt()
317+ late := <-outcome
318+ late.retire()
319+ s.transition(supervisorEvent{kind: supervisorTerminal, err: termErr})
320+ return termErr
321+ }
322+ cancelAttempt()
323+ adopted, cause := s.adoptAttempt(ctx, result)
324+ if !adopted {
325+ // The cancelled or superseded attempt is retired rather than adopted, so
326+ // it cannot drive the state of a newer attempt and a cancelled run never
327+ // publishes a retry cadence, renders ConnectivityRetryWait, or notifies a
328+ // connectivity failure for it.
329+ result.retire()
330+ if cause != nil {
331+ s.transition(supervisorEvent{kind: supervisorTerminal, err: cause})
332+ return cause
333+ }
334+ continue
335+ }
336+
337+ if result.err != nil {
338+ if supervisorTerminalFailure(result.err) {
339+ s.transition(supervisorEvent{kind: supervisorTerminal, err: result.err})
340+ return result.err
341+ }
342+ if !supervisorRetryable(result.err) {
343+ // Retrying can never fix this. Return to the picker with the
344+ // visible error and stop reconnecting; the process still owns
345+ // the terminal until the user exits or it is cancelled.
346+ s.transition(supervisorEvent{kind: supervisorNonRetryable, err: result.err})
347+ s.notify(result.err)
348+ terminalErr := s.awaitTermination(ctx, input)
349+ s.transition(supervisorEvent{kind: supervisorTerminal, err: terminalErr})
350+ return terminalErr
351+ }
352+ if terminated, termErr := s.retryFailure(ctx, input, supervisorTransientFailure, result.err); terminated {
353+ return termErr
354+ }
355+ continue
356+ }
357+
358+ service, sub = result.service, result.sub
359+ s.transition(supervisorEvent{kind: supervisorReady})
360+
361+ lossErr, terminated, termErr := s.awaitLoss(ctx, input, service)
362+ retire()
363+ if terminated {
364+ s.transition(supervisorEvent{kind: supervisorTerminal, err: termErr})
365+ return termErr
366+ }
367+ if cause := ctx.Err(); cause != nil {
368+ // Parent cancellation wins over a loss that settled at the same instant,
369+ // exactly as it does at the attempt settle. The connection was already
370+ // released above, so the loss is simply never turned into a retry
371+ // cadence, a ConnectivityRetryWait render, or a notification for a run
372+ // that was already cancelled.
373+ s.transition(supervisorEvent{kind: supervisorTerminal, err: cause})
374+ return cause
375+ }
376+ if terminated, termErr := s.retryFailure(ctx, input, supervisorBrokerLoss, normalizeUnavailable(lossErr)); terminated {
377+ return termErr
378+ }
379+ }
380+}
381+
382+// supervisorAttempt is the settled outcome of one connect/subscribe/first
383+// publication attempt. generation is the attempt's generation, claimed before
384+// the connector ran, so the completion is always attributable to the attempt
385+// that produced it (see adoptAttempt's generation fence).
386+type supervisorAttempt struct {
387+ generation uint64
388+ service ports.BrokerService
389+ sub ports.BrokerSubscription
390+ err error
391+}
392+
393+// retire closes whatever the attempt produced. It is safe on a partially built
394+// or empty attempt, and it treats a typed-nil service or subscription as absent
395+// rather than calling into it.
396+func (a supervisorAttempt) retire() {
397+ if !supervisorNil(a.sub) {
398+ a.sub.Close()
399+ }
400+ if !supervisorNil(a.service) {
401+ _ = a.service.Close()
402+ }
403+}
404+
405+// adoptAttempt decides whether one settled attempt may be adopted, and reports the
406+// parent cancellation that must settle the run instead. The decision is made under
407+// the state lock, atomically with respect to the state it consults, so an attempt
408+// that settled after the parent cancelled the run is never adopted even though
409+// awaitAttempt's select may have taken its outcome over an already-ready
410+// ctx.Done. The generation fence is enforced at the same decision point: a
411+// superseded attempt is rejected too. It is a defensive invariant, not a
412+// currently reachable production path, because Run joins each attempt before it
413+// advances; it is kept so a future driver that admits more than one attempt at a
414+// time can never let a late completion drive the state of a newer attempt.
415+func (s *Supervisor) adoptAttempt(ctx context.Context, result supervisorAttempt) (adopted bool, cause error) {
416+ s.mu.Lock()
417+ defer s.mu.Unlock()
418+ if cause := ctx.Err(); cause != nil {
419+ return false, cause
420+ }
421+ if result.generation != s.state.Generation {
422+ return false, nil
423+ }
424+ return true, nil
425+}
426+
427+// retryFailure records one retryable connectivity failure, surfaces it, and
428+// waits the cadence for the next attempt. It is the shared tail of the transient
429+// connect/subscribe failure and the established-connection loss paths, so both
430+// select the same observable transition, notification, and backoff.
431+func (s *Supervisor) retryFailure(ctx context.Context, input *terminalInputLifetime, kind supervisorEventKind, err error) (bool, error) {
432+ if cause := ctx.Err(); cause != nil {
433+ s.transition(supervisorEvent{kind: supervisorTerminal, err: cause})
434+ return true, cause
435+ }
436+ s.transition(supervisorEvent{kind: kind, err: err})
437+ s.notify(err)
438+ if terminated, termErr := s.waitBackoff(ctx, input); terminated {
439+ s.transition(supervisorEvent{kind: supervisorTerminal, err: termErr})
440+ return true, termErr
441+ }
442+ return false, nil
443+}
444+
445+// connectAttempt connects once, subscribes, and waits for the first committed
446+// publication. It returns a live service and subscription only when the
447+// connection is Ready; every failure closes whatever it opened before
448+// returning, so a caller has nothing to clean up on the error path. Every
449+// connect, subscribe, or first-publication failure is stamped with generation
450+// and normalized to a typed unavailable failure so the run loop never observes
451+// an untyped cause.
452+func (s *Supervisor) connectAttempt(ctx context.Context, generation uint64) supervisorAttempt {
453+ service, err := s.cfg.Connector.Connect(ctx)
454+ if err != nil {
455+ return supervisorAttempt{generation: generation, err: normalizeUnavailable(err)}
456+ }
457+ if supervisorNil(service) {
458+ return supervisorAttempt{generation: generation, err: ports.BrokerError{
459+ Code: ports.BrokerErrorUnavailable,
460+ Text: "broker connector returned no service",
461+ }}
462+ }
463+ sub, err := service.Subscribe()
464+ if err != nil {
465+ _ = service.Close()
466+ return supervisorAttempt{generation: generation, err: normalizeUnavailable(err)}
467+ }
468+ if supervisorNil(sub) {
469+ // A service that reports no subscription cannot ever become Ready, so it
470+ // is a typed unavailable failure rather than a nil dereference on the
471+ // first Changed() call.
472+ _ = service.Close()
473+ return supervisorAttempt{generation: generation, err: ports.BrokerError{
474+ Code: ports.BrokerErrorUnavailable,
475+ Text: "broker service returned no subscription",
476+ }}
477+ }
478+ for {
479+ if service.Snapshot().Epoch != 0 {
480+ return supervisorAttempt{generation: generation, service: service, sub: sub}
481+ }
482+ select {
483+ case <-sub.Changed():
484+ continue
485+ case <-service.Done():
486+ cause := normalizeUnavailable(service.Err())
487+ sub.Close()
488+ _ = service.Close()
489+ return supervisorAttempt{generation: generation, err: cause}
490+ case <-ctx.Done():
491+ sub.Close()
492+ _ = service.Close()
493+ return supervisorAttempt{generation: generation, err: ctx.Err()}
494+ }
495+ }
496+}
497+
498+// awaitAttempt waits for the in-flight attempt to settle, or for the run to be
499+// terminated first.
500+func (s *Supervisor) awaitAttempt(ctx context.Context, input *terminalInputLifetime, outcome <-chan supervisorAttempt) (supervisorAttempt, bool, error) {
501+ select {
502+ case result := <-outcome:
503+ return result, false, nil
504+ case <-ctx.Done():
505+ return supervisorAttempt{}, true, ctx.Err()
506+ case err := <-input.EOF():
507+ return supervisorAttempt{}, true, terminalReadCause(err)
508+ }
509+}
510+
511+// awaitLoss waits for an established connection to be lost, or for the run to
512+// be terminated first. A loss is reported with the service's stable terminal
513+// cause.
514+func (s *Supervisor) awaitLoss(ctx context.Context, input *terminalInputLifetime, service ports.BrokerService) (error, bool, error) {
515+ select {
516+ case <-service.Done():
517+ return service.Err(), false, nil
518+ case <-ctx.Done():
519+ return nil, true, ctx.Err()
520+ case err := <-input.EOF():
521+ return nil, true, terminalReadCause(err)
522+ }
523+}
524+
525+// awaitTermination parks the supervisor on the picker after a non-retryable
526+// failure until the terminal ends or the process is cancelled. A clean terminal
527+// EOF returns nil; every other cause is reported.
528+func (s *Supervisor) awaitTermination(ctx context.Context, input *terminalInputLifetime) error {
529+ select {
530+ case <-ctx.Done():
531+ return ctx.Err()
532+ case err := <-input.EOF():
533+ return terminalReadCause(err)
534+ }
535+}
536+
537+// waitBackoff waits one equal-jitter retry interval for the current failure
538+// count, or settles on cancellation or terminal EOF. The timer is always
539+// stopped and drained, so no timer outlives the attempt transition.
540+func (s *Supervisor) waitBackoff(ctx context.Context, input *terminalInputLifetime) (bool, error) {
541+ delay := supervisorBackoffDelay(s.State().Attempt, s.cfg.Jitter)
542+ if delay <= 0 {
543+ return false, nil
544+ }
545+ timer := s.cfg.Clock.NewTimer(delay)
546+ defer stopSupervisorTimer(timer)
547+ select {
548+ case <-timer.C():
549+ return false, nil
550+ case <-ctx.Done():
551+ return true, ctx.Err()
552+ case err := <-input.EOF():
553+ return true, terminalReadCause(err)
554+ }
555+}
556+
557+// supervisorBackoffDelay is the equal-jitter exponential interval for one
558+// failure count: the cap doubles from supervisorRetryInitial to
559+// supervisorRetryMax, and the interval is half the cap plus a uniform fraction
560+// of the other half, so attempts spread across [cap/2, cap). It is pure so the
561+// cadence can be tested exactly.
562+func supervisorBackoffDelay(attempt uint64, jitter func() float64) time.Duration {
563+ if attempt == 0 {
564+ return 0
565+ }
566+ limit := supervisorRetryInitial
567+ for i := uint64(1); i < attempt && limit < supervisorRetryMax; i++ {
568+ limit *= 2
569+ }
570+ if limit > supervisorRetryMax {
571+ limit = supervisorRetryMax
572+ }
573+ half := limit / 2
574+ fraction := jitter()
575+ if !(fraction >= 0) {
576+ // NaN is unordered: both fraction < 0 and fraction > 1 are false for
577+ // it, so an explicit non-negative test maps NaN and every negative value
578+ // to the minimum fraction. Without this, a NaN would reach the duration
579+ // conversion and yield an undefined interval instead of the documented
580+ // half-cap minimum.
581+ fraction = 0
582+ }
583+ if fraction > 1 {
584+ fraction = 1
585+ }
586+ return half + time.Duration(fraction*float64(half))
587+}
588+
589+// terminalReadCause maps a terminal read end to the Run result: a clean EOF is
590+// an orderly exit (nil), any other read failure is reported.
591+func terminalReadCause(err error) error {
592+ if err == nil || errors.Is(err, io.EOF) {
593+ return nil
594+ }
595+ return err
596+}
597+
598+// stopSupervisorTimer stops and drains a timer, supporting buffered timer
599+// adapters with pre-Go-1.23 reset semantics.
600+func stopSupervisorTimer(timer ports.Timer) {
601+ if timer == nil {
602+ return
603+ }
604+ if !timer.Stop() {
605+ select {
606+ case <-timer.C():
607+ default:
608+ }
609+ }
610+}
611+
612+// supervisorTerminalFailure reports whether a broker failure must end the
613+// process. Only the domain's terminal codes do; every other failure returns the
614+// client to the picker.
615+func supervisorTerminalFailure(err error) bool {
616+ var typed ports.BrokerError
617+ return errors.As(err, &typed) && typed.Terminal()
618+}
619+
620+// supervisorRetryable reports whether a failure is worth another attempt after
621+// backoff. An incompatible carriage or a conflicting policy can never succeed
622+// by retrying, so it is surfaced once and automatic reconnection stops; every
623+// other failure is treated as transient.
624+func supervisorRetryable(err error) bool {
625+ var typed ports.BrokerError
626+ if errors.As(err, &typed) {
627+ switch typed.Code {
628+ case ports.BrokerErrorIncompatible, ports.BrokerErrorConflictingPolicy:
629+ return false
630+ }
631+ }
632+ return true
633+}
634+
635+// normalizeUnavailable presents one connect, subscribe, or connection-loss
636+// failure as a typed ports.BrokerError. A cause that is already a BrokerError is
637+// returned unchanged, so its own code (for example incompatible or conflicting
638+// policy) still selects the non-retryable path; every untyped failure, including
639+// a nil cause, becomes an unavailable BrokerError that preserves the original
640+// error as its local-only Cause.
641+func normalizeUnavailable(err error) error {
642+ if err == nil {
643+ return ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "broker connection lost"}
644+ }
645+ var typed ports.BrokerError
646+ if errors.As(err, &typed) {
647+ return err
648+ }
649+ return ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "broker connection lost", Cause: err}
650+}
651+
652+// notify surfaces a connectivity failure through the optional notifier.
653+func (s *Supervisor) notify(err error) {
654+ if s.cfg.Notify == nil || err == nil {
655+ return
656+ }
657+ s.cfg.Notify(s.State(), err)
658+}
659+
660+// transition applies one reducer event under the supervisor mutex and renders
661+// the resulting state outside it, so a renderer that reads State back never
662+// deadlocks.
663+func (s *Supervisor) transition(event supervisorEvent) {
664+ s.mu.Lock()
665+ s.state = reduceSupervisor(s.state, event)
666+ state := s.state
667+ s.mu.Unlock()
668+ s.cfg.Render(state)
669+}
670+
671+// terminalInputLifetime owns the single read of the controlling terminal. It is
672+// started once and never replaced: a bare io.Reader cannot be interrupted, so
673+// the goroutine may outlive Run when stdin never reaches EOF, exactly like the
674+// attach client's input pump. It never closes caller-owned input, and P5.1a
675+// discards read bytes because picker interaction is a later slice.
676+type terminalInputLifetime struct {
677+ eof chan error
678+ once sync.Once
679+}
680+
681+// startTerminalInputLifetime starts the single terminal read. A nil reader is
682+// treated as an immediate orderly EOF.
683+func startTerminalInputLifetime(in io.Reader) *terminalInputLifetime {
684+ lifetime := &terminalInputLifetime{eof: make(chan error, 1)}
685+ if supervisorNil(in) {
686+ lifetime.finish(io.EOF)
687+ return lifetime
688+ }
689+ go func() {
690+ buffer := make([]byte, 4096)
691+ for {
692+ n, err := in.Read(buffer)
693+ if err != nil {
694+ lifetime.finish(err)
695+ return
696+ }
697+ if n == 0 {
698+ // A zero-byte, nil-error read is legal but cannot make
699+ // progress; keep reading so a well-behaved reader is never
700+ // mistaken for EOF.
701+ continue
702+ }
703+ }
704+ }()
705+ return lifetime
706+}
707+
708+// finish publishes the terminal cause exactly once.
709+func (l *terminalInputLifetime) finish(err error) {
710+ l.once.Do(func() { l.eof <- err })
711+}
712+
713+// EOF is closed-result channel delivering the terminal read cause exactly once.
714+func (l *terminalInputLifetime) EOF() <-chan error { return l.eof }
715+
716+// supervisorNil reports whether a dependency is nil in any shape. A typed nil
717+// satisfies a plain != nil comparison, so the guard would defer the failure to
718+// the first method call, which panics; the reflection check keeps it total.
719+func supervisorNil(dependency any) bool {
720+ if dependency == nil {
721+ return true
722+ }
723+ value := reflect.ValueOf(dependency)
724+ switch value.Kind() {
725+ case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
726+ return value.IsNil()
727+ default:
728+ return false
729+ }
730+}
@@ -0,0 +1,730 @@
1+package client
2+
3+import (
4+ "context"
5+ "errors"
6+ "fmt"
7+ "io"
8+ "math/rand/v2"
9+ "reflect"
10+ "sync"
11+ "time"
12+
13+ "github.com/bnema/vev/internal/ports"
14+)
15+
16+// Autonomous client supervisor (Plan 001 P5.1a, offline and unactivated).
17+//
18+// The supervisor is the attachment-free, terminal-owning owner of one
19+// autonomous client process. It enters raw mode once, before it ever reaches
20+// for the broker, and owns a single terminal input lifetime for the whole run.
21+// While the broker connection is down it renders the picker and keeps
22+// reconnecting; it never attaches a session and never interprets terminal
23+// input. Only the picker presentation and the broker-connectivity lifecycle are
24+// exercised by this slice: PresentConnecting and PresentAttached are defined so
25+// the attach slices can drive them later, but nothing here enters them.
26+//
27+// Connectivity is one attempt at a time. An attempt connects, subscribes, and
28+// waits for the first committed publication before it is Ready; readiness
29+// resets the equal-jitter exponential retry cadence (100ms doubling to a 2s
30+// cap). A transient connect, subscribe, or snapshot failure, and the loss of an
31+// established connection observed through BrokerService.Done/Err while no
32+// logical stream is held, both retire the old subscription and service, wait
33+// the cadence, and reconnect under a fresh generation. A failure that can never
34+// succeed by retrying (an incompatible carriage or a conflicting policy) is
35+// surfaced once and stops automatic reconnection; only the domain's terminal
36+// codes end the process.
37+//
38+// A connection attempt is joined before Run returns. Since the connector
39+// contract requires Connect to honor cancellation, a completion that arrives
40+// after the supervisor has begun terminating is still drained and its service
41+// and subscription are closed, so a late superseded success never leaks. Raw
42+// mode is restored exactly once on every exit path, terminal EOF and process
43+// cancellation included.
44+//
45+// Parent cancellation settles the run: an attempt that settles at the same
46+// instant as cancellation is retired instead of adopted, and the run ends with
47+// ctx.Err without publishing a retry cadence, rendering ConnectivityRetryWait,
48+// or notifying a connectivity failure.
49+
50+// Supervisor retry cadence bounds. The exponential cap doubles from the initial
51+// interval to the maximum; the equal-jitter interval is then half the cap plus
52+// a uniform fraction of the other half.
53+const (
54+ supervisorRetryInitial = 100 * time.Millisecond
55+ supervisorRetryMax = 2 * time.Second
56+)
57+
58+// Presentation is what the autonomous client is presenting on the terminal.
59+type Presentation uint8
60+
61+const (
62+ // PresentPicker shows the session picker. P5.1a exercises only this state.
63+ PresentPicker Presentation = iota + 1
64+ // PresentConnecting shows the attach transition. Reserved for the attach
65+ // slice; the P5.1a driver never enters it.
66+ PresentConnecting
67+ // PresentAttached shows a live attachment. Reserved for the attach slice.
68+ PresentAttached
69+ // PresentTerminating is the terminal state: raw mode is being restored and
70+ // the process is leaving.
71+ PresentTerminating
72+)
73+
74+func (p Presentation) String() string {
75+ switch p {
76+ case PresentPicker:
77+ return "picker"
78+ case PresentConnecting:
79+ return "connecting"
80+ case PresentAttached:
81+ return "attached"
82+ case PresentTerminating:
83+ return "terminating"
84+ default:
85+ return "unknown"
86+ }
87+}
88+
89+// Connectivity is the broker-connection lifecycle.
90+type Connectivity uint8
91+
92+const (
93+ // ConnectivityDisconnected holds no broker connection: either before the
94+ // first attempt, after a non-retryable failure, or while terminating.
95+ ConnectivityDisconnected Connectivity = iota + 1
96+ // ConnectivityConnectingBroker is one in-flight connect/subscribe/first
97+ // publication attempt.
98+ ConnectivityConnectingBroker
99+ // ConnectivityReady holds an established connection with its first
100+ // publication committed.
101+ ConnectivityReady
102+ // ConnectivityRetryWait waits the retry cadence before the next attempt.
103+ ConnectivityRetryWait
104+)
105+
106+func (c Connectivity) String() string {
107+ switch c {
108+ case ConnectivityDisconnected:
109+ return "disconnected"
110+ case ConnectivityConnectingBroker:
111+ return "connecting_broker"
112+ case ConnectivityReady:
113+ return "ready"
114+ case ConnectivityRetryWait:
115+ return "retry_wait"
116+ default:
117+ return "unknown"
118+ }
119+}
120+
121+// State is the immutable projection of the supervisor.
122+type State struct {
123+ Presentation Presentation
124+ Connectivity Connectivity
125+ // Attempt counts consecutive broker-connection failures since the last
126+ // Ready. It is zero before the first attempt and after Ready, and drives
127+ // the equal-jitter exponential backoff.
128+ Attempt uint64
129+ // Generation identifies the current connection attempt. It increases before
130+ // every attempt, so a settled attempt carries the generation that produced
131+ // it. Run joins each attempt before it advances, so every completion today
132+ // carries the current generation: the comparison in adoptAttempt is a
133+ // defensive invariant for a future concurrent driver, not a path this
134+ // driver can currently reach.
135+ Generation uint64
136+ // Err is the most recent visible connectivity failure, nil while healthy
137+ // and before any failure.
138+ Err error
139+}
140+
141+// supervisorEventKind is one input to the pure supervisor reducer.
142+type supervisorEventKind uint8
143+
144+const (
145+ // supervisorBeginAttempt starts one connect/subscribe attempt.
146+ supervisorBeginAttempt supervisorEventKind = iota + 1
147+ // supervisorReady marks the first committed publication of the current
148+ // attempt.
149+ supervisorReady
150+ // supervisorTransientFailure is a connect, subscribe, or first-publication
151+ // failure that is worth retrying.
152+ supervisorTransientFailure
153+ // supervisorBrokerLoss is the loss of an established connection observed
154+ // through BrokerService.Done/Err while no logical stream is held.
155+ supervisorBrokerLoss
156+ // supervisorNonRetryable is a failure that retrying can never fix.
157+ supervisorNonRetryable
158+ // supervisorTerminal ends the process: terminal EOF, process cancellation,
159+ // or a terminal broker failure.
160+ supervisorTerminal
161+)
162+
163+type supervisorEvent struct {
164+ kind supervisorEventKind
165+ err error
166+}
167+
168+// reduceSupervisor is the pure connectivity/presentation reducer. It is
169+// side-effect free so the state machine can be exhaustively table-tested; the
170+// driver applies it under a mutex and renders the result.
171+func reduceSupervisor(state State, event supervisorEvent) State {
172+ switch event.kind {
173+ case supervisorBeginAttempt:
174+ state.Presentation = PresentPicker
175+ state.Connectivity = ConnectivityConnectingBroker
176+ state.Generation++
177+ state.Err = nil
178+ case supervisorReady:
179+ state.Connectivity = ConnectivityReady
180+ state.Attempt = 0
181+ state.Err = nil
182+ case supervisorTransientFailure, supervisorBrokerLoss:
183+ state.Connectivity = ConnectivityRetryWait
184+ state.Attempt++
185+ state.Err = event.err
186+ case supervisorNonRetryable:
187+ state.Connectivity = ConnectivityDisconnected
188+ state.Err = event.err
189+ case supervisorTerminal:
190+ state.Presentation = PresentTerminating
191+ state.Connectivity = ConnectivityDisconnected
192+ state.Err = event.err
193+ }
194+ return state
195+}
196+
197+// SupervisorConfig supplies the autonomous client's dependencies. The
198+// connector, terminal, and clock are required; render, notify, and jitter are
199+// injectable seams with safe defaults. Render, notify, and jitter must not
200+// block: the supervisor drives them from its single run goroutine.
201+type SupervisorConfig struct {
202+ // Connector establishes the broker connection. Required.
203+ Connector ports.BrokerConnector
204+ // Terminal is the controlling terminal the supervisor owns. Required.
205+ Terminal ports.Terminal
206+ // Clock supplies the retry timers. Required; use ports.Clock, not the wall
207+ // clock.
208+ Clock ports.Clock
209+ // Render paints the current state. It is called once per visible
210+ // transition. Optional; the default renders nothing.
211+ Render func(State)
212+ // Notify surfaces a connectivity failure without leaving the picker.
213+ // Optional; the default notifies nothing.
214+ Notify func(State, error)
215+ // Jitter returns an equal-jitter fraction in [0,1). It is the only
216+ // randomness seam so tests can pin the retry cadence exactly. Optional; the
217+ // default is a process-local random source.
218+ Jitter func() float64
219+}
220+
221+// Supervisor owns one autonomous client process: raw mode, one terminal input
222+// lifetime, and the broker-connectivity lifecycle.
223+type Supervisor struct {
224+ cfg SupervisorConfig
225+
226+ mu sync.Mutex
227+ state State
228+}
229+
230+// NewSupervisor validates the required dependencies and returns a supervisor
231+// parked on the picker with no connection. It does not touch the terminal; Run
232+// enters raw mode.
233+func NewSupervisor(cfg SupervisorConfig) (*Supervisor, error) {
234+ if supervisorNil(cfg.Connector) {
235+ return nil, errors.New("vev: supervisor requires a broker connector")
236+ }
237+ if supervisorNil(cfg.Terminal) {
238+ return nil, errors.New("vev: supervisor requires a terminal")
239+ }
240+ if supervisorNil(cfg.Clock) {
241+ return nil, errors.New("vev: supervisor requires a clock")
242+ }
243+ if cfg.Render == nil {
244+ cfg.Render = func(State) {}
245+ }
246+ if cfg.Jitter == nil {
247+ cfg.Jitter = rand.Float64
248+ }
249+ return &Supervisor{
250+ cfg: cfg,
251+ state: State{Presentation: PresentPicker, Connectivity: ConnectivityDisconnected},
252+ }, nil
253+}
254+
255+// State returns the current immutable projection.
256+func (s *Supervisor) State() State {
257+ s.mu.Lock()
258+ defer s.mu.Unlock()
259+ return s.state
260+}
261+
262+// Run owns the terminal for exactly its lifetime. It enters raw mode before it
263+// connects to the broker, starts the single terminal input reader, and then
264+// drives connect/subscribe/retry until the terminal reaches EOF, the process
265+// context is cancelled, or a terminal broker failure settles the process. Raw
266+// mode is restored exactly once on every exit path.
267+func (s *Supervisor) Run(ctx context.Context) (retErr error) {
268+ restore, err := s.cfg.Terminal.EnterRaw()
269+ if err != nil {
270+ return fmt.Errorf("vev: entering raw mode: %w", err)
271+ }
272+ if restore == nil {
273+ restore = func() error { return nil }
274+ }
275+ restoreTerminal := sync.OnceValue(restore)
276+ defer func() {
277+ if rerr := restoreTerminal(); rerr != nil && retErr == nil {
278+ retErr = fmt.Errorf("vev: restoring terminal: %w", rerr)
279+ }
280+ }()
281+
282+ input := startTerminalInputLifetime(s.cfg.Terminal.In())
283+
284+ // The active connection is owned across loop iterations. retire is
285+ // idempotent so every exit path closes exactly the connection it adopted.
286+ var (
287+ service ports.BrokerService
288+ sub ports.BrokerSubscription
289+ )
290+ retire := func() {
291+ if sub != nil {
292+ sub.Close()
293+ sub = nil
294+ }
295+ if service != nil {
296+ _ = service.Close()
297+ service = nil
298+ }
299+ }
300+ defer retire()
301+
302+ for {
303+ // The generation is claimed before the connector runs, so the attempt is
304+ // always attributable to the state that started it and a completion can be
305+ // matched against the generation that produced it.
306+ s.transition(supervisorEvent{kind: supervisorBeginAttempt})
307+ generation := s.State().Generation
308+ attemptCtx, cancelAttempt := context.WithCancel(ctx)
309+ outcome := make(chan supervisorAttempt, 1)
310+ go func() { outcome <- s.connectAttempt(attemptCtx, generation) }()
311+
312+ result, terminated, termErr := s.awaitAttempt(ctx, input, outcome)
313+ if terminated {
314+ // Connect must honor cancellation, so joining the attempt here both
315+ // retires a late superseded success and leaves no goroutine behind.
316+ cancelAttempt()
317+ late := <-outcome
318+ late.retire()
319+ s.transition(supervisorEvent{kind: supervisorTerminal, err: termErr})
320+ return termErr
321+ }
322+ cancelAttempt()
323+ adopted, cause := s.adoptAttempt(ctx, result)
324+ if !adopted {
325+ // The cancelled or superseded attempt is retired rather than adopted, so
326+ // it cannot drive the state of a newer attempt and a cancelled run never
327+ // publishes a retry cadence, renders ConnectivityRetryWait, or notifies a
328+ // connectivity failure for it.
329+ result.retire()
330+ if cause != nil {
331+ s.transition(supervisorEvent{kind: supervisorTerminal, err: cause})
332+ return cause
333+ }
334+ continue
335+ }
336+
337+ if result.err != nil {
338+ if supervisorTerminalFailure(result.err) {
339+ s.transition(supervisorEvent{kind: supervisorTerminal, err: result.err})
340+ return result.err
341+ }
342+ if !supervisorRetryable(result.err) {
343+ // Retrying can never fix this. Return to the picker with the
344+ // visible error and stop reconnecting; the process still owns
345+ // the terminal until the user exits or it is cancelled.
346+ s.transition(supervisorEvent{kind: supervisorNonRetryable, err: result.err})
347+ s.notify(result.err)
348+ terminalErr := s.awaitTermination(ctx, input)
349+ s.transition(supervisorEvent{kind: supervisorTerminal, err: terminalErr})
350+ return terminalErr
351+ }
352+ if terminated, termErr := s.retryFailure(ctx, input, supervisorTransientFailure, result.err); terminated {
353+ return termErr
354+ }
355+ continue
356+ }
357+
358+ service, sub = result.service, result.sub
359+ s.transition(supervisorEvent{kind: supervisorReady})
360+
361+ lossErr, terminated, termErr := s.awaitLoss(ctx, input, service)
362+ retire()
363+ if terminated {
364+ s.transition(supervisorEvent{kind: supervisorTerminal, err: termErr})
365+ return termErr
366+ }
367+ if cause := ctx.Err(); cause != nil {
368+ // Parent cancellation wins over a loss that settled at the same instant,
369+ // exactly as it does at the attempt settle. The connection was already
370+ // released above, so the loss is simply never turned into a retry
371+ // cadence, a ConnectivityRetryWait render, or a notification for a run
372+ // that was already cancelled.
373+ s.transition(supervisorEvent{kind: supervisorTerminal, err: cause})
374+ return cause
375+ }
376+ if terminated, termErr := s.retryFailure(ctx, input, supervisorBrokerLoss, normalizeUnavailable(lossErr)); terminated {
377+ return termErr
378+ }
379+ }
380+}
381+
382+// supervisorAttempt is the settled outcome of one connect/subscribe/first
383+// publication attempt. generation is the attempt's generation, claimed before
384+// the connector ran, so the completion is always attributable to the attempt
385+// that produced it (see adoptAttempt's generation fence).
386+type supervisorAttempt struct {
387+ generation uint64
388+ service ports.BrokerService
389+ sub ports.BrokerSubscription
390+ err error
391+}
392+
393+// retire closes whatever the attempt produced. It is safe on a partially built
394+// or empty attempt, and it treats a typed-nil service or subscription as absent
395+// rather than calling into it.
396+func (a supervisorAttempt) retire() {
397+ if !supervisorNil(a.sub) {
398+ a.sub.Close()
399+ }
400+ if !supervisorNil(a.service) {
401+ _ = a.service.Close()
402+ }
403+}
404+
405+// adoptAttempt decides whether one settled attempt may be adopted, and reports the
406+// parent cancellation that must settle the run instead. The decision is made under
407+// the state lock, atomically with respect to the state it consults, so an attempt
408+// that settled after the parent cancelled the run is never adopted even though
409+// awaitAttempt's select may have taken its outcome over an already-ready
410+// ctx.Done. The generation fence is enforced at the same decision point: a
411+// superseded attempt is rejected too. It is a defensive invariant, not a
412+// currently reachable production path, because Run joins each attempt before it
413+// advances; it is kept so a future driver that admits more than one attempt at a
414+// time can never let a late completion drive the state of a newer attempt.
415+func (s *Supervisor) adoptAttempt(ctx context.Context, result supervisorAttempt) (adopted bool, cause error) {
416+ s.mu.Lock()
417+ defer s.mu.Unlock()
418+ if cause := ctx.Err(); cause != nil {
419+ return false, cause
420+ }
421+ if result.generation != s.state.Generation {
422+ return false, nil
423+ }
424+ return true, nil
425+}
426+
427+// retryFailure records one retryable connectivity failure, surfaces it, and
428+// waits the cadence for the next attempt. It is the shared tail of the transient
429+// connect/subscribe failure and the established-connection loss paths, so both
430+// select the same observable transition, notification, and backoff.
431+func (s *Supervisor) retryFailure(ctx context.Context, input *terminalInputLifetime, kind supervisorEventKind, err error) (bool, error) {
432+ if cause := ctx.Err(); cause != nil {
433+ s.transition(supervisorEvent{kind: supervisorTerminal, err: cause})
434+ return true, cause
435+ }
436+ s.transition(supervisorEvent{kind: kind, err: err})
437+ s.notify(err)
438+ if terminated, termErr := s.waitBackoff(ctx, input); terminated {
439+ s.transition(supervisorEvent{kind: supervisorTerminal, err: termErr})
440+ return true, termErr
441+ }
442+ return false, nil
443+}
444+
445+// connectAttempt connects once, subscribes, and waits for the first committed
446+// publication. It returns a live service and subscription only when the
447+// connection is Ready; every failure closes whatever it opened before
448+// returning, so a caller has nothing to clean up on the error path. Every
449+// connect, subscribe, or first-publication failure is stamped with generation
450+// and normalized to a typed unavailable failure so the run loop never observes
451+// an untyped cause.
452+func (s *Supervisor) connectAttempt(ctx context.Context, generation uint64) supervisorAttempt {
453+ service, err := s.cfg.Connector.Connect(ctx)
454+ if err != nil {
455+ return supervisorAttempt{generation: generation, err: normalizeUnavailable(err)}
456+ }
457+ if supervisorNil(service) {
458+ return supervisorAttempt{generation: generation, err: ports.BrokerError{
459+ Code: ports.BrokerErrorUnavailable,
460+ Text: "broker connector returned no service",
461+ }}
462+ }
463+ sub, err := service.Subscribe()
464+ if err != nil {
465+ _ = service.Close()
466+ return supervisorAttempt{generation: generation, err: normalizeUnavailable(err)}
467+ }
468+ if supervisorNil(sub) {
469+ // A service that reports no subscription cannot ever become Ready, so it
470+ // is a typed unavailable failure rather than a nil dereference on the
471+ // first Changed() call.
472+ _ = service.Close()
473+ return supervisorAttempt{generation: generation, err: ports.BrokerError{
474+ Code: ports.BrokerErrorUnavailable,
475+ Text: "broker service returned no subscription",
476+ }}
477+ }
478+ for {
479+ if service.Snapshot().Epoch != 0 {
480+ return supervisorAttempt{generation: generation, service: service, sub: sub}
481+ }
482+ select {
483+ case <-sub.Changed():
484+ continue
485+ case <-service.Done():
486+ cause := normalizeUnavailable(service.Err())
487+ sub.Close()
488+ _ = service.Close()
489+ return supervisorAttempt{generation: generation, err: cause}
490+ case <-ctx.Done():
491+ sub.Close()
492+ _ = service.Close()
493+ return supervisorAttempt{generation: generation, err: ctx.Err()}
494+ }
495+ }
496+}
497+
498+// awaitAttempt waits for the in-flight attempt to settle, or for the run to be
499+// terminated first.
500+func (s *Supervisor) awaitAttempt(ctx context.Context, input *terminalInputLifetime, outcome <-chan supervisorAttempt) (supervisorAttempt, bool, error) {
501+ select {
502+ case result := <-outcome:
503+ return result, false, nil
504+ case <-ctx.Done():
505+ return supervisorAttempt{}, true, ctx.Err()
506+ case err := <-input.EOF():
507+ return supervisorAttempt{}, true, terminalReadCause(err)
508+ }
509+}
510+
511+// awaitLoss waits for an established connection to be lost, or for the run to
512+// be terminated first. A loss is reported with the service's stable terminal
513+// cause.
514+func (s *Supervisor) awaitLoss(ctx context.Context, input *terminalInputLifetime, service ports.BrokerService) (error, bool, error) {
515+ select {
516+ case <-service.Done():
517+ return service.Err(), false, nil
518+ case <-ctx.Done():
519+ return nil, true, ctx.Err()
520+ case err := <-input.EOF():
521+ return nil, true, terminalReadCause(err)
522+ }
523+}
524+
525+// awaitTermination parks the supervisor on the picker after a non-retryable
526+// failure until the terminal ends or the process is cancelled. A clean terminal
527+// EOF returns nil; every other cause is reported.
528+func (s *Supervisor) awaitTermination(ctx context.Context, input *terminalInputLifetime) error {
529+ select {
530+ case <-ctx.Done():
531+ return ctx.Err()
532+ case err := <-input.EOF():
533+ return terminalReadCause(err)
534+ }
535+}
536+
537+// waitBackoff waits one equal-jitter retry interval for the current failure
538+// count, or settles on cancellation or terminal EOF. The timer is always
539+// stopped and drained, so no timer outlives the attempt transition.
540+func (s *Supervisor) waitBackoff(ctx context.Context, input *terminalInputLifetime) (bool, error) {
541+ delay := supervisorBackoffDelay(s.State().Attempt, s.cfg.Jitter)
542+ if delay <= 0 {
543+ return false, nil
544+ }
545+ timer := s.cfg.Clock.NewTimer(delay)
546+ defer stopSupervisorTimer(timer)
547+ select {
548+ case <-timer.C():
549+ return false, nil
550+ case <-ctx.Done():
551+ return true, ctx.Err()
552+ case err := <-input.EOF():
553+ return true, terminalReadCause(err)
554+ }
555+}
556+
557+// supervisorBackoffDelay is the equal-jitter exponential interval for one
558+// failure count: the cap doubles from supervisorRetryInitial to
559+// supervisorRetryMax, and the interval is half the cap plus a uniform fraction
560+// of the other half, so attempts spread across [cap/2, cap). It is pure so the
561+// cadence can be tested exactly.
562+func supervisorBackoffDelay(attempt uint64, jitter func() float64) time.Duration {
563+ if attempt == 0 {
564+ return 0
565+ }
566+ limit := supervisorRetryInitial
567+ for i := uint64(1); i < attempt && limit < supervisorRetryMax; i++ {
568+ limit *= 2
569+ }
570+ if limit > supervisorRetryMax {
571+ limit = supervisorRetryMax
572+ }
573+ half := limit / 2
574+ fraction := jitter()
575+ if !(fraction >= 0) {
576+ // NaN is unordered: both fraction < 0 and fraction > 1 are false for
577+ // it, so an explicit non-negative test maps NaN and every negative value
578+ // to the minimum fraction. Without this, a NaN would reach the duration
579+ // conversion and yield an undefined interval instead of the documented
580+ // half-cap minimum.
581+ fraction = 0
582+ }
583+ if fraction > 1 {
584+ fraction = 1
585+ }
586+ return half + time.Duration(fraction*float64(half))
587+}
588+
589+// terminalReadCause maps a terminal read end to the Run result: a clean EOF is
590+// an orderly exit (nil), any other read failure is reported.
591+func terminalReadCause(err error) error {
592+ if err == nil || errors.Is(err, io.EOF) {
593+ return nil
594+ }
595+ return err
596+}
597+
598+// stopSupervisorTimer stops and drains a timer, supporting buffered timer
599+// adapters with pre-Go-1.23 reset semantics.
600+func stopSupervisorTimer(timer ports.Timer) {
601+ if timer == nil {
602+ return
603+ }
604+ if !timer.Stop() {
605+ select {
606+ case <-timer.C():
607+ default:
608+ }
609+ }
610+}
611+
612+// supervisorTerminalFailure reports whether a broker failure must end the
613+// process. Only the domain's terminal codes do; every other failure returns the
614+// client to the picker.
615+func supervisorTerminalFailure(err error) bool {
616+ var typed ports.BrokerError
617+ return errors.As(err, &typed) && typed.Terminal()
618+}
619+
620+// supervisorRetryable reports whether a failure is worth another attempt after
621+// backoff. An incompatible carriage or a conflicting policy can never succeed
622+// by retrying, so it is surfaced once and automatic reconnection stops; every
623+// other failure is treated as transient.
624+func supervisorRetryable(err error) bool {
625+ var typed ports.BrokerError
626+ if errors.As(err, &typed) {
627+ switch typed.Code {
628+ case ports.BrokerErrorIncompatible, ports.BrokerErrorConflictingPolicy:
629+ return false
630+ }
631+ }
632+ return true
633+}
634+
635+// normalizeUnavailable presents one connect, subscribe, or connection-loss
636+// failure as a typed ports.BrokerError. A cause that is already a BrokerError is
637+// returned unchanged, so its own code (for example incompatible or conflicting
638+// policy) still selects the non-retryable path; every untyped failure, including
639+// a nil cause, becomes an unavailable BrokerError that preserves the original
640+// error as its local-only Cause.
641+func normalizeUnavailable(err error) error {
642+ if err == nil {
643+ return ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "broker connection lost"}
644+ }
645+ var typed ports.BrokerError
646+ if errors.As(err, &typed) {
647+ return err
648+ }
649+ return ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "broker connection lost", Cause: err}
650+}
651+
652+// notify surfaces a connectivity failure through the optional notifier.
653+func (s *Supervisor) notify(err error) {
654+ if s.cfg.Notify == nil || err == nil {
655+ return
656+ }
657+ s.cfg.Notify(s.State(), err)
658+}
659+
660+// transition applies one reducer event under the supervisor mutex and renders
661+// the resulting state outside it, so a renderer that reads State back never
662+// deadlocks.
663+func (s *Supervisor) transition(event supervisorEvent) {
664+ s.mu.Lock()
665+ s.state = reduceSupervisor(s.state, event)
666+ state := s.state
667+ s.mu.Unlock()
668+ s.cfg.Render(state)
669+}
670+
671+// terminalInputLifetime owns the single read of the controlling terminal. It is
672+// started once and never replaced: a bare io.Reader cannot be interrupted, so
673+// the goroutine may outlive Run when stdin never reaches EOF, exactly like the
674+// attach client's input pump. It never closes caller-owned input, and P5.1a
675+// discards read bytes because picker interaction is a later slice.
676+type terminalInputLifetime struct {
677+ eof chan error
678+ once sync.Once
679+}
680+
681+// startTerminalInputLifetime starts the single terminal read. A nil reader is
682+// treated as an immediate orderly EOF.
683+func startTerminalInputLifetime(in io.Reader) *terminalInputLifetime {
684+ lifetime := &terminalInputLifetime{eof: make(chan error, 1)}
685+ if supervisorNil(in) {
686+ lifetime.finish(io.EOF)
687+ return lifetime
688+ }
689+ go func() {
690+ buffer := make([]byte, 4096)
691+ for {
692+ n, err := in.Read(buffer)
693+ if err != nil {
694+ lifetime.finish(err)
695+ return
696+ }
697+ if n == 0 {
698+ // A zero-byte, nil-error read is legal but cannot make
699+ // progress; keep reading so a well-behaved reader is never
700+ // mistaken for EOF.
701+ continue
702+ }
703+ }
704+ }()
705+ return lifetime
706+}
707+
708+// finish publishes the terminal cause exactly once.
709+func (l *terminalInputLifetime) finish(err error) {
710+ l.once.Do(func() { l.eof <- err })
711+}
712+
713+// EOF is closed-result channel delivering the terminal read cause exactly once.
714+func (l *terminalInputLifetime) EOF() <-chan error { return l.eof }
715+
716+// supervisorNil reports whether a dependency is nil in any shape. A typed nil
717+// satisfies a plain != nil comparison, so the guard would defer the failure to
718+// the first method call, which panics; the reflection check keeps it total.
719+func supervisorNil(dependency any) bool {
720+ if dependency == nil {
721+ return true
722+ }
723+ value := reflect.ValueOf(dependency)
724+ switch value.Kind() {
725+ case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
726+ return value.IsNil()
727+ default:
728+ return false
729+ }
730+}
A internal/usecase/client/supervisor_integration_test.go +213 -0
@@ -0,0 +1,213 @@
1+package client
2+
3+import (
4+ "context"
5+ "encoding/binary"
6+ "errors"
7+ "os"
8+ "path/filepath"
9+ "sync"
10+ "testing"
11+ "time"
12+
13+ "github.com/stretchr/testify/require"
14+
15+ "github.com/bnema/vev/internal/adapters/brokeripc"
16+ "github.com/bnema/vev/internal/ports"
17+)
18+
19+// Real connector integration (Plan 001 P5.1a, GO-007). These tests wire the
20+// production brokeripc Connector into the autonomous supervisor over a real
21+// AF_UNIX broker endpoint, so the readiness handshake, the established-
22+// connection loss, and the cancellation lifetime are pinned against the true
23+// adapter and carriage rather than a scripted fake.
24+
25+const integrationEpoch = ports.BrokerEpoch(0x71)
26+
27+// integrationCore is the minimal admitted broker service one accepted
28+// connection is bound to: a fixed-epoch publication and a coalescing
29+// subscription, which is all the supervisor's readiness handshake needs.
30+type integrationCore struct {
31+ id ports.BrokerConnectionID
32+ epoch ports.BrokerEpoch
33+ changed chan struct{}
34+ done chan struct{}
35+ closed chan struct{}
36+ once sync.Once
37+}
38+
39+func (c *integrationCore) ConnectionID() ports.BrokerConnectionID { return c.id }
40+func (c *integrationCore) Done() <-chan struct{} { return c.done }
41+func (c *integrationCore) Err() error { return nil }
42+
43+func (c *integrationCore) Snapshot() ports.BrokerSnapshot {
44+ return ports.BrokerSnapshot{Epoch: c.epoch, Revision: 1}
45+}
46+
47+func (c *integrationCore) Subscribe() (ports.BrokerSubscription, error) {
48+ return integrationSubscription{changed: c.changed}, nil
49+}
50+
51+func (c *integrationCore) OpenStream(context.Context, ports.BrokerOpenStreamRequest) (ports.BrokerLogicalConnection, error) {
52+ return nil, errors.New("integration core does not support streams")
53+}
54+
55+func (c *integrationCore) CloseStream(ports.BrokerConnectionID, ports.BrokerStreamID) error {
56+ return nil
57+}
58+func (c *integrationCore) AddHost(context.Context, string) error { return nil }
59+func (c *integrationCore) RemoveHost(context.Context, string) (bool, error) { return false, nil }
60+func (c *integrationCore) RequestReconcile(string) {}
61+
62+func (c *integrationCore) Close() error {
63+ c.once.Do(func() {
64+ close(c.done)
65+ close(c.closed)
66+ })
67+ return nil
68+}
69+
70+// integrationSubscription is one idle coalescing subscription.
71+type integrationSubscription struct{ changed chan struct{} }
72+
73+func (s integrationSubscription) Changed() <-chan struct{} { return s.changed }
74+func (s integrationSubscription) Close() {}
75+
76+// integrationAuthority admits one connection per accepted carriage and records
77+// every admitted core, so a test can observe the broker-side end of the
78+// supervisor's connection lifetime.
79+type integrationAuthority struct {
80+ epoch ports.BrokerEpoch
81+
82+ mu sync.Mutex
83+ next uint64
84+ cores []*integrationCore
85+}
86+
87+var _ ports.BrokerAuthority = (*integrationAuthority)(nil)
88+
89+func (a *integrationAuthority) AdmitClient(ctx context.Context) (ports.BrokerService, error) {
90+ if err := ctx.Err(); err != nil {
91+ return nil, err
92+ }
93+ a.mu.Lock()
94+ a.next++
95+ sequence := a.next
96+ a.mu.Unlock()
97+ var id ports.BrokerConnectionID
98+ binary.BigEndian.PutUint64(id[:8], uint64(a.epoch))
99+ binary.BigEndian.PutUint64(id[8:], sequence)
100+ core := &integrationCore{
101+ id: id,
102+ epoch: a.epoch,
103+ changed: make(chan struct{}),
104+ done: make(chan struct{}),
105+ closed: make(chan struct{}),
106+ }
107+ a.mu.Lock()
108+ a.cores = append(a.cores, core)
109+ a.mu.Unlock()
110+ return core, nil
111+}
112+
113+func (a *integrationAuthority) lastCore() *integrationCore {
114+ a.mu.Lock()
115+ defer a.mu.Unlock()
116+ if len(a.cores) == 0 {
117+ return nil
118+ }
119+ return a.cores[len(a.cores)-1]
120+}
121+
122+// startIntegrationBroker binds one real broker IPC endpoint in a short private
123+// directory, so the AF_UNIX path stays well within the platform limit.
124+func startIntegrationBroker(t *testing.T) (*integrationAuthority, ports.BrokerListener, string) {
125+ t.Helper()
126+ root, err := os.MkdirTemp("/tmp", "v")
127+ require.NoError(t, err)
128+ t.Cleanup(func() { _ = os.RemoveAll(root) })
129+ path := brokeripc.SocketPath(filepath.Join(root, "vev"))
130+ authority := &integrationAuthority{epoch: integrationEpoch}
131+ listener, err := brokeripc.Listen(path, integrationEpoch, authority, brokeripc.Config{})
132+ require.NoError(t, err)
133+ t.Cleanup(func() { _ = listener.Close() })
134+ return authority, listener, path
135+}
136+
137+// integrationSupervisor builds one autonomous supervisor over the production
138+// connector and a pinned clock with a controlled terminal.
139+func integrationSupervisor(t *testing.T, path string) (*Supervisor, *supervisorTestClock, *supervisorTestTerminal) {
140+ t.Helper()
141+ clock := newSupervisorTestClock()
142+ reader := newSupervisorTestReader()
143+ t.Cleanup(reader.unblock)
144+ terminal := newSupervisorTestTerminal(reader)
145+ sup := mustSupervisor(t, SupervisorConfig{
146+ Connector: brokeripc.NewConnector(path, brokeripc.Config{}),
147+ Terminal: terminal,
148+ Clock: clock,
149+ Jitter: func() float64 { return 0 },
150+ })
151+ return sup, clock, terminal
152+}
153+
154+// TestSupervisorRealConnectorReadinessAndLoss pins the readiness handshake and
155+// the established-connection loss against a real broker endpoint: the
156+// supervisor becomes Ready only after the broker publishes the admitted
157+// connection's epoch, and a retired connection is observed through Done/Err and
158+// retried with a typed unavailable cause.
159+func TestSupervisorRealConnectorReadinessAndLoss(t *testing.T) {
160+ _, listener, path := startIntegrationBroker(t)
161+ sup, _, terminal := integrationSupervisor(t, path)
162+
163+ ctx, cancel := context.WithCancel(context.Background())
164+ defer cancel()
165+ runErr := make(chan error, 1)
166+ go func() { runErr <- sup.Run(ctx) }()
167+
168+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityReady }, 5*time.Second, time.Millisecond)
169+ require.Equal(t, uint64(1), sup.State().Generation)
170+ require.Equal(t, uint64(0), sup.State().Attempt)
171+ require.NoError(t, sup.State().Err)
172+
173+ // The broker retires the accepted connection; the supervisor observes the
174+ // carriage loss through Done/Err and waits the cadence before retrying.
175+ server, err := listener.Accept()
176+ require.NoError(t, err)
177+ require.NoError(t, server.Close())
178+
179+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityRetryWait }, 5*time.Second, time.Millisecond)
180+ var typed ports.BrokerError
181+ require.ErrorAs(t, sup.State().Err, &typed)
182+ require.Equal(t, ports.BrokerErrorUnavailable, typed.Code)
183+
184+ cancel()
185+ require.ErrorIs(t, <-runErr, context.Canceled)
186+ require.Equal(t, 1, terminal.restoreCount())
187+}
188+
189+// TestSupervisorRealConnectorCancelClosesConnection pins the cancellation
190+// lifetime: cancelling the run closes the supervisor's adopted connection, ends
191+// the broker-side session, and retires the admitted core exactly once.
192+func TestSupervisorRealConnectorCancelClosesConnection(t *testing.T) {
193+ authority, _, path := startIntegrationBroker(t)
194+ sup, _, terminal := integrationSupervisor(t, path)
195+
196+ ctx, cancel := context.WithCancel(context.Background())
197+ runErr := make(chan error, 1)
198+ go func() { runErr <- sup.Run(ctx) }()
199+
200+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityReady }, 5*time.Second, time.Millisecond)
201+ core := authority.lastCore()
202+ require.NotNil(t, core, "the broker must have admitted the connection")
203+
204+ cancel()
205+ require.ErrorIs(t, <-runErr, context.Canceled)
206+ require.Equal(t, 1, terminal.restoreCount())
207+
208+ select {
209+ case <-core.closed:
210+ case <-time.After(5 * time.Second):
211+ t.Fatal("cancelling the supervisor must close the broker connection")
212+ }
213+}
@@ -0,0 +1,213 @@
1+package client
2+
3+import (
4+ "context"
5+ "encoding/binary"
6+ "errors"
7+ "os"
8+ "path/filepath"
9+ "sync"
10+ "testing"
11+ "time"
12+
13+ "github.com/stretchr/testify/require"
14+
15+ "github.com/bnema/vev/internal/adapters/brokeripc"
16+ "github.com/bnema/vev/internal/ports"
17+)
18+
19+// Real connector integration (Plan 001 P5.1a, GO-007). These tests wire the
20+// production brokeripc Connector into the autonomous supervisor over a real
21+// AF_UNIX broker endpoint, so the readiness handshake, the established-
22+// connection loss, and the cancellation lifetime are pinned against the true
23+// adapter and carriage rather than a scripted fake.
24+
25+const integrationEpoch = ports.BrokerEpoch(0x71)
26+
27+// integrationCore is the minimal admitted broker service one accepted
28+// connection is bound to: a fixed-epoch publication and a coalescing
29+// subscription, which is all the supervisor's readiness handshake needs.
30+type integrationCore struct {
31+ id ports.BrokerConnectionID
32+ epoch ports.BrokerEpoch
33+ changed chan struct{}
34+ done chan struct{}
35+ closed chan struct{}
36+ once sync.Once
37+}
38+
39+func (c *integrationCore) ConnectionID() ports.BrokerConnectionID { return c.id }
40+func (c *integrationCore) Done() <-chan struct{} { return c.done }
41+func (c *integrationCore) Err() error { return nil }
42+
43+func (c *integrationCore) Snapshot() ports.BrokerSnapshot {
44+ return ports.BrokerSnapshot{Epoch: c.epoch, Revision: 1}
45+}
46+
47+func (c *integrationCore) Subscribe() (ports.BrokerSubscription, error) {
48+ return integrationSubscription{changed: c.changed}, nil
49+}
50+
51+func (c *integrationCore) OpenStream(context.Context, ports.BrokerOpenStreamRequest) (ports.BrokerLogicalConnection, error) {
52+ return nil, errors.New("integration core does not support streams")
53+}
54+
55+func (c *integrationCore) CloseStream(ports.BrokerConnectionID, ports.BrokerStreamID) error {
56+ return nil
57+}
58+func (c *integrationCore) AddHost(context.Context, string) error { return nil }
59+func (c *integrationCore) RemoveHost(context.Context, string) (bool, error) { return false, nil }
60+func (c *integrationCore) RequestReconcile(string) {}
61+
62+func (c *integrationCore) Close() error {
63+ c.once.Do(func() {
64+ close(c.done)
65+ close(c.closed)
66+ })
67+ return nil
68+}
69+
70+// integrationSubscription is one idle coalescing subscription.
71+type integrationSubscription struct{ changed chan struct{} }
72+
73+func (s integrationSubscription) Changed() <-chan struct{} { return s.changed }
74+func (s integrationSubscription) Close() {}
75+
76+// integrationAuthority admits one connection per accepted carriage and records
77+// every admitted core, so a test can observe the broker-side end of the
78+// supervisor's connection lifetime.
79+type integrationAuthority struct {
80+ epoch ports.BrokerEpoch
81+
82+ mu sync.Mutex
83+ next uint64
84+ cores []*integrationCore
85+}
86+
87+var _ ports.BrokerAuthority = (*integrationAuthority)(nil)
88+
89+func (a *integrationAuthority) AdmitClient(ctx context.Context) (ports.BrokerService, error) {
90+ if err := ctx.Err(); err != nil {
91+ return nil, err
92+ }
93+ a.mu.Lock()
94+ a.next++
95+ sequence := a.next
96+ a.mu.Unlock()
97+ var id ports.BrokerConnectionID
98+ binary.BigEndian.PutUint64(id[:8], uint64(a.epoch))
99+ binary.BigEndian.PutUint64(id[8:], sequence)
100+ core := &integrationCore{
101+ id: id,
102+ epoch: a.epoch,
103+ changed: make(chan struct{}),
104+ done: make(chan struct{}),
105+ closed: make(chan struct{}),
106+ }
107+ a.mu.Lock()
108+ a.cores = append(a.cores, core)
109+ a.mu.Unlock()
110+ return core, nil
111+}
112+
113+func (a *integrationAuthority) lastCore() *integrationCore {
114+ a.mu.Lock()
115+ defer a.mu.Unlock()
116+ if len(a.cores) == 0 {
117+ return nil
118+ }
119+ return a.cores[len(a.cores)-1]
120+}
121+
122+// startIntegrationBroker binds one real broker IPC endpoint in a short private
123+// directory, so the AF_UNIX path stays well within the platform limit.
124+func startIntegrationBroker(t *testing.T) (*integrationAuthority, ports.BrokerListener, string) {
125+ t.Helper()
126+ root, err := os.MkdirTemp("/tmp", "v")
127+ require.NoError(t, err)
128+ t.Cleanup(func() { _ = os.RemoveAll(root) })
129+ path := brokeripc.SocketPath(filepath.Join(root, "vev"))
130+ authority := &integrationAuthority{epoch: integrationEpoch}
131+ listener, err := brokeripc.Listen(path, integrationEpoch, authority, brokeripc.Config{})
132+ require.NoError(t, err)
133+ t.Cleanup(func() { _ = listener.Close() })
134+ return authority, listener, path
135+}
136+
137+// integrationSupervisor builds one autonomous supervisor over the production
138+// connector and a pinned clock with a controlled terminal.
139+func integrationSupervisor(t *testing.T, path string) (*Supervisor, *supervisorTestClock, *supervisorTestTerminal) {
140+ t.Helper()
141+ clock := newSupervisorTestClock()
142+ reader := newSupervisorTestReader()
143+ t.Cleanup(reader.unblock)
144+ terminal := newSupervisorTestTerminal(reader)
145+ sup := mustSupervisor(t, SupervisorConfig{
146+ Connector: brokeripc.NewConnector(path, brokeripc.Config{}),
147+ Terminal: terminal,
148+ Clock: clock,
149+ Jitter: func() float64 { return 0 },
150+ })
151+ return sup, clock, terminal
152+}
153+
154+// TestSupervisorRealConnectorReadinessAndLoss pins the readiness handshake and
155+// the established-connection loss against a real broker endpoint: the
156+// supervisor becomes Ready only after the broker publishes the admitted
157+// connection's epoch, and a retired connection is observed through Done/Err and
158+// retried with a typed unavailable cause.
159+func TestSupervisorRealConnectorReadinessAndLoss(t *testing.T) {
160+ _, listener, path := startIntegrationBroker(t)
161+ sup, _, terminal := integrationSupervisor(t, path)
162+
163+ ctx, cancel := context.WithCancel(context.Background())
164+ defer cancel()
165+ runErr := make(chan error, 1)
166+ go func() { runErr <- sup.Run(ctx) }()
167+
168+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityReady }, 5*time.Second, time.Millisecond)
169+ require.Equal(t, uint64(1), sup.State().Generation)
170+ require.Equal(t, uint64(0), sup.State().Attempt)
171+ require.NoError(t, sup.State().Err)
172+
173+ // The broker retires the accepted connection; the supervisor observes the
174+ // carriage loss through Done/Err and waits the cadence before retrying.
175+ server, err := listener.Accept()
176+ require.NoError(t, err)
177+ require.NoError(t, server.Close())
178+
179+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityRetryWait }, 5*time.Second, time.Millisecond)
180+ var typed ports.BrokerError
181+ require.ErrorAs(t, sup.State().Err, &typed)
182+ require.Equal(t, ports.BrokerErrorUnavailable, typed.Code)
183+
184+ cancel()
185+ require.ErrorIs(t, <-runErr, context.Canceled)
186+ require.Equal(t, 1, terminal.restoreCount())
187+}
188+
189+// TestSupervisorRealConnectorCancelClosesConnection pins the cancellation
190+// lifetime: cancelling the run closes the supervisor's adopted connection, ends
191+// the broker-side session, and retires the admitted core exactly once.
192+func TestSupervisorRealConnectorCancelClosesConnection(t *testing.T) {
193+ authority, _, path := startIntegrationBroker(t)
194+ sup, _, terminal := integrationSupervisor(t, path)
195+
196+ ctx, cancel := context.WithCancel(context.Background())
197+ runErr := make(chan error, 1)
198+ go func() { runErr <- sup.Run(ctx) }()
199+
200+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityReady }, 5*time.Second, time.Millisecond)
201+ core := authority.lastCore()
202+ require.NotNil(t, core, "the broker must have admitted the connection")
203+
204+ cancel()
205+ require.ErrorIs(t, <-runErr, context.Canceled)
206+ require.Equal(t, 1, terminal.restoreCount())
207+
208+ select {
209+ case <-core.closed:
210+ case <-time.After(5 * time.Second):
211+ t.Fatal("cancelling the supervisor must close the broker connection")
212+ }
213+}
A internal/usecase/client/supervisor_test.go +1284 -0
@@ -0,0 +1,1284 @@
1+package client
2+
3+import (
4+ "context"
5+ "errors"
6+ "io"
7+ "math"
8+ "runtime"
9+ "strings"
10+ "sync"
11+ "testing"
12+ "time"
13+
14+ "github.com/stretchr/testify/require"
15+
16+ "github.com/bnema/vev/internal/domain"
17+ "github.com/bnema/vev/internal/ports"
18+)
19+
20+// supervisorTestClock is a manually fired ports.Clock. NewTimer publishes each
21+// timer on a buffered channel so a test can fire it deterministically without a
22+// sleep, and Stop is recorded so a test can prove every retry timer was
23+// retired.
24+type supervisorTestClock struct {
25+ mu sync.Mutex
26+ now time.Time
27+ timers []*supervisorTestTimer
28+ created chan *supervisorTestTimer
29+}
30+
31+func newSupervisorTestClock() *supervisorTestClock {
32+ return &supervisorTestClock{now: time.Unix(1000, 0), created: make(chan *supervisorTestTimer, 32)}
33+}
34+
35+func (c *supervisorTestClock) Now() time.Time {
36+ c.mu.Lock()
37+ defer c.mu.Unlock()
38+ return c.now
39+}
40+
41+func (c *supervisorTestClock) NewTimer(delay time.Duration) ports.Timer {
42+ timer := &supervisorTestTimer{ch: make(chan time.Time, 1), delay: delay}
43+ c.mu.Lock()
44+ c.timers = append(c.timers, timer)
45+ c.mu.Unlock()
46+ select {
47+ case c.created <- timer:
48+ default:
49+ }
50+ return timer
51+}
52+
53+func (c *supervisorTestClock) awaitTimer(t *testing.T) *supervisorTestTimer {
54+ t.Helper()
55+ select {
56+ case timer := <-c.created:
57+ return timer
58+ case <-time.After(5 * time.Second):
59+ t.Fatal("supervisor did not schedule a retry timer")
60+ return nil
61+ }
62+}
63+
64+func (c *supervisorTestClock) timerCount() int {
65+ c.mu.Lock()
66+ defer c.mu.Unlock()
67+ return len(c.timers)
68+}
69+
70+func (c *supervisorTestClock) liveTimers() int {
71+ c.mu.Lock()
72+ defer c.mu.Unlock()
73+ live := 0
74+ for _, timer := range c.timers {
75+ if !timer.stopped() {
76+ live++
77+ }
78+ }
79+ return live
80+}
81+
82+func (c *supervisorTestClock) allStopped() bool {
83+ c.mu.Lock()
84+ defer c.mu.Unlock()
85+ for _, timer := range c.timers {
86+ if !timer.stopped() {
87+ return false
88+ }
89+ }
90+ return true
91+}
92+
93+type supervisorTestTimer struct {
94+ ch chan time.Time
95+ delay time.Duration
96+ mu sync.Mutex
97+ stoppedFlag bool
98+}
99+
100+func (t *supervisorTestTimer) C() <-chan time.Time { return t.ch }
101+
102+func (t *supervisorTestTimer) Reset(time.Duration) bool { return false }
103+
104+func (t *supervisorTestTimer) Stop() bool {
105+ t.mu.Lock()
106+ defer t.mu.Unlock()
107+ if t.stoppedFlag {
108+ return false
109+ }
110+ t.stoppedFlag = true
111+ return true
112+}
113+
114+func (t *supervisorTestTimer) stopped() bool {
115+ t.mu.Lock()
116+ defer t.mu.Unlock()
117+ return t.stoppedFlag
118+}
119+
120+// fire delivers the timer without blocking, so a test may fire before or after
121+// the supervisor reaches its select.
122+func (t *supervisorTestTimer) fire() {
123+ select {
124+ case t.ch <- time.Now():
125+ default:
126+ }
127+}
128+
129+// supervisorTestTerminal records raw-mode enters and restores. Its reader is
130+// shared so tests can model input EOF and a live terminal.
131+type supervisorTestTerminal struct {
132+ in io.Reader
133+ enterErr error
134+
135+ mu sync.Mutex
136+ restores int
137+}
138+
139+func newSupervisorTestTerminal(in io.Reader) *supervisorTestTerminal {
140+ return &supervisorTestTerminal{in: in}
141+}
142+
143+func (t *supervisorTestTerminal) EnterRaw() (func() error, error) {
144+ if t.enterErr != nil {
145+ return nil, t.enterErr
146+ }
147+ return func() error {
148+ t.mu.Lock()
149+ t.restores++
150+ t.mu.Unlock()
151+ return nil
152+ }, nil
153+}
154+
155+func (t *supervisorTestTerminal) Geometry() (domain.Geometry, error) {
156+ return domain.Geometry{}, nil
157+}
158+
159+func (t *supervisorTestTerminal) ResizeEvents() <-chan domain.Geometry { return nil }
160+
161+func (t *supervisorTestTerminal) In() io.Reader { return t.in }
162+
163+func (t *supervisorTestTerminal) Out() io.Writer { return io.Discard }
164+
165+func (t *supervisorTestTerminal) Flush() error { return nil }
166+
167+func (t *supervisorTestTerminal) restoreCount() int {
168+ t.mu.Lock()
169+ defer t.mu.Unlock()
170+ return t.restores
171+}
172+
173+// supervisorTestConnector scripts connection attempts and records concurrency,
174+// so a test can prove attempts never overlap.
175+type supervisorTestConnector struct {
176+ connect func(ctx context.Context, call int) (ports.BrokerService, error)
177+
178+ mu sync.Mutex
179+ calls int
180+ inFlight int
181+ maxInFlight int
182+ started chan int
183+}
184+
185+func newSupervisorTestConnector(connect func(ctx context.Context, call int) (ports.BrokerService, error)) *supervisorTestConnector {
186+ return &supervisorTestConnector{connect: connect, started: make(chan int, 32)}
187+}
188+
189+func (c *supervisorTestConnector) Connect(ctx context.Context) (ports.BrokerService, error) {
190+ c.mu.Lock()
191+ c.calls++
192+ call := c.calls
193+ c.inFlight++
194+ if c.inFlight > c.maxInFlight {
195+ c.maxInFlight = c.inFlight
196+ }
197+ c.mu.Unlock()
198+ select {
199+ case c.started <- call:
200+ default:
201+ }
202+ defer func() {
203+ c.mu.Lock()
204+ c.inFlight--
205+ c.mu.Unlock()
206+ }()
207+ if c.connect == nil {
208+ return nil, ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "no script"}
209+ }
210+ return c.connect(ctx, call)
211+}
212+
213+func (c *supervisorTestConnector) callCount() int {
214+ c.mu.Lock()
215+ defer c.mu.Unlock()
216+ return c.calls
217+}
218+
219+func (c *supervisorTestConnector) maxConcurrent() int {
220+ c.mu.Lock()
221+ defer c.mu.Unlock()
222+ return c.maxInFlight
223+}
224+
225+func (c *supervisorTestConnector) inFlightCount() int {
226+ c.mu.Lock()
227+ defer c.mu.Unlock()
228+ return c.inFlight
229+}
230+
231+func (c *supervisorTestConnector) awaitStart(t *testing.T) int {
232+ t.Helper()
233+ select {
234+ case call := <-c.started:
235+ return call
236+ case <-time.After(5 * time.Second):
237+ t.Fatal("supervisor did not start a connection attempt")
238+ return 0
239+ }
240+}
241+
242+// supervisorTestHub is one broker publication source. subscribe hands out the
243+// current change channel, and publish retires and replaces it, so a stale
244+// subscription can never observe a newer publication.
245+type supervisorTestHub struct {
246+ mu sync.Mutex
247+ snapshot ports.BrokerSnapshot
248+ changed chan struct{}
249+}
250+
251+func newSupervisorTestHub() *supervisorTestHub {
252+ return &supervisorTestHub{changed: make(chan struct{})}
253+}
254+
255+func (h *supervisorTestHub) publish(snapshot ports.BrokerSnapshot) {
256+ h.mu.Lock()
257+ h.snapshot = snapshot
258+ changed := h.changed
259+ h.changed = make(chan struct{})
260+ h.mu.Unlock()
261+ close(changed)
262+}
263+
264+func (h *supervisorTestHub) current() ports.BrokerSnapshot {
265+ h.mu.Lock()
266+ defer h.mu.Unlock()
267+ return h.snapshot
268+}
269+
270+func (h *supervisorTestHub) subscribe(closeFn func()) ports.BrokerSubscription {
271+ h.mu.Lock()
272+ changed := h.changed
273+ h.mu.Unlock()
274+ return supervisorTestSubscription{changed: changed, closeFn: closeFn}
275+}
276+
277+type supervisorTestSubscription struct {
278+ changed <-chan struct{}
279+ closeFn func()
280+}
281+
282+func (s supervisorTestSubscription) Changed() <-chan struct{} { return s.changed }
283+
284+func (s supervisorTestSubscription) Close() {
285+ if s.closeFn != nil {
286+ s.closeFn()
287+ }
288+}
289+
290+// supervisorTestService is one scripted BrokerService. Close is observable so a
291+// test can prove a retired or late service was closed, and Close can be held open
292+// so a test can order a parent cancellation strictly before a settle decision that
293+// runs after the connection is released.
294+type supervisorTestService struct {
295+ id ports.BrokerConnectionID
296+ hub *supervisorTestHub
297+
298+ mu sync.Mutex
299+ err error
300+ doneOnce sync.Once
301+ done chan struct{}
302+
303+ closeOnce sync.Once
304+ closed chan struct{}
305+ subClosed chan struct{}
306+
307+ // closeGate, when non-nil, holds Close until releaseClose is called, and
308+ // closeEntered reports that Close began waiting on it.
309+ closeGate chan struct{}
310+ closeEntered chan struct{}
311+}
312+
313+func newSupervisorTestService(id ports.BrokerConnectionID) *supervisorTestService {
314+ return &supervisorTestService{
315+ id: id,
316+ hub: newSupervisorTestHub(),
317+ done: make(chan struct{}),
318+ closed: make(chan struct{}),
319+ subClosed: make(chan struct{}, 1),
320+ }
321+}
322+
323+func (s *supervisorTestService) ConnectionID() ports.BrokerConnectionID { return s.id }
324+
325+func (s *supervisorTestService) Done() <-chan struct{} { return s.done }
326+
327+func (s *supervisorTestService) Err() error {
328+ s.mu.Lock()
329+ defer s.mu.Unlock()
330+ return s.err
331+}
332+
333+func (s *supervisorTestService) Snapshot() ports.BrokerSnapshot { return s.hub.current() }
334+
335+func (s *supervisorTestService) Subscribe() (ports.BrokerSubscription, error) {
336+ return s.hub.subscribe(func() {
337+ select {
338+ case s.subClosed <- struct{}{}:
339+ default:
340+ }
341+ }), nil
342+}
343+
344+func (s *supervisorTestService) OpenStream(context.Context, ports.BrokerOpenStreamRequest) (ports.BrokerLogicalConnection, error) {
345+ return nil, errors.New("supervisor test service does not support streams")
346+}
347+
348+func (s *supervisorTestService) CloseStream(ports.BrokerConnectionID, ports.BrokerStreamID) error {
349+ return nil
350+}
351+
352+func (s *supervisorTestService) AddHost(context.Context, string) error { return nil }
353+
354+func (s *supervisorTestService) RemoveHost(context.Context, string) (bool, error) {
355+ return false, nil
356+}
357+
358+func (s *supervisorTestService) RequestReconcile(string) {}
359+
360+func (s *supervisorTestService) Close() error {
361+ s.closeOnce.Do(func() {
362+ if s.closeGate != nil {
363+ select {
364+ case s.closeEntered <- struct{}{}:
365+ default:
366+ }
367+ <-s.closeGate
368+ }
369+ close(s.closed)
370+ s.terminate(nil)
371+ })
372+ return nil
373+}
374+
375+// holdClose makes Close block until releaseClose is called, so a test can order a
376+// parent cancellation strictly before a settle decision that runs after the
377+// connection is released. closingCh reports that the release was entered.
378+func (s *supervisorTestService) holdClose() {
379+ s.closeGate = make(chan struct{})
380+ s.closeEntered = make(chan struct{}, 1)
381+}
382+
383+// releaseClose releases a held Close.
384+func (s *supervisorTestService) releaseClose() {
385+ if s.closeGate != nil {
386+ close(s.closeGate)
387+ }
388+}
389+
390+// closingCh reports that the held release was entered.
391+func (s *supervisorTestService) closingCh() <-chan struct{} { return s.closeEntered }
392+
393+// terminate removes the connection, recording the first cause exactly once.
394+func (s *supervisorTestService) terminate(err error) {
395+ if err != nil {
396+ s.mu.Lock()
397+ if s.err == nil {
398+ s.err = err
399+ }
400+ s.mu.Unlock()
401+ }
402+ s.doneOnce.Do(func() { close(s.done) })
403+}
404+
405+// lose models an unexplained broker-side loss.
406+func (s *supervisorTestService) lose(err error) { s.terminate(err) }
407+
408+func (s *supervisorTestService) publish(epoch ports.BrokerEpoch, revision ports.BrokerRevision) {
409+ s.hub.publish(ports.BrokerSnapshot{Epoch: epoch, Revision: revision})
410+}
411+
412+func (s *supervisorTestService) closedCh() <-chan struct{} { return s.closed }
413+
414+func mustSupervisor(t *testing.T, cfg SupervisorConfig) *Supervisor {
415+ t.Helper()
416+ sup, err := NewSupervisor(cfg)
417+ require.NoError(t, err)
418+ return sup
419+}
420+
421+// supervisorTestReader blocks until unblocked, so a test controls exactly when
422+// the owned terminal input reaches EOF.
423+type supervisorTestReader struct {
424+ mu sync.Mutex
425+ done chan struct{}
426+}
427+
428+func newSupervisorTestReader() *supervisorTestReader {
429+ return &supervisorTestReader{done: make(chan struct{})}
430+}
431+
432+func (r *supervisorTestReader) Read([]byte) (int, error) {
433+ <-r.done
434+ return 0, io.EOF
435+}
436+
437+func (r *supervisorTestReader) unblock() {
438+ r.mu.Lock()
439+ defer r.mu.Unlock()
440+ select {
441+ case <-r.done:
442+ default:
443+ close(r.done)
444+ }
445+}
446+
447+func TestSupervisorReducerTransitions(t *testing.T) {
448+ transient := errors.New("unavailable")
449+ incompatible := ports.BrokerError{Code: ports.BrokerErrorIncompatible, Text: "version"}
450+ exited := ports.BrokerError{Code: ports.BrokerErrorExplicitExit, Text: "exit"}
451+ picker := State{Presentation: PresentPicker, Connectivity: ConnectivityDisconnected}
452+
453+ tests := []struct {
454+ name string
455+ state State
456+ event supervisorEvent
457+ want State
458+ }{
459+ {
460+ name: "begin attempt claims a generation and connects",
461+ state: picker,
462+ event: supervisorEvent{kind: supervisorBeginAttempt},
463+ want: State{Presentation: PresentPicker, Connectivity: ConnectivityConnectingBroker, Attempt: 0, Generation: 1},
464+ },
465+ {
466+ name: "ready resets the retry cadence",
467+ state: State{Presentation: PresentPicker, Connectivity: ConnectivityConnectingBroker, Attempt: 3, Generation: 4, Err: transient},
468+ event: supervisorEvent{kind: supervisorReady},
469+ want: State{Presentation: PresentPicker, Connectivity: ConnectivityReady, Attempt: 0, Generation: 4},
470+ },
471+ {
472+ name: "transient failure waits with a bumped attempt",
473+ state: State{Presentation: PresentPicker, Connectivity: ConnectivityConnectingBroker, Attempt: 0, Generation: 2},
474+ event: supervisorEvent{kind: supervisorTransientFailure, err: transient},
475+ want: State{Presentation: PresentPicker, Connectivity: ConnectivityRetryWait, Attempt: 1, Generation: 2, Err: transient},
476+ },
477+ {
478+ name: "broker loss returns to picker retry wait",
479+ state: State{Presentation: PresentPicker, Connectivity: ConnectivityReady, Attempt: 0, Generation: 5},
480+ event: supervisorEvent{kind: supervisorBrokerLoss, err: transient},
481+ want: State{Presentation: PresentPicker, Connectivity: ConnectivityRetryWait, Attempt: 1, Generation: 5, Err: transient},
482+ },
483+ {
484+ name: "non-retryable failure disconnects and stays on the picker",
485+ state: State{Presentation: PresentPicker, Connectivity: ConnectivityConnectingBroker, Attempt: 1, Generation: 3},
486+ event: supervisorEvent{kind: supervisorNonRetryable, err: incompatible},
487+ want: State{Presentation: PresentPicker, Connectivity: ConnectivityDisconnected, Attempt: 1, Generation: 3, Err: incompatible},
488+ },
489+ {
490+ name: "terminal failure terminates",
491+ state: State{Presentation: PresentPicker, Connectivity: ConnectivityReady, Generation: 7},
492+ event: supervisorEvent{kind: supervisorTerminal, err: exited},
493+ want: State{Presentation: PresentTerminating, Connectivity: ConnectivityDisconnected, Generation: 7, Err: exited},
494+ },
495+ }
496+
497+ for _, tt := range tests {
498+ t.Run(tt.name, func(t *testing.T) {
499+ require.Equal(t, tt.want, reduceSupervisor(tt.state, tt.event))
500+ })
501+ }
502+}
503+
504+func TestSupervisorBackoffCadence(t *testing.T) {
505+ zero := func() float64 { return 0 }
506+ half := func() float64 { return 0.5 }
507+ clamped := func() float64 { return 1.5 }
508+ nan := func() float64 { return math.NaN() }
509+ negInf := func() float64 { return math.Inf(-1) }
510+ posInf := func() float64 { return math.Inf(1) }
511+
512+ tests := []struct {
513+ name string
514+ attempt uint64
515+ jitter func() float64
516+ want time.Duration
517+ }{
518+ {"no attempt has no delay", 0, zero, 0},
519+ {"first failure halves 100ms", 1, zero, 50 * time.Millisecond},
520+ {"second failure halves 200ms", 2, zero, 100 * time.Millisecond},
521+ {"third failure halves 400ms", 3, zero, 200 * time.Millisecond},
522+ {"fourth failure halves 800ms", 4, zero, 400 * time.Millisecond},
523+ {"fifth failure halves 1.6s", 5, zero, 800 * time.Millisecond},
524+ {"cap reached at 2s", 6, zero, time.Second},
525+ {"cap stays at 2s", 9, zero, time.Second},
526+ {"equal jitter reaches three quarters", 1, half, 75 * time.Millisecond},
527+ {"equal jitter scales with the cap", 6, half, 1500 * time.Millisecond},
528+ {"full jitter reaches the cap", 1, clamped, 100 * time.Millisecond},
529+ {"full jitter stays capped", 6, clamped, 2 * time.Second},
530+ {"NaN jitter clamps to the minimum half", 1, nan, 50 * time.Millisecond},
531+ {"NaN jitter stays at half the cap", 6, nan, time.Second},
532+ {"negative jitter clamps to the minimum half", 1, negInf, 50 * time.Millisecond},
533+ {"negative jitter stays at half the cap", 6, negInf, time.Second},
534+ {"positive infinity clamps to the cap", 1, posInf, 100 * time.Millisecond},
535+ {"positive infinity stays capped", 6, posInf, 2 * time.Second},
536+ }
537+
538+ for _, tt := range tests {
539+ t.Run(tt.name, func(t *testing.T) {
540+ require.Equal(t, tt.want, supervisorBackoffDelay(tt.attempt, tt.jitter))
541+ })
542+ }
543+}
544+
545+func TestSupervisorPickerWithoutBroker(t *testing.T) {
546+ ctx, cancel := context.WithCancel(context.Background())
547+ defer cancel()
548+
549+ clock := newSupervisorTestClock()
550+ reader := newSupervisorTestReader()
551+ t.Cleanup(reader.unblock)
552+ terminal := newSupervisorTestTerminal(reader)
553+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
554+ return nil, ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "broker down"}
555+ })
556+
557+ var (
558+ renderMu sync.Mutex
559+ renders []State
560+ )
561+ sup := mustSupervisor(t, SupervisorConfig{
562+ Connector: connector,
563+ Terminal: terminal,
564+ Clock: clock,
565+ Jitter: func() float64 { return 0 },
566+ Render: func(state State) {
567+ renderMu.Lock()
568+ renders = append(renders, state)
569+ renderMu.Unlock()
570+ },
571+ })
572+
573+ runErr := make(chan error, 1)
574+ go func() { runErr <- sup.Run(ctx) }()
575+
576+ first := clock.awaitTimer(t)
577+ require.Equal(t, 50*time.Millisecond, first.delay, "the first retry is the equal-jitter half of 100ms")
578+ require.Equal(t, 1, connector.awaitStart(t))
579+ require.Equal(t, State{Presentation: PresentPicker, Connectivity: ConnectivityRetryWait, Attempt: 1, Generation: 1, Err: ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "broker down"}}, sup.State())
580+
581+ first.fire()
582+ second := clock.awaitTimer(t)
583+ require.Equal(t, 100*time.Millisecond, second.delay, "the second retry doubles the cap")
584+ require.Equal(t, 2, connector.awaitStart(t))
585+ require.Equal(t, PresentPicker, sup.State().Presentation, "the picker stays up while disconnected")
586+ require.Equal(t, ConnectivityRetryWait, sup.State().Connectivity)
587+ require.Equal(t, uint64(2), sup.State().Generation)
588+
589+ cancel()
590+ require.ErrorIs(t, <-runErr, context.Canceled)
591+ require.Equal(t, 1, terminal.restoreCount())
592+ require.Equal(t, 1, connector.maxConcurrent())
593+
594+ renderMu.Lock()
595+ defer renderMu.Unlock()
596+ require.NotEmpty(t, renders)
597+ require.Equal(t, PresentTerminating, renders[len(renders)-1].Presentation)
598+ for _, state := range renders[:len(renders)-1] {
599+ require.Equal(t, PresentPicker, state.Presentation, "P5.1a never leaves the picker")
600+ }
601+}
602+
603+func TestSupervisorBrokerLossReconnectsWithFreshGeneration(t *testing.T) {
604+ ctx, cancel := context.WithCancel(context.Background())
605+ defer cancel()
606+
607+ clock := newSupervisorTestClock()
608+ reader := newSupervisorTestReader()
609+ t.Cleanup(reader.unblock)
610+ terminal := newSupervisorTestTerminal(reader)
611+
612+ first := newSupervisorTestService(ports.BrokerConnectionID{1})
613+ first.publish(1, 1)
614+ second := newSupervisorTestService(ports.BrokerConnectionID{2})
615+
616+ connector := newSupervisorTestConnector(func(_ context.Context, call int) (ports.BrokerService, error) {
617+ switch call {
618+ case 1:
619+ return first, nil
620+ default:
621+ return second, nil
622+ }
623+ })
624+
625+ sup := mustSupervisor(t, SupervisorConfig{
626+ Connector: connector,
627+ Terminal: terminal,
628+ Clock: clock,
629+ Jitter: func() float64 { return 0 },
630+ })
631+
632+ runErr := make(chan error, 1)
633+ go func() { runErr <- sup.Run(ctx) }()
634+
635+ require.Equal(t, 1, connector.awaitStart(t))
636+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityReady }, 5*time.Second, time.Millisecond)
637+ require.Equal(t, uint64(1), sup.State().Generation)
638+ require.Equal(t, uint64(0), sup.State().Attempt)
639+
640+ // The idle broker connection is lost: the supervisor retires it and waits
641+ // the first cadence before reconnecting.
642+ first.lose(ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "idle loss"})
643+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityRetryWait }, 5*time.Second, time.Millisecond)
644+ require.Equal(t, uint64(1), sup.State().Attempt)
645+ select {
646+ case <-first.closedCh():
647+ default:
648+ t.Fatal("the lost connection was not retired")
649+ }
650+ select {
651+ case <-first.subClosed:
652+ default:
653+ t.Fatal("the lost subscription was not retired")
654+ }
655+
656+ retry := clock.awaitTimer(t)
657+ require.Equal(t, 50*time.Millisecond, retry.delay)
658+ retry.fire()
659+ require.Equal(t, 2, connector.awaitStart(t))
660+
661+ // Wait until the replacement is waiting for its first publication.
662+ require.Eventually(t, func() bool {
663+ return sup.State().Connectivity == ConnectivityConnectingBroker && sup.State().Generation == 2
664+ }, 5*time.Second, time.Millisecond)
665+
666+ // A publication on the retired connection must never make the replacement
667+ // Ready; only the replacement's own fresh epoch does.
668+ first.publish(1, 2)
669+ require.Never(t, func() bool { return sup.State().Connectivity == ConnectivityReady }, 50*time.Millisecond, 5*time.Millisecond)
670+
671+ second.publish(2, 1)
672+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityReady }, 5*time.Second, time.Millisecond)
673+ require.Equal(t, uint64(2), sup.State().Generation)
674+ require.Equal(t, uint64(0), sup.State().Attempt, "readiness resets the cadence")
675+
676+ cancel()
677+ require.ErrorIs(t, <-runErr, context.Canceled)
678+ require.Equal(t, 1, terminal.restoreCount())
679+ select {
680+ case <-second.closedCh():
681+ default:
682+ t.Fatal("the active connection was not retired on shutdown")
683+ }
684+}
685+
686+func TestSupervisorCancellationClosesLateConnection(t *testing.T) {
687+ ctx, cancel := context.WithCancel(context.Background())
688+
689+ clock := newSupervisorTestClock()
690+ reader := newSupervisorTestReader()
691+ t.Cleanup(reader.unblock)
692+ terminal := newSupervisorTestTerminal(reader)
693+
694+ release := make(chan struct{})
695+ late := newSupervisorTestService(ports.BrokerConnectionID{9})
696+ late.publish(1, 1)
697+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
698+ <-release
699+ return late, nil
700+ })
701+
702+ sup := mustSupervisor(t, SupervisorConfig{
703+ Connector: connector,
704+ Terminal: terminal,
705+ Clock: clock,
706+ Jitter: func() float64 { return 0 },
707+ })
708+
709+ runErr := make(chan error, 1)
710+ go func() { runErr <- sup.Run(ctx) }()
711+ require.Equal(t, 1, connector.awaitStart(t))
712+
713+ // Cancel while the connector ignores cancellation, then let it return a
714+ // live service: the supervisor must close the late superseded success.
715+ cancel()
716+ close(release)
717+
718+ require.ErrorIs(t, <-runErr, context.Canceled)
719+ select {
720+ case <-late.closedCh():
721+ default:
722+ t.Fatal("a late superseded success must be closed")
723+ }
724+ require.Equal(t, 1, terminal.restoreCount())
725+ require.Zero(t, connector.inFlightCount())
726+}
727+
728+func TestSupervisorIncompatibleStopsReconnecting(t *testing.T) {
729+ ctx, cancel := context.WithCancel(context.Background())
730+ defer cancel()
731+
732+ clock := newSupervisorTestClock()
733+ reader := newSupervisorTestReader()
734+ t.Cleanup(reader.unblock)
735+ terminal := newSupervisorTestTerminal(reader)
736+
737+ incompatible := ports.BrokerError{Code: ports.BrokerErrorIncompatible, Text: "protocol"}
738+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
739+ return nil, incompatible
740+ })
741+ notified := make(chan error, 4)
742+ sup := mustSupervisor(t, SupervisorConfig{
743+ Connector: connector,
744+ Terminal: terminal,
745+ Clock: clock,
746+ Jitter: func() float64 { return 0 },
747+ Notify: func(_ State, err error) { notified <- err },
748+ })
749+
750+ runErr := make(chan error, 1)
751+ go func() { runErr <- sup.Run(ctx) }()
752+
753+ select {
754+ case err := <-notified:
755+ var typed ports.BrokerError
756+ require.ErrorAs(t, err, &typed)
757+ require.Equal(t, ports.BrokerErrorIncompatible, typed.Code)
758+ case <-time.After(5 * time.Second):
759+ t.Fatal("the incompatible failure was not surfaced")
760+ }
761+
762+ require.Equal(t, State{Presentation: PresentPicker, Connectivity: ConnectivityDisconnected, Generation: 1, Err: incompatible}, sup.State())
763+ require.Never(t, func() bool { return connector.callCount() > 1 }, 50*time.Millisecond, 5*time.Millisecond)
764+ require.Zero(t, clock.timerCount(), "a non-retryable failure schedules no retry timer")
765+
766+ cancel()
767+ require.ErrorIs(t, <-runErr, context.Canceled)
768+ require.Equal(t, 1, terminal.restoreCount())
769+}
770+
771+func TestSupervisorTerminalEOFRestoresRawOnce(t *testing.T) {
772+ clock := newSupervisorTestClock()
773+ terminal := newSupervisorTestTerminal(strings.NewReader(""))
774+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
775+ return nil, ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "down"}
776+ })
777+
778+ sup := mustSupervisor(t, SupervisorConfig{
779+ Connector: connector,
780+ Terminal: terminal,
781+ Clock: clock,
782+ Jitter: func() float64 { return 0 },
783+ })
784+
785+ require.NoError(t, sup.Run(context.Background()), "a clean terminal EOF is an orderly exit")
786+ require.Equal(t, 1, terminal.restoreCount())
787+ require.Equal(t, PresentTerminating, sup.State().Presentation)
788+}
789+
790+func TestSupervisorRawModePrecedesBroker(t *testing.T) {
791+ clock := newSupervisorTestClock()
792+ reader := newSupervisorTestReader()
793+ t.Cleanup(reader.unblock)
794+ terminal := newSupervisorTestTerminal(reader)
795+
796+ enteredAfterRaw := make(chan bool, 1)
797+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
798+ enteredAfterRaw <- terminal.restoreCount() == 0
799+ return nil, ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "down"}
800+ })
801+
802+ sup := mustSupervisor(t, SupervisorConfig{
803+ Connector: connector,
804+ Terminal: terminal,
805+ Clock: clock,
806+ Jitter: func() float64 { return 0 },
807+ })
808+
809+ ctx, cancel := context.WithCancel(context.Background())
810+ runErr := make(chan error, 1)
811+ go func() { runErr <- sup.Run(ctx) }()
812+ require.Equal(t, 1, connector.awaitStart(t))
813+ require.True(t, <-enteredAfterRaw, "raw mode must be entered before the first broker connection")
814+ cancel()
815+ <-runErr
816+}
817+
818+func TestSupervisorSingleAttemptAndTimer(t *testing.T) {
819+ ctx, cancel := context.WithCancel(context.Background())
820+
821+ clock := newSupervisorTestClock()
822+ reader := newSupervisorTestReader()
823+ t.Cleanup(reader.unblock)
824+ terminal := newSupervisorTestTerminal(reader)
825+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
826+ return nil, ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "down"}
827+ })
828+
829+ sup := mustSupervisor(t, SupervisorConfig{
830+ Connector: connector,
831+ Terminal: terminal,
832+ Clock: clock,
833+ Jitter: func() float64 { return 0 },
834+ })
835+
836+ runErr := make(chan error, 1)
837+ go func() { runErr <- sup.Run(ctx) }()
838+
839+ first := clock.awaitTimer(t)
840+ first.fire()
841+ second := clock.awaitTimer(t)
842+ require.Equal(t, 100*time.Millisecond, second.delay)
843+ second.fire()
844+ clock.awaitTimer(t)
845+
846+ require.Equal(t, 1, connector.maxConcurrent(), "attempts never overlap")
847+ require.Equal(t, 3, connector.callCount())
848+ require.Equal(t, 1, clock.liveTimers(), "exactly one retry timer is live")
849+ require.Zero(t, connector.inFlightCount(), "the connector call has returned before the next wait")
850+
851+ cancel()
852+ require.ErrorIs(t, <-runErr, context.Canceled)
853+ require.Eventually(t, clock.allStopped, time.Second, time.Millisecond, "every retry timer is retired")
854+ require.Equal(t, terminal.restoreCount(), 1)
855+ require.Zero(t, connector.inFlightCount())
856+}
857+
858+// supervisorCancellationRaceRounds is how many times each cancellation race is
859+// replayed. Every round orders the parent cancellation strictly before the run's
860+// settle decision by construction, so the rounds are deterministic rather than
861+// timing-dependent; the repetition runs both settle points many times over.
862+const supervisorCancellationRaceRounds = 100
863+
864+// TestSupervisorCancellationRacingSettlementIsTerminal pins the parent
865+// cancellation fence at both settle points: an attempt that settles at the same
866+// instant as the parent cancellation is retired rather than adopted, and an
867+// established-connection loss that settles at that instant is never turned into a
868+// retry. Every round must end terminating with ctx.Err without scheduling a retry
869+// cadence, rendering ConnectivityRetryWait, or notifying a connectivity failure
870+// for a settle the parent had already given up on.
871+//
872+// The ordering is enforced, never hoped for: each round parks the settle at a
873+// point the round itself releases (the state lock the run decides adoption under,
874+// or the connection release the loss settle runs after) and cancels there, so the
875+// decision sees the cancellation whichever branch the run's select took.
876+func TestSupervisorCancellationRacingSettlementIsTerminal(t *testing.T) {
877+ transient := ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "raced"}
878+ exited := ports.BrokerError{Code: ports.BrokerErrorExplicitExit, Text: "raced exit"}
879+
880+ attempts := []struct {
881+ name string
882+ attempt func() (*supervisorTestService, error)
883+ }{
884+ {name: "retryable failure", attempt: func() (*supervisorTestService, error) { return nil, transient }},
885+ {
886+ name: "ready connection",
887+ attempt: func() (*supervisorTestService, error) {
888+ service := newSupervisorTestService(ports.BrokerConnectionID{3})
889+ service.publish(1, 1)
890+ return service, nil
891+ },
892+ },
893+ {name: "terminal failure", attempt: func() (*supervisorTestService, error) { return nil, exited }},
894+ }
895+
896+ for _, tc := range attempts {
897+ t.Run("attempt "+tc.name, func(t *testing.T) {
898+ // One carrier thread for the round: yielding after the attempt is
899+ // released then runs that goroutine to its settlement before this one
900+ // resumes, so the outcome is delivered while the parent is still live
901+ // and the run's select takes the settled attempt rather than ctx.Done.
902+ defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1))
903+ for round := 0; round < supervisorCancellationRaceRounds; round++ {
904+ release := make(chan struct{})
905+ var raced *supervisorTestService
906+ race := startCancellationRace(t, func(context.Context, int) (ports.BrokerService, error) {
907+ <-release
908+ service, err := tc.attempt()
909+ raced = service
910+ if service == nil {
911+ return nil, err
912+ }
913+ return service, err
914+ })
915+ require.Equal(t, 1, race.connector.awaitStart(t), "round %d", round)
916+
917+ // Take the state lock the run decides adoption under, release the
918+ // attempt, let it settle, and cancel the parent while still holding
919+ // the lock: the decision is ordered strictly after the cancellation
920+ // by construction.
921+ race.supervisor.mu.Lock()
922+ close(release)
923+ runtime.Gosched()
924+ race.cancel()
925+ race.supervisor.mu.Unlock()
926+
927+ race.assertSettledAsCancelled(t, round, false)
928+ if raced != nil {
929+ select {
930+ case <-raced.closedCh():
931+ default:
932+ t.Fatalf("round %d: an attempt that raced cancellation must be retired, not adopted", round)
933+ }
934+ }
935+ }
936+ })
937+ }
938+
939+ t.Run("established loss", func(t *testing.T) {
940+ for round := 0; round < supervisorCancellationRaceRounds; round++ {
941+ losing := newSupervisorTestService(ports.BrokerConnectionID{4})
942+ losing.publish(1, 1)
943+ losing.holdClose()
944+ race := startCancellationRace(t, func(context.Context, int) (ports.BrokerService, error) {
945+ return losing, nil
946+ })
947+ require.Equal(t, 1, race.connector.awaitStart(t), "round %d", round)
948+ require.Eventually(t, func() bool { return race.supervisor.State().Connectivity == ConnectivityReady }, 5*time.Second, time.Millisecond, "round %d", round)
949+
950+ // The loss wakes the run while the parent is still live, and its
951+ // connection release then parks the run before the settle decision, so
952+ // the cancellation below is ordered strictly before that decision.
953+ losing.lose(ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "raced loss"})
954+ select {
955+ case <-losing.closingCh():
956+ case <-time.After(5 * time.Second):
957+ t.Fatalf("round %d: the run never released the lost connection", round)
958+ }
959+ race.cancel()
960+ losing.releaseClose()
961+
962+ race.assertSettledAsCancelled(t, round, true)
963+ select {
964+ case <-losing.closedCh():
965+ case <-time.After(5 * time.Second):
966+ t.Fatalf("round %d: the lost connection must be retired", round)
967+ }
968+ }
969+ })
970+}
971+
972+// cancellationRace is one scripted round of the parent-cancellation race: the
973+// running supervisor with its fake clock, terminal, and recorder, plus the run's
974+// result channel. A round cancels the parent itself, at the point that orders the
975+// cancellation before the settle decision under test.
976+type cancellationRace struct {
977+ cancel context.CancelFunc
978+ clock *supervisorTestClock
979+ terminal *supervisorTestTerminal
980+ recorder *cancellationRaceRecorder
981+ connector *supervisorTestConnector
982+ supervisor *Supervisor
983+ runErr chan error
984+}
985+
986+// startCancellationRace starts one supervisor run over a scripted connector. The
987+// reader is shared and never unblocked until the test ends, so the run can only
988+// end through cancellation.
989+func startCancellationRace(t *testing.T, connect func(ctx context.Context, call int) (ports.BrokerService, error)) *cancellationRace {
990+ t.Helper()
991+ ctx, cancel := context.WithCancel(context.Background())
992+ reader := newSupervisorTestReader()
993+ t.Cleanup(reader.unblock)
994+
995+ race := &cancellationRace{
996+ cancel: cancel,
997+ clock: newSupervisorTestClock(),
998+ terminal: newSupervisorTestTerminal(reader),
999+ recorder: &cancellationRaceRecorder{},
1000+ connector: newSupervisorTestConnector(connect),
1001+ runErr: make(chan error, 1),
1002+ }
1003+ race.supervisor = mustSupervisor(t, SupervisorConfig{
1004+ Connector: race.connector,
1005+ Terminal: race.terminal,
1006+ Clock: race.clock,
1007+ Jitter: func() float64 { return 0 },
1008+ Render: race.recorder.render,
1009+ Notify: race.recorder.notify,
1010+ })
1011+ go func() { race.runErr <- race.supervisor.Run(ctx) }()
1012+ return race
1013+}
1014+
1015+// assertSettledAsCancelled asserts one round's cancellation invariants: the run
1016+// ends terminating with ctx.Err, raw mode is restored exactly once, no retry
1017+// cadence was scheduled or notified, and every earlier render is the attempt that
1018+// was still in flight. established marks a round whose connection was ready
1019+// before the race, so its earlier ready render is expected.
1020+func (r *cancellationRace) assertSettledAsCancelled(t *testing.T, round int, established bool) {
1021+ t.Helper()
1022+ require.ErrorIs(t, <-r.runErr, context.Canceled, "round %d: parent cancellation must settle the run", round)
1023+ renders, notified := r.recorder.recorded()
1024+ require.Empty(t, notified, "round %d: cancellation must not notify a connectivity failure", round)
1025+ require.Zero(t, r.clock.timerCount(), "round %d: cancellation must not schedule a retry timer", round)
1026+ require.Equal(t, 1, r.terminal.restoreCount(), "round %d: raw mode must be restored exactly once", round)
1027+ require.NotEmpty(t, renders, "round %d: the run must render its state", round)
1028+ require.Equal(t, PresentTerminating, renders[len(renders)-1].Presentation, "round %d: the run must end terminating", round)
1029+ for _, state := range renders[:len(renders)-1] {
1030+ require.Equal(t, PresentPicker, state.Presentation, "round %d: the run must stay in the picker", round)
1031+ require.NotEqual(t, ConnectivityRetryWait, state.Connectivity,
1032+ "round %d: no retry cadence may be published after the cancellation", round)
1033+ if established {
1034+ // The connection was ready before the race, so its ready render is the
1035+ // only settled state the round expects.
1036+ continue
1037+ }
1038+ require.Equal(t, ConnectivityConnectingBroker, state.Connectivity,
1039+ "round %d: no settled connection may be published for the cancelled attempt", round)
1040+ }
1041+}
1042+
1043+// cancellationRaceRecorder records what one round's supervisor published.
1044+type cancellationRaceRecorder struct {
1045+ mu sync.Mutex
1046+ renders []State
1047+ notified []error
1048+}
1049+
1050+func (r *cancellationRaceRecorder) render(state State) {
1051+ r.mu.Lock()
1052+ r.renders = append(r.renders, state)
1053+ r.mu.Unlock()
1054+}
1055+
1056+func (r *cancellationRaceRecorder) notify(_ State, err error) {
1057+ r.mu.Lock()
1058+ r.notified = append(r.notified, err)
1059+ r.mu.Unlock()
1060+}
1061+
1062+func (r *cancellationRaceRecorder) recorded() ([]State, []error) {
1063+ r.mu.Lock()
1064+ defer r.mu.Unlock()
1065+ return append([]State(nil), r.renders...), append([]error(nil), r.notified...)
1066+}
1067+
1068+// supervisorNilSub is a typed-nil subscription: it satisfies the interface but
1069+// is nil underneath, so only the reflection guard keeps the supervisor from
1070+// dereferencing it.
1071+type supervisorNilSub struct{}
1072+
1073+func (*supervisorNilSub) Changed() <-chan struct{} { return nil }
1074+func (*supervisorNilSub) Close() {}
1075+
1076+// supervisorNilSubService overrides Subscribe so a test can exercise the
1077+// supervisor's nil-subscription guard without a live subscription.
1078+type supervisorNilSubService struct {
1079+ *supervisorTestService
1080+ sub ports.BrokerSubscription
1081+}
1082+
1083+func (s *supervisorNilSubService) Subscribe() (ports.BrokerSubscription, error) {
1084+ return s.sub, nil
1085+}
1086+
1087+// TestSupervisorDefensiveGenerationFenceRejectsSupersededAttempt pins the
1088+// defensive generation fence directly: a settled attempt whose generation is not
1089+// the current one is retired and discarded, while the current generation is
1090+// adopted unchanged. Run joins every attempt before it advances, so no
1091+// superseded completion can reach adoptAttempt in the production driver today;
1092+// pinning the fence here keeps a future concurrent driver from dropping it.
1093+func TestSupervisorDefensiveGenerationFenceRejectsSupersededAttempt(t *testing.T) {
1094+ clock := newSupervisorTestClock()
1095+ terminal := newSupervisorTestTerminal(newSupervisorTestReader())
1096+ sup := mustSupervisor(t, SupervisorConfig{
1097+ Connector: newSupervisorTestConnector(nil),
1098+ Terminal: terminal,
1099+ Clock: clock,
1100+ Jitter: func() float64 { return 0 },
1101+ })
1102+
1103+ // Claim a generation exactly as Run does before it launches the connector.
1104+ sup.transition(supervisorEvent{kind: supervisorBeginAttempt})
1105+ current := sup.State().Generation
1106+ require.Equal(t, uint64(1), current)
1107+
1108+ service := newSupervisorTestService(ports.BrokerConnectionID{1})
1109+ subClosed := make(chan struct{})
1110+ stale := supervisorAttempt{
1111+ generation: current - 1,
1112+ service: service,
1113+ sub: supervisorTestSubscription{closeFn: func() { close(subClosed) }},
1114+ }
1115+ adopted, cause := sup.adoptAttempt(context.Background(), stale)
1116+ require.False(t, adopted, "a superseded generation must not be adopted")
1117+ require.NoError(t, cause, "a superseded generation is not a cancellation")
1118+ // Run retires every attempt it does not adopt.
1119+ stale.retire()
1120+ select {
1121+ case <-service.closedCh():
1122+ default:
1123+ t.Fatal("a superseded service must be closed")
1124+ }
1125+ select {
1126+ case <-subClosed:
1127+ default:
1128+ t.Fatal("a superseded subscription must be closed")
1129+ }
1130+
1131+ // The current generation is adopted, so its live service is left untouched.
1132+ live := newSupervisorTestService(ports.BrokerConnectionID{2})
1133+ adopted, cause = sup.adoptAttempt(context.Background(), supervisorAttempt{generation: current, service: live})
1134+ require.True(t, adopted, "the current generation must be adopted")
1135+ require.NoError(t, cause)
1136+ select {
1137+ case <-live.closedCh():
1138+ t.Fatal("the current attempt must be adopted, not retired")
1139+ default:
1140+ }
1141+}
1142+
1143+// TestSupervisorNormalizesUntypedConnectFailure proves an untyped connect
1144+// failure becomes a typed unavailable failure that preserves its cause, and that
1145+// the same typed value reaches the notifier.
1146+func TestSupervisorNormalizesUntypedConnectFailure(t *testing.T) {
1147+ ctx, cancel := context.WithCancel(context.Background())
1148+ defer cancel()
1149+
1150+ clock := newSupervisorTestClock()
1151+ reader := newSupervisorTestReader()
1152+ t.Cleanup(reader.unblock)
1153+ terminal := newSupervisorTestTerminal(reader)
1154+
1155+ cause := errors.New("dial: connection refused")
1156+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
1157+ return nil, cause
1158+ })
1159+ notified := make(chan error, 4)
1160+ sup := mustSupervisor(t, SupervisorConfig{
1161+ Connector: connector,
1162+ Terminal: terminal,
1163+ Clock: clock,
1164+ Jitter: func() float64 { return 0 },
1165+ Notify: func(_ State, err error) { notified <- err },
1166+ })
1167+
1168+ runErr := make(chan error, 1)
1169+ go func() { runErr <- sup.Run(ctx) }()
1170+ clock.awaitTimer(t)
1171+
1172+ var typed ports.BrokerError
1173+ require.ErrorAs(t, sup.State().Err, &typed)
1174+ require.Equal(t, ports.BrokerErrorUnavailable, typed.Code)
1175+ require.ErrorIs(t, typed.Cause, cause, "the untyped cause is preserved")
1176+
1177+ select {
1178+ case err := <-notified:
1179+ var surfaced ports.BrokerError
1180+ require.ErrorAs(t, err, &surfaced)
1181+ require.Equal(t, ports.BrokerErrorUnavailable, surfaced.Code)
1182+ require.ErrorIs(t, surfaced.Cause, cause)
1183+ case <-time.After(5 * time.Second):
1184+ t.Fatal("the normalized failure was not surfaced")
1185+ }
1186+
1187+ cancel()
1188+ require.ErrorIs(t, <-runErr, context.Canceled)
1189+ require.Equal(t, 1, terminal.restoreCount())
1190+}
1191+
1192+// TestSupervisorNormalizesUntypedLoss proves an untyped established-connection
1193+// loss is normalized to a typed unavailable failure that preserves its cause.
1194+func TestSupervisorNormalizesUntypedLoss(t *testing.T) {
1195+ ctx, cancel := context.WithCancel(context.Background())
1196+ defer cancel()
1197+
1198+ clock := newSupervisorTestClock()
1199+ reader := newSupervisorTestReader()
1200+ t.Cleanup(reader.unblock)
1201+ terminal := newSupervisorTestTerminal(reader)
1202+
1203+ service := newSupervisorTestService(ports.BrokerConnectionID{1})
1204+ service.publish(1, 1)
1205+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
1206+ return service, nil
1207+ })
1208+ sup := mustSupervisor(t, SupervisorConfig{
1209+ Connector: connector,
1210+ Terminal: terminal,
1211+ Clock: clock,
1212+ Jitter: func() float64 { return 0 },
1213+ })
1214+
1215+ runErr := make(chan error, 1)
1216+ go func() { runErr <- sup.Run(ctx) }()
1217+ require.Equal(t, 1, connector.awaitStart(t))
1218+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityReady }, 5*time.Second, time.Millisecond)
1219+
1220+ cause := errors.New("carriage reset")
1221+ service.lose(cause)
1222+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityRetryWait }, 5*time.Second, time.Millisecond)
1223+
1224+ var typed ports.BrokerError
1225+ require.ErrorAs(t, sup.State().Err, &typed)
1226+ require.Equal(t, ports.BrokerErrorUnavailable, typed.Code)
1227+ require.ErrorIs(t, typed.Cause, cause, "the untyped loss cause is preserved")
1228+
1229+ cancel()
1230+ require.ErrorIs(t, <-runErr, context.Canceled)
1231+}
1232+
1233+// TestSupervisorNilSubscriptionIsTypedUnavailable proves a service that reports
1234+// no subscription (a nil interface or a typed nil) is closed and treated as a
1235+// typed unavailable failure instead of panicking on the first Changed call.
1236+func TestSupervisorNilSubscriptionIsTypedUnavailable(t *testing.T) {
1237+ cases := []struct {
1238+ name string
1239+ sub ports.BrokerSubscription
1240+ }{
1241+ {"nil interface", nil},
1242+ {"typed nil", (*supervisorNilSub)(nil)},
1243+ }
1244+ for _, tc := range cases {
1245+ t.Run(tc.name, func(t *testing.T) {
1246+ ctx, cancel := context.WithCancel(context.Background())
1247+ defer cancel()
1248+
1249+ clock := newSupervisorTestClock()
1250+ reader := newSupervisorTestReader()
1251+ t.Cleanup(reader.unblock)
1252+ terminal := newSupervisorTestTerminal(reader)
1253+
1254+ base := newSupervisorTestService(ports.BrokerConnectionID{7})
1255+ svc := &supervisorNilSubService{supervisorTestService: base, sub: tc.sub}
1256+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
1257+ return svc, nil
1258+ })
1259+ sup := mustSupervisor(t, SupervisorConfig{
1260+ Connector: connector,
1261+ Terminal: terminal,
1262+ Clock: clock,
1263+ Jitter: func() float64 { return 0 },
1264+ })
1265+
1266+ runErr := make(chan error, 1)
1267+ go func() { runErr <- sup.Run(ctx) }()
1268+ clock.awaitTimer(t)
1269+
1270+ var typed ports.BrokerError
1271+ require.ErrorAs(t, sup.State().Err, &typed)
1272+ require.Equal(t, ports.BrokerErrorUnavailable, typed.Code)
1273+ select {
1274+ case <-base.closedCh():
1275+ default:
1276+ t.Fatal("a service that returned no subscription must be closed")
1277+ }
1278+
1279+ cancel()
1280+ require.ErrorIs(t, <-runErr, context.Canceled)
1281+ require.Equal(t, 1, terminal.restoreCount())
1282+ })
1283+ }
1284+}
@@ -0,0 +1,1284 @@
1+package client
2+
3+import (
4+ "context"
5+ "errors"
6+ "io"
7+ "math"
8+ "runtime"
9+ "strings"
10+ "sync"
11+ "testing"
12+ "time"
13+
14+ "github.com/stretchr/testify/require"
15+
16+ "github.com/bnema/vev/internal/domain"
17+ "github.com/bnema/vev/internal/ports"
18+)
19+
20+// supervisorTestClock is a manually fired ports.Clock. NewTimer publishes each
21+// timer on a buffered channel so a test can fire it deterministically without a
22+// sleep, and Stop is recorded so a test can prove every retry timer was
23+// retired.
24+type supervisorTestClock struct {
25+ mu sync.Mutex
26+ now time.Time
27+ timers []*supervisorTestTimer
28+ created chan *supervisorTestTimer
29+}
30+
31+func newSupervisorTestClock() *supervisorTestClock {
32+ return &supervisorTestClock{now: time.Unix(1000, 0), created: make(chan *supervisorTestTimer, 32)}
33+}
34+
35+func (c *supervisorTestClock) Now() time.Time {
36+ c.mu.Lock()
37+ defer c.mu.Unlock()
38+ return c.now
39+}
40+
41+func (c *supervisorTestClock) NewTimer(delay time.Duration) ports.Timer {
42+ timer := &supervisorTestTimer{ch: make(chan time.Time, 1), delay: delay}
43+ c.mu.Lock()
44+ c.timers = append(c.timers, timer)
45+ c.mu.Unlock()
46+ select {
47+ case c.created <- timer:
48+ default:
49+ }
50+ return timer
51+}
52+
53+func (c *supervisorTestClock) awaitTimer(t *testing.T) *supervisorTestTimer {
54+ t.Helper()
55+ select {
56+ case timer := <-c.created:
57+ return timer
58+ case <-time.After(5 * time.Second):
59+ t.Fatal("supervisor did not schedule a retry timer")
60+ return nil
61+ }
62+}
63+
64+func (c *supervisorTestClock) timerCount() int {
65+ c.mu.Lock()
66+ defer c.mu.Unlock()
67+ return len(c.timers)
68+}
69+
70+func (c *supervisorTestClock) liveTimers() int {
71+ c.mu.Lock()
72+ defer c.mu.Unlock()
73+ live := 0
74+ for _, timer := range c.timers {
75+ if !timer.stopped() {
76+ live++
77+ }
78+ }
79+ return live
80+}
81+
82+func (c *supervisorTestClock) allStopped() bool {
83+ c.mu.Lock()
84+ defer c.mu.Unlock()
85+ for _, timer := range c.timers {
86+ if !timer.stopped() {
87+ return false
88+ }
89+ }
90+ return true
91+}
92+
93+type supervisorTestTimer struct {
94+ ch chan time.Time
95+ delay time.Duration
96+ mu sync.Mutex
97+ stoppedFlag bool
98+}
99+
100+func (t *supervisorTestTimer) C() <-chan time.Time { return t.ch }
101+
102+func (t *supervisorTestTimer) Reset(time.Duration) bool { return false }
103+
104+func (t *supervisorTestTimer) Stop() bool {
105+ t.mu.Lock()
106+ defer t.mu.Unlock()
107+ if t.stoppedFlag {
108+ return false
109+ }
110+ t.stoppedFlag = true
111+ return true
112+}
113+
114+func (t *supervisorTestTimer) stopped() bool {
115+ t.mu.Lock()
116+ defer t.mu.Unlock()
117+ return t.stoppedFlag
118+}
119+
120+// fire delivers the timer without blocking, so a test may fire before or after
121+// the supervisor reaches its select.
122+func (t *supervisorTestTimer) fire() {
123+ select {
124+ case t.ch <- time.Now():
125+ default:
126+ }
127+}
128+
129+// supervisorTestTerminal records raw-mode enters and restores. Its reader is
130+// shared so tests can model input EOF and a live terminal.
131+type supervisorTestTerminal struct {
132+ in io.Reader
133+ enterErr error
134+
135+ mu sync.Mutex
136+ restores int
137+}
138+
139+func newSupervisorTestTerminal(in io.Reader) *supervisorTestTerminal {
140+ return &supervisorTestTerminal{in: in}
141+}
142+
143+func (t *supervisorTestTerminal) EnterRaw() (func() error, error) {
144+ if t.enterErr != nil {
145+ return nil, t.enterErr
146+ }
147+ return func() error {
148+ t.mu.Lock()
149+ t.restores++
150+ t.mu.Unlock()
151+ return nil
152+ }, nil
153+}
154+
155+func (t *supervisorTestTerminal) Geometry() (domain.Geometry, error) {
156+ return domain.Geometry{}, nil
157+}
158+
159+func (t *supervisorTestTerminal) ResizeEvents() <-chan domain.Geometry { return nil }
160+
161+func (t *supervisorTestTerminal) In() io.Reader { return t.in }
162+
163+func (t *supervisorTestTerminal) Out() io.Writer { return io.Discard }
164+
165+func (t *supervisorTestTerminal) Flush() error { return nil }
166+
167+func (t *supervisorTestTerminal) restoreCount() int {
168+ t.mu.Lock()
169+ defer t.mu.Unlock()
170+ return t.restores
171+}
172+
173+// supervisorTestConnector scripts connection attempts and records concurrency,
174+// so a test can prove attempts never overlap.
175+type supervisorTestConnector struct {
176+ connect func(ctx context.Context, call int) (ports.BrokerService, error)
177+
178+ mu sync.Mutex
179+ calls int
180+ inFlight int
181+ maxInFlight int
182+ started chan int
183+}
184+
185+func newSupervisorTestConnector(connect func(ctx context.Context, call int) (ports.BrokerService, error)) *supervisorTestConnector {
186+ return &supervisorTestConnector{connect: connect, started: make(chan int, 32)}
187+}
188+
189+func (c *supervisorTestConnector) Connect(ctx context.Context) (ports.BrokerService, error) {
190+ c.mu.Lock()
191+ c.calls++
192+ call := c.calls
193+ c.inFlight++
194+ if c.inFlight > c.maxInFlight {
195+ c.maxInFlight = c.inFlight
196+ }
197+ c.mu.Unlock()
198+ select {
199+ case c.started <- call:
200+ default:
201+ }
202+ defer func() {
203+ c.mu.Lock()
204+ c.inFlight--
205+ c.mu.Unlock()
206+ }()
207+ if c.connect == nil {
208+ return nil, ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "no script"}
209+ }
210+ return c.connect(ctx, call)
211+}
212+
213+func (c *supervisorTestConnector) callCount() int {
214+ c.mu.Lock()
215+ defer c.mu.Unlock()
216+ return c.calls
217+}
218+
219+func (c *supervisorTestConnector) maxConcurrent() int {
220+ c.mu.Lock()
221+ defer c.mu.Unlock()
222+ return c.maxInFlight
223+}
224+
225+func (c *supervisorTestConnector) inFlightCount() int {
226+ c.mu.Lock()
227+ defer c.mu.Unlock()
228+ return c.inFlight
229+}
230+
231+func (c *supervisorTestConnector) awaitStart(t *testing.T) int {
232+ t.Helper()
233+ select {
234+ case call := <-c.started:
235+ return call
236+ case <-time.After(5 * time.Second):
237+ t.Fatal("supervisor did not start a connection attempt")
238+ return 0
239+ }
240+}
241+
242+// supervisorTestHub is one broker publication source. subscribe hands out the
243+// current change channel, and publish retires and replaces it, so a stale
244+// subscription can never observe a newer publication.
245+type supervisorTestHub struct {
246+ mu sync.Mutex
247+ snapshot ports.BrokerSnapshot
248+ changed chan struct{}
249+}
250+
251+func newSupervisorTestHub() *supervisorTestHub {
252+ return &supervisorTestHub{changed: make(chan struct{})}
253+}
254+
255+func (h *supervisorTestHub) publish(snapshot ports.BrokerSnapshot) {
256+ h.mu.Lock()
257+ h.snapshot = snapshot
258+ changed := h.changed
259+ h.changed = make(chan struct{})
260+ h.mu.Unlock()
261+ close(changed)
262+}
263+
264+func (h *supervisorTestHub) current() ports.BrokerSnapshot {
265+ h.mu.Lock()
266+ defer h.mu.Unlock()
267+ return h.snapshot
268+}
269+
270+func (h *supervisorTestHub) subscribe(closeFn func()) ports.BrokerSubscription {
271+ h.mu.Lock()
272+ changed := h.changed
273+ h.mu.Unlock()
274+ return supervisorTestSubscription{changed: changed, closeFn: closeFn}
275+}
276+
277+type supervisorTestSubscription struct {
278+ changed <-chan struct{}
279+ closeFn func()
280+}
281+
282+func (s supervisorTestSubscription) Changed() <-chan struct{} { return s.changed }
283+
284+func (s supervisorTestSubscription) Close() {
285+ if s.closeFn != nil {
286+ s.closeFn()
287+ }
288+}
289+
290+// supervisorTestService is one scripted BrokerService. Close is observable so a
291+// test can prove a retired or late service was closed, and Close can be held open
292+// so a test can order a parent cancellation strictly before a settle decision that
293+// runs after the connection is released.
294+type supervisorTestService struct {
295+ id ports.BrokerConnectionID
296+ hub *supervisorTestHub
297+
298+ mu sync.Mutex
299+ err error
300+ doneOnce sync.Once
301+ done chan struct{}
302+
303+ closeOnce sync.Once
304+ closed chan struct{}
305+ subClosed chan struct{}
306+
307+ // closeGate, when non-nil, holds Close until releaseClose is called, and
308+ // closeEntered reports that Close began waiting on it.
309+ closeGate chan struct{}
310+ closeEntered chan struct{}
311+}
312+
313+func newSupervisorTestService(id ports.BrokerConnectionID) *supervisorTestService {
314+ return &supervisorTestService{
315+ id: id,
316+ hub: newSupervisorTestHub(),
317+ done: make(chan struct{}),
318+ closed: make(chan struct{}),
319+ subClosed: make(chan struct{}, 1),
320+ }
321+}
322+
323+func (s *supervisorTestService) ConnectionID() ports.BrokerConnectionID { return s.id }
324+
325+func (s *supervisorTestService) Done() <-chan struct{} { return s.done }
326+
327+func (s *supervisorTestService) Err() error {
328+ s.mu.Lock()
329+ defer s.mu.Unlock()
330+ return s.err
331+}
332+
333+func (s *supervisorTestService) Snapshot() ports.BrokerSnapshot { return s.hub.current() }
334+
335+func (s *supervisorTestService) Subscribe() (ports.BrokerSubscription, error) {
336+ return s.hub.subscribe(func() {
337+ select {
338+ case s.subClosed <- struct{}{}:
339+ default:
340+ }
341+ }), nil
342+}
343+
344+func (s *supervisorTestService) OpenStream(context.Context, ports.BrokerOpenStreamRequest) (ports.BrokerLogicalConnection, error) {
345+ return nil, errors.New("supervisor test service does not support streams")
346+}
347+
348+func (s *supervisorTestService) CloseStream(ports.BrokerConnectionID, ports.BrokerStreamID) error {
349+ return nil
350+}
351+
352+func (s *supervisorTestService) AddHost(context.Context, string) error { return nil }
353+
354+func (s *supervisorTestService) RemoveHost(context.Context, string) (bool, error) {
355+ return false, nil
356+}
357+
358+func (s *supervisorTestService) RequestReconcile(string) {}
359+
360+func (s *supervisorTestService) Close() error {
361+ s.closeOnce.Do(func() {
362+ if s.closeGate != nil {
363+ select {
364+ case s.closeEntered <- struct{}{}:
365+ default:
366+ }
367+ <-s.closeGate
368+ }
369+ close(s.closed)
370+ s.terminate(nil)
371+ })
372+ return nil
373+}
374+
375+// holdClose makes Close block until releaseClose is called, so a test can order a
376+// parent cancellation strictly before a settle decision that runs after the
377+// connection is released. closingCh reports that the release was entered.
378+func (s *supervisorTestService) holdClose() {
379+ s.closeGate = make(chan struct{})
380+ s.closeEntered = make(chan struct{}, 1)
381+}
382+
383+// releaseClose releases a held Close.
384+func (s *supervisorTestService) releaseClose() {
385+ if s.closeGate != nil {
386+ close(s.closeGate)
387+ }
388+}
389+
390+// closingCh reports that the held release was entered.
391+func (s *supervisorTestService) closingCh() <-chan struct{} { return s.closeEntered }
392+
393+// terminate removes the connection, recording the first cause exactly once.
394+func (s *supervisorTestService) terminate(err error) {
395+ if err != nil {
396+ s.mu.Lock()
397+ if s.err == nil {
398+ s.err = err
399+ }
400+ s.mu.Unlock()
401+ }
402+ s.doneOnce.Do(func() { close(s.done) })
403+}
404+
405+// lose models an unexplained broker-side loss.
406+func (s *supervisorTestService) lose(err error) { s.terminate(err) }
407+
408+func (s *supervisorTestService) publish(epoch ports.BrokerEpoch, revision ports.BrokerRevision) {
409+ s.hub.publish(ports.BrokerSnapshot{Epoch: epoch, Revision: revision})
410+}
411+
412+func (s *supervisorTestService) closedCh() <-chan struct{} { return s.closed }
413+
414+func mustSupervisor(t *testing.T, cfg SupervisorConfig) *Supervisor {
415+ t.Helper()
416+ sup, err := NewSupervisor(cfg)
417+ require.NoError(t, err)
418+ return sup
419+}
420+
421+// supervisorTestReader blocks until unblocked, so a test controls exactly when
422+// the owned terminal input reaches EOF.
423+type supervisorTestReader struct {
424+ mu sync.Mutex
425+ done chan struct{}
426+}
427+
428+func newSupervisorTestReader() *supervisorTestReader {
429+ return &supervisorTestReader{done: make(chan struct{})}
430+}
431+
432+func (r *supervisorTestReader) Read([]byte) (int, error) {
433+ <-r.done
434+ return 0, io.EOF
435+}
436+
437+func (r *supervisorTestReader) unblock() {
438+ r.mu.Lock()
439+ defer r.mu.Unlock()
440+ select {
441+ case <-r.done:
442+ default:
443+ close(r.done)
444+ }
445+}
446+
447+func TestSupervisorReducerTransitions(t *testing.T) {
448+ transient := errors.New("unavailable")
449+ incompatible := ports.BrokerError{Code: ports.BrokerErrorIncompatible, Text: "version"}
450+ exited := ports.BrokerError{Code: ports.BrokerErrorExplicitExit, Text: "exit"}
451+ picker := State{Presentation: PresentPicker, Connectivity: ConnectivityDisconnected}
452+
453+ tests := []struct {
454+ name string
455+ state State
456+ event supervisorEvent
457+ want State
458+ }{
459+ {
460+ name: "begin attempt claims a generation and connects",
461+ state: picker,
462+ event: supervisorEvent{kind: supervisorBeginAttempt},
463+ want: State{Presentation: PresentPicker, Connectivity: ConnectivityConnectingBroker, Attempt: 0, Generation: 1},
464+ },
465+ {
466+ name: "ready resets the retry cadence",
467+ state: State{Presentation: PresentPicker, Connectivity: ConnectivityConnectingBroker, Attempt: 3, Generation: 4, Err: transient},
468+ event: supervisorEvent{kind: supervisorReady},
469+ want: State{Presentation: PresentPicker, Connectivity: ConnectivityReady, Attempt: 0, Generation: 4},
470+ },
471+ {
472+ name: "transient failure waits with a bumped attempt",
473+ state: State{Presentation: PresentPicker, Connectivity: ConnectivityConnectingBroker, Attempt: 0, Generation: 2},
474+ event: supervisorEvent{kind: supervisorTransientFailure, err: transient},
475+ want: State{Presentation: PresentPicker, Connectivity: ConnectivityRetryWait, Attempt: 1, Generation: 2, Err: transient},
476+ },
477+ {
478+ name: "broker loss returns to picker retry wait",
479+ state: State{Presentation: PresentPicker, Connectivity: ConnectivityReady, Attempt: 0, Generation: 5},
480+ event: supervisorEvent{kind: supervisorBrokerLoss, err: transient},
481+ want: State{Presentation: PresentPicker, Connectivity: ConnectivityRetryWait, Attempt: 1, Generation: 5, Err: transient},
482+ },
483+ {
484+ name: "non-retryable failure disconnects and stays on the picker",
485+ state: State{Presentation: PresentPicker, Connectivity: ConnectivityConnectingBroker, Attempt: 1, Generation: 3},
486+ event: supervisorEvent{kind: supervisorNonRetryable, err: incompatible},
487+ want: State{Presentation: PresentPicker, Connectivity: ConnectivityDisconnected, Attempt: 1, Generation: 3, Err: incompatible},
488+ },
489+ {
490+ name: "terminal failure terminates",
491+ state: State{Presentation: PresentPicker, Connectivity: ConnectivityReady, Generation: 7},
492+ event: supervisorEvent{kind: supervisorTerminal, err: exited},
493+ want: State{Presentation: PresentTerminating, Connectivity: ConnectivityDisconnected, Generation: 7, Err: exited},
494+ },
495+ }
496+
497+ for _, tt := range tests {
498+ t.Run(tt.name, func(t *testing.T) {
499+ require.Equal(t, tt.want, reduceSupervisor(tt.state, tt.event))
500+ })
501+ }
502+}
503+
504+func TestSupervisorBackoffCadence(t *testing.T) {
505+ zero := func() float64 { return 0 }
506+ half := func() float64 { return 0.5 }
507+ clamped := func() float64 { return 1.5 }
508+ nan := func() float64 { return math.NaN() }
509+ negInf := func() float64 { return math.Inf(-1) }
510+ posInf := func() float64 { return math.Inf(1) }
511+
512+ tests := []struct {
513+ name string
514+ attempt uint64
515+ jitter func() float64
516+ want time.Duration
517+ }{
518+ {"no attempt has no delay", 0, zero, 0},
519+ {"first failure halves 100ms", 1, zero, 50 * time.Millisecond},
520+ {"second failure halves 200ms", 2, zero, 100 * time.Millisecond},
521+ {"third failure halves 400ms", 3, zero, 200 * time.Millisecond},
522+ {"fourth failure halves 800ms", 4, zero, 400 * time.Millisecond},
523+ {"fifth failure halves 1.6s", 5, zero, 800 * time.Millisecond},
524+ {"cap reached at 2s", 6, zero, time.Second},
525+ {"cap stays at 2s", 9, zero, time.Second},
526+ {"equal jitter reaches three quarters", 1, half, 75 * time.Millisecond},
527+ {"equal jitter scales with the cap", 6, half, 1500 * time.Millisecond},
528+ {"full jitter reaches the cap", 1, clamped, 100 * time.Millisecond},
529+ {"full jitter stays capped", 6, clamped, 2 * time.Second},
530+ {"NaN jitter clamps to the minimum half", 1, nan, 50 * time.Millisecond},
531+ {"NaN jitter stays at half the cap", 6, nan, time.Second},
532+ {"negative jitter clamps to the minimum half", 1, negInf, 50 * time.Millisecond},
533+ {"negative jitter stays at half the cap", 6, negInf, time.Second},
534+ {"positive infinity clamps to the cap", 1, posInf, 100 * time.Millisecond},
535+ {"positive infinity stays capped", 6, posInf, 2 * time.Second},
536+ }
537+
538+ for _, tt := range tests {
539+ t.Run(tt.name, func(t *testing.T) {
540+ require.Equal(t, tt.want, supervisorBackoffDelay(tt.attempt, tt.jitter))
541+ })
542+ }
543+}
544+
545+func TestSupervisorPickerWithoutBroker(t *testing.T) {
546+ ctx, cancel := context.WithCancel(context.Background())
547+ defer cancel()
548+
549+ clock := newSupervisorTestClock()
550+ reader := newSupervisorTestReader()
551+ t.Cleanup(reader.unblock)
552+ terminal := newSupervisorTestTerminal(reader)
553+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
554+ return nil, ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "broker down"}
555+ })
556+
557+ var (
558+ renderMu sync.Mutex
559+ renders []State
560+ )
561+ sup := mustSupervisor(t, SupervisorConfig{
562+ Connector: connector,
563+ Terminal: terminal,
564+ Clock: clock,
565+ Jitter: func() float64 { return 0 },
566+ Render: func(state State) {
567+ renderMu.Lock()
568+ renders = append(renders, state)
569+ renderMu.Unlock()
570+ },
571+ })
572+
573+ runErr := make(chan error, 1)
574+ go func() { runErr <- sup.Run(ctx) }()
575+
576+ first := clock.awaitTimer(t)
577+ require.Equal(t, 50*time.Millisecond, first.delay, "the first retry is the equal-jitter half of 100ms")
578+ require.Equal(t, 1, connector.awaitStart(t))
579+ require.Equal(t, State{Presentation: PresentPicker, Connectivity: ConnectivityRetryWait, Attempt: 1, Generation: 1, Err: ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "broker down"}}, sup.State())
580+
581+ first.fire()
582+ second := clock.awaitTimer(t)
583+ require.Equal(t, 100*time.Millisecond, second.delay, "the second retry doubles the cap")
584+ require.Equal(t, 2, connector.awaitStart(t))
585+ require.Equal(t, PresentPicker, sup.State().Presentation, "the picker stays up while disconnected")
586+ require.Equal(t, ConnectivityRetryWait, sup.State().Connectivity)
587+ require.Equal(t, uint64(2), sup.State().Generation)
588+
589+ cancel()
590+ require.ErrorIs(t, <-runErr, context.Canceled)
591+ require.Equal(t, 1, terminal.restoreCount())
592+ require.Equal(t, 1, connector.maxConcurrent())
593+
594+ renderMu.Lock()
595+ defer renderMu.Unlock()
596+ require.NotEmpty(t, renders)
597+ require.Equal(t, PresentTerminating, renders[len(renders)-1].Presentation)
598+ for _, state := range renders[:len(renders)-1] {
599+ require.Equal(t, PresentPicker, state.Presentation, "P5.1a never leaves the picker")
600+ }
601+}
602+
603+func TestSupervisorBrokerLossReconnectsWithFreshGeneration(t *testing.T) {
604+ ctx, cancel := context.WithCancel(context.Background())
605+ defer cancel()
606+
607+ clock := newSupervisorTestClock()
608+ reader := newSupervisorTestReader()
609+ t.Cleanup(reader.unblock)
610+ terminal := newSupervisorTestTerminal(reader)
611+
612+ first := newSupervisorTestService(ports.BrokerConnectionID{1})
613+ first.publish(1, 1)
614+ second := newSupervisorTestService(ports.BrokerConnectionID{2})
615+
616+ connector := newSupervisorTestConnector(func(_ context.Context, call int) (ports.BrokerService, error) {
617+ switch call {
618+ case 1:
619+ return first, nil
620+ default:
621+ return second, nil
622+ }
623+ })
624+
625+ sup := mustSupervisor(t, SupervisorConfig{
626+ Connector: connector,
627+ Terminal: terminal,
628+ Clock: clock,
629+ Jitter: func() float64 { return 0 },
630+ })
631+
632+ runErr := make(chan error, 1)
633+ go func() { runErr <- sup.Run(ctx) }()
634+
635+ require.Equal(t, 1, connector.awaitStart(t))
636+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityReady }, 5*time.Second, time.Millisecond)
637+ require.Equal(t, uint64(1), sup.State().Generation)
638+ require.Equal(t, uint64(0), sup.State().Attempt)
639+
640+ // The idle broker connection is lost: the supervisor retires it and waits
641+ // the first cadence before reconnecting.
642+ first.lose(ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "idle loss"})
643+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityRetryWait }, 5*time.Second, time.Millisecond)
644+ require.Equal(t, uint64(1), sup.State().Attempt)
645+ select {
646+ case <-first.closedCh():
647+ default:
648+ t.Fatal("the lost connection was not retired")
649+ }
650+ select {
651+ case <-first.subClosed:
652+ default:
653+ t.Fatal("the lost subscription was not retired")
654+ }
655+
656+ retry := clock.awaitTimer(t)
657+ require.Equal(t, 50*time.Millisecond, retry.delay)
658+ retry.fire()
659+ require.Equal(t, 2, connector.awaitStart(t))
660+
661+ // Wait until the replacement is waiting for its first publication.
662+ require.Eventually(t, func() bool {
663+ return sup.State().Connectivity == ConnectivityConnectingBroker && sup.State().Generation == 2
664+ }, 5*time.Second, time.Millisecond)
665+
666+ // A publication on the retired connection must never make the replacement
667+ // Ready; only the replacement's own fresh epoch does.
668+ first.publish(1, 2)
669+ require.Never(t, func() bool { return sup.State().Connectivity == ConnectivityReady }, 50*time.Millisecond, 5*time.Millisecond)
670+
671+ second.publish(2, 1)
672+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityReady }, 5*time.Second, time.Millisecond)
673+ require.Equal(t, uint64(2), sup.State().Generation)
674+ require.Equal(t, uint64(0), sup.State().Attempt, "readiness resets the cadence")
675+
676+ cancel()
677+ require.ErrorIs(t, <-runErr, context.Canceled)
678+ require.Equal(t, 1, terminal.restoreCount())
679+ select {
680+ case <-second.closedCh():
681+ default:
682+ t.Fatal("the active connection was not retired on shutdown")
683+ }
684+}
685+
686+func TestSupervisorCancellationClosesLateConnection(t *testing.T) {
687+ ctx, cancel := context.WithCancel(context.Background())
688+
689+ clock := newSupervisorTestClock()
690+ reader := newSupervisorTestReader()
691+ t.Cleanup(reader.unblock)
692+ terminal := newSupervisorTestTerminal(reader)
693+
694+ release := make(chan struct{})
695+ late := newSupervisorTestService(ports.BrokerConnectionID{9})
696+ late.publish(1, 1)
697+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
698+ <-release
699+ return late, nil
700+ })
701+
702+ sup := mustSupervisor(t, SupervisorConfig{
703+ Connector: connector,
704+ Terminal: terminal,
705+ Clock: clock,
706+ Jitter: func() float64 { return 0 },
707+ })
708+
709+ runErr := make(chan error, 1)
710+ go func() { runErr <- sup.Run(ctx) }()
711+ require.Equal(t, 1, connector.awaitStart(t))
712+
713+ // Cancel while the connector ignores cancellation, then let it return a
714+ // live service: the supervisor must close the late superseded success.
715+ cancel()
716+ close(release)
717+
718+ require.ErrorIs(t, <-runErr, context.Canceled)
719+ select {
720+ case <-late.closedCh():
721+ default:
722+ t.Fatal("a late superseded success must be closed")
723+ }
724+ require.Equal(t, 1, terminal.restoreCount())
725+ require.Zero(t, connector.inFlightCount())
726+}
727+
728+func TestSupervisorIncompatibleStopsReconnecting(t *testing.T) {
729+ ctx, cancel := context.WithCancel(context.Background())
730+ defer cancel()
731+
732+ clock := newSupervisorTestClock()
733+ reader := newSupervisorTestReader()
734+ t.Cleanup(reader.unblock)
735+ terminal := newSupervisorTestTerminal(reader)
736+
737+ incompatible := ports.BrokerError{Code: ports.BrokerErrorIncompatible, Text: "protocol"}
738+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
739+ return nil, incompatible
740+ })
741+ notified := make(chan error, 4)
742+ sup := mustSupervisor(t, SupervisorConfig{
743+ Connector: connector,
744+ Terminal: terminal,
745+ Clock: clock,
746+ Jitter: func() float64 { return 0 },
747+ Notify: func(_ State, err error) { notified <- err },
748+ })
749+
750+ runErr := make(chan error, 1)
751+ go func() { runErr <- sup.Run(ctx) }()
752+
753+ select {
754+ case err := <-notified:
755+ var typed ports.BrokerError
756+ require.ErrorAs(t, err, &typed)
757+ require.Equal(t, ports.BrokerErrorIncompatible, typed.Code)
758+ case <-time.After(5 * time.Second):
759+ t.Fatal("the incompatible failure was not surfaced")
760+ }
761+
762+ require.Equal(t, State{Presentation: PresentPicker, Connectivity: ConnectivityDisconnected, Generation: 1, Err: incompatible}, sup.State())
763+ require.Never(t, func() bool { return connector.callCount() > 1 }, 50*time.Millisecond, 5*time.Millisecond)
764+ require.Zero(t, clock.timerCount(), "a non-retryable failure schedules no retry timer")
765+
766+ cancel()
767+ require.ErrorIs(t, <-runErr, context.Canceled)
768+ require.Equal(t, 1, terminal.restoreCount())
769+}
770+
771+func TestSupervisorTerminalEOFRestoresRawOnce(t *testing.T) {
772+ clock := newSupervisorTestClock()
773+ terminal := newSupervisorTestTerminal(strings.NewReader(""))
774+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
775+ return nil, ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "down"}
776+ })
777+
778+ sup := mustSupervisor(t, SupervisorConfig{
779+ Connector: connector,
780+ Terminal: terminal,
781+ Clock: clock,
782+ Jitter: func() float64 { return 0 },
783+ })
784+
785+ require.NoError(t, sup.Run(context.Background()), "a clean terminal EOF is an orderly exit")
786+ require.Equal(t, 1, terminal.restoreCount())
787+ require.Equal(t, PresentTerminating, sup.State().Presentation)
788+}
789+
790+func TestSupervisorRawModePrecedesBroker(t *testing.T) {
791+ clock := newSupervisorTestClock()
792+ reader := newSupervisorTestReader()
793+ t.Cleanup(reader.unblock)
794+ terminal := newSupervisorTestTerminal(reader)
795+
796+ enteredAfterRaw := make(chan bool, 1)
797+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
798+ enteredAfterRaw <- terminal.restoreCount() == 0
799+ return nil, ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "down"}
800+ })
801+
802+ sup := mustSupervisor(t, SupervisorConfig{
803+ Connector: connector,
804+ Terminal: terminal,
805+ Clock: clock,
806+ Jitter: func() float64 { return 0 },
807+ })
808+
809+ ctx, cancel := context.WithCancel(context.Background())
810+ runErr := make(chan error, 1)
811+ go func() { runErr <- sup.Run(ctx) }()
812+ require.Equal(t, 1, connector.awaitStart(t))
813+ require.True(t, <-enteredAfterRaw, "raw mode must be entered before the first broker connection")
814+ cancel()
815+ <-runErr
816+}
817+
818+func TestSupervisorSingleAttemptAndTimer(t *testing.T) {
819+ ctx, cancel := context.WithCancel(context.Background())
820+
821+ clock := newSupervisorTestClock()
822+ reader := newSupervisorTestReader()
823+ t.Cleanup(reader.unblock)
824+ terminal := newSupervisorTestTerminal(reader)
825+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
826+ return nil, ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "down"}
827+ })
828+
829+ sup := mustSupervisor(t, SupervisorConfig{
830+ Connector: connector,
831+ Terminal: terminal,
832+ Clock: clock,
833+ Jitter: func() float64 { return 0 },
834+ })
835+
836+ runErr := make(chan error, 1)
837+ go func() { runErr <- sup.Run(ctx) }()
838+
839+ first := clock.awaitTimer(t)
840+ first.fire()
841+ second := clock.awaitTimer(t)
842+ require.Equal(t, 100*time.Millisecond, second.delay)
843+ second.fire()
844+ clock.awaitTimer(t)
845+
846+ require.Equal(t, 1, connector.maxConcurrent(), "attempts never overlap")
847+ require.Equal(t, 3, connector.callCount())
848+ require.Equal(t, 1, clock.liveTimers(), "exactly one retry timer is live")
849+ require.Zero(t, connector.inFlightCount(), "the connector call has returned before the next wait")
850+
851+ cancel()
852+ require.ErrorIs(t, <-runErr, context.Canceled)
853+ require.Eventually(t, clock.allStopped, time.Second, time.Millisecond, "every retry timer is retired")
854+ require.Equal(t, terminal.restoreCount(), 1)
855+ require.Zero(t, connector.inFlightCount())
856+}
857+
858+// supervisorCancellationRaceRounds is how many times each cancellation race is
859+// replayed. Every round orders the parent cancellation strictly before the run's
860+// settle decision by construction, so the rounds are deterministic rather than
861+// timing-dependent; the repetition runs both settle points many times over.
862+const supervisorCancellationRaceRounds = 100
863+
864+// TestSupervisorCancellationRacingSettlementIsTerminal pins the parent
865+// cancellation fence at both settle points: an attempt that settles at the same
866+// instant as the parent cancellation is retired rather than adopted, and an
867+// established-connection loss that settles at that instant is never turned into a
868+// retry. Every round must end terminating with ctx.Err without scheduling a retry
869+// cadence, rendering ConnectivityRetryWait, or notifying a connectivity failure
870+// for a settle the parent had already given up on.
871+//
872+// The ordering is enforced, never hoped for: each round parks the settle at a
873+// point the round itself releases (the state lock the run decides adoption under,
874+// or the connection release the loss settle runs after) and cancels there, so the
875+// decision sees the cancellation whichever branch the run's select took.
876+func TestSupervisorCancellationRacingSettlementIsTerminal(t *testing.T) {
877+ transient := ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "raced"}
878+ exited := ports.BrokerError{Code: ports.BrokerErrorExplicitExit, Text: "raced exit"}
879+
880+ attempts := []struct {
881+ name string
882+ attempt func() (*supervisorTestService, error)
883+ }{
884+ {name: "retryable failure", attempt: func() (*supervisorTestService, error) { return nil, transient }},
885+ {
886+ name: "ready connection",
887+ attempt: func() (*supervisorTestService, error) {
888+ service := newSupervisorTestService(ports.BrokerConnectionID{3})
889+ service.publish(1, 1)
890+ return service, nil
891+ },
892+ },
893+ {name: "terminal failure", attempt: func() (*supervisorTestService, error) { return nil, exited }},
894+ }
895+
896+ for _, tc := range attempts {
897+ t.Run("attempt "+tc.name, func(t *testing.T) {
898+ // One carrier thread for the round: yielding after the attempt is
899+ // released then runs that goroutine to its settlement before this one
900+ // resumes, so the outcome is delivered while the parent is still live
901+ // and the run's select takes the settled attempt rather than ctx.Done.
902+ defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1))
903+ for round := 0; round < supervisorCancellationRaceRounds; round++ {
904+ release := make(chan struct{})
905+ var raced *supervisorTestService
906+ race := startCancellationRace(t, func(context.Context, int) (ports.BrokerService, error) {
907+ <-release
908+ service, err := tc.attempt()
909+ raced = service
910+ if service == nil {
911+ return nil, err
912+ }
913+ return service, err
914+ })
915+ require.Equal(t, 1, race.connector.awaitStart(t), "round %d", round)
916+
917+ // Take the state lock the run decides adoption under, release the
918+ // attempt, let it settle, and cancel the parent while still holding
919+ // the lock: the decision is ordered strictly after the cancellation
920+ // by construction.
921+ race.supervisor.mu.Lock()
922+ close(release)
923+ runtime.Gosched()
924+ race.cancel()
925+ race.supervisor.mu.Unlock()
926+
927+ race.assertSettledAsCancelled(t, round, false)
928+ if raced != nil {
929+ select {
930+ case <-raced.closedCh():
931+ default:
932+ t.Fatalf("round %d: an attempt that raced cancellation must be retired, not adopted", round)
933+ }
934+ }
935+ }
936+ })
937+ }
938+
939+ t.Run("established loss", func(t *testing.T) {
940+ for round := 0; round < supervisorCancellationRaceRounds; round++ {
941+ losing := newSupervisorTestService(ports.BrokerConnectionID{4})
942+ losing.publish(1, 1)
943+ losing.holdClose()
944+ race := startCancellationRace(t, func(context.Context, int) (ports.BrokerService, error) {
945+ return losing, nil
946+ })
947+ require.Equal(t, 1, race.connector.awaitStart(t), "round %d", round)
948+ require.Eventually(t, func() bool { return race.supervisor.State().Connectivity == ConnectivityReady }, 5*time.Second, time.Millisecond, "round %d", round)
949+
950+ // The loss wakes the run while the parent is still live, and its
951+ // connection release then parks the run before the settle decision, so
952+ // the cancellation below is ordered strictly before that decision.
953+ losing.lose(ports.BrokerError{Code: ports.BrokerErrorUnavailable, Text: "raced loss"})
954+ select {
955+ case <-losing.closingCh():
956+ case <-time.After(5 * time.Second):
957+ t.Fatalf("round %d: the run never released the lost connection", round)
958+ }
959+ race.cancel()
960+ losing.releaseClose()
961+
962+ race.assertSettledAsCancelled(t, round, true)
963+ select {
964+ case <-losing.closedCh():
965+ case <-time.After(5 * time.Second):
966+ t.Fatalf("round %d: the lost connection must be retired", round)
967+ }
968+ }
969+ })
970+}
971+
972+// cancellationRace is one scripted round of the parent-cancellation race: the
973+// running supervisor with its fake clock, terminal, and recorder, plus the run's
974+// result channel. A round cancels the parent itself, at the point that orders the
975+// cancellation before the settle decision under test.
976+type cancellationRace struct {
977+ cancel context.CancelFunc
978+ clock *supervisorTestClock
979+ terminal *supervisorTestTerminal
980+ recorder *cancellationRaceRecorder
981+ connector *supervisorTestConnector
982+ supervisor *Supervisor
983+ runErr chan error
984+}
985+
986+// startCancellationRace starts one supervisor run over a scripted connector. The
987+// reader is shared and never unblocked until the test ends, so the run can only
988+// end through cancellation.
989+func startCancellationRace(t *testing.T, connect func(ctx context.Context, call int) (ports.BrokerService, error)) *cancellationRace {
990+ t.Helper()
991+ ctx, cancel := context.WithCancel(context.Background())
992+ reader := newSupervisorTestReader()
993+ t.Cleanup(reader.unblock)
994+
995+ race := &cancellationRace{
996+ cancel: cancel,
997+ clock: newSupervisorTestClock(),
998+ terminal: newSupervisorTestTerminal(reader),
999+ recorder: &cancellationRaceRecorder{},
1000+ connector: newSupervisorTestConnector(connect),
1001+ runErr: make(chan error, 1),
1002+ }
1003+ race.supervisor = mustSupervisor(t, SupervisorConfig{
1004+ Connector: race.connector,
1005+ Terminal: race.terminal,
1006+ Clock: race.clock,
1007+ Jitter: func() float64 { return 0 },
1008+ Render: race.recorder.render,
1009+ Notify: race.recorder.notify,
1010+ })
1011+ go func() { race.runErr <- race.supervisor.Run(ctx) }()
1012+ return race
1013+}
1014+
1015+// assertSettledAsCancelled asserts one round's cancellation invariants: the run
1016+// ends terminating with ctx.Err, raw mode is restored exactly once, no retry
1017+// cadence was scheduled or notified, and every earlier render is the attempt that
1018+// was still in flight. established marks a round whose connection was ready
1019+// before the race, so its earlier ready render is expected.
1020+func (r *cancellationRace) assertSettledAsCancelled(t *testing.T, round int, established bool) {
1021+ t.Helper()
1022+ require.ErrorIs(t, <-r.runErr, context.Canceled, "round %d: parent cancellation must settle the run", round)
1023+ renders, notified := r.recorder.recorded()
1024+ require.Empty(t, notified, "round %d: cancellation must not notify a connectivity failure", round)
1025+ require.Zero(t, r.clock.timerCount(), "round %d: cancellation must not schedule a retry timer", round)
1026+ require.Equal(t, 1, r.terminal.restoreCount(), "round %d: raw mode must be restored exactly once", round)
1027+ require.NotEmpty(t, renders, "round %d: the run must render its state", round)
1028+ require.Equal(t, PresentTerminating, renders[len(renders)-1].Presentation, "round %d: the run must end terminating", round)
1029+ for _, state := range renders[:len(renders)-1] {
1030+ require.Equal(t, PresentPicker, state.Presentation, "round %d: the run must stay in the picker", round)
1031+ require.NotEqual(t, ConnectivityRetryWait, state.Connectivity,
1032+ "round %d: no retry cadence may be published after the cancellation", round)
1033+ if established {
1034+ // The connection was ready before the race, so its ready render is the
1035+ // only settled state the round expects.
1036+ continue
1037+ }
1038+ require.Equal(t, ConnectivityConnectingBroker, state.Connectivity,
1039+ "round %d: no settled connection may be published for the cancelled attempt", round)
1040+ }
1041+}
1042+
1043+// cancellationRaceRecorder records what one round's supervisor published.
1044+type cancellationRaceRecorder struct {
1045+ mu sync.Mutex
1046+ renders []State
1047+ notified []error
1048+}
1049+
1050+func (r *cancellationRaceRecorder) render(state State) {
1051+ r.mu.Lock()
1052+ r.renders = append(r.renders, state)
1053+ r.mu.Unlock()
1054+}
1055+
1056+func (r *cancellationRaceRecorder) notify(_ State, err error) {
1057+ r.mu.Lock()
1058+ r.notified = append(r.notified, err)
1059+ r.mu.Unlock()
1060+}
1061+
1062+func (r *cancellationRaceRecorder) recorded() ([]State, []error) {
1063+ r.mu.Lock()
1064+ defer r.mu.Unlock()
1065+ return append([]State(nil), r.renders...), append([]error(nil), r.notified...)
1066+}
1067+
1068+// supervisorNilSub is a typed-nil subscription: it satisfies the interface but
1069+// is nil underneath, so only the reflection guard keeps the supervisor from
1070+// dereferencing it.
1071+type supervisorNilSub struct{}
1072+
1073+func (*supervisorNilSub) Changed() <-chan struct{} { return nil }
1074+func (*supervisorNilSub) Close() {}
1075+
1076+// supervisorNilSubService overrides Subscribe so a test can exercise the
1077+// supervisor's nil-subscription guard without a live subscription.
1078+type supervisorNilSubService struct {
1079+ *supervisorTestService
1080+ sub ports.BrokerSubscription
1081+}
1082+
1083+func (s *supervisorNilSubService) Subscribe() (ports.BrokerSubscription, error) {
1084+ return s.sub, nil
1085+}
1086+
1087+// TestSupervisorDefensiveGenerationFenceRejectsSupersededAttempt pins the
1088+// defensive generation fence directly: a settled attempt whose generation is not
1089+// the current one is retired and discarded, while the current generation is
1090+// adopted unchanged. Run joins every attempt before it advances, so no
1091+// superseded completion can reach adoptAttempt in the production driver today;
1092+// pinning the fence here keeps a future concurrent driver from dropping it.
1093+func TestSupervisorDefensiveGenerationFenceRejectsSupersededAttempt(t *testing.T) {
1094+ clock := newSupervisorTestClock()
1095+ terminal := newSupervisorTestTerminal(newSupervisorTestReader())
1096+ sup := mustSupervisor(t, SupervisorConfig{
1097+ Connector: newSupervisorTestConnector(nil),
1098+ Terminal: terminal,
1099+ Clock: clock,
1100+ Jitter: func() float64 { return 0 },
1101+ })
1102+
1103+ // Claim a generation exactly as Run does before it launches the connector.
1104+ sup.transition(supervisorEvent{kind: supervisorBeginAttempt})
1105+ current := sup.State().Generation
1106+ require.Equal(t, uint64(1), current)
1107+
1108+ service := newSupervisorTestService(ports.BrokerConnectionID{1})
1109+ subClosed := make(chan struct{})
1110+ stale := supervisorAttempt{
1111+ generation: current - 1,
1112+ service: service,
1113+ sub: supervisorTestSubscription{closeFn: func() { close(subClosed) }},
1114+ }
1115+ adopted, cause := sup.adoptAttempt(context.Background(), stale)
1116+ require.False(t, adopted, "a superseded generation must not be adopted")
1117+ require.NoError(t, cause, "a superseded generation is not a cancellation")
1118+ // Run retires every attempt it does not adopt.
1119+ stale.retire()
1120+ select {
1121+ case <-service.closedCh():
1122+ default:
1123+ t.Fatal("a superseded service must be closed")
1124+ }
1125+ select {
1126+ case <-subClosed:
1127+ default:
1128+ t.Fatal("a superseded subscription must be closed")
1129+ }
1130+
1131+ // The current generation is adopted, so its live service is left untouched.
1132+ live := newSupervisorTestService(ports.BrokerConnectionID{2})
1133+ adopted, cause = sup.adoptAttempt(context.Background(), supervisorAttempt{generation: current, service: live})
1134+ require.True(t, adopted, "the current generation must be adopted")
1135+ require.NoError(t, cause)
1136+ select {
1137+ case <-live.closedCh():
1138+ t.Fatal("the current attempt must be adopted, not retired")
1139+ default:
1140+ }
1141+}
1142+
1143+// TestSupervisorNormalizesUntypedConnectFailure proves an untyped connect
1144+// failure becomes a typed unavailable failure that preserves its cause, and that
1145+// the same typed value reaches the notifier.
1146+func TestSupervisorNormalizesUntypedConnectFailure(t *testing.T) {
1147+ ctx, cancel := context.WithCancel(context.Background())
1148+ defer cancel()
1149+
1150+ clock := newSupervisorTestClock()
1151+ reader := newSupervisorTestReader()
1152+ t.Cleanup(reader.unblock)
1153+ terminal := newSupervisorTestTerminal(reader)
1154+
1155+ cause := errors.New("dial: connection refused")
1156+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
1157+ return nil, cause
1158+ })
1159+ notified := make(chan error, 4)
1160+ sup := mustSupervisor(t, SupervisorConfig{
1161+ Connector: connector,
1162+ Terminal: terminal,
1163+ Clock: clock,
1164+ Jitter: func() float64 { return 0 },
1165+ Notify: func(_ State, err error) { notified <- err },
1166+ })
1167+
1168+ runErr := make(chan error, 1)
1169+ go func() { runErr <- sup.Run(ctx) }()
1170+ clock.awaitTimer(t)
1171+
1172+ var typed ports.BrokerError
1173+ require.ErrorAs(t, sup.State().Err, &typed)
1174+ require.Equal(t, ports.BrokerErrorUnavailable, typed.Code)
1175+ require.ErrorIs(t, typed.Cause, cause, "the untyped cause is preserved")
1176+
1177+ select {
1178+ case err := <-notified:
1179+ var surfaced ports.BrokerError
1180+ require.ErrorAs(t, err, &surfaced)
1181+ require.Equal(t, ports.BrokerErrorUnavailable, surfaced.Code)
1182+ require.ErrorIs(t, surfaced.Cause, cause)
1183+ case <-time.After(5 * time.Second):
1184+ t.Fatal("the normalized failure was not surfaced")
1185+ }
1186+
1187+ cancel()
1188+ require.ErrorIs(t, <-runErr, context.Canceled)
1189+ require.Equal(t, 1, terminal.restoreCount())
1190+}
1191+
1192+// TestSupervisorNormalizesUntypedLoss proves an untyped established-connection
1193+// loss is normalized to a typed unavailable failure that preserves its cause.
1194+func TestSupervisorNormalizesUntypedLoss(t *testing.T) {
1195+ ctx, cancel := context.WithCancel(context.Background())
1196+ defer cancel()
1197+
1198+ clock := newSupervisorTestClock()
1199+ reader := newSupervisorTestReader()
1200+ t.Cleanup(reader.unblock)
1201+ terminal := newSupervisorTestTerminal(reader)
1202+
1203+ service := newSupervisorTestService(ports.BrokerConnectionID{1})
1204+ service.publish(1, 1)
1205+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
1206+ return service, nil
1207+ })
1208+ sup := mustSupervisor(t, SupervisorConfig{
1209+ Connector: connector,
1210+ Terminal: terminal,
1211+ Clock: clock,
1212+ Jitter: func() float64 { return 0 },
1213+ })
1214+
1215+ runErr := make(chan error, 1)
1216+ go func() { runErr <- sup.Run(ctx) }()
1217+ require.Equal(t, 1, connector.awaitStart(t))
1218+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityReady }, 5*time.Second, time.Millisecond)
1219+
1220+ cause := errors.New("carriage reset")
1221+ service.lose(cause)
1222+ require.Eventually(t, func() bool { return sup.State().Connectivity == ConnectivityRetryWait }, 5*time.Second, time.Millisecond)
1223+
1224+ var typed ports.BrokerError
1225+ require.ErrorAs(t, sup.State().Err, &typed)
1226+ require.Equal(t, ports.BrokerErrorUnavailable, typed.Code)
1227+ require.ErrorIs(t, typed.Cause, cause, "the untyped loss cause is preserved")
1228+
1229+ cancel()
1230+ require.ErrorIs(t, <-runErr, context.Canceled)
1231+}
1232+
1233+// TestSupervisorNilSubscriptionIsTypedUnavailable proves a service that reports
1234+// no subscription (a nil interface or a typed nil) is closed and treated as a
1235+// typed unavailable failure instead of panicking on the first Changed call.
1236+func TestSupervisorNilSubscriptionIsTypedUnavailable(t *testing.T) {
1237+ cases := []struct {
1238+ name string
1239+ sub ports.BrokerSubscription
1240+ }{
1241+ {"nil interface", nil},
1242+ {"typed nil", (*supervisorNilSub)(nil)},
1243+ }
1244+ for _, tc := range cases {
1245+ t.Run(tc.name, func(t *testing.T) {
1246+ ctx, cancel := context.WithCancel(context.Background())
1247+ defer cancel()
1248+
1249+ clock := newSupervisorTestClock()
1250+ reader := newSupervisorTestReader()
1251+ t.Cleanup(reader.unblock)
1252+ terminal := newSupervisorTestTerminal(reader)
1253+
1254+ base := newSupervisorTestService(ports.BrokerConnectionID{7})
1255+ svc := &supervisorNilSubService{supervisorTestService: base, sub: tc.sub}
1256+ connector := newSupervisorTestConnector(func(context.Context, int) (ports.BrokerService, error) {
1257+ return svc, nil
1258+ })
1259+ sup := mustSupervisor(t, SupervisorConfig{
1260+ Connector: connector,
1261+ Terminal: terminal,
1262+ Clock: clock,
1263+ Jitter: func() float64 { return 0 },
1264+ })
1265+
1266+ runErr := make(chan error, 1)
1267+ go func() { runErr <- sup.Run(ctx) }()
1268+ clock.awaitTimer(t)
1269+
1270+ var typed ports.BrokerError
1271+ require.ErrorAs(t, sup.State().Err, &typed)
1272+ require.Equal(t, ports.BrokerErrorUnavailable, typed.Code)
1273+ select {
1274+ case <-base.closedCh():
1275+ default:
1276+ t.Fatal("a service that returned no subscription must be closed")
1277+ }
1278+
1279+ cancel()
1280+ require.ErrorIs(t, <-runErr, context.Canceled)
1281+ require.Equal(t, 1, terminal.restoreCount())
1282+ })
1283+ }
1284+}

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 12

Tags 11

↑↓ moveenter openesc close