detached ad3de98

feat(broker): add host observation registry

commit
ad3de98d9d72f9eb8635e395823139c4db449bc4
author
brice · · 2026-09-16 09:41 UTC
parent
01ca71d
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
5 files · +2671 -1
Unified
UnifiedSplit
M internal/ports/broker.go +26 -1
@@ -565,12 +565,37 @@ type BrokerService interface {
565565
566566 // BrokerSnapshotStore persists the durable broker snapshot independently
567567 // from session state. The broker is its sole writer; session persistence
568-// stays daemon-owned and is never moved here.
568+// stays daemon-owned and is never moved here. Load is called once while the
569+// broker is constructed; Store runs on the broker's serialized writer
570+// goroutine. Store must return promptly and must never block indefinitely:
571+// this seam deliberately carries no cancellation context in P2.1, and broker
572+// shutdown waits for the in-flight write to finish, so a Store that blocks
573+// without bound blocks shutdown. Implementations must bound their own write
574+// time and return an error rather than wait forever.
569575 type BrokerSnapshotStore interface {
570576 Load() (BrokerSnapshot, error)
571577 Store(BrokerSnapshot) error
572578 }
573579
580+// BrokerHostProbe observes one exact host registration without creating an
581+// attachment. The returned projection must describe the same registration,
582+// must not exceed BrokerMaxSessionsPerHost sessions (an over-bound projection
583+// is classified as an invalid response and never published), and every
584+// returned session and its catalogue values must already be fully validated
585+// and within the catalogue bounds: the registry re-checks only the session
586+// count, so malformed or over-bound session names, tab identities, states,
587+// and detail text would be published verbatim. Implementations should
588+// validate the projection with catalogue.ValidateRemoteCatalog against the
589+// RemoteCatalogMax* bounds before returning it. Probe must return promptly
590+// once ctx is cancelled: the registry cancels the attempt context when a
591+// registration is replaced or removed, and when the run settles. A probe that
592+// ignores cancellation delays that retirement and can overlap a replacement
593+// attempt for the same endpoint. Stale completions that arrive anyway are
594+// fenced by epoch and registration identity before publication.
595+type BrokerHostProbe interface {
596+ Probe(ctx context.Context, registration domain.RemoteRegistration) (RemoteHostSnapshot, error)
597+}
598+
574599 // BrokerPhysicalConnection is one pooled physical transport to an
575600 // authenticated daemon identity. It opens one independently cancellable
576601 // typed session connection per logical stream and never shares stream
@@ -565,12 +565,37 @@ type BrokerService interface {
565 565
566 // BrokerSnapshotStore persists the durable broker snapshot independently566 // BrokerSnapshotStore persists the durable broker snapshot independently
567 // from session state. The broker is its sole writer; session persistence567 // from session state. The broker is its sole writer; session persistence
568-// stays daemon-owned and is never moved here.568+// stays daemon-owned and is never moved here. Load is called once while the
569+// broker is constructed; Store runs on the broker's serialized writer
570+// goroutine. Store must return promptly and must never block indefinitely:
571+// this seam deliberately carries no cancellation context in P2.1, and broker
572+// shutdown waits for the in-flight write to finish, so a Store that blocks
573+// without bound blocks shutdown. Implementations must bound their own write
574+// time and return an error rather than wait forever.
569 type BrokerSnapshotStore interface {575 type BrokerSnapshotStore interface {
570 Load() (BrokerSnapshot, error)576 Load() (BrokerSnapshot, error)
571 Store(BrokerSnapshot) error577 Store(BrokerSnapshot) error
572 }578 }
573 579
580+// BrokerHostProbe observes one exact host registration without creating an
581+// attachment. The returned projection must describe the same registration,
582+// must not exceed BrokerMaxSessionsPerHost sessions (an over-bound projection
583+// is classified as an invalid response and never published), and every
584+// returned session and its catalogue values must already be fully validated
585+// and within the catalogue bounds: the registry re-checks only the session
586+// count, so malformed or over-bound session names, tab identities, states,
587+// and detail text would be published verbatim. Implementations should
588+// validate the projection with catalogue.ValidateRemoteCatalog against the
589+// RemoteCatalogMax* bounds before returning it. Probe must return promptly
590+// once ctx is cancelled: the registry cancels the attempt context when a
591+// registration is replaced or removed, and when the run settles. A probe that
592+// ignores cancellation delays that retirement and can overlap a replacement
593+// attempt for the same endpoint. Stale completions that arrive anyway are
594+// fenced by epoch and registration identity before publication.
595+type BrokerHostProbe interface {
596+ Probe(ctx context.Context, registration domain.RemoteRegistration) (RemoteHostSnapshot, error)
597+}
598+
574 // BrokerPhysicalConnection is one pooled physical transport to an599 // BrokerPhysicalConnection is one pooled physical transport to an
575 // authenticated daemon identity. It opens one independently cancellable600 // authenticated daemon identity. It opens one independently cancellable
576 // typed session connection per logical stream and never shares stream601 // typed session connection per logical stream and never shares stream
M internal/ports/broker_test.go +1 -0
@@ -467,6 +467,7 @@ func TestBrokerInterfacesArePorts(t *testing.T) {
467467 for name, iface := range map[string]reflect.Type{
468468 "BrokerService": reflect.TypeOf((*BrokerService)(nil)).Elem(),
469469 "BrokerSnapshotStore": reflect.TypeOf((*BrokerSnapshotStore)(nil)).Elem(),
470+ "BrokerHostProbe": reflect.TypeOf((*BrokerHostProbe)(nil)).Elem(),
470471 "BrokerPhysicalConnection": reflect.TypeOf((*BrokerPhysicalConnection)(nil)).Elem(),
471472 "BrokerEndpointConnector": reflect.TypeOf((*BrokerEndpointConnector)(nil)).Elem(),
472473 "BrokerListener": reflect.TypeOf((*BrokerListener)(nil)).Elem(),
@@ -467,6 +467,7 @@ func TestBrokerInterfacesArePorts(t *testing.T) {
467 for name, iface := range map[string]reflect.Type{467 for name, iface := range map[string]reflect.Type{
468 "BrokerService": reflect.TypeOf((*BrokerService)(nil)).Elem(),468 "BrokerService": reflect.TypeOf((*BrokerService)(nil)).Elem(),
469 "BrokerSnapshotStore": reflect.TypeOf((*BrokerSnapshotStore)(nil)).Elem(),469 "BrokerSnapshotStore": reflect.TypeOf((*BrokerSnapshotStore)(nil)).Elem(),
470+ "BrokerHostProbe": reflect.TypeOf((*BrokerHostProbe)(nil)).Elem(),
470 "BrokerPhysicalConnection": reflect.TypeOf((*BrokerPhysicalConnection)(nil)).Elem(),471 "BrokerPhysicalConnection": reflect.TypeOf((*BrokerPhysicalConnection)(nil)).Elem(),
471 "BrokerEndpointConnector": reflect.TypeOf((*BrokerEndpointConnector)(nil)).Elem(),472 "BrokerEndpointConnector": reflect.TypeOf((*BrokerEndpointConnector)(nil)).Elem(),
472 "BrokerListener": reflect.TypeOf((*BrokerListener)(nil)).Elem(),473 "BrokerListener": reflect.TypeOf((*BrokerListener)(nil)).Elem(),
M internal/ports/mocks/broker_mock.go +94 -0
@@ -7,6 +7,7 @@ package portsmocks
77 import (
88 "context"
99
10+ "github.com/bnema/vev/internal/domain"
1011 "github.com/bnema/vev/internal/ports"
1112 mock "github.com/stretchr/testify/mock"
1213 )
@@ -752,6 +753,99 @@ func (_c *MockBrokerSnapshotStore_Store_Call) RunAndReturn(run func(brokerSnapsh
752753 return _c
753754 }
754755
756+// NewMockBrokerHostProbe creates a new instance of MockBrokerHostProbe. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
757+// The first argument is typically a *testing.T value.
758+func NewMockBrokerHostProbe(t interface {
759+ mock.TestingT
760+ Cleanup(func())
761+}) *MockBrokerHostProbe {
762+ mock := &MockBrokerHostProbe{}
763+ mock.Mock.Test(t)
764+
765+ t.Cleanup(func() { mock.AssertExpectations(t) })
766+
767+ return mock
768+}
769+
770+// MockBrokerHostProbe is an autogenerated mock type for the BrokerHostProbe type
771+type MockBrokerHostProbe struct {
772+ mock.Mock
773+}
774+
775+type MockBrokerHostProbe_Expecter struct {
776+ mock *mock.Mock
777+}
778+
779+func (_m *MockBrokerHostProbe) EXPECT() *MockBrokerHostProbe_Expecter {
780+ return &MockBrokerHostProbe_Expecter{mock: &_m.Mock}
781+}
782+
783+// Probe provides a mock function for the type MockBrokerHostProbe
784+func (_mock *MockBrokerHostProbe) Probe(ctx context.Context, registration domain.RemoteRegistration) (ports.RemoteHostSnapshot, error) {
785+ ret := _mock.Called(ctx, registration)
786+
787+ if len(ret) == 0 {
788+ panic("no return value specified for Probe")
789+ }
790+
791+ var r0 ports.RemoteHostSnapshot
792+ var r1 error
793+ if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RemoteRegistration) (ports.RemoteHostSnapshot, error)); ok {
794+ return returnFunc(ctx, registration)
795+ }
796+ if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RemoteRegistration) ports.RemoteHostSnapshot); ok {
797+ r0 = returnFunc(ctx, registration)
798+ } else {
799+ r0 = ret.Get(0).(ports.RemoteHostSnapshot)
800+ }
801+ if returnFunc, ok := ret.Get(1).(func(context.Context, domain.RemoteRegistration) error); ok {
802+ r1 = returnFunc(ctx, registration)
803+ } else {
804+ r1 = ret.Error(1)
805+ }
806+ return r0, r1
807+}
808+
809+// MockBrokerHostProbe_Probe_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Probe'
810+type MockBrokerHostProbe_Probe_Call struct {
811+ *mock.Call
812+}
813+
814+// Probe is a helper method to define mock.On call
815+// - ctx context.Context
816+// - registration domain.RemoteRegistration
817+func (_e *MockBrokerHostProbe_Expecter) Probe(ctx any, registration any) *MockBrokerHostProbe_Probe_Call {
818+ return &MockBrokerHostProbe_Probe_Call{Call: _e.mock.On("Probe", ctx, registration)}
819+}
820+
821+func (_c *MockBrokerHostProbe_Probe_Call) Run(run func(ctx context.Context, registration domain.RemoteRegistration)) *MockBrokerHostProbe_Probe_Call {
822+ _c.Call.Run(func(args mock.Arguments) {
823+ var arg0 context.Context
824+ if args[0] != nil {
825+ arg0 = args[0].(context.Context)
826+ }
827+ var arg1 domain.RemoteRegistration
828+ if args[1] != nil {
829+ arg1 = args[1].(domain.RemoteRegistration)
830+ }
831+ run(
832+ arg0,
833+ arg1,
834+ )
835+ })
836+ return _c
837+}
838+
839+func (_c *MockBrokerHostProbe_Probe_Call) Return(remoteHostSnapshot ports.RemoteHostSnapshot, err error) *MockBrokerHostProbe_Probe_Call {
840+ _c.Call.Return(remoteHostSnapshot, err)
841+ return _c
842+}
843+
844+func (_c *MockBrokerHostProbe_Probe_Call) RunAndReturn(run func(ctx context.Context, registration domain.RemoteRegistration) (ports.RemoteHostSnapshot, error)) *MockBrokerHostProbe_Probe_Call {
845+ _c.Call.Return(run)
846+ return _c
847+}
848+
755849 // NewMockBrokerPhysicalConnection creates a new instance of MockBrokerPhysicalConnection. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
756850 // The first argument is typically a *testing.T value.
757851 func NewMockBrokerPhysicalConnection(t interface {
@@ -7,6 +7,7 @@ package portsmocks
7 import (7 import (
8 "context"8 "context"
9 9
10+ "github.com/bnema/vev/internal/domain"
10 "github.com/bnema/vev/internal/ports"11 "github.com/bnema/vev/internal/ports"
11 mock "github.com/stretchr/testify/mock"12 mock "github.com/stretchr/testify/mock"
12 )13 )
@@ -752,6 +753,99 @@ func (_c *MockBrokerSnapshotStore_Store_Call) RunAndReturn(run func(brokerSnapsh
752 return _c753 return _c
753 }754 }
754 755
756+// NewMockBrokerHostProbe creates a new instance of MockBrokerHostProbe. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
757+// The first argument is typically a *testing.T value.
758+func NewMockBrokerHostProbe(t interface {
759+ mock.TestingT
760+ Cleanup(func())
761+}) *MockBrokerHostProbe {
762+ mock := &MockBrokerHostProbe{}
763+ mock.Mock.Test(t)
764+
765+ t.Cleanup(func() { mock.AssertExpectations(t) })
766+
767+ return mock
768+}
769+
770+// MockBrokerHostProbe is an autogenerated mock type for the BrokerHostProbe type
771+type MockBrokerHostProbe struct {
772+ mock.Mock
773+}
774+
775+type MockBrokerHostProbe_Expecter struct {
776+ mock *mock.Mock
777+}
778+
779+func (_m *MockBrokerHostProbe) EXPECT() *MockBrokerHostProbe_Expecter {
780+ return &MockBrokerHostProbe_Expecter{mock: &_m.Mock}
781+}
782+
783+// Probe provides a mock function for the type MockBrokerHostProbe
784+func (_mock *MockBrokerHostProbe) Probe(ctx context.Context, registration domain.RemoteRegistration) (ports.RemoteHostSnapshot, error) {
785+ ret := _mock.Called(ctx, registration)
786+
787+ if len(ret) == 0 {
788+ panic("no return value specified for Probe")
789+ }
790+
791+ var r0 ports.RemoteHostSnapshot
792+ var r1 error
793+ if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RemoteRegistration) (ports.RemoteHostSnapshot, error)); ok {
794+ return returnFunc(ctx, registration)
795+ }
796+ if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RemoteRegistration) ports.RemoteHostSnapshot); ok {
797+ r0 = returnFunc(ctx, registration)
798+ } else {
799+ r0 = ret.Get(0).(ports.RemoteHostSnapshot)
800+ }
801+ if returnFunc, ok := ret.Get(1).(func(context.Context, domain.RemoteRegistration) error); ok {
802+ r1 = returnFunc(ctx, registration)
803+ } else {
804+ r1 = ret.Error(1)
805+ }
806+ return r0, r1
807+}
808+
809+// MockBrokerHostProbe_Probe_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Probe'
810+type MockBrokerHostProbe_Probe_Call struct {
811+ *mock.Call
812+}
813+
814+// Probe is a helper method to define mock.On call
815+// - ctx context.Context
816+// - registration domain.RemoteRegistration
817+func (_e *MockBrokerHostProbe_Expecter) Probe(ctx any, registration any) *MockBrokerHostProbe_Probe_Call {
818+ return &MockBrokerHostProbe_Probe_Call{Call: _e.mock.On("Probe", ctx, registration)}
819+}
820+
821+func (_c *MockBrokerHostProbe_Probe_Call) Run(run func(ctx context.Context, registration domain.RemoteRegistration)) *MockBrokerHostProbe_Probe_Call {
822+ _c.Call.Run(func(args mock.Arguments) {
823+ var arg0 context.Context
824+ if args[0] != nil {
825+ arg0 = args[0].(context.Context)
826+ }
827+ var arg1 domain.RemoteRegistration
828+ if args[1] != nil {
829+ arg1 = args[1].(domain.RemoteRegistration)
830+ }
831+ run(
832+ arg0,
833+ arg1,
834+ )
835+ })
836+ return _c
837+}
838+
839+func (_c *MockBrokerHostProbe_Probe_Call) Return(remoteHostSnapshot ports.RemoteHostSnapshot, err error) *MockBrokerHostProbe_Probe_Call {
840+ _c.Call.Return(remoteHostSnapshot, err)
841+ return _c
842+}
843+
844+func (_c *MockBrokerHostProbe_Probe_Call) RunAndReturn(run func(ctx context.Context, registration domain.RemoteRegistration) (ports.RemoteHostSnapshot, error)) *MockBrokerHostProbe_Probe_Call {
845+ _c.Call.Return(run)
846+ return _c
847+}
848+
755 // NewMockBrokerPhysicalConnection creates a new instance of MockBrokerPhysicalConnection. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.849 // NewMockBrokerPhysicalConnection creates a new instance of MockBrokerPhysicalConnection. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
756 // The first argument is typically a *testing.T value.850 // The first argument is typically a *testing.T value.
757 func NewMockBrokerPhysicalConnection(t interface {851 func NewMockBrokerPhysicalConnection(t interface {
A internal/usecase/broker/registry.go +770 -0
@@ -0,0 +1,770 @@
1+package broker
2+
3+import (
4+ "context"
5+ "encoding/binary"
6+ "errors"
7+ "fmt"
8+ "hash/fnv"
9+ "log/slog"
10+ "sort"
11+ "sync"
12+ "sync/atomic"
13+ "time"
14+
15+ "github.com/bnema/vev/internal/domain"
16+ "github.com/bnema/vev/internal/ports"
17+)
18+
19+// Observation cadence. The broker deliberately mirrors the legacy remotes
20+// monitor cadence (internal/usecase/remotes) so a fleet observed through
21+// either path refreshes on the same schedule. The duplication is temporary:
22+// the registry is intended to replace that monitor, and these constants move
23+// to a single owner once the old monitor is removed.
24+const (
25+ defaultFreshFor = 15 * time.Second
26+ defaultRetryBase = 5 * time.Second
27+ defaultRetryLimit = time.Minute
28+)
29+
30+// Run lifecycle states. A registry runs exactly once: runIdle is the zero
31+// value, runActive marks the single admitted run, and runStopped permanently
32+// refuses further runs.
33+const (
34+ runIdle int32 = iota
35+ runActive
36+ runStopped
37+)
38+
39+// Registry owns configured hosts and their immutable observation projection.
40+type Registry struct {
41+ epoch ports.BrokerEpoch
42+ probe ports.BrokerHostProbe
43+ clock ports.Clock
44+ log *slog.Logger
45+ store *snapshotWriter
46+
47+ current atomic.Pointer[ports.BrokerSnapshot]
48+ mu sync.Mutex
49+ revision ports.BrokerRevision
50+ // attempts is the registry-wide monotonic attempt token source. Every
51+ // started observation claims one token, and only the token currently
52+ // recorded as in flight may complete. Tokens are never reused, so a stale
53+ // completion is fenced even when the endpoint is removed and re-added with
54+ // the identical registration.
55+ attempts uint64
56+ hosts map[string]ports.RemoteHostSnapshot
57+ // inflight records the one admitted attempt per endpoint. The record
58+ // carries the attempt's derived context cancel so a retired attempt is
59+ // released the moment its registration is replaced, removed, or the run
60+ // settles; a ctx-honoring probe therefore stops promptly and churn can
61+ // never accumulate overlapping probes for one endpoint.
62+ inflight map[string]*probeAttempt
63+ pending map[string]bool
64+ tombstones map[string]ports.BrokerHostTombstone
65+ subs map[*subscription]struct{}
66+ wake chan struct{}
67+ running atomic.Int32
68+
69+ freshFor time.Duration
70+ retryBase time.Duration
71+ retryMax time.Duration
72+ jitter func(base time.Duration, endpoint string, attempt uint64) time.Duration
73+}
74+
75+// NewRegistry restores a validated durable snapshot under a fresh broker epoch.
76+func NewRegistry(epoch ports.BrokerEpoch, store ports.BrokerSnapshotStore, probe ports.BrokerHostProbe, clock ports.Clock, log *slog.Logger) (*Registry, error) {
77+ if epoch == 0 || probe == nil || clock == nil {
78+ return nil, errors.New("broker: invalid registry dependencies")
79+ }
80+ if log == nil {
81+ log = slog.Default()
82+ }
83+ r := &Registry{
84+ epoch: epoch, probe: probe, clock: clock, log: log,
85+ hosts: make(map[string]ports.RemoteHostSnapshot), inflight: make(map[string]*probeAttempt),
86+ pending: make(map[string]bool), tombstones: make(map[string]ports.BrokerHostTombstone),
87+ subs: make(map[*subscription]struct{}), wake: make(chan struct{}, 1),
88+ freshFor: defaultFreshFor, retryBase: defaultRetryBase, retryMax: defaultRetryLimit,
89+ jitter: jittered,
90+ }
91+ if store != nil {
92+ r.store = newSnapshotWriter(store, log)
93+ snapshot, err := store.Load()
94+ if err != nil {
95+ return nil, err
96+ }
97+ if err := r.restore(snapshot); err != nil {
98+ return nil, err
99+ }
100+ }
101+ r.publishLocked(false)
102+ return r, nil
103+}
104+
105+// restore adopts a validated durable snapshot. A zero-epoch snapshot is only
106+// the empty placeholder a store returns before its first write: any record
107+// without an epoch is rejected instead of silently dropped. A persisted
108+// snapshot from the registry's own epoch is rejected as well: every broker
109+// process samples a fresh epoch and restarts its revision series at one, so
110+// adopting same-epoch revisions would make the restored series non-monotonic
111+// (and a stale same-epoch record indistinguishable from current authority).
112+// In-flight observation state and retirement tombstones are per-process
113+// fencing state and never survive a restart.
114+func (r *Registry) restore(snapshot ports.BrokerSnapshot) error {
115+ if snapshot.Epoch == 0 {
116+ if len(snapshot.Hosts) > 0 || len(snapshot.Removed) > 0 || snapshot.Revision != 0 {
117+ return errors.New("broker: persisted snapshot has no epoch")
118+ }
119+ return nil
120+ }
121+ if snapshot.Epoch == r.epoch {
122+ return errors.New("broker: persisted snapshot epoch matches the new registry epoch")
123+ }
124+ if err := snapshot.Validate(); err != nil {
125+ return err
126+ }
127+ for _, host := range snapshot.Hosts {
128+ host = host.Clone()
129+ host.Checking = false
130+ r.hosts[host.Endpoint] = host
131+ }
132+ return nil
133+}
134+
135+// Snapshot returns a defensive immutable copy without I/O.
136+func (r *Registry) Snapshot() ports.BrokerSnapshot { return r.current.Load().Clone() }
137+
138+// Subscribe returns a capacity-one notification stream. Slow consumers never
139+// block publication and always re-read the newest snapshot.
140+func (r *Registry) Subscribe() ports.BrokerSubscription {
141+ s := &subscription{changed: make(chan struct{}, 1)}
142+ s.close = func() {
143+ r.mu.Lock()
144+ delete(r.subs, s)
145+ r.mu.Unlock()
146+ }
147+ r.mu.Lock()
148+ r.subs[s] = struct{}{}
149+ r.mu.Unlock()
150+ s.offer()
151+ return s
152+}
153+
154+// SetHosts atomically replaces configured membership. An unchanged membership
155+// is a no-op: it neither republishes nor restarts an in-flight observation.
156+// Removed registrations retire bounded tombstones so a stale completion can
157+// never revive them; replaced or re-added endpoints are fenced by the
158+// registry attempt token instead.
159+func (r *Registry) SetHosts(registrations []domain.RemoteRegistration) error {
160+ if len(registrations) > ports.BrokerMaxHosts {
161+ return fmt.Errorf("broker: %d host registrations exceed the limit of %d", len(registrations), ports.BrokerMaxHosts)
162+ }
163+ next := make(map[string]ports.RemoteHostSnapshot, len(registrations))
164+ for _, registration := range registrations {
165+ if err := registration.Validate(); err != nil {
166+ return err
167+ }
168+ if _, duplicate := next[registration.Endpoint]; duplicate {
169+ return errors.New("broker: duplicate host registration")
170+ }
171+ next[registration.Endpoint] = ports.RemoteHostSnapshot{Endpoint: registration.Endpoint, Registration: registration, Availability: domain.RemoteAvailabilityUnknown}
172+ }
173+
174+ r.mu.Lock()
175+ if registrationsUnchanged(r.hosts, next) {
176+ r.mu.Unlock()
177+ return nil
178+ }
179+ for endpoint, current := range r.hosts {
180+ candidate, present := next[endpoint]
181+ if present && current.Registration.Equal(candidate.Registration) {
182+ // Unchanged registration: keep the live observation as is.
183+ next[endpoint] = current
184+ continue
185+ }
186+ // A retired registration owns no observation: cancelling its attempt
187+ // releases a ctx-honoring probe immediately, and dropping its record
188+ // means the retired completion can never clear or answer the
189+ // replacement, and the replacement is never blocked behind it. The
190+ // cancellation happens under the registry lock, so no replacement
191+ // attempt can start before the retired context is cancelled.
192+ r.cancelInflightLocked(endpoint)
193+ delete(r.pending, endpoint)
194+ if present {
195+ // The endpoint stays live under a new registration, so its
196+ // retirement is never published as a tombstone alongside a live
197+ // host; the attempt token fences the stale completion.
198+ delete(r.tombstones, endpoint)
199+ continue
200+ }
201+ r.retireLocked(current.Registration)
202+ }
203+ for endpoint := range next {
204+ // Re-adding an endpoint supersedes its tombstone: a live host and a
205+ // tombstone for the same endpoint are mutually exclusive.
206+ delete(r.tombstones, endpoint)
207+ }
208+ r.hosts = next
209+ r.publishLocked(true)
210+ r.mu.Unlock()
211+ r.hint()
212+ return nil
213+}
214+
215+// cancelInflightLocked cancels the in-flight attempt for one endpoint and
216+// drops its record. Callers must hold r.mu: the retired attempt's context is
217+// cancelled before the record disappears, so no replacement attempt can start
218+// while the retired probe's context is still live.
219+func (r *Registry) cancelInflightLocked(endpoint string) {
220+ if attempt := r.inflight[endpoint]; attempt != nil {
221+ attempt.cancel()
222+ delete(r.inflight, endpoint)
223+ }
224+}
225+
226+// cancelAllInflightLocked cancels every in-flight attempt and drops the
227+// records. Callers must hold r.mu.
228+func (r *Registry) cancelAllInflightLocked() {
229+ for endpoint, attempt := range r.inflight {
230+ attempt.cancel()
231+ delete(r.inflight, endpoint)
232+ }
233+}
234+
235+// RequestProbe is non-blocking and coalesces concurrent demand per endpoint.
236+func (r *Registry) RequestProbe(endpoint string) {
237+ r.mu.Lock()
238+ if _, ok := r.hosts[endpoint]; ok {
239+ r.pending[endpoint] = true
240+ }
241+ r.mu.Unlock()
242+ r.hint()
243+}
244+
245+func (r *Registry) hint() {
246+ select {
247+ case r.wake <- struct{}{}:
248+ default:
249+ }
250+}
251+
252+// Run schedules observations until cancellation. Run owns the registry
253+// lifecycle and may execute exactly once: a second call is refused so a
254+// concurrent or restarted caller can never duplicate scheduling. On return it
255+// clears in-flight observation state, publishes the settled projection, and
256+// flushes durable persistence before any further write is refused.
257+func (r *Registry) Run(ctx context.Context) {
258+ if !r.running.CompareAndSwap(runIdle, runActive) {
259+ r.log.Warn("broker: registry Run refused; single-run already started or finished")
260+ return
261+ }
262+ defer r.settle()
263+ timer := r.clock.NewTimer(0)
264+ defer timer.Stop()
265+ results := make(chan probeResult, ports.BrokerMaxHosts)
266+ for {
267+ now := r.clock.Now()
268+ r.dispatch(ctx, now, results)
269+ timer.Reset(r.nextDelay(now))
270+ select {
271+ case <-ctx.Done():
272+ return
273+ case <-r.wake:
274+ case <-timer.C():
275+ case result := <-results:
276+ if ctx.Err() != nil {
277+ // Shutdown wins: a cancellation is never recorded as an
278+ // observation failure.
279+ return
280+ }
281+ r.apply(result)
282+ }
283+ }
284+}
285+
286+// settle is Run's cleanup: nothing stays in flight, the published projection
287+// carries no transient Checking flag, durable persistence is flushed, and
288+// every later Run call is refused.
289+func (r *Registry) settle() {
290+ r.mu.Lock()
291+ r.cancelAllInflightLocked()
292+ cleared := false
293+ for endpoint, host := range r.hosts {
294+ if !host.Checking {
295+ continue
296+ }
297+ host.Checking = false
298+ r.hosts[endpoint] = host
299+ cleared = true
300+ }
301+ if cleared {
302+ r.publishLocked(true)
303+ }
304+ r.mu.Unlock()
305+ if r.store != nil {
306+ r.store.close()
307+ }
308+ r.running.Store(runStopped)
309+}
310+
311+// probeAttempt is one admitted observation: the registry-wide token that
312+// fences its completion, and the cancel for the context derived from the run
313+// context. Cancelling the derived context retires the attempt: a probe that
314+// honors cancellation returns promptly, so a replaced or removed endpoint
315+// never accumulates overlapping probes, and settle releases every attempt
316+// before the run loop exits.
317+type probeAttempt struct {
318+ token uint64
319+ cancel context.CancelFunc
320+}
321+
322+// probeResult is one completed observation bound to the exact attempt token
323+// that started it.
324+type probeResult struct {
325+ endpoint string
326+ registration domain.RemoteRegistration
327+ attempt uint64
328+ snapshot ports.RemoteHostSnapshot
329+ err error
330+ at time.Time
331+}
332+
333+func (r *Registry) dispatch(ctx context.Context, now time.Time, results chan<- probeResult) {
334+ r.mu.Lock()
335+ started := false
336+ for endpoint, host := range r.hosts {
337+ if _, inflight := r.inflight[endpoint]; inflight {
338+ continue
339+ }
340+ if !r.pending[endpoint] && !host.NextDue.IsZero() && now.Before(host.NextDue) {
341+ continue
342+ }
343+ delete(r.pending, endpoint)
344+ r.attempts++
345+ attempt := r.attempts
346+ probeCtx, cancel := context.WithCancel(ctx)
347+ r.inflight[endpoint] = &probeAttempt{token: attempt, cancel: cancel}
348+ started = true
349+ host.Checking = true
350+ host.LastAttempt = now
351+ r.hosts[endpoint] = host
352+ registration := host.Registration
353+ go func(endpoint string, registration domain.RemoteRegistration, attempt uint64, probeCtx context.Context) {
354+ snapshot, err := r.probe.Probe(probeCtx, registration)
355+ result := probeResult{endpoint: endpoint, registration: registration, attempt: attempt, snapshot: snapshot, err: err, at: r.clock.Now()}
356+ select {
357+ case results <- result:
358+ case <-probeCtx.Done():
359+ // The attempt was retired (replaced, removed, or shutdown), so
360+ // its completion is already fenced: dropping it keeps churn from
361+ // queueing results the run loop would only discard.
362+ }
363+ }(endpoint, registration, attempt, probeCtx)
364+ }
365+ // Only an observation that actually started changes the projection; an
366+ // idle dispatch must not bump the revision or wake subscribers. The
367+ // in-flight projection is never persisted: Checking is transient.
368+ if started {
369+ r.publishLocked(false)
370+ }
371+ r.mu.Unlock()
372+}
373+
374+func (r *Registry) apply(result probeResult) {
375+ r.mu.Lock()
376+ defer r.mu.Unlock()
377+ current, ok := r.hosts[result.endpoint]
378+ attempt := r.inflight[result.endpoint]
379+ if !ok || !current.Registration.Equal(result.registration) || attempt == nil || attempt.token != result.attempt {
380+ // A stale completion touches neither current authority nor the
381+ // in-flight slot, which may already belong to a newer registration or
382+ // attempt.
383+ return
384+ }
385+ delete(r.inflight, result.endpoint)
386+ // The probe has returned, so releasing its derived context here only
387+ // frees the attempt's resources; the record is gone before any new
388+ // attempt can start.
389+ attempt.cancel()
390+ current.Checking = false
391+ current.LastAttempt = result.at
392+
393+ kind, availability, failed := observationOutcome(result)
394+ if !failed {
395+ // Reachable: adopt the observed inventory, but registry-owned
396+ // accounting (success time, attempt counters, failure episode) stays
397+ // local.
398+ observed := result.snapshot.Clone()
399+ observed.Endpoint = result.endpoint
400+ observed.Registration = result.registration
401+ observed.Checking = false
402+ observed.LastAttempt = result.at
403+ observed.LastSuccess = result.at
404+ observed.NextDue = result.at.Add(r.jitter(r.freshFor, result.endpoint, result.attempt))
405+ observed.ConsecutiveFailures = 0
406+ observed.LastFailure = domain.RemoteFailure{}
407+ observed.FailureEpisode = current.FailureEpisode
408+ r.hosts[result.endpoint] = observed
409+ r.publishLocked(true)
410+ return
411+ }
412+ // Every non-reachable outcome is a typed failure on the capped retry
413+ // cadence: the last success, last-known inventory, and failure episode
414+ // counter stay authoritative.
415+ current.Availability = availability
416+ if current.ConsecutiveFailures == 0 {
417+ current.FailureEpisode++
418+ }
419+ current.ConsecutiveFailures++
420+ current.LastFailure = domain.RemoteFailure{Kind: kind, Err: result.err}
421+ current.NextDue = result.at.Add(r.jitter(r.retryDelay(current.ConsecutiveFailures), result.endpoint, result.attempt))
422+ r.hosts[result.endpoint] = current
423+ r.publishLocked(true)
424+}
425+
426+// observationOutcome classifies a completed observation. It reports
427+// failed=false only for a confirmed reachable result; every other outcome
428+// carries the typed failure kind and the availability to publish.
429+func observationOutcome(result probeResult) (domain.RemoteFailureKind, domain.RemoteAvailability, bool) {
430+ if result.err != nil {
431+ kind := domain.RemoteFailureTransport
432+ if errors.Is(result.err, context.DeadlineExceeded) {
433+ kind = domain.RemoteFailureTimeout
434+ }
435+ return kind, availabilityFor(kind), true
436+ }
437+ // A projection carrying more sessions than the publication bound is a
438+ // malformed response: publishing it would produce a snapshot that fails
439+ // BrokerSnapshot.Validate. Classify the whole projection as an invalid
440+ // response so the last-known inventory stays authoritative.
441+ if len(result.snapshot.Sessions) > ports.BrokerMaxSessionsPerHost {
442+ return domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true
443+ }
444+ switch result.snapshot.Availability {
445+ case domain.RemoteAvailabilityReachable:
446+ return domain.RemoteFailureNone, domain.RemoteAvailabilityReachable, false
447+ case domain.RemoteAvailabilityIncompatible:
448+ return domain.RemoteFailureIncompatible, domain.RemoteAvailabilityIncompatible, true
449+ case domain.RemoteAvailabilityAuthFailed:
450+ return domain.RemoteFailureAuthentication, domain.RemoteAvailabilityAuthFailed, true
451+ case domain.RemoteAvailabilityInvalidResponse:
452+ return domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true
453+ default:
454+ // Unknown or Unreachable without an error still means reachability was
455+ // not confirmed, so it retries on the typed backoff cadence.
456+ return domain.RemoteFailureTransport, domain.RemoteAvailabilityUnreachable, true
457+ }
458+}
459+
460+// availabilityFor maps a typed failure kind to the published availability.
461+// The typed kind is preserved separately for notice policy.
462+func availabilityFor(kind domain.RemoteFailureKind) domain.RemoteAvailability {
463+ switch kind {
464+ case domain.RemoteFailureAuthentication:
465+ return domain.RemoteAvailabilityAuthFailed
466+ case domain.RemoteFailureIncompatible:
467+ return domain.RemoteAvailabilityIncompatible
468+ case domain.RemoteFailureInvalidResponse:
469+ return domain.RemoteAvailabilityInvalidResponse
470+ default:
471+ return domain.RemoteAvailabilityUnreachable
472+ }
473+}
474+
475+// retryDelay is the capped exponential retry cadence for a failure streak.
476+func (r *Registry) retryDelay(failures uint) time.Duration {
477+ delay := r.retryBase
478+ for i := uint(1); i < failures; i++ {
479+ if delay >= r.retryMax {
480+ return r.retryMax
481+ }
482+ delay *= 2
483+ }
484+ if delay > r.retryMax {
485+ return r.retryMax
486+ }
487+ return delay
488+}
489+
490+// jittered spreads a base interval deterministically across endpoint and
491+// attempt: identical inputs always yield the identical duration, and the
492+// result stays within 90%-110% of the base. No wall clock or global
493+// randomness participates, so retry and healthy-refresh schedules are
494+// reproducible instead of racing a shared PRNG.
495+func jittered(base time.Duration, endpoint string, attempt uint64) time.Duration {
496+ if base <= 0 {
497+ return base
498+ }
499+ h := fnv.New32a()
500+ h.Write([]byte(endpoint))
501+ var counter [8]byte
502+ binary.BigEndian.PutUint64(counter[:], attempt)
503+ h.Write(counter[:])
504+ spread := int64(h.Sum32() % 21)
505+ return base*90/100 + time.Duration(spread)*base/100
506+}
507+
508+func (r *Registry) nextDelay(now time.Time) time.Duration {
509+ r.mu.Lock()
510+ defer r.mu.Unlock()
511+ var earliest time.Time
512+ for endpoint, host := range r.hosts {
513+ // An observation already in flight is completed by its result, never
514+ // by the schedule; waiting on it here would spin the run loop.
515+ if _, inflight := r.inflight[endpoint]; inflight {
516+ continue
517+ }
518+ if r.pending[endpoint] || host.NextDue.IsZero() {
519+ return 0
520+ }
521+ if earliest.IsZero() || host.NextDue.Before(earliest) {
522+ earliest = host.NextDue
523+ }
524+ }
525+ if earliest.IsZero() {
526+ return time.Hour
527+ }
528+ if delay := earliest.Sub(now); delay > 0 {
529+ return delay
530+ }
531+ return 0
532+}
533+
534+// registrationsUnchanged reports exact membership equality: the same
535+// endpoints bound to the same registration identities. Projection fields such
536+// as availability, checking, or inventory never participate.
537+func registrationsUnchanged(current, next map[string]ports.RemoteHostSnapshot) bool {
538+ if len(current) != len(next) {
539+ return false
540+ }
541+ for endpoint, candidate := range next {
542+ existing, ok := current[endpoint]
543+ if !ok || !existing.Registration.Equal(candidate.Registration) {
544+ return false
545+ }
546+ }
547+ return true
548+}
549+
550+// retireLocked records the tombstone for one removed registration at the
551+// revision of the publication that carries it.
552+func (r *Registry) retireLocked(registration domain.RemoteRegistration) {
553+ retired := r.revision + 1
554+ if retired == 0 {
555+ retired = 1
556+ }
557+ r.tombstones[registration.Endpoint] = ports.BrokerHostTombstone{
558+ Endpoint: registration.Endpoint,
559+ Registration: registration,
560+ RetiredRevision: retired,
561+ }
562+ r.pruneTombstonesLocked()
563+}
564+
565+// pruneTombstonesLocked keeps only the newest BrokerMaxTombstones retirements
566+// so an endpoint that churns can never grow the published set without bound.
567+func (r *Registry) pruneTombstonesLocked() {
568+ for len(r.tombstones) > ports.BrokerMaxTombstones {
569+ oldest := ""
570+ var oldestRevision ports.BrokerRevision
571+ for endpoint, tombstone := range r.tombstones {
572+ if oldest == "" || tombstone.RetiredRevision < oldestRevision ||
573+ (tombstone.RetiredRevision == oldestRevision && endpoint < oldest) {
574+ oldest, oldestRevision = endpoint, tombstone.RetiredRevision
575+ }
576+ }
577+ delete(r.tombstones, oldest)
578+ }
579+}
580+
581+// tombstonesLocked renders the bounded tombstone publication, newest first.
582+// Live endpoints are never published as tombstones.
583+func (r *Registry) tombstonesLocked() []ports.BrokerHostTombstone {
584+ if len(r.tombstones) == 0 {
585+ return nil
586+ }
587+ out := make([]ports.BrokerHostTombstone, 0, len(r.tombstones))
588+ for endpoint, tombstone := range r.tombstones {
589+ if _, live := r.hosts[endpoint]; live {
590+ continue
591+ }
592+ out = append(out, tombstone)
593+ }
594+ sort.Slice(out, func(i, j int) bool {
595+ if out[i].RetiredRevision != out[j].RetiredRevision {
596+ return out[i].RetiredRevision > out[j].RetiredRevision
597+ }
598+ return out[i].Endpoint < out[j].Endpoint
599+ })
600+ if len(out) > ports.BrokerMaxTombstones {
601+ out = out[:ports.BrokerMaxTombstones]
602+ }
603+ return out
604+}
605+
606+func (r *Registry) publishLocked(persist bool) {
607+ r.revision++
608+ hosts := make([]ports.RemoteHostSnapshot, 0, len(r.hosts))
609+ for _, host := range r.hosts {
610+ hosts = append(hosts, host.Clone())
611+ }
612+ sort.Slice(hosts, func(i, j int) bool { return hosts[i].Endpoint < hosts[j].Endpoint })
613+ snapshot := ports.BrokerSnapshot{Epoch: r.epoch, Revision: r.revision, Hosts: hosts, Removed: r.tombstonesLocked()}
614+ r.current.Store(&snapshot)
615+ if persist && r.store != nil {
616+ // enqueue only stages an immutable publication: rendering the durable
617+ // copy and every store call happen on the writer goroutine, never
618+ // under the registry lock and never on the probe schedule.
619+ r.store.enqueue(snapshot)
620+ }
621+ for sub := range r.subs {
622+ sub.offer()
623+ }
624+}
625+
626+// durable returns the persistable copy of a publication, rendered by the
627+// writer goroutine. Checking is transient in-flight state and never reaches
628+// the durable snapshot.
629+func durable(snapshot ports.BrokerSnapshot) ports.BrokerSnapshot {
630+ out := snapshot.Clone()
631+ for i := range out.Hosts {
632+ out.Hosts[i].Checking = false
633+ }
634+ return out
635+}
636+
637+// snapshotWriter serializes durable snapshot writes off the registry lock.
638+// Enqueue never blocks on store I/O and always coalesces to the newest
639+// revision; Store calls run on one goroutine, so the durable store never sees
640+// an out-of-order write. A failed write is logged and retried by the next
641+// publication, so a store fault never stalls publication. Close flushes the
642+// newest pending snapshot, stops the writer, and waits, so no write outlives
643+// shutdown.
644+type snapshotWriter struct {
645+ store ports.BrokerSnapshotStore
646+ log *slog.Logger
647+
648+ mu sync.Mutex
649+ pending *ports.BrokerSnapshot
650+ written ports.BrokerRevision
651+ started bool
652+ closed bool
653+ wake chan struct{}
654+ stop chan struct{}
655+ done chan struct{}
656+}
657+
658+func newSnapshotWriter(store ports.BrokerSnapshotStore, log *slog.Logger) *snapshotWriter {
659+ return &snapshotWriter{
660+ store: store, log: log,
661+ wake: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{}),
662+ }
663+}
664+
665+// enqueue stages a snapshot for durable persistence. It coalesces to the
666+// newest revision, starts the writer on first use, and never blocks on the
667+// store or copies session payloads. Enqueue after Close is refused: nothing is
668+// written after shutdown.
669+func (w *snapshotWriter) enqueue(snapshot ports.BrokerSnapshot) {
670+ w.mu.Lock()
671+ if w.closed {
672+ w.mu.Unlock()
673+ w.log.Debug("broker: durable snapshot write refused after shutdown", "revision", snapshot.Revision)
674+ return
675+ }
676+ if w.pending != nil {
677+ if snapshot.Revision <= w.pending.Revision {
678+ w.mu.Unlock()
679+ return
680+ }
681+ } else if snapshot.Revision <= w.written {
682+ w.mu.Unlock()
683+ return
684+ }
685+ // The publication is immutable after publication, so staging it is a value
686+ // copy; the defensive durable copy is rendered on the writer goroutine.
687+ w.pending = &snapshot
688+ if !w.started {
689+ w.started = true
690+ go w.run()
691+ }
692+ w.mu.Unlock()
693+ select {
694+ case w.wake <- struct{}{}:
695+ default:
696+ }
697+}
698+
699+func (w *snapshotWriter) run() {
700+ defer close(w.done)
701+ for {
702+ select {
703+ case <-w.stop:
704+ w.flush()
705+ return
706+ case <-w.wake:
707+ w.flush()
708+ }
709+ }
710+}
711+
712+// flush writes the newest staged snapshot, skipping revisions the store
713+// already holds.
714+func (w *snapshotWriter) flush() {
715+ for {
716+ w.mu.Lock()
717+ snapshot := w.pending
718+ written := w.written
719+ w.pending = nil
720+ w.mu.Unlock()
721+ if snapshot == nil {
722+ return
723+ }
724+ if snapshot.Revision <= written {
725+ continue
726+ }
727+ if err := w.store.Store(durable(*snapshot)); err != nil {
728+ w.log.Error("broker: durable snapshot write failed", "revision", snapshot.Revision, "err", err)
729+ continue
730+ }
731+ w.mu.Lock()
732+ if snapshot.Revision > w.written {
733+ w.written = snapshot.Revision
734+ }
735+ w.mu.Unlock()
736+ }
737+}
738+
739+// close flushes the newest staged snapshot, stops the writer, and waits for
740+// the drain to finish. It never returns before the in-flight write completes.
741+func (w *snapshotWriter) close() {
742+ w.mu.Lock()
743+ if w.closed {
744+ w.mu.Unlock()
745+ return
746+ }
747+ w.closed = true
748+ started := w.started
749+ w.mu.Unlock()
750+ if !started {
751+ return
752+ }
753+ close(w.stop)
754+ <-w.done
755+}
756+
757+type subscription struct {
758+ changed chan struct{}
759+ once sync.Once
760+ close func()
761+}
762+
763+func (s *subscription) Changed() <-chan struct{} { return s.changed }
764+func (s *subscription) Close() { s.once.Do(s.close) }
765+func (s *subscription) offer() {
766+ select {
767+ case s.changed <- struct{}{}:
768+ default:
769+ }
770+}
@@ -0,0 +1,770 @@
1+package broker
2+
3+import (
4+ "context"
5+ "encoding/binary"
6+ "errors"
7+ "fmt"
8+ "hash/fnv"
9+ "log/slog"
10+ "sort"
11+ "sync"
12+ "sync/atomic"
13+ "time"
14+
15+ "github.com/bnema/vev/internal/domain"
16+ "github.com/bnema/vev/internal/ports"
17+)
18+
19+// Observation cadence. The broker deliberately mirrors the legacy remotes
20+// monitor cadence (internal/usecase/remotes) so a fleet observed through
21+// either path refreshes on the same schedule. The duplication is temporary:
22+// the registry is intended to replace that monitor, and these constants move
23+// to a single owner once the old monitor is removed.
24+const (
25+ defaultFreshFor = 15 * time.Second
26+ defaultRetryBase = 5 * time.Second
27+ defaultRetryLimit = time.Minute
28+)
29+
30+// Run lifecycle states. A registry runs exactly once: runIdle is the zero
31+// value, runActive marks the single admitted run, and runStopped permanently
32+// refuses further runs.
33+const (
34+ runIdle int32 = iota
35+ runActive
36+ runStopped
37+)
38+
39+// Registry owns configured hosts and their immutable observation projection.
40+type Registry struct {
41+ epoch ports.BrokerEpoch
42+ probe ports.BrokerHostProbe
43+ clock ports.Clock
44+ log *slog.Logger
45+ store *snapshotWriter
46+
47+ current atomic.Pointer[ports.BrokerSnapshot]
48+ mu sync.Mutex
49+ revision ports.BrokerRevision
50+ // attempts is the registry-wide monotonic attempt token source. Every
51+ // started observation claims one token, and only the token currently
52+ // recorded as in flight may complete. Tokens are never reused, so a stale
53+ // completion is fenced even when the endpoint is removed and re-added with
54+ // the identical registration.
55+ attempts uint64
56+ hosts map[string]ports.RemoteHostSnapshot
57+ // inflight records the one admitted attempt per endpoint. The record
58+ // carries the attempt's derived context cancel so a retired attempt is
59+ // released the moment its registration is replaced, removed, or the run
60+ // settles; a ctx-honoring probe therefore stops promptly and churn can
61+ // never accumulate overlapping probes for one endpoint.
62+ inflight map[string]*probeAttempt
63+ pending map[string]bool
64+ tombstones map[string]ports.BrokerHostTombstone
65+ subs map[*subscription]struct{}
66+ wake chan struct{}
67+ running atomic.Int32
68+
69+ freshFor time.Duration
70+ retryBase time.Duration
71+ retryMax time.Duration
72+ jitter func(base time.Duration, endpoint string, attempt uint64) time.Duration
73+}
74+
75+// NewRegistry restores a validated durable snapshot under a fresh broker epoch.
76+func NewRegistry(epoch ports.BrokerEpoch, store ports.BrokerSnapshotStore, probe ports.BrokerHostProbe, clock ports.Clock, log *slog.Logger) (*Registry, error) {
77+ if epoch == 0 || probe == nil || clock == nil {
78+ return nil, errors.New("broker: invalid registry dependencies")
79+ }
80+ if log == nil {
81+ log = slog.Default()
82+ }
83+ r := &Registry{
84+ epoch: epoch, probe: probe, clock: clock, log: log,
85+ hosts: make(map[string]ports.RemoteHostSnapshot), inflight: make(map[string]*probeAttempt),
86+ pending: make(map[string]bool), tombstones: make(map[string]ports.BrokerHostTombstone),
87+ subs: make(map[*subscription]struct{}), wake: make(chan struct{}, 1),
88+ freshFor: defaultFreshFor, retryBase: defaultRetryBase, retryMax: defaultRetryLimit,
89+ jitter: jittered,
90+ }
91+ if store != nil {
92+ r.store = newSnapshotWriter(store, log)
93+ snapshot, err := store.Load()
94+ if err != nil {
95+ return nil, err
96+ }
97+ if err := r.restore(snapshot); err != nil {
98+ return nil, err
99+ }
100+ }
101+ r.publishLocked(false)
102+ return r, nil
103+}
104+
105+// restore adopts a validated durable snapshot. A zero-epoch snapshot is only
106+// the empty placeholder a store returns before its first write: any record
107+// without an epoch is rejected instead of silently dropped. A persisted
108+// snapshot from the registry's own epoch is rejected as well: every broker
109+// process samples a fresh epoch and restarts its revision series at one, so
110+// adopting same-epoch revisions would make the restored series non-monotonic
111+// (and a stale same-epoch record indistinguishable from current authority).
112+// In-flight observation state and retirement tombstones are per-process
113+// fencing state and never survive a restart.
114+func (r *Registry) restore(snapshot ports.BrokerSnapshot) error {
115+ if snapshot.Epoch == 0 {
116+ if len(snapshot.Hosts) > 0 || len(snapshot.Removed) > 0 || snapshot.Revision != 0 {
117+ return errors.New("broker: persisted snapshot has no epoch")
118+ }
119+ return nil
120+ }
121+ if snapshot.Epoch == r.epoch {
122+ return errors.New("broker: persisted snapshot epoch matches the new registry epoch")
123+ }
124+ if err := snapshot.Validate(); err != nil {
125+ return err
126+ }
127+ for _, host := range snapshot.Hosts {
128+ host = host.Clone()
129+ host.Checking = false
130+ r.hosts[host.Endpoint] = host
131+ }
132+ return nil
133+}
134+
135+// Snapshot returns a defensive immutable copy without I/O.
136+func (r *Registry) Snapshot() ports.BrokerSnapshot { return r.current.Load().Clone() }
137+
138+// Subscribe returns a capacity-one notification stream. Slow consumers never
139+// block publication and always re-read the newest snapshot.
140+func (r *Registry) Subscribe() ports.BrokerSubscription {
141+ s := &subscription{changed: make(chan struct{}, 1)}
142+ s.close = func() {
143+ r.mu.Lock()
144+ delete(r.subs, s)
145+ r.mu.Unlock()
146+ }
147+ r.mu.Lock()
148+ r.subs[s] = struct{}{}
149+ r.mu.Unlock()
150+ s.offer()
151+ return s
152+}
153+
154+// SetHosts atomically replaces configured membership. An unchanged membership
155+// is a no-op: it neither republishes nor restarts an in-flight observation.
156+// Removed registrations retire bounded tombstones so a stale completion can
157+// never revive them; replaced or re-added endpoints are fenced by the
158+// registry attempt token instead.
159+func (r *Registry) SetHosts(registrations []domain.RemoteRegistration) error {
160+ if len(registrations) > ports.BrokerMaxHosts {
161+ return fmt.Errorf("broker: %d host registrations exceed the limit of %d", len(registrations), ports.BrokerMaxHosts)
162+ }
163+ next := make(map[string]ports.RemoteHostSnapshot, len(registrations))
164+ for _, registration := range registrations {
165+ if err := registration.Validate(); err != nil {
166+ return err
167+ }
168+ if _, duplicate := next[registration.Endpoint]; duplicate {
169+ return errors.New("broker: duplicate host registration")
170+ }
171+ next[registration.Endpoint] = ports.RemoteHostSnapshot{Endpoint: registration.Endpoint, Registration: registration, Availability: domain.RemoteAvailabilityUnknown}
172+ }
173+
174+ r.mu.Lock()
175+ if registrationsUnchanged(r.hosts, next) {
176+ r.mu.Unlock()
177+ return nil
178+ }
179+ for endpoint, current := range r.hosts {
180+ candidate, present := next[endpoint]
181+ if present && current.Registration.Equal(candidate.Registration) {
182+ // Unchanged registration: keep the live observation as is.
183+ next[endpoint] = current
184+ continue
185+ }
186+ // A retired registration owns no observation: cancelling its attempt
187+ // releases a ctx-honoring probe immediately, and dropping its record
188+ // means the retired completion can never clear or answer the
189+ // replacement, and the replacement is never blocked behind it. The
190+ // cancellation happens under the registry lock, so no replacement
191+ // attempt can start before the retired context is cancelled.
192+ r.cancelInflightLocked(endpoint)
193+ delete(r.pending, endpoint)
194+ if present {
195+ // The endpoint stays live under a new registration, so its
196+ // retirement is never published as a tombstone alongside a live
197+ // host; the attempt token fences the stale completion.
198+ delete(r.tombstones, endpoint)
199+ continue
200+ }
201+ r.retireLocked(current.Registration)
202+ }
203+ for endpoint := range next {
204+ // Re-adding an endpoint supersedes its tombstone: a live host and a
205+ // tombstone for the same endpoint are mutually exclusive.
206+ delete(r.tombstones, endpoint)
207+ }
208+ r.hosts = next
209+ r.publishLocked(true)
210+ r.mu.Unlock()
211+ r.hint()
212+ return nil
213+}
214+
215+// cancelInflightLocked cancels the in-flight attempt for one endpoint and
216+// drops its record. Callers must hold r.mu: the retired attempt's context is
217+// cancelled before the record disappears, so no replacement attempt can start
218+// while the retired probe's context is still live.
219+func (r *Registry) cancelInflightLocked(endpoint string) {
220+ if attempt := r.inflight[endpoint]; attempt != nil {
221+ attempt.cancel()
222+ delete(r.inflight, endpoint)
223+ }
224+}
225+
226+// cancelAllInflightLocked cancels every in-flight attempt and drops the
227+// records. Callers must hold r.mu.
228+func (r *Registry) cancelAllInflightLocked() {
229+ for endpoint, attempt := range r.inflight {
230+ attempt.cancel()
231+ delete(r.inflight, endpoint)
232+ }
233+}
234+
235+// RequestProbe is non-blocking and coalesces concurrent demand per endpoint.
236+func (r *Registry) RequestProbe(endpoint string) {
237+ r.mu.Lock()
238+ if _, ok := r.hosts[endpoint]; ok {
239+ r.pending[endpoint] = true
240+ }
241+ r.mu.Unlock()
242+ r.hint()
243+}
244+
245+func (r *Registry) hint() {
246+ select {
247+ case r.wake <- struct{}{}:
248+ default:
249+ }
250+}
251+
252+// Run schedules observations until cancellation. Run owns the registry
253+// lifecycle and may execute exactly once: a second call is refused so a
254+// concurrent or restarted caller can never duplicate scheduling. On return it
255+// clears in-flight observation state, publishes the settled projection, and
256+// flushes durable persistence before any further write is refused.
257+func (r *Registry) Run(ctx context.Context) {
258+ if !r.running.CompareAndSwap(runIdle, runActive) {
259+ r.log.Warn("broker: registry Run refused; single-run already started or finished")
260+ return
261+ }
262+ defer r.settle()
263+ timer := r.clock.NewTimer(0)
264+ defer timer.Stop()
265+ results := make(chan probeResult, ports.BrokerMaxHosts)
266+ for {
267+ now := r.clock.Now()
268+ r.dispatch(ctx, now, results)
269+ timer.Reset(r.nextDelay(now))
270+ select {
271+ case <-ctx.Done():
272+ return
273+ case <-r.wake:
274+ case <-timer.C():
275+ case result := <-results:
276+ if ctx.Err() != nil {
277+ // Shutdown wins: a cancellation is never recorded as an
278+ // observation failure.
279+ return
280+ }
281+ r.apply(result)
282+ }
283+ }
284+}
285+
286+// settle is Run's cleanup: nothing stays in flight, the published projection
287+// carries no transient Checking flag, durable persistence is flushed, and
288+// every later Run call is refused.
289+func (r *Registry) settle() {
290+ r.mu.Lock()
291+ r.cancelAllInflightLocked()
292+ cleared := false
293+ for endpoint, host := range r.hosts {
294+ if !host.Checking {
295+ continue
296+ }
297+ host.Checking = false
298+ r.hosts[endpoint] = host
299+ cleared = true
300+ }
301+ if cleared {
302+ r.publishLocked(true)
303+ }
304+ r.mu.Unlock()
305+ if r.store != nil {
306+ r.store.close()
307+ }
308+ r.running.Store(runStopped)
309+}
310+
311+// probeAttempt is one admitted observation: the registry-wide token that
312+// fences its completion, and the cancel for the context derived from the run
313+// context. Cancelling the derived context retires the attempt: a probe that
314+// honors cancellation returns promptly, so a replaced or removed endpoint
315+// never accumulates overlapping probes, and settle releases every attempt
316+// before the run loop exits.
317+type probeAttempt struct {
318+ token uint64
319+ cancel context.CancelFunc
320+}
321+
322+// probeResult is one completed observation bound to the exact attempt token
323+// that started it.
324+type probeResult struct {
325+ endpoint string
326+ registration domain.RemoteRegistration
327+ attempt uint64
328+ snapshot ports.RemoteHostSnapshot
329+ err error
330+ at time.Time
331+}
332+
333+func (r *Registry) dispatch(ctx context.Context, now time.Time, results chan<- probeResult) {
334+ r.mu.Lock()
335+ started := false
336+ for endpoint, host := range r.hosts {
337+ if _, inflight := r.inflight[endpoint]; inflight {
338+ continue
339+ }
340+ if !r.pending[endpoint] && !host.NextDue.IsZero() && now.Before(host.NextDue) {
341+ continue
342+ }
343+ delete(r.pending, endpoint)
344+ r.attempts++
345+ attempt := r.attempts
346+ probeCtx, cancel := context.WithCancel(ctx)
347+ r.inflight[endpoint] = &probeAttempt{token: attempt, cancel: cancel}
348+ started = true
349+ host.Checking = true
350+ host.LastAttempt = now
351+ r.hosts[endpoint] = host
352+ registration := host.Registration
353+ go func(endpoint string, registration domain.RemoteRegistration, attempt uint64, probeCtx context.Context) {
354+ snapshot, err := r.probe.Probe(probeCtx, registration)
355+ result := probeResult{endpoint: endpoint, registration: registration, attempt: attempt, snapshot: snapshot, err: err, at: r.clock.Now()}
356+ select {
357+ case results <- result:
358+ case <-probeCtx.Done():
359+ // The attempt was retired (replaced, removed, or shutdown), so
360+ // its completion is already fenced: dropping it keeps churn from
361+ // queueing results the run loop would only discard.
362+ }
363+ }(endpoint, registration, attempt, probeCtx)
364+ }
365+ // Only an observation that actually started changes the projection; an
366+ // idle dispatch must not bump the revision or wake subscribers. The
367+ // in-flight projection is never persisted: Checking is transient.
368+ if started {
369+ r.publishLocked(false)
370+ }
371+ r.mu.Unlock()
372+}
373+
374+func (r *Registry) apply(result probeResult) {
375+ r.mu.Lock()
376+ defer r.mu.Unlock()
377+ current, ok := r.hosts[result.endpoint]
378+ attempt := r.inflight[result.endpoint]
379+ if !ok || !current.Registration.Equal(result.registration) || attempt == nil || attempt.token != result.attempt {
380+ // A stale completion touches neither current authority nor the
381+ // in-flight slot, which may already belong to a newer registration or
382+ // attempt.
383+ return
384+ }
385+ delete(r.inflight, result.endpoint)
386+ // The probe has returned, so releasing its derived context here only
387+ // frees the attempt's resources; the record is gone before any new
388+ // attempt can start.
389+ attempt.cancel()
390+ current.Checking = false
391+ current.LastAttempt = result.at
392+
393+ kind, availability, failed := observationOutcome(result)
394+ if !failed {
395+ // Reachable: adopt the observed inventory, but registry-owned
396+ // accounting (success time, attempt counters, failure episode) stays
397+ // local.
398+ observed := result.snapshot.Clone()
399+ observed.Endpoint = result.endpoint
400+ observed.Registration = result.registration
401+ observed.Checking = false
402+ observed.LastAttempt = result.at
403+ observed.LastSuccess = result.at
404+ observed.NextDue = result.at.Add(r.jitter(r.freshFor, result.endpoint, result.attempt))
405+ observed.ConsecutiveFailures = 0
406+ observed.LastFailure = domain.RemoteFailure{}
407+ observed.FailureEpisode = current.FailureEpisode
408+ r.hosts[result.endpoint] = observed
409+ r.publishLocked(true)
410+ return
411+ }
412+ // Every non-reachable outcome is a typed failure on the capped retry
413+ // cadence: the last success, last-known inventory, and failure episode
414+ // counter stay authoritative.
415+ current.Availability = availability
416+ if current.ConsecutiveFailures == 0 {
417+ current.FailureEpisode++
418+ }
419+ current.ConsecutiveFailures++
420+ current.LastFailure = domain.RemoteFailure{Kind: kind, Err: result.err}
421+ current.NextDue = result.at.Add(r.jitter(r.retryDelay(current.ConsecutiveFailures), result.endpoint, result.attempt))
422+ r.hosts[result.endpoint] = current
423+ r.publishLocked(true)
424+}
425+
426+// observationOutcome classifies a completed observation. It reports
427+// failed=false only for a confirmed reachable result; every other outcome
428+// carries the typed failure kind and the availability to publish.
429+func observationOutcome(result probeResult) (domain.RemoteFailureKind, domain.RemoteAvailability, bool) {
430+ if result.err != nil {
431+ kind := domain.RemoteFailureTransport
432+ if errors.Is(result.err, context.DeadlineExceeded) {
433+ kind = domain.RemoteFailureTimeout
434+ }
435+ return kind, availabilityFor(kind), true
436+ }
437+ // A projection carrying more sessions than the publication bound is a
438+ // malformed response: publishing it would produce a snapshot that fails
439+ // BrokerSnapshot.Validate. Classify the whole projection as an invalid
440+ // response so the last-known inventory stays authoritative.
441+ if len(result.snapshot.Sessions) > ports.BrokerMaxSessionsPerHost {
442+ return domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true
443+ }
444+ switch result.snapshot.Availability {
445+ case domain.RemoteAvailabilityReachable:
446+ return domain.RemoteFailureNone, domain.RemoteAvailabilityReachable, false
447+ case domain.RemoteAvailabilityIncompatible:
448+ return domain.RemoteFailureIncompatible, domain.RemoteAvailabilityIncompatible, true
449+ case domain.RemoteAvailabilityAuthFailed:
450+ return domain.RemoteFailureAuthentication, domain.RemoteAvailabilityAuthFailed, true
451+ case domain.RemoteAvailabilityInvalidResponse:
452+ return domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true
453+ default:
454+ // Unknown or Unreachable without an error still means reachability was
455+ // not confirmed, so it retries on the typed backoff cadence.
456+ return domain.RemoteFailureTransport, domain.RemoteAvailabilityUnreachable, true
457+ }
458+}
459+
460+// availabilityFor maps a typed failure kind to the published availability.
461+// The typed kind is preserved separately for notice policy.
462+func availabilityFor(kind domain.RemoteFailureKind) domain.RemoteAvailability {
463+ switch kind {
464+ case domain.RemoteFailureAuthentication:
465+ return domain.RemoteAvailabilityAuthFailed
466+ case domain.RemoteFailureIncompatible:
467+ return domain.RemoteAvailabilityIncompatible
468+ case domain.RemoteFailureInvalidResponse:
469+ return domain.RemoteAvailabilityInvalidResponse
470+ default:
471+ return domain.RemoteAvailabilityUnreachable
472+ }
473+}
474+
475+// retryDelay is the capped exponential retry cadence for a failure streak.
476+func (r *Registry) retryDelay(failures uint) time.Duration {
477+ delay := r.retryBase
478+ for i := uint(1); i < failures; i++ {
479+ if delay >= r.retryMax {
480+ return r.retryMax
481+ }
482+ delay *= 2
483+ }
484+ if delay > r.retryMax {
485+ return r.retryMax
486+ }
487+ return delay
488+}
489+
490+// jittered spreads a base interval deterministically across endpoint and
491+// attempt: identical inputs always yield the identical duration, and the
492+// result stays within 90%-110% of the base. No wall clock or global
493+// randomness participates, so retry and healthy-refresh schedules are
494+// reproducible instead of racing a shared PRNG.
495+func jittered(base time.Duration, endpoint string, attempt uint64) time.Duration {
496+ if base <= 0 {
497+ return base
498+ }
499+ h := fnv.New32a()
500+ h.Write([]byte(endpoint))
501+ var counter [8]byte
502+ binary.BigEndian.PutUint64(counter[:], attempt)
503+ h.Write(counter[:])
504+ spread := int64(h.Sum32() % 21)
505+ return base*90/100 + time.Duration(spread)*base/100
506+}
507+
508+func (r *Registry) nextDelay(now time.Time) time.Duration {
509+ r.mu.Lock()
510+ defer r.mu.Unlock()
511+ var earliest time.Time
512+ for endpoint, host := range r.hosts {
513+ // An observation already in flight is completed by its result, never
514+ // by the schedule; waiting on it here would spin the run loop.
515+ if _, inflight := r.inflight[endpoint]; inflight {
516+ continue
517+ }
518+ if r.pending[endpoint] || host.NextDue.IsZero() {
519+ return 0
520+ }
521+ if earliest.IsZero() || host.NextDue.Before(earliest) {
522+ earliest = host.NextDue
523+ }
524+ }
525+ if earliest.IsZero() {
526+ return time.Hour
527+ }
528+ if delay := earliest.Sub(now); delay > 0 {
529+ return delay
530+ }
531+ return 0
532+}
533+
534+// registrationsUnchanged reports exact membership equality: the same
535+// endpoints bound to the same registration identities. Projection fields such
536+// as availability, checking, or inventory never participate.
537+func registrationsUnchanged(current, next map[string]ports.RemoteHostSnapshot) bool {
538+ if len(current) != len(next) {
539+ return false
540+ }
541+ for endpoint, candidate := range next {
542+ existing, ok := current[endpoint]
543+ if !ok || !existing.Registration.Equal(candidate.Registration) {
544+ return false
545+ }
546+ }
547+ return true
548+}
549+
550+// retireLocked records the tombstone for one removed registration at the
551+// revision of the publication that carries it.
552+func (r *Registry) retireLocked(registration domain.RemoteRegistration) {
553+ retired := r.revision + 1
554+ if retired == 0 {
555+ retired = 1
556+ }
557+ r.tombstones[registration.Endpoint] = ports.BrokerHostTombstone{
558+ Endpoint: registration.Endpoint,
559+ Registration: registration,
560+ RetiredRevision: retired,
561+ }
562+ r.pruneTombstonesLocked()
563+}
564+
565+// pruneTombstonesLocked keeps only the newest BrokerMaxTombstones retirements
566+// so an endpoint that churns can never grow the published set without bound.
567+func (r *Registry) pruneTombstonesLocked() {
568+ for len(r.tombstones) > ports.BrokerMaxTombstones {
569+ oldest := ""
570+ var oldestRevision ports.BrokerRevision
571+ for endpoint, tombstone := range r.tombstones {
572+ if oldest == "" || tombstone.RetiredRevision < oldestRevision ||
573+ (tombstone.RetiredRevision == oldestRevision && endpoint < oldest) {
574+ oldest, oldestRevision = endpoint, tombstone.RetiredRevision
575+ }
576+ }
577+ delete(r.tombstones, oldest)
578+ }
579+}
580+
581+// tombstonesLocked renders the bounded tombstone publication, newest first.
582+// Live endpoints are never published as tombstones.
583+func (r *Registry) tombstonesLocked() []ports.BrokerHostTombstone {
584+ if len(r.tombstones) == 0 {
585+ return nil
586+ }
587+ out := make([]ports.BrokerHostTombstone, 0, len(r.tombstones))
588+ for endpoint, tombstone := range r.tombstones {
589+ if _, live := r.hosts[endpoint]; live {
590+ continue
591+ }
592+ out = append(out, tombstone)
593+ }
594+ sort.Slice(out, func(i, j int) bool {
595+ if out[i].RetiredRevision != out[j].RetiredRevision {
596+ return out[i].RetiredRevision > out[j].RetiredRevision
597+ }
598+ return out[i].Endpoint < out[j].Endpoint
599+ })
600+ if len(out) > ports.BrokerMaxTombstones {
601+ out = out[:ports.BrokerMaxTombstones]
602+ }
603+ return out
604+}
605+
606+func (r *Registry) publishLocked(persist bool) {
607+ r.revision++
608+ hosts := make([]ports.RemoteHostSnapshot, 0, len(r.hosts))
609+ for _, host := range r.hosts {
610+ hosts = append(hosts, host.Clone())
611+ }
612+ sort.Slice(hosts, func(i, j int) bool { return hosts[i].Endpoint < hosts[j].Endpoint })
613+ snapshot := ports.BrokerSnapshot{Epoch: r.epoch, Revision: r.revision, Hosts: hosts, Removed: r.tombstonesLocked()}
614+ r.current.Store(&snapshot)
615+ if persist && r.store != nil {
616+ // enqueue only stages an immutable publication: rendering the durable
617+ // copy and every store call happen on the writer goroutine, never
618+ // under the registry lock and never on the probe schedule.
619+ r.store.enqueue(snapshot)
620+ }
621+ for sub := range r.subs {
622+ sub.offer()
623+ }
624+}
625+
626+// durable returns the persistable copy of a publication, rendered by the
627+// writer goroutine. Checking is transient in-flight state and never reaches
628+// the durable snapshot.
629+func durable(snapshot ports.BrokerSnapshot) ports.BrokerSnapshot {
630+ out := snapshot.Clone()
631+ for i := range out.Hosts {
632+ out.Hosts[i].Checking = false
633+ }
634+ return out
635+}
636+
637+// snapshotWriter serializes durable snapshot writes off the registry lock.
638+// Enqueue never blocks on store I/O and always coalesces to the newest
639+// revision; Store calls run on one goroutine, so the durable store never sees
640+// an out-of-order write. A failed write is logged and retried by the next
641+// publication, so a store fault never stalls publication. Close flushes the
642+// newest pending snapshot, stops the writer, and waits, so no write outlives
643+// shutdown.
644+type snapshotWriter struct {
645+ store ports.BrokerSnapshotStore
646+ log *slog.Logger
647+
648+ mu sync.Mutex
649+ pending *ports.BrokerSnapshot
650+ written ports.BrokerRevision
651+ started bool
652+ closed bool
653+ wake chan struct{}
654+ stop chan struct{}
655+ done chan struct{}
656+}
657+
658+func newSnapshotWriter(store ports.BrokerSnapshotStore, log *slog.Logger) *snapshotWriter {
659+ return &snapshotWriter{
660+ store: store, log: log,
661+ wake: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{}),
662+ }
663+}
664+
665+// enqueue stages a snapshot for durable persistence. It coalesces to the
666+// newest revision, starts the writer on first use, and never blocks on the
667+// store or copies session payloads. Enqueue after Close is refused: nothing is
668+// written after shutdown.
669+func (w *snapshotWriter) enqueue(snapshot ports.BrokerSnapshot) {
670+ w.mu.Lock()
671+ if w.closed {
672+ w.mu.Unlock()
673+ w.log.Debug("broker: durable snapshot write refused after shutdown", "revision", snapshot.Revision)
674+ return
675+ }
676+ if w.pending != nil {
677+ if snapshot.Revision <= w.pending.Revision {
678+ w.mu.Unlock()
679+ return
680+ }
681+ } else if snapshot.Revision <= w.written {
682+ w.mu.Unlock()
683+ return
684+ }
685+ // The publication is immutable after publication, so staging it is a value
686+ // copy; the defensive durable copy is rendered on the writer goroutine.
687+ w.pending = &snapshot
688+ if !w.started {
689+ w.started = true
690+ go w.run()
691+ }
692+ w.mu.Unlock()
693+ select {
694+ case w.wake <- struct{}{}:
695+ default:
696+ }
697+}
698+
699+func (w *snapshotWriter) run() {
700+ defer close(w.done)
701+ for {
702+ select {
703+ case <-w.stop:
704+ w.flush()
705+ return
706+ case <-w.wake:
707+ w.flush()
708+ }
709+ }
710+}
711+
712+// flush writes the newest staged snapshot, skipping revisions the store
713+// already holds.
714+func (w *snapshotWriter) flush() {
715+ for {
716+ w.mu.Lock()
717+ snapshot := w.pending
718+ written := w.written
719+ w.pending = nil
720+ w.mu.Unlock()
721+ if snapshot == nil {
722+ return
723+ }
724+ if snapshot.Revision <= written {
725+ continue
726+ }
727+ if err := w.store.Store(durable(*snapshot)); err != nil {
728+ w.log.Error("broker: durable snapshot write failed", "revision", snapshot.Revision, "err", err)
729+ continue
730+ }
731+ w.mu.Lock()
732+ if snapshot.Revision > w.written {
733+ w.written = snapshot.Revision
734+ }
735+ w.mu.Unlock()
736+ }
737+}
738+
739+// close flushes the newest staged snapshot, stops the writer, and waits for
740+// the drain to finish. It never returns before the in-flight write completes.
741+func (w *snapshotWriter) close() {
742+ w.mu.Lock()
743+ if w.closed {
744+ w.mu.Unlock()
745+ return
746+ }
747+ w.closed = true
748+ started := w.started
749+ w.mu.Unlock()
750+ if !started {
751+ return
752+ }
753+ close(w.stop)
754+ <-w.done
755+}
756+
757+type subscription struct {
758+ changed chan struct{}
759+ once sync.Once
760+ close func()
761+}
762+
763+func (s *subscription) Changed() <-chan struct{} { return s.changed }
764+func (s *subscription) Close() { s.once.Do(s.close) }
765+func (s *subscription) offer() {
766+ select {
767+ case s.changed <- struct{}{}:
768+ default:
769+ }
770+}
A internal/usecase/broker/registry_test.go +1780 -0
@@ -0,0 +1,1780 @@
1+package broker
2+
3+import (
4+ "bytes"
5+ "context"
6+ "errors"
7+ "fmt"
8+ "log/slog"
9+ "sort"
10+ "strings"
11+ "sync"
12+ "testing"
13+ "time"
14+
15+ "github.com/bnema/vev/internal/domain"
16+ "github.com/bnema/vev/internal/ports"
17+ "github.com/bnema/vev/internal/protocol/catalogue"
18+ "github.com/stretchr/testify/require"
19+)
20+
21+// probeGrace bounds a negative scheduling assertion: a probe may not start
22+// inside this window, so the test observes "nothing happened" without
23+// sleeping on the behavior under test.
24+const probeGrace = 25 * time.Millisecond
25+
26+// manualClock is a deterministic ports.Clock/ports.Timer pair: time only moves
27+// when Advance is called, and timers with equal deadlines fire in creation
28+// order. Scheduling assertions are therefore exact instead of racing wall time.
29+type manualClock struct {
30+ mu sync.Mutex
31+ now time.Time
32+ order uint64
33+ timers map[*manualTimer]struct{}
34+}
35+
36+func newManualClock(start time.Time) *manualClock {
37+ return &manualClock{now: start, timers: make(map[*manualTimer]struct{})}
38+}
39+
40+func (c *manualClock) Now() time.Time {
41+ c.mu.Lock()
42+ defer c.mu.Unlock()
43+ return c.now
44+}
45+
46+func (c *manualClock) NewTimer(d time.Duration) ports.Timer {
47+ timer := &manualTimer{clock: c, ch: make(chan time.Time, 1)}
48+ c.mu.Lock()
49+ defer c.mu.Unlock()
50+ timer.arm(d)
51+ return timer
52+}
53+
54+// Advance moves time forward and fires every timer that came due.
55+func (c *manualClock) Advance(d time.Duration) {
56+ if d < 0 {
57+ panic("broker: manualClock.Advance requires a non-negative duration")
58+ }
59+ c.mu.Lock()
60+ defer c.mu.Unlock()
61+ c.now = c.now.Add(d)
62+ due := make([]*manualTimer, 0, len(c.timers))
63+ for timer := range c.timers {
64+ if !timer.when.After(c.now) {
65+ due = append(due, timer)
66+ }
67+ }
68+ sort.Slice(due, func(i, j int) bool {
69+ if due[i].when.Equal(due[j].when) {
70+ return due[i].order < due[j].order
71+ }
72+ return due[i].when.Before(due[j].when)
73+ })
74+ for _, timer := range due {
75+ timer.active = false
76+ delete(c.timers, timer)
77+ select {
78+ case timer.ch <- c.now:
79+ default:
80+ }
81+ }
82+}
83+
84+type manualTimer struct {
85+ clock *manualClock
86+ ch chan time.Time
87+ when time.Time
88+ order uint64
89+ active bool
90+}
91+
92+func (t *manualTimer) C() <-chan time.Time { return t.ch }
93+
94+func (t *manualTimer) Stop() bool {
95+ t.clock.mu.Lock()
96+ defer t.clock.mu.Unlock()
97+ if !t.active {
98+ return false
99+ }
100+ t.active = false
101+ delete(t.clock.timers, t)
102+ return true
103+}
104+
105+func (t *manualTimer) Reset(d time.Duration) bool {
106+ t.clock.mu.Lock()
107+ defer t.clock.mu.Unlock()
108+ wasActive := t.active
109+ if t.active {
110+ t.active = false
111+ delete(t.clock.timers, t)
112+ }
113+ select {
114+ case <-t.ch:
115+ default:
116+ }
117+ t.arm(d)
118+ return wasActive
119+}
120+
121+// arm schedules the timer relative to the clock's current time. The caller
122+// must hold clock.mu.
123+func (t *manualTimer) arm(d time.Duration) {
124+ t.when = t.clock.now.Add(d)
125+ t.order = t.clock.order
126+ t.clock.order++
127+ if d <= 0 {
128+ t.active = false
129+ select {
130+ case t.ch <- t.clock.now:
131+ default:
132+ }
133+ return
134+ }
135+ t.active = true
136+ t.clock.timers[t] = struct{}{}
137+}
138+
139+// identityJitter removes scheduling jitter so a test can assert exact
140+// deadlines; the production jitter is covered separately.
141+func identityJitter(d time.Duration, _ string, _ uint64) time.Duration { return d }
142+
143+// syncBuffer captures log output written from the registry's persistence
144+// goroutine as well as the test goroutine.
145+type syncBuffer struct {
146+ mu sync.Mutex
147+ b bytes.Buffer
148+}
149+
150+func (w *syncBuffer) Write(p []byte) (int, error) {
151+ w.mu.Lock()
152+ defer w.mu.Unlock()
153+ return w.b.Write(p)
154+}
155+
156+func (w *syncBuffer) String() string {
157+ w.mu.Lock()
158+ defer w.mu.Unlock()
159+ return w.b.String()
160+}
161+
162+func testLogger() (*slog.Logger, *syncBuffer) {
163+ logs := &syncBuffer{}
164+ return slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})), logs
165+}
166+
167+// testStore is a race-safe in-memory durable store whose writes signal a
168+// capacity-one channel so tests wait on a write instead of on wall time.
169+type testStore struct {
170+ mu sync.Mutex
171+ loaded ports.BrokerSnapshot
172+ loadErr error
173+ stored ports.BrokerSnapshot
174+ storeErr error
175+ writes int
176+ changes chan struct{}
177+}
178+
179+func (s *testStore) Load() (ports.BrokerSnapshot, error) {
180+ s.mu.Lock()
181+ defer s.mu.Unlock()
182+ if s.loadErr != nil {
183+ return ports.BrokerSnapshot{}, s.loadErr
184+ }
185+ return s.loaded.Clone(), nil
186+}
187+
188+func (s *testStore) Store(snapshot ports.BrokerSnapshot) error {
189+ s.mu.Lock()
190+ s.stored = snapshot.Clone()
191+ s.writes++
192+ err := s.storeErr
193+ s.mu.Unlock()
194+ s.signal()
195+ return err
196+}
197+
198+func (s *testStore) storedSnapshot() ports.BrokerSnapshot {
199+ s.mu.Lock()
200+ defer s.mu.Unlock()
201+ return s.stored.Clone()
202+}
203+
204+func (s *testStore) writeCount() int {
205+ s.mu.Lock()
206+ defer s.mu.Unlock()
207+ return s.writes
208+}
209+
210+func (s *testStore) setStoreErr(err error) {
211+ s.mu.Lock()
212+ defer s.mu.Unlock()
213+ s.storeErr = err
214+}
215+
216+func (s *testStore) signal() {
217+ s.mu.Lock()
218+ if s.changes == nil {
219+ s.changes = make(chan struct{}, 1)
220+ }
221+ changes := s.changes
222+ s.mu.Unlock()
223+ select {
224+ case changes <- struct{}{}:
225+ default:
226+ }
227+}
228+
229+func (s *testStore) changesChan() chan struct{} {
230+ s.mu.Lock()
231+ defer s.mu.Unlock()
232+ if s.changes == nil {
233+ s.changes = make(chan struct{}, 1)
234+ }
235+ return s.changes
236+}
237+
238+// blockingStore holds the first durable write inside Store so a test can prove
239+// that publication never waits behind persistence.
240+type blockingStore struct {
241+ mu sync.Mutex
242+ revisions []ports.BrokerRevision
243+ entered chan ports.BrokerRevision
244+ released chan struct{}
245+ first sync.Once
246+}
247+
248+func newBlockingStore() *blockingStore {
249+ return &blockingStore{entered: make(chan ports.BrokerRevision, 1), released: make(chan struct{})}
250+}
251+
252+func (s *blockingStore) Load() (ports.BrokerSnapshot, error) { return ports.BrokerSnapshot{}, nil }
253+
254+func (s *blockingStore) Store(snapshot ports.BrokerSnapshot) error {
255+ s.mu.Lock()
256+ s.revisions = append(s.revisions, snapshot.Revision)
257+ s.mu.Unlock()
258+ s.first.Do(func() {
259+ select {
260+ case s.entered <- snapshot.Revision:
261+ default:
262+ }
263+ <-s.released
264+ })
265+ return nil
266+}
267+
268+func (s *blockingStore) releaseFirst() { close(s.released) }
269+
270+func (s *blockingStore) writtenRevisions() []ports.BrokerRevision {
271+ s.mu.Lock()
272+ defer s.mu.Unlock()
273+ return append([]ports.BrokerRevision(nil), s.revisions...)
274+}
275+
276+// testProbe scripts one observation per received call: the test owns the
277+// completion, so a probe never depends on wall time.
278+type testProbe struct {
279+ calls chan probeCall
280+ canceled chan struct{}
281+}
282+
283+func newTestProbe(capacity int) *testProbe {
284+ return &testProbe{calls: make(chan probeCall, capacity), canceled: make(chan struct{}, 1)}
285+}
286+
287+type probeCall struct {
288+ registration domain.RemoteRegistration
289+ result chan probeAnswer
290+}
291+
292+type probeAnswer struct {
293+ snapshot ports.RemoteHostSnapshot
294+ err error
295+}
296+
297+func (p *testProbe) Probe(ctx context.Context, registration domain.RemoteRegistration) (ports.RemoteHostSnapshot, error) {
298+ call := probeCall{registration: registration, result: make(chan probeAnswer, 1)}
299+ select {
300+ case p.calls <- call:
301+ case <-ctx.Done():
302+ p.noteCanceled()
303+ return ports.RemoteHostSnapshot{}, ctx.Err()
304+ }
305+ select {
306+ case answer := <-call.result:
307+ return answer.snapshot, answer.err
308+ case <-ctx.Done():
309+ p.noteCanceled()
310+ return ports.RemoteHostSnapshot{}, ctx.Err()
311+ }
312+}
313+
314+func (p *testProbe) noteCanceled() {
315+ select {
316+ case p.canceled <- struct{}{}:
317+ default:
318+ }
319+}
320+
321+// churnProbe is a ctx-honoring probe used by the churn regression test. It
322+// tracks, per endpoint, how many probes were simultaneously live with an
323+// uncancelled context: an attempt that is retired without cancelling its
324+// context stays live and shows up as an overlap with its replacement.
325+type churnProbe struct {
326+ release chan struct{}
327+ released sync.Once
328+ startedSig chan struct{}
329+
330+ mu sync.Mutex
331+ contexts map[context.Context]string
332+ maxLive map[string]int
333+ started int
334+}
335+
336+func newChurnProbe() *churnProbe {
337+ return &churnProbe{
338+ release: make(chan struct{}),
339+ startedSig: make(chan struct{}, 256),
340+ contexts: make(map[context.Context]string),
341+ maxLive: make(map[string]int),
342+ }
343+}
344+
345+func (p *churnProbe) Probe(ctx context.Context, registration domain.RemoteRegistration) (ports.RemoteHostSnapshot, error) {
346+ p.mu.Lock()
347+ live := 0
348+ for other, endpoint := range p.contexts {
349+ if endpoint == registration.Endpoint && other.Err() == nil {
350+ live++
351+ }
352+ }
353+ // The attempt is registered even when its context is already cancelled (a
354+ // probe launched into a settling run), but only an uncancelled context is
355+ // counted live: an attempt retired without cancelling its context stays in
356+ // the map with Err()==nil and still overlaps its replacement.
357+ p.contexts[ctx] = registration.Endpoint
358+ if ctx.Err() == nil {
359+ live++
360+ }
361+ if live > p.maxLive[registration.Endpoint] {
362+ p.maxLive[registration.Endpoint] = live
363+ }
364+ p.started++
365+ p.mu.Unlock()
366+ select {
367+ case p.startedSig <- struct{}{}:
368+ default:
369+ }
370+ defer func() {
371+ p.mu.Lock()
372+ delete(p.contexts, ctx)
373+ p.mu.Unlock()
374+ }()
375+ select {
376+ case <-ctx.Done():
377+ case <-p.release:
378+ }
379+ return ports.RemoteHostSnapshot{}, ctx.Err()
380+}
381+
382+// openRelease lets every still-blocked probe return, so a failing test never
383+// leaks probe goroutines.
384+func (p *churnProbe) openRelease() { p.released.Do(func() { close(p.release) }) }
385+
386+func (p *churnProbe) startedCount() int {
387+ p.mu.Lock()
388+ defer p.mu.Unlock()
389+ return p.started
390+}
391+
392+func (p *churnProbe) maxLiveFor(endpoint string) int {
393+ p.mu.Lock()
394+ defer p.mu.Unlock()
395+ return p.maxLive[endpoint]
396+}
397+
398+func (p *churnProbe) observedEndpoints() []string {
399+ p.mu.Lock()
400+ defer p.mu.Unlock()
401+ endpoints := make([]string, 0, len(p.maxLive))
402+ for endpoint := range p.maxLive {
403+ endpoints = append(endpoints, endpoint)
404+ }
405+ sort.Strings(endpoints)
406+ return endpoints
407+}
408+
409+func (p *churnProbe) inflight() int {
410+ p.mu.Lock()
411+ defer p.mu.Unlock()
412+ return len(p.contexts)
413+}
414+
415+func registration(t *testing.T, endpoint string, marker byte) domain.RemoteRegistration {
416+ t.Helper()
417+ var incarnation [16]byte
418+ incarnation[0] = marker
419+ result, err := domain.NewRemoteRegistration(endpoint, incarnation)
420+ require.NoError(t, err)
421+ return result
422+}
423+
424+// hostSessions builds count minimal catalogue sessions for the projection
425+// session bound tests.
426+func hostSessions(count int) []catalogue.RemoteCatalogSession {
427+ sessions := make([]catalogue.RemoteCatalogSession, 0, count)
428+ for index := 0; index < count; index++ {
429+ sessions = append(sessions, catalogue.RemoteCatalogSession{
430+ Name: fmt.Sprintf("session-%d", index),
431+ State: catalogue.RemoteCatalogSessionUp,
432+ })
433+ }
434+ return sessions
435+}
436+
437+// newTestRegistry builds a registry with exact scheduling and no logger so a
438+// failure never depends on jitter.
439+func newTestRegistry(t *testing.T, epoch ports.BrokerEpoch, store ports.BrokerSnapshotStore, probe ports.BrokerHostProbe, clock *manualClock) *Registry {
440+ t.Helper()
441+ registry, err := NewRegistry(epoch, store, probe, clock, nil)
442+ require.NoError(t, err)
443+ registry.jitter = identityJitter
444+ return registry
445+}
446+
447+func startRegistry(t *testing.T, r *Registry) context.CancelFunc {
448+ t.Helper()
449+ ctx, cancel := context.WithCancel(context.Background())
450+ done := make(chan struct{})
451+ go func() { defer close(done); r.Run(ctx) }()
452+ t.Cleanup(func() { cancel(); <-done })
453+ return cancel
454+}
455+
456+func receiveCall(t *testing.T, probe *testProbe) probeCall {
457+ t.Helper()
458+ select {
459+ case call := <-probe.calls:
460+ return call
461+ case <-time.After(time.Second):
462+ t.Fatal("timed out waiting for probe")
463+ return probeCall{}
464+ }
465+}
466+
467+func requireNoProbeWithin(t *testing.T, probe *testProbe, window time.Duration) {
468+ t.Helper()
469+ timer := time.NewTimer(window)
470+ defer timer.Stop()
471+ select {
472+ case call := <-probe.calls:
473+ t.Fatalf("unexpected probe for %q", call.registration.Endpoint)
474+ case <-timer.C:
475+ }
476+}
477+
478+func waitSnapshot(t *testing.T, r *Registry, predicate func(ports.BrokerSnapshot) bool) ports.BrokerSnapshot {
479+ t.Helper()
480+ sub := r.Subscribe()
481+ defer sub.Close()
482+ deadline := time.After(time.Second)
483+ for {
484+ snapshot := r.Snapshot()
485+ if predicate(snapshot) {
486+ return snapshot
487+ }
488+ select {
489+ case <-sub.Changed():
490+ case <-deadline:
491+ t.Fatal("timed out waiting for snapshot")
492+ }
493+ }
494+}
495+
496+func waitHost(t *testing.T, r *Registry, endpoint string, predicate func(ports.RemoteHostSnapshot) bool) ports.RemoteHostSnapshot {
497+ t.Helper()
498+ snapshot := waitSnapshot(t, r, func(snapshot ports.BrokerSnapshot) bool {
499+ host, ok := snapshot.Find(endpoint)
500+ return ok && predicate(host)
501+ })
502+ host, _ := snapshot.Find(endpoint)
503+ return host
504+}
505+
506+// requireHostStableWithin asserts a stale completion has no delayed effect:
507+// the published host stays exactly as observed for a bounded window, and any
508+// intervening publication fails immediately.
509+func requireHostStableWithin(t *testing.T, r *Registry, endpoint string, want ports.RemoteHostSnapshot, window time.Duration) {
510+ t.Helper()
511+ sub := r.Subscribe()
512+ defer sub.Close()
513+ deadline := time.After(window)
514+ for {
515+ got, ok := r.Snapshot().Find(endpoint)
516+ require.True(t, ok)
517+ require.Equal(t, want, got)
518+ select {
519+ case <-sub.Changed():
520+ case <-deadline:
521+ return
522+ }
523+ }
524+}
525+
526+// waitStored waits for a durable write satisfying predicate.
527+func waitStored(t *testing.T, store *testStore, predicate func(ports.BrokerSnapshot) bool) ports.BrokerSnapshot {
528+ t.Helper()
529+ deadline := time.After(2 * time.Second)
530+ for {
531+ if snapshot := store.storedSnapshot(); predicate(snapshot) {
532+ return snapshot
533+ }
534+ select {
535+ case <-store.changesChan():
536+ case <-deadline:
537+ t.Fatal("timed out waiting for durable snapshot")
538+ return ports.BrokerSnapshot{}
539+ }
540+ }
541+}
542+
543+// waitWriterRevision waits until the serializer reports the revision durable.
544+func waitWriterRevision(t *testing.T, r *Registry, revision ports.BrokerRevision) {
545+ t.Helper()
546+ deadline := time.Now().Add(2 * time.Second)
547+ for {
548+ r.store.mu.Lock()
549+ written := r.store.written
550+ r.store.mu.Unlock()
551+ if written >= revision {
552+ return
553+ }
554+ if time.Now().After(deadline) {
555+ t.Fatalf("timed out waiting for durable revision %d", revision)
556+ }
557+ time.Sleep(time.Millisecond)
558+ }
559+}
560+
561+// waitForLog waits until captured log output contains want, so a persistence
562+// fault is proven observable instead of silently dropped.
563+func waitForLog(t *testing.T, logs *syncBuffer, want string) {
564+ t.Helper()
565+ deadline := time.Now().Add(2 * time.Second)
566+ for !strings.Contains(logs.String(), want) {
567+ if time.Now().After(deadline) {
568+ t.Fatalf("timed out waiting for log %q in %q", want, logs.String())
569+ }
570+ time.Sleep(time.Millisecond)
571+ }
572+}
573+
574+// attemptFor returns the attempt token currently in flight for an endpoint,
575+// or zero when no attempt is admitted.
576+func attemptFor(r *Registry, endpoint string) uint64 {
577+ r.mu.Lock()
578+ defer r.mu.Unlock()
579+ if attempt := r.inflight[endpoint]; attempt != nil {
580+ return attempt.token
581+ }
582+ return 0
583+}
584+
585+func TestRegistryRestoresPersistedSnapshotUnderFreshEpoch(t *testing.T) {
586+ reg := registration(t, "example.test", 1)
587+ sessions := []catalogue.RemoteCatalogSession{{
588+ Name: "work",
589+ State: catalogue.RemoteCatalogSessionUp,
590+ Tabs: []catalogue.RemoteCatalogTab{{ID: "tab-1", Name: "shell"}},
591+ }}
592+ stored := ports.BrokerSnapshot{
593+ Epoch: 7,
594+ Revision: 9,
595+ Hosts: []ports.RemoteHostSnapshot{{
596+ Endpoint: reg.Endpoint,
597+ Registration: reg,
598+ Availability: domain.RemoteAvailabilityReachable,
599+ Checking: true,
600+ LastSuccess: time.Unix(50, 0),
601+ NextDue: time.Unix(80, 0),
602+ InventoryKnown: true,
603+ Sessions: sessions,
604+ }},
605+ }
606+ store := &testStore{loaded: stored}
607+ registry, err := NewRegistry(11, store, newTestProbe(1), newManualClock(time.Unix(100, 0)), nil)
608+ require.NoError(t, err)
609+
610+ snapshot := registry.Snapshot()
611+ require.Equal(t, ports.BrokerEpoch(11), snapshot.Epoch)
612+ // A restored broker process starts a fresh revision series: revisions
613+ // never compare across epochs.
614+ require.Equal(t, ports.BrokerRevision(1), snapshot.Revision)
615+ require.Len(t, snapshot.Hosts, 1)
616+ require.Empty(t, snapshot.Removed)
617+
618+ host := snapshot.Hosts[0]
619+ require.True(t, host.Registration.Equal(reg))
620+ require.Equal(t, domain.RemoteAvailabilityReachable, host.Availability)
621+ require.Equal(t, time.Unix(80, 0), host.NextDue)
622+ require.Equal(t, sessions, host.Sessions)
623+ // In-flight observation state never survives a restart.
624+ require.False(t, host.Checking)
625+
626+ // Restoring is read-only: the registry republishes under its own epoch
627+ // without rewriting the durable store.
628+ require.Equal(t, ports.BrokerRevision(0), store.storedSnapshot().Revision)
629+ require.Zero(t, store.writeCount())
630+}
631+
632+func TestRegistryRestoreRejectsInvalidInputs(t *testing.T) {
633+ clock := newManualClock(time.Unix(100, 0))
634+ reg := registration(t, "example.test", 1)
635+
636+ _, err := NewRegistry(0, nil, newTestProbe(1), clock, nil)
637+ require.Error(t, err)
638+ _, err = NewRegistry(1, nil, nil, clock, nil)
639+ require.Error(t, err)
640+ _, err = NewRegistry(1, nil, newTestProbe(1), nil, nil)
641+ require.Error(t, err)
642+
643+ loadErr := errors.New("load failed")
644+ _, err = NewRegistry(1, &testStore{loadErr: loadErr}, newTestProbe(1), clock, nil)
645+ require.ErrorIs(t, err, loadErr)
646+
647+ _, err = NewRegistry(1, &testStore{loaded: ports.BrokerSnapshot{Epoch: 7}}, newTestProbe(1), clock, nil)
648+ require.Error(t, err)
649+
650+ // A durable record without an epoch is inconsistent, never silently empty.
651+ _, err = NewRegistry(1, &testStore{loaded: ports.BrokerSnapshot{Epoch: 0, Hosts: []ports.RemoteHostSnapshot{{Endpoint: reg.Endpoint, Registration: reg}}}}, newTestProbe(1), clock, nil)
652+ require.Error(t, err)
653+ _, err = NewRegistry(1, &testStore{loaded: ports.BrokerSnapshot{Epoch: 0, Removed: []ports.BrokerHostTombstone{{Endpoint: reg.Endpoint, Registration: reg, RetiredRevision: 3}}}}, newTestProbe(1), clock, nil)
654+ require.Error(t, err)
655+ _, err = NewRegistry(1, &testStore{loaded: ports.BrokerSnapshot{Epoch: 0, Revision: 4}}, newTestProbe(1), clock, nil)
656+ require.Error(t, err)
657+
658+ empty, err := NewRegistry(1, &testStore{}, newTestProbe(1), clock, nil)
659+ require.NoError(t, err)
660+ require.Equal(t, ports.BrokerRevision(1), empty.Snapshot().Revision)
661+ require.Empty(t, empty.Snapshot().Hosts)
662+}
663+
664+func TestRegistryRestoreRejectsPersistedSnapshotFromItsOwnEpoch(t *testing.T) {
665+ clock := newManualClock(time.Unix(100, 0))
666+ reg := registration(t, "example.test", 1)
667+
668+ // A fresh broker process samples a fresh epoch and restarts its revision
669+ // series at one, so a durable record from its own epoch can never be
670+ // adopted: the restored series would not be monotonic and a stale record
671+ // would be indistinguishable from current authority.
672+ currentEpoch := ports.BrokerSnapshot{
673+ Epoch: 7,
674+ Revision: 9,
675+ Hosts: []ports.RemoteHostSnapshot{{
676+ Endpoint: reg.Endpoint,
677+ Registration: reg,
678+ Availability: domain.RemoteAvailabilityReachable,
679+ }},
680+ }
681+ _, err := NewRegistry(7, &testStore{loaded: currentEpoch}, newTestProbe(1), clock, nil)
682+ require.Error(t, err)
683+ require.Contains(t, err.Error(), "matches the new registry epoch")
684+
685+ // The same rejection holds for a tombstone-only and for an empty-hosts
686+ // record from the current epoch.
687+ _, err = NewRegistry(7, &testStore{loaded: ports.BrokerSnapshot{Epoch: 7, Revision: 1, Removed: []ports.BrokerHostTombstone{{
688+ Endpoint: reg.Endpoint, Registration: reg, RetiredRevision: 1,
689+ }}}}, newTestProbe(1), clock, nil)
690+ require.Error(t, err)
691+ _, err = NewRegistry(7, &testStore{loaded: ports.BrokerSnapshot{Epoch: 7, Revision: 1}}, newTestProbe(1), clock, nil)
692+ require.Error(t, err)
693+
694+ // A different epoch is still adopted normally.
695+ registry, err := NewRegistry(8, &testStore{loaded: currentEpoch}, newTestProbe(1), clock, nil)
696+ require.NoError(t, err)
697+ require.Len(t, registry.Snapshot().Hosts, 1)
698+}
699+
700+func TestRegistrySetHostsRejectsInvalidRegistrations(t *testing.T) {
701+ registry := newTestRegistry(t, 1, nil, newTestProbe(1), newManualClock(time.Unix(100, 0)))
702+
703+ reg := registration(t, "example.test", 1)
704+ require.Error(t, registry.SetHosts([]domain.RemoteRegistration{reg, reg}))
705+ require.Error(t, registry.SetHosts([]domain.RemoteRegistration{{Endpoint: reg.Endpoint}}))
706+ require.Empty(t, registry.Snapshot().Hosts)
707+}
708+
709+func TestRegistrySetHostsRejectsHostLimitWithoutMutation(t *testing.T) {
710+ registry := newTestRegistry(t, 1, nil, newTestProbe(1), newManualClock(time.Unix(100, 0)))
711+ reg := registration(t, "example.test", 1)
712+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
713+
714+ oversized := make([]domain.RemoteRegistration, 0, ports.BrokerMaxHosts+1)
715+ for index := 0; index <= ports.BrokerMaxHosts; index++ {
716+ oversized = append(oversized, registration(t, fmt.Sprintf("host-%d.test", index), byte(index+1)))
717+ }
718+ before := registry.Snapshot()
719+ require.Error(t, registry.SetHosts(oversized))
720+ // The rejected call mutates nothing: same membership, same revision.
721+ require.Equal(t, before, registry.Snapshot())
722+
723+ // The boundary itself is accepted.
724+ require.NoError(t, registry.SetHosts(oversized[:ports.BrokerMaxHosts]))
725+ require.Len(t, registry.Snapshot().Hosts, ports.BrokerMaxHosts)
726+}
727+
728+func TestRegistryUnchangedSetHostsIsNoOp(t *testing.T) {
729+ clock := newManualClock(time.Unix(100, 0))
730+ probe := newTestProbe(2)
731+ registry := newTestRegistry(t, 1, nil, probe, clock)
732+ first := registration(t, "example.test", 1)
733+ second := registration(t, "other.test", 2)
734+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{first, second}))
735+ startRegistry(t, registry)
736+
737+ // Every configured host is observed exactly once.
738+ calls := make(map[string]probeCall, 2)
739+ for range 2 {
740+ call := receiveCall(t, probe)
741+ calls[call.registration.Endpoint] = call
742+ }
743+ require.Len(t, calls, 2)
744+ before := registry.Snapshot()
745+ for _, host := range before.Hosts {
746+ require.True(t, host.Checking)
747+ }
748+
749+ // Identical membership, including in a different order, never republishes
750+ // and never restarts an in-flight observation.
751+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{second, first}))
752+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{first, second}))
753+ require.Equal(t, before, registry.Snapshot())
754+ requireNoProbeWithin(t, probe, probeGrace)
755+
756+ // The in-flight observations still complete normally.
757+ for endpoint, call := range calls {
758+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
759+ waitHost(t, registry, endpoint, func(host ports.RemoteHostSnapshot) bool {
760+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
761+ })
762+ }
763+}
764+
765+func TestRegistrySnapshotIsImmutable(t *testing.T) {
766+ clock := newManualClock(time.Unix(100, 0))
767+ probe := newTestProbe(1)
768+ registry := newTestRegistry(t, 1, nil, probe, clock)
769+ reg := registration(t, "example.test", 1)
770+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
771+ startRegistry(t, registry)
772+
773+ call := receiveCall(t, probe)
774+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
775+ Availability: domain.RemoteAvailabilityReachable,
776+ Sessions: []catalogue.RemoteCatalogSession{{
777+ Name: "work",
778+ State: catalogue.RemoteCatalogSessionUp,
779+ Tabs: []catalogue.RemoteCatalogTab{{ID: "tab-1", Name: "shell"}},
780+ }},
781+ }}
782+ observed := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
783+ return !host.Checking && len(host.Sessions) == 1
784+ })
785+ require.Equal(t, "work", observed.Sessions[0].Name)
786+
787+ // Mutating a returned projection never reaches the registry.
788+ observed.Endpoint = "mutated"
789+ observed.Sessions[0].Name = "mutated"
790+ observed.Sessions[0].Tabs[0].Name = "mutated"
791+ current := registry.Snapshot()
792+ require.Equal(t, reg.Endpoint, current.Hosts[0].Endpoint)
793+ require.Equal(t, "work", current.Hosts[0].Sessions[0].Name)
794+ require.Equal(t, "shell", current.Hosts[0].Sessions[0].Tabs[0].Name)
795+
796+ // A later publication never mutates a snapshot already handed out.
797+ earlier := registry.Snapshot()
798+ registry.RequestProbe(reg.Endpoint)
799+ next := receiveCall(t, probe)
800+ next.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
801+ Availability: domain.RemoteAvailabilityReachable,
802+ Sessions: []catalogue.RemoteCatalogSession{{Name: "other", State: catalogue.RemoteCatalogSessionUp}},
803+ }}
804+ waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
805+ return !host.Checking && len(host.Sessions) == 1 && host.Sessions[0].Name == "other"
806+ })
807+ require.Equal(t, "work", earlier.Hosts[0].Sessions[0].Name)
808+}
809+
810+func TestRegistrySchedulesFreshObservationAfterSuccess(t *testing.T) {
811+ start := time.Unix(100, 0)
812+ clock := newManualClock(start)
813+ probe := newTestProbe(1)
814+ registry := newTestRegistry(t, 1, nil, probe, clock)
815+ reg := registration(t, "example.test", 1)
816+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
817+ startRegistry(t, registry)
818+
819+ call := receiveCall(t, probe)
820+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
821+
822+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
823+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
824+ })
825+ require.Equal(t, start, host.LastAttempt)
826+ require.Equal(t, start, host.LastSuccess)
827+ require.Equal(t, start.Add(defaultFreshFor), host.NextDue)
828+ require.Zero(t, host.ConsecutiveFailures)
829+ require.Zero(t, host.FailureEpisode)
830+
831+ // A fresh host is not observed again before its scheduled deadline.
832+ clock.Advance(defaultFreshFor - time.Nanosecond)
833+ requireNoProbeWithin(t, probe, probeGrace)
834+ clock.Advance(time.Nanosecond)
835+ next := receiveCall(t, probe)
836+ require.True(t, next.registration.Equal(reg))
837+}
838+
839+func TestRegistryDemandOverridesFreshSchedule(t *testing.T) {
840+ start := time.Unix(100, 0)
841+ clock := newManualClock(start)
842+ probe := newTestProbe(1)
843+ registry := newTestRegistry(t, 1, nil, probe, clock)
844+ reg := registration(t, "example.test", 1)
845+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
846+ startRegistry(t, registry)
847+
848+ call := receiveCall(t, probe)
849+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
850+ waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
851+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
852+ })
853+
854+ // Explicit demand is admitted immediately, before the fresh deadline.
855+ registry.RequestProbe(reg.Endpoint)
856+ forced := receiveCall(t, probe)
857+ require.True(t, forced.registration.Equal(reg))
858+ require.Equal(t, start, clock.Now())
859+}
860+
861+func TestRegistryCoalescesConcurrentDemand(t *testing.T) {
862+ start := time.Unix(100, 0)
863+ clock := newManualClock(start)
864+ probe := newTestProbe(2)
865+ registry := newTestRegistry(t, 1, nil, probe, clock)
866+ reg := registration(t, "example.test", 1)
867+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
868+ startRegistry(t, registry)
869+
870+ call := receiveCall(t, probe)
871+ for range 10 {
872+ registry.RequestProbe(reg.Endpoint)
873+ }
874+ select {
875+ case duplicate := <-probe.calls:
876+ t.Fatalf("concurrent demand started a duplicate probe for %q", duplicate.registration.Endpoint)
877+ default:
878+ }
879+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
880+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
881+ return host.Availability == domain.RemoteAvailabilityReachable && host.LastSuccess.Equal(start)
882+ })
883+ require.Equal(t, start.Add(defaultFreshFor), host.NextDue)
884+
885+ // The single coalesced follow-up is admitted once the first completes.
886+ followUp := receiveCall(t, probe)
887+ require.True(t, followUp.registration.Equal(reg))
888+ followUp.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
889+ waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
890+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
891+ })
892+}
893+
894+func TestRegistryIgnoresDemandForUnknownEndpoint(t *testing.T) {
895+ clock := newManualClock(time.Unix(100, 0))
896+ probe := newTestProbe(1)
897+ registry := newTestRegistry(t, 1, nil, probe, clock)
898+ reg := registration(t, "example.test", 1)
899+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
900+ registry.RequestProbe("unknown.test")
901+ startRegistry(t, registry)
902+
903+ call := receiveCall(t, probe)
904+ require.True(t, call.registration.Equal(reg))
905+ // No probe is ever scheduled for an endpoint outside configured membership.
906+ requireNoProbeWithin(t, probe, probeGrace)
907+}
908+
909+func TestRegistryPublishesReachabilityAndPersists(t *testing.T) {
910+ start := time.Unix(100, 0)
911+ clock := newManualClock(start)
912+ probe := newTestProbe(1)
913+ store := &testStore{}
914+ registry := newTestRegistry(t, 3, store, probe, clock)
915+ reg := registration(t, "example.test", 1)
916+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
917+ startRegistry(t, registry)
918+
919+ call := receiveCall(t, probe)
920+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
921+ Availability: domain.RemoteAvailabilityReachable,
922+ InventoryKnown: true,
923+ Sessions: []catalogue.RemoteCatalogSession{{Name: "work", State: catalogue.RemoteCatalogSessionUp}},
924+ }}
925+ snapshot := waitSnapshot(t, registry, func(snapshot ports.BrokerSnapshot) bool {
926+ host, ok := snapshot.Find(reg.Endpoint)
927+ return ok && !host.Checking && host.Availability == domain.RemoteAvailabilityReachable && len(host.Sessions) == 1
928+ })
929+
930+ stored := waitStored(t, store, func(stored ports.BrokerSnapshot) bool {
931+ return stored.Revision >= snapshot.Revision && len(stored.Hosts) == 1
932+ })
933+ require.Equal(t, ports.BrokerEpoch(3), stored.Epoch)
934+ require.Len(t, stored.Hosts, 1)
935+ require.Equal(t, "work", stored.Hosts[0].Sessions[0].Name)
936+ require.Equal(t, reg.Endpoint, stored.Hosts[0].Endpoint)
937+ require.False(t, stored.Hosts[0].Checking)
938+}
939+
940+func TestRegistryNonReachableOutcomesUseTypedFailureBackoff(t *testing.T) {
941+ dialErr := errors.New("dial failed")
942+ tests := []struct {
943+ name string
944+ answer probeAnswer
945+ wantAvailability domain.RemoteAvailability
946+ wantKind domain.RemoteFailureKind
947+ wantErr error
948+ }{
949+ {
950+ name: "unreachable without error",
951+ answer: probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityUnreachable}},
952+ wantAvailability: domain.RemoteAvailabilityUnreachable,
953+ wantKind: domain.RemoteFailureTransport,
954+ },
955+ {
956+ name: "unknown outcome is not reachability",
957+ answer: probeAnswer{snapshot: ports.RemoteHostSnapshot{}},
958+ wantAvailability: domain.RemoteAvailabilityUnreachable,
959+ wantKind: domain.RemoteFailureTransport,
960+ },
961+ {
962+ name: "incompatible",
963+ answer: probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityIncompatible}},
964+ wantAvailability: domain.RemoteAvailabilityIncompatible,
965+ wantKind: domain.RemoteFailureIncompatible,
966+ },
967+ {
968+ name: "authentication",
969+ answer: probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityAuthFailed}},
970+ wantAvailability: domain.RemoteAvailabilityAuthFailed,
971+ wantKind: domain.RemoteFailureAuthentication,
972+ },
973+ {
974+ name: "invalid response",
975+ answer: probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityInvalidResponse}},
976+ wantAvailability: domain.RemoteAvailabilityInvalidResponse,
977+ wantKind: domain.RemoteFailureInvalidResponse,
978+ },
979+ {
980+ name: "transport error",
981+ answer: probeAnswer{err: dialErr},
982+ wantAvailability: domain.RemoteAvailabilityUnreachable,
983+ wantKind: domain.RemoteFailureTransport,
984+ wantErr: dialErr,
985+ },
986+ {
987+ name: "deadline exceeded",
988+ answer: probeAnswer{err: context.DeadlineExceeded},
989+ wantAvailability: domain.RemoteAvailabilityUnreachable,
990+ wantKind: domain.RemoteFailureTimeout,
991+ wantErr: context.DeadlineExceeded,
992+ },
993+ }
994+ for _, tt := range tests {
995+ t.Run(tt.name, func(t *testing.T) {
996+ start := time.Unix(100, 0)
997+ clock := newManualClock(start)
998+ probe := newTestProbe(1)
999+ registry := newTestRegistry(t, 1, nil, probe, clock)
1000+ reg := registration(t, "example.test", 1)
1001+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1002+ startRegistry(t, registry)
1003+
1004+ call := receiveCall(t, probe)
1005+ call.result <- tt.answer
1006+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1007+ return !host.Checking && host.ConsecutiveFailures == 1
1008+ })
1009+ require.Equal(t, tt.wantAvailability, host.Availability)
1010+ require.Equal(t, tt.wantKind, host.LastFailure.Kind)
1011+ require.Equal(t, uint(1), host.ConsecutiveFailures)
1012+ require.Equal(t, uint64(1), host.FailureEpisode)
1013+ // No observation succeeded yet, so there is no last success.
1014+ require.True(t, host.LastSuccess.IsZero())
1015+ // Every non-reachable outcome retries on the capped cadence.
1016+ require.Equal(t, start.Add(defaultRetryBase), host.NextDue)
1017+ if tt.wantErr != nil {
1018+ require.ErrorIs(t, host.LastFailure, tt.wantErr)
1019+ } else {
1020+ require.Nil(t, host.LastFailure.Err)
1021+ }
1022+ })
1023+ }
1024+}
1025+
1026+func TestRegistryClassifiesOversizedProbeProjectionAsInvalidResponse(t *testing.T) {
1027+ start := time.Unix(100, 0)
1028+ clock := newManualClock(start)
1029+ probe := newTestProbe(1)
1030+ registry := newTestRegistry(t, 1, nil, probe, clock)
1031+ reg := registration(t, "example.test", 1)
1032+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1033+ startRegistry(t, registry)
1034+ sub := registry.Subscribe()
1035+ defer sub.Close()
1036+
1037+ // A reachable observation establishes the last-known inventory.
1038+ call := receiveCall(t, probe)
1039+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
1040+ Availability: domain.RemoteAvailabilityReachable,
1041+ Sessions: []catalogue.RemoteCatalogSession{{Name: "work", State: catalogue.RemoteCatalogSessionUp}},
1042+ }}
1043+ waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1044+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
1045+ })
1046+
1047+ // A projection carrying one session more than the publication bound is a
1048+ // malformed response: it is classified as an invalid response and never
1049+ // reaches publication, because such a snapshot fails Validate. The loop
1050+ // checks every observed publication, so an invalid snapshot can never pass
1051+ // unnoticed.
1052+ clock.Advance(defaultFreshFor)
1053+ call = receiveCall(t, probe)
1054+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
1055+ Availability: domain.RemoteAvailabilityReachable,
1056+ Sessions: hostSessions(ports.BrokerMaxSessionsPerHost + 1),
1057+ }}
1058+ deadline := time.After(time.Second)
1059+ var host ports.RemoteHostSnapshot
1060+ for {
1061+ snapshot := registry.Snapshot()
1062+ require.NoError(t, snapshot.Validate())
1063+ current, ok := snapshot.Find(reg.Endpoint)
1064+ if ok && !current.Checking && current.ConsecutiveFailures == 1 {
1065+ host = current
1066+ break
1067+ }
1068+ select {
1069+ case <-sub.Changed():
1070+ case <-deadline:
1071+ t.Fatal("timed out waiting for the invalid-response failure")
1072+ }
1073+ }
1074+ require.Equal(t, domain.RemoteAvailabilityInvalidResponse, host.Availability)
1075+ require.Equal(t, domain.RemoteFailureInvalidResponse, host.LastFailure.Kind)
1076+ // The last-known inventory is retained: a malformed projection never
1077+ // replaces it.
1078+ require.Len(t, host.Sessions, 1)
1079+ require.Equal(t, "work", host.Sessions[0].Name)
1080+ require.Equal(t, clock.Now().Add(defaultRetryBase), host.NextDue)
1081+}
1082+
1083+func TestRegistryAcceptsProbeProjectionAtSessionBound(t *testing.T) {
1084+ start := time.Unix(100, 0)
1085+ clock := newManualClock(start)
1086+ probe := newTestProbe(1)
1087+ registry := newTestRegistry(t, 1, nil, probe, clock)
1088+ reg := registration(t, "example.test", 1)
1089+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1090+ startRegistry(t, registry)
1091+
1092+ // The bound itself is valid: exactly BrokerMaxSessionsPerHost sessions are
1093+ // accepted and published, and the resulting snapshot validates.
1094+ call := receiveCall(t, probe)
1095+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
1096+ Availability: domain.RemoteAvailabilityReachable,
1097+ Sessions: hostSessions(ports.BrokerMaxSessionsPerHost),
1098+ }}
1099+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1100+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
1101+ })
1102+ require.Len(t, host.Sessions, ports.BrokerMaxSessionsPerHost)
1103+ require.Zero(t, host.ConsecutiveFailures)
1104+ snapshot := registry.Snapshot()
1105+ require.NoError(t, snapshot.Validate())
1106+ require.Len(t, snapshot.Hosts[0].Sessions, ports.BrokerMaxSessionsPerHost)
1107+}
1108+
1109+func TestRegistryPreservesLastSuccessAndOwnsFailureEpisode(t *testing.T) {
1110+ start := time.Unix(100, 0)
1111+ clock := newManualClock(start)
1112+ probe := newTestProbe(1)
1113+ registry := newTestRegistry(t, 1, nil, probe, clock)
1114+ reg := registration(t, "example.test", 1)
1115+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1116+ startRegistry(t, registry)
1117+
1118+ // A successful observation owns the success time and opens no episode.
1119+ call := receiveCall(t, probe)
1120+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
1121+ Availability: domain.RemoteAvailabilityReachable,
1122+ Sessions: []catalogue.RemoteCatalogSession{{Name: "work", State: catalogue.RemoteCatalogSessionUp}},
1123+ }}
1124+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1125+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
1126+ })
1127+ require.Equal(t, start, host.LastSuccess)
1128+ require.Zero(t, host.FailureEpisode)
1129+
1130+ // A failure preserves the last success and opens exactly one episode.
1131+ clock.Advance(defaultFreshFor)
1132+ call = receiveCall(t, probe)
1133+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
1134+ Availability: domain.RemoteAvailabilityIncompatible,
1135+ Sessions: []catalogue.RemoteCatalogSession{{Name: "stale", State: catalogue.RemoteCatalogSessionUp}},
1136+ }}
1137+ host = waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1138+ return !host.Checking && host.ConsecutiveFailures == 1
1139+ })
1140+ require.Equal(t, start, host.LastSuccess)
1141+ require.Equal(t, uint64(1), host.FailureEpisode)
1142+ // The last-known inventory is retained: a failed observation never
1143+ // replaces it.
1144+ require.Equal(t, "work", host.Sessions[0].Name)
1145+
1146+ // Further failures in the same streak preserve the episode identity so
1147+ // notice policy does not re-notify.
1148+ clock.Advance(defaultRetryBase)
1149+ call = receiveCall(t, probe)
1150+ call.result <- probeAnswer{err: errors.New("offline")}
1151+ host = waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1152+ return !host.Checking && host.ConsecutiveFailures == 2
1153+ })
1154+ require.Equal(t, start, host.LastSuccess)
1155+ require.Equal(t, uint64(1), host.FailureEpisode)
1156+
1157+ // Recovery clears the streak, keeps the episode, and advances the success.
1158+ clock.Advance(2 * defaultRetryBase)
1159+ recovered := clock.Now()
1160+ call = receiveCall(t, probe)
1161+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
1162+ host = waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1163+ return !host.Checking && host.ConsecutiveFailures == 0
1164+ })
1165+ require.Equal(t, recovered, host.LastSuccess)
1166+ require.Equal(t, uint64(1), host.FailureEpisode)
1167+ require.Equal(t, domain.RemoteFailureNone, host.LastFailure.Kind)
1168+ require.Nil(t, host.LastFailure.Err)
1169+}
1170+
1171+func TestRegistryBackoffIsExponentialAndCapped(t *testing.T) {
1172+ start := time.Unix(100, 0)
1173+ clock := newManualClock(start)
1174+ probe := newTestProbe(1)
1175+ registry := newTestRegistry(t, 1, nil, probe, clock)
1176+ reg := registration(t, "example.test", 1)
1177+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1178+ startRegistry(t, registry)
1179+
1180+ probeErr := errors.New("offline")
1181+ // The retry cadence matches the legacy remotes monitor: a 5s base doubling
1182+ // to a 60s cap.
1183+ want := []time.Duration{
1184+ 5 * time.Second,
1185+ 10 * time.Second,
1186+ 20 * time.Second,
1187+ 40 * time.Second,
1188+ time.Minute, // 80s capped at defaultRetryLimit
1189+ time.Minute,
1190+ time.Minute,
1191+ }
1192+ for index, delay := range want {
1193+ call := receiveCall(t, probe)
1194+ at := clock.Now()
1195+ call.result <- probeAnswer{err: probeErr}
1196+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1197+ return !host.Checking && host.ConsecutiveFailures == uint(index+1)
1198+ })
1199+ require.Equal(t, domain.RemoteAvailabilityUnreachable, host.Availability)
1200+ require.Equal(t, domain.RemoteFailureTransport, host.LastFailure.Kind)
1201+ require.ErrorIs(t, host.LastFailure, probeErr)
1202+ require.Equal(t, at.Add(delay), host.NextDue)
1203+
1204+ if index == len(want)-1 {
1205+ break
1206+ }
1207+ // The retry is not admitted early: only the backoff deadline wakes it.
1208+ clock.Advance(delay - time.Nanosecond)
1209+ requireNoProbeWithin(t, probe, probeGrace)
1210+ clock.Advance(time.Nanosecond)
1211+ }
1212+}
1213+
1214+func TestRegistryJitterAppliesToRetryAndHealthyRefresh(t *testing.T) {
1215+ start := time.Unix(100, 0)
1216+ clock := newManualClock(start)
1217+ probe := newTestProbe(1)
1218+ registry := newTestRegistry(t, 1, nil, probe, clock)
1219+ registry.jitter = func(base time.Duration, _ string, _ uint64) time.Duration { return base / 2 }
1220+ reg := registration(t, "example.test", 1)
1221+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1222+ startRegistry(t, registry)
1223+
1224+ call := receiveCall(t, probe)
1225+ call.result <- probeAnswer{err: errors.New("offline")}
1226+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1227+ return !host.Checking && host.Availability == domain.RemoteAvailabilityUnreachable
1228+ })
1229+ require.Equal(t, start.Add(defaultRetryBase/2), host.NextDue)
1230+
1231+ clock.Advance(defaultRetryBase / 2)
1232+ next := receiveCall(t, probe)
1233+ successAt := clock.Now()
1234+ next.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
1235+ host = waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1236+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
1237+ })
1238+ // The healthy refresh is jittered too: a fleet of hosts never converges
1239+ // on one instant.
1240+ require.Equal(t, successAt.Add(defaultFreshFor/2), host.NextDue)
1241+}
1242+
1243+func TestRegistryJitterIsDeterministicAndBounded(t *testing.T) {
1244+ bases := []time.Duration{defaultRetryBase, defaultFreshFor, defaultRetryLimit}
1245+ endpoints := []string{"example.test", "other.test", "user@host:2222"}
1246+ for _, base := range bases {
1247+ minimum, maximum := base*9/10, base*11/10
1248+ for _, endpoint := range endpoints {
1249+ spread := make(map[time.Duration]struct{})
1250+ for attempt := uint64(1); attempt <= 64; attempt++ {
1251+ got := jittered(base, endpoint, attempt)
1252+ require.GreaterOrEqual(t, got, minimum)
1253+ require.LessOrEqual(t, got, maximum)
1254+ // Identical endpoint and attempt always yield the identical
1255+ // duration: no wall clock and no shared PRNG participate.
1256+ require.Equal(t, got, jittered(base, endpoint, attempt))
1257+ spread[got] = struct{}{}
1258+ }
1259+ require.Greater(t, len(spread), 1, "jitter must vary across attempts")
1260+ }
1261+ }
1262+ // A non-positive base has no spread to apply.
1263+ require.Zero(t, jittered(0, "example.test", 1))
1264+}
1265+
1266+func TestRegistryFencesStaleCompletionAcrossRegistrationChanges(t *testing.T) {
1267+ replacementAnswer := func(name string) probeAnswer {
1268+ return probeAnswer{snapshot: ports.RemoteHostSnapshot{
1269+ Availability: domain.RemoteAvailabilityReachable,
1270+ Sessions: []catalogue.RemoteCatalogSession{{Name: name, State: catalogue.RemoteCatalogSessionUp}},
1271+ }}
1272+ }
1273+ assertFenced := func(t *testing.T, current, replacement domain.RemoteRegistration) {
1274+ t.Helper()
1275+ clock := newManualClock(time.Unix(100, 0))
1276+ probe := newTestProbe(2)
1277+ registry := newTestRegistry(t, 1, nil, probe, clock)
1278+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{current}))
1279+ startRegistry(t, registry)
1280+ currentCall := receiveCall(t, probe)
1281+ require.True(t, currentCall.registration.Equal(current))
1282+
1283+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{replacement}))
1284+ replacementCall := receiveCall(t, probe)
1285+ require.True(t, replacementCall.registration.Equal(replacement))
1286+
1287+ // The replacement completes and becomes authoritative...
1288+ replacementCall.result <- replacementAnswer("replacement")
1289+ applied := waitHost(t, registry, replacement.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1290+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable && len(host.Sessions) == 1
1291+ })
1292+ require.True(t, applied.Registration.Equal(replacement))
1293+ require.Equal(t, "replacement", applied.Sessions[0].Name)
1294+
1295+ // ...and the completion bound to the retired registration, arriving
1296+ // afterwards, never touches it.
1297+ currentCall.result <- replacementAnswer("retired")
1298+ requireHostStableWithin(t, registry, replacement.Endpoint, applied, probeGrace)
1299+
1300+ // The retired completion also never holds the replacement's slot: a
1301+ // further observation is still admitted and still applies.
1302+ registry.RequestProbe(replacement.Endpoint)
1303+ next := receiveCall(t, probe)
1304+ require.True(t, next.registration.Equal(replacement))
1305+ next.result <- replacementAnswer("next")
1306+ waitHost(t, registry, replacement.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1307+ return !host.Checking && len(host.Sessions) == 1 && host.Sessions[0].Name == "next"
1308+ })
1309+ }
1310+
1311+ t.Run("incarnation", func(t *testing.T) {
1312+ assertFenced(t, registration(t, "example.test", 1), registration(t, "example.test", 2))
1313+ })
1314+ t.Run("generation", func(t *testing.T) {
1315+ current := registration(t, "example.test", 1)
1316+ replacement := current
1317+ replacement.Generation++
1318+ assertFenced(t, current, replacement)
1319+ })
1320+}
1321+
1322+func TestRegistryFencesRetiredAttemptsAcrossRemovalAndIdenticalReAdd(t *testing.T) {
1323+ clock := newManualClock(time.Unix(100, 0))
1324+ probe := newTestProbe(2)
1325+ registry := newTestRegistry(t, 1, nil, probe, clock)
1326+ reg := registration(t, "example.test", 1)
1327+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1328+
1329+ ctx, cancel := context.WithCancel(context.Background())
1330+ defer cancel()
1331+ results := make(chan probeResult, 1)
1332+ dispatch := func() { registry.dispatch(ctx, clock.Now(), results) }
1333+
1334+ dispatch()
1335+ retiredCall := receiveCall(t, probe)
1336+ require.True(t, retiredCall.registration.Equal(reg))
1337+ retired := attemptFor(registry, reg.Endpoint)
1338+ require.NotZero(t, retired)
1339+
1340+ // Removal retires a bounded tombstone and drops the in-flight token.
1341+ require.NoError(t, registry.SetHosts(nil))
1342+ removed := registry.Snapshot()
1343+ require.Empty(t, removed.Hosts)
1344+ require.Len(t, removed.Removed, 1)
1345+ require.Equal(t, removed.Revision, removed.Removed[0].RetiredRevision)
1346+ require.True(t, removed.Removed[0].Fences(reg))
1347+ require.NoError(t, removed.Validate())
1348+
1349+ // The retired attempt can never revive a removed endpoint.
1350+ registry.apply(probeResult{endpoint: reg.Endpoint, registration: reg, attempt: retired, snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}, at: clock.Now()})
1351+ require.Empty(t, registry.Snapshot().Hosts)
1352+ require.Len(t, registry.Snapshot().Removed, 1)
1353+
1354+ // Re-adding the identical registration supersedes the tombstone, so only
1355+ // the attempt token can fence the retired completion.
1356+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1357+ require.Empty(t, registry.Snapshot().Removed)
1358+ dispatch()
1359+ freshCall := receiveCall(t, probe)
1360+ require.True(t, freshCall.registration.Equal(reg))
1361+ fresh := attemptFor(registry, reg.Endpoint)
1362+ require.Greater(t, fresh, retired)
1363+
1364+ registry.apply(probeResult{endpoint: reg.Endpoint, registration: reg, attempt: retired, snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}, at: clock.Now()})
1365+ stale, ok := registry.Snapshot().Find(reg.Endpoint)
1366+ require.True(t, ok)
1367+ require.True(t, stale.Checking)
1368+ require.Equal(t, domain.RemoteAvailabilityUnknown, stale.Availability)
1369+
1370+ registry.apply(probeResult{endpoint: reg.Endpoint, registration: reg, attempt: fresh, snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}, at: clock.Now()})
1371+ applied, ok := registry.Snapshot().Find(reg.Endpoint)
1372+ require.True(t, ok)
1373+ require.False(t, applied.Checking)
1374+ require.Equal(t, domain.RemoteAvailabilityReachable, applied.Availability)
1375+}
1376+
1377+func TestRegistryRetiresInflightAttemptsOnReplacementAndRemoval(t *testing.T) {
1378+ current := registration(t, "example.test", 1)
1379+ replacement := registration(t, "example.test", 2)
1380+ tests := []struct {
1381+ name string
1382+ next []domain.RemoteRegistration
1383+ }{
1384+ {name: "replacement", next: []domain.RemoteRegistration{replacement}},
1385+ {name: "removal", next: nil},
1386+ }
1387+ for _, tt := range tests {
1388+ t.Run(tt.name, func(t *testing.T) {
1389+ clock := newManualClock(time.Unix(100, 0))
1390+ probe := newTestProbe(2)
1391+ registry := newTestRegistry(t, 1, nil, probe, clock)
1392+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{current}))
1393+ startRegistry(t, registry)
1394+
1395+ call := receiveCall(t, probe)
1396+ require.True(t, call.registration.Equal(current))
1397+
1398+ // Replacing or removing the registration retires the in-flight
1399+ // attempt: the probe's derived context is cancelled before the
1400+ // replacement can start, so a ctx-honoring probe stops instead of
1401+ // overlapping its successor.
1402+ require.NoError(t, registry.SetHosts(tt.next))
1403+ select {
1404+ case <-probe.canceled:
1405+ case <-time.After(time.Second):
1406+ t.Fatal("retired probe did not observe cancellation")
1407+ }
1408+ })
1409+ }
1410+}
1411+
1412+func TestRegistryChurnKeepsOneActiveProbePerEndpoint(t *testing.T) {
1413+ clock := newManualClock(time.Unix(100, 0))
1414+ probe := newChurnProbe()
1415+ defer probe.openRelease()
1416+ registry := newTestRegistry(t, 1, nil, probe, clock)
1417+
1418+ first := registration(t, "example.test", 1)
1419+ second := registration(t, "example.test", 2)
1420+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{first}))
1421+
1422+ ctx, cancel := context.WithCancel(context.Background())
1423+ defer cancel()
1424+ done := make(chan struct{})
1425+ go func() { defer close(done); registry.Run(ctx) }()
1426+
1427+ // Drop the initial attempt's start signal so each lockstep round below
1428+ // consumes the start of its own replacement attempt.
1429+ select {
1430+ case <-probe.startedSig:
1431+ case <-time.After(time.Second):
1432+ t.Fatal("timed out waiting for the initial probe")
1433+ }
1434+ require.Equal(t, 1, probe.startedCount())
1435+
1436+ // Lockstep churn: every round replaces the registration, which must cancel
1437+ // the retired attempt before the replacement starts. An uncancelled
1438+ // retirement would stay live and overlap its successor, and the exact start
1439+ // count catches an attempt that never restarts.
1440+ const rounds = 8
1441+ for round := 0; round < rounds; round++ {
1442+ next := first
1443+ if round%2 == 0 {
1444+ next = second
1445+ }
1446+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{next}))
1447+ select {
1448+ case <-probe.startedSig:
1449+ case <-time.After(time.Second):
1450+ t.Fatal("timed out waiting for the replacement attempt to start")
1451+ }
1452+ require.Equal(t, round+2, probe.startedCount())
1453+ }
1454+
1455+ // Concurrent churn and demand: the same invariant must hold under the race
1456+ // detector while membership and demand churn from several goroutines.
1457+ failures := make([]error, 4)
1458+ var wg sync.WaitGroup
1459+ for worker := 0; worker < len(failures); worker++ {
1460+ wg.Add(1)
1461+ go func(worker int) {
1462+ defer wg.Done()
1463+ for round := 0; round < 32; round++ {
1464+ next := first
1465+ if (worker+round)%2 == 0 {
1466+ next = second
1467+ }
1468+ if err := registry.SetHosts([]domain.RemoteRegistration{next}); err != nil {
1469+ failures[worker] = err
1470+ return
1471+ }
1472+ registry.RequestProbe(first.Endpoint)
1473+ registry.Snapshot()
1474+ }
1475+ }(worker)
1476+ }
1477+ wg.Wait()
1478+ for _, err := range failures {
1479+ require.NoError(t, err)
1480+ }
1481+
1482+ endpoints := probe.observedEndpoints()
1483+ require.NotEmpty(t, endpoints)
1484+ for _, endpoint := range endpoints {
1485+ require.Equal(t, 1, probe.maxLiveFor(endpoint), "endpoint %q had overlapping live probes", endpoint)
1486+ }
1487+ require.GreaterOrEqual(t, probe.startedCount(), rounds+1)
1488+
1489+ // Settling releases every attempt: no probe stays live once Run returns.
1490+ cancel()
1491+ select {
1492+ case <-done:
1493+ case <-time.After(time.Second):
1494+ t.Fatal("registry did not stop after cancellation")
1495+ }
1496+ deadline := time.Now().Add(time.Second)
1497+ for probe.inflight() != 0 {
1498+ if time.Now().After(deadline) {
1499+ t.Fatalf("%d probes still live after settle", probe.inflight())
1500+ }
1501+ time.Sleep(time.Millisecond)
1502+ }
1503+}
1504+
1505+func TestRegistryPublishesBoundedTombstones(t *testing.T) {
1506+ registry := newTestRegistry(t, 1, nil, newTestProbe(1), newManualClock(time.Unix(100, 0)))
1507+
1508+ // Two full membership rounds with distinct endpoints: without pruning the
1509+ // retired set would double.
1510+ for round := 0; round < 2; round++ {
1511+ registrations := make([]domain.RemoteRegistration, 0, ports.BrokerMaxHosts)
1512+ for index := 0; index < ports.BrokerMaxHosts; index++ {
1513+ registrations = append(registrations, registration(t, fmt.Sprintf("round-%d-host-%d.test", round, index), byte(round*ports.BrokerMaxHosts+index+1)))
1514+ }
1515+ require.NoError(t, registry.SetHosts(registrations))
1516+ require.NoError(t, registry.SetHosts(nil))
1517+ }
1518+
1519+ snapshot := registry.Snapshot()
1520+ require.NoError(t, snapshot.Validate())
1521+ require.Len(t, snapshot.Removed, ports.BrokerMaxTombstones)
1522+ for index, tombstone := range snapshot.Removed {
1523+ require.NoError(t, tombstone.Validate())
1524+ // Only the newest retirements survive the bound.
1525+ require.Equal(t, snapshot.Revision, tombstone.RetiredRevision)
1526+ if index > 0 {
1527+ require.GreaterOrEqual(t, snapshot.Removed[index-1].RetiredRevision, tombstone.RetiredRevision)
1528+ }
1529+ }
1530+}
1531+
1532+func TestRegistryPersistenceRunsOffLockAndCoalescesWrites(t *testing.T) {
1533+ clock := newManualClock(time.Unix(100, 0))
1534+ probe := newTestProbe(1)
1535+ store := newBlockingStore()
1536+ registry, err := NewRegistry(1, store, probe, clock, nil)
1537+ require.NoError(t, err)
1538+ defer registry.store.close()
1539+
1540+ reg := registration(t, "example.test", 1)
1541+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1542+ // The writer is now blocked inside the first durable write.
1543+ first := <-store.entered
1544+ require.Equal(t, ports.BrokerRevision(2), first)
1545+
1546+ // Publication is never blocked behind persistence and never holds the
1547+ // registry lock across a store call.
1548+ var setErr error
1549+ published := make(chan struct{})
1550+ go func() {
1551+ defer close(published)
1552+ for generation := domain.RemoteGeneration(2); generation <= 6; generation++ {
1553+ reg.Generation = generation
1554+ if err := registry.SetHosts([]domain.RemoteRegistration{reg}); err != nil {
1555+ setErr = err
1556+ return
1557+ }
1558+ }
1559+ }()
1560+ select {
1561+ case <-published:
1562+ case <-time.After(time.Second):
1563+ t.Fatal("publication blocked behind durable persistence")
1564+ }
1565+ require.NoError(t, setErr)
1566+
1567+ last := registry.Snapshot().Revision
1568+ store.releaseFirst()
1569+ waitWriterRevision(t, registry, last)
1570+
1571+ // Writes stay ordered and coalesced: only the first and the newest
1572+ // revision ever reach the store.
1573+ require.Equal(t, []ports.BrokerRevision{2, last}, store.writtenRevisions())
1574+}
1575+
1576+func TestRegistryStoreErrorsAreObservableAndDoNotStallPublication(t *testing.T) {
1577+ clock := newManualClock(time.Unix(100, 0))
1578+ probe := newTestProbe(1)
1579+ logger, logs := testLogger()
1580+ storeErr := errors.New("disk full")
1581+ store := &testStore{storeErr: storeErr}
1582+ registry, err := NewRegistry(1, store, probe, clock, logger)
1583+ require.NoError(t, err)
1584+ defer registry.store.close()
1585+
1586+ reg := registration(t, "example.test", 1)
1587+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1588+ waitForLog(t, logs, "durable snapshot write failed")
1589+ require.Contains(t, logs.String(), storeErr.Error())
1590+
1591+ // A failed write is not fatal and never blocks the next publication.
1592+ store.setStoreErr(nil)
1593+ reg.Generation++
1594+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1595+ stored := waitStored(t, store, func(snapshot ports.BrokerSnapshot) bool {
1596+ return snapshot.Revision == registry.Snapshot().Revision
1597+ })
1598+ require.Len(t, stored.Hosts, 1)
1599+ require.True(t, stored.Hosts[0].Registration.Equal(reg))
1600+}
1601+
1602+func TestRegistryPersistedCopyClearsChecking(t *testing.T) {
1603+ clock := newManualClock(time.Unix(100, 0))
1604+ probe := newTestProbe(1)
1605+ store := &testStore{}
1606+ registry := newTestRegistry(t, 3, store, probe, clock)
1607+ reg := registration(t, "example.test", 1)
1608+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1609+ startRegistry(t, registry)
1610+
1611+ // The observation is in flight: the published projection carries Checking,
1612+ // the durable copy never does.
1613+ require.True(t, waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1614+ return host.Checking
1615+ }).Checking)
1616+ stored := waitStored(t, store, func(snapshot ports.BrokerSnapshot) bool {
1617+ return len(snapshot.Hosts) == 1
1618+ })
1619+ require.False(t, stored.Hosts[0].Checking)
1620+
1621+ // Completing the observation persists the settled projection.
1622+ call := receiveCall(t, probe)
1623+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
1624+ settled := waitStored(t, store, func(snapshot ports.BrokerSnapshot) bool {
1625+ return len(snapshot.Hosts) == 1 && snapshot.Hosts[0].Availability == domain.RemoteAvailabilityReachable
1626+ })
1627+ require.False(t, settled.Hosts[0].Checking)
1628+}
1629+
1630+func TestRegistryBoundsSlowSubscribersAndStopsOnCancellation(t *testing.T) {
1631+ clock := newManualClock(time.Unix(100, 0))
1632+ probe := newTestProbe(1)
1633+ registry := newTestRegistry(t, 1, nil, probe, clock)
1634+
1635+ sub := registry.Subscribe()
1636+ reg := registration(t, "example.test", 1)
1637+ for generation := domain.RemoteGeneration(1); generation <= 10; generation++ {
1638+ reg.Generation = generation
1639+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1640+ }
1641+ // A subscriber that never drains is bounded to one pending wake and
1642+ // never blocks publication.
1643+ require.Len(t, sub.Changed(), 1)
1644+
1645+ // Drain the pending wake, then close: a closed subscriber never receives
1646+ // another notification, and closing is idempotent.
1647+ <-sub.Changed()
1648+ sub.Close()
1649+ sub.Close()
1650+ reg.Generation++
1651+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1652+ select {
1653+ case <-sub.Changed():
1654+ t.Fatal("closed subscription received a notification")
1655+ default:
1656+ }
1657+
1658+ ctx, cancel := context.WithCancel(context.Background())
1659+ done := make(chan struct{})
1660+ go func() { defer close(done); registry.Run(ctx) }()
1661+ receiveCall(t, probe)
1662+ cancel()
1663+ select {
1664+ case <-done:
1665+ case <-time.After(time.Second):
1666+ t.Fatal("registry did not stop after cancellation")
1667+ }
1668+ select {
1669+ case <-probe.canceled:
1670+ case <-time.After(time.Second):
1671+ t.Fatal("in-flight probe did not observe cancellation")
1672+ }
1673+ // Cleanup settles the projection: no host stays in flight.
1674+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1675+ return !host.Checking
1676+ })
1677+ require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability)
1678+}
1679+
1680+func TestRegistryRunIsSingleShotAndCleansUpInFlightState(t *testing.T) {
1681+ clock := newManualClock(time.Unix(100, 0))
1682+ probe := newTestProbe(1)
1683+ logger, logs := testLogger()
1684+ store := &testStore{}
1685+ registry, err := NewRegistry(1, store, probe, clock, logger)
1686+ require.NoError(t, err)
1687+ reg := registration(t, "example.test", 1)
1688+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1689+
1690+ ctx, cancel := context.WithCancel(context.Background())
1691+ done := make(chan struct{})
1692+ go func() { defer close(done); registry.Run(ctx) }()
1693+
1694+ call := receiveCall(t, probe)
1695+ require.True(t, waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1696+ return host.Checking
1697+ }).Checking)
1698+
1699+ // A concurrent Run is refused instead of duplicating scheduling.
1700+ refused := make(chan struct{})
1701+ go func() { defer close(refused); registry.Run(context.Background()) }()
1702+ select {
1703+ case <-refused:
1704+ case <-time.After(time.Second):
1705+ t.Fatal("second Run did not return")
1706+ }
1707+ require.Contains(t, logs.String(), "Run refused")
1708+
1709+ cancel()
1710+ select {
1711+ case <-done:
1712+ case <-time.After(time.Second):
1713+ t.Fatal("registry did not stop after cancellation")
1714+ }
1715+ // Cleanup publishes the settled projection: nothing stays in flight.
1716+ require.False(t, waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1717+ return !host.Checking
1718+ }).Checking)
1719+
1720+ // The retired completion can never be applied once the run is over.
1721+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
1722+ settled := registry.Snapshot()
1723+ require.Equal(t, domain.RemoteAvailabilityUnknown, settled.Hosts[0].Availability)
1724+ require.Zero(t, settled.Hosts[0].ConsecutiveFailures)
1725+
1726+ // Persistence is flushed before shutdown and no write outlives it.
1727+ stored := waitStored(t, store, func(snapshot ports.BrokerSnapshot) bool {
1728+ return snapshot.Revision >= settled.Revision
1729+ })
1730+ require.Equal(t, settled.Revision, stored.Revision)
1731+ require.False(t, stored.Hosts[0].Checking)
1732+ writes := store.writeCount()
1733+ require.NoError(t, registry.SetHosts(nil))
1734+ require.Equal(t, writes, store.writeCount())
1735+ require.Contains(t, logs.String(), "refused after shutdown")
1736+
1737+ // A restart is refused rather than racing a stale scheduler.
1738+ restarted := make(chan struct{})
1739+ go func() { defer close(restarted); registry.Run(context.Background()) }()
1740+ select {
1741+ case <-restarted:
1742+ case <-time.After(time.Second):
1743+ t.Fatal("restarted Run did not return")
1744+ }
1745+ requireNoProbeWithin(t, probe, probeGrace)
1746+}
1747+
1748+func TestRegistryRevisionsIncreaseMonotonicallyWithinEpoch(t *testing.T) {
1749+ start := time.Unix(100, 0)
1750+ clock := newManualClock(start)
1751+ probe := newTestProbe(2)
1752+ registry := newTestRegistry(t, 42, nil, probe, clock)
1753+
1754+ var last ports.BrokerRevision
1755+ observe := func() {
1756+ t.Helper()
1757+ snapshot := registry.Snapshot()
1758+ require.Equal(t, ports.BrokerEpoch(42), snapshot.Epoch)
1759+ require.Greater(t, snapshot.Revision, last)
1760+ last = snapshot.Revision
1761+ }
1762+ observe()
1763+
1764+ first := registration(t, "a.test", 1)
1765+ second := registration(t, "b.test", 2)
1766+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{first}))
1767+ observe()
1768+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{first, second}))
1769+ observe()
1770+
1771+ startRegistry(t, registry)
1772+ for range 2 {
1773+ call := receiveCall(t, probe)
1774+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
1775+ waitHost(t, registry, call.registration.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1776+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
1777+ })
1778+ observe()
1779+ }
1780+}
@@ -0,0 +1,1780 @@
1+package broker
2+
3+import (
4+ "bytes"
5+ "context"
6+ "errors"
7+ "fmt"
8+ "log/slog"
9+ "sort"
10+ "strings"
11+ "sync"
12+ "testing"
13+ "time"
14+
15+ "github.com/bnema/vev/internal/domain"
16+ "github.com/bnema/vev/internal/ports"
17+ "github.com/bnema/vev/internal/protocol/catalogue"
18+ "github.com/stretchr/testify/require"
19+)
20+
21+// probeGrace bounds a negative scheduling assertion: a probe may not start
22+// inside this window, so the test observes "nothing happened" without
23+// sleeping on the behavior under test.
24+const probeGrace = 25 * time.Millisecond
25+
26+// manualClock is a deterministic ports.Clock/ports.Timer pair: time only moves
27+// when Advance is called, and timers with equal deadlines fire in creation
28+// order. Scheduling assertions are therefore exact instead of racing wall time.
29+type manualClock struct {
30+ mu sync.Mutex
31+ now time.Time
32+ order uint64
33+ timers map[*manualTimer]struct{}
34+}
35+
36+func newManualClock(start time.Time) *manualClock {
37+ return &manualClock{now: start, timers: make(map[*manualTimer]struct{})}
38+}
39+
40+func (c *manualClock) Now() time.Time {
41+ c.mu.Lock()
42+ defer c.mu.Unlock()
43+ return c.now
44+}
45+
46+func (c *manualClock) NewTimer(d time.Duration) ports.Timer {
47+ timer := &manualTimer{clock: c, ch: make(chan time.Time, 1)}
48+ c.mu.Lock()
49+ defer c.mu.Unlock()
50+ timer.arm(d)
51+ return timer
52+}
53+
54+// Advance moves time forward and fires every timer that came due.
55+func (c *manualClock) Advance(d time.Duration) {
56+ if d < 0 {
57+ panic("broker: manualClock.Advance requires a non-negative duration")
58+ }
59+ c.mu.Lock()
60+ defer c.mu.Unlock()
61+ c.now = c.now.Add(d)
62+ due := make([]*manualTimer, 0, len(c.timers))
63+ for timer := range c.timers {
64+ if !timer.when.After(c.now) {
65+ due = append(due, timer)
66+ }
67+ }
68+ sort.Slice(due, func(i, j int) bool {
69+ if due[i].when.Equal(due[j].when) {
70+ return due[i].order < due[j].order
71+ }
72+ return due[i].when.Before(due[j].when)
73+ })
74+ for _, timer := range due {
75+ timer.active = false
76+ delete(c.timers, timer)
77+ select {
78+ case timer.ch <- c.now:
79+ default:
80+ }
81+ }
82+}
83+
84+type manualTimer struct {
85+ clock *manualClock
86+ ch chan time.Time
87+ when time.Time
88+ order uint64
89+ active bool
90+}
91+
92+func (t *manualTimer) C() <-chan time.Time { return t.ch }
93+
94+func (t *manualTimer) Stop() bool {
95+ t.clock.mu.Lock()
96+ defer t.clock.mu.Unlock()
97+ if !t.active {
98+ return false
99+ }
100+ t.active = false
101+ delete(t.clock.timers, t)
102+ return true
103+}
104+
105+func (t *manualTimer) Reset(d time.Duration) bool {
106+ t.clock.mu.Lock()
107+ defer t.clock.mu.Unlock()
108+ wasActive := t.active
109+ if t.active {
110+ t.active = false
111+ delete(t.clock.timers, t)
112+ }
113+ select {
114+ case <-t.ch:
115+ default:
116+ }
117+ t.arm(d)
118+ return wasActive
119+}
120+
121+// arm schedules the timer relative to the clock's current time. The caller
122+// must hold clock.mu.
123+func (t *manualTimer) arm(d time.Duration) {
124+ t.when = t.clock.now.Add(d)
125+ t.order = t.clock.order
126+ t.clock.order++
127+ if d <= 0 {
128+ t.active = false
129+ select {
130+ case t.ch <- t.clock.now:
131+ default:
132+ }
133+ return
134+ }
135+ t.active = true
136+ t.clock.timers[t] = struct{}{}
137+}
138+
139+// identityJitter removes scheduling jitter so a test can assert exact
140+// deadlines; the production jitter is covered separately.
141+func identityJitter(d time.Duration, _ string, _ uint64) time.Duration { return d }
142+
143+// syncBuffer captures log output written from the registry's persistence
144+// goroutine as well as the test goroutine.
145+type syncBuffer struct {
146+ mu sync.Mutex
147+ b bytes.Buffer
148+}
149+
150+func (w *syncBuffer) Write(p []byte) (int, error) {
151+ w.mu.Lock()
152+ defer w.mu.Unlock()
153+ return w.b.Write(p)
154+}
155+
156+func (w *syncBuffer) String() string {
157+ w.mu.Lock()
158+ defer w.mu.Unlock()
159+ return w.b.String()
160+}
161+
162+func testLogger() (*slog.Logger, *syncBuffer) {
163+ logs := &syncBuffer{}
164+ return slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})), logs
165+}
166+
167+// testStore is a race-safe in-memory durable store whose writes signal a
168+// capacity-one channel so tests wait on a write instead of on wall time.
169+type testStore struct {
170+ mu sync.Mutex
171+ loaded ports.BrokerSnapshot
172+ loadErr error
173+ stored ports.BrokerSnapshot
174+ storeErr error
175+ writes int
176+ changes chan struct{}
177+}
178+
179+func (s *testStore) Load() (ports.BrokerSnapshot, error) {
180+ s.mu.Lock()
181+ defer s.mu.Unlock()
182+ if s.loadErr != nil {
183+ return ports.BrokerSnapshot{}, s.loadErr
184+ }
185+ return s.loaded.Clone(), nil
186+}
187+
188+func (s *testStore) Store(snapshot ports.BrokerSnapshot) error {
189+ s.mu.Lock()
190+ s.stored = snapshot.Clone()
191+ s.writes++
192+ err := s.storeErr
193+ s.mu.Unlock()
194+ s.signal()
195+ return err
196+}
197+
198+func (s *testStore) storedSnapshot() ports.BrokerSnapshot {
199+ s.mu.Lock()
200+ defer s.mu.Unlock()
201+ return s.stored.Clone()
202+}
203+
204+func (s *testStore) writeCount() int {
205+ s.mu.Lock()
206+ defer s.mu.Unlock()
207+ return s.writes
208+}
209+
210+func (s *testStore) setStoreErr(err error) {
211+ s.mu.Lock()
212+ defer s.mu.Unlock()
213+ s.storeErr = err
214+}
215+
216+func (s *testStore) signal() {
217+ s.mu.Lock()
218+ if s.changes == nil {
219+ s.changes = make(chan struct{}, 1)
220+ }
221+ changes := s.changes
222+ s.mu.Unlock()
223+ select {
224+ case changes <- struct{}{}:
225+ default:
226+ }
227+}
228+
229+func (s *testStore) changesChan() chan struct{} {
230+ s.mu.Lock()
231+ defer s.mu.Unlock()
232+ if s.changes == nil {
233+ s.changes = make(chan struct{}, 1)
234+ }
235+ return s.changes
236+}
237+
238+// blockingStore holds the first durable write inside Store so a test can prove
239+// that publication never waits behind persistence.
240+type blockingStore struct {
241+ mu sync.Mutex
242+ revisions []ports.BrokerRevision
243+ entered chan ports.BrokerRevision
244+ released chan struct{}
245+ first sync.Once
246+}
247+
248+func newBlockingStore() *blockingStore {
249+ return &blockingStore{entered: make(chan ports.BrokerRevision, 1), released: make(chan struct{})}
250+}
251+
252+func (s *blockingStore) Load() (ports.BrokerSnapshot, error) { return ports.BrokerSnapshot{}, nil }
253+
254+func (s *blockingStore) Store(snapshot ports.BrokerSnapshot) error {
255+ s.mu.Lock()
256+ s.revisions = append(s.revisions, snapshot.Revision)
257+ s.mu.Unlock()
258+ s.first.Do(func() {
259+ select {
260+ case s.entered <- snapshot.Revision:
261+ default:
262+ }
263+ <-s.released
264+ })
265+ return nil
266+}
267+
268+func (s *blockingStore) releaseFirst() { close(s.released) }
269+
270+func (s *blockingStore) writtenRevisions() []ports.BrokerRevision {
271+ s.mu.Lock()
272+ defer s.mu.Unlock()
273+ return append([]ports.BrokerRevision(nil), s.revisions...)
274+}
275+
276+// testProbe scripts one observation per received call: the test owns the
277+// completion, so a probe never depends on wall time.
278+type testProbe struct {
279+ calls chan probeCall
280+ canceled chan struct{}
281+}
282+
283+func newTestProbe(capacity int) *testProbe {
284+ return &testProbe{calls: make(chan probeCall, capacity), canceled: make(chan struct{}, 1)}
285+}
286+
287+type probeCall struct {
288+ registration domain.RemoteRegistration
289+ result chan probeAnswer
290+}
291+
292+type probeAnswer struct {
293+ snapshot ports.RemoteHostSnapshot
294+ err error
295+}
296+
297+func (p *testProbe) Probe(ctx context.Context, registration domain.RemoteRegistration) (ports.RemoteHostSnapshot, error) {
298+ call := probeCall{registration: registration, result: make(chan probeAnswer, 1)}
299+ select {
300+ case p.calls <- call:
301+ case <-ctx.Done():
302+ p.noteCanceled()
303+ return ports.RemoteHostSnapshot{}, ctx.Err()
304+ }
305+ select {
306+ case answer := <-call.result:
307+ return answer.snapshot, answer.err
308+ case <-ctx.Done():
309+ p.noteCanceled()
310+ return ports.RemoteHostSnapshot{}, ctx.Err()
311+ }
312+}
313+
314+func (p *testProbe) noteCanceled() {
315+ select {
316+ case p.canceled <- struct{}{}:
317+ default:
318+ }
319+}
320+
321+// churnProbe is a ctx-honoring probe used by the churn regression test. It
322+// tracks, per endpoint, how many probes were simultaneously live with an
323+// uncancelled context: an attempt that is retired without cancelling its
324+// context stays live and shows up as an overlap with its replacement.
325+type churnProbe struct {
326+ release chan struct{}
327+ released sync.Once
328+ startedSig chan struct{}
329+
330+ mu sync.Mutex
331+ contexts map[context.Context]string
332+ maxLive map[string]int
333+ started int
334+}
335+
336+func newChurnProbe() *churnProbe {
337+ return &churnProbe{
338+ release: make(chan struct{}),
339+ startedSig: make(chan struct{}, 256),
340+ contexts: make(map[context.Context]string),
341+ maxLive: make(map[string]int),
342+ }
343+}
344+
345+func (p *churnProbe) Probe(ctx context.Context, registration domain.RemoteRegistration) (ports.RemoteHostSnapshot, error) {
346+ p.mu.Lock()
347+ live := 0
348+ for other, endpoint := range p.contexts {
349+ if endpoint == registration.Endpoint && other.Err() == nil {
350+ live++
351+ }
352+ }
353+ // The attempt is registered even when its context is already cancelled (a
354+ // probe launched into a settling run), but only an uncancelled context is
355+ // counted live: an attempt retired without cancelling its context stays in
356+ // the map with Err()==nil and still overlaps its replacement.
357+ p.contexts[ctx] = registration.Endpoint
358+ if ctx.Err() == nil {
359+ live++
360+ }
361+ if live > p.maxLive[registration.Endpoint] {
362+ p.maxLive[registration.Endpoint] = live
363+ }
364+ p.started++
365+ p.mu.Unlock()
366+ select {
367+ case p.startedSig <- struct{}{}:
368+ default:
369+ }
370+ defer func() {
371+ p.mu.Lock()
372+ delete(p.contexts, ctx)
373+ p.mu.Unlock()
374+ }()
375+ select {
376+ case <-ctx.Done():
377+ case <-p.release:
378+ }
379+ return ports.RemoteHostSnapshot{}, ctx.Err()
380+}
381+
382+// openRelease lets every still-blocked probe return, so a failing test never
383+// leaks probe goroutines.
384+func (p *churnProbe) openRelease() { p.released.Do(func() { close(p.release) }) }
385+
386+func (p *churnProbe) startedCount() int {
387+ p.mu.Lock()
388+ defer p.mu.Unlock()
389+ return p.started
390+}
391+
392+func (p *churnProbe) maxLiveFor(endpoint string) int {
393+ p.mu.Lock()
394+ defer p.mu.Unlock()
395+ return p.maxLive[endpoint]
396+}
397+
398+func (p *churnProbe) observedEndpoints() []string {
399+ p.mu.Lock()
400+ defer p.mu.Unlock()
401+ endpoints := make([]string, 0, len(p.maxLive))
402+ for endpoint := range p.maxLive {
403+ endpoints = append(endpoints, endpoint)
404+ }
405+ sort.Strings(endpoints)
406+ return endpoints
407+}
408+
409+func (p *churnProbe) inflight() int {
410+ p.mu.Lock()
411+ defer p.mu.Unlock()
412+ return len(p.contexts)
413+}
414+
415+func registration(t *testing.T, endpoint string, marker byte) domain.RemoteRegistration {
416+ t.Helper()
417+ var incarnation [16]byte
418+ incarnation[0] = marker
419+ result, err := domain.NewRemoteRegistration(endpoint, incarnation)
420+ require.NoError(t, err)
421+ return result
422+}
423+
424+// hostSessions builds count minimal catalogue sessions for the projection
425+// session bound tests.
426+func hostSessions(count int) []catalogue.RemoteCatalogSession {
427+ sessions := make([]catalogue.RemoteCatalogSession, 0, count)
428+ for index := 0; index < count; index++ {
429+ sessions = append(sessions, catalogue.RemoteCatalogSession{
430+ Name: fmt.Sprintf("session-%d", index),
431+ State: catalogue.RemoteCatalogSessionUp,
432+ })
433+ }
434+ return sessions
435+}
436+
437+// newTestRegistry builds a registry with exact scheduling and no logger so a
438+// failure never depends on jitter.
439+func newTestRegistry(t *testing.T, epoch ports.BrokerEpoch, store ports.BrokerSnapshotStore, probe ports.BrokerHostProbe, clock *manualClock) *Registry {
440+ t.Helper()
441+ registry, err := NewRegistry(epoch, store, probe, clock, nil)
442+ require.NoError(t, err)
443+ registry.jitter = identityJitter
444+ return registry
445+}
446+
447+func startRegistry(t *testing.T, r *Registry) context.CancelFunc {
448+ t.Helper()
449+ ctx, cancel := context.WithCancel(context.Background())
450+ done := make(chan struct{})
451+ go func() { defer close(done); r.Run(ctx) }()
452+ t.Cleanup(func() { cancel(); <-done })
453+ return cancel
454+}
455+
456+func receiveCall(t *testing.T, probe *testProbe) probeCall {
457+ t.Helper()
458+ select {
459+ case call := <-probe.calls:
460+ return call
461+ case <-time.After(time.Second):
462+ t.Fatal("timed out waiting for probe")
463+ return probeCall{}
464+ }
465+}
466+
467+func requireNoProbeWithin(t *testing.T, probe *testProbe, window time.Duration) {
468+ t.Helper()
469+ timer := time.NewTimer(window)
470+ defer timer.Stop()
471+ select {
472+ case call := <-probe.calls:
473+ t.Fatalf("unexpected probe for %q", call.registration.Endpoint)
474+ case <-timer.C:
475+ }
476+}
477+
478+func waitSnapshot(t *testing.T, r *Registry, predicate func(ports.BrokerSnapshot) bool) ports.BrokerSnapshot {
479+ t.Helper()
480+ sub := r.Subscribe()
481+ defer sub.Close()
482+ deadline := time.After(time.Second)
483+ for {
484+ snapshot := r.Snapshot()
485+ if predicate(snapshot) {
486+ return snapshot
487+ }
488+ select {
489+ case <-sub.Changed():
490+ case <-deadline:
491+ t.Fatal("timed out waiting for snapshot")
492+ }
493+ }
494+}
495+
496+func waitHost(t *testing.T, r *Registry, endpoint string, predicate func(ports.RemoteHostSnapshot) bool) ports.RemoteHostSnapshot {
497+ t.Helper()
498+ snapshot := waitSnapshot(t, r, func(snapshot ports.BrokerSnapshot) bool {
499+ host, ok := snapshot.Find(endpoint)
500+ return ok && predicate(host)
501+ })
502+ host, _ := snapshot.Find(endpoint)
503+ return host
504+}
505+
506+// requireHostStableWithin asserts a stale completion has no delayed effect:
507+// the published host stays exactly as observed for a bounded window, and any
508+// intervening publication fails immediately.
509+func requireHostStableWithin(t *testing.T, r *Registry, endpoint string, want ports.RemoteHostSnapshot, window time.Duration) {
510+ t.Helper()
511+ sub := r.Subscribe()
512+ defer sub.Close()
513+ deadline := time.After(window)
514+ for {
515+ got, ok := r.Snapshot().Find(endpoint)
516+ require.True(t, ok)
517+ require.Equal(t, want, got)
518+ select {
519+ case <-sub.Changed():
520+ case <-deadline:
521+ return
522+ }
523+ }
524+}
525+
526+// waitStored waits for a durable write satisfying predicate.
527+func waitStored(t *testing.T, store *testStore, predicate func(ports.BrokerSnapshot) bool) ports.BrokerSnapshot {
528+ t.Helper()
529+ deadline := time.After(2 * time.Second)
530+ for {
531+ if snapshot := store.storedSnapshot(); predicate(snapshot) {
532+ return snapshot
533+ }
534+ select {
535+ case <-store.changesChan():
536+ case <-deadline:
537+ t.Fatal("timed out waiting for durable snapshot")
538+ return ports.BrokerSnapshot{}
539+ }
540+ }
541+}
542+
543+// waitWriterRevision waits until the serializer reports the revision durable.
544+func waitWriterRevision(t *testing.T, r *Registry, revision ports.BrokerRevision) {
545+ t.Helper()
546+ deadline := time.Now().Add(2 * time.Second)
547+ for {
548+ r.store.mu.Lock()
549+ written := r.store.written
550+ r.store.mu.Unlock()
551+ if written >= revision {
552+ return
553+ }
554+ if time.Now().After(deadline) {
555+ t.Fatalf("timed out waiting for durable revision %d", revision)
556+ }
557+ time.Sleep(time.Millisecond)
558+ }
559+}
560+
561+// waitForLog waits until captured log output contains want, so a persistence
562+// fault is proven observable instead of silently dropped.
563+func waitForLog(t *testing.T, logs *syncBuffer, want string) {
564+ t.Helper()
565+ deadline := time.Now().Add(2 * time.Second)
566+ for !strings.Contains(logs.String(), want) {
567+ if time.Now().After(deadline) {
568+ t.Fatalf("timed out waiting for log %q in %q", want, logs.String())
569+ }
570+ time.Sleep(time.Millisecond)
571+ }
572+}
573+
574+// attemptFor returns the attempt token currently in flight for an endpoint,
575+// or zero when no attempt is admitted.
576+func attemptFor(r *Registry, endpoint string) uint64 {
577+ r.mu.Lock()
578+ defer r.mu.Unlock()
579+ if attempt := r.inflight[endpoint]; attempt != nil {
580+ return attempt.token
581+ }
582+ return 0
583+}
584+
585+func TestRegistryRestoresPersistedSnapshotUnderFreshEpoch(t *testing.T) {
586+ reg := registration(t, "example.test", 1)
587+ sessions := []catalogue.RemoteCatalogSession{{
588+ Name: "work",
589+ State: catalogue.RemoteCatalogSessionUp,
590+ Tabs: []catalogue.RemoteCatalogTab{{ID: "tab-1", Name: "shell"}},
591+ }}
592+ stored := ports.BrokerSnapshot{
593+ Epoch: 7,
594+ Revision: 9,
595+ Hosts: []ports.RemoteHostSnapshot{{
596+ Endpoint: reg.Endpoint,
597+ Registration: reg,
598+ Availability: domain.RemoteAvailabilityReachable,
599+ Checking: true,
600+ LastSuccess: time.Unix(50, 0),
601+ NextDue: time.Unix(80, 0),
602+ InventoryKnown: true,
603+ Sessions: sessions,
604+ }},
605+ }
606+ store := &testStore{loaded: stored}
607+ registry, err := NewRegistry(11, store, newTestProbe(1), newManualClock(time.Unix(100, 0)), nil)
608+ require.NoError(t, err)
609+
610+ snapshot := registry.Snapshot()
611+ require.Equal(t, ports.BrokerEpoch(11), snapshot.Epoch)
612+ // A restored broker process starts a fresh revision series: revisions
613+ // never compare across epochs.
614+ require.Equal(t, ports.BrokerRevision(1), snapshot.Revision)
615+ require.Len(t, snapshot.Hosts, 1)
616+ require.Empty(t, snapshot.Removed)
617+
618+ host := snapshot.Hosts[0]
619+ require.True(t, host.Registration.Equal(reg))
620+ require.Equal(t, domain.RemoteAvailabilityReachable, host.Availability)
621+ require.Equal(t, time.Unix(80, 0), host.NextDue)
622+ require.Equal(t, sessions, host.Sessions)
623+ // In-flight observation state never survives a restart.
624+ require.False(t, host.Checking)
625+
626+ // Restoring is read-only: the registry republishes under its own epoch
627+ // without rewriting the durable store.
628+ require.Equal(t, ports.BrokerRevision(0), store.storedSnapshot().Revision)
629+ require.Zero(t, store.writeCount())
630+}
631+
632+func TestRegistryRestoreRejectsInvalidInputs(t *testing.T) {
633+ clock := newManualClock(time.Unix(100, 0))
634+ reg := registration(t, "example.test", 1)
635+
636+ _, err := NewRegistry(0, nil, newTestProbe(1), clock, nil)
637+ require.Error(t, err)
638+ _, err = NewRegistry(1, nil, nil, clock, nil)
639+ require.Error(t, err)
640+ _, err = NewRegistry(1, nil, newTestProbe(1), nil, nil)
641+ require.Error(t, err)
642+
643+ loadErr := errors.New("load failed")
644+ _, err = NewRegistry(1, &testStore{loadErr: loadErr}, newTestProbe(1), clock, nil)
645+ require.ErrorIs(t, err, loadErr)
646+
647+ _, err = NewRegistry(1, &testStore{loaded: ports.BrokerSnapshot{Epoch: 7}}, newTestProbe(1), clock, nil)
648+ require.Error(t, err)
649+
650+ // A durable record without an epoch is inconsistent, never silently empty.
651+ _, err = NewRegistry(1, &testStore{loaded: ports.BrokerSnapshot{Epoch: 0, Hosts: []ports.RemoteHostSnapshot{{Endpoint: reg.Endpoint, Registration: reg}}}}, newTestProbe(1), clock, nil)
652+ require.Error(t, err)
653+ _, err = NewRegistry(1, &testStore{loaded: ports.BrokerSnapshot{Epoch: 0, Removed: []ports.BrokerHostTombstone{{Endpoint: reg.Endpoint, Registration: reg, RetiredRevision: 3}}}}, newTestProbe(1), clock, nil)
654+ require.Error(t, err)
655+ _, err = NewRegistry(1, &testStore{loaded: ports.BrokerSnapshot{Epoch: 0, Revision: 4}}, newTestProbe(1), clock, nil)
656+ require.Error(t, err)
657+
658+ empty, err := NewRegistry(1, &testStore{}, newTestProbe(1), clock, nil)
659+ require.NoError(t, err)
660+ require.Equal(t, ports.BrokerRevision(1), empty.Snapshot().Revision)
661+ require.Empty(t, empty.Snapshot().Hosts)
662+}
663+
664+func TestRegistryRestoreRejectsPersistedSnapshotFromItsOwnEpoch(t *testing.T) {
665+ clock := newManualClock(time.Unix(100, 0))
666+ reg := registration(t, "example.test", 1)
667+
668+ // A fresh broker process samples a fresh epoch and restarts its revision
669+ // series at one, so a durable record from its own epoch can never be
670+ // adopted: the restored series would not be monotonic and a stale record
671+ // would be indistinguishable from current authority.
672+ currentEpoch := ports.BrokerSnapshot{
673+ Epoch: 7,
674+ Revision: 9,
675+ Hosts: []ports.RemoteHostSnapshot{{
676+ Endpoint: reg.Endpoint,
677+ Registration: reg,
678+ Availability: domain.RemoteAvailabilityReachable,
679+ }},
680+ }
681+ _, err := NewRegistry(7, &testStore{loaded: currentEpoch}, newTestProbe(1), clock, nil)
682+ require.Error(t, err)
683+ require.Contains(t, err.Error(), "matches the new registry epoch")
684+
685+ // The same rejection holds for a tombstone-only and for an empty-hosts
686+ // record from the current epoch.
687+ _, err = NewRegistry(7, &testStore{loaded: ports.BrokerSnapshot{Epoch: 7, Revision: 1, Removed: []ports.BrokerHostTombstone{{
688+ Endpoint: reg.Endpoint, Registration: reg, RetiredRevision: 1,
689+ }}}}, newTestProbe(1), clock, nil)
690+ require.Error(t, err)
691+ _, err = NewRegistry(7, &testStore{loaded: ports.BrokerSnapshot{Epoch: 7, Revision: 1}}, newTestProbe(1), clock, nil)
692+ require.Error(t, err)
693+
694+ // A different epoch is still adopted normally.
695+ registry, err := NewRegistry(8, &testStore{loaded: currentEpoch}, newTestProbe(1), clock, nil)
696+ require.NoError(t, err)
697+ require.Len(t, registry.Snapshot().Hosts, 1)
698+}
699+
700+func TestRegistrySetHostsRejectsInvalidRegistrations(t *testing.T) {
701+ registry := newTestRegistry(t, 1, nil, newTestProbe(1), newManualClock(time.Unix(100, 0)))
702+
703+ reg := registration(t, "example.test", 1)
704+ require.Error(t, registry.SetHosts([]domain.RemoteRegistration{reg, reg}))
705+ require.Error(t, registry.SetHosts([]domain.RemoteRegistration{{Endpoint: reg.Endpoint}}))
706+ require.Empty(t, registry.Snapshot().Hosts)
707+}
708+
709+func TestRegistrySetHostsRejectsHostLimitWithoutMutation(t *testing.T) {
710+ registry := newTestRegistry(t, 1, nil, newTestProbe(1), newManualClock(time.Unix(100, 0)))
711+ reg := registration(t, "example.test", 1)
712+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
713+
714+ oversized := make([]domain.RemoteRegistration, 0, ports.BrokerMaxHosts+1)
715+ for index := 0; index <= ports.BrokerMaxHosts; index++ {
716+ oversized = append(oversized, registration(t, fmt.Sprintf("host-%d.test", index), byte(index+1)))
717+ }
718+ before := registry.Snapshot()
719+ require.Error(t, registry.SetHosts(oversized))
720+ // The rejected call mutates nothing: same membership, same revision.
721+ require.Equal(t, before, registry.Snapshot())
722+
723+ // The boundary itself is accepted.
724+ require.NoError(t, registry.SetHosts(oversized[:ports.BrokerMaxHosts]))
725+ require.Len(t, registry.Snapshot().Hosts, ports.BrokerMaxHosts)
726+}
727+
728+func TestRegistryUnchangedSetHostsIsNoOp(t *testing.T) {
729+ clock := newManualClock(time.Unix(100, 0))
730+ probe := newTestProbe(2)
731+ registry := newTestRegistry(t, 1, nil, probe, clock)
732+ first := registration(t, "example.test", 1)
733+ second := registration(t, "other.test", 2)
734+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{first, second}))
735+ startRegistry(t, registry)
736+
737+ // Every configured host is observed exactly once.
738+ calls := make(map[string]probeCall, 2)
739+ for range 2 {
740+ call := receiveCall(t, probe)
741+ calls[call.registration.Endpoint] = call
742+ }
743+ require.Len(t, calls, 2)
744+ before := registry.Snapshot()
745+ for _, host := range before.Hosts {
746+ require.True(t, host.Checking)
747+ }
748+
749+ // Identical membership, including in a different order, never republishes
750+ // and never restarts an in-flight observation.
751+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{second, first}))
752+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{first, second}))
753+ require.Equal(t, before, registry.Snapshot())
754+ requireNoProbeWithin(t, probe, probeGrace)
755+
756+ // The in-flight observations still complete normally.
757+ for endpoint, call := range calls {
758+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
759+ waitHost(t, registry, endpoint, func(host ports.RemoteHostSnapshot) bool {
760+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
761+ })
762+ }
763+}
764+
765+func TestRegistrySnapshotIsImmutable(t *testing.T) {
766+ clock := newManualClock(time.Unix(100, 0))
767+ probe := newTestProbe(1)
768+ registry := newTestRegistry(t, 1, nil, probe, clock)
769+ reg := registration(t, "example.test", 1)
770+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
771+ startRegistry(t, registry)
772+
773+ call := receiveCall(t, probe)
774+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
775+ Availability: domain.RemoteAvailabilityReachable,
776+ Sessions: []catalogue.RemoteCatalogSession{{
777+ Name: "work",
778+ State: catalogue.RemoteCatalogSessionUp,
779+ Tabs: []catalogue.RemoteCatalogTab{{ID: "tab-1", Name: "shell"}},
780+ }},
781+ }}
782+ observed := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
783+ return !host.Checking && len(host.Sessions) == 1
784+ })
785+ require.Equal(t, "work", observed.Sessions[0].Name)
786+
787+ // Mutating a returned projection never reaches the registry.
788+ observed.Endpoint = "mutated"
789+ observed.Sessions[0].Name = "mutated"
790+ observed.Sessions[0].Tabs[0].Name = "mutated"
791+ current := registry.Snapshot()
792+ require.Equal(t, reg.Endpoint, current.Hosts[0].Endpoint)
793+ require.Equal(t, "work", current.Hosts[0].Sessions[0].Name)
794+ require.Equal(t, "shell", current.Hosts[0].Sessions[0].Tabs[0].Name)
795+
796+ // A later publication never mutates a snapshot already handed out.
797+ earlier := registry.Snapshot()
798+ registry.RequestProbe(reg.Endpoint)
799+ next := receiveCall(t, probe)
800+ next.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
801+ Availability: domain.RemoteAvailabilityReachable,
802+ Sessions: []catalogue.RemoteCatalogSession{{Name: "other", State: catalogue.RemoteCatalogSessionUp}},
803+ }}
804+ waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
805+ return !host.Checking && len(host.Sessions) == 1 && host.Sessions[0].Name == "other"
806+ })
807+ require.Equal(t, "work", earlier.Hosts[0].Sessions[0].Name)
808+}
809+
810+func TestRegistrySchedulesFreshObservationAfterSuccess(t *testing.T) {
811+ start := time.Unix(100, 0)
812+ clock := newManualClock(start)
813+ probe := newTestProbe(1)
814+ registry := newTestRegistry(t, 1, nil, probe, clock)
815+ reg := registration(t, "example.test", 1)
816+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
817+ startRegistry(t, registry)
818+
819+ call := receiveCall(t, probe)
820+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
821+
822+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
823+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
824+ })
825+ require.Equal(t, start, host.LastAttempt)
826+ require.Equal(t, start, host.LastSuccess)
827+ require.Equal(t, start.Add(defaultFreshFor), host.NextDue)
828+ require.Zero(t, host.ConsecutiveFailures)
829+ require.Zero(t, host.FailureEpisode)
830+
831+ // A fresh host is not observed again before its scheduled deadline.
832+ clock.Advance(defaultFreshFor - time.Nanosecond)
833+ requireNoProbeWithin(t, probe, probeGrace)
834+ clock.Advance(time.Nanosecond)
835+ next := receiveCall(t, probe)
836+ require.True(t, next.registration.Equal(reg))
837+}
838+
839+func TestRegistryDemandOverridesFreshSchedule(t *testing.T) {
840+ start := time.Unix(100, 0)
841+ clock := newManualClock(start)
842+ probe := newTestProbe(1)
843+ registry := newTestRegistry(t, 1, nil, probe, clock)
844+ reg := registration(t, "example.test", 1)
845+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
846+ startRegistry(t, registry)
847+
848+ call := receiveCall(t, probe)
849+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
850+ waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
851+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
852+ })
853+
854+ // Explicit demand is admitted immediately, before the fresh deadline.
855+ registry.RequestProbe(reg.Endpoint)
856+ forced := receiveCall(t, probe)
857+ require.True(t, forced.registration.Equal(reg))
858+ require.Equal(t, start, clock.Now())
859+}
860+
861+func TestRegistryCoalescesConcurrentDemand(t *testing.T) {
862+ start := time.Unix(100, 0)
863+ clock := newManualClock(start)
864+ probe := newTestProbe(2)
865+ registry := newTestRegistry(t, 1, nil, probe, clock)
866+ reg := registration(t, "example.test", 1)
867+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
868+ startRegistry(t, registry)
869+
870+ call := receiveCall(t, probe)
871+ for range 10 {
872+ registry.RequestProbe(reg.Endpoint)
873+ }
874+ select {
875+ case duplicate := <-probe.calls:
876+ t.Fatalf("concurrent demand started a duplicate probe for %q", duplicate.registration.Endpoint)
877+ default:
878+ }
879+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
880+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
881+ return host.Availability == domain.RemoteAvailabilityReachable && host.LastSuccess.Equal(start)
882+ })
883+ require.Equal(t, start.Add(defaultFreshFor), host.NextDue)
884+
885+ // The single coalesced follow-up is admitted once the first completes.
886+ followUp := receiveCall(t, probe)
887+ require.True(t, followUp.registration.Equal(reg))
888+ followUp.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
889+ waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
890+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
891+ })
892+}
893+
894+func TestRegistryIgnoresDemandForUnknownEndpoint(t *testing.T) {
895+ clock := newManualClock(time.Unix(100, 0))
896+ probe := newTestProbe(1)
897+ registry := newTestRegistry(t, 1, nil, probe, clock)
898+ reg := registration(t, "example.test", 1)
899+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
900+ registry.RequestProbe("unknown.test")
901+ startRegistry(t, registry)
902+
903+ call := receiveCall(t, probe)
904+ require.True(t, call.registration.Equal(reg))
905+ // No probe is ever scheduled for an endpoint outside configured membership.
906+ requireNoProbeWithin(t, probe, probeGrace)
907+}
908+
909+func TestRegistryPublishesReachabilityAndPersists(t *testing.T) {
910+ start := time.Unix(100, 0)
911+ clock := newManualClock(start)
912+ probe := newTestProbe(1)
913+ store := &testStore{}
914+ registry := newTestRegistry(t, 3, store, probe, clock)
915+ reg := registration(t, "example.test", 1)
916+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
917+ startRegistry(t, registry)
918+
919+ call := receiveCall(t, probe)
920+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
921+ Availability: domain.RemoteAvailabilityReachable,
922+ InventoryKnown: true,
923+ Sessions: []catalogue.RemoteCatalogSession{{Name: "work", State: catalogue.RemoteCatalogSessionUp}},
924+ }}
925+ snapshot := waitSnapshot(t, registry, func(snapshot ports.BrokerSnapshot) bool {
926+ host, ok := snapshot.Find(reg.Endpoint)
927+ return ok && !host.Checking && host.Availability == domain.RemoteAvailabilityReachable && len(host.Sessions) == 1
928+ })
929+
930+ stored := waitStored(t, store, func(stored ports.BrokerSnapshot) bool {
931+ return stored.Revision >= snapshot.Revision && len(stored.Hosts) == 1
932+ })
933+ require.Equal(t, ports.BrokerEpoch(3), stored.Epoch)
934+ require.Len(t, stored.Hosts, 1)
935+ require.Equal(t, "work", stored.Hosts[0].Sessions[0].Name)
936+ require.Equal(t, reg.Endpoint, stored.Hosts[0].Endpoint)
937+ require.False(t, stored.Hosts[0].Checking)
938+}
939+
940+func TestRegistryNonReachableOutcomesUseTypedFailureBackoff(t *testing.T) {
941+ dialErr := errors.New("dial failed")
942+ tests := []struct {
943+ name string
944+ answer probeAnswer
945+ wantAvailability domain.RemoteAvailability
946+ wantKind domain.RemoteFailureKind
947+ wantErr error
948+ }{
949+ {
950+ name: "unreachable without error",
951+ answer: probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityUnreachable}},
952+ wantAvailability: domain.RemoteAvailabilityUnreachable,
953+ wantKind: domain.RemoteFailureTransport,
954+ },
955+ {
956+ name: "unknown outcome is not reachability",
957+ answer: probeAnswer{snapshot: ports.RemoteHostSnapshot{}},
958+ wantAvailability: domain.RemoteAvailabilityUnreachable,
959+ wantKind: domain.RemoteFailureTransport,
960+ },
961+ {
962+ name: "incompatible",
963+ answer: probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityIncompatible}},
964+ wantAvailability: domain.RemoteAvailabilityIncompatible,
965+ wantKind: domain.RemoteFailureIncompatible,
966+ },
967+ {
968+ name: "authentication",
969+ answer: probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityAuthFailed}},
970+ wantAvailability: domain.RemoteAvailabilityAuthFailed,
971+ wantKind: domain.RemoteFailureAuthentication,
972+ },
973+ {
974+ name: "invalid response",
975+ answer: probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityInvalidResponse}},
976+ wantAvailability: domain.RemoteAvailabilityInvalidResponse,
977+ wantKind: domain.RemoteFailureInvalidResponse,
978+ },
979+ {
980+ name: "transport error",
981+ answer: probeAnswer{err: dialErr},
982+ wantAvailability: domain.RemoteAvailabilityUnreachable,
983+ wantKind: domain.RemoteFailureTransport,
984+ wantErr: dialErr,
985+ },
986+ {
987+ name: "deadline exceeded",
988+ answer: probeAnswer{err: context.DeadlineExceeded},
989+ wantAvailability: domain.RemoteAvailabilityUnreachable,
990+ wantKind: domain.RemoteFailureTimeout,
991+ wantErr: context.DeadlineExceeded,
992+ },
993+ }
994+ for _, tt := range tests {
995+ t.Run(tt.name, func(t *testing.T) {
996+ start := time.Unix(100, 0)
997+ clock := newManualClock(start)
998+ probe := newTestProbe(1)
999+ registry := newTestRegistry(t, 1, nil, probe, clock)
1000+ reg := registration(t, "example.test", 1)
1001+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1002+ startRegistry(t, registry)
1003+
1004+ call := receiveCall(t, probe)
1005+ call.result <- tt.answer
1006+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1007+ return !host.Checking && host.ConsecutiveFailures == 1
1008+ })
1009+ require.Equal(t, tt.wantAvailability, host.Availability)
1010+ require.Equal(t, tt.wantKind, host.LastFailure.Kind)
1011+ require.Equal(t, uint(1), host.ConsecutiveFailures)
1012+ require.Equal(t, uint64(1), host.FailureEpisode)
1013+ // No observation succeeded yet, so there is no last success.
1014+ require.True(t, host.LastSuccess.IsZero())
1015+ // Every non-reachable outcome retries on the capped cadence.
1016+ require.Equal(t, start.Add(defaultRetryBase), host.NextDue)
1017+ if tt.wantErr != nil {
1018+ require.ErrorIs(t, host.LastFailure, tt.wantErr)
1019+ } else {
1020+ require.Nil(t, host.LastFailure.Err)
1021+ }
1022+ })
1023+ }
1024+}
1025+
1026+func TestRegistryClassifiesOversizedProbeProjectionAsInvalidResponse(t *testing.T) {
1027+ start := time.Unix(100, 0)
1028+ clock := newManualClock(start)
1029+ probe := newTestProbe(1)
1030+ registry := newTestRegistry(t, 1, nil, probe, clock)
1031+ reg := registration(t, "example.test", 1)
1032+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1033+ startRegistry(t, registry)
1034+ sub := registry.Subscribe()
1035+ defer sub.Close()
1036+
1037+ // A reachable observation establishes the last-known inventory.
1038+ call := receiveCall(t, probe)
1039+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
1040+ Availability: domain.RemoteAvailabilityReachable,
1041+ Sessions: []catalogue.RemoteCatalogSession{{Name: "work", State: catalogue.RemoteCatalogSessionUp}},
1042+ }}
1043+ waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1044+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
1045+ })
1046+
1047+ // A projection carrying one session more than the publication bound is a
1048+ // malformed response: it is classified as an invalid response and never
1049+ // reaches publication, because such a snapshot fails Validate. The loop
1050+ // checks every observed publication, so an invalid snapshot can never pass
1051+ // unnoticed.
1052+ clock.Advance(defaultFreshFor)
1053+ call = receiveCall(t, probe)
1054+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
1055+ Availability: domain.RemoteAvailabilityReachable,
1056+ Sessions: hostSessions(ports.BrokerMaxSessionsPerHost + 1),
1057+ }}
1058+ deadline := time.After(time.Second)
1059+ var host ports.RemoteHostSnapshot
1060+ for {
1061+ snapshot := registry.Snapshot()
1062+ require.NoError(t, snapshot.Validate())
1063+ current, ok := snapshot.Find(reg.Endpoint)
1064+ if ok && !current.Checking && current.ConsecutiveFailures == 1 {
1065+ host = current
1066+ break
1067+ }
1068+ select {
1069+ case <-sub.Changed():
1070+ case <-deadline:
1071+ t.Fatal("timed out waiting for the invalid-response failure")
1072+ }
1073+ }
1074+ require.Equal(t, domain.RemoteAvailabilityInvalidResponse, host.Availability)
1075+ require.Equal(t, domain.RemoteFailureInvalidResponse, host.LastFailure.Kind)
1076+ // The last-known inventory is retained: a malformed projection never
1077+ // replaces it.
1078+ require.Len(t, host.Sessions, 1)
1079+ require.Equal(t, "work", host.Sessions[0].Name)
1080+ require.Equal(t, clock.Now().Add(defaultRetryBase), host.NextDue)
1081+}
1082+
1083+func TestRegistryAcceptsProbeProjectionAtSessionBound(t *testing.T) {
1084+ start := time.Unix(100, 0)
1085+ clock := newManualClock(start)
1086+ probe := newTestProbe(1)
1087+ registry := newTestRegistry(t, 1, nil, probe, clock)
1088+ reg := registration(t, "example.test", 1)
1089+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1090+ startRegistry(t, registry)
1091+
1092+ // The bound itself is valid: exactly BrokerMaxSessionsPerHost sessions are
1093+ // accepted and published, and the resulting snapshot validates.
1094+ call := receiveCall(t, probe)
1095+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
1096+ Availability: domain.RemoteAvailabilityReachable,
1097+ Sessions: hostSessions(ports.BrokerMaxSessionsPerHost),
1098+ }}
1099+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1100+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
1101+ })
1102+ require.Len(t, host.Sessions, ports.BrokerMaxSessionsPerHost)
1103+ require.Zero(t, host.ConsecutiveFailures)
1104+ snapshot := registry.Snapshot()
1105+ require.NoError(t, snapshot.Validate())
1106+ require.Len(t, snapshot.Hosts[0].Sessions, ports.BrokerMaxSessionsPerHost)
1107+}
1108+
1109+func TestRegistryPreservesLastSuccessAndOwnsFailureEpisode(t *testing.T) {
1110+ start := time.Unix(100, 0)
1111+ clock := newManualClock(start)
1112+ probe := newTestProbe(1)
1113+ registry := newTestRegistry(t, 1, nil, probe, clock)
1114+ reg := registration(t, "example.test", 1)
1115+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1116+ startRegistry(t, registry)
1117+
1118+ // A successful observation owns the success time and opens no episode.
1119+ call := receiveCall(t, probe)
1120+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
1121+ Availability: domain.RemoteAvailabilityReachable,
1122+ Sessions: []catalogue.RemoteCatalogSession{{Name: "work", State: catalogue.RemoteCatalogSessionUp}},
1123+ }}
1124+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1125+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
1126+ })
1127+ require.Equal(t, start, host.LastSuccess)
1128+ require.Zero(t, host.FailureEpisode)
1129+
1130+ // A failure preserves the last success and opens exactly one episode.
1131+ clock.Advance(defaultFreshFor)
1132+ call = receiveCall(t, probe)
1133+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{
1134+ Availability: domain.RemoteAvailabilityIncompatible,
1135+ Sessions: []catalogue.RemoteCatalogSession{{Name: "stale", State: catalogue.RemoteCatalogSessionUp}},
1136+ }}
1137+ host = waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1138+ return !host.Checking && host.ConsecutiveFailures == 1
1139+ })
1140+ require.Equal(t, start, host.LastSuccess)
1141+ require.Equal(t, uint64(1), host.FailureEpisode)
1142+ // The last-known inventory is retained: a failed observation never
1143+ // replaces it.
1144+ require.Equal(t, "work", host.Sessions[0].Name)
1145+
1146+ // Further failures in the same streak preserve the episode identity so
1147+ // notice policy does not re-notify.
1148+ clock.Advance(defaultRetryBase)
1149+ call = receiveCall(t, probe)
1150+ call.result <- probeAnswer{err: errors.New("offline")}
1151+ host = waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1152+ return !host.Checking && host.ConsecutiveFailures == 2
1153+ })
1154+ require.Equal(t, start, host.LastSuccess)
1155+ require.Equal(t, uint64(1), host.FailureEpisode)
1156+
1157+ // Recovery clears the streak, keeps the episode, and advances the success.
1158+ clock.Advance(2 * defaultRetryBase)
1159+ recovered := clock.Now()
1160+ call = receiveCall(t, probe)
1161+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
1162+ host = waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1163+ return !host.Checking && host.ConsecutiveFailures == 0
1164+ })
1165+ require.Equal(t, recovered, host.LastSuccess)
1166+ require.Equal(t, uint64(1), host.FailureEpisode)
1167+ require.Equal(t, domain.RemoteFailureNone, host.LastFailure.Kind)
1168+ require.Nil(t, host.LastFailure.Err)
1169+}
1170+
1171+func TestRegistryBackoffIsExponentialAndCapped(t *testing.T) {
1172+ start := time.Unix(100, 0)
1173+ clock := newManualClock(start)
1174+ probe := newTestProbe(1)
1175+ registry := newTestRegistry(t, 1, nil, probe, clock)
1176+ reg := registration(t, "example.test", 1)
1177+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1178+ startRegistry(t, registry)
1179+
1180+ probeErr := errors.New("offline")
1181+ // The retry cadence matches the legacy remotes monitor: a 5s base doubling
1182+ // to a 60s cap.
1183+ want := []time.Duration{
1184+ 5 * time.Second,
1185+ 10 * time.Second,
1186+ 20 * time.Second,
1187+ 40 * time.Second,
1188+ time.Minute, // 80s capped at defaultRetryLimit
1189+ time.Minute,
1190+ time.Minute,
1191+ }
1192+ for index, delay := range want {
1193+ call := receiveCall(t, probe)
1194+ at := clock.Now()
1195+ call.result <- probeAnswer{err: probeErr}
1196+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1197+ return !host.Checking && host.ConsecutiveFailures == uint(index+1)
1198+ })
1199+ require.Equal(t, domain.RemoteAvailabilityUnreachable, host.Availability)
1200+ require.Equal(t, domain.RemoteFailureTransport, host.LastFailure.Kind)
1201+ require.ErrorIs(t, host.LastFailure, probeErr)
1202+ require.Equal(t, at.Add(delay), host.NextDue)
1203+
1204+ if index == len(want)-1 {
1205+ break
1206+ }
1207+ // The retry is not admitted early: only the backoff deadline wakes it.
1208+ clock.Advance(delay - time.Nanosecond)
1209+ requireNoProbeWithin(t, probe, probeGrace)
1210+ clock.Advance(time.Nanosecond)
1211+ }
1212+}
1213+
1214+func TestRegistryJitterAppliesToRetryAndHealthyRefresh(t *testing.T) {
1215+ start := time.Unix(100, 0)
1216+ clock := newManualClock(start)
1217+ probe := newTestProbe(1)
1218+ registry := newTestRegistry(t, 1, nil, probe, clock)
1219+ registry.jitter = func(base time.Duration, _ string, _ uint64) time.Duration { return base / 2 }
1220+ reg := registration(t, "example.test", 1)
1221+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1222+ startRegistry(t, registry)
1223+
1224+ call := receiveCall(t, probe)
1225+ call.result <- probeAnswer{err: errors.New("offline")}
1226+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1227+ return !host.Checking && host.Availability == domain.RemoteAvailabilityUnreachable
1228+ })
1229+ require.Equal(t, start.Add(defaultRetryBase/2), host.NextDue)
1230+
1231+ clock.Advance(defaultRetryBase / 2)
1232+ next := receiveCall(t, probe)
1233+ successAt := clock.Now()
1234+ next.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
1235+ host = waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1236+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
1237+ })
1238+ // The healthy refresh is jittered too: a fleet of hosts never converges
1239+ // on one instant.
1240+ require.Equal(t, successAt.Add(defaultFreshFor/2), host.NextDue)
1241+}
1242+
1243+func TestRegistryJitterIsDeterministicAndBounded(t *testing.T) {
1244+ bases := []time.Duration{defaultRetryBase, defaultFreshFor, defaultRetryLimit}
1245+ endpoints := []string{"example.test", "other.test", "user@host:2222"}
1246+ for _, base := range bases {
1247+ minimum, maximum := base*9/10, base*11/10
1248+ for _, endpoint := range endpoints {
1249+ spread := make(map[time.Duration]struct{})
1250+ for attempt := uint64(1); attempt <= 64; attempt++ {
1251+ got := jittered(base, endpoint, attempt)
1252+ require.GreaterOrEqual(t, got, minimum)
1253+ require.LessOrEqual(t, got, maximum)
1254+ // Identical endpoint and attempt always yield the identical
1255+ // duration: no wall clock and no shared PRNG participate.
1256+ require.Equal(t, got, jittered(base, endpoint, attempt))
1257+ spread[got] = struct{}{}
1258+ }
1259+ require.Greater(t, len(spread), 1, "jitter must vary across attempts")
1260+ }
1261+ }
1262+ // A non-positive base has no spread to apply.
1263+ require.Zero(t, jittered(0, "example.test", 1))
1264+}
1265+
1266+func TestRegistryFencesStaleCompletionAcrossRegistrationChanges(t *testing.T) {
1267+ replacementAnswer := func(name string) probeAnswer {
1268+ return probeAnswer{snapshot: ports.RemoteHostSnapshot{
1269+ Availability: domain.RemoteAvailabilityReachable,
1270+ Sessions: []catalogue.RemoteCatalogSession{{Name: name, State: catalogue.RemoteCatalogSessionUp}},
1271+ }}
1272+ }
1273+ assertFenced := func(t *testing.T, current, replacement domain.RemoteRegistration) {
1274+ t.Helper()
1275+ clock := newManualClock(time.Unix(100, 0))
1276+ probe := newTestProbe(2)
1277+ registry := newTestRegistry(t, 1, nil, probe, clock)
1278+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{current}))
1279+ startRegistry(t, registry)
1280+ currentCall := receiveCall(t, probe)
1281+ require.True(t, currentCall.registration.Equal(current))
1282+
1283+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{replacement}))
1284+ replacementCall := receiveCall(t, probe)
1285+ require.True(t, replacementCall.registration.Equal(replacement))
1286+
1287+ // The replacement completes and becomes authoritative...
1288+ replacementCall.result <- replacementAnswer("replacement")
1289+ applied := waitHost(t, registry, replacement.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1290+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable && len(host.Sessions) == 1
1291+ })
1292+ require.True(t, applied.Registration.Equal(replacement))
1293+ require.Equal(t, "replacement", applied.Sessions[0].Name)
1294+
1295+ // ...and the completion bound to the retired registration, arriving
1296+ // afterwards, never touches it.
1297+ currentCall.result <- replacementAnswer("retired")
1298+ requireHostStableWithin(t, registry, replacement.Endpoint, applied, probeGrace)
1299+
1300+ // The retired completion also never holds the replacement's slot: a
1301+ // further observation is still admitted and still applies.
1302+ registry.RequestProbe(replacement.Endpoint)
1303+ next := receiveCall(t, probe)
1304+ require.True(t, next.registration.Equal(replacement))
1305+ next.result <- replacementAnswer("next")
1306+ waitHost(t, registry, replacement.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1307+ return !host.Checking && len(host.Sessions) == 1 && host.Sessions[0].Name == "next"
1308+ })
1309+ }
1310+
1311+ t.Run("incarnation", func(t *testing.T) {
1312+ assertFenced(t, registration(t, "example.test", 1), registration(t, "example.test", 2))
1313+ })
1314+ t.Run("generation", func(t *testing.T) {
1315+ current := registration(t, "example.test", 1)
1316+ replacement := current
1317+ replacement.Generation++
1318+ assertFenced(t, current, replacement)
1319+ })
1320+}
1321+
1322+func TestRegistryFencesRetiredAttemptsAcrossRemovalAndIdenticalReAdd(t *testing.T) {
1323+ clock := newManualClock(time.Unix(100, 0))
1324+ probe := newTestProbe(2)
1325+ registry := newTestRegistry(t, 1, nil, probe, clock)
1326+ reg := registration(t, "example.test", 1)
1327+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1328+
1329+ ctx, cancel := context.WithCancel(context.Background())
1330+ defer cancel()
1331+ results := make(chan probeResult, 1)
1332+ dispatch := func() { registry.dispatch(ctx, clock.Now(), results) }
1333+
1334+ dispatch()
1335+ retiredCall := receiveCall(t, probe)
1336+ require.True(t, retiredCall.registration.Equal(reg))
1337+ retired := attemptFor(registry, reg.Endpoint)
1338+ require.NotZero(t, retired)
1339+
1340+ // Removal retires a bounded tombstone and drops the in-flight token.
1341+ require.NoError(t, registry.SetHosts(nil))
1342+ removed := registry.Snapshot()
1343+ require.Empty(t, removed.Hosts)
1344+ require.Len(t, removed.Removed, 1)
1345+ require.Equal(t, removed.Revision, removed.Removed[0].RetiredRevision)
1346+ require.True(t, removed.Removed[0].Fences(reg))
1347+ require.NoError(t, removed.Validate())
1348+
1349+ // The retired attempt can never revive a removed endpoint.
1350+ registry.apply(probeResult{endpoint: reg.Endpoint, registration: reg, attempt: retired, snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}, at: clock.Now()})
1351+ require.Empty(t, registry.Snapshot().Hosts)
1352+ require.Len(t, registry.Snapshot().Removed, 1)
1353+
1354+ // Re-adding the identical registration supersedes the tombstone, so only
1355+ // the attempt token can fence the retired completion.
1356+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1357+ require.Empty(t, registry.Snapshot().Removed)
1358+ dispatch()
1359+ freshCall := receiveCall(t, probe)
1360+ require.True(t, freshCall.registration.Equal(reg))
1361+ fresh := attemptFor(registry, reg.Endpoint)
1362+ require.Greater(t, fresh, retired)
1363+
1364+ registry.apply(probeResult{endpoint: reg.Endpoint, registration: reg, attempt: retired, snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}, at: clock.Now()})
1365+ stale, ok := registry.Snapshot().Find(reg.Endpoint)
1366+ require.True(t, ok)
1367+ require.True(t, stale.Checking)
1368+ require.Equal(t, domain.RemoteAvailabilityUnknown, stale.Availability)
1369+
1370+ registry.apply(probeResult{endpoint: reg.Endpoint, registration: reg, attempt: fresh, snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}, at: clock.Now()})
1371+ applied, ok := registry.Snapshot().Find(reg.Endpoint)
1372+ require.True(t, ok)
1373+ require.False(t, applied.Checking)
1374+ require.Equal(t, domain.RemoteAvailabilityReachable, applied.Availability)
1375+}
1376+
1377+func TestRegistryRetiresInflightAttemptsOnReplacementAndRemoval(t *testing.T) {
1378+ current := registration(t, "example.test", 1)
1379+ replacement := registration(t, "example.test", 2)
1380+ tests := []struct {
1381+ name string
1382+ next []domain.RemoteRegistration
1383+ }{
1384+ {name: "replacement", next: []domain.RemoteRegistration{replacement}},
1385+ {name: "removal", next: nil},
1386+ }
1387+ for _, tt := range tests {
1388+ t.Run(tt.name, func(t *testing.T) {
1389+ clock := newManualClock(time.Unix(100, 0))
1390+ probe := newTestProbe(2)
1391+ registry := newTestRegistry(t, 1, nil, probe, clock)
1392+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{current}))
1393+ startRegistry(t, registry)
1394+
1395+ call := receiveCall(t, probe)
1396+ require.True(t, call.registration.Equal(current))
1397+
1398+ // Replacing or removing the registration retires the in-flight
1399+ // attempt: the probe's derived context is cancelled before the
1400+ // replacement can start, so a ctx-honoring probe stops instead of
1401+ // overlapping its successor.
1402+ require.NoError(t, registry.SetHosts(tt.next))
1403+ select {
1404+ case <-probe.canceled:
1405+ case <-time.After(time.Second):
1406+ t.Fatal("retired probe did not observe cancellation")
1407+ }
1408+ })
1409+ }
1410+}
1411+
1412+func TestRegistryChurnKeepsOneActiveProbePerEndpoint(t *testing.T) {
1413+ clock := newManualClock(time.Unix(100, 0))
1414+ probe := newChurnProbe()
1415+ defer probe.openRelease()
1416+ registry := newTestRegistry(t, 1, nil, probe, clock)
1417+
1418+ first := registration(t, "example.test", 1)
1419+ second := registration(t, "example.test", 2)
1420+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{first}))
1421+
1422+ ctx, cancel := context.WithCancel(context.Background())
1423+ defer cancel()
1424+ done := make(chan struct{})
1425+ go func() { defer close(done); registry.Run(ctx) }()
1426+
1427+ // Drop the initial attempt's start signal so each lockstep round below
1428+ // consumes the start of its own replacement attempt.
1429+ select {
1430+ case <-probe.startedSig:
1431+ case <-time.After(time.Second):
1432+ t.Fatal("timed out waiting for the initial probe")
1433+ }
1434+ require.Equal(t, 1, probe.startedCount())
1435+
1436+ // Lockstep churn: every round replaces the registration, which must cancel
1437+ // the retired attempt before the replacement starts. An uncancelled
1438+ // retirement would stay live and overlap its successor, and the exact start
1439+ // count catches an attempt that never restarts.
1440+ const rounds = 8
1441+ for round := 0; round < rounds; round++ {
1442+ next := first
1443+ if round%2 == 0 {
1444+ next = second
1445+ }
1446+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{next}))
1447+ select {
1448+ case <-probe.startedSig:
1449+ case <-time.After(time.Second):
1450+ t.Fatal("timed out waiting for the replacement attempt to start")
1451+ }
1452+ require.Equal(t, round+2, probe.startedCount())
1453+ }
1454+
1455+ // Concurrent churn and demand: the same invariant must hold under the race
1456+ // detector while membership and demand churn from several goroutines.
1457+ failures := make([]error, 4)
1458+ var wg sync.WaitGroup
1459+ for worker := 0; worker < len(failures); worker++ {
1460+ wg.Add(1)
1461+ go func(worker int) {
1462+ defer wg.Done()
1463+ for round := 0; round < 32; round++ {
1464+ next := first
1465+ if (worker+round)%2 == 0 {
1466+ next = second
1467+ }
1468+ if err := registry.SetHosts([]domain.RemoteRegistration{next}); err != nil {
1469+ failures[worker] = err
1470+ return
1471+ }
1472+ registry.RequestProbe(first.Endpoint)
1473+ registry.Snapshot()
1474+ }
1475+ }(worker)
1476+ }
1477+ wg.Wait()
1478+ for _, err := range failures {
1479+ require.NoError(t, err)
1480+ }
1481+
1482+ endpoints := probe.observedEndpoints()
1483+ require.NotEmpty(t, endpoints)
1484+ for _, endpoint := range endpoints {
1485+ require.Equal(t, 1, probe.maxLiveFor(endpoint), "endpoint %q had overlapping live probes", endpoint)
1486+ }
1487+ require.GreaterOrEqual(t, probe.startedCount(), rounds+1)
1488+
1489+ // Settling releases every attempt: no probe stays live once Run returns.
1490+ cancel()
1491+ select {
1492+ case <-done:
1493+ case <-time.After(time.Second):
1494+ t.Fatal("registry did not stop after cancellation")
1495+ }
1496+ deadline := time.Now().Add(time.Second)
1497+ for probe.inflight() != 0 {
1498+ if time.Now().After(deadline) {
1499+ t.Fatalf("%d probes still live after settle", probe.inflight())
1500+ }
1501+ time.Sleep(time.Millisecond)
1502+ }
1503+}
1504+
1505+func TestRegistryPublishesBoundedTombstones(t *testing.T) {
1506+ registry := newTestRegistry(t, 1, nil, newTestProbe(1), newManualClock(time.Unix(100, 0)))
1507+
1508+ // Two full membership rounds with distinct endpoints: without pruning the
1509+ // retired set would double.
1510+ for round := 0; round < 2; round++ {
1511+ registrations := make([]domain.RemoteRegistration, 0, ports.BrokerMaxHosts)
1512+ for index := 0; index < ports.BrokerMaxHosts; index++ {
1513+ registrations = append(registrations, registration(t, fmt.Sprintf("round-%d-host-%d.test", round, index), byte(round*ports.BrokerMaxHosts+index+1)))
1514+ }
1515+ require.NoError(t, registry.SetHosts(registrations))
1516+ require.NoError(t, registry.SetHosts(nil))
1517+ }
1518+
1519+ snapshot := registry.Snapshot()
1520+ require.NoError(t, snapshot.Validate())
1521+ require.Len(t, snapshot.Removed, ports.BrokerMaxTombstones)
1522+ for index, tombstone := range snapshot.Removed {
1523+ require.NoError(t, tombstone.Validate())
1524+ // Only the newest retirements survive the bound.
1525+ require.Equal(t, snapshot.Revision, tombstone.RetiredRevision)
1526+ if index > 0 {
1527+ require.GreaterOrEqual(t, snapshot.Removed[index-1].RetiredRevision, tombstone.RetiredRevision)
1528+ }
1529+ }
1530+}
1531+
1532+func TestRegistryPersistenceRunsOffLockAndCoalescesWrites(t *testing.T) {
1533+ clock := newManualClock(time.Unix(100, 0))
1534+ probe := newTestProbe(1)
1535+ store := newBlockingStore()
1536+ registry, err := NewRegistry(1, store, probe, clock, nil)
1537+ require.NoError(t, err)
1538+ defer registry.store.close()
1539+
1540+ reg := registration(t, "example.test", 1)
1541+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1542+ // The writer is now blocked inside the first durable write.
1543+ first := <-store.entered
1544+ require.Equal(t, ports.BrokerRevision(2), first)
1545+
1546+ // Publication is never blocked behind persistence and never holds the
1547+ // registry lock across a store call.
1548+ var setErr error
1549+ published := make(chan struct{})
1550+ go func() {
1551+ defer close(published)
1552+ for generation := domain.RemoteGeneration(2); generation <= 6; generation++ {
1553+ reg.Generation = generation
1554+ if err := registry.SetHosts([]domain.RemoteRegistration{reg}); err != nil {
1555+ setErr = err
1556+ return
1557+ }
1558+ }
1559+ }()
1560+ select {
1561+ case <-published:
1562+ case <-time.After(time.Second):
1563+ t.Fatal("publication blocked behind durable persistence")
1564+ }
1565+ require.NoError(t, setErr)
1566+
1567+ last := registry.Snapshot().Revision
1568+ store.releaseFirst()
1569+ waitWriterRevision(t, registry, last)
1570+
1571+ // Writes stay ordered and coalesced: only the first and the newest
1572+ // revision ever reach the store.
1573+ require.Equal(t, []ports.BrokerRevision{2, last}, store.writtenRevisions())
1574+}
1575+
1576+func TestRegistryStoreErrorsAreObservableAndDoNotStallPublication(t *testing.T) {
1577+ clock := newManualClock(time.Unix(100, 0))
1578+ probe := newTestProbe(1)
1579+ logger, logs := testLogger()
1580+ storeErr := errors.New("disk full")
1581+ store := &testStore{storeErr: storeErr}
1582+ registry, err := NewRegistry(1, store, probe, clock, logger)
1583+ require.NoError(t, err)
1584+ defer registry.store.close()
1585+
1586+ reg := registration(t, "example.test", 1)
1587+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1588+ waitForLog(t, logs, "durable snapshot write failed")
1589+ require.Contains(t, logs.String(), storeErr.Error())
1590+
1591+ // A failed write is not fatal and never blocks the next publication.
1592+ store.setStoreErr(nil)
1593+ reg.Generation++
1594+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1595+ stored := waitStored(t, store, func(snapshot ports.BrokerSnapshot) bool {
1596+ return snapshot.Revision == registry.Snapshot().Revision
1597+ })
1598+ require.Len(t, stored.Hosts, 1)
1599+ require.True(t, stored.Hosts[0].Registration.Equal(reg))
1600+}
1601+
1602+func TestRegistryPersistedCopyClearsChecking(t *testing.T) {
1603+ clock := newManualClock(time.Unix(100, 0))
1604+ probe := newTestProbe(1)
1605+ store := &testStore{}
1606+ registry := newTestRegistry(t, 3, store, probe, clock)
1607+ reg := registration(t, "example.test", 1)
1608+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1609+ startRegistry(t, registry)
1610+
1611+ // The observation is in flight: the published projection carries Checking,
1612+ // the durable copy never does.
1613+ require.True(t, waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1614+ return host.Checking
1615+ }).Checking)
1616+ stored := waitStored(t, store, func(snapshot ports.BrokerSnapshot) bool {
1617+ return len(snapshot.Hosts) == 1
1618+ })
1619+ require.False(t, stored.Hosts[0].Checking)
1620+
1621+ // Completing the observation persists the settled projection.
1622+ call := receiveCall(t, probe)
1623+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
1624+ settled := waitStored(t, store, func(snapshot ports.BrokerSnapshot) bool {
1625+ return len(snapshot.Hosts) == 1 && snapshot.Hosts[0].Availability == domain.RemoteAvailabilityReachable
1626+ })
1627+ require.False(t, settled.Hosts[0].Checking)
1628+}
1629+
1630+func TestRegistryBoundsSlowSubscribersAndStopsOnCancellation(t *testing.T) {
1631+ clock := newManualClock(time.Unix(100, 0))
1632+ probe := newTestProbe(1)
1633+ registry := newTestRegistry(t, 1, nil, probe, clock)
1634+
1635+ sub := registry.Subscribe()
1636+ reg := registration(t, "example.test", 1)
1637+ for generation := domain.RemoteGeneration(1); generation <= 10; generation++ {
1638+ reg.Generation = generation
1639+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1640+ }
1641+ // A subscriber that never drains is bounded to one pending wake and
1642+ // never blocks publication.
1643+ require.Len(t, sub.Changed(), 1)
1644+
1645+ // Drain the pending wake, then close: a closed subscriber never receives
1646+ // another notification, and closing is idempotent.
1647+ <-sub.Changed()
1648+ sub.Close()
1649+ sub.Close()
1650+ reg.Generation++
1651+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1652+ select {
1653+ case <-sub.Changed():
1654+ t.Fatal("closed subscription received a notification")
1655+ default:
1656+ }
1657+
1658+ ctx, cancel := context.WithCancel(context.Background())
1659+ done := make(chan struct{})
1660+ go func() { defer close(done); registry.Run(ctx) }()
1661+ receiveCall(t, probe)
1662+ cancel()
1663+ select {
1664+ case <-done:
1665+ case <-time.After(time.Second):
1666+ t.Fatal("registry did not stop after cancellation")
1667+ }
1668+ select {
1669+ case <-probe.canceled:
1670+ case <-time.After(time.Second):
1671+ t.Fatal("in-flight probe did not observe cancellation")
1672+ }
1673+ // Cleanup settles the projection: no host stays in flight.
1674+ host := waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1675+ return !host.Checking
1676+ })
1677+ require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability)
1678+}
1679+
1680+func TestRegistryRunIsSingleShotAndCleansUpInFlightState(t *testing.T) {
1681+ clock := newManualClock(time.Unix(100, 0))
1682+ probe := newTestProbe(1)
1683+ logger, logs := testLogger()
1684+ store := &testStore{}
1685+ registry, err := NewRegistry(1, store, probe, clock, logger)
1686+ require.NoError(t, err)
1687+ reg := registration(t, "example.test", 1)
1688+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{reg}))
1689+
1690+ ctx, cancel := context.WithCancel(context.Background())
1691+ done := make(chan struct{})
1692+ go func() { defer close(done); registry.Run(ctx) }()
1693+
1694+ call := receiveCall(t, probe)
1695+ require.True(t, waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1696+ return host.Checking
1697+ }).Checking)
1698+
1699+ // A concurrent Run is refused instead of duplicating scheduling.
1700+ refused := make(chan struct{})
1701+ go func() { defer close(refused); registry.Run(context.Background()) }()
1702+ select {
1703+ case <-refused:
1704+ case <-time.After(time.Second):
1705+ t.Fatal("second Run did not return")
1706+ }
1707+ require.Contains(t, logs.String(), "Run refused")
1708+
1709+ cancel()
1710+ select {
1711+ case <-done:
1712+ case <-time.After(time.Second):
1713+ t.Fatal("registry did not stop after cancellation")
1714+ }
1715+ // Cleanup publishes the settled projection: nothing stays in flight.
1716+ require.False(t, waitHost(t, registry, reg.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1717+ return !host.Checking
1718+ }).Checking)
1719+
1720+ // The retired completion can never be applied once the run is over.
1721+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
1722+ settled := registry.Snapshot()
1723+ require.Equal(t, domain.RemoteAvailabilityUnknown, settled.Hosts[0].Availability)
1724+ require.Zero(t, settled.Hosts[0].ConsecutiveFailures)
1725+
1726+ // Persistence is flushed before shutdown and no write outlives it.
1727+ stored := waitStored(t, store, func(snapshot ports.BrokerSnapshot) bool {
1728+ return snapshot.Revision >= settled.Revision
1729+ })
1730+ require.Equal(t, settled.Revision, stored.Revision)
1731+ require.False(t, stored.Hosts[0].Checking)
1732+ writes := store.writeCount()
1733+ require.NoError(t, registry.SetHosts(nil))
1734+ require.Equal(t, writes, store.writeCount())
1735+ require.Contains(t, logs.String(), "refused after shutdown")
1736+
1737+ // A restart is refused rather than racing a stale scheduler.
1738+ restarted := make(chan struct{})
1739+ go func() { defer close(restarted); registry.Run(context.Background()) }()
1740+ select {
1741+ case <-restarted:
1742+ case <-time.After(time.Second):
1743+ t.Fatal("restarted Run did not return")
1744+ }
1745+ requireNoProbeWithin(t, probe, probeGrace)
1746+}
1747+
1748+func TestRegistryRevisionsIncreaseMonotonicallyWithinEpoch(t *testing.T) {
1749+ start := time.Unix(100, 0)
1750+ clock := newManualClock(start)
1751+ probe := newTestProbe(2)
1752+ registry := newTestRegistry(t, 42, nil, probe, clock)
1753+
1754+ var last ports.BrokerRevision
1755+ observe := func() {
1756+ t.Helper()
1757+ snapshot := registry.Snapshot()
1758+ require.Equal(t, ports.BrokerEpoch(42), snapshot.Epoch)
1759+ require.Greater(t, snapshot.Revision, last)
1760+ last = snapshot.Revision
1761+ }
1762+ observe()
1763+
1764+ first := registration(t, "a.test", 1)
1765+ second := registration(t, "b.test", 2)
1766+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{first}))
1767+ observe()
1768+ require.NoError(t, registry.SetHosts([]domain.RemoteRegistration{first, second}))
1769+ observe()
1770+
1771+ startRegistry(t, registry)
1772+ for range 2 {
1773+ call := receiveCall(t, probe)
1774+ call.result <- probeAnswer{snapshot: ports.RemoteHostSnapshot{Availability: domain.RemoteAvailabilityReachable}}
1775+ waitHost(t, registry, call.registration.Endpoint, func(host ports.RemoteHostSnapshot) bool {
1776+ return !host.Checking && host.Availability == domain.RemoteAvailabilityReachable
1777+ })
1778+ observe()
1779+ }
1780+}

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 12

Tags 11

↑↓ moveenter openesc close