feat(broker): own mutable host membership
- commit
- c9fb7d8e17faf8ad4f0d091706dbca4f7463fe1a
- author
- brice · · 2026-09-20 00:48 UTC
- parent
- 49f309d
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
9 files · +1492 -34
M internal/adapters/brokeripc/errors.go +7 -0
| @@ -118,7 +118,14 @@ func errorDetail(err error) brokerwire.ErrorDetail { | |||
| 118 | 118 | } | |
| 119 | 119 | return detail | |
| 120 | 120 | } | |
| 121 | + | // The ports type carries value-receiver methods, so both the value and the | |
| 122 | + | // pointer form are legal errors; classify either rather than letting the | |
| 123 | + | // pointer form fall through to a retryable-looking code. | |
| 124 | + | var outcomeUnknown ports.BrokerStoreOutcomeUnknownError | |
| 125 | + | var outcomeUnknownPtr *ports.BrokerStoreOutcomeUnknownError | |
| 121 | 126 | switch { | |
| 127 | + | case errors.As(err, &outcomeUnknown), errors.As(err, &outcomeUnknownPtr): | |
| 128 | + | return brokerwire.ErrorDetail{Code: ports.BrokerErrorOutcomeUnknown} | |
| 122 | 129 | case errors.Is(err, context.DeadlineExceeded): | |
| 123 | 130 | return brokerwire.ErrorDetail{Code: ports.BrokerErrorTimeout} | |
| 124 | 131 | case errors.Is(err, context.Canceled): | |
| @@ -118,7 +118,14 @@ func errorDetail(err error) brokerwire.ErrorDetail { | |||||
| 118 | } | 118 | } | ||
| 119 | return detail | 119 | return detail | ||
| 120 | } | 120 | } | ||
| 121 | + | // The ports type carries value-receiver methods, so both the value and the | |||
| 122 | + | // pointer form are legal errors; classify either rather than letting the | |||
| 123 | + | // pointer form fall through to a retryable-looking code. | |||
| 124 | + | var outcomeUnknown ports.BrokerStoreOutcomeUnknownError | |||
| 125 | + | var outcomeUnknownPtr *ports.BrokerStoreOutcomeUnknownError | |||
| 121 | switch { | 126 | switch { | ||
| 127 | + | case errors.As(err, &outcomeUnknown), errors.As(err, &outcomeUnknownPtr): | |||
| 128 | + | return brokerwire.ErrorDetail{Code: ports.BrokerErrorOutcomeUnknown} | |||
| 122 | case errors.Is(err, context.DeadlineExceeded): | 129 | case errors.Is(err, context.DeadlineExceeded): | ||
| 123 | return brokerwire.ErrorDetail{Code: ports.BrokerErrorTimeout} | 130 | return brokerwire.ErrorDetail{Code: ports.BrokerErrorTimeout} | ||
| 124 | case errors.Is(err, context.Canceled): | 131 | case errors.Is(err, context.Canceled): | ||
M internal/adapters/brokeripc/limits_test.go +8 -0
| @@ -217,6 +217,14 @@ func TestErrorDetailMapping(t *testing.T) { | |||
| 217 | 217 | {"invalid", ports.BrokerAdmissionInvalid, ports.BrokerErrorUnavailable, 4}, | |
| 218 | 218 | {"deadline", context.DeadlineExceeded, ports.BrokerErrorTimeout, 0}, | |
| 219 | 219 | {"cancelled", context.Canceled, ports.BrokerErrorCancelled, 0}, | |
| 220 | + | // An indeterminate store outcome is classified before the context and | |
| 221 | + | // transport rules: a failure after the durable commit point must never be | |
| 222 | + | // presented as a retryable timeout, however its cause would classify | |
| 223 | + | // alone. Both the value and the pointer form are legal errors because the | |
| 224 | + | // ports type carries value-receiver methods. | |
| 225 | + | {"outcome unknown value", ports.BrokerStoreOutcomeUnknownError{Err: errors.New("power loss")}, ports.BrokerErrorOutcomeUnknown, 0}, | |
| 226 | + | {"outcome unknown wrapping deadline", ports.BrokerStoreOutcomeUnknownError{Err: context.DeadlineExceeded}, ports.BrokerErrorOutcomeUnknown, 0}, | |
| 227 | + | {"outcome unknown pointer", &ports.BrokerStoreOutcomeUnknownError{Err: context.DeadlineExceeded}, ports.BrokerErrorOutcomeUnknown, 0}, | |
| 220 | 228 | {"unknown", errors.New("boom"), ports.BrokerErrorUnavailable, 0}, | |
| 221 | 229 | } | |
| 222 | 230 | for _, tc := range cases { | |
| @@ -217,6 +217,14 @@ func TestErrorDetailMapping(t *testing.T) { | |||||
| 217 | {"invalid", ports.BrokerAdmissionInvalid, ports.BrokerErrorUnavailable, 4}, | 217 | {"invalid", ports.BrokerAdmissionInvalid, ports.BrokerErrorUnavailable, 4}, | ||
| 218 | {"deadline", context.DeadlineExceeded, ports.BrokerErrorTimeout, 0}, | 218 | {"deadline", context.DeadlineExceeded, ports.BrokerErrorTimeout, 0}, | ||
| 219 | {"cancelled", context.Canceled, ports.BrokerErrorCancelled, 0}, | 219 | {"cancelled", context.Canceled, ports.BrokerErrorCancelled, 0}, | ||
| 220 | + | // An indeterminate store outcome is classified before the context and | |||
| 221 | + | // transport rules: a failure after the durable commit point must never be | |||
| 222 | + | // presented as a retryable timeout, however its cause would classify | |||
| 223 | + | // alone. Both the value and the pointer form are legal errors because the | |||
| 224 | + | // ports type carries value-receiver methods. | |||
| 225 | + | {"outcome unknown value", ports.BrokerStoreOutcomeUnknownError{Err: errors.New("power loss")}, ports.BrokerErrorOutcomeUnknown, 0}, | |||
| 226 | + | {"outcome unknown wrapping deadline", ports.BrokerStoreOutcomeUnknownError{Err: context.DeadlineExceeded}, ports.BrokerErrorOutcomeUnknown, 0}, | |||
| 227 | + | {"outcome unknown pointer", &ports.BrokerStoreOutcomeUnknownError{Err: context.DeadlineExceeded}, ports.BrokerErrorOutcomeUnknown, 0}, | |||
| 220 | {"unknown", errors.New("boom"), ports.BrokerErrorUnavailable, 0}, | 228 | {"unknown", errors.New("boom"), ports.BrokerErrorUnavailable, 0}, | ||
| 221 | } | 229 | } | ||
| 222 | for _, tc := range cases { | 230 | for _, tc := range cases { | ||
M internal/adapters/brokerstore/store.go +15 -6
| @@ -214,8 +214,11 @@ func (r recovery) verify() error { | |||
| 214 | 214 | } | |
| 215 | 215 | func digest(b []byte) string { sum := sha256.Sum256(b); return hex.EncodeToString(sum[:]) } | |
| 216 | 216 | func (s *Store) ready() error { | |
| 217 | - | if s.lock == nil || s.failed { | |
| 218 | - | return errors.New("brokerstore: closed or outcome unknown; reopen required") | |
| 217 | + | if s.failed { | |
| 218 | + | return ports.BrokerStoreOutcomeUnknownError{Err: errors.New("brokerstore: poisoned; reopen required")} | |
| 219 | + | } | |
| 220 | + | if s.lock == nil { | |
| 221 | + | return errors.New("brokerstore: closed") | |
| 219 | 222 | } | |
| 220 | 223 | return nil | |
| 221 | 224 | } | |
| @@ -510,16 +513,22 @@ func (s *Store) write(name string, value any) error { | |||
| 510 | 513 | if err = os.Rename(f.Name(), filepath.Join(s.dir, name)); err != nil { | |
| 511 | 514 | return err | |
| 512 | 515 | } | |
| 516 | + | unknown := func(err error) error { | |
| 517 | + | if err == nil { | |
| 518 | + | return nil | |
| 519 | + | } | |
| 520 | + | return ports.BrokerStoreOutcomeUnknownError{Err: err} | |
| 521 | + | } | |
| 513 | 522 | if err = s.boundary(name + ":rename"); err != nil { | |
| 514 | - | return err | |
| 523 | + | return unknown(err) | |
| 515 | 524 | } | |
| 516 | 525 | d, err := os.Open(s.dir) | |
| 517 | 526 | if err != nil { | |
| 518 | - | return err | |
| 527 | + | return unknown(err) | |
| 519 | 528 | } | |
| 520 | 529 | defer d.Close() | |
| 521 | 530 | if err = d.Sync(); err != nil { | |
| 522 | - | return err | |
| 531 | + | return unknown(err) | |
| 523 | 532 | } | |
| 524 | - | return s.boundary(name + ":dirsync") | |
| 533 | + | return unknown(s.boundary(name + ":dirsync")) | |
| 525 | 534 | } | |
| @@ -214,8 +214,11 @@ func (r recovery) verify() error { | |||||
| 214 | } | 214 | } | ||
| 215 | func digest(b []byte) string { sum := sha256.Sum256(b); return hex.EncodeToString(sum[:]) } | 215 | func digest(b []byte) string { sum := sha256.Sum256(b); return hex.EncodeToString(sum[:]) } | ||
| 216 | func (s *Store) ready() error { | 216 | func (s *Store) ready() error { | ||
| 217 | - | if s.lock == nil || s.failed { | 217 | + | if s.failed { |
| 218 | - | return errors.New("brokerstore: closed or outcome unknown; reopen required") | 218 | + | return ports.BrokerStoreOutcomeUnknownError{Err: errors.New("brokerstore: poisoned; reopen required")} |
| 219 | + | } | |||
| 220 | + | if s.lock == nil { | |||
| 221 | + | return errors.New("brokerstore: closed") | |||
| 219 | } | 222 | } | ||
| 220 | return nil | 223 | return nil | ||
| 221 | } | 224 | } | ||
| @@ -510,16 +513,22 @@ func (s *Store) write(name string, value any) error { | |||||
| 510 | if err = os.Rename(f.Name(), filepath.Join(s.dir, name)); err != nil { | 513 | if err = os.Rename(f.Name(), filepath.Join(s.dir, name)); err != nil { | ||
| 511 | return err | 514 | return err | ||
| 512 | } | 515 | } | ||
| 516 | + | unknown := func(err error) error { | |||
| 517 | + | if err == nil { | |||
| 518 | + | return nil | |||
| 519 | + | } | |||
| 520 | + | return ports.BrokerStoreOutcomeUnknownError{Err: err} | |||
| 521 | + | } | |||
| 513 | if err = s.boundary(name + ":rename"); err != nil { | 522 | if err = s.boundary(name + ":rename"); err != nil { | ||
| 514 | - | return err | 523 | + | return unknown(err) |
| 515 | } | 524 | } | ||
| 516 | d, err := os.Open(s.dir) | 525 | d, err := os.Open(s.dir) | ||
| 517 | if err != nil { | 526 | if err != nil { | ||
| 518 | - | return err | 527 | + | return unknown(err) |
| 519 | } | 528 | } | ||
| 520 | defer d.Close() | 529 | defer d.Close() | ||
| 521 | if err = d.Sync(); err != nil { | 530 | if err = d.Sync(); err != nil { | ||
| 522 | - | return err | 531 | + | return unknown(err) |
| 523 | } | 532 | } | ||
| 524 | - | return s.boundary(name + ":dirsync") | 533 | + | return unknown(s.boundary(name + ":dirsync")) |
| 525 | } | 534 | } | ||
M internal/ports/broker_store.go +15 -0
| @@ -13,6 +13,9 @@ import ( | |||
| 13 | 13 | // invalid state is resolved offline from the recovery record. None of them is | |
| 14 | 14 | // ever retried blindly inside the store. | |
| 15 | 15 | var ( | |
| 16 | + | // ErrBrokerMembershipImmutable reports that a registry was not explicitly | |
| 17 | + | // configured to accept membership mutations. | |
| 18 | + | ErrBrokerMembershipImmutable = errors.New("ports: broker membership is immutable") | |
| 16 | 19 | // ErrBrokerHostConflict reports that durable host authority changed under | |
| 17 | 20 | // the caller's expected revision, or that the supplied membership would | |
| 18 | 21 | // regress or re-trust an existing registration without a fresh identity. | |
| @@ -25,6 +28,18 @@ var ( | |||
| 25 | 28 | ErrBrokerStoreInvalidState = errors.New("ports: broker durable store state is invalid") | |
| 26 | 29 | ) | |
| 27 | 30 | ||
| 31 | + | // BrokerStoreOutcomeUnknownError reports a write failure after the durable | |
| 32 | + | // commit point. Callers must reopen and reload authority rather than retrying. | |
| 33 | + | type BrokerStoreOutcomeUnknownError struct{ Err error } | |
| 34 | + | ||
| 35 | + | func (e BrokerStoreOutcomeUnknownError) Error() string { | |
| 36 | + | if e.Err == nil { | |
| 37 | + | return "ports: broker store outcome unknown" | |
| 38 | + | } | |
| 39 | + | return "ports: broker store outcome unknown: " + e.Err.Error() | |
| 40 | + | } | |
| 41 | + | func (e BrokerStoreOutcomeUnknownError) Unwrap() error { return e.Err } | |
| 42 | + | ||
| 28 | 43 | // BrokerHostRecord is durable registration authority. Membership and policy | |
| 29 | 44 | // are never inferred from an observation or from the broker environment. | |
| 30 | 45 | type BrokerHostRecord struct { | |
| @@ -13,6 +13,9 @@ import ( | |||||
| 13 | // invalid state is resolved offline from the recovery record. None of them is | 13 | // invalid state is resolved offline from the recovery record. None of them is | ||
| 14 | // ever retried blindly inside the store. | 14 | // ever retried blindly inside the store. | ||
| 15 | var ( | 15 | var ( | ||
| 16 | + | // ErrBrokerMembershipImmutable reports that a registry was not explicitly | |||
| 17 | + | // configured to accept membership mutations. | |||
| 18 | + | ErrBrokerMembershipImmutable = errors.New("ports: broker membership is immutable") | |||
| 16 | // ErrBrokerHostConflict reports that durable host authority changed under | 19 | // ErrBrokerHostConflict reports that durable host authority changed under | ||
| 17 | // the caller's expected revision, or that the supplied membership would | 20 | // the caller's expected revision, or that the supplied membership would | ||
| 18 | // regress or re-trust an existing registration without a fresh identity. | 21 | // regress or re-trust an existing registration without a fresh identity. | ||
| @@ -25,6 +28,18 @@ var ( | |||||
| 25 | ErrBrokerStoreInvalidState = errors.New("ports: broker durable store state is invalid") | 28 | ErrBrokerStoreInvalidState = errors.New("ports: broker durable store state is invalid") | ||
| 26 | ) | 29 | ) | ||
| 27 | 30 | ||||
| 31 | + | // BrokerStoreOutcomeUnknownError reports a write failure after the durable | |||
| 32 | + | // commit point. Callers must reopen and reload authority rather than retrying. | |||
| 33 | + | type BrokerStoreOutcomeUnknownError struct{ Err error } | |||
| 34 | + | ||||
| 35 | + | func (e BrokerStoreOutcomeUnknownError) Error() string { | |||
| 36 | + | if e.Err == nil { | |||
| 37 | + | return "ports: broker store outcome unknown" | |||
| 38 | + | } | |||
| 39 | + | return "ports: broker store outcome unknown: " + e.Err.Error() | |||
| 40 | + | } | |||
| 41 | + | func (e BrokerStoreOutcomeUnknownError) Unwrap() error { return e.Err } | |||
| 42 | + | ||||
| 28 | // BrokerHostRecord is durable registration authority. Membership and policy | 43 | // BrokerHostRecord is durable registration authority. Membership and policy | ||
| 29 | // are never inferred from an observation or from the broker environment. | 44 | // are never inferred from an observation or from the broker environment. | ||
| 30 | type BrokerHostRecord struct { | 45 | type BrokerHostRecord struct { | ||
A internal/ports/broker_store_outcome_test.go +46 -0
| @@ -0,0 +1,46 @@ | |||
| 1 | + | package ports | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "errors" | |
| 5 | + | "fmt" | |
| 6 | + | "testing" | |
| 7 | + | ||
| 8 | + | "github.com/stretchr/testify/require" | |
| 9 | + | ) | |
| 10 | + | ||
| 11 | + | // TestBrokerStoreOutcomeUnknownErrorClassification pins the indeterminate-write | |
| 12 | + | // contract at the ports seam: the error reports a store failure that happened | |
| 13 | + | // after the durable commit point, stays classifiable after wrapping, and keeps | |
| 14 | + | // the underlying store cause inspectable so a caller can log or classify it | |
| 15 | + | // without importing the adapter. | |
| 16 | + | func TestBrokerStoreOutcomeUnknownErrorClassification(t *testing.T) { | |
| 17 | + | cause := errors.New("power loss") | |
| 18 | + | unknown := BrokerStoreOutcomeUnknownError{Err: cause} | |
| 19 | + | ||
| 20 | + | require.ErrorContains(t, unknown, "outcome unknown") | |
| 21 | + | require.ErrorContains(t, unknown, "power loss") | |
| 22 | + | require.ErrorIs(t, unknown, cause, "the underlying store cause stays inspectable") | |
| 23 | + | var classified BrokerStoreOutcomeUnknownError | |
| 24 | + | require.ErrorAs(t, unknown, &classified) | |
| 25 | + | require.Equal(t, cause, classified.Err) | |
| 26 | + | ||
| 27 | + | // Wrapping by a caller or a composing use case must not erase the | |
| 28 | + | // classification: that is the whole point of a typed error here. | |
| 29 | + | wrapped := fmt.Errorf("broker: replace hosts: %w", unknown) | |
| 30 | + | require.ErrorAs(t, wrapped, &classified) | |
| 31 | + | require.ErrorIs(t, wrapped, cause) | |
| 32 | + | ||
| 33 | + | // A definite store failure is never an unknown outcome, so a caller cannot | |
| 34 | + | // mistake a retryable conflict for a reopen-required indeterminate write. | |
| 35 | + | for name, definite := range map[string]error{ | |
| 36 | + | "conflict": ErrBrokerHostConflict, | |
| 37 | + | "locked": ErrBrokerStoreLocked, | |
| 38 | + | "invalid state": ErrBrokerStoreInvalidState, | |
| 39 | + | "immutable": ErrBrokerMembershipImmutable, | |
| 40 | + | } { | |
| 41 | + | t.Run(name, func(t *testing.T) { | |
| 42 | + | require.False(t, errors.As(definite, &classified)) | |
| 43 | + | require.False(t, errors.As(fmt.Errorf("broker: replace hosts: %w", definite), &classified)) | |
| 44 | + | }) | |
| 45 | + | } | |
| 46 | + | } | |
| @@ -0,0 +1,46 @@ | |||||
| 1 | + | package ports | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "errors" | |||
| 5 | + | "fmt" | |||
| 6 | + | "testing" | |||
| 7 | + | ||||
| 8 | + | "github.com/stretchr/testify/require" | |||
| 9 | + | ) | |||
| 10 | + | ||||
| 11 | + | // TestBrokerStoreOutcomeUnknownErrorClassification pins the indeterminate-write | |||
| 12 | + | // contract at the ports seam: the error reports a store failure that happened | |||
| 13 | + | // after the durable commit point, stays classifiable after wrapping, and keeps | |||
| 14 | + | // the underlying store cause inspectable so a caller can log or classify it | |||
| 15 | + | // without importing the adapter. | |||
| 16 | + | func TestBrokerStoreOutcomeUnknownErrorClassification(t *testing.T) { | |||
| 17 | + | cause := errors.New("power loss") | |||
| 18 | + | unknown := BrokerStoreOutcomeUnknownError{Err: cause} | |||
| 19 | + | ||||
| 20 | + | require.ErrorContains(t, unknown, "outcome unknown") | |||
| 21 | + | require.ErrorContains(t, unknown, "power loss") | |||
| 22 | + | require.ErrorIs(t, unknown, cause, "the underlying store cause stays inspectable") | |||
| 23 | + | var classified BrokerStoreOutcomeUnknownError | |||
| 24 | + | require.ErrorAs(t, unknown, &classified) | |||
| 25 | + | require.Equal(t, cause, classified.Err) | |||
| 26 | + | ||||
| 27 | + | // Wrapping by a caller or a composing use case must not erase the | |||
| 28 | + | // classification: that is the whole point of a typed error here. | |||
| 29 | + | wrapped := fmt.Errorf("broker: replace hosts: %w", unknown) | |||
| 30 | + | require.ErrorAs(t, wrapped, &classified) | |||
| 31 | + | require.ErrorIs(t, wrapped, cause) | |||
| 32 | + | ||||
| 33 | + | // A definite store failure is never an unknown outcome, so a caller cannot | |||
| 34 | + | // mistake a retryable conflict for a reopen-required indeterminate write. | |||
| 35 | + | for name, definite := range map[string]error{ | |||
| 36 | + | "conflict": ErrBrokerHostConflict, | |||
| 37 | + | "locked": ErrBrokerStoreLocked, | |||
| 38 | + | "invalid state": ErrBrokerStoreInvalidState, | |||
| 39 | + | "immutable": ErrBrokerMembershipImmutable, | |||
| 40 | + | } { | |||
| 41 | + | t.Run(name, func(t *testing.T) { | |||
| 42 | + | require.False(t, errors.As(definite, &classified)) | |||
| 43 | + | require.False(t, errors.As(fmt.Errorf("broker: replace hosts: %w", definite), &classified)) | |||
| 44 | + | }) | |||
| 45 | + | } | |||
| 46 | + | } | |||
M internal/usecase/broker/registry.go +202 -28
| @@ -2,6 +2,7 @@ package broker | |||
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | + | "crypto/rand" | |
| 5 | 6 | "encoding/binary" | |
| 6 | 7 | "errors" | |
| 7 | 8 | "fmt" | |
| @@ -39,8 +40,21 @@ const ( | |||
| 39 | 40 | runStopped | |
| 40 | 41 | ) | |
| 41 | 42 | ||
| 42 | - | // RegistryConfig selects how a Registry runs. The zero value observes hosts. | |
| 43 | + | // MembershipMode controls whether runtime membership mutation is admitted. | |
| 44 | + | type MembershipMode uint8 | |
| 45 | + | ||
| 46 | + | const ( | |
| 47 | + | MembershipImmutable MembershipMode = iota | |
| 48 | + | MembershipMutable | |
| 49 | + | ) | |
| 50 | + | ||
| 51 | + | // RegistryConfig selects how a Registry runs. The zero value observes hosts | |
| 52 | + | // with immutable membership. | |
| 43 | 53 | type RegistryConfig struct { | |
| 54 | + | MembershipMode MembershipMode | |
| 55 | + | // IncarnationGenerator is an injectable entropy seam for mutable additions. | |
| 56 | + | // Nil uses crypto/rand. | |
| 57 | + | IncarnationGenerator func() ([16]byte, error) | |
| 44 | 58 | // ObservationDisabled runs the registry as a read-only snapshot owner: it | |
| 45 | 59 | // restores and publishes durable state, owns and drains its writer, and | |
| 46 | 60 | // honors the Run lifecycle, but issues no probes, arms no timers, and never | |
| @@ -65,6 +79,8 @@ type Registry struct { | |||
| 65 | 79 | // observationDisabled is fixed at construction: a disabled registry never | |
| 66 | 80 | // observes, so it has no probe, timer, or reconcile work to do. | |
| 67 | 81 | observationDisabled bool | |
| 82 | + | membershipMode MembershipMode | |
| 83 | + | newIncarnation func() ([16]byte, error) | |
| 68 | 84 | ||
| 69 | 85 | // hostStore and authority are the synchronous membership owner. Only this | |
| 70 | 86 | // registry may mutate the store while it is live; the caller owns Close. | |
| @@ -144,6 +160,9 @@ func NewRegistry(epoch ports.BrokerEpoch, store ports.BrokerHostStore, probe por | |||
| 144 | 160 | // probe as long as it owns no remote membership. Every other dependency is | |
| 145 | 161 | // required exactly as NewRegistry requires it. | |
| 146 | 162 | func NewRegistryWithConfig(epoch ports.BrokerEpoch, store ports.BrokerHostStore, probe ports.BrokerHostProbe, clock ports.Clock, log *slog.Logger, cfg RegistryConfig) (*Registry, error) { | |
| 163 | + | if cfg.MembershipMode != MembershipImmutable && cfg.MembershipMode != MembershipMutable { | |
| 164 | + | return nil, errors.New("broker: invalid membership mode") | |
| 165 | + | } | |
| 147 | 166 | if epoch == 0 || nilDependency(store) || nilDependency(clock) { | |
| 148 | 167 | return nil, errors.New("broker: invalid registry dependencies") | |
| 149 | 168 | } | |
| @@ -157,10 +176,15 @@ func NewRegistryWithConfig(epoch ports.BrokerEpoch, store ports.BrokerHostStore, | |||
| 157 | 176 | if log == nil { | |
| 158 | 177 | log = slog.Default() | |
| 159 | 178 | } | |
| 179 | + | generator := cfg.IncarnationGenerator | |
| 180 | + | if generator == nil { | |
| 181 | + | generator = func() (id [16]byte, err error) { _, err = rand.Read(id[:]); return id, err } | |
| 182 | + | } | |
| 160 | 183 | r := &Registry{ | |
| 161 | 184 | epoch: epoch, probe: probe, clock: clock, log: log, | |
| 162 | 185 | observationDisabled: cfg.ObservationDisabled, | |
| 163 | - | hosts: make(map[string]ports.BrokerDaemonObservation), inflight: make(map[string]*probeAttempt), | |
| 186 | + | membershipMode: cfg.MembershipMode, newIncarnation: generator, | |
| 187 | + | hosts: make(map[string]ports.BrokerDaemonObservation), inflight: make(map[string]*probeAttempt), | |
| 164 | 188 | pending: make(map[string]bool), tombstones: make(map[string]ports.BrokerHostTombstone), | |
| 165 | 189 | subs: make(map[*subscription]struct{}), wake: make(chan struct{}, 1), | |
| 166 | 190 | freshFor: defaultFreshFor, retryBase: defaultRetryBase, retryMax: defaultRetryLimit, | |
| @@ -426,49 +450,199 @@ func (r *Registry) Subscribe() ports.BrokerSubscription { | |||
| 426 | 450 | return s | |
| 427 | 451 | } | |
| 428 | 452 | ||
| 429 | - | // ReplaceHosts durably replaces membership and explicit policy using the loaded | |
| 430 | - | // authority revision. Success means membership is committed, not that advisory | |
| 431 | - | // observations have flushed. Store errors (including stale authority) are | |
| 432 | - | // returned without changing the projection, probe attempts, or local token. | |
| 433 | - | // A settled run reports ports.ErrBrokerRegistryClosed and an exhausted revision | |
| 434 | - | // series reports ports.ErrBrokerRevisionExhausted; both refuse the mutation | |
| 435 | - | // before the store CAS, so a refusal never changes durable authority either. | |
| 436 | - | // Conflicts are not retried or refreshed implicitly: reconcile by reopening the | |
| 437 | - | // owner. Callers must supply fresh incarnations when removing and re-adding. | |
| 438 | - | // Even identical records perform CAS so stale authority cannot report success. | |
| 439 | - | // The synchronous write holds mu; Snapshot remains lock-free, but scheduling | |
| 440 | - | // waits for membership durability. Observation writes remain asynchronous. | |
| 441 | - | func (r *Registry) ReplaceHosts(records []ports.BrokerHostRecord) error { | |
| 442 | - | records = append([]ports.BrokerHostRecord(nil), records...) | |
| 443 | - | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |
| 444 | - | return err | |
| 453 | + | // AddHost adds or pins endpoint under policy. Duplicate same-policy additions | |
| 454 | + | // CAS-verify authority and preserve identity; a conflicting policy is refused. | |
| 455 | + | func (r *Registry) AddHost(ctx context.Context, endpoint string, policy ports.BrokerPolicy) (domain.RemoteRegistration, error) { | |
| 456 | + | if err := domain.ValidateRemoteHostTarget(endpoint); err != nil { | |
| 457 | + | return domain.RemoteRegistration{}, err | |
| 445 | 458 | } | |
| 446 | - | next := make(map[string]ports.BrokerDaemonObservation, len(records)) | |
| 447 | - | order := make([]string, 0, len(records)) | |
| 448 | - | for rank, record := range records { | |
| 449 | - | reg := record.Registration | |
| 450 | - | next[reg.Endpoint] = stampHostAuthority(ports.BrokerDaemonObservation{}, record, rank) | |
| 451 | - | order = append(order, reg.Endpoint) | |
| 459 | + | if err := policy.Validate(); err != nil { | |
| 460 | + | return domain.RemoteRegistration{}, err | |
| 452 | 461 | } | |
| 453 | 462 | r.mu.Lock() | |
| 454 | 463 | defer r.mu.Unlock() | |
| 464 | + | if err := r.mutableLocked(ctx); err != nil { | |
| 465 | + | return domain.RemoteRegistration{}, err | |
| 466 | + | } | |
| 467 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |
| 468 | + | for i := range records { | |
| 469 | + | if records[i].Registration.Endpoint != endpoint { | |
| 470 | + | continue | |
| 471 | + | } | |
| 472 | + | if records[i].Policy != policy { | |
| 473 | + | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |
| 474 | + | } | |
| 475 | + | records[i].Pinned = true | |
| 476 | + | if err := r.commitMembershipLocked(records); err != nil { | |
| 477 | + | return domain.RemoteRegistration{}, err | |
| 478 | + | } | |
| 479 | + | return records[i].Registration, nil | |
| 480 | + | } | |
| 481 | + | id, err := r.newIncarnation() | |
| 482 | + | if err != nil { | |
| 483 | + | return domain.RemoteRegistration{}, err | |
| 484 | + | } | |
| 485 | + | reg, err := domain.NewRemoteRegistration(endpoint, id) | |
| 486 | + | if err != nil { | |
| 487 | + | return domain.RemoteRegistration{}, err | |
| 488 | + | } | |
| 489 | + | if reg.IsZero() { | |
| 490 | + | return domain.RemoteRegistration{}, errors.New("broker: incarnation generator returned zero") | |
| 491 | + | } | |
| 492 | + | records = append(records, ports.BrokerHostRecord{Registration: reg, Pinned: true, Policy: policy}) | |
| 493 | + | if err := r.commitMembershipLocked(records); err != nil { | |
| 494 | + | return domain.RemoteRegistration{}, err | |
| 495 | + | } | |
| 496 | + | return reg, nil | |
| 497 | + | } | |
| 498 | + | ||
| 499 | + | // RemoveHost removes only the exact expected registration. | |
| 500 | + | func (r *Registry) RemoveHost(ctx context.Context, expected domain.RemoteRegistration) (bool, error) { | |
| 501 | + | if err := expected.Validate(); err != nil { | |
| 502 | + | return false, err | |
| 503 | + | } | |
| 504 | + | r.mu.Lock() | |
| 505 | + | defer r.mu.Unlock() | |
| 506 | + | if err := r.mutableLocked(ctx); err != nil { | |
| 507 | + | return false, err | |
| 508 | + | } | |
| 509 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |
| 510 | + | for i := range records { | |
| 511 | + | if records[i].Registration.Endpoint != expected.Endpoint { | |
| 512 | + | continue | |
| 513 | + | } | |
| 514 | + | if !records[i].Registration.Equal(expected) { | |
| 515 | + | return false, ports.ErrBrokerHostConflict | |
| 516 | + | } | |
| 517 | + | records = append(records[:i], records[i+1:]...) | |
| 518 | + | if err := r.commitMembershipLocked(records); err != nil { | |
| 519 | + | return false, err | |
| 520 | + | } | |
| 521 | + | return true, nil | |
| 522 | + | } | |
| 523 | + | // A no-op still verifies the loaded revision. | |
| 524 | + | return false, r.commitMembershipLocked(records) | |
| 525 | + | } | |
| 526 | + | ||
| 527 | + | // UpdateHostPolicy updates exact authority and advances its generation. | |
| 528 | + | func (r *Registry) UpdateHostPolicy(ctx context.Context, expected domain.RemoteRegistration, policy ports.BrokerPolicy) (domain.RemoteRegistration, error) { | |
| 529 | + | if err := expected.Validate(); err != nil { | |
| 530 | + | return domain.RemoteRegistration{}, err | |
| 531 | + | } | |
| 532 | + | if err := policy.Validate(); err != nil { | |
| 533 | + | return domain.RemoteRegistration{}, err | |
| 534 | + | } | |
| 535 | + | r.mu.Lock() | |
| 536 | + | defer r.mu.Unlock() | |
| 537 | + | if err := r.mutableLocked(ctx); err != nil { | |
| 538 | + | return domain.RemoteRegistration{}, err | |
| 539 | + | } | |
| 540 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |
| 541 | + | for i := range records { | |
| 542 | + | if records[i].Registration.Endpoint != expected.Endpoint { | |
| 543 | + | continue | |
| 544 | + | } | |
| 545 | + | if !records[i].Registration.Equal(expected) { | |
| 546 | + | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |
| 547 | + | } | |
| 548 | + | if records[i].Policy == policy { | |
| 549 | + | if err := r.commitMembershipLocked(records); err != nil { | |
| 550 | + | return domain.RemoteRegistration{}, err | |
| 551 | + | } | |
| 552 | + | return expected, nil | |
| 553 | + | } | |
| 554 | + | if expected.Generation == ^domain.RemoteGeneration(0) { | |
| 555 | + | return domain.RemoteRegistration{}, ports.ErrBrokerRevisionExhausted | |
| 556 | + | } | |
| 557 | + | records[i].Registration.Generation++ | |
| 558 | + | records[i].Policy = policy | |
| 559 | + | if err := r.commitMembershipLocked(records); err != nil { | |
| 560 | + | return domain.RemoteRegistration{}, err | |
| 561 | + | } | |
| 562 | + | return records[i].Registration, nil | |
| 563 | + | } | |
| 564 | + | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |
| 565 | + | } | |
| 566 | + | ||
| 567 | + | func (r *Registry) mutableLocked(ctx context.Context) error { | |
| 568 | + | if r.membershipMode != MembershipMutable { | |
| 569 | + | return ports.ErrBrokerMembershipImmutable | |
| 570 | + | } | |
| 571 | + | if err := ctx.Err(); err != nil { | |
| 572 | + | return err | |
| 573 | + | } | |
| 455 | 574 | if r.running.Load() == runStopped { | |
| 456 | - | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRegistryClosed) | |
| 575 | + | return ports.ErrBrokerRegistryClosed | |
| 576 | + | } | |
| 577 | + | if r.revisionExhaustedLocked() { | |
| 578 | + | return ports.ErrBrokerRevisionExhausted | |
| 579 | + | } | |
| 580 | + | return nil | |
| 581 | + | } | |
| 582 | + | ||
| 583 | + | // commitMembershipLocked is the sole mutable-membership durable commit path. | |
| 584 | + | // It installs projection only after CAS success. | |
| 585 | + | func (r *Registry) commitMembershipLocked(records []ports.BrokerHostRecord) error { | |
| 586 | + | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |
| 587 | + | return err | |
| 457 | 588 | } | |
| 458 | 589 | if len(records) > 0 && !r.observationDisabled && nilDependency(r.probe) { | |
| 459 | 590 | return errors.New("broker: remote hosts require a host probe") | |
| 460 | 591 | } | |
| 461 | - | if r.revisionExhaustedLocked() { | |
| 462 | - | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRevisionExhausted) | |
| 592 | + | next := make(map[string]ports.BrokerDaemonObservation, len(records)) | |
| 593 | + | order := make([]string, 0, len(records)) | |
| 594 | + | for rank, record := range records { | |
| 595 | + | observation := stampHostAuthority(ports.BrokerDaemonObservation{}, record, rank) | |
| 596 | + | // A membership the registry cannot publish is refused here, before the | |
| 597 | + | // store CAS, rather than persisted and left to freeze every later | |
| 598 | + | // publication: an endpoint whose derived display hint is empty after the | |
| 599 | + | // display rule drops its runes makes every snapshot carrying it invalid. | |
| 600 | + | if err := observation.Validate(); err != nil { | |
| 601 | + | return fmt.Errorf("broker: host record %q is not publishable: %w", record.Registration.Endpoint, err) | |
| 602 | + | } | |
| 603 | + | next[record.Registration.Endpoint] = observation | |
| 604 | + | order = append(order, record.Registration.Endpoint) | |
| 463 | 605 | } | |
| 464 | 606 | if err := r.hostStore.ReplaceHosts(r.authority.Revision, records); err != nil { | |
| 465 | 607 | return fmt.Errorf("broker: replace hosts: %w", err) | |
| 466 | 608 | } | |
| 467 | - | r.authority = ports.BrokerHosts{Revision: r.authority.Revision + 1, Hosts: records} | |
| 609 | + | r.authority = ports.BrokerHosts{Revision: r.authority.Revision + 1, Hosts: append([]ports.BrokerHostRecord(nil), records...)} | |
| 468 | 610 | r.setHostsLocked(next, order) | |
| 469 | 611 | return nil | |
| 470 | 612 | } | |
| 471 | 613 | ||
| 614 | + | // ReplaceHosts durably replaces membership and explicit policy using the loaded | |
| 615 | + | // authority revision. It delegates to commitMembershipLocked, the same | |
| 616 | + | // validated commit path the runtime mutations use, so membership whose | |
| 617 | + | // projection could never be published is refused before the store CAS instead | |
| 618 | + | // of being persisted and silently freezing every later publication. Success | |
| 619 | + | // means membership is committed, not that advisory observations have flushed. | |
| 620 | + | // Store errors (including stale authority) are returned without changing the | |
| 621 | + | // projection, probe attempts, or local token. A settled run reports | |
| 622 | + | // ports.ErrBrokerRegistryClosed and an exhausted revision series reports | |
| 623 | + | // ports.ErrBrokerRevisionExhausted; both refuse the mutation before the store | |
| 624 | + | // CAS, so a refusal never changes durable authority either. Conflicts are not | |
| 625 | + | // retried or refreshed implicitly: reconcile by reopening the owner. Callers | |
| 626 | + | // must supply fresh incarnations when removing and re-adding. Even identical | |
| 627 | + | // records perform CAS so stale authority cannot report success. The synchronous | |
| 628 | + | // write holds mu; Snapshot remains lock-free, but scheduling waits for | |
| 629 | + | // membership durability. Observation writes remain asynchronous. | |
| 630 | + | func (r *Registry) ReplaceHosts(records []ports.BrokerHostRecord) error { | |
| 631 | + | records = append([]ports.BrokerHostRecord(nil), records...) | |
| 632 | + | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |
| 633 | + | return err | |
| 634 | + | } | |
| 635 | + | r.mu.Lock() | |
| 636 | + | defer r.mu.Unlock() | |
| 637 | + | if r.running.Load() == runStopped { | |
| 638 | + | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRegistryClosed) | |
| 639 | + | } | |
| 640 | + | if r.revisionExhaustedLocked() { | |
| 641 | + | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRevisionExhausted) | |
| 642 | + | } | |
| 643 | + | return r.commitMembershipLocked(records) | |
| 644 | + | } | |
| 645 | + | ||
| 472 | 646 | // setHosts is projection-only; it is private so callers cannot bypass durable | |
| 473 | 647 | // membership and policy. Observation tests use it without claiming durability. | |
| 474 | 648 | // An unchanged membership is a no-op: it neither republishes nor restarts an | |
| @@ -2,6 +2,7 @@ package broker | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | + | "crypto/rand" | |||
| 5 | "encoding/binary" | 6 | "encoding/binary" | ||
| 6 | "errors" | 7 | "errors" | ||
| 7 | "fmt" | 8 | "fmt" | ||
| @@ -39,8 +40,21 @@ const ( | |||||
| 39 | runStopped | 40 | runStopped | ||
| 40 | ) | 41 | ) | ||
| 41 | 42 | ||||
| 42 | - | // RegistryConfig selects how a Registry runs. The zero value observes hosts. | 43 | + | // MembershipMode controls whether runtime membership mutation is admitted. |
| 44 | + | type MembershipMode uint8 | |||
| 45 | + | ||||
| 46 | + | const ( | |||
| 47 | + | MembershipImmutable MembershipMode = iota | |||
| 48 | + | MembershipMutable | |||
| 49 | + | ) | |||
| 50 | + | ||||
| 51 | + | // RegistryConfig selects how a Registry runs. The zero value observes hosts | |||
| 52 | + | // with immutable membership. | |||
| 43 | type RegistryConfig struct { | 53 | type RegistryConfig struct { | ||
| 54 | + | MembershipMode MembershipMode | |||
| 55 | + | // IncarnationGenerator is an injectable entropy seam for mutable additions. | |||
| 56 | + | // Nil uses crypto/rand. | |||
| 57 | + | IncarnationGenerator func() ([16]byte, error) | |||
| 44 | // ObservationDisabled runs the registry as a read-only snapshot owner: it | 58 | // ObservationDisabled runs the registry as a read-only snapshot owner: it | ||
| 45 | // restores and publishes durable state, owns and drains its writer, and | 59 | // restores and publishes durable state, owns and drains its writer, and | ||
| 46 | // honors the Run lifecycle, but issues no probes, arms no timers, and never | 60 | // honors the Run lifecycle, but issues no probes, arms no timers, and never | ||
| @@ -65,6 +79,8 @@ type Registry struct { | |||||
| 65 | // observationDisabled is fixed at construction: a disabled registry never | 79 | // observationDisabled is fixed at construction: a disabled registry never | ||
| 66 | // observes, so it has no probe, timer, or reconcile work to do. | 80 | // observes, so it has no probe, timer, or reconcile work to do. | ||
| 67 | observationDisabled bool | 81 | observationDisabled bool | ||
| 82 | + | membershipMode MembershipMode | |||
| 83 | + | newIncarnation func() ([16]byte, error) | |||
| 68 | 84 | ||||
| 69 | // hostStore and authority are the synchronous membership owner. Only this | 85 | // hostStore and authority are the synchronous membership owner. Only this | ||
| 70 | // registry may mutate the store while it is live; the caller owns Close. | 86 | // registry may mutate the store while it is live; the caller owns Close. | ||
| @@ -144,6 +160,9 @@ func NewRegistry(epoch ports.BrokerEpoch, store ports.BrokerHostStore, probe por | |||||
| 144 | // probe as long as it owns no remote membership. Every other dependency is | 160 | // probe as long as it owns no remote membership. Every other dependency is | ||
| 145 | // required exactly as NewRegistry requires it. | 161 | // required exactly as NewRegistry requires it. | ||
| 146 | func NewRegistryWithConfig(epoch ports.BrokerEpoch, store ports.BrokerHostStore, probe ports.BrokerHostProbe, clock ports.Clock, log *slog.Logger, cfg RegistryConfig) (*Registry, error) { | 162 | func NewRegistryWithConfig(epoch ports.BrokerEpoch, store ports.BrokerHostStore, probe ports.BrokerHostProbe, clock ports.Clock, log *slog.Logger, cfg RegistryConfig) (*Registry, error) { | ||
| 163 | + | if cfg.MembershipMode != MembershipImmutable && cfg.MembershipMode != MembershipMutable { | |||
| 164 | + | return nil, errors.New("broker: invalid membership mode") | |||
| 165 | + | } | |||
| 147 | if epoch == 0 || nilDependency(store) || nilDependency(clock) { | 166 | if epoch == 0 || nilDependency(store) || nilDependency(clock) { | ||
| 148 | return nil, errors.New("broker: invalid registry dependencies") | 167 | return nil, errors.New("broker: invalid registry dependencies") | ||
| 149 | } | 168 | } | ||
| @@ -157,10 +176,15 @@ func NewRegistryWithConfig(epoch ports.BrokerEpoch, store ports.BrokerHostStore, | |||||
| 157 | if log == nil { | 176 | if log == nil { | ||
| 158 | log = slog.Default() | 177 | log = slog.Default() | ||
| 159 | } | 178 | } | ||
| 179 | + | generator := cfg.IncarnationGenerator | |||
| 180 | + | if generator == nil { | |||
| 181 | + | generator = func() (id [16]byte, err error) { _, err = rand.Read(id[:]); return id, err } | |||
| 182 | + | } | |||
| 160 | r := &Registry{ | 183 | r := &Registry{ | ||
| 161 | epoch: epoch, probe: probe, clock: clock, log: log, | 184 | epoch: epoch, probe: probe, clock: clock, log: log, | ||
| 162 | observationDisabled: cfg.ObservationDisabled, | 185 | observationDisabled: cfg.ObservationDisabled, | ||
| 163 | - | hosts: make(map[string]ports.BrokerDaemonObservation), inflight: make(map[string]*probeAttempt), | 186 | + | membershipMode: cfg.MembershipMode, newIncarnation: generator, |
| 187 | + | hosts: make(map[string]ports.BrokerDaemonObservation), inflight: make(map[string]*probeAttempt), | |||
| 164 | pending: make(map[string]bool), tombstones: make(map[string]ports.BrokerHostTombstone), | 188 | pending: make(map[string]bool), tombstones: make(map[string]ports.BrokerHostTombstone), | ||
| 165 | subs: make(map[*subscription]struct{}), wake: make(chan struct{}, 1), | 189 | subs: make(map[*subscription]struct{}), wake: make(chan struct{}, 1), | ||
| 166 | freshFor: defaultFreshFor, retryBase: defaultRetryBase, retryMax: defaultRetryLimit, | 190 | freshFor: defaultFreshFor, retryBase: defaultRetryBase, retryMax: defaultRetryLimit, | ||
| @@ -426,49 +450,199 @@ func (r *Registry) Subscribe() ports.BrokerSubscription { | |||||
| 426 | return s | 450 | return s | ||
| 427 | } | 451 | } | ||
| 428 | 452 | ||||
| 429 | - | // ReplaceHosts durably replaces membership and explicit policy using the loaded | 453 | + | // AddHost adds or pins endpoint under policy. Duplicate same-policy additions |
| 430 | - | // authority revision. Success means membership is committed, not that advisory | 454 | + | // CAS-verify authority and preserve identity; a conflicting policy is refused. |
| 431 | - | // observations have flushed. Store errors (including stale authority) are | 455 | + | func (r *Registry) AddHost(ctx context.Context, endpoint string, policy ports.BrokerPolicy) (domain.RemoteRegistration, error) { |
| 432 | - | // returned without changing the projection, probe attempts, or local token. | 456 | + | if err := domain.ValidateRemoteHostTarget(endpoint); err != nil { |
| 433 | - | // A settled run reports ports.ErrBrokerRegistryClosed and an exhausted revision | 457 | + | return domain.RemoteRegistration{}, err |
| 434 | - | // series reports ports.ErrBrokerRevisionExhausted; both refuse the mutation | |||
| 435 | - | // before the store CAS, so a refusal never changes durable authority either. | |||
| 436 | - | // Conflicts are not retried or refreshed implicitly: reconcile by reopening the | |||
| 437 | - | // owner. Callers must supply fresh incarnations when removing and re-adding. | |||
| 438 | - | // Even identical records perform CAS so stale authority cannot report success. | |||
| 439 | - | // The synchronous write holds mu; Snapshot remains lock-free, but scheduling | |||
| 440 | - | // waits for membership durability. Observation writes remain asynchronous. | |||
| 441 | - | func (r *Registry) ReplaceHosts(records []ports.BrokerHostRecord) error { | |||
| 442 | - | records = append([]ports.BrokerHostRecord(nil), records...) | |||
| 443 | - | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |||
| 444 | - | return err | |||
| 445 | } | 458 | } | ||
| 446 | - | next := make(map[string]ports.BrokerDaemonObservation, len(records)) | 459 | + | if err := policy.Validate(); err != nil { |
| 447 | - | order := make([]string, 0, len(records)) | 460 | + | return domain.RemoteRegistration{}, err |
| 448 | - | for rank, record := range records { | |||
| 449 | - | reg := record.Registration | |||
| 450 | - | next[reg.Endpoint] = stampHostAuthority(ports.BrokerDaemonObservation{}, record, rank) | |||
| 451 | - | order = append(order, reg.Endpoint) | |||
| 452 | } | 461 | } | ||
| 453 | r.mu.Lock() | 462 | r.mu.Lock() | ||
| 454 | defer r.mu.Unlock() | 463 | defer r.mu.Unlock() | ||
| 464 | + | if err := r.mutableLocked(ctx); err != nil { | |||
| 465 | + | return domain.RemoteRegistration{}, err | |||
| 466 | + | } | |||
| 467 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |||
| 468 | + | for i := range records { | |||
| 469 | + | if records[i].Registration.Endpoint != endpoint { | |||
| 470 | + | continue | |||
| 471 | + | } | |||
| 472 | + | if records[i].Policy != policy { | |||
| 473 | + | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |||
| 474 | + | } | |||
| 475 | + | records[i].Pinned = true | |||
| 476 | + | if err := r.commitMembershipLocked(records); err != nil { | |||
| 477 | + | return domain.RemoteRegistration{}, err | |||
| 478 | + | } | |||
| 479 | + | return records[i].Registration, nil | |||
| 480 | + | } | |||
| 481 | + | id, err := r.newIncarnation() | |||
| 482 | + | if err != nil { | |||
| 483 | + | return domain.RemoteRegistration{}, err | |||
| 484 | + | } | |||
| 485 | + | reg, err := domain.NewRemoteRegistration(endpoint, id) | |||
| 486 | + | if err != nil { | |||
| 487 | + | return domain.RemoteRegistration{}, err | |||
| 488 | + | } | |||
| 489 | + | if reg.IsZero() { | |||
| 490 | + | return domain.RemoteRegistration{}, errors.New("broker: incarnation generator returned zero") | |||
| 491 | + | } | |||
| 492 | + | records = append(records, ports.BrokerHostRecord{Registration: reg, Pinned: true, Policy: policy}) | |||
| 493 | + | if err := r.commitMembershipLocked(records); err != nil { | |||
| 494 | + | return domain.RemoteRegistration{}, err | |||
| 495 | + | } | |||
| 496 | + | return reg, nil | |||
| 497 | + | } | |||
| 498 | + | ||||
| 499 | + | // RemoveHost removes only the exact expected registration. | |||
| 500 | + | func (r *Registry) RemoveHost(ctx context.Context, expected domain.RemoteRegistration) (bool, error) { | |||
| 501 | + | if err := expected.Validate(); err != nil { | |||
| 502 | + | return false, err | |||
| 503 | + | } | |||
| 504 | + | r.mu.Lock() | |||
| 505 | + | defer r.mu.Unlock() | |||
| 506 | + | if err := r.mutableLocked(ctx); err != nil { | |||
| 507 | + | return false, err | |||
| 508 | + | } | |||
| 509 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |||
| 510 | + | for i := range records { | |||
| 511 | + | if records[i].Registration.Endpoint != expected.Endpoint { | |||
| 512 | + | continue | |||
| 513 | + | } | |||
| 514 | + | if !records[i].Registration.Equal(expected) { | |||
| 515 | + | return false, ports.ErrBrokerHostConflict | |||
| 516 | + | } | |||
| 517 | + | records = append(records[:i], records[i+1:]...) | |||
| 518 | + | if err := r.commitMembershipLocked(records); err != nil { | |||
| 519 | + | return false, err | |||
| 520 | + | } | |||
| 521 | + | return true, nil | |||
| 522 | + | } | |||
| 523 | + | // A no-op still verifies the loaded revision. | |||
| 524 | + | return false, r.commitMembershipLocked(records) | |||
| 525 | + | } | |||
| 526 | + | ||||
| 527 | + | // UpdateHostPolicy updates exact authority and advances its generation. | |||
| 528 | + | func (r *Registry) UpdateHostPolicy(ctx context.Context, expected domain.RemoteRegistration, policy ports.BrokerPolicy) (domain.RemoteRegistration, error) { | |||
| 529 | + | if err := expected.Validate(); err != nil { | |||
| 530 | + | return domain.RemoteRegistration{}, err | |||
| 531 | + | } | |||
| 532 | + | if err := policy.Validate(); err != nil { | |||
| 533 | + | return domain.RemoteRegistration{}, err | |||
| 534 | + | } | |||
| 535 | + | r.mu.Lock() | |||
| 536 | + | defer r.mu.Unlock() | |||
| 537 | + | if err := r.mutableLocked(ctx); err != nil { | |||
| 538 | + | return domain.RemoteRegistration{}, err | |||
| 539 | + | } | |||
| 540 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |||
| 541 | + | for i := range records { | |||
| 542 | + | if records[i].Registration.Endpoint != expected.Endpoint { | |||
| 543 | + | continue | |||
| 544 | + | } | |||
| 545 | + | if !records[i].Registration.Equal(expected) { | |||
| 546 | + | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |||
| 547 | + | } | |||
| 548 | + | if records[i].Policy == policy { | |||
| 549 | + | if err := r.commitMembershipLocked(records); err != nil { | |||
| 550 | + | return domain.RemoteRegistration{}, err | |||
| 551 | + | } | |||
| 552 | + | return expected, nil | |||
| 553 | + | } | |||
| 554 | + | if expected.Generation == ^domain.RemoteGeneration(0) { | |||
| 555 | + | return domain.RemoteRegistration{}, ports.ErrBrokerRevisionExhausted | |||
| 556 | + | } | |||
| 557 | + | records[i].Registration.Generation++ | |||
| 558 | + | records[i].Policy = policy | |||
| 559 | + | if err := r.commitMembershipLocked(records); err != nil { | |||
| 560 | + | return domain.RemoteRegistration{}, err | |||
| 561 | + | } | |||
| 562 | + | return records[i].Registration, nil | |||
| 563 | + | } | |||
| 564 | + | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |||
| 565 | + | } | |||
| 566 | + | ||||
| 567 | + | func (r *Registry) mutableLocked(ctx context.Context) error { | |||
| 568 | + | if r.membershipMode != MembershipMutable { | |||
| 569 | + | return ports.ErrBrokerMembershipImmutable | |||
| 570 | + | } | |||
| 571 | + | if err := ctx.Err(); err != nil { | |||
| 572 | + | return err | |||
| 573 | + | } | |||
| 455 | if r.running.Load() == runStopped { | 574 | if r.running.Load() == runStopped { | ||
| 456 | - | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRegistryClosed) | 575 | + | return ports.ErrBrokerRegistryClosed |
| 576 | + | } | |||
| 577 | + | if r.revisionExhaustedLocked() { | |||
| 578 | + | return ports.ErrBrokerRevisionExhausted | |||
| 579 | + | } | |||
| 580 | + | return nil | |||
| 581 | + | } | |||
| 582 | + | ||||
| 583 | + | // commitMembershipLocked is the sole mutable-membership durable commit path. | |||
| 584 | + | // It installs projection only after CAS success. | |||
| 585 | + | func (r *Registry) commitMembershipLocked(records []ports.BrokerHostRecord) error { | |||
| 586 | + | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |||
| 587 | + | return err | |||
| 457 | } | 588 | } | ||
| 458 | if len(records) > 0 && !r.observationDisabled && nilDependency(r.probe) { | 589 | if len(records) > 0 && !r.observationDisabled && nilDependency(r.probe) { | ||
| 459 | return errors.New("broker: remote hosts require a host probe") | 590 | return errors.New("broker: remote hosts require a host probe") | ||
| 460 | } | 591 | } | ||
| 461 | - | if r.revisionExhaustedLocked() { | 592 | + | next := make(map[string]ports.BrokerDaemonObservation, len(records)) |
| 462 | - | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRevisionExhausted) | 593 | + | order := make([]string, 0, len(records)) |
| 594 | + | for rank, record := range records { | |||
| 595 | + | observation := stampHostAuthority(ports.BrokerDaemonObservation{}, record, rank) | |||
| 596 | + | // A membership the registry cannot publish is refused here, before the | |||
| 597 | + | // store CAS, rather than persisted and left to freeze every later | |||
| 598 | + | // publication: an endpoint whose derived display hint is empty after the | |||
| 599 | + | // display rule drops its runes makes every snapshot carrying it invalid. | |||
| 600 | + | if err := observation.Validate(); err != nil { | |||
| 601 | + | return fmt.Errorf("broker: host record %q is not publishable: %w", record.Registration.Endpoint, err) | |||
| 602 | + | } | |||
| 603 | + | next[record.Registration.Endpoint] = observation | |||
| 604 | + | order = append(order, record.Registration.Endpoint) | |||
| 463 | } | 605 | } | ||
| 464 | if err := r.hostStore.ReplaceHosts(r.authority.Revision, records); err != nil { | 606 | if err := r.hostStore.ReplaceHosts(r.authority.Revision, records); err != nil { | ||
| 465 | return fmt.Errorf("broker: replace hosts: %w", err) | 607 | return fmt.Errorf("broker: replace hosts: %w", err) | ||
| 466 | } | 608 | } | ||
| 467 | - | r.authority = ports.BrokerHosts{Revision: r.authority.Revision + 1, Hosts: records} | 609 | + | r.authority = ports.BrokerHosts{Revision: r.authority.Revision + 1, Hosts: append([]ports.BrokerHostRecord(nil), records...)} |
| 468 | r.setHostsLocked(next, order) | 610 | r.setHostsLocked(next, order) | ||
| 469 | return nil | 611 | return nil | ||
| 470 | } | 612 | } | ||
| 471 | 613 | ||||
| 614 | + | // ReplaceHosts durably replaces membership and explicit policy using the loaded | |||
| 615 | + | // authority revision. It delegates to commitMembershipLocked, the same | |||
| 616 | + | // validated commit path the runtime mutations use, so membership whose | |||
| 617 | + | // projection could never be published is refused before the store CAS instead | |||
| 618 | + | // of being persisted and silently freezing every later publication. Success | |||
| 619 | + | // means membership is committed, not that advisory observations have flushed. | |||
| 620 | + | // Store errors (including stale authority) are returned without changing the | |||
| 621 | + | // projection, probe attempts, or local token. A settled run reports | |||
| 622 | + | // ports.ErrBrokerRegistryClosed and an exhausted revision series reports | |||
| 623 | + | // ports.ErrBrokerRevisionExhausted; both refuse the mutation before the store | |||
| 624 | + | // CAS, so a refusal never changes durable authority either. Conflicts are not | |||
| 625 | + | // retried or refreshed implicitly: reconcile by reopening the owner. Callers | |||
| 626 | + | // must supply fresh incarnations when removing and re-adding. Even identical | |||
| 627 | + | // records perform CAS so stale authority cannot report success. The synchronous | |||
| 628 | + | // write holds mu; Snapshot remains lock-free, but scheduling waits for | |||
| 629 | + | // membership durability. Observation writes remain asynchronous. | |||
| 630 | + | func (r *Registry) ReplaceHosts(records []ports.BrokerHostRecord) error { | |||
| 631 | + | records = append([]ports.BrokerHostRecord(nil), records...) | |||
| 632 | + | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |||
| 633 | + | return err | |||
| 634 | + | } | |||
| 635 | + | r.mu.Lock() | |||
| 636 | + | defer r.mu.Unlock() | |||
| 637 | + | if r.running.Load() == runStopped { | |||
| 638 | + | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRegistryClosed) | |||
| 639 | + | } | |||
| 640 | + | if r.revisionExhaustedLocked() { | |||
| 641 | + | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRevisionExhausted) | |||
| 642 | + | } | |||
| 643 | + | return r.commitMembershipLocked(records) | |||
| 644 | + | } | |||
| 645 | + | ||||
| 472 | // setHosts is projection-only; it is private so callers cannot bypass durable | 646 | // setHosts is projection-only; it is private so callers cannot bypass durable | ||
| 473 | // membership and policy. Observation tests use it without claiming durability. | 647 | // membership and policy. Observation tests use it without claiming durability. | ||
| 474 | // An unchanged membership is a no-op: it neither republishes nor restarts an | 648 | // An unchanged membership is a no-op: it neither republishes nor restarts an | ||
M internal/usecase/broker/registry_local_test.go +48 -0
| @@ -458,3 +458,51 @@ func TestPoolLocalStreamsShareOnePhysicalEntry(t *testing.T) { | |||
| 458 | 458 | require.NoError(t, second.Close()) | |
| 459 | 459 | require.NoError(t, physical.Close()) | |
| 460 | 460 | } | |
| 461 | + | ||
| 462 | + | // TestRegistryMutableAddHostLocalOnlyNoRemoteProbe proves a mutable registry that | |
| 463 | + | // observes only the broker's own machine daemon (a configured local observation | |
| 464 | + | // and no remote probe) refuses a remote addition outright: the refusal is | |
| 465 | + | // typed, it never reaches the durable store CAS, authority and projection stay | |
| 466 | + | // exactly as loaded, and the registry keeps serving its local entry instead of | |
| 467 | + | // panicking on the absent remote probe. A mutable registry is the interesting | |
| 468 | + | // case because the immutable default refuses before the membership mode is even | |
| 469 | + | // consulted. | |
| 470 | + | func TestRegistryMutableAddHostLocalOnlyNoRemoteProbe(t *testing.T) { | |
| 471 | + | clock := newManualClock(time.Unix(100, 0)) | |
| 472 | + | store := newMembershipStore() | |
| 473 | + | local := newScriptedLocalProbe(1) | |
| 474 | + | registry, err := NewRegistryWithConfig(1, store, nil, clock, nil, RegistryConfig{ | |
| 475 | + | MembershipMode: MembershipMutable, | |
| 476 | + | IncarnationGenerator: (&incarnationSequencer{}).generate, | |
| 477 | + | Local: &LocalObservation{DisplayOrigin: "local", Policy: poolPolicy(), Probe: local}, | |
| 478 | + | }) | |
| 479 | + | require.NoError(t, err) | |
| 480 | + | registry.jitter = identityJitter | |
| 481 | + | ||
| 482 | + | authority := registry.authority | |
| 483 | + | before := registry.Snapshot() | |
| 484 | + | require.NotPanics(t, func() { | |
| 485 | + | _, err = registry.AddHost(context.Background(), "user@host:22", poolPolicy()) | |
| 486 | + | }, "an absent remote probe must be refused, never dereferenced") | |
| 487 | + | require.Error(t, err) | |
| 488 | + | require.Contains(t, err.Error(), "remote hosts require a host probe") | |
| 489 | + | require.Equal(t, 0, store.casCalls(), "the refusal happens before the store CAS") | |
| 490 | + | require.Equal(t, authority, registry.authority, "a refused add never moves authority") | |
| 491 | + | require.Equal(t, before, registry.Snapshot(), "a refused add never publishes") | |
| 492 | + | ||
| 493 | + | // The local entry stays observable: the refusal never leaves the registry | |
| 494 | + | // unable to publish what it still owns. | |
| 495 | + | startRegistry(t, registry) | |
| 496 | + | call := receiveLocalCall(t, local) | |
| 497 | + | call.result <- localProbeAnswer{snapshot: ports.BrokerDaemonObservation{ | |
| 498 | + | Identity: "local-daemon", | |
| 499 | + | Incarnation: ports.BrokerDaemonIncarnation{9}, | |
| 500 | + | ProtocolVersion: protocol.Version, | |
| 501 | + | Availability: domain.RemoteAvailabilityReachable, | |
| 502 | + | }} | |
| 503 | + | host := waitLocal(t, registry, func(o ports.BrokerDaemonObservation) bool { | |
| 504 | + | return !o.Checking && o.Availability == domain.RemoteAvailabilityReachable | |
| 505 | + | }) | |
| 506 | + | require.True(t, host.Local) | |
| 507 | + | require.Len(t, registry.Snapshot().Daemons, 1) | |
| 508 | + | } | |
| @@ -458,3 +458,51 @@ func TestPoolLocalStreamsShareOnePhysicalEntry(t *testing.T) { | |||||
| 458 | require.NoError(t, second.Close()) | 458 | require.NoError(t, second.Close()) | ||
| 459 | require.NoError(t, physical.Close()) | 459 | require.NoError(t, physical.Close()) | ||
| 460 | } | 460 | } | ||
| 461 | + | ||||
| 462 | + | // TestRegistryMutableAddHostLocalOnlyNoRemoteProbe proves a mutable registry that | |||
| 463 | + | // observes only the broker's own machine daemon (a configured local observation | |||
| 464 | + | // and no remote probe) refuses a remote addition outright: the refusal is | |||
| 465 | + | // typed, it never reaches the durable store CAS, authority and projection stay | |||
| 466 | + | // exactly as loaded, and the registry keeps serving its local entry instead of | |||
| 467 | + | // panicking on the absent remote probe. A mutable registry is the interesting | |||
| 468 | + | // case because the immutable default refuses before the membership mode is even | |||
| 469 | + | // consulted. | |||
| 470 | + | func TestRegistryMutableAddHostLocalOnlyNoRemoteProbe(t *testing.T) { | |||
| 471 | + | clock := newManualClock(time.Unix(100, 0)) | |||
| 472 | + | store := newMembershipStore() | |||
| 473 | + | local := newScriptedLocalProbe(1) | |||
| 474 | + | registry, err := NewRegistryWithConfig(1, store, nil, clock, nil, RegistryConfig{ | |||
| 475 | + | MembershipMode: MembershipMutable, | |||
| 476 | + | IncarnationGenerator: (&incarnationSequencer{}).generate, | |||
| 477 | + | Local: &LocalObservation{DisplayOrigin: "local", Policy: poolPolicy(), Probe: local}, | |||
| 478 | + | }) | |||
| 479 | + | require.NoError(t, err) | |||
| 480 | + | registry.jitter = identityJitter | |||
| 481 | + | ||||
| 482 | + | authority := registry.authority | |||
| 483 | + | before := registry.Snapshot() | |||
| 484 | + | require.NotPanics(t, func() { | |||
| 485 | + | _, err = registry.AddHost(context.Background(), "user@host:22", poolPolicy()) | |||
| 486 | + | }, "an absent remote probe must be refused, never dereferenced") | |||
| 487 | + | require.Error(t, err) | |||
| 488 | + | require.Contains(t, err.Error(), "remote hosts require a host probe") | |||
| 489 | + | require.Equal(t, 0, store.casCalls(), "the refusal happens before the store CAS") | |||
| 490 | + | require.Equal(t, authority, registry.authority, "a refused add never moves authority") | |||
| 491 | + | require.Equal(t, before, registry.Snapshot(), "a refused add never publishes") | |||
| 492 | + | ||||
| 493 | + | // The local entry stays observable: the refusal never leaves the registry | |||
| 494 | + | // unable to publish what it still owns. | |||
| 495 | + | startRegistry(t, registry) | |||
| 496 | + | call := receiveLocalCall(t, local) | |||
| 497 | + | call.result <- localProbeAnswer{snapshot: ports.BrokerDaemonObservation{ | |||
| 498 | + | Identity: "local-daemon", | |||
| 499 | + | Incarnation: ports.BrokerDaemonIncarnation{9}, | |||
| 500 | + | ProtocolVersion: protocol.Version, | |||
| 501 | + | Availability: domain.RemoteAvailabilityReachable, | |||
| 502 | + | }} | |||
| 503 | + | host := waitLocal(t, registry, func(o ports.BrokerDaemonObservation) bool { | |||
| 504 | + | return !o.Checking && o.Availability == domain.RemoteAvailabilityReachable | |||
| 505 | + | }) | |||
| 506 | + | require.True(t, host.Local) | |||
| 507 | + | require.Len(t, registry.Snapshot().Daemons, 1) | |||
| 508 | + | } | |||
A internal/usecase/broker/registry_membership_durable_test.go +280 -0
| @@ -0,0 +1,280 @@ | |||
| 1 | + | package broker | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "errors" | |
| 6 | + | "path/filepath" | |
| 7 | + | "sync/atomic" | |
| 8 | + | "testing" | |
| 9 | + | "time" | |
| 10 | + | ||
| 11 | + | "github.com/bnema/vev/internal/adapters/brokerstore" | |
| 12 | + | "github.com/bnema/vev/internal/domain" | |
| 13 | + | "github.com/bnema/vev/internal/ports" | |
| 14 | + | "github.com/stretchr/testify/require" | |
| 15 | + | ) | |
| 16 | + | ||
| 17 | + | // Durable mutable membership: the same public API exercised against the real | |
| 18 | + | // offline store, which owns the compare-and-swap and the snapshot validator. | |
| 19 | + | // Every stage reopens both owners, so membership and its revision token must | |
| 20 | + | // come from disk rather than from the previous process's memory. | |
| 21 | + | ||
| 22 | + | // newDurableMutableRegistry opens a mutable registry over a real store. The | |
| 23 | + | // registry is observation-disabled: no probe, timer, or reconciliation runs, so | |
| 24 | + | // the test observes only the durable membership path. | |
| 25 | + | func newDurableMutableRegistry(t *testing.T, epoch ports.BrokerEpoch, store ports.BrokerHostStore) *Registry { | |
| 26 | + | t.Helper() | |
| 27 | + | registry, err := NewRegistryWithConfig(epoch, store, nil, newManualClock(time.Unix(100, 0)), nil, RegistryConfig{ | |
| 28 | + | MembershipMode: MembershipMutable, | |
| 29 | + | ObservationDisabled: true, | |
| 30 | + | }) | |
| 31 | + | require.NoError(t, err) | |
| 32 | + | registry.jitter = identityJitter | |
| 33 | + | return registry | |
| 34 | + | } | |
| 35 | + | ||
| 36 | + | func openDurableStore(t *testing.T, dir string, fault func(string) error) *brokerstore.Store { | |
| 37 | + | t.Helper() | |
| 38 | + | store, err := brokerstore.OpenOffline(brokerstore.Options{Dir: dir, Fault: fault}) | |
| 39 | + | require.NoError(t, err) | |
| 40 | + | t.Cleanup(func() { require.NoError(t, store.Close()) }) | |
| 41 | + | return store | |
| 42 | + | } | |
| 43 | + | ||
| 44 | + | // TestRegistryMutableMembershipDurableRestart proves a committed add, update, | |
| 45 | + | // and remove all survive a restart through the real store, and that the | |
| 46 | + | // remove/re-add fence is minted fresh by the mutation path rather than restored. | |
| 47 | + | func TestRegistryMutableMembershipDurableRestart(t *testing.T) { | |
| 48 | + | dir := filepath.Join(t.TempDir(), "broker") | |
| 49 | + | const endpoint = "user@arch:22" | |
| 50 | + | first := poolPolicy() | |
| 51 | + | changed := policyWithTrust("changed-trust") | |
| 52 | + | ||
| 53 | + | var ( | |
| 54 | + | added domain.RemoteRegistration | |
| 55 | + | updated domain.RemoteRegistration | |
| 56 | + | readded domain.RemoteRegistration | |
| 57 | + | revision uint64 | |
| 58 | + | ) | |
| 59 | + | ||
| 60 | + | t.Run("add commits durably", func(t *testing.T) { | |
| 61 | + | store := openDurableStore(t, dir, nil) | |
| 62 | + | registry := newDurableMutableRegistry(t, 1, store) | |
| 63 | + | defer registry.settle() | |
| 64 | + | require.Empty(t, registry.authority.Hosts) | |
| 65 | + | revision = registry.authority.Revision | |
| 66 | + | ||
| 67 | + | reg, err := registry.AddHost(context.Background(), endpoint, first) | |
| 68 | + | require.NoError(t, err) | |
| 69 | + | require.Equal(t, domain.RemoteGeneration(1), reg.Generation) | |
| 70 | + | require.False(t, reg.IsZero()) | |
| 71 | + | added = reg | |
| 72 | + | ||
| 73 | + | revision++ | |
| 74 | + | require.Equal(t, revision, registry.authority.Revision) | |
| 75 | + | authority, err := store.LoadHosts() | |
| 76 | + | require.NoError(t, err) | |
| 77 | + | require.Equal(t, registry.authority, authority) | |
| 78 | + | require.Equal(t, []ports.BrokerHostRecord{{Registration: reg, Pinned: true, Policy: first}}, authority.Hosts) | |
| 79 | + | registry.settle() | |
| 80 | + | }) | |
| 81 | + | ||
| 82 | + | t.Run("restart restores the add and admits an update", func(t *testing.T) { | |
| 83 | + | store := openDurableStore(t, dir, nil) | |
| 84 | + | registry := newDurableMutableRegistry(t, 2, store) | |
| 85 | + | defer registry.settle() | |
| 86 | + | require.Equal(t, revision, registry.authority.Revision) | |
| 87 | + | require.Equal(t, []ports.BrokerHostRecord{{Registration: added, Pinned: true, Policy: first}}, registry.authority.Hosts) | |
| 88 | + | host, ok := registry.Snapshot().Find(endpoint) | |
| 89 | + | require.True(t, ok) | |
| 90 | + | require.Equal(t, added, host.Registration) | |
| 91 | + | require.Equal(t, first, host.Policy, "policy is re-stamped from restored membership") | |
| 92 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |
| 93 | + | ||
| 94 | + | reg, err := registry.UpdateHostPolicy(context.Background(), added, changed) | |
| 95 | + | require.NoError(t, err) | |
| 96 | + | require.Equal(t, added.Generation+1, reg.Generation) | |
| 97 | + | require.Equal(t, added.Incarnation, reg.Incarnation) | |
| 98 | + | updated = reg | |
| 99 | + | ||
| 100 | + | revision++ | |
| 101 | + | require.Equal(t, revision, registry.authority.Revision) | |
| 102 | + | authority, err := store.LoadHosts() | |
| 103 | + | require.NoError(t, err) | |
| 104 | + | require.Equal(t, registry.authority, authority) | |
| 105 | + | require.Equal(t, []ports.BrokerHostRecord{{Registration: updated, Pinned: true, Policy: changed}}, authority.Hosts) | |
| 106 | + | registry.settle() | |
| 107 | + | }) | |
| 108 | + | ||
| 109 | + | t.Run("restart restores the update and admits a remove", func(t *testing.T) { | |
| 110 | + | store := openDurableStore(t, dir, nil) | |
| 111 | + | registry := newDurableMutableRegistry(t, 3, store) | |
| 112 | + | defer registry.settle() | |
| 113 | + | require.Equal(t, revision, registry.authority.Revision) | |
| 114 | + | require.Equal(t, []ports.BrokerHostRecord{{Registration: updated, Pinned: true, Policy: changed}}, registry.authority.Hosts) | |
| 115 | + | ||
| 116 | + | removed, err := registry.RemoveHost(context.Background(), updated) | |
| 117 | + | require.NoError(t, err) | |
| 118 | + | require.True(t, removed) | |
| 119 | + | ||
| 120 | + | revision++ | |
| 121 | + | require.Equal(t, revision, registry.authority.Revision) | |
| 122 | + | require.Empty(t, registry.authority.Hosts) | |
| 123 | + | authority, err := store.LoadHosts() | |
| 124 | + | require.NoError(t, err) | |
| 125 | + | require.Equal(t, registry.authority, authority) | |
| 126 | + | require.Empty(t, authority.Hosts) | |
| 127 | + | require.Empty(t, registry.Snapshot().Daemons) | |
| 128 | + | registry.settle() | |
| 129 | + | }) | |
| 130 | + | ||
| 131 | + | t.Run("restart restores the removal and re-adds with a fresh incarnation", func(t *testing.T) { | |
| 132 | + | store := openDurableStore(t, dir, nil) | |
| 133 | + | registry := newDurableMutableRegistry(t, 4, store) | |
| 134 | + | defer registry.settle() | |
| 135 | + | require.Equal(t, revision, registry.authority.Revision) | |
| 136 | + | require.Empty(t, registry.authority.Hosts) | |
| 137 | + | ||
| 138 | + | reg, err := registry.AddHost(context.Background(), endpoint, changed) | |
| 139 | + | require.NoError(t, err) | |
| 140 | + | require.False(t, updated.Equal(reg), "a re-add mints a fresh identity instead of reviving the retired one") | |
| 141 | + | require.NotEqual(t, updated.Incarnation, reg.Incarnation) | |
| 142 | + | require.Equal(t, domain.RemoteGeneration(1), reg.Generation) | |
| 143 | + | readded = reg | |
| 144 | + | ||
| 145 | + | revision++ | |
| 146 | + | require.Equal(t, revision, registry.authority.Revision) | |
| 147 | + | authority, err := store.LoadHosts() | |
| 148 | + | require.NoError(t, err) | |
| 149 | + | require.Equal(t, registry.authority, authority) | |
| 150 | + | registry.settle() | |
| 151 | + | }) | |
| 152 | + | ||
| 153 | + | t.Run("restart restores the re-add", func(t *testing.T) { | |
| 154 | + | store := openDurableStore(t, dir, nil) | |
| 155 | + | registry := newDurableMutableRegistry(t, 5, store) | |
| 156 | + | defer registry.settle() | |
| 157 | + | require.Equal(t, revision, registry.authority.Revision) | |
| 158 | + | require.Equal(t, []ports.BrokerHostRecord{{Registration: readded, Pinned: true, Policy: changed}}, registry.authority.Hosts) | |
| 159 | + | host, ok := registry.Snapshot().Find(endpoint) | |
| 160 | + | require.True(t, ok) | |
| 161 | + | require.Equal(t, readded, host.Registration) | |
| 162 | + | require.False(t, updated.Equal(host.Registration), "the retired incarnation is never restored") | |
| 163 | + | }) | |
| 164 | + | } | |
| 165 | + | ||
| 166 | + | // TestRegistryReplaceHostsRefusesUnpublishableMembershipBeforeCAS proves the | |
| 167 | + | // public replacement path shares the runtime mutations' validated commit rule: a | |
| 168 | + | // caller-supplied target whose derived display hint is empty after the display | |
| 169 | + | // rule drops its bidi/control runes is refused by projection validation, before | |
| 170 | + | // the store CAS. Nothing is persisted, nothing is published, and a restart still | |
| 171 | + | // loads valid membership. The routing validator accepts the target, so this is | |
| 172 | + | // exactly the membership the store alone would have committed. | |
| 173 | + | func TestRegistryReplaceHostsRefusesUnpublishableMembershipBeforeCAS(t *testing.T) { | |
| 174 | + | dir := filepath.Join(t.TempDir(), "broker") | |
| 175 | + | valid := ports.BrokerHostRecord{Registration: registration(t, "user@arch:22", 1), Pinned: true, Policy: poolPolicy()} | |
| 176 | + | store := openDurableStore(t, dir, nil) | |
| 177 | + | require.NoError(t, store.ReplaceHosts(1, []ports.BrokerHostRecord{valid})) | |
| 178 | + | ||
| 179 | + | registry, err := NewRegistryWithConfig(1, store, nil, newManualClock(time.Unix(100, 0)), nil, RegistryConfig{ObservationDisabled: true}) | |
| 180 | + | require.NoError(t, err) | |
| 181 | + | defer registry.settle() | |
| 182 | + | authority := registry.authority | |
| 183 | + | before := registry.Snapshot() | |
| 184 | + | revision := registry.revision | |
| 185 | + | persistedBefore, err := store.Load() | |
| 186 | + | require.NoError(t, err) | |
| 187 | + | require.NotZero(t, authority.Revision) | |
| 188 | + | require.Len(t, before.Daemons, 1) | |
| 189 | + | ||
| 190 | + | poisoned := ports.BrokerHostRecord{Registration: registration(t, "user@\u202e", 2), Pinned: true, Policy: poolPolicy()} | |
| 191 | + | require.NoError(t, domain.ValidateRemoteHostTarget(poisoned.Registration.Endpoint), "the routing validator accepts the target") | |
| 192 | + | require.Empty(t, hostDisplayOrigin(poisoned.Registration.Endpoint), "only the derived hint is empty after sanitizing") | |
| 193 | + | ||
| 194 | + | require.ErrorContains(t, registry.ReplaceHosts([]ports.BrokerHostRecord{poisoned}), "not publishable") | |
| 195 | + | require.Equal(t, authority, registry.authority, "a refused replacement never moves authority") | |
| 196 | + | require.Equal(t, before, registry.Snapshot(), "a refused replacement never publishes") | |
| 197 | + | require.Equal(t, revision, registry.revision, "a refused replacement never advances the publication series") | |
| 198 | + | ||
| 199 | + | // Nothing reached the durable store, even after the writer flushes: the CAS | |
| 200 | + | // was never crossed, and the advisory snapshot is still the published one. | |
| 201 | + | registry.settle() | |
| 202 | + | durableHosts, err := store.LoadHosts() | |
| 203 | + | require.NoError(t, err) | |
| 204 | + | require.Equal(t, authority, durableHosts) | |
| 205 | + | require.Equal(t, authority.Revision, durableHosts.Revision, "the refused replacement never crossed the store CAS") | |
| 206 | + | persisted, err := store.Load() | |
| 207 | + | require.NoError(t, err) | |
| 208 | + | require.Equal(t, persistedBefore, persisted, "the refused replacement never wrote a snapshot") | |
| 209 | + | ||
| 210 | + | // A restart remains valid: the refused membership was never stored, so | |
| 211 | + | // reopening restores and publishes the original host. | |
| 212 | + | require.NoError(t, store.Close()) | |
| 213 | + | reopened := openDurableStore(t, dir, nil) | |
| 214 | + | restarted, err := NewRegistryWithConfig(2, reopened, nil, newManualClock(time.Unix(200, 0)), nil, RegistryConfig{ObservationDisabled: true}) | |
| 215 | + | require.NoError(t, err) | |
| 216 | + | defer restarted.settle() | |
| 217 | + | require.Equal(t, authority, restarted.authority) | |
| 218 | + | restored := restarted.Snapshot() | |
| 219 | + | require.NoError(t, restored.Validate()) | |
| 220 | + | require.Len(t, restored.Daemons, 1) | |
| 221 | + | require.Equal(t, valid.Registration, restored.Daemons[0].Registration) | |
| 222 | + | } | |
| 223 | + | ||
| 224 | + | // TestRegistryMutableOutcomeUnknownRequiresReopen pins the indeterminate-outcome | |
| 225 | + | // contract against the real store: a failure after the durable commit point is | |
| 226 | + | // classified through the registry, is never acknowledged as membership, and | |
| 227 | + | // leaves the store poisoned until it is reopened and authority is reloaded. | |
| 228 | + | // Reopening reveals that the mutation did commit, which is exactly why the | |
| 229 | + | // registry may not report it either way. | |
| 230 | + | func TestRegistryMutableOutcomeUnknownRequiresReopen(t *testing.T) { | |
| 231 | + | dir := filepath.Join(t.TempDir(), "broker") | |
| 232 | + | fault := errors.New("power loss") | |
| 233 | + | // The fault is armed only after OpenOffline so the store can be created and | |
| 234 | + | // seeded normally; state.json:rename fires after the rename, so the commit | |
| 235 | + | // point was already crossed when the failure is reported. | |
| 236 | + | var armed atomic.Bool | |
| 237 | + | store := openDurableStore(t, dir, func(point string) error { | |
| 238 | + | if armed.Load() && point == "state.json:rename" { | |
| 239 | + | return fault | |
| 240 | + | } | |
| 241 | + | return nil | |
| 242 | + | }) | |
| 243 | + | registry := newDurableMutableRegistry(t, 1, store) | |
| 244 | + | defer registry.settle() | |
| 245 | + | before := registry.Snapshot() | |
| 246 | + | authority := registry.authority | |
| 247 | + | armed.Store(true) | |
| 248 | + | ||
| 249 | + | _, err := registry.AddHost(context.Background(), "user@arch:22", poolPolicy()) | |
| 250 | + | require.Error(t, err) | |
| 251 | + | var unknown ports.BrokerStoreOutcomeUnknownError | |
| 252 | + | require.ErrorAs(t, err, &unknown, "a post-commit failure is classified as an unknown outcome") | |
| 253 | + | require.ErrorIs(t, err, fault, "the underlying store failure stays inspectable") | |
| 254 | + | require.Equal(t, authority, registry.authority, "an unknown outcome never becomes acknowledged membership") | |
| 255 | + | require.Equal(t, before, registry.Snapshot()) | |
| 256 | + | ||
| 257 | + | // The store handle is poisoned: every later operation refuses instead of | |
| 258 | + | // reading or writing a state it cannot vouch for. | |
| 259 | + | _, err = store.LoadHosts() | |
| 260 | + | require.ErrorContains(t, err, "reopen required") | |
| 261 | + | require.ErrorAs(t, err, &unknown) | |
| 262 | + | require.Empty(t, registry.Snapshot().Daemons) | |
| 263 | + | ||
| 264 | + | // Reopening reloads authority from disk: the rename already published the | |
| 265 | + | // new membership, so the only safe recovery is to reopen rather than retry. | |
| 266 | + | require.NoError(t, store.Close()) | |
| 267 | + | armed.Store(false) | |
| 268 | + | reopened := openDurableStore(t, dir, nil) | |
| 269 | + | hosts, err := reopened.LoadHosts() | |
| 270 | + | require.NoError(t, err) | |
| 271 | + | require.Equal(t, authority.Revision+1, hosts.Revision) | |
| 272 | + | require.Len(t, hosts.Hosts, 1) | |
| 273 | + | require.Equal(t, "user@arch:22", hosts.Hosts[0].Registration.Endpoint) | |
| 274 | + | require.Equal(t, poolPolicy(), hosts.Hosts[0].Policy) | |
| 275 | + | ||
| 276 | + | recovered := newDurableMutableRegistry(t, 2, reopened) | |
| 277 | + | defer recovered.settle() | |
| 278 | + | require.Len(t, recovered.authority.Hosts, 1) | |
| 279 | + | require.Equal(t, hosts.Hosts[0].Registration, recovered.authority.Hosts[0].Registration) | |
| 280 | + | } | |
| @@ -0,0 +1,280 @@ | |||||
| 1 | + | package broker | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "errors" | |||
| 6 | + | "path/filepath" | |||
| 7 | + | "sync/atomic" | |||
| 8 | + | "testing" | |||
| 9 | + | "time" | |||
| 10 | + | ||||
| 11 | + | "github.com/bnema/vev/internal/adapters/brokerstore" | |||
| 12 | + | "github.com/bnema/vev/internal/domain" | |||
| 13 | + | "github.com/bnema/vev/internal/ports" | |||
| 14 | + | "github.com/stretchr/testify/require" | |||
| 15 | + | ) | |||
| 16 | + | ||||
| 17 | + | // Durable mutable membership: the same public API exercised against the real | |||
| 18 | + | // offline store, which owns the compare-and-swap and the snapshot validator. | |||
| 19 | + | // Every stage reopens both owners, so membership and its revision token must | |||
| 20 | + | // come from disk rather than from the previous process's memory. | |||
| 21 | + | ||||
| 22 | + | // newDurableMutableRegistry opens a mutable registry over a real store. The | |||
| 23 | + | // registry is observation-disabled: no probe, timer, or reconciliation runs, so | |||
| 24 | + | // the test observes only the durable membership path. | |||
| 25 | + | func newDurableMutableRegistry(t *testing.T, epoch ports.BrokerEpoch, store ports.BrokerHostStore) *Registry { | |||
| 26 | + | t.Helper() | |||
| 27 | + | registry, err := NewRegistryWithConfig(epoch, store, nil, newManualClock(time.Unix(100, 0)), nil, RegistryConfig{ | |||
| 28 | + | MembershipMode: MembershipMutable, | |||
| 29 | + | ObservationDisabled: true, | |||
| 30 | + | }) | |||
| 31 | + | require.NoError(t, err) | |||
| 32 | + | registry.jitter = identityJitter | |||
| 33 | + | return registry | |||
| 34 | + | } | |||
| 35 | + | ||||
| 36 | + | func openDurableStore(t *testing.T, dir string, fault func(string) error) *brokerstore.Store { | |||
| 37 | + | t.Helper() | |||
| 38 | + | store, err := brokerstore.OpenOffline(brokerstore.Options{Dir: dir, Fault: fault}) | |||
| 39 | + | require.NoError(t, err) | |||
| 40 | + | t.Cleanup(func() { require.NoError(t, store.Close()) }) | |||
| 41 | + | return store | |||
| 42 | + | } | |||
| 43 | + | ||||
| 44 | + | // TestRegistryMutableMembershipDurableRestart proves a committed add, update, | |||
| 45 | + | // and remove all survive a restart through the real store, and that the | |||
| 46 | + | // remove/re-add fence is minted fresh by the mutation path rather than restored. | |||
| 47 | + | func TestRegistryMutableMembershipDurableRestart(t *testing.T) { | |||
| 48 | + | dir := filepath.Join(t.TempDir(), "broker") | |||
| 49 | + | const endpoint = "user@arch:22" | |||
| 50 | + | first := poolPolicy() | |||
| 51 | + | changed := policyWithTrust("changed-trust") | |||
| 52 | + | ||||
| 53 | + | var ( | |||
| 54 | + | added domain.RemoteRegistration | |||
| 55 | + | updated domain.RemoteRegistration | |||
| 56 | + | readded domain.RemoteRegistration | |||
| 57 | + | revision uint64 | |||
| 58 | + | ) | |||
| 59 | + | ||||
| 60 | + | t.Run("add commits durably", func(t *testing.T) { | |||
| 61 | + | store := openDurableStore(t, dir, nil) | |||
| 62 | + | registry := newDurableMutableRegistry(t, 1, store) | |||
| 63 | + | defer registry.settle() | |||
| 64 | + | require.Empty(t, registry.authority.Hosts) | |||
| 65 | + | revision = registry.authority.Revision | |||
| 66 | + | ||||
| 67 | + | reg, err := registry.AddHost(context.Background(), endpoint, first) | |||
| 68 | + | require.NoError(t, err) | |||
| 69 | + | require.Equal(t, domain.RemoteGeneration(1), reg.Generation) | |||
| 70 | + | require.False(t, reg.IsZero()) | |||
| 71 | + | added = reg | |||
| 72 | + | ||||
| 73 | + | revision++ | |||
| 74 | + | require.Equal(t, revision, registry.authority.Revision) | |||
| 75 | + | authority, err := store.LoadHosts() | |||
| 76 | + | require.NoError(t, err) | |||
| 77 | + | require.Equal(t, registry.authority, authority) | |||
| 78 | + | require.Equal(t, []ports.BrokerHostRecord{{Registration: reg, Pinned: true, Policy: first}}, authority.Hosts) | |||
| 79 | + | registry.settle() | |||
| 80 | + | }) | |||
| 81 | + | ||||
| 82 | + | t.Run("restart restores the add and admits an update", func(t *testing.T) { | |||
| 83 | + | store := openDurableStore(t, dir, nil) | |||
| 84 | + | registry := newDurableMutableRegistry(t, 2, store) | |||
| 85 | + | defer registry.settle() | |||
| 86 | + | require.Equal(t, revision, registry.authority.Revision) | |||
| 87 | + | require.Equal(t, []ports.BrokerHostRecord{{Registration: added, Pinned: true, Policy: first}}, registry.authority.Hosts) | |||
| 88 | + | host, ok := registry.Snapshot().Find(endpoint) | |||
| 89 | + | require.True(t, ok) | |||
| 90 | + | require.Equal(t, added, host.Registration) | |||
| 91 | + | require.Equal(t, first, host.Policy, "policy is re-stamped from restored membership") | |||
| 92 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |||
| 93 | + | ||||
| 94 | + | reg, err := registry.UpdateHostPolicy(context.Background(), added, changed) | |||
| 95 | + | require.NoError(t, err) | |||
| 96 | + | require.Equal(t, added.Generation+1, reg.Generation) | |||
| 97 | + | require.Equal(t, added.Incarnation, reg.Incarnation) | |||
| 98 | + | updated = reg | |||
| 99 | + | ||||
| 100 | + | revision++ | |||
| 101 | + | require.Equal(t, revision, registry.authority.Revision) | |||
| 102 | + | authority, err := store.LoadHosts() | |||
| 103 | + | require.NoError(t, err) | |||
| 104 | + | require.Equal(t, registry.authority, authority) | |||
| 105 | + | require.Equal(t, []ports.BrokerHostRecord{{Registration: updated, Pinned: true, Policy: changed}}, authority.Hosts) | |||
| 106 | + | registry.settle() | |||
| 107 | + | }) | |||
| 108 | + | ||||
| 109 | + | t.Run("restart restores the update and admits a remove", func(t *testing.T) { | |||
| 110 | + | store := openDurableStore(t, dir, nil) | |||
| 111 | + | registry := newDurableMutableRegistry(t, 3, store) | |||
| 112 | + | defer registry.settle() | |||
| 113 | + | require.Equal(t, revision, registry.authority.Revision) | |||
| 114 | + | require.Equal(t, []ports.BrokerHostRecord{{Registration: updated, Pinned: true, Policy: changed}}, registry.authority.Hosts) | |||
| 115 | + | ||||
| 116 | + | removed, err := registry.RemoveHost(context.Background(), updated) | |||
| 117 | + | require.NoError(t, err) | |||
| 118 | + | require.True(t, removed) | |||
| 119 | + | ||||
| 120 | + | revision++ | |||
| 121 | + | require.Equal(t, revision, registry.authority.Revision) | |||
| 122 | + | require.Empty(t, registry.authority.Hosts) | |||
| 123 | + | authority, err := store.LoadHosts() | |||
| 124 | + | require.NoError(t, err) | |||
| 125 | + | require.Equal(t, registry.authority, authority) | |||
| 126 | + | require.Empty(t, authority.Hosts) | |||
| 127 | + | require.Empty(t, registry.Snapshot().Daemons) | |||
| 128 | + | registry.settle() | |||
| 129 | + | }) | |||
| 130 | + | ||||
| 131 | + | t.Run("restart restores the removal and re-adds with a fresh incarnation", func(t *testing.T) { | |||
| 132 | + | store := openDurableStore(t, dir, nil) | |||
| 133 | + | registry := newDurableMutableRegistry(t, 4, store) | |||
| 134 | + | defer registry.settle() | |||
| 135 | + | require.Equal(t, revision, registry.authority.Revision) | |||
| 136 | + | require.Empty(t, registry.authority.Hosts) | |||
| 137 | + | ||||
| 138 | + | reg, err := registry.AddHost(context.Background(), endpoint, changed) | |||
| 139 | + | require.NoError(t, err) | |||
| 140 | + | require.False(t, updated.Equal(reg), "a re-add mints a fresh identity instead of reviving the retired one") | |||
| 141 | + | require.NotEqual(t, updated.Incarnation, reg.Incarnation) | |||
| 142 | + | require.Equal(t, domain.RemoteGeneration(1), reg.Generation) | |||
| 143 | + | readded = reg | |||
| 144 | + | ||||
| 145 | + | revision++ | |||
| 146 | + | require.Equal(t, revision, registry.authority.Revision) | |||
| 147 | + | authority, err := store.LoadHosts() | |||
| 148 | + | require.NoError(t, err) | |||
| 149 | + | require.Equal(t, registry.authority, authority) | |||
| 150 | + | registry.settle() | |||
| 151 | + | }) | |||
| 152 | + | ||||
| 153 | + | t.Run("restart restores the re-add", func(t *testing.T) { | |||
| 154 | + | store := openDurableStore(t, dir, nil) | |||
| 155 | + | registry := newDurableMutableRegistry(t, 5, store) | |||
| 156 | + | defer registry.settle() | |||
| 157 | + | require.Equal(t, revision, registry.authority.Revision) | |||
| 158 | + | require.Equal(t, []ports.BrokerHostRecord{{Registration: readded, Pinned: true, Policy: changed}}, registry.authority.Hosts) | |||
| 159 | + | host, ok := registry.Snapshot().Find(endpoint) | |||
| 160 | + | require.True(t, ok) | |||
| 161 | + | require.Equal(t, readded, host.Registration) | |||
| 162 | + | require.False(t, updated.Equal(host.Registration), "the retired incarnation is never restored") | |||
| 163 | + | }) | |||
| 164 | + | } | |||
| 165 | + | ||||
| 166 | + | // TestRegistryReplaceHostsRefusesUnpublishableMembershipBeforeCAS proves the | |||
| 167 | + | // public replacement path shares the runtime mutations' validated commit rule: a | |||
| 168 | + | // caller-supplied target whose derived display hint is empty after the display | |||
| 169 | + | // rule drops its bidi/control runes is refused by projection validation, before | |||
| 170 | + | // the store CAS. Nothing is persisted, nothing is published, and a restart still | |||
| 171 | + | // loads valid membership. The routing validator accepts the target, so this is | |||
| 172 | + | // exactly the membership the store alone would have committed. | |||
| 173 | + | func TestRegistryReplaceHostsRefusesUnpublishableMembershipBeforeCAS(t *testing.T) { | |||
| 174 | + | dir := filepath.Join(t.TempDir(), "broker") | |||
| 175 | + | valid := ports.BrokerHostRecord{Registration: registration(t, "user@arch:22", 1), Pinned: true, Policy: poolPolicy()} | |||
| 176 | + | store := openDurableStore(t, dir, nil) | |||
| 177 | + | require.NoError(t, store.ReplaceHosts(1, []ports.BrokerHostRecord{valid})) | |||
| 178 | + | ||||
| 179 | + | registry, err := NewRegistryWithConfig(1, store, nil, newManualClock(time.Unix(100, 0)), nil, RegistryConfig{ObservationDisabled: true}) | |||
| 180 | + | require.NoError(t, err) | |||
| 181 | + | defer registry.settle() | |||
| 182 | + | authority := registry.authority | |||
| 183 | + | before := registry.Snapshot() | |||
| 184 | + | revision := registry.revision | |||
| 185 | + | persistedBefore, err := store.Load() | |||
| 186 | + | require.NoError(t, err) | |||
| 187 | + | require.NotZero(t, authority.Revision) | |||
| 188 | + | require.Len(t, before.Daemons, 1) | |||
| 189 | + | ||||
| 190 | + | poisoned := ports.BrokerHostRecord{Registration: registration(t, "user@\u202e", 2), Pinned: true, Policy: poolPolicy()} | |||
| 191 | + | require.NoError(t, domain.ValidateRemoteHostTarget(poisoned.Registration.Endpoint), "the routing validator accepts the target") | |||
| 192 | + | require.Empty(t, hostDisplayOrigin(poisoned.Registration.Endpoint), "only the derived hint is empty after sanitizing") | |||
| 193 | + | ||||
| 194 | + | require.ErrorContains(t, registry.ReplaceHosts([]ports.BrokerHostRecord{poisoned}), "not publishable") | |||
| 195 | + | require.Equal(t, authority, registry.authority, "a refused replacement never moves authority") | |||
| 196 | + | require.Equal(t, before, registry.Snapshot(), "a refused replacement never publishes") | |||
| 197 | + | require.Equal(t, revision, registry.revision, "a refused replacement never advances the publication series") | |||
| 198 | + | ||||
| 199 | + | // Nothing reached the durable store, even after the writer flushes: the CAS | |||
| 200 | + | // was never crossed, and the advisory snapshot is still the published one. | |||
| 201 | + | registry.settle() | |||
| 202 | + | durableHosts, err := store.LoadHosts() | |||
| 203 | + | require.NoError(t, err) | |||
| 204 | + | require.Equal(t, authority, durableHosts) | |||
| 205 | + | require.Equal(t, authority.Revision, durableHosts.Revision, "the refused replacement never crossed the store CAS") | |||
| 206 | + | persisted, err := store.Load() | |||
| 207 | + | require.NoError(t, err) | |||
| 208 | + | require.Equal(t, persistedBefore, persisted, "the refused replacement never wrote a snapshot") | |||
| 209 | + | ||||
| 210 | + | // A restart remains valid: the refused membership was never stored, so | |||
| 211 | + | // reopening restores and publishes the original host. | |||
| 212 | + | require.NoError(t, store.Close()) | |||
| 213 | + | reopened := openDurableStore(t, dir, nil) | |||
| 214 | + | restarted, err := NewRegistryWithConfig(2, reopened, nil, newManualClock(time.Unix(200, 0)), nil, RegistryConfig{ObservationDisabled: true}) | |||
| 215 | + | require.NoError(t, err) | |||
| 216 | + | defer restarted.settle() | |||
| 217 | + | require.Equal(t, authority, restarted.authority) | |||
| 218 | + | restored := restarted.Snapshot() | |||
| 219 | + | require.NoError(t, restored.Validate()) | |||
| 220 | + | require.Len(t, restored.Daemons, 1) | |||
| 221 | + | require.Equal(t, valid.Registration, restored.Daemons[0].Registration) | |||
| 222 | + | } | |||
| 223 | + | ||||
| 224 | + | // TestRegistryMutableOutcomeUnknownRequiresReopen pins the indeterminate-outcome | |||
| 225 | + | // contract against the real store: a failure after the durable commit point is | |||
| 226 | + | // classified through the registry, is never acknowledged as membership, and | |||
| 227 | + | // leaves the store poisoned until it is reopened and authority is reloaded. | |||
| 228 | + | // Reopening reveals that the mutation did commit, which is exactly why the | |||
| 229 | + | // registry may not report it either way. | |||
| 230 | + | func TestRegistryMutableOutcomeUnknownRequiresReopen(t *testing.T) { | |||
| 231 | + | dir := filepath.Join(t.TempDir(), "broker") | |||
| 232 | + | fault := errors.New("power loss") | |||
| 233 | + | // The fault is armed only after OpenOffline so the store can be created and | |||
| 234 | + | // seeded normally; state.json:rename fires after the rename, so the commit | |||
| 235 | + | // point was already crossed when the failure is reported. | |||
| 236 | + | var armed atomic.Bool | |||
| 237 | + | store := openDurableStore(t, dir, func(point string) error { | |||
| 238 | + | if armed.Load() && point == "state.json:rename" { | |||
| 239 | + | return fault | |||
| 240 | + | } | |||
| 241 | + | return nil | |||
| 242 | + | }) | |||
| 243 | + | registry := newDurableMutableRegistry(t, 1, store) | |||
| 244 | + | defer registry.settle() | |||
| 245 | + | before := registry.Snapshot() | |||
| 246 | + | authority := registry.authority | |||
| 247 | + | armed.Store(true) | |||
| 248 | + | ||||
| 249 | + | _, err := registry.AddHost(context.Background(), "user@arch:22", poolPolicy()) | |||
| 250 | + | require.Error(t, err) | |||
| 251 | + | var unknown ports.BrokerStoreOutcomeUnknownError | |||
| 252 | + | require.ErrorAs(t, err, &unknown, "a post-commit failure is classified as an unknown outcome") | |||
| 253 | + | require.ErrorIs(t, err, fault, "the underlying store failure stays inspectable") | |||
| 254 | + | require.Equal(t, authority, registry.authority, "an unknown outcome never becomes acknowledged membership") | |||
| 255 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 256 | + | ||||
| 257 | + | // The store handle is poisoned: every later operation refuses instead of | |||
| 258 | + | // reading or writing a state it cannot vouch for. | |||
| 259 | + | _, err = store.LoadHosts() | |||
| 260 | + | require.ErrorContains(t, err, "reopen required") | |||
| 261 | + | require.ErrorAs(t, err, &unknown) | |||
| 262 | + | require.Empty(t, registry.Snapshot().Daemons) | |||
| 263 | + | ||||
| 264 | + | // Reopening reloads authority from disk: the rename already published the | |||
| 265 | + | // new membership, so the only safe recovery is to reopen rather than retry. | |||
| 266 | + | require.NoError(t, store.Close()) | |||
| 267 | + | armed.Store(false) | |||
| 268 | + | reopened := openDurableStore(t, dir, nil) | |||
| 269 | + | hosts, err := reopened.LoadHosts() | |||
| 270 | + | require.NoError(t, err) | |||
| 271 | + | require.Equal(t, authority.Revision+1, hosts.Revision) | |||
| 272 | + | require.Len(t, hosts.Hosts, 1) | |||
| 273 | + | require.Equal(t, "user@arch:22", hosts.Hosts[0].Registration.Endpoint) | |||
| 274 | + | require.Equal(t, poolPolicy(), hosts.Hosts[0].Policy) | |||
| 275 | + | ||||
| 276 | + | recovered := newDurableMutableRegistry(t, 2, reopened) | |||
| 277 | + | defer recovered.settle() | |||
| 278 | + | require.Len(t, recovered.authority.Hosts, 1) | |||
| 279 | + | require.Equal(t, hosts.Hosts[0].Registration, recovered.authority.Hosts[0].Registration) | |||
| 280 | + | } | |||
A internal/usecase/broker/registry_membership_test.go +871 -0
| @@ -0,0 +1,871 @@ | |||
| 1 | + | package broker | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "errors" | |
| 6 | + | "fmt" | |
| 7 | + | "sync" | |
| 8 | + | "testing" | |
| 9 | + | "time" | |
| 10 | + | ||
| 11 | + | "github.com/bnema/vev/internal/domain" | |
| 12 | + | "github.com/bnema/vev/internal/ports" | |
| 13 | + | portsmocks "github.com/bnema/vev/internal/ports/mocks" | |
| 14 | + | "github.com/bnema/vev/internal/protocol" | |
| 15 | + | "github.com/stretchr/testify/require" | |
| 16 | + | ) | |
| 17 | + | ||
| 18 | + | // Phase A mutable membership: the registry is the only synchronous owner of | |
| 19 | + | // durable authority, so every admitted mutation must CAS the store, install its | |
| 20 | + | // projection only after that CAS succeeds, and leave authority, projection, | |
| 21 | + | // probe attempts, and tombstones untouched on every refusal. The tests below | |
| 22 | + | // drive the public mutation API and the production fencing path directly, so a | |
| 23 | + | // regression in any of those rules is observable instead of being masked by the | |
| 24 | + | // observation scheduler. | |
| 25 | + | ||
| 26 | + | // incarnationSequencer is a deterministic entropy seam: every call yields a | |
| 27 | + | // distinct, never-zero incarnation, so a test can assert that a remove/re-add | |
| 28 | + | // really minted a fresh identity and that a refused mutation minted none. | |
| 29 | + | type incarnationSequencer struct { | |
| 30 | + | mu sync.Mutex | |
| 31 | + | next byte | |
| 32 | + | } | |
| 33 | + | ||
| 34 | + | func (s *incarnationSequencer) generate() ([16]byte, error) { | |
| 35 | + | s.mu.Lock() | |
| 36 | + | defer s.mu.Unlock() | |
| 37 | + | s.next++ | |
| 38 | + | var id [16]byte | |
| 39 | + | id[0] = 0xA5 | |
| 40 | + | id[15] = s.next | |
| 41 | + | return id, nil | |
| 42 | + | } | |
| 43 | + | ||
| 44 | + | func (s *incarnationSequencer) generated() int { | |
| 45 | + | s.mu.Lock() | |
| 46 | + | defer s.mu.Unlock() | |
| 47 | + | return int(s.next) | |
| 48 | + | } | |
| 49 | + | ||
| 50 | + | // membershipStore is a race-safe durable store with an observable CAS counter | |
| 51 | + | // and an injectable CAS failure. The counter proves a mutation reached the | |
| 52 | + | // store (including the no-op paths, which must still CAS), and the injected | |
| 53 | + | // failure proves a store fault is synchronous and never installs a projection. | |
| 54 | + | type membershipStore struct { | |
| 55 | + | *testStore | |
| 56 | + | casMu sync.Mutex | |
| 57 | + | casErr error | |
| 58 | + | casCall int | |
| 59 | + | } | |
| 60 | + | ||
| 61 | + | func newMembershipStore(hosts ...ports.BrokerHostRecord) *membershipStore { | |
| 62 | + | return &membershipStore{testStore: &testStore{hosts: ports.BrokerHosts{Revision: 1, Hosts: hosts}}} | |
| 63 | + | } | |
| 64 | + | ||
| 65 | + | func (s *membershipStore) ReplaceHosts(expected uint64, hosts []ports.BrokerHostRecord) error { | |
| 66 | + | s.casMu.Lock() | |
| 67 | + | s.casCall++ | |
| 68 | + | err := s.casErr | |
| 69 | + | s.casMu.Unlock() | |
| 70 | + | if err != nil { | |
| 71 | + | return err | |
| 72 | + | } | |
| 73 | + | return s.testStore.ReplaceHosts(expected, hosts) | |
| 74 | + | } | |
| 75 | + | ||
| 76 | + | func (s *membershipStore) failCAS(err error) { | |
| 77 | + | s.casMu.Lock() | |
| 78 | + | s.casErr = err | |
| 79 | + | s.casMu.Unlock() | |
| 80 | + | } | |
| 81 | + | ||
| 82 | + | func (s *membershipStore) casCalls() int { | |
| 83 | + | s.casMu.Lock() | |
| 84 | + | defer s.casMu.Unlock() | |
| 85 | + | return s.casCall | |
| 86 | + | } | |
| 87 | + | ||
| 88 | + | // durableAuthority reads the store's committed authority, exactly as a reopen | |
| 89 | + | // would: membership and its revision token come from the store, never from the | |
| 90 | + | // registry's in-memory copy. | |
| 91 | + | func (s *membershipStore) durableAuthority() ports.BrokerHosts { | |
| 92 | + | s.testStore.mu.Lock() | |
| 93 | + | defer s.testStore.mu.Unlock() | |
| 94 | + | return ports.BrokerHosts{Revision: s.testStore.hosts.Revision, Hosts: append([]ports.BrokerHostRecord(nil), s.testStore.hosts.Hosts...)} | |
| 95 | + | } | |
| 96 | + | ||
| 97 | + | // newMutableRegistry builds a mutable registry with exact scheduling, a | |
| 98 | + | // deterministic incarnation seam, and its scripted probe. | |
| 99 | + | func newMutableRegistry(t *testing.T, store ports.BrokerHostStore, clock *manualClock, generator func() ([16]byte, error)) (*Registry, *testProbe) { | |
| 100 | + | t.Helper() | |
| 101 | + | probe := newTestProbe(2) | |
| 102 | + | registry, err := NewRegistryWithConfig(1, store, probe, clock, nil, RegistryConfig{ | |
| 103 | + | MembershipMode: MembershipMutable, | |
| 104 | + | IncarnationGenerator: generator, | |
| 105 | + | ObservationDisabled: false, | |
| 106 | + | }) | |
| 107 | + | require.NoError(t, err) | |
| 108 | + | registry.jitter = identityJitter | |
| 109 | + | return registry, probe | |
| 110 | + | } | |
| 111 | + | ||
| 112 | + | func policyWithTrust(trust string) ports.BrokerPolicy { | |
| 113 | + | policy := poolPolicy() | |
| 114 | + | policy.Trust = trust | |
| 115 | + | return policy | |
| 116 | + | } | |
| 117 | + | ||
| 118 | + | func pinnedRecord(reg domain.RemoteRegistration, policy ports.BrokerPolicy) ports.BrokerHostRecord { | |
| 119 | + | return ports.BrokerHostRecord{Registration: reg, Pinned: true, Policy: policy} | |
| 120 | + | } | |
| 121 | + | ||
| 122 | + | // admittedAttempt is one in-flight probe attempt a test installs directly, so a | |
| 123 | + | // completion can be delivered to apply without racing the scheduler. | |
| 124 | + | type admittedAttempt struct { | |
| 125 | + | token uint64 | |
| 126 | + | ctx context.Context | |
| 127 | + | cancel context.CancelFunc | |
| 128 | + | } | |
| 129 | + | ||
| 130 | + | func admitMembershipAttempt(t *testing.T, r *Registry, endpoint string) admittedAttempt { | |
| 131 | + | t.Helper() | |
| 132 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 133 | + | t.Cleanup(cancel) | |
| 134 | + | r.mu.Lock() | |
| 135 | + | defer r.mu.Unlock() | |
| 136 | + | r.attempts++ | |
| 137 | + | attempt := admittedAttempt{token: r.attempts, ctx: ctx, cancel: cancel} | |
| 138 | + | r.inflight[endpoint] = &probeAttempt{token: attempt.token, cancel: cancel} | |
| 139 | + | return attempt | |
| 140 | + | } | |
| 141 | + | ||
| 142 | + | // membershipFences snapshots the tombstone set so a failed mutation can be | |
| 143 | + | // proven to have retired nothing. | |
| 144 | + | func membershipFences(r *Registry) map[string]ports.BrokerHostTombstone { | |
| 145 | + | r.mu.Lock() | |
| 146 | + | defer r.mu.Unlock() | |
| 147 | + | out := make(map[string]ports.BrokerHostTombstone, len(r.tombstones)) | |
| 148 | + | for endpoint, tombstone := range r.tombstones { | |
| 149 | + | out[endpoint] = tombstone | |
| 150 | + | } | |
| 151 | + | return out | |
| 152 | + | } | |
| 153 | + | ||
| 154 | + | func staleObservation() ports.BrokerDaemonObservation { | |
| 155 | + | return ports.BrokerDaemonObservation{ | |
| 156 | + | Identity: "stale-daemon", | |
| 157 | + | Incarnation: ports.BrokerDaemonIncarnation{7}, | |
| 158 | + | ProtocolVersion: protocol.Version, | |
| 159 | + | Availability: domain.RemoteAvailabilityReachable, | |
| 160 | + | InventoryKnown: true, | |
| 161 | + | } | |
| 162 | + | } | |
| 163 | + | ||
| 164 | + | func currentObservation() ports.BrokerDaemonObservation { | |
| 165 | + | return ports.BrokerDaemonObservation{ | |
| 166 | + | Identity: "current-daemon", | |
| 167 | + | Incarnation: ports.BrokerDaemonIncarnation{9}, | |
| 168 | + | ProtocolVersion: protocol.Version, | |
| 169 | + | Availability: domain.RemoteAvailabilityReachable, | |
| 170 | + | InventoryKnown: true, | |
| 171 | + | } | |
| 172 | + | } | |
| 173 | + | ||
| 174 | + | // TestRegistryMutableConcurrentAddsLoseNoUpdate proves concurrent membership | |
| 175 | + | // mutations serialize over durable authority: every added endpoint survives, | |
| 176 | + | // each addition advances the revision by exactly one, and the registry's copy | |
| 177 | + | // still equals what the store committed. | |
| 178 | + | func TestRegistryMutableConcurrentAddsLoseNoUpdate(t *testing.T) { | |
| 179 | + | const adds = 32 | |
| 180 | + | store := newMembershipStore() | |
| 181 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |
| 182 | + | start := registry.authority.Revision | |
| 183 | + | ||
| 184 | + | endpoints := make([]string, adds) | |
| 185 | + | for i := range endpoints { | |
| 186 | + | endpoints[i] = fmt.Sprintf("host-%d.test", i) | |
| 187 | + | } | |
| 188 | + | ||
| 189 | + | var wg sync.WaitGroup | |
| 190 | + | errs := make([]error, adds) | |
| 191 | + | regs := make([]domain.RemoteRegistration, adds) | |
| 192 | + | startGate := make(chan struct{}) | |
| 193 | + | for i, endpoint := range endpoints { | |
| 194 | + | wg.Add(1) | |
| 195 | + | go func(i int, endpoint string) { | |
| 196 | + | defer wg.Done() | |
| 197 | + | <-startGate | |
| 198 | + | regs[i], errs[i] = registry.AddHost(context.Background(), endpoint, poolPolicy()) | |
| 199 | + | }(i, endpoint) | |
| 200 | + | } | |
| 201 | + | close(startGate) | |
| 202 | + | wg.Wait() | |
| 203 | + | ||
| 204 | + | for i, err := range errs { | |
| 205 | + | require.NoErrorf(t, err, "concurrent add %q", endpoints[i]) | |
| 206 | + | } | |
| 207 | + | require.Equal(t, adds, store.casCalls(), "every admitted add must reach the store CAS") | |
| 208 | + | require.Equal(t, start+adds, registry.authority.Revision, "each add advances the revision exactly once") | |
| 209 | + | require.Len(t, registry.authority.Hosts, adds) | |
| 210 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |
| 211 | + | ||
| 212 | + | // Admission order is the lock order, not the launch order, so the | |
| 213 | + | // publication order is compared against the returned identities by endpoint. | |
| 214 | + | returned := make(map[string]domain.RemoteRegistration, adds) | |
| 215 | + | for i, endpoint := range endpoints { | |
| 216 | + | require.False(t, regs[i].IsZero(), "an admitted add returns its minted identity") | |
| 217 | + | require.Equal(t, endpoint, regs[i].Endpoint) | |
| 218 | + | require.Equal(t, domain.RemoteGeneration(1), regs[i].Generation) | |
| 219 | + | returned[endpoint] = regs[i] | |
| 220 | + | } | |
| 221 | + | require.Len(t, returned, adds, "every concurrent add minted a distinct identity") | |
| 222 | + | ||
| 223 | + | seen := make(map[string]domain.RemoteRegistration, adds) | |
| 224 | + | for _, record := range registry.authority.Hosts { | |
| 225 | + | require.NotContains(t, seen, record.Registration.Endpoint, "no endpoint may be added twice") | |
| 226 | + | require.Equal(t, returned[record.Registration.Endpoint], record.Registration) | |
| 227 | + | seen[record.Registration.Endpoint] = record.Registration | |
| 228 | + | require.Equal(t, poolPolicy(), record.Policy) | |
| 229 | + | require.True(t, record.Pinned) | |
| 230 | + | } | |
| 231 | + | require.Len(t, seen, adds) | |
| 232 | + | ||
| 233 | + | snapshot := registry.Snapshot() | |
| 234 | + | require.Len(t, snapshot.Daemons, adds) | |
| 235 | + | require.NoError(t, snapshot.Validate()) | |
| 236 | + | for _, endpoint := range endpoints { | |
| 237 | + | host, ok := snapshot.Find(endpoint) | |
| 238 | + | require.True(t, ok, "a concurrently added endpoint must be published") | |
| 239 | + | require.Equal(t, seen[endpoint], host.Registration) | |
| 240 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |
| 241 | + | } | |
| 242 | + | } | |
| 243 | + | ||
| 244 | + | // TestRegistryMembershipModeRejectsUnknownMode proves the run mode is a closed | |
| 245 | + | // set: an out-of-range mode is refused at construction, before any store call, | |
| 246 | + | // so a caller can never run a registry whose mutation policy is undefined. | |
| 247 | + | func TestRegistryMembershipModeRejectsUnknownMode(t *testing.T) { | |
| 248 | + | // The mock has no expectations: any store call during construction fails | |
| 249 | + | // the test, so a refusal must happen before the durable store is touched. | |
| 250 | + | store := portsmocks.NewMockBrokerHostStore(t) | |
| 251 | + | registry, err := NewRegistryWithConfig(1, store, newTestProbe(1), newManualClock(time.Unix(100, 0)), nil, | |
| 252 | + | RegistryConfig{MembershipMode: MembershipMode(9)}) | |
| 253 | + | require.ErrorContains(t, err, "invalid membership mode") | |
| 254 | + | require.Nil(t, registry) | |
| 255 | + | } | |
| 256 | + | ||
| 257 | + | // TestRegistryImmutableMembershipRefusesEveryMutation pins the default: a | |
| 258 | + | // registry that was not explicitly configured mutable refuses every membership | |
| 259 | + | // mutation, including the ones that would be no-ops, and refuses them without | |
| 260 | + | // touching the durable store or its own projection. | |
| 261 | + | func TestRegistryImmutableMembershipRefusesEveryMutation(t *testing.T) { | |
| 262 | + | reg := registration(t, "alpha.test", 1) | |
| 263 | + | store := portsmocks.NewMockBrokerHostStore(t) | |
| 264 | + | store.EXPECT().Load().Return(ports.BrokerSnapshot{}, nil).Once() | |
| 265 | + | store.EXPECT().LoadHosts().Return(ports.BrokerHosts{Revision: 3, Hosts: []ports.BrokerHostRecord{pinnedRecord(reg, poolPolicy())}}, nil).Once() | |
| 266 | + | registry, err := NewRegistry(1, store, newTestProbe(1), newManualClock(time.Unix(100, 0)), nil) | |
| 267 | + | require.NoError(t, err) | |
| 268 | + | authority := registry.authority | |
| 269 | + | before := registry.Snapshot() | |
| 270 | + | ||
| 271 | + | tests := []struct { | |
| 272 | + | name string | |
| 273 | + | run func(r *Registry) error | |
| 274 | + | }{ | |
| 275 | + | {"add new endpoint", func(r *Registry) error { | |
| 276 | + | _, err := r.AddHost(context.Background(), "beta.test", poolPolicy()) | |
| 277 | + | return err | |
| 278 | + | }}, | |
| 279 | + | {"add duplicate endpoint with equal policy is a no-op", func(r *Registry) error { | |
| 280 | + | _, err := r.AddHost(context.Background(), reg.Endpoint, poolPolicy()) | |
| 281 | + | return err | |
| 282 | + | }}, | |
| 283 | + | {"remove the exact registration", func(r *Registry) error { | |
| 284 | + | _, err := r.RemoveHost(context.Background(), reg) | |
| 285 | + | return err | |
| 286 | + | }}, | |
| 287 | + | {"remove an absent endpoint is a no-op", func(r *Registry) error { | |
| 288 | + | _, err := r.RemoveHost(context.Background(), registration(t, "gamma.test", 3)) | |
| 289 | + | return err | |
| 290 | + | }}, | |
| 291 | + | {"update to a different policy", func(r *Registry) error { | |
| 292 | + | _, err := r.UpdateHostPolicy(context.Background(), reg, policyWithTrust("other-trust")) | |
| 293 | + | return err | |
| 294 | + | }}, | |
| 295 | + | {"update to the equal policy is a no-op", func(r *Registry) error { | |
| 296 | + | _, err := r.UpdateHostPolicy(context.Background(), reg, poolPolicy()) | |
| 297 | + | return err | |
| 298 | + | }}, | |
| 299 | + | } | |
| 300 | + | for _, tc := range tests { | |
| 301 | + | t.Run(tc.name, func(t *testing.T) { | |
| 302 | + | require.ErrorIs(t, tc.run(registry), ports.ErrBrokerMembershipImmutable) | |
| 303 | + | require.Equal(t, authority, registry.authority) | |
| 304 | + | require.Equal(t, before, registry.Snapshot()) | |
| 305 | + | require.Empty(t, membershipFences(registry)) | |
| 306 | + | }) | |
| 307 | + | } | |
| 308 | + | ||
| 309 | + | // Control: the same duplicate add is admitted when the mode is mutable, so | |
| 310 | + | // the refusals above are the mode and not the validity of the operation. | |
| 311 | + | seq := &incarnationSequencer{} | |
| 312 | + | mutable, _ := newMutableRegistry(t, newMembershipStore(pinnedRecord(reg, poolPolicy())), newManualClock(time.Unix(100, 0)), seq.generate) | |
| 313 | + | added, err := mutable.AddHost(context.Background(), reg.Endpoint, poolPolicy()) | |
| 314 | + | require.NoError(t, err) | |
| 315 | + | require.Equal(t, reg, added) | |
| 316 | + | require.Equal(t, 0, seq.generated(), "a duplicate add must not mint an identity") | |
| 317 | + | } | |
| 318 | + | ||
| 319 | + | // TestRegistryMutableAddPinsDuplicatePreservingIdentity proves a duplicate | |
| 320 | + | // same-policy addition pins the existing record without changing its identity, | |
| 321 | + | // and that even that no-op CAS-verifies durable authority. | |
| 322 | + | func TestRegistryMutableAddPinsDuplicatePreservingIdentity(t *testing.T) { | |
| 323 | + | seq := &incarnationSequencer{} | |
| 324 | + | store := newMembershipStore(ports.BrokerHostRecord{Registration: registration(t, "alpha.test", 1), Learned: true, Policy: poolPolicy()}) | |
| 325 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), seq.generate) | |
| 326 | + | reg := registration(t, "alpha.test", 1) | |
| 327 | + | ||
| 328 | + | got, err := registry.AddHost(context.Background(), reg.Endpoint, poolPolicy()) | |
| 329 | + | require.NoError(t, err) | |
| 330 | + | require.Equal(t, reg, got, "a duplicate add preserves the registration identity") | |
| 331 | + | require.Equal(t, 1, store.casCalls()) | |
| 332 | + | require.Len(t, registry.authority.Hosts, 1) | |
| 333 | + | record := registry.authority.Hosts[0] | |
| 334 | + | require.Equal(t, reg, record.Registration) | |
| 335 | + | require.Equal(t, poolPolicy(), record.Policy) | |
| 336 | + | require.True(t, record.Pinned, "the duplicate add pins the learned registration") | |
| 337 | + | require.True(t, record.Learned, "pinning never discards learned provenance") | |
| 338 | + | require.Equal(t, uint64(2), registry.authority.Revision) | |
| 339 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |
| 340 | + | require.Equal(t, 0, seq.generated(), "pinning mints no identity") | |
| 341 | + | ||
| 342 | + | host, ok := registry.Snapshot().Find(reg.Endpoint) | |
| 343 | + | require.True(t, ok) | |
| 344 | + | require.Equal(t, reg, host.Registration) | |
| 345 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |
| 346 | + | ||
| 347 | + | // A second duplicate is a no-op that still CAS-verifies: authority moves | |
| 348 | + | // durably, the projection does not republish, and identity is unchanged. | |
| 349 | + | before := registry.Snapshot() | |
| 350 | + | again, err := registry.AddHost(context.Background(), reg.Endpoint, poolPolicy()) | |
| 351 | + | require.NoError(t, err) | |
| 352 | + | require.Equal(t, reg, again) | |
| 353 | + | require.Equal(t, 2, store.casCalls()) | |
| 354 | + | require.Equal(t, uint64(3), registry.authority.Revision) | |
| 355 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |
| 356 | + | require.Equal(t, before, registry.Snapshot()) | |
| 357 | + | } | |
| 358 | + | ||
| 359 | + | // TestRegistryMutableAddConflictingPolicyConflicts proves a conflicting policy | |
| 360 | + | // is a definite conflict: it is refused before the store CAS, and it mints no | |
| 361 | + | // identity. | |
| 362 | + | func TestRegistryMutableAddConflictingPolicyConflicts(t *testing.T) { | |
| 363 | + | seq := &incarnationSequencer{} | |
| 364 | + | reg := registration(t, "alpha.test", 1) | |
| 365 | + | store := newMembershipStore(pinnedRecord(reg, poolPolicy())) | |
| 366 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), seq.generate) | |
| 367 | + | authority := registry.authority | |
| 368 | + | before := registry.Snapshot() | |
| 369 | + | ||
| 370 | + | _, err := registry.AddHost(context.Background(), reg.Endpoint, policyWithTrust("other-trust")) | |
| 371 | + | require.ErrorIs(t, err, ports.ErrBrokerHostConflict) | |
| 372 | + | require.Equal(t, 0, store.casCalls(), "a conflict is refused before the CAS") | |
| 373 | + | require.Equal(t, authority, registry.authority) | |
| 374 | + | require.Equal(t, before, registry.Snapshot()) | |
| 375 | + | require.Equal(t, 0, seq.generated()) | |
| 376 | + | } | |
| 377 | + | ||
| 378 | + | // TestRegistryMutableAddEntropyFailureLeavesAuthorityUnchanged pins the entropy | |
| 379 | + | // seam: an unusable incarnation is refused before anything durable is staged. | |
| 380 | + | func TestRegistryMutableAddEntropyFailureLeavesAuthorityUnchanged(t *testing.T) { | |
| 381 | + | tests := []struct { | |
| 382 | + | name string | |
| 383 | + | generator func() ([16]byte, error) | |
| 384 | + | want string | |
| 385 | + | }{ | |
| 386 | + | {"entropy source fails", func() ([16]byte, error) { return [16]byte{}, errors.New("no entropy") }, "no entropy"}, | |
| 387 | + | {"entropy source returns a zero identity", func() ([16]byte, error) { return [16]byte{}, nil }, "zero"}, | |
| 388 | + | } | |
| 389 | + | for _, tc := range tests { | |
| 390 | + | t.Run(tc.name, func(t *testing.T) { | |
| 391 | + | store := newMembershipStore() | |
| 392 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), tc.generator) | |
| 393 | + | authority := registry.authority | |
| 394 | + | before := registry.Snapshot() | |
| 395 | + | ||
| 396 | + | _, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |
| 397 | + | require.ErrorContains(t, err, tc.want) | |
| 398 | + | require.Equal(t, 0, store.casCalls()) | |
| 399 | + | require.Equal(t, authority, registry.authority) | |
| 400 | + | require.Equal(t, before, registry.Snapshot()) | |
| 401 | + | }) | |
| 402 | + | } | |
| 403 | + | } | |
| 404 | + | ||
| 405 | + | // TestRegistryMutableRemoveExactStaleAndAbsent pins removal admission: only the | |
| 406 | + | // exact expected registration is removed, a stale generation or incarnation is | |
| 407 | + | // a definite conflict, and an absent endpoint reports false only after the no-op | |
| 408 | + | // CAS verified the loaded revision. | |
| 409 | + | func TestRegistryMutableRemoveExactStaleAndAbsent(t *testing.T) { | |
| 410 | + | alpha := registration(t, "alpha.test", 1) | |
| 411 | + | beta := registration(t, "beta.test", 2) | |
| 412 | + | store := newMembershipStore(pinnedRecord(alpha, poolPolicy()), pinnedRecord(beta, poolPolicy())) | |
| 413 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |
| 414 | + | authority := registry.authority | |
| 415 | + | before := registry.Snapshot() | |
| 416 | + | ||
| 417 | + | staleGeneration := alpha | |
| 418 | + | staleGeneration.Generation++ | |
| 419 | + | staleIncarnation := alpha | |
| 420 | + | staleIncarnation.Incarnation = [16]byte{0xEE} | |
| 421 | + | for name, expected := range map[string]domain.RemoteRegistration{ | |
| 422 | + | "stale generation": staleGeneration, | |
| 423 | + | "stale incarnation": staleIncarnation, | |
| 424 | + | } { | |
| 425 | + | t.Run(name, func(t *testing.T) { | |
| 426 | + | removed, err := registry.RemoveHost(context.Background(), expected) | |
| 427 | + | require.ErrorIs(t, err, ports.ErrBrokerHostConflict) | |
| 428 | + | require.False(t, removed) | |
| 429 | + | require.Equal(t, 0, store.casCalls(), "a mismatch is refused before the CAS") | |
| 430 | + | require.Equal(t, authority, registry.authority) | |
| 431 | + | require.Equal(t, before, registry.Snapshot()) | |
| 432 | + | require.Empty(t, membershipFences(registry)) | |
| 433 | + | }) | |
| 434 | + | } | |
| 435 | + | ||
| 436 | + | removed, err := registry.RemoveHost(context.Background(), alpha) | |
| 437 | + | require.NoError(t, err) | |
| 438 | + | require.True(t, removed) | |
| 439 | + | require.Equal(t, 1, store.casCalls()) | |
| 440 | + | require.Len(t, registry.authority.Hosts, 1) | |
| 441 | + | require.Equal(t, beta, registry.authority.Hosts[0].Registration) | |
| 442 | + | require.Equal(t, uint64(2), registry.authority.Revision) | |
| 443 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |
| 444 | + | afterRemove := registry.Snapshot() | |
| 445 | + | require.Len(t, afterRemove.Daemons, 1) | |
| 446 | + | require.Equal(t, beta, afterRemove.Daemons[0].Registration) | |
| 447 | + | require.Len(t, afterRemove.Removed, 1) | |
| 448 | + | require.Equal(t, alpha, afterRemove.Removed[0].Registration) | |
| 449 | + | require.Equal(t, afterRemove.Revision, afterRemove.Removed[0].RetiredRevision, | |
| 450 | + | "a tombstone is retired at the revision of the publication that carries it") | |
| 451 | + | ||
| 452 | + | // An absent endpoint is a no-op that still CAS-verifies: durable authority | |
| 453 | + | // advances, while nothing is republished and nothing is retired. | |
| 454 | + | absent, err := registry.RemoveHost(context.Background(), alpha) | |
| 455 | + | require.NoError(t, err) | |
| 456 | + | require.False(t, absent) | |
| 457 | + | require.Equal(t, 2, store.casCalls()) | |
| 458 | + | require.Equal(t, uint64(3), registry.authority.Revision) | |
| 459 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |
| 460 | + | require.Equal(t, afterRemove, registry.Snapshot()) | |
| 461 | + | } | |
| 462 | + | ||
| 463 | + | // TestRegistryMutableNoOpRemoveVerifiesCAS pins the no-op CAS itself: with | |
| 464 | + | // authority already moved by another mutator, an absent-endpoint removal must | |
| 465 | + | // report a conflict instead of a hollow success, because the loaded revision was | |
| 466 | + | // never verified. | |
| 467 | + | func TestRegistryMutableNoOpRemoveVerifiesCAS(t *testing.T) { | |
| 468 | + | alpha := registration(t, "alpha.test", 1) | |
| 469 | + | store := newMembershipStore(pinnedRecord(alpha, poolPolicy())) | |
| 470 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |
| 471 | + | authority := registry.authority | |
| 472 | + | before := registry.Snapshot() | |
| 473 | + | ||
| 474 | + | // An illicit second mutator moves durable authority behind the registry. | |
| 475 | + | require.NoError(t, store.testStore.ReplaceHosts(authority.Revision, []ports.BrokerHostRecord{pinnedRecord(alpha, poolPolicy())})) | |
| 476 | + | ||
| 477 | + | removed, err := registry.RemoveHost(context.Background(), registration(t, "gamma.test", 3)) | |
| 478 | + | require.ErrorIs(t, err, ports.ErrBrokerHostConflict) | |
| 479 | + | require.False(t, removed) | |
| 480 | + | require.Equal(t, 1, store.casCalls()) | |
| 481 | + | require.Equal(t, authority, registry.authority) | |
| 482 | + | require.Equal(t, before, registry.Snapshot()) | |
| 483 | + | } | |
| 484 | + | ||
| 485 | + | // TestRegistryMutableRemoveReAddGetsFreshIncarnation pins the ABA fence at the | |
| 486 | + | // mutation boundary: a full removal followed by a re-add mints a fresh | |
| 487 | + | // incarnation with a fresh generation series, and the retired registration is | |
| 488 | + | // fenced rather than revived. | |
| 489 | + | func TestRegistryMutableRemoveReAddGetsFreshIncarnation(t *testing.T) { | |
| 490 | + | seq := &incarnationSequencer{} | |
| 491 | + | store := newMembershipStore() | |
| 492 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), seq.generate) | |
| 493 | + | ||
| 494 | + | first, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |
| 495 | + | require.NoError(t, err) | |
| 496 | + | require.Equal(t, 1, seq.generated()) | |
| 497 | + | ||
| 498 | + | removed, err := registry.RemoveHost(context.Background(), first) | |
| 499 | + | require.NoError(t, err) | |
| 500 | + | require.True(t, removed) | |
| 501 | + | ||
| 502 | + | second, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |
| 503 | + | require.NoError(t, err) | |
| 504 | + | require.Equal(t, 2, seq.generated()) | |
| 505 | + | require.False(t, first.Equal(second), "a re-add must not reuse the retired registration") | |
| 506 | + | require.NotEqual(t, first.Incarnation, second.Incarnation) | |
| 507 | + | require.Equal(t, domain.RemoteGeneration(1), second.Generation) | |
| 508 | + | require.Len(t, registry.authority.Hosts, 1) | |
| 509 | + | require.Equal(t, second, registry.authority.Hosts[0].Registration) | |
| 510 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |
| 511 | + | ||
| 512 | + | snapshot := registry.Snapshot() | |
| 513 | + | require.Len(t, snapshot.Daemons, 1) | |
| 514 | + | require.Equal(t, second, snapshot.Daemons[0].Registration) | |
| 515 | + | require.Empty(t, snapshot.Removed, "a live endpoint is never published as a tombstone") | |
| 516 | + | host, ok := snapshot.Find(second.Endpoint) | |
| 517 | + | require.True(t, ok) | |
| 518 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |
| 519 | + | } | |
| 520 | + | ||
| 521 | + | // TestRegistryMutableUpdatePolicyIncrementsGenerationOnce pins the policy | |
| 522 | + | // update: a changed policy advances the registration generation exactly once and | |
| 523 | + | // is the only thing that does; an unmatchable expectation is a definite conflict | |
| 524 | + | // refused before the CAS. | |
| 525 | + | func TestRegistryMutableUpdatePolicyIncrementsGenerationOnce(t *testing.T) { | |
| 526 | + | alpha := registration(t, "alpha.test", 1) | |
| 527 | + | store := newMembershipStore(pinnedRecord(alpha, poolPolicy())) | |
| 528 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |
| 529 | + | ||
| 530 | + | // Not found and a stale expectation are definite conflicts, refused before | |
| 531 | + | // the store is touched. | |
| 532 | + | authority := registry.authority | |
| 533 | + | before := registry.Snapshot() | |
| 534 | + | _, err := registry.UpdateHostPolicy(context.Background(), registration(t, "gamma.test", 3), policyWithTrust("other-trust")) | |
| 535 | + | require.ErrorIs(t, err, ports.ErrBrokerHostConflict) | |
| 536 | + | stale := alpha | |
| 537 | + | stale.Generation++ | |
| 538 | + | _, err = registry.UpdateHostPolicy(context.Background(), stale, policyWithTrust("other-trust")) | |
| 539 | + | require.ErrorIs(t, err, ports.ErrBrokerHostConflict) | |
| 540 | + | require.Equal(t, 0, store.casCalls()) | |
| 541 | + | require.Equal(t, authority, registry.authority) | |
| 542 | + | require.Equal(t, before, registry.Snapshot()) | |
| 543 | + | ||
| 544 | + | updated, err := registry.UpdateHostPolicy(context.Background(), alpha, policyWithTrust("other-trust")) | |
| 545 | + | require.NoError(t, err) | |
| 546 | + | require.Equal(t, alpha.Generation+1, updated.Generation, "a policy change advances the generation exactly once") | |
| 547 | + | require.Equal(t, alpha.Incarnation, updated.Incarnation, "a policy change never touches the incarnation") | |
| 548 | + | require.Equal(t, 1, store.casCalls()) | |
| 549 | + | require.Equal(t, uint64(2), registry.authority.Revision) | |
| 550 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |
| 551 | + | require.Equal(t, []ports.BrokerHostRecord{pinnedRecord(updated, policyWithTrust("other-trust"))}, store.durableAuthority().Hosts) | |
| 552 | + | ||
| 553 | + | host, ok := registry.Snapshot().Find(alpha.Endpoint) | |
| 554 | + | require.True(t, ok) | |
| 555 | + | require.Equal(t, updated, host.Registration) | |
| 556 | + | require.Equal(t, policyWithTrust("other-trust"), host.Policy) | |
| 557 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |
| 558 | + | } | |
| 559 | + | ||
| 560 | + | // TestRegistryMutableUpdateEqualPolicyPreservesIdentityAndVerifiesCAS proves an | |
| 561 | + | // equal-policy update is an identity-preserving no-op that still CAS-verifies | |
| 562 | + | // durable authority. | |
| 563 | + | func TestRegistryMutableUpdateEqualPolicyPreservesIdentityAndVerifiesCAS(t *testing.T) { | |
| 564 | + | alpha := registration(t, "alpha.test", 1) | |
| 565 | + | store := newMembershipStore(pinnedRecord(alpha, poolPolicy())) | |
| 566 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |
| 567 | + | before := registry.Snapshot() | |
| 568 | + | ||
| 569 | + | got, err := registry.UpdateHostPolicy(context.Background(), alpha, poolPolicy()) | |
| 570 | + | require.NoError(t, err) | |
| 571 | + | require.Equal(t, alpha, got, "an equal policy preserves the registration identity") | |
| 572 | + | require.Equal(t, 1, store.casCalls()) | |
| 573 | + | require.Equal(t, uint64(2), registry.authority.Revision) | |
| 574 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |
| 575 | + | require.Equal(t, before, registry.Snapshot(), "an unchanged policy republishes nothing") | |
| 576 | + | ||
| 577 | + | // The no-op still verified the loaded revision: a revision moved behind the | |
| 578 | + | // registry turns the same call into a definite conflict. | |
| 579 | + | require.NoError(t, store.testStore.ReplaceHosts(registry.authority.Revision, []ports.BrokerHostRecord{pinnedRecord(alpha, poolPolicy())})) | |
| 580 | + | authority := registry.authority | |
| 581 | + | _, err = registry.UpdateHostPolicy(context.Background(), alpha, poolPolicy()) | |
| 582 | + | require.ErrorIs(t, err, ports.ErrBrokerHostConflict) | |
| 583 | + | require.Equal(t, 2, store.casCalls()) | |
| 584 | + | require.Equal(t, authority, registry.authority) | |
| 585 | + | require.Equal(t, before, registry.Snapshot()) | |
| 586 | + | } | |
| 587 | + | ||
| 588 | + | // TestRegistryMutableUpdatePolicyOverflowRefused pins the generation series: a | |
| 589 | + | // policy change at the last generation fails closed instead of wrapping to zero, | |
| 590 | + | // which would re-trust an already queued observation. | |
| 591 | + | func TestRegistryMutableUpdatePolicyOverflowRefused(t *testing.T) { | |
| 592 | + | alpha := registration(t, "alpha.test", 1) | |
| 593 | + | alpha.Generation = ^domain.RemoteGeneration(0) | |
| 594 | + | store := newMembershipStore(pinnedRecord(alpha, poolPolicy())) | |
| 595 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |
| 596 | + | authority := registry.authority | |
| 597 | + | before := registry.Snapshot() | |
| 598 | + | ||
| 599 | + | _, err := registry.UpdateHostPolicy(context.Background(), alpha, policyWithTrust("other-trust")) | |
| 600 | + | require.ErrorIs(t, err, ports.ErrBrokerRevisionExhausted) | |
| 601 | + | require.Equal(t, 0, store.casCalls()) | |
| 602 | + | require.Equal(t, authority, registry.authority) | |
| 603 | + | require.Equal(t, before, registry.Snapshot()) | |
| 604 | + | } | |
| 605 | + | ||
| 606 | + | // TestRegistryMutableProbeCompletionFencedByChangedGeneration proves a | |
| 607 | + | // completion whose registration was replaced by a policy update can never | |
| 608 | + | // populate the new generation, whether it carries the retired registration or | |
| 609 | + | // the current one with a retired attempt token. | |
| 610 | + | func TestRegistryMutableProbeCompletionFencedByChangedGeneration(t *testing.T) { | |
| 611 | + | clock := newManualClock(time.Unix(100, 0)) | |
| 612 | + | store := newMembershipStore() | |
| 613 | + | registry, _ := newMutableRegistry(t, store, clock, (&incarnationSequencer{}).generate) | |
| 614 | + | regA, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |
| 615 | + | require.NoError(t, err) | |
| 616 | + | retired := admitMembershipAttempt(t, registry, regA.Endpoint) | |
| 617 | + | regB, err := registry.UpdateHostPolicy(context.Background(), regA, policyWithTrust("other-trust")) | |
| 618 | + | require.NoError(t, err) | |
| 619 | + | require.Equal(t, regA.Generation+1, regB.Generation) | |
| 620 | + | require.ErrorIs(t, retired.ctx.Err(), context.Canceled, "replacing a registration retires its attempt") | |
| 621 | + | require.Zero(t, attemptFor(registry, regA.Endpoint)) | |
| 622 | + | ||
| 623 | + | before := registry.Snapshot() | |
| 624 | + | for _, tc := range []struct { | |
| 625 | + | name string | |
| 626 | + | result probeResult | |
| 627 | + | }{ | |
| 628 | + | {"retired registration", probeResult{endpoint: regA.Endpoint, registration: regA, attempt: retired.token, at: clock.Now(), snapshot: staleObservation()}}, | |
| 629 | + | {"current registration with a retired token", probeResult{endpoint: regA.Endpoint, registration: regB, attempt: retired.token, at: clock.Now(), snapshot: staleObservation()}}, | |
| 630 | + | {"retired registration with the current token", probeResult{endpoint: regA.Endpoint, registration: regA, attempt: retired.token + 1, at: clock.Now(), snapshot: staleObservation()}}, | |
| 631 | + | } { | |
| 632 | + | t.Run(tc.name, func(t *testing.T) { | |
| 633 | + | registry.apply(tc.result) | |
| 634 | + | require.Equal(t, before, registry.Snapshot()) | |
| 635 | + | host, ok := registry.Snapshot().Find(regA.Endpoint) | |
| 636 | + | require.True(t, ok) | |
| 637 | + | require.Equal(t, regB, host.Registration) | |
| 638 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |
| 639 | + | require.Empty(t, host.Identity) | |
| 640 | + | require.True(t, host.LastSuccess.IsZero()) | |
| 641 | + | require.Zero(t, attemptFor(registry, regA.Endpoint), "a stale completion never admits an attempt") | |
| 642 | + | }) | |
| 643 | + | } | |
| 644 | + | } | |
| 645 | + | ||
| 646 | + | // TestRegistryMutableStaleCompletionDoesNotClearLiveAttempt proves the fence on | |
| 647 | + | // the live side: a completion for the retired generation is dropped while a | |
| 648 | + | // current attempt is in flight, so it neither clears the in-flight slot nor | |
| 649 | + | // publishes under the retired registration. | |
| 650 | + | func TestRegistryMutableStaleCompletionDoesNotClearLiveAttempt(t *testing.T) { | |
| 651 | + | clock := newManualClock(time.Unix(100, 0)) | |
| 652 | + | registry, _ := newMutableRegistry(t, newMembershipStore(), clock, (&incarnationSequencer{}).generate) | |
| 653 | + | regA, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |
| 654 | + | require.NoError(t, err) | |
| 655 | + | retired := admitMembershipAttempt(t, registry, regA.Endpoint) | |
| 656 | + | regB, err := registry.UpdateHostPolicy(context.Background(), regA, policyWithTrust("other-trust")) | |
| 657 | + | require.NoError(t, err) | |
| 658 | + | ||
| 659 | + | live := admitMembershipAttempt(t, registry, regB.Endpoint) | |
| 660 | + | before := registry.Snapshot() | |
| 661 | + | for _, result := range []probeResult{ | |
| 662 | + | {endpoint: regB.Endpoint, registration: regA, attempt: retired.token, at: clock.Now(), snapshot: staleObservation()}, | |
| 663 | + | {endpoint: regB.Endpoint, registration: regB, attempt: retired.token, at: clock.Now(), snapshot: staleObservation()}, | |
| 664 | + | {endpoint: regB.Endpoint, registration: regA, attempt: live.token, at: clock.Now(), snapshot: staleObservation()}, | |
| 665 | + | } { | |
| 666 | + | registry.apply(result) | |
| 667 | + | } | |
| 668 | + | require.Equal(t, live.token, attemptFor(registry, regB.Endpoint), "a stale completion must not clear the live attempt") | |
| 669 | + | require.NoError(t, live.ctx.Err(), "a stale completion must not retire the live attempt") | |
| 670 | + | require.Equal(t, before, registry.Snapshot()) | |
| 671 | + | ||
| 672 | + | // The live attempt still completes normally and publishes its own result. | |
| 673 | + | registry.apply(probeResult{endpoint: regB.Endpoint, registration: regB, attempt: live.token, at: clock.Now(), snapshot: currentObservation()}) | |
| 674 | + | require.ErrorIs(t, live.ctx.Err(), context.Canceled) | |
| 675 | + | host, ok := registry.Snapshot().Find(regB.Endpoint) | |
| 676 | + | require.True(t, ok) | |
| 677 | + | require.Equal(t, regB, host.Registration) | |
| 678 | + | require.Equal(t, ports.BrokerDaemonIdentity("current-daemon"), host.Identity) | |
| 679 | + | require.Equal(t, domain.RemoteAvailabilityReachable, host.Availability) | |
| 680 | + | } | |
| 681 | + | ||
| 682 | + | // TestRegistryMutableStaleProbeCannotReviveRemovedRegistration proves the | |
| 683 | + | // remove/re-add fence: a completion for the retired incarnation is dropped after | |
| 684 | + | // the endpoint was re-added under a fresh incarnation, and the re-add supersedes | |
| 685 | + | // rather than revives the retirement. | |
| 686 | + | func TestRegistryMutableStaleProbeCannotReviveRemovedRegistration(t *testing.T) { | |
| 687 | + | clock := newManualClock(time.Unix(100, 0)) | |
| 688 | + | seq := &incarnationSequencer{} | |
| 689 | + | registry, _ := newMutableRegistry(t, newMembershipStore(), clock, seq.generate) | |
| 690 | + | regA, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |
| 691 | + | require.NoError(t, err) | |
| 692 | + | retired := admitMembershipAttempt(t, registry, regA.Endpoint) | |
| 693 | + | ||
| 694 | + | removed, err := registry.RemoveHost(context.Background(), regA) | |
| 695 | + | require.NoError(t, err) | |
| 696 | + | require.True(t, removed) | |
| 697 | + | require.ErrorIs(t, retired.ctx.Err(), context.Canceled) | |
| 698 | + | afterRemove := registry.Snapshot() | |
| 699 | + | require.Len(t, afterRemove.Removed, 1) | |
| 700 | + | require.Equal(t, regA, afterRemove.Removed[0].Registration) | |
| 701 | + | ||
| 702 | + | regB, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |
| 703 | + | require.NoError(t, err) | |
| 704 | + | require.NotEqual(t, regA.Incarnation, regB.Incarnation) | |
| 705 | + | before := registry.Snapshot() | |
| 706 | + | require.Len(t, before.Daemons, 1) | |
| 707 | + | require.Empty(t, before.Removed) | |
| 708 | + | ||
| 709 | + | registry.apply(probeResult{endpoint: regA.Endpoint, registration: regA, attempt: retired.token, at: clock.Now(), snapshot: staleObservation()}) | |
| 710 | + | require.Equal(t, before, registry.Snapshot()) | |
| 711 | + | host, ok := registry.Snapshot().Find(regA.Endpoint) | |
| 712 | + | require.True(t, ok) | |
| 713 | + | require.Equal(t, regB, host.Registration) | |
| 714 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |
| 715 | + | require.Empty(t, host.Identity) | |
| 716 | + | require.Zero(t, attemptFor(registry, regA.Endpoint)) | |
| 717 | + | } | |
| 718 | + | ||
| 719 | + | // TestRegistryMutablePolicyChangeRetiresInflightProbe drives the same fence | |
| 720 | + | // through the production run loop: a policy change retires the in-flight probe | |
| 721 | + | // promptly, a fresh attempt observes the new registration, and the retired | |
| 722 | + | // answer can never surface as the new generation's observation. | |
| 723 | + | func TestRegistryMutablePolicyChangeRetiresInflightProbe(t *testing.T) { | |
| 724 | + | registry, probe := newMutableRegistry(t, newMembershipStore(), newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |
| 725 | + | regA, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |
| 726 | + | require.NoError(t, err) | |
| 727 | + | startRegistry(t, registry) | |
| 728 | + | ||
| 729 | + | first := receiveCall(t, probe) | |
| 730 | + | require.Equal(t, regA, first.registration) | |
| 731 | + | ||
| 732 | + | regB, err := registry.UpdateHostPolicy(context.Background(), regA, policyWithTrust("other-trust")) | |
| 733 | + | require.NoError(t, err) | |
| 734 | + | require.Equal(t, regA.Generation+1, regB.Generation) | |
| 735 | + | select { | |
| 736 | + | case <-probe.canceled: | |
| 737 | + | case <-time.After(time.Second): | |
| 738 | + | t.Fatal("a policy change must retire the in-flight probe") | |
| 739 | + | } | |
| 740 | + | ||
| 741 | + | second := receiveCall(t, probe) | |
| 742 | + | require.Equal(t, regB, second.registration, "the replacement generation is observed") | |
| 743 | + | ||
| 744 | + | // The retired answer arrives after its attempt was retired; only the new | |
| 745 | + | // generation's answer may become the published observation. | |
| 746 | + | first.result <- probeAnswer{snapshot: staleObservation()} | |
| 747 | + | second.result <- probeAnswer{snapshot: currentObservation()} | |
| 748 | + | host := waitHost(t, registry, regB.Endpoint, func(host ports.BrokerDaemonObservation) bool { | |
| 749 | + | return host.Identity == ports.BrokerDaemonIdentity("current-daemon") | |
| 750 | + | }) | |
| 751 | + | require.Equal(t, regB, host.Registration) | |
| 752 | + | require.Equal(t, domain.RemoteAvailabilityReachable, host.Availability) | |
| 753 | + | requireHostStableWithin(t, registry, regB.Endpoint, host, 50*time.Millisecond) | |
| 754 | + | } | |
| 755 | + | ||
| 756 | + | // membershipFixture seeds the state a failed mutation must leave untouched: two | |
| 757 | + | // pinned hosts, one live probe attempt, and one retirement tombstone. | |
| 758 | + | type membershipFixture struct { | |
| 759 | + | registry *Registry | |
| 760 | + | store *membershipStore | |
| 761 | + | alpha domain.RemoteRegistration | |
| 762 | + | attempt admittedAttempt | |
| 763 | + | authority ports.BrokerHosts | |
| 764 | + | before ports.BrokerSnapshot | |
| 765 | + | fences map[string]ports.BrokerHostTombstone | |
| 766 | + | casBefore int | |
| 767 | + | } | |
| 768 | + | ||
| 769 | + | func newMembershipFixture(t *testing.T) *membershipFixture { | |
| 770 | + | t.Helper() | |
| 771 | + | alpha := registration(t, "alpha.test", 1) | |
| 772 | + | fixture := &membershipFixture{alpha: alpha} | |
| 773 | + | fixture.store = newMembershipStore(pinnedRecord(alpha, poolPolicy()), pinnedRecord(registration(t, "beta.test", 2), poolPolicy())) | |
| 774 | + | registry, _ := newMutableRegistry(t, fixture.store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |
| 775 | + | fixture.registry = registry | |
| 776 | + | fixture.attempt = admitMembershipAttempt(t, registry, alpha.Endpoint) | |
| 777 | + | ||
| 778 | + | // Retire beta so the fixture carries a live tombstone alongside the live | |
| 779 | + | // attempt at the moment the failure is injected. | |
| 780 | + | removed, err := registry.RemoveHost(context.Background(), registration(t, "beta.test", 2)) | |
| 781 | + | require.NoError(t, err) | |
| 782 | + | require.True(t, removed) | |
| 783 | + | ||
| 784 | + | fixture.authority = registry.authority | |
| 785 | + | fixture.before = registry.Snapshot() | |
| 786 | + | fixture.fences = membershipFences(registry) | |
| 787 | + | fixture.casBefore = fixture.store.casCalls() | |
| 788 | + | require.Len(t, fixture.fences, 1) | |
| 789 | + | return fixture | |
| 790 | + | } | |
| 791 | + | ||
| 792 | + | // TestRegistryMutableStoreFailureLeavesStateUnchanged pins the commit ordering: | |
| 793 | + | // a definite or indeterminate store failure returns before any projection, | |
| 794 | + | // authority, attempt, or tombstone changes, and the store was genuinely reached. | |
| 795 | + | func TestRegistryMutableStoreFailureLeavesStateUnchanged(t *testing.T) { | |
| 796 | + | definite := []struct { | |
| 797 | + | name string | |
| 798 | + | err error | |
| 799 | + | }{ | |
| 800 | + | {"conflict", ports.ErrBrokerHostConflict}, | |
| 801 | + | {"plain failure", context.DeadlineExceeded}, | |
| 802 | + | } | |
| 803 | + | indeterminate := errors.New("power loss") | |
| 804 | + | ||
| 805 | + | for _, failure := range definite { | |
| 806 | + | t.Run(failure.name, func(t *testing.T) { | |
| 807 | + | runMembershipStoreFailures(t, failure.err) | |
| 808 | + | }) | |
| 809 | + | } | |
| 810 | + | t.Run("indeterminate outcome", func(t *testing.T) { | |
| 811 | + | runMembershipStoreFailures(t, ports.BrokerStoreOutcomeUnknownError{Err: indeterminate}) | |
| 812 | + | }) | |
| 813 | + | } | |
| 814 | + | ||
| 815 | + | func runMembershipStoreFailures(t *testing.T, storeErr error) { | |
| 816 | + | t.Helper() | |
| 817 | + | ops := []struct { | |
| 818 | + | name string | |
| 819 | + | run func(fixture *membershipFixture) (bool, error) | |
| 820 | + | }{ | |
| 821 | + | {"add new", func(f *membershipFixture) (bool, error) { | |
| 822 | + | _, err := f.registry.AddHost(context.Background(), "gamma.test", poolPolicy()) | |
| 823 | + | return false, err | |
| 824 | + | }}, | |
| 825 | + | {"add duplicate", func(f *membershipFixture) (bool, error) { | |
| 826 | + | _, err := f.registry.AddHost(context.Background(), f.alpha.Endpoint, poolPolicy()) | |
| 827 | + | return false, err | |
| 828 | + | }}, | |
| 829 | + | {"remove exact", func(f *membershipFixture) (bool, error) { | |
| 830 | + | return f.registry.RemoveHost(context.Background(), f.alpha) | |
| 831 | + | }}, | |
| 832 | + | {"remove absent", func(f *membershipFixture) (bool, error) { | |
| 833 | + | return f.registry.RemoveHost(context.Background(), registration(t, "delta.test", 4)) | |
| 834 | + | }}, | |
| 835 | + | {"update policy", func(f *membershipFixture) (bool, error) { | |
| 836 | + | _, err := f.registry.UpdateHostPolicy(context.Background(), f.alpha, policyWithTrust("other-trust")) | |
| 837 | + | return false, err | |
| 838 | + | }}, | |
| 839 | + | {"update equal policy", func(f *membershipFixture) (bool, error) { | |
| 840 | + | _, err := f.registry.UpdateHostPolicy(context.Background(), f.alpha, poolPolicy()) | |
| 841 | + | return false, err | |
| 842 | + | }}, | |
| 843 | + | } | |
| 844 | + | for _, op := range ops { | |
| 845 | + | t.Run(op.name, func(t *testing.T) { | |
| 846 | + | fixture := newMembershipFixture(t) | |
| 847 | + | fixture.store.failCAS(storeErr) | |
| 848 | + | ||
| 849 | + | removed, err := op.run(fixture) | |
| 850 | + | require.Error(t, err) | |
| 851 | + | // A commit that never happened is never reported as a removal, even | |
| 852 | + | // for the one operation whose success is a boolean: a caller that | |
| 853 | + | // trusted removed=true from a failed commit would drop live authority. | |
| 854 | + | require.False(t, removed, "a failed commit must never report a removal") | |
| 855 | + | var unknown ports.BrokerStoreOutcomeUnknownError | |
| 856 | + | if errors.As(storeErr, &unknown) { | |
| 857 | + | require.ErrorAs(t, err, &unknown, "an indeterminate outcome stays classifiable through the registry") | |
| 858 | + | } else { | |
| 859 | + | require.ErrorIs(t, err, storeErr) | |
| 860 | + | require.False(t, errors.As(err, &unknown), "a definite failure is not an unknown outcome") | |
| 861 | + | } | |
| 862 | + | ||
| 863 | + | require.Equal(t, fixture.authority, fixture.registry.authority, "authority must not move") | |
| 864 | + | require.Equal(t, fixture.before, fixture.registry.Snapshot(), "projection must not move") | |
| 865 | + | require.Equal(t, fixture.attempt.token, attemptFor(fixture.registry, fixture.alpha.Endpoint), "the in-flight attempt must survive") | |
| 866 | + | require.NoError(t, fixture.attempt.ctx.Err(), "a failed store call must not retire the attempt") | |
| 867 | + | require.Equal(t, fixture.fences, membershipFences(fixture.registry), "tombstones must not move") | |
| 868 | + | require.Equal(t, fixture.casBefore+1, fixture.store.casCalls(), "the mutation reached the store CAS") | |
| 869 | + | }) | |
| 870 | + | } | |
| 871 | + | } | |
| @@ -0,0 +1,871 @@ | |||||
| 1 | + | package broker | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "errors" | |||
| 6 | + | "fmt" | |||
| 7 | + | "sync" | |||
| 8 | + | "testing" | |||
| 9 | + | "time" | |||
| 10 | + | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 12 | + | "github.com/bnema/vev/internal/ports" | |||
| 13 | + | portsmocks "github.com/bnema/vev/internal/ports/mocks" | |||
| 14 | + | "github.com/bnema/vev/internal/protocol" | |||
| 15 | + | "github.com/stretchr/testify/require" | |||
| 16 | + | ) | |||
| 17 | + | ||||
| 18 | + | // Phase A mutable membership: the registry is the only synchronous owner of | |||
| 19 | + | // durable authority, so every admitted mutation must CAS the store, install its | |||
| 20 | + | // projection only after that CAS succeeds, and leave authority, projection, | |||
| 21 | + | // probe attempts, and tombstones untouched on every refusal. The tests below | |||
| 22 | + | // drive the public mutation API and the production fencing path directly, so a | |||
| 23 | + | // regression in any of those rules is observable instead of being masked by the | |||
| 24 | + | // observation scheduler. | |||
| 25 | + | ||||
| 26 | + | // incarnationSequencer is a deterministic entropy seam: every call yields a | |||
| 27 | + | // distinct, never-zero incarnation, so a test can assert that a remove/re-add | |||
| 28 | + | // really minted a fresh identity and that a refused mutation minted none. | |||
| 29 | + | type incarnationSequencer struct { | |||
| 30 | + | mu sync.Mutex | |||
| 31 | + | next byte | |||
| 32 | + | } | |||
| 33 | + | ||||
| 34 | + | func (s *incarnationSequencer) generate() ([16]byte, error) { | |||
| 35 | + | s.mu.Lock() | |||
| 36 | + | defer s.mu.Unlock() | |||
| 37 | + | s.next++ | |||
| 38 | + | var id [16]byte | |||
| 39 | + | id[0] = 0xA5 | |||
| 40 | + | id[15] = s.next | |||
| 41 | + | return id, nil | |||
| 42 | + | } | |||
| 43 | + | ||||
| 44 | + | func (s *incarnationSequencer) generated() int { | |||
| 45 | + | s.mu.Lock() | |||
| 46 | + | defer s.mu.Unlock() | |||
| 47 | + | return int(s.next) | |||
| 48 | + | } | |||
| 49 | + | ||||
| 50 | + | // membershipStore is a race-safe durable store with an observable CAS counter | |||
| 51 | + | // and an injectable CAS failure. The counter proves a mutation reached the | |||
| 52 | + | // store (including the no-op paths, which must still CAS), and the injected | |||
| 53 | + | // failure proves a store fault is synchronous and never installs a projection. | |||
| 54 | + | type membershipStore struct { | |||
| 55 | + | *testStore | |||
| 56 | + | casMu sync.Mutex | |||
| 57 | + | casErr error | |||
| 58 | + | casCall int | |||
| 59 | + | } | |||
| 60 | + | ||||
| 61 | + | func newMembershipStore(hosts ...ports.BrokerHostRecord) *membershipStore { | |||
| 62 | + | return &membershipStore{testStore: &testStore{hosts: ports.BrokerHosts{Revision: 1, Hosts: hosts}}} | |||
| 63 | + | } | |||
| 64 | + | ||||
| 65 | + | func (s *membershipStore) ReplaceHosts(expected uint64, hosts []ports.BrokerHostRecord) error { | |||
| 66 | + | s.casMu.Lock() | |||
| 67 | + | s.casCall++ | |||
| 68 | + | err := s.casErr | |||
| 69 | + | s.casMu.Unlock() | |||
| 70 | + | if err != nil { | |||
| 71 | + | return err | |||
| 72 | + | } | |||
| 73 | + | return s.testStore.ReplaceHosts(expected, hosts) | |||
| 74 | + | } | |||
| 75 | + | ||||
| 76 | + | func (s *membershipStore) failCAS(err error) { | |||
| 77 | + | s.casMu.Lock() | |||
| 78 | + | s.casErr = err | |||
| 79 | + | s.casMu.Unlock() | |||
| 80 | + | } | |||
| 81 | + | ||||
| 82 | + | func (s *membershipStore) casCalls() int { | |||
| 83 | + | s.casMu.Lock() | |||
| 84 | + | defer s.casMu.Unlock() | |||
| 85 | + | return s.casCall | |||
| 86 | + | } | |||
| 87 | + | ||||
| 88 | + | // durableAuthority reads the store's committed authority, exactly as a reopen | |||
| 89 | + | // would: membership and its revision token come from the store, never from the | |||
| 90 | + | // registry's in-memory copy. | |||
| 91 | + | func (s *membershipStore) durableAuthority() ports.BrokerHosts { | |||
| 92 | + | s.testStore.mu.Lock() | |||
| 93 | + | defer s.testStore.mu.Unlock() | |||
| 94 | + | return ports.BrokerHosts{Revision: s.testStore.hosts.Revision, Hosts: append([]ports.BrokerHostRecord(nil), s.testStore.hosts.Hosts...)} | |||
| 95 | + | } | |||
| 96 | + | ||||
| 97 | + | // newMutableRegistry builds a mutable registry with exact scheduling, a | |||
| 98 | + | // deterministic incarnation seam, and its scripted probe. | |||
| 99 | + | func newMutableRegistry(t *testing.T, store ports.BrokerHostStore, clock *manualClock, generator func() ([16]byte, error)) (*Registry, *testProbe) { | |||
| 100 | + | t.Helper() | |||
| 101 | + | probe := newTestProbe(2) | |||
| 102 | + | registry, err := NewRegistryWithConfig(1, store, probe, clock, nil, RegistryConfig{ | |||
| 103 | + | MembershipMode: MembershipMutable, | |||
| 104 | + | IncarnationGenerator: generator, | |||
| 105 | + | ObservationDisabled: false, | |||
| 106 | + | }) | |||
| 107 | + | require.NoError(t, err) | |||
| 108 | + | registry.jitter = identityJitter | |||
| 109 | + | return registry, probe | |||
| 110 | + | } | |||
| 111 | + | ||||
| 112 | + | func policyWithTrust(trust string) ports.BrokerPolicy { | |||
| 113 | + | policy := poolPolicy() | |||
| 114 | + | policy.Trust = trust | |||
| 115 | + | return policy | |||
| 116 | + | } | |||
| 117 | + | ||||
| 118 | + | func pinnedRecord(reg domain.RemoteRegistration, policy ports.BrokerPolicy) ports.BrokerHostRecord { | |||
| 119 | + | return ports.BrokerHostRecord{Registration: reg, Pinned: true, Policy: policy} | |||
| 120 | + | } | |||
| 121 | + | ||||
| 122 | + | // admittedAttempt is one in-flight probe attempt a test installs directly, so a | |||
| 123 | + | // completion can be delivered to apply without racing the scheduler. | |||
| 124 | + | type admittedAttempt struct { | |||
| 125 | + | token uint64 | |||
| 126 | + | ctx context.Context | |||
| 127 | + | cancel context.CancelFunc | |||
| 128 | + | } | |||
| 129 | + | ||||
| 130 | + | func admitMembershipAttempt(t *testing.T, r *Registry, endpoint string) admittedAttempt { | |||
| 131 | + | t.Helper() | |||
| 132 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 133 | + | t.Cleanup(cancel) | |||
| 134 | + | r.mu.Lock() | |||
| 135 | + | defer r.mu.Unlock() | |||
| 136 | + | r.attempts++ | |||
| 137 | + | attempt := admittedAttempt{token: r.attempts, ctx: ctx, cancel: cancel} | |||
| 138 | + | r.inflight[endpoint] = &probeAttempt{token: attempt.token, cancel: cancel} | |||
| 139 | + | return attempt | |||
| 140 | + | } | |||
| 141 | + | ||||
| 142 | + | // membershipFences snapshots the tombstone set so a failed mutation can be | |||
| 143 | + | // proven to have retired nothing. | |||
| 144 | + | func membershipFences(r *Registry) map[string]ports.BrokerHostTombstone { | |||
| 145 | + | r.mu.Lock() | |||
| 146 | + | defer r.mu.Unlock() | |||
| 147 | + | out := make(map[string]ports.BrokerHostTombstone, len(r.tombstones)) | |||
| 148 | + | for endpoint, tombstone := range r.tombstones { | |||
| 149 | + | out[endpoint] = tombstone | |||
| 150 | + | } | |||
| 151 | + | return out | |||
| 152 | + | } | |||
| 153 | + | ||||
| 154 | + | func staleObservation() ports.BrokerDaemonObservation { | |||
| 155 | + | return ports.BrokerDaemonObservation{ | |||
| 156 | + | Identity: "stale-daemon", | |||
| 157 | + | Incarnation: ports.BrokerDaemonIncarnation{7}, | |||
| 158 | + | ProtocolVersion: protocol.Version, | |||
| 159 | + | Availability: domain.RemoteAvailabilityReachable, | |||
| 160 | + | InventoryKnown: true, | |||
| 161 | + | } | |||
| 162 | + | } | |||
| 163 | + | ||||
| 164 | + | func currentObservation() ports.BrokerDaemonObservation { | |||
| 165 | + | return ports.BrokerDaemonObservation{ | |||
| 166 | + | Identity: "current-daemon", | |||
| 167 | + | Incarnation: ports.BrokerDaemonIncarnation{9}, | |||
| 168 | + | ProtocolVersion: protocol.Version, | |||
| 169 | + | Availability: domain.RemoteAvailabilityReachable, | |||
| 170 | + | InventoryKnown: true, | |||
| 171 | + | } | |||
| 172 | + | } | |||
| 173 | + | ||||
| 174 | + | // TestRegistryMutableConcurrentAddsLoseNoUpdate proves concurrent membership | |||
| 175 | + | // mutations serialize over durable authority: every added endpoint survives, | |||
| 176 | + | // each addition advances the revision by exactly one, and the registry's copy | |||
| 177 | + | // still equals what the store committed. | |||
| 178 | + | func TestRegistryMutableConcurrentAddsLoseNoUpdate(t *testing.T) { | |||
| 179 | + | const adds = 32 | |||
| 180 | + | store := newMembershipStore() | |||
| 181 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |||
| 182 | + | start := registry.authority.Revision | |||
| 183 | + | ||||
| 184 | + | endpoints := make([]string, adds) | |||
| 185 | + | for i := range endpoints { | |||
| 186 | + | endpoints[i] = fmt.Sprintf("host-%d.test", i) | |||
| 187 | + | } | |||
| 188 | + | ||||
| 189 | + | var wg sync.WaitGroup | |||
| 190 | + | errs := make([]error, adds) | |||
| 191 | + | regs := make([]domain.RemoteRegistration, adds) | |||
| 192 | + | startGate := make(chan struct{}) | |||
| 193 | + | for i, endpoint := range endpoints { | |||
| 194 | + | wg.Add(1) | |||
| 195 | + | go func(i int, endpoint string) { | |||
| 196 | + | defer wg.Done() | |||
| 197 | + | <-startGate | |||
| 198 | + | regs[i], errs[i] = registry.AddHost(context.Background(), endpoint, poolPolicy()) | |||
| 199 | + | }(i, endpoint) | |||
| 200 | + | } | |||
| 201 | + | close(startGate) | |||
| 202 | + | wg.Wait() | |||
| 203 | + | ||||
| 204 | + | for i, err := range errs { | |||
| 205 | + | require.NoErrorf(t, err, "concurrent add %q", endpoints[i]) | |||
| 206 | + | } | |||
| 207 | + | require.Equal(t, adds, store.casCalls(), "every admitted add must reach the store CAS") | |||
| 208 | + | require.Equal(t, start+adds, registry.authority.Revision, "each add advances the revision exactly once") | |||
| 209 | + | require.Len(t, registry.authority.Hosts, adds) | |||
| 210 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |||
| 211 | + | ||||
| 212 | + | // Admission order is the lock order, not the launch order, so the | |||
| 213 | + | // publication order is compared against the returned identities by endpoint. | |||
| 214 | + | returned := make(map[string]domain.RemoteRegistration, adds) | |||
| 215 | + | for i, endpoint := range endpoints { | |||
| 216 | + | require.False(t, regs[i].IsZero(), "an admitted add returns its minted identity") | |||
| 217 | + | require.Equal(t, endpoint, regs[i].Endpoint) | |||
| 218 | + | require.Equal(t, domain.RemoteGeneration(1), regs[i].Generation) | |||
| 219 | + | returned[endpoint] = regs[i] | |||
| 220 | + | } | |||
| 221 | + | require.Len(t, returned, adds, "every concurrent add minted a distinct identity") | |||
| 222 | + | ||||
| 223 | + | seen := make(map[string]domain.RemoteRegistration, adds) | |||
| 224 | + | for _, record := range registry.authority.Hosts { | |||
| 225 | + | require.NotContains(t, seen, record.Registration.Endpoint, "no endpoint may be added twice") | |||
| 226 | + | require.Equal(t, returned[record.Registration.Endpoint], record.Registration) | |||
| 227 | + | seen[record.Registration.Endpoint] = record.Registration | |||
| 228 | + | require.Equal(t, poolPolicy(), record.Policy) | |||
| 229 | + | require.True(t, record.Pinned) | |||
| 230 | + | } | |||
| 231 | + | require.Len(t, seen, adds) | |||
| 232 | + | ||||
| 233 | + | snapshot := registry.Snapshot() | |||
| 234 | + | require.Len(t, snapshot.Daemons, adds) | |||
| 235 | + | require.NoError(t, snapshot.Validate()) | |||
| 236 | + | for _, endpoint := range endpoints { | |||
| 237 | + | host, ok := snapshot.Find(endpoint) | |||
| 238 | + | require.True(t, ok, "a concurrently added endpoint must be published") | |||
| 239 | + | require.Equal(t, seen[endpoint], host.Registration) | |||
| 240 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |||
| 241 | + | } | |||
| 242 | + | } | |||
| 243 | + | ||||
| 244 | + | // TestRegistryMembershipModeRejectsUnknownMode proves the run mode is a closed | |||
| 245 | + | // set: an out-of-range mode is refused at construction, before any store call, | |||
| 246 | + | // so a caller can never run a registry whose mutation policy is undefined. | |||
| 247 | + | func TestRegistryMembershipModeRejectsUnknownMode(t *testing.T) { | |||
| 248 | + | // The mock has no expectations: any store call during construction fails | |||
| 249 | + | // the test, so a refusal must happen before the durable store is touched. | |||
| 250 | + | store := portsmocks.NewMockBrokerHostStore(t) | |||
| 251 | + | registry, err := NewRegistryWithConfig(1, store, newTestProbe(1), newManualClock(time.Unix(100, 0)), nil, | |||
| 252 | + | RegistryConfig{MembershipMode: MembershipMode(9)}) | |||
| 253 | + | require.ErrorContains(t, err, "invalid membership mode") | |||
| 254 | + | require.Nil(t, registry) | |||
| 255 | + | } | |||
| 256 | + | ||||
| 257 | + | // TestRegistryImmutableMembershipRefusesEveryMutation pins the default: a | |||
| 258 | + | // registry that was not explicitly configured mutable refuses every membership | |||
| 259 | + | // mutation, including the ones that would be no-ops, and refuses them without | |||
| 260 | + | // touching the durable store or its own projection. | |||
| 261 | + | func TestRegistryImmutableMembershipRefusesEveryMutation(t *testing.T) { | |||
| 262 | + | reg := registration(t, "alpha.test", 1) | |||
| 263 | + | store := portsmocks.NewMockBrokerHostStore(t) | |||
| 264 | + | store.EXPECT().Load().Return(ports.BrokerSnapshot{}, nil).Once() | |||
| 265 | + | store.EXPECT().LoadHosts().Return(ports.BrokerHosts{Revision: 3, Hosts: []ports.BrokerHostRecord{pinnedRecord(reg, poolPolicy())}}, nil).Once() | |||
| 266 | + | registry, err := NewRegistry(1, store, newTestProbe(1), newManualClock(time.Unix(100, 0)), nil) | |||
| 267 | + | require.NoError(t, err) | |||
| 268 | + | authority := registry.authority | |||
| 269 | + | before := registry.Snapshot() | |||
| 270 | + | ||||
| 271 | + | tests := []struct { | |||
| 272 | + | name string | |||
| 273 | + | run func(r *Registry) error | |||
| 274 | + | }{ | |||
| 275 | + | {"add new endpoint", func(r *Registry) error { | |||
| 276 | + | _, err := r.AddHost(context.Background(), "beta.test", poolPolicy()) | |||
| 277 | + | return err | |||
| 278 | + | }}, | |||
| 279 | + | {"add duplicate endpoint with equal policy is a no-op", func(r *Registry) error { | |||
| 280 | + | _, err := r.AddHost(context.Background(), reg.Endpoint, poolPolicy()) | |||
| 281 | + | return err | |||
| 282 | + | }}, | |||
| 283 | + | {"remove the exact registration", func(r *Registry) error { | |||
| 284 | + | _, err := r.RemoveHost(context.Background(), reg) | |||
| 285 | + | return err | |||
| 286 | + | }}, | |||
| 287 | + | {"remove an absent endpoint is a no-op", func(r *Registry) error { | |||
| 288 | + | _, err := r.RemoveHost(context.Background(), registration(t, "gamma.test", 3)) | |||
| 289 | + | return err | |||
| 290 | + | }}, | |||
| 291 | + | {"update to a different policy", func(r *Registry) error { | |||
| 292 | + | _, err := r.UpdateHostPolicy(context.Background(), reg, policyWithTrust("other-trust")) | |||
| 293 | + | return err | |||
| 294 | + | }}, | |||
| 295 | + | {"update to the equal policy is a no-op", func(r *Registry) error { | |||
| 296 | + | _, err := r.UpdateHostPolicy(context.Background(), reg, poolPolicy()) | |||
| 297 | + | return err | |||
| 298 | + | }}, | |||
| 299 | + | } | |||
| 300 | + | for _, tc := range tests { | |||
| 301 | + | t.Run(tc.name, func(t *testing.T) { | |||
| 302 | + | require.ErrorIs(t, tc.run(registry), ports.ErrBrokerMembershipImmutable) | |||
| 303 | + | require.Equal(t, authority, registry.authority) | |||
| 304 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 305 | + | require.Empty(t, membershipFences(registry)) | |||
| 306 | + | }) | |||
| 307 | + | } | |||
| 308 | + | ||||
| 309 | + | // Control: the same duplicate add is admitted when the mode is mutable, so | |||
| 310 | + | // the refusals above are the mode and not the validity of the operation. | |||
| 311 | + | seq := &incarnationSequencer{} | |||
| 312 | + | mutable, _ := newMutableRegistry(t, newMembershipStore(pinnedRecord(reg, poolPolicy())), newManualClock(time.Unix(100, 0)), seq.generate) | |||
| 313 | + | added, err := mutable.AddHost(context.Background(), reg.Endpoint, poolPolicy()) | |||
| 314 | + | require.NoError(t, err) | |||
| 315 | + | require.Equal(t, reg, added) | |||
| 316 | + | require.Equal(t, 0, seq.generated(), "a duplicate add must not mint an identity") | |||
| 317 | + | } | |||
| 318 | + | ||||
| 319 | + | // TestRegistryMutableAddPinsDuplicatePreservingIdentity proves a duplicate | |||
| 320 | + | // same-policy addition pins the existing record without changing its identity, | |||
| 321 | + | // and that even that no-op CAS-verifies durable authority. | |||
| 322 | + | func TestRegistryMutableAddPinsDuplicatePreservingIdentity(t *testing.T) { | |||
| 323 | + | seq := &incarnationSequencer{} | |||
| 324 | + | store := newMembershipStore(ports.BrokerHostRecord{Registration: registration(t, "alpha.test", 1), Learned: true, Policy: poolPolicy()}) | |||
| 325 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), seq.generate) | |||
| 326 | + | reg := registration(t, "alpha.test", 1) | |||
| 327 | + | ||||
| 328 | + | got, err := registry.AddHost(context.Background(), reg.Endpoint, poolPolicy()) | |||
| 329 | + | require.NoError(t, err) | |||
| 330 | + | require.Equal(t, reg, got, "a duplicate add preserves the registration identity") | |||
| 331 | + | require.Equal(t, 1, store.casCalls()) | |||
| 332 | + | require.Len(t, registry.authority.Hosts, 1) | |||
| 333 | + | record := registry.authority.Hosts[0] | |||
| 334 | + | require.Equal(t, reg, record.Registration) | |||
| 335 | + | require.Equal(t, poolPolicy(), record.Policy) | |||
| 336 | + | require.True(t, record.Pinned, "the duplicate add pins the learned registration") | |||
| 337 | + | require.True(t, record.Learned, "pinning never discards learned provenance") | |||
| 338 | + | require.Equal(t, uint64(2), registry.authority.Revision) | |||
| 339 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |||
| 340 | + | require.Equal(t, 0, seq.generated(), "pinning mints no identity") | |||
| 341 | + | ||||
| 342 | + | host, ok := registry.Snapshot().Find(reg.Endpoint) | |||
| 343 | + | require.True(t, ok) | |||
| 344 | + | require.Equal(t, reg, host.Registration) | |||
| 345 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |||
| 346 | + | ||||
| 347 | + | // A second duplicate is a no-op that still CAS-verifies: authority moves | |||
| 348 | + | // durably, the projection does not republish, and identity is unchanged. | |||
| 349 | + | before := registry.Snapshot() | |||
| 350 | + | again, err := registry.AddHost(context.Background(), reg.Endpoint, poolPolicy()) | |||
| 351 | + | require.NoError(t, err) | |||
| 352 | + | require.Equal(t, reg, again) | |||
| 353 | + | require.Equal(t, 2, store.casCalls()) | |||
| 354 | + | require.Equal(t, uint64(3), registry.authority.Revision) | |||
| 355 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |||
| 356 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 357 | + | } | |||
| 358 | + | ||||
| 359 | + | // TestRegistryMutableAddConflictingPolicyConflicts proves a conflicting policy | |||
| 360 | + | // is a definite conflict: it is refused before the store CAS, and it mints no | |||
| 361 | + | // identity. | |||
| 362 | + | func TestRegistryMutableAddConflictingPolicyConflicts(t *testing.T) { | |||
| 363 | + | seq := &incarnationSequencer{} | |||
| 364 | + | reg := registration(t, "alpha.test", 1) | |||
| 365 | + | store := newMembershipStore(pinnedRecord(reg, poolPolicy())) | |||
| 366 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), seq.generate) | |||
| 367 | + | authority := registry.authority | |||
| 368 | + | before := registry.Snapshot() | |||
| 369 | + | ||||
| 370 | + | _, err := registry.AddHost(context.Background(), reg.Endpoint, policyWithTrust("other-trust")) | |||
| 371 | + | require.ErrorIs(t, err, ports.ErrBrokerHostConflict) | |||
| 372 | + | require.Equal(t, 0, store.casCalls(), "a conflict is refused before the CAS") | |||
| 373 | + | require.Equal(t, authority, registry.authority) | |||
| 374 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 375 | + | require.Equal(t, 0, seq.generated()) | |||
| 376 | + | } | |||
| 377 | + | ||||
| 378 | + | // TestRegistryMutableAddEntropyFailureLeavesAuthorityUnchanged pins the entropy | |||
| 379 | + | // seam: an unusable incarnation is refused before anything durable is staged. | |||
| 380 | + | func TestRegistryMutableAddEntropyFailureLeavesAuthorityUnchanged(t *testing.T) { | |||
| 381 | + | tests := []struct { | |||
| 382 | + | name string | |||
| 383 | + | generator func() ([16]byte, error) | |||
| 384 | + | want string | |||
| 385 | + | }{ | |||
| 386 | + | {"entropy source fails", func() ([16]byte, error) { return [16]byte{}, errors.New("no entropy") }, "no entropy"}, | |||
| 387 | + | {"entropy source returns a zero identity", func() ([16]byte, error) { return [16]byte{}, nil }, "zero"}, | |||
| 388 | + | } | |||
| 389 | + | for _, tc := range tests { | |||
| 390 | + | t.Run(tc.name, func(t *testing.T) { | |||
| 391 | + | store := newMembershipStore() | |||
| 392 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), tc.generator) | |||
| 393 | + | authority := registry.authority | |||
| 394 | + | before := registry.Snapshot() | |||
| 395 | + | ||||
| 396 | + | _, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |||
| 397 | + | require.ErrorContains(t, err, tc.want) | |||
| 398 | + | require.Equal(t, 0, store.casCalls()) | |||
| 399 | + | require.Equal(t, authority, registry.authority) | |||
| 400 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 401 | + | }) | |||
| 402 | + | } | |||
| 403 | + | } | |||
| 404 | + | ||||
| 405 | + | // TestRegistryMutableRemoveExactStaleAndAbsent pins removal admission: only the | |||
| 406 | + | // exact expected registration is removed, a stale generation or incarnation is | |||
| 407 | + | // a definite conflict, and an absent endpoint reports false only after the no-op | |||
| 408 | + | // CAS verified the loaded revision. | |||
| 409 | + | func TestRegistryMutableRemoveExactStaleAndAbsent(t *testing.T) { | |||
| 410 | + | alpha := registration(t, "alpha.test", 1) | |||
| 411 | + | beta := registration(t, "beta.test", 2) | |||
| 412 | + | store := newMembershipStore(pinnedRecord(alpha, poolPolicy()), pinnedRecord(beta, poolPolicy())) | |||
| 413 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |||
| 414 | + | authority := registry.authority | |||
| 415 | + | before := registry.Snapshot() | |||
| 416 | + | ||||
| 417 | + | staleGeneration := alpha | |||
| 418 | + | staleGeneration.Generation++ | |||
| 419 | + | staleIncarnation := alpha | |||
| 420 | + | staleIncarnation.Incarnation = [16]byte{0xEE} | |||
| 421 | + | for name, expected := range map[string]domain.RemoteRegistration{ | |||
| 422 | + | "stale generation": staleGeneration, | |||
| 423 | + | "stale incarnation": staleIncarnation, | |||
| 424 | + | } { | |||
| 425 | + | t.Run(name, func(t *testing.T) { | |||
| 426 | + | removed, err := registry.RemoveHost(context.Background(), expected) | |||
| 427 | + | require.ErrorIs(t, err, ports.ErrBrokerHostConflict) | |||
| 428 | + | require.False(t, removed) | |||
| 429 | + | require.Equal(t, 0, store.casCalls(), "a mismatch is refused before the CAS") | |||
| 430 | + | require.Equal(t, authority, registry.authority) | |||
| 431 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 432 | + | require.Empty(t, membershipFences(registry)) | |||
| 433 | + | }) | |||
| 434 | + | } | |||
| 435 | + | ||||
| 436 | + | removed, err := registry.RemoveHost(context.Background(), alpha) | |||
| 437 | + | require.NoError(t, err) | |||
| 438 | + | require.True(t, removed) | |||
| 439 | + | require.Equal(t, 1, store.casCalls()) | |||
| 440 | + | require.Len(t, registry.authority.Hosts, 1) | |||
| 441 | + | require.Equal(t, beta, registry.authority.Hosts[0].Registration) | |||
| 442 | + | require.Equal(t, uint64(2), registry.authority.Revision) | |||
| 443 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |||
| 444 | + | afterRemove := registry.Snapshot() | |||
| 445 | + | require.Len(t, afterRemove.Daemons, 1) | |||
| 446 | + | require.Equal(t, beta, afterRemove.Daemons[0].Registration) | |||
| 447 | + | require.Len(t, afterRemove.Removed, 1) | |||
| 448 | + | require.Equal(t, alpha, afterRemove.Removed[0].Registration) | |||
| 449 | + | require.Equal(t, afterRemove.Revision, afterRemove.Removed[0].RetiredRevision, | |||
| 450 | + | "a tombstone is retired at the revision of the publication that carries it") | |||
| 451 | + | ||||
| 452 | + | // An absent endpoint is a no-op that still CAS-verifies: durable authority | |||
| 453 | + | // advances, while nothing is republished and nothing is retired. | |||
| 454 | + | absent, err := registry.RemoveHost(context.Background(), alpha) | |||
| 455 | + | require.NoError(t, err) | |||
| 456 | + | require.False(t, absent) | |||
| 457 | + | require.Equal(t, 2, store.casCalls()) | |||
| 458 | + | require.Equal(t, uint64(3), registry.authority.Revision) | |||
| 459 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |||
| 460 | + | require.Equal(t, afterRemove, registry.Snapshot()) | |||
| 461 | + | } | |||
| 462 | + | ||||
| 463 | + | // TestRegistryMutableNoOpRemoveVerifiesCAS pins the no-op CAS itself: with | |||
| 464 | + | // authority already moved by another mutator, an absent-endpoint removal must | |||
| 465 | + | // report a conflict instead of a hollow success, because the loaded revision was | |||
| 466 | + | // never verified. | |||
| 467 | + | func TestRegistryMutableNoOpRemoveVerifiesCAS(t *testing.T) { | |||
| 468 | + | alpha := registration(t, "alpha.test", 1) | |||
| 469 | + | store := newMembershipStore(pinnedRecord(alpha, poolPolicy())) | |||
| 470 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |||
| 471 | + | authority := registry.authority | |||
| 472 | + | before := registry.Snapshot() | |||
| 473 | + | ||||
| 474 | + | // An illicit second mutator moves durable authority behind the registry. | |||
| 475 | + | require.NoError(t, store.testStore.ReplaceHosts(authority.Revision, []ports.BrokerHostRecord{pinnedRecord(alpha, poolPolicy())})) | |||
| 476 | + | ||||
| 477 | + | removed, err := registry.RemoveHost(context.Background(), registration(t, "gamma.test", 3)) | |||
| 478 | + | require.ErrorIs(t, err, ports.ErrBrokerHostConflict) | |||
| 479 | + | require.False(t, removed) | |||
| 480 | + | require.Equal(t, 1, store.casCalls()) | |||
| 481 | + | require.Equal(t, authority, registry.authority) | |||
| 482 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 483 | + | } | |||
| 484 | + | ||||
| 485 | + | // TestRegistryMutableRemoveReAddGetsFreshIncarnation pins the ABA fence at the | |||
| 486 | + | // mutation boundary: a full removal followed by a re-add mints a fresh | |||
| 487 | + | // incarnation with a fresh generation series, and the retired registration is | |||
| 488 | + | // fenced rather than revived. | |||
| 489 | + | func TestRegistryMutableRemoveReAddGetsFreshIncarnation(t *testing.T) { | |||
| 490 | + | seq := &incarnationSequencer{} | |||
| 491 | + | store := newMembershipStore() | |||
| 492 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), seq.generate) | |||
| 493 | + | ||||
| 494 | + | first, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |||
| 495 | + | require.NoError(t, err) | |||
| 496 | + | require.Equal(t, 1, seq.generated()) | |||
| 497 | + | ||||
| 498 | + | removed, err := registry.RemoveHost(context.Background(), first) | |||
| 499 | + | require.NoError(t, err) | |||
| 500 | + | require.True(t, removed) | |||
| 501 | + | ||||
| 502 | + | second, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |||
| 503 | + | require.NoError(t, err) | |||
| 504 | + | require.Equal(t, 2, seq.generated()) | |||
| 505 | + | require.False(t, first.Equal(second), "a re-add must not reuse the retired registration") | |||
| 506 | + | require.NotEqual(t, first.Incarnation, second.Incarnation) | |||
| 507 | + | require.Equal(t, domain.RemoteGeneration(1), second.Generation) | |||
| 508 | + | require.Len(t, registry.authority.Hosts, 1) | |||
| 509 | + | require.Equal(t, second, registry.authority.Hosts[0].Registration) | |||
| 510 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |||
| 511 | + | ||||
| 512 | + | snapshot := registry.Snapshot() | |||
| 513 | + | require.Len(t, snapshot.Daemons, 1) | |||
| 514 | + | require.Equal(t, second, snapshot.Daemons[0].Registration) | |||
| 515 | + | require.Empty(t, snapshot.Removed, "a live endpoint is never published as a tombstone") | |||
| 516 | + | host, ok := snapshot.Find(second.Endpoint) | |||
| 517 | + | require.True(t, ok) | |||
| 518 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |||
| 519 | + | } | |||
| 520 | + | ||||
| 521 | + | // TestRegistryMutableUpdatePolicyIncrementsGenerationOnce pins the policy | |||
| 522 | + | // update: a changed policy advances the registration generation exactly once and | |||
| 523 | + | // is the only thing that does; an unmatchable expectation is a definite conflict | |||
| 524 | + | // refused before the CAS. | |||
| 525 | + | func TestRegistryMutableUpdatePolicyIncrementsGenerationOnce(t *testing.T) { | |||
| 526 | + | alpha := registration(t, "alpha.test", 1) | |||
| 527 | + | store := newMembershipStore(pinnedRecord(alpha, poolPolicy())) | |||
| 528 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |||
| 529 | + | ||||
| 530 | + | // Not found and a stale expectation are definite conflicts, refused before | |||
| 531 | + | // the store is touched. | |||
| 532 | + | authority := registry.authority | |||
| 533 | + | before := registry.Snapshot() | |||
| 534 | + | _, err := registry.UpdateHostPolicy(context.Background(), registration(t, "gamma.test", 3), policyWithTrust("other-trust")) | |||
| 535 | + | require.ErrorIs(t, err, ports.ErrBrokerHostConflict) | |||
| 536 | + | stale := alpha | |||
| 537 | + | stale.Generation++ | |||
| 538 | + | _, err = registry.UpdateHostPolicy(context.Background(), stale, policyWithTrust("other-trust")) | |||
| 539 | + | require.ErrorIs(t, err, ports.ErrBrokerHostConflict) | |||
| 540 | + | require.Equal(t, 0, store.casCalls()) | |||
| 541 | + | require.Equal(t, authority, registry.authority) | |||
| 542 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 543 | + | ||||
| 544 | + | updated, err := registry.UpdateHostPolicy(context.Background(), alpha, policyWithTrust("other-trust")) | |||
| 545 | + | require.NoError(t, err) | |||
| 546 | + | require.Equal(t, alpha.Generation+1, updated.Generation, "a policy change advances the generation exactly once") | |||
| 547 | + | require.Equal(t, alpha.Incarnation, updated.Incarnation, "a policy change never touches the incarnation") | |||
| 548 | + | require.Equal(t, 1, store.casCalls()) | |||
| 549 | + | require.Equal(t, uint64(2), registry.authority.Revision) | |||
| 550 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |||
| 551 | + | require.Equal(t, []ports.BrokerHostRecord{pinnedRecord(updated, policyWithTrust("other-trust"))}, store.durableAuthority().Hosts) | |||
| 552 | + | ||||
| 553 | + | host, ok := registry.Snapshot().Find(alpha.Endpoint) | |||
| 554 | + | require.True(t, ok) | |||
| 555 | + | require.Equal(t, updated, host.Registration) | |||
| 556 | + | require.Equal(t, policyWithTrust("other-trust"), host.Policy) | |||
| 557 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |||
| 558 | + | } | |||
| 559 | + | ||||
| 560 | + | // TestRegistryMutableUpdateEqualPolicyPreservesIdentityAndVerifiesCAS proves an | |||
| 561 | + | // equal-policy update is an identity-preserving no-op that still CAS-verifies | |||
| 562 | + | // durable authority. | |||
| 563 | + | func TestRegistryMutableUpdateEqualPolicyPreservesIdentityAndVerifiesCAS(t *testing.T) { | |||
| 564 | + | alpha := registration(t, "alpha.test", 1) | |||
| 565 | + | store := newMembershipStore(pinnedRecord(alpha, poolPolicy())) | |||
| 566 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |||
| 567 | + | before := registry.Snapshot() | |||
| 568 | + | ||||
| 569 | + | got, err := registry.UpdateHostPolicy(context.Background(), alpha, poolPolicy()) | |||
| 570 | + | require.NoError(t, err) | |||
| 571 | + | require.Equal(t, alpha, got, "an equal policy preserves the registration identity") | |||
| 572 | + | require.Equal(t, 1, store.casCalls()) | |||
| 573 | + | require.Equal(t, uint64(2), registry.authority.Revision) | |||
| 574 | + | require.Equal(t, registry.authority, store.durableAuthority()) | |||
| 575 | + | require.Equal(t, before, registry.Snapshot(), "an unchanged policy republishes nothing") | |||
| 576 | + | ||||
| 577 | + | // The no-op still verified the loaded revision: a revision moved behind the | |||
| 578 | + | // registry turns the same call into a definite conflict. | |||
| 579 | + | require.NoError(t, store.testStore.ReplaceHosts(registry.authority.Revision, []ports.BrokerHostRecord{pinnedRecord(alpha, poolPolicy())})) | |||
| 580 | + | authority := registry.authority | |||
| 581 | + | _, err = registry.UpdateHostPolicy(context.Background(), alpha, poolPolicy()) | |||
| 582 | + | require.ErrorIs(t, err, ports.ErrBrokerHostConflict) | |||
| 583 | + | require.Equal(t, 2, store.casCalls()) | |||
| 584 | + | require.Equal(t, authority, registry.authority) | |||
| 585 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 586 | + | } | |||
| 587 | + | ||||
| 588 | + | // TestRegistryMutableUpdatePolicyOverflowRefused pins the generation series: a | |||
| 589 | + | // policy change at the last generation fails closed instead of wrapping to zero, | |||
| 590 | + | // which would re-trust an already queued observation. | |||
| 591 | + | func TestRegistryMutableUpdatePolicyOverflowRefused(t *testing.T) { | |||
| 592 | + | alpha := registration(t, "alpha.test", 1) | |||
| 593 | + | alpha.Generation = ^domain.RemoteGeneration(0) | |||
| 594 | + | store := newMembershipStore(pinnedRecord(alpha, poolPolicy())) | |||
| 595 | + | registry, _ := newMutableRegistry(t, store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |||
| 596 | + | authority := registry.authority | |||
| 597 | + | before := registry.Snapshot() | |||
| 598 | + | ||||
| 599 | + | _, err := registry.UpdateHostPolicy(context.Background(), alpha, policyWithTrust("other-trust")) | |||
| 600 | + | require.ErrorIs(t, err, ports.ErrBrokerRevisionExhausted) | |||
| 601 | + | require.Equal(t, 0, store.casCalls()) | |||
| 602 | + | require.Equal(t, authority, registry.authority) | |||
| 603 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 604 | + | } | |||
| 605 | + | ||||
| 606 | + | // TestRegistryMutableProbeCompletionFencedByChangedGeneration proves a | |||
| 607 | + | // completion whose registration was replaced by a policy update can never | |||
| 608 | + | // populate the new generation, whether it carries the retired registration or | |||
| 609 | + | // the current one with a retired attempt token. | |||
| 610 | + | func TestRegistryMutableProbeCompletionFencedByChangedGeneration(t *testing.T) { | |||
| 611 | + | clock := newManualClock(time.Unix(100, 0)) | |||
| 612 | + | store := newMembershipStore() | |||
| 613 | + | registry, _ := newMutableRegistry(t, store, clock, (&incarnationSequencer{}).generate) | |||
| 614 | + | regA, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |||
| 615 | + | require.NoError(t, err) | |||
| 616 | + | retired := admitMembershipAttempt(t, registry, regA.Endpoint) | |||
| 617 | + | regB, err := registry.UpdateHostPolicy(context.Background(), regA, policyWithTrust("other-trust")) | |||
| 618 | + | require.NoError(t, err) | |||
| 619 | + | require.Equal(t, regA.Generation+1, regB.Generation) | |||
| 620 | + | require.ErrorIs(t, retired.ctx.Err(), context.Canceled, "replacing a registration retires its attempt") | |||
| 621 | + | require.Zero(t, attemptFor(registry, regA.Endpoint)) | |||
| 622 | + | ||||
| 623 | + | before := registry.Snapshot() | |||
| 624 | + | for _, tc := range []struct { | |||
| 625 | + | name string | |||
| 626 | + | result probeResult | |||
| 627 | + | }{ | |||
| 628 | + | {"retired registration", probeResult{endpoint: regA.Endpoint, registration: regA, attempt: retired.token, at: clock.Now(), snapshot: staleObservation()}}, | |||
| 629 | + | {"current registration with a retired token", probeResult{endpoint: regA.Endpoint, registration: regB, attempt: retired.token, at: clock.Now(), snapshot: staleObservation()}}, | |||
| 630 | + | {"retired registration with the current token", probeResult{endpoint: regA.Endpoint, registration: regA, attempt: retired.token + 1, at: clock.Now(), snapshot: staleObservation()}}, | |||
| 631 | + | } { | |||
| 632 | + | t.Run(tc.name, func(t *testing.T) { | |||
| 633 | + | registry.apply(tc.result) | |||
| 634 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 635 | + | host, ok := registry.Snapshot().Find(regA.Endpoint) | |||
| 636 | + | require.True(t, ok) | |||
| 637 | + | require.Equal(t, regB, host.Registration) | |||
| 638 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |||
| 639 | + | require.Empty(t, host.Identity) | |||
| 640 | + | require.True(t, host.LastSuccess.IsZero()) | |||
| 641 | + | require.Zero(t, attemptFor(registry, regA.Endpoint), "a stale completion never admits an attempt") | |||
| 642 | + | }) | |||
| 643 | + | } | |||
| 644 | + | } | |||
| 645 | + | ||||
| 646 | + | // TestRegistryMutableStaleCompletionDoesNotClearLiveAttempt proves the fence on | |||
| 647 | + | // the live side: a completion for the retired generation is dropped while a | |||
| 648 | + | // current attempt is in flight, so it neither clears the in-flight slot nor | |||
| 649 | + | // publishes under the retired registration. | |||
| 650 | + | func TestRegistryMutableStaleCompletionDoesNotClearLiveAttempt(t *testing.T) { | |||
| 651 | + | clock := newManualClock(time.Unix(100, 0)) | |||
| 652 | + | registry, _ := newMutableRegistry(t, newMembershipStore(), clock, (&incarnationSequencer{}).generate) | |||
| 653 | + | regA, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |||
| 654 | + | require.NoError(t, err) | |||
| 655 | + | retired := admitMembershipAttempt(t, registry, regA.Endpoint) | |||
| 656 | + | regB, err := registry.UpdateHostPolicy(context.Background(), regA, policyWithTrust("other-trust")) | |||
| 657 | + | require.NoError(t, err) | |||
| 658 | + | ||||
| 659 | + | live := admitMembershipAttempt(t, registry, regB.Endpoint) | |||
| 660 | + | before := registry.Snapshot() | |||
| 661 | + | for _, result := range []probeResult{ | |||
| 662 | + | {endpoint: regB.Endpoint, registration: regA, attempt: retired.token, at: clock.Now(), snapshot: staleObservation()}, | |||
| 663 | + | {endpoint: regB.Endpoint, registration: regB, attempt: retired.token, at: clock.Now(), snapshot: staleObservation()}, | |||
| 664 | + | {endpoint: regB.Endpoint, registration: regA, attempt: live.token, at: clock.Now(), snapshot: staleObservation()}, | |||
| 665 | + | } { | |||
| 666 | + | registry.apply(result) | |||
| 667 | + | } | |||
| 668 | + | require.Equal(t, live.token, attemptFor(registry, regB.Endpoint), "a stale completion must not clear the live attempt") | |||
| 669 | + | require.NoError(t, live.ctx.Err(), "a stale completion must not retire the live attempt") | |||
| 670 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 671 | + | ||||
| 672 | + | // The live attempt still completes normally and publishes its own result. | |||
| 673 | + | registry.apply(probeResult{endpoint: regB.Endpoint, registration: regB, attempt: live.token, at: clock.Now(), snapshot: currentObservation()}) | |||
| 674 | + | require.ErrorIs(t, live.ctx.Err(), context.Canceled) | |||
| 675 | + | host, ok := registry.Snapshot().Find(regB.Endpoint) | |||
| 676 | + | require.True(t, ok) | |||
| 677 | + | require.Equal(t, regB, host.Registration) | |||
| 678 | + | require.Equal(t, ports.BrokerDaemonIdentity("current-daemon"), host.Identity) | |||
| 679 | + | require.Equal(t, domain.RemoteAvailabilityReachable, host.Availability) | |||
| 680 | + | } | |||
| 681 | + | ||||
| 682 | + | // TestRegistryMutableStaleProbeCannotReviveRemovedRegistration proves the | |||
| 683 | + | // remove/re-add fence: a completion for the retired incarnation is dropped after | |||
| 684 | + | // the endpoint was re-added under a fresh incarnation, and the re-add supersedes | |||
| 685 | + | // rather than revives the retirement. | |||
| 686 | + | func TestRegistryMutableStaleProbeCannotReviveRemovedRegistration(t *testing.T) { | |||
| 687 | + | clock := newManualClock(time.Unix(100, 0)) | |||
| 688 | + | seq := &incarnationSequencer{} | |||
| 689 | + | registry, _ := newMutableRegistry(t, newMembershipStore(), clock, seq.generate) | |||
| 690 | + | regA, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |||
| 691 | + | require.NoError(t, err) | |||
| 692 | + | retired := admitMembershipAttempt(t, registry, regA.Endpoint) | |||
| 693 | + | ||||
| 694 | + | removed, err := registry.RemoveHost(context.Background(), regA) | |||
| 695 | + | require.NoError(t, err) | |||
| 696 | + | require.True(t, removed) | |||
| 697 | + | require.ErrorIs(t, retired.ctx.Err(), context.Canceled) | |||
| 698 | + | afterRemove := registry.Snapshot() | |||
| 699 | + | require.Len(t, afterRemove.Removed, 1) | |||
| 700 | + | require.Equal(t, regA, afterRemove.Removed[0].Registration) | |||
| 701 | + | ||||
| 702 | + | regB, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |||
| 703 | + | require.NoError(t, err) | |||
| 704 | + | require.NotEqual(t, regA.Incarnation, regB.Incarnation) | |||
| 705 | + | before := registry.Snapshot() | |||
| 706 | + | require.Len(t, before.Daemons, 1) | |||
| 707 | + | require.Empty(t, before.Removed) | |||
| 708 | + | ||||
| 709 | + | registry.apply(probeResult{endpoint: regA.Endpoint, registration: regA, attempt: retired.token, at: clock.Now(), snapshot: staleObservation()}) | |||
| 710 | + | require.Equal(t, before, registry.Snapshot()) | |||
| 711 | + | host, ok := registry.Snapshot().Find(regA.Endpoint) | |||
| 712 | + | require.True(t, ok) | |||
| 713 | + | require.Equal(t, regB, host.Registration) | |||
| 714 | + | require.Equal(t, domain.RemoteAvailabilityUnknown, host.Availability) | |||
| 715 | + | require.Empty(t, host.Identity) | |||
| 716 | + | require.Zero(t, attemptFor(registry, regA.Endpoint)) | |||
| 717 | + | } | |||
| 718 | + | ||||
| 719 | + | // TestRegistryMutablePolicyChangeRetiresInflightProbe drives the same fence | |||
| 720 | + | // through the production run loop: a policy change retires the in-flight probe | |||
| 721 | + | // promptly, a fresh attempt observes the new registration, and the retired | |||
| 722 | + | // answer can never surface as the new generation's observation. | |||
| 723 | + | func TestRegistryMutablePolicyChangeRetiresInflightProbe(t *testing.T) { | |||
| 724 | + | registry, probe := newMutableRegistry(t, newMembershipStore(), newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |||
| 725 | + | regA, err := registry.AddHost(context.Background(), "alpha.test", poolPolicy()) | |||
| 726 | + | require.NoError(t, err) | |||
| 727 | + | startRegistry(t, registry) | |||
| 728 | + | ||||
| 729 | + | first := receiveCall(t, probe) | |||
| 730 | + | require.Equal(t, regA, first.registration) | |||
| 731 | + | ||||
| 732 | + | regB, err := registry.UpdateHostPolicy(context.Background(), regA, policyWithTrust("other-trust")) | |||
| 733 | + | require.NoError(t, err) | |||
| 734 | + | require.Equal(t, regA.Generation+1, regB.Generation) | |||
| 735 | + | select { | |||
| 736 | + | case <-probe.canceled: | |||
| 737 | + | case <-time.After(time.Second): | |||
| 738 | + | t.Fatal("a policy change must retire the in-flight probe") | |||
| 739 | + | } | |||
| 740 | + | ||||
| 741 | + | second := receiveCall(t, probe) | |||
| 742 | + | require.Equal(t, regB, second.registration, "the replacement generation is observed") | |||
| 743 | + | ||||
| 744 | + | // The retired answer arrives after its attempt was retired; only the new | |||
| 745 | + | // generation's answer may become the published observation. | |||
| 746 | + | first.result <- probeAnswer{snapshot: staleObservation()} | |||
| 747 | + | second.result <- probeAnswer{snapshot: currentObservation()} | |||
| 748 | + | host := waitHost(t, registry, regB.Endpoint, func(host ports.BrokerDaemonObservation) bool { | |||
| 749 | + | return host.Identity == ports.BrokerDaemonIdentity("current-daemon") | |||
| 750 | + | }) | |||
| 751 | + | require.Equal(t, regB, host.Registration) | |||
| 752 | + | require.Equal(t, domain.RemoteAvailabilityReachable, host.Availability) | |||
| 753 | + | requireHostStableWithin(t, registry, regB.Endpoint, host, 50*time.Millisecond) | |||
| 754 | + | } | |||
| 755 | + | ||||
| 756 | + | // membershipFixture seeds the state a failed mutation must leave untouched: two | |||
| 757 | + | // pinned hosts, one live probe attempt, and one retirement tombstone. | |||
| 758 | + | type membershipFixture struct { | |||
| 759 | + | registry *Registry | |||
| 760 | + | store *membershipStore | |||
| 761 | + | alpha domain.RemoteRegistration | |||
| 762 | + | attempt admittedAttempt | |||
| 763 | + | authority ports.BrokerHosts | |||
| 764 | + | before ports.BrokerSnapshot | |||
| 765 | + | fences map[string]ports.BrokerHostTombstone | |||
| 766 | + | casBefore int | |||
| 767 | + | } | |||
| 768 | + | ||||
| 769 | + | func newMembershipFixture(t *testing.T) *membershipFixture { | |||
| 770 | + | t.Helper() | |||
| 771 | + | alpha := registration(t, "alpha.test", 1) | |||
| 772 | + | fixture := &membershipFixture{alpha: alpha} | |||
| 773 | + | fixture.store = newMembershipStore(pinnedRecord(alpha, poolPolicy()), pinnedRecord(registration(t, "beta.test", 2), poolPolicy())) | |||
| 774 | + | registry, _ := newMutableRegistry(t, fixture.store, newManualClock(time.Unix(100, 0)), (&incarnationSequencer{}).generate) | |||
| 775 | + | fixture.registry = registry | |||
| 776 | + | fixture.attempt = admitMembershipAttempt(t, registry, alpha.Endpoint) | |||
| 777 | + | ||||
| 778 | + | // Retire beta so the fixture carries a live tombstone alongside the live | |||
| 779 | + | // attempt at the moment the failure is injected. | |||
| 780 | + | removed, err := registry.RemoveHost(context.Background(), registration(t, "beta.test", 2)) | |||
| 781 | + | require.NoError(t, err) | |||
| 782 | + | require.True(t, removed) | |||
| 783 | + | ||||
| 784 | + | fixture.authority = registry.authority | |||
| 785 | + | fixture.before = registry.Snapshot() | |||
| 786 | + | fixture.fences = membershipFences(registry) | |||
| 787 | + | fixture.casBefore = fixture.store.casCalls() | |||
| 788 | + | require.Len(t, fixture.fences, 1) | |||
| 789 | + | return fixture | |||
| 790 | + | } | |||
| 791 | + | ||||
| 792 | + | // TestRegistryMutableStoreFailureLeavesStateUnchanged pins the commit ordering: | |||
| 793 | + | // a definite or indeterminate store failure returns before any projection, | |||
| 794 | + | // authority, attempt, or tombstone changes, and the store was genuinely reached. | |||
| 795 | + | func TestRegistryMutableStoreFailureLeavesStateUnchanged(t *testing.T) { | |||
| 796 | + | definite := []struct { | |||
| 797 | + | name string | |||
| 798 | + | err error | |||
| 799 | + | }{ | |||
| 800 | + | {"conflict", ports.ErrBrokerHostConflict}, | |||
| 801 | + | {"plain failure", context.DeadlineExceeded}, | |||
| 802 | + | } | |||
| 803 | + | indeterminate := errors.New("power loss") | |||
| 804 | + | ||||
| 805 | + | for _, failure := range definite { | |||
| 806 | + | t.Run(failure.name, func(t *testing.T) { | |||
| 807 | + | runMembershipStoreFailures(t, failure.err) | |||
| 808 | + | }) | |||
| 809 | + | } | |||
| 810 | + | t.Run("indeterminate outcome", func(t *testing.T) { | |||
| 811 | + | runMembershipStoreFailures(t, ports.BrokerStoreOutcomeUnknownError{Err: indeterminate}) | |||
| 812 | + | }) | |||
| 813 | + | } | |||
| 814 | + | ||||
| 815 | + | func runMembershipStoreFailures(t *testing.T, storeErr error) { | |||
| 816 | + | t.Helper() | |||
| 817 | + | ops := []struct { | |||
| 818 | + | name string | |||
| 819 | + | run func(fixture *membershipFixture) (bool, error) | |||
| 820 | + | }{ | |||
| 821 | + | {"add new", func(f *membershipFixture) (bool, error) { | |||
| 822 | + | _, err := f.registry.AddHost(context.Background(), "gamma.test", poolPolicy()) | |||
| 823 | + | return false, err | |||
| 824 | + | }}, | |||
| 825 | + | {"add duplicate", func(f *membershipFixture) (bool, error) { | |||
| 826 | + | _, err := f.registry.AddHost(context.Background(), f.alpha.Endpoint, poolPolicy()) | |||
| 827 | + | return false, err | |||
| 828 | + | }}, | |||
| 829 | + | {"remove exact", func(f *membershipFixture) (bool, error) { | |||
| 830 | + | return f.registry.RemoveHost(context.Background(), f.alpha) | |||
| 831 | + | }}, | |||
| 832 | + | {"remove absent", func(f *membershipFixture) (bool, error) { | |||
| 833 | + | return f.registry.RemoveHost(context.Background(), registration(t, "delta.test", 4)) | |||
| 834 | + | }}, | |||
| 835 | + | {"update policy", func(f *membershipFixture) (bool, error) { | |||
| 836 | + | _, err := f.registry.UpdateHostPolicy(context.Background(), f.alpha, policyWithTrust("other-trust")) | |||
| 837 | + | return false, err | |||
| 838 | + | }}, | |||
| 839 | + | {"update equal policy", func(f *membershipFixture) (bool, error) { | |||
| 840 | + | _, err := f.registry.UpdateHostPolicy(context.Background(), f.alpha, poolPolicy()) | |||
| 841 | + | return false, err | |||
| 842 | + | }}, | |||
| 843 | + | } | |||
| 844 | + | for _, op := range ops { | |||
| 845 | + | t.Run(op.name, func(t *testing.T) { | |||
| 846 | + | fixture := newMembershipFixture(t) | |||
| 847 | + | fixture.store.failCAS(storeErr) | |||
| 848 | + | ||||
| 849 | + | removed, err := op.run(fixture) | |||
| 850 | + | require.Error(t, err) | |||
| 851 | + | // A commit that never happened is never reported as a removal, even | |||
| 852 | + | // for the one operation whose success is a boolean: a caller that | |||
| 853 | + | // trusted removed=true from a failed commit would drop live authority. | |||
| 854 | + | require.False(t, removed, "a failed commit must never report a removal") | |||
| 855 | + | var unknown ports.BrokerStoreOutcomeUnknownError | |||
| 856 | + | if errors.As(storeErr, &unknown) { | |||
| 857 | + | require.ErrorAs(t, err, &unknown, "an indeterminate outcome stays classifiable through the registry") | |||
| 858 | + | } else { | |||
| 859 | + | require.ErrorIs(t, err, storeErr) | |||
| 860 | + | require.False(t, errors.As(err, &unknown), "a definite failure is not an unknown outcome") | |||
| 861 | + | } | |||
| 862 | + | ||||
| 863 | + | require.Equal(t, fixture.authority, fixture.registry.authority, "authority must not move") | |||
| 864 | + | require.Equal(t, fixture.before, fixture.registry.Snapshot(), "projection must not move") | |||
| 865 | + | require.Equal(t, fixture.attempt.token, attemptFor(fixture.registry, fixture.alpha.Endpoint), "the in-flight attempt must survive") | |||
| 866 | + | require.NoError(t, fixture.attempt.ctx.Err(), "a failed store call must not retire the attempt") | |||
| 867 | + | require.Equal(t, fixture.fences, membershipFences(fixture.registry), "tombstones must not move") | |||
| 868 | + | require.Equal(t, fixture.casBefore+1, fixture.store.casCalls(), "the mutation reached the store CAS") | |||
| 869 | + | }) | |||
| 870 | + | } | |||
| 871 | + | } | |||