feat(daemonmux): add multiplexed daemon carriage
- commit
- 7bfefe29fadceee2ba53e82fc83061982b1e31c1
- author
- brice · · 2026-09-16 19:12 UTC
- parent
- 0471f4f
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
78 files · +23801 -90
M docs/architecture.md +1 -1
| @@ -110,7 +110,7 @@ Use cases exchange `protocol.ClientMessage` and `protocol.ServerMessage` values. | |||
| 110 | 110 | `go tool buf generate`, and handle the new variant in `sessionwire`. | |
| 111 | 111 | Never edit generated `*.pb.go`; never add manual IDs or dispatch tables. | |
| 112 | 112 | - Implement I/O, queues, workers, environment integration, or technology selection in an adapter or `internal/app`. | |
| 113 | - | - Bump `internal/protocol.Version` for negotiated wire layout changes (currently `53`). The preamble epoch (`wire.ProtocolEpoch`, QUIC ALPN `vev/1`) bumps only for an intentional clean break. | |
| 113 | + | - Bump `internal/protocol.Version` for negotiated wire layout changes (currently `54`). The preamble epoch (`wire.ProtocolEpoch`, QUIC ALPN `vev/1`) bumps only for an intentional clean break. | |
| 114 | 114 | ||
| 115 | 115 | ## Broker wire core (Plan 001 P3.1, not activated) | |
| 116 | 116 | ||
M docs/protocol.md +9 -1
| @@ -56,7 +56,15 @@ negotiated ceiling. | |||
| 56 | 56 | - Daemon (`internal/usecase/daemon/handshake.go`, `daemon.go:handleConn`): | |
| 57 | 57 | `newHandshakeContext` + `watchHandshakeTransport` close the exact | |
| 58 | 58 | admitted connection on timeout; `failHandshakeAttachment` retires only | |
| 59 | - | that connection and purges a route-created session only when empty. | |
| 59 | + | that connection and purges a route-created session only when empty. An | |
| 60 | + | accepted connection that implements the optional | |
| 61 | + | `ports.HandshakeDeadlineProvider` seam supplies the absolute deadline it | |
| 62 | + | was admitted with, which the daemon adopts verbatim: queue delay is | |
| 63 | + | consumed instead of restarting the budget, and an already-elapsed | |
| 64 | + | deadline fails promptly. Connections without the seam keep a fresh | |
| 65 | + | budget from accept. Completion stays where the daemon commits its | |
| 66 | + | initial publication; the sessionwire preamble's own completion is not | |
| 67 | + | the handshake deadline. | |
| 60 | 68 | ||
| 61 | 69 | First frame routing (`daemon.go:handleConn`): `Hello` → full attach; | |
| 62 | 70 | `List`, `CommandRequest`, `RemotePreviewRequest`, | |
A internal/adapters/daemonmux/aggregate.go +328 -0
| @@ -0,0 +1,328 @@ | |||
| 1 | + | // Aggregate listener over multiple physical listeners (P3.2). | |
| 2 | + | // | |
| 3 | + | // AggregateListener is the daemon's global ports.ServerListener over several | |
| 4 | + | // physical daemonmux connections. The daemon's shared accept loop must not stop | |
| 5 | + | // when one physical carriage is lost: a daemonmux.Listener Accept fails with | |
| 6 | + | // ErrListenerLost exactly when its one physical connection reached its terminal | |
| 7 | + | // outcome, and a raw accept loop treats that as the end of service. The | |
| 8 | + | // aggregate composes independent child listeners into one accept stream, so the | |
| 9 | + | // loss or closure of one child is isolated - the aggregate deregisters it and | |
| 10 | + | // keeps accepting from every other child and from children registered later - | |
| 11 | + | // and never becomes the aggregate's own terminal error. | |
| 12 | + | // | |
| 13 | + | // Each registered child is owned: the aggregate runs exactly one forwarding | |
| 14 | + | // goroutine per child (bounded by the number of registered children, never by | |
| 15 | + | // the number of accepted streams) that repeatedly takes one typed | |
| 16 | + | // ports.ServerConnection from the child and hands it to the aggregate's bounded | |
| 17 | + | // FIFO. The FIFO is a mutex-guarded slice signalled by a sync.Cond | |
| 18 | + | // (MaxAggregateQueue, 128, by default): a slow or stalled consumer applies | |
| 19 | + | // backpressure to the child forwarders instead of growing memory, so the | |
| 20 | + | // aggregate never holds more than its queue bound plus one in-flight connection | |
| 21 | + | // per child forwarder, and each child's own bounded accept queue still refuses | |
| 22 | + | // a stream whose slot it cannot hold. | |
| 23 | + | // | |
| 24 | + | // A child whose Accept fails for any terminal reason - ErrListenerLost, | |
| 25 | + | // ErrListenerClosed, or a transport-specific error - is deregistered and its | |
| 26 | + | // goroutine joined; the failure is swallowed rather than returned by the | |
| 27 | + | // aggregate's Accept, because one physical carriage is not the daemon's whole | |
| 28 | + | // listener. Register and Remove are safe for concurrent use, as are Accept and | |
| 29 | + | // Close. Close unblocks every blocked Accept with ErrAggregateClosed, closes | |
| 30 | + | // every owned child (which unblocks and joins that child's own accept loop), | |
| 31 | + | // joins every forwarding goroutine, and closes every connection that was taken | |
| 32 | + | // from a child but never handed to the daemon. Removing one child closes that | |
| 33 | + | // child and joins its goroutine without touching a sibling, a queued connection | |
| 34 | + | // already taken from it, or a connection already delivered to the daemon. | |
| 35 | + | // | |
| 36 | + | // The aggregate owns no framing, session conversion, or carrier: it consumes | |
| 37 | + | // only the typed ports.ServerListener and ports.ServerConnection contracts, so | |
| 38 | + | // it composes with the daemonmux Listener, the sessionwire listener, or any | |
| 39 | + | // later physical listener implementation. | |
| 40 | + | package daemonmux | |
| 41 | + | ||
| 42 | + | import ( | |
| 43 | + | "errors" | |
| 44 | + | "sync" | |
| 45 | + | ||
| 46 | + | "github.com/bnema/vev/internal/ports" | |
| 47 | + | ) | |
| 48 | + | ||
| 49 | + | // Aggregate-listener sentinels. They are typed so a caller can classify a | |
| 50 | + | // configuration, lifecycle, or registration outcome without matching on | |
| 51 | + | // message text. | |
| 52 | + | var ( | |
| 53 | + | // ErrAggregateConfig reports an aggregate listener operation on a nil | |
| 54 | + | // receiver, or a registration of a nil child. | |
| 55 | + | ErrAggregateConfig = errors.New("daemonmux: invalid aggregate listener configuration") | |
| 56 | + | // ErrAggregateClosed reports a registration on, or an Accept of, an | |
| 57 | + | // aggregate listener that was closed. | |
| 58 | + | ErrAggregateClosed = errors.New("daemonmux: aggregate listener is closed") | |
| 59 | + | // ErrAggregateChildTaken reports a registration of a child that is already | |
| 60 | + | // registered. | |
| 61 | + | ErrAggregateChildTaken = errors.New("daemonmux: aggregate child is already registered") | |
| 62 | + | // ErrAggregateUnknownChild reports a removal of a child that is not | |
| 63 | + | // registered, whether it was never registered or already deregistered | |
| 64 | + | // because its physical carriage reached its terminal outcome. | |
| 65 | + | ErrAggregateUnknownChild = errors.New("daemonmux: aggregate child is not registered") | |
| 66 | + | ) | |
| 67 | + | ||
| 68 | + | // MaxAggregateQueue bounds the accepted-but-undelivered typed connections the | |
| 69 | + | // aggregate listener holds across every child. It is the per-physical-listener | |
| 70 | + | // accept bound, so the aggregate's own memory is never larger than one child's. | |
| 71 | + | // A child forwarder may additionally hold one connection while the queue is | |
| 72 | + | // full, so the aggregate's total retained connections are bounded by this value | |
| 73 | + | // plus the number of registered children. | |
| 74 | + | const MaxAggregateQueue = MaxAcceptQueue | |
| 75 | + | ||
| 76 | + | // AggregateListener is the daemon's global ports.ServerListener over several | |
| 77 | + | // independent physical listeners. It is safe for concurrent use. | |
| 78 | + | type AggregateListener struct { | |
| 79 | + | limit int | |
| 80 | + | ||
| 81 | + | // mu guards the child registry, the accepted-but-undelivered FIFO, and the | |
| 82 | + | // closed flag. cond broadcasts every FIFO and lifecycle change so a blocked | |
| 83 | + | // Accept and a child forwarder waiting for space never miss a wakeup. | |
| 84 | + | mu sync.Mutex | |
| 85 | + | cond *sync.Cond | |
| 86 | + | children map[ports.ServerListener]*aggregateChild | |
| 87 | + | queue []ports.ServerConnection | |
| 88 | + | closed bool | |
| 89 | + | ||
| 90 | + | closeOnce sync.Once | |
| 91 | + | } | |
| 92 | + | ||
| 93 | + | // aggregateChild is one registered physical listener and the lifecycle of its | |
| 94 | + | // single forwarding goroutine. quit is the per-child stop signal Remove and the | |
| 95 | + | // forwarder's own shutdown observe; done closes when the forwarder returns. | |
| 96 | + | type aggregateChild struct { | |
| 97 | + | listener ports.ServerListener | |
| 98 | + | quit chan struct{} | |
| 99 | + | done chan struct{} | |
| 100 | + | } | |
| 101 | + | ||
| 102 | + | var _ ports.ServerListener = (*AggregateListener)(nil) | |
| 103 | + | ||
| 104 | + | // NewAggregateListener returns an empty aggregate listener under the package | |
| 105 | + | // policy: the aggregate queue is bounded by MaxAggregateQueue. Children are | |
| 106 | + | // registered and removed explicitly; the aggregate owns every registered child | |
| 107 | + | // and closes it on Close. | |
| 108 | + | func NewAggregateListener() *AggregateListener { | |
| 109 | + | return newAggregateListener(MaxAggregateQueue) | |
| 110 | + | } | |
| 111 | + | ||
| 112 | + | // newAggregateListener builds an aggregate listener from an explicit queue | |
| 113 | + | // bound, so a test can tighten it and prove the bound and its backpressure | |
| 114 | + | // without changing live behavior. A non-positive bound falls back to the | |
| 115 | + | // package policy. | |
| 116 | + | func newAggregateListener(limit int) *AggregateListener { | |
| 117 | + | if limit <= 0 { | |
| 118 | + | limit = MaxAggregateQueue | |
| 119 | + | } | |
| 120 | + | listener := &AggregateListener{ | |
| 121 | + | limit: limit, | |
| 122 | + | children: make(map[ports.ServerListener]*aggregateChild), | |
| 123 | + | } | |
| 124 | + | listener.cond = sync.NewCond(&listener.mu) | |
| 125 | + | return listener | |
| 126 | + | } | |
| 127 | + | ||
| 128 | + | // Addr names the aggregate's carriage for ports.ServerListener. The aggregate | |
| 129 | + | // multiplexes several physical carriages into one accept stream, so it has no | |
| 130 | + | // single per-child address. | |
| 131 | + | func (a *AggregateListener) Addr() string { return "daemonmux" } | |
| 132 | + | ||
| 133 | + | // Register takes ownership of one physical child listener and starts its single | |
| 134 | + | // forwarding goroutine. Registering a nil child is refused with | |
| 135 | + | // ErrAggregateConfig, a child that is already registered with | |
| 136 | + | // ErrAggregateChildTaken, and any child after Close with ErrAggregateClosed. A | |
| 137 | + | // child whose Accept is already terminal is deregistered by its forwarder as | |
| 138 | + | // soon as it runs, without failing the aggregate. The child is identified by | |
| 139 | + | // its interface value, so a child is a pointer-like, comparable listener as | |
| 140 | + | // every ports.ServerListener implementation in this repository is. | |
| 141 | + | func (a *AggregateListener) Register(child ports.ServerListener) error { | |
| 142 | + | if a == nil || child == nil { | |
| 143 | + | return ErrAggregateConfig | |
| 144 | + | } | |
| 145 | + | a.mu.Lock() | |
| 146 | + | if a.closed { | |
| 147 | + | a.mu.Unlock() | |
| 148 | + | return ErrAggregateClosed | |
| 149 | + | } | |
| 150 | + | if _, exists := a.children[child]; exists { | |
| 151 | + | a.mu.Unlock() | |
| 152 | + | return ErrAggregateChildTaken | |
| 153 | + | } | |
| 154 | + | entry := &aggregateChild{ | |
| 155 | + | listener: child, | |
| 156 | + | quit: make(chan struct{}), | |
| 157 | + | done: make(chan struct{}), | |
| 158 | + | } | |
| 159 | + | a.children[child] = entry | |
| 160 | + | a.mu.Unlock() | |
| 161 | + | go a.forward(entry) | |
| 162 | + | return nil | |
| 163 | + | } | |
| 164 | + | ||
| 165 | + | // Remove closes one registered child and joins its forwarding goroutine without | |
| 166 | + | // touching a sibling, the connections already queued from that child, or a | |
| 167 | + | // connection already delivered to the daemon. It is safe to call concurrently | |
| 168 | + | // with Accept, Close, and another Remove; a child that was never registered or | |
| 169 | + | // already deregistered because its carriage was lost reports | |
| 170 | + | // ErrAggregateUnknownChild. A removal that races Close may either win (and | |
| 171 | + | // close the child) or lose (and report the child already gone); either way the | |
| 172 | + | // child is closed exactly once. | |
| 173 | + | func (a *AggregateListener) Remove(child ports.ServerListener) error { | |
| 174 | + | if a == nil || child == nil { | |
| 175 | + | return ErrAggregateConfig | |
| 176 | + | } | |
| 177 | + | a.mu.Lock() | |
| 178 | + | entry, ok := a.children[child] | |
| 179 | + | if !ok { | |
| 180 | + | a.mu.Unlock() | |
| 181 | + | return ErrAggregateUnknownChild | |
| 182 | + | } | |
| 183 | + | delete(a.children, child) | |
| 184 | + | close(entry.quit) | |
| 185 | + | // Wake a forwarder that is waiting for queue space, so it observes quit and | |
| 186 | + | // releases the connection it was holding instead of waiting forever. | |
| 187 | + | a.cond.Broadcast() | |
| 188 | + | a.mu.Unlock() | |
| 189 | + | ||
| 190 | + | _ = entry.listener.Close() | |
| 191 | + | <-entry.done | |
| 192 | + | return nil | |
| 193 | + | } | |
| 194 | + | ||
| 195 | + | // Accept returns the oldest typed connection accepted from any registered child, | |
| 196 | + | // blocking until one is available, and fails with ErrAggregateClosed after | |
| 197 | + | // Close. It never surfaces a child's Accept failure: a lost or closed physical | |
| 198 | + | // carriage is isolated by its forwarder and cannot stop the aggregate's accept | |
| 199 | + | // stream. A nil receiver is refused with ErrAggregateConfig. | |
| 200 | + | func (a *AggregateListener) Accept() (ports.ServerConnection, error) { | |
| 201 | + | if a == nil { | |
| 202 | + | return nil, ErrAggregateConfig | |
| 203 | + | } | |
| 204 | + | a.mu.Lock() | |
| 205 | + | defer a.mu.Unlock() | |
| 206 | + | for len(a.queue) == 0 { | |
| 207 | + | if a.closed { | |
| 208 | + | return nil, ErrAggregateClosed | |
| 209 | + | } | |
| 210 | + | a.cond.Wait() | |
| 211 | + | } | |
| 212 | + | connection := a.queue[0] | |
| 213 | + | a.queue = a.queue[1:] | |
| 214 | + | return connection, nil | |
| 215 | + | } | |
| 216 | + | ||
| 217 | + | // Close stops accepting: it unblocks every blocked Accept with | |
| 218 | + | // ErrAggregateClosed, clears every owned child (which unblocks and joins that | |
| 219 | + | // child's own accept loop), joins every forwarding goroutine, and closes every | |
| 220 | + | // connection that was taken from a child but never handed to the daemon. Close | |
| 221 | + | // is idempotent and concurrent-safe; closeOnce guarantees a single drain across | |
| 222 | + | // concurrent callers. It never touches a connection already handed to the | |
| 223 | + | // daemon, which owns it from then on. | |
| 224 | + | func (a *AggregateListener) Close() error { | |
| 225 | + | if a == nil { | |
| 226 | + | return nil | |
| 227 | + | } | |
| 228 | + | a.closeOnce.Do(func() { | |
| 229 | + | a.mu.Lock() | |
| 230 | + | a.closed = true | |
| 231 | + | queued := a.queue | |
| 232 | + | a.queue = nil | |
| 233 | + | children := make([]*aggregateChild, 0, len(a.children)) | |
| 234 | + | for _, entry := range a.children { | |
| 235 | + | children = append(children, entry) | |
| 236 | + | } | |
| 237 | + | a.children = make(map[ports.ServerListener]*aggregateChild) | |
| 238 | + | // Wake every blocked Accept (which now returns ErrAggregateClosed) and | |
| 239 | + | // every forwarder waiting for queue space (which now releases its | |
| 240 | + | // in-flight connection). | |
| 241 | + | a.cond.Broadcast() | |
| 242 | + | a.mu.Unlock() | |
| 243 | + | ||
| 244 | + | // Close every owned child first: this unblocks a forwarder parked in | |
| 245 | + | // the child's Accept so it can observe the closed aggregate and exit. | |
| 246 | + | for _, entry := range children { | |
| 247 | + | _ = entry.listener.Close() | |
| 248 | + | } | |
| 249 | + | for _, entry := range children { | |
| 250 | + | <-entry.done | |
| 251 | + | } | |
| 252 | + | for _, connection := range queued { | |
| 253 | + | if connection != nil { | |
| 254 | + | _ = connection.Close() | |
| 255 | + | } | |
| 256 | + | } | |
| 257 | + | }) | |
| 258 | + | return nil | |
| 259 | + | } | |
| 260 | + | ||
| 261 | + | // forward is one child's single forwarding goroutine. It takes one typed | |
| 262 | + | // connection at a time from the child and hands it to the aggregate's bounded | |
| 263 | + | // FIFO; when the child's Accept fails for any reason it deregisters the child | |
| 264 | + | // and returns, so the failure is isolated from the aggregate and every sibling. | |
| 265 | + | // When the aggregate is closed or the child is removed it releases the | |
| 266 | + | // connection it is holding and returns. | |
| 267 | + | func (a *AggregateListener) forward(entry *aggregateChild) { | |
| 268 | + | defer close(entry.done) | |
| 269 | + | for { | |
| 270 | + | connection, err := entry.listener.Accept() | |
| 271 | + | if err != nil { | |
| 272 | + | if connection != nil { | |
| 273 | + | _ = connection.Close() | |
| 274 | + | } | |
| 275 | + | a.deregister(entry) | |
| 276 | + | return | |
| 277 | + | } | |
| 278 | + | if !a.enqueue(entry, connection) { | |
| 279 | + | if connection != nil { | |
| 280 | + | _ = connection.Close() | |
| 281 | + | } | |
| 282 | + | a.deregister(entry) | |
| 283 | + | return | |
| 284 | + | } | |
| 285 | + | } | |
| 286 | + | } | |
| 287 | + | ||
| 288 | + | // enqueue appends one accepted connection to the bounded FIFO, waiting for | |
| 289 | + | // space while the queue is full. It reports false when the aggregate was closed | |
| 290 | + | // or the child was removed before the connection could be queued, so the caller | |
| 291 | + | // releases it. It never blocks a child's own accept loop: while the queue is | |
| 292 | + | // full the child is left to its own bounded queue and refuses a stream it | |
| 293 | + | // cannot hold. | |
| 294 | + | func (a *AggregateListener) enqueue(entry *aggregateChild, connection ports.ServerConnection) bool { | |
| 295 | + | a.mu.Lock() | |
| 296 | + | defer a.mu.Unlock() | |
| 297 | + | for len(a.queue) >= a.limit && !a.closed && !childStopped(entry) { | |
| 298 | + | a.cond.Wait() | |
| 299 | + | } | |
| 300 | + | if a.closed || childStopped(entry) { | |
| 301 | + | return false | |
| 302 | + | } | |
| 303 | + | a.queue = append(a.queue, connection) | |
| 304 | + | a.cond.Broadcast() | |
| 305 | + | return true | |
| 306 | + | } | |
| 307 | + | ||
| 308 | + | // deregister removes one forwarder's child from the registry if it is still | |
| 309 | + | // registered. It is a no-op for a child already removed by Remove or by Close, | |
| 310 | + | // so a natural terminal outcome and an explicit removal never double-close. | |
| 311 | + | func (a *AggregateListener) deregister(entry *aggregateChild) { | |
| 312 | + | a.mu.Lock() | |
| 313 | + | if a.children[entry.listener] == entry { | |
| 314 | + | delete(a.children, entry.listener) | |
| 315 | + | } | |
| 316 | + | a.mu.Unlock() | |
| 317 | + | } | |
| 318 | + | ||
| 319 | + | // childStopped reports whether one child's forwarder was asked to stop. The | |
| 320 | + | // caller holds a.mu. | |
| 321 | + | func childStopped(entry *aggregateChild) bool { | |
| 322 | + | select { | |
| 323 | + | case <-entry.quit: | |
| 324 | + | return true | |
| 325 | + | default: | |
| 326 | + | return false | |
| 327 | + | } | |
| 328 | + | } | |
A internal/adapters/daemonmux/aggregate_test.go +291 -0
| @@ -0,0 +1,291 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "sync" | |
| 5 | + | "testing" | |
| 6 | + | "time" | |
| 7 | + | ||
| 8 | + | "github.com/stretchr/testify/require" | |
| 9 | + | ||
| 10 | + | "github.com/bnema/vev/internal/ports" | |
| 11 | + | ) | |
| 12 | + | ||
| 13 | + | // aggregateSink is the test's daemon-style consumer: it takes typed connections | |
| 14 | + | // from the aggregate until Accept fails and serves each one with servePongs. | |
| 15 | + | type aggregateSink struct { | |
| 16 | + | mu sync.Mutex | |
| 17 | + | conns []ports.ServerConnection | |
| 18 | + | errs []error | |
| 19 | + | } | |
| 20 | + | ||
| 21 | + | func startAggregateSink(a *AggregateListener) *aggregateSink { | |
| 22 | + | sink := &aggregateSink{} | |
| 23 | + | go func() { | |
| 24 | + | for { | |
| 25 | + | connection, err := a.Accept() | |
| 26 | + | if err != nil { | |
| 27 | + | sink.mu.Lock() | |
| 28 | + | sink.errs = append(sink.errs, err) | |
| 29 | + | sink.mu.Unlock() | |
| 30 | + | return | |
| 31 | + | } | |
| 32 | + | sink.mu.Lock() | |
| 33 | + | sink.conns = append(sink.conns, connection) | |
| 34 | + | sink.mu.Unlock() | |
| 35 | + | go func() { _ = servePongs(connection) }() | |
| 36 | + | } | |
| 37 | + | }() | |
| 38 | + | return sink | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | func (s *aggregateSink) await(t *testing.T, count int) []ports.ServerConnection { | |
| 42 | + | t.Helper() | |
| 43 | + | require.Eventually(t, func() bool { | |
| 44 | + | s.mu.Lock() | |
| 45 | + | defer s.mu.Unlock() | |
| 46 | + | return len(s.conns) >= count | |
| 47 | + | }, listenerTestDeadline, time.Millisecond, "aggregate accepted %d of %d connections", count, count) | |
| 48 | + | s.mu.Lock() | |
| 49 | + | defer s.mu.Unlock() | |
| 50 | + | out := make([]ports.ServerConnection, len(s.conns)) | |
| 51 | + | copy(out, s.conns) | |
| 52 | + | return out | |
| 53 | + | } | |
| 54 | + | ||
| 55 | + | func (s *aggregateSink) errors() []error { | |
| 56 | + | s.mu.Lock() | |
| 57 | + | defer s.mu.Unlock() | |
| 58 | + | return append([]error(nil), s.errs...) | |
| 59 | + | } | |
| 60 | + | ||
| 61 | + | // aggregateChildCount reports the children the aggregate still forwards from. | |
| 62 | + | func aggregateChildCount(a *AggregateListener) int { | |
| 63 | + | a.mu.Lock() | |
| 64 | + | defer a.mu.Unlock() | |
| 65 | + | return len(a.children) | |
| 66 | + | } | |
| 67 | + | ||
| 68 | + | // aggregateQueued reports the accepted-but-undelivered connections the | |
| 69 | + | // aggregate holds. | |
| 70 | + | func aggregateQueued(a *AggregateListener) int { | |
| 71 | + | a.mu.Lock() | |
| 72 | + | defer a.mu.Unlock() | |
| 73 | + | return len(a.queue) | |
| 74 | + | } | |
| 75 | + | ||
| 76 | + | // TestNewAggregateListenerValidation covers the nil-receiver and registration | |
| 77 | + | // guards without any physical listener. | |
| 78 | + | func TestNewAggregateListenerValidation(t *testing.T) { | |
| 79 | + | var nilAggregate *AggregateListener | |
| 80 | + | _, err := nilAggregate.Accept() | |
| 81 | + | require.ErrorIs(t, err, ErrAggregateConfig) | |
| 82 | + | require.ErrorIs(t, nilAggregate.Register(nil), ErrAggregateConfig) | |
| 83 | + | require.ErrorIs(t, nilAggregate.Remove(nil), ErrAggregateConfig) | |
| 84 | + | require.NoError(t, nilAggregate.Close()) | |
| 85 | + | ||
| 86 | + | a := newAggregateListener(MaxAggregateQueue) | |
| 87 | + | require.Equal(t, "daemonmux", a.Addr()) | |
| 88 | + | require.Equal(t, MaxAggregateQueue, a.limit) | |
| 89 | + | require.ErrorIs(t, a.Register(nil), ErrAggregateConfig) | |
| 90 | + | ||
| 91 | + | first := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 92 | + | second := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 93 | + | require.NoError(t, a.Register(first.listener)) | |
| 94 | + | require.ErrorIs(t, a.Register(first.listener), ErrAggregateChildTaken) | |
| 95 | + | require.NoError(t, a.Close()) | |
| 96 | + | require.ErrorIs(t, a.Register(second.listener), ErrAggregateClosed) | |
| 97 | + | _, err = a.Accept() | |
| 98 | + | require.ErrorIs(t, err, ErrAggregateClosed) | |
| 99 | + | } | |
| 100 | + | ||
| 101 | + | // TestAggregateTwoPhysicalChildrenAdmitIndependently proves the aggregate | |
| 102 | + | // delivers both children's independently accepted typed connections through one | |
| 103 | + | // ports.ServerListener, and that one child's stream lifecycle never touches the | |
| 104 | + | // other child or its physical carriage. | |
| 105 | + | func TestAggregateTwoPhysicalChildrenAdmitIndependently(t *testing.T) { | |
| 106 | + | a := newAggregateListener(MaxAggregateQueue) | |
| 107 | + | t.Cleanup(func() { _ = a.Close() }) | |
| 108 | + | ||
| 109 | + | first := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 110 | + | second := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 111 | + | require.NoError(t, a.Register(first.listener)) | |
| 112 | + | require.NoError(t, a.Register(second.listener)) | |
| 113 | + | ||
| 114 | + | sink := startAggregateSink(a) | |
| 115 | + | firstBroker := first.mustOpen(1) | |
| 116 | + | secondBroker := second.mustOpen(1) | |
| 117 | + | accepted := sink.await(t, 2) | |
| 118 | + | ||
| 119 | + | // Each physical child contributed exactly one typed daemonmux connection. | |
| 120 | + | pumps := map[*Pump]bool{} | |
| 121 | + | for _, connection := range accepted { | |
| 122 | + | listenerConnection, ok := connection.(*ListenerConnection) | |
| 123 | + | require.True(t, ok, "the aggregate must forward the child's typed daemonmux connection") | |
| 124 | + | pumps[listenerConnection.pump] = true | |
| 125 | + | } | |
| 126 | + | require.True(t, pumps[first.daemonPump], "the first child never contributed a connection") | |
| 127 | + | require.True(t, pumps[second.daemonPump], "the second child never contributed a connection") | |
| 128 | + | require.Len(t, pumps, 2) | |
| 129 | + | ||
| 130 | + | listenerRoundTrip(t, firstBroker) | |
| 131 | + | listenerRoundTrip(t, secondBroker) | |
| 132 | + | ||
| 133 | + | // One stream ending leaves its sibling and both physical carriages live. | |
| 134 | + | require.NoError(t, firstBroker.Close()) | |
| 135 | + | require.True(t, channelClosed(firstBroker.Done())) | |
| 136 | + | listenerRoundTrip(t, secondBroker) | |
| 137 | + | require.False(t, channelClosed(first.daemonPump.Done())) | |
| 138 | + | require.False(t, channelClosed(second.daemonPump.Done())) | |
| 139 | + | require.Empty(t, sink.errors()) | |
| 140 | + | } | |
| 141 | + | ||
| 142 | + | // TestAggregateChildLossDoesNotStopAccept proves a physical loss of one child | |
| 143 | + | // is isolated: the aggregate deregisters it without a terminal error, keeps | |
| 144 | + | // accepting from the surviving child, and accepts from a child registered after | |
| 145 | + | // the loss. | |
| 146 | + | func TestAggregateChildLossDoesNotStopAccept(t *testing.T) { | |
| 147 | + | a := newAggregateListener(MaxAggregateQueue) | |
| 148 | + | t.Cleanup(func() { _ = a.Close() }) | |
| 149 | + | ||
| 150 | + | first := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 151 | + | second := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 152 | + | require.NoError(t, a.Register(first.listener)) | |
| 153 | + | require.NoError(t, a.Register(second.listener)) | |
| 154 | + | ||
| 155 | + | sink := startAggregateSink(a) | |
| 156 | + | firstBroker := first.mustOpen(1) | |
| 157 | + | secondBroker := second.mustOpen(1) | |
| 158 | + | sink.await(t, 2) | |
| 159 | + | listenerRoundTrip(t, firstBroker) | |
| 160 | + | listenerRoundTrip(t, secondBroker) | |
| 161 | + | ||
| 162 | + | // Failing the first physical carriage makes its child's Accept terminal. | |
| 163 | + | require.NoError(t, first.brokerPump.Close()) | |
| 164 | + | require.Eventually(t, func() bool { return channelClosed(first.daemonPump.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 165 | + | ||
| 166 | + | // The aggregate isolates the loss: the child is deregistered and the loss | |
| 167 | + | // is never surfaced to the shared accept loop. | |
| 168 | + | require.Eventually(t, func() bool { return aggregateChildCount(a) == 1 }, listenerTestDeadline, time.Millisecond, | |
| 169 | + | "the lost child was not deregistered") | |
| 170 | + | require.Empty(t, sink.errors(), "a child loss must not surface as an aggregate Accept error") | |
| 171 | + | ||
| 172 | + | // The surviving physical child still admits. | |
| 173 | + | survivorBroker := second.mustOpen(2) | |
| 174 | + | survivorDaemon := sink.await(t, 3)[2].(*ListenerConnection) | |
| 175 | + | require.Equal(t, second.daemonPump, survivorDaemon.pump) | |
| 176 | + | listenerRoundTrip(t, survivorBroker) | |
| 177 | + | ||
| 178 | + | // A child registered after the loss also admits. | |
| 179 | + | third := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 180 | + | require.NoError(t, a.Register(third.listener)) | |
| 181 | + | thirdBroker := third.mustOpen(1) | |
| 182 | + | thirdDaemon := sink.await(t, 4)[3].(*ListenerConnection) | |
| 183 | + | require.Equal(t, third.daemonPump, thirdDaemon.pump) | |
| 184 | + | listenerRoundTrip(t, thirdBroker) | |
| 185 | + | require.Empty(t, sink.errors()) | |
| 186 | + | } | |
| 187 | + | ||
| 188 | + | // TestAggregateCloseDrainsQueuedConnections proves Close unblocks Accept, | |
| 189 | + | // closes the owned child, joins the forwarder, and cleans every connection it | |
| 190 | + | // was holding: the queued ones and the single in-flight one blocked on the | |
| 191 | + | // bounded queue. | |
| 192 | + | func TestAggregateCloseDrainsQueuedConnections(t *testing.T) { | |
| 193 | + | const limit = 2 | |
| 194 | + | a := newAggregateListener(limit) | |
| 195 | + | child := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 196 | + | require.NoError(t, a.Register(child.listener)) | |
| 197 | + | ||
| 198 | + | // Three admitted connections, none accepted: the aggregate holds its | |
| 199 | + | // bounded two, and the forwarder holds the third while backpressured. | |
| 200 | + | first := child.mustOpen(1) | |
| 201 | + | second := child.mustOpen(2) | |
| 202 | + | third := child.mustOpen(3) | |
| 203 | + | require.Eventually(t, func() bool { return aggregateQueued(a) == limit }, listenerTestDeadline, time.Millisecond, | |
| 204 | + | "the aggregate queue never filled to its bound") | |
| 205 | + | ||
| 206 | + | blocked := make(chan error, 1) | |
| 207 | + | go func() { | |
| 208 | + | _, err := a.Accept() | |
| 209 | + | blocked <- err | |
| 210 | + | }() | |
| 211 | + | ||
| 212 | + | require.NoError(t, a.Close()) | |
| 213 | + | require.NoError(t, a.Close(), "Close is idempotent") | |
| 214 | + | ||
| 215 | + | select { | |
| 216 | + | case err := <-blocked: | |
| 217 | + | require.ErrorIs(t, err, ErrAggregateClosed) | |
| 218 | + | case <-time.After(listenerTestDeadline): | |
| 219 | + | t.Fatal("Close did not unblock Accept") | |
| 220 | + | } | |
| 221 | + | _, err := a.Accept() | |
| 222 | + | require.ErrorIs(t, err, ErrAggregateClosed) | |
| 223 | + | require.Zero(t, aggregateQueued(a), "Close must clear the aggregate queue") | |
| 224 | + | require.Zero(t, aggregateChildCount(a), "Close must drop every owned child") | |
| 225 | + | ||
| 226 | + | // Every connection the aggregate held is released, including the one its | |
| 227 | + | // forwarder was holding while backpressured; the child itself is closed. | |
| 228 | + | for i, broker := range []*LogicalConnection{first, second, third} { | |
| 229 | + | require.Eventually(t, func() bool { return channelClosed(broker.Done()) }, listenerTestDeadline, time.Millisecond, | |
| 230 | + | "queued connection %d was not cleaned by Close", i+1) | |
| 231 | + | } | |
| 232 | + | _, err = child.listener.Accept() | |
| 233 | + | require.ErrorIs(t, err, ErrListenerClosed) | |
| 234 | + | } | |
| 235 | + | ||
| 236 | + | // TestAggregateRemoveStopsChildOnly proves Remove closes and joins exactly the | |
| 237 | + | // removed child, leaves the aggregate accepting from a sibling, and reports an | |
| 238 | + | // unknown child afterwards. | |
| 239 | + | func TestAggregateRemoveStopsChildOnly(t *testing.T) { | |
| 240 | + | a := newAggregateListener(MaxAggregateQueue) | |
| 241 | + | t.Cleanup(func() { _ = a.Close() }) | |
| 242 | + | ||
| 243 | + | first := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 244 | + | second := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 245 | + | require.NoError(t, a.Register(first.listener)) | |
| 246 | + | require.NoError(t, a.Register(second.listener)) | |
| 247 | + | sink := startAggregateSink(a) | |
| 248 | + | ||
| 249 | + | require.NoError(t, a.Remove(first.listener)) | |
| 250 | + | require.Equal(t, 1, aggregateChildCount(a)) | |
| 251 | + | _, err := first.listener.Accept() | |
| 252 | + | require.ErrorIs(t, err, ErrListenerClosed) | |
| 253 | + | require.ErrorIs(t, a.Remove(first.listener), ErrAggregateUnknownChild) | |
| 254 | + | ||
| 255 | + | broker := second.mustOpen(1) | |
| 256 | + | sink.await(t, 1) | |
| 257 | + | listenerRoundTrip(t, broker) | |
| 258 | + | require.Empty(t, sink.errors()) | |
| 259 | + | } | |
| 260 | + | ||
| 261 | + | // TestAggregateRemoveUnblocksBackpressuredForwarder proves Remove does not | |
| 262 | + | // deadlock when the removed child's forwarder is waiting for queue space: it | |
| 263 | + | // releases the in-flight connection, while a connection already queued stays | |
| 264 | + | // deliverable to the daemon. | |
| 265 | + | func TestAggregateRemoveUnblocksBackpressuredForwarder(t *testing.T) { | |
| 266 | + | a := newAggregateListener(1) | |
| 267 | + | t.Cleanup(func() { _ = a.Close() }) | |
| 268 | + | child := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 269 | + | require.NoError(t, a.Register(child.listener)) | |
| 270 | + | ||
| 271 | + | queuedBroker := child.mustOpen(1) | |
| 272 | + | heldBroker := child.mustOpen(2) | |
| 273 | + | require.Eventually(t, func() bool { return aggregateQueued(a) == 1 }, listenerTestDeadline, time.Millisecond, | |
| 274 | + | "the aggregate queue never filled to its bound") | |
| 275 | + | ||
| 276 | + | require.NoError(t, a.Remove(child.listener)) | |
| 277 | + | require.Zero(t, aggregateChildCount(a)) | |
| 278 | + | ||
| 279 | + | // The connection the forwarder was holding is released; the connection | |
| 280 | + | // already queued from the removed child stays deliverable. | |
| 281 | + | require.Eventually(t, func() bool { return channelClosed(heldBroker.Done()) }, listenerTestDeadline, time.Millisecond, | |
| 282 | + | "the backpressured in-flight connection was not released") | |
| 283 | + | queued, err := a.Accept() | |
| 284 | + | require.NoError(t, err) | |
| 285 | + | queuedConnection, ok := queued.(*ListenerConnection) | |
| 286 | + | require.True(t, ok, "the aggregate must forward the child's typed daemonmux connection") | |
| 287 | + | require.False(t, channelClosed(queuedConnection.Done()), "a queued connection from a removed child must stay deliverable") | |
| 288 | + | require.NoError(t, queued.Close()) | |
| 289 | + | require.True(t, channelClosed(queuedConnection.Done())) | |
| 290 | + | require.Eventually(t, func() bool { return channelClosed(queuedBroker.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 291 | + | } | |
A internal/adapters/daemonmux/binding.go +91 -0
| @@ -0,0 +1,91 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | // Server binding (P3.2a). | |
| 4 | + | // | |
| 5 | + | // ServerBinding is the daemon's validated, immutable authority for one | |
| 6 | + | // accepted daemonmux physical connection: the stable authenticated daemon | |
| 7 | + | // identity used for pooling, this process incarnation used to detect a | |
| 8 | + | // restart behind that identity, and the exact connection policy the daemon | |
| 9 | + | // requires. It is constructed once, validates its own invariants, exposes | |
| 10 | + | // only read accessors, and is never mutated, so a later change to a caller's | |
| 11 | + | // local value cannot weaken what the server handshake already enforced. | |
| 12 | + | // | |
| 13 | + | // The server handshake compares a client's requested policy against this | |
| 14 | + | // binding rather than echoing the request back: an accepted response carries | |
| 15 | + | // the binding's identity, incarnation, and policy. Identity is stable across | |
| 16 | + | // restarts; the incarnation is a freshly sampled nonzero 16-byte value. | |
| 17 | + | // | |
| 18 | + | // Authentication premise: the binding is only as trustworthy as the carrier | |
| 19 | + | // that carried the handshake. The carriage must already be authenticated | |
| 20 | + | // (for example an established QUIC channel with an exact peer pin) before a | |
| 21 | + | // connection reaches this handshake, and the expected endpoint binding is | |
| 22 | + | // authoritative. This package never authenticates a carrier and never treats | |
| 23 | + | // an in-band identity claim as proof. | |
| 24 | + | ||
| 25 | + | import ( | |
| 26 | + | "errors" | |
| 27 | + | ||
| 28 | + | "github.com/bnema/vev/internal/ports" | |
| 29 | + | ) | |
| 30 | + | ||
| 31 | + | // ErrInvalidBinding reports a daemon-side physical binding that is not | |
| 32 | + | // authoritative: an invalid or empty authenticated daemon identity, a zero | |
| 33 | + | // daemon incarnation, or an invalid connection policy. NewServerBinding | |
| 34 | + | // refuses it once, at construction, so the handshake never re-validates an | |
| 35 | + | // unusable binding. | |
| 36 | + | var ErrInvalidBinding = errors.New("daemonmux: invalid server binding") | |
| 37 | + | ||
| 38 | + | // ServerBinding is the daemon's validated, immutable physical binding for one | |
| 39 | + | // accepted daemonmux connection. Its fields are unexported and reachable only | |
| 40 | + | // through the Identity, Incarnation, and Policy accessors, so a ServerBinding | |
| 41 | + | // value is immutable once constructed and safe to copy and share. | |
| 42 | + | type ServerBinding struct { | |
| 43 | + | identity ports.BrokerDaemonIdentity | |
| 44 | + | incarnation ports.BrokerDaemonIncarnation | |
| 45 | + | policy ports.BrokerPolicy | |
| 46 | + | } | |
| 47 | + | ||
| 48 | + | // NewServerBinding validates and returns the daemon's immutable physical | |
| 49 | + | // binding. It refuses an invalid authenticated identity, a zero incarnation, | |
| 50 | + | // or an invalid policy with ErrInvalidBinding; the returned binding either | |
| 51 | + | // validates completely or is never handed out. | |
| 52 | + | func NewServerBinding(identity ports.BrokerDaemonIdentity, incarnation ports.BrokerDaemonIncarnation, policy ports.BrokerPolicy) (ServerBinding, error) { | |
| 53 | + | binding := ServerBinding{identity: identity, incarnation: incarnation, policy: policy} | |
| 54 | + | if err := binding.Validate(); err != nil { | |
| 55 | + | return ServerBinding{}, err | |
| 56 | + | } | |
| 57 | + | return binding, nil | |
| 58 | + | } | |
| 59 | + | ||
| 60 | + | // Identity returns the stable authenticated daemon identity the binding | |
| 61 | + | // advertises and pools on. | |
| 62 | + | func (b ServerBinding) Identity() ports.BrokerDaemonIdentity { return b.identity } | |
| 63 | + | ||
| 64 | + | // Incarnation returns the nonzero process incarnation the binding advertises | |
| 65 | + | // so a broker can detect a restart behind the stable identity. | |
| 66 | + | func (b ServerBinding) Incarnation() ports.BrokerDaemonIncarnation { return b.incarnation } | |
| 67 | + | ||
| 68 | + | // Policy returns the exact connection policy the binding requires. | |
| 69 | + | func (b ServerBinding) Policy() ports.BrokerPolicy { return b.policy } | |
| 70 | + | ||
| 71 | + | // Validate reports whether the binding is authoritative: a valid authenticated | |
| 72 | + | // identity, a nonzero incarnation, and a valid policy. | |
| 73 | + | func (b ServerBinding) Validate() error { | |
| 74 | + | if err := b.identity.Validate(); err != nil { | |
| 75 | + | return ErrInvalidBinding | |
| 76 | + | } | |
| 77 | + | if err := b.incarnation.Validate(); err != nil { | |
| 78 | + | return ErrInvalidBinding | |
| 79 | + | } | |
| 80 | + | if err := b.policy.Validate(); err != nil { | |
| 81 | + | return ErrInvalidBinding | |
| 82 | + | } | |
| 83 | + | return nil | |
| 84 | + | } | |
| 85 | + | ||
| 86 | + | // Accepts reports whether the requested policy exactly equals the binding's | |
| 87 | + | // policy. Pooling requires exact equality across every field, so a request | |
| 88 | + | // that differs in any field is refused rather than merged or inherited. | |
| 89 | + | func (b ServerBinding) Accepts(policy ports.BrokerPolicy) bool { | |
| 90 | + | return b.policy.Compatible(policy) | |
| 91 | + | } | |
A internal/adapters/daemonmux/binding_test.go +92 -0
| @@ -0,0 +1,92 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "testing" | |
| 5 | + | ||
| 6 | + | "github.com/stretchr/testify/require" | |
| 7 | + | ||
| 8 | + | "github.com/bnema/vev/internal/ports" | |
| 9 | + | "github.com/bnema/vev/internal/protocol" | |
| 10 | + | ) | |
| 11 | + | ||
| 12 | + | // TestServerBindingValidatesAndExposesAuthority proves the immutable binding | |
| 13 | + | // rejects every invalid field once, at construction, and exposes exactly the | |
| 14 | + | // validated authority through its accessors. | |
| 15 | + | func TestServerBindingValidatesAndExposesAuthority(t *testing.T) { | |
| 16 | + | binding, err := NewServerBinding(testIdentity(), testIncarnation(), testPolicy()) | |
| 17 | + | require.NoError(t, err) | |
| 18 | + | require.NoError(t, binding.Validate()) | |
| 19 | + | require.Equal(t, testIdentity(), binding.Identity()) | |
| 20 | + | require.Equal(t, testIncarnation(), binding.Incarnation()) | |
| 21 | + | require.Equal(t, testPolicy(), binding.Policy()) | |
| 22 | + | } | |
| 23 | + | ||
| 24 | + | // TestServerBindingRefusesInvalidAuthority proves an empty identity, a zero | |
| 25 | + | // incarnation, and every invalid policy shape are refused with the binding's | |
| 26 | + | // single sentinel. | |
| 27 | + | func TestServerBindingRefusesInvalidAuthority(t *testing.T) { | |
| 28 | + | invalidPolicy := testPolicy() | |
| 29 | + | invalidPolicy.ProtocolVersion = 0 | |
| 30 | + | ||
| 31 | + | tests := []struct { | |
| 32 | + | name string | |
| 33 | + | identity ports.BrokerDaemonIdentity | |
| 34 | + | incarnation ports.BrokerDaemonIncarnation | |
| 35 | + | policy ports.BrokerPolicy | |
| 36 | + | }{ | |
| 37 | + | {name: "empty identity", identity: "", incarnation: testIncarnation(), policy: testPolicy()}, | |
| 38 | + | {name: "zero incarnation", identity: testIdentity(), incarnation: ports.BrokerDaemonIncarnation{}, policy: testPolicy()}, | |
| 39 | + | {name: "zero policy", identity: testIdentity(), incarnation: testIncarnation(), policy: ports.BrokerPolicy{}}, | |
| 40 | + | {name: "policy without protocol version", identity: testIdentity(), incarnation: testIncarnation(), policy: invalidPolicy}, | |
| 41 | + | } | |
| 42 | + | for _, test := range tests { | |
| 43 | + | t.Run(test.name, func(t *testing.T) { | |
| 44 | + | _, err := NewServerBinding(test.identity, test.incarnation, test.policy) | |
| 45 | + | require.ErrorIs(t, err, ErrInvalidBinding) | |
| 46 | + | }) | |
| 47 | + | } | |
| 48 | + | } | |
| 49 | + | ||
| 50 | + | // TestServerBindingAcceptsOnlyExactPolicy proves the binding accepts the exact | |
| 51 | + | // policy and refuses a request differing in a single field. | |
| 52 | + | func TestServerBindingAcceptsOnlyExactPolicy(t *testing.T) { | |
| 53 | + | binding, err := NewServerBinding(testIdentity(), testIncarnation(), testPolicy()) | |
| 54 | + | require.NoError(t, err) | |
| 55 | + | require.True(t, binding.Accepts(testPolicy())) | |
| 56 | + | ||
| 57 | + | tests := []struct { | |
| 58 | + | name string | |
| 59 | + | mutate func(*ports.BrokerPolicy) | |
| 60 | + | }{ | |
| 61 | + | {name: "protocol version", mutate: func(p *ports.BrokerPolicy) { p.ProtocolVersion++ }}, | |
| 62 | + | {name: "catalogue schema version", mutate: func(p *ports.BrokerPolicy) { p.CatalogSchemaVersion++ }}, | |
| 63 | + | {name: "environment policy", mutate: func(p *ports.BrokerPolicy) { | |
| 64 | + | p.EnvironmentPolicy = protocol.EnvironmentPolicyClientOwned | |
| 65 | + | }}, | |
| 66 | + | {name: "transport", mutate: func(p *ports.BrokerPolicy) { p.Transport = "stdio" }}, | |
| 67 | + | {name: "trust", mutate: func(p *ports.BrokerPolicy) { p.Trust = "tofu" }}, | |
| 68 | + | {name: "launch", mutate: func(p *ports.BrokerPolicy) { p.Launch = "manual" }}, | |
| 69 | + | {name: "isolation", mutate: func(p *ports.BrokerPolicy) { p.Isolation = "system" }}, | |
| 70 | + | } | |
| 71 | + | for _, test := range tests { | |
| 72 | + | t.Run(test.name, func(t *testing.T) { | |
| 73 | + | requested := testPolicy() | |
| 74 | + | test.mutate(&requested) | |
| 75 | + | require.False(t, binding.Accepts(requested)) | |
| 76 | + | }) | |
| 77 | + | } | |
| 78 | + | } | |
| 79 | + | ||
| 80 | + | // TestServerBindingIsImmutable proves the binding copied its inputs at | |
| 81 | + | // construction: mutating the caller's local policy afterwards never changes | |
| 82 | + | // what the binding enforces. | |
| 83 | + | func TestServerBindingIsImmutable(t *testing.T) { | |
| 84 | + | local := testPolicy() | |
| 85 | + | binding, err := NewServerBinding(testIdentity(), testIncarnation(), local) | |
| 86 | + | require.NoError(t, err) | |
| 87 | + | ||
| 88 | + | local.Transport = "mutated" | |
| 89 | + | local.Trust = "mutated" | |
| 90 | + | require.Equal(t, testPolicy(), binding.Policy()) | |
| 91 | + | require.True(t, binding.Accepts(testPolicy())) | |
| 92 | + | } | |
A internal/adapters/daemonmux/carrier.go +267 -0
| @@ -0,0 +1,267 @@ | |||
| 1 | + | // Raw-carrier bridge (P3.2b). | |
| 2 | + | // | |
| 3 | + | // One daemonmux physical connection rides an existing raw framed transport: | |
| 4 | + | // IPC, QUIC, or SSH stdio, each of which already owns vev's shared 4-byte | |
| 5 | + | // big-endian length framing and a caller-bounded receive (wire.BoundedTransport). | |
| 6 | + | // This file adapts that raw carriage to the private FramedCarrier the pump | |
| 7 | + | // (pump.go) and the physical preamble handshake (handshake.go) drive. It | |
| 8 | + | // introduces no socket, no framing of its own, no listener, and no sessionwire. | |
| 9 | + | // | |
| 10 | + | // The bridge is narrow and bounded. Send is synchronous: it hands one envelope | |
| 11 | + | // to the raw transport and returns the raw write result, so no error means the | |
| 12 | + | // carriage accepted the frame. Receive is bounded to wire.PreambleLimit while | |
| 13 | + | // the connection is in its preamble phase, and the explicit, one-way Negotiate | |
| 14 | + | // transition raises that bound to the daemonmux envelope ceiling the physical | |
| 15 | + | // preamble negotiated: a receive never allocates a body larger than the bound | |
| 16 | + | // that applied when it started, and an oversized application frame after | |
| 17 | + | // negotiation is refused rather than accepted. Negotiate must run before the | |
| 18 | + | // pump that reads application frames; a receive already in flight keeps the | |
| 19 | + | // bound it started with. | |
| 20 | + | // | |
| 21 | + | // Close is idempotent and concurrent-safe: it marks the bridge closed, closes | |
| 22 | + | // the raw transport through the adapter prompt-close contract so a blocked | |
| 23 | + | // Send or RecvBounded releases, and then waits for every bridge operation that | |
| 24 | + | // was already admitted. Send and Receive honour their context the same way: | |
| 25 | + | // cancelling it closes the raw transport to release the blocked call - a | |
| 26 | + | // carriage that can no longer make progress is terminal - and joins the one | |
| 27 | + | // worker it started, so no goroutine is abandoned. Because Close joins the | |
| 28 | + | // operations it admitted and each operation joins its own worker, the bridge | |
| 29 | + | // owns no long-lived goroutine. | |
| 30 | + | package daemonmux | |
| 31 | + | ||
| 32 | + | import ( | |
| 33 | + | "context" | |
| 34 | + | "errors" | |
| 35 | + | "sync" | |
| 36 | + | "sync/atomic" | |
| 37 | + | ||
| 38 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 39 | + | ) | |
| 40 | + | ||
| 41 | + | var ( | |
| 42 | + | // ErrCarrierState reports an invalid construction or state transition: a | |
| 43 | + | // nil raw transport, or a second Negotiate once the preamble phase ended. | |
| 44 | + | ErrCarrierState = errors.New("daemonmux: invalid carrier state") | |
| 45 | + | // ErrCarrierClosed reports Send or Receive on a bridge that was closed. | |
| 46 | + | ErrCarrierClosed = errors.New("daemonmux: carrier closed") | |
| 47 | + | ) | |
| 48 | + | ||
| 49 | + | // RawFramedTransport is the narrow raw framed carriage the bridge adapts: Send | |
| 50 | + | // writes one complete envelope synchronously, RecvBounded reads one complete | |
| 51 | + | // envelope bounded to limit before allocating or reading its body, and Close | |
| 52 | + | // releases blocked I/O. Every wire.BoundedTransport satisfies it; the bridge | |
| 53 | + | // deliberately does not require the unbounded Recv it never calls. | |
| 54 | + | type RawFramedTransport interface { | |
| 55 | + | Send(envelope wire.Envelope) error | |
| 56 | + | RecvBounded(limit uint64) (wire.Envelope, error) | |
| 57 | + | Close() error | |
| 58 | + | } | |
| 59 | + | ||
| 60 | + | // Every raw wire.BoundedTransport satisfies the bridge's narrow capability. | |
| 61 | + | var _ RawFramedTransport = (wire.BoundedTransport)(nil) | |
| 62 | + | ||
| 63 | + | // FramedCarrierBridge adapts a RawFramedTransport to the private FramedCarrier | |
| 64 | + | // contract. It is safe for concurrent use: Send and Receive may run from | |
| 65 | + | // different goroutines, Close is idempotent and concurrent-safe, and the | |
| 66 | + | // receive bound is read atomically so Negotiate never races a receive. | |
| 67 | + | type FramedCarrierBridge struct { | |
| 68 | + | raw RawFramedTransport | |
| 69 | + | ||
| 70 | + | // limit is the receive bound the next Receive applies: wire.PreambleLimit | |
| 71 | + | // until Negotiate records the negotiated envelope ceiling. | |
| 72 | + | limit atomic.Uint64 | |
| 73 | + | negotiated atomic.Bool | |
| 74 | + | ||
| 75 | + | // mu guards closed and closeErr and serializes operation admission against | |
| 76 | + | // Close, so ops is never Add-ed concurrently with Wait. | |
| 77 | + | mu sync.Mutex | |
| 78 | + | closed bool | |
| 79 | + | closeErr error | |
| 80 | + | ||
| 81 | + | // done is closed exactly once by Close so a blocked operation wakes. | |
| 82 | + | done chan struct{} | |
| 83 | + | ||
| 84 | + | // ops counts admitted operations between admission and return. Close waits | |
| 85 | + | // for it after closing the raw transport, which is what guarantees no | |
| 86 | + | // bridge worker outlives Close. | |
| 87 | + | ops sync.WaitGroup | |
| 88 | + | ||
| 89 | + | rawCloseOnce sync.Once | |
| 90 | + | closeOnce sync.Once | |
| 91 | + | } | |
| 92 | + | ||
| 93 | + | var _ FramedCarrier = (*FramedCarrierBridge)(nil) | |
| 94 | + | ||
| 95 | + | // NewPreambleCarrier wraps raw in its preamble phase: every Receive is bounded | |
| 96 | + | // to wire.PreambleLimit until Negotiate raises the bound to the negotiated | |
| 97 | + | // daemonmux envelope ceiling. A nil raw transport is refused with | |
| 98 | + | // ErrCarrierState. | |
| 99 | + | func NewPreambleCarrier(raw RawFramedTransport) (*FramedCarrierBridge, error) { | |
| 100 | + | if raw == nil { | |
| 101 | + | return nil, ErrCarrierState | |
| 102 | + | } | |
| 103 | + | c := &FramedCarrierBridge{raw: raw, done: make(chan struct{})} | |
| 104 | + | c.limit.Store(wire.PreambleLimit) | |
| 105 | + | return c, nil | |
| 106 | + | } | |
| 107 | + | ||
| 108 | + | // Negotiate records the connection's negotiated ceilings and raises the receive | |
| 109 | + | // bound from wire.PreambleLimit to ceilings.MaxReceiveEnvelopeBytes. It is the | |
| 110 | + | // explicit, one-way state transition out of the preamble phase: an invalid | |
| 111 | + | // advertisement, a second call, or a call after Close is refused rather than | |
| 112 | + | // silently widening the bound. The transition must precede the pump that reads | |
| 113 | + | // application frames; a receive already in flight keeps its old bound. | |
| 114 | + | func (c *FramedCarrierBridge) Negotiate(ceilings MuxCeilings) error { | |
| 115 | + | if c == nil || c.raw == nil { | |
| 116 | + | return ErrCarrierState | |
| 117 | + | } | |
| 118 | + | if err := ceilings.Validate(); err != nil { | |
| 119 | + | return err | |
| 120 | + | } | |
| 121 | + | c.mu.Lock() | |
| 122 | + | defer c.mu.Unlock() | |
| 123 | + | if c.closed { | |
| 124 | + | return ErrCarrierClosed | |
| 125 | + | } | |
| 126 | + | if c.negotiated.Load() { | |
| 127 | + | return ErrCarrierState | |
| 128 | + | } | |
| 129 | + | c.limit.Store(ceilings.MaxReceiveEnvelopeBytes) | |
| 130 | + | c.negotiated.Store(true) | |
| 131 | + | return nil | |
| 132 | + | } | |
| 133 | + | ||
| 134 | + | // ReceiveLimit returns the bound the next Receive applies: wire.PreambleLimit | |
| 135 | + | // before Negotiate and the negotiated envelope ceiling afterwards. | |
| 136 | + | func (c *FramedCarrierBridge) ReceiveLimit() uint64 { | |
| 137 | + | if c == nil { | |
| 138 | + | return 0 | |
| 139 | + | } | |
| 140 | + | return c.limit.Load() | |
| 141 | + | } | |
| 142 | + | ||
| 143 | + | // Preamble reports whether the carrier is still bounded to wire.PreambleLimit. | |
| 144 | + | func (c *FramedCarrierBridge) Preamble() bool { | |
| 145 | + | if c == nil { | |
| 146 | + | return false | |
| 147 | + | } | |
| 148 | + | return !c.negotiated.Load() | |
| 149 | + | } | |
| 150 | + | ||
| 151 | + | // Send hands one envelope to the raw transport and returns its write result, so | |
| 152 | + | // it is synchronous: a nil error means the raw write attempt completed. A | |
| 153 | + | // closed bridge reports ErrCarrierClosed. Cancelling ctx closes the raw | |
| 154 | + | // transport to release a blocked write and joins the worker before returning, | |
| 155 | + | // so a send that cannot finish never strands a goroutine. | |
| 156 | + | func (c *FramedCarrierBridge) Send(ctx context.Context, payload []byte) error { | |
| 157 | + | if c == nil || c.raw == nil { | |
| 158 | + | return ErrCarrierState | |
| 159 | + | } | |
| 160 | + | if ctx == nil { | |
| 161 | + | ctx = context.Background() | |
| 162 | + | } | |
| 163 | + | if err := c.enter(); err != nil { | |
| 164 | + | return err | |
| 165 | + | } | |
| 166 | + | defer c.ops.Done() | |
| 167 | + | result := make(chan error, 1) | |
| 168 | + | go func() { result <- c.raw.Send(wire.Envelope{Payload: payload}) }() | |
| 169 | + | select { | |
| 170 | + | case err := <-result: | |
| 171 | + | return err | |
| 172 | + | case <-c.done: | |
| 173 | + | <-result | |
| 174 | + | return ErrCarrierClosed | |
| 175 | + | case <-ctx.Done(): | |
| 176 | + | c.closeRaw() | |
| 177 | + | <-result | |
| 178 | + | return ctx.Err() | |
| 179 | + | } | |
| 180 | + | } | |
| 181 | + | ||
| 182 | + | // Receive blocks for the next complete envelope, validated against the | |
| 183 | + | // carrier's current bound (wire.PreambleLimit in the preamble phase, the | |
| 184 | + | // negotiated envelope ceiling afterwards) before its body is allocated. A | |
| 185 | + | // closed bridge reports ErrCarrierClosed. Cancelling ctx closes the raw | |
| 186 | + | // transport to release the blocked read and joins the worker before returning. | |
| 187 | + | func (c *FramedCarrierBridge) Receive(ctx context.Context) ([]byte, error) { | |
| 188 | + | if c == nil || c.raw == nil { | |
| 189 | + | return nil, ErrCarrierState | |
| 190 | + | } | |
| 191 | + | if ctx == nil { | |
| 192 | + | ctx = context.Background() | |
| 193 | + | } | |
| 194 | + | if err := c.enter(); err != nil { | |
| 195 | + | return nil, err | |
| 196 | + | } | |
| 197 | + | defer c.ops.Done() | |
| 198 | + | limit := c.limit.Load() | |
| 199 | + | result := make(chan carrierRecv, 1) | |
| 200 | + | go func() { | |
| 201 | + | envelope, err := c.raw.RecvBounded(limit) | |
| 202 | + | result <- carrierRecv{payload: envelope.Payload, err: err} | |
| 203 | + | }() | |
| 204 | + | select { | |
| 205 | + | case r := <-result: | |
| 206 | + | return r.payload, r.err | |
| 207 | + | case <-c.done: | |
| 208 | + | <-result | |
| 209 | + | return nil, ErrCarrierClosed | |
| 210 | + | case <-ctx.Done(): | |
| 211 | + | c.closeRaw() | |
| 212 | + | <-result | |
| 213 | + | return nil, ctx.Err() | |
| 214 | + | } | |
| 215 | + | } | |
| 216 | + | ||
| 217 | + | // Close interrupts a blocked Send or Receive and is idempotent and | |
| 218 | + | // concurrent-safe. It marks the bridge closed, closes the raw transport | |
| 219 | + | // through the adapter prompt-close contract, then waits for every admitted | |
| 220 | + | // operation, so it never returns with a bridge worker still running. It returns | |
| 221 | + | // the raw transport's close error; every caller observes the same result. | |
| 222 | + | func (c *FramedCarrierBridge) Close() error { | |
| 223 | + | if c == nil || c.raw == nil { | |
| 224 | + | return ErrCarrierState | |
| 225 | + | } | |
| 226 | + | c.closeOnce.Do(func() { | |
| 227 | + | c.mu.Lock() | |
| 228 | + | c.closed = true | |
| 229 | + | c.mu.Unlock() | |
| 230 | + | close(c.done) | |
| 231 | + | c.closeRaw() | |
| 232 | + | c.ops.Wait() | |
| 233 | + | }) | |
| 234 | + | c.mu.Lock() | |
| 235 | + | defer c.mu.Unlock() | |
| 236 | + | return c.closeErr | |
| 237 | + | } | |
| 238 | + | ||
| 239 | + | // enter admits one operation unless the bridge already closed. It holds mu so | |
| 240 | + | // an admission can never race Close's Wait. | |
| 241 | + | func (c *FramedCarrierBridge) enter() error { | |
| 242 | + | c.mu.Lock() | |
| 243 | + | defer c.mu.Unlock() | |
| 244 | + | if c.closed { | |
| 245 | + | return ErrCarrierClosed | |
| 246 | + | } | |
| 247 | + | c.ops.Add(1) | |
| 248 | + | return nil | |
| 249 | + | } | |
| 250 | + | ||
| 251 | + | // closeRaw closes the raw transport exactly once through the adapter | |
| 252 | + | // prompt-close contract, unblocking a blocked Send or RecvBounded. The first | |
| 253 | + | // close's error is retained for Close. | |
| 254 | + | func (c *FramedCarrierBridge) closeRaw() { | |
| 255 | + | c.rawCloseOnce.Do(func() { | |
| 256 | + | err := c.raw.Close() | |
| 257 | + | c.mu.Lock() | |
| 258 | + | c.closeErr = err | |
| 259 | + | c.mu.Unlock() | |
| 260 | + | }) | |
| 261 | + | } | |
| 262 | + | ||
| 263 | + | // carrierRecv is one bounded receive result delivered from a bridge worker. | |
| 264 | + | type carrierRecv struct { | |
| 265 | + | payload []byte | |
| 266 | + | err error | |
| 267 | + | } | |
A internal/adapters/daemonmux/carrier_test.go +346 -0
| @@ -0,0 +1,346 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "encoding/binary" | |
| 6 | + | "errors" | |
| 7 | + | "net" | |
| 8 | + | "runtime" | |
| 9 | + | "sync" | |
| 10 | + | "sync/atomic" | |
| 11 | + | "testing" | |
| 12 | + | "time" | |
| 13 | + | ||
| 14 | + | "github.com/stretchr/testify/require" | |
| 15 | + | ||
| 16 | + | "github.com/bnema/vev/internal/adapters/ipc" | |
| 17 | + | "github.com/bnema/vev/internal/adapters/streamframe" | |
| 18 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 19 | + | ) | |
| 20 | + | ||
| 21 | + | // errFakeRawClosed is the synthetic raw-carriage error a fakeRawTransport | |
| 22 | + | // returns once Close released its blocked I/O, mirroring the adapter | |
| 23 | + | // prompt-close contract. | |
| 24 | + | var errFakeRawClosed = errors.New("fakeRawTransport: closed") | |
| 25 | + | ||
| 26 | + | // fakeRawTransport is one end of a scripted raw framed carriage. RecvBounded | |
| 27 | + | // records every caller limit and refuses a body above it exactly like a real | |
| 28 | + | // streamframe framer, so a bridge test observes the bound the bridge applied. | |
| 29 | + | type fakeRawTransport struct { | |
| 30 | + | mu sync.Mutex | |
| 31 | + | limits []uint64 | |
| 32 | + | sends [][]byte | |
| 33 | + | inbound chan []byte | |
| 34 | + | closedCh chan struct{} | |
| 35 | + | closeOnce sync.Once | |
| 36 | + | closes atomic.Int64 | |
| 37 | + | closeErr error | |
| 38 | + | // sendHold, when non-nil, blocks Send until released or the carriage | |
| 39 | + | // closes, so a test can observe a synchronous send in flight. | |
| 40 | + | sendHold chan struct{} | |
| 41 | + | } | |
| 42 | + | ||
| 43 | + | func newFakeRawTransport() *fakeRawTransport { | |
| 44 | + | return &fakeRawTransport{ | |
| 45 | + | inbound: make(chan []byte, 8), | |
| 46 | + | closedCh: make(chan struct{}), | |
| 47 | + | } | |
| 48 | + | } | |
| 49 | + | ||
| 50 | + | func (f *fakeRawTransport) Send(envelope wire.Envelope) error { | |
| 51 | + | f.mu.Lock() | |
| 52 | + | hold := f.sendHold | |
| 53 | + | f.mu.Unlock() | |
| 54 | + | if hold != nil { | |
| 55 | + | select { | |
| 56 | + | case <-hold: | |
| 57 | + | case <-f.closedCh: | |
| 58 | + | return errFakeRawClosed | |
| 59 | + | } | |
| 60 | + | } | |
| 61 | + | select { | |
| 62 | + | case <-f.closedCh: | |
| 63 | + | return errFakeRawClosed | |
| 64 | + | default: | |
| 65 | + | } | |
| 66 | + | f.mu.Lock() | |
| 67 | + | f.sends = append(f.sends, append([]byte(nil), envelope.Payload...)) | |
| 68 | + | f.mu.Unlock() | |
| 69 | + | return nil | |
| 70 | + | } | |
| 71 | + | ||
| 72 | + | func (f *fakeRawTransport) RecvBounded(limit uint64) (wire.Envelope, error) { | |
| 73 | + | f.mu.Lock() | |
| 74 | + | f.limits = append(f.limits, limit) | |
| 75 | + | f.mu.Unlock() | |
| 76 | + | select { | |
| 77 | + | case payload := <-f.inbound: | |
| 78 | + | if uint64(len(payload)) > limit { | |
| 79 | + | return wire.Envelope{}, streamframe.ErrTooLarge | |
| 80 | + | } | |
| 81 | + | return wire.Envelope{Payload: payload}, nil | |
| 82 | + | case <-f.closedCh: | |
| 83 | + | return wire.Envelope{}, errFakeRawClosed | |
| 84 | + | } | |
| 85 | + | } | |
| 86 | + | ||
| 87 | + | func (f *fakeRawTransport) Close() error { | |
| 88 | + | f.closes.Add(1) | |
| 89 | + | f.closeOnce.Do(func() { close(f.closedCh) }) | |
| 90 | + | return f.closeErr | |
| 91 | + | } | |
| 92 | + | ||
| 93 | + | func (f *fakeRawTransport) recordedLimits() []uint64 { | |
| 94 | + | f.mu.Lock() | |
| 95 | + | defer f.mu.Unlock() | |
| 96 | + | return append([]uint64(nil), f.limits...) | |
| 97 | + | } | |
| 98 | + | ||
| 99 | + | func (f *fakeRawTransport) recordedSends() [][]byte { | |
| 100 | + | f.mu.Lock() | |
| 101 | + | defer f.mu.Unlock() | |
| 102 | + | return append([][]byte(nil), f.sends...) | |
| 103 | + | } | |
| 104 | + | ||
| 105 | + | // carrierTestCeilings is a valid advertisement with the smallest envelope | |
| 106 | + | // ceiling, so the negotiated bound stays clearly above wire.PreambleLimit. | |
| 107 | + | func carrierTestCeilings() MuxCeilings { | |
| 108 | + | ceilings := DefaultMuxCeilings() | |
| 109 | + | ceilings.MaxReceiveEnvelopeBytes = MinMuxEnvelopeBytes | |
| 110 | + | return ceilings | |
| 111 | + | } | |
| 112 | + | ||
| 113 | + | // TestFramedCarrierBridgeBoundsPreambleThenNegotiated proves the bridge bounds | |
| 114 | + | // every receive to wire.PreambleLimit before Negotiate and to the negotiated | |
| 115 | + | // envelope ceiling afterwards, refusing an oversized frame at each phase. | |
| 116 | + | func TestFramedCarrierBridgeBoundsPreambleThenNegotiated(t *testing.T) { | |
| 117 | + | raw := newFakeRawTransport() | |
| 118 | + | bridge, err := NewPreambleCarrier(raw) | |
| 119 | + | require.NoError(t, err) | |
| 120 | + | require.True(t, bridge.Preamble()) | |
| 121 | + | require.Equal(t, uint64(wire.PreambleLimit), bridge.ReceiveLimit()) | |
| 122 | + | ||
| 123 | + | small := []byte("preamble-sized request") | |
| 124 | + | raw.inbound <- small | |
| 125 | + | got, err := bridge.Receive(context.Background()) | |
| 126 | + | require.NoError(t, err) | |
| 127 | + | require.Equal(t, small, got) | |
| 128 | + | require.Equal(t, []uint64{wire.PreambleLimit}, raw.recordedLimits()) | |
| 129 | + | ||
| 130 | + | oversize := make([]byte, wire.PreambleLimit+1) | |
| 131 | + | raw.inbound <- oversize | |
| 132 | + | _, err = bridge.Receive(context.Background()) | |
| 133 | + | require.ErrorIs(t, err, streamframe.ErrTooLarge) | |
| 134 | + | ||
| 135 | + | ceilings := carrierTestCeilings() | |
| 136 | + | require.NoError(t, bridge.Negotiate(ceilings)) | |
| 137 | + | require.False(t, bridge.Preamble()) | |
| 138 | + | require.Equal(t, ceilings.MaxReceiveEnvelopeBytes, bridge.ReceiveLimit()) | |
| 139 | + | ||
| 140 | + | raw.inbound <- oversize | |
| 141 | + | got, err = bridge.Receive(context.Background()) | |
| 142 | + | require.NoError(t, err) | |
| 143 | + | require.Equal(t, oversize, got) | |
| 144 | + | limits := raw.recordedLimits() | |
| 145 | + | require.Equal(t, ceilings.MaxReceiveEnvelopeBytes, limits[len(limits)-1]) | |
| 146 | + | } | |
| 147 | + | ||
| 148 | + | // TestFramedCarrierBridgeNegotiateIsExplicitAndOneWay proves the state | |
| 149 | + | // transition is explicit: an invalid advertisement is refused, a second | |
| 150 | + | // negotiation is refused, and negotiation after Close is refused. | |
| 151 | + | func TestFramedCarrierBridgeNegotiateIsExplicitAndOneWay(t *testing.T) { | |
| 152 | + | raw := newFakeRawTransport() | |
| 153 | + | bridge, err := NewPreambleCarrier(raw) | |
| 154 | + | require.NoError(t, err) | |
| 155 | + | ||
| 156 | + | require.ErrorIs(t, bridge.Negotiate(MuxCeilings{}), ErrInvalidCeilings) | |
| 157 | + | ||
| 158 | + | require.NoError(t, bridge.Negotiate(carrierTestCeilings())) | |
| 159 | + | require.ErrorIs(t, bridge.Negotiate(carrierTestCeilings()), ErrCarrierState) | |
| 160 | + | ||
| 161 | + | require.NoError(t, bridge.Close()) | |
| 162 | + | require.ErrorIs(t, bridge.Negotiate(carrierTestCeilings()), ErrCarrierClosed) | |
| 163 | + | } | |
| 164 | + | ||
| 165 | + | // TestNewPreambleCarrierRefusesNil proves construction validates its raw | |
| 166 | + | // transport. | |
| 167 | + | func TestNewPreambleCarrierRefusesNil(t *testing.T) { | |
| 168 | + | _, err := NewPreambleCarrier(nil) | |
| 169 | + | require.ErrorIs(t, err, ErrCarrierState) | |
| 170 | + | } | |
| 171 | + | ||
| 172 | + | // TestFramedCarrierBridgeSendIsSynchronousAndExact proves Send returns the raw | |
| 173 | + | // write result: it does not return while the write is held, records the payload | |
| 174 | + | // byte for byte, and propagates the raw error. | |
| 175 | + | func TestFramedCarrierBridgeSendIsSynchronousAndExact(t *testing.T) { | |
| 176 | + | raw := newFakeRawTransport() | |
| 177 | + | bridge, err := NewPreambleCarrier(raw) | |
| 178 | + | require.NoError(t, err) | |
| 179 | + | defer func() { _ = bridge.Close() }() | |
| 180 | + | ||
| 181 | + | raw.sendHold = make(chan struct{}) | |
| 182 | + | payload := []byte("exact outbound envelope") | |
| 183 | + | sendDone := make(chan error, 1) | |
| 184 | + | go func() { sendDone <- bridge.Send(context.Background(), payload) }() | |
| 185 | + | ||
| 186 | + | select { | |
| 187 | + | case err := <-sendDone: | |
| 188 | + | t.Fatalf("Send returned %v while the raw write was held", err) | |
| 189 | + | case <-time.After(50 * time.Millisecond): | |
| 190 | + | } | |
| 191 | + | require.Empty(t, raw.recordedSends()) | |
| 192 | + | ||
| 193 | + | close(raw.sendHold) | |
| 194 | + | require.NoError(t, <-sendDone) | |
| 195 | + | require.Equal(t, [][]byte{payload}, raw.recordedSends()) | |
| 196 | + | ||
| 197 | + | raw.closeErr = errors.New("raw close failed") | |
| 198 | + | require.ErrorIs(t, bridge.Close(), raw.closeErr) | |
| 199 | + | require.ErrorIs(t, bridge.Send(context.Background(), payload), ErrCarrierClosed) | |
| 200 | + | _, err = bridge.Receive(context.Background()) | |
| 201 | + | require.ErrorIs(t, err, ErrCarrierClosed) | |
| 202 | + | } | |
| 203 | + | ||
| 204 | + | // TestFramedCarrierBridgeReceiveContextCancelReleasesRead proves a cancelled | |
| 205 | + | // receive closes the raw carriage to release the blocked call, returns the | |
| 206 | + | // context error, and leaves no goroutine behind. | |
| 207 | + | func TestFramedCarrierBridgeReceiveContextCancelReleasesRead(t *testing.T) { | |
| 208 | + | raw := newFakeRawTransport() | |
| 209 | + | bridge, err := NewPreambleCarrier(raw) | |
| 210 | + | require.NoError(t, err) | |
| 211 | + | ||
| 212 | + | before := runtime.NumGoroutine() | |
| 213 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 214 | + | recvDone := make(chan error, 1) | |
| 215 | + | go func() { | |
| 216 | + | _, err := bridge.Receive(ctx) | |
| 217 | + | recvDone <- err | |
| 218 | + | }() | |
| 219 | + | require.Eventually(t, func() bool { return len(raw.recordedLimits()) == 1 }, time.Second, time.Millisecond) | |
| 220 | + | cancel() | |
| 221 | + | select { | |
| 222 | + | case err := <-recvDone: | |
| 223 | + | require.ErrorIs(t, err, context.Canceled) | |
| 224 | + | case <-time.After(5 * time.Second): | |
| 225 | + | t.Fatal("Receive did not return after context cancellation") | |
| 226 | + | } | |
| 227 | + | require.Equal(t, int64(1), raw.closes.Load(), "cancellation must release the blocked raw read") | |
| 228 | + | require.NoError(t, bridge.Close()) | |
| 229 | + | require.Eventually(t, func() bool { return runtime.NumGoroutine() <= before+1 }, 5*time.Second, time.Millisecond, | |
| 230 | + | "the bridge must not abandon its receive worker (before=%d now=%d)", before, runtime.NumGoroutine()) | |
| 231 | + | } | |
| 232 | + | ||
| 233 | + | // TestFramedCarrierBridgeCloseInterruptsBlockedReceiveAndIsConcurrentSafe | |
| 234 | + | // proves Close unblocks a blocked receive, is idempotent across concurrent | |
| 235 | + | // callers, closes the raw carriage exactly once, and joins its workers. | |
| 236 | + | func TestFramedCarrierBridgeCloseInterruptsBlockedReceiveAndIsConcurrentSafe(t *testing.T) { | |
| 237 | + | raw := newFakeRawTransport() | |
| 238 | + | bridge, err := NewPreambleCarrier(raw) | |
| 239 | + | require.NoError(t, err) | |
| 240 | + | ||
| 241 | + | before := runtime.NumGoroutine() | |
| 242 | + | recvDone := make(chan error, 1) | |
| 243 | + | go func() { | |
| 244 | + | _, err := bridge.Receive(context.Background()) | |
| 245 | + | recvDone <- err | |
| 246 | + | }() | |
| 247 | + | require.Eventually(t, func() bool { return len(raw.recordedLimits()) == 1 }, time.Second, time.Millisecond) | |
| 248 | + | ||
| 249 | + | const closers = 8 | |
| 250 | + | var wg sync.WaitGroup | |
| 251 | + | closeErrs := make([]error, closers) | |
| 252 | + | for i := range closers { | |
| 253 | + | wg.Add(1) | |
| 254 | + | go func() { | |
| 255 | + | defer wg.Done() | |
| 256 | + | closeErrs[i] = bridge.Close() | |
| 257 | + | }() | |
| 258 | + | } | |
| 259 | + | wg.Wait() | |
| 260 | + | for _, err := range closeErrs { | |
| 261 | + | require.NoError(t, err) | |
| 262 | + | } | |
| 263 | + | require.Equal(t, int64(1), raw.closes.Load(), "Close must close the raw carriage exactly once") | |
| 264 | + | ||
| 265 | + | select { | |
| 266 | + | case err := <-recvDone: | |
| 267 | + | require.ErrorIs(t, err, ErrCarrierClosed) | |
| 268 | + | case <-time.After(5 * time.Second): | |
| 269 | + | t.Fatal("Close did not unblock the receive") | |
| 270 | + | } | |
| 271 | + | require.Eventually(t, func() bool { return runtime.NumGoroutine() <= before+1 }, 5*time.Second, time.Millisecond, | |
| 272 | + | "Close must join every bridge worker (before=%d now=%d)", before, runtime.NumGoroutine()) | |
| 273 | + | } | |
| 274 | + | ||
| 275 | + | // TestFramedCarrierBridgeOverIPCFramer proves the bridge drives a real vev raw | |
| 276 | + | // framed transport: the shared framing round-trips through the bridge, the | |
| 277 | + | // preamble bound refuses an oversized frame before its body, and Negotiate | |
| 278 | + | // raises the bound for a larger negotiated frame. | |
| 279 | + | func TestFramedCarrierBridgeOverIPCFramer(t *testing.T) { | |
| 280 | + | t.Run("preamble and negotiated frames round trip", func(t *testing.T) { | |
| 281 | + | left, right := net.Pipe() | |
| 282 | + | t.Cleanup(func() { | |
| 283 | + | _ = left.Close() | |
| 284 | + | _ = right.Close() | |
| 285 | + | }) | |
| 286 | + | raw, ok := ipc.NewTransport(left).(RawFramedTransport) | |
| 287 | + | require.True(t, ok, "the IPC transport must satisfy the bounded raw capability") | |
| 288 | + | bridge, err := NewPreambleCarrier(raw) | |
| 289 | + | require.NoError(t, err) | |
| 290 | + | peer := ipc.NewTransport(right) | |
| 291 | + | defer func() { | |
| 292 | + | _ = bridge.Close() | |
| 293 | + | _ = peer.Close() | |
| 294 | + | }() | |
| 295 | + | ||
| 296 | + | small := []byte("small preamble frame") | |
| 297 | + | go func() { _ = peer.Send(wire.Envelope{Payload: small}) }() | |
| 298 | + | got, err := bridge.Receive(context.Background()) | |
| 299 | + | require.NoError(t, err) | |
| 300 | + | require.Equal(t, small, got) | |
| 301 | + | ||
| 302 | + | require.NoError(t, bridge.Negotiate(carrierTestCeilings())) | |
| 303 | + | large := make([]byte, wire.PreambleLimit+4096) | |
| 304 | + | for i := range large { | |
| 305 | + | large[i] = byte('a' + i%26) | |
| 306 | + | } | |
| 307 | + | go func() { _ = peer.Send(wire.Envelope{Payload: large}) }() | |
| 308 | + | got, err = bridge.Receive(context.Background()) | |
| 309 | + | require.NoError(t, err) | |
| 310 | + | require.Equal(t, large, got) | |
| 311 | + | ||
| 312 | + | outbound := []byte("exact outbound through the real framer") | |
| 313 | + | type peerRecv struct { | |
| 314 | + | envelope wire.Envelope | |
| 315 | + | err error | |
| 316 | + | } | |
| 317 | + | peerResult := make(chan peerRecv, 1) | |
| 318 | + | go func() { | |
| 319 | + | envelope, err := peer.Recv() | |
| 320 | + | peerResult <- peerRecv{envelope: envelope, err: err} | |
| 321 | + | }() | |
| 322 | + | require.NoError(t, bridge.Send(context.Background(), outbound)) | |
| 323 | + | result := <-peerResult | |
| 324 | + | require.NoError(t, result.err) | |
| 325 | + | require.Equal(t, outbound, result.envelope.Payload) | |
| 326 | + | }) | |
| 327 | + | ||
| 328 | + | t.Run("oversized preamble frame is refused before its body", func(t *testing.T) { | |
| 329 | + | left, right := net.Pipe() | |
| 330 | + | t.Cleanup(func() { | |
| 331 | + | _ = left.Close() | |
| 332 | + | _ = right.Close() | |
| 333 | + | }) | |
| 334 | + | raw, ok := ipc.NewTransport(left).(RawFramedTransport) | |
| 335 | + | require.True(t, ok) | |
| 336 | + | bridge, err := NewPreambleCarrier(raw) | |
| 337 | + | require.NoError(t, err) | |
| 338 | + | defer func() { _ = bridge.Close() }() | |
| 339 | + | ||
| 340 | + | // Advertise a 1 MiB frame and never send its body: only the four-byte | |
| 341 | + | // length prefix exists on the wire. | |
| 342 | + | go func() { _, _ = right.Write(binary.BigEndian.AppendUint32(nil, 1<<20)) }() | |
| 343 | + | _, err = bridge.Receive(context.Background()) | |
| 344 | + | require.ErrorIs(t, err, streamframe.ErrTooLarge) | |
| 345 | + | }) | |
| 346 | + | } | |
A internal/adapters/daemonmux/closing_test.go +138 -0
| @@ -0,0 +1,138 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "io" | |
| 6 | + | "testing" | |
| 7 | + | ||
| 8 | + | "github.com/stretchr/testify/require" | |
| 9 | + | ) | |
| 10 | + | ||
| 11 | + | // TestPumpInboundDataClosePipelineDrainsThenEOF proves a pipelined inbound | |
| 12 | + | // Data followed by an orderly Close stays readable: the consumer drains the | |
| 13 | + | // accepted bytes and only then observes io.EOF, at which point the stream is | |
| 14 | + | // terminal and its bytes are released. | |
| 15 | + | func TestPumpInboundDataClosePipelineDrainsThenEOF(t *testing.T) { | |
| 16 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 17 | + | pump.Start(context.Background()) | |
| 18 | + | require.NoError(t, pump.Engine().Open(openFor(1))) | |
| 19 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 20 | + | status := mustStatus(t, pump.Engine(), 1) | |
| 21 | + | pipe := newMuxStreamPipe(pump, 1, pump.Ceilings().StreamChunkLimit, status.Done) | |
| 22 | + | ||
| 23 | + | // The peer pipelines one Data frame and its Close before the consumer reads. | |
| 24 | + | deliverClientFrame(t, carrier, Data{Physical: 1, Data: []byte("hello")}) | |
| 25 | + | deliverClientFrame(t, carrier, Close{Physical: 1}) | |
| 26 | + | ||
| 27 | + | buffer := make([]byte, len("hello")) | |
| 28 | + | _, err := io.ReadFull(pipe, buffer) | |
| 29 | + | require.NoError(t, err) | |
| 30 | + | require.Equal(t, "hello", string(buffer)) | |
| 31 | + | // The stream ends only after the preserved bytes are drained. | |
| 32 | + | _, err = pipe.Read(make([]byte, 1)) | |
| 33 | + | require.ErrorIs(t, err, io.EOF) | |
| 34 | + | ||
| 35 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == 0 }) | |
| 36 | + | status = mustStatus(t, pump.Engine(), 1) | |
| 37 | + | require.Equal(t, StreamTerminal, status.State) | |
| 38 | + | require.True(t, channelClosed(status.Done)) | |
| 39 | + | require.Zero(t, status.QueuedChunks) | |
| 40 | + | require.Zero(t, pump.Engine().AggregateBytes()) | |
| 41 | + | require.NoError(t, status.Err) | |
| 42 | + | require.False(t, pump.Engine().Closed()) | |
| 43 | + | require.False(t, channelClosed(pump.Done())) | |
| 44 | + | } | |
| 45 | + | ||
| 46 | + | // TestPumpInboundCloseSiblingIsolation proves an orderly Close with unread data | |
| 47 | + | // settles only its own stream once drained: a sibling keeps its state, its | |
| 48 | + | // queued chunk, and its admission slot while the closing stream's data is | |
| 49 | + | // preserved. Draining the sibling never closes it. | |
| 50 | + | func TestPumpInboundCloseSiblingIsolation(t *testing.T) { | |
| 51 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 52 | + | pump.Start(context.Background()) | |
| 53 | + | require.NoError(t, pump.Engine().Open(openFor(1))) | |
| 54 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 55 | + | require.NoError(t, pump.Engine().Open(openFor(2))) | |
| 56 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(2)})) | |
| 57 | + | ||
| 58 | + | deliverClientFrame(t, carrier, Data{Physical: 1, Data: []byte("one")}) | |
| 59 | + | deliverClientFrame(t, carrier, Data{Physical: 2, Data: []byte("two")}) | |
| 60 | + | deliverClientFrame(t, carrier, Close{Physical: 1}) | |
| 61 | + | ||
| 62 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { | |
| 63 | + | status, ok := e.Status(1) | |
| 64 | + | return ok && status.State == StreamClosing | |
| 65 | + | }) | |
| 66 | + | sibling := mustStatus(t, pump.Engine(), 2) | |
| 67 | + | require.Equal(t, StreamOpen, sibling.State, "the sibling is untouched by the close") | |
| 68 | + | require.Equal(t, 1, sibling.QueuedChunks) | |
| 69 | + | ||
| 70 | + | chunk, ok := takeEventually(t, pump, 2) | |
| 71 | + | require.True(t, ok) | |
| 72 | + | require.Equal(t, []byte("two"), chunk) | |
| 73 | + | require.Equal(t, StreamOpen, mustStatus(t, pump.Engine(), 2).State, "draining a sibling never closes it") | |
| 74 | + | ||
| 75 | + | chunk, ok = takeEventually(t, pump, 1) | |
| 76 | + | require.True(t, ok) | |
| 77 | + | require.Equal(t, []byte("one"), chunk) | |
| 78 | + | require.Equal(t, StreamTerminal, mustStatus(t, pump.Engine(), 1).State) | |
| 79 | + | require.Equal(t, StreamOpen, mustStatus(t, pump.Engine(), 2).State) | |
| 80 | + | require.Equal(t, 1, pump.Engine().Live()) | |
| 81 | + | require.False(t, pump.Engine().Closed()) | |
| 82 | + | require.False(t, channelClosed(pump.Done())) | |
| 83 | + | } | |
| 84 | + | ||
| 85 | + | // TestPumpInboundCloseChurnAccountingBounded proves the closing accounting is | |
| 86 | + | // balanced and bounded across many inbound Data-then-Close cycles: each closing | |
| 87 | + | // stream holds exactly its unread bytes until the draining Take releases them, | |
| 88 | + | // and the engine's retained record set never grows past its retention bound. | |
| 89 | + | func TestPumpInboundCloseChurnAccountingBounded(t *testing.T) { | |
| 90 | + | pump, _ := newTestPump(t, DirectionClient) | |
| 91 | + | pump.Start(context.Background()) | |
| 92 | + | const churn = int(MaxMuxStreams) * 8 | |
| 93 | + | for i := 1; i <= churn; i++ { | |
| 94 | + | id := PhysicalStreamID(i) | |
| 95 | + | require.NoError(t, pump.Engine().Open(openFor(id))) | |
| 96 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(id)})) | |
| 97 | + | require.NoError(t, pump.applyClient(Data{Physical: id, Data: []byte("payload")})) | |
| 98 | + | require.NoError(t, pump.applyClient(Close{Physical: id})) | |
| 99 | + | ||
| 100 | + | require.Equal(t, StreamClosing, mustStatus(t, pump.Engine(), id).State) | |
| 101 | + | require.Equal(t, 1, pump.Engine().Live(), "the closing stream still holds its slot") | |
| 102 | + | require.Equal(t, len("payload"), pump.Engine().AggregateBytes()) | |
| 103 | + | ||
| 104 | + | chunk, ok := pump.Take(id) | |
| 105 | + | require.True(t, ok) | |
| 106 | + | require.Equal(t, []byte("payload"), chunk) | |
| 107 | + | require.Equal(t, StreamTerminal, mustStatus(t, pump.Engine(), id).State) | |
| 108 | + | require.Zero(t, pump.Engine().Live()) | |
| 109 | + | require.Zero(t, pump.Engine().AggregateBytes()) | |
| 110 | + | } | |
| 111 | + | require.Len(t, pump.Engine().retired, MaxRetiredStreamRecords) | |
| 112 | + | require.LessOrEqual(t, len(pump.Engine().streams), MaxRetiredStreamRecords) | |
| 113 | + | require.False(t, pump.Engine().Closed()) | |
| 114 | + | require.False(t, channelClosed(pump.Done())) | |
| 115 | + | } | |
| 116 | + | ||
| 117 | + | // TestPumpInboundResetDiscardsClosingQueue proves a Reset still abandons a | |
| 118 | + | // closing stream's preserved queue immediately, releasing the bytes and the | |
| 119 | + | // slot without waiting for a consumer. | |
| 120 | + | func TestPumpInboundResetDiscardsClosingQueue(t *testing.T) { | |
| 121 | + | pump, _ := newTestPump(t, DirectionClient) | |
| 122 | + | require.NoError(t, pump.Engine().Open(openFor(1))) | |
| 123 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 124 | + | require.NoError(t, pump.applyClient(Data{Physical: 1, Data: []byte("queued")})) | |
| 125 | + | require.NoError(t, pump.applyClient(Close{Physical: 1})) | |
| 126 | + | require.Equal(t, StreamClosing, mustStatus(t, pump.Engine(), 1).State) | |
| 127 | + | ||
| 128 | + | require.NoError(t, pump.applyClient(Reset{Physical: 1})) | |
| 129 | + | ||
| 130 | + | status := mustStatus(t, pump.Engine(), 1) | |
| 131 | + | require.Equal(t, StreamTerminal, status.State) | |
| 132 | + | require.True(t, channelClosed(status.Done)) | |
| 133 | + | require.Zero(t, status.QueuedChunks) | |
| 134 | + | require.Zero(t, pump.Engine().AggregateBytes()) | |
| 135 | + | require.Zero(t, pump.Engine().Live()) | |
| 136 | + | _, ok := pump.Take(1) | |
| 137 | + | require.False(t, ok, "the reset discarded the preserved queue") | |
| 138 | + | } | |
A internal/adapters/daemonmux/codec.go +527 -0
| @@ -0,0 +1,527 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | // Directional envelope adaptation (P3.2). | |
| 4 | + | // | |
| 5 | + | // EncodeClient wraps one adapter-local client value in its directional | |
| 6 | + | // MuxClientEnvelope; DecodeClient unwraps and validates. EncodeServer and | |
| 7 | + | // DecodeServer do the same for the daemon direction. Every convert path uses | |
| 8 | + | // an explicit taxonomy switch: unknown variants and wrong-direction payloads | |
| 9 | + | // fail with ErrWrongDirection before any mutation. Decode paths run strict | |
| 10 | + | // wire.ScanEnvelope before generated unmarshal, then validate stream | |
| 11 | + | // identity, UTF-8/control/bidi text, bounds, environment/policy, purpose, | |
| 12 | + | // chunks, and error details. | |
| 13 | + | // | |
| 14 | + | // The codecs are stateless: no stream table, admission decision, scheduler, | |
| 15 | + | // queue, pump, socket, or framing engine lives here. Envelope sizes ride | |
| 16 | + | // under the caller's negotiated ceiling, every application envelope is | |
| 17 | + | // additionally capped at the fixed 1 MiB control-class bound | |
| 18 | + | // (MaxMuxApplicationBytes), and opaque stream frames ride under the | |
| 19 | + | // negotiated chunk ceiling. Data, Close, and Reset are shared payloads: the | |
| 20 | + | // same committed value converts into either direction, and direction is fixed | |
| 21 | + | // by the closed envelope oneof. | |
| 22 | + | ||
| 23 | + | import ( | |
| 24 | + | "google.golang.org/protobuf/proto" | |
| 25 | + | ||
| 26 | + | "github.com/bnema/vev/internal/domain" | |
| 27 | + | "github.com/bnema/vev/internal/ports" | |
| 28 | + | "github.com/bnema/vev/internal/protocol" | |
| 29 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 30 | + | ) | |
| 31 | + | ||
| 32 | + | // EncodeClient wraps one broker-to-daemon message in its serialized | |
| 33 | + | // directional envelope. The payload must fit maxEnvelopeBytes and the fixed | |
| 34 | + | // 1 MiB application ceiling; stream chunks must fit maxChunkBytes. | |
| 35 | + | func EncodeClient(message ClientMessage, maxEnvelopeBytes, maxChunkBytes uint64) ([]byte, error) { | |
| 36 | + | envelope, err := encodeClientEnvelope(message, maxChunkBytes) | |
| 37 | + | if err != nil { | |
| 38 | + | return nil, err | |
| 39 | + | } | |
| 40 | + | raw, err := proto.Marshal(envelope) | |
| 41 | + | if err != nil { | |
| 42 | + | return nil, err | |
| 43 | + | } | |
| 44 | + | if err := checkMuxEnvelopeCeiling(raw, maxEnvelopeBytes); err != nil { | |
| 45 | + | return nil, err | |
| 46 | + | } | |
| 47 | + | return raw, nil | |
| 48 | + | } | |
| 49 | + | ||
| 50 | + | // DecodeClient unwraps one serialized broker-to-daemon envelope into its | |
| 51 | + | // adapter-local value. Strict scanning runs before generated unmarshal; | |
| 52 | + | // wrong-direction payloads fail with ErrWrongDirection. | |
| 53 | + | func DecodeClient(payload []byte, maxEnvelopeBytes, maxChunkBytes uint64) (ClientMessage, error) { | |
| 54 | + | if err := checkMuxEnvelopeCeiling(payload, maxEnvelopeBytes); err != nil { | |
| 55 | + | return nil, err | |
| 56 | + | } | |
| 57 | + | envelope := &wire.MuxClientEnvelope{} | |
| 58 | + | if err := wire.ScanEnvelope(envelope, payload); err != nil { | |
| 59 | + | return nil, err | |
| 60 | + | } | |
| 61 | + | if err := (proto.UnmarshalOptions{DiscardUnknown: false}.Unmarshal(payload, envelope)); err != nil { | |
| 62 | + | return nil, err | |
| 63 | + | } | |
| 64 | + | return decodeClientEnvelope(envelope, maxChunkBytes) | |
| 65 | + | } | |
| 66 | + | ||
| 67 | + | // EncodeServer wraps one daemon-to-broker message in its serialized | |
| 68 | + | // directional envelope under the same envelope and chunk bounds. | |
| 69 | + | func EncodeServer(message ServerMessage, maxEnvelopeBytes, maxChunkBytes uint64) ([]byte, error) { | |
| 70 | + | envelope, err := encodeServerEnvelope(message, maxChunkBytes) | |
| 71 | + | if err != nil { | |
| 72 | + | return nil, err | |
| 73 | + | } | |
| 74 | + | raw, err := proto.Marshal(envelope) | |
| 75 | + | if err != nil { | |
| 76 | + | return nil, err | |
| 77 | + | } | |
| 78 | + | if err := checkMuxEnvelopeCeiling(raw, maxEnvelopeBytes); err != nil { | |
| 79 | + | return nil, err | |
| 80 | + | } | |
| 81 | + | return raw, nil | |
| 82 | + | } | |
| 83 | + | ||
| 84 | + | // DecodeServer unwraps one serialized daemon-to-broker envelope into its | |
| 85 | + | // adapter-local value. | |
| 86 | + | func DecodeServer(payload []byte, maxEnvelopeBytes, maxChunkBytes uint64) (ServerMessage, error) { | |
| 87 | + | if err := checkMuxEnvelopeCeiling(payload, maxEnvelopeBytes); err != nil { | |
| 88 | + | return nil, err | |
| 89 | + | } | |
| 90 | + | envelope := &wire.MuxServerEnvelope{} | |
| 91 | + | if err := wire.ScanEnvelope(envelope, payload); err != nil { | |
| 92 | + | return nil, err | |
| 93 | + | } | |
| 94 | + | if err := (proto.UnmarshalOptions{DiscardUnknown: false}.Unmarshal(payload, envelope)); err != nil { | |
| 95 | + | return nil, err | |
| 96 | + | } | |
| 97 | + | return decodeServerEnvelope(envelope, maxChunkBytes) | |
| 98 | + | } | |
| 99 | + | ||
| 100 | + | func encodeClientEnvelope(message ClientMessage, maxChunkBytes uint64) (*wire.MuxClientEnvelope, error) { | |
| 101 | + | if message == nil { | |
| 102 | + | return nil, ErrInvalidMessage | |
| 103 | + | } | |
| 104 | + | switch m := message.(type) { | |
| 105 | + | case Open: | |
| 106 | + | converted, err := openToWire(m) | |
| 107 | + | if err != nil { | |
| 108 | + | return nil, err | |
| 109 | + | } | |
| 110 | + | return &wire.MuxClientEnvelope{Payload: &wire.MuxClientEnvelope_Open{Open: converted}}, nil | |
| 111 | + | case *Open: | |
| 112 | + | if m == nil { | |
| 113 | + | return nil, ErrInvalidMessage | |
| 114 | + | } | |
| 115 | + | return encodeClientEnvelope(*m, maxChunkBytes) | |
| 116 | + | case Data: | |
| 117 | + | converted, err := dataToWire(m, maxChunkBytes) | |
| 118 | + | if err != nil { | |
| 119 | + | return nil, err | |
| 120 | + | } | |
| 121 | + | return &wire.MuxClientEnvelope{Payload: &wire.MuxClientEnvelope_Data{Data: converted}}, nil | |
| 122 | + | case *Data: | |
| 123 | + | if m == nil { | |
| 124 | + | return nil, ErrInvalidMessage | |
| 125 | + | } | |
| 126 | + | return encodeClientEnvelope(*m, maxChunkBytes) | |
| 127 | + | case Close: | |
| 128 | + | converted, err := closeToWire(m) | |
| 129 | + | if err != nil { | |
| 130 | + | return nil, err | |
| 131 | + | } | |
| 132 | + | return &wire.MuxClientEnvelope{Payload: &wire.MuxClientEnvelope_Close{Close: converted}}, nil | |
| 133 | + | case *Close: | |
| 134 | + | if m == nil { | |
| 135 | + | return nil, ErrInvalidMessage | |
| 136 | + | } | |
| 137 | + | return encodeClientEnvelope(*m, maxChunkBytes) | |
| 138 | + | case Reset: | |
| 139 | + | converted, err := resetToWire(m) | |
| 140 | + | if err != nil { | |
| 141 | + | return nil, err | |
| 142 | + | } | |
| 143 | + | return &wire.MuxClientEnvelope{Payload: &wire.MuxClientEnvelope_Reset_{Reset_: converted}}, nil | |
| 144 | + | case *Reset: | |
| 145 | + | if m == nil { | |
| 146 | + | return nil, ErrInvalidMessage | |
| 147 | + | } | |
| 148 | + | return encodeClientEnvelope(*m, maxChunkBytes) | |
| 149 | + | default: | |
| 150 | + | return nil, ErrWrongDirection | |
| 151 | + | } | |
| 152 | + | } | |
| 153 | + | ||
| 154 | + | func decodeClientEnvelope(envelope *wire.MuxClientEnvelope, maxChunkBytes uint64) (ClientMessage, error) { | |
| 155 | + | if envelope == nil { | |
| 156 | + | return nil, ErrInvalidMessage | |
| 157 | + | } | |
| 158 | + | switch payload := envelope.Payload.(type) { | |
| 159 | + | case *wire.MuxClientEnvelope_Open: | |
| 160 | + | if payload.Open == nil { | |
| 161 | + | return nil, ErrInvalidMessage | |
| 162 | + | } | |
| 163 | + | return openFromWire(payload.Open) | |
| 164 | + | case *wire.MuxClientEnvelope_Data: | |
| 165 | + | return dataFromWire(payload.Data, maxChunkBytes) | |
| 166 | + | case *wire.MuxClientEnvelope_Close: | |
| 167 | + | return closeFromWire(payload.Close) | |
| 168 | + | case *wire.MuxClientEnvelope_Reset_: | |
| 169 | + | return resetFromWire(payload.Reset_) | |
| 170 | + | default: | |
| 171 | + | return nil, ErrWrongDirection | |
| 172 | + | } | |
| 173 | + | } | |
| 174 | + | ||
| 175 | + | func encodeServerEnvelope(message ServerMessage, maxChunkBytes uint64) (*wire.MuxServerEnvelope, error) { | |
| 176 | + | if message == nil { | |
| 177 | + | return nil, ErrInvalidMessage | |
| 178 | + | } | |
| 179 | + | switch m := message.(type) { | |
| 180 | + | case Opened: | |
| 181 | + | if err := m.Ref.Validate(); err != nil { | |
| 182 | + | return nil, ErrInvalidMessage | |
| 183 | + | } | |
| 184 | + | return &wire.MuxServerEnvelope{Payload: &wire.MuxServerEnvelope_Opened{Opened: &wire.MuxOpened{ | |
| 185 | + | Ref: refToWire(m.Ref), | |
| 186 | + | }}}, nil | |
| 187 | + | case *Opened: | |
| 188 | + | if m == nil { | |
| 189 | + | return nil, ErrInvalidMessage | |
| 190 | + | } | |
| 191 | + | return encodeServerEnvelope(*m, maxChunkBytes) | |
| 192 | + | case Refused: | |
| 193 | + | converted, err := refusedToWire(m) | |
| 194 | + | if err != nil { | |
| 195 | + | return nil, err | |
| 196 | + | } | |
| 197 | + | return &wire.MuxServerEnvelope{Payload: &wire.MuxServerEnvelope_Refused{Refused: converted}}, nil | |
| 198 | + | case *Refused: | |
| 199 | + | if m == nil { | |
| 200 | + | return nil, ErrInvalidMessage | |
| 201 | + | } | |
| 202 | + | return encodeServerEnvelope(*m, maxChunkBytes) | |
| 203 | + | case Data: | |
| 204 | + | converted, err := dataToWire(m, maxChunkBytes) | |
| 205 | + | if err != nil { | |
| 206 | + | return nil, err | |
| 207 | + | } | |
| 208 | + | return &wire.MuxServerEnvelope{Payload: &wire.MuxServerEnvelope_Data{Data: converted}}, nil | |
| 209 | + | case *Data: | |
| 210 | + | if m == nil { | |
| 211 | + | return nil, ErrInvalidMessage | |
| 212 | + | } | |
| 213 | + | return encodeServerEnvelope(*m, maxChunkBytes) | |
| 214 | + | case Close: | |
| 215 | + | converted, err := closeToWire(m) | |
| 216 | + | if err != nil { | |
| 217 | + | return nil, err | |
| 218 | + | } | |
| 219 | + | return &wire.MuxServerEnvelope{Payload: &wire.MuxServerEnvelope_Close{Close: converted}}, nil | |
| 220 | + | case *Close: | |
| 221 | + | if m == nil { | |
| 222 | + | return nil, ErrInvalidMessage | |
| 223 | + | } | |
| 224 | + | return encodeServerEnvelope(*m, maxChunkBytes) | |
| 225 | + | case Reset: | |
| 226 | + | converted, err := resetToWire(m) | |
| 227 | + | if err != nil { | |
| 228 | + | return nil, err | |
| 229 | + | } | |
| 230 | + | return &wire.MuxServerEnvelope{Payload: &wire.MuxServerEnvelope_Reset_{Reset_: converted}}, nil | |
| 231 | + | case *Reset: | |
| 232 | + | if m == nil { | |
| 233 | + | return nil, ErrInvalidMessage | |
| 234 | + | } | |
| 235 | + | return encodeServerEnvelope(*m, maxChunkBytes) | |
| 236 | + | default: | |
| 237 | + | return nil, ErrWrongDirection | |
| 238 | + | } | |
| 239 | + | } | |
| 240 | + | ||
| 241 | + | func decodeServerEnvelope(envelope *wire.MuxServerEnvelope, maxChunkBytes uint64) (ServerMessage, error) { | |
| 242 | + | if envelope == nil { | |
| 243 | + | return nil, ErrInvalidMessage | |
| 244 | + | } | |
| 245 | + | switch payload := envelope.Payload.(type) { | |
| 246 | + | case *wire.MuxServerEnvelope_Opened: | |
| 247 | + | if payload.Opened == nil { | |
| 248 | + | return nil, ErrInvalidMessage | |
| 249 | + | } | |
| 250 | + | ref, err := refFromWire(payload.Opened.GetRef()) | |
| 251 | + | if err != nil { | |
| 252 | + | return nil, ErrInvalidMessage | |
| 253 | + | } | |
| 254 | + | return Opened{Ref: ref}, nil | |
| 255 | + | case *wire.MuxServerEnvelope_Refused: | |
| 256 | + | return refusedFromWire(payload.Refused) | |
| 257 | + | case *wire.MuxServerEnvelope_Data: | |
| 258 | + | return dataFromWire(payload.Data, maxChunkBytes) | |
| 259 | + | case *wire.MuxServerEnvelope_Close: | |
| 260 | + | return closeFromWire(payload.Close) | |
| 261 | + | case *wire.MuxServerEnvelope_Reset_: | |
| 262 | + | return resetFromWire(payload.Reset_) | |
| 263 | + | default: | |
| 264 | + | return nil, ErrWrongDirection | |
| 265 | + | } | |
| 266 | + | } | |
| 267 | + | ||
| 268 | + | func openToWire(m Open) (*wire.MuxOpen, error) { | |
| 269 | + | if err := m.Ref.Validate(); err != nil { | |
| 270 | + | return nil, ErrInvalidMessage | |
| 271 | + | } | |
| 272 | + | request := ports.BrokerOpenStreamRequest{ | |
| 273 | + | Epoch: m.Ref.Epoch, Purpose: m.Purpose, Local: m.Local, | |
| 274 | + | Connection: m.Ref.Connection, Stream: m.Ref.Client, | |
| 275 | + | Endpoint: m.Endpoint, Registration: m.Registration, | |
| 276 | + | Target: m.Target, Env: m.Env, Policy: m.Policy, | |
| 277 | + | } | |
| 278 | + | if err := request.Validate(); err != nil { | |
| 279 | + | // Distinguish bound refusals from semantic refusals. | |
| 280 | + | if len(m.Env) > ports.BrokerMaxEnvEntries { | |
| 281 | + | return nil, ErrTooLarge | |
| 282 | + | } | |
| 283 | + | for _, entry := range m.Env { | |
| 284 | + | if len(entry) > ports.BrokerMaxEnvEntryBytes { | |
| 285 | + | return nil, ErrTooLarge | |
| 286 | + | } | |
| 287 | + | } | |
| 288 | + | return nil, ErrInvalidMessage | |
| 289 | + | } | |
| 290 | + | if err := checkEnvEntries(m.Env); err != nil { | |
| 291 | + | return nil, err | |
| 292 | + | } | |
| 293 | + | purpose, err := purposeToWire(m.Purpose) | |
| 294 | + | if err != nil { | |
| 295 | + | return nil, err | |
| 296 | + | } | |
| 297 | + | out := &wire.MuxOpen{ | |
| 298 | + | Ref: refToWire(m.Ref), | |
| 299 | + | Purpose: purpose, Local: m.Local, Endpoint: m.Endpoint, | |
| 300 | + | Env: append([]string(nil), m.Env...), | |
| 301 | + | Policy: policyToWire(m.Policy), | |
| 302 | + | } | |
| 303 | + | if !m.Local { | |
| 304 | + | out.Registration = registrationToWire(m.Registration) | |
| 305 | + | } | |
| 306 | + | if m.Purpose == ports.BrokerStreamAttachment { | |
| 307 | + | out.Target = exactTargetToWire(m.Target) | |
| 308 | + | } | |
| 309 | + | return out, nil | |
| 310 | + | } | |
| 311 | + | ||
| 312 | + | func openFromWire(message *wire.MuxOpen) (Open, error) { | |
| 313 | + | var out Open | |
| 314 | + | if message == nil { | |
| 315 | + | return out, ErrInvalidMessage | |
| 316 | + | } | |
| 317 | + | ref, err := refFromWire(message.GetRef()) | |
| 318 | + | if err != nil { | |
| 319 | + | return Open{}, ErrInvalidMessage | |
| 320 | + | } | |
| 321 | + | purpose, err := purposeFromWire(message.GetPurpose()) | |
| 322 | + | if err != nil { | |
| 323 | + | return Open{}, err | |
| 324 | + | } | |
| 325 | + | local := message.GetLocal() | |
| 326 | + | var registration domain.RemoteRegistration | |
| 327 | + | if !local { | |
| 328 | + | registration, err = registrationFromWire(message.GetRegistration()) | |
| 329 | + | if err != nil { | |
| 330 | + | return Open{}, ErrInvalidMessage | |
| 331 | + | } | |
| 332 | + | } else if message.GetRegistration() != nil { | |
| 333 | + | return Open{}, ErrInvalidMessage | |
| 334 | + | } | |
| 335 | + | var target protocol.ExactSessionTarget | |
| 336 | + | if purpose == ports.BrokerStreamAttachment { | |
| 337 | + | target, err = exactTargetFromWire(message.GetTarget()) | |
| 338 | + | if err != nil { | |
| 339 | + | return Open{}, ErrInvalidMessage | |
| 340 | + | } | |
| 341 | + | } else if message.GetTarget() != nil { | |
| 342 | + | return Open{}, ErrInvalidMessage | |
| 343 | + | } | |
| 344 | + | env := append([]string(nil), message.GetEnv()...) | |
| 345 | + | if err := checkEnvEntries(env); err != nil { | |
| 346 | + | return Open{}, err | |
| 347 | + | } | |
| 348 | + | if purpose != ports.BrokerStreamAttachment && len(env) != 0 { | |
| 349 | + | return Open{}, ErrInvalidMessage | |
| 350 | + | } | |
| 351 | + | policy, err := policyFromWire(message.GetPolicy()) | |
| 352 | + | if err != nil { | |
| 353 | + | return Open{}, ErrInvalidMessage | |
| 354 | + | } | |
| 355 | + | endpoint := message.GetEndpoint() | |
| 356 | + | candidate := Open{ | |
| 357 | + | Ref: ref, Purpose: purpose, Local: local, Endpoint: endpoint, | |
| 358 | + | Registration: registration, Target: target, Env: env, Policy: policy, | |
| 359 | + | } | |
| 360 | + | request := ports.BrokerOpenStreamRequest{ | |
| 361 | + | Epoch: candidate.Ref.Epoch, Purpose: candidate.Purpose, Local: candidate.Local, | |
| 362 | + | Connection: candidate.Ref.Connection, Stream: candidate.Ref.Client, | |
| 363 | + | Endpoint: candidate.Endpoint, Registration: candidate.Registration, | |
| 364 | + | Target: candidate.Target, Env: candidate.Env, Policy: candidate.Policy, | |
| 365 | + | } | |
| 366 | + | if err := request.Validate(); err != nil { | |
| 367 | + | return Open{}, ErrInvalidMessage | |
| 368 | + | } | |
| 369 | + | return candidate, nil | |
| 370 | + | } | |
| 371 | + | ||
| 372 | + | func refusedToWire(m Refused) (*wire.MuxRefused, error) { | |
| 373 | + | if err := m.Ref.Validate(); err != nil { | |
| 374 | + | return nil, ErrInvalidMessage | |
| 375 | + | } | |
| 376 | + | if err := m.Error.validate(); err != nil { | |
| 377 | + | return nil, err | |
| 378 | + | } | |
| 379 | + | return &wire.MuxRefused{Ref: refToWire(m.Ref), Error: errorDetailToWire(m.Error)}, nil | |
| 380 | + | } | |
| 381 | + | ||
| 382 | + | func refusedFromWire(message *wire.MuxRefused) (Refused, error) { | |
| 383 | + | var out Refused | |
| 384 | + | if message == nil { | |
| 385 | + | return out, ErrInvalidMessage | |
| 386 | + | } | |
| 387 | + | ref, err := refFromWire(message.GetRef()) | |
| 388 | + | if err != nil { | |
| 389 | + | return Refused{}, ErrInvalidMessage | |
| 390 | + | } | |
| 391 | + | detail, err := errorDetailFromWire(message.GetError()) | |
| 392 | + | if err != nil { | |
| 393 | + | return Refused{}, ErrInvalidMessage | |
| 394 | + | } | |
| 395 | + | return Refused{Ref: ref, Error: detail}, nil | |
| 396 | + | } | |
| 397 | + | ||
| 398 | + | func dataToWire(m Data, maxChunkBytes uint64) (*wire.MuxData, error) { | |
| 399 | + | if err := m.Physical.Validate(); err != nil { | |
| 400 | + | return nil, ErrInvalidMessage | |
| 401 | + | } | |
| 402 | + | if uint64(len(m.Data)) > maxChunkBytes { | |
| 403 | + | return nil, ErrTooLarge | |
| 404 | + | } | |
| 405 | + | if uint64(len(m.Data)) > MaxMuxChunkBytes { | |
| 406 | + | return nil, ErrTooLarge | |
| 407 | + | } | |
| 408 | + | if len(m.Data) == 0 { | |
| 409 | + | return nil, ErrInvalidMessage | |
| 410 | + | } | |
| 411 | + | return &wire.MuxData{ | |
| 412 | + | PhysicalStreamId: physicalToWire(m.Physical), | |
| 413 | + | Data: append([]byte(nil), m.Data...), | |
| 414 | + | }, nil | |
| 415 | + | } | |
| 416 | + | ||
| 417 | + | func dataFromWire(message *wire.MuxData, maxChunkBytes uint64) (Data, error) { | |
| 418 | + | var out Data | |
| 419 | + | if message == nil { | |
| 420 | + | return out, ErrInvalidMessage | |
| 421 | + | } | |
| 422 | + | physical, err := physicalFromWire(message.GetPhysicalStreamId()) | |
| 423 | + | if err != nil { | |
| 424 | + | return Data{}, ErrInvalidMessage | |
| 425 | + | } | |
| 426 | + | data := message.GetData() | |
| 427 | + | if uint64(len(data)) > maxChunkBytes { | |
| 428 | + | return Data{}, ErrTooLarge | |
| 429 | + | } | |
| 430 | + | if uint64(len(data)) > MaxMuxChunkBytes { | |
| 431 | + | return Data{}, ErrTooLarge | |
| 432 | + | } | |
| 433 | + | if len(data) == 0 { | |
| 434 | + | return Data{}, ErrInvalidMessage | |
| 435 | + | } | |
| 436 | + | return Data{Physical: physical, Data: append([]byte(nil), data...)}, nil | |
| 437 | + | } | |
| 438 | + | ||
| 439 | + | func closeToWire(m Close) (*wire.MuxClose, error) { | |
| 440 | + | if err := m.Physical.Validate(); err != nil { | |
| 441 | + | return nil, ErrInvalidMessage | |
| 442 | + | } | |
| 443 | + | return &wire.MuxClose{PhysicalStreamId: physicalToWire(m.Physical)}, nil | |
| 444 | + | } | |
| 445 | + | ||
| 446 | + | func closeFromWire(message *wire.MuxClose) (Close, error) { | |
| 447 | + | var out Close | |
| 448 | + | if message == nil { | |
| 449 | + | return out, ErrInvalidMessage | |
| 450 | + | } | |
| 451 | + | physical, err := physicalFromWire(message.GetPhysicalStreamId()) | |
| 452 | + | if err != nil { | |
| 453 | + | return Close{}, ErrInvalidMessage | |
| 454 | + | } | |
| 455 | + | return Close{Physical: physical}, nil | |
| 456 | + | } | |
| 457 | + | ||
| 458 | + | func resetToWire(m Reset) (*wire.MuxReset, error) { | |
| 459 | + | if err := m.Physical.Validate(); err != nil { | |
| 460 | + | return nil, ErrInvalidMessage | |
| 461 | + | } | |
| 462 | + | if m.HasError { | |
| 463 | + | if err := m.Error.validate(); err != nil { | |
| 464 | + | return nil, err | |
| 465 | + | } | |
| 466 | + | } else if m.Error != (ErrorDetail{}) { | |
| 467 | + | // A reset without a declared detail carries no detail at all; a | |
| 468 | + | // nonzero one under HasError=false is a caller bug that must fail | |
| 469 | + | // closed rather than be silently dropped. | |
| 470 | + | return nil, ErrInvalidMessage | |
| 471 | + | } | |
| 472 | + | out := &wire.MuxReset{PhysicalStreamId: physicalToWire(m.Physical)} | |
| 473 | + | if m.HasError { | |
| 474 | + | out.Error = errorDetailToWire(m.Error) | |
| 475 | + | } | |
| 476 | + | return out, nil | |
| 477 | + | } | |
| 478 | + | ||
| 479 | + | func resetFromWire(message *wire.MuxReset) (Reset, error) { | |
| 480 | + | var out Reset | |
| 481 | + | if message == nil { | |
| 482 | + | return out, ErrInvalidMessage | |
| 483 | + | } | |
| 484 | + | physical, err := physicalFromWire(message.GetPhysicalStreamId()) | |
| 485 | + | if err != nil { | |
| 486 | + | return Reset{}, ErrInvalidMessage | |
| 487 | + | } | |
| 488 | + | result := Reset{Physical: physical} | |
| 489 | + | if message.GetError() != nil { | |
| 490 | + | detail, err := errorDetailFromWire(message.GetError()) | |
| 491 | + | if err != nil { | |
| 492 | + | return Reset{}, ErrInvalidMessage | |
| 493 | + | } | |
| 494 | + | result.Error = detail | |
| 495 | + | result.HasError = true | |
| 496 | + | } | |
| 497 | + | return result, nil | |
| 498 | + | } | |
| 499 | + | ||
| 500 | + | // purposeToWire maps the closed ports stream-purpose taxonomy onto the wire | |
| 501 | + | // taxonomy. The switch is explicit: an unknown purpose never defaults to a | |
| 502 | + | // wire value. | |
| 503 | + | func purposeToWire(purpose ports.BrokerStreamPurpose) (uint32, error) { | |
| 504 | + | switch purpose { | |
| 505 | + | case ports.BrokerStreamAttachment: | |
| 506 | + | return 1, nil | |
| 507 | + | case ports.BrokerStreamControl: | |
| 508 | + | return 2, nil | |
| 509 | + | case ports.BrokerStreamObservation: | |
| 510 | + | return 3, nil | |
| 511 | + | default: | |
| 512 | + | return 0, ErrInvalidMessage | |
| 513 | + | } | |
| 514 | + | } | |
| 515 | + | ||
| 516 | + | func purposeFromWire(value uint32) (ports.BrokerStreamPurpose, error) { | |
| 517 | + | switch value { | |
| 518 | + | case 1: | |
| 519 | + | return ports.BrokerStreamAttachment, nil | |
| 520 | + | case 2: | |
| 521 | + | return ports.BrokerStreamControl, nil | |
| 522 | + | case 3: | |
| 523 | + | return ports.BrokerStreamObservation, nil | |
| 524 | + | default: | |
| 525 | + | return 0, ErrInvalidMessage | |
| 526 | + | } | |
| 527 | + | } | |
A internal/adapters/daemonmux/codec_test.go +664 -0
| @@ -0,0 +1,664 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "bytes" | |
| 5 | + | "strings" | |
| 6 | + | "testing" | |
| 7 | + | ||
| 8 | + | "google.golang.org/protobuf/encoding/protowire" | |
| 9 | + | "google.golang.org/protobuf/proto" | |
| 10 | + | ||
| 11 | + | "github.com/bnema/vev/internal/domain" | |
| 12 | + | "github.com/bnema/vev/internal/ports" | |
| 13 | + | "github.com/bnema/vev/internal/protocol" | |
| 14 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 15 | + | "github.com/stretchr/testify/require" | |
| 16 | + | ) | |
| 17 | + | ||
| 18 | + | func testConnectionID(seed byte) ports.BrokerConnectionID { | |
| 19 | + | var id ports.BrokerConnectionID | |
| 20 | + | for i := range id { | |
| 21 | + | id[i] = seed + byte(i) | |
| 22 | + | } | |
| 23 | + | return id | |
| 24 | + | } | |
| 25 | + | ||
| 26 | + | func testRegistration() domain.RemoteRegistration { | |
| 27 | + | return domain.RemoteRegistration{ | |
| 28 | + | Endpoint: "dev@host:22", | |
| 29 | + | Incarnation: [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, | |
| 30 | + | Generation: 9, | |
| 31 | + | } | |
| 32 | + | } | |
| 33 | + | ||
| 34 | + | func testPolicy() ports.BrokerPolicy { | |
| 35 | + | return ports.BrokerPolicy{ | |
| 36 | + | ProtocolVersion: protocol.Version, | |
| 37 | + | CatalogSchemaVersion: 3, | |
| 38 | + | EnvironmentPolicy: protocol.EnvironmentPolicyDaemonOwned, | |
| 39 | + | Transport: "quic", | |
| 40 | + | Trust: "pinned", | |
| 41 | + | Launch: "agent", | |
| 42 | + | Isolation: "user", | |
| 43 | + | } | |
| 44 | + | } | |
| 45 | + | ||
| 46 | + | func testTarget() protocol.ExactSessionTarget { | |
| 47 | + | return protocol.ExactSessionTarget{ | |
| 48 | + | LifecycleID: domain.SessionLifecycleID{1, 2, 3}, | |
| 49 | + | SessionName: "work", | |
| 50 | + | } | |
| 51 | + | } | |
| 52 | + | ||
| 53 | + | func testRef(physical PhysicalStreamID) StreamRef { | |
| 54 | + | return StreamRef{ | |
| 55 | + | Physical: physical, | |
| 56 | + | Epoch: 7, | |
| 57 | + | Connection: testConnectionID(0x11), | |
| 58 | + | Client: 3, | |
| 59 | + | } | |
| 60 | + | } | |
| 61 | + | ||
| 62 | + | func testErrorDetail() ErrorDetail { | |
| 63 | + | return ErrorDetail{Code: ports.BrokerErrorUnavailable, Text: "dial refused", AdmissionCode: 1, FailureKind: domain.RemoteFailureTransport} | |
| 64 | + | } | |
| 65 | + | ||
| 66 | + | // unknownClientMessage and unknownServerMessage prove the directional codecs | |
| 67 | + | // fail closed on concrete types outside their closed union. | |
| 68 | + | type unknownClientMessage struct{} | |
| 69 | + | ||
| 70 | + | func (unknownClientMessage) muxClientMessage() {} | |
| 71 | + | ||
| 72 | + | type unknownServerMessage struct{} | |
| 73 | + | ||
| 74 | + | func (unknownServerMessage) muxServerMessage() {} | |
| 75 | + | ||
| 76 | + | const ( | |
| 77 | + | testEnvelopeCeiling = wire.AbsoluteEnvelopeLimit | |
| 78 | + | testChunkCeiling = MaxMuxChunkBytes | |
| 79 | + | ) | |
| 80 | + | ||
| 81 | + | func mustEncodeClient(t *testing.T, message ClientMessage) []byte { | |
| 82 | + | t.Helper() | |
| 83 | + | raw, err := EncodeClient(message, testEnvelopeCeiling, testChunkCeiling) | |
| 84 | + | require.NoError(t, err) | |
| 85 | + | return raw | |
| 86 | + | } | |
| 87 | + | ||
| 88 | + | func mustEncodeServer(t *testing.T, message ServerMessage) []byte { | |
| 89 | + | t.Helper() | |
| 90 | + | raw, err := EncodeServer(message, testEnvelopeCeiling, testChunkCeiling) | |
| 91 | + | require.NoError(t, err) | |
| 92 | + | return raw | |
| 93 | + | } | |
| 94 | + | ||
| 95 | + | func testOpen() Open { | |
| 96 | + | return Open{ | |
| 97 | + | Ref: testRef(5), Purpose: ports.BrokerStreamAttachment, Local: false, | |
| 98 | + | Endpoint: "dev@host:22", Registration: testRegistration(), Target: testTarget(), | |
| 99 | + | Env: []string{"TERM=xterm-256color"}, Policy: testPolicy(), | |
| 100 | + | } | |
| 101 | + | } | |
| 102 | + | ||
| 103 | + | // TestMuxClientVariants inventories every client variant tag and proves each | |
| 104 | + | // one round-trips through encode and decode. | |
| 105 | + | func TestMuxClientVariants(t *testing.T) { | |
| 106 | + | messages := []struct { | |
| 107 | + | name string | |
| 108 | + | message ClientMessage | |
| 109 | + | tag protowire.Number | |
| 110 | + | }{ | |
| 111 | + | {"open", testOpen(), 301}, | |
| 112 | + | {"data", Data{Physical: 5, Data: []byte("frame")}, 302}, | |
| 113 | + | {"close", Close{Physical: 5}, 303}, | |
| 114 | + | {"reset", Reset{Physical: 5, Error: testErrorDetail(), HasError: true}, 304}, | |
| 115 | + | {"reset_orderly", Reset{Physical: 5}, 304}, | |
| 116 | + | } | |
| 117 | + | for _, tc := range messages { | |
| 118 | + | t.Run(tc.name, func(t *testing.T) { | |
| 119 | + | raw := mustEncodeClient(t, tc.message) | |
| 120 | + | number, wireType, consumed := protowire.ConsumeTag(raw) | |
| 121 | + | require.Greater(t, consumed, 0) | |
| 122 | + | require.Equal(t, tc.tag, number) | |
| 123 | + | require.Equal(t, protowire.BytesType, wireType) | |
| 124 | + | decoded, err := DecodeClient(raw, testEnvelopeCeiling, testChunkCeiling) | |
| 125 | + | require.NoError(t, err) | |
| 126 | + | require.Equal(t, tc.message, decoded) | |
| 127 | + | }) | |
| 128 | + | } | |
| 129 | + | } | |
| 130 | + | ||
| 131 | + | // TestMuxServerVariants inventories every server variant tag and proves each | |
| 132 | + | // one round-trips through encode and decode. | |
| 133 | + | func TestMuxServerVariants(t *testing.T) { | |
| 134 | + | messages := []struct { | |
| 135 | + | name string | |
| 136 | + | message ServerMessage | |
| 137 | + | tag protowire.Number | |
| 138 | + | }{ | |
| 139 | + | {"opened", Opened{Ref: testRef(5)}, 401}, | |
| 140 | + | {"refused", Refused{Ref: testRef(5), Error: testErrorDetail()}, 402}, | |
| 141 | + | {"data", Data{Physical: 5, Data: []byte("frame")}, 403}, | |
| 142 | + | {"close", Close{Physical: 5}, 404}, | |
| 143 | + | {"reset", Reset{Physical: 5, Error: testErrorDetail(), HasError: true}, 405}, | |
| 144 | + | {"reset_orderly", Reset{Physical: 5}, 405}, | |
| 145 | + | } | |
| 146 | + | for _, tc := range messages { | |
| 147 | + | t.Run(tc.name, func(t *testing.T) { | |
| 148 | + | raw := mustEncodeServer(t, tc.message) | |
| 149 | + | number, wireType, consumed := protowire.ConsumeTag(raw) | |
| 150 | + | require.Greater(t, consumed, 0) | |
| 151 | + | require.Equal(t, tc.tag, number) | |
| 152 | + | require.Equal(t, protowire.BytesType, wireType) | |
| 153 | + | decoded, err := DecodeServer(raw, testEnvelopeCeiling, testChunkCeiling) | |
| 154 | + | require.NoError(t, err) | |
| 155 | + | require.Equal(t, tc.message, decoded) | |
| 156 | + | }) | |
| 157 | + | } | |
| 158 | + | } | |
| 159 | + | ||
| 160 | + | // TestMuxSharedPayloadDirections proves the shared Data/Close/Reset payloads | |
| 161 | + | // convert into both directions: the same value produces the direction's own | |
| 162 | + | // tag and decodes back identically on either side. | |
| 163 | + | func TestMuxSharedPayloadDirections(t *testing.T) { | |
| 164 | + | pairs := []struct { | |
| 165 | + | name string | |
| 166 | + | client ClientMessage | |
| 167 | + | server ServerMessage | |
| 168 | + | clientTag protowire.Number | |
| 169 | + | serverTag protowire.Number | |
| 170 | + | expectedSame any | |
| 171 | + | }{ | |
| 172 | + | {"data", Data{Physical: 5, Data: []byte("frame")}, Data{Physical: 5, Data: []byte("frame")}, 302, 403, Data{Physical: 5, Data: []byte("frame")}}, | |
| 173 | + | {"close", Close{Physical: 5}, Close{Physical: 5}, 303, 404, Close{Physical: 5}}, | |
| 174 | + | {"reset", Reset{Physical: 5}, Reset{Physical: 5}, 304, 405, Reset{Physical: 5}}, | |
| 175 | + | } | |
| 176 | + | for _, tc := range pairs { | |
| 177 | + | t.Run(tc.name, func(t *testing.T) { | |
| 178 | + | clientRaw := mustEncodeClient(t, tc.client) | |
| 179 | + | clientTag, _, _ := protowire.ConsumeTag(clientRaw) | |
| 180 | + | require.Equal(t, tc.clientTag, clientTag) | |
| 181 | + | clientDecoded, err := DecodeClient(clientRaw, testEnvelopeCeiling, testChunkCeiling) | |
| 182 | + | require.NoError(t, err) | |
| 183 | + | require.Equal(t, tc.expectedSame, clientDecoded) | |
| 184 | + | ||
| 185 | + | serverRaw := mustEncodeServer(t, tc.server) | |
| 186 | + | serverTag, _, _ := protowire.ConsumeTag(serverRaw) | |
| 187 | + | require.Equal(t, tc.serverTag, serverTag) | |
| 188 | + | serverDecoded, err := DecodeServer(serverRaw, testEnvelopeCeiling, testChunkCeiling) | |
| 189 | + | require.NoError(t, err) | |
| 190 | + | require.Equal(t, tc.expectedSame, serverDecoded) | |
| 191 | + | }) | |
| 192 | + | } | |
| 193 | + | } | |
| 194 | + | ||
| 195 | + | // TestMuxGoldenVectors pins literal byte encodings where the wire layout is | |
| 196 | + | // stable: tag, length, and one varint field. | |
| 197 | + | func TestMuxGoldenVectors(t *testing.T) { | |
| 198 | + | vectors := []struct { | |
| 199 | + | name string | |
| 200 | + | message ClientMessage | |
| 201 | + | want []byte | |
| 202 | + | }{ | |
| 203 | + | {"close", Close{Physical: 1}, []byte{0xFA, 0x12, 0x02, 0x08, 0x01}}, | |
| 204 | + | {"data", Data{Physical: 1, Data: []byte("hi")}, []byte{0xF2, 0x12, 0x06, 0x08, 0x01, 0x12, 0x02, 0x68, 0x69}}, | |
| 205 | + | {"reset_orderly", Reset{Physical: 1}, []byte{0x82, 0x13, 0x02, 0x08, 0x01}}, | |
| 206 | + | } | |
| 207 | + | for _, tc := range vectors { | |
| 208 | + | t.Run(tc.name, func(t *testing.T) { | |
| 209 | + | raw := mustEncodeClient(t, tc.message) | |
| 210 | + | require.Equal(t, tc.want, raw) | |
| 211 | + | // Re-encode is deterministic: the same message marshals | |
| 212 | + | // identically, so a receiver can rely on a stable byte layout. | |
| 213 | + | again := mustEncodeClient(t, tc.message) | |
| 214 | + | require.Equal(t, tc.want, again) | |
| 215 | + | decoded, err := DecodeClient(tc.want, testEnvelopeCeiling, testChunkCeiling) | |
| 216 | + | require.NoError(t, err) | |
| 217 | + | require.Equal(t, tc.message, decoded) | |
| 218 | + | }) | |
| 219 | + | } | |
| 220 | + | ||
| 221 | + | // The server direction has its own tags: the same value never produces | |
| 222 | + | // the client bytes. | |
| 223 | + | serverRaw := mustEncodeServer(t, Close{Physical: 1}) | |
| 224 | + | require.Equal(t, []byte{0xA2, 0x19, 0x02, 0x08, 0x01}, serverRaw) | |
| 225 | + | ||
| 226 | + | // A populated ref is stable too: MuxOpened carries the four-field ref in | |
| 227 | + | // numeric order. | |
| 228 | + | ref := StreamRef{Physical: 1, Epoch: 1, Connection: testConnectionID(1), Client: 1} | |
| 229 | + | opened := mustEncodeServer(t, Opened{Ref: ref}) | |
| 230 | + | back, err := DecodeServer(opened, testEnvelopeCeiling, testChunkCeiling) | |
| 231 | + | require.NoError(t, err) | |
| 232 | + | require.Equal(t, Opened{Ref: ref}, back) | |
| 233 | + | } | |
| 234 | + | ||
| 235 | + | // TestMuxBounds proves stateless bound refusals at max-1, max, and max+1. | |
| 236 | + | func TestMuxBounds(t *testing.T) { | |
| 237 | + | t.Run("envelope ceiling", func(t *testing.T) { | |
| 238 | + | raw := mustEncodeClient(t, Close{Physical: 1}) | |
| 239 | + | _, err := DecodeClient(raw, testEnvelopeCeiling, testChunkCeiling) | |
| 240 | + | require.NoError(t, err) | |
| 241 | + | // One byte below the exact serialized length refuses the envelope. | |
| 242 | + | _, err = DecodeClient(raw, uint64(len(raw)-1), testChunkCeiling) | |
| 243 | + | require.ErrorIs(t, err, wire.ErrScanLength) | |
| 244 | + | // At the ceiling the envelope decodes; one byte above the ceiling is | |
| 245 | + | // refused before scanning. | |
| 246 | + | _, err = DecodeClient(raw, uint64(len(raw)), testChunkCeiling) | |
| 247 | + | require.NoError(t, err) | |
| 248 | + | _, err = DecodeClient(make([]byte, MaxMuxEnvelopeBytes+1), MaxMuxEnvelopeBytes, testChunkCeiling) | |
| 249 | + | require.ErrorIs(t, err, wire.ErrScanLength) | |
| 250 | + | // Encoding above the local ceiling fails closed. | |
| 251 | + | _, err = EncodeClient(Close{Physical: 1}, uint64(len(raw)-1), testChunkCeiling) | |
| 252 | + | require.ErrorIs(t, err, wire.ErrScanLength) | |
| 253 | + | _, err = EncodeClient(Close{Physical: 1}, uint64(len(raw)), testChunkCeiling) | |
| 254 | + | require.NoError(t, err) | |
| 255 | + | }) | |
| 256 | + | t.Run("application envelope ceiling is 1 MiB", func(t *testing.T) { | |
| 257 | + | require.Equal(t, uint64(1<<20), MaxMuxApplicationBytes) | |
| 258 | + | require.Equal(t, uint64(1<<20), MinMuxEnvelopeBytes) | |
| 259 | + | // The fixed application ceiling applies at max and refuses at max+1, | |
| 260 | + | // whatever the negotiated receive ceiling advertises. | |
| 261 | + | require.NoError(t, checkMuxEnvelopeCeiling(make([]byte, MaxMuxApplicationBytes), testEnvelopeCeiling)) | |
| 262 | + | require.ErrorIs(t, checkMuxEnvelopeCeiling(make([]byte, MaxMuxApplicationBytes+1), testEnvelopeCeiling), wire.ErrScanLength) | |
| 263 | + | require.ErrorIs(t, checkMuxEnvelopeCeiling(make([]byte, MaxMuxApplicationBytes+1), MaxMuxEnvelopeBytes), wire.ErrScanLength) | |
| 264 | + | // A hostile open envelope above the application ceiling is refused | |
| 265 | + | // before scan even though the negotiated ceiling admits 16 MiB. | |
| 266 | + | env := make([]string, 400) | |
| 267 | + | for i := range env { | |
| 268 | + | env[i] = "K=" + strings.Repeat("x", ports.BrokerMaxEnvEntryBytes-2) | |
| 269 | + | } | |
| 270 | + | hostile := &wire.MuxClientEnvelope{Payload: &wire.MuxClientEnvelope_Open{Open: &wire.MuxOpen{ | |
| 271 | + | Ref: &wire.MuxStreamRef{PhysicalStreamId: 1, BrokerEpoch: 1, ConnectionId: make([]byte, 16), ClientStreamId: 1}, | |
| 272 | + | Purpose: 1, Local: true, Env: env, | |
| 273 | + | }}} | |
| 274 | + | payload, err := proto.Marshal(hostile) | |
| 275 | + | require.NoError(t, err) | |
| 276 | + | require.Greater(t, len(payload), int(MaxMuxApplicationBytes)) | |
| 277 | + | _, err = DecodeClient(payload, testEnvelopeCeiling, testChunkCeiling) | |
| 278 | + | require.ErrorIs(t, err, wire.ErrScanLength) | |
| 279 | + | }) | |
| 280 | + | t.Run("chunk window 1..64KiB", func(t *testing.T) { | |
| 281 | + | require.Equal(t, uint64(64<<10), MaxMuxChunkBytes) | |
| 282 | + | atMax := bytes.Repeat([]byte{0x66}, int(MaxMuxChunkBytes)) | |
| 283 | + | belowMax := bytes.Repeat([]byte{0x66}, int(MaxMuxChunkBytes)-1) | |
| 284 | + | aboveMax := bytes.Repeat([]byte{0x66}, int(MaxMuxChunkBytes)+1) | |
| 285 | + | for _, data := range [][]byte{belowMax, atMax} { | |
| 286 | + | clientRaw, err := EncodeClient(Data{Physical: 5, Data: data}, testEnvelopeCeiling, testChunkCeiling) | |
| 287 | + | require.NoError(t, err) | |
| 288 | + | clientDecoded, err := DecodeClient(clientRaw, testEnvelopeCeiling, testChunkCeiling) | |
| 289 | + | require.NoError(t, err) | |
| 290 | + | require.Equal(t, data, clientDecoded.(Data).Data) | |
| 291 | + | ||
| 292 | + | serverRaw, err := EncodeServer(Data{Physical: 5, Data: data}, testEnvelopeCeiling, testChunkCeiling) | |
| 293 | + | require.NoError(t, err) | |
| 294 | + | serverDecoded, err := DecodeServer(serverRaw, testEnvelopeCeiling, testChunkCeiling) | |
| 295 | + | require.NoError(t, err) | |
| 296 | + | require.Equal(t, data, serverDecoded.(Data).Data) | |
| 297 | + | } | |
| 298 | + | _, err := EncodeClient(Data{Physical: 5, Data: aboveMax}, testEnvelopeCeiling, testChunkCeiling) | |
| 299 | + | require.ErrorIs(t, err, ErrTooLarge) | |
| 300 | + | _, err = EncodeServer(Data{Physical: 5, Data: aboveMax}, testEnvelopeCeiling, testChunkCeiling) | |
| 301 | + | require.ErrorIs(t, err, ErrTooLarge) | |
| 302 | + | // A negotiated chunk below the absolute maximum also refuses. | |
| 303 | + | raw := mustEncodeClient(t, Data{Physical: 5, Data: []byte("12345678")}) | |
| 304 | + | _, err = DecodeClient(raw, testEnvelopeCeiling, 7) | |
| 305 | + | require.ErrorIs(t, err, ErrTooLarge) | |
| 306 | + | _, err = DecodeClient(raw, testEnvelopeCeiling, 8) | |
| 307 | + | require.NoError(t, err) | |
| 308 | + | // Empty chunk is invalid, not too large. | |
| 309 | + | _, err = EncodeClient(Data{Physical: 5}, testEnvelopeCeiling, testChunkCeiling) | |
| 310 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 311 | + | }) | |
| 312 | + | t.Run("environment entries", func(t *testing.T) { | |
| 313 | + | atMax := "K=" + strings.Repeat("x", ports.BrokerMaxEnvEntryBytes-2) | |
| 314 | + | require.Len(t, atMax, ports.BrokerMaxEnvEntryBytes) | |
| 315 | + | aboveMax := atMax + "x" | |
| 316 | + | message := testOpen() | |
| 317 | + | message.Env = []string{atMax} | |
| 318 | + | _, err := EncodeClient(message, testEnvelopeCeiling, testChunkCeiling) | |
| 319 | + | require.NoError(t, err) | |
| 320 | + | message.Env = []string{aboveMax} | |
| 321 | + | _, err = EncodeClient(message, testEnvelopeCeiling, testChunkCeiling) | |
| 322 | + | require.ErrorIs(t, err, ErrTooLarge) | |
| 323 | + | ||
| 324 | + | many := make([]string, ports.BrokerMaxEnvEntries) | |
| 325 | + | for i := range many { | |
| 326 | + | many[i] = "A=B" | |
| 327 | + | } | |
| 328 | + | message = testOpen() | |
| 329 | + | message.Env = many | |
| 330 | + | _, err = EncodeClient(message, testEnvelopeCeiling, testChunkCeiling) | |
| 331 | + | require.NoError(t, err) | |
| 332 | + | message.Env = append(append([]string(nil), many...), "C=D") | |
| 333 | + | _, err = EncodeClient(message, testEnvelopeCeiling, testChunkCeiling) | |
| 334 | + | require.ErrorIs(t, err, ErrTooLarge) | |
| 335 | + | }) | |
| 336 | + | t.Run("error text", func(t *testing.T) { | |
| 337 | + | atMax := testErrorDetail() | |
| 338 | + | atMax.Text = strings.Repeat("e", ports.BrokerMaxErrorBytes) | |
| 339 | + | _, err := EncodeServer(Refused{Ref: testRef(5), Error: atMax}, testEnvelopeCeiling, testChunkCeiling) | |
| 340 | + | require.NoError(t, err) | |
| 341 | + | over := testErrorDetail() | |
| 342 | + | over.Text = strings.Repeat("e", ports.BrokerMaxErrorBytes+1) | |
| 343 | + | _, err = EncodeServer(Refused{Ref: testRef(5), Error: over}, testEnvelopeCeiling, testChunkCeiling) | |
| 344 | + | require.ErrorIs(t, err, ErrTooLarge) | |
| 345 | + | }) | |
| 346 | + | } | |
| 347 | + | ||
| 348 | + | // TestMuxValidationNegatives is the table of semantic refusals: stream | |
| 349 | + | // identity, taxonomy, text, environment, policy, and error details. | |
| 350 | + | func TestMuxValidationNegatives(t *testing.T) { | |
| 351 | + | t.Run("zero stream identities refused", func(t *testing.T) { | |
| 352 | + | _, err := EncodeClient(Close{}, testEnvelopeCeiling, testChunkCeiling) | |
| 353 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 354 | + | _, err = EncodeClient(Data{Physical: 0, Data: []byte("x")}, testEnvelopeCeiling, testChunkCeiling) | |
| 355 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 356 | + | _, err = EncodeServer(Reset{Physical: 0}, testEnvelopeCeiling, testChunkCeiling) | |
| 357 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 358 | + | ||
| 359 | + | zeroEpoch := testRef(5) | |
| 360 | + | zeroEpoch.Epoch = 0 | |
| 361 | + | _, err = EncodeServer(Opened{Ref: zeroEpoch}, testEnvelopeCeiling, testChunkCeiling) | |
| 362 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 363 | + | ||
| 364 | + | zeroConnection := testRef(5) | |
| 365 | + | zeroConnection.Connection = ports.BrokerConnectionID{} | |
| 366 | + | _, err = EncodeServer(Opened{Ref: zeroConnection}, testEnvelopeCeiling, testChunkCeiling) | |
| 367 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 368 | + | ||
| 369 | + | zeroClient := testRef(5) | |
| 370 | + | zeroClient.Client = 0 | |
| 371 | + | _, err = EncodeServer(Opened{Ref: zeroClient}, testEnvelopeCeiling, testChunkCeiling) | |
| 372 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 373 | + | ||
| 374 | + | _, err = EncodeClient(Open{Ref: zeroClient, Purpose: ports.BrokerStreamControl, Local: true, Policy: testPolicy()}, testEnvelopeCeiling, testChunkCeiling) | |
| 375 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 376 | + | }) | |
| 377 | + | t.Run("open taxonomy and shape", func(t *testing.T) { | |
| 378 | + | badPurpose := testOpen() | |
| 379 | + | badPurpose.Purpose = ports.BrokerStreamPurpose(9) | |
| 380 | + | _, err := EncodeClient(badPurpose, testEnvelopeCeiling, testChunkCeiling) | |
| 381 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 382 | + | ||
| 383 | + | badEndpoint := testOpen() | |
| 384 | + | badEndpoint.Endpoint = "not a host" | |
| 385 | + | _, err = EncodeClient(badEndpoint, testEnvelopeCeiling, testChunkCeiling) | |
| 386 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 387 | + | ||
| 388 | + | local := testOpen() | |
| 389 | + | local.Local = true | |
| 390 | + | _, err = EncodeClient(local, testEnvelopeCeiling, testChunkCeiling) | |
| 391 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 392 | + | ||
| 393 | + | controlWithTarget := testOpen() | |
| 394 | + | controlWithTarget.Purpose = ports.BrokerStreamControl | |
| 395 | + | _, err = EncodeClient(controlWithTarget, testEnvelopeCeiling, testChunkCeiling) | |
| 396 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 397 | + | ||
| 398 | + | controlWithEnv := testOpen() | |
| 399 | + | controlWithEnv.Purpose = ports.BrokerStreamControl | |
| 400 | + | controlWithEnv.Target = protocol.ExactSessionTarget{} | |
| 401 | + | _, err = EncodeClient(controlWithEnv, testEnvelopeCeiling, testChunkCeiling) | |
| 402 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 403 | + | ||
| 404 | + | noEquals := testOpen() | |
| 405 | + | noEquals.Env = []string{"NOEQUALS"} | |
| 406 | + | _, err = EncodeClient(noEquals, testEnvelopeCeiling, testChunkCeiling) | |
| 407 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 408 | + | ||
| 409 | + | emptyPolicy := testOpen() | |
| 410 | + | emptyPolicy.Policy = ports.BrokerPolicy{} | |
| 411 | + | _, err = EncodeClient(emptyPolicy, testEnvelopeCeiling, testChunkCeiling) | |
| 412 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 413 | + | ||
| 414 | + | // A local control open with no attachment state is legal. | |
| 415 | + | valid := Open{Ref: testRef(5), Purpose: ports.BrokerStreamControl, Local: true, Policy: testPolicy()} | |
| 416 | + | raw, err := EncodeClient(valid, testEnvelopeCeiling, testChunkCeiling) | |
| 417 | + | require.NoError(t, err) | |
| 418 | + | decoded, err := DecodeClient(raw, testEnvelopeCeiling, testChunkCeiling) | |
| 419 | + | require.NoError(t, err) | |
| 420 | + | require.Equal(t, valid, decoded) | |
| 421 | + | }) | |
| 422 | + | t.Run("taxonomy switches refuse mutated wire values", func(t *testing.T) { | |
| 423 | + | raw := mustEncodeClient(t, testOpen()) | |
| 424 | + | envelope := &wire.MuxClientEnvelope{} | |
| 425 | + | require.NoError(t, wire.ScanEnvelope(envelope, raw)) | |
| 426 | + | require.NoError(t, proto.Unmarshal(raw, envelope)) | |
| 427 | + | envelope.GetOpen().Purpose = 9 | |
| 428 | + | mutated, err := proto.Marshal(envelope) | |
| 429 | + | require.NoError(t, err) | |
| 430 | + | _, err = DecodeClient(mutated, testEnvelopeCeiling, testChunkCeiling) | |
| 431 | + | require.Error(t, err) | |
| 432 | + | ||
| 433 | + | rawErr := mustEncodeServer(t, Refused{Ref: testRef(5), Error: testErrorDetail()}) | |
| 434 | + | server := &wire.MuxServerEnvelope{} | |
| 435 | + | require.NoError(t, wire.ScanEnvelope(server, rawErr)) | |
| 436 | + | require.NoError(t, proto.Unmarshal(rawErr, server)) | |
| 437 | + | server.GetRefused().GetError().Code = 99 | |
| 438 | + | mutated, err = proto.Marshal(server) | |
| 439 | + | require.NoError(t, err) | |
| 440 | + | _, err = DecodeServer(mutated, testEnvelopeCeiling, testChunkCeiling) | |
| 441 | + | require.Error(t, err) | |
| 442 | + | }) | |
| 443 | + | t.Run("error detail is mandatory on refusal", func(t *testing.T) { | |
| 444 | + | _, err := EncodeServer(Refused{Ref: testRef(5)}, testEnvelopeCeiling, testChunkCeiling) | |
| 445 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 446 | + | // The zero detail has no valid code, so it can never travel. | |
| 447 | + | _, err = EncodeServer(Refused{Ref: testRef(5), Error: ErrorDetail{AdmissionCode: 1}}, testEnvelopeCeiling, testChunkCeiling) | |
| 448 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 449 | + | }) | |
| 450 | + | t.Run("reset never drops a declared detail", func(t *testing.T) { | |
| 451 | + | for name, stray := range map[string]ErrorDetail{ | |
| 452 | + | "full detail": testErrorDetail(), | |
| 453 | + | "code only": {Code: ports.BrokerErrorTimeout}, | |
| 454 | + | "text only": {Text: "late"}, | |
| 455 | + | "admission only": {AdmissionCode: 2}, | |
| 456 | + | "failure kind only": {FailureKind: domain.RemoteFailureTransport}, | |
| 457 | + | } { | |
| 458 | + | t.Run(name, func(t *testing.T) { | |
| 459 | + | _, err := EncodeClient(Reset{Physical: 5, Error: stray}, testEnvelopeCeiling, testChunkCeiling) | |
| 460 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 461 | + | _, err = EncodeServer(Reset{Physical: 5, Error: stray}, testEnvelopeCeiling, testChunkCeiling) | |
| 462 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 463 | + | }) | |
| 464 | + | } | |
| 465 | + | // The zero detail is the canonical absent value and still encodes; a | |
| 466 | + | // declared detail still round-trips exactly. | |
| 467 | + | _, err := EncodeServer(Reset{Physical: 5}, testEnvelopeCeiling, testChunkCeiling) | |
| 468 | + | require.NoError(t, err) | |
| 469 | + | declared := Reset{Physical: 5, Error: testErrorDetail(), HasError: true} | |
| 470 | + | decoded, err := DecodeServer(mustEncodeServer(t, declared), testEnvelopeCeiling, testChunkCeiling) | |
| 471 | + | require.NoError(t, err) | |
| 472 | + | require.Equal(t, declared, decoded) | |
| 473 | + | }) | |
| 474 | + | t.Run("invalid error detail refused on every path", func(t *testing.T) { | |
| 475 | + | badAdmission := testErrorDetail() | |
| 476 | + | badAdmission.AdmissionCode = MaxAdmissionCode + 1 | |
| 477 | + | badKind := testErrorDetail() | |
| 478 | + | badKind.FailureKind = MaxFailureKind + 1 | |
| 479 | + | badCode := testErrorDetail() | |
| 480 | + | badCode.Code = 0 | |
| 481 | + | for name, detail := range map[string]ErrorDetail{ | |
| 482 | + | "admission over maximum": badAdmission, | |
| 483 | + | "failure kind over bound": badKind, | |
| 484 | + | "invalid error code": badCode, | |
| 485 | + | } { | |
| 486 | + | t.Run(name, func(t *testing.T) { | |
| 487 | + | _, err := EncodeServer(Refused{Ref: testRef(5), Error: detail}, testEnvelopeCeiling, testChunkCeiling) | |
| 488 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 489 | + | _, err = EncodeServer(Reset{Physical: 5, Error: detail, HasError: true}, testEnvelopeCeiling, testChunkCeiling) | |
| 490 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 491 | + | _, err = EncodeClient(Reset{Physical: 5, Error: detail, HasError: true}, testEnvelopeCeiling, testChunkCeiling) | |
| 492 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 493 | + | }) | |
| 494 | + | } | |
| 495 | + | // The closed taxonomy boundaries are accepted. | |
| 496 | + | boundary := testErrorDetail() | |
| 497 | + | boundary.AdmissionCode = MaxAdmissionCode | |
| 498 | + | boundary.FailureKind = MaxFailureKind | |
| 499 | + | _, err := EncodeServer(Refused{Ref: testRef(5), Error: boundary}, testEnvelopeCeiling, testChunkCeiling) | |
| 500 | + | require.NoError(t, err) | |
| 501 | + | }) | |
| 502 | + | t.Run("bidi and control text refused", func(t *testing.T) { | |
| 503 | + | evil := testErrorDetail() | |
| 504 | + | evil.Text = "timeout\u202erorrE" | |
| 505 | + | _, err := EncodeServer(Refused{Ref: testRef(5), Error: evil}, testEnvelopeCeiling, testChunkCeiling) | |
| 506 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 507 | + | evil.Text = "bad\x01text" | |
| 508 | + | _, err = EncodeServer(Refused{Ref: testRef(5), Error: evil}, testEnvelopeCeiling, testChunkCeiling) | |
| 509 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 510 | + | ||
| 511 | + | badEnv := testOpen() | |
| 512 | + | badEnv.Env = []string{"K=\u202eevil"} | |
| 513 | + | _, err = EncodeClient(badEnv, testEnvelopeCeiling, testChunkCeiling) | |
| 514 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 515 | + | }) | |
| 516 | + | t.Run("nil messages refused", func(t *testing.T) { | |
| 517 | + | var openNil *Open | |
| 518 | + | _, err := EncodeClient(openNil, testEnvelopeCeiling, testChunkCeiling) | |
| 519 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 520 | + | var dataNil *Data | |
| 521 | + | _, err = EncodeServer(dataNil, testEnvelopeCeiling, testChunkCeiling) | |
| 522 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 523 | + | var refusedNil *Refused | |
| 524 | + | _, err = EncodeServer(refusedNil, testEnvelopeCeiling, testChunkCeiling) | |
| 525 | + | require.ErrorIs(t, err, ErrInvalidMessage) | |
| 526 | + | }) | |
| 527 | + | } | |
| 528 | + | ||
| 529 | + | // TestMuxTruncatedAndTrailing proves truncated prefixes and trailing garbage | |
| 530 | + | // are rejected on both directions. | |
| 531 | + | func TestMuxTruncatedAndTrailing(t *testing.T) { | |
| 532 | + | raw := mustEncodeClient(t, testOpen()) | |
| 533 | + | for size := range len(raw) { | |
| 534 | + | _, err := DecodeClient(raw[:size], testEnvelopeCeiling, testChunkCeiling) | |
| 535 | + | require.Error(t, err, "client prefix[:%d] accepted", size) | |
| 536 | + | } | |
| 537 | + | _, err := DecodeClient(append(append([]byte(nil), raw...), 0xFF), testEnvelopeCeiling, testChunkCeiling) | |
| 538 | + | require.Error(t, err) | |
| 539 | + | ||
| 540 | + | serverRaw := mustEncodeServer(t, Opened{Ref: testRef(5)}) | |
| 541 | + | for size := range len(serverRaw) { | |
| 542 | + | _, err := DecodeServer(serverRaw[:size], testEnvelopeCeiling, testChunkCeiling) | |
| 543 | + | require.Error(t, err, "server prefix[:%d] accepted", size) | |
| 544 | + | } | |
| 545 | + | concatenated := append(append([]byte(nil), serverRaw...), serverRaw...) | |
| 546 | + | _, err = DecodeServer(concatenated, testEnvelopeCeiling, testChunkCeiling) | |
| 547 | + | require.Error(t, err) | |
| 548 | + | } | |
| 549 | + | ||
| 550 | + | // TestMuxWrongDirection proves wrong-direction decode is rejected: client | |
| 551 | + | // bytes never decode as daemon output and vice versa. | |
| 552 | + | func TestMuxWrongDirection(t *testing.T) { | |
| 553 | + | clientRaw := mustEncodeClient(t, Close{Physical: 1}) | |
| 554 | + | _, err := DecodeServer(clientRaw, testEnvelopeCeiling, testChunkCeiling) | |
| 555 | + | require.Error(t, err) | |
| 556 | + | ||
| 557 | + | serverRaw := mustEncodeServer(t, Close{Physical: 1}) | |
| 558 | + | _, err = DecodeClient(serverRaw, testEnvelopeCeiling, testChunkCeiling) | |
| 559 | + | require.Error(t, err) | |
| 560 | + | ||
| 561 | + | // Go-typed misuse fails with ErrWrongDirection at encode time: unknown | |
| 562 | + | // concrete types presented to either directional codec fail closed. | |
| 563 | + | _, err = encodeClientEnvelope(unknownClientMessage{}, testChunkCeiling) | |
| 564 | + | require.ErrorIs(t, err, ErrWrongDirection) | |
| 565 | + | _, err = encodeServerEnvelope(unknownServerMessage{}, testChunkCeiling) | |
| 566 | + | require.ErrorIs(t, err, ErrWrongDirection) | |
| 567 | + | ||
| 568 | + | // Nil-payload envelopes decode as wrong-direction. | |
| 569 | + | _, err = decodeClientEnvelope(&wire.MuxClientEnvelope{}, testChunkCeiling) | |
| 570 | + | require.ErrorIs(t, err, ErrWrongDirection) | |
| 571 | + | _, err = decodeServerEnvelope(&wire.MuxServerEnvelope{}, testChunkCeiling) | |
| 572 | + | require.ErrorIs(t, err, ErrWrongDirection) | |
| 573 | + | } | |
| 574 | + | ||
| 575 | + | // TestMuxCorruptInputSmoke feeds representative corruptions through both | |
| 576 | + | // decoders as a smoke test: bit flips, truncation, unknown fields, and | |
| 577 | + | // direction swaps must never panic and must never be accepted silently. | |
| 578 | + | func TestMuxCorruptInputSmoke(t *testing.T) { | |
| 579 | + | seeds := [][]byte{ | |
| 580 | + | mustEncodeClient(t, testOpen()), | |
| 581 | + | mustEncodeClient(t, Data{Physical: 5, Data: []byte("frame")}), | |
| 582 | + | mustEncodeClient(t, Close{Physical: 5}), | |
| 583 | + | mustEncodeServer(t, Opened{Ref: testRef(5)}), | |
| 584 | + | mustEncodeServer(t, Refused{Ref: testRef(5), Error: testErrorDetail()}), | |
| 585 | + | mustEncodeServer(t, Reset{Physical: 5, Error: testErrorDetail(), HasError: true}), | |
| 586 | + | } | |
| 587 | + | for _, seed := range seeds { | |
| 588 | + | mutated := append([]byte(nil), seed...) | |
| 589 | + | if len(mutated) > 2 { | |
| 590 | + | mutated[len(mutated)-1] ^= 0xFF | |
| 591 | + | mutated[len(mutated)-2] ^= 0x01 | |
| 592 | + | } | |
| 593 | + | _, _ = DecodeClient(mutated, testEnvelopeCeiling, testChunkCeiling) | |
| 594 | + | _, _ = DecodeServer(mutated, testEnvelopeCeiling, testChunkCeiling) | |
| 595 | + | unknown := protowire.AppendTag(append([]byte(nil), seed...), 399, protowire.VarintType) | |
| 596 | + | unknown = protowire.AppendVarint(unknown, 1) | |
| 597 | + | _, _ = DecodeClient(unknown, testEnvelopeCeiling, testChunkCeiling) | |
| 598 | + | _, _ = DecodeServer(unknown, testEnvelopeCeiling, testChunkCeiling) | |
| 599 | + | } | |
| 600 | + | require.NotEmpty(t, seeds) | |
| 601 | + | } | |
| 602 | + | ||
| 603 | + | // appendStrictScanSeeds adds raw vectors that must be refused by the strict | |
| 604 | + | // scanner rather than reaching generated unmarshal. | |
| 605 | + | func appendStrictScanSeeds(seeds [][]byte) [][]byte { | |
| 606 | + | unknown := protowire.AppendTag(nil, 399, protowire.VarintType) | |
| 607 | + | unknown = protowire.AppendVarint(unknown, 1) | |
| 608 | + | wrongType := protowire.AppendTag(nil, 301, protowire.VarintType) | |
| 609 | + | wrongType = protowire.AppendVarint(wrongType, 1) | |
| 610 | + | return append(seeds, unknown, wrongType, []byte{0xFF}, []byte{0x80}, nil) | |
| 611 | + | } | |
| 612 | + | ||
| 613 | + | func fuzzClientSeeds(t testing.TB) [][]byte { | |
| 614 | + | t.Helper() | |
| 615 | + | seeds := [][]byte{} | |
| 616 | + | for _, message := range []ClientMessage{ | |
| 617 | + | testOpen(), | |
| 618 | + | Data{Physical: 5, Data: []byte("frame")}, | |
| 619 | + | Close{Physical: 5}, | |
| 620 | + | Reset{Physical: 5, Error: testErrorDetail(), HasError: true}, | |
| 621 | + | } { | |
| 622 | + | raw, err := EncodeClient(message, testEnvelopeCeiling, testChunkCeiling) | |
| 623 | + | require.NoError(t, err) | |
| 624 | + | seeds = append(seeds, raw) | |
| 625 | + | } | |
| 626 | + | return appendStrictScanSeeds(seeds) | |
| 627 | + | } | |
| 628 | + | ||
| 629 | + | func fuzzServerSeeds(t testing.TB) [][]byte { | |
| 630 | + | t.Helper() | |
| 631 | + | seeds := [][]byte{} | |
| 632 | + | for _, message := range []ServerMessage{ | |
| 633 | + | Opened{Ref: testRef(5)}, | |
| 634 | + | Refused{Ref: testRef(5), Error: testErrorDetail()}, | |
| 635 | + | Data{Physical: 5, Data: []byte("frame")}, | |
| 636 | + | Close{Physical: 5}, | |
| 637 | + | Reset{Physical: 5, Error: testErrorDetail(), HasError: true}, | |
| 638 | + | } { | |
| 639 | + | raw, err := EncodeServer(message, testEnvelopeCeiling, testChunkCeiling) | |
| 640 | + | require.NoError(t, err) | |
| 641 | + | seeds = append(seeds, raw) | |
| 642 | + | } | |
| 643 | + | return appendStrictScanSeeds(seeds) | |
| 644 | + | } | |
| 645 | + | ||
| 646 | + | // FuzzDecodeClient proves DecodeClient never panics on arbitrary input. | |
| 647 | + | func FuzzDecodeClient(f *testing.F) { | |
| 648 | + | for _, seed := range fuzzClientSeeds(f) { | |
| 649 | + | f.Add(seed) | |
| 650 | + | } | |
| 651 | + | f.Fuzz(func(t *testing.T, payload []byte) { | |
| 652 | + | _, _ = DecodeClient(payload, testEnvelopeCeiling, testChunkCeiling) | |
| 653 | + | }) | |
| 654 | + | } | |
| 655 | + | ||
| 656 | + | // FuzzDecodeServer proves DecodeServer never panics on arbitrary input. | |
| 657 | + | func FuzzDecodeServer(f *testing.F) { | |
| 658 | + | for _, seed := range fuzzServerSeeds(f) { | |
| 659 | + | f.Add(seed) | |
| 660 | + | } | |
| 661 | + | f.Fuzz(func(t *testing.T, payload []byte) { | |
| 662 | + | _, _ = DecodeServer(payload, testEnvelopeCeiling, testChunkCeiling) | |
| 663 | + | }) | |
| 664 | + | } | |
A internal/adapters/daemonmux/connector.go +307 -0
| @@ -0,0 +1,307 @@ | |||
| 1 | + | // Broker endpoint connector and physical connection (P3.2c). | |
| 2 | + | // | |
| 3 | + | // EndpointConnector is the broker-side bridge between the transport-neutral | |
| 4 | + | // ports.BrokerEndpointConnector contract and the daemonmux multiplexer. It owns | |
| 5 | + | // the setup half of one pooled physical connection: dial one already | |
| 6 | + | // authenticated raw framed carriage for the resolved endpoint, run the | |
| 7 | + | // daemonmux physical preamble over the FramedCarrierBridge, negotiate the | |
| 8 | + | // effective ceilings, and hand the live pump and logical connector to a | |
| 9 | + | // PhysicalConnection. | |
| 10 | + | // | |
| 11 | + | // The dial function is injected, not chosen here: the connector never knows | |
| 12 | + | // whether the carriage is IPC, QUIC, SSH stdio, or a test pipe, and it never | |
| 13 | + | // authenticates it. The carriage must already be authenticated before this | |
| 14 | + | // connector sees it; the resolved endpoint's identity and policy are the | |
| 15 | + | // authoritative values the handshake verifies the daemon's response against. | |
| 16 | + | // | |
| 17 | + | // One caller-supplied setup context (or its deadline) bounds dial and handshake | |
| 18 | + | // as a single unit: the same context goes to the dial function and to | |
| 19 | + | // RunClientHandshake, and no timer is created here. A successful connection is | |
| 20 | + | // deliberately detached from that setup context: the pump is started on a | |
| 21 | + | // context that preserves the setup values but not its cancellation, so a caller | |
| 22 | + | // that cancels or lets its setup deadline elapse after Connect returned never | |
| 23 | + | // tears down a pooled physical connection. The connection's lifetime is owned | |
| 24 | + | // by PhysicalConnection.Close. | |
| 25 | + | // | |
| 26 | + | // PhysicalConnection is the broker-side ports.BrokerPhysicalConnection over | |
| 27 | + | // that live pump. Its Identity, Incarnation, and Policy are immutable values | |
| 28 | + | // captured from the verified handshake; Done, Err, and FailureKind delegate to | |
| 29 | + | // the pump's publish-once terminal authority, so a physical loss closes Done | |
| 30 | + | // before every logical stream is terminalized; OpenStream exact-checks the | |
| 31 | + | // request policy against the accepted physical policy and then defers to the | |
| 32 | + | // LogicalConnector; Close tears the pump (and therefore the carrier) down | |
| 33 | + | // exactly once. | |
| 34 | + | package daemonmux | |
| 35 | + | ||
| 36 | + | import ( | |
| 37 | + | "context" | |
| 38 | + | "errors" | |
| 39 | + | "fmt" | |
| 40 | + | "sync" | |
| 41 | + | ||
| 42 | + | "github.com/bnema/vev/internal/domain" | |
| 43 | + | "github.com/bnema/vev/internal/ports" | |
| 44 | + | ) | |
| 45 | + | ||
| 46 | + | // ErrConnectorConfig reports an endpoint connector constructed without a dial | |
| 47 | + | // function or with an invalid ceiling advertisement, or a Connect call with an | |
| 48 | + | // invalid resolved endpoint. It is a caller bug, not a peer refusal. | |
| 49 | + | var ErrConnectorConfig = errors.New("daemonmux: invalid endpoint connector configuration") | |
| 50 | + | ||
| 51 | + | // RawCarrierDialer establishes one already authenticated raw framed carriage | |
| 52 | + | // for an opaque endpoint address. It returns a carriage the daemonmux | |
| 53 | + | // FramedCarrierBridge can adapt; it returns promptly once ctx is done. The | |
| 54 | + | // carriage's authentication and transport selection live entirely outside this | |
| 55 | + | // package, so the connector carries no dial policy of its own. | |
| 56 | + | type RawCarrierDialer func(ctx context.Context, address string) (RawFramedTransport, error) | |
| 57 | + | ||
| 58 | + | // EndpointConnector implements ports.BrokerEndpointConnector over daemonmux. It | |
| 59 | + | // owns no carriage state itself: every Connect produces one independent | |
| 60 | + | // PhysicalConnection that owns its own pump, carrier, and logical connector. | |
| 61 | + | // It is safe for concurrent use. | |
| 62 | + | type EndpointConnector struct { | |
| 63 | + | dial RawCarrierDialer | |
| 64 | + | ceilings MuxCeilings | |
| 65 | + | } | |
| 66 | + | ||
| 67 | + | var _ ports.BrokerEndpointConnector = (*EndpointConnector)(nil) | |
| 68 | + | ||
| 69 | + | // NewEndpointConnector returns a connector that dials carriage through dial and | |
| 70 | + | // advertises ceilings in the daemonmux physical preamble. A nil dial or an | |
| 71 | + | // invalid ceiling advertisement is refused with ErrConnectorConfig; the | |
| 72 | + | // ceilings are validated and copied, so a later change to the caller's value | |
| 73 | + | // never changes what a connection negotiates. | |
| 74 | + | func NewEndpointConnector(dial RawCarrierDialer, ceilings MuxCeilings) (*EndpointConnector, error) { | |
| 75 | + | if dial == nil { | |
| 76 | + | return nil, ErrConnectorConfig | |
| 77 | + | } | |
| 78 | + | if err := ceilings.Validate(); err != nil { | |
| 79 | + | return nil, fmt.Errorf("%w: %w", ErrConnectorConfig, err) | |
| 80 | + | } | |
| 81 | + | return &EndpointConnector{dial: dial, ceilings: ceilings}, nil | |
| 82 | + | } | |
| 83 | + | ||
| 84 | + | // Connect dials the resolved endpoint, completes the daemonmux physical | |
| 85 | + | // preamble against the endpoint's authoritative identity and policy, and | |
| 86 | + | // returns a live ports.BrokerPhysicalConnection. The caller's ctx (or its | |
| 87 | + | // deadline) bounds dial and handshake together; every failure closes the | |
| 88 | + | // carriage, so no half-open physical connection survives. A successful | |
| 89 | + | // connection is detached from ctx: cancelling or expiring ctx afterwards never | |
| 90 | + | // stops the pooled physical connection, which Close alone owns. | |
| 91 | + | func (c *EndpointConnector) Connect(ctx context.Context, endpoint ports.BrokerResolvedEndpoint) (ports.BrokerPhysicalConnection, error) { | |
| 92 | + | if c == nil || c.dial == nil { | |
| 93 | + | return nil, ErrConnectorConfig | |
| 94 | + | } | |
| 95 | + | if ctx == nil { | |
| 96 | + | ctx = context.Background() | |
| 97 | + | } | |
| 98 | + | if err := endpoint.Validate(); err != nil { | |
| 99 | + | return nil, ports.BrokerError{Code: ports.BrokerErrorIncompatible, Cause: err} | |
| 100 | + | } | |
| 101 | + | ||
| 102 | + | raw, err := c.dial(ctx, endpoint.Address) | |
| 103 | + | if err != nil { | |
| 104 | + | if raw != nil { | |
| 105 | + | _ = raw.Close() | |
| 106 | + | } | |
| 107 | + | return nil, dialError(err) | |
| 108 | + | } | |
| 109 | + | if raw == nil { | |
| 110 | + | return nil, ports.BrokerError{Code: ports.BrokerErrorIncompatible, Cause: ErrCarrierState} | |
| 111 | + | } | |
| 112 | + | ||
| 113 | + | bridge, err := NewPreambleCarrier(raw) | |
| 114 | + | if err != nil { | |
| 115 | + | _ = raw.Close() | |
| 116 | + | return nil, ports.BrokerError{Code: ports.BrokerErrorIncompatible, Cause: err} | |
| 117 | + | } | |
| 118 | + | ||
| 119 | + | result, err := RunClientHandshake(ctx, bridge, endpoint, c.ceilings) | |
| 120 | + | if err != nil { | |
| 121 | + | // The handshake already closed a refused carriage; Close is idempotent. | |
| 122 | + | _ = bridge.Close() | |
| 123 | + | return nil, handshakeError(err) | |
| 124 | + | } | |
| 125 | + | // The endpoint stays authoritative: re-verify the accepted authority even | |
| 126 | + | // though the handshake already checked it, so a future handshake change can | |
| 127 | + | // never weaken what the connector publishes. | |
| 128 | + | if result.Identity != endpoint.Identity || !result.Policy.Compatible(endpoint.Policy) || result.Incarnation.Validate() != nil { | |
| 129 | + | _ = bridge.Close() | |
| 130 | + | return nil, ports.BrokerError{Code: ports.BrokerErrorIncompatible} | |
| 131 | + | } | |
| 132 | + | if err := bridge.Negotiate(result.Ceilings); err != nil { | |
| 133 | + | _ = bridge.Close() | |
| 134 | + | return nil, ports.BrokerError{Code: ports.BrokerErrorIncompatible, Cause: err} | |
| 135 | + | } | |
| 136 | + | ||
| 137 | + | pump, err := NewPump(bridge, DirectionServer, result.Ceilings) | |
| 138 | + | if err != nil { | |
| 139 | + | _ = bridge.Close() | |
| 140 | + | return nil, ports.BrokerError{Code: ports.BrokerErrorIncompatible, Cause: err} | |
| 141 | + | } | |
| 142 | + | logical, err := NewLogicalConnector(pump) | |
| 143 | + | if err != nil { | |
| 144 | + | _ = pump.Close() | |
| 145 | + | return nil, ports.BrokerError{Code: ports.BrokerErrorIncompatible, Cause: err} | |
| 146 | + | } | |
| 147 | + | ||
| 148 | + | physical := &PhysicalConnection{ | |
| 149 | + | pump: pump, | |
| 150 | + | logical: logical, | |
| 151 | + | identity: result.Identity, | |
| 152 | + | incarnation: result.Incarnation, | |
| 153 | + | policy: result.Policy, | |
| 154 | + | } | |
| 155 | + | // The setup context ends here. WithoutCancel keeps the setup values while | |
| 156 | + | // removing its cancellation and deadline, so Close - not the caller's setup | |
| 157 | + | // context - ends the pooled connection. | |
| 158 | + | pump.Start(context.WithoutCancel(ctx)) | |
| 159 | + | return physical, nil | |
| 160 | + | } | |
| 161 | + | ||
| 162 | + | // dialError classifies one dial failure: cancellation and deadline stay | |
| 163 | + | // recognizable for the pool's precedence, a validated broker error passes | |
| 164 | + | // through, and anything else becomes an unavailable adapter failure with its | |
| 165 | + | // cause retained locally. | |
| 166 | + | func dialError(err error) error { | |
| 167 | + | if err == nil { | |
| 168 | + | return nil | |
| 169 | + | } | |
| 170 | + | if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { | |
| 171 | + | return err | |
| 172 | + | } | |
| 173 | + | var typed ports.BrokerError | |
| 174 | + | if errors.As(err, &typed) && typed.Validate() == nil { | |
| 175 | + | return err | |
| 176 | + | } | |
| 177 | + | return ports.BrokerError{Code: ports.BrokerErrorUnavailable, Cause: err} | |
| 178 | + | } | |
| 179 | + | ||
| 180 | + | // handshakeError classifies one handshake failure: cancellation and deadline | |
| 181 | + | // stay recognizable, a peer refusal is a policy conflict, a local authority | |
| 182 | + | // mismatch is an incompatibility, and anything else is unavailable. | |
| 183 | + | func handshakeError(err error) error { | |
| 184 | + | if err == nil { | |
| 185 | + | return nil | |
| 186 | + | } | |
| 187 | + | if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { | |
| 188 | + | return err | |
| 189 | + | } | |
| 190 | + | var refusal *HandshakeRefusal | |
| 191 | + | if errors.As(err, &refusal) { | |
| 192 | + | return ports.BrokerError{Code: ports.BrokerErrorConflictingPolicy, Cause: err} | |
| 193 | + | } | |
| 194 | + | if errors.Is(err, ErrHandshakeRejected) { | |
| 195 | + | return ports.BrokerError{Code: ports.BrokerErrorIncompatible, Cause: err} | |
| 196 | + | } | |
| 197 | + | if errors.Is(err, ErrHandshakeConfig) { | |
| 198 | + | return ports.BrokerError{Code: ports.BrokerErrorIncompatible, Cause: err} | |
| 199 | + | } | |
| 200 | + | return ports.BrokerError{Code: ports.BrokerErrorUnavailable, Cause: err} | |
| 201 | + | } | |
| 202 | + | ||
| 203 | + | // PhysicalConnection is one pooled physical daemonmux connection: the live | |
| 204 | + | // pump, its logical connector, and the immutable daemon authority the physical | |
| 205 | + | // preamble verified. It implements ports.BrokerPhysicalConnection and is safe | |
| 206 | + | // for concurrent use. | |
| 207 | + | type PhysicalConnection struct { | |
| 208 | + | pump *Pump | |
| 209 | + | logical *LogicalConnector | |
| 210 | + | ||
| 211 | + | identity ports.BrokerDaemonIdentity | |
| 212 | + | incarnation ports.BrokerDaemonIncarnation | |
| 213 | + | policy ports.BrokerPolicy | |
| 214 | + | ||
| 215 | + | closeOnce sync.Once | |
| 216 | + | closeErr error | |
| 217 | + | } | |
| 218 | + | ||
| 219 | + | var _ ports.BrokerPhysicalConnection = (*PhysicalConnection)(nil) | |
| 220 | + | ||
| 221 | + | // Done returns the pump's terminal channel: closed exactly once, before every | |
| 222 | + | // logical stream of the connection is terminalized. | |
| 223 | + | func (c *PhysicalConnection) Done() <-chan struct{} { | |
| 224 | + | if c == nil || c.pump == nil { | |
| 225 | + | return nil | |
| 226 | + | } | |
| 227 | + | return c.pump.Done() | |
| 228 | + | } | |
| 229 | + | ||
| 230 | + | // Err returns the pump's terminal cause: nil while open and for an orderly | |
| 231 | + | // local Close. | |
| 232 | + | func (c *PhysicalConnection) Err() error { | |
| 233 | + | if c == nil || c.pump == nil { | |
| 234 | + | return nil | |
| 235 | + | } | |
| 236 | + | return c.pump.Err() | |
| 237 | + | } | |
| 238 | + | ||
| 239 | + | // FailureKind returns the pump's terminal classification: RemoteFailureNone | |
| 240 | + | // while open and for an orderly local Close. | |
| 241 | + | func (c *PhysicalConnection) FailureKind() domain.RemoteFailureKind { | |
| 242 | + | if c == nil || c.pump == nil { | |
| 243 | + | return domain.RemoteFailureNone | |
| 244 | + | } | |
| 245 | + | return c.pump.FailureKind() | |
| 246 | + | } | |
| 247 | + | ||
| 248 | + | // Identity returns the authenticated daemon identity the handshake verified | |
| 249 | + | // against the resolved endpoint. It is immutable for the connection's lifetime. | |
| 250 | + | func (c *PhysicalConnection) Identity() ports.BrokerDaemonIdentity { | |
| 251 | + | if c == nil { | |
| 252 | + | return "" | |
| 253 | + | } | |
| 254 | + | return c.identity | |
| 255 | + | } | |
| 256 | + | ||
| 257 | + | // Incarnation returns the daemon process incarnation captured at handshake: a | |
| 258 | + | // stable identity across restarts carries a fresh, nonzero incarnation. | |
| 259 | + | func (c *PhysicalConnection) Incarnation() ports.BrokerDaemonIncarnation { | |
| 260 | + | if c == nil { | |
| 261 | + | return ports.BrokerDaemonIncarnation{} | |
| 262 | + | } | |
| 263 | + | return c.incarnation | |
| 264 | + | } | |
| 265 | + | ||
| 266 | + | // Policy returns the exact accepted connection policy this physical connection | |
| 267 | + | // was pooled under. It is immutable for the connection's lifetime. | |
| 268 | + | func (c *PhysicalConnection) Policy() ports.BrokerPolicy { | |
| 269 | + | if c == nil { | |
| 270 | + | return ports.BrokerPolicy{} | |
| 271 | + | } | |
| 272 | + | return c.policy | |
| 273 | + | } | |
| 274 | + | ||
| 275 | + | // OpenStream exact-checks one open request against the accepted physical | |
| 276 | + | // policy and then opens one independently cancellable logical stream through | |
| 277 | + | // the LogicalConnector. A policy that differs in any field is refused as a | |
| 278 | + | // conflicting policy without touching the carriage, so a physical connection | |
| 279 | + | // can never multiplex a stream under a policy it did not negotiate. | |
| 280 | + | func (c *PhysicalConnection) OpenStream(ctx context.Context, request ports.BrokerOpenStreamRequest) (ports.BrokerLogicalConnection, error) { | |
| 281 | + | if c == nil || c.logical == nil { | |
| 282 | + | return nil, ports.BrokerError{Code: ports.BrokerErrorIncompatible, Cause: ErrLogicalConfig} | |
| 283 | + | } | |
| 284 | + | if err := request.Validate(); err != nil { | |
| 285 | + | return nil, ports.BrokerError{Code: ports.BrokerErrorIncompatible, Cause: errors.Join(ports.BrokerAdmissionInvalid, err)} | |
| 286 | + | } | |
| 287 | + | if !request.Policy.Compatible(c.policy) { | |
| 288 | + | return nil, ports.BrokerError{Code: ports.BrokerErrorConflictingPolicy} | |
| 289 | + | } | |
| 290 | + | return c.logical.Open(ctx, request) | |
| 291 | + | } | |
| 292 | + | ||
| 293 | + | // Close tears the physical connection down exactly once: it closes the pump, | |
| 294 | + | // which cancels both carriage goroutines and closes the carrier through the | |
| 295 | + | // adapter prompt-close contract. Every caller observes the same carrier close | |
| 296 | + | // error. Close is concurrent-safe and prompt. | |
| 297 | + | func (c *PhysicalConnection) Close() error { | |
| 298 | + | if c == nil { | |
| 299 | + | return nil | |
| 300 | + | } | |
| 301 | + | c.closeOnce.Do(func() { | |
| 302 | + | if c.pump != nil { | |
| 303 | + | c.closeErr = c.pump.Close() | |
| 304 | + | } | |
| 305 | + | }) | |
| 306 | + | return c.closeErr | |
| 307 | + | } | |
A internal/adapters/daemonmux/connector_test.go +384 -0
| @@ -0,0 +1,384 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "net" | |
| 6 | + | "sync" | |
| 7 | + | "sync/atomic" | |
| 8 | + | "testing" | |
| 9 | + | "time" | |
| 10 | + | ||
| 11 | + | "github.com/stretchr/testify/require" | |
| 12 | + | ||
| 13 | + | "github.com/bnema/vev/internal/adapters/clock" | |
| 14 | + | "github.com/bnema/vev/internal/adapters/ipc" | |
| 15 | + | "github.com/bnema/vev/internal/domain" | |
| 16 | + | "github.com/bnema/vev/internal/ports" | |
| 17 | + | "github.com/bnema/vev/internal/protocol" | |
| 18 | + | ) | |
| 19 | + | ||
| 20 | + | // p3cTestDeadline bounds every p3c integration wait. It is deliberately | |
| 21 | + | // generous wall time: the only timing under test is a real carriage loss or an | |
| 22 | + | // admission handshake, never a handshake budget. | |
| 23 | + | const p3cTestDeadline = 5 * time.Second | |
| 24 | + | ||
| 25 | + | // rawCarriagePair returns two independent real raw framed transports joined by | |
| 26 | + | // an in-memory duplex pipe. Each end frames through the shared streamframe | |
| 27 | + | // carriage via the IPC adapter, so the bridge, physical preamble, pump, and | |
| 28 | + | // sessionwire all run over real framing rather than an in-memory envelope | |
| 29 | + | // channel. | |
| 30 | + | func rawCarriagePair(t *testing.T) (RawFramedTransport, RawFramedTransport) { | |
| 31 | + | t.Helper() | |
| 32 | + | client, server := net.Pipe() | |
| 33 | + | t.Cleanup(func() { | |
| 34 | + | _ = client.Close() | |
| 35 | + | _ = server.Close() | |
| 36 | + | }) | |
| 37 | + | return rawFramedEnd(t, client), rawFramedEnd(t, server) | |
| 38 | + | } | |
| 39 | + | ||
| 40 | + | // rawFramedEnd adapts one net.Conn to the raw framed carriage the bridge and | |
| 41 | + | // supervisor consume. | |
| 42 | + | func rawFramedEnd(t *testing.T, conn net.Conn) RawFramedTransport { | |
| 43 | + | t.Helper() | |
| 44 | + | raw, ok := ipc.NewTransport(conn).(RawFramedTransport) | |
| 45 | + | require.True(t, ok, "ipc transport must satisfy the raw framed carriage") | |
| 46 | + | return raw | |
| 47 | + | } | |
| 48 | + | ||
| 49 | + | // alternatePolicy returns a valid policy that differs from base in exactly one | |
| 50 | + | // field, so a mismatch is never a validation failure. | |
| 51 | + | func alternatePolicy(base ports.BrokerPolicy) ports.BrokerPolicy { | |
| 52 | + | other := base | |
| 53 | + | other.Transport = base.Transport + "-alt" | |
| 54 | + | return other | |
| 55 | + | } | |
| 56 | + | ||
| 57 | + | // muxOpenRequest builds one valid local control-purpose open request under | |
| 58 | + | // policy, so a client can open a stream without a remote registration. | |
| 59 | + | func muxOpenRequest(stream uint64, policy ports.BrokerPolicy) ports.BrokerOpenStreamRequest { | |
| 60 | + | return ports.BrokerOpenStreamRequest{ | |
| 61 | + | Epoch: 1, | |
| 62 | + | Purpose: ports.BrokerStreamControl, | |
| 63 | + | Local: true, | |
| 64 | + | Connection: ports.BrokerConnectionID{1}, | |
| 65 | + | Stream: ports.BrokerStreamID(stream), | |
| 66 | + | Policy: policy, | |
| 67 | + | } | |
| 68 | + | } | |
| 69 | + | ||
| 70 | + | // muxEndpoint builds the resolved endpoint the broker verifies against. | |
| 71 | + | func muxEndpoint(identity ports.BrokerDaemonIdentity, policy ports.BrokerPolicy, address string) ports.BrokerResolvedEndpoint { | |
| 72 | + | return ports.BrokerResolvedEndpoint{Identity: identity, Policy: policy, Address: address} | |
| 73 | + | } | |
| 74 | + | ||
| 75 | + | // superviseServer is a test daemon: an AggregateListener plus a | |
| 76 | + | // ServerSupervisor, with an optional accept loop that records and serves every | |
| 77 | + | // typed connection the aggregate delivers. Its dial method produces the broker | |
| 78 | + | // end of one real carriage and hands the daemon end to the supervisor. | |
| 79 | + | type superviseServer struct { | |
| 80 | + | t *testing.T | |
| 81 | + | aggregate *AggregateListener | |
| 82 | + | supervisor *ServerSupervisor | |
| 83 | + | dials atomic.Int64 | |
| 84 | + | adoptions chan error | |
| 85 | + | ||
| 86 | + | mu sync.Mutex | |
| 87 | + | accepted []ports.ServerConnection | |
| 88 | + | serveOne sync.Once | |
| 89 | + | } | |
| 90 | + | ||
| 91 | + | func newSuperviseServer(t *testing.T, binding ServerBinding, ceilings MuxCeilings, limit int, timeSource ports.Clock) *superviseServer { | |
| 92 | + | t.Helper() | |
| 93 | + | if ceilings == (MuxCeilings{}) { | |
| 94 | + | ceilings = DefaultMuxCeilings() | |
| 95 | + | } | |
| 96 | + | if timeSource == nil { | |
| 97 | + | timeSource = clock.New() | |
| 98 | + | } | |
| 99 | + | aggregate := NewAggregateListener() | |
| 100 | + | supervisor, err := newServerSupervisor(aggregate, binding, ceilings, limit, timeSource) | |
| 101 | + | require.NoError(t, err) | |
| 102 | + | t.Cleanup(func() { | |
| 103 | + | _ = supervisor.Close() | |
| 104 | + | _ = aggregate.Close() | |
| 105 | + | }) | |
| 106 | + | return &superviseServer{ | |
| 107 | + | t: t, | |
| 108 | + | aggregate: aggregate, | |
| 109 | + | supervisor: supervisor, | |
| 110 | + | adoptions: make(chan error, 16), | |
| 111 | + | } | |
| 112 | + | } | |
| 113 | + | ||
| 114 | + | // dial returns a RawCarrierDialer that produces one fresh real carriage per | |
| 115 | + | // call and adopts its daemon end. | |
| 116 | + | func (s *superviseServer) dial() RawCarrierDialer { | |
| 117 | + | return func(_ context.Context, _ string) (RawFramedTransport, error) { | |
| 118 | + | s.dials.Add(1) | |
| 119 | + | client, server := rawCarriagePair(s.t) | |
| 120 | + | go func() { s.adoptions <- s.supervisor.Adopt(context.Background(), server) }() | |
| 121 | + | return client, nil | |
| 122 | + | } | |
| 123 | + | } | |
| 124 | + | ||
| 125 | + | // adopt hands one already built carriage to the supervisor and reports the | |
| 126 | + | // outcome through the shared adoption channel. | |
| 127 | + | func (s *superviseServer) adopt(ctx context.Context, server RawFramedTransport) { | |
| 128 | + | go func() { s.adoptions <- s.supervisor.Adopt(ctx, server) }() | |
| 129 | + | } | |
| 130 | + | ||
| 131 | + | // awaitAdoption returns the next Adopt outcome. | |
| 132 | + | func (s *superviseServer) awaitAdoption(t *testing.T) error { | |
| 133 | + | t.Helper() | |
| 134 | + | select { | |
| 135 | + | case err := <-s.adoptions: | |
| 136 | + | return err | |
| 137 | + | case <-time.After(p3cTestDeadline): | |
| 138 | + | t.Fatal("supervisor: Adopt did not return") | |
| 139 | + | return nil | |
| 140 | + | } | |
| 141 | + | } | |
| 142 | + | ||
| 143 | + | // serve starts the accept loop: every delivered typed connection is recorded | |
| 144 | + | // and answered with Pong per Ping until its stream ends. | |
| 145 | + | func (s *superviseServer) serve() { | |
| 146 | + | s.serveOne.Do(func() { | |
| 147 | + | go func() { | |
| 148 | + | for { | |
| 149 | + | connection, err := s.aggregate.Accept() | |
| 150 | + | if err != nil { | |
| 151 | + | return | |
| 152 | + | } | |
| 153 | + | s.mu.Lock() | |
| 154 | + | s.accepted = append(s.accepted, connection) | |
| 155 | + | s.mu.Unlock() | |
| 156 | + | go func() { _ = servePongs(connection) }() | |
| 157 | + | } | |
| 158 | + | }() | |
| 159 | + | }) | |
| 160 | + | } | |
| 161 | + | ||
| 162 | + | // acceptedCount reports how many typed connections the aggregate delivered. | |
| 163 | + | func (s *superviseServer) acceptedCount() int { | |
| 164 | + | s.mu.Lock() | |
| 165 | + | defer s.mu.Unlock() | |
| 166 | + | return len(s.accepted) | |
| 167 | + | } | |
| 168 | + | ||
| 169 | + | func (s *superviseServer) awaitAccepted(t *testing.T, count int) { | |
| 170 | + | t.Helper() | |
| 171 | + | require.Eventually(t, func() bool { return s.acceptedCount() >= count }, p3cTestDeadline, time.Millisecond, | |
| 172 | + | "server accepted %d of %d streams", s.acceptedCount(), count) | |
| 173 | + | } | |
| 174 | + | ||
| 175 | + | // TestNewEndpointConnectorValidation proves a connector refuses a nil dial | |
| 176 | + | // function or an invalid ceiling advertisement, and Connect refuses an invalid | |
| 177 | + | // resolved endpoint. | |
| 178 | + | func TestNewEndpointConnectorValidation(t *testing.T) { | |
| 179 | + | _, err := NewEndpointConnector(nil, DefaultMuxCeilings()) | |
| 180 | + | require.ErrorIs(t, err, ErrConnectorConfig) | |
| 181 | + | ||
| 182 | + | _, err = NewEndpointConnector(func(context.Context, string) (RawFramedTransport, error) { return nil, nil }, MuxCeilings{}) | |
| 183 | + | require.ErrorIs(t, err, ErrConnectorConfig) | |
| 184 | + | ||
| 185 | + | connector, err := NewEndpointConnector(func(context.Context, string) (RawFramedTransport, error) { | |
| 186 | + | t.Fatal("dial must not run for an invalid endpoint") | |
| 187 | + | return nil, nil | |
| 188 | + | }, DefaultMuxCeilings()) | |
| 189 | + | require.NoError(t, err) | |
| 190 | + | ||
| 191 | + | _, err = connector.Connect(context.Background(), ports.BrokerResolvedEndpoint{}) | |
| 192 | + | require.Error(t, err) | |
| 193 | + | var typed ports.BrokerError | |
| 194 | + | require.ErrorAs(t, err, &typed) | |
| 195 | + | require.Equal(t, ports.BrokerErrorIncompatible, typed.Code) | |
| 196 | + | } | |
| 197 | + | ||
| 198 | + | // TestEndpointConnectorRefusesPolicyMismatch proves the resolved endpoint stays | |
| 199 | + | // authoritative: a daemon that does not accept the endpoint policy is refused, | |
| 200 | + | // no physical connection is published, and the daemon admits no child. | |
| 201 | + | func TestEndpointConnectorRefusesPolicyMismatch(t *testing.T) { | |
| 202 | + | binding := mustServerBinding(t) | |
| 203 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 204 | + | ||
| 205 | + | connector, err := NewEndpointConnector(server.dial(), DefaultMuxCeilings()) | |
| 206 | + | require.NoError(t, err) | |
| 207 | + | physical, err := connector.Connect(context.Background(), muxEndpoint(binding.Identity(), alternatePolicy(binding.Policy()), "raw://mismatch")) | |
| 208 | + | require.Error(t, err) | |
| 209 | + | require.Nil(t, physical) | |
| 210 | + | require.ErrorIs(t, err, ErrHandshakeRejected) | |
| 211 | + | var typed ports.BrokerError | |
| 212 | + | require.ErrorAs(t, err, &typed) | |
| 213 | + | require.Equal(t, ports.BrokerErrorConflictingPolicy, typed.Code) | |
| 214 | + | ||
| 215 | + | require.ErrorIs(t, server.awaitAdoption(t), ErrHandshakeRejected) | |
| 216 | + | require.Zero(t, server.supervisor.Children(), "a refused handshake admits no physical child") | |
| 217 | + | require.Equal(t, int64(1), server.dials.Load()) | |
| 218 | + | } | |
| 219 | + | ||
| 220 | + | // TestEndpointConnectorExposesAuthorityAndPolicyGuard proves one successful | |
| 221 | + | // connection exposes the immutable accepted authority, delegates its terminal | |
| 222 | + | // results to the pump, refuses a mismatched Open without touching the carriage, | |
| 223 | + | // and admits a matching Open over the real carriage. | |
| 224 | + | func TestEndpointConnectorExposesAuthorityAndPolicyGuard(t *testing.T) { | |
| 225 | + | binding := mustServerBinding(t) | |
| 226 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 227 | + | server.serve() | |
| 228 | + | ||
| 229 | + | connector, err := NewEndpointConnector(server.dial(), DefaultMuxCeilings()) | |
| 230 | + | require.NoError(t, err) | |
| 231 | + | physical, err := connector.Connect(context.Background(), muxEndpoint(binding.Identity(), binding.Policy(), "raw://authority")) | |
| 232 | + | require.NoError(t, err) | |
| 233 | + | require.NotNil(t, physical) | |
| 234 | + | t.Cleanup(func() { _ = physical.Close() }) | |
| 235 | + | require.NoError(t, server.awaitAdoption(t)) | |
| 236 | + | ||
| 237 | + | require.Equal(t, binding.Identity(), physical.Identity()) | |
| 238 | + | require.Equal(t, binding.Incarnation(), physical.Incarnation()) | |
| 239 | + | require.Equal(t, binding.Policy(), physical.Policy()) | |
| 240 | + | require.False(t, channelClosed(physical.Done())) | |
| 241 | + | require.NoError(t, physical.Err()) | |
| 242 | + | require.Equal(t, domain.RemoteFailureNone, physical.FailureKind()) | |
| 243 | + | ||
| 244 | + | // A mismatched request policy is refused before the wire is touched. | |
| 245 | + | _, err = physical.OpenStream(context.Background(), muxOpenRequest(1, alternatePolicy(binding.Policy()))) | |
| 246 | + | require.Error(t, err) | |
| 247 | + | var typed ports.BrokerError | |
| 248 | + | require.ErrorAs(t, err, &typed) | |
| 249 | + | require.Equal(t, ports.BrokerErrorConflictingPolicy, typed.Code) | |
| 250 | + | require.Zero(t, server.acceptedCount(), "a policy-mismatched open is never admitted") | |
| 251 | + | ||
| 252 | + | // The matching open is admitted and usable over the real carriage. | |
| 253 | + | logical, err := physical.OpenStream(context.Background(), muxOpenRequest(1, binding.Policy())) | |
| 254 | + | require.NoError(t, err) | |
| 255 | + | server.awaitAccepted(t, 1) | |
| 256 | + | require.NoError(t, logical.SendClient(protocol.Ping{})) | |
| 257 | + | message, err := logical.ReceiveServer() | |
| 258 | + | require.NoError(t, err) | |
| 259 | + | require.Equal(t, protocol.Pong{}, message) | |
| 260 | + | ||
| 261 | + | require.NoError(t, logical.Close()) | |
| 262 | + | require.NoError(t, physical.Close()) | |
| 263 | + | require.True(t, channelClosed(physical.Done())) | |
| 264 | + | require.NoError(t, physical.Err(), "an orderly local Close is not a failure") | |
| 265 | + | require.Equal(t, domain.RemoteFailureNone, physical.FailureKind()) | |
| 266 | + | } | |
| 267 | + | ||
| 268 | + | // TestEndpointConnectorDetachesSetupContext proves a successful connection is | |
| 269 | + | // not coupled to the caller's setup context: cancelling it after Connect | |
| 270 | + | // returned does not stop the pooled physical connection. | |
| 271 | + | func TestEndpointConnectorDetachesSetupContext(t *testing.T) { | |
| 272 | + | binding := mustServerBinding(t) | |
| 273 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 274 | + | server.serve() | |
| 275 | + | ||
| 276 | + | connector, err := NewEndpointConnector(server.dial(), DefaultMuxCeilings()) | |
| 277 | + | require.NoError(t, err) | |
| 278 | + | setupCtx, cancel := context.WithCancel(context.Background()) | |
| 279 | + | physical, err := connector.Connect(setupCtx, muxEndpoint(binding.Identity(), binding.Policy(), "raw://detach")) | |
| 280 | + | require.NoError(t, err) | |
| 281 | + | require.NoError(t, server.awaitAdoption(t)) | |
| 282 | + | t.Cleanup(func() { _ = physical.Close() }) | |
| 283 | + | ||
| 284 | + | cancel() | |
| 285 | + | // The setup context ended; the pooled connection must still work. | |
| 286 | + | require.False(t, channelClosed(physical.Done())) | |
| 287 | + | logical, err := physical.OpenStream(context.Background(), muxOpenRequest(1, binding.Policy())) | |
| 288 | + | require.NoError(t, err) | |
| 289 | + | server.awaitAccepted(t, 1) | |
| 290 | + | require.NoError(t, logical.SendClient(protocol.Ping{})) | |
| 291 | + | message, err := logical.ReceiveServer() | |
| 292 | + | require.NoError(t, err) | |
| 293 | + | require.Equal(t, protocol.Pong{}, message) | |
| 294 | + | require.NoError(t, logical.Close()) | |
| 295 | + | } | |
| 296 | + | ||
| 297 | + | // TestPhysicalDoneBeforeLogicalLoss proves a carriage loss publishes the | |
| 298 | + | // physical terminal outcome before the affected logical stream is terminalized: | |
| 299 | + | // by the time the logical Done fires, the physical Done is already closed, and | |
| 300 | + | // both carry the loss cause. | |
| 301 | + | func TestPhysicalDoneBeforeLogicalLoss(t *testing.T) { | |
| 302 | + | binding := mustServerBinding(t) | |
| 303 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 304 | + | server.serve() | |
| 305 | + | ||
| 306 | + | connector, err := NewEndpointConnector(server.dial(), DefaultMuxCeilings()) | |
| 307 | + | require.NoError(t, err) | |
| 308 | + | physical, err := connector.Connect(context.Background(), muxEndpoint(binding.Identity(), binding.Policy(), "raw://loss")) | |
| 309 | + | require.NoError(t, err) | |
| 310 | + | t.Cleanup(func() { _ = physical.Close() }) | |
| 311 | + | require.NoError(t, server.awaitAdoption(t)) | |
| 312 | + | ||
| 313 | + | logical, err := physical.OpenStream(context.Background(), muxOpenRequest(1, binding.Policy())) | |
| 314 | + | require.NoError(t, err) | |
| 315 | + | server.awaitAccepted(t, 1) | |
| 316 | + | require.False(t, channelClosed(physical.Done())) | |
| 317 | + | ||
| 318 | + | ordered := make(chan bool, 1) | |
| 319 | + | go func() { | |
| 320 | + | select { | |
| 321 | + | case <-logical.Done(): | |
| 322 | + | case <-time.After(p3cTestDeadline): | |
| 323 | + | ordered <- false | |
| 324 | + | return | |
| 325 | + | } | |
| 326 | + | select { | |
| 327 | + | case <-physical.Done(): | |
| 328 | + | ordered <- true | |
| 329 | + | case <-time.After(p3cTestDeadline): | |
| 330 | + | ordered <- false | |
| 331 | + | } | |
| 332 | + | }() | |
| 333 | + | ||
| 334 | + | // Kill the daemon's carriage: the broker pump observes the carrier error. | |
| 335 | + | server.supervisor.mu.Lock() | |
| 336 | + | children := make([]*supervisedPhysical, 0, len(server.supervisor.children)) | |
| 337 | + | for child := range server.supervisor.children { | |
| 338 | + | children = append(children, child) | |
| 339 | + | } | |
| 340 | + | server.supervisor.mu.Unlock() | |
| 341 | + | require.Len(t, children, 1) | |
| 342 | + | require.NoError(t, children[0].Close()) | |
| 343 | + | ||
| 344 | + | require.True(t, <-ordered, "the logical Done fired before the physical Done") | |
| 345 | + | require.True(t, channelClosed(physical.Done())) | |
| 346 | + | require.Error(t, physical.Err()) | |
| 347 | + | require.NotEqual(t, domain.RemoteFailureNone, physical.FailureKind()) | |
| 348 | + | require.Error(t, logical.Err()) | |
| 349 | + | } | |
| 350 | + | ||
| 351 | + | // TestEndpointConnectorStableIdentityDifferentIncarnationRestart proves a | |
| 352 | + | // restart behind a stable authenticated identity surfaces a fresh incarnation: | |
| 353 | + | // the identity and policy stay equal, the incarnation changes, and each | |
| 354 | + | // connection reports its own captured authority. | |
| 355 | + | func TestEndpointConnectorStableIdentityDifferentIncarnationRestart(t *testing.T) { | |
| 356 | + | first, err := NewServerBinding(testIdentity(), testIncarnation(), testPolicy()) | |
| 357 | + | require.NoError(t, err) | |
| 358 | + | second, err := NewServerBinding(testIdentity(), ports.BrokerDaemonIncarnation{9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 1, 2, 3, 4, 5, 6}, testPolicy()) | |
| 359 | + | require.NoError(t, err) | |
| 360 | + | require.Equal(t, first.Identity(), second.Identity()) | |
| 361 | + | require.NotEqual(t, first.Incarnation(), second.Incarnation()) | |
| 362 | + | ||
| 363 | + | serverA := newSuperviseServer(t, first, DefaultMuxCeilings(), 0, nil) | |
| 364 | + | serverB := newSuperviseServer(t, second, DefaultMuxCeilings(), 0, nil) | |
| 365 | + | ||
| 366 | + | connectorA, err := NewEndpointConnector(serverA.dial(), DefaultMuxCeilings()) | |
| 367 | + | require.NoError(t, err) | |
| 368 | + | physicalA, err := connectorA.Connect(context.Background(), muxEndpoint(first.Identity(), first.Policy(), "raw://restart-a")) | |
| 369 | + | require.NoError(t, err) | |
| 370 | + | require.NoError(t, serverA.awaitAdoption(t)) | |
| 371 | + | require.Equal(t, first.Incarnation(), physicalA.Incarnation()) | |
| 372 | + | require.NoError(t, physicalA.Close()) | |
| 373 | + | ||
| 374 | + | connectorB, err := NewEndpointConnector(serverB.dial(), DefaultMuxCeilings()) | |
| 375 | + | require.NoError(t, err) | |
| 376 | + | physicalB, err := connectorB.Connect(context.Background(), muxEndpoint(second.Identity(), second.Policy(), "raw://restart-b")) | |
| 377 | + | require.NoError(t, err) | |
| 378 | + | require.NoError(t, serverB.awaitAdoption(t)) | |
| 379 | + | t.Cleanup(func() { _ = physicalB.Close() }) | |
| 380 | + | ||
| 381 | + | require.Equal(t, physicalA.Identity(), physicalB.Identity(), "the authenticated identity is stable across a restart") | |
| 382 | + | require.Equal(t, physicalA.Policy(), physicalB.Policy()) | |
| 383 | + | require.NotEqual(t, physicalA.Incarnation(), physicalB.Incarnation(), "a restart carries a fresh incarnation") | |
| 384 | + | } | |
A internal/adapters/daemonmux/convert.go +296 -0
| @@ -0,0 +1,296 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | // Shared semantic converters (P3.2). | |
| 4 | + | // | |
| 5 | + | // Conversion mirrors the port, domain, and protocol types losslessly and | |
| 6 | + | // validates every narrowing numeric cast before it happens. Wire uint32 | |
| 7 | + | // taxonomy fields never truncate into a smaller semantic enum: values whose | |
| 8 | + | // high bits would alias a legitimate code are refused with errConvertRange | |
| 9 | + | // before any cast. Identity lengths are exact (16-byte connection, | |
| 10 | + | // incarnation, and lifecycle values); text fields enforce UTF-8, | |
| 11 | + | // control/bidi, and byte bounds. | |
| 12 | + | // | |
| 13 | + | // The daemonmux codec deliberately imports no brokerwire symbol: it reuses | |
| 14 | + | // the generated shared wire messages and the ports/domain semantic values | |
| 15 | + | // directly, so the two conversations stay independently evolvable. | |
| 16 | + | ||
| 17 | + | import ( | |
| 18 | + | "math" | |
| 19 | + | "strings" | |
| 20 | + | "unicode" | |
| 21 | + | "unicode/utf8" | |
| 22 | + | ||
| 23 | + | "github.com/bnema/vev/internal/domain" | |
| 24 | + | "github.com/bnema/vev/internal/ports" | |
| 25 | + | "github.com/bnema/vev/internal/protocol" | |
| 26 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 27 | + | ) | |
| 28 | + | ||
| 29 | + | // muxEnum8 narrows one wire uint32 into an 8-bit semantic enum. Values whose | |
| 30 | + | // high bits would otherwise truncate into a valid enum range are refused | |
| 31 | + | // before any cast. | |
| 32 | + | func muxEnum8[T ~uint8](value uint32) (T, error) { | |
| 33 | + | if value > math.MaxUint8 { | |
| 34 | + | return 0, errConvertRange | |
| 35 | + | } | |
| 36 | + | return T(value), nil | |
| 37 | + | } | |
| 38 | + | ||
| 39 | + | // muxEnum16 narrows one wire uint32 into a 16-bit semantic value. | |
| 40 | + | func muxEnum16[T ~uint16](value uint32) (T, error) { | |
| 41 | + | if value > math.MaxUint16 { | |
| 42 | + | return 0, errConvertRange | |
| 43 | + | } | |
| 44 | + | return T(value), nil | |
| 45 | + | } | |
| 46 | + | ||
| 47 | + | func physicalToWire(id PhysicalStreamID) uint64 { return uint64(id) } | |
| 48 | + | ||
| 49 | + | func physicalFromWire(value uint64) (PhysicalStreamID, error) { | |
| 50 | + | id := PhysicalStreamID(value) | |
| 51 | + | if err := id.Validate(); err != nil { | |
| 52 | + | return 0, errConvertRange | |
| 53 | + | } | |
| 54 | + | return id, nil | |
| 55 | + | } | |
| 56 | + | ||
| 57 | + | func refToWire(ref StreamRef) *wire.MuxStreamRef { | |
| 58 | + | return &wire.MuxStreamRef{ | |
| 59 | + | PhysicalStreamId: physicalToWire(ref.Physical), | |
| 60 | + | BrokerEpoch: uint64(ref.Epoch), | |
| 61 | + | ConnectionId: append([]byte(nil), ref.Connection[:]...), | |
| 62 | + | ClientStreamId: uint64(ref.Client), | |
| 63 | + | } | |
| 64 | + | } | |
| 65 | + | ||
| 66 | + | func refFromWire(message *wire.MuxStreamRef) (StreamRef, error) { | |
| 67 | + | var ref StreamRef | |
| 68 | + | if message == nil { | |
| 69 | + | return ref, errConvertRange | |
| 70 | + | } | |
| 71 | + | physical, err := physicalFromWire(message.GetPhysicalStreamId()) | |
| 72 | + | if err != nil { | |
| 73 | + | return StreamRef{}, err | |
| 74 | + | } | |
| 75 | + | if message.GetBrokerEpoch() == 0 { | |
| 76 | + | return StreamRef{}, errConvertRange | |
| 77 | + | } | |
| 78 | + | raw := message.GetConnectionId() | |
| 79 | + | if len(raw) != len(ref.Connection) { | |
| 80 | + | return StreamRef{}, errConvertRange | |
| 81 | + | } | |
| 82 | + | copy(ref.Connection[:], raw) | |
| 83 | + | if err := ref.Connection.Validate(); err != nil { | |
| 84 | + | return StreamRef{}, errConvertRange | |
| 85 | + | } | |
| 86 | + | client := ports.BrokerStreamID(message.GetClientStreamId()) | |
| 87 | + | if err := client.Validate(); err != nil { | |
| 88 | + | return StreamRef{}, errConvertRange | |
| 89 | + | } | |
| 90 | + | return StreamRef{ | |
| 91 | + | Physical: physical, | |
| 92 | + | Epoch: ports.BrokerEpoch(message.GetBrokerEpoch()), | |
| 93 | + | Connection: ref.Connection, | |
| 94 | + | Client: client, | |
| 95 | + | }, nil | |
| 96 | + | } | |
| 97 | + | ||
| 98 | + | // registrationToWire mirrors domain.RemoteRegistration: an endpoint, an exact | |
| 99 | + | // 16-byte incarnation, and a generation. | |
| 100 | + | func registrationToWire(registration domain.RemoteRegistration) *wire.RemoteRegistration { | |
| 101 | + | incarnation := append([]byte(nil), registration.Incarnation[:]...) | |
| 102 | + | return &wire.RemoteRegistration{ | |
| 103 | + | Endpoint: registration.Endpoint, | |
| 104 | + | Incarnation: incarnation, | |
| 105 | + | Generation: uint64(registration.Generation), | |
| 106 | + | } | |
| 107 | + | } | |
| 108 | + | ||
| 109 | + | func registrationFromWire(message *wire.RemoteRegistration) (domain.RemoteRegistration, error) { | |
| 110 | + | var registration domain.RemoteRegistration | |
| 111 | + | if message == nil { | |
| 112 | + | return registration, errConvertRange | |
| 113 | + | } | |
| 114 | + | registration.Endpoint = message.GetEndpoint() | |
| 115 | + | if err := domain.ValidateRemoteHostTarget(registration.Endpoint); err != nil { | |
| 116 | + | return domain.RemoteRegistration{}, errConvertRange | |
| 117 | + | } | |
| 118 | + | raw := message.GetIncarnation() | |
| 119 | + | if len(raw) != len(registration.Incarnation) { | |
| 120 | + | return domain.RemoteRegistration{}, errConvertRange | |
| 121 | + | } | |
| 122 | + | copy(registration.Incarnation[:], raw) | |
| 123 | + | registration.Generation = domain.RemoteGeneration(message.GetGeneration()) | |
| 124 | + | if err := registration.Validate(); err != nil { | |
| 125 | + | return domain.RemoteRegistration{}, errConvertRange | |
| 126 | + | } | |
| 127 | + | return registration, nil | |
| 128 | + | } | |
| 129 | + | ||
| 130 | + | func exactTargetToWire(target protocol.ExactSessionTarget) *wire.ExactTarget { | |
| 131 | + | lifecycle := append([]byte(nil), target.LifecycleID[:]...) | |
| 132 | + | return &wire.ExactTarget{ | |
| 133 | + | LifecycleId: &wire.LifecycleID{Value: lifecycle}, | |
| 134 | + | SessionName: target.SessionName, | |
| 135 | + | } | |
| 136 | + | } | |
| 137 | + | ||
| 138 | + | func exactTargetFromWire(message *wire.ExactTarget) (protocol.ExactSessionTarget, error) { | |
| 139 | + | var target protocol.ExactSessionTarget | |
| 140 | + | if message == nil { | |
| 141 | + | return target, errConvertRange | |
| 142 | + | } | |
| 143 | + | lifecycle := message.GetLifecycleId() | |
| 144 | + | if lifecycle == nil || len(lifecycle.GetValue()) != len(target.LifecycleID) { | |
| 145 | + | return protocol.ExactSessionTarget{}, errConvertRange | |
| 146 | + | } | |
| 147 | + | copy(target.LifecycleID[:], lifecycle.GetValue()) | |
| 148 | + | target.SessionName = message.GetSessionName() | |
| 149 | + | if err := target.Validate(); err != nil { | |
| 150 | + | return protocol.ExactSessionTarget{}, errConvertRange | |
| 151 | + | } | |
| 152 | + | return target, nil | |
| 153 | + | } | |
| 154 | + | ||
| 155 | + | func policyToWire(policy ports.BrokerPolicy) *wire.BrokerWirePolicy { | |
| 156 | + | return &wire.BrokerWirePolicy{ | |
| 157 | + | ProtocolVersion: uint32(policy.ProtocolVersion), | |
| 158 | + | CatalogueVersion: uint32(policy.CatalogSchemaVersion), | |
| 159 | + | EnvironmentPolicy: uint32(policy.EnvironmentPolicy), | |
| 160 | + | Transport: policy.Transport, | |
| 161 | + | Trust: policy.Trust, | |
| 162 | + | Launch: policy.Launch, | |
| 163 | + | Isolation: policy.Isolation, | |
| 164 | + | } | |
| 165 | + | } | |
| 166 | + | ||
| 167 | + | func policyFromWire(message *wire.BrokerWirePolicy) (ports.BrokerPolicy, error) { | |
| 168 | + | var policy ports.BrokerPolicy | |
| 169 | + | if message == nil { | |
| 170 | + | return policy, errConvertRange | |
| 171 | + | } | |
| 172 | + | protocolVersion, err := muxEnum16[uint16](message.GetProtocolVersion()) | |
| 173 | + | if err != nil { | |
| 174 | + | return ports.BrokerPolicy{}, err | |
| 175 | + | } | |
| 176 | + | catalogueVersion, err := muxEnum16[uint16](message.GetCatalogueVersion()) | |
| 177 | + | if err != nil { | |
| 178 | + | return ports.BrokerPolicy{}, err | |
| 179 | + | } | |
| 180 | + | env, err := muxEnum8[protocol.EnvironmentPolicy](message.GetEnvironmentPolicy()) | |
| 181 | + | if err != nil { | |
| 182 | + | return ports.BrokerPolicy{}, err | |
| 183 | + | } | |
| 184 | + | policy = ports.BrokerPolicy{ | |
| 185 | + | ProtocolVersion: protocolVersion, | |
| 186 | + | CatalogSchemaVersion: catalogueVersion, | |
| 187 | + | EnvironmentPolicy: env, | |
| 188 | + | Transport: message.GetTransport(), | |
| 189 | + | Trust: message.GetTrust(), | |
| 190 | + | Launch: message.GetLaunch(), | |
| 191 | + | Isolation: message.GetIsolation(), | |
| 192 | + | } | |
| 193 | + | if err := policy.Validate(); err != nil { | |
| 194 | + | return ports.BrokerPolicy{}, ErrInvalidMessage | |
| 195 | + | } | |
| 196 | + | return policy, nil | |
| 197 | + | } | |
| 198 | + | ||
| 199 | + | func errorDetailToWire(detail ErrorDetail) *wire.BrokerErrorDetail { | |
| 200 | + | return &wire.BrokerErrorDetail{ | |
| 201 | + | Code: uint32(detail.Code), | |
| 202 | + | Text: detail.Text, | |
| 203 | + | AdmissionCode: detail.AdmissionCode, | |
| 204 | + | FailureKind: uint32(detail.FailureKind), | |
| 205 | + | } | |
| 206 | + | } | |
| 207 | + | ||
| 208 | + | func errorDetailFromWire(message *wire.BrokerErrorDetail) (ErrorDetail, error) { | |
| 209 | + | var detail ErrorDetail | |
| 210 | + | if message == nil { | |
| 211 | + | return detail, errConvertRange | |
| 212 | + | } | |
| 213 | + | code, err := muxEnum8[ports.BrokerErrorCode](message.GetCode()) | |
| 214 | + | if err != nil { | |
| 215 | + | return ErrorDetail{}, err | |
| 216 | + | } | |
| 217 | + | if err := code.Validate(); err != nil { | |
| 218 | + | return ErrorDetail{}, ErrInvalidMessage | |
| 219 | + | } | |
| 220 | + | detail.Code = code | |
| 221 | + | if err := validateDisplayText(message.GetText(), ports.BrokerMaxErrorBytes); err != nil { | |
| 222 | + | return ErrorDetail{}, err | |
| 223 | + | } | |
| 224 | + | detail.Text = message.GetText() | |
| 225 | + | admission := message.GetAdmissionCode() | |
| 226 | + | if admission > MaxAdmissionCode { | |
| 227 | + | return ErrorDetail{}, errConvertRange | |
| 228 | + | } | |
| 229 | + | detail.AdmissionCode = admission | |
| 230 | + | kind, err := muxEnum8[domain.RemoteFailureKind](message.GetFailureKind()) | |
| 231 | + | if err != nil { | |
| 232 | + | return ErrorDetail{}, err | |
| 233 | + | } | |
| 234 | + | if kind > MaxFailureKind { | |
| 235 | + | return ErrorDetail{}, ErrInvalidMessage | |
| 236 | + | } | |
| 237 | + | detail.FailureKind = kind | |
| 238 | + | failure := ports.BrokerError{Code: code, Text: detail.Text} | |
| 239 | + | if err := failure.Validate(); err != nil { | |
| 240 | + | return ErrorDetail{}, ErrInvalidMessage | |
| 241 | + | } | |
| 242 | + | return detail, nil | |
| 243 | + | } | |
| 244 | + | ||
| 245 | + | // validateDisplayText enforces bounded, presentation-safe text: valid UTF-8, | |
| 246 | + | // no control/bidi/line-separator characters, within maxBytes. | |
| 247 | + | func validateDisplayText(value string, maxBytes int) error { | |
| 248 | + | if len(value) > maxBytes { | |
| 249 | + | return ErrTooLarge | |
| 250 | + | } | |
| 251 | + | if !utf8.ValidString(value) { | |
| 252 | + | return ErrInvalidMessage | |
| 253 | + | } | |
| 254 | + | for _, r := range value { | |
| 255 | + | if unicode.IsControl(r) || unicode.Is(unicode.Bidi_Control, r) || r == '\u2028' || r == '\u2029' { | |
| 256 | + | return ErrInvalidMessage | |
| 257 | + | } | |
| 258 | + | } | |
| 259 | + | return nil | |
| 260 | + | } | |
| 261 | + | ||
| 262 | + | // validateEnvEntry enforces the per-request environment contract: valid | |
| 263 | + | // UTF-8, no control/bidi characters, bounded bytes. The daemonmux client | |
| 264 | + | // never merges these into the broker or daemon process environment. | |
| 265 | + | func validateEnvEntry(entry string) error { | |
| 266 | + | if len(entry) > ports.BrokerMaxEnvEntryBytes { | |
| 267 | + | return ErrTooLarge | |
| 268 | + | } | |
| 269 | + | if !utf8.ValidString(entry) { | |
| 270 | + | return ErrInvalidMessage | |
| 271 | + | } | |
| 272 | + | for _, r := range entry { | |
| 273 | + | if unicode.IsControl(r) || unicode.Is(unicode.Bidi_Control, r) || r == '\u2028' || r == '\u2029' { | |
| 274 | + | return ErrInvalidMessage | |
| 275 | + | } | |
| 276 | + | } | |
| 277 | + | return nil | |
| 278 | + | } | |
| 279 | + | ||
| 280 | + | // checkEnvEntries enforces the per-request environment bound: at most | |
| 281 | + | // BrokerMaxEnvEntries entries, each valid, bounded, and an explicit | |
| 282 | + | // KEY=VALUE assignment. | |
| 283 | + | func checkEnvEntries(env []string) error { | |
| 284 | + | if len(env) > ports.BrokerMaxEnvEntries { | |
| 285 | + | return ErrTooLarge | |
| 286 | + | } | |
| 287 | + | for _, entry := range env { | |
| 288 | + | if err := validateEnvEntry(entry); err != nil { | |
| 289 | + | return err | |
| 290 | + | } | |
| 291 | + | if !strings.Contains(entry, "=") { | |
| 292 | + | return ErrInvalidMessage | |
| 293 | + | } | |
| 294 | + | } | |
| 295 | + | return nil | |
| 296 | + | } | |
A internal/adapters/daemonmux/doc.go +214 -0
| @@ -0,0 +1,214 @@ | |||
| 1 | + | // Package daemonmux is the Plan 001 P3.2 daemon transport adapter. | |
| 2 | + | // | |
| 3 | + | // daemonmux will own one pooled physical connection to an authenticated | |
| 4 | + | // daemon and the independently cancellable logical streams multiplexed over | |
| 5 | + | // it: carriage framing, per-stream identity and bounded queues, admission, and | |
| 6 | + | // the terminal publication that fences every affected stream. Like the | |
| 7 | + | // brokerwire codec (P3.1) and the broker pool core (P2.2), it is not yet | |
| 8 | + | // activated by production composition. | |
| 9 | + | // | |
| 10 | + | // The stateless wire adaptation is additively complete: the private | |
| 11 | + | // directional codec over the daemonmux Protobuf envelopes (codec.go), its | |
| 12 | + | // bounded ceilings (limits.go), its adapter-local message values | |
| 13 | + | // (messages.go), the lossless conversions (convert.go), and the physical | |
| 14 | + | // preamble that negotiates the per-connection ceilings, the authenticated | |
| 15 | + | // daemon identity, and the accepted policy once (preamble.go). The negotiated | |
| 16 | + | // effective minima are exported as MuxCeilings and are the seam the engine and | |
| 17 | + | // scheduler constructors consume (NewStreamEngineWithCeilings, | |
| 18 | + | // NewSchedulerWithCeilings); the default constructors delegate with the | |
| 19 | + | // package policy. Encode and decode operate on one complete serialized | |
| 20 | + | // envelope at a time over strict wire.ScanEnvelope. The in-memory stream | |
| 21 | + | // table, admission, bounded inbound queues, and terminal publication live in | |
| 22 | + | // engine_state.go. No socket, framing engine, or composition is introduced | |
| 23 | + | // here. | |
| 24 | + | // | |
| 25 | + | // The daemon-side physical binding (binding.go) and the stateful physical | |
| 26 | + | // preamble handshake (handshake.go) are the connection's authority layer. A | |
| 27 | + | // ServerBinding is a validated, immutable (identity, incarnation, policy) | |
| 28 | + | // value. The broker-side handshake verifies the accepted response against the | |
| 29 | + | // independently resolved endpoint identity and policy, the nonzero | |
| 30 | + | // incarnation, and the ceilings it offered; the daemon-side handshake enforces | |
| 31 | + | // the immutable binding instead of echoing the requested policy, negotiates | |
| 32 | + | // the element-wise minima, and refuses with the precise shared rejection code. | |
| 33 | + | // One caller-supplied context or deadline bounds the single request/response | |
| 34 | + | // exchange, and every failure closes the carrier; a success leaves it open for | |
| 35 | + | // the pump its owner starts afterwards. Neither file starts a pump, admits a | |
| 36 | + | // stream, or introduces a socket, framing engine, or production composition. | |
| 37 | + | // | |
| 38 | + | // The private terminal-state primitive (terminal.go) is a deliberate engine | |
| 39 | + | // foundation, not a stateless-codec concern: it introduces the publish-once | |
| 40 | + | // terminal outcome that backs Done, Err, and FailureKind on a physical | |
| 41 | + | // connection and on every stream, because that contract is the engine's | |
| 42 | + | // foundation and defining it up front keeps the multiplexer carriage layered | |
| 43 | + | // on top without changing it. It carries no framing, streams, queues, or | |
| 44 | + | // socket. | |
| 45 | + | // | |
| 46 | + | // The outbound fair scheduler (scheduler.go) is the engine's outbound half: it | |
| 47 | + | // owns one bounded, independently cancellable FIFO queue per admitted stream | |
| 48 | + | // and orders already-encoded envelopes with byte-deficit round robin, a small | |
| 49 | + | // bounded control priority, and immediate per-stream discard on Reset. It | |
| 50 | + | // performs no I/O and owns no goroutine; the physical writer that drains it is | |
| 51 | + | // pump.go. No socket, framing engine, listener, or composition is introduced | |
| 52 | + | // here. | |
| 53 | + | // | |
| 54 | + | // The physical pump (pump.go) is the connection's carriage half: it composes | |
| 55 | + | // the stateless directional codec, the stream engine, the fair scheduler, and | |
| 56 | + | // the publish-once terminal state over the private FramedCarrier abstraction | |
| 57 | + | // with exactly one reader and one writer goroutine. The reader strict-decodes | |
| 58 | + | // each inbound direction and applies the engine; the writer drains the | |
| 59 | + | // scheduler's fairly ordered immutable envelopes; a coalescing wake channel | |
| 60 | + | // keeps the writer from polling or sleeping. A malformed outer envelope or a | |
| 61 | + | // carrier Receive/Send error publishes the physical terminal outcome before | |
| 62 | + | // every stream is terminalized, a stream-local semantic, reference, or queue | |
| 63 | + | // refusal schedules exactly one Reset without touching siblings, and | |
| 64 | + | // cancellation is an orderly close. An inbound Open that duplicates or replays | |
| 65 | + | // an identity the engine already owns is ignored, so it never disturbs the | |
| 66 | + | // live or retired stream it names; a fresh Open the engine refuses for | |
| 67 | + | // capacity, a closed connection, or an invalid authority is answered with | |
| 68 | + | // exactly one typed Mux Refused carrying its admission code and records no | |
| 69 | + | // stream. An outbound queue overflow is settled the same way: the overflowing | |
| 70 | + | // stream's engine record is reset and its one Reset is scheduled, and an | |
| 71 | + | // inbound Close or Reset retires the matching scheduler record and discards | |
| 72 | + | // its queued outbound frames. An orderly inbound Close preserves the inbound | |
| 73 | + | // data that was already accepted: the stream moves to closing, accepts no new | |
| 74 | + | // data, and stays readable until Take drains the last queued chunk, which is | |
| 75 | + | // when it terminalizes and releases its slot; a Reset always discards the | |
| 76 | + | // preserved queue at once. Flush is the explicit write barrier for callers that | |
| 77 | + | // must publish a final frame before closing; Close itself cancels the writer | |
| 78 | + | // and does not drain. Close unblocks the carrier through the adapter | |
| 79 | + | // prompt-close contract and joins both goroutines. The carrier stays an | |
| 80 | + | // abstract, testable seam: no socket, listener, streamframe, sessionwire, or | |
| 81 | + | // ports physical implementation is introduced here. | |
| 82 | + | // | |
| 83 | + | // The raw-carrier bridge (carrier.go) is the connection's framing boundary: it | |
| 84 | + | // adapts an existing raw framed transport - IPC, QUIC, or SSH stdio, each a | |
| 85 | + | // wire.BoundedTransport - to the private FramedCarrier the pump and the | |
| 86 | + | // preamble handshake drive. Send is synchronous; Receive is bounded to | |
| 87 | + | // wire.PreambleLimit until the explicit, one-way Negotiate transition records | |
| 88 | + | // the negotiated daemonmux envelope ceiling, so no receive allocates a body a | |
| 89 | + | // peer did not bound. Close is idempotent and concurrent-safe, interrupts a | |
| 90 | + | // blocked read or write, and joins every worker. It reuses the shared | |
| 91 | + | // streamframe framing and introduces no socket, listener, sessionwire, or frame | |
| 92 | + | // format of its own. | |
| 93 | + | // | |
| 94 | + | // The typed server-side listener (listener.go) is the daemon half of one | |
| 95 | + | // physical connection: the pump's reader admits each incoming MuxOpen and | |
| 96 | + | // notifies the listener through the pump's narrow admission-observer seam, the | |
| 97 | + | // listener queues admitted streams in a bounded FIFO accept queue (at most | |
| 98 | + | // MaxAcceptQueue, 128), and Accept hands each one back as an independent typed | |
| 99 | + | // ports.ServerConnection built with sessionwire over that mux stream. Every | |
| 100 | + | // stream carries one absolute local handshake deadline fixed at Open admission | |
| 101 | + | // (protocol.HandshakeTimeout) that queue delay and the session handshake share; | |
| 102 | + | // the deadline and the handshake completion signal are exposed to the daemon | |
| 103 | + | // through the narrow optional HandshakePlumbing interface, so the daemon adopts | |
| 104 | + | // this one deadline instead of starting a second one. A stream that reaches its | |
| 105 | + | // deadline, is refused because the accept queue is full, or is reset by the | |
| 106 | + | // peer is settled on its own: exactly one mux Reset, its admission slot | |
| 107 | + | // released, and no effect on a sibling or on the physical connection. An | |
| 108 | + | // admission whose absolute deadline elapsed is settled by Accept itself, even | |
| 109 | + | // before the watchdog observes it, and a connection whose inner session | |
| 110 | + | // carriage closes on its own during the handshake (a sessionwire preamble | |
| 111 | + | // failure) is settled by its terminal watcher with exactly one stream-local | |
| 112 | + | // Reset, so no consumer Close is needed. Accept | |
| 113 | + | // therefore never fails for a stream-local failure; it fails only when the | |
| 114 | + | // listener was closed or when the one physical connection reached its terminal | |
| 115 | + | // outcome, which is exactly when the shared daemon accept loop must stop. Close | |
| 116 | + | // unblocks Accept, drains every admitted-but-unaccepted stream through its one | |
| 117 | + | // stream-local Reset (releasing each slot and unblocking the broker's pending | |
| 118 | + | // Open), and joins the deadline watchdog without closing the pooled physical | |
| 119 | + | // pump. The listener is constructed before its pump starts: it registers the | |
| 120 | + | // pump's single admission observer, which the pump refuses after Start. No | |
| 121 | + | // socket, framing engine, or composition is introduced here. | |
| 122 | + | // | |
| 123 | + | // The aggregate listener (aggregate.go) is the daemon's global | |
| 124 | + | // ports.ServerListener over several physical listeners: it owns one forwarding | |
| 125 | + | // goroutine per registered child, delivers each child's independently accepted | |
| 126 | + | // typed connection through one bounded FIFO, and isolates a child's terminal | |
| 127 | + | // Accept failure - a lost or closed physical carriage - instead of surfacing it | |
| 128 | + | // as the aggregate's own error. So the loss of one physical daemonmux transport | |
| 129 | + | // never stops the daemon's shared accept loop while another or a newly | |
| 130 | + | // registered child can still serve. Register and Remove take and release | |
| 131 | + | // ownership of a child; Close unblocks Accept, closes every owned child, joins | |
| 132 | + | // every forwarder, and closes every accepted-but-unaccepted connection. The | |
| 133 | + | // aggregate consumes the typed ports.ServerListener/ports.ServerConnection | |
| 134 | + | // contracts only, so it composes with the daemonmux Listener, the sessionwire | |
| 135 | + | // listener, or any later physical listener. No socket, framing engine, or | |
| 136 | + | // production composition is introduced here. | |
| 137 | + | // | |
| 138 | + | // The broker endpoint connector (connector.go) is the broker half of one | |
| 139 | + | // pooled physical connection: EndpointConnector takes an injected narrow | |
| 140 | + | // authenticated raw-carrier dial function, runs the physical preamble over the | |
| 141 | + | // FramedCarrierBridge with the resolved endpoint's identity and policy as the | |
| 142 | + | // authoritative values, negotiates the effective ceilings, and hands the live | |
| 143 | + | // pump and LogicalConnector to a PhysicalConnection. One caller-supplied setup | |
| 144 | + | // context or deadline bounds dial and handshake together; a successful | |
| 145 | + | // connection is deliberately detached from it, so cancelling or expiring the | |
| 146 | + | // setup context never stops a pooled physical connection, which Close alone | |
| 147 | + | // owns. PhysicalConnection reports the immutable accepted identity, | |
| 148 | + | // incarnation, and policy, delegates Done, Err, and FailureKind to the pump's | |
| 149 | + | // publish-once terminal authority, exact-policy-checks every OpenStream before | |
| 150 | + | // delegating to the LogicalConnector, and tears the pump and carrier down | |
| 151 | + | // exactly once. | |
| 152 | + | // | |
| 153 | + | // The server supervisor (supervisor.go) is the daemon's owner of accepted raw | |
| 154 | + | // physical carriage: Adopt bounds concurrent handshakes and admissions, | |
| 155 | + | // completes the physical preamble against the immutable ServerBinding, | |
| 156 | + | // restricts the pump's engine to the accepted physical policy so every inbound | |
| 157 | + | // Open is checked before fresh stream admission, builds the typed Listener | |
| 158 | + | // before the pump starts, and registers one owned child - which closes | |
| 159 | + | // listener, pump, and carrier together - with the AggregateListener. One | |
| 160 | + | // reaper per child closes and deregisters it on its own terminal outcome, | |
| 161 | + | // isolating a lost or failed physical child from every sibling and from the | |
| 162 | + | // aggregate, and Close tears every owned child down and joins the reapers | |
| 163 | + | // without closing the caller-owned aggregate. The carriage stays an injected, | |
| 164 | + | // abstract seam: daemonmux never dials, listens, authenticates, or inspects a | |
| 165 | + | // socket, and no identity persistence or production activation is introduced | |
| 166 | + | // here. | |
| 167 | + | // | |
| 168 | + | // The private Unix carriage (P3.2d) lives in internal/adapters/ipc, not here: | |
| 169 | + | // ListenMux and DialMuxContext bind and connect a caller-supplied isolated | |
| 170 | + | // AF_UNIX path (never the live daemon.sock) and return a raw bounded transport | |
| 171 | + | // the EndpointConnector and ServerSupervisor consume unmodified. Listener setup | |
| 172 | + | // creates or validates an owner-only parent and socket, recovers race-safely | |
| 173 | + | // from a stale socket, refuses a foreign non-socket path, and both accept and | |
| 174 | + | // dial verify same-user kernel peer credentials (SO_PEERCRED on Linux), failing | |
| 175 | + | // closed on a platform or build without the check. Close unblocks Accept and | |
| 176 | + | // I/O and unlinks only the socket inode the listener created. daemonmux gains | |
| 177 | + | // no socket, dial/listen, or peer-credential code from this slice. | |
| 178 | + | // | |
| 179 | + | // The private QUIC carriage (P3.2e) lives in internal/adapters/quic, not here: | |
| 180 | + | // quic.DialMuxContext and quic.Server.AcceptMux reuse the existing | |
| 181 | + | // authenticated bootstrap contracts unchanged - the ephemeral certificate and | |
| 182 | + | // its exact SHA-256 pin, the single-use 32-byte token, and the single-use | |
| 183 | + | // nonce - and hand the connection's one bidirectional stream to the same | |
| 184 | + | // EndpointConnector and ServerSupervisor as a raw bounded transport | |
| 185 | + | // (wire.BoundedTransport). One QUIC connection is therefore one physical | |
| 186 | + | // daemonmux connection and exactly one native QUIC stream: logical attachments | |
| 187 | + | // are multiplexed inside that stream and are never mapped to native QUIC | |
| 188 | + | // streams, the one-stream guard stays in force for the whole connection | |
| 189 | + | // lifetime, and one freshly minted bootstrap server admits exactly one | |
| 190 | + | // authenticated carriage. The caller's setup context bounds the credential | |
| 191 | + | // wait, the pinned dial, the bounded auth record, and the physical preamble | |
| 192 | + | // the connector runs afterwards; a refused pin, token, or nonce closes the | |
| 193 | + | // carriage and hands nothing out, and a successful carriage detaches from the | |
| 194 | + | // setup context exactly as the Unix and in-memory carriages do. daemonmux | |
| 195 | + | // gains no socket, dial, TLS, or credential code from this slice. | |
| 196 | + | // | |
| 197 | + | // The private SSH stdio carriage lives in internal/adapters/sshstdio, not here: | |
| 198 | + | // sshstdio.DialMuxContext starts one caller-supplied explicit command over a | |
| 199 | + | // local ssh child process whose stdin/stdout are the raw bounded carriage, and | |
| 200 | + | // sshstdio.NewStdioTransport is its helper half over the remote process' own | |
| 201 | + | // stdio. The command is supplied by the caller, never the `vev _stdio` session | |
| 202 | + | // command, so a mux carriage cannot become a second navigation path to a | |
| 203 | + | // session; host-key and authentication policy stay owned by the target's | |
| 204 | + | // effective OpenSSH configuration, and the carriage requests no remote PTY | |
| 205 | + | // (`-T`). One child process is one physical daemonmux connection, exactly like | |
| 206 | + | // the Unix and QUIC carriages: every logical attachment is multiplexed inside | |
| 207 | + | // its stdio stream and is never mapped to another process. The caller's setup | |
| 208 | + | // context bounds subprocess start and the physical preamble the connector runs | |
| 209 | + | // afterwards together; a successful carriage detaches from it, and Close | |
| 210 | + | // terminates and joins the subprocess. The child's stderr is captured bounded | |
| 211 | + | // and sanitized, and a non-clean exit surfaces a typed sanitized error, never | |
| 212 | + | // raw remote stderr. daemonmux gains no process, pipe, or framing code from | |
| 213 | + | // this slice. | |
| 214 | + | package daemonmux | |
A internal/adapters/daemonmux/engine_policy_test.go +56 -0
| @@ -0,0 +1,56 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "testing" | |
| 5 | + | ||
| 6 | + | "github.com/stretchr/testify/require" | |
| 7 | + | ||
| 8 | + | "github.com/bnema/vev/internal/ports" | |
| 9 | + | ) | |
| 10 | + | ||
| 11 | + | // TestStreamEngineRestrictAdmissions proves the accepted physical policy seam: | |
| 12 | + | // it is configured once before any admission, refuses a mismatched fresh Open | |
| 13 | + | // before that Open records anything, leaves a replayed identity classified as a | |
| 14 | + | // duplicate rather than a policy refusal, and admits a matching Open. | |
| 15 | + | func TestStreamEngineRestrictAdmissions(t *testing.T) { | |
| 16 | + | engine := NewStreamEngine() | |
| 17 | + | require.NoError(t, engine.RestrictAdmissions(testPolicy())) | |
| 18 | + | require.ErrorIs(t, engine.RestrictAdmissions(testPolicy()), ErrStreamState, "the policy is fixed once") | |
| 19 | + | ||
| 20 | + | mismatched := openFor(1) | |
| 21 | + | mismatched.Policy = alternatePolicy(testPolicy()) | |
| 22 | + | require.ErrorIs(t, engine.Open(mismatched), ErrOpenPolicyRefused) | |
| 23 | + | _, tracked := engine.Status(1) | |
| 24 | + | require.False(t, tracked, "a policy refusal records no stream") | |
| 25 | + | require.Zero(t, engine.Live()) | |
| 26 | + | ||
| 27 | + | matching := openFor(1) | |
| 28 | + | matching.Policy = testPolicy() | |
| 29 | + | require.NoError(t, engine.Open(matching)) | |
| 30 | + | require.Equal(t, 1, engine.Live()) | |
| 31 | + | ||
| 32 | + | // A duplicate identity is still fenced by reference/identity first, so a | |
| 33 | + | // mismatched replay never disturbs the live stream it names. | |
| 34 | + | replay := openFor(1) | |
| 35 | + | replay.Policy = alternatePolicy(testPolicy()) | |
| 36 | + | require.ErrorIs(t, engine.Open(replay), ErrStreamIDReused) | |
| 37 | + | require.Equal(t, 1, engine.Live()) | |
| 38 | + | } | |
| 39 | + | ||
| 40 | + | // TestStreamEngineRestrictAdmissionsGuards proves the seam refuses an invalid | |
| 41 | + | // policy, a terminal engine, and a nil receiver, and that a policy gate is | |
| 42 | + | // optional: an unrestricted engine admits any validly encoded Open. | |
| 43 | + | func TestStreamEngineRestrictAdmissionsGuards(t *testing.T) { | |
| 44 | + | var nilEngine *StreamEngine | |
| 45 | + | require.ErrorIs(t, nilEngine.RestrictAdmissions(testPolicy()), ErrPhysicalClosed) | |
| 46 | + | ||
| 47 | + | engine := NewStreamEngine() | |
| 48 | + | require.Error(t, engine.RestrictAdmissions(ports.BrokerPolicy{}), "an invalid policy is refused") | |
| 49 | + | ||
| 50 | + | unrestricted := NewStreamEngine() | |
| 51 | + | require.NoError(t, unrestricted.Open(openFor(1))) | |
| 52 | + | ||
| 53 | + | terminated := NewStreamEngine() | |
| 54 | + | terminated.Terminate() | |
| 55 | + | require.ErrorIs(t, terminated.RestrictAdmissions(testPolicy()), ErrPhysicalClosed) | |
| 56 | + | } | |
A internal/adapters/daemonmux/engine_state.go +754 -0
| @@ -0,0 +1,754 @@ | |||
| 1 | + | // In-memory stream state and admission (P3.2). | |
| 2 | + | // | |
| 3 | + | // StreamEngine owns the admission and lifecycle of the logical streams | |
| 4 | + | // multiplexed over one physical daemon connection. It performs no I/O and | |
| 5 | + | // carries no framing, pump, socket, scheduler, or writer-fairness policy: it | |
| 6 | + | // holds the stream table, the strictly-increasing physical stream identity | |
| 7 | + | // fence, the per-stream lifecycle, the bounded inbound queues, and the | |
| 8 | + | // terminal publication that fences the physical connection and every stream | |
| 9 | + | // it owns. | |
| 10 | + | // | |
| 11 | + | // One physical connection admits at most the negotiated stream ceiling | |
| 12 | + | // (MuxCeilings.MaxStreams, 1..128) pending plus live streams: opening, open, | |
| 13 | + | // or closing - a closing stream keeps its slot until Take drains its preserved | |
| 14 | + | // data. A fresh incoming Open must carry a nonzero physical stream ID that | |
| 15 | + | // is strictly above every ID already admitted; an ID is never reused, | |
| 16 | + | // including after its stream reached a terminal state. Traffic at or below | |
| 17 | + | // the high-water mark that is no longer live belongs to a retired stream and | |
| 18 | + | // is ignored; traffic above the high-water mark was never opened and is | |
| 19 | + | // refused. The high-water mark alone fences a retired identity, so it keeps | |
| 20 | + | // classifying late traffic even after its record was evicted past the | |
| 21 | + | // retention bound. | |
| 22 | + | // | |
| 23 | + | // Every stream walks one exact lifecycle: opening (admitted, not yet | |
| 24 | + | // confirmed), open (confirmed), closing (an orderly inbound Close arrived | |
| 25 | + | // while the stream still held unread data), and terminal (closed, reset, | |
| 26 | + | // refused, or lost with the physical connection). Data is queued only while | |
| 27 | + | // open. An orderly Close that finds the inbound queue empty terminalizes the | |
| 28 | + | // stream immediately; a Close that finds already-queued data marks the stream | |
| 29 | + | // closing instead, so the consumer keeps draining exactly the chunks that were | |
| 30 | + | // accepted before the Close while no new chunk is accepted, and the stream | |
| 31 | + | // reaches terminal - releasing its admission slot and Done - only once Take | |
| 32 | + | // drains the last queued chunk. Reset is an abnormal, stream-local termination | |
| 33 | + | // that always discards the queue immediately. Neither Close nor Reset ever | |
| 34 | + | // terminalizes the physical connection or a sibling stream. | |
| 35 | + | // | |
| 36 | + | // The inbound queue is bounded, and bytes are charged against every budget | |
| 37 | + | // before the chunk is copied into the queue: at most MaxMuxStreamQueueChunks | |
| 38 | + | // chunks and MaxMuxStreamQueueBytes bytes per stream, the negotiated aggregate | |
| 39 | + | // ceiling (MuxCeilings.MaxAggregateBytes) across the connection, and the | |
| 40 | + | // negotiated chunk ceiling (MuxCeilings.StreamChunkLimit) per chunk. A chunk | |
| 41 | + | // above the negotiated chunk ceiling is refused statelessly with ErrTooLarge | |
| 42 | + | // and mutates nothing, exactly as the codec refuses it. A chunk that would | |
| 43 | + | // exceed a queue bound resets exactly that stream and releases the bytes it | |
| 44 | + | // held; sibling streams keep their budget and keep progressing. The per-stream | |
| 45 | + | // byte bound is defensive: the codec already refuses a chunk above the | |
| 46 | + | // negotiated chunk ceiling, so a conforming caller never reaches it. | |
| 47 | + | // | |
| 48 | + | // Terminal publication is deliberately ordered: a physical terminal outcome | |
| 49 | + | // publishes its own Done first and only then terminalizes every live stream | |
| 50 | + | // independently, so a reader woken by any stream always observes the physical | |
| 51 | + | // connection already terminal. A stream that already reached its own outcome | |
| 52 | + | // keeps it. | |
| 53 | + | // | |
| 54 | + | // A settled stream stays in the table so its reference, state, Done, and cause | |
| 55 | + | // remain observable; only an opening, open, or closing stream holds a slot | |
| 56 | + | // against the admission ceiling, and a closing stream retains that slot until | |
| 57 | + | // Take drains its preserved data. Retired records are themselves bounded: the | |
| 58 | + | // engine retains at most MaxRetiredStreamRecords settled records and evicts the | |
| 59 | + | // oldest first, so a long-lived connection cannot accumulate an unbounded | |
| 60 | + | // stream table. An evicted identity is still classified as retired rather than | |
| 61 | + | // future because the high-water mark is never rolled back. | |
| 62 | + | package daemonmux | |
| 63 | + | ||
| 64 | + | import ( | |
| 65 | + | "errors" | |
| 66 | + | "sync" | |
| 67 | + | ||
| 68 | + | "github.com/bnema/vev/internal/domain" | |
| 69 | + | "github.com/bnema/vev/internal/ports" | |
| 70 | + | ) | |
| 71 | + | ||
| 72 | + | // Stream queue bounds. The stream-count and aggregate-bytes ceilings are the | |
| 73 | + | // negotiated maxima from limits.go (MaxMuxStreams, MaxMuxAggregateBytes); the | |
| 74 | + | // per-stream chunk and byte ceilings are engine-local. | |
| 75 | + | const ( | |
| 76 | + | // MaxMuxStreamQueueChunks bounds the inbound chunks queued for one stream. | |
| 77 | + | MaxMuxStreamQueueChunks = 8 | |
| 78 | + | // MaxMuxStreamQueueBytes bounds the inbound bytes queued for one stream: | |
| 79 | + | // 32 MiB. It is only reachable by a caller that hands the engine a chunk | |
| 80 | + | // above the negotiated chunk ceiling, which the codec already refuses. | |
| 81 | + | MaxMuxStreamQueueBytes = 32 << 20 | |
| 82 | + | ||
| 83 | + | // MaxRetiredStreamRecords bounds the settled stream records one engine | |
| 84 | + | // retains so a late frame is still classified as retired instead of | |
| 85 | + | // mistaken for a future stream. It mirrors the negotiated stream ceiling | |
| 86 | + | // and is evicted oldest-first, so a long-lived connection's stream table | |
| 87 | + | // stays bounded by the live ceiling plus this retention bound. | |
| 88 | + | MaxRetiredStreamRecords = 128 | |
| 89 | + | ) | |
| 90 | + | ||
| 91 | + | // Stream admission and lifecycle sentinels. They are typed so a caller can | |
| 92 | + | // classify a refused frame without matching on message text. | |
| 93 | + | var ( | |
| 94 | + | // ErrInvalidStream reports a missing or zero physical stream identity. | |
| 95 | + | ErrInvalidStream = errors.New("daemonmux: invalid physical stream") | |
| 96 | + | // ErrStreamIDReused reports a physical stream identity that is not | |
| 97 | + | // strictly above every admitted identity: IDs strictly increase and are | |
| 98 | + | // never reused, including after a stream reached a terminal state. | |
| 99 | + | ErrStreamIDReused = errors.New("daemonmux: stream ID is not fresh") | |
| 100 | + | // ErrFutureStream reports a frame for a physical stream identity that was | |
| 101 | + | // never opened. The physical connection's high-water mark fences it. | |
| 102 | + | ErrFutureStream = errors.New("daemonmux: stream was never opened") | |
| 103 | + | // ErrTooManyStreams reports the pending-plus-live stream ceiling of one | |
| 104 | + | // physical connection. | |
| 105 | + | ErrTooManyStreams = errors.New("daemonmux: too many live streams") | |
| 106 | + | // ErrStreamState reports an event that is illegal in the stream's current | |
| 107 | + | // state. | |
| 108 | + | ErrStreamState = errors.New("daemonmux: invalid stream state") | |
| 109 | + | // ErrStreamRefMismatch reports an Opened or Refused whose full stream | |
| 110 | + | // reference does not match the one recorded at admission. | |
| 111 | + | ErrStreamRefMismatch = errors.New("daemonmux: stream reference mismatch") | |
| 112 | + | // ErrStreamQueueFull reports an inbound chunk that would exceed a queue | |
| 113 | + | // bound. The offending stream is reset and its budget released. | |
| 114 | + | ErrStreamQueueFull = errors.New("daemonmux: stream queue overflow") | |
| 115 | + | // ErrPhysicalClosed reports admission or stream work presented to a | |
| 116 | + | // physical connection that already reached its terminal outcome. | |
| 117 | + | ErrPhysicalClosed = errors.New("daemonmux: physical connection is closed") | |
| 118 | + | // ErrOpenPolicyRefused reports a fresh inbound Open whose requested policy | |
| 119 | + | // does not exactly match the accepted physical policy an engine was | |
| 120 | + | // restricted to. The stream is refused before it is recorded or admitted, | |
| 121 | + | // so it never holds an admission slot and never reaches a consumer. | |
| 122 | + | ErrOpenPolicyRefused = errors.New("daemonmux: open policy refused") | |
| 123 | + | ) | |
| 124 | + | ||
| 125 | + | // StreamState is the closed lifecycle of one logical stream over a physical | |
| 126 | + | // daemon connection. | |
| 127 | + | type StreamState uint8 | |
| 128 | + | ||
| 129 | + | const ( | |
| 130 | + | // StreamOpening is an admitted stream the opening side has not confirmed. | |
| 131 | + | StreamOpening StreamState = iota + 1 | |
| 132 | + | // StreamOpen is a confirmed stream that accepts data. | |
| 133 | + | StreamOpen | |
| 134 | + | // StreamClosing is a stream whose opening side sent an orderly Close while | |
| 135 | + | // inbound data was still queued. It accepts no new data; its already-queued | |
| 136 | + | // chunks stay readable until Take drains them, at which point the stream | |
| 137 | + | // reaches terminal and releases its slot. | |
| 138 | + | StreamClosing | |
| 139 | + | // StreamTerminal is a settled stream: no further frame is applied. | |
| 140 | + | StreamTerminal | |
| 141 | + | ) | |
| 142 | + | ||
| 143 | + | func (s StreamState) String() string { | |
| 144 | + | switch s { | |
| 145 | + | case StreamOpening: | |
| 146 | + | return "opening" | |
| 147 | + | case StreamOpen: | |
| 148 | + | return "open" | |
| 149 | + | case StreamClosing: | |
| 150 | + | return "closing" | |
| 151 | + | case StreamTerminal: | |
| 152 | + | return "terminal" | |
| 153 | + | default: | |
| 154 | + | return "unknown" | |
| 155 | + | } | |
| 156 | + | } | |
| 157 | + | ||
| 158 | + | // StreamDisposition classifies one stream frame: applied, or a late frame for | |
| 159 | + | // a retired stream that is ignored without effect. | |
| 160 | + | type StreamDisposition uint8 | |
| 161 | + | ||
| 162 | + | const ( | |
| 163 | + | // StreamAccepted reports the frame was applied. | |
| 164 | + | StreamAccepted StreamDisposition = iota + 1 | |
| 165 | + | // StreamDiscarded reports a late frame for a retired stream. It is | |
| 166 | + | // ignored and never revives, reopens, or terminalizes anything. | |
| 167 | + | StreamDiscarded | |
| 168 | + | ) | |
| 169 | + | ||
| 170 | + | // StreamStatus is one immutable read of a tracked stream. | |
| 171 | + | type StreamStatus struct { | |
| 172 | + | // Ref is the full reference recorded at admission. | |
| 173 | + | Ref StreamRef | |
| 174 | + | // State is the stream lifecycle state. | |
| 175 | + | State StreamState | |
| 176 | + | // Done closes exactly once when the stream reaches its terminal state. | |
| 177 | + | Done <-chan struct{} | |
| 178 | + | // Err is the stream's terminal cause: nil while live and for an orderly | |
| 179 | + | // Close, and the failure cause for a Reset, a queue overflow, or a | |
| 180 | + | // physical loss. | |
| 181 | + | Err error | |
| 182 | + | // FailureKind classifies Err: RemoteFailureNone while live, for an | |
| 183 | + | // orderly Close, and for a refusal. | |
| 184 | + | FailureKind domain.RemoteFailureKind | |
| 185 | + | // Refused reports that the opening side refused the stream. | |
| 186 | + | Refused bool | |
| 187 | + | // Refusal is the typed refusal carried by Refused when Refused is true. | |
| 188 | + | Refusal ErrorDetail | |
| 189 | + | // QueuedChunks and QueuedBytes are the currently queued inbound chunks | |
| 190 | + | // and bytes. | |
| 191 | + | QueuedChunks int | |
| 192 | + | QueuedBytes int | |
| 193 | + | } | |
| 194 | + | ||
| 195 | + | // muxStream is one tracked logical stream. Every field is guarded by the | |
| 196 | + | // owning StreamEngine's lock; terminal is its own publish-once authority and | |
| 197 | + | // is safe to read independently once published. | |
| 198 | + | type muxStream struct { | |
| 199 | + | ref StreamRef | |
| 200 | + | state StreamState | |
| 201 | + | terminal *terminalState | |
| 202 | + | queue [][]byte | |
| 203 | + | bytes int | |
| 204 | + | refused bool | |
| 205 | + | refusal ErrorDetail | |
| 206 | + | } | |
| 207 | + | ||
| 208 | + | // StreamEngine is the in-memory stream table and admission authority of one | |
| 209 | + | // physical daemon connection. It is safe for concurrent use. It owns no I/O. | |
| 210 | + | // The admission, chunk, and aggregate budgets are copied from the negotiated | |
| 211 | + | // MuxCeilings at construction; the per-stream queue bounds are engine-local. | |
| 212 | + | // They are fields so the engine's private ceilings are fixed per instance and | |
| 213 | + | // a test can tighten them. | |
| 214 | + | type StreamEngine struct { | |
| 215 | + | mu sync.Mutex | |
| 216 | + | physical *terminalState | |
| 217 | + | streams map[PhysicalStreamID]*muxStream | |
| 218 | + | retired []PhysicalStreamID | |
| 219 | + | highest PhysicalStreamID | |
| 220 | + | live int | |
| 221 | + | aggregate int | |
| 222 | + | ||
| 223 | + | maxStreams int | |
| 224 | + | maxChunkBytes uint64 | |
| 225 | + | maxStreamBytes int | |
| 226 | + | maxStreamFrames int | |
| 227 | + | maxAggregateBytes int | |
| 228 | + | ||
| 229 | + | // restricted, when set, binds every fresh inbound Open to the exact | |
| 230 | + | // accepted physical policy in policy. It is configured once, before any | |
| 231 | + | // stream is admitted, by a daemon-side consumer that owns the connection's | |
| 232 | + | // authoritative binding (RestrictAdmissions). | |
| 233 | + | restricted bool | |
| 234 | + | policy ports.BrokerPolicy | |
| 235 | + | } | |
| 236 | + | ||
| 237 | + | // NewStreamEngine returns an open engine bound to one physical connection | |
| 238 | + | // under the default MuxCeilings: the full envelope ceiling, the maximum | |
| 239 | + | // chunk, the maximum stream count (pending-plus-live), and the maximum | |
| 240 | + | // aggregate buffered bytes. | |
| 241 | + | func NewStreamEngine() *StreamEngine { | |
| 242 | + | return newStreamEngine(DefaultMuxCeilings()) | |
| 243 | + | } | |
| 244 | + | ||
| 245 | + | // NewStreamEngineWithCeilings returns an open engine bound to one physical | |
| 246 | + | // connection whose admission, chunk, and aggregate budgets enforce the given | |
| 247 | + | // negotiated MuxCeilings. The value is validated and copied, so the ceilings | |
| 248 | + | // an engine enforces are fixed for its lifetime. An invalid advertisement is | |
| 249 | + | // refused with ErrInvalidCeilings. | |
| 250 | + | func NewStreamEngineWithCeilings(ceilings MuxCeilings) (*StreamEngine, error) { | |
| 251 | + | if err := ceilings.Validate(); err != nil { | |
| 252 | + | return nil, err | |
| 253 | + | } | |
| 254 | + | return newStreamEngine(ceilings), nil | |
| 255 | + | } | |
| 256 | + | ||
| 257 | + | // newStreamEngine builds an engine from already-validated ceilings. The | |
| 258 | + | // exported constructors validate first; NewStreamEngine delegates with the | |
| 259 | + | // package defaults, which are valid by construction. | |
| 260 | + | func newStreamEngine(ceilings MuxCeilings) *StreamEngine { | |
| 261 | + | return &StreamEngine{ | |
| 262 | + | physical: newTerminalState(), | |
| 263 | + | streams: make(map[PhysicalStreamID]*muxStream, int(ceilings.MaxStreams)), | |
| 264 | + | maxStreams: int(ceilings.MaxStreams), | |
| 265 | + | maxChunkBytes: ceilings.StreamChunkLimit, | |
| 266 | + | maxStreamBytes: MaxMuxStreamQueueBytes, | |
| 267 | + | maxStreamFrames: MaxMuxStreamQueueChunks, | |
| 268 | + | maxAggregateBytes: int(ceilings.MaxAggregateBytes), | |
| 269 | + | } | |
| 270 | + | } | |
| 271 | + | ||
| 272 | + | // Done returns the physical connection's terminal channel. It is closed | |
| 273 | + | // exactly once, before every live stream is terminalized. | |
| 274 | + | func (e *StreamEngine) Done() <-chan struct{} { | |
| 275 | + | if e == nil || e.physical == nil { | |
| 276 | + | return nil | |
| 277 | + | } | |
| 278 | + | return e.physical.Done() | |
| 279 | + | } | |
| 280 | + | ||
| 281 | + | // Err returns the physical connection's terminal cause: nil while open and | |
| 282 | + | // for an orderly Terminate. | |
| 283 | + | func (e *StreamEngine) Err() error { | |
| 284 | + | if e == nil || e.physical == nil { | |
| 285 | + | return nil | |
| 286 | + | } | |
| 287 | + | return e.physical.Err() | |
| 288 | + | } | |
| 289 | + | ||
| 290 | + | // FailureKind returns the physical connection's terminal classification: | |
| 291 | + | // RemoteFailureNone while open and for an orderly Terminate. | |
| 292 | + | func (e *StreamEngine) FailureKind() domain.RemoteFailureKind { | |
| 293 | + | if e == nil || e.physical == nil { | |
| 294 | + | return domain.RemoteFailureNone | |
| 295 | + | } | |
| 296 | + | return e.physical.FailureKind() | |
| 297 | + | } | |
| 298 | + | ||
| 299 | + | // Closed reports whether the physical connection already reached its terminal | |
| 300 | + | // outcome. | |
| 301 | + | func (e *StreamEngine) Closed() bool { | |
| 302 | + | if e == nil { | |
| 303 | + | return true | |
| 304 | + | } | |
| 305 | + | e.mu.Lock() | |
| 306 | + | defer e.mu.Unlock() | |
| 307 | + | return e.closedLocked() | |
| 308 | + | } | |
| 309 | + | ||
| 310 | + | // Live returns the number of admitted streams that have not yet reached their | |
| 311 | + | // terminal state: opening, open, or closing (a closing stream still holds the | |
| 312 | + | // admission slot and its queued bytes until Take drains them). | |
| 313 | + | func (e *StreamEngine) Live() int { | |
| 314 | + | if e == nil { | |
| 315 | + | return 0 | |
| 316 | + | } | |
| 317 | + | e.mu.Lock() | |
| 318 | + | defer e.mu.Unlock() | |
| 319 | + | return e.live | |
| 320 | + | } | |
| 321 | + | ||
| 322 | + | // AggregateBytes returns the inbound bytes currently charged across every | |
| 323 | + | // stream. It never exceeds MaxMuxAggregateBytes. | |
| 324 | + | func (e *StreamEngine) AggregateBytes() int { | |
| 325 | + | if e == nil { | |
| 326 | + | return 0 | |
| 327 | + | } | |
| 328 | + | e.mu.Lock() | |
| 329 | + | defer e.mu.Unlock() | |
| 330 | + | return e.aggregate | |
| 331 | + | } | |
| 332 | + | ||
| 333 | + | // RestrictAdmissions binds the engine to one exact accepted physical policy: | |
| 334 | + | // every fresh inbound Open must then carry a policy exactly compatible with | |
| 335 | + | // policy, and an Open that differs in any field is refused with | |
| 336 | + | // ErrOpenPolicyRefused before it is recorded or admitted. A replay or | |
| 337 | + | // duplicate Open is still classified by the identity fence first, so a | |
| 338 | + | // restriction never resets or disturbs the live stream a duplicate names. | |
| 339 | + | // | |
| 340 | + | // It must be configured before the first inbound Open - that is, before the | |
| 341 | + | // pump driving the engine starts - because changing the accepted policy after | |
| 342 | + | // admission would leave earlier streams bound to a policy the engine no longer | |
| 343 | + | // enforces. A second call, a call after any admission, an already-terminal | |
| 344 | + | // connection, or an invalid policy is refused without changing enforcement. | |
| 345 | + | func (e *StreamEngine) RestrictAdmissions(policy ports.BrokerPolicy) error { | |
| 346 | + | if e == nil { | |
| 347 | + | return ErrPhysicalClosed | |
| 348 | + | } | |
| 349 | + | if err := policy.Validate(); err != nil { | |
| 350 | + | return err | |
| 351 | + | } | |
| 352 | + | e.mu.Lock() | |
| 353 | + | defer e.mu.Unlock() | |
| 354 | + | if e.closedLocked() { | |
| 355 | + | return ErrPhysicalClosed | |
| 356 | + | } | |
| 357 | + | if e.restricted || e.highest != 0 { | |
| 358 | + | return ErrStreamState | |
| 359 | + | } | |
| 360 | + | e.policy = policy | |
| 361 | + | e.restricted = true | |
| 362 | + | return nil | |
| 363 | + | } | |
| 364 | + | ||
| 365 | + | // Open admits one incoming logical stream in opening state. The reference | |
| 366 | + | // must be complete, its physical ID must be strictly above every admitted ID | |
| 367 | + | // (IDs are never reused), and the pending-plus-live ceiling must not be | |
| 368 | + | // reached. When the engine is restricted to an accepted physical policy | |
| 369 | + | // (RestrictAdmissions), a fresh Open whose requested policy does not exactly | |
| 370 | + | // match is refused before it takes an admission slot. A physical connection | |
| 371 | + | // that already reached its terminal outcome refuses all admission. | |
| 372 | + | func (e *StreamEngine) Open(msg Open) error { | |
| 373 | + | if e == nil { | |
| 374 | + | return ErrPhysicalClosed | |
| 375 | + | } | |
| 376 | + | e.mu.Lock() | |
| 377 | + | defer e.mu.Unlock() | |
| 378 | + | if e.closedLocked() { | |
| 379 | + | return ErrPhysicalClosed | |
| 380 | + | } | |
| 381 | + | if err := msg.Ref.Validate(); err != nil { | |
| 382 | + | return ErrInvalidStream | |
| 383 | + | } | |
| 384 | + | id := msg.Ref.Physical | |
| 385 | + | if id <= e.highest { | |
| 386 | + | return ErrStreamIDReused | |
| 387 | + | } | |
| 388 | + | if e.restricted && !e.policy.Compatible(msg.Policy) { | |
| 389 | + | return ErrOpenPolicyRefused | |
| 390 | + | } | |
| 391 | + | if e.live >= e.maxStreams { | |
| 392 | + | return ErrTooManyStreams | |
| 393 | + | } | |
| 394 | + | e.streams[id] = &muxStream{ref: msg.Ref, state: StreamOpening, terminal: newTerminalState()} | |
| 395 | + | e.highest = id | |
| 396 | + | e.live++ | |
| 397 | + | return nil | |
| 398 | + | } | |
| 399 | + | ||
| 400 | + | // Opened confirms one admitted stream and moves it to open. It is legal only | |
| 401 | + | // for an opening stream, and the full reference must match the one recorded | |
| 402 | + | // at admission. | |
| 403 | + | func (e *StreamEngine) Opened(msg Opened) error { | |
| 404 | + | if e == nil { | |
| 405 | + | return ErrPhysicalClosed | |
| 406 | + | } | |
| 407 | + | e.mu.Lock() | |
| 408 | + | defer e.mu.Unlock() | |
| 409 | + | stream, state, err := e.lookupLocked(msg.Ref.Physical) | |
| 410 | + | if err != nil { | |
| 411 | + | return err | |
| 412 | + | } | |
| 413 | + | if state != StreamOpening { | |
| 414 | + | return ErrStreamState | |
| 415 | + | } | |
| 416 | + | if stream.ref != msg.Ref { | |
| 417 | + | return ErrStreamRefMismatch | |
| 418 | + | } | |
| 419 | + | stream.state = StreamOpen | |
| 420 | + | return nil | |
| 421 | + | } | |
| 422 | + | ||
| 423 | + | // Refused settles one admitted stream after the opening side refused it. It is | |
| 424 | + | // legal only for an opening stream, and the full reference must match. A | |
| 425 | + | // refusal is an orderly settlement: the stream's Done closes with a nil Err, | |
| 426 | + | // and the typed refusal is retained for the caller. | |
| 427 | + | func (e *StreamEngine) Refused(msg Refused) error { | |
| 428 | + | if e == nil { | |
| 429 | + | return ErrPhysicalClosed | |
| 430 | + | } | |
| 431 | + | e.mu.Lock() | |
| 432 | + | defer e.mu.Unlock() | |
| 433 | + | stream, state, err := e.lookupLocked(msg.Ref.Physical) | |
| 434 | + | if err != nil { | |
| 435 | + | return err | |
| 436 | + | } | |
| 437 | + | if state != StreamOpening { | |
| 438 | + | return ErrStreamState | |
| 439 | + | } | |
| 440 | + | if stream.ref != msg.Ref { | |
| 441 | + | return ErrStreamRefMismatch | |
| 442 | + | } | |
| 443 | + | stream.refused = true | |
| 444 | + | stream.refusal = msg.Error | |
| 445 | + | e.retireLocked(stream) | |
| 446 | + | stream.terminal.Close() | |
| 447 | + | return nil | |
| 448 | + | } | |
| 449 | + | ||
| 450 | + | // Data queues one already-validated chunk of an open stream. Data is legal | |
| 451 | + | // only while open: a retired stream discards it, and a stream that was never | |
| 452 | + | // opened is refused. A stream that already received an orderly Close | |
| 453 | + | // (closing) accepts no new data and refuses it with ErrStreamState, exactly as | |
| 454 | + | // an unconfirmed stream refuses data. A chunk above the negotiated chunk | |
| 455 | + | // ceiling is refused statelessly with ErrTooLarge and mutates nothing. A chunk | |
| 456 | + | // that would exceed the per-stream chunk, the per-stream byte, or the | |
| 457 | + | // aggregate byte bound resets exactly this stream and releases the bytes it | |
| 458 | + | // held, leaving every sibling untouched. | |
| 459 | + | func (e *StreamEngine) Data(msg Data) (StreamDisposition, error) { | |
| 460 | + | if e == nil { | |
| 461 | + | return 0, ErrPhysicalClosed | |
| 462 | + | } | |
| 463 | + | if uint64(len(msg.Data)) > e.maxChunkBytes { | |
| 464 | + | return 0, ErrTooLarge | |
| 465 | + | } | |
| 466 | + | e.mu.Lock() | |
| 467 | + | defer e.mu.Unlock() | |
| 468 | + | stream, state, err := e.lookupLocked(msg.Physical) | |
| 469 | + | if err != nil { | |
| 470 | + | return 0, err | |
| 471 | + | } | |
| 472 | + | switch state { | |
| 473 | + | case StreamTerminal: | |
| 474 | + | return StreamDiscarded, nil | |
| 475 | + | case StreamOpening, StreamClosing: | |
| 476 | + | return 0, ErrStreamState | |
| 477 | + | } | |
| 478 | + | if e.queueOverflowsLocked(stream, len(msg.Data)) { | |
| 479 | + | e.retireLocked(stream) | |
| 480 | + | stream.terminal.Fail(domain.RemoteFailureTransport, ErrStreamQueueFull) | |
| 481 | + | return 0, ErrStreamQueueFull | |
| 482 | + | } | |
| 483 | + | e.enqueueLocked(stream, msg.Data) | |
| 484 | + | return StreamAccepted, nil | |
| 485 | + | } | |
| 486 | + | ||
| 487 | + | // Close orderly terminates one opening, open, or closing stream. A late close | |
| 488 | + | // for an already terminal or closing stream is discarded. It never | |
| 489 | + | // terminalizes the physical connection or a sibling stream. | |
| 490 | + | // | |
| 491 | + | // An orderly Close preserves data that was already accepted: when the inbound | |
| 492 | + | // queue holds unread chunks, the stream moves to closing, accepts no new data, | |
| 493 | + | // and keeps its queue readable until Take drains it; the stream only reaches | |
| 494 | + | // terminal, releasing its admission slot and closing Done, on the Take that | |
| 495 | + | // empties the queue. A Close with an empty queue terminalizes immediately. | |
| 496 | + | func (e *StreamEngine) Close(msg Close) (StreamDisposition, error) { | |
| 497 | + | if e == nil { | |
| 498 | + | return 0, ErrPhysicalClosed | |
| 499 | + | } | |
| 500 | + | e.mu.Lock() | |
| 501 | + | defer e.mu.Unlock() | |
| 502 | + | stream, state, err := e.lookupLocked(msg.Physical) | |
| 503 | + | if err != nil { | |
| 504 | + | return 0, err | |
| 505 | + | } | |
| 506 | + | if state == StreamTerminal || state == StreamClosing { | |
| 507 | + | return StreamDiscarded, nil | |
| 508 | + | } | |
| 509 | + | if len(stream.queue) == 0 { | |
| 510 | + | e.retireLocked(stream) | |
| 511 | + | stream.terminal.Close() | |
| 512 | + | return StreamAccepted, nil | |
| 513 | + | } | |
| 514 | + | stream.state = StreamClosing | |
| 515 | + | return StreamAccepted, nil | |
| 516 | + | } | |
| 517 | + | ||
| 518 | + | // Reset abnormally terminates one opening or open stream. The termination is | |
| 519 | + | // stream-local: it never terminalizes the physical connection or a sibling | |
| 520 | + | // stream. A late reset for a retired stream is discarded, so cancellation is | |
| 521 | + | // idempotent. | |
| 522 | + | func (e *StreamEngine) Reset(msg Reset) (StreamDisposition, error) { | |
| 523 | + | if e == nil { | |
| 524 | + | return 0, ErrPhysicalClosed | |
| 525 | + | } | |
| 526 | + | e.mu.Lock() | |
| 527 | + | defer e.mu.Unlock() | |
| 528 | + | stream, state, err := e.lookupLocked(msg.Physical) | |
| 529 | + | if err != nil { | |
| 530 | + | return 0, err | |
| 531 | + | } | |
| 532 | + | if state == StreamTerminal { | |
| 533 | + | return StreamDiscarded, nil | |
| 534 | + | } | |
| 535 | + | e.retireLocked(stream) | |
| 536 | + | stream.terminal.Fail(msg.Error.FailureKind, resetCause(msg)) | |
| 537 | + | return StreamAccepted, nil | |
| 538 | + | } | |
| 539 | + | ||
| 540 | + | // Fail publishes the physical connection's terminal failure and then | |
| 541 | + | // terminalizes every live stream independently with the same cause. Done is | |
| 542 | + | // closed before any stream is terminalized; a stream that already settled | |
| 543 | + | // keeps its own outcome. All queued bytes are released. | |
| 544 | + | // | |
| 545 | + | // Physical publication is first-wins: a Fail after a Terminate is a no-op | |
| 546 | + | // that leaves the orderly nil Err and RemoteFailureNone, exactly as a | |
| 547 | + | // Terminate after a Fail leaves the failure cause. Streams that were live at | |
| 548 | + | // the winning publication all observe that one outcome. | |
| 549 | + | func (e *StreamEngine) Fail(kind domain.RemoteFailureKind, err error) { | |
| 550 | + | if e == nil { | |
| 551 | + | return | |
| 552 | + | } | |
| 553 | + | e.mu.Lock() | |
| 554 | + | defer e.mu.Unlock() | |
| 555 | + | e.physical.Fail(kind, err) | |
| 556 | + | e.terminalizeAllLocked(func(stream *muxStream) { stream.terminal.Fail(kind, err) }) | |
| 557 | + | } | |
| 558 | + | ||
| 559 | + | // Terminate orderly closes the physical connection and then terminalizes every | |
| 560 | + | // live stream independently with a nil error. Done is closed before any stream | |
| 561 | + | // is terminalized. All queued bytes are released. | |
| 562 | + | // | |
| 563 | + | // Physical publication is first-wins: a Terminate after a Fail is a no-op | |
| 564 | + | // that leaves the failure cause and its kind, exactly as a Fail after a | |
| 565 | + | // Terminate leaves the orderly nil Err and RemoteFailureNone. | |
| 566 | + | func (e *StreamEngine) Terminate() { | |
| 567 | + | if e == nil { | |
| 568 | + | return | |
| 569 | + | } | |
| 570 | + | e.mu.Lock() | |
| 571 | + | defer e.mu.Unlock() | |
| 572 | + | e.physical.Close() | |
| 573 | + | e.terminalizeAllLocked(func(stream *muxStream) { stream.terminal.Close() }) | |
| 574 | + | } | |
| 575 | + | ||
| 576 | + | // Status returns one immutable read of a tracked stream and whether it exists. | |
| 577 | + | func (e *StreamEngine) Status(physical PhysicalStreamID) (StreamStatus, bool) { | |
| 578 | + | if e == nil { | |
| 579 | + | return StreamStatus{}, false | |
| 580 | + | } | |
| 581 | + | e.mu.Lock() | |
| 582 | + | defer e.mu.Unlock() | |
| 583 | + | stream := e.streams[physical] | |
| 584 | + | if stream == nil { | |
| 585 | + | return StreamStatus{}, false | |
| 586 | + | } | |
| 587 | + | return StreamStatus{ | |
| 588 | + | Ref: stream.ref, | |
| 589 | + | State: stream.state, | |
| 590 | + | Done: stream.terminal.Done(), | |
| 591 | + | Err: stream.terminal.Err(), | |
| 592 | + | FailureKind: stream.terminal.FailureKind(), | |
| 593 | + | Refused: stream.refused, | |
| 594 | + | Refusal: stream.refusal, | |
| 595 | + | QueuedChunks: len(stream.queue), | |
| 596 | + | QueuedBytes: stream.bytes, | |
| 597 | + | }, true | |
| 598 | + | } | |
| 599 | + | ||
| 600 | + | // cause returns the retained publish-once terminal authority of one admitted | |
| 601 | + | // stream. The pointer stays valid for the connection's lifetime even after the | |
| 602 | + | // engine evicts the bounded record, so a settled stream keeps its exact cause | |
| 603 | + | // independent of retention. It reports false for a physical identity the | |
| 604 | + | // engine never admitted. | |
| 605 | + | func (e *StreamEngine) cause(physical PhysicalStreamID) (*terminalState, bool) { | |
| 606 | + | if e == nil { | |
| 607 | + | return nil, false | |
| 608 | + | } | |
| 609 | + | e.mu.Lock() | |
| 610 | + | defer e.mu.Unlock() | |
| 611 | + | stream := e.streams[physical] | |
| 612 | + | if stream == nil { | |
| 613 | + | return nil, false | |
| 614 | + | } | |
| 615 | + | return stream.terminal, true | |
| 616 | + | } | |
| 617 | + | ||
| 618 | + | // Take removes and returns the oldest queued inbound chunk of one stream. | |
| 619 | + | // It reports false when the stream does not exist or its queue is empty. The | |
| 620 | + | // returned chunk is an engine-owned copy: it releases the chunk's bytes from | |
| 621 | + | // both budgets. Draining the last queued chunk of a closing stream terminalizes | |
| 622 | + | // it, releasing its admission slot and closing its Done. | |
| 623 | + | func (e *StreamEngine) Take(physical PhysicalStreamID) ([]byte, bool) { | |
| 624 | + | chunk, ok, _ := e.take(physical) | |
| 625 | + | return chunk, ok | |
| 626 | + | } | |
| 627 | + | ||
| 628 | + | // take is Take plus whether this pop drained a closing stream to its terminal | |
| 629 | + | // state, so the pump can rotate the stream's watch channel and wake a consumer | |
| 630 | + | // blocked on the queue transition. It holds the engine lock only for the pop | |
| 631 | + | // and the terminal publication. | |
| 632 | + | func (e *StreamEngine) take(physical PhysicalStreamID) ([]byte, bool, bool) { | |
| 633 | + | if e == nil { | |
| 634 | + | return nil, false, false | |
| 635 | + | } | |
| 636 | + | e.mu.Lock() | |
| 637 | + | defer e.mu.Unlock() | |
| 638 | + | stream := e.streams[physical] | |
| 639 | + | if stream == nil || len(stream.queue) == 0 { | |
| 640 | + | return nil, false, false | |
| 641 | + | } | |
| 642 | + | chunk := stream.queue[0] | |
| 643 | + | stream.queue[0] = nil | |
| 644 | + | stream.queue = stream.queue[1:] | |
| 645 | + | stream.bytes -= len(chunk) | |
| 646 | + | e.aggregate -= len(chunk) | |
| 647 | + | if stream.state == StreamClosing && len(stream.queue) == 0 { | |
| 648 | + | e.retireLocked(stream) | |
| 649 | + | stream.terminal.Close() | |
| 650 | + | return chunk, true, true | |
| 651 | + | } | |
| 652 | + | return chunk, true, false | |
| 653 | + | } | |
| 654 | + | ||
| 655 | + | // lookupLocked classifies one frame: the tracked record of a live or settled | |
| 656 | + | // stream, a retired stream at or below the high-water mark, or a future | |
| 657 | + | // identity that was never opened. | |
| 658 | + | func (e *StreamEngine) lookupLocked(id PhysicalStreamID) (*muxStream, StreamState, error) { | |
| 659 | + | if err := id.Validate(); err != nil { | |
| 660 | + | return nil, 0, ErrInvalidStream | |
| 661 | + | } | |
| 662 | + | if stream, ok := e.streams[id]; ok { | |
| 663 | + | return stream, stream.state, nil | |
| 664 | + | } | |
| 665 | + | if id > e.highest { | |
| 666 | + | return nil, 0, ErrFutureStream | |
| 667 | + | } | |
| 668 | + | return nil, StreamTerminal, nil | |
| 669 | + | } | |
| 670 | + | ||
| 671 | + | // terminalizeAllLocked publishes one terminal outcome on every live stream. | |
| 672 | + | // The physical outcome is published by the caller before this runs; settled | |
| 673 | + | // streams are left untouched so they keep their own cause. | |
| 674 | + | func (e *StreamEngine) terminalizeAllLocked(publish func(*muxStream)) { | |
| 675 | + | for _, stream := range e.streams { | |
| 676 | + | if stream.state == StreamTerminal { | |
| 677 | + | continue | |
| 678 | + | } | |
| 679 | + | e.retireLocked(stream) | |
| 680 | + | publish(stream) | |
| 681 | + | } | |
| 682 | + | } | |
| 683 | + | ||
| 684 | + | // retireLocked moves one stream to terminal, releases the bytes it holds, and | |
| 685 | + | // records it for bounded retention: the oldest retired record is evicted past | |
| 686 | + | // MaxRetiredStreamRecords. The high-water mark is never rolled back, so an | |
| 687 | + | // evicted identity still classifies as retired rather than future. | |
| 688 | + | func (e *StreamEngine) retireLocked(stream *muxStream) { | |
| 689 | + | e.releaseQueueLocked(stream) | |
| 690 | + | stream.state = StreamTerminal | |
| 691 | + | e.live-- | |
| 692 | + | e.retired = append(e.retired, stream.ref.Physical) | |
| 693 | + | for len(e.retired) > MaxRetiredStreamRecords { | |
| 694 | + | evicted := e.retired[0] | |
| 695 | + | e.retired = e.retired[1:] | |
| 696 | + | if tracked := e.streams[evicted]; tracked != nil && tracked.state == StreamTerminal { | |
| 697 | + | delete(e.streams, evicted) | |
| 698 | + | } | |
| 699 | + | } | |
| 700 | + | } | |
| 701 | + | ||
| 702 | + | // releaseQueueLocked drops every queued chunk and returns its bytes to the | |
| 703 | + | // aggregate budget. | |
| 704 | + | func (e *StreamEngine) releaseQueueLocked(stream *muxStream) { | |
| 705 | + | for i := range stream.queue { | |
| 706 | + | stream.queue[i] = nil | |
| 707 | + | } | |
| 708 | + | stream.queue = nil | |
| 709 | + | if stream.bytes != 0 { | |
| 710 | + | e.aggregate -= stream.bytes | |
| 711 | + | stream.bytes = 0 | |
| 712 | + | } | |
| 713 | + | } | |
| 714 | + | ||
| 715 | + | // queueOverflowsLocked reports whether one inbound chunk would exceed the | |
| 716 | + | // per-stream chunk bound, the per-stream byte bound, or the negotiated | |
| 717 | + | // aggregate byte bound. | |
| 718 | + | func (e *StreamEngine) queueOverflowsLocked(stream *muxStream, n int) bool { | |
| 719 | + | return len(stream.queue) >= e.maxStreamFrames || | |
| 720 | + | stream.bytes+n > e.maxStreamBytes || | |
| 721 | + | e.aggregate+n > e.maxAggregateBytes | |
| 722 | + | } | |
| 723 | + | ||
| 724 | + | // enqueueLocked charges the chunk against both budgets and only then copies it | |
| 725 | + | // into the queue, so a refused chunk is never allocated a queue-owned copy. | |
| 726 | + | func (e *StreamEngine) enqueueLocked(stream *muxStream, chunk []byte) { | |
| 727 | + | n := len(chunk) | |
| 728 | + | e.aggregate += n | |
| 729 | + | stream.bytes += n | |
| 730 | + | owned := make([]byte, n) | |
| 731 | + | copy(owned, chunk) | |
| 732 | + | stream.queue = append(stream.queue, owned) | |
| 733 | + | } | |
| 734 | + | ||
| 735 | + | // closedLocked reports whether the physical connection already reached its | |
| 736 | + | // terminal outcome. The caller holds the engine lock. | |
| 737 | + | func (e *StreamEngine) closedLocked() bool { | |
| 738 | + | select { | |
| 739 | + | case <-e.physical.Done(): | |
| 740 | + | return true | |
| 741 | + | default: | |
| 742 | + | return false | |
| 743 | + | } | |
| 744 | + | } | |
| 745 | + | ||
| 746 | + | // resetCause builds the stream-local terminal cause of one reset. A reset | |
| 747 | + | // without a declared detail publishes no cause and lets terminalState | |
| 748 | + | // substitute its failure sentinel. | |
| 749 | + | func resetCause(msg Reset) error { | |
| 750 | + | if !msg.HasError { | |
| 751 | + | return nil | |
| 752 | + | } | |
| 753 | + | return ports.BrokerError{Code: msg.Error.Code, Text: msg.Error.Text} | |
| 754 | + | } | |
A internal/adapters/daemonmux/engine_state_test.go +1160 -0
| @@ -0,0 +1,1160 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "errors" | |
| 5 | + | "fmt" | |
| 6 | + | "sync" | |
| 7 | + | "testing" | |
| 8 | + | ||
| 9 | + | "github.com/stretchr/testify/assert" | |
| 10 | + | "github.com/stretchr/testify/require" | |
| 11 | + | ||
| 12 | + | "github.com/bnema/vev/internal/domain" | |
| 13 | + | "github.com/bnema/vev/internal/ports" | |
| 14 | + | ) | |
| 15 | + | ||
| 16 | + | func testEngine() *StreamEngine { return NewStreamEngine() } | |
| 17 | + | ||
| 18 | + | // admitTestStream admits one stream in opening state. | |
| 19 | + | func admitTestStream(t *testing.T, e *StreamEngine, id PhysicalStreamID) StreamRef { | |
| 20 | + | t.Helper() | |
| 21 | + | ref := testRef(id) | |
| 22 | + | require.NoError(t, e.Open(Open{Ref: ref})) | |
| 23 | + | return ref | |
| 24 | + | } | |
| 25 | + | ||
| 26 | + | // openTestStream admits and confirms one stream. | |
| 27 | + | func openTestStream(t *testing.T, e *StreamEngine, id PhysicalStreamID) StreamRef { | |
| 28 | + | t.Helper() | |
| 29 | + | ref := admitTestStream(t, e, id) | |
| 30 | + | require.NoError(t, e.Opened(Opened{Ref: ref})) | |
| 31 | + | return ref | |
| 32 | + | } | |
| 33 | + | ||
| 34 | + | func mustStatus(t *testing.T, e *StreamEngine, id PhysicalStreamID) StreamStatus { | |
| 35 | + | t.Helper() | |
| 36 | + | status, ok := e.Status(id) | |
| 37 | + | require.True(t, ok, "stream %d is not tracked", id) | |
| 38 | + | return status | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | func channelClosed(ch <-chan struct{}) bool { | |
| 42 | + | select { | |
| 43 | + | case <-ch: | |
| 44 | + | return true | |
| 45 | + | default: | |
| 46 | + | return false | |
| 47 | + | } | |
| 48 | + | } | |
| 49 | + | ||
| 50 | + | func TestNewStreamEngine(t *testing.T) { | |
| 51 | + | e := NewStreamEngine() | |
| 52 | + | require.Zero(t, e.Live()) | |
| 53 | + | require.Zero(t, e.AggregateBytes()) | |
| 54 | + | require.False(t, e.Closed()) | |
| 55 | + | require.NoError(t, e.Err()) | |
| 56 | + | require.Equal(t, domain.RemoteFailureNone, e.FailureKind()) | |
| 57 | + | require.NotNil(t, e.Done()) | |
| 58 | + | } | |
| 59 | + | ||
| 60 | + | func TestStreamStateStrings(t *testing.T) { | |
| 61 | + | require.Equal(t, "opening", StreamOpening.String()) | |
| 62 | + | require.Equal(t, "open", StreamOpen.String()) | |
| 63 | + | require.Equal(t, "closing", StreamClosing.String()) | |
| 64 | + | require.Equal(t, "terminal", StreamTerminal.String()) | |
| 65 | + | require.Equal(t, "unknown", StreamState(0).String()) | |
| 66 | + | require.Equal(t, "unknown", StreamState(200).String()) | |
| 67 | + | } | |
| 68 | + | ||
| 69 | + | // TestStreamEngineOpenAdmission is the admission table: complete references, | |
| 70 | + | // strictly increasing identities that are never reused, and one shared | |
| 71 | + | // pending-plus-live ceiling. | |
| 72 | + | func TestStreamEngineOpenAdmission(t *testing.T) { | |
| 73 | + | t.Run("incomplete identities refused", func(t *testing.T) { | |
| 74 | + | e := testEngine() | |
| 75 | + | cases := []struct { | |
| 76 | + | name string | |
| 77 | + | ref StreamRef | |
| 78 | + | }{ | |
| 79 | + | {"zero ref", StreamRef{}}, | |
| 80 | + | {"zero physical", StreamRef{Physical: 0, Epoch: 7, Connection: testConnectionID(0x11), Client: 3}}, | |
| 81 | + | {"zero epoch", StreamRef{Physical: 1, Epoch: 0, Connection: testConnectionID(0x11), Client: 3}}, | |
| 82 | + | {"zero connection", StreamRef{Physical: 1, Epoch: 7, Client: 3}}, | |
| 83 | + | {"zero client", StreamRef{Physical: 1, Epoch: 7, Connection: testConnectionID(0x11), Client: 0}}, | |
| 84 | + | } | |
| 85 | + | for _, tc := range cases { | |
| 86 | + | require.ErrorIs(t, e.Open(Open{Ref: tc.ref}), ErrInvalidStream, tc.name) | |
| 87 | + | } | |
| 88 | + | require.Zero(t, e.Live()) | |
| 89 | + | _, ok := e.Status(1) | |
| 90 | + | require.False(t, ok) | |
| 91 | + | }) | |
| 92 | + | ||
| 93 | + | t.Run("strictly increasing and never reused", func(t *testing.T) { | |
| 94 | + | e := testEngine() | |
| 95 | + | admitTestStream(t, e, 5) | |
| 96 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(5)}), ErrStreamIDReused) | |
| 97 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(1)}), ErrStreamIDReused) | |
| 98 | + | require.NoError(t, e.Opened(Opened{Ref: testRef(5)})) | |
| 99 | + | admitTestStream(t, e, 6) | |
| 100 | + | ||
| 101 | + | disposition, err := e.Close(Close{Physical: 5}) | |
| 102 | + | require.NoError(t, err) | |
| 103 | + | require.Equal(t, StreamAccepted, disposition) | |
| 104 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(5)}), ErrStreamIDReused, "a settled ID is never reused") | |
| 105 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(6)}), ErrStreamIDReused) | |
| 106 | + | require.NoError(t, e.Open(Open{Ref: testRef(7)})) | |
| 107 | + | require.Equal(t, 2, e.Live()) | |
| 108 | + | }) | |
| 109 | + | ||
| 110 | + | t.Run("pending and live share the ceiling", func(t *testing.T) { | |
| 111 | + | e := testEngine() | |
| 112 | + | for i := 1; i <= int(MaxMuxStreams); i++ { | |
| 113 | + | require.NoError(t, e.Open(Open{Ref: testRef(PhysicalStreamID(i))})) | |
| 114 | + | } | |
| 115 | + | require.Equal(t, int(MaxMuxStreams), e.Live()) | |
| 116 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(PhysicalStreamID(MaxMuxStreams + 1))}), ErrTooManyStreams) | |
| 117 | + | ||
| 118 | + | // Confirming a pending stream does not free a slot: pending and live | |
| 119 | + | // share the one ceiling. | |
| 120 | + | require.NoError(t, e.Opened(Opened{Ref: testRef(1)})) | |
| 121 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(PhysicalStreamID(MaxMuxStreams + 2))}), ErrTooManyStreams) | |
| 122 | + | ||
| 123 | + | // Settling one stream frees exactly one slot for a fresh higher ID. | |
| 124 | + | disposition, err := e.Reset(Reset{Physical: 1}) | |
| 125 | + | require.NoError(t, err) | |
| 126 | + | require.Equal(t, StreamAccepted, disposition) | |
| 127 | + | require.Equal(t, int(MaxMuxStreams)-1, e.Live()) | |
| 128 | + | require.NoError(t, e.Open(Open{Ref: testRef(PhysicalStreamID(MaxMuxStreams + 2))})) | |
| 129 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(PhysicalStreamID(MaxMuxStreams + 3))}), ErrTooManyStreams) | |
| 130 | + | }) | |
| 131 | + | } | |
| 132 | + | ||
| 133 | + | // TestStreamEngineFutureAndRetired proves the high-water fence: an identity | |
| 134 | + | // above it was never opened and is refused, while traffic at or below it that | |
| 135 | + | // is no longer live belongs to a retired stream and is ignored. | |
| 136 | + | func TestStreamEngineFutureAndRetired(t *testing.T) { | |
| 137 | + | t.Run("future identity refused", func(t *testing.T) { | |
| 138 | + | e := testEngine() | |
| 139 | + | openTestStream(t, e, 2) | |
| 140 | + | future := PhysicalStreamID(3) | |
| 141 | + | require.ErrorIs(t, e.Opened(Opened{Ref: testRef(future)}), ErrFutureStream) | |
| 142 | + | require.ErrorIs(t, e.Refused(Refused{Ref: testRef(future)}), ErrFutureStream) | |
| 143 | + | disposition, err := e.Data(Data{Physical: future, Data: []byte{'x'}}) | |
| 144 | + | require.ErrorIs(t, err, ErrFutureStream) | |
| 145 | + | require.Zero(t, disposition) | |
| 146 | + | disposition, err = e.Close(Close{Physical: future}) | |
| 147 | + | require.ErrorIs(t, err, ErrFutureStream) | |
| 148 | + | require.Zero(t, disposition) | |
| 149 | + | disposition, err = e.Reset(Reset{Physical: future}) | |
| 150 | + | require.ErrorIs(t, err, ErrFutureStream) | |
| 151 | + | require.Zero(t, disposition) | |
| 152 | + | require.Equal(t, 1, e.Live()) | |
| 153 | + | }) | |
| 154 | + | ||
| 155 | + | t.Run("retired traffic ignored including gaps", func(t *testing.T) { | |
| 156 | + | e := testEngine() | |
| 157 | + | openTestStream(t, e, 5) | |
| 158 | + | disposition, err := e.Reset(Reset{Physical: 5}) | |
| 159 | + | require.NoError(t, err) | |
| 160 | + | require.Equal(t, StreamAccepted, disposition) | |
| 161 | + | openTestStream(t, e, 9) | |
| 162 | + | ||
| 163 | + | // 6 and 8 were never admitted but sit below the high-water mark: they | |
| 164 | + | // are retired traffic, not future streams. | |
| 165 | + | for _, id := range []PhysicalStreamID{5, 6, 8} { | |
| 166 | + | disposition, err := e.Data(Data{Physical: id, Data: []byte{'x'}}) | |
| 167 | + | require.NoError(t, err) | |
| 168 | + | require.Equal(t, StreamDiscarded, disposition) | |
| 169 | + | disposition, err = e.Close(Close{Physical: id}) | |
| 170 | + | require.NoError(t, err) | |
| 171 | + | require.Equal(t, StreamDiscarded, disposition) | |
| 172 | + | disposition, err = e.Reset(Reset{Physical: id}) | |
| 173 | + | require.NoError(t, err) | |
| 174 | + | require.Equal(t, StreamDiscarded, disposition) | |
| 175 | + | } | |
| 176 | + | ||
| 177 | + | // A retired identity can never be confirmed or refused again. | |
| 178 | + | require.ErrorIs(t, e.Opened(Opened{Ref: testRef(5)}), ErrStreamState) | |
| 179 | + | require.ErrorIs(t, e.Refused(Refused{Ref: testRef(5)}), ErrStreamState) | |
| 180 | + | require.ErrorIs(t, e.Opened(Opened{Ref: testRef(6)}), ErrStreamState) | |
| 181 | + | require.Equal(t, 1, e.Live()) | |
| 182 | + | }) | |
| 183 | + | } | |
| 184 | + | ||
| 185 | + | type engineStep struct { | |
| 186 | + | name string | |
| 187 | + | run func(*StreamEngine, PhysicalStreamID) (StreamDisposition, error) | |
| 188 | + | want StreamDisposition | |
| 189 | + | wantErr error | |
| 190 | + | } | |
| 191 | + | ||
| 192 | + | func stepAdmit(e *StreamEngine, id PhysicalStreamID) (StreamDisposition, error) { | |
| 193 | + | if err := e.Open(Open{Ref: testRef(id)}); err != nil { | |
| 194 | + | return 0, err | |
| 195 | + | } | |
| 196 | + | return StreamAccepted, nil | |
| 197 | + | } | |
| 198 | + | ||
| 199 | + | func stepOpened(e *StreamEngine, id PhysicalStreamID) (StreamDisposition, error) { | |
| 200 | + | if err := e.Opened(Opened{Ref: testRef(id)}); err != nil { | |
| 201 | + | return 0, err | |
| 202 | + | } | |
| 203 | + | return StreamAccepted, nil | |
| 204 | + | } | |
| 205 | + | ||
| 206 | + | func stepRefused(e *StreamEngine, id PhysicalStreamID) (StreamDisposition, error) { | |
| 207 | + | if err := e.Refused(Refused{Ref: testRef(id)}); err != nil { | |
| 208 | + | return 0, err | |
| 209 | + | } | |
| 210 | + | return StreamAccepted, nil | |
| 211 | + | } | |
| 212 | + | ||
| 213 | + | func stepData(e *StreamEngine, id PhysicalStreamID) (StreamDisposition, error) { | |
| 214 | + | return e.Data(Data{Physical: id, Data: []byte{'d'}}) | |
| 215 | + | } | |
| 216 | + | ||
| 217 | + | func stepClose(e *StreamEngine, id PhysicalStreamID) (StreamDisposition, error) { | |
| 218 | + | return e.Close(Close{Physical: id}) | |
| 219 | + | } | |
| 220 | + | ||
| 221 | + | func stepReset(e *StreamEngine, id PhysicalStreamID) (StreamDisposition, error) { | |
| 222 | + | return e.Reset(Reset{Physical: id}) | |
| 223 | + | } | |
| 224 | + | ||
| 225 | + | // stepTake drains one queued chunk, reporting StreamAccepted when a chunk was | |
| 226 | + | // taken and StreamDiscarded when the queue was already empty. It is the drain | |
| 227 | + | // half of the orderly-close lifecycle: taking the last chunk of a closing | |
| 228 | + | // stream terminalizes it. | |
| 229 | + | func stepTake(e *StreamEngine, id PhysicalStreamID) (StreamDisposition, error) { | |
| 230 | + | if _, ok := e.Take(id); ok { | |
| 231 | + | return StreamAccepted, nil | |
| 232 | + | } | |
| 233 | + | return StreamDiscarded, nil | |
| 234 | + | } | |
| 235 | + | ||
| 236 | + | // TestStreamEngineLifecycle is the stream state table: opening -> open -> | |
| 237 | + | // terminal, with legal Opened/Refused only from opening, data only while open, | |
| 238 | + | // and late traffic discarded. | |
| 239 | + | func TestStreamEngineLifecycle(t *testing.T) { | |
| 240 | + | cases := []struct { | |
| 241 | + | name string | |
| 242 | + | steps []engineStep | |
| 243 | + | }{ | |
| 244 | + | { | |
| 245 | + | name: "confirmed then closed", | |
| 246 | + | steps: []engineStep{ | |
| 247 | + | {"admit", stepAdmit, StreamAccepted, nil}, | |
| 248 | + | {"data while opening", stepData, 0, ErrStreamState}, | |
| 249 | + | {"opened", stepOpened, StreamAccepted, nil}, | |
| 250 | + | {"opened twice", stepOpened, 0, ErrStreamState}, | |
| 251 | + | {"refused when open", stepRefused, 0, ErrStreamState}, | |
| 252 | + | {"close", stepClose, StreamAccepted, nil}, | |
| 253 | + | {"late data", stepData, StreamDiscarded, nil}, | |
| 254 | + | {"late close", stepClose, StreamDiscarded, nil}, | |
| 255 | + | {"late reset", stepReset, StreamDiscarded, nil}, | |
| 256 | + | {"opened after close", stepOpened, 0, ErrStreamState}, | |
| 257 | + | {"refused after close", stepRefused, 0, ErrStreamState}, | |
| 258 | + | }, | |
| 259 | + | }, | |
| 260 | + | { | |
| 261 | + | name: "orderly close preserves queued data until drained", | |
| 262 | + | steps: []engineStep{ | |
| 263 | + | {"admit", stepAdmit, StreamAccepted, nil}, | |
| 264 | + | {"opened", stepOpened, StreamAccepted, nil}, | |
| 265 | + | {"data when open", stepData, StreamAccepted, nil}, | |
| 266 | + | {"close with unread data", stepClose, StreamAccepted, nil}, | |
| 267 | + | {"close twice", stepClose, StreamDiscarded, nil}, | |
| 268 | + | {"data after close refused", stepData, 0, ErrStreamState}, | |
| 269 | + | {"drain the preserved chunk", stepTake, StreamAccepted, nil}, | |
| 270 | + | {"late close", stepClose, StreamDiscarded, nil}, | |
| 271 | + | {"late data", stepData, StreamDiscarded, nil}, | |
| 272 | + | {"late reset", stepReset, StreamDiscarded, nil}, | |
| 273 | + | {"opened after close", stepOpened, 0, ErrStreamState}, | |
| 274 | + | }, | |
| 275 | + | }, | |
| 276 | + | { | |
| 277 | + | name: "refused from opening", | |
| 278 | + | steps: []engineStep{ | |
| 279 | + | {"admit", stepAdmit, StreamAccepted, nil}, | |
| 280 | + | {"refused", stepRefused, StreamAccepted, nil}, | |
| 281 | + | {"refused twice", stepRefused, 0, ErrStreamState}, | |
| 282 | + | {"opened after refusal", stepOpened, 0, ErrStreamState}, | |
| 283 | + | {"data after refusal", stepData, StreamDiscarded, nil}, | |
| 284 | + | }, | |
| 285 | + | }, | |
| 286 | + | { | |
| 287 | + | name: "close while opening is terminal", | |
| 288 | + | steps: []engineStep{ | |
| 289 | + | {"admit", stepAdmit, StreamAccepted, nil}, | |
| 290 | + | {"close", stepClose, StreamAccepted, nil}, | |
| 291 | + | {"data after close", stepData, StreamDiscarded, nil}, | |
| 292 | + | {"opened after close", stepOpened, 0, ErrStreamState}, | |
| 293 | + | }, | |
| 294 | + | }, | |
| 295 | + | { | |
| 296 | + | name: "reset while opening is terminal and idempotent", | |
| 297 | + | steps: []engineStep{ | |
| 298 | + | {"admit", stepAdmit, StreamAccepted, nil}, | |
| 299 | + | {"reset", stepReset, StreamAccepted, nil}, | |
| 300 | + | {"reset idempotent", stepReset, StreamDiscarded, nil}, | |
| 301 | + | {"close idempotent", stepClose, StreamDiscarded, nil}, | |
| 302 | + | }, | |
| 303 | + | }, | |
| 304 | + | } | |
| 305 | + | for _, tc := range cases { | |
| 306 | + | t.Run(tc.name, func(t *testing.T) { | |
| 307 | + | e := testEngine() | |
| 308 | + | id := PhysicalStreamID(4) | |
| 309 | + | for _, s := range tc.steps { | |
| 310 | + | disposition, err := s.run(e, id) | |
| 311 | + | if s.wantErr != nil { | |
| 312 | + | require.ErrorIs(t, err, s.wantErr, s.name) | |
| 313 | + | require.Zero(t, disposition, s.name) | |
| 314 | + | continue | |
| 315 | + | } | |
| 316 | + | require.NoError(t, err, s.name) | |
| 317 | + | require.Equal(t, s.want, disposition, s.name) | |
| 318 | + | } | |
| 319 | + | require.Zero(t, e.Live()) | |
| 320 | + | require.Equal(t, StreamTerminal, mustStatus(t, e, id).State) | |
| 321 | + | }) | |
| 322 | + | } | |
| 323 | + | } | |
| 324 | + | ||
| 325 | + | // TestStreamEngineRejectedAdmissionLeavesNothing proves a refused admission | |
| 326 | + | // records nothing and cannot resurrect a settled identity. | |
| 327 | + | func TestStreamEngineRejectedAdmissionLeavesNothing(t *testing.T) { | |
| 328 | + | e := testEngine() | |
| 329 | + | require.ErrorIs(t, e.Open(Open{}), ErrInvalidStream) | |
| 330 | + | require.Zero(t, e.Live()) | |
| 331 | + | _, ok := e.Status(1) | |
| 332 | + | require.False(t, ok) | |
| 333 | + | ||
| 334 | + | admitTestStream(t, e, 1) | |
| 335 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(1)}), ErrStreamIDReused) | |
| 336 | + | require.Equal(t, 1, e.Live()) | |
| 337 | + | require.Equal(t, StreamOpening, mustStatus(t, e, 1).State) | |
| 338 | + | _, ok = e.Status(2) | |
| 339 | + | require.False(t, ok) | |
| 340 | + | } | |
| 341 | + | ||
| 342 | + | // TestStreamEngineRefFence proves Opened and Refused are legal only for the | |
| 343 | + | // opening stream whose full reference was recorded at admission. | |
| 344 | + | func TestStreamEngineRefFence(t *testing.T) { | |
| 345 | + | e := testEngine() | |
| 346 | + | ref := admitTestStream(t, e, 4) | |
| 347 | + | ||
| 348 | + | mismatched := ref | |
| 349 | + | mismatched.Epoch++ | |
| 350 | + | require.ErrorIs(t, e.Opened(Opened{Ref: mismatched}), ErrStreamRefMismatch) | |
| 351 | + | require.Equal(t, StreamOpening, mustStatus(t, e, 4).State) | |
| 352 | + | require.ErrorIs(t, e.Refused(Refused{Ref: mismatched}), ErrStreamRefMismatch) | |
| 353 | + | require.Equal(t, StreamOpening, mustStatus(t, e, 4).State) | |
| 354 | + | ||
| 355 | + | require.NoError(t, e.Opened(Opened{Ref: ref})) | |
| 356 | + | require.Equal(t, StreamOpen, mustStatus(t, e, 4).State) | |
| 357 | + | ||
| 358 | + | // Legality is checked before reference identity: a settled stream refuses | |
| 359 | + | // both a matching and a mismatched confirmation. | |
| 360 | + | require.ErrorIs(t, e.Opened(Opened{Ref: ref}), ErrStreamState) | |
| 361 | + | require.ErrorIs(t, e.Opened(Opened{Ref: mismatched}), ErrStreamState) | |
| 362 | + | } | |
| 363 | + | ||
| 364 | + | // TestStreamEngineRefusalRetained proves a refusal settles the stream | |
| 365 | + | // orderly and retains the typed refusal for the caller. | |
| 366 | + | func TestStreamEngineRefusalRetained(t *testing.T) { | |
| 367 | + | e := testEngine() | |
| 368 | + | admitTestStream(t, e, 2) | |
| 369 | + | detail := testErrorDetail() | |
| 370 | + | require.NoError(t, e.Refused(Refused{Ref: testRef(2), Error: detail})) | |
| 371 | + | ||
| 372 | + | status := mustStatus(t, e, 2) | |
| 373 | + | require.Equal(t, StreamTerminal, status.State) | |
| 374 | + | require.True(t, status.Refused) | |
| 375 | + | require.Equal(t, detail, status.Refusal) | |
| 376 | + | require.True(t, channelClosed(status.Done)) | |
| 377 | + | require.NoError(t, status.Err) | |
| 378 | + | require.Equal(t, domain.RemoteFailureNone, status.FailureKind) | |
| 379 | + | require.Zero(t, e.Live()) | |
| 380 | + | } | |
| 381 | + | ||
| 382 | + | // TestStreamEngineTwoAndHundredStreams proves admission, confirmation, data, | |
| 383 | + | // and orderly close for a small and a large concurrent set. A stream whose | |
| 384 | + | // already-queued chunk was drained before the Close terminalizes immediately; | |
| 385 | + | // every other stream preserves its queued chunk while closing until Take | |
| 386 | + | // drains it. | |
| 387 | + | func TestStreamEngineTwoAndHundredStreams(t *testing.T) { | |
| 388 | + | for _, count := range []int{2, 100} { | |
| 389 | + | t.Run(fmt.Sprintf("%d streams", count), func(t *testing.T) { | |
| 390 | + | e := testEngine() | |
| 391 | + | for i := 1; i <= count; i++ { | |
| 392 | + | ref := openTestStream(t, e, PhysicalStreamID(i)) | |
| 393 | + | require.Equal(t, ref, mustStatus(t, e, PhysicalStreamID(i)).Ref) | |
| 394 | + | } | |
| 395 | + | require.Equal(t, count, e.Live()) | |
| 396 | + | ||
| 397 | + | for i := 1; i <= count; i++ { | |
| 398 | + | disposition, err := e.Data(Data{Physical: PhysicalStreamID(i), Data: []byte{byte(i)}}) | |
| 399 | + | require.NoError(t, err) | |
| 400 | + | require.Equal(t, StreamAccepted, disposition) | |
| 401 | + | } | |
| 402 | + | require.Equal(t, count, e.AggregateBytes()) | |
| 403 | + | ||
| 404 | + | taken, ok := e.Take(PhysicalStreamID(1)) | |
| 405 | + | require.True(t, ok) | |
| 406 | + | require.Equal(t, []byte{1}, taken) | |
| 407 | + | require.Equal(t, count-1, e.AggregateBytes()) | |
| 408 | + | require.Zero(t, mustStatus(t, e, 1).QueuedChunks) | |
| 409 | + | ||
| 410 | + | for i := 1; i <= count; i++ { | |
| 411 | + | disposition, err := e.Close(Close{Physical: PhysicalStreamID(i)}) | |
| 412 | + | require.NoError(t, err) | |
| 413 | + | require.Equal(t, StreamAccepted, disposition) | |
| 414 | + | } | |
| 415 | + | // The drained stream is terminal; every other stream is closing and | |
| 416 | + | // still holds exactly its one unread, charged chunk. | |
| 417 | + | require.Equal(t, count-1, e.Live()) | |
| 418 | + | require.Equal(t, count-1, e.AggregateBytes()) | |
| 419 | + | require.False(t, e.Closed(), "closing every stream never terminalizes the physical connection") | |
| 420 | + | terminal := mustStatus(t, e, 1) | |
| 421 | + | require.Equal(t, StreamTerminal, terminal.State) | |
| 422 | + | require.True(t, channelClosed(terminal.Done)) | |
| 423 | + | for i := 2; i <= count; i++ { | |
| 424 | + | status := mustStatus(t, e, PhysicalStreamID(i)) | |
| 425 | + | require.Equal(t, StreamClosing, status.State, "stream %d", i) | |
| 426 | + | require.False(t, channelClosed(status.Done), "stream %d", i) | |
| 427 | + | require.Equal(t, 1, status.QueuedChunks, "stream %d", i) | |
| 428 | + | } | |
| 429 | + | ||
| 430 | + | // Draining every preserved chunk releases its bytes and terminalizes | |
| 431 | + | // the stream. | |
| 432 | + | for i := 2; i <= count; i++ { | |
| 433 | + | chunk, ok := e.Take(PhysicalStreamID(i)) | |
| 434 | + | require.True(t, ok, "stream %d", i) | |
| 435 | + | require.Equal(t, []byte{byte(i)}, chunk, "stream %d", i) | |
| 436 | + | } | |
| 437 | + | require.Zero(t, e.Live()) | |
| 438 | + | require.Zero(t, e.AggregateBytes()) | |
| 439 | + | require.False(t, e.Closed(), "closing every stream never terminalizes the physical connection") | |
| 440 | + | for i := 1; i <= count; i++ { | |
| 441 | + | status := mustStatus(t, e, PhysicalStreamID(i)) | |
| 442 | + | require.Equal(t, StreamTerminal, status.State) | |
| 443 | + | require.True(t, channelClosed(status.Done)) | |
| 444 | + | require.NoError(t, status.Err) | |
| 445 | + | } | |
| 446 | + | }) | |
| 447 | + | } | |
| 448 | + | } | |
| 449 | + | ||
| 450 | + | // TestStreamEngineIsolation proves one stream's close or reset terminalizes | |
| 451 | + | // neither the physical connection nor a sibling. | |
| 452 | + | func TestStreamEngineIsolation(t *testing.T) { | |
| 453 | + | e := testEngine() | |
| 454 | + | openTestStream(t, e, 1) | |
| 455 | + | openTestStream(t, e, 2) | |
| 456 | + | openTestStream(t, e, 3) | |
| 457 | + | _, err := e.Data(Data{Physical: 1, Data: []byte("a")}) | |
| 458 | + | require.NoError(t, err) | |
| 459 | + | ||
| 460 | + | disposition, err := e.Close(Close{Physical: 2}) | |
| 461 | + | require.NoError(t, err) | |
| 462 | + | require.Equal(t, StreamAccepted, disposition) | |
| 463 | + | require.False(t, e.Closed()) | |
| 464 | + | require.NoError(t, e.Err()) | |
| 465 | + | require.Equal(t, StreamOpen, mustStatus(t, e, 1).State) | |
| 466 | + | require.Equal(t, StreamOpen, mustStatus(t, e, 3).State) | |
| 467 | + | require.Equal(t, 2, e.Live()) | |
| 468 | + | require.Equal(t, 1, e.AggregateBytes()) | |
| 469 | + | ||
| 470 | + | disposition, err = e.Reset(Reset{Physical: 3}) | |
| 471 | + | require.NoError(t, err) | |
| 472 | + | require.Equal(t, StreamAccepted, disposition) | |
| 473 | + | require.False(t, e.Closed()) | |
| 474 | + | require.Equal(t, StreamOpen, mustStatus(t, e, 1).State) | |
| 475 | + | require.Equal(t, 1, e.Live()) | |
| 476 | + | ||
| 477 | + | disposition, err = e.Data(Data{Physical: 1, Data: []byte("b")}) | |
| 478 | + | require.NoError(t, err) | |
| 479 | + | require.Equal(t, StreamAccepted, disposition) | |
| 480 | + | require.Equal(t, 2, e.AggregateBytes()) | |
| 481 | + | } | |
| 482 | + | ||
| 483 | + | // TestStreamEngineQueueOverflow proves a stalled queue resets exactly its own | |
| 484 | + | // stream, frees that stream's budget, and leaves a sibling progressing. | |
| 485 | + | func TestStreamEngineQueueOverflow(t *testing.T) { | |
| 486 | + | e := testEngine() | |
| 487 | + | openTestStream(t, e, 1) | |
| 488 | + | openTestStream(t, e, 2) | |
| 489 | + | _, err := e.Data(Data{Physical: 2, Data: []byte("sibling")}) | |
| 490 | + | require.NoError(t, err) | |
| 491 | + | ||
| 492 | + | for i := 0; i < MaxMuxStreamQueueChunks; i++ { | |
| 493 | + | disposition, err := e.Data(Data{Physical: 1, Data: []byte{byte(i)}}) | |
| 494 | + | require.NoError(t, err) | |
| 495 | + | require.Equal(t, StreamAccepted, disposition) | |
| 496 | + | } | |
| 497 | + | require.Equal(t, MaxMuxStreamQueueChunks, mustStatus(t, e, 1).QueuedChunks) | |
| 498 | + | ||
| 499 | + | disposition, err := e.Data(Data{Physical: 1, Data: []byte{'x'}}) | |
| 500 | + | require.ErrorIs(t, err, ErrStreamQueueFull) | |
| 501 | + | require.Zero(t, disposition) | |
| 502 | + | ||
| 503 | + | status := mustStatus(t, e, 1) | |
| 504 | + | require.Equal(t, StreamTerminal, status.State) | |
| 505 | + | require.True(t, channelClosed(status.Done)) | |
| 506 | + | require.ErrorIs(t, status.Err, ErrStreamQueueFull) | |
| 507 | + | require.Equal(t, domain.RemoteFailureTransport, status.FailureKind) | |
| 508 | + | require.Zero(t, status.QueuedChunks) | |
| 509 | + | require.Zero(t, status.QueuedBytes) | |
| 510 | + | require.Equal(t, len("sibling"), e.AggregateBytes(), "the overflowed stream released its budget") | |
| 511 | + | require.Equal(t, 1, e.Live()) | |
| 512 | + | require.False(t, e.Closed()) | |
| 513 | + | ||
| 514 | + | sibling := mustStatus(t, e, 2) | |
| 515 | + | require.Equal(t, StreamOpen, sibling.State) | |
| 516 | + | require.False(t, channelClosed(sibling.Done)) | |
| 517 | + | disposition, err = e.Data(Data{Physical: 2, Data: []byte("more")}) | |
| 518 | + | require.NoError(t, err) | |
| 519 | + | require.Equal(t, StreamAccepted, disposition) | |
| 520 | + | require.Equal(t, len("sibling")+len("more"), e.AggregateBytes()) | |
| 521 | + | ||
| 522 | + | // Cancellation of an already terminal stream stays idempotent and never | |
| 523 | + | // rewrites its cause. | |
| 524 | + | disposition, err = e.Reset(Reset{Physical: 1}) | |
| 525 | + | require.NoError(t, err) | |
| 526 | + | require.Equal(t, StreamDiscarded, disposition) | |
| 527 | + | require.ErrorIs(t, mustStatus(t, e, 1).Err, ErrStreamQueueFull) | |
| 528 | + | } | |
| 529 | + | ||
| 530 | + | // TestStreamEngineQueueByteBounds proves the per-stream and aggregate byte | |
| 531 | + | // ceilings reset only the offending stream. | |
| 532 | + | func TestStreamEngineQueueByteBounds(t *testing.T) { | |
| 533 | + | t.Run("per-stream byte bound", func(t *testing.T) { | |
| 534 | + | e := testEngine() | |
| 535 | + | // The codec never admits a chunk above the negotiated chunk ceiling; | |
| 536 | + | // raise the engine's private ceiling so this test can reach the | |
| 537 | + | // defensive per-stream byte bound directly. | |
| 538 | + | e.maxChunkBytes = MaxMuxStreamQueueBytes | |
| 539 | + | openTestStream(t, e, 1) | |
| 540 | + | chunk := make([]byte, MaxMuxStreamQueueBytes/2+1) | |
| 541 | + | disposition, err := e.Data(Data{Physical: 1, Data: chunk}) | |
| 542 | + | require.NoError(t, err) | |
| 543 | + | require.Equal(t, StreamAccepted, disposition) | |
| 544 | + | require.Equal(t, len(chunk), e.AggregateBytes()) | |
| 545 | + | ||
| 546 | + | // The second chunk fits the chunk-count and aggregate bounds but | |
| 547 | + | // exceeds the per-stream byte bound. | |
| 548 | + | disposition, err = e.Data(Data{Physical: 1, Data: chunk}) | |
| 549 | + | require.ErrorIs(t, err, ErrStreamQueueFull) | |
| 550 | + | require.Zero(t, disposition) | |
| 551 | + | require.Zero(t, e.AggregateBytes(), "the reset stream released its bytes") | |
| 552 | + | require.Equal(t, StreamTerminal, mustStatus(t, e, 1).State) | |
| 553 | + | }) | |
| 554 | + | ||
| 555 | + | t.Run("aggregate byte bound", func(t *testing.T) { | |
| 556 | + | e := testEngine() | |
| 557 | + | // As above: bypass the codec-level chunk ceiling to reach the | |
| 558 | + | // defensive aggregate byte bound with oversized chunks. | |
| 559 | + | e.maxChunkBytes = 16 << 20 | |
| 560 | + | chunk := make([]byte, 16<<20) | |
| 561 | + | for i := 1; i <= 4; i++ { | |
| 562 | + | openTestStream(t, e, PhysicalStreamID(i)) | |
| 563 | + | disposition, err := e.Data(Data{Physical: PhysicalStreamID(i), Data: chunk}) | |
| 564 | + | require.NoError(t, err) | |
| 565 | + | require.Equal(t, StreamAccepted, disposition) | |
| 566 | + | } | |
| 567 | + | require.Equal(t, int(MaxMuxAggregateBytes), e.AggregateBytes()) | |
| 568 | + | ||
| 569 | + | openTestStream(t, e, 5) | |
| 570 | + | disposition, err := e.Data(Data{Physical: 5, Data: chunk}) | |
| 571 | + | require.ErrorIs(t, err, ErrStreamQueueFull) | |
| 572 | + | require.Zero(t, disposition) | |
| 573 | + | require.Equal(t, StreamTerminal, mustStatus(t, e, 5).State) | |
| 574 | + | require.Equal(t, int(MaxMuxAggregateBytes), e.AggregateBytes(), "siblings keep their budget") | |
| 575 | + | for i := 1; i <= 4; i++ { | |
| 576 | + | require.Equal(t, StreamOpen, mustStatus(t, e, PhysicalStreamID(i)).State) | |
| 577 | + | } | |
| 578 | + | require.Equal(t, 4, e.Live()) | |
| 579 | + | ||
| 580 | + | // Taking one sibling chunk releases exactly its bytes, so a fresh | |
| 581 | + | // stream takes the freed capacity. | |
| 582 | + | _, ok := e.Take(PhysicalStreamID(1)) | |
| 583 | + | require.True(t, ok) | |
| 584 | + | require.Equal(t, int(MaxMuxAggregateBytes)-len(chunk), e.AggregateBytes()) | |
| 585 | + | openTestStream(t, e, 6) | |
| 586 | + | disposition, err = e.Data(Data{Physical: 6, Data: chunk}) | |
| 587 | + | require.NoError(t, err) | |
| 588 | + | require.Equal(t, StreamAccepted, disposition) | |
| 589 | + | require.Equal(t, int(MaxMuxAggregateBytes), e.AggregateBytes()) | |
| 590 | + | }) | |
| 591 | + | } | |
| 592 | + | ||
| 593 | + | // TestStreamEngineTake proves the FIFO pop returns an engine-owned copy and | |
| 594 | + | // releases the chunk's bytes. | |
| 595 | + | func TestStreamEngineTake(t *testing.T) { | |
| 596 | + | e := testEngine() | |
| 597 | + | openTestStream(t, e, 1) | |
| 598 | + | chunk := []byte("payload") | |
| 599 | + | _, err := e.Data(Data{Physical: 1, Data: chunk}) | |
| 600 | + | require.NoError(t, err) | |
| 601 | + | ||
| 602 | + | chunk[0] = 'P' // mutating the caller's slice must not change the queue | |
| 603 | + | taken, ok := e.Take(PhysicalStreamID(1)) | |
| 604 | + | require.True(t, ok) | |
| 605 | + | require.Equal(t, []byte("payload"), taken) | |
| 606 | + | require.Zero(t, e.AggregateBytes()) | |
| 607 | + | ||
| 608 | + | _, ok = e.Take(PhysicalStreamID(1)) | |
| 609 | + | require.False(t, ok) | |
| 610 | + | _, ok = e.Take(PhysicalStreamID(9)) | |
| 611 | + | require.False(t, ok) | |
| 612 | + | } | |
| 613 | + | ||
| 614 | + | // TestStreamEngineResetCause proves a reset is abnormal: it carries the typed | |
| 615 | + | // cause when present and the terminal sentinel otherwise. | |
| 616 | + | func TestStreamEngineResetCause(t *testing.T) { | |
| 617 | + | t.Run("typed detail", func(t *testing.T) { | |
| 618 | + | e := testEngine() | |
| 619 | + | openTestStream(t, e, 1) | |
| 620 | + | detail := ErrorDetail{ | |
| 621 | + | Code: ports.BrokerErrorUnavailable, Text: "reset by peer", | |
| 622 | + | AdmissionCode: 3, FailureKind: domain.RemoteFailureTimeout, | |
| 623 | + | } | |
| 624 | + | disposition, err := e.Reset(Reset{Physical: 1, HasError: true, Error: detail}) | |
| 625 | + | require.NoError(t, err) | |
| 626 | + | require.Equal(t, StreamAccepted, disposition) | |
| 627 | + | ||
| 628 | + | status := mustStatus(t, e, 1) | |
| 629 | + | require.Error(t, status.Err) | |
| 630 | + | require.Equal(t, domain.RemoteFailureTimeout, status.FailureKind) | |
| 631 | + | var brokerErr ports.BrokerError | |
| 632 | + | require.ErrorAs(t, status.Err, &brokerErr) | |
| 633 | + | require.Equal(t, ports.BrokerErrorUnavailable, brokerErr.Code) | |
| 634 | + | require.Equal(t, "reset by peer", brokerErr.Text) | |
| 635 | + | }) | |
| 636 | + | ||
| 637 | + | t.Run("no detail", func(t *testing.T) { | |
| 638 | + | e := testEngine() | |
| 639 | + | openTestStream(t, e, 1) | |
| 640 | + | disposition, err := e.Reset(Reset{Physical: 1}) | |
| 641 | + | require.NoError(t, err) | |
| 642 | + | require.Equal(t, StreamAccepted, disposition) | |
| 643 | + | ||
| 644 | + | status := mustStatus(t, e, 1) | |
| 645 | + | require.ErrorIs(t, status.Err, ErrTerminalFailure) | |
| 646 | + | require.Equal(t, domain.RemoteFailureTransport, status.FailureKind) | |
| 647 | + | }) | |
| 648 | + | } | |
| 649 | + | ||
| 650 | + | // TestStreamEnginePhysicalFailureOrdering proves a physical failure closes | |
| 651 | + | // Done before terminalizing every live stream, that every stream observes the | |
| 652 | + | // physical outcome, and that a stream settled beforehand keeps its own cause. | |
| 653 | + | func TestStreamEnginePhysicalFailureOrdering(t *testing.T) { | |
| 654 | + | e := testEngine() | |
| 655 | + | const live = 5 | |
| 656 | + | for i := 1; i <= live; i++ { | |
| 657 | + | openTestStream(t, e, PhysicalStreamID(i)) | |
| 658 | + | } | |
| 659 | + | _, err := e.Reset(Reset{Physical: 5, HasError: true, Error: testErrorDetail()}) | |
| 660 | + | require.NoError(t, err) | |
| 661 | + | siblingCause := mustStatus(t, e, 5).Err | |
| 662 | + | require.Error(t, siblingCause) | |
| 663 | + | ||
| 664 | + | cause := fmt.Errorf("physical loss: %w", errTerminalSentinel) | |
| 665 | + | observed := make([]bool, live-1) | |
| 666 | + | var wg sync.WaitGroup | |
| 667 | + | for i := 1; i < live; i++ { | |
| 668 | + | status := mustStatus(t, e, PhysicalStreamID(i)) | |
| 669 | + | wg.Add(1) | |
| 670 | + | go func(i int, done <-chan struct{}) { | |
| 671 | + | defer wg.Done() | |
| 672 | + | <-done | |
| 673 | + | // A reader woken by a stream must observe the physical | |
| 674 | + | // connection already terminal. | |
| 675 | + | observed[i-1] = e.Closed() | |
| 676 | + | }(i, status.Done) | |
| 677 | + | } | |
| 678 | + | e.Fail(domain.RemoteFailureTimeout, cause) | |
| 679 | + | wg.Wait() | |
| 680 | + | ||
| 681 | + | require.True(t, e.Closed()) | |
| 682 | + | require.Equal(t, cause, e.Err()) | |
| 683 | + | require.ErrorIs(t, e.Err(), errTerminalSentinel) | |
| 684 | + | require.Equal(t, domain.RemoteFailureTimeout, e.FailureKind()) | |
| 685 | + | require.Zero(t, e.Live()) | |
| 686 | + | require.Zero(t, e.AggregateBytes()) | |
| 687 | + | ||
| 688 | + | for i := 1; i < live; i++ { | |
| 689 | + | status := mustStatus(t, e, PhysicalStreamID(i)) | |
| 690 | + | require.Equal(t, StreamTerminal, status.State) | |
| 691 | + | require.True(t, channelClosed(status.Done)) | |
| 692 | + | require.Equal(t, cause, status.Err) | |
| 693 | + | require.Equal(t, domain.RemoteFailureTimeout, status.FailureKind) | |
| 694 | + | require.True(t, observed[i-1], "stream %d was terminalized before the physical Done", i) | |
| 695 | + | } | |
| 696 | + | ||
| 697 | + | // The stream settled before the loss keeps its own, independent cause. | |
| 698 | + | settled := mustStatus(t, e, 5) | |
| 699 | + | require.Equal(t, StreamTerminal, settled.State) | |
| 700 | + | require.Equal(t, siblingCause, settled.Err) | |
| 701 | + | } | |
| 702 | + | ||
| 703 | + | // TestStreamEngineTerminate proves an orderly physical close terminalizes | |
| 704 | + | // every stream with a nil error and refuses further admission. | |
| 705 | + | func TestStreamEngineTerminate(t *testing.T) { | |
| 706 | + | e := testEngine() | |
| 707 | + | for i := 1; i <= 3; i++ { | |
| 708 | + | openTestStream(t, e, PhysicalStreamID(i)) | |
| 709 | + | } | |
| 710 | + | _, err := e.Data(Data{Physical: 1, Data: []byte("queued")}) | |
| 711 | + | require.NoError(t, err) | |
| 712 | + | ||
| 713 | + | e.Terminate() | |
| 714 | + | require.True(t, e.Closed()) | |
| 715 | + | require.NoError(t, e.Err()) | |
| 716 | + | require.Equal(t, domain.RemoteFailureNone, e.FailureKind()) | |
| 717 | + | require.Zero(t, e.Live()) | |
| 718 | + | require.Zero(t, e.AggregateBytes()) | |
| 719 | + | for i := 1; i <= 3; i++ { | |
| 720 | + | status := mustStatus(t, e, PhysicalStreamID(i)) | |
| 721 | + | require.Equal(t, StreamTerminal, status.State) | |
| 722 | + | require.True(t, channelClosed(status.Done)) | |
| 723 | + | require.NoError(t, status.Err) | |
| 724 | + | } | |
| 725 | + | ||
| 726 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(9)}), ErrPhysicalClosed) | |
| 727 | + | e.Terminate() | |
| 728 | + | require.True(t, e.Closed()) | |
| 729 | + | } | |
| 730 | + | ||
| 731 | + | // TestStreamEnginePhysicalClosedFrames proves frames after a physical terminal | |
| 732 | + | // outcome stay classified: settled streams discard traffic, admission is | |
| 733 | + | // closed, and a future identity is still refused. | |
| 734 | + | func TestStreamEnginePhysicalClosedFrames(t *testing.T) { | |
| 735 | + | e := testEngine() | |
| 736 | + | openTestStream(t, e, 1) | |
| 737 | + | e.Fail(domain.RemoteFailureTransport, errors.New("gone")) | |
| 738 | + | ||
| 739 | + | for _, run := range []struct { | |
| 740 | + | name string | |
| 741 | + | run func() (StreamDisposition, error) | |
| 742 | + | }{ | |
| 743 | + | {"data", func() (StreamDisposition, error) { return e.Data(Data{Physical: 1, Data: []byte{'x'}}) }}, | |
| 744 | + | {"close", func() (StreamDisposition, error) { return e.Close(Close{Physical: 1}) }}, | |
| 745 | + | {"reset", func() (StreamDisposition, error) { return e.Reset(Reset{Physical: 1}) }}, | |
| 746 | + | } { | |
| 747 | + | disposition, err := run.run() | |
| 748 | + | require.NoError(t, err, run.name) | |
| 749 | + | require.Equal(t, StreamDiscarded, disposition, run.name) | |
| 750 | + | } | |
| 751 | + | ||
| 752 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(2)}), ErrPhysicalClosed) | |
| 753 | + | _, err := e.Data(Data{Physical: 3, Data: []byte{'x'}}) | |
| 754 | + | require.ErrorIs(t, err, ErrFutureStream) | |
| 755 | + | } | |
| 756 | + | ||
| 757 | + | // TestStreamEngineConcurrentStreams exercises concurrent confirmation, data, | |
| 758 | + | // orderly close, abnormal reset, and admission over one engine. | |
| 759 | + | func TestStreamEngineConcurrentStreams(t *testing.T) { | |
| 760 | + | const streams = 64 | |
| 761 | + | const admitted = 32 | |
| 762 | + | e := testEngine() | |
| 763 | + | for i := 1; i <= streams; i++ { | |
| 764 | + | admitTestStream(t, e, PhysicalStreamID(i)) | |
| 765 | + | } | |
| 766 | + | ||
| 767 | + | var wg sync.WaitGroup | |
| 768 | + | start := make(chan struct{}) | |
| 769 | + | for i := 1; i <= streams; i++ { | |
| 770 | + | wg.Add(1) | |
| 771 | + | go func(i int) { | |
| 772 | + | defer wg.Done() | |
| 773 | + | <-start | |
| 774 | + | id := PhysicalStreamID(i) | |
| 775 | + | assert.NoError(t, e.Opened(Opened{Ref: testRef(id)})) | |
| 776 | + | disposition, err := e.Data(Data{Physical: id, Data: []byte{byte(i)}}) | |
| 777 | + | assert.NoError(t, err) | |
| 778 | + | assert.Equal(t, StreamAccepted, disposition) | |
| 779 | + | if i%2 == 0 { | |
| 780 | + | disposition, err = e.Close(Close{Physical: id}) | |
| 781 | + | } else { | |
| 782 | + | disposition, err = e.Reset(Reset{Physical: id}) | |
| 783 | + | } | |
| 784 | + | assert.NoError(t, err) | |
| 785 | + | assert.Equal(t, StreamAccepted, disposition) | |
| 786 | + | }(i) | |
| 787 | + | } | |
| 788 | + | // One goroutine keeps admitting fresh, strictly increasing identities | |
| 789 | + | // while the others settle theirs. | |
| 790 | + | wg.Add(1) | |
| 791 | + | go func() { | |
| 792 | + | defer wg.Done() | |
| 793 | + | <-start | |
| 794 | + | for i := 1; i <= admitted; i++ { | |
| 795 | + | id := PhysicalStreamID(streams + i) | |
| 796 | + | assert.NoError(t, e.Open(Open{Ref: testRef(id)})) | |
| 797 | + | assert.NoError(t, e.Opened(Opened{Ref: testRef(id)})) | |
| 798 | + | disposition, err := e.Close(Close{Physical: id}) | |
| 799 | + | assert.NoError(t, err) | |
| 800 | + | assert.Equal(t, StreamAccepted, disposition) | |
| 801 | + | } | |
| 802 | + | }() | |
| 803 | + | close(start) | |
| 804 | + | wg.Wait() | |
| 805 | + | ||
| 806 | + | // The even streams were closed with one chunk still unread, so they are | |
| 807 | + | // closing and still charged; draining each queue terminalizes it. Odd | |
| 808 | + | // streams and the admitted-only goroutine are already terminal. | |
| 809 | + | for i := 1; i <= streams+admitted; i++ { | |
| 810 | + | for { | |
| 811 | + | if _, ok := e.Take(PhysicalStreamID(i)); !ok { | |
| 812 | + | break | |
| 813 | + | } | |
| 814 | + | } | |
| 815 | + | } | |
| 816 | + | ||
| 817 | + | require.Zero(t, e.Live()) | |
| 818 | + | require.Zero(t, e.AggregateBytes()) | |
| 819 | + | require.False(t, e.Closed()) | |
| 820 | + | for i := 1; i <= streams+admitted; i++ { | |
| 821 | + | require.Equal(t, StreamTerminal, mustStatus(t, e, PhysicalStreamID(i)).State) | |
| 822 | + | } | |
| 823 | + | } | |
| 824 | + | ||
| 825 | + | // TestStreamEngineConcurrentPhysicalFailure races stream work against a | |
| 826 | + | // physical failure and proves the engine settles consistently. | |
| 827 | + | func TestStreamEngineConcurrentPhysicalFailure(t *testing.T) { | |
| 828 | + | const streams = 64 | |
| 829 | + | e := testEngine() | |
| 830 | + | for i := 1; i <= streams; i++ { | |
| 831 | + | openTestStream(t, e, PhysicalStreamID(i)) | |
| 832 | + | } | |
| 833 | + | cause := errors.New("physical loss") | |
| 834 | + | ||
| 835 | + | var wg sync.WaitGroup | |
| 836 | + | start := make(chan struct{}) | |
| 837 | + | for i := 1; i <= streams; i++ { | |
| 838 | + | wg.Add(1) | |
| 839 | + | go func(i int) { | |
| 840 | + | defer wg.Done() | |
| 841 | + | <-start | |
| 842 | + | id := PhysicalStreamID(i) | |
| 843 | + | switch i % 3 { | |
| 844 | + | case 0: | |
| 845 | + | _, _ = e.Data(Data{Physical: id, Data: []byte{byte(i)}}) | |
| 846 | + | case 1: | |
| 847 | + | _, _ = e.Close(Close{Physical: id}) | |
| 848 | + | default: | |
| 849 | + | _, _ = e.Reset(Reset{Physical: id}) | |
| 850 | + | } | |
| 851 | + | }(i) | |
| 852 | + | } | |
| 853 | + | wg.Add(1) | |
| 854 | + | go func() { | |
| 855 | + | defer wg.Done() | |
| 856 | + | <-start | |
| 857 | + | e.Fail(domain.RemoteFailureTimeout, cause) | |
| 858 | + | }() | |
| 859 | + | close(start) | |
| 860 | + | wg.Wait() | |
| 861 | + | ||
| 862 | + | require.True(t, e.Closed()) | |
| 863 | + | require.Equal(t, cause, e.Err()) | |
| 864 | + | require.Zero(t, e.Live()) | |
| 865 | + | require.Zero(t, e.AggregateBytes()) | |
| 866 | + | for i := 1; i <= streams; i++ { | |
| 867 | + | status := mustStatus(t, e, PhysicalStreamID(i)) | |
| 868 | + | require.Equal(t, StreamTerminal, status.State) | |
| 869 | + | require.True(t, channelClosed(status.Done)) | |
| 870 | + | } | |
| 871 | + | } | |
| 872 | + | ||
| 873 | + | // TestStreamEngineNilSafety proves the nil-receiver paths. | |
| 874 | + | func TestStreamEngineNilSafety(t *testing.T) { | |
| 875 | + | var e *StreamEngine | |
| 876 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(1)}), ErrPhysicalClosed) | |
| 877 | + | require.ErrorIs(t, e.Opened(Opened{Ref: testRef(1)}), ErrPhysicalClosed) | |
| 878 | + | require.ErrorIs(t, e.Refused(Refused{Ref: testRef(1)}), ErrPhysicalClosed) | |
| 879 | + | _, err := e.Data(Data{Physical: 1, Data: []byte{'x'}}) | |
| 880 | + | require.ErrorIs(t, err, ErrPhysicalClosed) | |
| 881 | + | _, err = e.Close(Close{Physical: 1}) | |
| 882 | + | require.ErrorIs(t, err, ErrPhysicalClosed) | |
| 883 | + | _, err = e.Reset(Reset{Physical: 1}) | |
| 884 | + | require.ErrorIs(t, err, ErrPhysicalClosed) | |
| 885 | + | require.Zero(t, e.Live()) | |
| 886 | + | require.Zero(t, e.AggregateBytes()) | |
| 887 | + | require.True(t, e.Closed()) | |
| 888 | + | require.Nil(t, e.Done()) | |
| 889 | + | require.NoError(t, e.Err()) | |
| 890 | + | require.Equal(t, domain.RemoteFailureNone, e.FailureKind()) | |
| 891 | + | _, ok := e.Status(1) | |
| 892 | + | require.False(t, ok) | |
| 893 | + | _, ok = e.Take(1) | |
| 894 | + | require.False(t, ok) | |
| 895 | + | e.Fail(domain.RemoteFailureTransport, errors.New("ignored")) | |
| 896 | + | e.Terminate() | |
| 897 | + | } | |
| 898 | + | ||
| 899 | + | // TestNewStreamEngineWithCeilings proves the constructor validates the | |
| 900 | + | // negotiated advertisement, enforces the negotiated maxima, and copies the | |
| 901 | + | // value so a caller cannot mutate live enforcement. | |
| 902 | + | func TestNewStreamEngineWithCeilings(t *testing.T) { | |
| 903 | + | t.Run("invalid advertisement refused", func(t *testing.T) { | |
| 904 | + | bad := DefaultMuxCeilings() | |
| 905 | + | bad.MaxStreams = 0 | |
| 906 | + | engine, err := NewStreamEngineWithCeilings(bad) | |
| 907 | + | require.ErrorIs(t, err, ErrInvalidCeilings) | |
| 908 | + | require.Nil(t, engine) | |
| 909 | + | ||
| 910 | + | bad = DefaultMuxCeilings() | |
| 911 | + | bad.StreamChunkLimit = MaxMuxChunkBytes + 1 | |
| 912 | + | _, err = NewStreamEngineWithCeilings(bad) | |
| 913 | + | require.ErrorIs(t, err, ErrInvalidCeilings) | |
| 914 | + | ||
| 915 | + | _, err = NewStreamEngineWithCeilings(MuxCeilings{}) | |
| 916 | + | require.ErrorIs(t, err, ErrInvalidCeilings) | |
| 917 | + | }) | |
| 918 | + | ||
| 919 | + | t.Run("negotiated small limits enforced", func(t *testing.T) { | |
| 920 | + | ceilings := MuxCeilings{ | |
| 921 | + | MaxReceiveEnvelopeBytes: MinMuxEnvelopeBytes, | |
| 922 | + | StreamChunkLimit: 32768, | |
| 923 | + | MaxStreams: 2, | |
| 924 | + | MaxAggregateBytes: MinMuxAggregateBytes, | |
| 925 | + | } | |
| 926 | + | e, err := NewStreamEngineWithCeilings(ceilings) | |
| 927 | + | require.NoError(t, err) | |
| 928 | + | openTestStream(t, e, 1) | |
| 929 | + | openTestStream(t, e, 2) | |
| 930 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(3)}), ErrTooManyStreams) | |
| 931 | + | ||
| 932 | + | // The negotiated chunk ceiling is refused statelessly, mutating | |
| 933 | + | // nothing. | |
| 934 | + | _, err = e.Data(Data{Physical: 1, Data: make([]byte, 32769)}) | |
| 935 | + | require.ErrorIs(t, err, ErrTooLarge) | |
| 936 | + | require.Equal(t, StreamOpen, mustStatus(t, e, 1).State) | |
| 937 | + | require.Zero(t, e.AggregateBytes()) | |
| 938 | + | ||
| 939 | + | // Two full chunks fit the aggregate floor; a third would exceed it and | |
| 940 | + | // resets exactly the offending stream. | |
| 941 | + | _, err = e.Data(Data{Physical: 1, Data: make([]byte, 32768)}) | |
| 942 | + | require.NoError(t, err) | |
| 943 | + | require.Equal(t, 32768, e.AggregateBytes()) | |
| 944 | + | _, err = e.Data(Data{Physical: 2, Data: make([]byte, 32768)}) | |
| 945 | + | require.NoError(t, err) | |
| 946 | + | require.Equal(t, 65536, e.AggregateBytes()) | |
| 947 | + | _, err = e.Data(Data{Physical: 1, Data: make([]byte, 32768)}) | |
| 948 | + | require.ErrorIs(t, err, ErrStreamQueueFull) | |
| 949 | + | require.Equal(t, StreamTerminal, mustStatus(t, e, 1).State) | |
| 950 | + | require.Equal(t, 32768, e.AggregateBytes(), "the reset stream released its bytes") | |
| 951 | + | require.Equal(t, StreamOpen, mustStatus(t, e, 2).State) | |
| 952 | + | }) | |
| 953 | + | ||
| 954 | + | t.Run("ceilings are copied", func(t *testing.T) { | |
| 955 | + | ceilings := DefaultMuxCeilings() | |
| 956 | + | ceilings.MaxStreams = 1 | |
| 957 | + | e, err := NewStreamEngineWithCeilings(ceilings) | |
| 958 | + | require.NoError(t, err) | |
| 959 | + | // A caller mutating its own copy must not raise the engine's ceiling. | |
| 960 | + | ceilings.MaxStreams = MaxMuxStreams | |
| 961 | + | openTestStream(t, e, 1) | |
| 962 | + | require.ErrorIs(t, e.Open(Open{Ref: testRef(2)}), ErrTooManyStreams) | |
| 963 | + | }) | |
| 964 | + | } | |
| 965 | + | ||
| 966 | + | // TestStreamEngineBoundedRetiredRecords proves a churned long-lived engine | |
| 967 | + | // keeps a bounded table: the oldest retired records are evicted, while the | |
| 968 | + | // high-water mark still classifies them as retired rather than future. | |
| 969 | + | func TestStreamEngineBoundedRetiredRecords(t *testing.T) { | |
| 970 | + | e := testEngine() | |
| 971 | + | const churn = 4000 | |
| 972 | + | for i := 1; i <= churn; i++ { | |
| 973 | + | id := PhysicalStreamID(i) | |
| 974 | + | openTestStream(t, e, id) | |
| 975 | + | _, err := e.Close(Close{Physical: id}) | |
| 976 | + | require.NoError(t, err) | |
| 977 | + | } | |
| 978 | + | require.LessOrEqual(t, len(e.streams), MaxRetiredStreamRecords) | |
| 979 | + | require.Len(t, e.retired, MaxRetiredStreamRecords) | |
| 980 | + | ||
| 981 | + | // The newest identity is retained; the oldest record was evicted but its | |
| 982 | + | // high-water mark still classifies late traffic as retired. | |
| 983 | + | require.Equal(t, StreamTerminal, mustStatus(t, e, PhysicalStreamID(churn)).State) | |
| 984 | + | _, ok := e.Status(PhysicalStreamID(1)) | |
| 985 | + | require.False(t, ok) | |
| 986 | + | disposition, err := e.Data(Data{Physical: PhysicalStreamID(1), Data: []byte{'x'}}) | |
| 987 | + | require.NoError(t, err) | |
| 988 | + | require.Equal(t, StreamDiscarded, disposition) | |
| 989 | + | ||
| 990 | + | // An identity above the high-water mark is still refused, not discarded. | |
| 991 | + | _, err = e.Data(Data{Physical: PhysicalStreamID(churn + 1), Data: []byte{'x'}}) | |
| 992 | + | require.ErrorIs(t, err, ErrFutureStream) | |
| 993 | + | } | |
| 994 | + | ||
| 995 | + | // TestStreamEnginePhysicalFirstWins proves the physical terminal publication | |
| 996 | + | // is first-wins: whichever of Fail or Terminate publishes first fixes the | |
| 997 | + | // connection and every live stream's outcome. | |
| 998 | + | func TestStreamEnginePhysicalFirstWins(t *testing.T) { | |
| 999 | + | t.Run("failure wins an orderly close", func(t *testing.T) { | |
| 1000 | + | e := testEngine() | |
| 1001 | + | openTestStream(t, e, 1) | |
| 1002 | + | cause := errors.New("physical loss") | |
| 1003 | + | e.Fail(domain.RemoteFailureTimeout, cause) | |
| 1004 | + | e.Terminate() | |
| 1005 | + | require.True(t, e.Closed()) | |
| 1006 | + | require.Equal(t, cause, e.Err()) | |
| 1007 | + | require.Equal(t, domain.RemoteFailureTimeout, e.FailureKind()) | |
| 1008 | + | status := mustStatus(t, e, 1) | |
| 1009 | + | require.Equal(t, cause, status.Err) | |
| 1010 | + | require.Equal(t, domain.RemoteFailureTimeout, status.FailureKind) | |
| 1011 | + | }) | |
| 1012 | + | ||
| 1013 | + | t.Run("orderly close wins a failure", func(t *testing.T) { | |
| 1014 | + | e := testEngine() | |
| 1015 | + | openTestStream(t, e, 1) | |
| 1016 | + | e.Terminate() | |
| 1017 | + | e.Fail(domain.RemoteFailureTimeout, errors.New("late")) | |
| 1018 | + | require.True(t, e.Closed()) | |
| 1019 | + | require.NoError(t, e.Err()) | |
| 1020 | + | require.Equal(t, domain.RemoteFailureNone, e.FailureKind()) | |
| 1021 | + | status := mustStatus(t, e, 1) | |
| 1022 | + | require.NoError(t, status.Err) | |
| 1023 | + | require.Equal(t, domain.RemoteFailureNone, status.FailureKind) | |
| 1024 | + | }) | |
| 1025 | + | } | |
| 1026 | + | ||
| 1027 | + | // TestStreamEngineClosingPreservesQueuedData proves the orderly-close | |
| 1028 | + | // semantics: a Close that finds unread data marks the stream closing instead of | |
| 1029 | + | // discarding it, no new data is accepted, already-accepted chunks stay readable | |
| 1030 | + | // in order, and the first Take that empties the queue terminalizes the stream | |
| 1031 | + | // and releases its slot and bytes. A sibling is untouched throughout. | |
| 1032 | + | func TestStreamEngineClosingPreservesQueuedData(t *testing.T) { | |
| 1033 | + | e := testEngine() | |
| 1034 | + | closingRef := openTestStream(t, e, 1) | |
| 1035 | + | siblingRef := openTestStream(t, e, 2) | |
| 1036 | + | ||
| 1037 | + | for _, chunk := range []struct { | |
| 1038 | + | physical PhysicalStreamID | |
| 1039 | + | data string | |
| 1040 | + | }{{1, "first"}, {1, "second"}, {2, "sibling"}} { | |
| 1041 | + | disposition, err := e.Data(Data{Physical: chunk.physical, Data: []byte(chunk.data)}) | |
| 1042 | + | require.NoError(t, err) | |
| 1043 | + | require.Equal(t, StreamAccepted, disposition) | |
| 1044 | + | } | |
| 1045 | + | require.Equal(t, len("first")+len("second")+len("sibling"), e.AggregateBytes()) | |
| 1046 | + | ||
| 1047 | + | // The orderly Close preserves both accepted chunks. | |
| 1048 | + | disposition, err := e.Close(Close{Physical: 1}) | |
| 1049 | + | require.NoError(t, err) | |
| 1050 | + | require.Equal(t, StreamAccepted, disposition) | |
| 1051 | + | status := mustStatus(t, e, 1) | |
| 1052 | + | require.Equal(t, StreamClosing, status.State) | |
| 1053 | + | require.Equal(t, closingRef, status.Ref) | |
| 1054 | + | require.False(t, channelClosed(status.Done), "a closing stream is not terminal yet") | |
| 1055 | + | require.Equal(t, 2, status.QueuedChunks) | |
| 1056 | + | require.Equal(t, len("first")+len("second"), status.QueuedBytes) | |
| 1057 | + | require.Equal(t, 2, e.Live(), "the closing stream still holds its admission slot") | |
| 1058 | + | require.Equal(t, len("first")+len("second")+len("sibling"), e.AggregateBytes()) | |
| 1059 | + | ||
| 1060 | + | // No new data is accepted while closing, and a second Close changes nothing. | |
| 1061 | + | _, err = e.Data(Data{Physical: 1, Data: []byte("late")}) | |
| 1062 | + | require.ErrorIs(t, err, ErrStreamState) | |
| 1063 | + | disposition, err = e.Close(Close{Physical: 1}) | |
| 1064 | + | require.NoError(t, err) | |
| 1065 | + | require.Equal(t, StreamDiscarded, disposition) | |
| 1066 | + | require.Equal(t, 2, mustStatus(t, e, 1).QueuedChunks, "a duplicate close preserved the queue") | |
| 1067 | + | ||
| 1068 | + | // Draining preserves order and releases bytes chunk by chunk; only the last | |
| 1069 | + | // take terminalizes the stream. | |
| 1070 | + | chunk, ok := e.Take(1) | |
| 1071 | + | require.True(t, ok) | |
| 1072 | + | require.Equal(t, []byte("first"), chunk) | |
| 1073 | + | require.Equal(t, StreamClosing, mustStatus(t, e, 1).State, "still closing with one chunk left") | |
| 1074 | + | require.Equal(t, 2, e.Live()) | |
| 1075 | + | require.Equal(t, len("second")+len("sibling"), e.AggregateBytes()) | |
| 1076 | + | ||
| 1077 | + | chunk, ok = e.Take(1) | |
| 1078 | + | require.True(t, ok) | |
| 1079 | + | require.Equal(t, []byte("second"), chunk) | |
| 1080 | + | status = mustStatus(t, e, 1) | |
| 1081 | + | require.Equal(t, StreamTerminal, status.State) | |
| 1082 | + | require.True(t, channelClosed(status.Done)) | |
| 1083 | + | require.Zero(t, status.QueuedChunks) | |
| 1084 | + | require.NoError(t, status.Err) | |
| 1085 | + | require.Equal(t, 1, e.Live(), "the drained stream released its slot") | |
| 1086 | + | require.Equal(t, len("sibling"), e.AggregateBytes(), "only the sibling's bytes remain") | |
| 1087 | + | ||
| 1088 | + | // A drained closing stream accepts no second drain and no new frame. | |
| 1089 | + | _, ok = e.Take(1) | |
| 1090 | + | require.False(t, ok) | |
| 1091 | + | disposition, err = e.Close(Close{Physical: 1}) | |
| 1092 | + | require.NoError(t, err) | |
| 1093 | + | require.Equal(t, StreamDiscarded, disposition) | |
| 1094 | + | ||
| 1095 | + | // The sibling is untouched and still holds its own queued chunk. | |
| 1096 | + | siblingStatus := mustStatus(t, e, 2) | |
| 1097 | + | require.Equal(t, StreamOpen, siblingStatus.State) | |
| 1098 | + | require.Equal(t, siblingRef, siblingStatus.Ref) | |
| 1099 | + | require.Equal(t, 1, siblingStatus.QueuedChunks) | |
| 1100 | + | chunk, ok = e.Take(2) | |
| 1101 | + | require.True(t, ok) | |
| 1102 | + | require.Equal(t, []byte("sibling"), chunk) | |
| 1103 | + | } | |
| 1104 | + | ||
| 1105 | + | // TestStreamEngineClosingResetDiscardsImmediately proves a Reset abandons a | |
| 1106 | + | // closing stream's preserved queue at once: the bytes, the admission slot, and | |
| 1107 | + | // the terminal outcome are released without waiting for a consumer. | |
| 1108 | + | func TestStreamEngineClosingResetDiscardsImmediately(t *testing.T) { | |
| 1109 | + | e := testEngine() | |
| 1110 | + | openTestStream(t, e, 1) | |
| 1111 | + | _, err := e.Data(Data{Physical: 1, Data: []byte("queued")}) | |
| 1112 | + | require.NoError(t, err) | |
| 1113 | + | disposition, err := e.Close(Close{Physical: 1}) | |
| 1114 | + | require.NoError(t, err) | |
| 1115 | + | require.Equal(t, StreamAccepted, disposition) | |
| 1116 | + | require.Equal(t, StreamClosing, mustStatus(t, e, 1).State) | |
| 1117 | + | ||
| 1118 | + | disposition, err = e.Reset(Reset{Physical: 1}) | |
| 1119 | + | require.NoError(t, err) | |
| 1120 | + | require.Equal(t, StreamAccepted, disposition) | |
| 1121 | + | status := mustStatus(t, e, 1) | |
| 1122 | + | require.Equal(t, StreamTerminal, status.State) | |
| 1123 | + | require.True(t, channelClosed(status.Done)) | |
| 1124 | + | require.Zero(t, status.QueuedChunks) | |
| 1125 | + | require.Zero(t, status.QueuedBytes) | |
| 1126 | + | require.Zero(t, e.AggregateBytes()) | |
| 1127 | + | require.Zero(t, e.Live()) | |
| 1128 | + | _, ok := e.Take(1) | |
| 1129 | + | require.False(t, ok, "a reset discarded the preserved queue") | |
| 1130 | + | } | |
| 1131 | + | ||
| 1132 | + | // TestStreamEngineClosingAccountingBounded proves the closing accounting is | |
| 1133 | + | // balanced across many cycles and stays bounded: a closing stream keeps its | |
| 1134 | + | // slot and bytes until the draining Take releases them, and the retired record | |
| 1135 | + | // set never grows past its retention bound. | |
| 1136 | + | func TestStreamEngineClosingAccountingBounded(t *testing.T) { | |
| 1137 | + | e := testEngine() | |
| 1138 | + | const churn = MaxRetiredStreamRecords * 3 | |
| 1139 | + | for i := 1; i <= churn; i++ { | |
| 1140 | + | id := PhysicalStreamID(i) | |
| 1141 | + | openTestStream(t, e, id) | |
| 1142 | + | _, err := e.Data(Data{Physical: id, Data: []byte{byte(i)}}) | |
| 1143 | + | require.NoError(t, err) | |
| 1144 | + | disposition, err := e.Close(Close{Physical: id}) | |
| 1145 | + | require.NoError(t, err) | |
| 1146 | + | require.Equal(t, StreamAccepted, disposition) | |
| 1147 | + | require.Equal(t, StreamClosing, mustStatus(t, e, id).State) | |
| 1148 | + | require.Equal(t, 1, e.Live()) | |
| 1149 | + | require.Equal(t, 1, e.AggregateBytes()) | |
| 1150 | + | ||
| 1151 | + | chunk, ok := e.Take(id) | |
| 1152 | + | require.True(t, ok) | |
| 1153 | + | require.Equal(t, []byte{byte(i)}, chunk) | |
| 1154 | + | require.Equal(t, StreamTerminal, mustStatus(t, e, id).State) | |
| 1155 | + | require.Zero(t, e.Live()) | |
| 1156 | + | require.Zero(t, e.AggregateBytes()) | |
| 1157 | + | } | |
| 1158 | + | require.Len(t, e.retired, MaxRetiredStreamRecords) | |
| 1159 | + | require.LessOrEqual(t, len(e.streams), MaxRetiredStreamRecords) | |
| 1160 | + | } | |
A internal/adapters/daemonmux/handshake.go +291 -0
| @@ -0,0 +1,291 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | // Physical preamble handshake (P3.2a). | |
| 4 | + | // | |
| 5 | + | // The daemonmux connection begins with exactly one physical preamble exchange | |
| 6 | + | // over the private FramedCarrier, before any application envelope. The client | |
| 7 | + | // (broker) sends one MuxPreambleRequest carrying its advertised ceilings and | |
| 8 | + | // the connection policy it needs; the server (daemon) answers with exactly one | |
| 9 | + | // MuxPreambleResponse carrying either the effective (minimum) ceilings, the | |
| 10 | + | // authoritative daemon identity and incarnation, and the accepted policy, or a | |
| 11 | + | // typed refusal. The message pair is the existing stateless preamble codec | |
| 12 | + | // (preamble.go): this file is only the stateful exchange and the authority | |
| 13 | + | // checks on top of it. Magic, epoch, exact protocol.Version, roles 5/6, zero | |
| 14 | + | // capability bits, the 4 KiB preamble bound, and valid/refusal exclusivity are | |
| 15 | + | // enforced by that codec. | |
| 16 | + | // | |
| 17 | + | // The client verifies the accepted response against the independently resolved | |
| 18 | + | // endpoint it dialed: the response identity must exactly equal | |
| 19 | + | // ports.BrokerResolvedEndpoint.Identity, the accepted policy must exactly equal | |
| 20 | + | // the endpoint policy, the incarnation must be nonzero, and the effective | |
| 21 | + | // ceilings must never exceed what the client offered. The response is the | |
| 22 | + | // server's authority, never a value the client supplied and got echoed back. | |
| 23 | + | // | |
| 24 | + | // The server verifies the request policy against its immutable ServerBinding | |
| 25 | + | // (binding.go) instead of echoing the requested policy: an accepted answer | |
| 26 | + | // carries the binding's identity, incarnation, and policy, and any request | |
| 27 | + | // whose policy differs in a single field is refused. The server negotiates the | |
| 28 | + | // element-wise minima of its local advertisement and the request, so a peer | |
| 29 | + | // can never grant or demand more than either side offered. | |
| 30 | + | // | |
| 31 | + | // One caller-supplied context (or its deadline) bounds the whole exchange: the | |
| 32 | + | // same value goes to the carrier's Send and Receive, no timer is created here, | |
| 33 | + | // and there is no internal retry or grace period. Every failed handshake - | |
| 34 | + | // local validation failure, carrier error, peer refusal, identity or policy | |
| 35 | + | // mismatch, or an inadmissible advertisement - closes the carrier before | |
| 36 | + | // returning, so no half-open physical connection survives a refusal. A | |
| 37 | + | // successful handshake leaves the carrier open and untouched for its owner: | |
| 38 | + | // the handshake never starts a Pump, never admits a stream, and never wraps | |
| 39 | + | // the connection in ports values. The caller starts the Pump only after this | |
| 40 | + | // function returns nil. | |
| 41 | + | // | |
| 42 | + | // Authentication premise: the handshake does not authenticate the carrier and | |
| 43 | + | // does not prove an identity in band. The carriage must already be | |
| 44 | + | // authenticated, and the expected endpoint binding is authoritative. The | |
| 45 | + | // identity, incarnation, and policy here are authorization and pooling | |
| 46 | + | // metadata on top of that premise, not a substitute for it. | |
| 47 | + | ||
| 48 | + | import ( | |
| 49 | + | "context" | |
| 50 | + | "errors" | |
| 51 | + | "fmt" | |
| 52 | + | ||
| 53 | + | "google.golang.org/protobuf/proto" | |
| 54 | + | ||
| 55 | + | "github.com/bnema/vev/internal/ports" | |
| 56 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 57 | + | ) | |
| 58 | + | ||
| 59 | + | var ( | |
| 60 | + | // ErrHandshakeRejected reports a physical preamble handshake that was | |
| 61 | + | // refused by the peer or failed a local authority check on an otherwise | |
| 62 | + | // well-formed response. A peer refusal is reported as a *HandshakeRefusal | |
| 63 | + | // carrying its precise code and also matches errors.Is(ErrHandshakeRejected). | |
| 64 | + | ErrHandshakeRejected = errors.New("daemonmux: physical handshake rejected") | |
| 65 | + | // ErrHandshakeConfig reports a handshake called without a carrier, with an | |
| 66 | + | // invalid endpoint or binding, or with an invalid local ceiling | |
| 67 | + | // advertisement. It is a caller bug, not a peer refusal. | |
| 68 | + | ErrHandshakeConfig = errors.New("daemonmux: invalid handshake configuration") | |
| 69 | + | ) | |
| 70 | + | ||
| 71 | + | // HandshakeRefusal is the typed refusal a physical preamble handshake returns: | |
| 72 | + | // the precise PreambleRejectionCode the peer sent or the server sent. | |
| 73 | + | type HandshakeRefusal struct { | |
| 74 | + | Code uint32 | |
| 75 | + | } | |
| 76 | + | ||
| 77 | + | func (r *HandshakeRefusal) Error() string { | |
| 78 | + | return fmt.Sprintf("daemonmux: physical handshake refused with code %d", r.Code) | |
| 79 | + | } | |
| 80 | + | ||
| 81 | + | // Is classifies every typed refusal as ErrHandshakeRejected so a caller can | |
| 82 | + | // test with errors.Is without losing the precise code carried on the value. | |
| 83 | + | func (r *HandshakeRefusal) Is(target error) bool { return target == ErrHandshakeRejected } | |
| 84 | + | ||
| 85 | + | // ClientHandshakeResult is the broker's validated physical handshake outcome: | |
| 86 | + | // the negotiated (effective) ceilings, and the daemon binding the server | |
| 87 | + | // proved against the independently resolved endpoint. | |
| 88 | + | type ClientHandshakeResult struct { | |
| 89 | + | Ceilings MuxCeilings | |
| 90 | + | Identity ports.BrokerDaemonIdentity | |
| 91 | + | Incarnation ports.BrokerDaemonIncarnation | |
| 92 | + | Policy ports.BrokerPolicy | |
| 93 | + | } | |
| 94 | + | ||
| 95 | + | // ServerHandshakeResult is the daemon's validated physical handshake outcome: | |
| 96 | + | // the negotiated (effective) ceilings and the binding the server enforced. | |
| 97 | + | type ServerHandshakeResult struct { | |
| 98 | + | Ceilings MuxCeilings | |
| 99 | + | Binding ServerBinding | |
| 100 | + | } | |
| 101 | + | ||
| 102 | + | // RunClientHandshake performs the broker side of the daemonmux physical | |
| 103 | + | // preamble over carrier. It offers offered and the resolved endpoint's policy, | |
| 104 | + | // then accepts exactly one response and verifies it against the endpoint: | |
| 105 | + | // identity and policy must match exactly, the incarnation must be nonzero, and | |
| 106 | + | // the effective ceilings must not exceed the offer. It returns a typed | |
| 107 | + | // *HandshakeRefusal for a peer refusal and ErrHandshakeRejected for any local | |
| 108 | + | // authority mismatch. The caller's ctx (or its deadline) bounds both the send | |
| 109 | + | // and the receive. Every failure closes carrier; success leaves it open and | |
| 110 | + | // does not start a pump. | |
| 111 | + | func RunClientHandshake(ctx context.Context, carrier FramedCarrier, endpoint ports.BrokerResolvedEndpoint, offered MuxCeilings) (ClientHandshakeResult, error) { | |
| 112 | + | if ctx == nil { | |
| 113 | + | ctx = context.Background() | |
| 114 | + | } | |
| 115 | + | if carrier == nil { | |
| 116 | + | return ClientHandshakeResult{}, ErrHandshakeConfig | |
| 117 | + | } | |
| 118 | + | if err := endpoint.Validate(); err != nil { | |
| 119 | + | closeFailedHandshake(carrier) | |
| 120 | + | return ClientHandshakeResult{}, ErrHandshakeConfig | |
| 121 | + | } | |
| 122 | + | if err := offered.Validate(); err != nil { | |
| 123 | + | closeFailedHandshake(carrier) | |
| 124 | + | return ClientHandshakeResult{}, ErrHandshakeConfig | |
| 125 | + | } | |
| 126 | + | ||
| 127 | + | request, err := EncodePreambleRequest(offered, endpoint.Policy) | |
| 128 | + | if err != nil { | |
| 129 | + | closeFailedHandshake(carrier) | |
| 130 | + | return ClientHandshakeResult{}, ErrHandshakeConfig | |
| 131 | + | } | |
| 132 | + | offer, err := proto.Marshal(request) | |
| 133 | + | if err != nil { | |
| 134 | + | closeFailedHandshake(carrier) | |
| 135 | + | return ClientHandshakeResult{}, errors.Join(ErrHandshakeConfig, err) | |
| 136 | + | } | |
| 137 | + | if err := carrier.Send(ctx, offer); err != nil { | |
| 138 | + | closeFailedHandshake(carrier) | |
| 139 | + | return ClientHandshakeResult{}, err | |
| 140 | + | } | |
| 141 | + | payload, err := carrier.Receive(ctx) | |
| 142 | + | if err != nil { | |
| 143 | + | closeFailedHandshake(carrier) | |
| 144 | + | return ClientHandshakeResult{}, err | |
| 145 | + | } | |
| 146 | + | response, err := DecodePreambleResponseBytes(payload) | |
| 147 | + | if err != nil { | |
| 148 | + | closeFailedHandshake(carrier) | |
| 149 | + | if !response.Accepted && response.Code != 0 { | |
| 150 | + | return ClientHandshakeResult{}, &HandshakeRefusal{Code: response.Code} | |
| 151 | + | } | |
| 152 | + | return ClientHandshakeResult{}, ErrHandshakeRejected | |
| 153 | + | } | |
| 154 | + | if err := validateClientResponse(response, endpoint, offered); err != nil { | |
| 155 | + | closeFailedHandshake(carrier) | |
| 156 | + | return ClientHandshakeResult{}, err | |
| 157 | + | } | |
| 158 | + | return ClientHandshakeResult{ | |
| 159 | + | Ceilings: EffectiveMuxCeilings(offered, response.Ceilings), | |
| 160 | + | Identity: response.Identity, | |
| 161 | + | Incarnation: response.Incarnation, | |
| 162 | + | Policy: response.Policy, | |
| 163 | + | }, nil | |
| 164 | + | } | |
| 165 | + | ||
| 166 | + | // validateClientResponse is the broker's authority check on one accepted | |
| 167 | + | // response: identity and policy exactly equal the resolved endpoint, the | |
| 168 | + | // incarnation is nonzero, and the effective ceilings never exceed the offer. | |
| 169 | + | // A non-acceptance is refused too, so a caller can never treat a refusal as an | |
| 170 | + | // accepted empty response. | |
| 171 | + | func validateClientResponse(response MuxPreambleResponse, endpoint ports.BrokerResolvedEndpoint, offered MuxCeilings) error { | |
| 172 | + | if !response.Accepted { | |
| 173 | + | return ErrHandshakeRejected | |
| 174 | + | } | |
| 175 | + | if response.Identity != endpoint.Identity { | |
| 176 | + | return ErrHandshakeRejected | |
| 177 | + | } | |
| 178 | + | if !response.Policy.Compatible(endpoint.Policy) { | |
| 179 | + | return ErrHandshakeRejected | |
| 180 | + | } | |
| 181 | + | if err := response.Incarnation.Validate(); err != nil { | |
| 182 | + | return ErrHandshakeRejected | |
| 183 | + | } | |
| 184 | + | if err := ValidatePreambleResponseAgainstOffer(response, offered); err != nil { | |
| 185 | + | return ErrHandshakeRejected | |
| 186 | + | } | |
| 187 | + | return nil | |
| 188 | + | } | |
| 189 | + | ||
| 190 | + | // RunServerHandshake performs the daemon side of the daemonmux physical | |
| 191 | + | // preamble over carrier. It accepts exactly one request, refuses it with the | |
| 192 | + | // precise rejection code when it is malformed or its policy differs from the | |
| 193 | + | // immutable binding, and otherwise answers with the binding's identity, | |
| 194 | + | // incarnation, and policy over the effective ceilings. The caller's ctx (or | |
| 195 | + | // its deadline) bounds both the receive and the send. Every failure sends the | |
| 196 | + | // typed refusal where possible, closes carrier, and returns a typed | |
| 197 | + | // *HandshakeRefusal; success leaves carrier open and does not start a pump. | |
| 198 | + | func RunServerHandshake(ctx context.Context, carrier FramedCarrier, binding ServerBinding, offered MuxCeilings) (ServerHandshakeResult, error) { | |
| 199 | + | if ctx == nil { | |
| 200 | + | ctx = context.Background() | |
| 201 | + | } | |
| 202 | + | if carrier == nil { | |
| 203 | + | return ServerHandshakeResult{}, ErrHandshakeConfig | |
| 204 | + | } | |
| 205 | + | if err := binding.Validate(); err != nil { | |
| 206 | + | closeFailedHandshake(carrier) | |
| 207 | + | return ServerHandshakeResult{}, ErrHandshakeConfig | |
| 208 | + | } | |
| 209 | + | if err := offered.Validate(); err != nil { | |
| 210 | + | closeFailedHandshake(carrier) | |
| 211 | + | return ServerHandshakeResult{}, ErrHandshakeConfig | |
| 212 | + | } | |
| 213 | + | ||
| 214 | + | payload, err := carrier.Receive(ctx) | |
| 215 | + | if err != nil { | |
| 216 | + | closeFailedHandshake(carrier) | |
| 217 | + | return ServerHandshakeResult{}, err | |
| 218 | + | } | |
| 219 | + | request, code, err := decodePreambleRequestWithCode(payload) | |
| 220 | + | if err != nil { | |
| 221 | + | _ = sendHandshakeRefusal(ctx, carrier, offered, code) | |
| 222 | + | closeFailedHandshake(carrier) | |
| 223 | + | return ServerHandshakeResult{}, &HandshakeRefusal{Code: code} | |
| 224 | + | } | |
| 225 | + | if !binding.Accepts(request.Policy) { | |
| 226 | + | code := uint32(RejectionLimitRefused) | |
| 227 | + | _ = sendHandshakeRefusal(ctx, carrier, offered, code) | |
| 228 | + | closeFailedHandshake(carrier) | |
| 229 | + | return ServerHandshakeResult{}, &HandshakeRefusal{Code: code} | |
| 230 | + | } | |
| 231 | + | ||
| 232 | + | effective := EffectiveMuxCeilings(offered, request.Ceilings) | |
| 233 | + | response, err := EncodePreambleResponse(true, effective, binding.Policy(), binding.Identity(), binding.Incarnation(), 0) | |
| 234 | + | if err != nil { | |
| 235 | + | closeFailedHandshake(carrier) | |
| 236 | + | return ServerHandshakeResult{}, ErrHandshakeConfig | |
| 237 | + | } | |
| 238 | + | answer, err := proto.Marshal(response) | |
| 239 | + | if err != nil { | |
| 240 | + | closeFailedHandshake(carrier) | |
| 241 | + | return ServerHandshakeResult{}, errors.Join(ErrHandshakeConfig, err) | |
| 242 | + | } | |
| 243 | + | if err := carrier.Send(ctx, answer); err != nil { | |
| 244 | + | closeFailedHandshake(carrier) | |
| 245 | + | return ServerHandshakeResult{}, err | |
| 246 | + | } | |
| 247 | + | return ServerHandshakeResult{Ceilings: effective, Binding: binding}, nil | |
| 248 | + | } | |
| 249 | + | ||
| 250 | + | // decodePreambleRequestWithCode is the server's strict request decode paired | |
| 251 | + | // with the precise refusal code for its failure, so a refusal always carries | |
| 252 | + | // the shared PreambleRejectionCode taxonomy rather than a generic limit code. | |
| 253 | + | func decodePreambleRequestWithCode(payload []byte) (MuxPreambleRequest, uint32, error) { | |
| 254 | + | if err := CheckPreambleSize(payload); err != nil { | |
| 255 | + | return MuxPreambleRequest{}, RejectionLimitRefused, err | |
| 256 | + | } | |
| 257 | + | message := &wire.MuxPreambleRequest{} | |
| 258 | + | if err := wire.ScanEnvelope(message, payload); err != nil { | |
| 259 | + | return MuxPreambleRequest{}, RejectionCodeFor(nil, err), err | |
| 260 | + | } | |
| 261 | + | if err := (proto.UnmarshalOptions{DiscardUnknown: false}.Unmarshal(payload, message)); err != nil { | |
| 262 | + | return MuxPreambleRequest{}, RejectionCodeFor(message, err), err | |
| 263 | + | } | |
| 264 | + | request, err := DecodePreambleRequest(message) | |
| 265 | + | if err != nil { | |
| 266 | + | return MuxPreambleRequest{}, RejectionCodeFor(message, err), err | |
| 267 | + | } | |
| 268 | + | return request, 0, nil | |
| 269 | + | } | |
| 270 | + | ||
| 271 | + | // sendHandshakeRefusal best-effort sends one typed refusal. The caller closes | |
| 272 | + | // the carrier immediately afterwards regardless of the send result, so a | |
| 273 | + | // failed refusal write never leaves the connection half-open. | |
| 274 | + | func sendHandshakeRefusal(ctx context.Context, carrier FramedCarrier, offered MuxCeilings, code uint32) error { | |
| 275 | + | response, err := EncodePreambleResponse(false, offered, ports.BrokerPolicy{}, "", ports.BrokerDaemonIncarnation{}, code) | |
| 276 | + | if err != nil { | |
| 277 | + | return err | |
| 278 | + | } | |
| 279 | + | answer, err := proto.Marshal(response) | |
| 280 | + | if err != nil { | |
| 281 | + | return err | |
| 282 | + | } | |
| 283 | + | return carrier.Send(ctx, answer) | |
| 284 | + | } | |
| 285 | + | ||
| 286 | + | // closeFailedHandshake closes the carrier on a failed handshake. The close | |
| 287 | + | // error is deliberately dropped: the handshake result is the caller's signal, | |
| 288 | + | // and a carrier that refuses to close must not replace or mask it. | |
| 289 | + | func closeFailedHandshake(carrier FramedCarrier) { | |
| 290 | + | _ = carrier.Close() | |
| 291 | + | } | |
A internal/adapters/daemonmux/handshake_test.go +562 -0
| @@ -0,0 +1,562 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "testing" | |
| 6 | + | "time" | |
| 7 | + | ||
| 8 | + | "github.com/stretchr/testify/require" | |
| 9 | + | "google.golang.org/protobuf/proto" | |
| 10 | + | ||
| 11 | + | "github.com/bnema/vev/internal/ports" | |
| 12 | + | "github.com/bnema/vev/internal/protocol" | |
| 13 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 14 | + | ) | |
| 15 | + | ||
| 16 | + | // handshakeEndpoint is the independently resolved endpoint the client verifies | |
| 17 | + | // the daemon response against. | |
| 18 | + | func handshakeEndpoint(identity ports.BrokerDaemonIdentity, policy ports.BrokerPolicy) ports.BrokerResolvedEndpoint { | |
| 19 | + | return ports.BrokerResolvedEndpoint{Identity: identity, Policy: policy, Address: "quic://127.0.0.1:7777"} | |
| 20 | + | } | |
| 21 | + | ||
| 22 | + | func mustServerBinding(t *testing.T) ServerBinding { | |
| 23 | + | t.Helper() | |
| 24 | + | binding, err := NewServerBinding(testIdentity(), testIncarnation(), testPolicy()) | |
| 25 | + | require.NoError(t, err) | |
| 26 | + | return binding | |
| 27 | + | } | |
| 28 | + | ||
| 29 | + | // encodedAcceptedResponse builds one valid acceptance and lets the caller | |
| 30 | + | // corrupt exactly one field before marshaling. | |
| 31 | + | func encodedAcceptedResponse(t *testing.T, mutate func(*wire.MuxPreambleResponse)) []byte { | |
| 32 | + | t.Helper() | |
| 33 | + | response, err := EncodePreambleResponse(true, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), 0) | |
| 34 | + | require.NoError(t, err) | |
| 35 | + | if mutate != nil { | |
| 36 | + | mutate(response) | |
| 37 | + | } | |
| 38 | + | raw, err := proto.Marshal(response) | |
| 39 | + | require.NoError(t, err) | |
| 40 | + | return raw | |
| 41 | + | } | |
| 42 | + | ||
| 43 | + | // encodedAcceptedPolicy corrupts one field of the accepted policy. | |
| 44 | + | func encodedAcceptedPolicy(t *testing.T, mutate func(*wire.BrokerWirePolicy)) []byte { | |
| 45 | + | t.Helper() | |
| 46 | + | return encodedAcceptedResponse(t, func(r *wire.MuxPreambleResponse) { mutate(r.AcceptedPolicy) }) | |
| 47 | + | } | |
| 48 | + | ||
| 49 | + | func encodedRefusal(t *testing.T, code uint32) []byte { | |
| 50 | + | t.Helper() | |
| 51 | + | response, err := EncodePreambleResponse(false, DefaultMuxCeilings(), ports.BrokerPolicy{}, "", ports.BrokerDaemonIncarnation{}, code) | |
| 52 | + | require.NoError(t, err) | |
| 53 | + | raw, err := proto.Marshal(response) | |
| 54 | + | require.NoError(t, err) | |
| 55 | + | return raw | |
| 56 | + | } | |
| 57 | + | ||
| 58 | + | // encodedRequest builds one valid request and lets the caller corrupt exactly | |
| 59 | + | // one field before marshaling. | |
| 60 | + | func encodedRequest(t *testing.T, ceilings MuxCeilings, policy ports.BrokerPolicy, mutate func(*wire.MuxPreambleRequest)) []byte { | |
| 61 | + | t.Helper() | |
| 62 | + | request, err := EncodePreambleRequest(ceilings, policy) | |
| 63 | + | require.NoError(t, err) | |
| 64 | + | if mutate != nil { | |
| 65 | + | mutate(request) | |
| 66 | + | } | |
| 67 | + | raw, err := proto.Marshal(request) | |
| 68 | + | require.NoError(t, err) | |
| 69 | + | return raw | |
| 70 | + | } | |
| 71 | + | ||
| 72 | + | func awaitSentFrame(t *testing.T, carrier *fakeCarrier) []byte { | |
| 73 | + | t.Helper() | |
| 74 | + | select { | |
| 75 | + | case payload := <-carrier.sent: | |
| 76 | + | return payload | |
| 77 | + | case <-time.After(time.Second): | |
| 78 | + | t.Fatal("handshake sent no frame") | |
| 79 | + | return nil | |
| 80 | + | } | |
| 81 | + | } | |
| 82 | + | ||
| 83 | + | // TestClientHandshakeAcceptsVerifiedResponse proves a matching acceptance is | |
| 84 | + | // accepted, yields the server authority, leaves the carrier open, and offers | |
| 85 | + | // exactly the requested policy and ceilings. | |
| 86 | + | func TestClientHandshakeAcceptsVerifiedResponse(t *testing.T) { | |
| 87 | + | carrier := newFakeCarrier() | |
| 88 | + | carrier.inbound <- fakeFrame{payload: encodedAcceptedResponse(t, nil)} | |
| 89 | + | ||
| 90 | + | result, err := RunClientHandshake(context.Background(), carrier, handshakeEndpoint(testIdentity(), testPolicy()), DefaultMuxCeilings()) | |
| 91 | + | require.NoError(t, err) | |
| 92 | + | require.Equal(t, testIdentity(), result.Identity) | |
| 93 | + | require.Equal(t, testIncarnation(), result.Incarnation) | |
| 94 | + | require.Equal(t, testPolicy(), result.Policy) | |
| 95 | + | require.Equal(t, DefaultMuxCeilings(), result.Ceilings) | |
| 96 | + | require.Equal(t, 0, carrier.closeCalls()) | |
| 97 | + | ||
| 98 | + | request, err := DecodePreambleRequestBytes(awaitSentFrame(t, carrier)) | |
| 99 | + | require.NoError(t, err) | |
| 100 | + | require.Equal(t, DefaultMuxCeilings(), request.Ceilings) | |
| 101 | + | require.Equal(t, testPolicy(), request.Policy) | |
| 102 | + | } | |
| 103 | + | ||
| 104 | + | // TestClientHandshakeRejectsUntrustedResponses is the client authority table: | |
| 105 | + | // wrong identity, every policy field, zero incarnation, wrong role/version/ | |
| 106 | + | // magic/epoch, a typed refusal, an acceptance carrying a rejection field, and | |
| 107 | + | // excess ceilings are each refused and each close the carrier. | |
| 108 | + | func TestClientHandshakeRejectsUntrustedResponses(t *testing.T) { | |
| 109 | + | smallOffer := MuxCeilings{ | |
| 110 | + | MaxReceiveEnvelopeBytes: MinMuxEnvelopeBytes, | |
| 111 | + | StreamChunkLimit: MinMuxChunkBytes, | |
| 112 | + | MaxStreams: MinMuxStreams, | |
| 113 | + | MaxAggregateBytes: MinMuxAggregateBytes, | |
| 114 | + | } | |
| 115 | + | ||
| 116 | + | tests := []struct { | |
| 117 | + | name string | |
| 118 | + | offered MuxCeilings | |
| 119 | + | response func(t *testing.T) []byte | |
| 120 | + | code uint32 | |
| 121 | + | }{ | |
| 122 | + | { | |
| 123 | + | name: "wrong identity", | |
| 124 | + | offered: DefaultMuxCeilings(), | |
| 125 | + | response: func(t *testing.T) []byte { | |
| 126 | + | return encodedAcceptedResponse(t, func(r *wire.MuxPreambleResponse) { r.DaemonIdentity = "other-identity" }) | |
| 127 | + | }, | |
| 128 | + | }, | |
| 129 | + | { | |
| 130 | + | name: "policy protocol version", | |
| 131 | + | offered: DefaultMuxCeilings(), | |
| 132 | + | response: func(t *testing.T) []byte { | |
| 133 | + | return encodedAcceptedPolicy(t, func(p *wire.BrokerWirePolicy) { p.ProtocolVersion++ }) | |
| 134 | + | }, | |
| 135 | + | }, | |
| 136 | + | { | |
| 137 | + | name: "policy catalogue version", | |
| 138 | + | offered: DefaultMuxCeilings(), | |
| 139 | + | response: func(t *testing.T) []byte { | |
| 140 | + | return encodedAcceptedPolicy(t, func(p *wire.BrokerWirePolicy) { p.CatalogueVersion++ }) | |
| 141 | + | }, | |
| 142 | + | }, | |
| 143 | + | { | |
| 144 | + | name: "policy environment", | |
| 145 | + | offered: DefaultMuxCeilings(), | |
| 146 | + | response: func(t *testing.T) []byte { | |
| 147 | + | return encodedAcceptedPolicy(t, func(p *wire.BrokerWirePolicy) { p.EnvironmentPolicy++ }) | |
| 148 | + | }, | |
| 149 | + | }, | |
| 150 | + | { | |
| 151 | + | name: "policy transport", | |
| 152 | + | offered: DefaultMuxCeilings(), | |
| 153 | + | response: func(t *testing.T) []byte { | |
| 154 | + | return encodedAcceptedPolicy(t, func(p *wire.BrokerWirePolicy) { p.Transport = "stdio" }) | |
| 155 | + | }, | |
| 156 | + | }, | |
| 157 | + | { | |
| 158 | + | name: "policy trust", | |
| 159 | + | offered: DefaultMuxCeilings(), | |
| 160 | + | response: func(t *testing.T) []byte { | |
| 161 | + | return encodedAcceptedPolicy(t, func(p *wire.BrokerWirePolicy) { p.Trust = "tofu" }) | |
| 162 | + | }, | |
| 163 | + | }, | |
| 164 | + | { | |
| 165 | + | name: "policy launch", | |
| 166 | + | offered: DefaultMuxCeilings(), | |
| 167 | + | response: func(t *testing.T) []byte { | |
| 168 | + | return encodedAcceptedPolicy(t, func(p *wire.BrokerWirePolicy) { p.Launch = "manual" }) | |
| 169 | + | }, | |
| 170 | + | }, | |
| 171 | + | { | |
| 172 | + | name: "policy isolation", | |
| 173 | + | offered: DefaultMuxCeilings(), | |
| 174 | + | response: func(t *testing.T) []byte { | |
| 175 | + | return encodedAcceptedPolicy(t, func(p *wire.BrokerWirePolicy) { p.Isolation = "system" }) | |
| 176 | + | }, | |
| 177 | + | }, | |
| 178 | + | { | |
| 179 | + | name: "zero incarnation", | |
| 180 | + | offered: DefaultMuxCeilings(), | |
| 181 | + | response: func(t *testing.T) []byte { | |
| 182 | + | return encodedAcceptedResponse(t, func(r *wire.MuxPreambleResponse) { r.Incarnation = make([]byte, 16) }) | |
| 183 | + | }, | |
| 184 | + | }, | |
| 185 | + | { | |
| 186 | + | name: "wrong role", | |
| 187 | + | offered: DefaultMuxCeilings(), | |
| 188 | + | response: func(t *testing.T) []byte { | |
| 189 | + | return encodedAcceptedResponse(t, func(r *wire.MuxPreambleResponse) { r.Role = &wire.PreambleRole{Role: MuxRoleClient} }) | |
| 190 | + | }, | |
| 191 | + | }, | |
| 192 | + | { | |
| 193 | + | name: "wrong version", | |
| 194 | + | offered: DefaultMuxCeilings(), | |
| 195 | + | response: func(t *testing.T) []byte { | |
| 196 | + | return encodedAcceptedResponse(t, func(r *wire.MuxPreambleResponse) { r.Version = uint32(protocol.Version) + 1 }) | |
| 197 | + | }, | |
| 198 | + | }, | |
| 199 | + | { | |
| 200 | + | name: "bad magic", | |
| 201 | + | offered: DefaultMuxCeilings(), | |
| 202 | + | response: func(t *testing.T) []byte { | |
| 203 | + | return encodedAcceptedResponse(t, func(r *wire.MuxPreambleResponse) { r.Magic = 0 }) | |
| 204 | + | }, | |
| 205 | + | }, | |
| 206 | + | { | |
| 207 | + | name: "epoch mismatch", | |
| 208 | + | offered: DefaultMuxCeilings(), | |
| 209 | + | response: func(t *testing.T) []byte { | |
| 210 | + | return encodedAcceptedResponse(t, func(r *wire.MuxPreambleResponse) { r.Epoch = wire.ProtocolEpoch + 1 }) | |
| 211 | + | }, | |
| 212 | + | }, | |
| 213 | + | { | |
| 214 | + | name: "refused response", | |
| 215 | + | offered: DefaultMuxCeilings(), | |
| 216 | + | response: func(t *testing.T) []byte { | |
| 217 | + | return encodedRefusal(t, RejectionVersionMismatch) | |
| 218 | + | }, | |
| 219 | + | code: RejectionVersionMismatch, | |
| 220 | + | }, | |
| 221 | + | { | |
| 222 | + | name: "acceptance carrying rejection", | |
| 223 | + | offered: DefaultMuxCeilings(), | |
| 224 | + | response: func(t *testing.T) []byte { | |
| 225 | + | return encodedAcceptedResponse(t, func(r *wire.MuxPreambleResponse) { | |
| 226 | + | r.Rejection = &wire.PreambleRejectionCode{Code: RejectionLimitRefused} | |
| 227 | + | }) | |
| 228 | + | }, | |
| 229 | + | }, | |
| 230 | + | { | |
| 231 | + | name: "excess ceilings", | |
| 232 | + | offered: smallOffer, | |
| 233 | + | response: func(t *testing.T) []byte { | |
| 234 | + | return encodedAcceptedResponse(t, nil) | |
| 235 | + | }, | |
| 236 | + | }, | |
| 237 | + | } | |
| 238 | + | ||
| 239 | + | for _, test := range tests { | |
| 240 | + | t.Run(test.name, func(t *testing.T) { | |
| 241 | + | carrier := newFakeCarrier() | |
| 242 | + | carrier.inbound <- fakeFrame{payload: test.response(t)} | |
| 243 | + | ||
| 244 | + | _, err := RunClientHandshake(context.Background(), carrier, handshakeEndpoint(testIdentity(), testPolicy()), test.offered) | |
| 245 | + | require.ErrorIs(t, err, ErrHandshakeRejected) | |
| 246 | + | if test.code != 0 { | |
| 247 | + | var refusal *HandshakeRefusal | |
| 248 | + | require.ErrorAs(t, err, &refusal) | |
| 249 | + | require.Equal(t, test.code, refusal.Code) | |
| 250 | + | } | |
| 251 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 252 | + | }) | |
| 253 | + | } | |
| 254 | + | } | |
| 255 | + | ||
| 256 | + | // TestServerHandshakeAcceptsAndNegotiates proves the server answers an exact | |
| 257 | + | // request with the immutable binding and the element-wise minima, and leaves | |
| 258 | + | // the carrier open. | |
| 259 | + | func TestServerHandshakeAcceptsAndNegotiates(t *testing.T) { | |
| 260 | + | binding := mustServerBinding(t) | |
| 261 | + | clientCeilings := MuxCeilings{ | |
| 262 | + | MaxReceiveEnvelopeBytes: MinMuxEnvelopeBytes, | |
| 263 | + | StreamChunkLimit: 8, | |
| 264 | + | MaxStreams: 4, | |
| 265 | + | MaxAggregateBytes: MinMuxAggregateBytes, | |
| 266 | + | } | |
| 267 | + | serverCeilings := DefaultMuxCeilings() | |
| 268 | + | serverCeilings.MaxStreams = 2 | |
| 269 | + | ||
| 270 | + | carrier := newFakeCarrier() | |
| 271 | + | carrier.inbound <- fakeFrame{payload: encodedRequest(t, clientCeilings, testPolicy(), nil)} | |
| 272 | + | ||
| 273 | + | result, err := RunServerHandshake(context.Background(), carrier, binding, serverCeilings) | |
| 274 | + | require.NoError(t, err) | |
| 275 | + | require.Equal(t, binding, result.Binding) | |
| 276 | + | require.Equal(t, EffectiveMuxCeilings(clientCeilings, serverCeilings), result.Ceilings) | |
| 277 | + | require.Equal(t, 0, carrier.closeCalls()) | |
| 278 | + | ||
| 279 | + | decoded, err := DecodePreambleResponseBytes(awaitSentFrame(t, carrier)) | |
| 280 | + | require.NoError(t, err) | |
| 281 | + | require.True(t, decoded.Accepted) | |
| 282 | + | require.Equal(t, testIdentity(), decoded.Identity) | |
| 283 | + | require.Equal(t, testIncarnation(), decoded.Incarnation) | |
| 284 | + | require.Equal(t, testPolicy(), decoded.Policy) | |
| 285 | + | require.Equal(t, result.Ceilings, decoded.Ceilings) | |
| 286 | + | } | |
| 287 | + | ||
| 288 | + | // TestServerHandshakeEnforcesBindingPolicy proves a request differing in one | |
| 289 | + | // policy field is refused with code 7, no accepted authority is echoed, and | |
| 290 | + | // the carrier is closed. | |
| 291 | + | func TestServerHandshakeEnforcesBindingPolicy(t *testing.T) { | |
| 292 | + | binding := mustServerBinding(t) | |
| 293 | + | tests := []struct { | |
| 294 | + | name string | |
| 295 | + | mutate func(*ports.BrokerPolicy) | |
| 296 | + | }{ | |
| 297 | + | {name: "protocol version", mutate: func(p *ports.BrokerPolicy) { p.ProtocolVersion++ }}, | |
| 298 | + | {name: "catalogue schema version", mutate: func(p *ports.BrokerPolicy) { p.CatalogSchemaVersion++ }}, | |
| 299 | + | {name: "environment policy", mutate: func(p *ports.BrokerPolicy) { | |
| 300 | + | p.EnvironmentPolicy = protocol.EnvironmentPolicyClientOwned | |
| 301 | + | }}, | |
| 302 | + | {name: "transport", mutate: func(p *ports.BrokerPolicy) { p.Transport = "stdio" }}, | |
| 303 | + | {name: "trust", mutate: func(p *ports.BrokerPolicy) { p.Trust = "tofu" }}, | |
| 304 | + | {name: "launch", mutate: func(p *ports.BrokerPolicy) { p.Launch = "manual" }}, | |
| 305 | + | {name: "isolation", mutate: func(p *ports.BrokerPolicy) { p.Isolation = "system" }}, | |
| 306 | + | } | |
| 307 | + | for _, test := range tests { | |
| 308 | + | t.Run(test.name, func(t *testing.T) { | |
| 309 | + | requested := testPolicy() | |
| 310 | + | test.mutate(&requested) | |
| 311 | + | ||
| 312 | + | carrier := newFakeCarrier() | |
| 313 | + | carrier.inbound <- fakeFrame{payload: encodedRequest(t, DefaultMuxCeilings(), requested, nil)} | |
| 314 | + | ||
| 315 | + | _, err := RunServerHandshake(context.Background(), carrier, binding, DefaultMuxCeilings()) | |
| 316 | + | var refusal *HandshakeRefusal | |
| 317 | + | require.ErrorAs(t, err, &refusal) | |
| 318 | + | require.Equal(t, uint32(RejectionLimitRefused), refusal.Code) | |
| 319 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 320 | + | ||
| 321 | + | raw := &wire.MuxPreambleResponse{} | |
| 322 | + | require.NoError(t, proto.Unmarshal(awaitSentFrame(t, carrier), raw)) | |
| 323 | + | require.False(t, raw.GetAccepted()) | |
| 324 | + | require.Nil(t, raw.GetAcceptedPolicy()) | |
| 325 | + | require.Empty(t, raw.GetDaemonIdentity()) | |
| 326 | + | require.Empty(t, raw.GetIncarnation()) | |
| 327 | + | }) | |
| 328 | + | } | |
| 329 | + | } | |
| 330 | + | ||
| 331 | + | // TestServerHandshakeRefusesMalformedRequest proves the precise refusal code | |
| 332 | + | // taxonomy for a bad magic, epoch, version, or role, plus oversize and | |
| 333 | + | // undecodable payloads, and that every refusal closes the carrier. | |
| 334 | + | func TestServerHandshakeRefusesMalformedRequest(t *testing.T) { | |
| 335 | + | binding := mustServerBinding(t) | |
| 336 | + | tests := []struct { | |
| 337 | + | name string | |
| 338 | + | request func(t *testing.T) []byte | |
| 339 | + | code uint32 | |
| 340 | + | }{ | |
| 341 | + | { | |
| 342 | + | name: "bad magic", | |
| 343 | + | request: func(t *testing.T) []byte { | |
| 344 | + | return encodedRequest(t, DefaultMuxCeilings(), testPolicy(), func(r *wire.MuxPreambleRequest) { r.Magic = 0 }) | |
| 345 | + | }, | |
| 346 | + | code: RejectionBadMagic, | |
| 347 | + | }, | |
| 348 | + | { | |
| 349 | + | name: "epoch mismatch", | |
| 350 | + | request: func(t *testing.T) []byte { | |
| 351 | + | return encodedRequest(t, DefaultMuxCeilings(), testPolicy(), func(r *wire.MuxPreambleRequest) { r.Epoch = wire.ProtocolEpoch + 1 }) | |
| 352 | + | }, | |
| 353 | + | code: RejectionEpochMismatch, | |
| 354 | + | }, | |
| 355 | + | { | |
| 356 | + | name: "wrong version", | |
| 357 | + | request: func(t *testing.T) []byte { | |
| 358 | + | return encodedRequest(t, DefaultMuxCeilings(), testPolicy(), func(r *wire.MuxPreambleRequest) { r.Version = uint32(protocol.Version) + 1 }) | |
| 359 | + | }, | |
| 360 | + | code: RejectionVersionMismatch, | |
| 361 | + | }, | |
| 362 | + | { | |
| 363 | + | name: "wrong role", | |
| 364 | + | request: func(t *testing.T) []byte { | |
| 365 | + | return encodedRequest(t, DefaultMuxCeilings(), testPolicy(), func(r *wire.MuxPreambleRequest) { r.Role = &wire.PreambleRole{Role: MuxRoleServer} }) | |
| 366 | + | }, | |
| 367 | + | code: RejectionWrongRole, | |
| 368 | + | }, | |
| 369 | + | { | |
| 370 | + | name: "oversize payload", | |
| 371 | + | request: func(t *testing.T) []byte { | |
| 372 | + | return make([]byte, MuxPreambleLimit+1) | |
| 373 | + | }, | |
| 374 | + | code: RejectionLimitRefused, | |
| 375 | + | }, | |
| 376 | + | { | |
| 377 | + | name: "undecodable payload", | |
| 378 | + | request: func(t *testing.T) []byte { | |
| 379 | + | return []byte{0xff, 0xff, 0xff} | |
| 380 | + | }, | |
| 381 | + | code: RejectionLimitRefused, | |
| 382 | + | }, | |
| 383 | + | } | |
| 384 | + | for _, test := range tests { | |
| 385 | + | t.Run(test.name, func(t *testing.T) { | |
| 386 | + | carrier := newFakeCarrier() | |
| 387 | + | carrier.inbound <- fakeFrame{payload: test.request(t)} | |
| 388 | + | ||
| 389 | + | _, err := RunServerHandshake(context.Background(), carrier, binding, DefaultMuxCeilings()) | |
| 390 | + | var refusal *HandshakeRefusal | |
| 391 | + | require.ErrorAs(t, err, &refusal) | |
| 392 | + | require.Equal(t, test.code, refusal.Code) | |
| 393 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 394 | + | ||
| 395 | + | decoded, err := DecodePreambleResponseBytes(awaitSentFrame(t, carrier)) | |
| 396 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 397 | + | require.Equal(t, test.code, decoded.Code) | |
| 398 | + | }) | |
| 399 | + | } | |
| 400 | + | } | |
| 401 | + | ||
| 402 | + | // TestHandshakeRefusesInvalidConfiguration proves nil carriers, invalid | |
| 403 | + | // endpoints, invalid bindings, and invalid local ceilings fail closed with the | |
| 404 | + | // configuration sentinel. | |
| 405 | + | func TestHandshakeRefusesInvalidConfiguration(t *testing.T) { | |
| 406 | + | t.Run("client nil carrier", func(t *testing.T) { | |
| 407 | + | _, err := RunClientHandshake(context.Background(), nil, handshakeEndpoint(testIdentity(), testPolicy()), DefaultMuxCeilings()) | |
| 408 | + | require.ErrorIs(t, err, ErrHandshakeConfig) | |
| 409 | + | }) | |
| 410 | + | t.Run("client invalid endpoint", func(t *testing.T) { | |
| 411 | + | carrier := newFakeCarrier() | |
| 412 | + | _, err := RunClientHandshake(context.Background(), carrier, handshakeEndpoint("", testPolicy()), DefaultMuxCeilings()) | |
| 413 | + | require.ErrorIs(t, err, ErrHandshakeConfig) | |
| 414 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 415 | + | }) | |
| 416 | + | t.Run("client invalid ceilings", func(t *testing.T) { | |
| 417 | + | carrier := newFakeCarrier() | |
| 418 | + | _, err := RunClientHandshake(context.Background(), carrier, handshakeEndpoint(testIdentity(), testPolicy()), MuxCeilings{}) | |
| 419 | + | require.ErrorIs(t, err, ErrHandshakeConfig) | |
| 420 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 421 | + | }) | |
| 422 | + | t.Run("server nil carrier", func(t *testing.T) { | |
| 423 | + | _, err := RunServerHandshake(context.Background(), nil, mustServerBinding(t), DefaultMuxCeilings()) | |
| 424 | + | require.ErrorIs(t, err, ErrHandshakeConfig) | |
| 425 | + | }) | |
| 426 | + | t.Run("server invalid binding", func(t *testing.T) { | |
| 427 | + | carrier := newFakeCarrier() | |
| 428 | + | _, err := RunServerHandshake(context.Background(), carrier, ServerBinding{}, DefaultMuxCeilings()) | |
| 429 | + | require.ErrorIs(t, err, ErrHandshakeConfig) | |
| 430 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 431 | + | }) | |
| 432 | + | t.Run("server invalid ceilings", func(t *testing.T) { | |
| 433 | + | carrier := newFakeCarrier() | |
| 434 | + | _, err := RunServerHandshake(context.Background(), carrier, mustServerBinding(t), MuxCeilings{}) | |
| 435 | + | require.ErrorIs(t, err, ErrHandshakeConfig) | |
| 436 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 437 | + | }) | |
| 438 | + | } | |
| 439 | + | ||
| 440 | + | // TestHandshakeHonorsCallerContext proves the caller's context is the single | |
| 441 | + | // absolute bound on the exchange: a cancelled or already-expired context ends | |
| 442 | + | // the handshake promptly and closes the carrier. | |
| 443 | + | func TestHandshakeHonorsCallerContext(t *testing.T) { | |
| 444 | + | endpoint := handshakeEndpoint(testIdentity(), testPolicy()) | |
| 445 | + | binding := mustServerBinding(t) | |
| 446 | + | ||
| 447 | + | tests := []struct { | |
| 448 | + | name string | |
| 449 | + | ctx func() (context.Context, context.CancelFunc) | |
| 450 | + | run func(context.Context, *fakeCarrier) error | |
| 451 | + | want error | |
| 452 | + | }{ | |
| 453 | + | { | |
| 454 | + | name: "client cancelled", | |
| 455 | + | ctx: func() (context.Context, context.CancelFunc) { | |
| 456 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 457 | + | cancel() | |
| 458 | + | return ctx, func() {} | |
| 459 | + | }, | |
| 460 | + | run: func(ctx context.Context, carrier *fakeCarrier) error { | |
| 461 | + | _, err := RunClientHandshake(ctx, carrier, endpoint, DefaultMuxCeilings()) | |
| 462 | + | return err | |
| 463 | + | }, | |
| 464 | + | want: context.Canceled, | |
| 465 | + | }, | |
| 466 | + | { | |
| 467 | + | name: "client deadline exceeded", | |
| 468 | + | ctx: func() (context.Context, context.CancelFunc) { | |
| 469 | + | return context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) | |
| 470 | + | }, | |
| 471 | + | run: func(ctx context.Context, carrier *fakeCarrier) error { | |
| 472 | + | _, err := RunClientHandshake(ctx, carrier, endpoint, DefaultMuxCeilings()) | |
| 473 | + | return err | |
| 474 | + | }, | |
| 475 | + | want: context.DeadlineExceeded, | |
| 476 | + | }, | |
| 477 | + | { | |
| 478 | + | name: "server cancelled", | |
| 479 | + | ctx: func() (context.Context, context.CancelFunc) { | |
| 480 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 481 | + | cancel() | |
| 482 | + | return ctx, func() {} | |
| 483 | + | }, | |
| 484 | + | run: func(ctx context.Context, carrier *fakeCarrier) error { | |
| 485 | + | _, err := RunServerHandshake(ctx, carrier, binding, DefaultMuxCeilings()) | |
| 486 | + | return err | |
| 487 | + | }, | |
| 488 | + | want: context.Canceled, | |
| 489 | + | }, | |
| 490 | + | { | |
| 491 | + | name: "server deadline exceeded", | |
| 492 | + | ctx: func() (context.Context, context.CancelFunc) { | |
| 493 | + | return context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) | |
| 494 | + | }, | |
| 495 | + | run: func(ctx context.Context, carrier *fakeCarrier) error { | |
| 496 | + | _, err := RunServerHandshake(ctx, carrier, binding, DefaultMuxCeilings()) | |
| 497 | + | return err | |
| 498 | + | }, | |
| 499 | + | want: context.DeadlineExceeded, | |
| 500 | + | }, | |
| 501 | + | } | |
| 502 | + | for _, test := range tests { | |
| 503 | + | t.Run(test.name, func(t *testing.T) { | |
| 504 | + | ctx, cancel := test.ctx() | |
| 505 | + | defer cancel() | |
| 506 | + | carrier := newFakeCarrier() | |
| 507 | + | err := test.run(ctx, carrier) | |
| 508 | + | require.ErrorIs(t, err, test.want) | |
| 509 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 510 | + | }) | |
| 511 | + | } | |
| 512 | + | } | |
| 513 | + | ||
| 514 | + | // TestHandshakeEndToEndNegotiates runs the broker and daemon sides over one | |
| 515 | + | // in-memory link and proves they agree on the element-wise minima and the | |
| 516 | + | // binding the daemon enforced. | |
| 517 | + | func TestHandshakeEndToEndNegotiates(t *testing.T) { | |
| 518 | + | clientCarrier, serverCarrier := newMemCarrierPair(4) | |
| 519 | + | clientCeilings := MuxCeilings{ | |
| 520 | + | MaxReceiveEnvelopeBytes: MinMuxEnvelopeBytes, | |
| 521 | + | StreamChunkLimit: 8, | |
| 522 | + | MaxStreams: 4, | |
| 523 | + | MaxAggregateBytes: MinMuxAggregateBytes, | |
| 524 | + | } | |
| 525 | + | serverCeilings := DefaultMuxCeilings() | |
| 526 | + | serverCeilings.MaxStreams = 2 | |
| 527 | + | binding := mustServerBinding(t) | |
| 528 | + | endpoint := handshakeEndpoint(testIdentity(), testPolicy()) | |
| 529 | + | ||
| 530 | + | type clientOutcome struct { | |
| 531 | + | result ClientHandshakeResult | |
| 532 | + | err error | |
| 533 | + | } | |
| 534 | + | type serverOutcome struct { | |
| 535 | + | result ServerHandshakeResult | |
| 536 | + | err error | |
| 537 | + | } | |
| 538 | + | clientCh := make(chan clientOutcome, 1) | |
| 539 | + | serverCh := make(chan serverOutcome, 1) | |
| 540 | + | ||
| 541 | + | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) | |
| 542 | + | defer cancel() | |
| 543 | + | go func() { | |
| 544 | + | result, err := RunClientHandshake(ctx, clientCarrier, endpoint, clientCeilings) | |
| 545 | + | clientCh <- clientOutcome{result, err} | |
| 546 | + | }() | |
| 547 | + | go func() { | |
| 548 | + | result, err := RunServerHandshake(ctx, serverCarrier, binding, serverCeilings) | |
| 549 | + | serverCh <- serverOutcome{result, err} | |
| 550 | + | }() | |
| 551 | + | ||
| 552 | + | client := <-clientCh | |
| 553 | + | server := <-serverCh | |
| 554 | + | require.NoError(t, client.err) | |
| 555 | + | require.NoError(t, server.err) | |
| 556 | + | require.Equal(t, testIdentity(), client.result.Identity) | |
| 557 | + | require.Equal(t, testIncarnation(), client.result.Incarnation) | |
| 558 | + | require.Equal(t, testPolicy(), client.result.Policy) | |
| 559 | + | require.Equal(t, binding, server.result.Binding) | |
| 560 | + | require.Equal(t, EffectiveMuxCeilings(clientCeilings, serverCeilings), client.result.Ceilings) | |
| 561 | + | require.Equal(t, server.result.Ceilings, client.result.Ceilings) | |
| 562 | + | } | |
A internal/adapters/daemonmux/limits.go +200 -0
| @@ -0,0 +1,200 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | // Stateless bound refusals (P3.2). | |
| 4 | + | // | |
| 5 | + | // One physical daemon connection negotiates one immutable set of ceilings | |
| 6 | + | // exactly once, in the daemonmux physical preamble: a receive envelope | |
| 7 | + | // ceiling (1 MiB..16 MiB), a stream chunk ceiling (1..64 KiB), a stream | |
| 8 | + | // count ceiling (1..128), and an aggregate buffered-bytes ceiling | |
| 9 | + | // (MinMuxAggregateBytes..64 MiB, one maximum chunk plus envelope overhead at | |
| 10 | + | // the floor). Every application envelope and every opaque data frame is | |
| 11 | + | // bounded statelessly before it allocates; the strictly-increasing stream | |
| 12 | + | // ID rule and the aggregate accounting are state-layer concerns owned by | |
| 13 | + | // the later engine slice, so this codec only enforces nonzero stream IDs. | |
| 14 | + | // | |
| 15 | + | // The daemonmux conversation carries no bulk category: every application | |
| 16 | + | // envelope is control-class and never exceeds 1 MiB, so one envelope ceiling | |
| 17 | + | // and one chunk ceiling cover the whole conversation. The negotiated receive | |
| 18 | + | // ceiling still bounds the physical carriage's own envelope reads, which may | |
| 19 | + | // sit above what a mux application envelope can ever need. | |
| 20 | + | ||
| 21 | + | import ( | |
| 22 | + | "errors" | |
| 23 | + | ||
| 24 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 25 | + | ) | |
| 26 | + | ||
| 27 | + | const ( | |
| 28 | + | // MuxRoleClient is the daemonmux preamble client role: the broker side | |
| 29 | + | // that owns the physical connection. It is disjoint from the session | |
| 30 | + | // roles 1/2 and the broker roles 3/4. | |
| 31 | + | MuxRoleClient uint32 = 5 | |
| 32 | + | // MuxRoleServer is the daemonmux preamble server role: the daemon that | |
| 33 | + | // accepts the multiplexed streams. | |
| 34 | + | MuxRoleServer uint32 = 6 | |
| 35 | + | ||
| 36 | + | // MuxPreambleLimit bounds the serialized daemonmux preamble. It is the | |
| 37 | + | // same 4 KiB bound the session and broker preambles use. | |
| 38 | + | MuxPreambleLimit = 4 << 10 | |
| 39 | + | ||
| 40 | + | // MinMuxEnvelopeBytes is the advertised-envelope floor: a daemonmux peer | |
| 41 | + | // must accept at least 1 MiB per application envelope | |
| 42 | + | // (wire.ControlEnvelopeLimit, the control-class ceiling). | |
| 43 | + | MinMuxEnvelopeBytes uint64 = wire.ControlEnvelopeLimit | |
| 44 | + | // MaxMuxEnvelopeBytes is the advertised-envelope ceiling: a daemonmux | |
| 45 | + | // peer must never advertise above 16 MiB (wire.AbsoluteEnvelopeLimit). | |
| 46 | + | MaxMuxEnvelopeBytes uint64 = wire.AbsoluteEnvelopeLimit | |
| 47 | + | ||
| 48 | + | // MaxMuxApplicationBytes is the absolute bound on one daemonmux | |
| 49 | + | // application envelope. Every mux envelope is control-class metadata, so | |
| 50 | + | // no application envelope ever exceeds 1 MiB (wire.ControlEnvelopeLimit) | |
| 51 | + | // even when the negotiated receive ceiling admits a larger carriage | |
| 52 | + | // envelope. The codec therefore enforces both bounds at once. | |
| 53 | + | MaxMuxApplicationBytes uint64 = wire.ControlEnvelopeLimit | |
| 54 | + | ||
| 55 | + | // MinMuxChunkBytes is the smallest legal stream chunk ceiling. | |
| 56 | + | MinMuxChunkBytes uint64 = 1 | |
| 57 | + | // MaxMuxChunkBytes is the largest legal stream chunk ceiling: 64 KiB per | |
| 58 | + | // opaque data frame. | |
| 59 | + | MaxMuxChunkBytes uint64 = 64 << 10 | |
| 60 | + | ||
| 61 | + | // MinMuxStreams is the smallest legal concurrent-stream ceiling. | |
| 62 | + | MinMuxStreams uint64 = 1 | |
| 63 | + | // MaxMuxStreams is the largest legal concurrent-stream ceiling. | |
| 64 | + | MaxMuxStreams uint64 = 128 | |
| 65 | + | ||
| 66 | + | // MuxEnvelopeOverheadBytes is the largest carriage overhead one daemonmux | |
| 67 | + | // data frame adds around its opaque payload: the outer directional oneof | |
| 68 | + | // tag and length plus the MuxData field tags, the varint physical stream | |
| 69 | + | // ID, and the varint payload length. The scheduler charges a frame's full | |
| 70 | + | // serialized envelope against the aggregate byte budget, so an aggregate | |
| 71 | + | // ceiling below one maximum chunk plus this overhead could refuse even a | |
| 72 | + | // single legal frame. 64 bytes is a deliberate upper bound on that | |
| 73 | + | // overhead (the worst case is under 32 bytes). | |
| 74 | + | MuxEnvelopeOverheadBytes uint64 = 64 | |
| 75 | + | ||
| 76 | + | // MinMuxAggregateBytes is the smallest legal aggregate buffered-bytes | |
| 77 | + | // ceiling: one maximum stream chunk plus the envelope overhead, so a peer | |
| 78 | + | // that advertises the floor can always buffer at least one full-size | |
| 79 | + | // frame in the direction it accepts. | |
| 80 | + | MinMuxAggregateBytes uint64 = MaxMuxChunkBytes + MuxEnvelopeOverheadBytes | |
| 81 | + | // MaxMuxAggregateBytes is the largest legal aggregate buffered-bytes | |
| 82 | + | // ceiling: 64 MiB across every live stream of one physical connection. | |
| 83 | + | MaxMuxAggregateBytes uint64 = 64 << 20 | |
| 84 | + | ) | |
| 85 | + | ||
| 86 | + | // ErrInvalidCeilings reports an advertisement outside its window: a zero, | |
| 87 | + | // below-floor, or above-ceiling envelope, chunk, stream-count, or aggregate | |
| 88 | + | // bound. The preamble maps it to its own ErrPreambleRejected sentinel; the | |
| 89 | + | // engine and scheduler constructors return it directly. | |
| 90 | + | var ErrInvalidCeilings = errors.New("daemonmux: invalid ceilings") | |
| 91 | + | ||
| 92 | + | // MuxCeilings is the validated, immutable set of per-connection ceilings | |
| 93 | + | // negotiated exactly once by the daemonmux physical preamble: the receive | |
| 94 | + | // envelope ceiling (1 MiB..16 MiB), the stream chunk ceiling (1..64 KiB), the | |
| 95 | + | // concurrent stream ceiling (1..128), and the aggregate buffered-bytes | |
| 96 | + | // ceiling (MinMuxAggregateBytes..64 MiB). The preamble adopts the | |
| 97 | + | // element-wise minima of both peers' advertisements as its effective value. | |
| 98 | + | // | |
| 99 | + | // The engine and scheduler constructors take a MuxCeilings value and copy it, | |
| 100 | + | // so the ceilings an instance enforces are fixed for its lifetime: a caller | |
| 101 | + | // that mutates its own value after construction never changes live | |
| 102 | + | // enforcement. The zero value is invalid and is refused by Validate. | |
| 103 | + | type MuxCeilings struct { | |
| 104 | + | // MaxReceiveEnvelopeBytes is the largest serialized application envelope | |
| 105 | + | // either peer accepts, 1 MiB..16 MiB. | |
| 106 | + | MaxReceiveEnvelopeBytes uint64 | |
| 107 | + | // StreamChunkLimit is the largest opaque stream chunk, 1..64 KiB. | |
| 108 | + | StreamChunkLimit uint64 | |
| 109 | + | // MaxStreams is the largest number of pending plus live streams, 1..128: | |
| 110 | + | // opening, open, or closing (a closing stream keeps its slot until its | |
| 111 | + | // preserved inbound data is drained). | |
| 112 | + | MaxStreams uint64 | |
| 113 | + | // MaxAggregateBytes is the largest number of buffered bytes across every | |
| 114 | + | // live stream, MinMuxAggregateBytes..64 MiB. It is a per-direction budget: | |
| 115 | + | // the inbound engine and the outbound scheduler each enforce it | |
| 116 | + | // independently, so the two buffered directions never share one | |
| 117 | + | // accounting. The floor admits one maximum chunk plus envelope overhead. | |
| 118 | + | MaxAggregateBytes uint64 | |
| 119 | + | } | |
| 120 | + | ||
| 121 | + | // Validate refuses a zero, below-floor, or above-ceiling advertisement. | |
| 122 | + | // Envelope bounds are the 1 MiB..16 MiB window, chunk bounds are 1..64 KiB, | |
| 123 | + | // stream counts are 1..128, and aggregate bounds are | |
| 124 | + | // MinMuxAggregateBytes..64 MiB (one maximum chunk plus envelope overhead at | |
| 125 | + | // the floor). | |
| 126 | + | func (c MuxCeilings) Validate() error { | |
| 127 | + | if c.MaxReceiveEnvelopeBytes < MinMuxEnvelopeBytes || | |
| 128 | + | c.MaxReceiveEnvelopeBytes > MaxMuxEnvelopeBytes { | |
| 129 | + | return ErrInvalidCeilings | |
| 130 | + | } | |
| 131 | + | if c.StreamChunkLimit < MinMuxChunkBytes || | |
| 132 | + | c.StreamChunkLimit > MaxMuxChunkBytes { | |
| 133 | + | return ErrInvalidCeilings | |
| 134 | + | } | |
| 135 | + | if c.MaxStreams < MinMuxStreams || c.MaxStreams > MaxMuxStreams { | |
| 136 | + | return ErrInvalidCeilings | |
| 137 | + | } | |
| 138 | + | if c.MaxAggregateBytes < MinMuxAggregateBytes || | |
| 139 | + | c.MaxAggregateBytes > MaxMuxAggregateBytes { | |
| 140 | + | return ErrInvalidCeilings | |
| 141 | + | } | |
| 142 | + | return nil | |
| 143 | + | } | |
| 144 | + | ||
| 145 | + | // DefaultMuxCeilings advertises the local daemonmux receive policy: the full | |
| 146 | + | // absolute envelope ceiling, the maximum chunk, the maximum stream count, | |
| 147 | + | // and the maximum aggregate buffered bytes. | |
| 148 | + | func DefaultMuxCeilings() MuxCeilings { | |
| 149 | + | return MuxCeilings{ | |
| 150 | + | MaxReceiveEnvelopeBytes: MaxMuxEnvelopeBytes, | |
| 151 | + | StreamChunkLimit: MaxMuxChunkBytes, | |
| 152 | + | MaxStreams: MaxMuxStreams, | |
| 153 | + | MaxAggregateBytes: MaxMuxAggregateBytes, | |
| 154 | + | } | |
| 155 | + | } | |
| 156 | + | ||
| 157 | + | // EffectiveMuxCeilings takes the element-wise minima of both sides' | |
| 158 | + | // advertisements: the value the preamble adopts when a peer offers less than | |
| 159 | + | // the local policy. Each field is independent, so a peer that offers less in | |
| 160 | + | // one dimension lowers only that dimension. Both inputs are assumed | |
| 161 | + | // individually valid; the minima of two valid advertisements is valid. | |
| 162 | + | func EffectiveMuxCeilings(local, remote MuxCeilings) MuxCeilings { | |
| 163 | + | return MuxCeilings{ | |
| 164 | + | MaxReceiveEnvelopeBytes: min(local.MaxReceiveEnvelopeBytes, remote.MaxReceiveEnvelopeBytes), | |
| 165 | + | StreamChunkLimit: min(local.StreamChunkLimit, remote.StreamChunkLimit), | |
| 166 | + | MaxStreams: min(local.MaxStreams, remote.MaxStreams), | |
| 167 | + | MaxAggregateBytes: min(local.MaxAggregateBytes, remote.MaxAggregateBytes), | |
| 168 | + | } | |
| 169 | + | } | |
| 170 | + | ||
| 171 | + | // checkMuxCeilings refuses an invalid advertisement as a preamble refusal so | |
| 172 | + | // the preamble keeps its single ErrPreambleRejected sentinel. | |
| 173 | + | func checkMuxCeilings(ceilings MuxCeilings) error { | |
| 174 | + | if err := ceilings.Validate(); err != nil { | |
| 175 | + | return ErrPreambleRejected | |
| 176 | + | } | |
| 177 | + | return nil | |
| 178 | + | } | |
| 179 | + | ||
| 180 | + | // checkMuxEnvelopeCeiling rejects one serialized daemonmux application | |
| 181 | + | // envelope above either bound: the negotiated receive ceiling or the fixed | |
| 182 | + | // 1 MiB application ceiling. Every daemonmux envelope is control-class, so | |
| 183 | + | // the application ceiling always applies in full. | |
| 184 | + | func checkMuxEnvelopeCeiling(payload []byte, ceiling uint64) error { | |
| 185 | + | if uint64(len(payload)) > min(ceiling, MaxMuxApplicationBytes) { | |
| 186 | + | return wire.ErrScanLength | |
| 187 | + | } | |
| 188 | + | return nil | |
| 189 | + | } | |
| 190 | + | ||
| 191 | + | // CheckPreambleSize refuses one serialized daemonmux preamble above the | |
| 192 | + | // 4 KiB bound before scanning or allocation. Callers apply it to an encoded | |
| 193 | + | // preamble before sending and to a received payload before decode; the | |
| 194 | + | // session and broker contracts apply the identical wire.PreambleLimit. | |
| 195 | + | func CheckPreambleSize(payload []byte) error { | |
| 196 | + | if len(payload) > MuxPreambleLimit { | |
| 197 | + | return ErrPreambleRejected | |
| 198 | + | } | |
| 199 | + | return nil | |
| 200 | + | } | |
A internal/adapters/daemonmux/listener.go +946 -0
| @@ -0,0 +1,946 @@ | |||
| 1 | + | // Typed server-side listener over one physical Pump (P3.2). | |
| 2 | + | // | |
| 3 | + | // Listener is the daemon half of one physical daemonmux connection. The pump's | |
| 4 | + | // reader admits each incoming MuxOpen; the listener turns every admitted stream | |
| 5 | + | // into one independent typed ports.ServerConnection through sessionwire over | |
| 6 | + | // the mux stream, exactly as LogicalConnector does on the broker side. Like the | |
| 7 | + | // connector it owns neither framing nor composition: the carriage is the pump's | |
| 8 | + | // private byte stream and the typed protocol is sessionwire's. | |
| 9 | + | // | |
| 10 | + | // Admission and acceptance are decoupled by a bounded FIFO accept queue: an | |
| 11 | + | // admitted stream is queued, Accept hands back the oldest still-live one, and a | |
| 12 | + | // slow or stalled sibling never blocks the daemon's accept loop. One physical | |
| 13 | + | // connection admits at most its negotiated stream ceiling, so the queue is | |
| 14 | + | // bounded by MaxAcceptQueue (<=128) by construction; a stream admitted while | |
| 15 | + | // the queue is already full is refused on its own stream - exactly one mux | |
| 16 | + | // Reset, with no effect on a sibling or on the physical connection - instead of | |
| 17 | + | // growing the queue. | |
| 18 | + | // | |
| 19 | + | // Every stream carries one absolute local handshake deadline fixed at Open | |
| 20 | + | // admission (admission time + protocol.HandshakeTimeout). Queue delay and the | |
| 21 | + | // session handshake spend that one budget: the listener never restarts it, the | |
| 22 | + | // session connection built for the stream adopts the same absolute deadline | |
| 23 | + | // instead of starting a second one, and the deadline and the completion signal | |
| 24 | + | // are exposed to the daemon through the narrow optional HandshakePlumbing | |
| 25 | + | // interface. When the deadline elapses before the session handshake completed, | |
| 26 | + | // the deadline watchdog resets exactly that stream (one mux Reset, its | |
| 27 | + | // admission slot released) and leaves every sibling and the physical connection | |
| 28 | + | // untouched. A completed handshake retires the watchdog, so a stream that | |
| 29 | + | // finished its handshake is never expired afterwards. | |
| 30 | + | // | |
| 31 | + | // Failures stay stream-local and are never fatal to the shared accept loop: a | |
| 32 | + | // stream the peer resets or overflows, and a malformed inner session frame, all | |
| 33 | + | // settle that stream alone, and Accept skips settled streams instead of | |
| 34 | + | // returning an error for them. Accept fails only when it can never again | |
| 35 | + | // produce a stream: the listener was closed (ErrListenerClosed) or the one | |
| 36 | + | // physical connection reached its terminal outcome (ErrListenerLost, wrapping | |
| 37 | + | // the pump's cause), which is exactly when the daemon's accept loop must stop. | |
| 38 | + | // | |
| 39 | + | // Close unblocks every blocked Accept, drains every admitted-but-unaccepted | |
| 40 | + | // stream through its one stream-local Reset (releasing each admission slot and | |
| 41 | + | // unblocking the broker's pending Open), ends the deadline watchdog, and joins | |
| 42 | + | // it. It closes neither the physical pump - the pooled physical connection | |
| 43 | + | // outlives the listener - nor a connection already handed to the daemon. | |
| 44 | + | // | |
| 45 | + | // The listener must be constructed before the pump starts: it registers the | |
| 46 | + | // pump's single admission observer, which the pump refuses after Start with | |
| 47 | + | // ErrAdmissionObserverLate. That ordering is what guarantees the listener sees | |
| 48 | + | // every admitted Open. A stream whose inner session carriage closes on its own | |
| 49 | + | // during the handshake (a sessionwire preamble failure) is settled by the | |
| 50 | + | // connection's terminal watcher with exactly one stream-local Reset, releasing | |
| 51 | + | // its admission slot and watchdog without any consumer Close. | |
| 52 | + | package daemonmux | |
| 53 | + | ||
| 54 | + | import ( | |
| 55 | + | "errors" | |
| 56 | + | "fmt" | |
| 57 | + | "io" | |
| 58 | + | "sync" | |
| 59 | + | "time" | |
| 60 | + | ||
| 61 | + | "github.com/bnema/vev/internal/adapters/clock" | |
| 62 | + | "github.com/bnema/vev/internal/adapters/sessionwire" | |
| 63 | + | "github.com/bnema/vev/internal/domain" | |
| 64 | + | "github.com/bnema/vev/internal/ports" | |
| 65 | + | "github.com/bnema/vev/internal/protocol" | |
| 66 | + | ) | |
| 67 | + | ||
| 68 | + | // Listener sentinels. They are typed so a caller can classify the two fatal | |
| 69 | + | // Accept outcomes without matching on message text. | |
| 70 | + | var ( | |
| 71 | + | // ErrListenerConfig reports a listener constructed without a pump or over a | |
| 72 | + | // pump that does not read broker-to-daemon frames. | |
| 73 | + | ErrListenerConfig = errors.New("daemonmux: invalid listener configuration") | |
| 74 | + | // ErrListenerClosed reports Accept on a listener that was closed. | |
| 75 | + | ErrListenerClosed = errors.New("daemonmux: listener is closed") | |
| 76 | + | // ErrListenerLost reports Accept once the one physical connection reached | |
| 77 | + | // its terminal outcome. The pump's cause is wrapped when it has one; the | |
| 78 | + | // outcome is stable for every later Accept. | |
| 79 | + | ErrListenerLost = errors.New("daemonmux: listener physical connection lost") | |
| 80 | + | ) | |
| 81 | + | ||
| 82 | + | // errSessionCarriageClosed is the stream-local terminal cause published when a | |
| 83 | + | // listener connection's inner session carriage closed on its own during the | |
| 84 | + | // handshake - a sessionwire preamble failure closes the raw transport - without | |
| 85 | + | // a stream cause already recorded and without any consumer Close. It never | |
| 86 | + | // escapes as a physical failure: the offending stream is reset on its own. | |
| 87 | + | var errSessionCarriageClosed = errors.New("daemonmux: session carriage closed during handshake") | |
| 88 | + | ||
| 89 | + | const ( | |
| 90 | + | // MaxAcceptQueue bounds the admitted-but-unaccepted streams one listener | |
| 91 | + | // holds: the negotiated per-connection stream ceiling (1..128). The engine | |
| 92 | + | // admits at most that many opening or open streams, so the listener's FIFO | |
| 93 | + | // can never exceed it; a stream admitted while the queue is full is refused | |
| 94 | + | // on its own stream rather than queued. | |
| 95 | + | MaxAcceptQueue = int(MaxMuxStreams) | |
| 96 | + | ||
| 97 | + | // acceptQueueFullText and listenerClosedText are the bounded, | |
| 98 | + | // presentation-safe texts carried by the stream-local Reset a listener | |
| 99 | + | // sends when it refuses a stream it cannot queue, and | |
| 100 | + | // handshakeDeadlineText is the text of the stream-local Reset it sends when | |
| 101 | + | // a stream's absolute handshake deadline elapses. | |
| 102 | + | acceptQueueFullText = "accept queue full" | |
| 103 | + | listenerClosedText = "listener closed" | |
| 104 | + | handshakeDeadlineText = "handshake deadline exceeded" | |
| 105 | + | ||
| 106 | + | // sessionCarriageClosedText is the bounded, presentation-safe text carried | |
| 107 | + | // by the stream-local Reset a listener connection sends when its inner | |
| 108 | + | // session carriage closed on its own during the handshake (a sessionwire | |
| 109 | + | // preamble failure) without any consumer Close. | |
| 110 | + | sessionCarriageClosedText = "session handshake failed" | |
| 111 | + | ) | |
| 112 | + | ||
| 113 | + | // HandshakePlumbing is the narrow, optional seam a typed daemonmux connection | |
| 114 | + | // exposes so its consumer can adopt the exact absolute handshake deadline the | |
| 115 | + | // stream was admitted with and observe the session handshake's completion | |
| 116 | + | // without restarting the deadline. Both the broker-side LogicalConnection and | |
| 117 | + | // the daemon-side ListenerConnection implement it. It deliberately lives here, | |
| 118 | + | // not in ports, so the session contract is unchanged by this plumbing; a | |
| 119 | + | // consumer asserts it structurally. | |
| 120 | + | type HandshakePlumbing interface { | |
| 121 | + | // HandshakeDeadline is the absolute local deadline of the handshake, | |
| 122 | + | // fixed at Open admission and stable for the connection's lifetime. | |
| 123 | + | HandshakeDeadline() time.Time | |
| 124 | + | // OnHandshakeComplete registers fn to run exactly once when the session | |
| 125 | + | // handshake has run, whether it succeeded or failed. Registering after | |
| 126 | + | // completion runs fn immediately. | |
| 127 | + | OnHandshakeComplete(func()) | |
| 128 | + | } | |
| 129 | + | ||
| 130 | + | // Listener accepts the logical streams the daemon side of one physical | |
| 131 | + | // daemonmux connection admitted. It implements ports.ServerListener over the | |
| 132 | + | // pump's carriage: Accept returns independent typed ports.ServerConnection | |
| 133 | + | // values, Close unblocks Accept and joins the listener's goroutine, and Addr | |
| 134 | + | // names the multiplexed carriage. It is safe for concurrent use. | |
| 135 | + | type Listener struct { | |
| 136 | + | pump *Pump | |
| 137 | + | clock ports.Clock | |
| 138 | + | budget time.Duration | |
| 139 | + | queueLimit int | |
| 140 | + | ||
| 141 | + | // mu guards the accept queue, the deadline registry, and the closed flag. | |
| 142 | + | // change is the closed-and-replaced broadcast channel every blocked Accept | |
| 143 | + | // and the watchdog select on, so a state change never loses a wakeup. | |
| 144 | + | mu sync.Mutex | |
| 145 | + | queue []*admittedStream | |
| 146 | + | active map[PhysicalStreamID]*admittedStream | |
| 147 | + | closed bool | |
| 148 | + | change chan struct{} | |
| 149 | + | ||
| 150 | + | done chan struct{} | |
| 151 | + | watchDone chan struct{} | |
| 152 | + | ||
| 153 | + | closeOnce sync.Once | |
| 154 | + | } | |
| 155 | + | ||
| 156 | + | var ( | |
| 157 | + | _ ports.ServerListener = (*Listener)(nil) | |
| 158 | + | _ ports.ServerConnection = (*ListenerConnection)(nil) | |
| 159 | + | _ HandshakePlumbing = (*ListenerConnection)(nil) | |
| 160 | + | ) | |
| 161 | + | ||
| 162 | + | // admittedStream is one inbound logical stream the engine admitted and the | |
| 163 | + | // listener tracks until it is settled: delivered with a completed handshake, or | |
| 164 | + | // terminal (expired, peer-settled, refused, or released). | |
| 165 | + | type admittedStream struct { | |
| 166 | + | ref StreamRef | |
| 167 | + | deadline time.Time | |
| 168 | + | done <-chan struct{} | |
| 169 | + | queued bool | |
| 170 | + | settled bool | |
| 171 | + | } | |
| 172 | + | ||
| 173 | + | // NewListener returns a listener over one daemon-side pump under the package | |
| 174 | + | // policy: the accepted absolute handshake deadline budget is | |
| 175 | + | // protocol.HandshakeTimeout, and the accept queue is bounded by | |
| 176 | + | // MaxAcceptQueue. The pump must decode broker-to-daemon frames | |
| 177 | + | // (Inbound == DirectionClient) and emit daemon-to-broker frames, as the daemon | |
| 178 | + | // side of a pooled physical connection does; a nil, engine-less, or | |
| 179 | + | // broker-side pump is refused. The pump must not have started: the listener | |
| 180 | + | // registers the pump's admission observer, and a pump that already started | |
| 181 | + | // refuses the registration with ErrAdmissionObserverLate (wrapped in | |
| 182 | + | // ErrListenerConfig), because an observer registered after Start could miss an | |
| 183 | + | // already-admitted Open. | |
| 184 | + | func NewListener(pump *Pump) (*Listener, error) { | |
| 185 | + | return newListener(pump, protocol.HandshakeTimeout, MaxAcceptQueue, clock.New()) | |
| 186 | + | } | |
| 187 | + | ||
| 188 | + | // newListener builds a listener from explicit policy. budget is the handshake | |
| 189 | + | // deadline budget each stream receives at admission, limit is the accept-queue | |
| 190 | + | // bound, and clock supplies the admission time and the deadline timer. The | |
| 191 | + | // exported constructor delegates with the package policy; a test tightens the | |
| 192 | + | // budget, the queue bound, or the clock without changing live behavior. | |
| 193 | + | func newListener(pump *Pump, budget time.Duration, limit int, timeSource ports.Clock) (*Listener, error) { | |
| 194 | + | if pump == nil || pump.Engine() == nil || pump.Inbound() != DirectionClient { | |
| 195 | + | return nil, ErrListenerConfig | |
| 196 | + | } | |
| 197 | + | if budget <= 0 { | |
| 198 | + | budget = protocol.HandshakeTimeout | |
| 199 | + | } | |
| 200 | + | if limit <= 0 { | |
| 201 | + | limit = MaxAcceptQueue | |
| 202 | + | } | |
| 203 | + | if timeSource == nil { | |
| 204 | + | timeSource = clock.New() | |
| 205 | + | } | |
| 206 | + | listener := &Listener{ | |
| 207 | + | pump: pump, | |
| 208 | + | clock: timeSource, | |
| 209 | + | budget: budget, | |
| 210 | + | queueLimit: limit, | |
| 211 | + | active: make(map[PhysicalStreamID]*admittedStream), | |
| 212 | + | change: make(chan struct{}), | |
| 213 | + | done: make(chan struct{}), | |
| 214 | + | watchDone: make(chan struct{}), | |
| 215 | + | } | |
| 216 | + | if err := pump.OnAdmitted(listener.onAdmitted); err != nil { | |
| 217 | + | return nil, fmt.Errorf("%w: %w", ErrListenerConfig, err) | |
| 218 | + | } | |
| 219 | + | go listener.watchLoop() | |
| 220 | + | return listener, nil | |
| 221 | + | } | |
| 222 | + | ||
| 223 | + | // Addr names the listener's carriage for ports.ServerListener. One physical | |
| 224 | + | // daemonmux connection multiplexes every logical stream, so the listener has no | |
| 225 | + | // per-stream address. | |
| 226 | + | func (l *Listener) Addr() string { return "daemonmux" } | |
| 227 | + | ||
| 228 | + | // Accept returns the oldest admitted stream as one independent typed session | |
| 229 | + | // connection, blocking until one is available. It hands a stream back at most | |
| 230 | + | // once and skips streams that already reached a terminal outcome - expired, | |
| 231 | + | // peer-reset, overflowed, or refused - instead of returning them. It fails with | |
| 232 | + | // ErrListenerClosed after Close and with ErrListenerLost once the physical | |
| 233 | + | // connection reached its terminal outcome; both are stable, so the shared | |
| 234 | + | // daemon accept loop stops exactly when no stream can ever arrive again. A | |
| 235 | + | // malformed or failed individual stream never becomes an Accept error. | |
| 236 | + | func (l *Listener) Accept() (ports.ServerConnection, error) { | |
| 237 | + | if l == nil { | |
| 238 | + | return nil, ErrListenerConfig | |
| 239 | + | } | |
| 240 | + | for { | |
| 241 | + | entry, wait, err := l.nextAdmitted() | |
| 242 | + | if err != nil { | |
| 243 | + | return nil, err | |
| 244 | + | } | |
| 245 | + | if entry != nil { | |
| 246 | + | if connection := l.deliver(entry); connection != nil { | |
| 247 | + | return connection, nil | |
| 248 | + | } | |
| 249 | + | continue | |
| 250 | + | } | |
| 251 | + | select { | |
| 252 | + | case <-wait: | |
| 253 | + | case <-l.done: | |
| 254 | + | case <-l.pump.Done(): | |
| 255 | + | } | |
| 256 | + | } | |
| 257 | + | } | |
| 258 | + | ||
| 259 | + | // Close stops accepting: it unblocks every blocked Accept with | |
| 260 | + | // ErrListenerClosed, drains every admitted-but-unaccepted stream through its | |
| 261 | + | // one stream-local Reset (releasing its admission slot and unblocking the | |
| 262 | + | // broker's pending Open), ends the deadline watchdog, and joins it. Close is | |
| 263 | + | // idempotent and concurrent-safe; closeOnce guarantees a single drain across | |
| 264 | + | // concurrent callers. It does not close the physical pump, which the pooled | |
| 265 | + | // connection owns, and it does not close or abort a connection already handed | |
| 266 | + | // to the daemon, which owns that stream from then on. | |
| 267 | + | func (l *Listener) Close() error { | |
| 268 | + | if l == nil { | |
| 269 | + | return nil | |
| 270 | + | } | |
| 271 | + | l.closeOnce.Do(func() { | |
| 272 | + | l.mu.Lock() | |
| 273 | + | l.closed = true | |
| 274 | + | queued := l.queue | |
| 275 | + | l.queue = nil | |
| 276 | + | for _, entry := range queued { | |
| 277 | + | if entry.settled { | |
| 278 | + | continue | |
| 279 | + | } | |
| 280 | + | entry.settled = true | |
| 281 | + | delete(l.active, entry.ref.Physical) | |
| 282 | + | } | |
| 283 | + | l.signalLocked() | |
| 284 | + | l.mu.Unlock() | |
| 285 | + | // Settle every drained admission on its own stream: one Reset releases | |
| 286 | + | // the admission slot and unblocks the broker's pending Open. Every abort | |
| 287 | + | // runs outside the listener lock, and closeOnce makes it exactly once. | |
| 288 | + | for _, entry := range queued { | |
| 289 | + | l.abort(entry.ref.Physical, listenerClosedText) | |
| 290 | + | } | |
| 291 | + | close(l.done) | |
| 292 | + | <-l.watchDone | |
| 293 | + | }) | |
| 294 | + | return nil | |
| 295 | + | } | |
| 296 | + | ||
| 297 | + | // onAdmitted is the pump's admission observer. It runs on the pump's reader | |
| 298 | + | // goroutine immediately after one inbound Open was admitted: it queues the | |
| 299 | + | // stream under its admission deadline, or refuses exactly that stream when the | |
| 300 | + | // accept queue is full or the listener is closed. It never blocks - the reader | |
| 301 | + | // is the one goroutine that applies every inbound frame of the connection. | |
| 302 | + | func (l *Listener) onAdmitted(open Open) { | |
| 303 | + | admitted := l.clock.Now() | |
| 304 | + | entry := &admittedStream{ | |
| 305 | + | ref: open.Ref, | |
| 306 | + | deadline: admitted.Add(l.budget), | |
| 307 | + | } | |
| 308 | + | if status, ok := l.pump.Engine().Status(open.Ref.Physical); ok { | |
| 309 | + | entry.done = status.Done | |
| 310 | + | } | |
| 311 | + | ||
| 312 | + | l.mu.Lock() | |
| 313 | + | if l.closed { | |
| 314 | + | l.mu.Unlock() | |
| 315 | + | l.abort(open.Ref.Physical, listenerClosedText) | |
| 316 | + | return | |
| 317 | + | } | |
| 318 | + | if len(l.queue) >= l.queueLimit { | |
| 319 | + | l.mu.Unlock() | |
| 320 | + | l.abort(open.Ref.Physical, acceptQueueFullText) | |
| 321 | + | return | |
| 322 | + | } | |
| 323 | + | entry.queued = true | |
| 324 | + | l.queue = append(l.queue, entry) | |
| 325 | + | l.active[open.Ref.Physical] = entry | |
| 326 | + | l.signalLocked() | |
| 327 | + | l.mu.Unlock() | |
| 328 | + | } | |
| 329 | + | ||
| 330 | + | // nextAdmitted pops the oldest still-live admitted stream, or reports the | |
| 331 | + | // channel to wait on and any stable fatal outcome. A stream that already | |
| 332 | + | // reached a terminal outcome is dropped instead of handed to the daemon. | |
| 333 | + | func (l *Listener) nextAdmitted() (*admittedStream, <-chan struct{}, error) { | |
| 334 | + | l.mu.Lock() | |
| 335 | + | defer l.mu.Unlock() | |
| 336 | + | if l.closed { | |
| 337 | + | return nil, nil, ErrListenerClosed | |
| 338 | + | } | |
| 339 | + | for len(l.queue) > 0 { | |
| 340 | + | entry := l.queue[0] | |
| 341 | + | l.queue = l.queue[1:] | |
| 342 | + | entry.queued = false | |
| 343 | + | if entry.settled { | |
| 344 | + | continue | |
| 345 | + | } | |
| 346 | + | if watchClosed(entry.done) { | |
| 347 | + | entry.settled = true | |
| 348 | + | delete(l.active, entry.ref.Physical) | |
| 349 | + | continue | |
| 350 | + | } | |
| 351 | + | return entry, nil, nil | |
| 352 | + | } | |
| 353 | + | if err := l.lostLocked(); err != nil { | |
| 354 | + | return nil, nil, err | |
| 355 | + | } | |
| 356 | + | return nil, l.change, nil | |
| 357 | + | } | |
| 358 | + | ||
| 359 | + | // deliver confirms one admitted stream towards the peer and builds its typed | |
| 360 | + | // session connection. It reports nil when the stream can no longer be | |
| 361 | + | // delivered - its absolute admission deadline elapsed before the watchdog ran, | |
| 362 | + | // it was reset by the peer, the listener was closed, or the physical connection | |
| 363 | + | // reached its terminal outcome between admission and acceptance - so the caller | |
| 364 | + | // moves on to the next stream instead of surfacing a fatal Accept error. An | |
| 365 | + | // entry it finds expired is settled on its own here, without waiting for the | |
| 366 | + | // watchdog to observe the same deadline. | |
| 367 | + | func (l *Listener) deliver(entry *admittedStream) ports.ServerConnection { | |
| 368 | + | if ready, abortText := l.claimForDelivery(entry); !ready { | |
| 369 | + | if abortText != "" { | |
| 370 | + | l.abort(entry.ref.Physical, abortText) | |
| 371 | + | } | |
| 372 | + | return nil | |
| 373 | + | } | |
| 374 | + | if err := l.pump.Engine().Opened(Opened{Ref: entry.ref}); err != nil { | |
| 375 | + | l.finishStream(entry.ref.Physical) | |
| 376 | + | return nil | |
| 377 | + | } | |
| 378 | + | connection := newListenerConnection(l, entry) | |
| 379 | + | if err := l.pump.Send(Opened{Ref: entry.ref}); err != nil { | |
| 380 | + | // The peer can never be confirmed: settle this stream alone so its | |
| 381 | + | // carriage is released instead of leaking a half-delivered stream. | |
| 382 | + | _ = connection.Close() | |
| 383 | + | return nil | |
| 384 | + | } | |
| 385 | + | return connection | |
| 386 | + | } | |
| 387 | + | ||
| 388 | + | // claimForDelivery classifies one popped admission under the listener lock. It | |
| 389 | + | // returns true when the stream may still be delivered. A stream the watchdog | |
| 390 | + | // already settled is simply reported as not ready; a stream whose absolute | |
| 391 | + | // admission deadline elapsed before the watchdog observed it, or whose listener | |
| 392 | + | // was closed after the stream left the queue, is retired from the watchdog here | |
| 393 | + | // and reported with the bounded Reset text its caller must signal the peer | |
| 394 | + | // with, so the admission slot is released even when the watchdog never ran. | |
| 395 | + | func (l *Listener) claimForDelivery(entry *admittedStream) (ready bool, abortText string) { | |
| 396 | + | l.mu.Lock() | |
| 397 | + | defer l.mu.Unlock() | |
| 398 | + | if entry.settled { | |
| 399 | + | return false, "" | |
| 400 | + | } | |
| 401 | + | switch { | |
| 402 | + | case l.closed: | |
| 403 | + | l.retireLocked(entry) | |
| 404 | + | return false, listenerClosedText | |
| 405 | + | case !entry.deadline.After(l.clock.Now()): | |
| 406 | + | l.retireLocked(entry) | |
| 407 | + | return false, handshakeDeadlineText | |
| 408 | + | default: | |
| 409 | + | return true, "" | |
| 410 | + | } | |
| 411 | + | } | |
| 412 | + | ||
| 413 | + | // retireLocked marks one popped admission settled and removes it from the | |
| 414 | + | // deadline watchdog. The caller holds the listener lock. | |
| 415 | + | func (l *Listener) retireLocked(entry *admittedStream) { | |
| 416 | + | entry.settled = true | |
| 417 | + | delete(l.active, entry.ref.Physical) | |
| 418 | + | l.signalLocked() | |
| 419 | + | } | |
| 420 | + | ||
| 421 | + | // finishStream retires one stream from the deadline watchdog: its session | |
| 422 | + | // handshake completed, its connection was closed, or it reached a terminal | |
| 423 | + | // outcome. A retired stream is never expired afterwards, so a completed | |
| 424 | + | // handshake never triggers a late stream-local reset. | |
| 425 | + | func (l *Listener) finishStream(physical PhysicalStreamID) { | |
| 426 | + | if l == nil || physical.Validate() != nil { | |
| 427 | + | return | |
| 428 | + | } | |
| 429 | + | l.mu.Lock() | |
| 430 | + | defer l.mu.Unlock() | |
| 431 | + | entry, ok := l.active[physical] | |
| 432 | + | if !ok { | |
| 433 | + | return | |
| 434 | + | } | |
| 435 | + | entry.settled = true | |
| 436 | + | delete(l.active, physical) | |
| 437 | + | l.signalLocked() | |
| 438 | + | } | |
| 439 | + | ||
| 440 | + | // abort settles one locally refused, expired, or malformed stream and | |
| 441 | + | // best-effort signals the peer with exactly one mux Reset. Only the offending | |
| 442 | + | // stream is touched: siblings and the physical connection keep flowing. An | |
| 443 | + | // identity the engine never admitted, and an identity that already reached its | |
| 444 | + | // terminal outcome, are left alone by the engine itself. | |
| 445 | + | func (l *Listener) abort(physical PhysicalStreamID, text string) { | |
| 446 | + | if physical.Validate() != nil { | |
| 447 | + | return | |
| 448 | + | } | |
| 449 | + | reset := Reset{ | |
| 450 | + | Physical: physical, | |
| 451 | + | Error: ErrorDetail{ | |
| 452 | + | Code: ports.BrokerErrorUnavailable, | |
| 453 | + | Text: text, | |
| 454 | + | FailureKind: domain.RemoteFailureTransport, | |
| 455 | + | }, | |
| 456 | + | HasError: true, | |
| 457 | + | } | |
| 458 | + | _, _ = l.pump.Engine().Reset(reset) | |
| 459 | + | _ = l.pump.Send(reset) | |
| 460 | + | } | |
| 461 | + | ||
| 462 | + | // lostLocked returns the stable fatal Accept outcome once the physical | |
| 463 | + | // connection reached its terminal outcome, or nil while it is still live. The | |
| 464 | + | // caller holds the listener lock. | |
| 465 | + | func (l *Listener) lostLocked() error { | |
| 466 | + | select { | |
| 467 | + | case <-l.pump.Done(): | |
| 468 | + | if err := l.pump.Err(); err != nil { | |
| 469 | + | return fmt.Errorf("%w: %w", ErrListenerLost, err) | |
| 470 | + | } | |
| 471 | + | return ErrListenerLost | |
| 472 | + | default: | |
| 473 | + | return nil | |
| 474 | + | } | |
| 475 | + | } | |
| 476 | + | ||
| 477 | + | // signalLocked wakes every blocked Accept and the deadline watchdog. The | |
| 478 | + | // broadcast channel is closed and replaced, so no waiter can miss the change. | |
| 479 | + | // The caller holds the listener lock. | |
| 480 | + | func (l *Listener) signalLocked() { | |
| 481 | + | close(l.change) | |
| 482 | + | l.change = make(chan struct{}) | |
| 483 | + | } | |
| 484 | + | ||
| 485 | + | // watchLoop is the deadline watchdog. It never polls: it waits for the earliest | |
| 486 | + | // active stream deadline, expires every stream that reached it, and re-scans | |
| 487 | + | // whenever the registry changed. It ends when the listener is closed or the | |
| 488 | + | // physical connection reached its terminal outcome. | |
| 489 | + | func (l *Listener) watchLoop() { | |
| 490 | + | defer close(l.watchDone) | |
| 491 | + | for { | |
| 492 | + | deadline, change, ok := l.watchSnapshot() | |
| 493 | + | var timer ports.Timer | |
| 494 | + | var fire <-chan time.Time | |
| 495 | + | if ok { | |
| 496 | + | wait := deadline.Sub(l.clock.Now()) | |
| 497 | + | if wait < 0 { | |
| 498 | + | wait = 0 | |
| 499 | + | } | |
| 500 | + | timer = l.clock.NewTimer(wait) | |
| 501 | + | fire = timer.C() | |
| 502 | + | } | |
| 503 | + | select { | |
| 504 | + | case <-fire: | |
| 505 | + | l.expireDue() | |
| 506 | + | case <-change: | |
| 507 | + | case <-l.done: | |
| 508 | + | stopTimer(timer) | |
| 509 | + | return | |
| 510 | + | case <-l.pump.Done(): | |
| 511 | + | stopTimer(timer) | |
| 512 | + | return | |
| 513 | + | } | |
| 514 | + | stopTimer(timer) | |
| 515 | + | } | |
| 516 | + | } | |
| 517 | + | ||
| 518 | + | // watchSnapshot returns the earliest tracked deadline and the broadcast channel | |
| 519 | + | // to wait on atomically, so a state change published between the scan and the | |
| 520 | + | // wait is never missed: the captured channel is already closed when it does. | |
| 521 | + | func (l *Listener) watchSnapshot() (time.Time, <-chan struct{}, bool) { | |
| 522 | + | l.mu.Lock() | |
| 523 | + | defer l.mu.Unlock() | |
| 524 | + | earliest, found := l.earliestDeadlineLocked() | |
| 525 | + | return earliest, l.change, found | |
| 526 | + | } | |
| 527 | + | ||
| 528 | + | // earliestDeadlineLocked reports the earliest tracked deadline. The caller | |
| 529 | + | // holds the listener lock. | |
| 530 | + | func (l *Listener) earliestDeadlineLocked() (time.Time, bool) { | |
| 531 | + | var earliest time.Time | |
| 532 | + | found := false | |
| 533 | + | for _, entry := range l.active { | |
| 534 | + | if entry.settled { | |
| 535 | + | continue | |
| 536 | + | } | |
| 537 | + | if !found || entry.deadline.Before(earliest) { | |
| 538 | + | earliest = entry.deadline | |
| 539 | + | found = true | |
| 540 | + | } | |
| 541 | + | } | |
| 542 | + | return earliest, found | |
| 543 | + | } | |
| 544 | + | ||
| 545 | + | // expireDue resets every tracked stream whose absolute admission deadline | |
| 546 | + | // elapsed. Each reset is stream-local: the stream's admission slot is released | |
| 547 | + | // and its delivered connection observes its own terminal outcome, while every | |
| 548 | + | // sibling and the physical connection stay untouched. A stream that already | |
| 549 | + | // reached a terminal outcome is retired without a second reset. | |
| 550 | + | func (l *Listener) expireDue() { | |
| 551 | + | now := l.clock.Now() | |
| 552 | + | l.mu.Lock() | |
| 553 | + | due := make([]*admittedStream, 0, len(l.active)) | |
| 554 | + | for id, entry := range l.active { | |
| 555 | + | if entry.settled || entry.deadline.After(now) { | |
| 556 | + | continue | |
| 557 | + | } | |
| 558 | + | entry.settled = true | |
| 559 | + | delete(l.active, id) | |
| 560 | + | if entry.queued { | |
| 561 | + | l.dropLocked(entry) | |
| 562 | + | } | |
| 563 | + | due = append(due, entry) | |
| 564 | + | } | |
| 565 | + | if len(due) > 0 { | |
| 566 | + | l.signalLocked() | |
| 567 | + | } | |
| 568 | + | l.mu.Unlock() | |
| 569 | + | for _, entry := range due { | |
| 570 | + | if watchClosed(entry.done) { | |
| 571 | + | continue | |
| 572 | + | } | |
| 573 | + | l.abort(entry.ref.Physical, handshakeDeadlineText) | |
| 574 | + | } | |
| 575 | + | } | |
| 576 | + | ||
| 577 | + | // dropLocked removes one queued stream from the FIFO, so Accept never hands out | |
| 578 | + | // a stream the watchdog already expired. The caller holds the listener lock. | |
| 579 | + | func (l *Listener) dropLocked(target *admittedStream) { | |
| 580 | + | for i, entry := range l.queue { | |
| 581 | + | if entry == target { | |
| 582 | + | l.queue = append(l.queue[:i], l.queue[i+1:]...) | |
| 583 | + | return | |
| 584 | + | } | |
| 585 | + | } | |
| 586 | + | } | |
| 587 | + | ||
| 588 | + | // stopTimer releases a deadline timer, tolerating a loop that never armed one. | |
| 589 | + | func stopTimer(timer ports.Timer) { | |
| 590 | + | if timer != nil { | |
| 591 | + | timer.Stop() | |
| 592 | + | } | |
| 593 | + | } | |
| 594 | + | ||
| 595 | + | // ListenerConnection is one typed daemon-side logical stream accepted from a | |
| 596 | + | // Listener: the sessionwire server connection over the mux byte stream, plus | |
| 597 | + | // the stream lifecycle the daemon owns. It implements ports.ServerConnection | |
| 598 | + | // and exposes the narrow optional HandshakePlumbing seam - the absolute | |
| 599 | + | // handshake deadline fixed at admission and the completion signal - without | |
| 600 | + | // restarting the deadline. Done, Err, and FailureKind publish the stream's | |
| 601 | + | // terminal outcome independently of reads. It is safe for concurrent use. | |
| 602 | + | type ListenerConnection struct { | |
| 603 | + | ports.ServerConnection | |
| 604 | + | ||
| 605 | + | listener *Listener | |
| 606 | + | pump *Pump | |
| 607 | + | ref StreamRef | |
| 608 | + | pipe *muxStreamPipe | |
| 609 | + | raw *muxTransport | |
| 610 | + | stream <-chan struct{} | |
| 611 | + | // cause is the stream's retained publish-once terminal authority, captured | |
| 612 | + | // at acceptance. It outlives the engine's bounded record, so a settled | |
| 613 | + | // stream keeps its exact cause even after the record is evicted. | |
| 614 | + | cause *terminalState | |
| 615 | + | ||
| 616 | + | // plumbing is the wrapped session connection's handshake seam, resolved | |
| 617 | + | // once at construction; nil when the wrapped connection lacks it. | |
| 618 | + | plumbing HandshakePlumbing | |
| 619 | + | // deadline is the exact absolute handshake deadline fixed at Open | |
| 620 | + | // admission, resolved once at construction. | |
| 621 | + | deadline time.Time | |
| 622 | + | ||
| 623 | + | terminal *terminalState | |
| 624 | + | watchDone chan struct{} | |
| 625 | + | closeOnce sync.Once | |
| 626 | + | closeErr error | |
| 627 | + | } | |
| 628 | + | ||
| 629 | + | // newListenerConnection builds the typed session channel over one confirmed | |
| 630 | + | // open stream and starts the terminal watcher. It never blocks. The session | |
| 631 | + | // connection adopts the stream's admission deadline, so accepting a stream late | |
| 632 | + | // consumes that budget instead of starting a second one, and the listener | |
| 633 | + | // retires its watchdog exactly when the session handshake completes. | |
| 634 | + | func newListenerConnection(l *Listener, entry *admittedStream) *ListenerConnection { | |
| 635 | + | limit := l.pump.Ceilings().StreamChunkLimit | |
| 636 | + | pipe := newMuxStreamPipe(l.pump, entry.ref.Physical, limit, entry.done) | |
| 637 | + | raw := newMuxTransport(pipe) | |
| 638 | + | connection := &ListenerConnection{ | |
| 639 | + | ServerConnection: sessionwire.NewServerConnectionWithDeadline(raw, entry.deadline), | |
| 640 | + | listener: l, | |
| 641 | + | pump: l.pump, | |
| 642 | + | ref: entry.ref, | |
| 643 | + | pipe: pipe, | |
| 644 | + | raw: raw, | |
| 645 | + | stream: entry.done, | |
| 646 | + | cause: l.pump.cause(entry.ref.Physical), | |
| 647 | + | deadline: entry.deadline, | |
| 648 | + | terminal: newTerminalState(), | |
| 649 | + | watchDone: make(chan struct{}), | |
| 650 | + | } | |
| 651 | + | if plumbing, ok := connection.ServerConnection.(HandshakePlumbing); ok { | |
| 652 | + | connection.plumbing = plumbing | |
| 653 | + | connection.deadline = plumbing.HandshakeDeadline() | |
| 654 | + | } | |
| 655 | + | connection.OnHandshakeComplete(func() { l.finishStream(entry.ref.Physical) }) | |
| 656 | + | go connection.watch() | |
| 657 | + | return connection | |
| 658 | + | } | |
| 659 | + | ||
| 660 | + | // Ref returns the complete stream reference recorded at admission. | |
| 661 | + | func (c *ListenerConnection) Ref() StreamRef { return c.ref } | |
| 662 | + | ||
| 663 | + | // HandshakeDeadline returns the accepted absolute local deadline of the | |
| 664 | + | // session handshake, fixed at Open admission and plumbing the exact deadline | |
| 665 | + | // the wrapped session connection started with. Queue delay and the handshake | |
| 666 | + | // spend this one budget; the value is stable for the connection's lifetime. | |
| 667 | + | func (c *ListenerConnection) HandshakeDeadline() time.Time { return c.deadline } | |
| 668 | + | ||
| 669 | + | // OnHandshakeComplete registers fn to run exactly once when the session | |
| 670 | + | // handshake completes, plumbing the wrapped session connection's completion | |
| 671 | + | // hook. Registering after completion runs fn immediately. | |
| 672 | + | func (c *ListenerConnection) OnHandshakeComplete(fn func()) { | |
| 673 | + | if c.plumbing != nil { | |
| 674 | + | c.plumbing.OnHandshakeComplete(fn) | |
| 675 | + | return | |
| 676 | + | } | |
| 677 | + | if fn != nil { | |
| 678 | + | fn() | |
| 679 | + | } | |
| 680 | + | } | |
| 681 | + | ||
| 682 | + | // Done returns the logical terminal channel independent of reads. It closes | |
| 683 | + | // exactly once, before Err becomes stable. | |
| 684 | + | func (c *ListenerConnection) Done() <-chan struct{} { return c.terminal.Done() } | |
| 685 | + | ||
| 686 | + | // Err returns the stable terminal cause: nil for an orderly local or peer | |
| 687 | + | // Close, the retained stream cause for a peer Reset, a queue overflow, or a | |
| 688 | + | // watchdog expiry, and the pump's cause for a physical loss. | |
| 689 | + | func (c *ListenerConnection) Err() error { return c.terminal.Err() } | |
| 690 | + | ||
| 691 | + | // FailureKind classifies Err. It is None while open and for an orderly Close. | |
| 692 | + | func (c *ListenerConnection) FailureKind() domain.RemoteFailureKind { return c.terminal.FailureKind() } | |
| 693 | + | ||
| 694 | + | // ReceiveClient receives one typed client message. An orderly end of the inner | |
| 695 | + | // stream returns io.EOF; any other inner-stream error is treated as a malformed | |
| 696 | + | // or failed session frame: exactly this stream is reset towards the peer, its | |
| 697 | + | // terminal outcome is published, and the error is returned. Neither outcome | |
| 698 | + | // touches a sibling stream or the physical connection. | |
| 699 | + | func (c *ListenerConnection) ReceiveClient() (protocol.ClientMessage, error) { | |
| 700 | + | if c.settled() { | |
| 701 | + | return nil, c.terminalError() | |
| 702 | + | } | |
| 703 | + | message, err := c.ServerConnection.ReceiveClient() | |
| 704 | + | if err == nil { | |
| 705 | + | return message, nil | |
| 706 | + | } | |
| 707 | + | if errors.Is(err, io.EOF) { | |
| 708 | + | c.syncFromStream() | |
| 709 | + | return nil, c.terminalError() | |
| 710 | + | } | |
| 711 | + | return nil, c.abortMalformed(err) | |
| 712 | + | } | |
| 713 | + | ||
| 714 | + | // Close independently closes the logical stream: it discards the inbound | |
| 715 | + | // chunks the engine already accepted, settles this stream locally, sends | |
| 716 | + | // exactly one mux Close, publishes the orderly nil outcome, and unblocks every | |
| 717 | + | // read and write. It never touches the physical connection or a sibling | |
| 718 | + | // stream. Close is idempotent and concurrent-safe; every caller observes the | |
| 719 | + | // same stored transport Close error. | |
| 720 | + | func (c *ListenerConnection) Close() error { | |
| 721 | + | c.closeOnce.Do(func() { | |
| 722 | + | // Discard what the engine already accepted before closing locally: this | |
| 723 | + | // side stops reading as soon as it closes, so draining first is what lets | |
| 724 | + | // the engine's orderly Close reach its terminal state and release the | |
| 725 | + | // stream's admission slot and aggregate bytes instead of parking the | |
| 726 | + | // stream in closing with a queue nothing will ever drain. | |
| 727 | + | discardQueuedInbound(c.pump, c.ref.Physical) | |
| 728 | + | _, _ = c.pump.Engine().Close(Close{Physical: c.ref.Physical}) | |
| 729 | + | // A chunk the reader accepted between the drain and the Close parked the | |
| 730 | + | // stream in closing instead of terminal; no new chunk is accepted after | |
| 731 | + | // the Close, so this drain terminalizes the stream and releases its slot | |
| 732 | + | // and bytes. | |
| 733 | + | discardQueuedInbound(c.pump, c.ref.Physical) | |
| 734 | + | _ = c.pump.Send(Close{Physical: c.ref.Physical}) | |
| 735 | + | c.terminal.Close() | |
| 736 | + | c.closeErr = c.raw.Close() | |
| 737 | + | // The watcher retires the stream from the deadline watchdog and the | |
| 738 | + | // pump's watch table before it closes watchDone. | |
| 739 | + | <-c.watchDone | |
| 740 | + | }) | |
| 741 | + | return c.closeErr | |
| 742 | + | } | |
| 743 | + | ||
| 744 | + | // watch publishes the logical terminal outcome when the stream settles on its | |
| 745 | + | // own (peer Close, peer Reset, queue overflow, watchdog expiry, or physical | |
| 746 | + | // loss) independently of reads, and unblocks the carriage. A local Close | |
| 747 | + | // settles first and makes this a no-op. | |
| 748 | + | func (c *ListenerConnection) watch() { | |
| 749 | + | defer c.pump.ReleaseWatch(c.ref.Physical) | |
| 750 | + | defer c.listener.finishStream(c.ref.Physical) | |
| 751 | + | defer close(c.watchDone) | |
| 752 | + | select { | |
| 753 | + | case <-c.stream: | |
| 754 | + | // The stream reached its terminal state. Publish the retained cause, | |
| 755 | + | // not a fresh engine lookup: the engine record may already be evicted. | |
| 756 | + | c.syncFromStream() | |
| 757 | + | case <-c.pump.Done(): | |
| 758 | + | // A physical terminal outcome wakes every stream. A stream-local cause | |
| 759 | + | // recorded before the physical failure stays authoritative. | |
| 760 | + | if !c.settleFromCause() { | |
| 761 | + | if err := c.pump.Err(); err != nil { | |
| 762 | + | c.settleLoss(c.pump.FailureKind(), err) | |
| 763 | + | } else { | |
| 764 | + | c.terminal.Close() | |
| 765 | + | } | |
| 766 | + | } | |
| 767 | + | case <-c.pipe.Done(): | |
| 768 | + | // The inner session carriage closed on its own. A sessionwire preamble | |
| 769 | + | // failure closes the raw transport, which closes the pipe, before any | |
| 770 | + | // consumer Close: settle the stream from the retained or physical cause | |
| 771 | + | // when one already exists, and otherwise issue exactly one stream-local | |
| 772 | + | // Reset so the admission slot and watchdog are released without a | |
| 773 | + | // consumer Close. | |
| 774 | + | if !c.settleFromCarriage() { | |
| 775 | + | _ = c.abortClosedCarriage() | |
| 776 | + | } | |
| 777 | + | } | |
| 778 | + | // Closing the framed transport also closes the byte stream and joins | |
| 779 | + | // streamframe's writer, so a self-terminating stream leaves no goroutine | |
| 780 | + | // behind. | |
| 781 | + | _ = c.raw.Close() | |
| 782 | + | } | |
| 783 | + | ||
| 784 | + | // syncFromStream settles this connection from the stream's retained terminal | |
| 785 | + | // authority when it published, and otherwise from the pump's own terminal | |
| 786 | + | // outcome. It closes the race between a blocked read observing the inner-stream | |
| 787 | + | // end and the watcher publishing the same outcome, and it never lets a physical | |
| 788 | + | // failure surface as a clean io.EOF: the retained authority outlives the | |
| 789 | + | // bounded engine record, so an evicted stream still reports its exact cause. | |
| 790 | + | func (c *ListenerConnection) syncFromStream() { | |
| 791 | + | if c.settled() { | |
| 792 | + | return | |
| 793 | + | } | |
| 794 | + | if c.settleFromCause() { | |
| 795 | + | return | |
| 796 | + | } | |
| 797 | + | if err := c.pump.Err(); err != nil { | |
| 798 | + | c.settleLoss(c.pump.FailureKind(), err) | |
| 799 | + | return | |
| 800 | + | } | |
| 801 | + | if status, ok := c.pump.Engine().Status(c.ref.Physical); ok && status.State == StreamTerminal { | |
| 802 | + | c.settleFromStatus(status) | |
| 803 | + | return | |
| 804 | + | } | |
| 805 | + | c.terminal.Close() | |
| 806 | + | } | |
| 807 | + | ||
| 808 | + | // settleFromCause publishes the terminal outcome recorded by the stream's | |
| 809 | + | // retained publish-once authority. The authority is captured when the stream is | |
| 810 | + | // accepted and stays valid for the connection's lifetime even after the engine | |
| 811 | + | // evicts the bounded record, so a settled stream keeps its exact cause | |
| 812 | + | // independent of retention: a Reset or an expiry never degrades into an orderly | |
| 813 | + | // close. It reports whether the authority had published. | |
| 814 | + | func (c *ListenerConnection) settleFromCause() bool { | |
| 815 | + | cause := c.cause | |
| 816 | + | if cause == nil { | |
| 817 | + | return false | |
| 818 | + | } | |
| 819 | + | select { | |
| 820 | + | case <-cause.Done(): | |
| 821 | + | default: | |
| 822 | + | return false | |
| 823 | + | } | |
| 824 | + | if err := cause.Err(); err != nil { | |
| 825 | + | c.settleLoss(cause.FailureKind(), err) | |
| 826 | + | } else { | |
| 827 | + | c.terminal.Close() | |
| 828 | + | } | |
| 829 | + | return true | |
| 830 | + | } | |
| 831 | + | ||
| 832 | + | // settleFromStatus publishes the terminal outcome recorded by the engine. | |
| 833 | + | func (c *ListenerConnection) settleFromStatus(status StreamStatus) { | |
| 834 | + | if status.Err == nil { | |
| 835 | + | c.terminal.Close() | |
| 836 | + | return | |
| 837 | + | } | |
| 838 | + | c.settleLoss(status.FailureKind, status.Err) | |
| 839 | + | } | |
| 840 | + | ||
| 841 | + | // settleFromCarriage publishes this stream's terminal outcome when its inner | |
| 842 | + | // session carriage closed on its own while the stream was still live. It syncs | |
| 843 | + | // the retained stream authority or the pump's physical cause first, and reports | |
| 844 | + | // whether one of them published. It deliberately does not fall back to an | |
| 845 | + | // orderly close, because a carriage that closed without a recorded cause is a | |
| 846 | + | // failed handshake, not an orderly end; the caller resets the stream instead. | |
| 847 | + | func (c *ListenerConnection) settleFromCarriage() bool { | |
| 848 | + | if c.settled() { | |
| 849 | + | return true | |
| 850 | + | } | |
| 851 | + | if c.settleFromCause() { | |
| 852 | + | return true | |
| 853 | + | } | |
| 854 | + | if err := c.pump.Err(); err != nil { | |
| 855 | + | c.settleLoss(c.pump.FailureKind(), err) | |
| 856 | + | return true | |
| 857 | + | } | |
| 858 | + | if status, ok := c.pump.Engine().Status(c.ref.Physical); ok && status.State == StreamTerminal { | |
| 859 | + | c.settleFromStatus(status) | |
| 860 | + | return true | |
| 861 | + | } | |
| 862 | + | return false | |
| 863 | + | } | |
| 864 | + | ||
| 865 | + | // abortMalformed resets exactly this stream after a malformed or failed inner | |
| 866 | + | // session frame: it settles the local engine record, best-effort signals the | |
| 867 | + | // peer with one Reset, publishes the stream-local failure, unblocks the byte | |
| 868 | + | // stream, and returns the stable terminal error. | |
| 869 | + | func (c *ListenerConnection) abortMalformed(cause error) error { | |
| 870 | + | return c.resetStream(malformedResetText, domain.RemoteFailureInvalidResponse, cause) | |
| 871 | + | } | |
| 872 | + | ||
| 873 | + | // abortClosedCarriage resets exactly this stream when its inner session | |
| 874 | + | // carriage closed on its own during the handshake with no stream cause recorded | |
| 875 | + | // and no consumer Close. It issues one stream-local Reset so the peer learns the | |
| 876 | + | // stream is gone, publishes the stream-local failure, and returns the stable | |
| 877 | + | // terminal error, so the watchdog and the admission slot are released without | |
| 878 | + | // waiting for a read or a Close. | |
| 879 | + | func (c *ListenerConnection) abortClosedCarriage() error { | |
| 880 | + | return c.resetStream(sessionCarriageClosedText, domain.RemoteFailureInvalidResponse, errSessionCarriageClosed) | |
| 881 | + | } | |
| 882 | + | ||
| 883 | + | // resetStream settles this stream with exactly one stream-local Reset. It claims | |
| 884 | + | // the publish-once terminal outcome before signalling the peer, so a consumer | |
| 885 | + | // read and the terminal watcher that observe the same failure never send a | |
| 886 | + | // second Reset for one stream. The engine record is settled and the carriage is | |
| 887 | + | // unblocked. It returns the caller's own cause - the specific read or decode | |
| 888 | + | // error - so a consumer that lost the publish race to the terminal watcher still | |
| 889 | + | // reports its precise diagnostic; the stable terminal error is its fallback. | |
| 890 | + | func (c *ListenerConnection) resetStream(text string, kind domain.RemoteFailureKind, cause error) error { | |
| 891 | + | if c.settled() || !c.settleLoss(kind, cause) { | |
| 892 | + | // Another publisher already claimed this stream's outcome and issued its | |
| 893 | + | // one Reset; never signal the peer twice. | |
| 894 | + | if cause != nil { | |
| 895 | + | return cause | |
| 896 | + | } | |
| 897 | + | return c.terminalError() | |
| 898 | + | } | |
| 899 | + | reset := Reset{ | |
| 900 | + | Physical: c.ref.Physical, | |
| 901 | + | Error: ErrorDetail{ | |
| 902 | + | Code: ports.BrokerErrorUnavailable, | |
| 903 | + | Text: text, | |
| 904 | + | FailureKind: kind, | |
| 905 | + | }, | |
| 906 | + | HasError: true, | |
| 907 | + | } | |
| 908 | + | _, _ = c.pump.Engine().Reset(reset) | |
| 909 | + | _ = c.pump.Send(reset) | |
| 910 | + | _ = c.raw.Close() | |
| 911 | + | if cause != nil { | |
| 912 | + | return cause | |
| 913 | + | } | |
| 914 | + | return c.terminalError() | |
| 915 | + | } | |
| 916 | + | ||
| 917 | + | // settleLoss publishes one stream-local loss carrying the given diagnostic | |
| 918 | + | // cause, normalized to the transport-failure taxonomy. It reports whether this | |
| 919 | + | // call won the publish-once outcome, so a caller that must signal the peer | |
| 920 | + | // exactly once knows it is the one publisher. | |
| 921 | + | func (c *ListenerConnection) settleLoss(kind domain.RemoteFailureKind, cause error) bool { | |
| 922 | + | if kind < domain.RemoteFailureTransport || kind > domain.RemoteFailureInvalidResponse { | |
| 923 | + | kind = domain.RemoteFailureTransport | |
| 924 | + | } | |
| 925 | + | return c.terminal.publishFail(kind, cause) | |
| 926 | + | } | |
| 927 | + | ||
| 928 | + | // settled reports whether the logical connection already reached its terminal | |
| 929 | + | // outcome. | |
| 930 | + | func (c *ListenerConnection) settled() bool { | |
| 931 | + | select { | |
| 932 | + | case <-c.terminal.Done(): | |
| 933 | + | return true | |
| 934 | + | default: | |
| 935 | + | return false | |
| 936 | + | } | |
| 937 | + | } | |
| 938 | + | ||
| 939 | + | // terminalError is the stable error of a settled connection: the published | |
| 940 | + | // cause, or io.EOF for an orderly close. | |
| 941 | + | func (c *ListenerConnection) terminalError() error { | |
| 942 | + | if err := c.terminal.Err(); err != nil { | |
| 943 | + | return err | |
| 944 | + | } | |
| 945 | + | return io.EOF | |
| 946 | + | } | |
A internal/adapters/daemonmux/listener_test.go +1218 -0
| @@ -0,0 +1,1218 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "io" | |
| 6 | + | "runtime" | |
| 7 | + | "sync" | |
| 8 | + | "testing" | |
| 9 | + | "time" | |
| 10 | + | ||
| 11 | + | "github.com/stretchr/testify/require" | |
| 12 | + | ||
| 13 | + | "github.com/bnema/vev/internal/adapters/sessionwire" | |
| 14 | + | "github.com/bnema/vev/internal/domain" | |
| 15 | + | "github.com/bnema/vev/internal/ports" | |
| 16 | + | "github.com/bnema/vev/internal/protocol" | |
| 17 | + | ) | |
| 18 | + | ||
| 19 | + | // listenerTestDeadline bounds every eventually/select assertion on the listener | |
| 20 | + | // tests. It is wall time and generous: the handshake budget under test is spent | |
| 21 | + | // by advancing the deterministic clock, never by waiting it out. | |
| 22 | + | const listenerTestDeadline = 5 * time.Second | |
| 23 | + | ||
| 24 | + | // listenerIdleBudget is the handshake budget of every test that must never | |
| 25 | + | // observe an expiry: only advancing the deterministic clock past that budget | |
| 26 | + | // can expire a stream, so an unexpected expiry is a real failure instead of a | |
| 27 | + | // slow assertion. | |
| 28 | + | const listenerIdleBudget = time.Hour | |
| 29 | + | ||
| 30 | + | // listenerClock is a deterministic ports.Clock/ports.Timer pair: time only | |
| 31 | + | // moves when Advance is called, so a stream's absolute handshake deadline and | |
| 32 | + | // its expiry are asserted exactly instead of racing wall time. Timers that come | |
| 33 | + | // due at the same instant fire in creation order. | |
| 34 | + | type listenerClock struct { | |
| 35 | + | mu sync.Mutex | |
| 36 | + | now time.Time | |
| 37 | + | order uint64 | |
| 38 | + | timers map[*listenerTimer]struct{} | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | func newListenerClock(start time.Time) *listenerClock { | |
| 42 | + | return &listenerClock{now: start, timers: make(map[*listenerTimer]struct{})} | |
| 43 | + | } | |
| 44 | + | ||
| 45 | + | func (c *listenerClock) Now() time.Time { | |
| 46 | + | c.mu.Lock() | |
| 47 | + | defer c.mu.Unlock() | |
| 48 | + | return c.now | |
| 49 | + | } | |
| 50 | + | ||
| 51 | + | func (c *listenerClock) NewTimer(d time.Duration) ports.Timer { | |
| 52 | + | timer := &listenerTimer{clock: c, ch: make(chan time.Time, 1)} | |
| 53 | + | c.mu.Lock() | |
| 54 | + | defer c.mu.Unlock() | |
| 55 | + | timer.arm(d) | |
| 56 | + | return timer | |
| 57 | + | } | |
| 58 | + | ||
| 59 | + | // Advance moves time forward and fires every timer that came due. | |
| 60 | + | func (c *listenerClock) Advance(d time.Duration) { | |
| 61 | + | if d < 0 { | |
| 62 | + | panic("daemonmux test: listenerClock.Advance requires a non-negative duration") | |
| 63 | + | } | |
| 64 | + | c.mu.Lock() | |
| 65 | + | defer c.mu.Unlock() | |
| 66 | + | c.now = c.now.Add(d) | |
| 67 | + | for timer := range c.timers { | |
| 68 | + | if timer.when.After(c.now) { | |
| 69 | + | continue | |
| 70 | + | } | |
| 71 | + | timer.active = false | |
| 72 | + | delete(c.timers, timer) | |
| 73 | + | select { | |
| 74 | + | case timer.ch <- c.now: | |
| 75 | + | default: | |
| 76 | + | } | |
| 77 | + | } | |
| 78 | + | } | |
| 79 | + | ||
| 80 | + | // SetNow moves the clock to now without firing any due timer, so a test can | |
| 81 | + | // drive a consumer's own deadline check in isolation from the watchdog | |
| 82 | + | // goroutine. A later Advance still fires any timer whose deadline passed. | |
| 83 | + | func (c *listenerClock) SetNow(now time.Time) { | |
| 84 | + | c.mu.Lock() | |
| 85 | + | defer c.mu.Unlock() | |
| 86 | + | c.now = now | |
| 87 | + | } | |
| 88 | + | ||
| 89 | + | // timerCount reports how many timers are currently armed, so a test can wait | |
| 90 | + | // until the watchdog is blocked on a deadline before moving the clock without | |
| 91 | + | // firing it. | |
| 92 | + | func (c *listenerClock) timerCount() int { | |
| 93 | + | c.mu.Lock() | |
| 94 | + | defer c.mu.Unlock() | |
| 95 | + | return len(c.timers) | |
| 96 | + | } | |
| 97 | + | ||
| 98 | + | type listenerTimer struct { | |
| 99 | + | clock *listenerClock | |
| 100 | + | ch chan time.Time | |
| 101 | + | when time.Time | |
| 102 | + | order uint64 | |
| 103 | + | active bool | |
| 104 | + | } | |
| 105 | + | ||
| 106 | + | func (t *listenerTimer) C() <-chan time.Time { return t.ch } | |
| 107 | + | ||
| 108 | + | func (t *listenerTimer) Stop() bool { | |
| 109 | + | t.clock.mu.Lock() | |
| 110 | + | defer t.clock.mu.Unlock() | |
| 111 | + | if !t.active { | |
| 112 | + | return false | |
| 113 | + | } | |
| 114 | + | t.active = false | |
| 115 | + | delete(t.clock.timers, t) | |
| 116 | + | return true | |
| 117 | + | } | |
| 118 | + | ||
| 119 | + | func (t *listenerTimer) Reset(d time.Duration) bool { | |
| 120 | + | t.clock.mu.Lock() | |
| 121 | + | defer t.clock.mu.Unlock() | |
| 122 | + | wasActive := t.active | |
| 123 | + | if t.active { | |
| 124 | + | t.active = false | |
| 125 | + | delete(t.clock.timers, t) | |
| 126 | + | } | |
| 127 | + | select { | |
| 128 | + | case <-t.ch: | |
| 129 | + | default: | |
| 130 | + | } | |
| 131 | + | t.arm(d) | |
| 132 | + | return wasActive | |
| 133 | + | } | |
| 134 | + | ||
| 135 | + | // arm schedules the timer relative to the clock's current time. The caller | |
| 136 | + | // holds clock.mu. | |
| 137 | + | func (t *listenerTimer) arm(d time.Duration) { | |
| 138 | + | t.when = t.clock.now.Add(d) | |
| 139 | + | t.order = t.clock.order | |
| 140 | + | t.clock.order++ | |
| 141 | + | if d <= 0 { | |
| 142 | + | t.active = false | |
| 143 | + | select { | |
| 144 | + | case t.ch <- t.clock.now: | |
| 145 | + | default: | |
| 146 | + | } | |
| 147 | + | return | |
| 148 | + | } | |
| 149 | + | t.active = true | |
| 150 | + | t.clock.timers[t] = struct{}{} | |
| 151 | + | } | |
| 152 | + | ||
| 153 | + | // listenerHarness joins one paired pump set, one broker-side connector, one | |
| 154 | + | // deterministic clock, and one daemon-side listener under test. | |
| 155 | + | type listenerHarness struct { | |
| 156 | + | t *testing.T | |
| 157 | + | brokerPump *Pump | |
| 158 | + | daemonPump *Pump | |
| 159 | + | connector *LogicalConnector | |
| 160 | + | listener *Listener | |
| 161 | + | clock *listenerClock | |
| 162 | + | } | |
| 163 | + | ||
| 164 | + | // newListenerHarness builds a listener over a fresh physical pump pair under an | |
| 165 | + | // explicit handshake budget and accept-queue bound, driven by a deterministic | |
| 166 | + | // clock. The budget is deliberately long: a test that wants an expiry advances | |
| 167 | + | // the clock instead of waiting. The listener is constructed before the pumps | |
| 168 | + | // start, because it registers the daemon pump's admission observer, which the | |
| 169 | + | // pump refuses after Start. | |
| 170 | + | func newListenerHarness(t *testing.T, budget time.Duration, queueLimit int) *listenerHarness { | |
| 171 | + | t.Helper() | |
| 172 | + | return newListenerHarnessWithCeilings(t, budget, queueLimit, DefaultMuxCeilings()) | |
| 173 | + | } | |
| 174 | + | ||
| 175 | + | // newListenerHarnessWithCeilings builds the same harness under an explicit | |
| 176 | + | // ceiling set, so a test can tighten the stream-count ceiling and prove a | |
| 177 | + | // closed stream genuinely released its admission slot. | |
| 178 | + | func newListenerHarnessWithCeilings(t *testing.T, budget time.Duration, queueLimit int, ceilings MuxCeilings) *listenerHarness { | |
| 179 | + | t.Helper() | |
| 180 | + | brokerPump, daemonPump, start := newPairedPumpsUnstarted(t, ceilings) | |
| 181 | + | clock := newListenerClock(time.Now()) | |
| 182 | + | listener, err := newListener(daemonPump, budget, queueLimit, clock) | |
| 183 | + | require.NoError(t, err) | |
| 184 | + | t.Cleanup(func() { _ = listener.Close() }) | |
| 185 | + | start() | |
| 186 | + | return &listenerHarness{ | |
| 187 | + | t: t, | |
| 188 | + | brokerPump: brokerPump, | |
| 189 | + | daemonPump: daemonPump, | |
| 190 | + | connector: mustConnector(t, brokerPump), | |
| 191 | + | listener: listener, | |
| 192 | + | clock: clock, | |
| 193 | + | } | |
| 194 | + | } | |
| 195 | + | ||
| 196 | + | // open admits one broker-side logical stream and waits for the daemon's | |
| 197 | + | // Opened, which the listener sends at acceptance. | |
| 198 | + | func (h *listenerHarness) open(stream int) (*LogicalConnection, error) { | |
| 199 | + | ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | |
| 200 | + | defer cancel() | |
| 201 | + | return h.connector.Open(ctx, controlRequest(stream)) | |
| 202 | + | } | |
| 203 | + | ||
| 204 | + | func (h *listenerHarness) mustOpen(stream int) *LogicalConnection { | |
| 205 | + | h.t.Helper() | |
| 206 | + | connection, err := h.open(stream) | |
| 207 | + | require.NoError(h.t, err, "open stream %d", stream) | |
| 208 | + | return connection | |
| 209 | + | } | |
| 210 | + | ||
| 211 | + | // pending reports the admitted streams waiting in the accept queue. | |
| 212 | + | func (h *listenerHarness) pending() int { | |
| 213 | + | h.listener.mu.Lock() | |
| 214 | + | defer h.listener.mu.Unlock() | |
| 215 | + | return len(h.listener.queue) | |
| 216 | + | } | |
| 217 | + | ||
| 218 | + | func (h *listenerHarness) awaitPending(count int) { | |
| 219 | + | h.t.Helper() | |
| 220 | + | require.Eventually(h.t, func() bool { return h.pending() == count }, listenerTestDeadline, time.Millisecond, | |
| 221 | + | "accept queue never held %d admitted stream(s)", count) | |
| 222 | + | } | |
| 223 | + | ||
| 224 | + | // activeWatchdog reports how many admitted streams the deadline watchdog still | |
| 225 | + | // tracks. | |
| 226 | + | func (h *listenerHarness) activeWatchdog() int { | |
| 227 | + | h.listener.mu.Lock() | |
| 228 | + | defer h.listener.mu.Unlock() | |
| 229 | + | return len(h.listener.active) | |
| 230 | + | } | |
| 231 | + | ||
| 232 | + | // moveClock moves the clock forward without firing any pending watchdog timer, | |
| 233 | + | // so a test can observe a consumer's own deadline check in isolation from the | |
| 234 | + | // watchdog goroutine. | |
| 235 | + | func (h *listenerHarness) moveClock(now time.Time) { | |
| 236 | + | h.clock.SetNow(now) | |
| 237 | + | } | |
| 238 | + | ||
| 239 | + | type acceptOutcome struct { | |
| 240 | + | conn *ListenerConnection | |
| 241 | + | err error | |
| 242 | + | } | |
| 243 | + | ||
| 244 | + | // acceptAsync starts one Accept call; the outcome is delivered exactly once. | |
| 245 | + | func (h *listenerHarness) acceptAsync() <-chan acceptOutcome { | |
| 246 | + | out := make(chan acceptOutcome, 1) | |
| 247 | + | go func() { | |
| 248 | + | raw, err := h.listener.Accept() | |
| 249 | + | if err != nil { | |
| 250 | + | out <- acceptOutcome{err: err} | |
| 251 | + | return | |
| 252 | + | } | |
| 253 | + | out <- acceptOutcome{conn: raw.(*ListenerConnection)} | |
| 254 | + | }() | |
| 255 | + | return out | |
| 256 | + | } | |
| 257 | + | ||
| 258 | + | func (h *listenerHarness) awaitAccept(out <-chan acceptOutcome) *ListenerConnection { | |
| 259 | + | h.t.Helper() | |
| 260 | + | select { | |
| 261 | + | case outcome := <-out: | |
| 262 | + | require.NoError(h.t, outcome.err) | |
| 263 | + | return outcome.conn | |
| 264 | + | case <-time.After(listenerTestDeadline): | |
| 265 | + | h.t.Fatal("listener: Accept did not return an admitted stream") | |
| 266 | + | return nil | |
| 267 | + | } | |
| 268 | + | } | |
| 269 | + | ||
| 270 | + | // servePongs answers every typed Ping of one accepted connection with exactly | |
| 271 | + | // one Pong until the stream ends. | |
| 272 | + | func servePongs(conn ports.ServerConnection) error { | |
| 273 | + | for { | |
| 274 | + | message, err := conn.ReceiveClient() | |
| 275 | + | if err != nil { | |
| 276 | + | return err | |
| 277 | + | } | |
| 278 | + | if _, ok := message.(protocol.Ping); !ok { | |
| 279 | + | continue | |
| 280 | + | } | |
| 281 | + | if err := conn.SendServer(protocol.Pong{}); err != nil { | |
| 282 | + | return err | |
| 283 | + | } | |
| 284 | + | } | |
| 285 | + | } | |
| 286 | + | ||
| 287 | + | // listenerRoundTrip proves one typed exchange on a logical connection. | |
| 288 | + | func listenerRoundTrip(t *testing.T, conn ports.ClientConnection) { | |
| 289 | + | t.Helper() | |
| 290 | + | require.NoError(t, conn.SendClient(protocol.Ping{})) | |
| 291 | + | message, err := conn.ReceiveServer() | |
| 292 | + | require.NoError(t, err) | |
| 293 | + | require.Equal(t, protocol.Pong{}, message) | |
| 294 | + | } | |
| 295 | + | ||
| 296 | + | // requireStreamState waits until one stream of the daemon pump reached state. | |
| 297 | + | func requireStreamState(t *testing.T, pump *Pump, id PhysicalStreamID, state StreamState) StreamStatus { | |
| 298 | + | t.Helper() | |
| 299 | + | var status StreamStatus | |
| 300 | + | require.Eventually(t, func() bool { | |
| 301 | + | current, ok := pump.Engine().Status(id) | |
| 302 | + | if !ok || current.State != state { | |
| 303 | + | return false | |
| 304 | + | } | |
| 305 | + | status = current | |
| 306 | + | return true | |
| 307 | + | }, listenerTestDeadline, time.Millisecond, "daemon stream %d never reached %s", id, state) | |
| 308 | + | return status | |
| 309 | + | } | |
| 310 | + | ||
| 311 | + | // listenerDaemon is the test's daemon accept loop: it takes connections from | |
| 312 | + | // the listener in acceptance order and serves each one with servePongs. | |
| 313 | + | type listenerDaemon struct { | |
| 314 | + | listener *Listener | |
| 315 | + | ||
| 316 | + | mu sync.Mutex | |
| 317 | + | conns []*ListenerConnection | |
| 318 | + | } | |
| 319 | + | ||
| 320 | + | func startListenerDaemon(l *Listener) *listenerDaemon { | |
| 321 | + | daemon := &listenerDaemon{listener: l} | |
| 322 | + | go func() { | |
| 323 | + | for { | |
| 324 | + | raw, err := l.Accept() | |
| 325 | + | if err != nil { | |
| 326 | + | return | |
| 327 | + | } | |
| 328 | + | connection := raw.(*ListenerConnection) | |
| 329 | + | daemon.mu.Lock() | |
| 330 | + | daemon.conns = append(daemon.conns, connection) | |
| 331 | + | daemon.mu.Unlock() | |
| 332 | + | go func() { _ = servePongs(connection) }() | |
| 333 | + | } | |
| 334 | + | }() | |
| 335 | + | return daemon | |
| 336 | + | } | |
| 337 | + | ||
| 338 | + | func (d *listenerDaemon) connection(index int) *ListenerConnection { | |
| 339 | + | d.mu.Lock() | |
| 340 | + | defer d.mu.Unlock() | |
| 341 | + | return d.conns[index] | |
| 342 | + | } | |
| 343 | + | ||
| 344 | + | func (d *listenerDaemon) await(t *testing.T, count int) { | |
| 345 | + | t.Helper() | |
| 346 | + | require.Eventually(t, func() bool { | |
| 347 | + | d.mu.Lock() | |
| 348 | + | defer d.mu.Unlock() | |
| 349 | + | return len(d.conns) >= count | |
| 350 | + | }, listenerTestDeadline, time.Millisecond, "daemon accepted %d of %d streams", count, count) | |
| 351 | + | } | |
| 352 | + | ||
| 353 | + | func TestNewListenerValidation(t *testing.T) { | |
| 354 | + | t.Run("nil pump", func(t *testing.T) { | |
| 355 | + | listener, err := NewListener(nil) | |
| 356 | + | require.ErrorIs(t, err, ErrListenerConfig) | |
| 357 | + | require.Nil(t, listener) | |
| 358 | + | }) | |
| 359 | + | t.Run("broker-side pump is refused", func(t *testing.T) { | |
| 360 | + | pump, _ := newTestPump(t, DirectionServer) | |
| 361 | + | listener, err := NewListener(pump) | |
| 362 | + | require.ErrorIs(t, err, ErrListenerConfig) | |
| 363 | + | require.Nil(t, listener) | |
| 364 | + | }) | |
| 365 | + | t.Run("daemon-side pump is accepted", func(t *testing.T) { | |
| 366 | + | pump, _ := newTestPump(t, DirectionClient) | |
| 367 | + | listener, err := NewListener(pump) | |
| 368 | + | require.NoError(t, err) | |
| 369 | + | require.NotNil(t, listener) | |
| 370 | + | require.Equal(t, "daemonmux", listener.Addr()) | |
| 371 | + | require.NoError(t, listener.Close()) | |
| 372 | + | _, err = listener.Accept() | |
| 373 | + | require.ErrorIs(t, err, ErrListenerClosed) | |
| 374 | + | }) | |
| 375 | + | t.Run("accept queue is bounded by the stream ceiling", func(t *testing.T) { | |
| 376 | + | require.Equal(t, 128, MaxAcceptQueue) | |
| 377 | + | require.LessOrEqual(t, MaxAcceptQueue, int(MaxMuxStreams)) | |
| 378 | + | }) | |
| 379 | + | } | |
| 380 | + | ||
| 381 | + | // TestPumpAdmittedObserver proves the pump notifies its single admission | |
| 382 | + | // observer for every admitted inbound Open, and never for a refused one. | |
| 383 | + | func TestPumpAdmittedObserver(t *testing.T) { | |
| 384 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 385 | + | observed := make(chan Open, 4) | |
| 386 | + | require.NoError(t, pump.OnAdmitted(func(open Open) { observed <- open })) | |
| 387 | + | pump.Start(context.Background()) | |
| 388 | + | ||
| 389 | + | deliverClientFrame(t, carrier, openFor(1)) | |
| 390 | + | select { | |
| 391 | + | case open := <-observed: | |
| 392 | + | require.Equal(t, testRef(1), open.Ref) | |
| 393 | + | require.Equal(t, openFor(1), open, "the observer receives the decoded admission request") | |
| 394 | + | case <-time.After(listenerTestDeadline): | |
| 395 | + | t.Fatal("admission observer did not run") | |
| 396 | + | } | |
| 397 | + | ||
| 398 | + | // A refused admission (a reused physical identity) never notifies. | |
| 399 | + | deliverClientFrame(t, carrier, openFor(1)) | |
| 400 | + | select { | |
| 401 | + | case open := <-observed: | |
| 402 | + | t.Fatalf("refused admission notified the observer: %+v", open) | |
| 403 | + | case <-time.After(50 * time.Millisecond): | |
| 404 | + | } | |
| 405 | + | ||
| 406 | + | deliverClientFrame(t, carrier, openFor(2)) | |
| 407 | + | select { | |
| 408 | + | case open := <-observed: | |
| 409 | + | require.Equal(t, testRef(2), open.Ref) | |
| 410 | + | case <-time.After(listenerTestDeadline): | |
| 411 | + | t.Fatal("admission observer did not run for a fresh stream") | |
| 412 | + | } | |
| 413 | + | } | |
| 414 | + | ||
| 415 | + | // TestListenerTwoTypedAdmissions proves two admitted streams become two | |
| 416 | + | // independent typed session channels over one physical pair, each with its own | |
| 417 | + | // lifecycle. | |
| 418 | + | func TestListenerTwoTypedAdmissions(t *testing.T) { | |
| 419 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 420 | + | daemon := startListenerDaemon(h.listener) | |
| 421 | + | ||
| 422 | + | first := h.mustOpen(1) | |
| 423 | + | daemon.await(t, 1) | |
| 424 | + | second := h.mustOpen(2) | |
| 425 | + | daemon.await(t, 2) | |
| 426 | + | ||
| 427 | + | require.NotEqual(t, daemon.connection(0).Ref().Physical, daemon.connection(1).Ref().Physical) | |
| 428 | + | listenerRoundTrip(t, first) | |
| 429 | + | listenerRoundTrip(t, second) | |
| 430 | + | ||
| 431 | + | // Independence: closing one stream leaves the sibling and the physical | |
| 432 | + | // connection untouched. | |
| 433 | + | require.NoError(t, first.Close()) | |
| 434 | + | require.True(t, channelClosed(first.Done())) | |
| 435 | + | require.Nil(t, first.Err()) | |
| 436 | + | require.False(t, channelClosed(second.Done())) | |
| 437 | + | listenerRoundTrip(t, second) | |
| 438 | + | require.False(t, channelClosed(h.brokerPump.Done())) | |
| 439 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 440 | + | require.Equal(t, 1, h.daemonPump.Engine().Live()) | |
| 441 | + | } | |
| 442 | + | ||
| 443 | + | // TestListenerHundredTypedAdmissions concurrently admits 100 independent typed | |
| 444 | + | // streams over one physical pair, proves them all live before traffic, and | |
| 445 | + | // interleaves a typed round trip on every one. | |
| 446 | + | func TestListenerHundredTypedAdmissions(t *testing.T) { | |
| 447 | + | const count = 100 | |
| 448 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 449 | + | daemon := startListenerDaemon(h.listener) | |
| 450 | + | ||
| 451 | + | connections := make([]*LogicalConnection, count) | |
| 452 | + | errs := make([]error, count) | |
| 453 | + | var wg sync.WaitGroup | |
| 454 | + | for i := 0; i < count; i++ { | |
| 455 | + | wg.Add(1) | |
| 456 | + | go func(i int) { | |
| 457 | + | defer wg.Done() | |
| 458 | + | connections[i], errs[i] = h.open(i + 1) | |
| 459 | + | }(i) | |
| 460 | + | } | |
| 461 | + | wg.Wait() | |
| 462 | + | for i, err := range errs { | |
| 463 | + | require.NoError(t, err, "stream %d", i+1) | |
| 464 | + | } | |
| 465 | + | daemon.await(t, count) | |
| 466 | + | ||
| 467 | + | // Every stream is admitted and live before any traffic, and the physical | |
| 468 | + | // connection is not terminal. | |
| 469 | + | require.Equal(t, count, h.daemonPump.Engine().Live()) | |
| 470 | + | require.False(t, channelClosed(h.brokerPump.Done())) | |
| 471 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 472 | + | for i, connection := range connections { | |
| 473 | + | require.False(t, channelClosed(connection.Done()), "stream %d settled before traffic", i+1) | |
| 474 | + | } | |
| 475 | + | ||
| 476 | + | for _, connection := range connections { | |
| 477 | + | listenerRoundTrip(t, connection) | |
| 478 | + | } | |
| 479 | + | ||
| 480 | + | for i, connection := range connections { | |
| 481 | + | require.NoError(t, connection.Close(), "stream %d", i+1) | |
| 482 | + | } | |
| 483 | + | require.Eventually(t, func() bool { return h.daemonPump.Engine().Live() == 0 }, listenerTestDeadline, time.Millisecond) | |
| 484 | + | for i, connection := range connections { | |
| 485 | + | require.True(t, channelClosed(connection.Done()), "stream %d", i+1) | |
| 486 | + | } | |
| 487 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 488 | + | } | |
| 489 | + | ||
| 490 | + | // TestListenerBlockedAcceptAndReaderSiblingProgress proves a blocked Accept and | |
| 491 | + | // a blocked reader never stall a sibling: an established stream keeps its typed | |
| 492 | + | // round trips while the accept queue is empty, and while another accepted | |
| 493 | + | // stream's reader is parked. | |
| 494 | + | func TestListenerBlockedAcceptAndReaderSiblingProgress(t *testing.T) { | |
| 495 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 496 | + | ||
| 497 | + | firstAccept := h.acceptAsync() | |
| 498 | + | first := h.mustOpen(1) | |
| 499 | + | daemonFirst := h.awaitAccept(firstAccept) | |
| 500 | + | go func() { _ = servePongs(daemonFirst) }() | |
| 501 | + | listenerRoundTrip(t, first) | |
| 502 | + | ||
| 503 | + | // The accept queue is empty: Accept is blocked, and a sibling still | |
| 504 | + | // exchanges typed traffic. | |
| 505 | + | probe := h.acceptAsync() | |
| 506 | + | select { | |
| 507 | + | case outcome := <-probe: | |
| 508 | + | t.Fatalf("Accept returned with an empty queue: %+v", outcome) | |
| 509 | + | default: | |
| 510 | + | } | |
| 511 | + | listenerRoundTrip(t, first) | |
| 512 | + | listenerRoundTrip(t, first) | |
| 513 | + | select { | |
| 514 | + | case outcome := <-probe: | |
| 515 | + | t.Fatalf("Accept returned with an empty queue: %+v", outcome) | |
| 516 | + | default: | |
| 517 | + | } | |
| 518 | + | ||
| 519 | + | // A newly admitted stream wakes the blocked Accept in admission order and | |
| 520 | + | // is delivered as its own typed connection. Its reader is parked, so it | |
| 521 | + | // never answers, while the first stream keeps progressing. | |
| 522 | + | second := h.mustOpen(2) | |
| 523 | + | daemonSecond := h.awaitAccept(probe) | |
| 524 | + | require.Equal(t, PhysicalStreamID(2), daemonSecond.Ref().Physical) | |
| 525 | + | blockedReader := make(chan error, 1) | |
| 526 | + | go func() { | |
| 527 | + | _, err := daemonSecond.ReceiveClient() | |
| 528 | + | blockedReader <- err | |
| 529 | + | }() | |
| 530 | + | listenerRoundTrip(t, first) | |
| 531 | + | select { | |
| 532 | + | case err := <-blockedReader: | |
| 533 | + | t.Fatalf("parked reader returned early: %v", err) | |
| 534 | + | default: | |
| 535 | + | } | |
| 536 | + | ||
| 537 | + | // Closing the parked stream unblocks exactly its reader. | |
| 538 | + | require.NoError(t, second.Close()) | |
| 539 | + | select { | |
| 540 | + | case err := <-blockedReader: | |
| 541 | + | require.Error(t, err) | |
| 542 | + | case <-time.After(listenerTestDeadline): | |
| 543 | + | t.Fatal("closing the stream did not unblock its reader") | |
| 544 | + | } | |
| 545 | + | require.Eventually(t, func() bool { return channelClosed(daemonSecond.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 546 | + | listenerRoundTrip(t, first) | |
| 547 | + | } | |
| 548 | + | ||
| 549 | + | // TestListenerQueueFullRefusesOnlyOffendingStream proves a stream admitted | |
| 550 | + | // while the accept queue is full is refused and reset on its own stream, while | |
| 551 | + | // the queued sibling and the physical connection are untouched. | |
| 552 | + | func TestListenerQueueFullRefusesOnlyOffendingStream(t *testing.T) { | |
| 553 | + | h := newListenerHarness(t, listenerTestDeadline, 1) | |
| 554 | + | ||
| 555 | + | // The first stream is admitted but never accepted: it fills the queue. | |
| 556 | + | opened := make(chan *LogicalConnection, 1) | |
| 557 | + | openErrs := make(chan error, 1) | |
| 558 | + | go func() { | |
| 559 | + | connection, err := h.open(1) | |
| 560 | + | if err != nil { | |
| 561 | + | openErrs <- err | |
| 562 | + | return | |
| 563 | + | } | |
| 564 | + | opened <- connection | |
| 565 | + | }() | |
| 566 | + | h.awaitPending(1) | |
| 567 | + | ||
| 568 | + | // The second stream is refused on its own stream: one Reset, no queue | |
| 569 | + | // growth, no physical effect. | |
| 570 | + | _, err := h.open(2) | |
| 571 | + | require.Error(t, err) | |
| 572 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 573 | + | require.False(t, channelClosed(h.brokerPump.Done())) | |
| 574 | + | ||
| 575 | + | refused := requireStreamState(t, h.daemonPump, 2, StreamTerminal) | |
| 576 | + | require.Equal(t, domain.RemoteFailureTransport, refused.FailureKind) | |
| 577 | + | var detail ports.BrokerError | |
| 578 | + | require.ErrorAs(t, refused.Err, &detail) | |
| 579 | + | require.Equal(t, acceptQueueFullText, detail.Text) | |
| 580 | + | require.Equal(t, 1, h.daemonPump.Engine().Live(), "the queued sibling must stay live") | |
| 581 | + | ||
| 582 | + | // The queued sibling is still deliverable and fully usable. | |
| 583 | + | first := h.awaitAccept(h.acceptAsync()) | |
| 584 | + | go func() { _ = servePongs(first) }() | |
| 585 | + | select { | |
| 586 | + | case connection := <-opened: | |
| 587 | + | listenerRoundTrip(t, connection) | |
| 588 | + | case err := <-openErrs: | |
| 589 | + | t.Fatalf("queued stream did not open: %v", err) | |
| 590 | + | case <-time.After(listenerTestDeadline): | |
| 591 | + | t.Fatal("queued stream did not open") | |
| 592 | + | } | |
| 593 | + | require.Equal(t, PhysicalStreamID(1), first.Ref().Physical) | |
| 594 | + | } | |
| 595 | + | ||
| 596 | + | // TestListenerHandshakeExpiryResetsOnlyStream proves one stream whose session | |
| 597 | + | // handshake never ran is reset exactly at its admission deadline, while a | |
| 598 | + | // sibling that completed its handshake stays alive and usable. | |
| 599 | + | func TestListenerHandshakeExpiryResetsOnlyStream(t *testing.T) { | |
| 600 | + | const budget = 30 * time.Second | |
| 601 | + | h := newListenerHarness(t, budget, MaxAcceptQueue) | |
| 602 | + | ||
| 603 | + | firstAccept := h.acceptAsync() | |
| 604 | + | first := h.mustOpen(1) | |
| 605 | + | daemonFirst := h.awaitAccept(firstAccept) | |
| 606 | + | secondAccept := h.acceptAsync() | |
| 607 | + | second := h.mustOpen(2) | |
| 608 | + | daemonSecond := h.awaitAccept(secondAccept) | |
| 609 | + | go func() { _ = servePongs(daemonSecond) }() | |
| 610 | + | listenerRoundTrip(t, second) | |
| 611 | + | require.Equal(t, 2, h.daemonPump.Engine().Live()) | |
| 612 | + | ||
| 613 | + | // The first stream's session handshake never runs: its one absolute | |
| 614 | + | // deadline expires, and only that stream is reset. | |
| 615 | + | h.clock.Advance(budget + time.Second) | |
| 616 | + | ||
| 617 | + | expired := requireStreamState(t, h.daemonPump, 1, StreamTerminal) | |
| 618 | + | require.Equal(t, domain.RemoteFailureTransport, expired.FailureKind) | |
| 619 | + | var detail ports.BrokerError | |
| 620 | + | require.ErrorAs(t, expired.Err, &detail) | |
| 621 | + | require.Equal(t, handshakeDeadlineText, detail.Text) | |
| 622 | + | require.Eventually(t, func() bool { return channelClosed(daemonFirst.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 623 | + | require.Error(t, daemonFirst.Err()) | |
| 624 | + | require.Equal(t, domain.RemoteFailureTransport, daemonFirst.FailureKind()) | |
| 625 | + | ||
| 626 | + | // The broker observes the stream-local loss with its exact scope. | |
| 627 | + | require.Eventually(t, func() bool { return channelClosed(first.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 628 | + | var loss ports.BrokerStreamLost | |
| 629 | + | require.ErrorAs(t, first.Err(), &loss) | |
| 630 | + | require.Equal(t, ports.BrokerStreamID(1), loss.Stream) | |
| 631 | + | require.Equal(t, domain.RemoteFailureTransport, loss.Cause) | |
| 632 | + | ||
| 633 | + | // The sibling is untouched: still open, still serving, physical alive. | |
| 634 | + | require.Equal(t, 1, h.daemonPump.Engine().Live()) | |
| 635 | + | require.Equal(t, StreamOpen, requireStreamState(t, h.daemonPump, 2, StreamOpen).State) | |
| 636 | + | require.False(t, channelClosed(second.Done())) | |
| 637 | + | listenerRoundTrip(t, second) | |
| 638 | + | require.False(t, channelClosed(daemonSecond.Done())) | |
| 639 | + | require.False(t, channelClosed(h.brokerPump.Done())) | |
| 640 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 641 | + | } | |
| 642 | + | ||
| 643 | + | // TestListenerHandshakeCompletionCancelsWatchdog proves a completed session | |
| 644 | + | // handshake retires the stream's watchdog: advancing past the deadline never | |
| 645 | + | // resets a stream that already handshook. | |
| 646 | + | func TestListenerHandshakeCompletionCancelsWatchdog(t *testing.T) { | |
| 647 | + | const budget = 30 * time.Second | |
| 648 | + | h := newListenerHarness(t, budget, MaxAcceptQueue) | |
| 649 | + | ||
| 650 | + | probe := h.acceptAsync() | |
| 651 | + | connection := h.mustOpen(1) | |
| 652 | + | daemonConnection := h.awaitAccept(probe) | |
| 653 | + | go func() { _ = servePongs(daemonConnection) }() | |
| 654 | + | listenerRoundTrip(t, connection) | |
| 655 | + | ||
| 656 | + | completed := make(chan struct{}, 1) | |
| 657 | + | daemonConnection.OnHandshakeComplete(func() { completed <- struct{}{} }) | |
| 658 | + | select { | |
| 659 | + | case <-completed: | |
| 660 | + | case <-time.After(listenerTestDeadline): | |
| 661 | + | t.Fatal("session handshake completion was not published") | |
| 662 | + | } | |
| 663 | + | ||
| 664 | + | h.clock.Advance(2 * budget) | |
| 665 | + | ||
| 666 | + | require.Equal(t, StreamOpen, requireStreamState(t, h.daemonPump, 1, StreamOpen).State) | |
| 667 | + | require.False(t, channelClosed(connection.Done())) | |
| 668 | + | require.False(t, channelClosed(daemonConnection.Done())) | |
| 669 | + | require.Nil(t, daemonConnection.Err()) | |
| 670 | + | listenerRoundTrip(t, connection) | |
| 671 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 672 | + | } | |
| 673 | + | ||
| 674 | + | // TestListenerDeadlineConsumesQueueDelay proves the accepted absolute deadline | |
| 675 | + | // is fixed at Open admission, so queue delay spends the one budget instead of | |
| 676 | + | // starting a second one. | |
| 677 | + | func TestListenerDeadlineConsumesQueueDelay(t *testing.T) { | |
| 678 | + | const budget = 30 * time.Second | |
| 679 | + | h := newListenerHarness(t, budget, MaxAcceptQueue) | |
| 680 | + | admittedAt := h.clock.Now() | |
| 681 | + | ||
| 682 | + | opened := make(chan *LogicalConnection, 1) | |
| 683 | + | go func() { | |
| 684 | + | connection, err := h.open(1) | |
| 685 | + | if err == nil { | |
| 686 | + | opened <- connection | |
| 687 | + | } | |
| 688 | + | }() | |
| 689 | + | h.awaitPending(1) | |
| 690 | + | ||
| 691 | + | // Half the budget is spent while the stream waits in the accept queue. | |
| 692 | + | const queued = 15 * time.Second | |
| 693 | + | h.clock.Advance(queued) | |
| 694 | + | daemonConnection := h.awaitAccept(h.acceptAsync()) | |
| 695 | + | ||
| 696 | + | require.Equal(t, admittedAt.Add(budget), daemonConnection.HandshakeDeadline()) | |
| 697 | + | require.Equal(t, daemonConnection.HandshakeDeadline(), daemonConnection.HandshakeDeadline(), "the deadline is stable") | |
| 698 | + | require.True(t, daemonConnection.HandshakeDeadline().Before(h.clock.Now().Add(budget)), | |
| 699 | + | "accepting late must not start a second full budget") | |
| 700 | + | ||
| 701 | + | select { | |
| 702 | + | case connection := <-opened: | |
| 703 | + | go func() { _ = servePongs(daemonConnection) }() | |
| 704 | + | listenerRoundTrip(t, connection) | |
| 705 | + | case <-time.After(listenerTestDeadline): | |
| 706 | + | t.Fatal("queued stream did not open") | |
| 707 | + | } | |
| 708 | + | } | |
| 709 | + | ||
| 710 | + | // TestListenerWithdrawnPendingStreamIsSkipped proves a pending stream the peer | |
| 711 | + | // withdrew is never handed to the daemon: Accept skips it and delivers the next | |
| 712 | + | // admitted stream. | |
| 713 | + | func TestListenerWithdrawnPendingStreamIsSkipped(t *testing.T) { | |
| 714 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 715 | + | ||
| 716 | + | ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) | |
| 717 | + | defer cancel() | |
| 718 | + | _, err := h.connector.Open(ctx, controlRequest(1)) | |
| 719 | + | require.ErrorIs(t, err, context.DeadlineExceeded) | |
| 720 | + | h.awaitPending(1) | |
| 721 | + | requireStreamState(t, h.daemonPump, 1, StreamTerminal) | |
| 722 | + | ||
| 723 | + | // The withdrawn stream is skipped; the next admitted stream is delivered. | |
| 724 | + | probe := h.acceptAsync() | |
| 725 | + | second := h.mustOpen(2) | |
| 726 | + | daemonConnection := h.awaitAccept(probe) | |
| 727 | + | require.Equal(t, PhysicalStreamID(2), daemonConnection.Ref().Physical) | |
| 728 | + | go func() { _ = servePongs(daemonConnection) }() | |
| 729 | + | listenerRoundTrip(t, second) | |
| 730 | + | } | |
| 731 | + | ||
| 732 | + | // TestListenerMalformedStreamDoesNotStopAccept proves a malformed inner session | |
| 733 | + | // frame resets exactly its own stream and never surfaces as a fatal Accept | |
| 734 | + | // error that would stop the shared daemon accept loop. | |
| 735 | + | func TestListenerMalformedStreamDoesNotStopAccept(t *testing.T) { | |
| 736 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 737 | + | daemon := startListenerDaemon(h.listener) | |
| 738 | + | ||
| 739 | + | first := h.mustOpen(1) | |
| 740 | + | daemon.await(t, 1) | |
| 741 | + | ||
| 742 | + | // The peer speaks garbage on the first frame of its stream. | |
| 743 | + | require.NoError(t, h.brokerPump.Send(Data{Physical: 1, Data: []byte("not-a-session-frame")})) | |
| 744 | + | ||
| 745 | + | malformed := requireStreamState(t, h.daemonPump, 1, StreamTerminal) | |
| 746 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, malformed.FailureKind) | |
| 747 | + | require.Eventually(t, func() bool { return channelClosed(first.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 748 | + | ||
| 749 | + | // The accept loop survived: a fresh stream is admitted and served. | |
| 750 | + | second := h.mustOpen(2) | |
| 751 | + | daemon.await(t, 2) | |
| 752 | + | listenerRoundTrip(t, second) | |
| 753 | + | require.False(t, channelClosed(h.brokerPump.Done())) | |
| 754 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 755 | + | } | |
| 756 | + | ||
| 757 | + | // TestListenerPhysicalLossUnblocksAll proves a physical loss fails the blocked | |
| 758 | + | // Accept with its stable fatal outcome and unblocks every delivered stream. | |
| 759 | + | func TestListenerPhysicalLossUnblocksAll(t *testing.T) { | |
| 760 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 761 | + | ||
| 762 | + | connections := make([]*LogicalConnection, 0, 2) | |
| 763 | + | daemonConnections := make([]*ListenerConnection, 0, 2) | |
| 764 | + | for i := 1; i <= 2; i++ { | |
| 765 | + | probe := h.acceptAsync() | |
| 766 | + | connection := h.mustOpen(i) | |
| 767 | + | daemonConnection := h.awaitAccept(probe) | |
| 768 | + | connections = append(connections, connection) | |
| 769 | + | daemonConnections = append(daemonConnections, daemonConnection) | |
| 770 | + | } | |
| 771 | + | ||
| 772 | + | readers := make([]chan error, 0, 2) | |
| 773 | + | for _, daemonConnection := range daemonConnections { | |
| 774 | + | blocked := make(chan error, 1) | |
| 775 | + | readers = append(readers, blocked) | |
| 776 | + | conn := daemonConnection | |
| 777 | + | go func() { | |
| 778 | + | _, err := conn.ReceiveClient() | |
| 779 | + | blocked <- err | |
| 780 | + | }() | |
| 781 | + | } | |
| 782 | + | blockedAccept := h.acceptAsync() | |
| 783 | + | ||
| 784 | + | // Failing the broker carrier is a physical loss on the daemon side. | |
| 785 | + | require.NoError(t, h.brokerPump.Close()) | |
| 786 | + | require.Eventually(t, func() bool { return channelClosed(h.daemonPump.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 787 | + | require.Error(t, h.daemonPump.Err(), "the daemon pump must publish the carrier failure") | |
| 788 | + | ||
| 789 | + | select { | |
| 790 | + | case outcome := <-blockedAccept: | |
| 791 | + | require.ErrorIs(t, outcome.err, ErrListenerLost) | |
| 792 | + | require.ErrorIs(t, outcome.err, h.daemonPump.Err(), "the fatal Accept outcome wraps the pump cause") | |
| 793 | + | case <-time.After(listenerTestDeadline): | |
| 794 | + | t.Fatal("physical loss did not unblock Accept") | |
| 795 | + | } | |
| 796 | + | for i, blocked := range readers { | |
| 797 | + | select { | |
| 798 | + | case err := <-blocked: | |
| 799 | + | require.Error(t, err, "stream %d", i+1) | |
| 800 | + | case <-time.After(listenerTestDeadline): | |
| 801 | + | t.Fatalf("physical loss did not unblock stream %d", i+1) | |
| 802 | + | } | |
| 803 | + | require.Eventually(t, func() bool { return channelClosed(daemonConnections[i].Done()) }, listenerTestDeadline, time.Millisecond, "stream %d", i+1) | |
| 804 | + | require.Error(t, daemonConnections[i].Err(), "stream %d", i+1) | |
| 805 | + | require.Equal(t, domain.RemoteFailureTransport, daemonConnections[i].FailureKind(), "stream %d", i+1) | |
| 806 | + | } | |
| 807 | + | for i, connection := range connections { | |
| 808 | + | require.Eventually(t, func() bool { return channelClosed(connection.Done()) }, listenerTestDeadline, time.Millisecond, "stream %d", i+1) | |
| 809 | + | } | |
| 810 | + | ||
| 811 | + | // The fatal outcome is stable, so a later Accept cannot restart the loop. | |
| 812 | + | _, err := h.listener.Accept() | |
| 813 | + | require.ErrorIs(t, err, ErrListenerLost) | |
| 814 | + | } | |
| 815 | + | ||
| 816 | + | // TestListenerCloseUnblocksAcceptAndJoinsGoroutines proves Close unblocks every | |
| 817 | + | // blocked Accept, is idempotent, and leaves no listener goroutine or pump watch | |
| 818 | + | // entry behind. | |
| 819 | + | func TestListenerCloseUnblocksAcceptAndJoinsGoroutines(t *testing.T) { | |
| 820 | + | before := runtime.NumGoroutine() | |
| 821 | + | ||
| 822 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 823 | + | probe := h.acceptAsync() | |
| 824 | + | connection := h.mustOpen(1) | |
| 825 | + | daemonConnection := h.awaitAccept(probe) | |
| 826 | + | go func() { _ = servePongs(daemonConnection) }() | |
| 827 | + | listenerRoundTrip(t, connection) | |
| 828 | + | ||
| 829 | + | blocked := h.acceptAsync() | |
| 830 | + | select { | |
| 831 | + | case outcome := <-blocked: | |
| 832 | + | t.Fatalf("Accept returned before Close: %+v", outcome) | |
| 833 | + | default: | |
| 834 | + | } | |
| 835 | + | ||
| 836 | + | require.NoError(t, h.listener.Close()) | |
| 837 | + | require.NoError(t, h.listener.Close(), "Close is idempotent") | |
| 838 | + | select { | |
| 839 | + | case outcome := <-blocked: | |
| 840 | + | require.ErrorIs(t, outcome.err, ErrListenerClosed) | |
| 841 | + | case <-time.After(listenerTestDeadline): | |
| 842 | + | t.Fatal("Close did not unblock Accept") | |
| 843 | + | } | |
| 844 | + | _, err := h.listener.Accept() | |
| 845 | + | require.ErrorIs(t, err, ErrListenerClosed) | |
| 846 | + | ||
| 847 | + | require.NoError(t, connection.Close()) | |
| 848 | + | require.Eventually(t, func() bool { return channelClosed(daemonConnection.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 849 | + | require.NoError(t, h.brokerPump.Close()) | |
| 850 | + | require.NoError(t, h.daemonPump.Close()) | |
| 851 | + | ||
| 852 | + | require.Eventually(t, func() bool { return !watchRegistered(h.daemonPump, 1) }, listenerTestDeadline, time.Millisecond, | |
| 853 | + | "listener connection leaked a pump watch entry") | |
| 854 | + | require.Eventually(t, func() bool { return runtime.NumGoroutine() <= before+3 }, listenerTestDeadline, time.Millisecond, | |
| 855 | + | "listener goroutines leaked") | |
| 856 | + | } | |
| 857 | + | ||
| 858 | + | // TestListenerClosedListenerRefusesFreshStreams proves a listener closed while | |
| 859 | + | // its physical connection is still live refuses a later admission on that | |
| 860 | + | // stream alone. | |
| 861 | + | func TestListenerClosedListenerRefusesFreshStreams(t *testing.T) { | |
| 862 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 863 | + | require.NoError(t, h.listener.Close()) | |
| 864 | + | ||
| 865 | + | _, err := h.open(1) | |
| 866 | + | require.Error(t, err) | |
| 867 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 868 | + | require.False(t, channelClosed(h.brokerPump.Done())) | |
| 869 | + | refused := requireStreamState(t, h.daemonPump, 1, StreamTerminal) | |
| 870 | + | require.Equal(t, domain.RemoteFailureTransport, refused.FailureKind) | |
| 871 | + | var detail ports.BrokerError | |
| 872 | + | require.ErrorAs(t, refused.Err, &detail) | |
| 873 | + | require.Equal(t, listenerClosedText, detail.Text) | |
| 874 | + | require.Equal(t, 0, h.daemonPump.Engine().Live()) | |
| 875 | + | } | |
| 876 | + | ||
| 877 | + | // TestListenerConnectionAdoptsAdmissionDeadline proves the typed session | |
| 878 | + | // connection of one accepted stream spends the stream's admission budget and | |
| 879 | + | // never starts a second one: a stream whose session handshake never runs fails | |
| 880 | + | // its preamble after the short admission budget, and only that stream is reset. | |
| 881 | + | func TestListenerConnectionAdoptsAdmissionDeadline(t *testing.T) { | |
| 882 | + | const budget = 300 * time.Millisecond | |
| 883 | + | h := newListenerHarness(t, budget, MaxAcceptQueue) | |
| 884 | + | ||
| 885 | + | probe := h.acceptAsync() | |
| 886 | + | started := time.Now() | |
| 887 | + | connection := h.mustOpen(1) | |
| 888 | + | daemonConnection := h.awaitAccept(probe) | |
| 889 | + | ||
| 890 | + | // The peer never starts its session preamble, so the accepted connection's | |
| 891 | + | // own deadline is what ends the read. | |
| 892 | + | _, err := daemonConnection.ReceiveClient() | |
| 893 | + | require.ErrorIs(t, err, sessionwire.ErrPreambleTimeout) | |
| 894 | + | require.Less(t, time.Since(started), listenerTestDeadline, | |
| 895 | + | "the connection started a second handshake budget instead of adopting the admission deadline") | |
| 896 | + | ||
| 897 | + | malformed := requireStreamState(t, h.daemonPump, 1, StreamTerminal) | |
| 898 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, malformed.FailureKind) | |
| 899 | + | require.Eventually(t, func() bool { return channelClosed(daemonConnection.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 900 | + | require.Eventually(t, func() bool { return channelClosed(connection.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 901 | + | require.False(t, channelClosed(h.brokerPump.Done())) | |
| 902 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 903 | + | ||
| 904 | + | // The failed handshake leaves no live stream and no watchdog entry behind. | |
| 905 | + | require.Equal(t, 0, h.daemonPump.Engine().Live()) | |
| 906 | + | require.Eventually(t, func() bool { return h.activeWatchdog() == 0 }, listenerTestDeadline, time.Millisecond, | |
| 907 | + | "the failed handshake leaked a watchdog entry") | |
| 908 | + | } | |
| 909 | + | ||
| 910 | + | // TestListenerConnectionPreambleFailureLeavesNoResidue proves a session | |
| 911 | + | // preamble that fails on the daemon side - the peer's first frame is not a | |
| 912 | + | // valid preamble - settles only its own stream: no live stream and no watchdog | |
| 913 | + | // entry are left behind, and a sibling stream is still accepted and served over | |
| 914 | + | // the live physical connection. | |
| 915 | + | func TestListenerConnectionPreambleFailureLeavesNoResidue(t *testing.T) { | |
| 916 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 917 | + | ||
| 918 | + | probe := h.acceptAsync() | |
| 919 | + | connection := h.mustOpen(1) | |
| 920 | + | daemonConnection := h.awaitAccept(probe) | |
| 921 | + | ||
| 922 | + | // The peer's first frame is not a valid session preamble; the failed | |
| 923 | + | // handshake closes the stream's carriage. | |
| 924 | + | require.NoError(t, h.brokerPump.Send(Data{Physical: 1, Data: []byte("not-a-session-preamble")})) | |
| 925 | + | _, err := daemonConnection.ReceiveClient() | |
| 926 | + | require.Error(t, err) | |
| 927 | + | ||
| 928 | + | failed := requireStreamState(t, h.daemonPump, 1, StreamTerminal) | |
| 929 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, failed.FailureKind) | |
| 930 | + | require.Eventually(t, func() bool { return channelClosed(daemonConnection.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 931 | + | require.Eventually(t, func() bool { return channelClosed(connection.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 932 | + | require.Equal(t, 0, h.daemonPump.Engine().Live()) | |
| 933 | + | require.Eventually(t, func() bool { return h.activeWatchdog() == 0 }, listenerTestDeadline, time.Millisecond, | |
| 934 | + | "the failed handshake leaked a watchdog entry") | |
| 935 | + | ||
| 936 | + | siblingProbe := h.acceptAsync() | |
| 937 | + | sibling := h.mustOpen(2) | |
| 938 | + | siblingDaemon := h.awaitAccept(siblingProbe) | |
| 939 | + | go func() { _ = servePongs(siblingDaemon) }() | |
| 940 | + | listenerRoundTrip(t, sibling) | |
| 941 | + | require.False(t, channelClosed(h.brokerPump.Done())) | |
| 942 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 943 | + | } | |
| 944 | + | ||
| 945 | + | // TestListenerConnectionCarriageErrorsAreStreamLocal proves the typed | |
| 946 | + | // connection's own error paths stay stream-local: a settled connection reports | |
| 947 | + | // its stable cause, and an orderly peer close reports io.EOF. | |
| 948 | + | func TestListenerConnectionCarriageErrorsAreStreamLocal(t *testing.T) { | |
| 949 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 950 | + | ||
| 951 | + | probe := h.acceptAsync() | |
| 952 | + | connection := h.mustOpen(1) | |
| 953 | + | daemonConnection := h.awaitAccept(probe) | |
| 954 | + | ||
| 955 | + | // An orderly peer close unblocks the daemon reader with io.EOF and a nil | |
| 956 | + | // terminal cause. | |
| 957 | + | require.NoError(t, connection.Close()) | |
| 958 | + | _, err := daemonConnection.ReceiveClient() | |
| 959 | + | require.ErrorIs(t, err, io.EOF) | |
| 960 | + | require.Eventually(t, func() bool { return channelClosed(daemonConnection.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 961 | + | require.Nil(t, daemonConnection.Err()) | |
| 962 | + | require.Equal(t, domain.RemoteFailureNone, daemonConnection.FailureKind()) | |
| 963 | + | ||
| 964 | + | // The settled connection keeps reporting its stable outcome. | |
| 965 | + | _, err = daemonConnection.ReceiveClient() | |
| 966 | + | require.Error(t, err) | |
| 967 | + | require.NoError(t, daemonConnection.Close()) | |
| 968 | + | require.NoError(t, daemonConnection.Close()) | |
| 969 | + | } | |
| 970 | + | ||
| 971 | + | // TestListenerConnectionInternalCarriageCloseSettlesStream proves a listener | |
| 972 | + | // connection whose inner session carriage closes on its own during the | |
| 973 | + | // handshake - the sessionwire preamble failing closes the raw transport - is | |
| 974 | + | // settled without any consumer read or Close: exactly one stream-local Reset is | |
| 975 | + | // issued, the admission slot and watchdog are released, and a sibling stream is | |
| 976 | + | // still accepted over the live physical connection. | |
| 977 | + | func TestListenerConnectionInternalCarriageCloseSettlesStream(t *testing.T) { | |
| 978 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 979 | + | ||
| 980 | + | probe := h.acceptAsync() | |
| 981 | + | connection := h.mustOpen(1) | |
| 982 | + | daemonConnection := h.awaitAccept(probe) | |
| 983 | + | ||
| 984 | + | // No consumer touches the connection: the internal carriage close alone must | |
| 985 | + | // settle it, publish its Done/Err, and release the slot and watchdog. | |
| 986 | + | require.NoError(t, daemonConnection.raw.Close()) | |
| 987 | + | ||
| 988 | + | closed := requireStreamState(t, h.daemonPump, 1, StreamTerminal) | |
| 989 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, closed.FailureKind) | |
| 990 | + | var detail ports.BrokerError | |
| 991 | + | require.ErrorAs(t, closed.Err, &detail) | |
| 992 | + | require.Equal(t, sessionCarriageClosedText, detail.Text) | |
| 993 | + | require.Eventually(t, func() bool { return channelClosed(daemonConnection.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 994 | + | require.Error(t, daemonConnection.Err()) | |
| 995 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, daemonConnection.FailureKind()) | |
| 996 | + | require.Equal(t, 0, h.daemonPump.Engine().Live()) | |
| 997 | + | require.Eventually(t, func() bool { return h.activeWatchdog() == 0 }, listenerTestDeadline, time.Millisecond, | |
| 998 | + | "the closed carriage leaked a watchdog entry") | |
| 999 | + | ||
| 1000 | + | // The peer observes the stream-local Reset and its Open fails, while the | |
| 1001 | + | // physical connection stays live. | |
| 1002 | + | require.Eventually(t, func() bool { return channelClosed(connection.Done()) }, listenerTestDeadline, time.Millisecond) | |
| 1003 | + | require.Error(t, connection.Err()) | |
| 1004 | + | require.False(t, channelClosed(h.brokerPump.Done())) | |
| 1005 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 1006 | + | ||
| 1007 | + | // A sibling is still accepted and served. | |
| 1008 | + | probe2 := h.acceptAsync() | |
| 1009 | + | sibling := h.mustOpen(2) | |
| 1010 | + | siblingDaemon := h.awaitAccept(probe2) | |
| 1011 | + | go func() { _ = servePongs(siblingDaemon) }() | |
| 1012 | + | listenerRoundTrip(t, sibling) | |
| 1013 | + | } | |
| 1014 | + | ||
| 1015 | + | // TestListenerConnectionCloseDiscardsQueuedInboundReleasesAccounting proves a | |
| 1016 | + | // daemon-side local Close first discards the inbound chunks the engine already | |
| 1017 | + | // accepted for its stream: the engine's orderly Close reaches its terminal | |
| 1018 | + | // state, releasing both the stream's admission slot and its queued aggregate | |
| 1019 | + | // bytes, retiring its watchdog, and leaving the broker with exactly one orderly | |
| 1020 | + | // mux Close instead of a leak or a Reset. Every cycle closes with one queued | |
| 1021 | + | // inbound chunk under a two-stream ceiling, so a leaked slot would refuse the | |
| 1022 | + | // third admission. | |
| 1023 | + | func TestListenerConnectionCloseDiscardsQueuedInboundReleasesAccounting(t *testing.T) { | |
| 1024 | + | ceilings := DefaultMuxCeilings() | |
| 1025 | + | ceilings.MaxStreams = 2 | |
| 1026 | + | h := newListenerHarnessWithCeilings(t, listenerIdleBudget, MaxAcceptQueue, ceilings) | |
| 1027 | + | ||
| 1028 | + | chunk := []byte("queued") | |
| 1029 | + | const cycles = 3 | |
| 1030 | + | for i := 1; i <= cycles; i++ { | |
| 1031 | + | id := PhysicalStreamID(i) | |
| 1032 | + | probe := h.acceptAsync() | |
| 1033 | + | connection := h.mustOpen(i) | |
| 1034 | + | daemonConnection := h.awaitAccept(probe) | |
| 1035 | + | ||
| 1036 | + | // The broker sent one chunk the daemon never reads, so the stream holds | |
| 1037 | + | // accepted inbound data when the local Close arrives. | |
| 1038 | + | require.NoError(t, h.brokerPump.Send(Data{Physical: id, Data: chunk})) | |
| 1039 | + | requireEngineEventually(t, h.daemonPump, func(e *StreamEngine) bool { return e.AggregateBytes() > 0 }, | |
| 1040 | + | "cycle %d: the accepted chunk was never queued", i) | |
| 1041 | + | ||
| 1042 | + | require.NoError(t, daemonConnection.Close()) | |
| 1043 | + | require.NoError(t, daemonConnection.Close(), "Close is idempotent") | |
| 1044 | + | require.True(t, channelClosed(daemonConnection.Done())) | |
| 1045 | + | require.NoError(t, daemonConnection.Err()) | |
| 1046 | + | ||
| 1047 | + | closed := mustStatus(t, h.daemonPump.Engine(), id) | |
| 1048 | + | require.Equal(t, StreamTerminal, closed.State, "cycle %d: the closed stream must be terminal", i) | |
| 1049 | + | require.NoError(t, closed.Err, "cycle %d: an orderly local close carries no cause", i) | |
| 1050 | + | require.Zero(t, closed.QueuedChunks, "cycle %d", i) | |
| 1051 | + | require.Zero(t, h.daemonPump.Engine().Live(), "cycle %d: a closing stream leaked its admission slot", i) | |
| 1052 | + | require.Zero(t, h.daemonPump.Engine().AggregateBytes(), "cycle %d: a closing stream leaked its queued bytes", i) | |
| 1053 | + | require.Eventually(t, func() bool { return h.activeWatchdog() == 0 }, listenerTestDeadline, time.Millisecond, | |
| 1054 | + | "cycle %d: the closed connection leaked a watchdog entry", i) | |
| 1055 | + | ||
| 1056 | + | // The broker observes one orderly mux Close, never a stream-local Reset, | |
| 1057 | + | // and the physical connection keeps flowing into the next cycle. | |
| 1058 | + | require.Eventually(t, func() bool { return channelClosed(connection.Done()) }, listenerTestDeadline, time.Millisecond, | |
| 1059 | + | "cycle %d: the broker never observed the mux Close", i) | |
| 1060 | + | require.NoError(t, connection.Err(), "cycle %d: the mux Close must stay orderly", i) | |
| 1061 | + | require.False(t, channelClosed(h.brokerPump.Done())) | |
| 1062 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 1063 | + | } | |
| 1064 | + | } | |
| 1065 | + | ||
| 1066 | + | // TestListenerDeliverSkipsExpiredAdmissionWithoutWatchdog proves Accept settles | |
| 1067 | + | // an admission whose absolute deadline already elapsed even when the watchdog | |
| 1068 | + | // goroutine has not run: the clock is moved past the deadline without firing the | |
| 1069 | + | // watchdog timer, yet delivering the stream still aborts it and accepts the | |
| 1070 | + | // next one. | |
| 1071 | + | func TestListenerDeliverSkipsExpiredAdmissionWithoutWatchdog(t *testing.T) { | |
| 1072 | + | const budget = 30 * time.Second | |
| 1073 | + | h := newListenerHarness(t, budget, MaxAcceptQueue) | |
| 1074 | + | ||
| 1075 | + | expired := make(chan error, 1) | |
| 1076 | + | go func() { | |
| 1077 | + | _, err := h.open(1) | |
| 1078 | + | expired <- err | |
| 1079 | + | }() | |
| 1080 | + | h.awaitPending(1) | |
| 1081 | + | ||
| 1082 | + | // Wait until the watchdog is blocked on stream 1's timer, then move the | |
| 1083 | + | // clock past the deadline without firing that timer: only the consumer's own | |
| 1084 | + | // deadline check can settle the stream here. | |
| 1085 | + | require.Eventually(t, func() bool { return h.clock.timerCount() == 1 }, listenerTestDeadline, time.Millisecond, | |
| 1086 | + | "watchdog never armed the admission deadline") | |
| 1087 | + | h.moveClock(h.clock.Now().Add(budget + time.Second)) | |
| 1088 | + | ||
| 1089 | + | probe := h.acceptAsync() | |
| 1090 | + | ||
| 1091 | + | requireStreamState(t, h.daemonPump, 1, StreamTerminal) | |
| 1092 | + | select { | |
| 1093 | + | case err := <-expired: | |
| 1094 | + | require.Error(t, err) | |
| 1095 | + | case <-time.After(listenerTestDeadline): | |
| 1096 | + | t.Fatal("an expired admission was never aborted") | |
| 1097 | + | } | |
| 1098 | + | expiredStatus := mustStatus(t, h.daemonPump.Engine(), 1) | |
| 1099 | + | var detail ports.BrokerError | |
| 1100 | + | require.ErrorAs(t, expiredStatus.Err, &detail) | |
| 1101 | + | require.Equal(t, handshakeDeadlineText, detail.Text) | |
| 1102 | + | require.Equal(t, 0, h.daemonPump.Engine().Live()) | |
| 1103 | + | require.Equal(t, 0, h.activeWatchdog()) | |
| 1104 | + | ||
| 1105 | + | sibling := h.mustOpen(2) | |
| 1106 | + | siblingDaemon := h.awaitAccept(probe) | |
| 1107 | + | require.Equal(t, PhysicalStreamID(2), siblingDaemon.Ref().Physical) | |
| 1108 | + | go func() { _ = servePongs(siblingDaemon) }() | |
| 1109 | + | listenerRoundTrip(t, sibling) | |
| 1110 | + | } | |
| 1111 | + | ||
| 1112 | + | // TestListenerCloseDrainsQueuedAdmissions proves Close settles every | |
| 1113 | + | // admitted-but-unaccepted stream on its own: each broker Open is unblocked with | |
| 1114 | + | // a stream-local error, each admission slot is released, the queue is cleared, | |
| 1115 | + | // and the watchdog holds nothing afterwards. | |
| 1116 | + | func TestListenerCloseDrainsQueuedAdmissions(t *testing.T) { | |
| 1117 | + | const queued = 3 | |
| 1118 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 1119 | + | ||
| 1120 | + | opens := make([]chan error, queued) | |
| 1121 | + | for i := 0; i < queued; i++ { | |
| 1122 | + | opens[i] = make(chan error, 1) | |
| 1123 | + | stream := i + 1 | |
| 1124 | + | go func() { | |
| 1125 | + | _, err := h.open(stream) | |
| 1126 | + | opens[i] <- err | |
| 1127 | + | }() | |
| 1128 | + | } | |
| 1129 | + | h.awaitPending(queued) | |
| 1130 | + | ||
| 1131 | + | require.NoError(t, h.listener.Close()) | |
| 1132 | + | ||
| 1133 | + | for i := 0; i < queued; i++ { | |
| 1134 | + | select { | |
| 1135 | + | case err := <-opens[i]: | |
| 1136 | + | require.Error(t, err, "queued open %d must fail when the listener closes", i+1) | |
| 1137 | + | case <-time.After(listenerTestDeadline): | |
| 1138 | + | t.Fatalf("Close did not unblock queued open %d", i+1) | |
| 1139 | + | } | |
| 1140 | + | } | |
| 1141 | + | require.Equal(t, 0, h.daemonPump.Engine().Live(), "Close must release every admission slot") | |
| 1142 | + | require.Equal(t, 0, h.activeWatchdog(), "Close must clear the deadline watchdog") | |
| 1143 | + | h.listener.mu.Lock() | |
| 1144 | + | require.Empty(t, h.listener.queue) | |
| 1145 | + | h.listener.mu.Unlock() | |
| 1146 | + | require.False(t, channelClosed(h.daemonPump.Done())) | |
| 1147 | + | require.False(t, channelClosed(h.brokerPump.Done())) | |
| 1148 | + | } | |
| 1149 | + | ||
| 1150 | + | // TestListenerConcurrentCloseDrainsOnce proves concurrent Close calls are all | |
| 1151 | + | // successful and drain the queued admissions exactly once. | |
| 1152 | + | func TestListenerConcurrentCloseDrainsOnce(t *testing.T) { | |
| 1153 | + | const queued = 4 | |
| 1154 | + | h := newListenerHarness(t, listenerIdleBudget, MaxAcceptQueue) | |
| 1155 | + | ||
| 1156 | + | opens := make([]chan error, queued) | |
| 1157 | + | for i := 0; i < queued; i++ { | |
| 1158 | + | opens[i] = make(chan error, 1) | |
| 1159 | + | stream := i + 1 | |
| 1160 | + | go func() { | |
| 1161 | + | _, err := h.open(stream) | |
| 1162 | + | opens[i] <- err | |
| 1163 | + | }() | |
| 1164 | + | } | |
| 1165 | + | h.awaitPending(queued) | |
| 1166 | + | ||
| 1167 | + | const closers = 8 | |
| 1168 | + | errs := make([]error, closers) | |
| 1169 | + | var wg sync.WaitGroup | |
| 1170 | + | for i := 0; i < closers; i++ { | |
| 1171 | + | wg.Add(1) | |
| 1172 | + | go func(i int) { | |
| 1173 | + | defer wg.Done() | |
| 1174 | + | errs[i] = h.listener.Close() | |
| 1175 | + | }(i) | |
| 1176 | + | } | |
| 1177 | + | wg.Wait() | |
| 1178 | + | for i, err := range errs { | |
| 1179 | + | require.NoError(t, err, "closer %d", i) | |
| 1180 | + | } | |
| 1181 | + | for i := 0; i < queued; i++ { | |
| 1182 | + | select { | |
| 1183 | + | case err := <-opens[i]: | |
| 1184 | + | require.Error(t, err, "queued open %d must fail when the listener closes", i+1) | |
| 1185 | + | case <-time.After(listenerTestDeadline): | |
| 1186 | + | t.Fatalf("concurrent Close did not unblock queued open %d", i+1) | |
| 1187 | + | } | |
| 1188 | + | } | |
| 1189 | + | require.Equal(t, 0, h.daemonPump.Engine().Live()) | |
| 1190 | + | require.Equal(t, 0, h.activeWatchdog()) | |
| 1191 | + | } | |
| 1192 | + | ||
| 1193 | + | // TestPumpOnAdmittedRejectsLateRegistration proves the pump refuses an | |
| 1194 | + | // admission-observer registration after Start, so a listener can never miss an | |
| 1195 | + | // already-admitted Open. | |
| 1196 | + | func TestPumpOnAdmittedRejectsLateRegistration(t *testing.T) { | |
| 1197 | + | pump, _ := newTestPump(t, DirectionClient) | |
| 1198 | + | require.NoError(t, pump.OnAdmitted(func(Open) {})) | |
| 1199 | + | pump.Start(context.Background()) | |
| 1200 | + | require.ErrorIs(t, pump.OnAdmitted(func(Open) {}), ErrAdmissionObserverLate) | |
| 1201 | + | require.ErrorIs(t, pump.OnAdmitted(nil), ErrAdmissionObserverLate) | |
| 1202 | + | ||
| 1203 | + | var nilPump *Pump | |
| 1204 | + | require.ErrorIs(t, nilPump.OnAdmitted(func(Open) {}), ErrPumpConfig) | |
| 1205 | + | } | |
| 1206 | + | ||
| 1207 | + | // TestNewListenerRefusesStartedPump proves the listener constructor surfaces a | |
| 1208 | + | // late admission-observer registration: it must be constructed before the pump | |
| 1209 | + | // starts. | |
| 1210 | + | func TestNewListenerRefusesStartedPump(t *testing.T) { | |
| 1211 | + | pump, _ := newTestPump(t, DirectionClient) | |
| 1212 | + | pump.Start(context.Background()) | |
| 1213 | + | ||
| 1214 | + | listener, err := NewListener(pump) | |
| 1215 | + | require.ErrorIs(t, err, ErrListenerConfig) | |
| 1216 | + | require.ErrorIs(t, err, ErrAdmissionObserverLate) | |
| 1217 | + | require.Nil(t, listener) | |
| 1218 | + | } | |
A internal/adapters/daemonmux/logical_conn.go +703 -0
| @@ -0,0 +1,703 @@ | |||
| 1 | + | // Typed logical connections over one physical Pump (P3.2). | |
| 2 | + | // | |
| 3 | + | // A LogicalConnection is one independently cancellable typed session stream | |
| 4 | + | // multiplexed over one physical daemonmux Pump. It implements | |
| 5 | + | // ports.BrokerLogicalConnection by embedding the typed ports.ClientConnection | |
| 6 | + | // that sessionwire builds on top of the mux stream, so the typed session | |
| 7 | + | // protocol (4-byte framed Protobuf envelopes, the session preamble, and every | |
| 8 | + | // semantic conversion) is reused verbatim: daemonmux never reinterprets a | |
| 9 | + | // session message. | |
| 10 | + | // | |
| 11 | + | // The carriage between the two is a private, ordered, bounded byte stream | |
| 12 | + | // (muxStreamPipe). Its read side pulls already-queued chunks from the pump | |
| 13 | + | // engine with Take, waking on the pump's per-stream Watch; the pump reader is | |
| 14 | + | // therefore never blocked by a slow or stalled consumer, and a consumer that | |
| 15 | + | // never reads lets the engine reset only its own stream. Its write side splits | |
| 16 | + | // each outbound byte slice into chunks at or below the negotiated | |
| 17 | + | // StreamChunkLimit and hands them to Pump.Send as Data frames, so streamframe | |
| 18 | + | // sees one ordered byte stream and applies the 4-byte session framing on top. | |
| 19 | + | // | |
| 20 | + | // The open side maps a ports.BrokerOpenStreamRequest to one mux Open and its | |
| 21 | + | // StreamRef, admits the stream locally, sends the Open, and waits under the | |
| 22 | + | // caller's context for the daemon's Opened or Refused. An independent Close | |
| 23 | + | // sends one mux Close and settles orderly; a malformed inner session frame | |
| 24 | + | // resets exactly that stream (one mux Reset) and leaves every sibling and the | |
| 25 | + | // physical connection untouched. Done, Err, and a physical-loss | |
| 26 | + | // ports.BrokerStreamLost are published once and stay stable afterwards. | |
| 27 | + | // | |
| 28 | + | // The narrow HandshakePlumbing seam exposes the accepted absolute 15 s | |
| 29 | + | // handshake deadline and a completion hook from the wrapped session | |
| 30 | + | // connection without changing ports or the daemon. | |
| 31 | + | package daemonmux | |
| 32 | + | ||
| 33 | + | import ( | |
| 34 | + | "context" | |
| 35 | + | "errors" | |
| 36 | + | "io" | |
| 37 | + | "sync" | |
| 38 | + | "time" | |
| 39 | + | ||
| 40 | + | "github.com/bnema/vev/internal/adapters/sessionwire" | |
| 41 | + | "github.com/bnema/vev/internal/adapters/streamframe" | |
| 42 | + | "github.com/bnema/vev/internal/domain" | |
| 43 | + | "github.com/bnema/vev/internal/ports" | |
| 44 | + | "github.com/bnema/vev/internal/protocol" | |
| 45 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 46 | + | ) | |
| 47 | + | ||
| 48 | + | // Logical connection sentinels. They are typed so a caller can classify a | |
| 49 | + | // rejection without matching on message text. | |
| 50 | + | var ( | |
| 51 | + | // ErrLogicalConfig reports an invalid logical-connection configuration: a | |
| 52 | + | // nil or server-side pump, or an unencodable open request. | |
| 53 | + | ErrLogicalConfig = errors.New("daemonmux: invalid logical connection configuration") | |
| 54 | + | // ErrLogicalClosed reports session work presented to a logical connection | |
| 55 | + | // that already reached its terminal outcome. | |
| 56 | + | ErrLogicalClosed = errors.New("daemonmux: logical stream is closed") | |
| 57 | + | ) | |
| 58 | + | ||
| 59 | + | // malformedResetText is the bounded, presentation-safe refusal text carried by | |
| 60 | + | // the stream-local Reset a logical connection sends when the inner session | |
| 61 | + | // stream is malformed. The diagnostic cause stays local in the terminal error. | |
| 62 | + | const malformedResetText = "malformed session stream" | |
| 63 | + | ||
| 64 | + | // LogicalConnector opens typed logical streams over one client-side Pump. It | |
| 65 | + | // owns the mux-local physical stream identity allocator: physical IDs are | |
| 66 | + | // strictly increasing and never reused, as the engine requires. One connector | |
| 67 | + | // is bound to exactly one physical connection. Its Open is safe for concurrent | |
| 68 | + | // use; ID allocation and local admission are serialized so engine admission | |
| 69 | + | // always sees strictly increasing IDs, while the Opened/Refused wait runs | |
| 70 | + | // outside that lock. | |
| 71 | + | type LogicalConnector struct { | |
| 72 | + | pump *Pump | |
| 73 | + | ||
| 74 | + | openMu sync.Mutex | |
| 75 | + | next uint64 | |
| 76 | + | } | |
| 77 | + | ||
| 78 | + | // NewLogicalConnector returns a connector over one broker-side pump: the pump | |
| 79 | + | // must decode daemon-to-broker frames (Inbound == DirectionServer) and emit | |
| 80 | + | // broker-to-daemon frames. A nil, engine-less, or server-side pump is refused. | |
| 81 | + | func NewLogicalConnector(pump *Pump) (*LogicalConnector, error) { | |
| 82 | + | if pump == nil || pump.Engine() == nil || pump.Local() != DirectionClient { | |
| 83 | + | return nil, ErrLogicalConfig | |
| 84 | + | } | |
| 85 | + | return &LogicalConnector{pump: pump}, nil | |
| 86 | + | } | |
| 87 | + | ||
| 88 | + | // Pump returns the physical pump this connector opens streams over. | |
| 89 | + | func (c *LogicalConnector) Pump() *Pump { return c.pump } | |
| 90 | + | ||
| 91 | + | // Open maps request to one mux Open and StreamRef, admits the stream locally, | |
| 92 | + | // sends the Open, and waits under ctx for the daemon's Opened or Refused. A | |
| 93 | + | // locally admitted refusal, a stream-reset, or a physical loss during the wait | |
| 94 | + | // is returned as a typed error and the local reservation is released. A | |
| 95 | + | // cancelled context releases the local reservation and returns ctx.Err(). | |
| 96 | + | func (c *LogicalConnector) Open(ctx context.Context, request ports.BrokerOpenStreamRequest) (*LogicalConnection, error) { | |
| 97 | + | if c == nil || c.pump == nil { | |
| 98 | + | return nil, ErrLogicalConfig | |
| 99 | + | } | |
| 100 | + | if err := request.Validate(); err != nil { | |
| 101 | + | return nil, ErrLogicalConfig | |
| 102 | + | } | |
| 103 | + | if ctx == nil { | |
| 104 | + | ctx = context.Background() | |
| 105 | + | } | |
| 106 | + | physical, err := c.reserve(request) | |
| 107 | + | if err != nil { | |
| 108 | + | return nil, err | |
| 109 | + | } | |
| 110 | + | status, err := c.awaitOpened(ctx, physical) | |
| 111 | + | if err != nil { | |
| 112 | + | c.pump.ReleaseWatch(physical) | |
| 113 | + | _ = c.abortReserved(physical) | |
| 114 | + | return nil, err | |
| 115 | + | } | |
| 116 | + | return newLogicalConnection(c.pump, status.Ref, status.Done, c.pump.cause(physical)), nil | |
| 117 | + | } | |
| 118 | + | ||
| 119 | + | // awaitOpened blocks until the admitted stream is confirmed open, refused, or | |
| 120 | + | // settles. It watches the pump's per-stream channel so it never polls, and it | |
| 121 | + | // also honors ctx and a physical terminal outcome. | |
| 122 | + | func (c *LogicalConnector) awaitOpened(ctx context.Context, physical PhysicalStreamID) (StreamStatus, error) { | |
| 123 | + | engine := c.pump.Engine() | |
| 124 | + | for { | |
| 125 | + | watch := c.pump.Watch(physical) | |
| 126 | + | status, ok := engine.Status(physical) | |
| 127 | + | if !ok { | |
| 128 | + | return StreamStatus{}, openFailure(ErrInvalidStream, c.pump.Err()) | |
| 129 | + | } | |
| 130 | + | switch status.State { | |
| 131 | + | case StreamOpen: | |
| 132 | + | return status, nil | |
| 133 | + | case StreamTerminal: | |
| 134 | + | return StreamStatus{}, refusedOrLost(status, c.pump.Err()) | |
| 135 | + | } | |
| 136 | + | select { | |
| 137 | + | case <-watch: | |
| 138 | + | case <-ctx.Done(): | |
| 139 | + | return StreamStatus{}, ctx.Err() | |
| 140 | + | case <-c.pump.Done(): | |
| 141 | + | status, ok := engine.Status(physical) | |
| 142 | + | if !ok { | |
| 143 | + | return StreamStatus{}, openFailure(c.pump.Err(), c.pump.Err()) | |
| 144 | + | } | |
| 145 | + | return StreamStatus{}, refusedOrLost(status, c.pump.Err()) | |
| 146 | + | } | |
| 147 | + | } | |
| 148 | + | } | |
| 149 | + | ||
| 150 | + | // abortReserved releases one locally admitted reservation that never became a | |
| 151 | + | // live connection: it settles the engine record and best-effort signals the | |
| 152 | + | // peer with a Reset, without touching the physical connection or a sibling. | |
| 153 | + | func (c *LogicalConnector) abortReserved(physical PhysicalStreamID) error { | |
| 154 | + | reset := Reset{Physical: physical} | |
| 155 | + | _, _ = c.pump.Engine().Reset(reset) | |
| 156 | + | _ = c.pump.Send(reset) | |
| 157 | + | return nil | |
| 158 | + | } | |
| 159 | + | ||
| 160 | + | // openMessage maps one validated request to its mux Open and StreamRef. | |
| 161 | + | func openMessage(physical PhysicalStreamID, request ports.BrokerOpenStreamRequest) Open { | |
| 162 | + | return Open{ | |
| 163 | + | Ref: StreamRef{ | |
| 164 | + | Physical: physical, | |
| 165 | + | Epoch: request.Epoch, | |
| 166 | + | Connection: request.Connection, | |
| 167 | + | Client: request.Stream, | |
| 168 | + | }, | |
| 169 | + | Purpose: request.Purpose, | |
| 170 | + | Local: request.Local, | |
| 171 | + | Endpoint: request.Endpoint, | |
| 172 | + | Registration: request.Registration, | |
| 173 | + | Target: request.Target, | |
| 174 | + | Env: append([]string(nil), request.Env...), | |
| 175 | + | Policy: request.Policy, | |
| 176 | + | } | |
| 177 | + | } | |
| 178 | + | ||
| 179 | + | // openAdmissionError classifies one refused local admission as a typed | |
| 180 | + | // admission error so the caller can retry with a fresh identity. | |
| 181 | + | func openAdmissionError(err error) error { | |
| 182 | + | switch { | |
| 183 | + | case errors.Is(err, ErrTooManyStreams): | |
| 184 | + | return ports.BrokerAdmissionLimit | |
| 185 | + | case errors.Is(err, ErrPhysicalClosed): | |
| 186 | + | return ports.BrokerAdmissionClosed | |
| 187 | + | case errors.Is(err, ErrStreamIDReused): | |
| 188 | + | return ports.BrokerAdmissionStale | |
| 189 | + | default: | |
| 190 | + | return ports.BrokerAdmissionInvalid | |
| 191 | + | } | |
| 192 | + | } | |
| 193 | + | ||
| 194 | + | // refusedOrLost classifies one terminal status observed while opening: an | |
| 195 | + | // explicit refusal, or a stream reset / physical loss. | |
| 196 | + | func refusedOrLost(status StreamStatus, physicalErr error) error { | |
| 197 | + | if status.Refused { | |
| 198 | + | if status.Refusal.AdmissionCode != 0 { | |
| 199 | + | switch status.Refusal.AdmissionCode { | |
| 200 | + | case 1: | |
| 201 | + | return ports.BrokerAdmissionLimit | |
| 202 | + | case 2: | |
| 203 | + | return ports.BrokerAdmissionClosed | |
| 204 | + | case 3: | |
| 205 | + | return ports.BrokerAdmissionStale | |
| 206 | + | case 4: | |
| 207 | + | return ports.BrokerAdmissionInvalid | |
| 208 | + | } | |
| 209 | + | } | |
| 210 | + | return ports.BrokerError{Code: status.Refusal.Code, Text: status.Refusal.Text} | |
| 211 | + | } | |
| 212 | + | if physicalErr != nil { | |
| 213 | + | return ports.BrokerError{Code: ports.BrokerErrorUnavailable, Cause: physicalErr} | |
| 214 | + | } | |
| 215 | + | return ports.BrokerError{Code: ports.BrokerErrorUnavailable, Cause: status.Err} | |
| 216 | + | } | |
| 217 | + | ||
| 218 | + | // openFailure maps one non-status transport failure on open. | |
| 219 | + | func openFailure(err error, physicalErr error) error { | |
| 220 | + | if physicalErr != nil { | |
| 221 | + | return ports.BrokerError{Code: ports.BrokerErrorUnavailable, Cause: physicalErr} | |
| 222 | + | } | |
| 223 | + | return ports.BrokerError{Code: ports.BrokerErrorUnavailable, Cause: err} | |
| 224 | + | } | |
| 225 | + | ||
| 226 | + | // discardQueuedInbound drains every inbound chunk the engine already accepted | |
| 227 | + | // for one physical stream, releasing its bytes from the per-stream and | |
| 228 | + | // aggregate budgets. A locally closing stream calls it before the engine's | |
| 229 | + | // orderly Close, because a local Close stops this side's reader: nothing else | |
| 230 | + | // would ever drain a preserved queue, and the engine only reaches its terminal | |
| 231 | + | // state - releasing the admission slot and the queued bytes - once that queue | |
| 232 | + | // is empty. It is a no-op for an unknown, empty, or already terminal stream. | |
| 233 | + | func discardQueuedInbound(pump *Pump, physical PhysicalStreamID) { | |
| 234 | + | for { | |
| 235 | + | if _, ok := pump.Take(physical); !ok { | |
| 236 | + | return | |
| 237 | + | } | |
| 238 | + | } | |
| 239 | + | } | |
| 240 | + | ||
| 241 | + | // LogicalConnection is one typed client-side logical stream over a physical | |
| 242 | + | // Pump. It implements ports.BrokerLogicalConnection: the embedded typed | |
| 243 | + | // ports.ClientConnection carries the session protocol, while Close, Done, and | |
| 244 | + | // Err add the stream lifecycle. It is safe for concurrent use. | |
| 245 | + | type LogicalConnection struct { | |
| 246 | + | ports.ClientConnection | |
| 247 | + | ||
| 248 | + | pump *Pump | |
| 249 | + | ref StreamRef | |
| 250 | + | pipe *muxStreamPipe | |
| 251 | + | raw *muxTransport | |
| 252 | + | stream <-chan struct{} | |
| 253 | + | // cause is the stream's retained publish-once terminal authority, captured | |
| 254 | + | // at open. It outlives the engine's bounded record, so a settled stream | |
| 255 | + | // keeps its exact cause even after the record is evicted. | |
| 256 | + | cause *terminalState | |
| 257 | + | ||
| 258 | + | terminal *terminalState | |
| 259 | + | // plumbing is the wrapped session connection's handshake seam, resolved | |
| 260 | + | // once at construction; nil when the wrapped connection lacks it. | |
| 261 | + | plumbing HandshakePlumbing | |
| 262 | + | // deadline is the exact absolute handshake deadline of the wrapped session | |
| 263 | + | // connection, resolved once at construction. | |
| 264 | + | deadline time.Time | |
| 265 | + | watchDone chan struct{} | |
| 266 | + | closeOnce sync.Once | |
| 267 | + | closeErr error | |
| 268 | + | } | |
| 269 | + | ||
| 270 | + | var _ ports.BrokerLogicalConnection = (*LogicalConnection)(nil) | |
| 271 | + | ||
| 272 | + | // newLogicalConnection builds the typed session channel over one confirmed | |
| 273 | + | // open stream and starts the terminal watcher. It never blocks. cause is the | |
| 274 | + | // stream's retained terminal authority, captured at open so the connection | |
| 275 | + | // reports its exact outcome independent of the engine's bounded record. | |
| 276 | + | func newLogicalConnection(pump *Pump, ref StreamRef, streamDone <-chan struct{}, cause *terminalState) *LogicalConnection { | |
| 277 | + | limit := pump.Ceilings().StreamChunkLimit | |
| 278 | + | pipe := newMuxStreamPipe(pump, ref.Physical, limit, streamDone) | |
| 279 | + | raw := newMuxTransport(pipe) | |
| 280 | + | connection := &LogicalConnection{ | |
| 281 | + | ClientConnection: sessionwire.NewClientConnection(raw), | |
| 282 | + | pump: pump, | |
| 283 | + | ref: ref, | |
| 284 | + | pipe: pipe, | |
| 285 | + | raw: raw, | |
| 286 | + | stream: streamDone, | |
| 287 | + | cause: cause, | |
| 288 | + | terminal: newTerminalState(), | |
| 289 | + | watchDone: make(chan struct{}), | |
| 290 | + | } | |
| 291 | + | if plumbing, ok := connection.ClientConnection.(HandshakePlumbing); ok { | |
| 292 | + | connection.plumbing = plumbing | |
| 293 | + | connection.deadline = plumbing.HandshakeDeadline() | |
| 294 | + | } else { | |
| 295 | + | connection.deadline = time.Now().Add(protocol.HandshakeTimeout) | |
| 296 | + | } | |
| 297 | + | go connection.watch() | |
| 298 | + | return connection | |
| 299 | + | } | |
| 300 | + | ||
| 301 | + | // Ref returns the complete stream reference recorded at admission. | |
| 302 | + | func (c *LogicalConnection) Ref() StreamRef { return c.ref } | |
| 303 | + | ||
| 304 | + | // HandshakeDeadline returns the accepted absolute local deadline of the | |
| 305 | + | // session handshake, plumbing the exact 15 s deadline the wrapped session | |
| 306 | + | // connection started with. It is resolved once at construction and stable for | |
| 307 | + | // the connection's lifetime. | |
| 308 | + | func (c *LogicalConnection) HandshakeDeadline() time.Time { return c.deadline } | |
| 309 | + | ||
| 310 | + | // OnHandshakeComplete registers fn to run exactly once when the session | |
| 311 | + | // handshake completes, plumbing the wrapped session connection's completion | |
| 312 | + | // hook. Registering after completion runs fn immediately. | |
| 313 | + | func (c *LogicalConnection) OnHandshakeComplete(fn func()) { | |
| 314 | + | if c.plumbing != nil { | |
| 315 | + | c.plumbing.OnHandshakeComplete(fn) | |
| 316 | + | return | |
| 317 | + | } | |
| 318 | + | if fn != nil { | |
| 319 | + | fn() | |
| 320 | + | } | |
| 321 | + | } | |
| 322 | + | ||
| 323 | + | // Done returns the logical terminal channel independent of reads. It closes | |
| 324 | + | // exactly once, before Err becomes stable. | |
| 325 | + | func (c *LogicalConnection) Done() <-chan struct{} { return c.terminal.Done() } | |
| 326 | + | ||
| 327 | + | // Err returns the stable terminal cause: nil for an orderly local or peer | |
| 328 | + | // Close, a ports.BrokerStreamLost for a peer Reset, a local malformed-inner | |
| 329 | + | // reset, or a physical loss. | |
| 330 | + | func (c *LogicalConnection) Err() error { return c.terminal.Err() } | |
| 331 | + | ||
| 332 | + | // FailureKind classifies Err. It is None while open and for an orderly Close. | |
| 333 | + | func (c *LogicalConnection) FailureKind() domain.RemoteFailureKind { return c.terminal.FailureKind() } | |
| 334 | + | ||
| 335 | + | // SendClient sends one typed client message over the logical stream. It | |
| 336 | + | // classifies a terminal transport failure and otherwise returns the session | |
| 337 | + | // error verbatim. | |
| 338 | + | func (c *LogicalConnection) SendClient(message protocol.ClientMessage) error { | |
| 339 | + | if c.settled() { | |
| 340 | + | return c.terminalError() | |
| 341 | + | } | |
| 342 | + | if err := c.ClientConnection.SendClient(message); err != nil { | |
| 343 | + | if c.settled() { | |
| 344 | + | return c.terminalError() | |
| 345 | + | } | |
| 346 | + | return err | |
| 347 | + | } | |
| 348 | + | return nil | |
| 349 | + | } | |
| 350 | + | ||
| 351 | + | // ReceiveServer receives one typed server message. An orderly end of the inner | |
| 352 | + | // stream returns io.EOF; any other inner-stream error is treated as a | |
| 353 | + | // malformed session frame: exactly this stream is reset towards the peer, its | |
| 354 | + | // terminal outcome is published, and the error is returned. | |
| 355 | + | func (c *LogicalConnection) ReceiveServer() (protocol.ServerMessage, error) { | |
| 356 | + | if c.settled() { | |
| 357 | + | return nil, c.terminalError() | |
| 358 | + | } | |
| 359 | + | message, err := c.ClientConnection.ReceiveServer() | |
| 360 | + | if err == nil { | |
| 361 | + | return message, nil | |
| 362 | + | } | |
| 363 | + | if errors.Is(err, io.EOF) { | |
| 364 | + | c.syncFromEngine() | |
| 365 | + | return nil, c.terminalError() | |
| 366 | + | } | |
| 367 | + | return nil, c.abortMalformed(err) | |
| 368 | + | } | |
| 369 | + | ||
| 370 | + | // Close independently closes the logical stream: it discards the inbound | |
| 371 | + | // chunks the engine already accepted, settles this stream locally, sends | |
| 372 | + | // exactly one mux Close, publishes the orderly nil outcome, and unblocks every | |
| 373 | + | // read and write. It never touches the physical connection or a sibling | |
| 374 | + | // stream. Close is idempotent and concurrent-safe; every caller observes the | |
| 375 | + | // same stored transport Close error. | |
| 376 | + | func (c *LogicalConnection) Close() error { | |
| 377 | + | c.closeOnce.Do(func() { | |
| 378 | + | // Discard what the engine already accepted before closing locally: this | |
| 379 | + | // side stops reading as soon as it closes, so draining first is what lets | |
| 380 | + | // the engine's orderly Close reach its terminal state and release the | |
| 381 | + | // stream's admission slot and aggregate bytes instead of parking the | |
| 382 | + | // stream in closing with a queue nothing will ever drain. | |
| 383 | + | discardQueuedInbound(c.pump, c.ref.Physical) | |
| 384 | + | _, _ = c.pump.Engine().Close(Close{Physical: c.ref.Physical}) | |
| 385 | + | // A chunk the reader accepted between the drain and the Close parked the | |
| 386 | + | // stream in closing instead of terminal; no new chunk is accepted after | |
| 387 | + | // the Close, so this drain terminalizes the stream and releases its slot | |
| 388 | + | // and bytes. | |
| 389 | + | discardQueuedInbound(c.pump, c.ref.Physical) | |
| 390 | + | _ = c.pump.Send(Close{Physical: c.ref.Physical}) | |
| 391 | + | c.terminal.Close() | |
| 392 | + | c.closeErr = c.raw.Close() | |
| 393 | + | <-c.watchDone | |
| 394 | + | c.pump.ReleaseWatch(c.ref.Physical) | |
| 395 | + | }) | |
| 396 | + | return c.closeErr | |
| 397 | + | } | |
| 398 | + | ||
| 399 | + | // watch publishes the logical terminal outcome when the stream settles on its | |
| 400 | + | // own (peer Close, peer Reset, queue overflow, or physical loss) independently | |
| 401 | + | // of reads, and unblocks the byte stream. A local Close settles first and makes | |
| 402 | + | // this a no-op. | |
| 403 | + | func (c *LogicalConnection) watch() { | |
| 404 | + | defer c.pump.ReleaseWatch(c.ref.Physical) | |
| 405 | + | defer close(c.watchDone) | |
| 406 | + | select { | |
| 407 | + | case <-c.stream: | |
| 408 | + | // The stream reached its terminal state. Publish the retained cause, | |
| 409 | + | // not a fresh engine lookup: the engine record may already be evicted. | |
| 410 | + | c.syncFromEngine() | |
| 411 | + | case <-c.pump.Done(): | |
| 412 | + | // A physical terminal outcome wakes every stream. A stream-local cause | |
| 413 | + | // recorded before the physical failure stays authoritative. | |
| 414 | + | if !c.settleFromCause() { | |
| 415 | + | if err := c.pump.Err(); err != nil { | |
| 416 | + | c.settleLoss(c.pump.FailureKind(), err) | |
| 417 | + | } else { | |
| 418 | + | c.terminal.Close() | |
| 419 | + | } | |
| 420 | + | } | |
| 421 | + | case <-c.pipe.Done(): | |
| 422 | + | return | |
| 423 | + | } | |
| 424 | + | // Closing the framed transport also closes the byte stream and joins | |
| 425 | + | // streamframe's writer, so a self-terminating stream leaves no goroutine | |
| 426 | + | // behind. | |
| 427 | + | _ = c.raw.Close() | |
| 428 | + | } | |
| 429 | + | ||
| 430 | + | // syncFromEngine settles this connection from the stream's retained terminal | |
| 431 | + | // authority when it published, and otherwise from the pump's own terminal | |
| 432 | + | // outcome. It closes the race between a blocked read observing the inner-stream | |
| 433 | + | // end and the watcher publishing the same outcome, and it never lets a physical | |
| 434 | + | // failure surface as a clean io.EOF: the retained authority outlives the | |
| 435 | + | // bounded engine record, so an evicted stream still reports its exact cause. | |
| 436 | + | func (c *LogicalConnection) syncFromEngine() { | |
| 437 | + | if c.settled() { | |
| 438 | + | return | |
| 439 | + | } | |
| 440 | + | if c.settleFromCause() { | |
| 441 | + | return | |
| 442 | + | } | |
| 443 | + | if err := c.pump.Err(); err != nil { | |
| 444 | + | c.settleLoss(c.pump.FailureKind(), err) | |
| 445 | + | return | |
| 446 | + | } | |
| 447 | + | if status, ok := c.pump.Engine().Status(c.ref.Physical); ok && status.State == StreamTerminal { | |
| 448 | + | c.settleFromStatus(status) | |
| 449 | + | return | |
| 450 | + | } | |
| 451 | + | c.terminal.Close() | |
| 452 | + | } | |
| 453 | + | ||
| 454 | + | // settleFromCause publishes the terminal outcome recorded by the stream's | |
| 455 | + | // retained publish-once authority. The authority is captured when the stream | |
| 456 | + | // opens and stays valid for the connection's lifetime even after the engine | |
| 457 | + | // evicts the bounded record, so a settled stream keeps its exact cause | |
| 458 | + | // independent of retention: a Reset never degrades into an orderly close. It | |
| 459 | + | // reports whether the authority had published. | |
| 460 | + | func (c *LogicalConnection) settleFromCause() bool { | |
| 461 | + | cause := c.cause | |
| 462 | + | if cause == nil { | |
| 463 | + | return false | |
| 464 | + | } | |
| 465 | + | select { | |
| 466 | + | case <-cause.Done(): | |
| 467 | + | default: | |
| 468 | + | return false | |
| 469 | + | } | |
| 470 | + | if err := cause.Err(); err != nil { | |
| 471 | + | c.settleLoss(cause.FailureKind(), err) | |
| 472 | + | } else { | |
| 473 | + | c.terminal.Close() | |
| 474 | + | } | |
| 475 | + | return true | |
| 476 | + | } | |
| 477 | + | ||
| 478 | + | // settleFromStatus publishes the terminal outcome recorded by the engine. | |
| 479 | + | func (c *LogicalConnection) settleFromStatus(status StreamStatus) { | |
| 480 | + | switch { | |
| 481 | + | case status.Refused: | |
| 482 | + | c.terminal.Fail(domain.RemoteFailureInvalidResponse, ports.BrokerError{Code: status.Refusal.Code, Text: status.Refusal.Text}) | |
| 483 | + | case status.Err == nil: | |
| 484 | + | c.terminal.Close() | |
| 485 | + | default: | |
| 486 | + | c.settleLoss(status.FailureKind, status.Err) | |
| 487 | + | } | |
| 488 | + | } | |
| 489 | + | ||
| 490 | + | // abortMalformed resets exactly this stream after a malformed inner session | |
| 491 | + | // frame: it settles the local engine record, best-effort signals the peer with | |
| 492 | + | // one Reset, publishes the stream-local failure, unblocks the byte stream, and | |
| 493 | + | // returns the stable terminal error. | |
| 494 | + | func (c *LogicalConnection) abortMalformed(cause error) error { | |
| 495 | + | reset := Reset{ | |
| 496 | + | Physical: c.ref.Physical, | |
| 497 | + | Error: ErrorDetail{ | |
| 498 | + | Code: ports.BrokerErrorUnavailable, | |
| 499 | + | Text: malformedResetText, | |
| 500 | + | FailureKind: domain.RemoteFailureInvalidResponse, | |
| 501 | + | }, | |
| 502 | + | HasError: true, | |
| 503 | + | } | |
| 504 | + | _, _ = c.pump.Engine().Reset(reset) | |
| 505 | + | _ = c.pump.Send(reset) | |
| 506 | + | c.settleLoss(domain.RemoteFailureInvalidResponse, cause) | |
| 507 | + | _ = c.raw.Close() | |
| 508 | + | return c.terminalError() | |
| 509 | + | } | |
| 510 | + | ||
| 511 | + | // settleLoss publishes one stream-local loss carrying the exact | |
| 512 | + | // epoch/connection/stream scope of this logical connection. | |
| 513 | + | func (c *LogicalConnection) settleLoss(kind domain.RemoteFailureKind, cause error) { | |
| 514 | + | if kind < domain.RemoteFailureTransport || kind > domain.RemoteFailureInvalidResponse { | |
| 515 | + | kind = domain.RemoteFailureTransport | |
| 516 | + | } | |
| 517 | + | c.terminal.Fail(kind, ports.BrokerStreamLost{ | |
| 518 | + | Connection: c.ref.Connection, Stream: c.ref.Client, Epoch: c.ref.Epoch, | |
| 519 | + | Cause: kind, Err: cause, | |
| 520 | + | }) | |
| 521 | + | } | |
| 522 | + | ||
| 523 | + | // settled reports whether the logical connection already reached its terminal | |
| 524 | + | // outcome. | |
| 525 | + | func (c *LogicalConnection) settled() bool { | |
| 526 | + | select { | |
| 527 | + | case <-c.terminal.Done(): | |
| 528 | + | return true | |
| 529 | + | default: | |
| 530 | + | return false | |
| 531 | + | } | |
| 532 | + | } | |
| 533 | + | ||
| 534 | + | // terminalError is the stable error of a settled connection: the published | |
| 535 | + | // cause, or io.EOF for an orderly close. | |
| 536 | + | func (c *LogicalConnection) terminalError() error { | |
| 537 | + | if err := c.terminal.Err(); err != nil { | |
| 538 | + | return err | |
| 539 | + | } | |
| 540 | + | return io.EOF | |
| 541 | + | } | |
| 542 | + | ||
| 543 | + | // muxStreamPipe is the private ordered, bounded byte stream of one logical mux | |
| 544 | + | // stream. Read pulls already-queued inbound chunks from the pump engine with | |
| 545 | + | // Take, blocking on the pump's per-stream Watch and ending on the stream's | |
| 546 | + | // terminal channel or the pump's Done; Write splits an outbound slice at the | |
| 547 | + | // negotiated chunk ceiling and sends each piece as one Data frame. Close | |
| 548 | + | // unblocks both directions. Exactly one reader is expected (streamframe | |
| 549 | + | // serializes Recv), and concurrent writes are serialized by streamframe too. | |
| 550 | + | type muxStreamPipe struct { | |
| 551 | + | pump *Pump | |
| 552 | + | id PhysicalStreamID | |
| 553 | + | limit uint64 | |
| 554 | + | streamDone <-chan struct{} | |
| 555 | + | done chan struct{} | |
| 556 | + | once sync.Once | |
| 557 | + | mu sync.Mutex | |
| 558 | + | pending []byte | |
| 559 | + | } | |
| 560 | + | ||
| 561 | + | // newMuxStreamPipe builds the ordered byte stream of one mux stream. limit is | |
| 562 | + | // the negotiated StreamChunkLimit; it must be nonzero. | |
| 563 | + | func newMuxStreamPipe(pump *Pump, id PhysicalStreamID, limit uint64, streamDone <-chan struct{}) *muxStreamPipe { | |
| 564 | + | if limit == 0 { | |
| 565 | + | limit = MinMuxChunkBytes | |
| 566 | + | } | |
| 567 | + | return &muxStreamPipe{pump: pump, id: id, limit: limit, streamDone: streamDone, done: make(chan struct{})} | |
| 568 | + | } | |
| 569 | + | ||
| 570 | + | // done returns the pipe's closed-on-close channel. | |
| 571 | + | func (s *muxStreamPipe) Done() <-chan struct{} { return s.done } | |
| 572 | + | ||
| 573 | + | func (s *muxStreamPipe) closed() bool { | |
| 574 | + | select { | |
| 575 | + | case <-s.done: | |
| 576 | + | return true | |
| 577 | + | default: | |
| 578 | + | return false | |
| 579 | + | } | |
| 580 | + | } | |
| 581 | + | ||
| 582 | + | // Read returns the next ordered inbound bytes. It drains whatever the engine | |
| 583 | + | // still holds and returns io.EOF once the stream or the physical connection | |
| 584 | + | // reaches its terminal outcome. | |
| 585 | + | func (s *muxStreamPipe) Read(p []byte) (int, error) { | |
| 586 | + | if len(p) == 0 { | |
| 587 | + | return 0, nil | |
| 588 | + | } | |
| 589 | + | for { | |
| 590 | + | s.mu.Lock() | |
| 591 | + | if len(s.pending) > 0 { | |
| 592 | + | n := copy(p, s.pending) | |
| 593 | + | s.pending = s.pending[n:] | |
| 594 | + | s.mu.Unlock() | |
| 595 | + | return n, nil | |
| 596 | + | } | |
| 597 | + | s.mu.Unlock() | |
| 598 | + | if s.closed() { | |
| 599 | + | return 0, io.EOF | |
| 600 | + | } | |
| 601 | + | watch := s.pump.Watch(s.id) | |
| 602 | + | if chunk, ok := s.pump.Take(s.id); ok { | |
| 603 | + | s.mu.Lock() | |
| 604 | + | s.pending = chunk | |
| 605 | + | s.mu.Unlock() | |
| 606 | + | continue | |
| 607 | + | } | |
| 608 | + | if watchClosed(watch) { | |
| 609 | + | // Watch returns an already-closed channel only for a terminal or | |
| 610 | + | // unknown stream, so no further inbound chunk can ever be queued. | |
| 611 | + | return 0, io.EOF | |
| 612 | + | } | |
| 613 | + | select { | |
| 614 | + | case <-watch: | |
| 615 | + | case <-s.done: | |
| 616 | + | return 0, io.EOF | |
| 617 | + | case <-s.streamDone: | |
| 618 | + | return 0, io.EOF | |
| 619 | + | case <-s.pump.Done(): | |
| 620 | + | return 0, io.EOF | |
| 621 | + | } | |
| 622 | + | } | |
| 623 | + | } | |
| 624 | + | ||
| 625 | + | // Write splits p into ordered chunks at or below the negotiated chunk ceiling | |
| 626 | + | // and sends each as one Data frame. It reports the bytes fully sent before an | |
| 627 | + | // error. | |
| 628 | + | func (s *muxStreamPipe) Write(p []byte) (int, error) { | |
| 629 | + | written := 0 | |
| 630 | + | for len(p) > 0 { | |
| 631 | + | if s.closed() { | |
| 632 | + | return written, ErrLogicalClosed | |
| 633 | + | } | |
| 634 | + | n := len(p) | |
| 635 | + | if uint64(n) > s.limit { | |
| 636 | + | n = int(s.limit) | |
| 637 | + | } | |
| 638 | + | if err := s.pump.Send(Data{Physical: s.id, Data: p[:n]}); err != nil { | |
| 639 | + | if s.closed() { | |
| 640 | + | return written, ErrLogicalClosed | |
| 641 | + | } | |
| 642 | + | return written, err | |
| 643 | + | } | |
| 644 | + | written += n | |
| 645 | + | p = p[n:] | |
| 646 | + | } | |
| 647 | + | return written, nil | |
| 648 | + | } | |
| 649 | + | ||
| 650 | + | // Close unblocks every read and write on the byte stream exactly once. | |
| 651 | + | func (s *muxStreamPipe) Close() error { | |
| 652 | + | s.once.Do(func() { close(s.done) }) | |
| 653 | + | return nil | |
| 654 | + | } | |
| 655 | + | ||
| 656 | + | // muxTransport adapts the ordered byte stream to wire.Transport: streamframe | |
| 657 | + | // owns the 4-byte length framing, and sessionwire owns the typed conversion on | |
| 658 | + | // top. Closing it closes the byte stream and joins streamframe's writer. | |
| 659 | + | type muxTransport struct { | |
| 660 | + | pipe *muxStreamPipe | |
| 661 | + | framer *streamframe.Framer | |
| 662 | + | } | |
| 663 | + | ||
| 664 | + | var _ wire.Transport = (*muxTransport)(nil) | |
| 665 | + | ||
| 666 | + | // newMuxTransport frames the byte stream into typed envelopes. | |
| 667 | + | func newMuxTransport(pipe *muxStreamPipe) *muxTransport { | |
| 668 | + | return &muxTransport{pipe: pipe, framer: streamframe.NewFramer(pipe, pipe, pipe.Close)} | |
| 669 | + | } | |
| 670 | + | ||
| 671 | + | func (t *muxTransport) Send(envelope wire.Envelope) error { return t.framer.Send(envelope.Payload) } | |
| 672 | + | ||
| 673 | + | func (t *muxTransport) Recv() (wire.Envelope, error) { | |
| 674 | + | payload, err := t.framer.Recv() | |
| 675 | + | if err != nil { | |
| 676 | + | return wire.Envelope{}, err | |
| 677 | + | } | |
| 678 | + | return wire.Envelope{Payload: payload}, nil | |
| 679 | + | } | |
| 680 | + | ||
| 681 | + | func (t *muxTransport) Close() error { return t.framer.Close() } | |
| 682 | + | ||
| 683 | + | // reserve allocates the next strictly increasing mux-local physical stream ID, | |
| 684 | + | // admits it locally, and enqueues its Open frame under one lock, so both the | |
| 685 | + | // engine's strictly increasing admission order and the outbound frame order | |
| 686 | + | // match the allocation order even when Opens run concurrently. The lock is | |
| 687 | + | // released before the open waits on the round trip, so a slow peer never blocks | |
| 688 | + | // a sibling reservation. | |
| 689 | + | func (c *LogicalConnector) reserve(request ports.BrokerOpenStreamRequest) (PhysicalStreamID, error) { | |
| 690 | + | c.openMu.Lock() | |
| 691 | + | defer c.openMu.Unlock() | |
| 692 | + | c.next++ | |
| 693 | + | physical := PhysicalStreamID(c.next) | |
| 694 | + | open := openMessage(physical, request) | |
| 695 | + | if err := c.pump.Engine().Open(open); err != nil { | |
| 696 | + | return 0, openAdmissionError(err) | |
| 697 | + | } | |
| 698 | + | if err := c.pump.Send(open); err != nil { | |
| 699 | + | _ = c.abortReserved(physical) | |
| 700 | + | return 0, openFailure(err, c.pump.Err()) | |
| 701 | + | } | |
| 702 | + | return physical, nil | |
| 703 | + | } | |
A internal/adapters/daemonmux/logical_conn_test.go +1136 -0
| @@ -0,0 +1,1136 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "errors" | |
| 6 | + | "fmt" | |
| 7 | + | "io" | |
| 8 | + | "runtime" | |
| 9 | + | "sync" | |
| 10 | + | "testing" | |
| 11 | + | "time" | |
| 12 | + | ||
| 13 | + | "github.com/stretchr/testify/require" | |
| 14 | + | ||
| 15 | + | "github.com/bnema/vev/internal/adapters/sessionwire" | |
| 16 | + | "github.com/bnema/vev/internal/domain" | |
| 17 | + | "github.com/bnema/vev/internal/ports" | |
| 18 | + | "github.com/bnema/vev/internal/protocol" | |
| 19 | + | ) | |
| 20 | + | ||
| 21 | + | // errMemCarrierClosed is the synthetic carrier error an in-memory endpoint | |
| 22 | + | // returns once its shared link is closed, mirroring the adapter prompt-close | |
| 23 | + | // contract. | |
| 24 | + | var errMemCarrierClosed = errors.New("memCarrier: closed") | |
| 25 | + | ||
| 26 | + | // memCarrier is one end of a paired, in-memory FramedCarrier link. Two | |
| 27 | + | // endpoints share a closed channel and exchange complete envelopes over two | |
| 28 | + | // buffered channels, so one pump's reader is the other pump's writer. | |
| 29 | + | type memCarrier struct { | |
| 30 | + | out chan []byte | |
| 31 | + | in chan []byte | |
| 32 | + | done chan struct{} | |
| 33 | + | once *sync.Once | |
| 34 | + | } | |
| 35 | + | ||
| 36 | + | func newMemCarrierPair(buffer int) (*memCarrier, *memCarrier) { | |
| 37 | + | aToB := make(chan []byte, buffer) | |
| 38 | + | bToA := make(chan []byte, buffer) | |
| 39 | + | done := make(chan struct{}) | |
| 40 | + | once := &sync.Once{} | |
| 41 | + | a := &memCarrier{out: aToB, in: bToA, done: done, once: once} | |
| 42 | + | b := &memCarrier{out: bToA, in: aToB, done: done, once: once} | |
| 43 | + | return a, b | |
| 44 | + | } | |
| 45 | + | ||
| 46 | + | func (m *memCarrier) Send(ctx context.Context, payload []byte) error { | |
| 47 | + | select { | |
| 48 | + | case m.out <- append([]byte(nil), payload...): | |
| 49 | + | return nil | |
| 50 | + | case <-m.done: | |
| 51 | + | return errMemCarrierClosed | |
| 52 | + | case <-ctx.Done(): | |
| 53 | + | return ctx.Err() | |
| 54 | + | } | |
| 55 | + | } | |
| 56 | + | ||
| 57 | + | func (m *memCarrier) Receive(ctx context.Context) ([]byte, error) { | |
| 58 | + | select { | |
| 59 | + | case payload := <-m.in: | |
| 60 | + | return payload, nil | |
| 61 | + | case <-m.done: | |
| 62 | + | return nil, errMemCarrierClosed | |
| 63 | + | case <-ctx.Done(): | |
| 64 | + | return nil, ctx.Err() | |
| 65 | + | } | |
| 66 | + | } | |
| 67 | + | ||
| 68 | + | func (m *memCarrier) Close() error { | |
| 69 | + | m.once.Do(func() { close(m.done) }) | |
| 70 | + | return nil | |
| 71 | + | } | |
| 72 | + | ||
| 73 | + | // newPairedPumps builds a broker-side pump (decodes server frames, emits client | |
| 74 | + | // frames) and a daemon-side pump (decodes client frames, emits server frames) | |
| 75 | + | // joined by one in-memory link, starts both, and closes them at test end. | |
| 76 | + | func newPairedPumps(t *testing.T, ceilings MuxCeilings) (*Pump, *Pump) { | |
| 77 | + | t.Helper() | |
| 78 | + | broker, daemon, start := newPairedPumpsUnstarted(t, ceilings) | |
| 79 | + | start() | |
| 80 | + | return broker, daemon | |
| 81 | + | } | |
| 82 | + | ||
| 83 | + | // newPairedPumpsUnstarted builds the same paired pump link as newPairedPumps | |
| 84 | + | // but leaves both pumps unstarted, so a daemon-side consumer can register its | |
| 85 | + | // admission observer before the pump's reader begins (the pump refuses a late | |
| 86 | + | // registration with ErrAdmissionObserverLate). The returned start function | |
| 87 | + | // starts both pumps exactly where the caller needs them running; the pumps are | |
| 88 | + | // closed at test end. | |
| 89 | + | func newPairedPumpsUnstarted(t *testing.T, ceilings MuxCeilings) (*Pump, *Pump, func()) { | |
| 90 | + | t.Helper() | |
| 91 | + | brokerCarrier, daemonCarrier := newMemCarrierPair(4096) | |
| 92 | + | broker, err := NewPump(brokerCarrier, DirectionServer, ceilings) | |
| 93 | + | require.NoError(t, err) | |
| 94 | + | daemon, err := NewPump(daemonCarrier, DirectionClient, ceilings) | |
| 95 | + | require.NoError(t, err) | |
| 96 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 97 | + | t.Cleanup(func() { | |
| 98 | + | cancel() | |
| 99 | + | _ = broker.Close() | |
| 100 | + | _ = daemon.Close() | |
| 101 | + | }) | |
| 102 | + | start := func() { | |
| 103 | + | broker.Start(ctx) | |
| 104 | + | daemon.Start(ctx) | |
| 105 | + | } | |
| 106 | + | return broker, daemon, start | |
| 107 | + | } | |
| 108 | + | ||
| 109 | + | func mustConnector(t *testing.T, pump *Pump) *LogicalConnector { | |
| 110 | + | t.Helper() | |
| 111 | + | connector, err := NewLogicalConnector(pump) | |
| 112 | + | require.NoError(t, err) | |
| 113 | + | return connector | |
| 114 | + | } | |
| 115 | + | ||
| 116 | + | func logicalTestPolicy() ports.BrokerPolicy { | |
| 117 | + | return ports.BrokerPolicy{ | |
| 118 | + | ProtocolVersion: protocol.Version, | |
| 119 | + | CatalogSchemaVersion: 1, | |
| 120 | + | EnvironmentPolicy: protocol.EnvironmentPolicyClientOwned, | |
| 121 | + | Transport: "mux", | |
| 122 | + | Trust: "trust", | |
| 123 | + | Launch: "launch", | |
| 124 | + | Isolation: "isolation", | |
| 125 | + | } | |
| 126 | + | } | |
| 127 | + | ||
| 128 | + | // controlRequest builds one valid control-purpose open request for stream. | |
| 129 | + | func controlRequest(stream int) ports.BrokerOpenStreamRequest { | |
| 130 | + | return ports.BrokerOpenStreamRequest{ | |
| 131 | + | Epoch: 1, | |
| 132 | + | Purpose: ports.BrokerStreamControl, | |
| 133 | + | Local: true, | |
| 134 | + | Connection: ports.BrokerConnectionID{1}, | |
| 135 | + | Stream: ports.BrokerStreamID(stream), | |
| 136 | + | Policy: logicalTestPolicy(), | |
| 137 | + | } | |
| 138 | + | } | |
| 139 | + | ||
| 140 | + | // awaitStream waits until a stream identity has been admitted on the daemon | |
| 141 | + | // side. | |
| 142 | + | func awaitStream(pump *Pump, id PhysicalStreamID, deadline time.Time) (StreamStatus, error) { | |
| 143 | + | for time.Now().Before(deadline) { | |
| 144 | + | if status, ok := pump.Engine().Status(id); ok && status.State != StreamTerminal { | |
| 145 | + | return status, nil | |
| 146 | + | } | |
| 147 | + | time.Sleep(time.Millisecond) | |
| 148 | + | } | |
| 149 | + | return StreamStatus{}, fmt.Errorf("daemonmux test: stream %d was not admitted", id) | |
| 150 | + | } | |
| 151 | + | ||
| 152 | + | // sessionPeer is a test-only daemon-side typed session: it frames the same mux | |
| 153 | + | // byte stream with streamframe and wraps it with sessionwire's server | |
| 154 | + | // connection, so the client's typed connection speaks to a real typed peer. | |
| 155 | + | type sessionPeer struct { | |
| 156 | + | raw *muxTransport | |
| 157 | + | srv ports.ServerConnection | |
| 158 | + | flood int | |
| 159 | + | } | |
| 160 | + | ||
| 161 | + | func newSessionPeer(pump *Pump, ref StreamRef, flood int) *sessionPeer { | |
| 162 | + | status, _ := pump.Engine().Status(ref.Physical) | |
| 163 | + | pipe := newMuxStreamPipe(pump, ref.Physical, pump.Ceilings().StreamChunkLimit, status.Done) | |
| 164 | + | raw := newMuxTransport(pipe) | |
| 165 | + | peer := &sessionPeer{raw: raw, srv: sessionwire.NewServerConnection(raw), flood: flood} | |
| 166 | + | go peer.serve() | |
| 167 | + | return peer | |
| 168 | + | } | |
| 169 | + | ||
| 170 | + | // serve answers each client message with p.flood Pong frames, triggering the | |
| 171 | + | // server preamble on the first call. | |
| 172 | + | func (p *sessionPeer) serve() { | |
| 173 | + | for { | |
| 174 | + | message, err := p.srv.ReceiveClient() | |
| 175 | + | if err != nil { | |
| 176 | + | return | |
| 177 | + | } | |
| 178 | + | switch message.(type) { | |
| 179 | + | case protocol.Ping, protocol.Input, protocol.Resize, protocol.Detach: | |
| 180 | + | for i := 0; i < p.flood; i++ { | |
| 181 | + | if err := p.srv.SendServer(protocol.Pong{}); err != nil { | |
| 182 | + | return | |
| 183 | + | } | |
| 184 | + | } | |
| 185 | + | } | |
| 186 | + | } | |
| 187 | + | } | |
| 188 | + | ||
| 189 | + | func (p *sessionPeer) close() { | |
| 190 | + | if p.raw != nil { | |
| 191 | + | _ = p.raw.Close() | |
| 192 | + | } | |
| 193 | + | } | |
| 194 | + | ||
| 195 | + | // daemonAcceptor accepts mux opens on the daemon side: it confirms each one | |
| 196 | + | // with Opened (or refuses it), and optionally wraps it as a typed session | |
| 197 | + | // peer. It never uses testing.T from its goroutines. | |
| 198 | + | type daemonAcceptor struct { | |
| 199 | + | pump *Pump | |
| 200 | + | flood int | |
| 201 | + | ||
| 202 | + | mu sync.Mutex | |
| 203 | + | count int | |
| 204 | + | peers map[PhysicalStreamID]*sessionPeer | |
| 205 | + | errCh chan error | |
| 206 | + | } | |
| 207 | + | ||
| 208 | + | func newDaemonAcceptor(pump *Pump, flood int) *daemonAcceptor { | |
| 209 | + | return &daemonAcceptor{pump: pump, flood: flood, peers: map[PhysicalStreamID]*sessionPeer{}, errCh: make(chan error, 256)} | |
| 210 | + | } | |
| 211 | + | ||
| 212 | + | func (a *daemonAcceptor) start(id PhysicalStreamID, deadline time.Time) { | |
| 213 | + | go func() { | |
| 214 | + | status, err := awaitStream(a.pump, id, deadline) | |
| 215 | + | if err != nil { | |
| 216 | + | a.fail(err) | |
| 217 | + | return | |
| 218 | + | } | |
| 219 | + | if err := a.pump.Engine().Opened(Opened{Ref: status.Ref}); err != nil { | |
| 220 | + | a.fail(err) | |
| 221 | + | return | |
| 222 | + | } | |
| 223 | + | if err := a.pump.Send(Opened{Ref: status.Ref}); err != nil { | |
| 224 | + | a.fail(err) | |
| 225 | + | return | |
| 226 | + | } | |
| 227 | + | a.add(id, newSessionPeer(a.pump, status.Ref, a.flood)) | |
| 228 | + | }() | |
| 229 | + | } | |
| 230 | + | ||
| 231 | + | func (a *daemonAcceptor) startBare(id PhysicalStreamID, deadline time.Time) { | |
| 232 | + | go func() { | |
| 233 | + | status, err := awaitStream(a.pump, id, deadline) | |
| 234 | + | if err != nil { | |
| 235 | + | a.fail(err) | |
| 236 | + | return | |
| 237 | + | } | |
| 238 | + | if err := a.pump.Engine().Opened(Opened{Ref: status.Ref}); err != nil { | |
| 239 | + | a.fail(err) | |
| 240 | + | return | |
| 241 | + | } | |
| 242 | + | if err := a.pump.Send(Opened{Ref: status.Ref}); err != nil { | |
| 243 | + | a.fail(err) | |
| 244 | + | return | |
| 245 | + | } | |
| 246 | + | a.add(id, nil) | |
| 247 | + | }() | |
| 248 | + | } | |
| 249 | + | ||
| 250 | + | func (a *daemonAcceptor) startRefused(id PhysicalStreamID, deadline time.Time) { | |
| 251 | + | go func() { | |
| 252 | + | status, err := awaitStream(a.pump, id, deadline) | |
| 253 | + | if err != nil { | |
| 254 | + | a.fail(err) | |
| 255 | + | return | |
| 256 | + | } | |
| 257 | + | refusal := ErrorDetail{Code: ports.BrokerErrorUnavailable} | |
| 258 | + | if err := a.pump.Send(Refused{Ref: status.Ref, Error: refusal}); err != nil { | |
| 259 | + | a.fail(err) | |
| 260 | + | return | |
| 261 | + | } | |
| 262 | + | if err := a.pump.Engine().Refused(Refused{Ref: status.Ref, Error: refusal}); err != nil { | |
| 263 | + | a.fail(err) | |
| 264 | + | return | |
| 265 | + | } | |
| 266 | + | a.add(id, nil) | |
| 267 | + | }() | |
| 268 | + | } | |
| 269 | + | ||
| 270 | + | func (a *daemonAcceptor) add(id PhysicalStreamID, peer *sessionPeer) { | |
| 271 | + | a.mu.Lock() | |
| 272 | + | defer a.mu.Unlock() | |
| 273 | + | a.count++ | |
| 274 | + | if peer != nil { | |
| 275 | + | a.peers[id] = peer | |
| 276 | + | } | |
| 277 | + | } | |
| 278 | + | ||
| 279 | + | func (a *daemonAcceptor) fail(err error) { | |
| 280 | + | select { | |
| 281 | + | case a.errCh <- err: | |
| 282 | + | default: | |
| 283 | + | } | |
| 284 | + | } | |
| 285 | + | ||
| 286 | + | func (a *daemonAcceptor) wait(count int, deadline time.Time) error { | |
| 287 | + | for time.Now().Before(deadline) { | |
| 288 | + | select { | |
| 289 | + | case err := <-a.errCh: | |
| 290 | + | return err | |
| 291 | + | default: | |
| 292 | + | } | |
| 293 | + | a.mu.Lock() | |
| 294 | + | accepted := a.count | |
| 295 | + | a.mu.Unlock() | |
| 296 | + | if accepted >= count { | |
| 297 | + | return nil | |
| 298 | + | } | |
| 299 | + | time.Sleep(time.Millisecond) | |
| 300 | + | } | |
| 301 | + | a.mu.Lock() | |
| 302 | + | accepted := a.count | |
| 303 | + | a.mu.Unlock() | |
| 304 | + | select { | |
| 305 | + | case err := <-a.errCh: | |
| 306 | + | return err | |
| 307 | + | default: | |
| 308 | + | } | |
| 309 | + | return fmt.Errorf("daemonmux test: accepted %d of %d streams", accepted, count) | |
| 310 | + | } | |
| 311 | + | ||
| 312 | + | func (a *daemonAcceptor) closePeers() { | |
| 313 | + | a.mu.Lock() | |
| 314 | + | defer a.mu.Unlock() | |
| 315 | + | for _, peer := range a.peers { | |
| 316 | + | peer.close() | |
| 317 | + | } | |
| 318 | + | } | |
| 319 | + | ||
| 320 | + | func openWithDeadline(t *testing.T, connector *LogicalConnector, stream int) *LogicalConnection { | |
| 321 | + | t.Helper() | |
| 322 | + | ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | |
| 323 | + | defer cancel() | |
| 324 | + | connection, err := connector.Open(ctx, controlRequest(stream)) | |
| 325 | + | require.NoError(t, err) | |
| 326 | + | return connection | |
| 327 | + | } | |
| 328 | + | ||
| 329 | + | // watchRegistered reports whether the pump still holds an inbound watch entry | |
| 330 | + | // for one physical stream. | |
| 331 | + | func watchRegistered(pump *Pump, physical PhysicalStreamID) bool { | |
| 332 | + | pump.watchMu.Lock() | |
| 333 | + | defer pump.watchMu.Unlock() | |
| 334 | + | _, ok := pump.watch[physical] | |
| 335 | + | return ok | |
| 336 | + | } | |
| 337 | + | ||
| 338 | + | func TestNewLogicalConnectorValidation(t *testing.T) { | |
| 339 | + | t.Run("nil pump", func(t *testing.T) { | |
| 340 | + | connector, err := NewLogicalConnector(nil) | |
| 341 | + | require.ErrorIs(t, err, ErrLogicalConfig) | |
| 342 | + | require.Nil(t, connector) | |
| 343 | + | }) | |
| 344 | + | t.Run("server-side pump is refused", func(t *testing.T) { | |
| 345 | + | pump, _ := newTestPump(t, DirectionClient) | |
| 346 | + | connector, err := NewLogicalConnector(pump) | |
| 347 | + | require.ErrorIs(t, err, ErrLogicalConfig) | |
| 348 | + | require.Nil(t, connector) | |
| 349 | + | }) | |
| 350 | + | t.Run("broker-side pump is accepted", func(t *testing.T) { | |
| 351 | + | pump, _ := newTestPump(t, DirectionServer) | |
| 352 | + | connector, err := NewLogicalConnector(pump) | |
| 353 | + | require.NoError(t, err) | |
| 354 | + | require.NotNil(t, connector) | |
| 355 | + | }) | |
| 356 | + | } | |
| 357 | + | ||
| 358 | + | // TestMuxStreamPipeChunkAndOrder proves the byte stream chunks outbound writes | |
| 359 | + | // at the negotiated ceiling in order and drains inbound chunks in order. | |
| 360 | + | func TestMuxStreamPipeChunkAndOrder(t *testing.T) { | |
| 361 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 362 | + | pump.Start(context.Background()) | |
| 363 | + | require.NoError(t, pump.Engine().Open(openFor(1))) | |
| 364 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 365 | + | ||
| 366 | + | pipe := newMuxStreamPipe(pump, 1, 4, mustStatus(t, pump.Engine(), 1).Done) | |
| 367 | + | written, err := pipe.Write([]byte("0123456789")) | |
| 368 | + | require.NoError(t, err) | |
| 369 | + | require.Equal(t, 10, written) | |
| 370 | + | ||
| 371 | + | var chunks [][]byte | |
| 372 | + | for i := 0; i < 3; i++ { | |
| 373 | + | message, err := DecodeServer(carrier.nextSent(t), testEnvelopeCeiling, testChunkCeiling) | |
| 374 | + | require.NoError(t, err) | |
| 375 | + | data, ok := message.(Data) | |
| 376 | + | require.True(t, ok) | |
| 377 | + | require.LessOrEqual(t, uint64(len(data.Data)), uint64(4)) | |
| 378 | + | chunks = append(chunks, data.Data) | |
| 379 | + | } | |
| 380 | + | require.Equal(t, [][]byte{[]byte("0123"), []byte("4567"), []byte("89")}, chunks) | |
| 381 | + | ||
| 382 | + | deliverClientFrame(t, carrier, Data{Physical: 1, Data: []byte("abc")}) | |
| 383 | + | deliverClientFrame(t, carrier, Data{Physical: 1, Data: []byte("de")}) | |
| 384 | + | buffer := make([]byte, 3) | |
| 385 | + | _, err = io.ReadFull(pipe, buffer) | |
| 386 | + | require.NoError(t, err) | |
| 387 | + | require.Equal(t, []byte("abc"), buffer) | |
| 388 | + | buffer = make([]byte, 2) | |
| 389 | + | _, err = io.ReadFull(pipe, buffer) | |
| 390 | + | require.NoError(t, err) | |
| 391 | + | require.Equal(t, []byte("de"), buffer) | |
| 392 | + | } | |
| 393 | + | ||
| 394 | + | // TestLogicalTwoConnections opens two typed logical connections over one | |
| 395 | + | // physical pair, exchanges a typed round trip on each, and closes them | |
| 396 | + | // independently. | |
| 397 | + | func TestLogicalTwoConnections(t *testing.T) { | |
| 398 | + | const count = 2 | |
| 399 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 400 | + | connector := mustConnector(t, brokerPump) | |
| 401 | + | deadline := time.Now().Add(30 * time.Second) | |
| 402 | + | ||
| 403 | + | acceptor := newDaemonAcceptor(daemonPump, 1) | |
| 404 | + | for i := 1; i <= count; i++ { | |
| 405 | + | acceptor.start(PhysicalStreamID(i), deadline) | |
| 406 | + | } | |
| 407 | + | ||
| 408 | + | connections := make([]*LogicalConnection, 0, count) | |
| 409 | + | for i := 1; i <= count; i++ { | |
| 410 | + | connection := openWithDeadline(t, connector, i) | |
| 411 | + | require.NoError(t, connection.SendClient(protocol.Ping{}), "connection %d", i) | |
| 412 | + | message, err := connection.ReceiveServer() | |
| 413 | + | require.NoError(t, err, "connection %d", i) | |
| 414 | + | require.Equal(t, protocol.Pong{}, message, "connection %d", i) | |
| 415 | + | connections = append(connections, connection) | |
| 416 | + | } | |
| 417 | + | require.NoError(t, acceptor.wait(count, deadline)) | |
| 418 | + | ||
| 419 | + | for i, connection := range connections { | |
| 420 | + | require.False(t, channelClosed(connection.Done()), "connection %d", i) | |
| 421 | + | require.NoError(t, connection.Close(), "connection %d", i) | |
| 422 | + | require.True(t, channelClosed(connection.Done()), "connection %d", i) | |
| 423 | + | require.NoError(t, connection.Err(), "connection %d", i) | |
| 424 | + | } | |
| 425 | + | acceptor.closePeers() | |
| 426 | + | require.False(t, channelClosed(brokerPump.Done())) | |
| 427 | + | require.False(t, channelClosed(daemonPump.Done())) | |
| 428 | + | } | |
| 429 | + | ||
| 430 | + | // TestLogicalHundredConnectionsConcurrentAdmission opens 100 typed logical | |
| 431 | + | // connections concurrently over one physical pair, proves all 100 are live | |
| 432 | + | // before any traffic, interleaves a typed round trip on every stream, and then | |
| 433 | + | // drains and closes them. It fills the physical connection to its 128-stream | |
| 434 | + | // ceiling, proves the 129th stream is refused with the typed admission limit, | |
| 435 | + | // and proves a retry after a Close succeeds with a fresh monotonic identity. | |
| 436 | + | func TestLogicalHundredConnectionsConcurrentAdmission(t *testing.T) { | |
| 437 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 438 | + | connector := mustConnector(t, brokerPump) | |
| 439 | + | deadline := time.Now().Add(60 * time.Second) | |
| 440 | + | ||
| 441 | + | const count = 100 | |
| 442 | + | const ceiling = int(MaxMuxStreams) | |
| 443 | + | require.Equal(t, 128, ceiling, "the default physical connection admits 128 streams") | |
| 444 | + | ||
| 445 | + | acceptor := newDaemonAcceptor(daemonPump, 1) | |
| 446 | + | // Every physical identity the connector can allocate, plus the retry after | |
| 447 | + | // the refused 129th attempt (which still consumes its monotonic identity). | |
| 448 | + | for i := 1; i <= ceiling; i++ { | |
| 449 | + | acceptor.start(PhysicalStreamID(i), deadline) | |
| 450 | + | } | |
| 451 | + | acceptor.start(PhysicalStreamID(ceiling+2), deadline) | |
| 452 | + | ||
| 453 | + | openConcurrently := func(requests []ports.BrokerOpenStreamRequest) ([]*LogicalConnection, []error) { | |
| 454 | + | connections := make([]*LogicalConnection, len(requests)) | |
| 455 | + | errs := make([]error, len(requests)) | |
| 456 | + | var wg sync.WaitGroup | |
| 457 | + | for i, request := range requests { | |
| 458 | + | wg.Add(1) | |
| 459 | + | go func(index int, request ports.BrokerOpenStreamRequest) { | |
| 460 | + | defer wg.Done() | |
| 461 | + | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) | |
| 462 | + | defer cancel() | |
| 463 | + | connections[index], errs[index] = connector.Open(ctx, request) | |
| 464 | + | }(i, request) | |
| 465 | + | } | |
| 466 | + | wg.Wait() | |
| 467 | + | return connections, errs | |
| 468 | + | } | |
| 469 | + | ||
| 470 | + | requests := make([]ports.BrokerOpenStreamRequest, count) | |
| 471 | + | for i := range requests { | |
| 472 | + | requests[i] = controlRequest(i + 1) | |
| 473 | + | } | |
| 474 | + | connections, errs := openConcurrently(requests) | |
| 475 | + | for i, err := range errs { | |
| 476 | + | require.NoError(t, err, "connection %d", i) | |
| 477 | + | require.NotNil(t, connections[i], "connection %d", i) | |
| 478 | + | } | |
| 479 | + | require.Equal(t, count, brokerPump.Engine().Live(), "all connections are open before any traffic") | |
| 480 | + | require.NoError(t, acceptor.wait(count, deadline)) | |
| 481 | + | ||
| 482 | + | // Interleave a typed round trip across every stream, then drain it. | |
| 483 | + | trafficErrs := make([]error, count) | |
| 484 | + | var traffic sync.WaitGroup | |
| 485 | + | for i := range connections { | |
| 486 | + | traffic.Add(1) | |
| 487 | + | go func(index int) { | |
| 488 | + | defer traffic.Done() | |
| 489 | + | if err := connections[index].SendClient(protocol.Ping{}); err != nil { | |
| 490 | + | trafficErrs[index] = err | |
| 491 | + | return | |
| 492 | + | } | |
| 493 | + | message, err := connections[index].ReceiveServer() | |
| 494 | + | if err != nil { | |
| 495 | + | trafficErrs[index] = err | |
| 496 | + | return | |
| 497 | + | } | |
| 498 | + | if message != (protocol.Pong{}) { | |
| 499 | + | trafficErrs[index] = fmt.Errorf("connection %d: received %v", index, message) | |
| 500 | + | } | |
| 501 | + | }(i) | |
| 502 | + | } | |
| 503 | + | traffic.Wait() | |
| 504 | + | for i, err := range trafficErrs { | |
| 505 | + | require.NoError(t, err, "connection %d", i) | |
| 506 | + | } | |
| 507 | + | require.Equal(t, count, brokerPump.Engine().Live(), "traffic never settles a live stream") | |
| 508 | + | ||
| 509 | + | // Fill the physical connection to its concurrent-stream ceiling. | |
| 510 | + | fill := make([]ports.BrokerOpenStreamRequest, ceiling-count) | |
| 511 | + | for i := range fill { | |
| 512 | + | fill[i] = controlRequest(count + i + 1) | |
| 513 | + | } | |
| 514 | + | extra, fillErrs := openConcurrently(fill) | |
| 515 | + | for i, err := range fillErrs { | |
| 516 | + | require.NoError(t, err, "extra connection %d", i) | |
| 517 | + | } | |
| 518 | + | require.Equal(t, ceiling, brokerPump.Engine().Live()) | |
| 519 | + | require.NoError(t, acceptor.wait(ceiling, deadline)) | |
| 520 | + | ||
| 521 | + | // The 129th stream is refused by the admission ceiling before it reaches | |
| 522 | + | // the daemon. | |
| 523 | + | ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | |
| 524 | + | defer cancel() | |
| 525 | + | refused, err := connector.Open(ctx, controlRequest(1000)) | |
| 526 | + | require.ErrorIs(t, err, ports.BrokerAdmissionLimit) | |
| 527 | + | require.Nil(t, refused) | |
| 528 | + | require.Equal(t, ceiling, brokerPump.Engine().Live(), "a refused admission frees nothing") | |
| 529 | + | ||
| 530 | + | // A Close frees one slot; the retry uses a fresh monotonic identity. | |
| 531 | + | require.NoError(t, connections[0].Close()) | |
| 532 | + | require.True(t, channelClosed(connections[0].Done())) | |
| 533 | + | require.Equal(t, ceiling-1, brokerPump.Engine().Live()) | |
| 534 | + | retry := openWithDeadline(t, connector, 1001) | |
| 535 | + | require.NoError(t, retry.SendClient(protocol.Ping{})) | |
| 536 | + | message, err := retry.ReceiveServer() | |
| 537 | + | require.NoError(t, err) | |
| 538 | + | require.Equal(t, protocol.Pong{}, message) | |
| 539 | + | require.Equal(t, ceiling, brokerPump.Engine().Live()) | |
| 540 | + | require.NoError(t, acceptor.wait(ceiling+1, deadline)) | |
| 541 | + | ||
| 542 | + | for i := 1; i < len(connections); i++ { | |
| 543 | + | require.False(t, channelClosed(connections[i].Done()), "connection %d", i) | |
| 544 | + | require.NoError(t, connections[i].Close(), "connection %d", i) | |
| 545 | + | require.True(t, channelClosed(connections[i].Done()), "connection %d", i) | |
| 546 | + | require.NoError(t, connections[i].Err(), "connection %d", i) | |
| 547 | + | } | |
| 548 | + | for i := range extra { | |
| 549 | + | require.NoError(t, extra[i].Close(), "extra connection %d", i) | |
| 550 | + | } | |
| 551 | + | require.NoError(t, retry.Close()) | |
| 552 | + | require.Zero(t, brokerPump.Engine().Live(), "every stream is retired") | |
| 553 | + | acceptor.closePeers() | |
| 554 | + | require.False(t, channelClosed(brokerPump.Done())) | |
| 555 | + | require.False(t, channelClosed(daemonPump.Done())) | |
| 556 | + | } | |
| 557 | + | ||
| 558 | + | // TestLogicalBlockedReaderSiblingProgresses proves a logical connection whose | |
| 559 | + | // consumer never reads lets only its own bounded stream reset while a sibling | |
| 560 | + | // keeps exchanging typed messages and the physical connection stays open. | |
| 561 | + | func TestLogicalBlockedReaderSiblingProgresses(t *testing.T) { | |
| 562 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 563 | + | connector := mustConnector(t, brokerPump) | |
| 564 | + | deadline := time.Now().Add(15 * time.Second) | |
| 565 | + | ||
| 566 | + | blockedAcceptor := newDaemonAcceptor(daemonPump, MaxMuxStreamQueueChunks+4) | |
| 567 | + | blockedAcceptor.start(1, deadline) | |
| 568 | + | siblingAcceptor := newDaemonAcceptor(daemonPump, 1) | |
| 569 | + | siblingAcceptor.start(2, deadline) | |
| 570 | + | ||
| 571 | + | blocked := openWithDeadline(t, connector, 1) | |
| 572 | + | sibling := openWithDeadline(t, connector, 2) | |
| 573 | + | require.NoError(t, blockedAcceptor.wait(1, deadline)) | |
| 574 | + | require.NoError(t, siblingAcceptor.wait(1, deadline)) | |
| 575 | + | ||
| 576 | + | require.NoError(t, sibling.SendClient(protocol.Ping{})) | |
| 577 | + | message, err := sibling.ReceiveServer() | |
| 578 | + | require.NoError(t, err) | |
| 579 | + | require.Equal(t, protocol.Pong{}, message) | |
| 580 | + | ||
| 581 | + | // Run the blocked stream's handshake, then never read it; the daemon floods | |
| 582 | + | // Pong frames until the blocked stream's own bounded queue overflows. | |
| 583 | + | require.NoError(t, blocked.SendClient(protocol.Ping{})) | |
| 584 | + | require.Eventually(t, func() bool { return channelClosed(blocked.Done()) }, 5*time.Second, time.Millisecond) | |
| 585 | + | require.Error(t, blocked.Err()) | |
| 586 | + | ||
| 587 | + | // The sibling still progresses and the physical connection is untouched. | |
| 588 | + | require.NoError(t, sibling.SendClient(protocol.Ping{})) | |
| 589 | + | message, err = sibling.ReceiveServer() | |
| 590 | + | require.NoError(t, err) | |
| 591 | + | require.Equal(t, protocol.Pong{}, message) | |
| 592 | + | require.False(t, channelClosed(brokerPump.Done())) | |
| 593 | + | require.Equal(t, StreamOpen, mustStatus(t, brokerPump.Engine(), 2).State) | |
| 594 | + | } | |
| 595 | + | ||
| 596 | + | // TestLogicalMalformedInnerFrameResetsOnlyStream proves a malformed inner | |
| 597 | + | // session frame resets exactly its own stream and leaves the physical | |
| 598 | + | // connection and a sibling untouched. | |
| 599 | + | func TestLogicalMalformedInnerFrameResetsOnlyStream(t *testing.T) { | |
| 600 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 601 | + | connector := mustConnector(t, brokerPump) | |
| 602 | + | deadline := time.Now().Add(15 * time.Second) | |
| 603 | + | ||
| 604 | + | bareAcceptor := newDaemonAcceptor(daemonPump, 1) | |
| 605 | + | bareAcceptor.startBare(1, deadline) | |
| 606 | + | siblingAcceptor := newDaemonAcceptor(daemonPump, 1) | |
| 607 | + | siblingAcceptor.start(2, deadline) | |
| 608 | + | ||
| 609 | + | malformed := openWithDeadline(t, connector, 1) | |
| 610 | + | sibling := openWithDeadline(t, connector, 2) | |
| 611 | + | require.NoError(t, bareAcceptor.wait(1, deadline)) | |
| 612 | + | require.NoError(t, siblingAcceptor.wait(1, deadline)) | |
| 613 | + | ||
| 614 | + | require.NoError(t, sibling.SendClient(protocol.Ping{})) | |
| 615 | + | message, err := sibling.ReceiveServer() | |
| 616 | + | require.NoError(t, err) | |
| 617 | + | require.Equal(t, protocol.Pong{}, message) | |
| 618 | + | ||
| 619 | + | // A zero-length inner frame is malformed: the client's first read fails and | |
| 620 | + | // must reset only this stream. | |
| 621 | + | require.NoError(t, daemonPump.Send(Data{Physical: 1, Data: []byte{0, 0, 0, 0}})) | |
| 622 | + | _, err = malformed.ReceiveServer() | |
| 623 | + | require.Error(t, err) | |
| 624 | + | require.True(t, channelClosed(malformed.Done())) | |
| 625 | + | ||
| 626 | + | var loss ports.BrokerStreamLost | |
| 627 | + | require.ErrorAs(t, malformed.Err(), &loss) | |
| 628 | + | require.Equal(t, ports.BrokerStreamID(1), loss.Stream) | |
| 629 | + | require.Equal(t, ports.BrokerConnectionID{1}, loss.Connection) | |
| 630 | + | require.Equal(t, ports.BrokerEpoch(1), loss.Epoch) | |
| 631 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, loss.Cause) | |
| 632 | + | ||
| 633 | + | // The daemon observed exactly a stream-local reset. | |
| 634 | + | require.Eventually(t, func() bool { | |
| 635 | + | status, ok := daemonPump.Engine().Status(1) | |
| 636 | + | return ok && status.State == StreamTerminal | |
| 637 | + | }, 5*time.Second, time.Millisecond) | |
| 638 | + | reset := mustStatus(t, daemonPump.Engine(), 1) | |
| 639 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, reset.FailureKind) | |
| 640 | + | ||
| 641 | + | // The sibling still progresses and the physical pumps stay open. | |
| 642 | + | require.NoError(t, sibling.SendClient(protocol.Ping{})) | |
| 643 | + | message, err = sibling.ReceiveServer() | |
| 644 | + | require.NoError(t, err) | |
| 645 | + | require.Equal(t, protocol.Pong{}, message) | |
| 646 | + | require.False(t, channelClosed(brokerPump.Done())) | |
| 647 | + | require.False(t, channelClosed(daemonPump.Done())) | |
| 648 | + | } | |
| 649 | + | ||
| 650 | + | // TestLogicalOpenCancellationReleasesReservation proves a cancelled open | |
| 651 | + | // releases its local reservation and a later open still succeeds. | |
| 652 | + | func TestLogicalOpenCancellationReleasesReservation(t *testing.T) { | |
| 653 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 654 | + | connector := mustConnector(t, brokerPump) | |
| 655 | + | deadline := time.Now().Add(15 * time.Second) | |
| 656 | + | ||
| 657 | + | // No daemon acceptor: the open waits for an Opened that never arrives. | |
| 658 | + | ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) | |
| 659 | + | defer cancel() | |
| 660 | + | connection, err := connector.Open(ctx, controlRequest(1)) | |
| 661 | + | require.ErrorIs(t, err, context.DeadlineExceeded) | |
| 662 | + | require.Nil(t, connection) | |
| 663 | + | ||
| 664 | + | require.Eventually(t, func() bool { | |
| 665 | + | status, ok := brokerPump.Engine().Status(1) | |
| 666 | + | return ok && status.State == StreamTerminal | |
| 667 | + | }, 5*time.Second, time.Millisecond) | |
| 668 | + | require.False(t, watchRegistered(brokerPump, 1), "cancelled open leaked a watch entry") | |
| 669 | + | ||
| 670 | + | // A sibling stream on the same physical connection still opens and works. | |
| 671 | + | acceptor := newDaemonAcceptor(daemonPump, 1) | |
| 672 | + | acceptor.start(2, deadline) | |
| 673 | + | sibling := openWithDeadline(t, connector, 2) | |
| 674 | + | require.NoError(t, acceptor.wait(1, deadline)) | |
| 675 | + | require.NoError(t, sibling.SendClient(protocol.Ping{})) | |
| 676 | + | message, err := sibling.ReceiveServer() | |
| 677 | + | require.NoError(t, err) | |
| 678 | + | require.Equal(t, protocol.Pong{}, message) | |
| 679 | + | require.False(t, channelClosed(brokerPump.Done())) | |
| 680 | + | } | |
| 681 | + | ||
| 682 | + | // TestLogicalOpenRefused proves a daemon refusal fails the open with the typed | |
| 683 | + | // refusal and leaves the physical connection open. | |
| 684 | + | func TestLogicalOpenRefused(t *testing.T) { | |
| 685 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 686 | + | connector := mustConnector(t, brokerPump) | |
| 687 | + | deadline := time.Now().Add(15 * time.Second) | |
| 688 | + | ||
| 689 | + | acceptor := newDaemonAcceptor(daemonPump, 1) | |
| 690 | + | acceptor.startRefused(1, deadline) | |
| 691 | + | ||
| 692 | + | ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | |
| 693 | + | defer cancel() | |
| 694 | + | connection, err := connector.Open(ctx, controlRequest(1)) | |
| 695 | + | require.Error(t, err) | |
| 696 | + | require.Nil(t, connection) | |
| 697 | + | require.NoError(t, acceptor.wait(1, deadline)) | |
| 698 | + | require.False(t, channelClosed(brokerPump.Done())) | |
| 699 | + | } | |
| 700 | + | ||
| 701 | + | // TestLogicalCloseIsolation proves closing one logical connection settles only | |
| 702 | + | // itself and leaves a sibling and the physical connection untouched. | |
| 703 | + | func TestLogicalCloseIsolation(t *testing.T) { | |
| 704 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 705 | + | connector := mustConnector(t, brokerPump) | |
| 706 | + | deadline := time.Now().Add(15 * time.Second) | |
| 707 | + | ||
| 708 | + | firstAcceptor := newDaemonAcceptor(daemonPump, 1) | |
| 709 | + | firstAcceptor.start(1, deadline) | |
| 710 | + | secondAcceptor := newDaemonAcceptor(daemonPump, 1) | |
| 711 | + | secondAcceptor.start(2, deadline) | |
| 712 | + | ||
| 713 | + | first := openWithDeadline(t, connector, 1) | |
| 714 | + | second := openWithDeadline(t, connector, 2) | |
| 715 | + | require.NoError(t, firstAcceptor.wait(1, deadline)) | |
| 716 | + | require.NoError(t, secondAcceptor.wait(1, deadline)) | |
| 717 | + | ||
| 718 | + | require.NoError(t, first.Close()) | |
| 719 | + | require.True(t, channelClosed(first.Done())) | |
| 720 | + | require.NoError(t, first.Err()) | |
| 721 | + | require.Equal(t, domain.RemoteFailureNone, first.FailureKind()) | |
| 722 | + | require.False(t, watchRegistered(brokerPump, 1), "closed logical connection leaked a watch entry") | |
| 723 | + | ||
| 724 | + | // The daemon observed the orderly Close for stream 1. | |
| 725 | + | require.Eventually(t, func() bool { | |
| 726 | + | status, ok := daemonPump.Engine().Status(1) | |
| 727 | + | return ok && status.State == StreamTerminal && status.Err == nil | |
| 728 | + | }, 5*time.Second, time.Millisecond) | |
| 729 | + | ||
| 730 | + | require.NoError(t, second.SendClient(protocol.Ping{})) | |
| 731 | + | message, err := second.ReceiveServer() | |
| 732 | + | require.NoError(t, err) | |
| 733 | + | require.Equal(t, protocol.Pong{}, message) | |
| 734 | + | require.False(t, channelClosed(brokerPump.Done())) | |
| 735 | + | require.False(t, channelClosed(daemonPump.Done())) | |
| 736 | + | } | |
| 737 | + | ||
| 738 | + | // TestLogicalCloseDiscardsQueuedInboundReleasesAccounting proves a local Close | |
| 739 | + | // first discards the inbound chunks the engine already accepted for its stream: | |
| 740 | + | // the engine's orderly Close reaches its terminal state, releasing both the | |
| 741 | + | // stream's admission slot and its queued aggregate bytes, instead of parking | |
| 742 | + | // the stream in closing with a queue this side will never read again. Every | |
| 743 | + | // cycle closes with one queued inbound chunk under a two-stream ceiling, so a | |
| 744 | + | // leaked slot would refuse the third admission, and the peer still observes | |
| 745 | + | // exactly one orderly mux Close for each cycle. | |
| 746 | + | func TestLogicalCloseDiscardsQueuedInboundReleasesAccounting(t *testing.T) { | |
| 747 | + | ceilings := DefaultMuxCeilings() | |
| 748 | + | ceilings.MaxStreams = 2 | |
| 749 | + | pump, carrier := newTestPumpWithCeilings(t, DirectionServer, ceilings) | |
| 750 | + | pump.Start(context.Background()) | |
| 751 | + | ||
| 752 | + | chunk := []byte("queued") | |
| 753 | + | const cycles = 3 | |
| 754 | + | for i := 1; i <= cycles; i++ { | |
| 755 | + | id := PhysicalStreamID(i) | |
| 756 | + | require.NoError(t, pump.Engine().Open(openFor(id)), "cycle %d admission", i) | |
| 757 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(id)})) | |
| 758 | + | status := mustStatus(t, pump.Engine(), id) | |
| 759 | + | connection := newLogicalConnection(pump, status.Ref, status.Done, pump.cause(id)) | |
| 760 | + | ||
| 761 | + | // The peer pipelined one chunk that this side never reads, so the stream | |
| 762 | + | // holds accepted inbound data when the local Close arrives. | |
| 763 | + | deliverServerFrame(t, carrier, Data{Physical: id, Data: chunk}) | |
| 764 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.AggregateBytes() > 0 }, | |
| 765 | + | "cycle %d: the accepted chunk was never queued", i) | |
| 766 | + | ||
| 767 | + | require.NoError(t, connection.Close()) | |
| 768 | + | require.NoError(t, connection.Close(), "Close is idempotent") | |
| 769 | + | require.True(t, channelClosed(connection.Done())) | |
| 770 | + | require.NoError(t, connection.Err()) | |
| 771 | + | ||
| 772 | + | closed := mustStatus(t, pump.Engine(), id) | |
| 773 | + | require.Equal(t, StreamTerminal, closed.State, "cycle %d: the closed stream must be terminal", i) | |
| 774 | + | require.Zero(t, closed.QueuedChunks, "cycle %d", i) | |
| 775 | + | require.Zero(t, pump.Engine().Live(), "cycle %d: a closing stream leaked its admission slot", i) | |
| 776 | + | require.Zero(t, pump.Engine().AggregateBytes(), "cycle %d: a closing stream leaked its queued bytes", i) | |
| 777 | + | require.False(t, watchRegistered(pump, id), "cycle %d: the closed connection leaked a watch entry", i) | |
| 778 | + | ||
| 779 | + | // Exactly one orderly mux Close reached the peer; the drain is local and | |
| 780 | + | // never replaces it with a Reset. | |
| 781 | + | message, err := DecodeClient(carrier.nextSent(t), testEnvelopeCeiling, testChunkCeiling) | |
| 782 | + | require.NoError(t, err) | |
| 783 | + | closedFrame, ok := message.(Close) | |
| 784 | + | require.True(t, ok, "cycle %d: the peer frame is not a Close", i) | |
| 785 | + | require.Equal(t, Close{Physical: id}, closedFrame) | |
| 786 | + | carrier.requireNoSent(t, 20*time.Millisecond) | |
| 787 | + | } | |
| 788 | + | require.False(t, pump.Engine().Closed()) | |
| 789 | + | require.False(t, channelClosed(pump.Done())) | |
| 790 | + | } | |
| 791 | + | ||
| 792 | + | // TestLogicalPhysicalLossFanout proves a physical loss publishes one stable | |
| 793 | + | // ports.BrokerStreamLost per affected logical stream with its exact scope. | |
| 794 | + | func TestLogicalPhysicalLossFanout(t *testing.T) { | |
| 795 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 796 | + | connector := mustConnector(t, brokerPump) | |
| 797 | + | deadline := time.Now().Add(15 * time.Second) | |
| 798 | + | ||
| 799 | + | const count = 3 | |
| 800 | + | acceptor := newDaemonAcceptor(daemonPump, 1) | |
| 801 | + | for i := 1; i <= count; i++ { | |
| 802 | + | acceptor.start(PhysicalStreamID(i), deadline) | |
| 803 | + | } | |
| 804 | + | connections := make([]*LogicalConnection, 0, count) | |
| 805 | + | for i := 1; i <= count; i++ { | |
| 806 | + | connection := openWithDeadline(t, connector, i) | |
| 807 | + | require.NoError(t, connection.SendClient(protocol.Ping{})) | |
| 808 | + | message, err := connection.ReceiveServer() | |
| 809 | + | require.NoError(t, err) | |
| 810 | + | require.Equal(t, protocol.Pong{}, message) | |
| 811 | + | connections = append(connections, connection) | |
| 812 | + | } | |
| 813 | + | require.NoError(t, acceptor.wait(count, deadline)) | |
| 814 | + | ||
| 815 | + | // Failing the daemon-side carrier is a physical loss on the broker pump. | |
| 816 | + | require.NoError(t, daemonPump.Close()) | |
| 817 | + | require.Eventually(t, func() bool { return channelClosed(brokerPump.Done()) }, 5*time.Second, time.Millisecond) | |
| 818 | + | ||
| 819 | + | for i, connection := range connections { | |
| 820 | + | require.Eventually(t, func() bool { return channelClosed(connection.Done()) }, 5*time.Second, time.Millisecond) | |
| 821 | + | var loss ports.BrokerStreamLost | |
| 822 | + | require.ErrorAs(t, connection.Err(), &loss, "connection %d", i) | |
| 823 | + | require.Equal(t, ports.BrokerStreamID(i+1), loss.Stream, "connection %d", i) | |
| 824 | + | require.Equal(t, ports.BrokerConnectionID{1}, loss.Connection, "connection %d", i) | |
| 825 | + | require.Equal(t, domain.RemoteFailureTransport, loss.Cause, "connection %d", i) | |
| 826 | + | var boxed ports.BrokerError | |
| 827 | + | require.ErrorAs(t, connection.Err(), &boxed, "connection %d", i) | |
| 828 | + | require.Equal(t, ports.BrokerErrorAttachmentLost, boxed.Code, "connection %d", i) | |
| 829 | + | } | |
| 830 | + | require.Error(t, brokerPump.Err()) | |
| 831 | + | require.Equal(t, domain.RemoteFailureTransport, brokerPump.FailureKind()) | |
| 832 | + | } | |
| 833 | + | ||
| 834 | + | // TestLogicalHandshakeDeadlinePlumbing proves the accepted absolute 15 s | |
| 835 | + | // handshake deadline and the completion hook are exposed without restarting a | |
| 836 | + | // deadline. | |
| 837 | + | func TestLogicalHandshakeDeadlinePlumbing(t *testing.T) { | |
| 838 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 839 | + | connector := mustConnector(t, brokerPump) | |
| 840 | + | deadline := time.Now().Add(15 * time.Second) | |
| 841 | + | ||
| 842 | + | acceptor := newDaemonAcceptor(daemonPump, 1) | |
| 843 | + | acceptor.start(1, deadline) | |
| 844 | + | ||
| 845 | + | connection := openWithDeadline(t, connector, 1) | |
| 846 | + | require.NoError(t, acceptor.wait(1, deadline)) | |
| 847 | + | ||
| 848 | + | accepted := connection.HandshakeDeadline() | |
| 849 | + | require.False(t, accepted.IsZero()) | |
| 850 | + | require.WithinDuration(t, time.Now().Add(protocol.HandshakeTimeout), accepted, 5*time.Second) | |
| 851 | + | require.Equal(t, accepted, connection.HandshakeDeadline(), "deadline is stable") | |
| 852 | + | ||
| 853 | + | completed := make(chan struct{}, 2) | |
| 854 | + | connection.OnHandshakeComplete(func() { completed <- struct{}{} }) | |
| 855 | + | select { | |
| 856 | + | case <-completed: | |
| 857 | + | t.Fatal("completion hook fired before the handshake ran") | |
| 858 | + | default: | |
| 859 | + | } | |
| 860 | + | ||
| 861 | + | require.NoError(t, connection.SendClient(protocol.Ping{})) | |
| 862 | + | message, err := connection.ReceiveServer() | |
| 863 | + | require.NoError(t, err) | |
| 864 | + | require.Equal(t, protocol.Pong{}, message) | |
| 865 | + | ||
| 866 | + | select { | |
| 867 | + | case <-completed: | |
| 868 | + | case <-time.After(5 * time.Second): | |
| 869 | + | t.Fatal("completion hook did not fire after the handshake") | |
| 870 | + | } | |
| 871 | + | ||
| 872 | + | // Registering after completion runs immediately. | |
| 873 | + | late := make(chan struct{}, 1) | |
| 874 | + | connection.OnHandshakeComplete(func() { late <- struct{}{} }) | |
| 875 | + | select { | |
| 876 | + | case <-late: | |
| 877 | + | case <-time.After(5 * time.Second): | |
| 878 | + | t.Fatal("late completion hook did not run") | |
| 879 | + | } | |
| 880 | + | } | |
| 881 | + | ||
| 882 | + | // TestLogicalNoGoroutineLeak proves opening, using, and closing logical | |
| 883 | + | // connections leaves no goroutine behind. | |
| 884 | + | func TestLogicalNoGoroutineLeak(t *testing.T) { | |
| 885 | + | before := runtime.NumGoroutine() | |
| 886 | + | ||
| 887 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 888 | + | connector := mustConnector(t, brokerPump) | |
| 889 | + | deadline := time.Now().Add(15 * time.Second) | |
| 890 | + | ||
| 891 | + | const count = 5 | |
| 892 | + | acceptor := newDaemonAcceptor(daemonPump, 1) | |
| 893 | + | for i := 1; i <= count; i++ { | |
| 894 | + | acceptor.start(PhysicalStreamID(i), deadline) | |
| 895 | + | } | |
| 896 | + | connections := make([]*LogicalConnection, 0, count) | |
| 897 | + | for i := 1; i <= count; i++ { | |
| 898 | + | connection := openWithDeadline(t, connector, i) | |
| 899 | + | require.NoError(t, connection.SendClient(protocol.Ping{})) | |
| 900 | + | message, err := connection.ReceiveServer() | |
| 901 | + | require.NoError(t, err) | |
| 902 | + | require.Equal(t, protocol.Pong{}, message) | |
| 903 | + | connections = append(connections, connection) | |
| 904 | + | } | |
| 905 | + | require.NoError(t, acceptor.wait(count, deadline)) | |
| 906 | + | ||
| 907 | + | for _, connection := range connections { | |
| 908 | + | require.NoError(t, connection.Close()) | |
| 909 | + | } | |
| 910 | + | acceptor.closePeers() | |
| 911 | + | require.NoError(t, brokerPump.Close()) | |
| 912 | + | require.NoError(t, daemonPump.Close()) | |
| 913 | + | ||
| 914 | + | require.Eventually(t, func() bool { return runtime.NumGoroutine() <= before+3 }, 5*time.Second, time.Millisecond, | |
| 915 | + | "logical connection goroutines leaked") | |
| 916 | + | } | |
| 917 | + | ||
| 918 | + | // TestLogicalPeerCloseReleasesGoroutines proves a stream terminated by a peer | |
| 919 | + | // mux Close joins its watcher and framed-transport goroutines without an | |
| 920 | + | // explicit local Close. | |
| 921 | + | func TestLogicalPeerCloseReleasesGoroutines(t *testing.T) { | |
| 922 | + | before := runtime.NumGoroutine() | |
| 923 | + | ||
| 924 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 925 | + | connector := mustConnector(t, brokerPump) | |
| 926 | + | deadline := time.Now().Add(15 * time.Second) | |
| 927 | + | ||
| 928 | + | acceptor := newDaemonAcceptor(daemonPump, 1) | |
| 929 | + | acceptor.start(1, deadline) | |
| 930 | + | connection := openWithDeadline(t, connector, 1) | |
| 931 | + | require.NoError(t, acceptor.wait(1, deadline)) | |
| 932 | + | require.NoError(t, connection.SendClient(protocol.Ping{})) | |
| 933 | + | message, err := connection.ReceiveServer() | |
| 934 | + | require.NoError(t, err) | |
| 935 | + | require.Equal(t, protocol.Pong{}, message) | |
| 936 | + | ||
| 937 | + | // The daemon ends the stream orderly; the logical connection must settle | |
| 938 | + | // itself and join its goroutines without a local Close. | |
| 939 | + | if _, err := daemonPump.Engine().Close(Close{Physical: 1}); err != nil { | |
| 940 | + | t.Fatalf("daemon close: %v", err) | |
| 941 | + | } | |
| 942 | + | require.NoError(t, daemonPump.Send(Close{Physical: 1})) | |
| 943 | + | require.Eventually(t, func() bool { return channelClosed(connection.Done()) }, 5*time.Second, time.Millisecond) | |
| 944 | + | require.NoError(t, connection.Err()) | |
| 945 | + | ||
| 946 | + | acceptor.closePeers() | |
| 947 | + | require.NoError(t, brokerPump.Close()) | |
| 948 | + | require.NoError(t, daemonPump.Close()) | |
| 949 | + | ||
| 950 | + | require.Eventually(t, func() bool { return runtime.NumGoroutine() <= before+3 }, 5*time.Second, time.Millisecond, | |
| 951 | + | "peer-closed logical connection goroutines leaked") | |
| 952 | + | } | |
| 953 | + | ||
| 954 | + | // TestLogicalPhysicalLossBlockedReadReturnsLoss proves a read blocked on an | |
| 955 | + | // idle logical stream wakes when the physical connection fails and reports a | |
| 956 | + | // typed ports.BrokerStreamLost rather than a clean io.EOF. The pump publishes | |
| 957 | + | // its terminal state before the engine terminalizes each stream, so the read | |
| 958 | + | // must consult the pump's physical cause instead of assuming an orderly end. | |
| 959 | + | func TestLogicalPhysicalLossBlockedReadReturnsLoss(t *testing.T) { | |
| 960 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 961 | + | connector := mustConnector(t, brokerPump) | |
| 962 | + | deadline := time.Now().Add(15 * time.Second) | |
| 963 | + | ||
| 964 | + | acceptor := newDaemonAcceptor(daemonPump, 1) | |
| 965 | + | acceptor.start(1, deadline) | |
| 966 | + | connection := openWithDeadline(t, connector, 1) | |
| 967 | + | require.NoError(t, acceptor.wait(1, deadline)) | |
| 968 | + | ||
| 969 | + | require.NoError(t, connection.SendClient(protocol.Ping{})) | |
| 970 | + | message, err := connection.ReceiveServer() | |
| 971 | + | require.NoError(t, err) | |
| 972 | + | require.Equal(t, protocol.Pong{}, message) | |
| 973 | + | ||
| 974 | + | type result struct { | |
| 975 | + | message protocol.ServerMessage | |
| 976 | + | err error | |
| 977 | + | } | |
| 978 | + | blocked := make(chan result, 1) | |
| 979 | + | go func() { | |
| 980 | + | msg, err := connection.ReceiveServer() | |
| 981 | + | blocked <- result{message: msg, err: err} | |
| 982 | + | }() | |
| 983 | + | // Let the reader reach the inner stream and block there. | |
| 984 | + | time.Sleep(50 * time.Millisecond) | |
| 985 | + | require.False(t, channelClosed(connection.Done()), "the read is blocked, not settled") | |
| 986 | + | ||
| 987 | + | // Fail the daemon side; the broker observes a physical loss. | |
| 988 | + | require.NoError(t, daemonPump.Close()) | |
| 989 | + | ||
| 990 | + | select { | |
| 991 | + | case got := <-blocked: | |
| 992 | + | require.Nil(t, got.message) | |
| 993 | + | require.Error(t, got.err) | |
| 994 | + | require.NotErrorIs(t, got.err, io.EOF, "a physical failure must not surface as a clean EOF") | |
| 995 | + | var loss ports.BrokerStreamLost | |
| 996 | + | require.ErrorAs(t, got.err, &loss) | |
| 997 | + | require.Equal(t, ports.BrokerStreamID(1), loss.Stream) | |
| 998 | + | require.Equal(t, domain.RemoteFailureTransport, loss.Cause) | |
| 999 | + | case <-time.After(5 * time.Second): | |
| 1000 | + | t.Fatal("blocked read did not return after the physical failure") | |
| 1001 | + | } | |
| 1002 | + | require.True(t, channelClosed(connection.Done())) | |
| 1003 | + | } | |
| 1004 | + | ||
| 1005 | + | // TestLogicalTerminalCauseSurvivesEngineEviction proves a settled logical | |
| 1006 | + | // connection reports its exact stream-local cause after the engine evicts the | |
| 1007 | + | // bounded record. The retained authority outlives the record, so a Reset can | |
| 1008 | + | // never degrade into an orderly close for a reader that observes the end late. | |
| 1009 | + | func TestLogicalTerminalCauseSurvivesEngineEviction(t *testing.T) { | |
| 1010 | + | pump, _ := newTestPump(t, DirectionServer) | |
| 1011 | + | require.NoError(t, pump.Engine().Open(openFor(1))) | |
| 1012 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 1013 | + | status := mustStatus(t, pump.Engine(), 1) | |
| 1014 | + | ||
| 1015 | + | connection := &LogicalConnection{ | |
| 1016 | + | pump: pump, | |
| 1017 | + | ref: status.Ref, | |
| 1018 | + | terminal: newTerminalState(), | |
| 1019 | + | cause: pump.cause(status.Ref.Physical), | |
| 1020 | + | } | |
| 1021 | + | require.NotNil(t, connection.cause) | |
| 1022 | + | ||
| 1023 | + | // Retire stream 1 with a stream-local Reset, then retire more streams than | |
| 1024 | + | // the engine retains so its record is evicted before any reader consults it. | |
| 1025 | + | _, err := pump.Engine().Reset(Reset{ | |
| 1026 | + | Physical: 1, | |
| 1027 | + | HasError: true, | |
| 1028 | + | Error: ErrorDetail{Code: ports.BrokerErrorUnavailable, Text: "reset", FailureKind: domain.RemoteFailureInvalidResponse}, | |
| 1029 | + | }) | |
| 1030 | + | require.NoError(t, err) | |
| 1031 | + | for i := 2; i <= MaxRetiredStreamRecords+2; i++ { | |
| 1032 | + | id := PhysicalStreamID(i) | |
| 1033 | + | require.NoError(t, pump.Engine().Open(openFor(id))) | |
| 1034 | + | _, _ = pump.Engine().Close(Close{Physical: id}) | |
| 1035 | + | } | |
| 1036 | + | _, tracked := pump.Engine().Status(1) | |
| 1037 | + | require.False(t, tracked, "stream 1 record was evicted") | |
| 1038 | + | ||
| 1039 | + | connection.syncFromEngine() | |
| 1040 | + | ||
| 1041 | + | require.True(t, channelClosed(connection.Done())) | |
| 1042 | + | var loss ports.BrokerStreamLost | |
| 1043 | + | require.ErrorAs(t, connection.Err(), &loss) | |
| 1044 | + | require.Equal(t, status.Ref.Client, loss.Stream) | |
| 1045 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, loss.Cause) | |
| 1046 | + | require.NotEqual(t, domain.RemoteFailureNone, connection.FailureKind(), | |
| 1047 | + | "a Reset never degrades into an orderly close") | |
| 1048 | + | } | |
| 1049 | + | ||
| 1050 | + | // TestLogicalWatcherCauseSurvivesEvictionStress retires a stream and then more | |
| 1051 | + | // streams than the engine retains before the watcher is scheduled, proving a | |
| 1052 | + | // late watcher still publishes the stream-local Reset as a loss instead of an | |
| 1053 | + | // orderly close. | |
| 1054 | + | func TestLogicalWatcherCauseSurvivesEvictionStress(t *testing.T) { | |
| 1055 | + | const rounds = 12 | |
| 1056 | + | for round := 0; round < rounds; round++ { | |
| 1057 | + | t.Run(fmt.Sprintf("round %d", round), func(t *testing.T) { | |
| 1058 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 1059 | + | connector := mustConnector(t, brokerPump) | |
| 1060 | + | deadline := time.Now().Add(15 * time.Second) | |
| 1061 | + | ||
| 1062 | + | acceptor := newDaemonAcceptor(daemonPump, 1) | |
| 1063 | + | acceptor.start(1, deadline) | |
| 1064 | + | connection := openWithDeadline(t, connector, 1) | |
| 1065 | + | require.NoError(t, acceptor.wait(1, deadline)) | |
| 1066 | + | ||
| 1067 | + | // Retire the stream with a stream-local Reset, then immediately | |
| 1068 | + | // retire more stream records than the engine retains so a watcher | |
| 1069 | + | // scheduled late observes an evicted record. | |
| 1070 | + | _, err := brokerPump.Engine().Reset(Reset{ | |
| 1071 | + | Physical: 1, | |
| 1072 | + | HasError: true, | |
| 1073 | + | Error: ErrorDetail{Code: ports.BrokerErrorUnavailable, Text: "reset", FailureKind: domain.RemoteFailureInvalidResponse}, | |
| 1074 | + | }) | |
| 1075 | + | require.NoError(t, err) | |
| 1076 | + | for i := 2; i <= MaxRetiredStreamRecords+2; i++ { | |
| 1077 | + | id := PhysicalStreamID(i) | |
| 1078 | + | require.NoError(t, brokerPump.Engine().Open(openFor(id))) | |
| 1079 | + | _, _ = brokerPump.Engine().Close(Close{Physical: id}) | |
| 1080 | + | } | |
| 1081 | + | _, tracked := brokerPump.Engine().Status(1) | |
| 1082 | + | require.False(t, tracked, "stream 1 record was evicted") | |
| 1083 | + | ||
| 1084 | + | require.Eventually(t, func() bool { return channelClosed(connection.Done()) }, 5*time.Second, time.Millisecond) | |
| 1085 | + | var loss ports.BrokerStreamLost | |
| 1086 | + | require.ErrorAs(t, connection.Err(), &loss, | |
| 1087 | + | "a late watcher must not degrade a Reset into an orderly close") | |
| 1088 | + | require.Equal(t, ports.BrokerStreamID(1), loss.Stream) | |
| 1089 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, loss.Cause) | |
| 1090 | + | require.Zero(t, brokerPump.Engine().Live()) | |
| 1091 | + | acceptor.closePeers() | |
| 1092 | + | }) | |
| 1093 | + | } | |
| 1094 | + | } | |
| 1095 | + | ||
| 1096 | + | // TestLogicalFreshOpenRefusedByDaemonCeiling proves the daemon's fresh-refusal | |
| 1097 | + | // path reaches the broker: the 129th physical identity is refused by the | |
| 1098 | + | // daemon's stream ceiling with one typed Mux Refused, and LogicalConnector | |
| 1099 | + | // classifies it as the admission limit without handing back a connection or | |
| 1100 | + | // recording a daemon-side stream. | |
| 1101 | + | func TestLogicalFreshOpenRefusedByDaemonCeiling(t *testing.T) { | |
| 1102 | + | brokerPump, daemonPump := newPairedPumps(t, DefaultMuxCeilings()) | |
| 1103 | + | connector := mustConnector(t, brokerPump) | |
| 1104 | + | ||
| 1105 | + | // Fill the daemon's stream ceiling directly so the broker's own admission | |
| 1106 | + | // stays free: the connector's next allocated identity is the 129th. | |
| 1107 | + | for i := 1; i <= int(MaxMuxStreams); i++ { | |
| 1108 | + | require.NoError(t, daemonPump.Engine().Open(openFor(PhysicalStreamID(i)))) | |
| 1109 | + | } | |
| 1110 | + | require.Equal(t, int(MaxMuxStreams), daemonPump.Engine().Live()) | |
| 1111 | + | connector.next = uint64(MaxMuxStreams) | |
| 1112 | + | ||
| 1113 | + | ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | |
| 1114 | + | defer cancel() | |
| 1115 | + | connection, err := connector.Open(ctx, controlRequest(1)) | |
| 1116 | + | require.ErrorIs(t, err, ports.BrokerAdmissionLimit) | |
| 1117 | + | require.Nil(t, connection) | |
| 1118 | + | ||
| 1119 | + | refusedID := PhysicalStreamID(MaxMuxStreams + 1) | |
| 1120 | + | require.Eventually(t, func() bool { | |
| 1121 | + | status, ok := brokerPump.Engine().Status(refusedID) | |
| 1122 | + | return ok && status.State == StreamTerminal | |
| 1123 | + | }, 15*time.Second, time.Millisecond) | |
| 1124 | + | status := mustStatus(t, brokerPump.Engine(), refusedID) | |
| 1125 | + | require.True(t, status.Refused, "the daemon's fresh refusal reached the broker") | |
| 1126 | + | require.Equal(t, uint32(1), status.Refusal.AdmissionCode) | |
| 1127 | + | ||
| 1128 | + | // The daemon recorded no stream for the refused identity, the broker | |
| 1129 | + | // released its local reservation, and the physical connection stays open. | |
| 1130 | + | _, tracked := daemonPump.Engine().Status(refusedID) | |
| 1131 | + | require.False(t, tracked, "a refused fresh open records no daemon stream") | |
| 1132 | + | require.Equal(t, int(MaxMuxStreams), daemonPump.Engine().Live()) | |
| 1133 | + | require.Zero(t, brokerPump.Engine().Live(), "the broker released its local reservation") | |
| 1134 | + | require.False(t, channelClosed(brokerPump.Done())) | |
| 1135 | + | require.False(t, channelClosed(daemonPump.Done())) | |
| 1136 | + | } | |
A internal/adapters/daemonmux/messages.go +177 -0
| @@ -0,0 +1,177 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "errors" | |
| 5 | + | ||
| 6 | + | "github.com/bnema/vev/internal/domain" | |
| 7 | + | "github.com/bnema/vev/internal/ports" | |
| 8 | + | "github.com/bnema/vev/internal/protocol" | |
| 9 | + | ) | |
| 10 | + | ||
| 11 | + | var ( | |
| 12 | + | // ErrWrongDirection reports a daemonmux message presented to the wrong | |
| 13 | + | // directional codec. Direction is a wire property of the envelope oneof: | |
| 14 | + | // a client envelope never decodes as daemon output and vice versa. | |
| 15 | + | ErrWrongDirection = errors.New("daemonmux: wrong message direction") | |
| 16 | + | // ErrInvalidMessage reports a daemonmux message that fails semantic | |
| 17 | + | // validation: bad stream identity, text, taxonomy, bound, environment, | |
| 18 | + | // policy, or error detail. | |
| 19 | + | ErrInvalidMessage = errors.New("daemonmux: invalid message") | |
| 20 | + | // ErrTooLarge reports a stateless bound refusal established by limits.go: | |
| 21 | + | // an envelope, chunk, environment, policy token, or error text above its | |
| 22 | + | // ceiling. | |
| 23 | + | ErrTooLarge = errors.New("daemonmux: message exceeds bound") | |
| 24 | + | errConvertRange = errors.New("daemonmux: wire value out of semantic range") | |
| 25 | + | ) | |
| 26 | + | ||
| 27 | + | // ClientMessage is the closed set of broker-to-daemon daemonmux messages. | |
| 28 | + | // Every variant converts to exactly one MuxClientEnvelope payload. | |
| 29 | + | type ClientMessage interface{ muxClientMessage() } | |
| 30 | + | ||
| 31 | + | // ServerMessage is the closed set of daemon-to-broker daemonmux messages. | |
| 32 | + | // Every variant converts to exactly one MuxServerEnvelope payload. | |
| 33 | + | type ServerMessage interface{ muxServerMessage() } | |
| 34 | + | ||
| 35 | + | // PhysicalStreamID identifies one logical stream multiplexed over one | |
| 36 | + | // physical daemon connection. It is asserted by the opening side and never | |
| 37 | + | // crosses a physical connection; the engine (not this codec) enforces that a | |
| 38 | + | // connection's IDs are strictly increasing, so the codec only rejects zero. | |
| 39 | + | type PhysicalStreamID uint64 | |
| 40 | + | ||
| 41 | + | // Validate rejects the zero value: a stream is never routed without an | |
| 42 | + | // explicit physical identity. | |
| 43 | + | func (id PhysicalStreamID) Validate() error { | |
| 44 | + | if id == 0 { | |
| 45 | + | return errors.New("daemonmux: physical stream ID is zero") | |
| 46 | + | } | |
| 47 | + | return nil | |
| 48 | + | } | |
| 49 | + | ||
| 50 | + | // StreamRef is the full identity of one multiplexed logical stream: the | |
| 51 | + | // mux-local physical stream, the broker epoch and client connection that own | |
| 52 | + | // it, and the broker client stream it carries. It is carried by MuxOpen, | |
| 53 | + | // MuxOpened, and MuxRefused only; later frames are routed by the physical | |
| 54 | + | // stream ID alone. | |
| 55 | + | type StreamRef struct { | |
| 56 | + | Physical PhysicalStreamID | |
| 57 | + | Epoch ports.BrokerEpoch | |
| 58 | + | Connection ports.BrokerConnectionID | |
| 59 | + | Client ports.BrokerStreamID | |
| 60 | + | } | |
| 61 | + | ||
| 62 | + | // Validate enforces the complete reference: a nonzero physical and broker | |
| 63 | + | // stream, a nonzero epoch, and a nonzero 16-byte connection identity. | |
| 64 | + | func (r StreamRef) Validate() error { | |
| 65 | + | if err := r.Physical.Validate(); err != nil { | |
| 66 | + | return err | |
| 67 | + | } | |
| 68 | + | if r.Epoch == 0 { | |
| 69 | + | return errors.New("daemonmux: stream ref has no broker epoch") | |
| 70 | + | } | |
| 71 | + | if err := r.Connection.Validate(); err != nil { | |
| 72 | + | return err | |
| 73 | + | } | |
| 74 | + | return r.Client.Validate() | |
| 75 | + | } | |
| 76 | + | ||
| 77 | + | // Open asks the daemon to open one independently cancellable logical stream | |
| 78 | + | // over the physical connection. Env is the per-request session environment | |
| 79 | + | // and is never inherited from the broker process environment. | |
| 80 | + | type Open struct { | |
| 81 | + | Ref StreamRef | |
| 82 | + | Purpose ports.BrokerStreamPurpose | |
| 83 | + | Local bool | |
| 84 | + | Endpoint string | |
| 85 | + | Registration domain.RemoteRegistration | |
| 86 | + | Target protocol.ExactSessionTarget | |
| 87 | + | Env []string | |
| 88 | + | Policy ports.BrokerPolicy | |
| 89 | + | } | |
| 90 | + | ||
| 91 | + | func (Open) muxClientMessage() {} | |
| 92 | + | ||
| 93 | + | // Data carries one opaque stream frame. It travels in both directions over | |
| 94 | + | // the same wire payload. | |
| 95 | + | type Data struct { | |
| 96 | + | Physical PhysicalStreamID | |
| 97 | + | Data []byte | |
| 98 | + | } | |
| 99 | + | ||
| 100 | + | func (Data) muxClientMessage() {} | |
| 101 | + | func (Data) muxServerMessage() {} | |
| 102 | + | ||
| 103 | + | // Close ends one logical stream. It travels in both directions over the same | |
| 104 | + | // wire payload. | |
| 105 | + | type Close struct { | |
| 106 | + | Physical PhysicalStreamID | |
| 107 | + | } | |
| 108 | + | ||
| 109 | + | func (Close) muxClientMessage() {} | |
| 110 | + | func (Close) muxServerMessage() {} | |
| 111 | + | ||
| 112 | + | // Reset aborts one logical stream. It travels in both directions over the | |
| 113 | + | // same wire payload. HasError=false means no detail at all: encode refuses a | |
| 114 | + | // nonzero Error rather than silently dropping it. | |
| 115 | + | type Reset struct { | |
| 116 | + | Physical PhysicalStreamID | |
| 117 | + | Error ErrorDetail | |
| 118 | + | HasError bool | |
| 119 | + | } | |
| 120 | + | ||
| 121 | + | func (Reset) muxClientMessage() {} | |
| 122 | + | func (Reset) muxServerMessage() {} | |
| 123 | + | ||
| 124 | + | // Opened confirms one logical stream is established. | |
| 125 | + | type Opened struct { | |
| 126 | + | Ref StreamRef | |
| 127 | + | } | |
| 128 | + | ||
| 129 | + | func (Opened) muxServerMessage() {} | |
| 130 | + | ||
| 131 | + | // Refused refuses one logical stream open. The typed error is mandatory: a | |
| 132 | + | // refusal without a reason could not be classified. | |
| 133 | + | type Refused struct { | |
| 134 | + | Ref StreamRef | |
| 135 | + | Error ErrorDetail | |
| 136 | + | } | |
| 137 | + | ||
| 138 | + | func (Refused) muxServerMessage() {} | |
| 139 | + | ||
| 140 | + | // MaxAdmissionCode is the highest admission-refusal code carried on | |
| 141 | + | // ErrorDetail. The taxonomy is closed: 0 is none, 1 limit, 2 closed, 3 stale | |
| 142 | + | // connection or stream, 4 invalid request (mirroring | |
| 143 | + | // ports.BrokerAdmissionError). | |
| 144 | + | const MaxAdmissionCode uint32 = 4 | |
| 145 | + | ||
| 146 | + | // MaxFailureKind bounds ErrorDetail.FailureKind to the domain | |
| 147 | + | // transport-failure taxonomy. The zero value (RemoteFailureNone) is valid. | |
| 148 | + | const MaxFailureKind = domain.RemoteFailureInvalidResponse | |
| 149 | + | ||
| 150 | + | // ErrorDetail is the typed daemonmux failure carried to the broker: a closed | |
| 151 | + | // error code plus bounded display text, the admission refusal (if any), and | |
| 152 | + | // the sanitized transport cause. The underlying diagnostic cause is local-only | |
| 153 | + | // and never travels on this wire. | |
| 154 | + | type ErrorDetail struct { | |
| 155 | + | Code ports.BrokerErrorCode | |
| 156 | + | Text string | |
| 157 | + | // AdmissionCode is the closed admission-refusal taxonomy: 0 is none, | |
| 158 | + | // 1 limit, 2 closed, 3 stale connection or stream, 4 invalid request. | |
| 159 | + | AdmissionCode uint32 | |
| 160 | + | FailureKind domain.RemoteFailureKind | |
| 161 | + | } | |
| 162 | + | ||
| 163 | + | // validate enforces the transport contract of one ErrorDetail: a closed | |
| 164 | + | // error code, bounded presentation-safe text, an admission code within the | |
| 165 | + | // closed refusal taxonomy, and a failure kind within the domain taxonomy. | |
| 166 | + | func (d ErrorDetail) validate() error { | |
| 167 | + | if err := d.Code.Validate(); err != nil { | |
| 168 | + | return ErrInvalidMessage | |
| 169 | + | } | |
| 170 | + | if d.AdmissionCode > MaxAdmissionCode { | |
| 171 | + | return ErrInvalidMessage | |
| 172 | + | } | |
| 173 | + | if d.FailureKind > MaxFailureKind { | |
| 174 | + | return ErrInvalidMessage | |
| 175 | + | } | |
| 176 | + | return validateDisplayText(d.Text, ports.BrokerMaxErrorBytes) | |
| 177 | + | } | |
A internal/adapters/daemonmux/pool_integration_test.go +162 -0
| @@ -0,0 +1,162 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "testing" | |
| 6 | + | "time" | |
| 7 | + | ||
| 8 | + | "github.com/stretchr/testify/require" | |
| 9 | + | ||
| 10 | + | "github.com/bnema/vev/internal/domain" | |
| 11 | + | "github.com/bnema/vev/internal/ports" | |
| 12 | + | "github.com/bnema/vev/internal/protocol" | |
| 13 | + | "github.com/bnema/vev/internal/usecase/broker" | |
| 14 | + | ) | |
| 15 | + | ||
| 16 | + | // p3cResolver is a test BrokerEndpointResolver: it returns one authenticated | |
| 17 | + | // endpoint per request, keyed on the request's registration endpoint so two | |
| 18 | + | // compatible aliases resolve to the same (identity, policy) physical key with | |
| 19 | + | // distinct adapter routes. | |
| 20 | + | type p3cResolver func(context.Context, ports.BrokerOpenStreamRequest) (ports.BrokerResolvedEndpoint, error) | |
| 21 | + | ||
| 22 | + | func (f p3cResolver) Resolve(ctx context.Context, request ports.BrokerOpenStreamRequest) (ports.BrokerResolvedEndpoint, error) { | |
| 23 | + | return f(ctx, request) | |
| 24 | + | } | |
| 25 | + | ||
| 26 | + | // p3cRemoteRequest builds one valid remote control-purpose request. | |
| 27 | + | func p3cRemoteRequest(connection ports.BrokerConnectionID, endpoint string, stream uint64, policy ports.BrokerPolicy) ports.BrokerOpenStreamRequest { | |
| 28 | + | registration := domain.RemoteRegistration{ | |
| 29 | + | Endpoint: endpoint, | |
| 30 | + | Incarnation: [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, | |
| 31 | + | Generation: 1, | |
| 32 | + | } | |
| 33 | + | return ports.BrokerOpenStreamRequest{ | |
| 34 | + | Epoch: 1, | |
| 35 | + | Purpose: ports.BrokerStreamControl, | |
| 36 | + | Local: false, | |
| 37 | + | Connection: connection, | |
| 38 | + | Stream: ports.BrokerStreamID(stream), | |
| 39 | + | Endpoint: endpoint, | |
| 40 | + | Registration: registration, | |
| 41 | + | Policy: policy, | |
| 42 | + | } | |
| 43 | + | } | |
| 44 | + | ||
| 45 | + | // p3cPoolLimits bounds one pool with room for two aliases and two streams. | |
| 46 | + | func p3cPoolLimits() broker.PoolLimits { | |
| 47 | + | return broker.PoolLimits{Physical: 2, Clients: 2, Streams: 4, StreamsPerClient: 4, Idle: time.Hour} | |
| 48 | + | } | |
| 49 | + | ||
| 50 | + | // TestPoolSharesCompatibleAliasesOverOnePhysical proves the pool pools two | |
| 51 | + | // compatible aliases over exactly one daemonmux physical connection: only one | |
| 52 | + | // carriage is dialed, and both logical streams flow over it. | |
| 53 | + | func TestPoolSharesCompatibleAliasesOverOnePhysical(t *testing.T) { | |
| 54 | + | binding := mustServerBinding(t) | |
| 55 | + | policy := binding.Policy() | |
| 56 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 57 | + | server.serve() | |
| 58 | + | resolver := p3cResolver(func(_ context.Context, request ports.BrokerOpenStreamRequest) (ports.BrokerResolvedEndpoint, error) { | |
| 59 | + | return muxEndpoint(binding.Identity(), policy, "raw://"+request.Endpoint), nil | |
| 60 | + | }) | |
| 61 | + | ||
| 62 | + | connector, err := NewEndpointConnector(server.dial(), DefaultMuxCeilings()) | |
| 63 | + | require.NoError(t, err) | |
| 64 | + | pool, err := broker.NewPool(1, resolver, connector, newListenerClock(time.Now()), p3cPoolLimits()) | |
| 65 | + | require.NoError(t, err) | |
| 66 | + | t.Cleanup(func() { _ = pool.Close() }) | |
| 67 | + | ||
| 68 | + | client, err := pool.RegisterClient() | |
| 69 | + | require.NoError(t, err) | |
| 70 | + | ||
| 71 | + | first, err := pool.OpenStream(context.Background(), p3cRemoteRequest(client, "aliasA@host:22", 1, policy)) | |
| 72 | + | require.NoError(t, err) | |
| 73 | + | require.NotNil(t, first) | |
| 74 | + | t.Cleanup(func() { _ = first.Close() }) | |
| 75 | + | ||
| 76 | + | second, err := pool.OpenStream(context.Background(), p3cRemoteRequest(client, "aliasB@host:22", 2, policy)) | |
| 77 | + | require.NoError(t, err) | |
| 78 | + | require.NotNil(t, second) | |
| 79 | + | t.Cleanup(func() { _ = second.Close() }) | |
| 80 | + | ||
| 81 | + | require.Equal(t, int64(1), server.dials.Load(), "compatible aliases must share one physical carriage") | |
| 82 | + | server.awaitAccepted(t, 2) | |
| 83 | + | ||
| 84 | + | for _, stream := range []ports.BrokerLogicalConnection{first, second} { | |
| 85 | + | require.NoError(t, stream.SendClient(protocol.Ping{})) | |
| 86 | + | message, err := stream.ReceiveServer() | |
| 87 | + | require.NoError(t, err) | |
| 88 | + | require.Equal(t, protocol.Pong{}, message) | |
| 89 | + | } | |
| 90 | + | } | |
| 91 | + | ||
| 92 | + | // TestPoolRefusesConflictingPolicyWithoutDial proves the pool rejects a request | |
| 93 | + | // whose policy conflicts with the resolved endpoint policy before any carriage | |
| 94 | + | // is dialed. | |
| 95 | + | func TestPoolRefusesConflictingPolicyWithoutDial(t *testing.T) { | |
| 96 | + | binding := mustServerBinding(t) | |
| 97 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 98 | + | resolver := p3cResolver(func(_ context.Context, _ ports.BrokerOpenStreamRequest) (ports.BrokerResolvedEndpoint, error) { | |
| 99 | + | return muxEndpoint(binding.Identity(), binding.Policy(), "raw://conflict"), nil | |
| 100 | + | }) | |
| 101 | + | ||
| 102 | + | connector, err := NewEndpointConnector(server.dial(), DefaultMuxCeilings()) | |
| 103 | + | require.NoError(t, err) | |
| 104 | + | pool, err := broker.NewPool(1, resolver, connector, newListenerClock(time.Now()), p3cPoolLimits()) | |
| 105 | + | require.NoError(t, err) | |
| 106 | + | t.Cleanup(func() { _ = pool.Close() }) | |
| 107 | + | ||
| 108 | + | client, err := pool.RegisterClient() | |
| 109 | + | require.NoError(t, err) | |
| 110 | + | _, err = pool.OpenStream(context.Background(), p3cRemoteRequest(client, "aliasA@host:22", 1, alternatePolicy(binding.Policy()))) | |
| 111 | + | require.Error(t, err) | |
| 112 | + | var typed ports.BrokerError | |
| 113 | + | require.ErrorAs(t, err, &typed) | |
| 114 | + | require.Equal(t, ports.BrokerErrorConflictingPolicy, typed.Code) | |
| 115 | + | require.Zero(t, server.dials.Load(), "a conflicting policy never dials a carriage") | |
| 116 | + | } | |
| 117 | + | ||
| 118 | + | // TestPoolSurfacesPhysicalLossPerStream proves a carriage loss is published | |
| 119 | + | // per affected logical stream through the pool: the pooled stream terminates | |
| 120 | + | // with a broker stream-lost error once the accepted physical carriage dies. | |
| 121 | + | func TestPoolSurfacesPhysicalLossPerStream(t *testing.T) { | |
| 122 | + | binding := mustServerBinding(t) | |
| 123 | + | policy := binding.Policy() | |
| 124 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 125 | + | server.serve() | |
| 126 | + | resolver := p3cResolver(func(_ context.Context, request ports.BrokerOpenStreamRequest) (ports.BrokerResolvedEndpoint, error) { | |
| 127 | + | return muxEndpoint(binding.Identity(), policy, "raw://"+request.Endpoint), nil | |
| 128 | + | }) | |
| 129 | + | ||
| 130 | + | connector, err := NewEndpointConnector(server.dial(), DefaultMuxCeilings()) | |
| 131 | + | require.NoError(t, err) | |
| 132 | + | pool, err := broker.NewPool(1, resolver, connector, newListenerClock(time.Now()), p3cPoolLimits()) | |
| 133 | + | require.NoError(t, err) | |
| 134 | + | t.Cleanup(func() { _ = pool.Close() }) | |
| 135 | + | ||
| 136 | + | client, err := pool.RegisterClient() | |
| 137 | + | require.NoError(t, err) | |
| 138 | + | stream, err := pool.OpenStream(context.Background(), p3cRemoteRequest(client, "aliasA@host:22", 1, policy)) | |
| 139 | + | require.NoError(t, err) | |
| 140 | + | server.awaitAccepted(t, 1) | |
| 141 | + | ||
| 142 | + | // Lose the daemon carriage: the supervised child is closed, so the broker | |
| 143 | + | // pump publishes the physical loss and the pool settles the stream. | |
| 144 | + | server.supervisor.mu.Lock() | |
| 145 | + | children := make([]*supervisedPhysical, 0, len(server.supervisor.children)) | |
| 146 | + | for child := range server.supervisor.children { | |
| 147 | + | children = append(children, child) | |
| 148 | + | } | |
| 149 | + | server.supervisor.mu.Unlock() | |
| 150 | + | require.Len(t, children, 1) | |
| 151 | + | require.NoError(t, children[0].Close()) | |
| 152 | + | ||
| 153 | + | select { | |
| 154 | + | case <-stream.Done(): | |
| 155 | + | case <-time.After(p3cTestDeadline): | |
| 156 | + | t.Fatal("the pooled logical stream never observed the physical loss") | |
| 157 | + | } | |
| 158 | + | var lost ports.BrokerStreamLost | |
| 159 | + | require.ErrorAs(t, stream.Err(), &lost) | |
| 160 | + | require.Equal(t, ports.BrokerStreamID(1), lost.Stream) | |
| 161 | + | require.NotEqual(t, domain.RemoteFailureNone, lost.Cause) | |
| 162 | + | } | |
A internal/adapters/daemonmux/preamble.go +422 -0
| @@ -0,0 +1,422 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | // Daemonmux physical preamble (P3.2). | |
| 4 | + | // | |
| 5 | + | // The first daemonmux client frame on a physical connection is always | |
| 6 | + | // MuxPreambleRequest; the first daemon frame is always MuxPreambleResponse. | |
| 7 | + | // The preamble is an adapter message pair, not an envelope variant. It | |
| 8 | + | // reuses the shared session preamble magic, epoch, and exact protocol | |
| 9 | + | // version, and the shared refusal taxonomy, with daemonmux-specific roles: 5 | |
| 10 | + | // names the broker client, 6 names the daemon server. | |
| 11 | + | // | |
| 12 | + | // The preamble negotiates immutable per-connection ceilings once: the | |
| 13 | + | // receive envelope ceiling (1 MiB..16 MiB), the stream chunk ceiling | |
| 14 | + | // (1..64 KiB), the concurrent stream ceiling (1..128), and the aggregate | |
| 15 | + | // buffered-bytes ceiling (1..64 MiB). The response carries the effective | |
| 16 | + | // minimum of both advertisements, the authenticated daemon identity, its | |
| 17 | + | // 16-byte incarnation, and the accepted connection policy. The effective | |
| 18 | + | // minima are the exported MuxCeilings value (limits.go) that | |
| 19 | + | // NewStreamEngineWithCeilings and NewSchedulerWithCeilings consume. Capability | |
| 20 | + | // bits must be zero: daemonmux negotiates no optional capabilities. | |
| 21 | + | // | |
| 22 | + | // Refusals carry the precise shared PreambleRejectionCode taxonomy: 1 bad | |
| 23 | + | // magic, 2 epoch mismatch, 3 version mismatch, 4 wrong role, 5 duplicate, 6 | |
| 24 | + | // out of order, 7 limit refused. Code 5 means a duplicate observed inside one | |
| 25 | + | // frame: the strict scanner found a repeated singular field or oneof member | |
| 26 | + | // (wire.ErrScanDuplicate) in the serialized preamble this codec decodes. The | |
| 27 | + | // connection dispatcher separately refuses a second preamble frame on one | |
| 28 | + | // connection, also with code 5, but that stateful, cross-frame duplicate is | |
| 29 | + | // never observed here. Code 6 is reserved for the dispatcher too: out-of-order | |
| 30 | + | // application frames are a framing-order failure this stateless codec never | |
| 31 | + | // sees. A refused policy is also a code 7 refusal, because daemonmux negotiates | |
| 32 | + | // exact policy equality rather than a capability set. | |
| 33 | + | // | |
| 34 | + | // A response is either an acceptance or a refusal and never a blend of the | |
| 35 | + | // two: an acceptance carries no rejection field, and a refusal carries none of | |
| 36 | + | // the accepted-only authority fields (accepted policy, daemon identity, | |
| 37 | + | // incarnation) and a rejection code in 1..7. Both directions are enforced on | |
| 38 | + | // encode and decode so a peer can never smuggle accepted authority into a | |
| 39 | + | // refusal or a stale refusal code into an acceptance. | |
| 40 | + | ||
| 41 | + | import ( | |
| 42 | + | "errors" | |
| 43 | + | ||
| 44 | + | "google.golang.org/protobuf/proto" | |
| 45 | + | ||
| 46 | + | "github.com/bnema/vev/internal/ports" | |
| 47 | + | "github.com/bnema/vev/internal/protocol" | |
| 48 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 49 | + | ) | |
| 50 | + | ||
| 51 | + | // ErrPreambleRejected reports a daemonmux preamble that fails validation: | |
| 52 | + | // bad magic, epoch, version, role, capability bits, limit advertisement, | |
| 53 | + | // policy, identity, or incarnation; a refusal code outside 1..7; an | |
| 54 | + | // acceptance that also carries a rejection field; or a refusal that also | |
| 55 | + | // carries accepted-only authority. Callers map it to a precise rejection code | |
| 56 | + | // with RejectionCodeFor. | |
| 57 | + | var ErrPreambleRejected = errors.New("daemonmux: preamble rejected") | |
| 58 | + | ||
| 59 | + | // Preamble rejection codes. They mirror the session preamble refusal | |
| 60 | + | // taxonomy carried on wire.PreambleRejectionCode. | |
| 61 | + | const ( | |
| 62 | + | // RejectionBadMagic reports a preamble magic mismatch. | |
| 63 | + | RejectionBadMagic uint32 = 1 | |
| 64 | + | // RejectionEpochMismatch reports a protocol epoch mismatch. | |
| 65 | + | RejectionEpochMismatch uint32 = 2 | |
| 66 | + | // RejectionVersionMismatch reports a session protocol version mismatch. | |
| 67 | + | RejectionVersionMismatch uint32 = 3 | |
| 68 | + | // RejectionWrongRole reports a preamble role mismatch. | |
| 69 | + | RejectionWrongRole uint32 = 4 | |
| 70 | + | // RejectionDuplicate reports a duplicate within one frame: the strict | |
| 71 | + | // scanner found a repeated singular field or oneof member | |
| 72 | + | // (wire.ErrScanDuplicate) in the serialized preamble. The connection | |
| 73 | + | // dispatcher separately refuses a second preamble frame on one connection | |
| 74 | + | // with this same code; that stateful, cross-frame duplicate is never | |
| 75 | + | // observed by this stateless codec. | |
| 76 | + | RejectionDuplicate uint32 = 5 | |
| 77 | + | // RejectionOutOfOrder is reserved for the connection dispatcher, which | |
| 78 | + | // owns framing-order detection. The stateless codec never emits it, but it | |
| 79 | + | // accepts it as a well-formed refusal code on the wire. | |
| 80 | + | RejectionOutOfOrder uint32 = 6 | |
| 81 | + | // RejectionLimitRefused reports an unnegotiable advertisement. It also | |
| 82 | + | // covers nonzero capability bits and a policy the daemon does not | |
| 83 | + | // accept: P3.2 negotiates neither capabilities nor a merged policy | |
| 84 | + | // server, so either is an unnegotiable advertisement. | |
| 85 | + | RejectionLimitRefused uint32 = 7 | |
| 86 | + | ) | |
| 87 | + | ||
| 88 | + | // validRejectionCode reports whether code is one of the shared preamble | |
| 89 | + | // refusal codes 1..7. The stateless codec accepts code 6 (out of order) as a | |
| 90 | + | // well-formed code on the wire even though it never emits it, because the | |
| 91 | + | // dispatcher owns framing-order detection. | |
| 92 | + | func validRejectionCode(code uint32) bool { | |
| 93 | + | return code >= RejectionBadMagic && code <= RejectionLimitRefused | |
| 94 | + | } | |
| 95 | + | ||
| 96 | + | // MuxPreambleRequest is the validated daemonmux client preamble: the | |
| 97 | + | // requested ceilings plus the broker policy the client asks the daemon to | |
| 98 | + | // accept. | |
| 99 | + | type MuxPreambleRequest struct { | |
| 100 | + | Ceilings MuxCeilings | |
| 101 | + | Policy ports.BrokerPolicy | |
| 102 | + | } | |
| 103 | + | ||
| 104 | + | // MuxPreambleResponse is the validated daemonmux server preamble: the | |
| 105 | + | // effective ceilings, the authenticated daemon identity and incarnation, and | |
| 106 | + | // the accepted policy on acceptance; a precise refusal code otherwise. | |
| 107 | + | type MuxPreambleResponse struct { | |
| 108 | + | Ceilings MuxCeilings | |
| 109 | + | Policy ports.BrokerPolicy | |
| 110 | + | Identity ports.BrokerDaemonIdentity | |
| 111 | + | Incarnation ports.BrokerDaemonIncarnation | |
| 112 | + | Accepted bool | |
| 113 | + | Code uint32 | |
| 114 | + | } | |
| 115 | + | ||
| 116 | + | // EncodePreambleRequest builds the local daemonmux client preamble offer. It | |
| 117 | + | // fails closed on a ceiling advertisement outside its window or a policy that | |
| 118 | + | // could not be accepted. | |
| 119 | + | func EncodePreambleRequest(ceilings MuxCeilings, policy ports.BrokerPolicy) (*wire.MuxPreambleRequest, error) { | |
| 120 | + | if err := checkMuxCeilings(ceilings); err != nil { | |
| 121 | + | return nil, err | |
| 122 | + | } | |
| 123 | + | if err := policy.Validate(); err != nil { | |
| 124 | + | return nil, ErrPreambleRejected | |
| 125 | + | } | |
| 126 | + | return &wire.MuxPreambleRequest{ | |
| 127 | + | Magic: wire.PreambleMagic, | |
| 128 | + | Epoch: wire.ProtocolEpoch, | |
| 129 | + | Version: uint32(protocol.Version), | |
| 130 | + | Role: &wire.PreambleRole{Role: MuxRoleClient}, | |
| 131 | + | MaxReceiveEnvelopeBytes: ceilings.MaxReceiveEnvelopeBytes, | |
| 132 | + | StreamChunkLimit: ceilings.StreamChunkLimit, | |
| 133 | + | MaxStreams: ceilings.MaxStreams, | |
| 134 | + | MaxAggregateBytes: ceilings.MaxAggregateBytes, | |
| 135 | + | Policy: policyToWire(policy), | |
| 136 | + | }, nil | |
| 137 | + | } | |
| 138 | + | ||
| 139 | + | // EncodePreambleResponse builds the daemonmux server preamble answer: | |
| 140 | + | // acceptance with the effective ceilings, the authenticated daemon identity | |
| 141 | + | // and incarnation, and the accepted policy, or a typed refusal carrying the | |
| 142 | + | // precise rejection code. The answer is either an acceptance or a refusal and | |
| 143 | + | // never a blend: a refusal demands a code in 1..7 and carries no accepted | |
| 144 | + | // authority, while an acceptance demands a zero code and never carries a | |
| 145 | + | // rejection field. | |
| 146 | + | func EncodePreambleResponse(accepted bool, ceilings MuxCeilings, policy ports.BrokerPolicy, identity ports.BrokerDaemonIdentity, incarnation ports.BrokerDaemonIncarnation, code uint32) (*wire.MuxPreambleResponse, error) { | |
| 147 | + | response := &wire.MuxPreambleResponse{ | |
| 148 | + | Magic: wire.PreambleMagic, | |
| 149 | + | Epoch: wire.ProtocolEpoch, | |
| 150 | + | Version: uint32(protocol.Version), | |
| 151 | + | Role: &wire.PreambleRole{Role: MuxRoleServer}, | |
| 152 | + | MaxReceiveEnvelopeBytes: ceilings.MaxReceiveEnvelopeBytes, | |
| 153 | + | StreamChunkLimit: ceilings.StreamChunkLimit, | |
| 154 | + | MaxStreams: ceilings.MaxStreams, | |
| 155 | + | MaxAggregateBytes: ceilings.MaxAggregateBytes, | |
| 156 | + | Accepted: accepted, | |
| 157 | + | } | |
| 158 | + | if !accepted { | |
| 159 | + | if !validRejectionCode(code) { | |
| 160 | + | return nil, ErrPreambleRejected | |
| 161 | + | } | |
| 162 | + | response.Rejection = &wire.PreambleRejectionCode{Code: code} | |
| 163 | + | return response, nil | |
| 164 | + | } | |
| 165 | + | if code != 0 { | |
| 166 | + | // An acceptance carries no rejection field; a nonzero code is a | |
| 167 | + | // caller bug that must fail closed rather than be dropped silently. | |
| 168 | + | return nil, ErrPreambleRejected | |
| 169 | + | } | |
| 170 | + | if err := checkMuxCeilings(ceilings); err != nil { | |
| 171 | + | return nil, err | |
| 172 | + | } | |
| 173 | + | if err := policy.Validate(); err != nil { | |
| 174 | + | return nil, ErrPreambleRejected | |
| 175 | + | } | |
| 176 | + | if err := identity.Validate(); err != nil { | |
| 177 | + | return nil, ErrPreambleRejected | |
| 178 | + | } | |
| 179 | + | if err := incarnation.Validate(); err != nil { | |
| 180 | + | return nil, ErrPreambleRejected | |
| 181 | + | } | |
| 182 | + | response.AcceptedPolicy = policyToWire(policy) | |
| 183 | + | response.DaemonIdentity = string(identity) | |
| 184 | + | response.Incarnation = append([]byte(nil), incarnation[:]...) | |
| 185 | + | return response, nil | |
| 186 | + | } | |
| 187 | + | ||
| 188 | + | // DecodePreambleRequest validates one raw daemonmux client preamble. Magic, | |
| 189 | + | // epoch, version, role, and zero capability bits are checked before the | |
| 190 | + | // ceiling advertisement and the requested policy; the refusal code is | |
| 191 | + | // precise. | |
| 192 | + | func DecodePreambleRequest(message *wire.MuxPreambleRequest) (MuxPreambleRequest, error) { | |
| 193 | + | if message == nil { | |
| 194 | + | return MuxPreambleRequest{}, ErrPreambleRejected | |
| 195 | + | } | |
| 196 | + | if message.GetMagic() != wire.PreambleMagic { | |
| 197 | + | return MuxPreambleRequest{}, ErrPreambleRejected | |
| 198 | + | } | |
| 199 | + | if message.GetEpoch() != wire.ProtocolEpoch { | |
| 200 | + | return MuxPreambleRequest{}, ErrPreambleRejected | |
| 201 | + | } | |
| 202 | + | if message.GetVersion() != uint32(protocol.Version) { | |
| 203 | + | return MuxPreambleRequest{}, ErrPreambleRejected | |
| 204 | + | } | |
| 205 | + | if message.GetRole().GetRole() != MuxRoleClient { | |
| 206 | + | return MuxPreambleRequest{}, ErrPreambleRejected | |
| 207 | + | } | |
| 208 | + | if message.GetCapabilityBits() != 0 { | |
| 209 | + | return MuxPreambleRequest{}, ErrPreambleRejected | |
| 210 | + | } | |
| 211 | + | ceilings := MuxCeilings{ | |
| 212 | + | MaxReceiveEnvelopeBytes: message.GetMaxReceiveEnvelopeBytes(), | |
| 213 | + | StreamChunkLimit: message.GetStreamChunkLimit(), | |
| 214 | + | MaxStreams: message.GetMaxStreams(), | |
| 215 | + | MaxAggregateBytes: message.GetMaxAggregateBytes(), | |
| 216 | + | } | |
| 217 | + | if err := checkMuxCeilings(ceilings); err != nil { | |
| 218 | + | return MuxPreambleRequest{}, err | |
| 219 | + | } | |
| 220 | + | policy, err := policyFromWire(message.GetPolicy()) | |
| 221 | + | if err != nil { | |
| 222 | + | return MuxPreambleRequest{}, ErrPreambleRejected | |
| 223 | + | } | |
| 224 | + | return MuxPreambleRequest{Ceilings: ceilings, Policy: policy}, nil | |
| 225 | + | } | |
| 226 | + | ||
| 227 | + | // DecodePreambleResponse validates one raw daemonmux server preamble. | |
| 228 | + | // Refusals fail with ErrPreambleRejected and carry their code; acceptance | |
| 229 | + | // additionally validates the server role, the effective ceilings, the | |
| 230 | + | // accepted policy, the authenticated daemon identity, and the 16-byte | |
| 231 | + | // incarnation. An acceptance carrying a rejection field and a refusal carrying | |
| 232 | + | // accepted-only authority are both rejected, as is any refusal code outside | |
| 233 | + | // 1..7. | |
| 234 | + | func DecodePreambleResponse(message *wire.MuxPreambleResponse) (MuxPreambleResponse, error) { | |
| 235 | + | if message == nil { | |
| 236 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 237 | + | } | |
| 238 | + | if message.GetMagic() != wire.PreambleMagic { | |
| 239 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 240 | + | } | |
| 241 | + | if message.GetEpoch() != wire.ProtocolEpoch { | |
| 242 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 243 | + | } | |
| 244 | + | if message.GetVersion() != uint32(protocol.Version) { | |
| 245 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 246 | + | } | |
| 247 | + | if message.GetRole().GetRole() != MuxRoleServer { | |
| 248 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 249 | + | } | |
| 250 | + | if message.GetCapabilityBits() != 0 { | |
| 251 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 252 | + | } | |
| 253 | + | if !message.GetAccepted() { | |
| 254 | + | code := message.GetRejection().GetCode() | |
| 255 | + | if !validRejectionCode(code) { | |
| 256 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 257 | + | } | |
| 258 | + | if message.GetAcceptedPolicy() != nil || message.GetDaemonIdentity() != "" || len(message.GetIncarnation()) != 0 { | |
| 259 | + | // A refusal carries no accepted-only authority: accepted policy, | |
| 260 | + | // daemon identity, and incarnation belong to an acceptance alone. | |
| 261 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 262 | + | } | |
| 263 | + | return MuxPreambleResponse{Code: code}, ErrPreambleRejected | |
| 264 | + | } | |
| 265 | + | if message.GetRejection() != nil { | |
| 266 | + | // An acceptance carries no rejection field. | |
| 267 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 268 | + | } | |
| 269 | + | ceilings := MuxCeilings{ | |
| 270 | + | MaxReceiveEnvelopeBytes: message.GetMaxReceiveEnvelopeBytes(), | |
| 271 | + | StreamChunkLimit: message.GetStreamChunkLimit(), | |
| 272 | + | MaxStreams: message.GetMaxStreams(), | |
| 273 | + | MaxAggregateBytes: message.GetMaxAggregateBytes(), | |
| 274 | + | } | |
| 275 | + | if err := checkMuxCeilings(ceilings); err != nil { | |
| 276 | + | return MuxPreambleResponse{}, err | |
| 277 | + | } | |
| 278 | + | policy, err := policyFromWire(message.GetAcceptedPolicy()) | |
| 279 | + | if err != nil { | |
| 280 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 281 | + | } | |
| 282 | + | identity := ports.BrokerDaemonIdentity(message.GetDaemonIdentity()) | |
| 283 | + | if err := identity.Validate(); err != nil { | |
| 284 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 285 | + | } | |
| 286 | + | var incarnation ports.BrokerDaemonIncarnation | |
| 287 | + | raw := message.GetIncarnation() | |
| 288 | + | if len(raw) != len(incarnation) { | |
| 289 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 290 | + | } | |
| 291 | + | copy(incarnation[:], raw) | |
| 292 | + | if err := incarnation.Validate(); err != nil { | |
| 293 | + | return MuxPreambleResponse{}, ErrPreambleRejected | |
| 294 | + | } | |
| 295 | + | return MuxPreambleResponse{ | |
| 296 | + | Ceilings: ceilings, | |
| 297 | + | Policy: policy, | |
| 298 | + | Identity: identity, | |
| 299 | + | Incarnation: incarnation, | |
| 300 | + | Accepted: true, | |
| 301 | + | }, nil | |
| 302 | + | } | |
| 303 | + | ||
| 304 | + | // ValidatePreambleResponseAgainstOffer refuses an accepted daemonmux response | |
| 305 | + | // whose effective ceilings exceed the ceilings the local side offered. The | |
| 306 | + | // effective value is the element-wise minimum of both advertisements, so a | |
| 307 | + | // conforming daemon never grants more than the requester offered; a response | |
| 308 | + | // above the offer is a protocol violation. A refusal (which carries no | |
| 309 | + | // accepted ceilings), or an invalid advertisement on either side, is refused | |
| 310 | + | // with ErrPreambleRejected. A response strictly below the offer is accepted: | |
| 311 | + | // the daemon may lower any ceiling by its own policy. | |
| 312 | + | func ValidatePreambleResponseAgainstOffer(response MuxPreambleResponse, offered MuxCeilings) error { | |
| 313 | + | if err := checkMuxCeilings(offered); err != nil { | |
| 314 | + | return err | |
| 315 | + | } | |
| 316 | + | if !response.Accepted { | |
| 317 | + | return ErrPreambleRejected | |
| 318 | + | } | |
| 319 | + | if err := checkMuxCeilings(response.Ceilings); err != nil { | |
| 320 | + | return err | |
| 321 | + | } | |
| 322 | + | if response.Ceilings.MaxReceiveEnvelopeBytes > offered.MaxReceiveEnvelopeBytes || | |
| 323 | + | response.Ceilings.StreamChunkLimit > offered.StreamChunkLimit || | |
| 324 | + | response.Ceilings.MaxStreams > offered.MaxStreams || | |
| 325 | + | response.Ceilings.MaxAggregateBytes > offered.MaxAggregateBytes { | |
| 326 | + | return ErrPreambleRejected | |
| 327 | + | } | |
| 328 | + | return nil | |
| 329 | + | } | |
| 330 | + | ||
| 331 | + | // DecodePreambleRequestBytes strictly decodes one serialized daemonmux client | |
| 332 | + | // preamble. The 4 KiB bound is enforced before allocation, wire.ScanEnvelope | |
| 333 | + | // is the shared strict scanner that rejects unknown fields, duplicate singular | |
| 334 | + | // or oneof fields, trailing bytes, truncation, and empty payloads before | |
| 335 | + | // generated unmarshal, proto.UnmarshalOptions{DiscardUnknown: false} runs next, | |
| 336 | + | // and DecodePreambleRequest applies the semantic checks. A carriage failure | |
| 337 | + | // returns the precise wire sentinel (wire.ErrScanUnknown, wire.ErrScanDuplicate, | |
| 338 | + | // wire.ErrScanTrailing, wire.ErrScanTruncated, ...); an oversize payload or a | |
| 339 | + | // semantic failure returns ErrPreambleRejected. | |
| 340 | + | func DecodePreambleRequestBytes(payload []byte) (MuxPreambleRequest, error) { | |
| 341 | + | if err := CheckPreambleSize(payload); err != nil { | |
| 342 | + | return MuxPreambleRequest{}, err | |
| 343 | + | } | |
| 344 | + | message := &wire.MuxPreambleRequest{} | |
| 345 | + | if err := wire.ScanEnvelope(message, payload); err != nil { | |
| 346 | + | return MuxPreambleRequest{}, err | |
| 347 | + | } | |
| 348 | + | if err := (proto.UnmarshalOptions{DiscardUnknown: false}.Unmarshal(payload, message)); err != nil { | |
| 349 | + | return MuxPreambleRequest{}, err | |
| 350 | + | } | |
| 351 | + | return DecodePreambleRequest(message) | |
| 352 | + | } | |
| 353 | + | ||
| 354 | + | // DecodePreambleResponseBytes strictly decodes one serialized daemonmux server | |
| 355 | + | // preamble under the same 4 KiB bound, strict scan, generated unmarshal, and | |
| 356 | + | // semantic decode as DecodePreambleRequestBytes. Carriage failures return the | |
| 357 | + | // precise wire sentinel; oversize payloads and semantic failures return | |
| 358 | + | // ErrPreambleRejected. | |
| 359 | + | func DecodePreambleResponseBytes(payload []byte) (MuxPreambleResponse, error) { | |
| 360 | + | if err := CheckPreambleSize(payload); err != nil { | |
| 361 | + | return MuxPreambleResponse{}, err | |
| 362 | + | } | |
| 363 | + | message := &wire.MuxPreambleResponse{} | |
| 364 | + | if err := wire.ScanEnvelope(message, payload); err != nil { | |
| 365 | + | return MuxPreambleResponse{}, err | |
| 366 | + | } | |
| 367 | + | if err := (proto.UnmarshalOptions{DiscardUnknown: false}.Unmarshal(payload, message)); err != nil { | |
| 368 | + | return MuxPreambleResponse{}, err | |
| 369 | + | } | |
| 370 | + | return DecodePreambleResponse(message) | |
| 371 | + | } | |
| 372 | + | ||
| 373 | + | // RejectionCodeFor maps a preamble validation failure to its precise refusal | |
| 374 | + | // code. Limit, magic, epoch, version, role, and policy failures each carry | |
| 375 | + | // their own code; nonzero capability bits are refused as | |
| 376 | + | // RejectionLimitRefused because P3.2 negotiates no capabilities; unknown scan | |
| 377 | + | // failures are limit refusals. A duplicate inside one frame | |
| 378 | + | // (wire.ErrScanDuplicate) maps to RejectionDuplicate with the in-frame meaning | |
| 379 | + | // described above, never the dispatcher's cross-frame duplicate. Code 6 is | |
| 380 | + | // never returned here: the dispatcher owns out-of-order framing. | |
| 381 | + | func RejectionCodeFor(request *wire.MuxPreambleRequest, err error) uint32 { | |
| 382 | + | if err == nil { | |
| 383 | + | return 0 | |
| 384 | + | } | |
| 385 | + | if request != nil { | |
| 386 | + | if request.GetMagic() != wire.PreambleMagic { | |
| 387 | + | return RejectionBadMagic | |
| 388 | + | } | |
| 389 | + | if request.GetEpoch() != wire.ProtocolEpoch { | |
| 390 | + | return RejectionEpochMismatch | |
| 391 | + | } | |
| 392 | + | if request.GetVersion() != uint32(protocol.Version) { | |
| 393 | + | return RejectionVersionMismatch | |
| 394 | + | } | |
| 395 | + | if request.GetRole().GetRole() != MuxRoleClient { | |
| 396 | + | return RejectionWrongRole | |
| 397 | + | } | |
| 398 | + | if request.GetCapabilityBits() != 0 { | |
| 399 | + | // No capability is negotiable in P3.2, so any bit is a refused | |
| 400 | + | // limit advertisement (code 7), documented explicitly. | |
| 401 | + | return RejectionLimitRefused | |
| 402 | + | } | |
| 403 | + | remote := MuxCeilings{ | |
| 404 | + | MaxReceiveEnvelopeBytes: request.GetMaxReceiveEnvelopeBytes(), | |
| 405 | + | StreamChunkLimit: request.GetStreamChunkLimit(), | |
| 406 | + | MaxStreams: request.GetMaxStreams(), | |
| 407 | + | MaxAggregateBytes: request.GetMaxAggregateBytes(), | |
| 408 | + | } | |
| 409 | + | if checkMuxCeilings(remote) != nil { | |
| 410 | + | return RejectionLimitRefused | |
| 411 | + | } | |
| 412 | + | if _, err := policyFromWire(request.GetPolicy()); err != nil { | |
| 413 | + | // A malformed or unnegotiable requested policy is a refused | |
| 414 | + | // advertisement, not a transport failure. | |
| 415 | + | return RejectionLimitRefused | |
| 416 | + | } | |
| 417 | + | } | |
| 418 | + | if errors.Is(err, wire.ErrScanDuplicate) { | |
| 419 | + | return RejectionDuplicate | |
| 420 | + | } | |
| 421 | + | return RejectionLimitRefused | |
| 422 | + | } | |
A internal/adapters/daemonmux/preamble_test.go +735 -0
| @@ -0,0 +1,735 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "testing" | |
| 5 | + | ||
| 6 | + | "google.golang.org/protobuf/encoding/protowire" | |
| 7 | + | "google.golang.org/protobuf/proto" | |
| 8 | + | ||
| 9 | + | "github.com/bnema/vev/internal/ports" | |
| 10 | + | "github.com/bnema/vev/internal/protocol" | |
| 11 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 12 | + | "github.com/stretchr/testify/require" | |
| 13 | + | ) | |
| 14 | + | ||
| 15 | + | func testIdentity() ports.BrokerDaemonIdentity { | |
| 16 | + | return ports.BrokerDaemonIdentity("daemon-identity-1") | |
| 17 | + | } | |
| 18 | + | ||
| 19 | + | func testIncarnation() ports.BrokerDaemonIncarnation { | |
| 20 | + | return ports.BrokerDaemonIncarnation{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} | |
| 21 | + | } | |
| 22 | + | ||
| 23 | + | func mustEncodePreambleRequest(t *testing.T, ceilings MuxCeilings, policy ports.BrokerPolicy) *wire.MuxPreambleRequest { | |
| 24 | + | t.Helper() | |
| 25 | + | request, err := EncodePreambleRequest(ceilings, policy) | |
| 26 | + | require.NoError(t, err) | |
| 27 | + | return request | |
| 28 | + | } | |
| 29 | + | ||
| 30 | + | func mustEncodePreambleResponse(t *testing.T, accepted bool, ceilings MuxCeilings, policy ports.BrokerPolicy, identity ports.BrokerDaemonIdentity, incarnation ports.BrokerDaemonIncarnation, code uint32) *wire.MuxPreambleResponse { | |
| 31 | + | t.Helper() | |
| 32 | + | response, err := EncodePreambleResponse(accepted, ceilings, policy, identity, incarnation, code) | |
| 33 | + | require.NoError(t, err) | |
| 34 | + | return response | |
| 35 | + | } | |
| 36 | + | ||
| 37 | + | // TestMuxPreambleByteLengthHelper proves the exposed 4 KiB byte-length helper | |
| 38 | + | // accepts at the boundary and refuses one byte over, for both encoded and | |
| 39 | + | // received preambles. | |
| 40 | + | func TestMuxPreambleByteLengthHelper(t *testing.T) { | |
| 41 | + | require.Equal(t, wire.PreambleLimit, MuxPreambleLimit) | |
| 42 | + | require.Equal(t, 4<<10, MuxPreambleLimit) | |
| 43 | + | require.NoError(t, CheckPreambleSize(nil)) | |
| 44 | + | require.NoError(t, CheckPreambleSize(make([]byte, MuxPreambleLimit))) | |
| 45 | + | require.ErrorIs(t, CheckPreambleSize(make([]byte, MuxPreambleLimit+1)), ErrPreambleRejected) | |
| 46 | + | ||
| 47 | + | request, err := proto.Marshal(mustEncodePreambleRequest(t, DefaultMuxCeilings(), testPolicy())) | |
| 48 | + | require.NoError(t, err) | |
| 49 | + | require.NoError(t, CheckPreambleSize(request)) | |
| 50 | + | response, err := proto.Marshal(mustEncodePreambleResponse(t, true, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), 0)) | |
| 51 | + | require.NoError(t, err) | |
| 52 | + | require.NoError(t, CheckPreambleSize(response)) | |
| 53 | + | } | |
| 54 | + | ||
| 55 | + | // TestMuxPreambleRoles proves the daemonmux preamble uses roles 5/6, the | |
| 56 | + | // shared magic/epoch/exact version, zero capability bits, the 4 KiB preamble | |
| 57 | + | // bound, and precise refusal mapping. | |
| 58 | + | func TestMuxPreambleRoles(t *testing.T) { | |
| 59 | + | t.Run("client role is 5 and server role is 6", func(t *testing.T) { | |
| 60 | + | request := mustEncodePreambleRequest(t, DefaultMuxCeilings(), testPolicy()) | |
| 61 | + | require.Equal(t, MuxRoleClient, request.GetRole().GetRole()) | |
| 62 | + | require.Equal(t, uint32(5), request.GetRole().GetRole()) | |
| 63 | + | response := mustEncodePreambleResponse(t, true, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), 0) | |
| 64 | + | require.Equal(t, MuxRoleServer, response.GetRole().GetRole()) | |
| 65 | + | require.Equal(t, uint32(6), response.GetRole().GetRole()) | |
| 66 | + | // The daemonmux roles are disjoint from the session (1/2) and broker | |
| 67 | + | // (3/4) roles. | |
| 68 | + | require.NotEqual(t, MuxRoleClient, MuxRoleServer) | |
| 69 | + | require.NotContains(t, []uint32{1, 2, 3, 4}, MuxRoleClient) | |
| 70 | + | require.NotContains(t, []uint32{1, 2, 3, 4}, MuxRoleServer) | |
| 71 | + | }) | |
| 72 | + | t.Run("magic epoch version match the shared contract", func(t *testing.T) { | |
| 73 | + | request := mustEncodePreambleRequest(t, DefaultMuxCeilings(), testPolicy()) | |
| 74 | + | require.Equal(t, wire.PreambleMagic, request.GetMagic()) | |
| 75 | + | require.Equal(t, wire.ProtocolEpoch, request.GetEpoch()) | |
| 76 | + | require.Equal(t, uint32(protocol.Version), request.GetVersion()) | |
| 77 | + | require.Zero(t, request.GetCapabilityBits()) | |
| 78 | + | response := mustEncodePreambleResponse(t, true, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), 0) | |
| 79 | + | require.Equal(t, wire.PreambleMagic, response.GetMagic()) | |
| 80 | + | require.Equal(t, wire.ProtocolEpoch, response.GetEpoch()) | |
| 81 | + | require.Equal(t, uint32(protocol.Version), response.GetVersion()) | |
| 82 | + | require.Zero(t, response.GetCapabilityBits()) | |
| 83 | + | }) | |
| 84 | + | t.Run("preamble fits 4KiB with a maximum identity", func(t *testing.T) { | |
| 85 | + | identity := ports.BrokerDaemonIdentity(string(make([]byte, 0, ports.BrokerMaxIdentityBytes))) | |
| 86 | + | for range ports.BrokerMaxIdentityBytes { | |
| 87 | + | identity += "a" | |
| 88 | + | } | |
| 89 | + | require.NoError(t, identity.Validate()) | |
| 90 | + | raw, err := proto.Marshal(mustEncodePreambleResponse(t, true, DefaultMuxCeilings(), testPolicy(), identity, testIncarnation(), 0)) | |
| 91 | + | require.NoError(t, err) | |
| 92 | + | require.LessOrEqual(t, len(raw), MuxPreambleLimit) | |
| 93 | + | require.Equal(t, 4<<10, MuxPreambleLimit) | |
| 94 | + | raw, err = proto.Marshal(mustEncodePreambleRequest(t, DefaultMuxCeilings(), testPolicy())) | |
| 95 | + | require.NoError(t, err) | |
| 96 | + | require.LessOrEqual(t, len(raw), MuxPreambleLimit) | |
| 97 | + | }) | |
| 98 | + | t.Run("foreign roles rejected", func(t *testing.T) { | |
| 99 | + | request := mustEncodePreambleRequest(t, DefaultMuxCeilings(), testPolicy()) | |
| 100 | + | for _, role := range []uint32{1, 2, 3, 4, 6, 9} { | |
| 101 | + | request.Role.Role = role | |
| 102 | + | _, err := DecodePreambleRequest(request) | |
| 103 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 104 | + | require.Equal(t, RejectionWrongRole, RejectionCodeFor(request, err)) | |
| 105 | + | } | |
| 106 | + | response := mustEncodePreambleResponse(t, true, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), 0) | |
| 107 | + | for _, role := range []uint32{1, 2, 3, 4, 5, 9} { | |
| 108 | + | response.Role.Role = role | |
| 109 | + | _, err := DecodePreambleResponse(response) | |
| 110 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 111 | + | } | |
| 112 | + | }) | |
| 113 | + | t.Run("caps zero", func(t *testing.T) { | |
| 114 | + | request := mustEncodePreambleRequest(t, DefaultMuxCeilings(), testPolicy()) | |
| 115 | + | request.CapabilityBits = 1 | |
| 116 | + | _, err := DecodePreambleRequest(request) | |
| 117 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 118 | + | require.Equal(t, RejectionLimitRefused, RejectionCodeFor(request, err)) | |
| 119 | + | response := mustEncodePreambleResponse(t, true, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), 0) | |
| 120 | + | response.CapabilityBits = 7 | |
| 121 | + | _, err = DecodePreambleResponse(response) | |
| 122 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 123 | + | }) | |
| 124 | + | t.Run("nil preambles rejected", func(t *testing.T) { | |
| 125 | + | _, err := DecodePreambleRequest(nil) | |
| 126 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 127 | + | _, err = DecodePreambleResponse(nil) | |
| 128 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 129 | + | }) | |
| 130 | + | } | |
| 131 | + | ||
| 132 | + | // TestMuxPreambleCeilingWindows proves every advertised window is refused at | |
| 133 | + | // max-1 outside the floor, accepted at the window edges, and refused one byte | |
| 134 | + | // above the ceiling. | |
| 135 | + | func TestMuxPreambleCeilingWindows(t *testing.T) { | |
| 136 | + | mutate := func(name string, mutate func(*wire.MuxPreambleRequest)) { | |
| 137 | + | t.Run(name, func(t *testing.T) { | |
| 138 | + | request := mustEncodePreambleRequest(t, DefaultMuxCeilings(), testPolicy()) | |
| 139 | + | mutate(request) | |
| 140 | + | _, err := DecodePreambleRequest(request) | |
| 141 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 142 | + | require.Equal(t, RejectionLimitRefused, RejectionCodeFor(request, err)) | |
| 143 | + | }) | |
| 144 | + | } | |
| 145 | + | accept := func(name string, mutate func(*wire.MuxPreambleRequest), want MuxCeilings) { | |
| 146 | + | t.Run(name, func(t *testing.T) { | |
| 147 | + | request := mustEncodePreambleRequest(t, DefaultMuxCeilings(), testPolicy()) | |
| 148 | + | mutate(request) | |
| 149 | + | decoded, err := DecodePreambleRequest(request) | |
| 150 | + | require.NoError(t, err) | |
| 151 | + | require.Equal(t, want, decoded.Ceilings) | |
| 152 | + | }) | |
| 153 | + | } | |
| 154 | + | defaults := DefaultMuxCeilings() | |
| 155 | + | ||
| 156 | + | t.Run("envelope window 1MiB..16MiB", func(t *testing.T) { | |
| 157 | + | require.Equal(t, uint64(1<<20), MinMuxEnvelopeBytes) | |
| 158 | + | require.Equal(t, uint64(16<<20), MaxMuxEnvelopeBytes) | |
| 159 | + | mutate("below floor", func(r *wire.MuxPreambleRequest) { r.MaxReceiveEnvelopeBytes = MinMuxEnvelopeBytes - 1 }) | |
| 160 | + | mutate("above ceiling", func(r *wire.MuxPreambleRequest) { r.MaxReceiveEnvelopeBytes = MaxMuxEnvelopeBytes + 1 }) | |
| 161 | + | mutate("zero", func(r *wire.MuxPreambleRequest) { r.MaxReceiveEnvelopeBytes = 0 }) | |
| 162 | + | accept("at floor", func(r *wire.MuxPreambleRequest) { r.MaxReceiveEnvelopeBytes = MinMuxEnvelopeBytes }, MuxCeilings{ | |
| 163 | + | MaxReceiveEnvelopeBytes: MinMuxEnvelopeBytes, | |
| 164 | + | StreamChunkLimit: defaults.StreamChunkLimit, | |
| 165 | + | MaxStreams: defaults.MaxStreams, | |
| 166 | + | MaxAggregateBytes: defaults.MaxAggregateBytes, | |
| 167 | + | }) | |
| 168 | + | accept("floor+1", func(r *wire.MuxPreambleRequest) { r.MaxReceiveEnvelopeBytes = MinMuxEnvelopeBytes + 1 }, MuxCeilings{ | |
| 169 | + | MaxReceiveEnvelopeBytes: MinMuxEnvelopeBytes + 1, | |
| 170 | + | StreamChunkLimit: defaults.StreamChunkLimit, | |
| 171 | + | MaxStreams: defaults.MaxStreams, | |
| 172 | + | MaxAggregateBytes: defaults.MaxAggregateBytes, | |
| 173 | + | }) | |
| 174 | + | accept("at ceiling", func(r *wire.MuxPreambleRequest) { r.MaxReceiveEnvelopeBytes = MaxMuxEnvelopeBytes }, defaults) | |
| 175 | + | accept("ceiling-1", func(r *wire.MuxPreambleRequest) { r.MaxReceiveEnvelopeBytes = MaxMuxEnvelopeBytes - 1 }, MuxCeilings{ | |
| 176 | + | MaxReceiveEnvelopeBytes: MaxMuxEnvelopeBytes - 1, | |
| 177 | + | StreamChunkLimit: defaults.StreamChunkLimit, | |
| 178 | + | MaxStreams: defaults.MaxStreams, | |
| 179 | + | MaxAggregateBytes: defaults.MaxAggregateBytes, | |
| 180 | + | }) | |
| 181 | + | }) | |
| 182 | + | t.Run("chunk window 1..64KiB", func(t *testing.T) { | |
| 183 | + | require.Equal(t, uint64(1), MinMuxChunkBytes) | |
| 184 | + | require.Equal(t, uint64(64<<10), MaxMuxChunkBytes) | |
| 185 | + | mutate("zero", func(r *wire.MuxPreambleRequest) { r.StreamChunkLimit = 0 }) | |
| 186 | + | mutate("above ceiling", func(r *wire.MuxPreambleRequest) { r.StreamChunkLimit = MaxMuxChunkBytes + 1 }) | |
| 187 | + | accept("at floor", func(r *wire.MuxPreambleRequest) { r.StreamChunkLimit = MinMuxChunkBytes }, MuxCeilings{ | |
| 188 | + | MaxReceiveEnvelopeBytes: defaults.MaxReceiveEnvelopeBytes, | |
| 189 | + | StreamChunkLimit: MinMuxChunkBytes, | |
| 190 | + | MaxStreams: defaults.MaxStreams, | |
| 191 | + | MaxAggregateBytes: defaults.MaxAggregateBytes, | |
| 192 | + | }) | |
| 193 | + | accept("floor+1", func(r *wire.MuxPreambleRequest) { r.StreamChunkLimit = MinMuxChunkBytes + 1 }, MuxCeilings{ | |
| 194 | + | MaxReceiveEnvelopeBytes: defaults.MaxReceiveEnvelopeBytes, | |
| 195 | + | StreamChunkLimit: MinMuxChunkBytes + 1, | |
| 196 | + | MaxStreams: defaults.MaxStreams, | |
| 197 | + | MaxAggregateBytes: defaults.MaxAggregateBytes, | |
| 198 | + | }) | |
| 199 | + | accept("at ceiling", func(r *wire.MuxPreambleRequest) { r.StreamChunkLimit = MaxMuxChunkBytes }, defaults) | |
| 200 | + | accept("ceiling-1", func(r *wire.MuxPreambleRequest) { r.StreamChunkLimit = MaxMuxChunkBytes - 1 }, MuxCeilings{ | |
| 201 | + | MaxReceiveEnvelopeBytes: defaults.MaxReceiveEnvelopeBytes, | |
| 202 | + | StreamChunkLimit: MaxMuxChunkBytes - 1, | |
| 203 | + | MaxStreams: defaults.MaxStreams, | |
| 204 | + | MaxAggregateBytes: defaults.MaxAggregateBytes, | |
| 205 | + | }) | |
| 206 | + | }) | |
| 207 | + | t.Run("stream window 1..128", func(t *testing.T) { | |
| 208 | + | require.Equal(t, uint64(1), MinMuxStreams) | |
| 209 | + | require.Equal(t, uint64(128), MaxMuxStreams) | |
| 210 | + | mutate("zero", func(r *wire.MuxPreambleRequest) { r.MaxStreams = 0 }) | |
| 211 | + | mutate("above ceiling", func(r *wire.MuxPreambleRequest) { r.MaxStreams = MaxMuxStreams + 1 }) | |
| 212 | + | accept("at floor", func(r *wire.MuxPreambleRequest) { r.MaxStreams = MinMuxStreams }, MuxCeilings{ | |
| 213 | + | MaxReceiveEnvelopeBytes: defaults.MaxReceiveEnvelopeBytes, | |
| 214 | + | StreamChunkLimit: defaults.StreamChunkLimit, | |
| 215 | + | MaxStreams: MinMuxStreams, | |
| 216 | + | MaxAggregateBytes: defaults.MaxAggregateBytes, | |
| 217 | + | }) | |
| 218 | + | accept("floor+1", func(r *wire.MuxPreambleRequest) { r.MaxStreams = MinMuxStreams + 1 }, MuxCeilings{ | |
| 219 | + | MaxReceiveEnvelopeBytes: defaults.MaxReceiveEnvelopeBytes, | |
| 220 | + | StreamChunkLimit: defaults.StreamChunkLimit, | |
| 221 | + | MaxStreams: MinMuxStreams + 1, | |
| 222 | + | MaxAggregateBytes: defaults.MaxAggregateBytes, | |
| 223 | + | }) | |
| 224 | + | accept("at ceiling", func(r *wire.MuxPreambleRequest) { r.MaxStreams = MaxMuxStreams }, defaults) | |
| 225 | + | accept("ceiling-1", func(r *wire.MuxPreambleRequest) { r.MaxStreams = MaxMuxStreams - 1 }, MuxCeilings{ | |
| 226 | + | MaxReceiveEnvelopeBytes: defaults.MaxReceiveEnvelopeBytes, | |
| 227 | + | StreamChunkLimit: defaults.StreamChunkLimit, | |
| 228 | + | MaxStreams: MaxMuxStreams - 1, | |
| 229 | + | MaxAggregateBytes: defaults.MaxAggregateBytes, | |
| 230 | + | }) | |
| 231 | + | }) | |
| 232 | + | t.Run("aggregate window floor..64MiB", func(t *testing.T) { | |
| 233 | + | require.Equal(t, MaxMuxChunkBytes+MuxEnvelopeOverheadBytes, MinMuxAggregateBytes) | |
| 234 | + | require.Equal(t, uint64(64<<20), MaxMuxAggregateBytes) | |
| 235 | + | mutate("zero", func(r *wire.MuxPreambleRequest) { r.MaxAggregateBytes = 0 }) | |
| 236 | + | mutate("above ceiling", func(r *wire.MuxPreambleRequest) { r.MaxAggregateBytes = MaxMuxAggregateBytes + 1 }) | |
| 237 | + | accept("at floor", func(r *wire.MuxPreambleRequest) { r.MaxAggregateBytes = MinMuxAggregateBytes }, MuxCeilings{ | |
| 238 | + | MaxReceiveEnvelopeBytes: defaults.MaxReceiveEnvelopeBytes, | |
| 239 | + | StreamChunkLimit: defaults.StreamChunkLimit, | |
| 240 | + | MaxStreams: defaults.MaxStreams, | |
| 241 | + | MaxAggregateBytes: MinMuxAggregateBytes, | |
| 242 | + | }) | |
| 243 | + | accept("at ceiling", func(r *wire.MuxPreambleRequest) { r.MaxAggregateBytes = MaxMuxAggregateBytes }, defaults) | |
| 244 | + | accept("ceiling-1", func(r *wire.MuxPreambleRequest) { r.MaxAggregateBytes = MaxMuxAggregateBytes - 1 }, MuxCeilings{ | |
| 245 | + | MaxReceiveEnvelopeBytes: defaults.MaxReceiveEnvelopeBytes, | |
| 246 | + | StreamChunkLimit: defaults.StreamChunkLimit, | |
| 247 | + | MaxStreams: defaults.MaxStreams, | |
| 248 | + | MaxAggregateBytes: MaxMuxAggregateBytes - 1, | |
| 249 | + | }) | |
| 250 | + | }) | |
| 251 | + | } | |
| 252 | + | ||
| 253 | + | // TestMuxPreambleNegotiatesMinima proves the effective ceilings are the | |
| 254 | + | // element-wise minima of both advertisements, round-tripped through the wire. | |
| 255 | + | func TestMuxPreambleNegotiatesMinima(t *testing.T) { | |
| 256 | + | local := DefaultMuxCeilings() | |
| 257 | + | remote := MuxCeilings{ | |
| 258 | + | MaxReceiveEnvelopeBytes: MinMuxEnvelopeBytes, | |
| 259 | + | StreamChunkLimit: 1024, | |
| 260 | + | MaxStreams: 8, | |
| 261 | + | MaxAggregateBytes: 1 << 20, | |
| 262 | + | } | |
| 263 | + | decoded, err := DecodePreambleRequest(mustEncodePreambleRequest(t, remote, testPolicy())) | |
| 264 | + | require.NoError(t, err) | |
| 265 | + | require.Equal(t, remote, decoded.Ceilings) | |
| 266 | + | require.Equal(t, remote, EffectiveMuxCeilings(local, decoded.Ceilings)) | |
| 267 | + | ||
| 268 | + | // A response advertising the same minima round-trips them. | |
| 269 | + | response := mustEncodePreambleResponse(t, true, remote, testPolicy(), testIdentity(), testIncarnation(), 0) | |
| 270 | + | decodedResponse, err := DecodePreambleResponse(response) | |
| 271 | + | require.NoError(t, err) | |
| 272 | + | require.True(t, decodedResponse.Accepted) | |
| 273 | + | require.Equal(t, remote, decodedResponse.Ceilings) | |
| 274 | + | ||
| 275 | + | // The local default is never above the absolute ceilings. | |
| 276 | + | require.Equal(t, MaxMuxEnvelopeBytes, local.MaxReceiveEnvelopeBytes) | |
| 277 | + | require.Equal(t, MaxMuxChunkBytes, local.StreamChunkLimit) | |
| 278 | + | require.Equal(t, MaxMuxStreams, local.MaxStreams) | |
| 279 | + | require.Equal(t, MaxMuxAggregateBytes, local.MaxAggregateBytes) | |
| 280 | + | } | |
| 281 | + | ||
| 282 | + | // TestEffectiveMuxCeilings proves the effective ceilings are the element-wise | |
| 283 | + | // minima of both advertisements: a lower remote offer lowers only that field, | |
| 284 | + | // and the result is symmetric and idempotent. | |
| 285 | + | func TestEffectiveMuxCeilings(t *testing.T) { | |
| 286 | + | local := DefaultMuxCeilings() | |
| 287 | + | remote := MuxCeilings{ | |
| 288 | + | MaxReceiveEnvelopeBytes: local.MaxReceiveEnvelopeBytes - 1, | |
| 289 | + | StreamChunkLimit: local.StreamChunkLimit, | |
| 290 | + | MaxStreams: 4, | |
| 291 | + | MaxAggregateBytes: local.MaxAggregateBytes, | |
| 292 | + | } | |
| 293 | + | effective := EffectiveMuxCeilings(local, remote) | |
| 294 | + | require.Equal(t, remote.MaxReceiveEnvelopeBytes, effective.MaxReceiveEnvelopeBytes) | |
| 295 | + | require.Equal(t, local.StreamChunkLimit, effective.StreamChunkLimit) | |
| 296 | + | require.Equal(t, uint64(4), effective.MaxStreams) | |
| 297 | + | require.Equal(t, local.MaxAggregateBytes, effective.MaxAggregateBytes) | |
| 298 | + | ||
| 299 | + | require.Equal(t, effective, EffectiveMuxCeilings(remote, local)) | |
| 300 | + | require.Equal(t, effective, EffectiveMuxCeilings(effective, effective)) | |
| 301 | + | } | |
| 302 | + | ||
| 303 | + | // TestValidatePreambleResponseAgainstOffer proves an accepted response is | |
| 304 | + | // refused when any effective ceiling exceeds the local offer, and accepted | |
| 305 | + | // when every ceiling is at or below it. A refusal, or an invalid offer, is a | |
| 306 | + | // protocol violation. | |
| 307 | + | func TestValidatePreambleResponseAgainstOffer(t *testing.T) { | |
| 308 | + | offered := MuxCeilings{ | |
| 309 | + | MaxReceiveEnvelopeBytes: MinMuxEnvelopeBytes + 1, | |
| 310 | + | StreamChunkLimit: MinMuxChunkBytes + 1, | |
| 311 | + | MaxStreams: MinMuxStreams + 1, | |
| 312 | + | MaxAggregateBytes: MinMuxAggregateBytes + 1, | |
| 313 | + | } | |
| 314 | + | accepted := func(mutate func(*MuxCeilings)) MuxPreambleResponse { | |
| 315 | + | ceilings := offered | |
| 316 | + | if mutate != nil { | |
| 317 | + | mutate(&ceilings) | |
| 318 | + | } | |
| 319 | + | return MuxPreambleResponse{Accepted: true, Ceilings: ceilings} | |
| 320 | + | } | |
| 321 | + | ||
| 322 | + | cases := []struct { | |
| 323 | + | name string | |
| 324 | + | response MuxPreambleResponse | |
| 325 | + | wantErr bool | |
| 326 | + | }{ | |
| 327 | + | {"at offer", accepted(nil), false}, | |
| 328 | + | {"below offer", accepted(func(c *MuxCeilings) { | |
| 329 | + | c.MaxReceiveEnvelopeBytes = MinMuxEnvelopeBytes | |
| 330 | + | c.StreamChunkLimit = MinMuxChunkBytes | |
| 331 | + | c.MaxStreams = MinMuxStreams | |
| 332 | + | c.MaxAggregateBytes = MinMuxAggregateBytes | |
| 333 | + | }), false}, | |
| 334 | + | {"envelope above offer", accepted(func(c *MuxCeilings) { c.MaxReceiveEnvelopeBytes++ }), true}, | |
| 335 | + | {"chunk above offer", accepted(func(c *MuxCeilings) { c.StreamChunkLimit++ }), true}, | |
| 336 | + | {"streams above offer", accepted(func(c *MuxCeilings) { c.MaxStreams++ }), true}, | |
| 337 | + | {"aggregate above offer", accepted(func(c *MuxCeilings) { c.MaxAggregateBytes++ }), true}, | |
| 338 | + | } | |
| 339 | + | for _, tc := range cases { | |
| 340 | + | t.Run(tc.name, func(t *testing.T) { | |
| 341 | + | err := ValidatePreambleResponseAgainstOffer(tc.response, offered) | |
| 342 | + | if tc.wantErr { | |
| 343 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 344 | + | return | |
| 345 | + | } | |
| 346 | + | require.NoError(t, err) | |
| 347 | + | }) | |
| 348 | + | } | |
| 349 | + | ||
| 350 | + | t.Run("refusal", func(t *testing.T) { | |
| 351 | + | response := MuxPreambleResponse{Accepted: false, Code: RejectionLimitRefused} | |
| 352 | + | require.ErrorIs(t, ValidatePreambleResponseAgainstOffer(response, offered), ErrPreambleRejected) | |
| 353 | + | }) | |
| 354 | + | ||
| 355 | + | t.Run("invalid offer", func(t *testing.T) { | |
| 356 | + | require.ErrorIs(t, ValidatePreambleResponseAgainstOffer(accepted(nil), MuxCeilings{}), ErrPreambleRejected) | |
| 357 | + | }) | |
| 358 | + | } | |
| 359 | + | ||
| 360 | + | // TestMuxPreambleAcceptCarriesAuthority proves an acceptance carries the | |
| 361 | + | // authenticated daemon identity, the 16-byte incarnation, and the accepted | |
| 362 | + | // policy losslessly, and that a request carries the requested policy. | |
| 363 | + | func TestMuxPreambleAcceptCarriesAuthority(t *testing.T) { | |
| 364 | + | request, err := proto.Marshal(mustEncodePreambleRequest(t, DefaultMuxCeilings(), testPolicy())) | |
| 365 | + | require.NoError(t, err) | |
| 366 | + | decodedRequest := &wire.MuxPreambleRequest{} | |
| 367 | + | require.NoError(t, proto.Unmarshal(request, decodedRequest)) | |
| 368 | + | parsedRequest, err := DecodePreambleRequest(decodedRequest) | |
| 369 | + | require.NoError(t, err) | |
| 370 | + | require.Equal(t, testPolicy(), parsedRequest.Policy) | |
| 371 | + | ||
| 372 | + | response, err := proto.Marshal(mustEncodePreambleResponse(t, true, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), 0)) | |
| 373 | + | require.NoError(t, err) | |
| 374 | + | decodedResponse := &wire.MuxPreambleResponse{} | |
| 375 | + | require.NoError(t, proto.Unmarshal(response, decodedResponse)) | |
| 376 | + | parsedResponse, err := DecodePreambleResponse(decodedResponse) | |
| 377 | + | require.NoError(t, err) | |
| 378 | + | require.True(t, parsedResponse.Accepted) | |
| 379 | + | require.Equal(t, testPolicy(), parsedResponse.Policy) | |
| 380 | + | require.Equal(t, testIdentity(), parsedResponse.Identity) | |
| 381 | + | require.Equal(t, testIncarnation(), parsedResponse.Incarnation) | |
| 382 | + | require.Zero(t, parsedResponse.Code) | |
| 383 | + | ||
| 384 | + | // The accepted policy is exactly the requested policy when they match; | |
| 385 | + | // a different accepted policy stays visible to the caller. | |
| 386 | + | other := testPolicy() | |
| 387 | + | other.Trust = "tofu" | |
| 388 | + | acceptedOther := mustEncodePreambleResponse(t, true, DefaultMuxCeilings(), other, testIdentity(), testIncarnation(), 0) | |
| 389 | + | parsedOther, err := DecodePreambleResponse(acceptedOther) | |
| 390 | + | require.NoError(t, err) | |
| 391 | + | require.Equal(t, other, parsedOther.Policy) | |
| 392 | + | require.NotEqual(t, parsedRequest.Policy, parsedOther.Policy) | |
| 393 | + | require.False(t, parsedRequest.Policy.Compatible(parsedOther.Policy)) | |
| 394 | + | } | |
| 395 | + | ||
| 396 | + | // TestMuxPreambleIdentityChecks proves the accepted identity and incarnation | |
| 397 | + | // are mandatory, exact, and validated: they authorize the physical | |
| 398 | + | // connection, so a malformed one is never accepted silently. | |
| 399 | + | func TestMuxPreambleIdentityChecks(t *testing.T) { | |
| 400 | + | t.Run("encode refuses a missing or invalid authority", func(t *testing.T) { | |
| 401 | + | _, err := EncodePreambleResponse(true, DefaultMuxCeilings(), testPolicy(), "", testIncarnation(), 0) | |
| 402 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 403 | + | _, err = EncodePreambleResponse(true, DefaultMuxCeilings(), testPolicy(), ports.BrokerDaemonIdentity("has space"), testIncarnation(), 0) | |
| 404 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 405 | + | _, err = EncodePreambleResponse(true, DefaultMuxCeilings(), testPolicy(), testIdentity(), ports.BrokerDaemonIncarnation{}, 0) | |
| 406 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 407 | + | }) | |
| 408 | + | t.Run("decode refuses mutated authority", func(t *testing.T) { | |
| 409 | + | cases := map[string]func(*wire.MuxPreambleResponse){ | |
| 410 | + | "empty identity": func(r *wire.MuxPreambleResponse) { r.DaemonIdentity = "" }, | |
| 411 | + | "blank identity": func(r *wire.MuxPreambleResponse) { r.DaemonIdentity = " " }, | |
| 412 | + | "control identity": func(r *wire.MuxPreambleResponse) { r.DaemonIdentity = "id\x01" }, | |
| 413 | + | "short incarnation": func(r *wire.MuxPreambleResponse) { r.Incarnation = r.Incarnation[:15] }, | |
| 414 | + | "long incarnation": func(r *wire.MuxPreambleResponse) { r.Incarnation = append(r.Incarnation, 0) }, | |
| 415 | + | "zero incarnation": func(r *wire.MuxPreambleResponse) { r.Incarnation = make([]byte, 16) }, | |
| 416 | + | "missing policy": func(r *wire.MuxPreambleResponse) { r.AcceptedPolicy = nil }, | |
| 417 | + | "invalid policy token": func(r *wire.MuxPreambleResponse) { r.AcceptedPolicy.Trust = "" }, | |
| 418 | + | } | |
| 419 | + | for name, mutate := range cases { | |
| 420 | + | t.Run(name, func(t *testing.T) { | |
| 421 | + | response := mustEncodePreambleResponse(t, true, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), 0) | |
| 422 | + | mutate(response) | |
| 423 | + | _, err := DecodePreambleResponse(response) | |
| 424 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 425 | + | }) | |
| 426 | + | } | |
| 427 | + | }) | |
| 428 | + | t.Run("oversize authority is refused", func(t *testing.T) { | |
| 429 | + | identity := ports.BrokerDaemonIdentity(string(make([]byte, 0, ports.BrokerMaxIdentityBytes+1))) | |
| 430 | + | for range ports.BrokerMaxIdentityBytes + 1 { | |
| 431 | + | identity += "a" | |
| 432 | + | } | |
| 433 | + | _, err := EncodePreambleResponse(true, DefaultMuxCeilings(), testPolicy(), identity, testIncarnation(), 0) | |
| 434 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 435 | + | }) | |
| 436 | + | } | |
| 437 | + | ||
| 438 | + | // TestMuxPreambleRefusal proves refusals carry the precise code, that a | |
| 439 | + | // code-less refusal is refused locally, and that a request with an | |
| 440 | + | // unnegotiable policy is a refusal, not a decode error class of its own. | |
| 441 | + | func TestMuxPreambleRefusal(t *testing.T) { | |
| 442 | + | for name, code := range map[string]uint32{ | |
| 443 | + | "bad magic": RejectionBadMagic, | |
| 444 | + | "epoch": RejectionEpochMismatch, | |
| 445 | + | "version": RejectionVersionMismatch, | |
| 446 | + | "wrong role": RejectionWrongRole, | |
| 447 | + | "duplicate": RejectionDuplicate, | |
| 448 | + | "out of order": RejectionOutOfOrder, | |
| 449 | + | "limit refused": RejectionLimitRefused, | |
| 450 | + | "invalid policy": RejectionLimitRefused, | |
| 451 | + | "capability bits": RejectionLimitRefused, | |
| 452 | + | "envelope floor": RejectionLimitRefused, | |
| 453 | + | "chunk limit": RejectionLimitRefused, | |
| 454 | + | "stream limit": RejectionLimitRefused, | |
| 455 | + | "aggregate limit": RejectionLimitRefused, | |
| 456 | + | "zero refusal code": RejectionLimitRefused, | |
| 457 | + | } { | |
| 458 | + | t.Run("response/"+name, func(t *testing.T) { | |
| 459 | + | response := mustEncodePreambleResponse(t, false, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), code) | |
| 460 | + | require.False(t, response.GetAccepted()) | |
| 461 | + | require.Equal(t, code, response.GetRejection().GetCode()) | |
| 462 | + | decoded, err := DecodePreambleResponse(response) | |
| 463 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 464 | + | require.Equal(t, code, decoded.Code) | |
| 465 | + | require.False(t, decoded.Accepted) | |
| 466 | + | // A refusal never claims accepted authority. | |
| 467 | + | require.Nil(t, response.GetAcceptedPolicy()) | |
| 468 | + | require.Empty(t, response.GetDaemonIdentity()) | |
| 469 | + | }) | |
| 470 | + | } | |
| 471 | + | t.Run("code-less refusal is refused locally", func(t *testing.T) { | |
| 472 | + | _, err := EncodePreambleResponse(false, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), 0) | |
| 473 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 474 | + | }) | |
| 475 | + | t.Run("accepted-flag refusal without a code is refused", func(t *testing.T) { | |
| 476 | + | response := mustEncodePreambleResponse(t, false, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), RejectionVersionMismatch) | |
| 477 | + | response.Rejection = nil | |
| 478 | + | _, err := DecodePreambleResponse(response) | |
| 479 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 480 | + | }) | |
| 481 | + | t.Run("request refusals map precisely", func(t *testing.T) { | |
| 482 | + | for name, tc := range map[string]struct { | |
| 483 | + | mutate func(*wire.MuxPreambleRequest) | |
| 484 | + | want uint32 | |
| 485 | + | }{ | |
| 486 | + | "bad magic": {func(r *wire.MuxPreambleRequest) { r.Magic = 0xDEAD }, RejectionBadMagic}, | |
| 487 | + | "epoch": {func(r *wire.MuxPreambleRequest) { r.Epoch++ }, RejectionEpochMismatch}, | |
| 488 | + | "version": {func(r *wire.MuxPreambleRequest) { r.Version++ }, RejectionVersionMismatch}, | |
| 489 | + | "role": {func(r *wire.MuxPreambleRequest) { r.Role.Role = 9 }, RejectionWrongRole}, | |
| 490 | + | "caps": {func(r *wire.MuxPreambleRequest) { r.CapabilityBits = 1 }, RejectionLimitRefused}, | |
| 491 | + | "envelope floor": {func(r *wire.MuxPreambleRequest) { r.MaxReceiveEnvelopeBytes = 1 }, RejectionLimitRefused}, | |
| 492 | + | "chunk limit": {func(r *wire.MuxPreambleRequest) { r.StreamChunkLimit = 0 }, RejectionLimitRefused}, | |
| 493 | + | "stream limit": {func(r *wire.MuxPreambleRequest) { r.MaxStreams = 0 }, RejectionLimitRefused}, | |
| 494 | + | "aggregate limit": {func(r *wire.MuxPreambleRequest) { r.MaxAggregateBytes = 0 }, RejectionLimitRefused}, | |
| 495 | + | "missing policy": {func(r *wire.MuxPreambleRequest) { r.Policy = nil }, RejectionLimitRefused}, | |
| 496 | + | "invalid policy": {func(r *wire.MuxPreambleRequest) { r.Policy.Transport = "" }, RejectionLimitRefused}, | |
| 497 | + | } { | |
| 498 | + | t.Run(name, func(t *testing.T) { | |
| 499 | + | request := mustEncodePreambleRequest(t, DefaultMuxCeilings(), testPolicy()) | |
| 500 | + | tc.mutate(request) | |
| 501 | + | _, err := DecodePreambleRequest(request) | |
| 502 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 503 | + | require.Equal(t, tc.want, RejectionCodeFor(request, err)) | |
| 504 | + | }) | |
| 505 | + | } | |
| 506 | + | }) | |
| 507 | + | t.Run("nil request maps to limit refusal", func(t *testing.T) { | |
| 508 | + | require.Equal(t, RejectionLimitRefused, RejectionCodeFor(nil, ErrPreambleRejected)) | |
| 509 | + | require.Zero(t, RejectionCodeFor(nil, nil)) | |
| 510 | + | }) | |
| 511 | + | } | |
| 512 | + | ||
| 513 | + | // TestMuxPreambleEncodeFailClosed proves the local encoder refuses an | |
| 514 | + | // out-of-window advertisement or an unacceptably empty policy instead of | |
| 515 | + | // emitting a preamble the peer must reject. | |
| 516 | + | func TestMuxPreambleEncodeFailClosed(t *testing.T) { | |
| 517 | + | bad := DefaultMuxCeilings() | |
| 518 | + | bad.MaxStreams = MaxMuxStreams + 1 | |
| 519 | + | _, err := EncodePreambleRequest(bad, testPolicy()) | |
| 520 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 521 | + | ||
| 522 | + | _, err = EncodePreambleRequest(DefaultMuxCeilings(), ports.BrokerPolicy{}) | |
| 523 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 524 | + | ||
| 525 | + | _, err = EncodePreambleResponse(true, bad, testPolicy(), testIdentity(), testIncarnation(), 0) | |
| 526 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 527 | + | ||
| 528 | + | _, err = EncodePreambleResponse(true, DefaultMuxCeilings(), ports.BrokerPolicy{}, testIdentity(), testIncarnation(), 0) | |
| 529 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 530 | + | ||
| 531 | + | // A refusal is legal without any accepted authority, but it still | |
| 532 | + | // carries the shared magic, epoch, version, and server role. | |
| 533 | + | refusal := mustEncodePreambleResponse(t, false, MuxCeilings{}, ports.BrokerPolicy{}, "", ports.BrokerDaemonIncarnation{}, RejectionLimitRefused) | |
| 534 | + | require.Equal(t, wire.PreambleMagic, refusal.GetMagic()) | |
| 535 | + | require.Equal(t, MuxRoleServer, refusal.GetRole().GetRole()) | |
| 536 | + | require.Equal(t, RejectionLimitRefused, refusal.GetRejection().GetCode()) | |
| 537 | + | } | |
| 538 | + | ||
| 539 | + | // TestMuxPreambleRefusalCodeRange proves a refusal code is exactly 1..7 on | |
| 540 | + | // both encode and decode: 0 and anything above 7 fail closed, 6 is accepted as | |
| 541 | + | // a well-formed wire code even though this codec never emits it. | |
| 542 | + | func TestMuxPreambleRefusalCodeRange(t *testing.T) { | |
| 543 | + | for code := RejectionBadMagic; code <= RejectionLimitRefused; code++ { | |
| 544 | + | response, err := EncodePreambleResponse(false, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), code) | |
| 545 | + | require.NoError(t, err, "code %d", code) | |
| 546 | + | require.False(t, response.GetAccepted()) | |
| 547 | + | require.Equal(t, code, response.GetRejection().GetCode()) | |
| 548 | + | ||
| 549 | + | decoded, err := DecodePreambleResponse(response) | |
| 550 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 551 | + | require.Equal(t, code, decoded.Code) | |
| 552 | + | } | |
| 553 | + | ||
| 554 | + | for _, code := range []uint32{0, 8, 99, 0xFFFFFFFF} { | |
| 555 | + | _, err := EncodePreambleResponse(false, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), code) | |
| 556 | + | require.ErrorIs(t, err, ErrPreambleRejected, "encode code %d", code) | |
| 557 | + | } | |
| 558 | + | ||
| 559 | + | // A well-formed refusal frame with an out-of-range code still fails the | |
| 560 | + | // semantic decode. | |
| 561 | + | base := mustEncodePreambleResponse(t, false, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), RejectionLimitRefused) | |
| 562 | + | for _, code := range []uint32{0, 8, 99} { | |
| 563 | + | mutated := proto.Clone(base).(*wire.MuxPreambleResponse) | |
| 564 | + | mutated.Rejection = &wire.PreambleRejectionCode{Code: code} | |
| 565 | + | _, err := DecodePreambleResponse(mutated) | |
| 566 | + | require.ErrorIs(t, err, ErrPreambleRejected, "decode code %d", code) | |
| 567 | + | } | |
| 568 | + | ||
| 569 | + | // An acceptance never carries a rejection code. | |
| 570 | + | _, err := EncodePreambleResponse(true, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), RejectionDuplicate) | |
| 571 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 572 | + | } | |
| 573 | + | ||
| 574 | + | // TestMuxPreambleResponseExclusivity proves an answer is an acceptance or a | |
| 575 | + | // refusal but never a blend: an acceptance carrying a rejection field and a | |
| 576 | + | // refusal carrying accepted-only authority are both rejected. | |
| 577 | + | func TestMuxPreambleResponseExclusivity(t *testing.T) { | |
| 578 | + | t.Run("acceptance rejects a rejection field", func(t *testing.T) { | |
| 579 | + | response := mustEncodePreambleResponse(t, true, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), 0) | |
| 580 | + | require.NotNil(t, response.GetAcceptedPolicy()) | |
| 581 | + | response.Rejection = &wire.PreambleRejectionCode{Code: RejectionDuplicate} | |
| 582 | + | _, err := DecodePreambleResponse(response) | |
| 583 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 584 | + | }) | |
| 585 | + | ||
| 586 | + | t.Run("refusal rejects accepted-only authority", func(t *testing.T) { | |
| 587 | + | incarnation := testIncarnation() | |
| 588 | + | mutations := map[string]func(*wire.MuxPreambleResponse){ | |
| 589 | + | "accepted policy": func(r *wire.MuxPreambleResponse) { r.AcceptedPolicy = policyToWire(testPolicy()) }, | |
| 590 | + | "daemon identity": func(r *wire.MuxPreambleResponse) { | |
| 591 | + | r.DaemonIdentity = string(testIdentity()) | |
| 592 | + | }, | |
| 593 | + | "incarnation": func(r *wire.MuxPreambleResponse) { | |
| 594 | + | r.Incarnation = append([]byte(nil), incarnation[:]...) | |
| 595 | + | }, | |
| 596 | + | } | |
| 597 | + | for name, mutate := range mutations { | |
| 598 | + | t.Run(name, func(t *testing.T) { | |
| 599 | + | response := mustEncodePreambleResponse(t, false, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), RejectionLimitRefused) | |
| 600 | + | mutate(response) | |
| 601 | + | _, err := DecodePreambleResponse(response) | |
| 602 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 603 | + | }) | |
| 604 | + | } | |
| 605 | + | }) | |
| 606 | + | } | |
| 607 | + | ||
| 608 | + | // TestMuxPreambleStrictBytes proves the byte helpers enforce the 4 KiB bound, | |
| 609 | + | // the strict scanner, generated unmarshal, and semantic decode for both | |
| 610 | + | // directions: oversize, empty, unknown, duplicate, trailing/concatenated, | |
| 611 | + | // truncated, wrong-wire-type, and semantic failures are all refused. | |
| 612 | + | func TestMuxPreambleStrictBytes(t *testing.T) { | |
| 613 | + | requestRaw, err := proto.Marshal(mustEncodePreambleRequest(t, DefaultMuxCeilings(), testPolicy())) | |
| 614 | + | require.NoError(t, err) | |
| 615 | + | responseRaw, err := proto.Marshal(mustEncodePreambleResponse(t, true, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), 0)) | |
| 616 | + | require.NoError(t, err) | |
| 617 | + | ||
| 618 | + | t.Run("valid request round-trips", func(t *testing.T) { | |
| 619 | + | decoded, err := DecodePreambleRequestBytes(requestRaw) | |
| 620 | + | require.NoError(t, err) | |
| 621 | + | require.Equal(t, DefaultMuxCeilings(), decoded.Ceilings) | |
| 622 | + | require.Equal(t, testPolicy(), decoded.Policy) | |
| 623 | + | }) | |
| 624 | + | t.Run("valid response round-trips", func(t *testing.T) { | |
| 625 | + | decoded, err := DecodePreambleResponseBytes(responseRaw) | |
| 626 | + | require.NoError(t, err) | |
| 627 | + | require.True(t, decoded.Accepted) | |
| 628 | + | require.Equal(t, testIdentity(), decoded.Identity) | |
| 629 | + | require.Equal(t, testIncarnation(), decoded.Incarnation) | |
| 630 | + | }) | |
| 631 | + | t.Run("refusal frame surfaces its code", func(t *testing.T) { | |
| 632 | + | refusal, err := proto.Marshal(mustEncodePreambleResponse(t, false, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), RejectionLimitRefused)) | |
| 633 | + | require.NoError(t, err) | |
| 634 | + | decoded, err := DecodePreambleResponseBytes(refusal) | |
| 635 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 636 | + | require.Equal(t, RejectionLimitRefused, decoded.Code) | |
| 637 | + | }) | |
| 638 | + | ||
| 639 | + | corruptions := []struct { | |
| 640 | + | name string | |
| 641 | + | mutate func(raw []byte) []byte | |
| 642 | + | want error | |
| 643 | + | }{ | |
| 644 | + | {"oversize", func([]byte) []byte { return make([]byte, MuxPreambleLimit+1) }, ErrPreambleRejected}, | |
| 645 | + | {"empty", func([]byte) []byte { return nil }, wire.ErrScanEmpty}, | |
| 646 | + | {"unknown field", func(raw []byte) []byte { | |
| 647 | + | out := protowire.AppendTag(append([]byte(nil), raw...), 399, protowire.VarintType) | |
| 648 | + | return protowire.AppendVarint(out, 1) | |
| 649 | + | }, wire.ErrScanUnknown}, | |
| 650 | + | {"duplicate singular field", func(raw []byte) []byte { | |
| 651 | + | out := protowire.AppendTag(append([]byte(nil), raw...), 1, protowire.VarintType) | |
| 652 | + | return protowire.AppendVarint(out, uint64(wire.PreambleMagic)) | |
| 653 | + | }, wire.ErrScanDuplicate}, | |
| 654 | + | {"concatenated frame", func(raw []byte) []byte { | |
| 655 | + | return append(append([]byte(nil), raw...), raw...) | |
| 656 | + | }, wire.ErrScanDuplicate}, | |
| 657 | + | {"wrong wire type", func(raw []byte) []byte { | |
| 658 | + | out := protowire.AppendTag(append([]byte(nil), raw...), 1, protowire.BytesType) | |
| 659 | + | return protowire.AppendString(out, "x") | |
| 660 | + | }, wire.ErrScanWireType}, | |
| 661 | + | {"truncated varint", func(raw []byte) []byte { | |
| 662 | + | return append(append([]byte(nil), raw...), 0x80) | |
| 663 | + | }, wire.ErrScanTruncated}, | |
| 664 | + | } | |
| 665 | + | ||
| 666 | + | directions := []struct { | |
| 667 | + | name string | |
| 668 | + | raw []byte | |
| 669 | + | decode func([]byte) error | |
| 670 | + | }{ | |
| 671 | + | {"request", requestRaw, func(payload []byte) error { _, err := DecodePreambleRequestBytes(payload); return err }}, | |
| 672 | + | {"response", responseRaw, func(payload []byte) error { _, err := DecodePreambleResponseBytes(payload); return err }}, | |
| 673 | + | } | |
| 674 | + | for _, direction := range directions { | |
| 675 | + | for _, corruption := range corruptions { | |
| 676 | + | t.Run(direction.name+"/"+corruption.name, func(t *testing.T) { | |
| 677 | + | err := direction.decode(corruption.mutate(direction.raw)) | |
| 678 | + | require.Error(t, err) | |
| 679 | + | require.ErrorIs(t, err, corruption.want) | |
| 680 | + | }) | |
| 681 | + | } | |
| 682 | + | } | |
| 683 | + | ||
| 684 | + | t.Run("truncated prefixes", func(t *testing.T) { | |
| 685 | + | for size := range len(requestRaw) { | |
| 686 | + | _, err := DecodePreambleRequestBytes(requestRaw[:size]) | |
| 687 | + | require.Error(t, err, "request prefix[:%d] accepted", size) | |
| 688 | + | } | |
| 689 | + | _, err := DecodePreambleResponseBytes(requestRaw) | |
| 690 | + | require.Error(t, err, "client preamble accepted by the response helper") | |
| 691 | + | for size := range len(responseRaw) { | |
| 692 | + | _, err := DecodePreambleResponseBytes(responseRaw[:size]) | |
| 693 | + | require.Error(t, err, "response prefix[:%d] accepted", size) | |
| 694 | + | } | |
| 695 | + | }) | |
| 696 | + | ||
| 697 | + | t.Run("semantic failure from valid carriage", func(t *testing.T) { | |
| 698 | + | bad := proto.Clone(mustEncodePreambleRequest(t, DefaultMuxCeilings(), testPolicy())).(*wire.MuxPreambleRequest) | |
| 699 | + | bad.Magic = 0xDEADBEEF | |
| 700 | + | raw, err := proto.Marshal(bad) | |
| 701 | + | require.NoError(t, err) | |
| 702 | + | _, err = DecodePreambleRequestBytes(raw) | |
| 703 | + | require.ErrorIs(t, err, ErrPreambleRejected) | |
| 704 | + | }) | |
| 705 | + | } | |
| 706 | + | ||
| 707 | + | // FuzzDecodePreambleRequestBytes proves the strict byte helper never panics on | |
| 708 | + | // arbitrary input. | |
| 709 | + | func FuzzDecodePreambleRequestBytes(f *testing.F) { | |
| 710 | + | request, err := EncodePreambleRequest(DefaultMuxCeilings(), testPolicy()) | |
| 711 | + | require.NoError(f, err) | |
| 712 | + | raw, err := proto.Marshal(request) | |
| 713 | + | require.NoError(f, err) | |
| 714 | + | for _, seed := range appendStrictScanSeeds([][]byte{raw}) { | |
| 715 | + | f.Add(seed) | |
| 716 | + | } | |
| 717 | + | f.Fuzz(func(t *testing.T, payload []byte) { | |
| 718 | + | _, _ = DecodePreambleRequestBytes(payload) | |
| 719 | + | }) | |
| 720 | + | } | |
| 721 | + | ||
| 722 | + | // FuzzDecodePreambleResponseBytes proves the strict byte helper never panics on | |
| 723 | + | // arbitrary input. | |
| 724 | + | func FuzzDecodePreambleResponseBytes(f *testing.F) { | |
| 725 | + | response, err := EncodePreambleResponse(true, DefaultMuxCeilings(), testPolicy(), testIdentity(), testIncarnation(), 0) | |
| 726 | + | require.NoError(f, err) | |
| 727 | + | raw, err := proto.Marshal(response) | |
| 728 | + | require.NoError(f, err) | |
| 729 | + | for _, seed := range appendStrictScanSeeds([][]byte{raw}) { | |
| 730 | + | f.Add(seed) | |
| 731 | + | } | |
| 732 | + | f.Fuzz(func(t *testing.T, payload []byte) { | |
| 733 | + | _, _ = DecodePreambleResponseBytes(payload) | |
| 734 | + | }) | |
| 735 | + | } | |
A internal/adapters/daemonmux/pump.go +1039 -0
| @@ -0,0 +1,1039 @@ | |||
| 1 | + | // Physical pump over an abstract framed carrier (P3.2). | |
| 2 | + | // | |
| 3 | + | // Pump is the in-memory physical half of one daemonmux connection. It | |
| 4 | + | // composes the stateless directional codec (codec.go), the StreamEngine | |
| 5 | + | // (engine_state.go), the outbound fair Scheduler (scheduler.go), and the | |
| 6 | + | // publish-once terminalState (terminal.go) over a private FramedCarrier | |
| 7 | + | // abstraction, and drives exactly one reader goroutine and one writer | |
| 8 | + | // goroutine. It owns no socket, listener, sessionwire conversion, or stream | |
| 9 | + | // framing: carriage framing is the carrier's, and the pump never imports a | |
| 10 | + | // concrete physical implementation. | |
| 11 | + | // | |
| 12 | + | // The reader strict-decodes every inbound directional envelope with the | |
| 13 | + | // negotiated MuxCeilings and applies it to the engine: Open/Data/Close/Reset | |
| 14 | + | // when the inbound direction is DirectionClient (the local side is the | |
| 15 | + | // daemon), Opened/Refused/Data/Close/Reset when it is DirectionServer (the | |
| 16 | + | // local side is the broker client). The writer dequeues the scheduler's | |
| 17 | + | // fairly ordered, already-encoded immutable envelopes and sends them | |
| 18 | + | // verbatim; it never re-encodes, mutates, or reorders them. Between frames it | |
| 19 | + | // blocks on a coalescing wake channel that every successful Send signals | |
| 20 | + | // after enqueuing, so the writer never polls or sleeps and a lost wakeup is | |
| 21 | + | // impossible: a signal raised while the writer drains is re-tested by the | |
| 22 | + | // next dequeue. Flush is the explicit write barrier: it waits until the | |
| 23 | + | // scheduler is empty and no send is in flight (or the context or the | |
| 24 | + | // connection ends), because an accepted Send only means the frame is queued, | |
| 25 | + | // never that it reached the carrier. Close is not a flush; a caller that must | |
| 26 | + | // publish a final frame flushes first. | |
| 27 | + | // | |
| 28 | + | // The reader retires the outbound half of a stream the peer terminalizes: a | |
| 29 | + | // successful inbound Close or Reset resets the matching scheduler record, so | |
| 30 | + | // queued outbound frames are discarded rather than sent after the peer | |
| 31 | + | // stopped reading. A queue overflow in the outbound scheduler is settled | |
| 32 | + | // symmetrically: the overflowing stream's engine record is reset to free its | |
| 33 | + | // admission slot and exactly one terminal Reset is scheduled for the peer. | |
| 34 | + | // | |
| 35 | + | // Failures are split into two classes, exactly as the engine does. A | |
| 36 | + | // malformed outer envelope (a strict-scan, direction, or bound refusal of a | |
| 37 | + | // complete payload) and any carrier Receive/Send error are physical-fatal: | |
| 38 | + | // the pump publishes its own terminal state first, then the engine's, so | |
| 39 | + | // Done closes before every stream is terminalized, and the carrier is closed | |
| 40 | + | // to unblock the peer goroutine. A stream-local semantic, reference, or queue | |
| 41 | + | // refusal (an engine error that leaves the physical connection healthy) | |
| 42 | + | // schedules exactly one Reset for the offending stream, settles that stream | |
| 43 | + | // locally, and lets every sibling keep flowing: one stream's refusal never | |
| 44 | + | // terminalizes the physical connection or another stream. | |
| 45 | + | // | |
| 46 | + | // Cancellation is an orderly close. Cancelling the context passed to Start | |
| 47 | + | // (the carrier returns the context error from its blocked Send or Receive) or | |
| 48 | + | // calling Close stops both goroutines, closes the carrier, joins the reader | |
| 49 | + | // and writer, and publishes an orderly outcome: a nil Err and | |
| 50 | + | // RemoteFailureNone. A physical failure that already won keeps its cause; the | |
| 51 | + | // pump's terminal authority and the engine's are both first-wins, and the | |
| 52 | + | // pump serializes its two publications so they can never disagree. Close | |
| 53 | + | // unblocks an in-flight carrier Send or Receive through the adapter | |
| 54 | + | // prompt-close contract and joins both goroutines bounded by it: a carrier | |
| 55 | + | // whose Close promptly releases its blocked I/O lets Close return as soon as | |
| 56 | + | // the reader and writer observe the release. | |
| 57 | + | // | |
| 58 | + | // The pump itself performs no admission and no outbound encoding policy: a | |
| 59 | + | // locally initiated stream is admitted through Engine (Open) and confirmed | |
| 60 | + | // through Engine (Opened) by its owner before Send queues the matching frame, | |
| 61 | + | // exactly as the scheduler's admission gate requires. | |
| 62 | + | package daemonmux | |
| 63 | + | ||
| 64 | + | import ( | |
| 65 | + | "context" | |
| 66 | + | "errors" | |
| 67 | + | "sync" | |
| 68 | + | ||
| 69 | + | "github.com/bnema/vev/internal/domain" | |
| 70 | + | "github.com/bnema/vev/internal/ports" | |
| 71 | + | ) | |
| 72 | + | ||
| 73 | + | // FramedCarrier is the private physical carriage one Pump drives: one | |
| 74 | + | // complete framed application envelope in each direction over one connection. | |
| 75 | + | // Send writes one immutable envelope and reports the write result; it must not | |
| 76 | + | // retain or mutate the payload. Receive blocks for the next complete envelope; | |
| 77 | + | // it returns the receiver's own copy. Both must return promptly once their | |
| 78 | + | // context is done, which is what makes context cancellation an orderly close | |
| 79 | + | // without a concurrent Close. Close is safe to call concurrently with Send, | |
| 80 | + | // Receive, and itself, and it must unblock an in-flight Send or Receive | |
| 81 | + | // promptly: that is the adapter prompt-close contract Pump.Close relies on to | |
| 82 | + | // join both goroutines in bounded time. | |
| 83 | + | type FramedCarrier interface { | |
| 84 | + | Send(ctx context.Context, payload []byte) error | |
| 85 | + | Receive(ctx context.Context) ([]byte, error) | |
| 86 | + | Close() error | |
| 87 | + | } | |
| 88 | + | ||
| 89 | + | // ErrPumpConfig reports a pump constructed without a carrier or with an | |
| 90 | + | // unknown inbound direction. | |
| 91 | + | var ErrPumpConfig = errors.New("daemonmux: invalid pump configuration") | |
| 92 | + | ||
| 93 | + | // ErrAdmissionObserverLate reports an admission-observer registration attempted | |
| 94 | + | // after Start. The reader goroutine is already running and may have admitted an | |
| 95 | + | // inbound Open, so a late observer would silently miss it; a consumer must | |
| 96 | + | // register before Start. | |
| 97 | + | var ErrAdmissionObserverLate = errors.New("daemonmux: admission observer registered after start") | |
| 98 | + | ||
| 99 | + | // Pump is the physical pump of one daemonmux connection. It is safe for | |
| 100 | + | // concurrent use: the engine and scheduler it owns are themselves | |
| 101 | + | // concurrency-safe, and Send, Close, Start, and the accessors may be called | |
| 102 | + | // from any goroutine. It performs no stream framing and owns no socket, | |
| 103 | + | // listener, or session conversion. | |
| 104 | + | type Pump struct { | |
| 105 | + | carrier FramedCarrier | |
| 106 | + | engine *StreamEngine | |
| 107 | + | scheduler *Scheduler | |
| 108 | + | ceilings MuxCeilings | |
| 109 | + | inbound EnvelopeDirection | |
| 110 | + | local EnvelopeDirection | |
| 111 | + | terminal *terminalState | |
| 112 | + | ||
| 113 | + | // wake coalesces outbound wakeups: one pending token means the writer | |
| 114 | + | // must re-check the scheduler, and a signal raised while a token is | |
| 115 | + | // already pending is redundant rather than lost. | |
| 116 | + | wake chan struct{} | |
| 117 | + | ||
| 118 | + | // flushMu guards the flush bookkeeping. inflight counts frames the writer | |
| 119 | + | // has dequeued but not finished sending, and flushCh is closed and replaced | |
| 120 | + | // whenever the pump may have become idle (no in-flight send and an empty | |
| 121 | + | // scheduler), so Flush can wait for the transition without polling. The | |
| 122 | + | // scheduler lock is always taken after flushMu, never before it. | |
| 123 | + | flushMu sync.Mutex | |
| 124 | + | inflight int | |
| 125 | + | flushCh chan struct{} | |
| 126 | + | ||
| 127 | + | // watchMu guards the per-stream inbound watch table. Each admitted stream | |
| 128 | + | // a consumer is watching owns one channel that the reader closes and | |
| 129 | + | // replaces whenever it applies an inbound frame for that stream, so a | |
| 130 | + | // consumer can block for new inbound data or a stream state change without | |
| 131 | + | // polling. The table only holds entries for streams a consumer created with | |
| 132 | + | // Watch and are dropped again by ReleaseWatch, so it stays bounded by the | |
| 133 | + | // number of live logical connections rather than by every ID ever opened. | |
| 134 | + | watchMu sync.Mutex | |
| 135 | + | watch map[PhysicalStreamID]chan struct{} | |
| 136 | + | ||
| 137 | + | // admitMu guards admitFn, the optional inbound-admission observer. The | |
| 138 | + | // reader reads it on every admitted Open; its owner sets it before Start. | |
| 139 | + | admitMu sync.Mutex | |
| 140 | + | admitFn func(Open) | |
| 141 | + | ||
| 142 | + | mu sync.Mutex | |
| 143 | + | started bool | |
| 144 | + | closed bool | |
| 145 | + | cancel context.CancelFunc | |
| 146 | + | ||
| 147 | + | startOnce sync.Once | |
| 148 | + | closeOnce sync.Once | |
| 149 | + | stopOnce sync.Once | |
| 150 | + | ||
| 151 | + | // termMu serializes the pump-level and engine-level terminal | |
| 152 | + | // publications so both always carry the same outcome and only the first | |
| 153 | + | // publisher applies. | |
| 154 | + | termMu sync.Mutex | |
| 155 | + | settled bool | |
| 156 | + | ||
| 157 | + | readerDone chan struct{} | |
| 158 | + | writerDone chan struct{} | |
| 159 | + | closeErr error | |
| 160 | + | } | |
| 161 | + | ||
| 162 | + | // NewPump returns a pump over carrier that reads inbound envelopes of the | |
| 163 | + | // given direction and writes the opposite one. The negotiated MuxCeilings are | |
| 164 | + | // validated and copied: they build the engine with NewStreamEngineWithCeilings | |
| 165 | + | // and the scheduler (gated on that engine) with NewSchedulerWithCeilings, so | |
| 166 | + | // the pump enforces exactly the ceilings one physical connection negotiated | |
| 167 | + | // for its lifetime. An invalid advertisement is refused with ErrInvalidCeilings | |
| 168 | + | // and a missing carrier or unknown direction with ErrPumpConfig. Goroutines | |
| 169 | + | // start on Start. | |
| 170 | + | func NewPump(carrier FramedCarrier, inbound EnvelopeDirection, ceilings MuxCeilings) (*Pump, error) { | |
| 171 | + | if carrier == nil { | |
| 172 | + | return nil, ErrPumpConfig | |
| 173 | + | } | |
| 174 | + | if inbound != DirectionClient && inbound != DirectionServer { | |
| 175 | + | return nil, ErrPumpConfig | |
| 176 | + | } | |
| 177 | + | if err := ceilings.Validate(); err != nil { | |
| 178 | + | return nil, err | |
| 179 | + | } | |
| 180 | + | engine, err := NewStreamEngineWithCeilings(ceilings) | |
| 181 | + | if err != nil { | |
| 182 | + | return nil, err | |
| 183 | + | } | |
| 184 | + | scheduler, err := NewSchedulerWithCeilings(engine, ceilings) | |
| 185 | + | if err != nil { | |
| 186 | + | return nil, err | |
| 187 | + | } | |
| 188 | + | return &Pump{ | |
| 189 | + | carrier: carrier, | |
| 190 | + | engine: engine, | |
| 191 | + | scheduler: scheduler, | |
| 192 | + | ceilings: ceilings, | |
| 193 | + | inbound: inbound, | |
| 194 | + | local: oppositeDirection(inbound), | |
| 195 | + | terminal: newTerminalState(), | |
| 196 | + | wake: make(chan struct{}, 1), | |
| 197 | + | flushCh: make(chan struct{}), | |
| 198 | + | readerDone: make(chan struct{}), | |
| 199 | + | writerDone: make(chan struct{}), | |
| 200 | + | }, nil | |
| 201 | + | } | |
| 202 | + | ||
| 203 | + | // Start launches the reader and writer goroutines under ctx. It is | |
| 204 | + | // idempotent: only the first call takes effect, so a later call with a | |
| 205 | + | // different context is ignored rather than starting a second pair of | |
| 206 | + | // goroutines. A nil context is replaced with context.Background. Start after | |
| 207 | + | // Close does nothing, because the connection already reached its terminal | |
| 208 | + | // outcome. | |
| 209 | + | func (p *Pump) Start(ctx context.Context) { | |
| 210 | + | if p == nil { | |
| 211 | + | return | |
| 212 | + | } | |
| 213 | + | p.startOnce.Do(func() { | |
| 214 | + | if ctx == nil { | |
| 215 | + | ctx = context.Background() | |
| 216 | + | } | |
| 217 | + | p.mu.Lock() | |
| 218 | + | if p.closed { | |
| 219 | + | p.mu.Unlock() | |
| 220 | + | return | |
| 221 | + | } | |
| 222 | + | runCtx, cancel := context.WithCancel(ctx) | |
| 223 | + | p.cancel = cancel | |
| 224 | + | p.started = true | |
| 225 | + | p.mu.Unlock() | |
| 226 | + | go p.readLoop(runCtx) | |
| 227 | + | go p.writeLoop(runCtx) | |
| 228 | + | }) | |
| 229 | + | } | |
| 230 | + | ||
| 231 | + | // Close orderly closes the pump: it cancels the run context, closes the | |
| 232 | + | // carrier through the adapter prompt-close contract to unblock the reader and | |
| 233 | + | // writer, joins both goroutines, and publishes an orderly terminal outcome | |
| 234 | + | // unless a physical failure already won. It returns the carrier's Close error. | |
| 235 | + | // Close is idempotent and safe to call concurrently; every caller observes the | |
| 236 | + | // same result. A Close before Start never starts goroutines and never joins. | |
| 237 | + | // | |
| 238 | + | // Close is not a flush: it cancels the writer rather than draining it, so | |
| 239 | + | // frames accepted by Send but not yet written are dropped. A caller that must | |
| 240 | + | // publish a final frame (for example the last Close) calls Flush before Close; | |
| 241 | + | // a caller that races Close against the writer may lose the queued frame. | |
| 242 | + | func (p *Pump) Close() error { | |
| 243 | + | if p == nil { | |
| 244 | + | return ErrPumpConfig | |
| 245 | + | } | |
| 246 | + | p.closeOnce.Do(func() { | |
| 247 | + | p.mu.Lock() | |
| 248 | + | p.closed = true | |
| 249 | + | started := p.started | |
| 250 | + | p.mu.Unlock() | |
| 251 | + | p.stop() | |
| 252 | + | if started { | |
| 253 | + | <-p.readerDone | |
| 254 | + | <-p.writerDone | |
| 255 | + | } | |
| 256 | + | p.settleOrderly() | |
| 257 | + | }) | |
| 258 | + | return p.closeErr | |
| 259 | + | } | |
| 260 | + | ||
| 261 | + | // Done returns the pump's terminal channel: closed exactly once, before every | |
| 262 | + | // stream of the connection is terminalized. | |
| 263 | + | func (p *Pump) Done() <-chan struct{} { | |
| 264 | + | if p == nil || p.terminal == nil { | |
| 265 | + | return nil | |
| 266 | + | } | |
| 267 | + | return p.terminal.Done() | |
| 268 | + | } | |
| 269 | + | ||
| 270 | + | // Err returns the pump's terminal cause: nil while open and for an orderly | |
| 271 | + | // Close or context cancellation. | |
| 272 | + | func (p *Pump) Err() error { | |
| 273 | + | if p == nil || p.terminal == nil { | |
| 274 | + | return nil | |
| 275 | + | } | |
| 276 | + | return p.terminal.Err() | |
| 277 | + | } | |
| 278 | + | ||
| 279 | + | // FailureKind returns the pump's terminal classification: RemoteFailureNone | |
| 280 | + | // while open and for an orderly Close. | |
| 281 | + | func (p *Pump) FailureKind() domain.RemoteFailureKind { | |
| 282 | + | if p == nil || p.terminal == nil { | |
| 283 | + | return domain.RemoteFailureNone | |
| 284 | + | } | |
| 285 | + | return p.terminal.FailureKind() | |
| 286 | + | } | |
| 287 | + | ||
| 288 | + | // Engine returns the stream engine the pump drives. The reader applies every | |
| 289 | + | // inbound envelope to it, and a stream's owner uses it to admit a locally | |
| 290 | + | // initiated stream and to consume inbound chunks with Take. | |
| 291 | + | func (p *Pump) Engine() *StreamEngine { | |
| 292 | + | if p == nil { | |
| 293 | + | return nil | |
| 294 | + | } | |
| 295 | + | return p.engine | |
| 296 | + | } | |
| 297 | + | ||
| 298 | + | // Take removes the oldest queued inbound chunk of one stream. It is the | |
| 299 | + | // pump's convenience delegation to the engine and reports false when the | |
| 300 | + | // stream is unknown or its queue is empty. Draining the last queued chunk of a | |
| 301 | + | // stream that already received an orderly Close terminalizes it here, and the | |
| 302 | + | // pump rotates that stream's watch channel so a consumer blocked on the queue | |
| 303 | + | // transition wakes and observes the end instead of waiting for a frame that | |
| 304 | + | // can never arrive. | |
| 305 | + | func (p *Pump) Take(physical PhysicalStreamID) ([]byte, bool) { | |
| 306 | + | if p == nil || p.engine == nil { | |
| 307 | + | return nil, false | |
| 308 | + | } | |
| 309 | + | chunk, ok, finalized := p.engine.take(physical) | |
| 310 | + | if finalized { | |
| 311 | + | p.signalWatch(physical) | |
| 312 | + | } | |
| 313 | + | return chunk, ok | |
| 314 | + | } | |
| 315 | + | ||
| 316 | + | // Ceilings returns the negotiated ceilings the pump enforces. The value is | |
| 317 | + | // immutable for the pump's lifetime. | |
| 318 | + | func (p *Pump) Ceilings() MuxCeilings { | |
| 319 | + | if p == nil { | |
| 320 | + | return MuxCeilings{} | |
| 321 | + | } | |
| 322 | + | return p.ceilings | |
| 323 | + | } | |
| 324 | + | ||
| 325 | + | // Inbound returns the envelope direction the pump's reader decodes. | |
| 326 | + | func (p *Pump) Inbound() EnvelopeDirection { | |
| 327 | + | if p == nil { | |
| 328 | + | return 0 | |
| 329 | + | } | |
| 330 | + | return p.inbound | |
| 331 | + | } | |
| 332 | + | ||
| 333 | + | // Local returns the envelope direction the pump's writer emits: the opposite | |
| 334 | + | // of Inbound. | |
| 335 | + | func (p *Pump) Local() EnvelopeDirection { | |
| 336 | + | if p == nil { | |
| 337 | + | return 0 | |
| 338 | + | } | |
| 339 | + | return p.local | |
| 340 | + | } | |
| 341 | + | ||
| 342 | + | // OnAdmitted registers fn to observe every inbound logical stream the pump's | |
| 343 | + | // reader admits: it runs once per admitted Open, immediately after the engine | |
| 344 | + | // accepted it and before any later frame for that stream can be applied (and | |
| 345 | + | // before the opening side is confirmed with Opened). It carries the decoded | |
| 346 | + | // admission request, which the reader never mutates afterwards. | |
| 347 | + | // | |
| 348 | + | // fn runs on the reader goroutine, so it must not block: it is the one | |
| 349 | + | // goroutine that applies every inbound frame of the physical connection. The | |
| 350 | + | // daemon-side Listener registers its accept-queue admission here. Registering | |
| 351 | + | // a second observer replaces the first - one physical connection has exactly | |
| 352 | + | // one daemon-side admission consumer - and a nil fn clears it. | |
| 353 | + | // | |
| 354 | + | // The observer must be registered before the first inbound Open, that is before | |
| 355 | + | // Start: a registration attempted on a pump that already started fails with | |
| 356 | + | // ErrAdmissionObserverLate rather than silently missing the Opens the reader | |
| 357 | + | // may already have admitted. A nil pump fails with ErrPumpConfig. | |
| 358 | + | func (p *Pump) OnAdmitted(fn func(Open)) error { | |
| 359 | + | if p == nil { | |
| 360 | + | return ErrPumpConfig | |
| 361 | + | } | |
| 362 | + | p.mu.Lock() | |
| 363 | + | defer p.mu.Unlock() | |
| 364 | + | if p.started { | |
| 365 | + | return ErrAdmissionObserverLate | |
| 366 | + | } | |
| 367 | + | p.admitMu.Lock() | |
| 368 | + | p.admitFn = fn | |
| 369 | + | p.admitMu.Unlock() | |
| 370 | + | return nil | |
| 371 | + | } | |
| 372 | + | ||
| 373 | + | // notifyAdmitted runs the registered admission observer, if any. | |
| 374 | + | func (p *Pump) notifyAdmitted(message Open) { | |
| 375 | + | p.admitMu.Lock() | |
| 376 | + | fn := p.admitFn | |
| 377 | + | p.admitMu.Unlock() | |
| 378 | + | if fn != nil { | |
| 379 | + | fn(message) | |
| 380 | + | } | |
| 381 | + | } | |
| 382 | + | ||
| 383 | + | // Watch returns a channel the pump's reader closes and replaces whenever it | |
| 384 | + | // applies an inbound frame for the physical stream: Open, Opened, Refused, | |
| 385 | + | // Data, Close, or Reset. A consumer blocks on it to wait for new inbound data | |
| 386 | + | // or a stream state change without polling. The channel is stable until the | |
| 387 | + | // next applied frame, so a consumer must re-read the stream after every wake | |
| 388 | + | // and must also select on the stream's terminal channel and the connection's | |
| 389 | + | // Done to observe a terminal outcome. A zero, unknown, or evicted identity, | |
| 390 | + | // and a stream that already reached its terminal state, return an | |
| 391 | + | // already-closed channel and retain no watch entry, so the watch table stays | |
| 392 | + | // bounded by the number of live logical connections. Watch is safe for | |
| 393 | + | // concurrent use and the same channel is returned to every caller until the | |
| 394 | + | // next applied frame. | |
| 395 | + | func (p *Pump) Watch(physical PhysicalStreamID) <-chan struct{} { | |
| 396 | + | if p == nil || p.engine == nil || physical.Validate() != nil { | |
| 397 | + | return closedChannel() | |
| 398 | + | } | |
| 399 | + | p.watchMu.Lock() | |
| 400 | + | defer p.watchMu.Unlock() | |
| 401 | + | if status, ok := p.engine.Status(physical); !ok || status.State == StreamTerminal { | |
| 402 | + | // No further frame can ever be applied for this stream: drop any stale | |
| 403 | + | // entry and hand back a closed channel instead of retaining a watch | |
| 404 | + | // nobody will ever signal. | |
| 405 | + | delete(p.watch, physical) | |
| 406 | + | return closedChannel() | |
| 407 | + | } | |
| 408 | + | if ch, ok := p.watch[physical]; ok { | |
| 409 | + | return ch | |
| 410 | + | } | |
| 411 | + | ch := make(chan struct{}) | |
| 412 | + | if p.watch == nil { | |
| 413 | + | p.watch = make(map[PhysicalStreamID]chan struct{}) | |
| 414 | + | } | |
| 415 | + | p.watch[physical] = ch | |
| 416 | + | return ch | |
| 417 | + | } | |
| 418 | + | ||
| 419 | + | // cause returns the retained publish-once terminal authority of one admitted | |
| 420 | + | // stream, or nil when the pump is unset or the identity was never admitted. | |
| 421 | + | // The pointer outlives the engine's bounded record, so a stream's owner | |
| 422 | + | // captures it at open and keeps reporting the exact terminal cause even after | |
| 423 | + | // eviction. | |
| 424 | + | func (p *Pump) cause(physical PhysicalStreamID) *terminalState { | |
| 425 | + | if p == nil || p.engine == nil { | |
| 426 | + | return nil | |
| 427 | + | } | |
| 428 | + | cause, _ := p.engine.cause(physical) | |
| 429 | + | return cause | |
| 430 | + | } | |
| 431 | + | ||
| 432 | + | // ReleaseWatch drops the watch channel of one terminal or abandoned physical | |
| 433 | + | // stream so the watch table stays bounded by the number of live logical | |
| 434 | + | // connections. It is safe to call concurrently with Watch; a later Watch of | |
| 435 | + | // an already-terminal stream returns a closed channel without retaining an | |
| 436 | + | // entry. | |
| 437 | + | func (p *Pump) ReleaseWatch(physical PhysicalStreamID) { | |
| 438 | + | if p == nil { | |
| 439 | + | return | |
| 440 | + | } | |
| 441 | + | p.watchMu.Lock() | |
| 442 | + | defer p.watchMu.Unlock() | |
| 443 | + | delete(p.watch, physical) | |
| 444 | + | } | |
| 445 | + | ||
| 446 | + | // signalWatch closes and replaces the current watch channel of one stream so | |
| 447 | + | // every blocked watcher wakes and re-reads the engine. It is a no-op for a | |
| 448 | + | // stream nobody watches. The caller must not hold the watch lock, and the | |
| 449 | + | // engine lock is never held when it runs. | |
| 450 | + | func (p *Pump) signalWatch(physical PhysicalStreamID) { | |
| 451 | + | if p == nil || physical.Validate() != nil { | |
| 452 | + | return | |
| 453 | + | } | |
| 454 | + | p.watchMu.Lock() | |
| 455 | + | defer p.watchMu.Unlock() | |
| 456 | + | ch, ok := p.watch[physical] | |
| 457 | + | if !ok { | |
| 458 | + | return | |
| 459 | + | } | |
| 460 | + | close(ch) | |
| 461 | + | p.watch[physical] = make(chan struct{}) | |
| 462 | + | } | |
| 463 | + | ||
| 464 | + | // closedChannel returns a shared, already-closed channel for watchers of an | |
| 465 | + | // unknown or terminal stream. | |
| 466 | + | var closedWatchChan = func() <-chan struct{} { | |
| 467 | + | ch := make(chan struct{}) | |
| 468 | + | close(ch) | |
| 469 | + | return ch | |
| 470 | + | }() | |
| 471 | + | ||
| 472 | + | func closedChannel() <-chan struct{} { return closedWatchChan } | |
| 473 | + | ||
| 474 | + | // watchClosed reports whether a channel returned by Watch is already closed, | |
| 475 | + | // which means its stream is terminal or unknown and can never receive another | |
| 476 | + | // applied frame. | |
| 477 | + | func watchClosed(ch <-chan struct{}) bool { | |
| 478 | + | select { | |
| 479 | + | case <-ch: | |
| 480 | + | return true | |
| 481 | + | default: | |
| 482 | + | return false | |
| 483 | + | } | |
| 484 | + | } | |
| 485 | + | ||
| 486 | + | // Send encodes one outbound message with the negotiated ceilings, queues it | |
| 487 | + | // on the fair scheduler, and wakes the writer. The message's direction must | |
| 488 | + | // match the pump's local side: DirectionClient accepts ClientMessage and | |
| 489 | + | // DirectionServer accepts ServerMessage; the shared Data, Close, and Reset | |
| 490 | + | // values satisfy both. A wrong-direction value fails with ErrWrongDirection | |
| 491 | + | // before anything is encoded or queued, a message the codec refuses propagates | |
| 492 | + | // the codec's error, and a pump that already reached its terminal outcome | |
| 493 | + | // fails with ErrPhysicalClosed. On success the encoded bytes are immutable and | |
| 494 | + | // owned by the writer. | |
| 495 | + | // | |
| 496 | + | // An accepted Send only means the frame is queued, never that it reached the | |
| 497 | + | // wire: the writer drains asynchronously. Call Flush to wait until every | |
| 498 | + | // accepted frame has been handed to the carrier. | |
| 499 | + | // | |
| 500 | + | // A stream-local scheduler refusal propagates the scheduler's error. A queue | |
| 501 | + | // overflow is additionally isolated: the stream's engine record is reset to | |
| 502 | + | // free its admission slot and exactly one terminal Reset is scheduled so the | |
| 503 | + | // peer learns the stream was aborted; the caller still observes the overflow | |
| 504 | + | // error for the frame it tried to send. | |
| 505 | + | func (p *Pump) Send(message any) error { | |
| 506 | + | if p == nil { | |
| 507 | + | return ErrPumpConfig | |
| 508 | + | } | |
| 509 | + | if message == nil { | |
| 510 | + | return ErrInvalidMessage | |
| 511 | + | } | |
| 512 | + | if p.isTerminal() { | |
| 513 | + | return ErrPhysicalClosed | |
| 514 | + | } | |
| 515 | + | switch p.local { | |
| 516 | + | case DirectionClient: | |
| 517 | + | client, ok := message.(ClientMessage) | |
| 518 | + | if !ok { | |
| 519 | + | return ErrWrongDirection | |
| 520 | + | } | |
| 521 | + | physical, _, err := clientFrame(client) | |
| 522 | + | if err != nil { | |
| 523 | + | return err | |
| 524 | + | } | |
| 525 | + | return p.admitOutbound(physical, func() error { return p.scheduler.EnqueueClient(client) }) | |
| 526 | + | case DirectionServer: | |
| 527 | + | server, ok := message.(ServerMessage) | |
| 528 | + | if !ok { | |
| 529 | + | return ErrWrongDirection | |
| 530 | + | } | |
| 531 | + | physical, _, err := serverFrame(server) | |
| 532 | + | if err != nil { | |
| 533 | + | return err | |
| 534 | + | } | |
| 535 | + | return p.admitOutbound(physical, func() error { return p.scheduler.EnqueueServer(server) }) | |
| 536 | + | default: | |
| 537 | + | return ErrPumpConfig | |
| 538 | + | } | |
| 539 | + | } | |
| 540 | + | ||
| 541 | + | // Flush blocks until every frame the scheduler accepted has been handed to the | |
| 542 | + | // carrier and no send is in flight, or until ctx is done. Send accepted means | |
| 543 | + | // queued only, so Flush is the explicit barrier that makes a frame observable | |
| 544 | + | // on the wire before the connection is closed. It returns ctx.Err() when the | |
| 545 | + | // deadline or cancellation wins, and the pump's terminal outcome when the | |
| 546 | + | // connection became terminal before the scheduler drained (a physical failure | |
| 547 | + | // carries its cause; an orderly Close carries ErrPhysicalClosed when frames | |
| 548 | + | // were still queued). It never polls: the writer signals an idle transition. | |
| 549 | + | func (p *Pump) Flush(ctx context.Context) error { | |
| 550 | + | if p == nil { | |
| 551 | + | return ErrPumpConfig | |
| 552 | + | } | |
| 553 | + | if ctx == nil { | |
| 554 | + | ctx = context.Background() | |
| 555 | + | } | |
| 556 | + | for { | |
| 557 | + | p.flushMu.Lock() | |
| 558 | + | if p.inflight == 0 && !p.scheduler.Pending() { | |
| 559 | + | p.flushMu.Unlock() | |
| 560 | + | return nil | |
| 561 | + | } | |
| 562 | + | wait := p.flushCh | |
| 563 | + | p.flushMu.Unlock() | |
| 564 | + | select { | |
| 565 | + | case <-wait: | |
| 566 | + | case <-ctx.Done(): | |
| 567 | + | return ctx.Err() | |
| 568 | + | case <-p.terminal.Done(): | |
| 569 | + | p.flushMu.Lock() | |
| 570 | + | idle := p.inflight == 0 && !p.scheduler.Pending() | |
| 571 | + | p.flushMu.Unlock() | |
| 572 | + | if idle { | |
| 573 | + | return nil | |
| 574 | + | } | |
| 575 | + | if err := p.Err(); err != nil { | |
| 576 | + | return err | |
| 577 | + | } | |
| 578 | + | return ErrPhysicalClosed | |
| 579 | + | } | |
| 580 | + | } | |
| 581 | + | } | |
| 582 | + | ||
| 583 | + | // admitOutbound queues one already-typed outbound frame, isolates a queue | |
| 584 | + | // overflow, and wakes the writer only when the scheduler accepted the frame, | |
| 585 | + | // so a refused frame never produces a spurious wakeup. | |
| 586 | + | func (p *Pump) admitOutbound(physical PhysicalStreamID, admit func() error) error { | |
| 587 | + | if err := admit(); err != nil { | |
| 588 | + | p.handleOutboundRefusal(physical, err) | |
| 589 | + | return err | |
| 590 | + | } | |
| 591 | + | p.signal() | |
| 592 | + | return nil | |
| 593 | + | } | |
| 594 | + | ||
| 595 | + | // handleOutboundRefusal isolates one refused outbound frame. A queue overflow | |
| 596 | + | // is settled exactly like an inbound stream-local refusal: the stream's engine | |
| 597 | + | // record is reset to free its admission slot and exactly one terminal Reset is | |
| 598 | + | // scheduled towards the peer. Every other refusal (a settled or unadmitted | |
| 599 | + | // identity) is left untouched. | |
| 600 | + | func (p *Pump) handleOutboundRefusal(physical PhysicalStreamID, err error) { | |
| 601 | + | if errors.Is(err, ErrSchedulerFull) { | |
| 602 | + | p.scheduleReset(physical, err) | |
| 603 | + | } | |
| 604 | + | } | |
| 605 | + | ||
| 606 | + | // signalFlushLocked broadcasts one possible idle transition to every Flush | |
| 607 | + | // waiter by closing the current channel and installing a fresh one. The caller | |
| 608 | + | // holds flushMu. | |
| 609 | + | func (p *Pump) signalFlushLocked() { | |
| 610 | + | if p.flushCh != nil { | |
| 611 | + | close(p.flushCh) | |
| 612 | + | } | |
| 613 | + | p.flushCh = make(chan struct{}) | |
| 614 | + | } | |
| 615 | + | ||
| 616 | + | // maybeSignalFlush broadcasts an idle transition when the scheduler is empty | |
| 617 | + | // and no send is in flight, so a pending Flush can return. It is called after | |
| 618 | + | // the writer finishes a send and after a scheduler reset discards queued | |
| 619 | + | // frames. | |
| 620 | + | func (p *Pump) maybeSignalFlush() { | |
| 621 | + | p.flushMu.Lock() | |
| 622 | + | if p.inflight == 0 && !p.scheduler.Pending() { | |
| 623 | + | p.signalFlushLocked() | |
| 624 | + | } | |
| 625 | + | p.flushMu.Unlock() | |
| 626 | + | } | |
| 627 | + | ||
| 628 | + | // signal raises one coalesced writer wakeup without ever blocking. | |
| 629 | + | func (p *Pump) signal() { | |
| 630 | + | select { | |
| 631 | + | case p.wake <- struct{}{}: | |
| 632 | + | default: | |
| 633 | + | } | |
| 634 | + | } | |
| 635 | + | ||
| 636 | + | // readLoop is the reader goroutine: it receives one complete framed envelope | |
| 637 | + | // at a time, strict-decodes the inbound direction, and applies it to the | |
| 638 | + | // engine. A carrier Receive error that is not the local cancellation is | |
| 639 | + | // physical-fatal; so is a malformed outer envelope. Either way Done closes | |
| 640 | + | // before any stream is terminalized and the peer goroutine is unblocked. | |
| 641 | + | func (p *Pump) readLoop(ctx context.Context) { | |
| 642 | + | defer p.settleOrderly() | |
| 643 | + | defer close(p.readerDone) | |
| 644 | + | for { | |
| 645 | + | if ctx.Err() != nil { | |
| 646 | + | return | |
| 647 | + | } | |
| 648 | + | payload, err := p.carrier.Receive(ctx) | |
| 649 | + | if err != nil { | |
| 650 | + | if ctx.Err() != nil { | |
| 651 | + | return | |
| 652 | + | } | |
| 653 | + | p.settleFailure(domain.RemoteFailureTransport, err) | |
| 654 | + | return | |
| 655 | + | } | |
| 656 | + | if fatal := p.handleInbound(payload); fatal != nil { | |
| 657 | + | p.settleFailure(domain.RemoteFailureInvalidResponse, fatal) | |
| 658 | + | return | |
| 659 | + | } | |
| 660 | + | } | |
| 661 | + | } | |
| 662 | + | ||
| 663 | + | // writeLoop is the writer goroutine: it dequeues the scheduler's fairly | |
| 664 | + | // ordered immutable envelopes and sends them verbatim, then blocks on the | |
| 665 | + | // coalescing wake channel until more work arrives or the run context ends. It | |
| 666 | + | // never polls, sleeps, or reorders. | |
| 667 | + | func (p *Pump) writeLoop(ctx context.Context) { | |
| 668 | + | defer p.settleOrderly() | |
| 669 | + | defer close(p.writerDone) | |
| 670 | + | for { | |
| 671 | + | p.flushMu.Lock() | |
| 672 | + | envelope, ok := p.scheduler.Dequeue() | |
| 673 | + | if ok { | |
| 674 | + | p.inflight++ | |
| 675 | + | } | |
| 676 | + | p.flushMu.Unlock() | |
| 677 | + | if ok { | |
| 678 | + | err := p.carrier.Send(ctx, envelope.Bytes) | |
| 679 | + | p.flushMu.Lock() | |
| 680 | + | if p.inflight > 0 { | |
| 681 | + | p.inflight-- | |
| 682 | + | } | |
| 683 | + | idle := p.inflight == 0 && !p.scheduler.Pending() | |
| 684 | + | if idle { | |
| 685 | + | p.signalFlushLocked() | |
| 686 | + | } | |
| 687 | + | p.flushMu.Unlock() | |
| 688 | + | if err != nil { | |
| 689 | + | if ctx.Err() != nil { | |
| 690 | + | return | |
| 691 | + | } | |
| 692 | + | p.settleFailure(domain.RemoteFailureTransport, err) | |
| 693 | + | return | |
| 694 | + | } | |
| 695 | + | continue | |
| 696 | + | } | |
| 697 | + | // The scheduler is empty and no send is in flight: release any Flush | |
| 698 | + | // waiter before blocking on the coalescing wake channel. | |
| 699 | + | p.maybeSignalFlush() | |
| 700 | + | select { | |
| 701 | + | case <-p.wake: | |
| 702 | + | case <-ctx.Done(): | |
| 703 | + | return | |
| 704 | + | } | |
| 705 | + | } | |
| 706 | + | } | |
| 707 | + | ||
| 708 | + | // handleInbound strict-decodes one complete inbound payload in the pump's | |
| 709 | + | // direction and applies it to the engine. It returns the physical-fatal cause | |
| 710 | + | // of a malformed outer envelope, or nil when the payload was applied, when a | |
| 711 | + | // stream-local refusal was isolated with one Reset, or when the physical | |
| 712 | + | // connection was already terminal. | |
| 713 | + | func (p *Pump) handleInbound(payload []byte) error { | |
| 714 | + | switch p.inbound { | |
| 715 | + | case DirectionClient: | |
| 716 | + | message, err := DecodeClient(payload, p.ceilings.MaxReceiveEnvelopeBytes, p.ceilings.StreamChunkLimit) | |
| 717 | + | if err != nil { | |
| 718 | + | return err | |
| 719 | + | } | |
| 720 | + | return p.applyClient(message) | |
| 721 | + | case DirectionServer: | |
| 722 | + | message, err := DecodeServer(payload, p.ceilings.MaxReceiveEnvelopeBytes, p.ceilings.StreamChunkLimit) | |
| 723 | + | if err != nil { | |
| 724 | + | return err | |
| 725 | + | } | |
| 726 | + | return p.applyServer(message) | |
| 727 | + | default: | |
| 728 | + | return ErrPumpConfig | |
| 729 | + | } | |
| 730 | + | } | |
| 731 | + | ||
| 732 | + | // applyClient applies one broker-to-daemon message to the engine. | |
| 733 | + | func (p *Pump) applyClient(message ClientMessage) error { | |
| 734 | + | switch m := message.(type) { | |
| 735 | + | case Open: | |
| 736 | + | return p.applyOpen(m) | |
| 737 | + | case *Open: | |
| 738 | + | if m == nil { | |
| 739 | + | return ErrInvalidMessage | |
| 740 | + | } | |
| 741 | + | return p.applyClient(*m) | |
| 742 | + | case Data: | |
| 743 | + | return p.applyData(m) | |
| 744 | + | case *Data: | |
| 745 | + | if m == nil { | |
| 746 | + | return ErrInvalidMessage | |
| 747 | + | } | |
| 748 | + | return p.applyData(*m) | |
| 749 | + | case Close: | |
| 750 | + | disposition, err := p.engine.Close(m) | |
| 751 | + | return p.applyStreamTerminal(m.Physical, disposition, err) | |
| 752 | + | case *Close: | |
| 753 | + | if m == nil { | |
| 754 | + | return ErrInvalidMessage | |
| 755 | + | } | |
| 756 | + | return p.applyClient(*m) | |
| 757 | + | case Reset: | |
| 758 | + | disposition, err := p.engine.Reset(m) | |
| 759 | + | return p.applyStreamTerminal(m.Physical, disposition, err) | |
| 760 | + | case *Reset: | |
| 761 | + | if m == nil { | |
| 762 | + | return ErrInvalidMessage | |
| 763 | + | } | |
| 764 | + | return p.applyClient(*m) | |
| 765 | + | default: | |
| 766 | + | return ErrWrongDirection | |
| 767 | + | } | |
| 768 | + | } | |
| 769 | + | ||
| 770 | + | // applyOpen admits one incoming logical stream and notifies the admission | |
| 771 | + | // observer exactly when the engine accepted it. A refusal never notifies: a | |
| 772 | + | // stream that was never admitted has no daemon-side consumer. The observer runs | |
| 773 | + | // on the reader goroutine, between admission and any later frame for that | |
| 774 | + | // stream. | |
| 775 | + | // | |
| 776 | + | // The two refusal classes are deliberately different. A duplicate or replayed | |
| 777 | + | // Open (an identity at or below the high-water mark the engine already owns) | |
| 778 | + | // is a protocol violation that must not disturb the stream it names: it is | |
| 779 | + | // ignored, so it neither resets nor alters the existing live or retired stream, | |
| 780 | + | // and it never produces an outbound frame. A fresh Open the engine refused | |
| 781 | + | // before recording anything (capacity, a closed physical connection, or an | |
| 782 | + | // invalid authority) is signalled with exactly one typed Mux Refused carrying | |
| 783 | + | // its admission code; no stream is recorded for it, and a sibling continues | |
| 784 | + | // unaffected. | |
| 785 | + | func (p *Pump) applyOpen(message Open) error { | |
| 786 | + | err := p.engine.Open(message) | |
| 787 | + | switch { | |
| 788 | + | case err == nil: | |
| 789 | + | p.notifyAdmitted(message) | |
| 790 | + | return nil | |
| 791 | + | case errors.Is(err, ErrStreamIDReused): | |
| 792 | + | return nil | |
| 793 | + | default: | |
| 794 | + | p.refuseFreshOpen(message.Ref, err) | |
| 795 | + | return nil | |
| 796 | + | } | |
| 797 | + | } | |
| 798 | + | ||
| 799 | + | // refuseFreshOpen signals the peer that one fresh Open was refused before the | |
| 800 | + | // engine admitted anything. It schedules exactly one typed Mux Refused through | |
| 801 | + | // the scheduler's fresh-refusal exception, because the engine recorded no | |
| 802 | + | // stream to confirm, and it records no engine stream itself. A pump that | |
| 803 | + | // already reached its terminal outcome, or a Refused the scheduler refuses, | |
| 804 | + | // leaves the peer unsignalled rather than faulting the shared physical | |
| 805 | + | // connection. | |
| 806 | + | func (p *Pump) refuseFreshOpen(ref StreamRef, cause error) { | |
| 807 | + | if p == nil || p.isTerminal() { | |
| 808 | + | return | |
| 809 | + | } | |
| 810 | + | refused := Refused{Ref: ref, Error: openRefusalDetail(cause)} | |
| 811 | + | if err := p.scheduler.Refuse(refused); err != nil { | |
| 812 | + | return | |
| 813 | + | } | |
| 814 | + | p.signal() | |
| 815 | + | } | |
| 816 | + | ||
| 817 | + | // applyServer applies one daemon-to-broker message to the engine. | |
| 818 | + | func (p *Pump) applyServer(message ServerMessage) error { | |
| 819 | + | switch m := message.(type) { | |
| 820 | + | case Opened: | |
| 821 | + | return p.applyStreamError(m.Ref.Physical, p.engine.Opened(m)) | |
| 822 | + | case *Opened: | |
| 823 | + | if m == nil { | |
| 824 | + | return ErrInvalidMessage | |
| 825 | + | } | |
| 826 | + | return p.applyServer(*m) | |
| 827 | + | case Refused: | |
| 828 | + | return p.applyStreamError(m.Ref.Physical, p.engine.Refused(m)) | |
| 829 | + | case *Refused: | |
| 830 | + | if m == nil { | |
| 831 | + | return ErrInvalidMessage | |
| 832 | + | } | |
| 833 | + | return p.applyServer(*m) | |
| 834 | + | case Data: | |
| 835 | + | return p.applyData(m) | |
| 836 | + | case *Data: | |
| 837 | + | if m == nil { | |
| 838 | + | return ErrInvalidMessage | |
| 839 | + | } | |
| 840 | + | return p.applyData(*m) | |
| 841 | + | case Close: | |
| 842 | + | disposition, err := p.engine.Close(m) | |
| 843 | + | return p.applyStreamTerminal(m.Physical, disposition, err) | |
| 844 | + | case *Close: | |
| 845 | + | if m == nil { | |
| 846 | + | return ErrInvalidMessage | |
| 847 | + | } | |
| 848 | + | return p.applyServer(*m) | |
| 849 | + | case Reset: | |
| 850 | + | disposition, err := p.engine.Reset(m) | |
| 851 | + | return p.applyStreamTerminal(m.Physical, disposition, err) | |
| 852 | + | case *Reset: | |
| 853 | + | if m == nil { | |
| 854 | + | return ErrInvalidMessage | |
| 855 | + | } | |
| 856 | + | return p.applyServer(*m) | |
| 857 | + | default: | |
| 858 | + | return ErrWrongDirection | |
| 859 | + | } | |
| 860 | + | } | |
| 861 | + | ||
| 862 | + | // applyStreamTerminal applies one inbound Close or Reset. When the engine | |
| 863 | + | // accepted it, the matching outbound scheduler record is reset so queued | |
| 864 | + | // outbound frames for the peer-terminalized stream are discarded rather than | |
| 865 | + | // sent after the peer stopped reading it, and the record is retired. A refused | |
| 866 | + | // terminal falls back to the stream-local refusal path. | |
| 867 | + | func (p *Pump) applyStreamTerminal(physical PhysicalStreamID, disposition StreamDisposition, err error) error { | |
| 868 | + | p.signalWatch(physical) | |
| 869 | + | if err != nil { | |
| 870 | + | return p.applyStreamError(physical, err) | |
| 871 | + | } | |
| 872 | + | if disposition == StreamAccepted { | |
| 873 | + | p.scheduler.Reset(physical) | |
| 874 | + | p.maybeSignalFlush() | |
| 875 | + | } | |
| 876 | + | return nil | |
| 877 | + | } | |
| 878 | + | ||
| 879 | + | // applyData queues one inbound chunk and classifies the engine's refusal. | |
| 880 | + | func (p *Pump) applyData(message Data) error { | |
| 881 | + | _, err := p.engine.Data(message) | |
| 882 | + | return p.applyStreamError(message.Physical, err) | |
| 883 | + | } | |
| 884 | + | ||
| 885 | + | // applyStreamError classifies one engine refusal. A stream-local refusal | |
| 886 | + | // schedules exactly one Reset for the offending stream and returns nil so the | |
| 887 | + | // reader continues; the physical connection stays open. ErrPhysicalClosed only | |
| 888 | + | // occurs once the physical connection is already terminal, so the reader stops | |
| 889 | + | // without a second outcome. A nil refusal applies nothing. | |
| 890 | + | func (p *Pump) applyStreamError(physical PhysicalStreamID, err error) error { | |
| 891 | + | p.signalWatch(physical) | |
| 892 | + | if err == nil { | |
| 893 | + | return nil | |
| 894 | + | } | |
| 895 | + | if errors.Is(err, ErrPhysicalClosed) { | |
| 896 | + | return nil | |
| 897 | + | } | |
| 898 | + | p.scheduleReset(physical, err) | |
| 899 | + | return nil | |
| 900 | + | } | |
| 901 | + | ||
| 902 | + | // scheduleReset isolates one stream-local refusal: it settles the offending | |
| 903 | + | // stream locally, freeing its admission slot, and schedules exactly one Reset | |
| 904 | + | // frame towards the peer. An identity the engine never admitted cannot be | |
| 905 | + | // signalled and is left alone. The engine settlement is always applied first | |
| 906 | + | // and unconditionally, so the slot is freed even when the terminal frame | |
| 907 | + | // cannot be scheduled; the scheduler enqueue result decides only whether the | |
| 908 | + | // peer can be signalled, so a stream that already drained its one Reset is not | |
| 909 | + | // reset twice and a repeated refusal stays idempotent. Siblings and the | |
| 910 | + | // physical connection are untouched. | |
| 911 | + | func (p *Pump) scheduleReset(physical PhysicalStreamID, cause error) { | |
| 912 | + | if physical.Validate() != nil { | |
| 913 | + | return | |
| 914 | + | } | |
| 915 | + | if _, known := p.engine.Status(physical); !known { | |
| 916 | + | return | |
| 917 | + | } | |
| 918 | + | detail := resetErrorDetail(cause) | |
| 919 | + | reset := Reset{Physical: physical, Error: detail, HasError: true} | |
| 920 | + | _, _ = p.engine.Reset(reset) | |
| 921 | + | var err error | |
| 922 | + | if p.local == DirectionServer { | |
| 923 | + | err = p.scheduler.EnqueueServer(reset) | |
| 924 | + | } else { | |
| 925 | + | err = p.scheduler.EnqueueClient(reset) | |
| 926 | + | } | |
| 927 | + | if err != nil { | |
| 928 | + | // The peer cannot be signalled (the record is sealed or the identity | |
| 929 | + | // is not schedulable); the stream is already settled locally. | |
| 930 | + | return | |
| 931 | + | } | |
| 932 | + | p.signal() | |
| 933 | + | } | |
| 934 | + | ||
| 935 | + | // settleFailure publishes one physical failure outcome: the pump's terminal | |
| 936 | + | // state first, then the engine's, so Done closes before every stream is | |
| 937 | + | // terminalized, and then closes the carrier to unblock the peer goroutine. | |
| 938 | + | // The two publications are serialized and first-wins, so a concurrent orderly | |
| 939 | + | // close or a second failure can never make them disagree. | |
| 940 | + | func (p *Pump) settleFailure(kind domain.RemoteFailureKind, cause error) { | |
| 941 | + | p.termMu.Lock() | |
| 942 | + | if !p.settled { | |
| 943 | + | p.settled = true | |
| 944 | + | p.terminal.Fail(kind, cause) | |
| 945 | + | p.engine.Fail(kind, cause) | |
| 946 | + | } | |
| 947 | + | p.termMu.Unlock() | |
| 948 | + | p.stop() | |
| 949 | + | } | |
| 950 | + | ||
| 951 | + | // settleOrderly publishes the orderly local outcome unless a failure already | |
| 952 | + | // won, then closes the carrier. The pump's terminal state is published before | |
| 953 | + | // the engine's, exactly as a failure publishes it. | |
| 954 | + | func (p *Pump) settleOrderly() { | |
| 955 | + | p.termMu.Lock() | |
| 956 | + | if !p.settled { | |
| 957 | + | p.settled = true | |
| 958 | + | p.terminal.Close() | |
| 959 | + | p.engine.Terminate() | |
| 960 | + | } | |
| 961 | + | p.termMu.Unlock() | |
| 962 | + | p.stop() | |
| 963 | + | } | |
| 964 | + | ||
| 965 | + | // stop cancels the run context exactly once and closes the carrier exactly | |
| 966 | + | // once, unblocking an in-flight Send or Receive so both goroutines can exit. | |
| 967 | + | func (p *Pump) stop() { | |
| 968 | + | p.stopOnce.Do(func() { | |
| 969 | + | p.mu.Lock() | |
| 970 | + | cancel := p.cancel | |
| 971 | + | p.mu.Unlock() | |
| 972 | + | if cancel != nil { | |
| 973 | + | cancel() | |
| 974 | + | } | |
| 975 | + | p.closeErr = p.carrier.Close() | |
| 976 | + | }) | |
| 977 | + | } | |
| 978 | + | ||
| 979 | + | // isTerminal reports whether the pump already reached its terminal outcome. | |
| 980 | + | func (p *Pump) isTerminal() bool { | |
| 981 | + | select { | |
| 982 | + | case <-p.terminal.Done(): | |
| 983 | + | return true | |
| 984 | + | default: | |
| 985 | + | return false | |
| 986 | + | } | |
| 987 | + | } | |
| 988 | + | ||
| 989 | + | // oppositeDirection returns the direction a pump writes when it reads inbound. | |
| 990 | + | func oppositeDirection(inbound EnvelopeDirection) EnvelopeDirection { | |
| 991 | + | if inbound == DirectionClient { | |
| 992 | + | return DirectionServer | |
| 993 | + | } | |
| 994 | + | return DirectionClient | |
| 995 | + | } | |
| 996 | + | ||
| 997 | + | // openRefusalDetail builds the typed refusal one fresh Open carries. The | |
| 998 | + | // admission code is the closed taxonomy the broker classifies without matching | |
| 999 | + | // message text: capacity is a limit (1), a closed physical connection is closed | |
| 1000 | + | // (2), and every other refusal is an invalid request (4). The text is a fixed, | |
| 1001 | + | // bounded, presentation-safe summary; the underlying diagnostic cause stays | |
| 1002 | + | // local and never travels on this wire. | |
| 1003 | + | func openRefusalDetail(cause error) ErrorDetail { | |
| 1004 | + | detail := ErrorDetail{ | |
| 1005 | + | Code: ports.BrokerErrorUnavailable, | |
| 1006 | + | AdmissionCode: 4, | |
| 1007 | + | Text: "invalid open request", | |
| 1008 | + | FailureKind: domain.RemoteFailureNone, | |
| 1009 | + | } | |
| 1010 | + | switch { | |
| 1011 | + | case errors.Is(cause, ErrTooManyStreams): | |
| 1012 | + | detail.AdmissionCode = 1 | |
| 1013 | + | detail.Text = "stream limit reached" | |
| 1014 | + | case errors.Is(cause, ErrPhysicalClosed): | |
| 1015 | + | detail.AdmissionCode = 2 | |
| 1016 | + | detail.Text = "physical connection closed" | |
| 1017 | + | } | |
| 1018 | + | return detail | |
| 1019 | + | } | |
| 1020 | + | ||
| 1021 | + | // resetErrorDetail builds the typed detail one scheduled Reset carries. The | |
| 1022 | + | // cause is always one of the package's bounded sentinels, so its text is | |
| 1023 | + | // presentation-safe and within the display bound; a queue overflow is a | |
| 1024 | + | // transport classification and every other stream-local refusal is an invalid | |
| 1025 | + | // response. | |
| 1026 | + | func resetErrorDetail(cause error) ErrorDetail { | |
| 1027 | + | kind := domain.RemoteFailureInvalidResponse | |
| 1028 | + | if errors.Is(cause, ErrStreamQueueFull) || errors.Is(cause, ErrSchedulerFull) { | |
| 1029 | + | // A local byte-budget overflow is a transport-class abort in either | |
| 1030 | + | // direction: the stream was reset because a bounded queue could not | |
| 1031 | + | // hold the frame, not because the peer sent something invalid. | |
| 1032 | + | kind = domain.RemoteFailureTransport | |
| 1033 | + | } | |
| 1034 | + | return ErrorDetail{ | |
| 1035 | + | Code: ports.BrokerErrorUnavailable, | |
| 1036 | + | Text: cause.Error(), | |
| 1037 | + | FailureKind: kind, | |
| 1038 | + | } | |
| 1039 | + | } | |
A internal/adapters/daemonmux/pump_admission_test.go +146 -0
| @@ -0,0 +1,146 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "errors" | |
| 6 | + | "testing" | |
| 7 | + | "time" | |
| 8 | + | ||
| 9 | + | "github.com/stretchr/testify/require" | |
| 10 | + | ||
| 11 | + | "github.com/bnema/vev/internal/domain" | |
| 12 | + | "github.com/bnema/vev/internal/ports" | |
| 13 | + | ) | |
| 14 | + | ||
| 15 | + | // TestPumpApplyOpenDuplicateIsolation proves a duplicate or replayed inbound | |
| 16 | + | // Open never disturbs the stream it names and never produces an outbound frame: | |
| 17 | + | // a live stream keeps its state, its watch, and its queued outbound work, a | |
| 18 | + | // retired stream stays terminal, and a replayed identity below the high-water | |
| 19 | + | // mark stays unrecorded. | |
| 20 | + | func TestPumpApplyOpenDuplicateIsolation(t *testing.T) { | |
| 21 | + | t.Run("live stream", func(t *testing.T) { | |
| 22 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 23 | + | require.NoError(t, pump.Engine().Open(openFor(1))) | |
| 24 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 25 | + | // A queued outbound frame proves the duplicate Open neither resets the | |
| 26 | + | // engine record nor discards the scheduler's queued work. | |
| 27 | + | require.NoError(t, pump.Send(ServerMessage(Data{Physical: 1, Data: []byte("out")}))) | |
| 28 | + | before := mustStatus(t, pump.Engine(), 1) | |
| 29 | + | watch := pump.Watch(1) | |
| 30 | + | ||
| 31 | + | require.NoError(t, pump.applyOpen(openFor(1))) | |
| 32 | + | ||
| 33 | + | after := mustStatus(t, pump.Engine(), 1) | |
| 34 | + | require.Equal(t, before, after, "a duplicate open left the live stream untouched") | |
| 35 | + | require.Equal(t, StreamOpen, after.State) | |
| 36 | + | require.Equal(t, 1, pump.Engine().Live()) | |
| 37 | + | require.False(t, channelClosed(after.Done)) | |
| 38 | + | require.Equal(t, watch, pump.Watch(1), "a duplicate open never rotated the watch channel") | |
| 39 | + | require.Equal(t, 1, pump.scheduler.QueuedFrames(1), "the queued outbound frame survived") | |
| 40 | + | carrier.requireNoSent(t, 50*time.Millisecond) | |
| 41 | + | ||
| 42 | + | pump.Start(context.Background()) | |
| 43 | + | message, err := DecodeServer(carrier.nextSent(t), testEnvelopeCeiling, testChunkCeiling) | |
| 44 | + | require.NoError(t, err) | |
| 45 | + | require.Equal(t, Data{Physical: 1, Data: []byte("out")}, message) | |
| 46 | + | carrier.requireNoSent(t, 50*time.Millisecond) | |
| 47 | + | }) | |
| 48 | + | ||
| 49 | + | t.Run("retired stream", func(t *testing.T) { | |
| 50 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 51 | + | require.NoError(t, pump.Engine().Open(openFor(1))) | |
| 52 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 53 | + | disposition, err := pump.Engine().Close(Close{Physical: 1}) | |
| 54 | + | require.NoError(t, err) | |
| 55 | + | require.Equal(t, StreamAccepted, disposition) | |
| 56 | + | require.Equal(t, StreamTerminal, mustStatus(t, pump.Engine(), 1).State) | |
| 57 | + | ||
| 58 | + | require.NoError(t, pump.applyOpen(openFor(1))) | |
| 59 | + | ||
| 60 | + | status := mustStatus(t, pump.Engine(), 1) | |
| 61 | + | require.Equal(t, StreamTerminal, status.State, "a replayed open never revives a retired stream") | |
| 62 | + | require.NoError(t, status.Err) | |
| 63 | + | require.Zero(t, pump.Engine().Live()) | |
| 64 | + | carrier.requireNoSent(t, 50*time.Millisecond) | |
| 65 | + | }) | |
| 66 | + | ||
| 67 | + | t.Run("replayed identity below the high-water mark", func(t *testing.T) { | |
| 68 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 69 | + | require.NoError(t, pump.Engine().Open(openFor(3))) | |
| 70 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(3)})) | |
| 71 | + | ||
| 72 | + | // Identity 1 is at or below the high-water mark but was never admitted: | |
| 73 | + | // it can only be a replay, so it is ignored instead of refused. | |
| 74 | + | require.NoError(t, pump.applyOpen(openFor(1))) | |
| 75 | + | _, tracked := pump.Engine().Status(1) | |
| 76 | + | require.False(t, tracked, "a replayed identity is never recorded") | |
| 77 | + | require.Equal(t, 1, pump.Engine().Live()) | |
| 78 | + | carrier.requireNoSent(t, 50*time.Millisecond) | |
| 79 | + | }) | |
| 80 | + | } | |
| 81 | + | ||
| 82 | + | // TestPumpFreshOpenRefusedSendsTypedRefused proves a fresh Open the engine | |
| 83 | + | // refused for capacity is signalled with exactly one typed Mux Refused carrying | |
| 84 | + | // its admission code, records no stream, and leaves the live sibling and the | |
| 85 | + | // physical connection untouched. | |
| 86 | + | func TestPumpFreshOpenRefusedSendsTypedRefused(t *testing.T) { | |
| 87 | + | ceilings := DefaultMuxCeilings() | |
| 88 | + | ceilings.MaxStreams = 1 | |
| 89 | + | pump, carrier := newTestPumpWithCeilings(t, DirectionClient, ceilings) | |
| 90 | + | pump.Start(context.Background()) | |
| 91 | + | require.NoError(t, pump.Engine().Open(openFor(1))) | |
| 92 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 93 | + | require.Equal(t, 1, pump.Engine().Live()) | |
| 94 | + | ||
| 95 | + | // The fresh identity is refused by the stream ceiling before it is recorded. | |
| 96 | + | deliverClientFrame(t, carrier, openFor(2)) | |
| 97 | + | ||
| 98 | + | message, err := DecodeServer(carrier.nextSent(t), testEnvelopeCeiling, testChunkCeiling) | |
| 99 | + | require.NoError(t, err) | |
| 100 | + | refused, ok := message.(Refused) | |
| 101 | + | require.True(t, ok, "a fresh refusal is a Mux Refused") | |
| 102 | + | require.Equal(t, testRef(2), refused.Ref) | |
| 103 | + | require.Equal(t, ports.BrokerErrorUnavailable, refused.Error.Code) | |
| 104 | + | require.Equal(t, uint32(1), refused.Error.AdmissionCode, "capacity refusal carries the limit code") | |
| 105 | + | require.Equal(t, domain.RemoteFailureNone, refused.Error.FailureKind) | |
| 106 | + | require.NotEmpty(t, refused.Error.Text) | |
| 107 | + | require.LessOrEqual(t, len(refused.Error.Text), ports.BrokerMaxErrorBytes) | |
| 108 | + | require.NoError(t, refused.Error.validate()) | |
| 109 | + | ||
| 110 | + | // No stream was recorded for the refused identity, and the sibling and the | |
| 111 | + | // physical connection are untouched. | |
| 112 | + | _, tracked := pump.Engine().Status(2) | |
| 113 | + | require.False(t, tracked, "a fresh refusal records no stream") | |
| 114 | + | require.Equal(t, 1, pump.Engine().Live()) | |
| 115 | + | require.False(t, channelClosed(mustStatus(t, pump.Engine(), 1).Done)) | |
| 116 | + | require.False(t, pump.Engine().Closed()) | |
| 117 | + | require.False(t, channelClosed(pump.Done())) | |
| 118 | + | carrier.requireNoSent(t, 50*time.Millisecond) | |
| 119 | + | } | |
| 120 | + | ||
| 121 | + | // TestOpenRefusalDetail proves the closed admission taxonomy the broker | |
| 122 | + | // classifies without matching text: capacity is a limit, a closed physical | |
| 123 | + | // connection is closed, and every other refusal is an invalid request. | |
| 124 | + | func TestOpenRefusalDetail(t *testing.T) { | |
| 125 | + | tests := []struct { | |
| 126 | + | name string | |
| 127 | + | cause error | |
| 128 | + | code uint32 | |
| 129 | + | text string | |
| 130 | + | }{ | |
| 131 | + | {"capacity", ErrTooManyStreams, 1, "stream limit reached"}, | |
| 132 | + | {"closed", ErrPhysicalClosed, 2, "physical connection closed"}, | |
| 133 | + | {"invalid authority", ErrInvalidStream, 4, "invalid open request"}, | |
| 134 | + | {"unknown refusal", errors.New("boom"), 4, "invalid open request"}, | |
| 135 | + | } | |
| 136 | + | for _, tt := range tests { | |
| 137 | + | t.Run(tt.name, func(t *testing.T) { | |
| 138 | + | detail := openRefusalDetail(tt.cause) | |
| 139 | + | require.Equal(t, tt.code, detail.AdmissionCode) | |
| 140 | + | require.Equal(t, tt.text, detail.Text) | |
| 141 | + | require.Equal(t, ports.BrokerErrorUnavailable, detail.Code) | |
| 142 | + | require.Equal(t, domain.RemoteFailureNone, detail.FailureKind) | |
| 143 | + | require.NoError(t, detail.validate()) | |
| 144 | + | }) | |
| 145 | + | } | |
| 146 | + | } | |
A internal/adapters/daemonmux/pump_test.go +1017 -0
| @@ -0,0 +1,1017 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "errors" | |
| 6 | + | "fmt" | |
| 7 | + | "runtime" | |
| 8 | + | "sync" | |
| 9 | + | "sync/atomic" | |
| 10 | + | "testing" | |
| 11 | + | "time" | |
| 12 | + | ||
| 13 | + | "github.com/stretchr/testify/require" | |
| 14 | + | ||
| 15 | + | "github.com/bnema/vev/internal/domain" | |
| 16 | + | ) | |
| 17 | + | ||
| 18 | + | // errFakeCarrierClosed is the synthetic carrier error a fakeCarrier returns | |
| 19 | + | // once Close released its blocked I/O. It mirrors the adapter prompt-close | |
| 20 | + | // contract: Close unblocks an in-flight Send or Receive. | |
| 21 | + | var errFakeCarrierClosed = errors.New("fakeCarrier: closed") | |
| 22 | + | ||
| 23 | + | // fakeFrame is one inbound carrier event: a complete envelope payload, or the | |
| 24 | + | // error the next Receive must report. | |
| 25 | + | type fakeFrame struct { | |
| 26 | + | payload []byte | |
| 27 | + | err error | |
| 28 | + | } | |
| 29 | + | ||
| 30 | + | // fakeCarrier is the deterministic FramedCarrier the pump tests drive. Inbound | |
| 31 | + | // frames are delivered in order through a buffered channel; outbound frames are | |
| 32 | + | // recorded in the order the writer sent them. Close closes a channel that every | |
| 33 | + | // blocked Receive and Send observes, so it unblocks both callers exactly once. | |
| 34 | + | type fakeCarrier struct { | |
| 35 | + | mu sync.Mutex | |
| 36 | + | calls int | |
| 37 | + | // hold, when non-nil, makes Send wait for its release before recording. | |
| 38 | + | hold chan struct{} | |
| 39 | + | holdOnce sync.Once | |
| 40 | + | closeErr error | |
| 41 | + | closeOnce sync.Once | |
| 42 | + | entered atomic.Int64 | |
| 43 | + | ||
| 44 | + | inbound chan fakeFrame | |
| 45 | + | sent chan []byte | |
| 46 | + | closed chan struct{} | |
| 47 | + | } | |
| 48 | + | ||
| 49 | + | func newFakeCarrier() *fakeCarrier { | |
| 50 | + | return &fakeCarrier{ | |
| 51 | + | inbound: make(chan fakeFrame, 1024), | |
| 52 | + | sent: make(chan []byte, 1024), | |
| 53 | + | closed: make(chan struct{}), | |
| 54 | + | } | |
| 55 | + | } | |
| 56 | + | ||
| 57 | + | func (c *fakeCarrier) Receive(ctx context.Context) ([]byte, error) { | |
| 58 | + | select { | |
| 59 | + | case frame := <-c.inbound: | |
| 60 | + | if frame.err != nil { | |
| 61 | + | return nil, frame.err | |
| 62 | + | } | |
| 63 | + | return frame.payload, nil | |
| 64 | + | case <-c.closed: | |
| 65 | + | return nil, errFakeCarrierClosed | |
| 66 | + | case <-ctx.Done(): | |
| 67 | + | return nil, ctx.Err() | |
| 68 | + | } | |
| 69 | + | } | |
| 70 | + | ||
| 71 | + | func (c *fakeCarrier) Send(ctx context.Context, payload []byte) error { | |
| 72 | + | c.entered.Add(1) | |
| 73 | + | c.mu.Lock() | |
| 74 | + | hold := c.hold | |
| 75 | + | c.mu.Unlock() | |
| 76 | + | if hold != nil { | |
| 77 | + | select { | |
| 78 | + | case <-hold: | |
| 79 | + | case <-c.closed: | |
| 80 | + | return errFakeCarrierClosed | |
| 81 | + | case <-ctx.Done(): | |
| 82 | + | return ctx.Err() | |
| 83 | + | } | |
| 84 | + | } | |
| 85 | + | select { | |
| 86 | + | case c.sent <- append([]byte(nil), payload...): | |
| 87 | + | return nil | |
| 88 | + | case <-c.closed: | |
| 89 | + | return errFakeCarrierClosed | |
| 90 | + | case <-ctx.Done(): | |
| 91 | + | return ctx.Err() | |
| 92 | + | } | |
| 93 | + | } | |
| 94 | + | ||
| 95 | + | func (c *fakeCarrier) Close() error { | |
| 96 | + | c.closeOnce.Do(func() { | |
| 97 | + | c.mu.Lock() | |
| 98 | + | c.calls++ | |
| 99 | + | c.mu.Unlock() | |
| 100 | + | close(c.closed) | |
| 101 | + | }) | |
| 102 | + | c.mu.Lock() | |
| 103 | + | defer c.mu.Unlock() | |
| 104 | + | return c.closeErr | |
| 105 | + | } | |
| 106 | + | ||
| 107 | + | func (c *fakeCarrier) closeCalls() int { | |
| 108 | + | c.mu.Lock() | |
| 109 | + | defer c.mu.Unlock() | |
| 110 | + | return c.calls | |
| 111 | + | } | |
| 112 | + | ||
| 113 | + | // setHold makes the next Send wait until release is called. | |
| 114 | + | func (c *fakeCarrier) setHold(hold chan struct{}) { | |
| 115 | + | c.mu.Lock() | |
| 116 | + | defer c.mu.Unlock() | |
| 117 | + | c.hold = hold | |
| 118 | + | } | |
| 119 | + | ||
| 120 | + | // release unblocks a held Send exactly once. | |
| 121 | + | func (c *fakeCarrier) release(hold chan struct{}) { | |
| 122 | + | c.holdOnce.Do(func() { close(hold) }) | |
| 123 | + | } | |
| 124 | + | ||
| 125 | + | // enteredSends reports how many Send calls the writer has started. | |
| 126 | + | func (c *fakeCarrier) enteredSends() int64 { return c.entered.Load() } | |
| 127 | + | ||
| 128 | + | // deliver queues one inbound frame for the next Receive, in order. | |
| 129 | + | func (c *fakeCarrier) deliver(t *testing.T, frame fakeFrame) { | |
| 130 | + | t.Helper() | |
| 131 | + | select { | |
| 132 | + | case c.inbound <- frame: | |
| 133 | + | case <-time.After(5 * time.Second): | |
| 134 | + | t.Fatal("fakeCarrier: inbound delivery timed out") | |
| 135 | + | } | |
| 136 | + | } | |
| 137 | + | ||
| 138 | + | // nextSent returns the next outbound frame the writer recorded, in order. | |
| 139 | + | func (c *fakeCarrier) nextSent(t *testing.T) []byte { | |
| 140 | + | t.Helper() | |
| 141 | + | select { | |
| 142 | + | case payload := <-c.sent: | |
| 143 | + | return payload | |
| 144 | + | case <-time.After(5 * time.Second): | |
| 145 | + | t.Fatal("fakeCarrier: expected an outbound frame") | |
| 146 | + | return nil | |
| 147 | + | } | |
| 148 | + | } | |
| 149 | + | ||
| 150 | + | // requireNoSent fails when the writer recorded an outbound frame within d. | |
| 151 | + | func (c *fakeCarrier) requireNoSent(t *testing.T, d time.Duration) { | |
| 152 | + | t.Helper() | |
| 153 | + | select { | |
| 154 | + | case payload := <-c.sent: | |
| 155 | + | t.Fatalf("fakeCarrier: unexpected outbound frame %q", payload) | |
| 156 | + | case <-time.After(d): | |
| 157 | + | } | |
| 158 | + | } | |
| 159 | + | ||
| 160 | + | // newTestPump builds a pump over a fresh fakeCarrier under the default | |
| 161 | + | // ceilings and closes it when the test ends. | |
| 162 | + | func newTestPump(t *testing.T, inbound EnvelopeDirection) (*Pump, *fakeCarrier) { | |
| 163 | + | t.Helper() | |
| 164 | + | return newTestPumpWithCeilings(t, inbound, DefaultMuxCeilings()) | |
| 165 | + | } | |
| 166 | + | ||
| 167 | + | func newTestPumpWithCeilings(t *testing.T, inbound EnvelopeDirection, ceilings MuxCeilings) (*Pump, *fakeCarrier) { | |
| 168 | + | t.Helper() | |
| 169 | + | carrier := newFakeCarrier() | |
| 170 | + | pump, err := NewPump(carrier, inbound, ceilings) | |
| 171 | + | require.NoError(t, err) | |
| 172 | + | t.Cleanup(func() { _ = pump.Close() }) | |
| 173 | + | return pump, carrier | |
| 174 | + | } | |
| 175 | + | ||
| 176 | + | // openFor returns one complete, encodable attachment Open for a physical | |
| 177 | + | // identity: the wire codec validates the whole request, not just the reference. | |
| 178 | + | func openFor(id PhysicalStreamID) Open { | |
| 179 | + | open := testOpen() | |
| 180 | + | open.Ref = testRef(id) | |
| 181 | + | return open | |
| 182 | + | } | |
| 183 | + | ||
| 184 | + | func deliverClientFrame(t *testing.T, carrier *fakeCarrier, message ClientMessage) { | |
| 185 | + | t.Helper() | |
| 186 | + | carrier.deliver(t, fakeFrame{payload: mustEncodeClient(t, message)}) | |
| 187 | + | } | |
| 188 | + | ||
| 189 | + | func deliverServerFrame(t *testing.T, carrier *fakeCarrier, message ServerMessage) { | |
| 190 | + | t.Helper() | |
| 191 | + | carrier.deliver(t, fakeFrame{payload: mustEncodeServer(t, message)}) | |
| 192 | + | } | |
| 193 | + | ||
| 194 | + | // requireEngineEventually waits until the engine satisfies cond. | |
| 195 | + | func requireEngineEventually(t *testing.T, pump *Pump, cond func(*StreamEngine) bool, msgAndArgs ...any) { | |
| 196 | + | t.Helper() | |
| 197 | + | require.Eventually(t, func() bool { return cond(pump.Engine()) }, 5*time.Second, time.Millisecond, msgAndArgs...) | |
| 198 | + | } | |
| 199 | + | ||
| 200 | + | // takeEventually waits until one stream's inbound queue yields a chunk. | |
| 201 | + | func takeEventually(t *testing.T, pump *Pump, id PhysicalStreamID) ([]byte, bool) { | |
| 202 | + | t.Helper() | |
| 203 | + | var chunk []byte | |
| 204 | + | require.Eventually(t, func() bool { | |
| 205 | + | taken, ok := pump.Take(id) | |
| 206 | + | if ok { | |
| 207 | + | chunk = taken | |
| 208 | + | } | |
| 209 | + | return ok | |
| 210 | + | }, 5*time.Second, time.Millisecond) | |
| 211 | + | return chunk, chunk != nil | |
| 212 | + | } | |
| 213 | + | ||
| 214 | + | // decodeReset decodes one outbound Reset frame in the given direction. | |
| 215 | + | func decodeReset(t *testing.T, payload []byte, direction EnvelopeDirection) Reset { | |
| 216 | + | t.Helper() | |
| 217 | + | var message any | |
| 218 | + | var err error | |
| 219 | + | if direction == DirectionServer { | |
| 220 | + | message, err = DecodeServer(payload, testEnvelopeCeiling, testChunkCeiling) | |
| 221 | + | } else { | |
| 222 | + | message, err = DecodeClient(payload, testEnvelopeCeiling, testChunkCeiling) | |
| 223 | + | } | |
| 224 | + | require.NoError(t, err) | |
| 225 | + | reset, ok := message.(Reset) | |
| 226 | + | require.True(t, ok, "outbound frame is not a Reset") | |
| 227 | + | return reset | |
| 228 | + | } | |
| 229 | + | ||
| 230 | + | func TestNewPumpValidation(t *testing.T) { | |
| 231 | + | t.Run("missing carrier", func(t *testing.T) { | |
| 232 | + | pump, err := NewPump(nil, DirectionClient, DefaultMuxCeilings()) | |
| 233 | + | require.ErrorIs(t, err, ErrPumpConfig) | |
| 234 | + | require.Nil(t, pump) | |
| 235 | + | }) | |
| 236 | + | t.Run("unknown inbound direction", func(t *testing.T) { | |
| 237 | + | for _, direction := range []EnvelopeDirection{0, EnvelopeDirection(200)} { | |
| 238 | + | pump, err := NewPump(newFakeCarrier(), direction, DefaultMuxCeilings()) | |
| 239 | + | require.ErrorIs(t, err, ErrPumpConfig) | |
| 240 | + | require.Nil(t, pump) | |
| 241 | + | } | |
| 242 | + | }) | |
| 243 | + | t.Run("invalid ceilings", func(t *testing.T) { | |
| 244 | + | pump, err := NewPump(newFakeCarrier(), DirectionClient, MuxCeilings{}) | |
| 245 | + | require.ErrorIs(t, err, ErrInvalidCeilings) | |
| 246 | + | require.Nil(t, pump) | |
| 247 | + | }) | |
| 248 | + | t.Run("valid", func(t *testing.T) { | |
| 249 | + | pump, carrier := newTestPumpWithCeilings(t, DirectionServer, MuxCeilings{ | |
| 250 | + | MaxReceiveEnvelopeBytes: MinMuxEnvelopeBytes, | |
| 251 | + | StreamChunkLimit: MinMuxChunkBytes, | |
| 252 | + | MaxStreams: MinMuxStreams, | |
| 253 | + | MaxAggregateBytes: MinMuxAggregateBytes, | |
| 254 | + | }) | |
| 255 | + | require.NotNil(t, pump.Engine()) | |
| 256 | + | require.False(t, channelClosed(pump.Done())) | |
| 257 | + | require.NoError(t, pump.Err()) | |
| 258 | + | require.Equal(t, domain.RemoteFailureNone, pump.FailureKind()) | |
| 259 | + | require.NoError(t, pump.Close()) | |
| 260 | + | require.True(t, channelClosed(pump.Done())) | |
| 261 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 262 | + | require.ErrorIs(t, pump.Send(Data{Physical: 1, Data: []byte("x")}), ErrPhysicalClosed) | |
| 263 | + | }) | |
| 264 | + | } | |
| 265 | + | ||
| 266 | + | // TestPumpTwoAndHundredStreams drives inbound and outbound traffic over two | |
| 267 | + | // and a hundred concurrent streams: every inbound envelope is applied, every | |
| 268 | + | // locally admitted stream's frame is written, and closing every stream leaves | |
| 269 | + | // the physical connection open. | |
| 270 | + | func TestPumpTwoAndHundredStreams(t *testing.T) { | |
| 271 | + | for _, count := range []int{2, 100} { | |
| 272 | + | t.Run(fmt.Sprintf("%d streams", count), func(t *testing.T) { | |
| 273 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 274 | + | pump.Start(context.Background()) | |
| 275 | + | ||
| 276 | + | ids := make([]PhysicalStreamID, count) | |
| 277 | + | for i := range ids { | |
| 278 | + | ids[i] = PhysicalStreamID(i + 1) | |
| 279 | + | deliverClientFrame(t, carrier, openFor(ids[i])) | |
| 280 | + | } | |
| 281 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == count }) | |
| 282 | + | ||
| 283 | + | for _, id := range ids { | |
| 284 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(id)})) | |
| 285 | + | deliverClientFrame(t, carrier, Data{Physical: id, Data: []byte{byte(id)}}) | |
| 286 | + | } | |
| 287 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.AggregateBytes() == count }) | |
| 288 | + | ||
| 289 | + | taken, ok := pump.Take(ids[0]) | |
| 290 | + | require.True(t, ok) | |
| 291 | + | require.Equal(t, []byte{byte(ids[0])}, taken) | |
| 292 | + | require.Equal(t, count-1, pump.Engine().AggregateBytes()) | |
| 293 | + | ||
| 294 | + | for _, id := range ids { | |
| 295 | + | require.NoError(t, pump.Send(ServerMessage(Data{Physical: id, Data: []byte("out")}))) | |
| 296 | + | } | |
| 297 | + | seen := make(map[PhysicalStreamID]int, count) | |
| 298 | + | for i := 0; i < count; i++ { | |
| 299 | + | message, err := DecodeServer(carrier.nextSent(t), testEnvelopeCeiling, testChunkCeiling) | |
| 300 | + | require.NoError(t, err) | |
| 301 | + | data, ok := message.(Data) | |
| 302 | + | require.True(t, ok) | |
| 303 | + | require.Equal(t, []byte("out"), data.Data) | |
| 304 | + | seen[data.Physical]++ | |
| 305 | + | } | |
| 306 | + | require.Len(t, seen, count) | |
| 307 | + | ||
| 308 | + | for _, id := range ids { | |
| 309 | + | deliverClientFrame(t, carrier, Close{Physical: id}) | |
| 310 | + | } | |
| 311 | + | // The drained stream settles at once; every other stream is closing | |
| 312 | + | // and still holds its one unread, charged chunk until it is drained. | |
| 313 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == count-1 }) | |
| 314 | + | require.Equal(t, count-1, pump.Engine().AggregateBytes()) | |
| 315 | + | require.False(t, pump.Engine().Closed()) | |
| 316 | + | require.False(t, channelClosed(pump.Done())) | |
| 317 | + | terminal := mustStatus(t, pump.Engine(), ids[0]) | |
| 318 | + | require.Equal(t, StreamTerminal, terminal.State) | |
| 319 | + | require.True(t, channelClosed(terminal.Done)) | |
| 320 | + | for _, id := range ids[1:] { | |
| 321 | + | status := mustStatus(t, pump.Engine(), id) | |
| 322 | + | require.Equal(t, StreamClosing, status.State) | |
| 323 | + | require.False(t, channelClosed(status.Done)) | |
| 324 | + | require.Equal(t, 1, status.QueuedChunks) | |
| 325 | + | } | |
| 326 | + | ||
| 327 | + | // Draining every preserved chunk releases its bytes and terminalizes | |
| 328 | + | // the stream; the physical connection stays open. | |
| 329 | + | for _, id := range ids[1:] { | |
| 330 | + | chunk, ok := takeEventually(t, pump, id) | |
| 331 | + | require.True(t, ok) | |
| 332 | + | require.Equal(t, []byte{byte(id)}, chunk) | |
| 333 | + | } | |
| 334 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == 0 }) | |
| 335 | + | require.Zero(t, pump.Engine().AggregateBytes()) | |
| 336 | + | require.False(t, pump.Engine().Closed()) | |
| 337 | + | require.False(t, channelClosed(pump.Done())) | |
| 338 | + | for _, id := range ids { | |
| 339 | + | status := mustStatus(t, pump.Engine(), id) | |
| 340 | + | require.Equal(t, StreamTerminal, status.State) | |
| 341 | + | require.True(t, channelClosed(status.Done)) | |
| 342 | + | require.NoError(t, status.Err) | |
| 343 | + | } | |
| 344 | + | }) | |
| 345 | + | } | |
| 346 | + | } | |
| 347 | + | ||
| 348 | + | // TestPumpBlockedConsumerDoesNotBlockReaderOrSibling proves a logical consumer | |
| 349 | + | // that never drains one stream's inbound queue neither blocks the reader nor a | |
| 350 | + | // sibling: the stalled stream is reset by its own bound while the sibling keeps | |
| 351 | + | // receiving. | |
| 352 | + | func TestPumpBlockedConsumerDoesNotBlockReaderOrSibling(t *testing.T) { | |
| 353 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 354 | + | pump.Start(context.Background()) | |
| 355 | + | ||
| 356 | + | const siblings = 2 | |
| 357 | + | for i := 1; i <= siblings; i++ { | |
| 358 | + | deliverClientFrame(t, carrier, openFor(PhysicalStreamID(i))) | |
| 359 | + | } | |
| 360 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == siblings }) | |
| 361 | + | for i := 1; i <= siblings; i++ { | |
| 362 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(PhysicalStreamID(i))})) | |
| 363 | + | } | |
| 364 | + | ||
| 365 | + | // The consumer never Takes stream 1: its bounded queue fills. | |
| 366 | + | for i := 0; i < MaxMuxStreamQueueChunks; i++ { | |
| 367 | + | deliverClientFrame(t, carrier, Data{Physical: 1, Data: []byte{byte(i)}}) | |
| 368 | + | } | |
| 369 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { | |
| 370 | + | status, ok := e.Status(1) | |
| 371 | + | return ok && status.QueuedChunks == MaxMuxStreamQueueChunks | |
| 372 | + | }) | |
| 373 | + | ||
| 374 | + | // One more chunk overflows stream 1's own bound; a sibling chunk still | |
| 375 | + | // lands, so neither the reader nor the sibling was blocked. | |
| 376 | + | deliverClientFrame(t, carrier, Data{Physical: 1, Data: []byte{'x'}}) | |
| 377 | + | deliverClientFrame(t, carrier, Data{Physical: 2, Data: []byte("sibling")}) | |
| 378 | + | ||
| 379 | + | taken, ok := takeEventually(t, pump, 2) | |
| 380 | + | require.True(t, ok) | |
| 381 | + | require.Equal(t, []byte("sibling"), taken) | |
| 382 | + | ||
| 383 | + | stalled := mustStatus(t, pump.Engine(), 1) | |
| 384 | + | require.Equal(t, StreamTerminal, stalled.State) | |
| 385 | + | require.ErrorIs(t, stalled.Err, ErrStreamQueueFull) | |
| 386 | + | require.Equal(t, domain.RemoteFailureTransport, stalled.FailureKind) | |
| 387 | + | require.Zero(t, stalled.QueuedChunks) | |
| 388 | + | require.Equal(t, StreamOpen, mustStatus(t, pump.Engine(), 2).State) | |
| 389 | + | require.False(t, pump.Engine().Closed()) | |
| 390 | + | require.False(t, channelClosed(pump.Done())) | |
| 391 | + | ||
| 392 | + | reset := decodeReset(t, carrier.nextSent(t), DirectionServer) | |
| 393 | + | require.Equal(t, PhysicalStreamID(1), reset.Physical) | |
| 394 | + | require.Equal(t, domain.RemoteFailureTransport, reset.Error.FailureKind) | |
| 395 | + | } | |
| 396 | + | ||
| 397 | + | // TestPumpBlockedWriterDoesNotBlockReader proves the reader keeps applying | |
| 398 | + | // inbound frames while the writer is blocked inside a slow carrier Send. | |
| 399 | + | func TestPumpBlockedWriterDoesNotBlockReader(t *testing.T) { | |
| 400 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 401 | + | pump.Start(context.Background()) | |
| 402 | + | ||
| 403 | + | deliverClientFrame(t, carrier, openFor(1)) | |
| 404 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == 1 }) | |
| 405 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 406 | + | ||
| 407 | + | hold := make(chan struct{}) | |
| 408 | + | carrier.setHold(hold) | |
| 409 | + | require.NoError(t, pump.Send(Reset{Physical: 1})) | |
| 410 | + | require.Eventually(t, func() bool { return carrier.enteredSends() >= 1 }, 5*time.Second, time.Millisecond) | |
| 411 | + | ||
| 412 | + | for i := 0; i < 3; i++ { | |
| 413 | + | deliverClientFrame(t, carrier, Data{Physical: 1, Data: []byte{byte(i)}}) | |
| 414 | + | } | |
| 415 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.AggregateBytes() == 3 }) | |
| 416 | + | ||
| 417 | + | // Nothing reaches the wire until the carrier releases the blocked write. | |
| 418 | + | carrier.requireNoSent(t, 50*time.Millisecond) | |
| 419 | + | carrier.release(hold) | |
| 420 | + | ||
| 421 | + | message, err := DecodeServer(carrier.nextSent(t), testEnvelopeCeiling, testChunkCeiling) | |
| 422 | + | require.NoError(t, err) | |
| 423 | + | _, ok := message.(Reset) | |
| 424 | + | require.True(t, ok) | |
| 425 | + | } | |
| 426 | + | ||
| 427 | + | // TestPumpFairWriterProgress proves the writer preserves the scheduler's | |
| 428 | + | // deterministic byte-deficit round-robin order and each stream's own order. | |
| 429 | + | func TestPumpFairWriterProgress(t *testing.T) { | |
| 430 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 431 | + | pump.Start(context.Background()) | |
| 432 | + | ||
| 433 | + | const count = 4 | |
| 434 | + | ids := make([]PhysicalStreamID, count) | |
| 435 | + | for i := range ids { | |
| 436 | + | ids[i] = PhysicalStreamID(i + 1) | |
| 437 | + | deliverClientFrame(t, carrier, openFor(ids[i])) | |
| 438 | + | } | |
| 439 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == count }) | |
| 440 | + | ||
| 441 | + | const rounds = 2 | |
| 442 | + | for round := 0; round < rounds; round++ { | |
| 443 | + | for _, id := range ids { | |
| 444 | + | require.NoError(t, pump.Send(ServerMessage(Data{Physical: id, Data: []byte{byte(id), byte(round)}}))) | |
| 445 | + | } | |
| 446 | + | } | |
| 447 | + | ||
| 448 | + | seen := make(map[PhysicalStreamID]int, count) | |
| 449 | + | for served := 0; served < rounds*count; served++ { | |
| 450 | + | want := ids[served%count] | |
| 451 | + | message, err := DecodeServer(carrier.nextSent(t), testEnvelopeCeiling, testChunkCeiling) | |
| 452 | + | require.NoError(t, err) | |
| 453 | + | data, ok := message.(Data) | |
| 454 | + | require.True(t, ok) | |
| 455 | + | require.Equal(t, want, data.Physical, "dequeue %d", served) | |
| 456 | + | require.Equal(t, []byte{byte(want), byte(seen[want])}, data.Data, "per-stream order preserved") | |
| 457 | + | seen[want]++ | |
| 458 | + | } | |
| 459 | + | for _, id := range ids { | |
| 460 | + | require.Equal(t, rounds, seen[id]) | |
| 461 | + | } | |
| 462 | + | } | |
| 463 | + | ||
| 464 | + | // TestPumpResetIsolation proves one stream-local refusal schedules exactly one | |
| 465 | + | // Reset for its own stream, settles it locally, and leaves the physical | |
| 466 | + | // connection and every sibling healthy. | |
| 467 | + | func TestPumpResetIsolation(t *testing.T) { | |
| 468 | + | t.Run("semantic refusal", func(t *testing.T) { | |
| 469 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 470 | + | pump.Start(context.Background()) | |
| 471 | + | ||
| 472 | + | for _, id := range []PhysicalStreamID{1, 2, 3} { | |
| 473 | + | deliverClientFrame(t, carrier, openFor(id)) | |
| 474 | + | } | |
| 475 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == 3 }) | |
| 476 | + | for _, id := range []PhysicalStreamID{2, 3} { | |
| 477 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(id)})) | |
| 478 | + | } | |
| 479 | + | ||
| 480 | + | // Stream 1 is still opening: inbound data is a stream-local state | |
| 481 | + | // refusal. | |
| 482 | + | deliverClientFrame(t, carrier, Data{Physical: 1, Data: []byte("early")}) | |
| 483 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { | |
| 484 | + | status, ok := e.Status(1) | |
| 485 | + | return ok && status.State == StreamTerminal | |
| 486 | + | }) | |
| 487 | + | settled := mustStatus(t, pump.Engine(), 1) | |
| 488 | + | require.ErrorContains(t, settled.Err, "invalid stream state") | |
| 489 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, settled.FailureKind) | |
| 490 | + | ||
| 491 | + | deliverClientFrame(t, carrier, Data{Physical: 2, Data: []byte("sibling")}) | |
| 492 | + | taken, ok := takeEventually(t, pump, 2) | |
| 493 | + | require.True(t, ok) | |
| 494 | + | require.Equal(t, []byte("sibling"), taken) | |
| 495 | + | ||
| 496 | + | reset := decodeReset(t, carrier.nextSent(t), DirectionServer) | |
| 497 | + | require.Equal(t, PhysicalStreamID(1), reset.Physical) | |
| 498 | + | require.True(t, reset.HasError) | |
| 499 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, reset.Error.FailureKind) | |
| 500 | + | ||
| 501 | + | // Exactly one Reset: a later frame for the settled stream is | |
| 502 | + | // discarded, so no second frame is scheduled. | |
| 503 | + | deliverClientFrame(t, carrier, Data{Physical: 1, Data: []byte("late")}) | |
| 504 | + | carrier.requireNoSent(t, 50*time.Millisecond) | |
| 505 | + | ||
| 506 | + | require.False(t, pump.Engine().Closed()) | |
| 507 | + | require.False(t, channelClosed(pump.Done())) | |
| 508 | + | require.Equal(t, StreamOpen, mustStatus(t, pump.Engine(), 2).State) | |
| 509 | + | require.Equal(t, StreamOpen, mustStatus(t, pump.Engine(), 3).State) | |
| 510 | + | require.Equal(t, 2, pump.Engine().Live()) | |
| 511 | + | }) | |
| 512 | + | ||
| 513 | + | t.Run("queue overflow", func(t *testing.T) { | |
| 514 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 515 | + | pump.Start(context.Background()) | |
| 516 | + | ||
| 517 | + | for _, id := range []PhysicalStreamID{1, 2} { | |
| 518 | + | deliverClientFrame(t, carrier, openFor(id)) | |
| 519 | + | } | |
| 520 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == 2 }) | |
| 521 | + | for _, id := range []PhysicalStreamID{1, 2} { | |
| 522 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(id)})) | |
| 523 | + | } | |
| 524 | + | ||
| 525 | + | for i := 0; i < MaxMuxStreamQueueChunks; i++ { | |
| 526 | + | deliverClientFrame(t, carrier, Data{Physical: 1, Data: []byte{byte(i)}}) | |
| 527 | + | } | |
| 528 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { | |
| 529 | + | status, ok := e.Status(1) | |
| 530 | + | return ok && status.QueuedChunks == MaxMuxStreamQueueChunks | |
| 531 | + | }) | |
| 532 | + | deliverClientFrame(t, carrier, Data{Physical: 1, Data: []byte{'x'}}) | |
| 533 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { | |
| 534 | + | status, ok := e.Status(1) | |
| 535 | + | return ok && status.State == StreamTerminal | |
| 536 | + | }) | |
| 537 | + | ||
| 538 | + | deliverClientFrame(t, carrier, Data{Physical: 2, Data: []byte("sibling")}) | |
| 539 | + | taken, ok := takeEventually(t, pump, 2) | |
| 540 | + | require.True(t, ok) | |
| 541 | + | require.Equal(t, []byte("sibling"), taken) | |
| 542 | + | ||
| 543 | + | reset := decodeReset(t, carrier.nextSent(t), DirectionServer) | |
| 544 | + | require.Equal(t, PhysicalStreamID(1), reset.Physical) | |
| 545 | + | require.Equal(t, domain.RemoteFailureTransport, reset.Error.FailureKind) | |
| 546 | + | require.False(t, pump.Engine().Closed()) | |
| 547 | + | }) | |
| 548 | + | } | |
| 549 | + | ||
| 550 | + | // TestPumpMalformedPhysicalFatal proves a malformed outer envelope is | |
| 551 | + | // physical-fatal: Done closes before every stream is terminalized, the carrier | |
| 552 | + | // is closed, and both goroutines stop. | |
| 553 | + | func TestPumpMalformedPhysicalFatal(t *testing.T) { | |
| 554 | + | tight := MuxCeilings{ | |
| 555 | + | MaxReceiveEnvelopeBytes: MinMuxEnvelopeBytes, | |
| 556 | + | StreamChunkLimit: MinMuxChunkBytes, | |
| 557 | + | MaxStreams: MinMuxStreams, | |
| 558 | + | MaxAggregateBytes: MinMuxAggregateBytes, | |
| 559 | + | } | |
| 560 | + | cases := []struct { | |
| 561 | + | name string | |
| 562 | + | ceilings MuxCeilings | |
| 563 | + | payload []byte | |
| 564 | + | }{ | |
| 565 | + | {"garbage payload", DefaultMuxCeilings(), []byte{0xde, 0xad, 0xbe, 0xef}}, | |
| 566 | + | {"empty payload", DefaultMuxCeilings(), nil}, | |
| 567 | + | {"wrong direction", DefaultMuxCeilings(), mustEncodeServer(t, Opened{Ref: testRef(1)})}, | |
| 568 | + | {"chunk above negotiated ceiling", tight, mustEncodeClient(t, Data{Physical: 1, Data: []byte("ab")})}, | |
| 569 | + | } | |
| 570 | + | for _, tc := range cases { | |
| 571 | + | t.Run(tc.name, func(t *testing.T) { | |
| 572 | + | pump, carrier := newTestPumpWithCeilings(t, DirectionClient, tc.ceilings) | |
| 573 | + | pump.Start(context.Background()) | |
| 574 | + | deliverClientFrame(t, carrier, openFor(1)) | |
| 575 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == 1 }) | |
| 576 | + | ||
| 577 | + | carrier.deliver(t, fakeFrame{payload: tc.payload}) | |
| 578 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Closed() }) | |
| 579 | + | ||
| 580 | + | require.Error(t, pump.Err()) | |
| 581 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, pump.FailureKind()) | |
| 582 | + | require.True(t, channelClosed(pump.Engine().Done())) | |
| 583 | + | status := mustStatus(t, pump.Engine(), 1) | |
| 584 | + | require.Equal(t, StreamTerminal, status.State) | |
| 585 | + | require.True(t, channelClosed(status.Done)) | |
| 586 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, status.FailureKind) | |
| 587 | + | ||
| 588 | + | require.Eventually(t, func() bool { | |
| 589 | + | return channelClosed(pump.readerDone) && channelClosed(pump.writerDone) | |
| 590 | + | }, 5*time.Second, time.Millisecond) | |
| 591 | + | require.Eventually(t, func() bool { return carrier.closeCalls() == 1 }, 5*time.Second, time.Millisecond) | |
| 592 | + | }) | |
| 593 | + | } | |
| 594 | + | } | |
| 595 | + | ||
| 596 | + | // TestPumpCarrierLossFanoutOrdering proves a carrier error is physical-fatal, | |
| 597 | + | // that the physical Done closes before every stream Done, and that every live | |
| 598 | + | // stream observes the same transport cause. | |
| 599 | + | func TestPumpCarrierLossFanoutOrdering(t *testing.T) { | |
| 600 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 601 | + | pump.Start(context.Background()) | |
| 602 | + | ||
| 603 | + | const count = 3 | |
| 604 | + | for i := 1; i <= count; i++ { | |
| 605 | + | deliverClientFrame(t, carrier, openFor(PhysicalStreamID(i))) | |
| 606 | + | } | |
| 607 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == count }) | |
| 608 | + | ||
| 609 | + | // Each watcher records whether the physical Done was already closed when | |
| 610 | + | // its stream's Done fired. | |
| 611 | + | ordered := make(chan bool, count) | |
| 612 | + | for i := 1; i <= count; i++ { | |
| 613 | + | done := mustStatus(t, pump.Engine(), PhysicalStreamID(i)).Done | |
| 614 | + | go func(streamDone <-chan struct{}) { | |
| 615 | + | select { | |
| 616 | + | case <-streamDone: | |
| 617 | + | case <-time.After(5 * time.Second): | |
| 618 | + | ordered <- false | |
| 619 | + | return | |
| 620 | + | } | |
| 621 | + | select { | |
| 622 | + | case <-pump.Done(): | |
| 623 | + | ordered <- true | |
| 624 | + | case <-time.After(5 * time.Second): | |
| 625 | + | ordered <- false | |
| 626 | + | } | |
| 627 | + | }(done) | |
| 628 | + | } | |
| 629 | + | ||
| 630 | + | cause := errors.New("carrier lost") | |
| 631 | + | carrier.deliver(t, fakeFrame{err: cause}) | |
| 632 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Closed() }) | |
| 633 | + | ||
| 634 | + | for i := 0; i < count; i++ { | |
| 635 | + | require.True(t, <-ordered, "stream Done fired before the physical Done") | |
| 636 | + | } | |
| 637 | + | require.ErrorIs(t, pump.Err(), cause) | |
| 638 | + | require.Equal(t, domain.RemoteFailureTransport, pump.FailureKind()) | |
| 639 | + | require.Zero(t, pump.Engine().Live()) | |
| 640 | + | for i := 1; i <= count; i++ { | |
| 641 | + | status := mustStatus(t, pump.Engine(), PhysicalStreamID(i)) | |
| 642 | + | require.Equal(t, StreamTerminal, status.State) | |
| 643 | + | require.True(t, channelClosed(status.Done)) | |
| 644 | + | require.ErrorIs(t, status.Err, cause) | |
| 645 | + | require.Equal(t, domain.RemoteFailureTransport, status.FailureKind) | |
| 646 | + | } | |
| 647 | + | require.Eventually(t, func() bool { | |
| 648 | + | return channelClosed(pump.readerDone) && channelClosed(pump.writerDone) | |
| 649 | + | }, 5*time.Second, time.Millisecond) | |
| 650 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 651 | + | } | |
| 652 | + | ||
| 653 | + | // TestPumpCancellationNoLeak proves cancelling the run context is an orderly | |
| 654 | + | // close: the physical connection settles with a nil cause, streams settle with | |
| 655 | + | // nil, the carrier is closed, and no pump goroutine survives. | |
| 656 | + | func TestPumpCancellationNoLeak(t *testing.T) { | |
| 657 | + | before := runtime.NumGoroutine() | |
| 658 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 659 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 660 | + | pump.Start(ctx) | |
| 661 | + | ||
| 662 | + | for _, id := range []PhysicalStreamID{1, 2} { | |
| 663 | + | deliverClientFrame(t, carrier, openFor(id)) | |
| 664 | + | } | |
| 665 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == 2 }) | |
| 666 | + | ||
| 667 | + | cancel() | |
| 668 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Closed() }) | |
| 669 | + | require.NoError(t, pump.Err()) | |
| 670 | + | require.Equal(t, domain.RemoteFailureNone, pump.FailureKind()) | |
| 671 | + | for _, id := range []PhysicalStreamID{1, 2} { | |
| 672 | + | status := mustStatus(t, pump.Engine(), id) | |
| 673 | + | require.Equal(t, StreamTerminal, status.State) | |
| 674 | + | require.True(t, channelClosed(status.Done)) | |
| 675 | + | require.NoError(t, status.Err) | |
| 676 | + | require.Equal(t, domain.RemoteFailureNone, status.FailureKind) | |
| 677 | + | } | |
| 678 | + | require.Eventually(t, func() bool { return channelClosed(pump.readerDone) }, 5*time.Second, time.Millisecond) | |
| 679 | + | require.Eventually(t, func() bool { return channelClosed(pump.writerDone) }, 5*time.Second, time.Millisecond) | |
| 680 | + | require.Eventually(t, func() bool { return carrier.closeCalls() == 1 }, 5*time.Second, time.Millisecond) | |
| 681 | + | require.Eventually(t, func() bool { return runtime.NumGoroutine() <= before+2 }, 5*time.Second, time.Millisecond, | |
| 682 | + | "pump goroutines leaked after cancellation") | |
| 683 | + | } | |
| 684 | + | ||
| 685 | + | // TestPumpConcurrentClose proves concurrent Close calls are idempotent, each | |
| 686 | + | // returns the same result, the carrier is closed exactly once, and Close | |
| 687 | + | // unblocks a writer blocked inside the carrier and joins both goroutines | |
| 688 | + | // within the adapter prompt-close contract. | |
| 689 | + | func TestPumpConcurrentClose(t *testing.T) { | |
| 690 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 691 | + | pump.Start(context.Background()) | |
| 692 | + | ||
| 693 | + | deliverClientFrame(t, carrier, openFor(1)) | |
| 694 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == 1 }) | |
| 695 | + | ||
| 696 | + | hold := make(chan struct{}) | |
| 697 | + | carrier.setHold(hold) | |
| 698 | + | require.NoError(t, pump.Send(Reset{Physical: 1})) | |
| 699 | + | require.Eventually(t, func() bool { return carrier.enteredSends() >= 1 }, 5*time.Second, time.Millisecond) | |
| 700 | + | ||
| 701 | + | const closers = 12 | |
| 702 | + | errs := make([]error, closers) | |
| 703 | + | var wg sync.WaitGroup | |
| 704 | + | for i := 0; i < closers; i++ { | |
| 705 | + | wg.Add(1) | |
| 706 | + | go func(index int) { | |
| 707 | + | defer wg.Done() | |
| 708 | + | errs[index] = pump.Close() | |
| 709 | + | }(i) | |
| 710 | + | } | |
| 711 | + | joined := make(chan struct{}) | |
| 712 | + | go func() { | |
| 713 | + | wg.Wait() | |
| 714 | + | close(joined) | |
| 715 | + | }() | |
| 716 | + | select { | |
| 717 | + | case <-joined: | |
| 718 | + | case <-time.After(5 * time.Second): | |
| 719 | + | t.Fatal("concurrent Close did not unblock the carrier and join both goroutines") | |
| 720 | + | } | |
| 721 | + | ||
| 722 | + | for i, err := range errs { | |
| 723 | + | require.NoError(t, err, "closer %d", i) | |
| 724 | + | } | |
| 725 | + | require.True(t, channelClosed(pump.Done())) | |
| 726 | + | require.NoError(t, pump.Err()) | |
| 727 | + | require.Equal(t, domain.RemoteFailureNone, pump.FailureKind()) | |
| 728 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 729 | + | require.True(t, channelClosed(pump.readerDone)) | |
| 730 | + | require.True(t, channelClosed(pump.writerDone)) | |
| 731 | + | require.ErrorIs(t, pump.Send(Data{Physical: 1, Data: []byte("x")}), ErrPhysicalClosed) | |
| 732 | + | } | |
| 733 | + | ||
| 734 | + | // TestPumpCloseBeforeStart proves Close never starts or joins goroutines that | |
| 735 | + | // were never launched, and that Start after Close is a no-op. | |
| 736 | + | func TestPumpCloseBeforeStart(t *testing.T) { | |
| 737 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 738 | + | require.NoError(t, pump.Close()) | |
| 739 | + | require.True(t, channelClosed(pump.Done())) | |
| 740 | + | require.NoError(t, pump.Err()) | |
| 741 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 742 | + | ||
| 743 | + | pump.Start(context.Background()) | |
| 744 | + | require.False(t, channelClosed(pump.readerDone)) | |
| 745 | + | require.False(t, channelClosed(pump.writerDone)) | |
| 746 | + | require.NoError(t, pump.Close()) | |
| 747 | + | require.Equal(t, 1, carrier.closeCalls()) | |
| 748 | + | } | |
| 749 | + | ||
| 750 | + | // TestPumpSendDirection proves Send refuses a message whose direction does not | |
| 751 | + | // match the pump's local side before it encodes or queues anything. | |
| 752 | + | func TestPumpSendDirection(t *testing.T) { | |
| 753 | + | t.Run("daemon side writes server messages", func(t *testing.T) { | |
| 754 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 755 | + | pump.Start(context.Background()) | |
| 756 | + | deliverClientFrame(t, carrier, openFor(1)) | |
| 757 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == 1 }) | |
| 758 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 759 | + | ||
| 760 | + | require.ErrorIs(t, pump.Send(Open{Ref: testRef(2)}), ErrWrongDirection) | |
| 761 | + | require.ErrorIs(t, pump.Send(nil), ErrInvalidMessage) | |
| 762 | + | require.NoError(t, pump.Send(Opened{Ref: testRef(1)})) | |
| 763 | + | message, err := DecodeServer(carrier.nextSent(t), testEnvelopeCeiling, testChunkCeiling) | |
| 764 | + | require.NoError(t, err) | |
| 765 | + | _, ok := message.(Opened) | |
| 766 | + | require.True(t, ok) | |
| 767 | + | carrier.requireNoSent(t, 20*time.Millisecond) | |
| 768 | + | }) | |
| 769 | + | ||
| 770 | + | t.Run("client side writes client messages", func(t *testing.T) { | |
| 771 | + | pump, carrier := newTestPump(t, DirectionServer) | |
| 772 | + | pump.Start(context.Background()) | |
| 773 | + | require.NoError(t, pump.Engine().Open(Open{Ref: testRef(1)})) | |
| 774 | + | ||
| 775 | + | require.ErrorIs(t, pump.Send(Opened{Ref: testRef(1)}), ErrWrongDirection) | |
| 776 | + | require.NoError(t, pump.Send(openFor(1))) | |
| 777 | + | message, err := DecodeClient(carrier.nextSent(t), testEnvelopeCeiling, testChunkCeiling) | |
| 778 | + | require.NoError(t, err) | |
| 779 | + | _, ok := message.(Open) | |
| 780 | + | require.True(t, ok) | |
| 781 | + | ||
| 782 | + | // The client side reads daemon frames: an inbound Opened confirms the | |
| 783 | + | // locally admitted stream through the reader. | |
| 784 | + | deliverServerFrame(t, carrier, Opened{Ref: testRef(1)}) | |
| 785 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { | |
| 786 | + | status, ok := e.Status(1) | |
| 787 | + | return ok && status.State == StreamOpen | |
| 788 | + | }) | |
| 789 | + | }) | |
| 790 | + | } | |
| 791 | + | ||
| 792 | + | // TestPumpFlushWaitsForInflightSend proves Flush does not return while a send | |
| 793 | + | // is in flight inside the carrier, and returns once the writer finished it. | |
| 794 | + | func TestPumpFlushWaitsForInflightSend(t *testing.T) { | |
| 795 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 796 | + | pump.Start(context.Background()) | |
| 797 | + | require.NoError(t, pump.Engine().Open(Open{Ref: testRef(1)})) | |
| 798 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 799 | + | ||
| 800 | + | hold := make(chan struct{}) | |
| 801 | + | carrier.setHold(hold) | |
| 802 | + | require.NoError(t, pump.Send(ServerMessage(Close{Physical: 1}))) | |
| 803 | + | require.Eventually(t, func() bool { return carrier.enteredSends() >= 1 }, 5*time.Second, time.Millisecond) | |
| 804 | + | ||
| 805 | + | flushed := make(chan error, 1) | |
| 806 | + | go func() { flushed <- pump.Flush(context.Background()) }() | |
| 807 | + | select { | |
| 808 | + | case <-flushed: | |
| 809 | + | t.Fatal("Flush returned while a send was still in flight") | |
| 810 | + | case <-time.After(50 * time.Millisecond): | |
| 811 | + | } | |
| 812 | + | ||
| 813 | + | carrier.release(hold) | |
| 814 | + | select { | |
| 815 | + | case err := <-flushed: | |
| 816 | + | require.NoError(t, err) | |
| 817 | + | case <-time.After(5 * time.Second): | |
| 818 | + | t.Fatal("Flush did not return after the write drained") | |
| 819 | + | } | |
| 820 | + | message, err := DecodeServer(carrier.nextSent(t), testEnvelopeCeiling, testChunkCeiling) | |
| 821 | + | require.NoError(t, err) | |
| 822 | + | closed, ok := message.(Close) | |
| 823 | + | require.True(t, ok) | |
| 824 | + | require.Equal(t, PhysicalStreamID(1), closed.Physical) | |
| 825 | + | } | |
| 826 | + | ||
| 827 | + | // TestPumpFlushPublishesFinalClose proves a flushes-before-close caller makes | |
| 828 | + | // the final Close observable on the wire: Send accepted means queued only, and | |
| 829 | + | // Flush is the explicit barrier that drains it. | |
| 830 | + | func TestPumpFlushPublishesFinalClose(t *testing.T) { | |
| 831 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 832 | + | pump.Start(context.Background()) | |
| 833 | + | require.NoError(t, pump.Engine().Open(Open{Ref: testRef(1)})) | |
| 834 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 835 | + | ||
| 836 | + | require.NoError(t, pump.Send(ServerMessage(Close{Physical: 1}))) | |
| 837 | + | require.NoError(t, pump.Flush(context.Background())) | |
| 838 | + | message, err := DecodeServer(carrier.nextSent(t), testEnvelopeCeiling, testChunkCeiling) | |
| 839 | + | require.NoError(t, err) | |
| 840 | + | closed, ok := message.(Close) | |
| 841 | + | require.True(t, ok) | |
| 842 | + | require.Equal(t, PhysicalStreamID(1), closed.Physical) | |
| 843 | + | ||
| 844 | + | // Flush of an already-drained pump returns immediately. | |
| 845 | + | require.NoError(t, pump.Flush(context.Background())) | |
| 846 | + | } | |
| 847 | + | ||
| 848 | + | // TestPumpFlushHonoursContext proves Flush returns the context error when the | |
| 849 | + | // deadline wins before the scheduler drains. | |
| 850 | + | func TestPumpFlushHonoursContext(t *testing.T) { | |
| 851 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 852 | + | require.NoError(t, pump.Engine().Open(Open{Ref: testRef(1)})) | |
| 853 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 854 | + | // No writer is started, so the queued frame can never drain. | |
| 855 | + | require.NoError(t, pump.Send(ServerMessage(Data{Physical: 1, Data: []byte("queued")}))) | |
| 856 | + | ||
| 857 | + | ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) | |
| 858 | + | defer cancel() | |
| 859 | + | require.ErrorIs(t, pump.Flush(ctx), context.DeadlineExceeded) | |
| 860 | + | ||
| 861 | + | // Starting the pump drains the frame and a fresh Flush succeeds. | |
| 862 | + | pump.Start(context.Background()) | |
| 863 | + | require.NoError(t, pump.Flush(context.Background())) | |
| 864 | + | message, err := DecodeServer(carrier.nextSent(t), testEnvelopeCeiling, testChunkCeiling) | |
| 865 | + | require.NoError(t, err) | |
| 866 | + | data, ok := message.(Data) | |
| 867 | + | require.True(t, ok) | |
| 868 | + | require.Equal(t, []byte("queued"), data.Data) | |
| 869 | + | } | |
| 870 | + | ||
| 871 | + | // TestPumpFlushCancellationMayDrop proves Close is not a flush: a frame queued | |
| 872 | + | // behind a blocked write is dropped when the context is cancelled rather than | |
| 873 | + | // drained, so a caller that must publish a final frame flushes first. | |
| 874 | + | func TestPumpFlushCancellationMayDrop(t *testing.T) { | |
| 875 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 876 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 877 | + | pump.Start(ctx) | |
| 878 | + | require.NoError(t, pump.Engine().Open(Open{Ref: testRef(1)})) | |
| 879 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 880 | + | ||
| 881 | + | hold := make(chan struct{}) | |
| 882 | + | carrier.setHold(hold) | |
| 883 | + | require.NoError(t, pump.Send(ServerMessage(Data{Physical: 1, Data: []byte("first")}))) | |
| 884 | + | require.Eventually(t, func() bool { return carrier.enteredSends() >= 1 }, 5*time.Second, time.Millisecond) | |
| 885 | + | // The final Close is queued behind the blocked write. | |
| 886 | + | require.NoError(t, pump.Send(ServerMessage(Close{Physical: 1}))) | |
| 887 | + | ||
| 888 | + | cancel() | |
| 889 | + | require.Eventually(t, func() bool { return channelClosed(pump.writerDone) }, 5*time.Second, time.Millisecond) | |
| 890 | + | carrier.release(hold) | |
| 891 | + | carrier.requireNoSent(t, 50*time.Millisecond) | |
| 892 | + | } | |
| 893 | + | ||
| 894 | + | // TestPumpOutboundOverflowSchedulesReset proves an outbound queue overflow | |
| 895 | + | // discards the stream's queued frames, resets its engine record to free the | |
| 896 | + | // live admission slot, and schedules exactly one Reset for the peer. | |
| 897 | + | func TestPumpOutboundOverflowSchedulesReset(t *testing.T) { | |
| 898 | + | pump, carrier := newTestPump(t, DirectionClient) | |
| 899 | + | require.NoError(t, pump.Engine().Open(Open{Ref: testRef(1)})) | |
| 900 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 901 | + | ||
| 902 | + | // Fill the per-stream frame bound with no writer draining it. | |
| 903 | + | for i := 0; i < MaxMuxStreamQueueChunks; i++ { | |
| 904 | + | require.NoError(t, pump.Send(ServerMessage(Data{Physical: 1, Data: []byte{byte(i)}}))) | |
| 905 | + | } | |
| 906 | + | require.Equal(t, MaxMuxStreamQueueChunks, pump.scheduler.QueuedFrames(1)) | |
| 907 | + | ||
| 908 | + | require.ErrorIs(t, pump.Send(ServerMessage(Data{Physical: 1, Data: []byte("overflow")})), ErrSchedulerFull) | |
| 909 | + | require.Equal(t, 0, pump.Engine().Live(), "the overflowed stream released its admission slot") | |
| 910 | + | status := mustStatus(t, pump.Engine(), 1) | |
| 911 | + | require.Equal(t, StreamTerminal, status.State) | |
| 912 | + | require.ErrorContains(t, status.Err, "scheduler queue overflow") | |
| 913 | + | require.Equal(t, 1, pump.scheduler.QueuedFrames(1), "exactly the one Reset remains queued") | |
| 914 | + | require.False(t, pump.Engine().Closed()) | |
| 915 | + | ||
| 916 | + | pump.Start(context.Background()) | |
| 917 | + | reset := decodeReset(t, carrier.nextSent(t), DirectionServer) | |
| 918 | + | require.Equal(t, PhysicalStreamID(1), reset.Physical) | |
| 919 | + | require.True(t, reset.HasError) | |
| 920 | + | require.Equal(t, domain.RemoteFailureTransport, reset.Error.FailureKind) | |
| 921 | + | ||
| 922 | + | // The record is sealed after its one reset: a later frame for the retired | |
| 923 | + | // stream is refused and never dequeued again. | |
| 924 | + | require.ErrorIs(t, pump.Send(ServerMessage(Data{Physical: 1, Data: []byte("late")})), ErrSchedulerSettled) | |
| 925 | + | carrier.requireNoSent(t, 50*time.Millisecond) | |
| 926 | + | } | |
| 927 | + | ||
| 928 | + | // TestPumpInboundTerminalRetiresQueuedOutbound proves an inbound Close or Reset | |
| 929 | + | // discards the stream's queued outbound frames and retires the scheduler | |
| 930 | + | // record, and that 3000 stream cycles stay bounded with no post-terminal | |
| 931 | + | // dequeue. | |
| 932 | + | func TestPumpInboundTerminalRetiresQueuedOutbound(t *testing.T) { | |
| 933 | + | pump, _ := newTestPump(t, DirectionClient) | |
| 934 | + | const churn = 3000 | |
| 935 | + | for i := 1; i <= churn; i++ { | |
| 936 | + | id := PhysicalStreamID(i) | |
| 937 | + | require.NoError(t, pump.Engine().Open(Open{Ref: testRef(id)})) | |
| 938 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(id)})) | |
| 939 | + | require.NoError(t, pump.Send(ServerMessage(Data{Physical: id, Data: []byte("out")}))) | |
| 940 | + | require.Equal(t, 1, pump.scheduler.QueuedFrames(id)) | |
| 941 | + | ||
| 942 | + | if i%2 == 0 { | |
| 943 | + | require.NoError(t, pump.applyClient(Close{Physical: id})) | |
| 944 | + | } else { | |
| 945 | + | require.NoError(t, pump.applyClient(Reset{Physical: id})) | |
| 946 | + | } | |
| 947 | + | require.Zero(t, pump.scheduler.QueuedFrames(id), "queued outbound discarded") | |
| 948 | + | require.Zero(t, pump.scheduler.AggregateBytes()) | |
| 949 | + | require.False(t, pump.Engine().Closed()) | |
| 950 | + | } | |
| 951 | + | require.Zero(t, pump.Engine().Live()) | |
| 952 | + | require.Zero(t, pump.Engine().AggregateBytes()) | |
| 953 | + | require.LessOrEqual(t, len(pump.scheduler.streams), MaxRetiredSchedulerRecords) | |
| 954 | + | _, ok := pump.scheduler.Dequeue() | |
| 955 | + | require.False(t, ok, "no post-terminal dequeue") | |
| 956 | + | } | |
| 957 | + | ||
| 958 | + | // watchEntries reports how many inbound watch entries the pump retains. | |
| 959 | + | func watchEntries(pump *Pump) int { | |
| 960 | + | pump.watchMu.Lock() | |
| 961 | + | defer pump.watchMu.Unlock() | |
| 962 | + | return len(pump.watch) | |
| 963 | + | } | |
| 964 | + | ||
| 965 | + | // TestPumpWatchUnknownAndTerminalClosed proves Watch hands back an | |
| 966 | + | // already-closed channel for zero, unknown, and terminal identities without | |
| 967 | + | // retaining a watch entry, and returns one stable open channel for a live | |
| 968 | + | // stream that rotates on every applied frame. | |
| 969 | + | func TestPumpWatchUnknownAndTerminalClosed(t *testing.T) { | |
| 970 | + | pump, _ := newTestPump(t, DirectionClient) | |
| 971 | + | ||
| 972 | + | require.True(t, channelClosed(pump.Watch(0)), "zero identity is closed") | |
| 973 | + | require.True(t, channelClosed(pump.Watch(9)), "unknown identity is closed") | |
| 974 | + | require.Zero(t, watchEntries(pump), "unknown identities retain no entry") | |
| 975 | + | ||
| 976 | + | require.NoError(t, pump.Engine().Open(openFor(1))) | |
| 977 | + | live := pump.Watch(1) | |
| 978 | + | require.False(t, channelClosed(live)) | |
| 979 | + | require.Equal(t, 1, watchEntries(pump)) | |
| 980 | + | require.Equal(t, live, pump.Watch(1), "a live watch channel is stable") | |
| 981 | + | ||
| 982 | + | // An applied frame rotates the watch channel: the old one closes and a | |
| 983 | + | // fresh open one is installed. | |
| 984 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(1)})) | |
| 985 | + | require.NoError(t, pump.applyClient(Data{Physical: 1, Data: []byte("x")})) | |
| 986 | + | rotated := pump.Watch(1) | |
| 987 | + | require.NotEqual(t, live, rotated, "an applied frame rotates the channel") | |
| 988 | + | require.True(t, channelClosed(live)) | |
| 989 | + | ||
| 990 | + | // A closing stream keeps its queue readable: the applied Close rotates the | |
| 991 | + | // watch channel, but the stream is not terminal until its preserved chunk is | |
| 992 | + | // taken. | |
| 993 | + | require.NoError(t, pump.applyClient(Close{Physical: 1})) | |
| 994 | + | require.True(t, channelClosed(rotated), "the close frame rotated the current channel") | |
| 995 | + | require.Equal(t, StreamClosing, mustStatus(t, pump.Engine(), 1).State) | |
| 996 | + | closingWatch := pump.Watch(1) | |
| 997 | + | require.False(t, channelClosed(closingWatch), "a closing stream is still readable") | |
| 998 | + | require.Equal(t, closingWatch, pump.Watch(1), "the closing watch channel is stable") | |
| 999 | + | ||
| 1000 | + | chunk, ok := pump.Take(1) | |
| 1001 | + | require.True(t, ok) | |
| 1002 | + | require.Equal(t, []byte("x"), chunk) | |
| 1003 | + | require.True(t, channelClosed(closingWatch), "draining the last chunk rotated the watch channel") | |
| 1004 | + | require.True(t, channelClosed(pump.Watch(1)), "terminal identity is closed") | |
| 1005 | + | require.Zero(t, watchEntries(pump), "terminal Watch retains no entry") | |
| 1006 | + | ||
| 1007 | + | // An evicted identity is closed and retains no entry either. | |
| 1008 | + | for i := 2; i <= MaxRetiredStreamRecords+2; i++ { | |
| 1009 | + | id := PhysicalStreamID(i) | |
| 1010 | + | require.NoError(t, pump.Engine().Open(openFor(id))) | |
| 1011 | + | _, _ = pump.Engine().Close(Close{Physical: id}) | |
| 1012 | + | } | |
| 1013 | + | _, tracked := pump.Engine().Status(1) | |
| 1014 | + | require.False(t, tracked, "stream 1 record was evicted") | |
| 1015 | + | require.True(t, channelClosed(pump.Watch(1)), "evicted identity is closed") | |
| 1016 | + | require.Zero(t, watchEntries(pump), "evicted Watch retains no entry") | |
| 1017 | + | } | |
A internal/adapters/daemonmux/quic_carriage_test.go +505 -0
| @@ -0,0 +1,505 @@ | |||
| 1 | + | // Integration coverage for the real private QUIC daemonmux carriage (P3.2e). | |
| 2 | + | // | |
| 3 | + | // These tests exercise the daemonmux multiplexer over an actual loopback QUIC | |
| 4 | + | // connection authenticated by quic.NewServer's ephemeral certificate plus | |
| 5 | + | // one-time token/nonce and dialed by quic.DialMuxContext, not over the | |
| 6 | + | // in-memory carrier used by the unit tests or the Unix socket of P3.2d. One | |
| 7 | + | // QUIC connection is one physical daemonmux connection: every logical stream | |
| 8 | + | // rides its single bidirectional stream, and no logical attachment is ever | |
| 9 | + | // mapped to a native QUIC stream. | |
| 10 | + | // | |
| 11 | + | // The full daemon path (EndpointConnector -> ServerSupervisor -> | |
| 12 | + | // AggregateListener -> typed Listener) is exercised for the two-then-hundred | |
| 13 | + | // admissions and physical-failure tests, and the raw pump path is exercised for | |
| 14 | + | // the blocked-consumer and reset-isolation checks so the stream accounting is | |
| 15 | + | // asserted directly. | |
| 16 | + | package daemonmux | |
| 17 | + | ||
| 18 | + | import ( | |
| 19 | + | "context" | |
| 20 | + | "net" | |
| 21 | + | "strconv" | |
| 22 | + | "sync" | |
| 23 | + | "testing" | |
| 24 | + | "time" | |
| 25 | + | ||
| 26 | + | "github.com/stretchr/testify/require" | |
| 27 | + | ||
| 28 | + | "github.com/bnema/vev/internal/adapters/quic" | |
| 29 | + | "github.com/bnema/vev/internal/domain" | |
| 30 | + | "github.com/bnema/vev/internal/ports" | |
| 31 | + | "github.com/bnema/vev/internal/protocol" | |
| 32 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 33 | + | ) | |
| 34 | + | ||
| 35 | + | // quicCarriageDeadline bounds every real loopback QUIC mux carriage wait. QUIC | |
| 36 | + | // pays a TLS handshake and a pinned dial per physical carriage, so the parity | |
| 37 | + | // waits stay deliberately generous. | |
| 38 | + | const quicCarriageDeadline = 10 * time.Second | |
| 39 | + | ||
| 40 | + | // quicCarriageEndpoint mints one ephemeral bootstrap endpoint and returns it | |
| 41 | + | // with its readiness record and the dial address composed from loopback plus | |
| 42 | + | // the readiness port. The one-time token and nonce stay in the readiness | |
| 43 | + | // record: daemonmux only ever sees the host-independent address. | |
| 44 | + | func quicCarriageEndpoint(t *testing.T) (*quic.Server, quic.Readiness, string) { | |
| 45 | + | t.Helper() | |
| 46 | + | server, readiness, err := quic.NewServer() | |
| 47 | + | require.NoError(t, err) | |
| 48 | + | t.Cleanup(func() { _ = server.Close() }) | |
| 49 | + | return server, readiness, net.JoinHostPort("127.0.0.1", strconv.Itoa(readiness.Port)) | |
| 50 | + | } | |
| 51 | + | ||
| 52 | + | // quicDialer returns a RawCarrierDialer that establishes the real QUIC mux | |
| 53 | + | // carriage through quic.DialMuxContext - exact ephemeral certificate pin plus | |
| 54 | + | // single-use token/nonce - and hands the raw bounded transport to the connector | |
| 55 | + | // unmodified. | |
| 56 | + | func quicDialer(readiness quic.Readiness) RawCarrierDialer { | |
| 57 | + | return func(ctx context.Context, address string) (RawFramedTransport, error) { | |
| 58 | + | raw, err := quic.DialMuxContext(ctx, address, readiness, quic.Config{}, quicCarriageDeadline) | |
| 59 | + | if err != nil { | |
| 60 | + | return nil, err | |
| 61 | + | } | |
| 62 | + | return raw, nil | |
| 63 | + | } | |
| 64 | + | } | |
| 65 | + | ||
| 66 | + | // quicDialCarriage establishes the broker end of one real QUIC mux carriage. | |
| 67 | + | func quicDialCarriage(t *testing.T, ctx context.Context, addr string, readiness quic.Readiness) RawFramedTransport { | |
| 68 | + | t.Helper() | |
| 69 | + | raw, err := quicDialer(readiness)(ctx, addr) | |
| 70 | + | require.NoError(t, err) | |
| 71 | + | t.Cleanup(func() { _ = raw.Close() }) | |
| 72 | + | return raw | |
| 73 | + | } | |
| 74 | + | ||
| 75 | + | // serveQUIC adopts every carriage the bootstrap endpoint admits into the test | |
| 76 | + | // supervisor until the endpoint closes. There is exactly one: the credential is | |
| 77 | + | // one-time. | |
| 78 | + | func (s *superviseServer) serveQUIC(t *testing.T, server *quic.Server) { | |
| 79 | + | t.Helper() | |
| 80 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 81 | + | t.Cleanup(cancel) | |
| 82 | + | go func() { | |
| 83 | + | for { | |
| 84 | + | raw, err := server.AcceptMux(ctx) | |
| 85 | + | if err != nil { | |
| 86 | + | return | |
| 87 | + | } | |
| 88 | + | s.adopt(ctx, raw) | |
| 89 | + | } | |
| 90 | + | }() | |
| 91 | + | } | |
| 92 | + | ||
| 93 | + | // serveQUICCapturing adopts the bootstrap endpoint's one authenticated carriage | |
| 94 | + | // and publishes the accepted daemon carriage, so a test can fail that physical | |
| 95 | + | // connection deterministically. | |
| 96 | + | func (s *superviseServer) serveQUICCapturing(t *testing.T, server *quic.Server) <-chan RawFramedTransport { | |
| 97 | + | t.Helper() | |
| 98 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 99 | + | t.Cleanup(cancel) | |
| 100 | + | captured := make(chan RawFramedTransport, 1) | |
| 101 | + | go func() { | |
| 102 | + | raw, err := server.AcceptMux(ctx) | |
| 103 | + | if err != nil { | |
| 104 | + | return | |
| 105 | + | } | |
| 106 | + | captured <- raw | |
| 107 | + | s.adopt(ctx, raw) | |
| 108 | + | }() | |
| 109 | + | return captured | |
| 110 | + | } | |
| 111 | + | ||
| 112 | + | // awaitQUICAccepted waits for the aggregate to deliver count typed admissions. | |
| 113 | + | func (s *superviseServer) awaitQUICAccepted(t *testing.T, count int) { | |
| 114 | + | t.Helper() | |
| 115 | + | require.Eventually(t, func() bool { return s.acceptedCount() >= count }, quicCarriageDeadline, time.Millisecond, | |
| 116 | + | "server accepted %d of %d streams", s.acceptedCount(), count) | |
| 117 | + | } | |
| 118 | + | ||
| 119 | + | // quicDaemonPump builds a daemon-side pump over the daemon end of a real QUIC | |
| 120 | + | // mux carriage and returns the raw broker end for direct frame injection, so a | |
| 121 | + | // test can drive deterministic inbound traffic over the actual QUIC stream. | |
| 122 | + | func quicDaemonPump(t *testing.T) (*Pump, RawFramedTransport) { | |
| 123 | + | t.Helper() | |
| 124 | + | server, readiness, addr := quicCarriageEndpoint(t) | |
| 125 | + | ctx, cancel := context.WithTimeout(context.Background(), quicCarriageDeadline) | |
| 126 | + | t.Cleanup(cancel) | |
| 127 | + | ||
| 128 | + | type acceptOutcome struct { | |
| 129 | + | raw RawFramedTransport | |
| 130 | + | err error | |
| 131 | + | } | |
| 132 | + | accepted := make(chan acceptOutcome, 1) | |
| 133 | + | go func() { | |
| 134 | + | raw, err := server.AcceptMux(ctx) | |
| 135 | + | accepted <- acceptOutcome{raw: raw, err: err} | |
| 136 | + | }() | |
| 137 | + | ||
| 138 | + | client := quicDialCarriage(t, ctx, addr, readiness) | |
| 139 | + | ||
| 140 | + | var daemon RawFramedTransport | |
| 141 | + | select { | |
| 142 | + | case outcome := <-accepted: | |
| 143 | + | require.NoError(t, outcome.err) | |
| 144 | + | require.NotNil(t, outcome.raw) | |
| 145 | + | daemon = outcome.raw | |
| 146 | + | t.Cleanup(func() { _ = daemon.Close() }) | |
| 147 | + | case <-time.After(quicCarriageDeadline): | |
| 148 | + | t.Fatal("AcceptMux did not return") | |
| 149 | + | } | |
| 150 | + | ||
| 151 | + | carrier, err := NewPreambleCarrier(daemon) | |
| 152 | + | require.NoError(t, err) | |
| 153 | + | require.NoError(t, carrier.Negotiate(DefaultMuxCeilings())) | |
| 154 | + | pump, err := NewPump(carrier, DirectionClient, DefaultMuxCeilings()) | |
| 155 | + | require.NoError(t, err) | |
| 156 | + | t.Cleanup(func() { _ = pump.Close() }) | |
| 157 | + | pump.Start(context.Background()) | |
| 158 | + | return pump, client | |
| 159 | + | } | |
| 160 | + | ||
| 161 | + | // injectQUICClientFrame writes one client frame over the real QUIC carriage. | |
| 162 | + | func injectQUICClientFrame(t *testing.T, client RawFramedTransport, message ClientMessage) { | |
| 163 | + | t.Helper() | |
| 164 | + | require.NoError(t, client.Send(wire.Envelope{Payload: mustEncodeClient(t, message)})) | |
| 165 | + | } | |
| 166 | + | ||
| 167 | + | // nextQUICServerFrame reads one daemon frame from the real QUIC carriage. | |
| 168 | + | func nextQUICServerFrame(t *testing.T, client RawFramedTransport) []byte { | |
| 169 | + | t.Helper() | |
| 170 | + | envelope, err := client.RecvBounded(testEnvelopeCeiling) | |
| 171 | + | require.NoError(t, err) | |
| 172 | + | return envelope.Payload | |
| 173 | + | } | |
| 174 | + | ||
| 175 | + | // TestQUICCarriageSupervisorTwoThenHundredTypedAdmissions drives the full daemon | |
| 176 | + | // path over one real QUIC physical connection: EndpointConnector dials the | |
| 177 | + | // authenticated carriage, ServerSupervisor adopts it, the AggregateListener | |
| 178 | + | // delivers independent typed daemon admissions, and every stream's traffic is | |
| 179 | + | // isolated. Only one physical connection, one physical child, and one QUIC | |
| 180 | + | // stream exist for all 102 streams. | |
| 181 | + | func TestQUICCarriageSupervisorTwoThenHundredTypedAdmissions(t *testing.T) { | |
| 182 | + | binding := mustServerBinding(t) | |
| 183 | + | policy := binding.Policy() | |
| 184 | + | server, readiness, addr := quicCarriageEndpoint(t) | |
| 185 | + | ||
| 186 | + | daemon := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 187 | + | daemon.serve() | |
| 188 | + | daemon.serveQUIC(t, server) | |
| 189 | + | ||
| 190 | + | connector, err := NewEndpointConnector(quicDialer(readiness), DefaultMuxCeilings()) | |
| 191 | + | require.NoError(t, err) | |
| 192 | + | physical, err := connector.Connect(context.Background(), muxEndpoint(binding.Identity(), policy, addr)) | |
| 193 | + | require.NoError(t, err) | |
| 194 | + | t.Cleanup(func() { _ = physical.Close() }) | |
| 195 | + | require.NoError(t, daemon.awaitAdoption(t)) | |
| 196 | + | ||
| 197 | + | // Two independent typed admissions over the one physical carriage. | |
| 198 | + | first, err := physical.OpenStream(context.Background(), muxOpenRequest(1, policy)) | |
| 199 | + | require.NoError(t, err) | |
| 200 | + | t.Cleanup(func() { _ = first.Close() }) | |
| 201 | + | second, err := physical.OpenStream(context.Background(), muxOpenRequest(2, policy)) | |
| 202 | + | require.NoError(t, err) | |
| 203 | + | t.Cleanup(func() { _ = second.Close() }) | |
| 204 | + | daemon.awaitQUICAccepted(t, 2) | |
| 205 | + | for _, stream := range []ports.BrokerLogicalConnection{first, second} { | |
| 206 | + | require.NoError(t, stream.SendClient(protocol.Ping{})) | |
| 207 | + | message, err := stream.ReceiveServer() | |
| 208 | + | require.NoError(t, err) | |
| 209 | + | require.Equal(t, protocol.Pong{}, message) | |
| 210 | + | } | |
| 211 | + | ||
| 212 | + | // One hundred more independently admitted typed streams, opened | |
| 213 | + | // concurrently over the same physical carriage. | |
| 214 | + | const extra = 100 | |
| 215 | + | connections := make([]ports.BrokerLogicalConnection, extra) | |
| 216 | + | openErrs := make([]error, extra) | |
| 217 | + | var wg sync.WaitGroup | |
| 218 | + | for i := 0; i < extra; i++ { | |
| 219 | + | wg.Add(1) | |
| 220 | + | go func(i int) { | |
| 221 | + | defer wg.Done() | |
| 222 | + | connections[i], openErrs[i] = physical.OpenStream(context.Background(), muxOpenRequest(uint64(3+i), policy)) | |
| 223 | + | }(i) | |
| 224 | + | } | |
| 225 | + | wg.Wait() | |
| 226 | + | for i, openErr := range openErrs { | |
| 227 | + | require.NoError(t, openErr, "stream %d", i+3) | |
| 228 | + | } | |
| 229 | + | for i, connection := range connections { | |
| 230 | + | require.NotNil(t, connection, "stream %d", i+3) | |
| 231 | + | t.Cleanup(func() { _ = connection.Close() }) | |
| 232 | + | } | |
| 233 | + | ||
| 234 | + | daemon.awaitQUICAccepted(t, 2+extra) | |
| 235 | + | for i, connection := range connections { | |
| 236 | + | require.NoError(t, connection.SendClient(protocol.Ping{}), "stream %d", i+3) | |
| 237 | + | message, err := connection.ReceiveServer() | |
| 238 | + | require.NoError(t, err, "stream %d", i+3) | |
| 239 | + | require.Equal(t, protocol.Pong{}, message) | |
| 240 | + | } | |
| 241 | + | ||
| 242 | + | // All 102 streams share exactly one physical child and one physical QUIC | |
| 243 | + | // connection, and the daemon engine accounts for every admitted stream. | |
| 244 | + | require.Equal(t, 1, daemon.supervisor.Children(), "all streams must share one physical QUIC connection") | |
| 245 | + | pump := daemon.soleChildPump(t) | |
| 246 | + | require.Eventually(t, func() bool { return pump.Engine().Live() == 2+extra }, quicCarriageDeadline, time.Millisecond, | |
| 247 | + | "daemon engine admitted %d of %d streams", pump.Engine().Live(), 2+extra) | |
| 248 | + | ||
| 249 | + | all := make([]ports.BrokerLogicalConnection, 0, 2+extra) | |
| 250 | + | all = append(all, first, second) | |
| 251 | + | all = append(all, connections...) | |
| 252 | + | for _, connection := range all { | |
| 253 | + | require.NoError(t, connection.Close()) | |
| 254 | + | } | |
| 255 | + | require.Eventually(t, func() bool { return pump.Engine().Live() == 0 }, quicCarriageDeadline, time.Millisecond, | |
| 256 | + | "daemon engine still holds %d live streams after close", pump.Engine().Live()) | |
| 257 | + | require.False(t, channelClosed(physical.Done()), "closing every stream must leave the physical carriage open") | |
| 258 | + | } | |
| 259 | + | ||
| 260 | + | // TestQUICCarriageBlockedConsumerSiblingProgress proves, over a real QUIC | |
| 261 | + | // carriage, that a logical consumer which never drains one stream's inbound | |
| 262 | + | // queue neither blocks the reader nor a sibling: the stalled stream is reset by | |
| 263 | + | // its own bound while the sibling keeps receiving. | |
| 264 | + | func TestQUICCarriageBlockedConsumerSiblingProgress(t *testing.T) { | |
| 265 | + | pump, client := quicDaemonPump(t) | |
| 266 | + | ||
| 267 | + | const siblings = 2 | |
| 268 | + | for i := 1; i <= siblings; i++ { | |
| 269 | + | injectQUICClientFrame(t, client, openFor(PhysicalStreamID(i))) | |
| 270 | + | } | |
| 271 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == siblings }) | |
| 272 | + | for i := 1; i <= siblings; i++ { | |
| 273 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(PhysicalStreamID(i))})) | |
| 274 | + | } | |
| 275 | + | ||
| 276 | + | // The consumer never Takes stream 1: its bounded queue fills. | |
| 277 | + | for i := 0; i < MaxMuxStreamQueueChunks; i++ { | |
| 278 | + | injectQUICClientFrame(t, client, Data{Physical: 1, Data: []byte{byte(i)}}) | |
| 279 | + | } | |
| 280 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { | |
| 281 | + | status, ok := e.Status(1) | |
| 282 | + | return ok && status.QueuedChunks == MaxMuxStreamQueueChunks | |
| 283 | + | }) | |
| 284 | + | ||
| 285 | + | // One more chunk overflows stream 1's own bound; a sibling chunk still | |
| 286 | + | // lands, so neither the reader nor the sibling was blocked. | |
| 287 | + | injectQUICClientFrame(t, client, Data{Physical: 1, Data: []byte{'x'}}) | |
| 288 | + | injectQUICClientFrame(t, client, Data{Physical: 2, Data: []byte("sibling")}) | |
| 289 | + | ||
| 290 | + | taken, ok := takeEventually(t, pump, 2) | |
| 291 | + | require.True(t, ok) | |
| 292 | + | require.Equal(t, []byte("sibling"), taken) | |
| 293 | + | ||
| 294 | + | stalled := mustStatus(t, pump.Engine(), 1) | |
| 295 | + | require.Equal(t, StreamTerminal, stalled.State) | |
| 296 | + | require.ErrorIs(t, stalled.Err, ErrStreamQueueFull) | |
| 297 | + | require.Equal(t, domain.RemoteFailureTransport, stalled.FailureKind) | |
| 298 | + | require.Equal(t, StreamOpen, mustStatus(t, pump.Engine(), 2).State) | |
| 299 | + | require.False(t, pump.Engine().Closed()) | |
| 300 | + | require.False(t, channelClosed(pump.Done())) | |
| 301 | + | ||
| 302 | + | reset := decodeReset(t, nextQUICServerFrame(t, client), DirectionServer) | |
| 303 | + | require.Equal(t, PhysicalStreamID(1), reset.Physical) | |
| 304 | + | require.Equal(t, domain.RemoteFailureTransport, reset.Error.FailureKind) | |
| 305 | + | } | |
| 306 | + | ||
| 307 | + | // TestQUICCarriageResetIsolation proves, over a real QUIC carriage, that one | |
| 308 | + | // stream-local refusal schedules exactly one Reset for its own stream and | |
| 309 | + | // leaves the physical connection and every sibling healthy. | |
| 310 | + | func TestQUICCarriageResetIsolation(t *testing.T) { | |
| 311 | + | pump, client := quicDaemonPump(t) | |
| 312 | + | ||
| 313 | + | for _, id := range []PhysicalStreamID{1, 2, 3} { | |
| 314 | + | injectQUICClientFrame(t, client, openFor(id)) | |
| 315 | + | } | |
| 316 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == 3 }) | |
| 317 | + | for _, id := range []PhysicalStreamID{2, 3} { | |
| 318 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(id)})) | |
| 319 | + | } | |
| 320 | + | ||
| 321 | + | // Stream 1 is still opening: inbound data is a stream-local state refusal. | |
| 322 | + | injectQUICClientFrame(t, client, Data{Physical: 1, Data: []byte("early")}) | |
| 323 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { | |
| 324 | + | status, ok := e.Status(1) | |
| 325 | + | return ok && status.State == StreamTerminal | |
| 326 | + | }) | |
| 327 | + | settled := mustStatus(t, pump.Engine(), 1) | |
| 328 | + | require.ErrorContains(t, settled.Err, "invalid stream state") | |
| 329 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, settled.FailureKind) | |
| 330 | + | ||
| 331 | + | injectQUICClientFrame(t, client, Data{Physical: 2, Data: []byte("sibling")}) | |
| 332 | + | taken, ok := takeEventually(t, pump, 2) | |
| 333 | + | require.True(t, ok) | |
| 334 | + | require.Equal(t, []byte("sibling"), taken) | |
| 335 | + | ||
| 336 | + | reset := decodeReset(t, nextQUICServerFrame(t, client), DirectionServer) | |
| 337 | + | require.Equal(t, PhysicalStreamID(1), reset.Physical) | |
| 338 | + | require.True(t, reset.HasError) | |
| 339 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, reset.Error.FailureKind) | |
| 340 | + | require.False(t, pump.Engine().Closed()) | |
| 341 | + | require.False(t, channelClosed(pump.Done())) | |
| 342 | + | } | |
| 343 | + | ||
| 344 | + | // TestQUICCarriagePhysicalFailureFansOutAndIsIsolated proves a lost QUIC | |
| 345 | + | // physical carriage publishes its terminal outcome before fanning out to every | |
| 346 | + | // logical stream it carried, while a sibling physical QUIC connection on the | |
| 347 | + | // same supervisor is untouched and a fresh stream on the lost physical is | |
| 348 | + | // refused. | |
| 349 | + | func TestQUICCarriagePhysicalFailureFansOutAndIsIsolated(t *testing.T) { | |
| 350 | + | binding := mustServerBinding(t) | |
| 351 | + | policy := binding.Policy() | |
| 352 | + | serverA, readinessA, addrA := quicCarriageEndpoint(t) | |
| 353 | + | serverB, readinessB, addrB := quicCarriageEndpoint(t) | |
| 354 | + | ||
| 355 | + | daemon := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 356 | + | daemon.serve() | |
| 357 | + | carriageA := daemon.serveQUICCapturing(t, serverA) | |
| 358 | + | daemon.serveQUIC(t, serverB) | |
| 359 | + | ||
| 360 | + | connectorA, err := NewEndpointConnector(quicDialer(readinessA), DefaultMuxCeilings()) | |
| 361 | + | require.NoError(t, err) | |
| 362 | + | physicalA, err := connectorA.Connect(context.Background(), muxEndpoint(binding.Identity(), policy, addrA)) | |
| 363 | + | require.NoError(t, err) | |
| 364 | + | t.Cleanup(func() { _ = physicalA.Close() }) | |
| 365 | + | ||
| 366 | + | connectorB, err := NewEndpointConnector(quicDialer(readinessB), DefaultMuxCeilings()) | |
| 367 | + | require.NoError(t, err) | |
| 368 | + | physicalB, err := connectorB.Connect(context.Background(), muxEndpoint(binding.Identity(), policy, addrB)) | |
| 369 | + | require.NoError(t, err) | |
| 370 | + | t.Cleanup(func() { _ = physicalB.Close() }) | |
| 371 | + | ||
| 372 | + | require.NoError(t, daemon.awaitAdoption(t)) | |
| 373 | + | require.NoError(t, daemon.awaitAdoption(t)) | |
| 374 | + | require.Equal(t, 2, daemon.supervisor.Children()) | |
| 375 | + | ||
| 376 | + | lost := make([]ports.BrokerLogicalConnection, 3) | |
| 377 | + | for i := range lost { | |
| 378 | + | lost[i], err = physicalA.OpenStream(context.Background(), muxOpenRequest(uint64(i+1), policy)) | |
| 379 | + | require.NoError(t, err, "stream %d", i+1) | |
| 380 | + | t.Cleanup(func() { _ = lost[i].Close() }) | |
| 381 | + | } | |
| 382 | + | survivor, err := physicalB.OpenStream(context.Background(), muxOpenRequest(1, policy)) | |
| 383 | + | require.NoError(t, err) | |
| 384 | + | t.Cleanup(func() { _ = survivor.Close() }) | |
| 385 | + | daemon.awaitQUICAccepted(t, 4) | |
| 386 | + | ||
| 387 | + | // The physical terminal outcome is ordered before its logical fan-out. | |
| 388 | + | ordered := make(chan bool, 1) | |
| 389 | + | go func() { | |
| 390 | + | select { | |
| 391 | + | case <-lost[0].Done(): | |
| 392 | + | case <-time.After(quicCarriageDeadline): | |
| 393 | + | ordered <- false | |
| 394 | + | return | |
| 395 | + | } | |
| 396 | + | select { | |
| 397 | + | case <-physicalA.Done(): | |
| 398 | + | ordered <- true | |
| 399 | + | case <-time.After(quicCarriageDeadline): | |
| 400 | + | ordered <- false | |
| 401 | + | } | |
| 402 | + | }() | |
| 403 | + | ||
| 404 | + | // Lose the daemon end of physical A's QUIC carriage entirely. | |
| 405 | + | rawA := <-carriageA | |
| 406 | + | require.NoError(t, rawA.Close()) | |
| 407 | + | ||
| 408 | + | require.True(t, <-ordered, "the logical Done fired before the physical Done") | |
| 409 | + | require.True(t, channelClosed(physicalA.Done())) | |
| 410 | + | require.Error(t, physicalA.Err()) | |
| 411 | + | require.NotEqual(t, domain.RemoteFailureNone, physicalA.FailureKind()) | |
| 412 | + | // The physical outcome fans out to every logical stream it carried. Each | |
| 413 | + | // stream publishes on its own watcher goroutine, so the fan-out is | |
| 414 | + | // asynchronous even though it can never precede the physical Done. | |
| 415 | + | for i, stream := range lost { | |
| 416 | + | require.Eventually(t, func() bool { | |
| 417 | + | return channelClosed(stream.Done()) && stream.Err() != nil | |
| 418 | + | }, quicCarriageDeadline, time.Millisecond, "lost stream %d never received the physical failure", i+1) | |
| 419 | + | } | |
| 420 | + | ||
| 421 | + | // The sibling physical QUIC connection is unaffected. | |
| 422 | + | require.False(t, channelClosed(physicalB.Done())) | |
| 423 | + | require.False(t, channelClosed(survivor.Done())) | |
| 424 | + | require.NoError(t, survivor.SendClient(protocol.Ping{})) | |
| 425 | + | message, err := survivor.ReceiveServer() | |
| 426 | + | require.NoError(t, err) | |
| 427 | + | require.Equal(t, protocol.Pong{}, message) | |
| 428 | + | require.Eventually(t, func() bool { return daemon.supervisor.Children() == 1 }, quicCarriageDeadline, time.Millisecond, | |
| 429 | + | "the lost QUIC physical child was never reaped") | |
| 430 | + | ||
| 431 | + | // The lost physical refuses a fresh stream. | |
| 432 | + | _, err = physicalA.OpenStream(context.Background(), muxOpenRequest(9, policy)) | |
| 433 | + | require.Error(t, err) | |
| 434 | + | } | |
| 435 | + | ||
| 436 | + | // TestQUICCarriageSetupContextDetachedFromPhysicalLifetime proves the setup | |
| 437 | + | // context covers only setup: once EndpointConnector.Connect returned over a real | |
| 438 | + | // QUIC carriage, cancelling the setup context never stops the pooled physical | |
| 439 | + | // connection, which Close alone owns. | |
| 440 | + | func TestQUICCarriageSetupContextDetachedFromPhysicalLifetime(t *testing.T) { | |
| 441 | + | binding := mustServerBinding(t) | |
| 442 | + | server, readiness, addr := quicCarriageEndpoint(t) | |
| 443 | + | ||
| 444 | + | daemon := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 445 | + | daemon.serve() | |
| 446 | + | daemon.serveQUIC(t, server) | |
| 447 | + | ||
| 448 | + | connector, err := NewEndpointConnector(quicDialer(readiness), DefaultMuxCeilings()) | |
| 449 | + | require.NoError(t, err) | |
| 450 | + | setupCtx, cancel := context.WithCancel(context.Background()) | |
| 451 | + | physical, err := connector.Connect(setupCtx, muxEndpoint(binding.Identity(), binding.Policy(), addr)) | |
| 452 | + | require.NoError(t, err) | |
| 453 | + | require.NoError(t, daemon.awaitAdoption(t)) | |
| 454 | + | t.Cleanup(func() { _ = physical.Close() }) | |
| 455 | + | ||
| 456 | + | cancel() | |
| 457 | + | require.False(t, channelClosed(physical.Done())) | |
| 458 | + | logical, err := physical.OpenStream(context.Background(), muxOpenRequest(1, binding.Policy())) | |
| 459 | + | require.NoError(t, err) | |
| 460 | + | t.Cleanup(func() { _ = logical.Close() }) | |
| 461 | + | daemon.awaitQUICAccepted(t, 1) | |
| 462 | + | require.NoError(t, logical.SendClient(protocol.Ping{})) | |
| 463 | + | message, err := logical.ReceiveServer() | |
| 464 | + | require.NoError(t, err) | |
| 465 | + | require.Equal(t, protocol.Pong{}, message) | |
| 466 | + | require.False(t, channelClosed(physical.Done())) | |
| 467 | + | } | |
| 468 | + | ||
| 469 | + | // TestQUICCarriageCanceledSetupAdmitsNothing proves one setup context covers | |
| 470 | + | // bootstrap, dial, auth, and the mux handshake: a canceled setup admits no | |
| 471 | + | // physical connection, no child, and no stream. | |
| 472 | + | func TestQUICCarriageCanceledSetupAdmitsNothing(t *testing.T) { | |
| 473 | + | binding := mustServerBinding(t) | |
| 474 | + | server, readiness, addr := quicCarriageEndpoint(t) | |
| 475 | + | ||
| 476 | + | daemon := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 477 | + | daemon.serveQUIC(t, server) | |
| 478 | + | ||
| 479 | + | connector, err := NewEndpointConnector(quicDialer(readiness), DefaultMuxCeilings()) | |
| 480 | + | require.NoError(t, err) | |
| 481 | + | setupCtx, cancel := context.WithCancel(context.Background()) | |
| 482 | + | cancel() | |
| 483 | + | physical, err := connector.Connect(setupCtx, muxEndpoint(binding.Identity(), binding.Policy(), addr)) | |
| 484 | + | require.ErrorIs(t, err, context.Canceled) | |
| 485 | + | require.Nil(t, physical) | |
| 486 | + | require.Zero(t, daemon.supervisor.Children()) | |
| 487 | + | require.Zero(t, daemon.acceptedCount()) | |
| 488 | + | } | |
| 489 | + | ||
| 490 | + | // TestQUICCarriageSetupDeadlineBoundsTheMuxHandshake proves the setup deadline | |
| 491 | + | // covers the daemonmux physical handshake over a real QUIC carriage: a daemon | |
| 492 | + | // that never answers the preamble fails the whole Connect within the setup | |
| 493 | + | // deadline and publishes no physical connection. | |
| 494 | + | func TestQUICCarriageSetupDeadlineBoundsTheMuxHandshake(t *testing.T) { | |
| 495 | + | binding := mustServerBinding(t) | |
| 496 | + | _, readiness, addr := quicCarriageEndpoint(t) | |
| 497 | + | ||
| 498 | + | connector, err := NewEndpointConnector(quicDialer(readiness), DefaultMuxCeilings()) | |
| 499 | + | require.NoError(t, err) | |
| 500 | + | setupCtx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond) | |
| 501 | + | defer cancel() | |
| 502 | + | physical, err := connector.Connect(setupCtx, muxEndpoint(binding.Identity(), binding.Policy(), addr)) | |
| 503 | + | require.ErrorIs(t, err, context.DeadlineExceeded) | |
| 504 | + | require.Nil(t, physical) | |
| 505 | + | } | |
A internal/adapters/daemonmux/scheduler.go +951 -0
| @@ -0,0 +1,951 @@ | |||
| 1 | + | // Outbound fair scheduling (P3.2). | |
| 2 | + | // | |
| 3 | + | // Scheduler is the outbound companion of the StreamEngine on one physical | |
| 4 | + | // daemonmux connection. It owns one bounded, independently cancellable FIFO | |
| 5 | + | // queue per admitted stream and orders the already-encoded application | |
| 6 | + | // envelopes a future writer drains. It performs no I/O, owns no goroutine, and | |
| 7 | + | // carries no framing, pump, socket, or listener: it only decides which stream's | |
| 8 | + | // next immutable envelope is written next. | |
| 9 | + | // | |
| 10 | + | // A frame is enqueued only for a stream the engine has admitted and not yet | |
| 11 | + | // terminalized (an opening or open stream), or, when no engine gate is | |
| 12 | + | // installed, for any nonzero physical identity the caller already admitted. A | |
| 13 | + | // closing stream (an orderly inbound Close with unread data) admits no new | |
| 14 | + | // frame even though it is not yet terminal. Two exceptions are deliberate. | |
| 15 | + | // Exactly one terminal Close or Reset is accepted for a known identity the | |
| 16 | + | // engine has already settled, bypassing normal admission, so an overflow or | |
| 17 | + | // peer-reset path can still signal the peer. And Refuse schedules exactly one | |
| 18 | + | // Mux Refused for a fresh identity the engine rejected before recording it, so | |
| 19 | + | // a refused Open is still answered; that refusal seals its own record. A | |
| 20 | + | // duplicate terminal, a duplicate refusal, and data for a settled, unadmitted, | |
| 21 | + | // or closing identity stay refused while the settling record is retained (see | |
| 22 | + | // below). Each stream queues at most MaxMuxStreamQueueChunks (8) envelopes and | |
| 23 | + | // MaxMuxStreamQueueBytes (32 MiB), and all streams together queue at most | |
| 24 | + | // MaxMuxAggregateBytes (64 MiB) of application data. A frame is charged | |
| 25 | + | // against both byte budgets before it is inserted, and a frame that would | |
| 26 | + | // exceed the per-stream frame bound, the per-stream byte bound, or the | |
| 27 | + | // aggregate byte bound discards exactly its own stream queue, frees that | |
| 28 | + | // budget, and leaves every sibling untouched. | |
| 29 | + | // | |
| 30 | + | // An overflowed stream is not sealed outright: the discard settles its data | |
| 31 | + | // but leaves its record open for exactly one terminal Close or Reset, so the | |
| 32 | + | // caller can still signal the peer that the stream was aborted. That terminal | |
| 33 | + | // is always retained: it is inserted without re-running the overflow check, so | |
| 34 | + | // it may exceed the aggregate data budget by at most one bounded terminal | |
| 35 | + | // envelope per live or retained overflowed stream. That bounded excess is | |
| 36 | + | // peer notification, never application data: the data bounds above still hold | |
| 37 | + | // for every non-terminal frame. Until that one terminal is enqueued the record | |
| 38 | + | // refuses every non-terminal frame with ErrSchedulerFull; the terminal seals | |
| 39 | + | // it, after which every later frame is refused (a duplicate terminal with | |
| 40 | + | // ErrSchedulerSettled). | |
| 41 | + | // | |
| 42 | + | // Across active streams the scheduler runs byte-deficit round robin with a | |
| 43 | + | // 64 KiB quantum (SchedulerQuantum): active streams are visited in turn, each | |
| 44 | + | // visit tops up the stream's deficit by one quantum unconditionally and caps | |
| 45 | + | // it at one quantum plus the largest envelope the codec admits, and its oldest | |
| 46 | + | // frame is written once the head fits the accumulated deficit. A head frame | |
| 47 | + | // larger than the quantum therefore accumulates deficit across visits instead | |
| 48 | + | // of stalling the ring, so every admitted frame up to the 1 MiB application | |
| 49 | + | // ceiling makes bounded progress (at most one quantum of accumulation per | |
| 50 | + | // visit) and can never spin the scheduler. Per-stream order is always | |
| 51 | + | // preserved: the scheduler only ever writes a stream's oldest queued frame, so | |
| 52 | + | // a Close never overtakes earlier accepted data of its own stream. Control | |
| 53 | + | // frames carry a small bounded priority (MaxMuxSchedulerControlBurst): they | |
| 54 | + | // may be written ahead of data frames of other streams, but the burst is | |
| 55 | + | // capped so control metadata can never starve data. A single hot stream | |
| 56 | + | // therefore never starves the others, and the schedule is deterministic: the | |
| 57 | + | // same enqueue sequence always yields the same dequeue order. | |
| 58 | + | // | |
| 59 | + | // Settled stream records are bounded: the scheduler retains at most | |
| 60 | + | // MaxRetiredSchedulerRecords sealed-or-overflowed, empty records and evicts | |
| 61 | + | // the oldest first, so a long-lived connection cannot accumulate an unbounded | |
| 62 | + | // record table. The ErrSchedulerSettled refusals above are guaranteed within | |
| 63 | + | // exactly that retention window. Once a record is evicted its identity falls | |
| 64 | + | // back to the engine gate: an identity the engine gate no longer tracks fails | |
| 65 | + | // that check and is reported unadmitted (ErrSchedulerUnadmitted), while a late | |
| 66 | + | // terminal that is still accepted is harmlessly discarded by the peer's | |
| 67 | + | // stream fencing, which ignores a Close or Reset for an already terminal | |
| 68 | + | // stream. An overflowed record that accepts its one terminal after being | |
| 69 | + | // listed keeps that queued frame past eviction, but draining the frame | |
| 70 | + | // re-lists the record so a later eviction reclaims it and retention stays | |
| 71 | + | // bounded. | |
| 72 | + | // | |
| 73 | + | // Reset settles one stream locally: it discards that stream's queued frames, | |
| 74 | + | // frees their bytes immediately, and refuses every later frame for that | |
| 75 | + | // identity. It never touches the physical connection or a sibling stream, so | |
| 76 | + | // cancelling one stream is isolated. Every dequeued OutboundEnvelope carries | |
| 77 | + | // the complete serialized directional envelope in Bytes, owned by the caller | |
| 78 | + | // and never reused or mutated by the scheduler, together with the committed | |
| 79 | + | // client or server message. | |
| 80 | + | package daemonmux | |
| 81 | + | ||
| 82 | + | import ( | |
| 83 | + | "errors" | |
| 84 | + | "sync" | |
| 85 | + | ) | |
| 86 | + | ||
| 87 | + | const ( | |
| 88 | + | // SchedulerQuantum is the byte-deficit round-robin quantum: 64 KiB, the | |
| 89 | + | // largest opaque stream chunk the daemonmux codec admits. A stream with a | |
| 90 | + | // head frame at or below the quantum is served once per visit; a larger | |
| 91 | + | // frame accumulates one quantum per visit up to the overflow cap and is | |
| 92 | + | // served once its deficit covers it. | |
| 93 | + | SchedulerQuantum = int64(MaxMuxChunkBytes) | |
| 94 | + | ||
| 95 | + | // MaxRetiredSchedulerRecords bounds the settled scheduler records retained | |
| 96 | + | // so a duplicate terminal frame is still refused and a repeated Reset | |
| 97 | + | // stays idempotent; that refusal is guaranteed only within this retention | |
| 98 | + | // window, and an evicted identity may degrade to unadmitted. An overflowed, | |
| 99 | + | // not-yet-sealed record is retained the same way. It mirrors the negotiated | |
| 100 | + | // stream ceiling and is evicted oldest-first. | |
| 101 | + | MaxRetiredSchedulerRecords = 128 | |
| 102 | + | ||
| 103 | + | // MaxMuxSchedulerControlBurst bounds how many control frames the | |
| 104 | + | // scheduler writes ahead of data before it forces a data frame. It keeps | |
| 105 | + | // control priority small and bounded so control metadata can never starve | |
| 106 | + | // data. | |
| 107 | + | MaxMuxSchedulerControlBurst = 4 | |
| 108 | + | ) | |
| 109 | + | ||
| 110 | + | // Outbound scheduling sentinels. They are typed so a caller can classify a | |
| 111 | + | // refused frame without matching on message text. | |
| 112 | + | var ( | |
| 113 | + | // ErrSchedulerUnadmitted reports an outbound frame for a physical stream | |
| 114 | + | // the admission gate has not admitted or has already terminalized. An | |
| 115 | + | // identity whose settled scheduler record was evicted, and which the engine | |
| 116 | + | // gate no longer tracks, degrades to this refusal (see ErrSchedulerSettled). | |
| 117 | + | ErrSchedulerUnadmitted = errors.New("daemonmux: stream is not admitted for scheduling") | |
| 118 | + | // ErrSchedulerSettled reports an outbound frame for a stream the scheduler | |
| 119 | + | // already sealed: a queued terminal Close or Reset, a local Reset, or a | |
| 120 | + | // record that took the one terminal a queue overflow leaves open. | |
| 121 | + | // | |
| 122 | + | // The refusal is guaranteed only while the settled record is retained | |
| 123 | + | // (bounded by MaxRetiredSchedulerRecords). Once the record is evicted the | |
| 124 | + | // identity may instead be unadmitted (ErrSchedulerUnadmitted), or a late | |
| 125 | + | // terminal may be accepted once more and harmlessly discarded by the peer's | |
| 126 | + | // stream fencing. | |
| 127 | + | ErrSchedulerSettled = errors.New("daemonmux: stream is settled for scheduling") | |
| 128 | + | // ErrSchedulerFull reports an outbound frame that would exceed the | |
| 129 | + | // per-stream frame bound, the per-stream byte bound, or the aggregate byte | |
| 130 | + | // bound. The offending stream's queue is discarded and its budget | |
| 131 | + | // released. It also reports a non-terminal frame for a stream that already | |
| 132 | + | // overflowed and is awaiting its one terminal Close or Reset: the data was | |
| 133 | + | // discarded and the record is not yet sealed. | |
| 134 | + | ErrSchedulerFull = errors.New("daemonmux: scheduler queue overflow") | |
| 135 | + | ) | |
| 136 | + | ||
| 137 | + | // FrameClass classifies one outbound frame for scheduling priority. Control | |
| 138 | + | // frames may be written ahead of data frames under a bounded burst; data | |
| 139 | + | // frames carry the opaque stream payload. | |
| 140 | + | type FrameClass uint8 | |
| 141 | + | ||
| 142 | + | const ( | |
| 143 | + | // FrameData is an opaque stream chunk: a daemonmux Data payload. | |
| 144 | + | FrameData FrameClass = iota + 1 | |
| 145 | + | // FrameControl is connection or stream metadata: Open, Opened, Refused, | |
| 146 | + | // Close, or Reset. | |
| 147 | + | FrameControl | |
| 148 | + | ) | |
| 149 | + | ||
| 150 | + | func (c FrameClass) String() string { | |
| 151 | + | switch c { | |
| 152 | + | case FrameData: | |
| 153 | + | return "data" | |
| 154 | + | case FrameControl: | |
| 155 | + | return "control" | |
| 156 | + | default: | |
| 157 | + | return "unknown" | |
| 158 | + | } | |
| 159 | + | } | |
| 160 | + | ||
| 161 | + | // EnvelopeDirection names the daemonmux direction of one outbound envelope. | |
| 162 | + | // DirectionClient is broker-to-daemon (the codec's client direction); | |
| 163 | + | // DirectionServer is daemon-to-broker (the codec's server direction). | |
| 164 | + | type EnvelopeDirection uint8 | |
| 165 | + | ||
| 166 | + | const ( | |
| 167 | + | // DirectionClient is the broker-to-daemon direction. | |
| 168 | + | DirectionClient EnvelopeDirection = iota + 1 | |
| 169 | + | // DirectionServer is the daemon-to-broker direction. | |
| 170 | + | DirectionServer | |
| 171 | + | ) | |
| 172 | + | ||
| 173 | + | func (d EnvelopeDirection) String() string { | |
| 174 | + | switch d { | |
| 175 | + | case DirectionClient: | |
| 176 | + | return "client" | |
| 177 | + | case DirectionServer: | |
| 178 | + | return "server" | |
| 179 | + | default: | |
| 180 | + | return "unknown" | |
| 181 | + | } | |
| 182 | + | } | |
| 183 | + | ||
| 184 | + | // OutboundEnvelope is one immutable outbound frame ready for a future writer. | |
| 185 | + | // Bytes is the complete serialized directional envelope, owned by the caller | |
| 186 | + | // and never reused or mutated by the scheduler. Exactly one of Client and | |
| 187 | + | // Server is non-nil and matches Direction; it is a scheduler-owned committed | |
| 188 | + | // copy, so a caller that mutates its own message after enqueue never changes a | |
| 189 | + | // dequeued value. Class is the scheduling class the frame was enqueued under. | |
| 190 | + | type OutboundEnvelope struct { | |
| 191 | + | Physical PhysicalStreamID | |
| 192 | + | Direction EnvelopeDirection | |
| 193 | + | Class FrameClass | |
| 194 | + | Bytes []byte | |
| 195 | + | Client ClientMessage | |
| 196 | + | Server ServerMessage | |
| 197 | + | } | |
| 198 | + | ||
| 199 | + | // schedFrame is one queued outbound frame. Its encoded bytes are the | |
| 200 | + | // immutable carriage the future writer drains; the committed message is | |
| 201 | + | // retained for the caller that wants the typed value. | |
| 202 | + | type schedFrame struct { | |
| 203 | + | direction EnvelopeDirection | |
| 204 | + | class FrameClass | |
| 205 | + | bytes []byte | |
| 206 | + | seals bool | |
| 207 | + | client ClientMessage | |
| 208 | + | server ServerMessage | |
| 209 | + | } | |
| 210 | + | ||
| 211 | + | // schedStream is one tracked outbound stream. It is the scheduling view of a | |
| 212 | + | // stream the engine admitted: the bounded FIFO queue, its charged bytes, the | |
| 213 | + | // byte-deficit account, whether it still participates in the active ring, and | |
| 214 | + | // whether it is sealed against further frames. Every field is guarded by the | |
| 215 | + | // owning Scheduler's lock. | |
| 216 | + | type schedStream struct { | |
| 217 | + | physical PhysicalStreamID | |
| 218 | + | queue []schedFrame | |
| 219 | + | bytes int | |
| 220 | + | deficit int64 | |
| 221 | + | sealed bool | |
| 222 | + | // overflowed marks a stream whose queue overflowed and was discarded: it | |
| 223 | + | // is settled for data but not yet sealed, so exactly one terminal Close or | |
| 224 | + | // Reset may still be enqueued to signal the peer; that one terminal is | |
| 225 | + | // always retained and is the only frame that may carry the aggregate | |
| 226 | + | // charge past MaxMuxAggregateBytes (by at most one bounded envelope). | |
| 227 | + | overflowed bool | |
| 228 | + | active bool | |
| 229 | + | // retired marks the record as listed for bounded eviction once it is | |
| 230 | + | // sealed and empty; it is set at most once. | |
| 231 | + | retired bool | |
| 232 | + | } | |
| 233 | + | ||
| 234 | + | // Scheduler is the outbound fair-scheduling authority of one physical | |
| 235 | + | // daemonmux connection. It is safe for concurrent use and owns no I/O. The | |
| 236 | + | // optional StreamEngine gate reads admission; a nil engine leaves admission to | |
| 237 | + | // the caller. | |
| 238 | + | type Scheduler struct { | |
| 239 | + | mu sync.Mutex | |
| 240 | + | engine *StreamEngine | |
| 241 | + | streams map[PhysicalStreamID]*schedStream | |
| 242 | + | active []*schedStream | |
| 243 | + | retired []*schedStream | |
| 244 | + | aggregate int | |
| 245 | + | // controlBurst is the remaining number of control frames that may be | |
| 246 | + | // written ahead of data. It refills to MaxMuxSchedulerControlBurst after | |
| 247 | + | // every data frame. | |
| 248 | + | controlBurst int | |
| 249 | + | ||
| 250 | + | maxEnvelopeBytes uint64 | |
| 251 | + | maxChunkBytes uint64 | |
| 252 | + | ||
| 253 | + | // Bounds and quantum are constructed from the package defaults and are | |
| 254 | + | // owned by the scheduler; they are fields so a test can tighten them. | |
| 255 | + | maxAggregateBytes int | |
| 256 | + | maxStreamBytes int | |
| 257 | + | maxStreamFrames int | |
| 258 | + | quantum int64 | |
| 259 | + | } | |
| 260 | + | ||
| 261 | + | // NewScheduler returns an empty outbound scheduler under the default | |
| 262 | + | // MuxCeilings. When engine is non-nil the scheduler enqueues a frame only for | |
| 263 | + | // an opening or open stream the engine has admitted; when it is nil, the | |
| 264 | + | // caller owns admission and any nonzero physical identity is accepted. The | |
| 265 | + | // engine does not need to share the scheduler's ceilings: the scheduler's | |
| 266 | + | // envelope, chunk, and aggregate budgets come from its own copy. | |
| 267 | + | func NewScheduler(engine *StreamEngine) *Scheduler { | |
| 268 | + | return newScheduler(engine, DefaultMuxCeilings()) | |
| 269 | + | } | |
| 270 | + | ||
| 271 | + | // NewSchedulerWithCeilings returns an empty outbound scheduler whose envelope, | |
| 272 | + | // chunk, and aggregate budgets enforce the given negotiated MuxCeilings. The | |
| 273 | + | // value is validated and copied, so the ceilings a scheduler enforces are | |
| 274 | + | // fixed for its lifetime. An invalid advertisement is refused with | |
| 275 | + | // ErrInvalidCeilings. | |
| 276 | + | func NewSchedulerWithCeilings(engine *StreamEngine, ceilings MuxCeilings) (*Scheduler, error) { | |
| 277 | + | if err := ceilings.Validate(); err != nil { | |
| 278 | + | return nil, err | |
| 279 | + | } | |
| 280 | + | return newScheduler(engine, ceilings), nil | |
| 281 | + | } | |
| 282 | + | ||
| 283 | + | // newScheduler builds a scheduler from already-validated ceilings. The | |
| 284 | + | // exported constructors validate first; NewScheduler delegates with the | |
| 285 | + | // package defaults, which are valid by construction. | |
| 286 | + | func newScheduler(engine *StreamEngine, ceilings MuxCeilings) *Scheduler { | |
| 287 | + | return &Scheduler{ | |
| 288 | + | engine: engine, | |
| 289 | + | streams: make(map[PhysicalStreamID]*schedStream, int(ceilings.MaxStreams)), | |
| 290 | + | maxEnvelopeBytes: ceilings.MaxReceiveEnvelopeBytes, | |
| 291 | + | maxChunkBytes: ceilings.StreamChunkLimit, | |
| 292 | + | maxAggregateBytes: int(ceilings.MaxAggregateBytes), | |
| 293 | + | maxStreamBytes: MaxMuxStreamQueueBytes, | |
| 294 | + | maxStreamFrames: MaxMuxStreamQueueChunks, | |
| 295 | + | quantum: SchedulerQuantum, | |
| 296 | + | controlBurst: MaxMuxSchedulerControlBurst, | |
| 297 | + | } | |
| 298 | + | } | |
| 299 | + | ||
| 300 | + | // EnqueueClient encodes and enqueues one broker-to-daemon envelope. | |
| 301 | + | func (s *Scheduler) EnqueueClient(message ClientMessage) error { | |
| 302 | + | if s == nil || message == nil { | |
| 303 | + | return ErrInvalidMessage | |
| 304 | + | } | |
| 305 | + | physical, class, err := clientFrame(message) | |
| 306 | + | if err != nil { | |
| 307 | + | return err | |
| 308 | + | } | |
| 309 | + | raw, err := EncodeClient(message, s.maxEnvelopeBytes, s.maxChunkBytes) | |
| 310 | + | if err != nil { | |
| 311 | + | return err | |
| 312 | + | } | |
| 313 | + | return s.enqueue(schedFrame{ | |
| 314 | + | direction: DirectionClient, | |
| 315 | + | class: class, | |
| 316 | + | bytes: raw, | |
| 317 | + | seals: clientSeals(message), | |
| 318 | + | client: cloneClientMessage(message), | |
| 319 | + | }, physical) | |
| 320 | + | } | |
| 321 | + | ||
| 322 | + | // Refuse enqueues exactly one Mux Refused for a fresh inbound Open the engine | |
| 323 | + | // refused before it recorded any stream. It is the deliberate second admission | |
| 324 | + | // exception, alongside the one terminal Close or Reset a settled identity | |
| 325 | + | // accepts: a Refused is final for its physical identity (the daemon never sends | |
| 326 | + | // another frame for a stream it never admitted), so the record is sealed and | |
| 327 | + | // bounded by the same retired-record eviction as every other settled stream. | |
| 328 | + | // | |
| 329 | + | // The frame bypasses the engine gate because a rejected Open leaves no engine | |
| 330 | + | // record to confirm; the caller is the pump, and it only reaches here for an | |
| 331 | + | // identity above the engine's high-water mark that the engine refused. A | |
| 332 | + | // duplicate Refused, or any frame after the refusal drained, is refused with | |
| 333 | + | // ErrSchedulerSettled. | |
| 334 | + | func (s *Scheduler) Refuse(message Refused) error { | |
| 335 | + | if s == nil { | |
| 336 | + | return ErrInvalidMessage | |
| 337 | + | } | |
| 338 | + | physical, class, err := serverFrame(message) | |
| 339 | + | if err != nil { | |
| 340 | + | return err | |
| 341 | + | } | |
| 342 | + | raw, err := EncodeServer(message, s.maxEnvelopeBytes, s.maxChunkBytes) | |
| 343 | + | if err != nil { | |
| 344 | + | return err | |
| 345 | + | } | |
| 346 | + | if err := physical.Validate(); err != nil { | |
| 347 | + | return err | |
| 348 | + | } | |
| 349 | + | frame := schedFrame{ | |
| 350 | + | direction: DirectionServer, | |
| 351 | + | class: class, | |
| 352 | + | bytes: raw, | |
| 353 | + | seals: true, | |
| 354 | + | server: cloneServerMessage(message), | |
| 355 | + | } | |
| 356 | + | s.mu.Lock() | |
| 357 | + | defer s.mu.Unlock() | |
| 358 | + | stream := s.streams[physical] | |
| 359 | + | if stream != nil { | |
| 360 | + | if stream.sealed || stream.overflowed { | |
| 361 | + | return ErrSchedulerSettled | |
| 362 | + | } | |
| 363 | + | } else { | |
| 364 | + | stream = &schedStream{physical: physical} | |
| 365 | + | s.streams[physical] = stream | |
| 366 | + | } | |
| 367 | + | if s.overflowLocked(stream, len(frame.bytes)) { | |
| 368 | + | s.overflowLockedStream(stream) | |
| 369 | + | return ErrSchedulerFull | |
| 370 | + | } | |
| 371 | + | s.insertLocked(stream, frame) | |
| 372 | + | return nil | |
| 373 | + | } | |
| 374 | + | ||
| 375 | + | // EnqueueServer encodes and enqueues one daemon-to-broker envelope. | |
| 376 | + | func (s *Scheduler) EnqueueServer(message ServerMessage) error { | |
| 377 | + | if s == nil || message == nil { | |
| 378 | + | return ErrInvalidMessage | |
| 379 | + | } | |
| 380 | + | physical, class, err := serverFrame(message) | |
| 381 | + | if err != nil { | |
| 382 | + | return err | |
| 383 | + | } | |
| 384 | + | raw, err := EncodeServer(message, s.maxEnvelopeBytes, s.maxChunkBytes) | |
| 385 | + | if err != nil { | |
| 386 | + | return err | |
| 387 | + | } | |
| 388 | + | return s.enqueue(schedFrame{ | |
| 389 | + | direction: DirectionServer, | |
| 390 | + | class: class, | |
| 391 | + | bytes: raw, | |
| 392 | + | seals: serverSeals(message), | |
| 393 | + | server: cloneServerMessage(message), | |
| 394 | + | }, physical) | |
| 395 | + | } | |
| 396 | + | ||
| 397 | + | // enqueue inserts one already-encoded frame under the scheduler lock. It | |
| 398 | + | // confirms admission, then charges the frame against both byte budgets before | |
| 399 | + | // the queue adopts it. A frame that would overflow any bound discards its own | |
| 400 | + | // stream queue and frees that budget without touching a sibling. A terminal | |
| 401 | + | // frame is the one exception to admission: it is accepted exactly once for a | |
| 402 | + | // known identity, so an overflow or reset path can still signal the peer even | |
| 403 | + | // after the engine settled the stream. Data for such an identity stays | |
| 404 | + | // refused, and a duplicate terminal is refused with ErrSchedulerSettled. | |
| 405 | + | func (s *Scheduler) enqueue(frame schedFrame, physical PhysicalStreamID) error { | |
| 406 | + | if err := physical.Validate(); err != nil { | |
| 407 | + | return ErrInvalidStream | |
| 408 | + | } | |
| 409 | + | s.mu.Lock() | |
| 410 | + | defer s.mu.Unlock() | |
| 411 | + | stream := s.streams[physical] | |
| 412 | + | if stream != nil { | |
| 413 | + | if stream.sealed { | |
| 414 | + | // A duplicate terminal, and every frame after a terminal, is | |
| 415 | + | // refused: the segment is already settled. | |
| 416 | + | return ErrSchedulerSettled | |
| 417 | + | } | |
| 418 | + | if stream.overflowed { | |
| 419 | + | // The queue overflowed and was discarded, but the record is not yet | |
| 420 | + | // sealed: exactly one terminal Close or Reset may still signal the | |
| 421 | + | // peer. Any other frame is refused with ErrSchedulerFull; the | |
| 422 | + | // terminal is inserted without re-running the overflow check because | |
| 423 | + | // the queue is already empty and its bytes released. It is therefore | |
| 424 | + | // the one frame that may carry the aggregate charge past | |
| 425 | + | // MaxMuxAggregateBytes, by at most one bounded terminal per | |
| 426 | + | // overflowed stream, so peer notification is never lost. | |
| 427 | + | if !frame.seals { | |
| 428 | + | return ErrSchedulerFull | |
| 429 | + | } | |
| 430 | + | s.insertLocked(stream, frame) | |
| 431 | + | return nil | |
| 432 | + | } | |
| 433 | + | if !frame.seals && !s.admittedLocked(physical) { | |
| 434 | + | return ErrSchedulerUnadmitted | |
| 435 | + | } | |
| 436 | + | } else { | |
| 437 | + | if !frame.seals { | |
| 438 | + | if !s.admittedLocked(physical) { | |
| 439 | + | return ErrSchedulerUnadmitted | |
| 440 | + | } | |
| 441 | + | } else if !s.knownLocked(physical) { | |
| 442 | + | // A terminal frame still needs a known identity: it may bypass a | |
| 443 | + | // settled admission, never a stream that was never admitted. | |
| 444 | + | return ErrSchedulerUnadmitted | |
| 445 | + | } | |
| 446 | + | stream = &schedStream{physical: physical} | |
| 447 | + | s.streams[physical] = stream | |
| 448 | + | } | |
| 449 | + | if s.overflowLocked(stream, len(frame.bytes)) { | |
| 450 | + | s.overflowLockedStream(stream) | |
| 451 | + | return ErrSchedulerFull | |
| 452 | + | } | |
| 453 | + | s.insertLocked(stream, frame) | |
| 454 | + | return nil | |
| 455 | + | } | |
| 456 | + | ||
| 457 | + | // insertLocked charges one already-admitted frame against both byte budgets | |
| 458 | + | // and appends it to the stream's queue, sealing the record when the frame is a | |
| 459 | + | // terminal Close or Reset and entering the stream into the active ring. The | |
| 460 | + | // caller has already tested every bound, except for the one terminal an | |
| 461 | + | // overflowed stream still retains: that frame is charged here deliberately, so | |
| 462 | + | // peer notification survives the overflow at the cost of at most one bounded | |
| 463 | + | // envelope above the aggregate data budget. | |
| 464 | + | func (s *Scheduler) insertLocked(stream *schedStream, frame schedFrame) { | |
| 465 | + | n := len(frame.bytes) | |
| 466 | + | s.aggregate += n | |
| 467 | + | stream.bytes += n | |
| 468 | + | stream.queue = append(stream.queue, frame) | |
| 469 | + | if frame.seals { | |
| 470 | + | stream.sealed = true | |
| 471 | + | } | |
| 472 | + | if !stream.active { | |
| 473 | + | stream.active = true | |
| 474 | + | s.active = append(s.active, stream) | |
| 475 | + | } | |
| 476 | + | } | |
| 477 | + | ||
| 478 | + | // overflowLockedStream settles one stream whose frame exceeded a bound: it | |
| 479 | + | // discards the queued frames, frees their budget, and marks the record as | |
| 480 | + | // overflowed rather than sealed, so exactly one terminal Close or Reset may | |
| 481 | + | // still be enqueued. The record is listed for bounded eviction while it stays | |
| 482 | + | // empty and unsettled, so a long-lived connection cannot accumulate overflow | |
| 483 | + | // records. | |
| 484 | + | func (s *Scheduler) overflowLockedStream(stream *schedStream) { | |
| 485 | + | s.discardLocked(stream) | |
| 486 | + | stream.overflowed = true | |
| 487 | + | s.listRetiredLocked(stream) | |
| 488 | + | } | |
| 489 | + | ||
| 490 | + | // Dequeue removes and returns the next ready outbound envelope, or false when | |
| 491 | + | // no stream has a queued frame. The returned Bytes are immutable and owned by | |
| 492 | + | // the caller. | |
| 493 | + | func (s *Scheduler) Dequeue() (OutboundEnvelope, bool) { | |
| 494 | + | if s == nil { | |
| 495 | + | return OutboundEnvelope{}, false | |
| 496 | + | } | |
| 497 | + | s.mu.Lock() | |
| 498 | + | defer s.mu.Unlock() | |
| 499 | + | if len(s.active) == 0 { | |
| 500 | + | return OutboundEnvelope{}, false | |
| 501 | + | } | |
| 502 | + | if s.controlBurst > 0 { | |
| 503 | + | if index := s.firstControlLocked(); index >= 0 { | |
| 504 | + | s.controlBurst-- | |
| 505 | + | return s.serveIndexLocked(index), true | |
| 506 | + | } | |
| 507 | + | } | |
| 508 | + | envelope := s.serveDRRLocked() | |
| 509 | + | if envelope.Class == FrameData { | |
| 510 | + | s.controlBurst = MaxMuxSchedulerControlBurst | |
| 511 | + | } | |
| 512 | + | return envelope, true | |
| 513 | + | } | |
| 514 | + | ||
| 515 | + | // Reset settles one stream locally: it discards the stream's queued frames, | |
| 516 | + | // frees their bytes immediately, and refuses every later frame for that | |
| 517 | + | // identity. It reports whether it settled a stream that was not already | |
| 518 | + | // settled, so a repeated cancel is idempotent. It never touches the physical | |
| 519 | + | // connection or a sibling stream. | |
| 520 | + | func (s *Scheduler) Reset(physical PhysicalStreamID) bool { | |
| 521 | + | if s == nil { | |
| 522 | + | return false | |
| 523 | + | } | |
| 524 | + | s.mu.Lock() | |
| 525 | + | defer s.mu.Unlock() | |
| 526 | + | stream := s.streams[physical] | |
| 527 | + | if stream == nil { | |
| 528 | + | if !s.admittedLocked(physical) { | |
| 529 | + | return false | |
| 530 | + | } | |
| 531 | + | stream = &schedStream{physical: physical, sealed: true} | |
| 532 | + | s.streams[physical] = stream | |
| 533 | + | s.listRetiredLocked(stream) | |
| 534 | + | return true | |
| 535 | + | } | |
| 536 | + | if stream.sealed && len(stream.queue) == 0 { | |
| 537 | + | return false | |
| 538 | + | } | |
| 539 | + | s.discardLocked(stream) | |
| 540 | + | stream.sealed = true | |
| 541 | + | s.listRetiredLocked(stream) | |
| 542 | + | return true | |
| 543 | + | } | |
| 544 | + | ||
| 545 | + | // AggregateBytes returns the outbound bytes currently charged across every | |
| 546 | + | // stream. Application data never exceeds MaxMuxAggregateBytes; the only | |
| 547 | + | // possible excess is the terminal an overflowed stream retains, at most one | |
| 548 | + | // bounded envelope per live or retained overflowed stream, which is what keeps | |
| 549 | + | // peer notification possible. | |
| 550 | + | func (s *Scheduler) AggregateBytes() int { | |
| 551 | + | if s == nil { | |
| 552 | + | return 0 | |
| 553 | + | } | |
| 554 | + | s.mu.Lock() | |
| 555 | + | defer s.mu.Unlock() | |
| 556 | + | return s.aggregate | |
| 557 | + | } | |
| 558 | + | ||
| 559 | + | // QueuedFrames returns the number of frames queued for one stream. | |
| 560 | + | func (s *Scheduler) QueuedFrames(physical PhysicalStreamID) int { | |
| 561 | + | if s == nil { | |
| 562 | + | return 0 | |
| 563 | + | } | |
| 564 | + | s.mu.Lock() | |
| 565 | + | defer s.mu.Unlock() | |
| 566 | + | if stream := s.streams[physical]; stream != nil { | |
| 567 | + | return len(stream.queue) | |
| 568 | + | } | |
| 569 | + | return 0 | |
| 570 | + | } | |
| 571 | + | ||
| 572 | + | // QueuedBytes returns the outbound bytes currently queued for one stream. | |
| 573 | + | func (s *Scheduler) QueuedBytes(physical PhysicalStreamID) int { | |
| 574 | + | if s == nil { | |
| 575 | + | return 0 | |
| 576 | + | } | |
| 577 | + | s.mu.Lock() | |
| 578 | + | defer s.mu.Unlock() | |
| 579 | + | if stream := s.streams[physical]; stream != nil { | |
| 580 | + | return stream.bytes | |
| 581 | + | } | |
| 582 | + | return 0 | |
| 583 | + | } | |
| 584 | + | ||
| 585 | + | // Active returns the number of streams with a queued frame. | |
| 586 | + | func (s *Scheduler) Active() int { | |
| 587 | + | if s == nil { | |
| 588 | + | return 0 | |
| 589 | + | } | |
| 590 | + | s.mu.Lock() | |
| 591 | + | defer s.mu.Unlock() | |
| 592 | + | return len(s.active) | |
| 593 | + | } | |
| 594 | + | ||
| 595 | + | // Pending reports whether any stream has a queued frame. | |
| 596 | + | func (s *Scheduler) Pending() bool { return s.Active() > 0 } | |
| 597 | + | ||
| 598 | + | // admittedLocked reports whether the admission gate permits scheduling for one | |
| 599 | + | // physical stream: a nil engine admits any nonzero identity, and an installed | |
| 600 | + | // engine admits a stream it tracks in opening or open state. A closing stream | |
| 601 | + | // (an orderly inbound Close with unread data) accepts no new outbound frame, | |
| 602 | + | // so it is not admitted; its one terminal Close or Reset still bypasses the | |
| 603 | + | // gate through knownLocked. | |
| 604 | + | func (s *Scheduler) admittedLocked(physical PhysicalStreamID) bool { | |
| 605 | + | if physical == 0 { | |
| 606 | + | return false | |
| 607 | + | } | |
| 608 | + | if s.engine == nil { | |
| 609 | + | return true | |
| 610 | + | } | |
| 611 | + | status, ok := s.engine.Status(physical) | |
| 612 | + | return ok && (status.State == StreamOpening || status.State == StreamOpen) | |
| 613 | + | } | |
| 614 | + | ||
| 615 | + | // knownLocked reports whether one physical stream identity is known to the | |
| 616 | + | // admission gate at all: a nil engine trusts any nonzero identity, and an | |
| 617 | + | // installed engine knows a stream it tracks in any state, including a settled | |
| 618 | + | // one. It backs the single terminal-frame exception: a terminal Close or Reset | |
| 619 | + | // may be enqueued once for a settled identity to signal the peer. | |
| 620 | + | func (s *Scheduler) knownLocked(physical PhysicalStreamID) bool { | |
| 621 | + | if physical == 0 { | |
| 622 | + | return false | |
| 623 | + | } | |
| 624 | + | if s.engine == nil { | |
| 625 | + | return true | |
| 626 | + | } | |
| 627 | + | _, ok := s.engine.Status(physical) | |
| 628 | + | return ok | |
| 629 | + | } | |
| 630 | + | ||
| 631 | + | // listRetiredLocked records one settled or overflowed, empty stream for | |
| 632 | + | // bounded eviction and evicts the oldest retired record past | |
| 633 | + | // MaxRetiredSchedulerRecords. An evicted identity falls back to the engine | |
| 634 | + | // gate for admission, so the ErrSchedulerSettled refusal is guaranteed only | |
| 635 | + | // within this retention window and a late terminal may instead be accepted | |
| 636 | + | // once more and harmlessly discarded by the peer's stream fencing. A record | |
| 637 | + | // is listed at most once and only while it is | |
| 638 | + | // sealed or overflowed and empty; a still-draining record stays until it | |
| 639 | + | // empties, and an overflowed record that later takes its one terminal is | |
| 640 | + | // sealed in place. A record that took that terminal after it was listed keeps | |
| 641 | + | // its queued frame past eviction instead of being forgotten: eviction clears | |
| 642 | + | // its retired flag, so draining its last frame re-lists it and a later | |
| 643 | + | // eviction reclaims the record rather than orphaning it in the stream table. | |
| 644 | + | func (s *Scheduler) listRetiredLocked(stream *schedStream) { | |
| 645 | + | if stream.retired || !(stream.sealed || stream.overflowed) || len(stream.queue) != 0 { | |
| 646 | + | return | |
| 647 | + | } | |
| 648 | + | stream.retired = true | |
| 649 | + | s.retired = append(s.retired, stream) | |
| 650 | + | for len(s.retired) > MaxRetiredSchedulerRecords { | |
| 651 | + | evicted := s.retired[0] | |
| 652 | + | s.retired = s.retired[1:] | |
| 653 | + | if (evicted.sealed || evicted.overflowed) && len(evicted.queue) == 0 { | |
| 654 | + | delete(s.streams, evicted.physical) | |
| 655 | + | continue | |
| 656 | + | } | |
| 657 | + | // The terminal this record accepted after it was listed is still | |
| 658 | + | // queued, so the record cannot be reclaimed yet. Clear the flag so | |
| 659 | + | // draining its last frame re-lists it for a later eviction. | |
| 660 | + | evicted.retired = false | |
| 661 | + | } | |
| 662 | + | } | |
| 663 | + | ||
| 664 | + | // overflowLocked reports whether one frame would exceed the per-stream frame | |
| 665 | + | // bound, the per-stream byte bound, or the aggregate byte bound. | |
| 666 | + | func (s *Scheduler) overflowLocked(stream *schedStream, n int) bool { | |
| 667 | + | return len(stream.queue) >= s.maxStreamFrames || | |
| 668 | + | stream.bytes+n > s.maxStreamBytes || | |
| 669 | + | s.aggregate+n > s.maxAggregateBytes | |
| 670 | + | } | |
| 671 | + | ||
| 672 | + | // discardLocked drops every queued frame of one stream and returns its bytes | |
| 673 | + | // to the aggregate budget. The stream record stays so a repeat cancel is | |
| 674 | + | // idempotent and a later frame is refused. | |
| 675 | + | func (s *Scheduler) discardLocked(stream *schedStream) { | |
| 676 | + | s.aggregate -= stream.bytes | |
| 677 | + | stream.bytes = 0 | |
| 678 | + | for i := range stream.queue { | |
| 679 | + | stream.queue[i] = schedFrame{} | |
| 680 | + | } | |
| 681 | + | stream.queue = nil | |
| 682 | + | if stream.active { | |
| 683 | + | s.removeActiveLocked(stream) | |
| 684 | + | } | |
| 685 | + | } | |
| 686 | + | ||
| 687 | + | // removeActiveLocked takes one stream out of the active ring. | |
| 688 | + | func (s *Scheduler) removeActiveLocked(stream *schedStream) { | |
| 689 | + | for i, candidate := range s.active { | |
| 690 | + | if candidate != stream { | |
| 691 | + | continue | |
| 692 | + | } | |
| 693 | + | last := len(s.active) - 1 | |
| 694 | + | copy(s.active[i:], s.active[i+1:]) | |
| 695 | + | s.active[last] = nil | |
| 696 | + | s.active = s.active[:last] | |
| 697 | + | break | |
| 698 | + | } | |
| 699 | + | stream.active = false | |
| 700 | + | } | |
| 701 | + | ||
| 702 | + | // firstControlLocked returns the index of the first active stream whose oldest | |
| 703 | + | // frame is control, or -1 when none is. It prefers the head position in the | |
| 704 | + | // ring so the control priority stays round-robin. | |
| 705 | + | func (s *Scheduler) firstControlLocked() int { | |
| 706 | + | for i, stream := range s.active { | |
| 707 | + | if stream.queue[0].class == FrameControl { | |
| 708 | + | return i | |
| 709 | + | } | |
| 710 | + | } | |
| 711 | + | return -1 | |
| 712 | + | } | |
| 713 | + | ||
| 714 | + | // serveIndexLocked writes the oldest frame of the active stream at index. The | |
| 715 | + | // stream's byte-deficit account is topped up and charged before the frame is | |
| 716 | + | // removed, and the stream rotates to the back of the ring. | |
| 717 | + | func (s *Scheduler) serveIndexLocked(index int) OutboundEnvelope { | |
| 718 | + | stream := s.active[index] | |
| 719 | + | s.topUpLocked(stream) | |
| 720 | + | stream.deficit -= int64(len(stream.queue[0].bytes)) | |
| 721 | + | s.rotateToBackLocked(index) | |
| 722 | + | return s.popHeadLocked(stream) | |
| 723 | + | } | |
| 724 | + | ||
| 725 | + | // serveDRRLocked runs one byte-deficit round-robin step: it visits active | |
| 726 | + | // streams in ring order, tops up and tests the head of each, and writes the | |
| 727 | + | // first head that fits its accumulated deficit. When no head fits in a pass, | |
| 728 | + | // every stream was topped up, so another pass is guaranteed to make progress. | |
| 729 | + | func (s *Scheduler) serveDRRLocked() OutboundEnvelope { | |
| 730 | + | for { | |
| 731 | + | for i, stream := range s.active { | |
| 732 | + | s.topUpLocked(stream) | |
| 733 | + | size := int64(len(stream.queue[0].bytes)) | |
| 734 | + | if size <= stream.deficit { | |
| 735 | + | stream.deficit -= size | |
| 736 | + | s.rotateToBackLocked(i) | |
| 737 | + | return s.popHeadLocked(stream) | |
| 738 | + | } | |
| 739 | + | } | |
| 740 | + | } | |
| 741 | + | } | |
| 742 | + | ||
| 743 | + | // topUpLocked adds one quantum to a stream's deficit unconditionally and caps | |
| 744 | + | // the result at one quantum plus the largest frame the codec admits | |
| 745 | + | // (MaxMuxApplicationBytes). Unconditional top-up is what guarantees progress: | |
| 746 | + | // a head frame larger than the quantum accumulates one quantum per visit until | |
| 747 | + | // it fits, while the cap bounds the deficit a stream can bank and keeps the | |
| 748 | + | // schedule fair. | |
| 749 | + | func (s *Scheduler) topUpLocked(stream *schedStream) { | |
| 750 | + | stream.deficit += s.quantum | |
| 751 | + | if maxDeficit := s.quantum + int64(min(s.maxEnvelopeBytes, MaxMuxApplicationBytes)); stream.deficit > maxDeficit { | |
| 752 | + | stream.deficit = maxDeficit | |
| 753 | + | } | |
| 754 | + | } | |
| 755 | + | ||
| 756 | + | // rotateToBackLocked moves the active stream at index to the back of the ring, | |
| 757 | + | // preserving the relative order of every other stream and allocating nothing. | |
| 758 | + | func (s *Scheduler) rotateToBackLocked(index int) { | |
| 759 | + | last := len(s.active) - 1 | |
| 760 | + | if index == last { | |
| 761 | + | return | |
| 762 | + | } | |
| 763 | + | stream := s.active[index] | |
| 764 | + | copy(s.active[index:], s.active[index+1:]) | |
| 765 | + | s.active[last] = stream | |
| 766 | + | } | |
| 767 | + | ||
| 768 | + | // popHeadLocked removes and returns the head frame of a stream that the caller | |
| 769 | + | // already rotated to the back of the ring, releasing its bytes from both | |
| 770 | + | // budgets. An emptied stream leaves the ring. | |
| 771 | + | func (s *Scheduler) popHeadLocked(stream *schedStream) OutboundEnvelope { | |
| 772 | + | frame := stream.queue[0] | |
| 773 | + | stream.queue[0] = schedFrame{} | |
| 774 | + | stream.queue = stream.queue[1:] | |
| 775 | + | n := len(frame.bytes) | |
| 776 | + | stream.bytes -= n | |
| 777 | + | s.aggregate -= n | |
| 778 | + | if len(stream.queue) == 0 { | |
| 779 | + | last := len(s.active) - 1 | |
| 780 | + | s.active[last] = nil | |
| 781 | + | s.active = s.active[:last] | |
| 782 | + | stream.active = false | |
| 783 | + | s.listRetiredLocked(stream) | |
| 784 | + | } | |
| 785 | + | return OutboundEnvelope{ | |
| 786 | + | Physical: stream.physical, | |
| 787 | + | Direction: frame.direction, | |
| 788 | + | Class: frame.class, | |
| 789 | + | Bytes: frame.bytes, | |
| 790 | + | Client: frame.client, | |
| 791 | + | Server: frame.server, | |
| 792 | + | } | |
| 793 | + | } | |
| 794 | + | ||
| 795 | + | // clientSeals reports whether a client message ends its stream for scheduling: | |
| 796 | + | // a Close or Reset frame is the last frame the stream may enqueue, so a later | |
| 797 | + | // frame is refused while the terminal frame drains in per-stream order. | |
| 798 | + | func clientSeals(message ClientMessage) bool { | |
| 799 | + | switch message.(type) { | |
| 800 | + | case Close, *Close, Reset, *Reset: | |
| 801 | + | return true | |
| 802 | + | default: | |
| 803 | + | return false | |
| 804 | + | } | |
| 805 | + | } | |
| 806 | + | ||
| 807 | + | // serverSeals reports whether a server message ends its stream for scheduling. | |
| 808 | + | func serverSeals(message ServerMessage) bool { | |
| 809 | + | switch message.(type) { | |
| 810 | + | case Close, *Close, Reset, *Reset: | |
| 811 | + | return true | |
| 812 | + | default: | |
| 813 | + | return false | |
| 814 | + | } | |
| 815 | + | } | |
| 816 | + | ||
| 817 | + | // cloneClientMessage returns a committed copy of one client message. Byte and | |
| 818 | + | // string slices the caller still owns are copied so a dequeued envelope never | |
| 819 | + | // aliases caller memory; every other field is a value type and is copied by | |
| 820 | + | // assignment. Unknown concrete types are returned unchanged (they were already | |
| 821 | + | // refused by the codec). | |
| 822 | + | func cloneClientMessage(message ClientMessage) ClientMessage { | |
| 823 | + | switch m := message.(type) { | |
| 824 | + | case Data: | |
| 825 | + | return Data{Physical: m.Physical, Data: append([]byte(nil), m.Data...)} | |
| 826 | + | case *Data: | |
| 827 | + | if m == nil { | |
| 828 | + | return m | |
| 829 | + | } | |
| 830 | + | committed := *m | |
| 831 | + | committed.Data = append([]byte(nil), m.Data...) | |
| 832 | + | return &committed | |
| 833 | + | case Open: | |
| 834 | + | return m.clone() | |
| 835 | + | case *Open: | |
| 836 | + | if m == nil { | |
| 837 | + | return m | |
| 838 | + | } | |
| 839 | + | committed := m.clone() | |
| 840 | + | return &committed | |
| 841 | + | default: | |
| 842 | + | return message | |
| 843 | + | } | |
| 844 | + | } | |
| 845 | + | ||
| 846 | + | // clone returns an Open whose environment slice is owned by the scheduler. | |
| 847 | + | func (m Open) clone() Open { | |
| 848 | + | m.Env = append([]string(nil), m.Env...) | |
| 849 | + | return m | |
| 850 | + | } | |
| 851 | + | ||
| 852 | + | // cloneServerMessage returns a committed copy of one server message under the | |
| 853 | + | // same rule as cloneClientMessage: only the opaque data payload can alias | |
| 854 | + | // caller memory. | |
| 855 | + | func cloneServerMessage(message ServerMessage) ServerMessage { | |
| 856 | + | switch m := message.(type) { | |
| 857 | + | case Data: | |
| 858 | + | return Data{Physical: m.Physical, Data: append([]byte(nil), m.Data...)} | |
| 859 | + | case *Data: | |
| 860 | + | if m == nil { | |
| 861 | + | return m | |
| 862 | + | } | |
| 863 | + | committed := *m | |
| 864 | + | committed.Data = append([]byte(nil), m.Data...) | |
| 865 | + | return &committed | |
| 866 | + | default: | |
| 867 | + | return message | |
| 868 | + | } | |
| 869 | + | } | |
| 870 | + | ||
| 871 | + | // clientFrame resolves the physical stream identity and scheduling class of one | |
| 872 | + | // broker-to-daemon message before it is encoded. Unknown concrete types fail | |
| 873 | + | // closed with ErrWrongDirection, mirroring the codec. | |
| 874 | + | func clientFrame(message ClientMessage) (PhysicalStreamID, FrameClass, error) { | |
| 875 | + | switch m := message.(type) { | |
| 876 | + | case Open: | |
| 877 | + | return m.Ref.Physical, FrameControl, nil | |
| 878 | + | case *Open: | |
| 879 | + | if m == nil { | |
| 880 | + | return 0, 0, ErrInvalidMessage | |
| 881 | + | } | |
| 882 | + | return clientFrame(*m) | |
| 883 | + | case Data: | |
| 884 | + | return m.Physical, FrameData, nil | |
| 885 | + | case *Data: | |
| 886 | + | if m == nil { | |
| 887 | + | return 0, 0, ErrInvalidMessage | |
| 888 | + | } | |
| 889 | + | return clientFrame(*m) | |
| 890 | + | case Close: | |
| 891 | + | return m.Physical, FrameControl, nil | |
| 892 | + | case *Close: | |
| 893 | + | if m == nil { | |
| 894 | + | return 0, 0, ErrInvalidMessage | |
| 895 | + | } | |
| 896 | + | return clientFrame(*m) | |
| 897 | + | case Reset: | |
| 898 | + | return m.Physical, FrameControl, nil | |
| 899 | + | case *Reset: | |
| 900 | + | if m == nil { | |
| 901 | + | return 0, 0, ErrInvalidMessage | |
| 902 | + | } | |
| 903 | + | return clientFrame(*m) | |
| 904 | + | default: | |
| 905 | + | return 0, 0, ErrWrongDirection | |
| 906 | + | } | |
| 907 | + | } | |
| 908 | + | ||
| 909 | + | // serverFrame resolves the physical stream identity and scheduling class of one | |
| 910 | + | // daemon-to-broker message before it is encoded. | |
| 911 | + | func serverFrame(message ServerMessage) (PhysicalStreamID, FrameClass, error) { | |
| 912 | + | switch m := message.(type) { | |
| 913 | + | case Opened: | |
| 914 | + | return m.Ref.Physical, FrameControl, nil | |
| 915 | + | case *Opened: | |
| 916 | + | if m == nil { | |
| 917 | + | return 0, 0, ErrInvalidMessage | |
| 918 | + | } | |
| 919 | + | return serverFrame(*m) | |
| 920 | + | case Refused: | |
| 921 | + | return m.Ref.Physical, FrameControl, nil | |
| 922 | + | case *Refused: | |
| 923 | + | if m == nil { | |
| 924 | + | return 0, 0, ErrInvalidMessage | |
| 925 | + | } | |
| 926 | + | return serverFrame(*m) | |
| 927 | + | case Data: | |
| 928 | + | return m.Physical, FrameData, nil | |
| 929 | + | case *Data: | |
| 930 | + | if m == nil { | |
| 931 | + | return 0, 0, ErrInvalidMessage | |
| 932 | + | } | |
| 933 | + | return serverFrame(*m) | |
| 934 | + | case Close: | |
| 935 | + | return m.Physical, FrameControl, nil | |
| 936 | + | case *Close: | |
| 937 | + | if m == nil { | |
| 938 | + | return 0, 0, ErrInvalidMessage | |
| 939 | + | } | |
| 940 | + | return serverFrame(*m) | |
| 941 | + | case Reset: | |
| 942 | + | return m.Physical, FrameControl, nil | |
| 943 | + | case *Reset: | |
| 944 | + | if m == nil { | |
| 945 | + | return 0, 0, ErrInvalidMessage | |
| 946 | + | } | |
| 947 | + | return serverFrame(*m) | |
| 948 | + | default: | |
| 949 | + | return 0, 0, ErrWrongDirection | |
| 950 | + | } | |
| 951 | + | } | |
A internal/adapters/daemonmux/scheduler_test.go +872 -0
| @@ -0,0 +1,872 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "fmt" | |
| 5 | + | "strings" | |
| 6 | + | "sync" | |
| 7 | + | "testing" | |
| 8 | + | "time" | |
| 9 | + | ||
| 10 | + | "github.com/stretchr/testify/require" | |
| 11 | + | ||
| 12 | + | "github.com/bnema/vev/internal/ports" | |
| 13 | + | ) | |
| 14 | + | ||
| 15 | + | // testScheduler returns a scheduler with an open admission gate and one open | |
| 16 | + | // stream per requested physical identity, in ascending order. | |
| 17 | + | func testScheduler(t *testing.T, streams int) (*StreamEngine, *Scheduler, []PhysicalStreamID) { | |
| 18 | + | t.Helper() | |
| 19 | + | engine := testEngine() | |
| 20 | + | scheduler := NewScheduler(engine) | |
| 21 | + | ids := make([]PhysicalStreamID, streams) | |
| 22 | + | for i := range ids { | |
| 23 | + | ids[i] = PhysicalStreamID(i + 1) | |
| 24 | + | openTestStream(t, engine, ids[i]) | |
| 25 | + | } | |
| 26 | + | return engine, scheduler, ids | |
| 27 | + | } | |
| 28 | + | ||
| 29 | + | func countActive(scheduler *Scheduler, ids []PhysicalStreamID) int { | |
| 30 | + | active := 0 | |
| 31 | + | for _, id := range ids { | |
| 32 | + | if scheduler.QueuedFrames(id) > 0 { | |
| 33 | + | active++ | |
| 34 | + | } | |
| 35 | + | } | |
| 36 | + | return active | |
| 37 | + | } | |
| 38 | + | ||
| 39 | + | // TestSchedulerConstants pins the negotiated bounds the scheduler is specified | |
| 40 | + | // to enforce. | |
| 41 | + | func TestSchedulerConstants(t *testing.T) { | |
| 42 | + | require.Equal(t, 8, MaxMuxStreamQueueChunks) | |
| 43 | + | require.Equal(t, 32<<20, MaxMuxStreamQueueBytes) | |
| 44 | + | require.Equal(t, 64<<20, int(MaxMuxAggregateBytes)) | |
| 45 | + | require.Equal(t, int64(64<<10), SchedulerQuantum) | |
| 46 | + | require.Equal(t, 4, MaxMuxSchedulerControlBurst) | |
| 47 | + | } | |
| 48 | + | ||
| 49 | + | func TestSchedulerStrings(t *testing.T) { | |
| 50 | + | require.Equal(t, "data", FrameData.String()) | |
| 51 | + | require.Equal(t, "control", FrameControl.String()) | |
| 52 | + | require.Equal(t, "unknown", FrameClass(0).String()) | |
| 53 | + | require.Equal(t, "client", DirectionClient.String()) | |
| 54 | + | require.Equal(t, "server", DirectionServer.String()) | |
| 55 | + | require.Equal(t, "unknown", EnvelopeDirection(0).String()) | |
| 56 | + | } | |
| 57 | + | ||
| 58 | + | func TestSchedulerNilSafety(t *testing.T) { | |
| 59 | + | var scheduler *Scheduler | |
| 60 | + | _, ok := scheduler.Dequeue() | |
| 61 | + | require.False(t, ok) | |
| 62 | + | require.False(t, scheduler.Pending()) | |
| 63 | + | require.Zero(t, scheduler.Active()) | |
| 64 | + | require.Zero(t, scheduler.AggregateBytes()) | |
| 65 | + | require.Zero(t, scheduler.QueuedFrames(1)) | |
| 66 | + | require.Zero(t, scheduler.QueuedBytes(1)) | |
| 67 | + | require.False(t, scheduler.Reset(1)) | |
| 68 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: 1, Data: []byte("x")}), ErrInvalidMessage) | |
| 69 | + | require.ErrorIs(t, scheduler.EnqueueServer(Opened{Ref: testRef(1)}), ErrInvalidMessage) | |
| 70 | + | } | |
| 71 | + | ||
| 72 | + | // TestSchedulerAdmissionGate proves a frame is schedulable only for a stream the | |
| 73 | + | // gate admitted and has not terminalized, and that a nil gate trusts the | |
| 74 | + | // caller. | |
| 75 | + | func TestSchedulerAdmissionGate(t *testing.T) { | |
| 76 | + | t.Run("engine gate", func(t *testing.T) { | |
| 77 | + | engine := testEngine() | |
| 78 | + | scheduler := NewScheduler(engine) | |
| 79 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: 9, Data: []byte("x")}), ErrSchedulerUnadmitted) | |
| 80 | + | ||
| 81 | + | require.NoError(t, engine.Open(Open{Ref: testRef(9)})) | |
| 82 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: 9, Data: []byte("x")})) | |
| 83 | + | ||
| 84 | + | require.NoError(t, engine.Opened(Opened{Ref: testRef(9)})) | |
| 85 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: 9, Data: []byte("y")})) | |
| 86 | + | ||
| 87 | + | _, err := engine.Close(Close{Physical: 9}) | |
| 88 | + | require.NoError(t, err) | |
| 89 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: 9, Data: []byte("z")}), ErrSchedulerUnadmitted) | |
| 90 | + | }) | |
| 91 | + | ||
| 92 | + | t.Run("nil gate", func(t *testing.T) { | |
| 93 | + | scheduler := NewScheduler(nil) | |
| 94 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: 7, Data: []byte("x")})) | |
| 95 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: 0, Data: []byte("x")}), ErrInvalidMessage) | |
| 96 | + | }) | |
| 97 | + | ||
| 98 | + | t.Run("unknown variant fails closed", func(t *testing.T) { | |
| 99 | + | scheduler := NewScheduler(nil) | |
| 100 | + | require.ErrorIs(t, scheduler.EnqueueClient(unknownClientMessage{}), ErrWrongDirection) | |
| 101 | + | require.ErrorIs(t, scheduler.EnqueueServer(unknownServerMessage{}), ErrWrongDirection) | |
| 102 | + | }) | |
| 103 | + | } | |
| 104 | + | ||
| 105 | + | // TestSchedulerEnvelopeRoundTrip proves a dequeued envelope carries the | |
| 106 | + | // complete serialized directional envelope, decoded by the matching codec, and | |
| 107 | + | // that neither the bytes nor the committed message aliases caller memory. | |
| 108 | + | func TestSchedulerEnvelopeRoundTrip(t *testing.T) { | |
| 109 | + | ceiling := DefaultMuxCeilings() | |
| 110 | + | _, scheduler, ids := testScheduler(t, 1) | |
| 111 | + | ||
| 112 | + | payload := []byte("payload") | |
| 113 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: payload})) | |
| 114 | + | payload[0] = 'X' // a caller mutation must never reach the queued frame | |
| 115 | + | ||
| 116 | + | envelope, ok := scheduler.Dequeue() | |
| 117 | + | require.True(t, ok) | |
| 118 | + | require.Equal(t, ids[0], envelope.Physical) | |
| 119 | + | require.Equal(t, DirectionClient, envelope.Direction) | |
| 120 | + | require.Equal(t, FrameData, envelope.Class) | |
| 121 | + | require.Nil(t, envelope.Server) | |
| 122 | + | data, ok := envelope.Client.(Data) | |
| 123 | + | require.True(t, ok) | |
| 124 | + | require.Equal(t, []byte("payload"), data.Data) | |
| 125 | + | ||
| 126 | + | decoded, err := DecodeClient(envelope.Bytes, ceiling.MaxReceiveEnvelopeBytes, ceiling.StreamChunkLimit) | |
| 127 | + | require.NoError(t, err) | |
| 128 | + | require.Equal(t, Data{Physical: ids[0], Data: []byte("payload")}, decoded) | |
| 129 | + | ||
| 130 | + | require.NoError(t, scheduler.EnqueueServer(Opened{Ref: testRef(ids[0])})) | |
| 131 | + | serverEnvelope, ok := scheduler.Dequeue() | |
| 132 | + | require.True(t, ok) | |
| 133 | + | require.Equal(t, DirectionServer, serverEnvelope.Direction) | |
| 134 | + | require.Equal(t, FrameControl, serverEnvelope.Class) | |
| 135 | + | require.Nil(t, serverEnvelope.Client) | |
| 136 | + | decodedServer, err := DecodeServer(serverEnvelope.Bytes, ceiling.MaxReceiveEnvelopeBytes, ceiling.StreamChunkLimit) | |
| 137 | + | require.NoError(t, err) | |
| 138 | + | require.Equal(t, Opened{Ref: testRef(ids[0])}, decodedServer) | |
| 139 | + | } | |
| 140 | + | ||
| 141 | + | // TestSchedulerTwoAndHundredStreamsFairness proves deterministic round-robin | |
| 142 | + | // over a small and a large concurrent set: every stream is served once per | |
| 143 | + | // round, in admission order, with its own frames in order. | |
| 144 | + | func TestSchedulerTwoAndHundredStreamsFairness(t *testing.T) { | |
| 145 | + | for _, count := range []int{2, 100} { | |
| 146 | + | t.Run(fmt.Sprintf("%d streams", count), func(t *testing.T) { | |
| 147 | + | _, scheduler, ids := testScheduler(t, count) | |
| 148 | + | const rounds = 2 | |
| 149 | + | for round := 0; round < rounds; round++ { | |
| 150 | + | for _, id := range ids { | |
| 151 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte{byte(id), byte(round)}})) | |
| 152 | + | } | |
| 153 | + | } | |
| 154 | + | require.Equal(t, count, scheduler.Active()) | |
| 155 | + | ||
| 156 | + | seen := make(map[PhysicalStreamID]int, count) | |
| 157 | + | for served := 0; served < rounds*count; served++ { | |
| 158 | + | envelope, ok := scheduler.Dequeue() | |
| 159 | + | require.True(t, ok, "dequeue %d", served) | |
| 160 | + | want := ids[served%count] | |
| 161 | + | require.Equal(t, want, envelope.Physical, "dequeue %d", served) | |
| 162 | + | require.Equal(t, FrameData, envelope.Class) | |
| 163 | + | require.Equal(t, DirectionClient, envelope.Direction) | |
| 164 | + | data, ok := envelope.Client.(Data) | |
| 165 | + | require.True(t, ok) | |
| 166 | + | require.Equal(t, []byte{byte(want), byte(seen[want])}, data.Data, "per-stream order") | |
| 167 | + | seen[want]++ | |
| 168 | + | } | |
| 169 | + | require.False(t, scheduler.Pending()) | |
| 170 | + | require.Zero(t, scheduler.AggregateBytes()) | |
| 171 | + | }) | |
| 172 | + | } | |
| 173 | + | } | |
| 174 | + | ||
| 175 | + | // TestSchedulerStalledAndHotStream proves a continuously backlogged hot stream | |
| 176 | + | // never starves a sibling, and a full stalled backlog never blocks another | |
| 177 | + | // stream: every stream is served once per round while all queues stay full. | |
| 178 | + | func TestSchedulerStalledAndHotStream(t *testing.T) { | |
| 179 | + | _, scheduler, ids := testScheduler(t, 4) | |
| 180 | + | ||
| 181 | + | // The hot stream keeps a maximal backlog; the siblings are refilled to full | |
| 182 | + | // whenever the round drains a slot. | |
| 183 | + | fill := func(id PhysicalStreamID, seed byte) { | |
| 184 | + | for scheduler.QueuedFrames(id) < MaxMuxStreamQueueChunks { | |
| 185 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte{seed}})) | |
| 186 | + | } | |
| 187 | + | } | |
| 188 | + | ||
| 189 | + | const iterations = 400 | |
| 190 | + | served := make(map[PhysicalStreamID]int, len(ids)) | |
| 191 | + | for i := 0; i < iterations; i++ { | |
| 192 | + | fill(ids[0], byte(i)) | |
| 193 | + | for _, id := range ids[1:] { | |
| 194 | + | fill(id, byte(i)) | |
| 195 | + | } | |
| 196 | + | envelope, ok := scheduler.Dequeue() | |
| 197 | + | require.True(t, ok, "iteration %d", i) | |
| 198 | + | served[envelope.Physical]++ | |
| 199 | + | } | |
| 200 | + | ||
| 201 | + | for _, id := range ids { | |
| 202 | + | require.Greater(t, served[id], 0, "stream %d starved", id) | |
| 203 | + | } | |
| 204 | + | fair := iterations / (len(ids) + 1) | |
| 205 | + | for _, id := range ids { | |
| 206 | + | require.GreaterOrEqual(t, served[id], fair, "stream %d fell behind the hot stream", id) | |
| 207 | + | } | |
| 208 | + | } | |
| 209 | + | ||
| 210 | + | // TestSchedulerCloseOrdering proves a Close never overtakes earlier accepted | |
| 211 | + | // data of its own stream while control metadata still gets bounded priority | |
| 212 | + | // over another stream's data. | |
| 213 | + | func TestSchedulerCloseOrdering(t *testing.T) { | |
| 214 | + | _, scheduler, ids := testScheduler(t, 2) | |
| 215 | + | ||
| 216 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: []byte("a")})) | |
| 217 | + | require.NoError(t, scheduler.EnqueueClient(Close{Physical: ids[1]})) | |
| 218 | + | ||
| 219 | + | first, ok := scheduler.Dequeue() | |
| 220 | + | require.True(t, ok) | |
| 221 | + | require.Equal(t, ids[1], first.Physical, "control metadata takes bounded priority") | |
| 222 | + | require.Equal(t, FrameControl, first.Class) | |
| 223 | + | _, isClose := first.Client.(Close) | |
| 224 | + | require.True(t, isClose) | |
| 225 | + | ||
| 226 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: []byte("b")})) | |
| 227 | + | require.NoError(t, scheduler.EnqueueClient(Close{Physical: ids[0]})) | |
| 228 | + | ||
| 229 | + | var order []string | |
| 230 | + | for scheduler.Pending() { | |
| 231 | + | envelope, ok := scheduler.Dequeue() | |
| 232 | + | require.True(t, ok) | |
| 233 | + | if envelope.Physical != ids[0] { | |
| 234 | + | continue | |
| 235 | + | } | |
| 236 | + | switch message := envelope.Client.(type) { | |
| 237 | + | case Data: | |
| 238 | + | order = append(order, string(message.Data)) | |
| 239 | + | case Close: | |
| 240 | + | order = append(order, "close") | |
| 241 | + | default: | |
| 242 | + | t.Fatalf("unexpected message %T", message) | |
| 243 | + | } | |
| 244 | + | } | |
| 245 | + | require.Equal(t, []string{"a", "b", "close"}, order) | |
| 246 | + | ||
| 247 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: []byte("c")}), ErrSchedulerSettled) | |
| 248 | + | require.ErrorIs(t, scheduler.EnqueueClient(Close{Physical: ids[0]}), ErrSchedulerSettled) | |
| 249 | + | } | |
| 250 | + | ||
| 251 | + | // TestSchedulerControlPriorityBounded proves control frames are prioritized | |
| 252 | + | // over data but only for a bounded burst, so control metadata can never starve | |
| 253 | + | // data. | |
| 254 | + | func TestSchedulerControlPriorityBounded(t *testing.T) { | |
| 255 | + | _, scheduler, ids := testScheduler(t, 5) | |
| 256 | + | const backlog = 8 | |
| 257 | + | for round := 0; round < backlog; round++ { | |
| 258 | + | for _, id := range ids[:4] { | |
| 259 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte{byte(id), byte(round)}})) | |
| 260 | + | } | |
| 261 | + | require.NoError(t, scheduler.EnqueueServer(Opened{Ref: testRef(ids[4])})) | |
| 262 | + | } | |
| 263 | + | ||
| 264 | + | dataServed := make(map[PhysicalStreamID]int, 4) | |
| 265 | + | controls := 0 | |
| 266 | + | run := 0 | |
| 267 | + | for scheduler.Pending() { | |
| 268 | + | envelope, ok := scheduler.Dequeue() | |
| 269 | + | require.True(t, ok) | |
| 270 | + | if envelope.Class == FrameControl { | |
| 271 | + | require.Equal(t, ids[4], envelope.Physical) | |
| 272 | + | controls++ | |
| 273 | + | run++ | |
| 274 | + | require.LessOrEqual(t, run, MaxMuxSchedulerControlBurst, "control burst exceeded") | |
| 275 | + | continue | |
| 276 | + | } | |
| 277 | + | run = 0 | |
| 278 | + | dataServed[envelope.Physical]++ | |
| 279 | + | } | |
| 280 | + | require.Equal(t, backlog, controls) | |
| 281 | + | for _, id := range ids[:4] { | |
| 282 | + | require.Equal(t, backlog, dataServed[id], "stream %d starved by control", id) | |
| 283 | + | } | |
| 284 | + | } | |
| 285 | + | ||
| 286 | + | // TestSchedulerResetDiscard proves a reset discards exactly its own queue, | |
| 287 | + | // frees its bytes immediately, is idempotent, and never affects a sibling. | |
| 288 | + | func TestSchedulerResetDiscard(t *testing.T) { | |
| 289 | + | engine, scheduler, ids := testScheduler(t, 2) | |
| 290 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: []byte("one")})) | |
| 291 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: []byte("two")})) | |
| 292 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[1], Data: []byte("sibling")})) | |
| 293 | + | ||
| 294 | + | siblingBytes := scheduler.QueuedBytes(ids[1]) | |
| 295 | + | require.Greater(t, scheduler.QueuedBytes(ids[0]), 0) | |
| 296 | + | ||
| 297 | + | require.True(t, scheduler.Reset(ids[0])) | |
| 298 | + | require.Zero(t, scheduler.QueuedFrames(ids[0])) | |
| 299 | + | require.Zero(t, scheduler.QueuedBytes(ids[0])) | |
| 300 | + | require.Equal(t, siblingBytes, scheduler.AggregateBytes(), "the sibling keeps its budget") | |
| 301 | + | require.Equal(t, 1, scheduler.Active()) | |
| 302 | + | ||
| 303 | + | require.False(t, scheduler.Reset(ids[0]), "cancel is idempotent") | |
| 304 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: []byte("later")}), ErrSchedulerSettled) | |
| 305 | + | ||
| 306 | + | envelope, ok := scheduler.Dequeue() | |
| 307 | + | require.True(t, ok) | |
| 308 | + | require.Equal(t, ids[1], envelope.Physical) | |
| 309 | + | require.Zero(t, scheduler.AggregateBytes()) | |
| 310 | + | ||
| 311 | + | require.False(t, scheduler.Reset(99), "an unadmitted stream is a no-op") | |
| 312 | + | ||
| 313 | + | openTestStream(t, engine, 3) | |
| 314 | + | require.True(t, scheduler.Reset(3), "an admitted but idle stream settles") | |
| 315 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: 3, Data: []byte("x")}), ErrSchedulerSettled) | |
| 316 | + | } | |
| 317 | + | ||
| 318 | + | // TestSchedulerAggregateAndPerStreamCaps proves each bound discards exactly the | |
| 319 | + | // offending stream and leaves its siblings' budget intact. | |
| 320 | + | func TestSchedulerAggregateAndPerStreamCaps(t *testing.T) { | |
| 321 | + | t.Run("per-stream frame bound", func(t *testing.T) { | |
| 322 | + | _, scheduler, ids := testScheduler(t, 2) | |
| 323 | + | for i := 0; i < MaxMuxStreamQueueChunks; i++ { | |
| 324 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: []byte{byte(i)}})) | |
| 325 | + | } | |
| 326 | + | require.Equal(t, MaxMuxStreamQueueChunks, scheduler.QueuedFrames(ids[0])) | |
| 327 | + | ||
| 328 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: []byte("x")}), ErrSchedulerFull) | |
| 329 | + | require.Zero(t, scheduler.QueuedFrames(ids[0])) | |
| 330 | + | require.Zero(t, scheduler.QueuedBytes(ids[0])) | |
| 331 | + | // The overflowed record is not yet sealed: data stays refused with the | |
| 332 | + | // overflow sentinel until its one terminal Close or Reset is enqueued. | |
| 333 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: []byte("y")}), ErrSchedulerFull) | |
| 334 | + | ||
| 335 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[1], Data: []byte("sibling")})) | |
| 336 | + | require.Equal(t, scheduler.QueuedBytes(ids[1]), scheduler.AggregateBytes()) | |
| 337 | + | }) | |
| 338 | + | ||
| 339 | + | t.Run("per-stream byte bound", func(t *testing.T) { | |
| 340 | + | _, scheduler, ids := testScheduler(t, 1) | |
| 341 | + | payload := make([]byte, 300) | |
| 342 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: payload})) | |
| 343 | + | unit := scheduler.QueuedBytes(ids[0]) | |
| 344 | + | require.Positive(t, unit) | |
| 345 | + | ||
| 346 | + | scheduler.maxStreamFrames = 100 | |
| 347 | + | scheduler.maxStreamBytes = unit * 3 | |
| 348 | + | for i := 0; i < 2; i++ { | |
| 349 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: payload})) | |
| 350 | + | } | |
| 351 | + | require.Equal(t, unit*3, scheduler.QueuedBytes(ids[0])) | |
| 352 | + | ||
| 353 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: payload}), ErrSchedulerFull) | |
| 354 | + | require.Zero(t, scheduler.AggregateBytes()) | |
| 355 | + | }) | |
| 356 | + | ||
| 357 | + | t.Run("aggregate byte bound", func(t *testing.T) { | |
| 358 | + | _, scheduler, ids := testScheduler(t, 3) | |
| 359 | + | payload := make([]byte, 300) | |
| 360 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: payload})) | |
| 361 | + | unit := scheduler.QueuedBytes(ids[0]) | |
| 362 | + | require.Positive(t, unit) | |
| 363 | + | ||
| 364 | + | scheduler.maxStreamFrames = 100 | |
| 365 | + | scheduler.maxStreamBytes = unit * 100 | |
| 366 | + | scheduler.maxAggregateBytes = unit * 3 | |
| 367 | + | ||
| 368 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: payload})) | |
| 369 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[1], Data: payload})) | |
| 370 | + | require.Equal(t, unit*3, scheduler.AggregateBytes()) | |
| 371 | + | ||
| 372 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: ids[2], Data: payload}), ErrSchedulerFull) | |
| 373 | + | require.Zero(t, scheduler.QueuedBytes(ids[2])) | |
| 374 | + | require.Equal(t, unit*3, scheduler.AggregateBytes(), "siblings keep their budget") | |
| 375 | + | require.Equal(t, 2, scheduler.Active()) | |
| 376 | + | }) | |
| 377 | + | } | |
| 378 | + | ||
| 379 | + | // TestSchedulerConcurrentEnqueueDequeueReset exercises every scheduler method | |
| 380 | + | // from many goroutines so the race detector can prove the shared state is | |
| 381 | + | // serialized and the accounting stays consistent. | |
| 382 | + | func TestSchedulerConcurrentEnqueueDequeueReset(t *testing.T) { | |
| 383 | + | const streams = 8 | |
| 384 | + | _, scheduler, ids := testScheduler(t, streams) | |
| 385 | + | ||
| 386 | + | const ( | |
| 387 | + | producers = 4 | |
| 388 | + | perProducer = 300 | |
| 389 | + | consumers = 2 | |
| 390 | + | perConsumer = 600 | |
| 391 | + | resets = 50 | |
| 392 | + | ) | |
| 393 | + | ||
| 394 | + | var wg sync.WaitGroup | |
| 395 | + | wg.Add(producers + consumers + 1) | |
| 396 | + | for producer := 0; producer < producers; producer++ { | |
| 397 | + | go func(producer int) { | |
| 398 | + | defer wg.Done() | |
| 399 | + | for i := 0; i < perProducer; i++ { | |
| 400 | + | id := ids[(producer+i)%streams] | |
| 401 | + | _ = scheduler.EnqueueClient(Data{Physical: id, Data: []byte{byte(producer), byte(i)}}) | |
| 402 | + | } | |
| 403 | + | }(producer) | |
| 404 | + | } | |
| 405 | + | for consumer := 0; consumer < consumers; consumer++ { | |
| 406 | + | go func() { | |
| 407 | + | defer wg.Done() | |
| 408 | + | for i := 0; i < perConsumer; i++ { | |
| 409 | + | _, _ = scheduler.Dequeue() | |
| 410 | + | } | |
| 411 | + | }() | |
| 412 | + | } | |
| 413 | + | go func() { | |
| 414 | + | defer wg.Done() | |
| 415 | + | for i := 0; i < resets; i++ { | |
| 416 | + | _ = scheduler.Reset(ids[i%streams]) | |
| 417 | + | } | |
| 418 | + | }() | |
| 419 | + | wg.Wait() | |
| 420 | + | ||
| 421 | + | require.GreaterOrEqual(t, scheduler.AggregateBytes(), 0) | |
| 422 | + | require.LessOrEqual(t, scheduler.AggregateBytes(), int(MaxMuxAggregateBytes)) | |
| 423 | + | ||
| 424 | + | total := 0 | |
| 425 | + | for _, id := range ids { | |
| 426 | + | total += scheduler.QueuedBytes(id) | |
| 427 | + | } | |
| 428 | + | require.Equal(t, total, scheduler.AggregateBytes(), "aggregate accounting is exact") | |
| 429 | + | require.Equal(t, countActive(scheduler, ids), scheduler.Active()) | |
| 430 | + | } | |
| 431 | + | ||
| 432 | + | // BenchmarkSchedulerEnqueueDequeue measures the steady-state cost of encoding, | |
| 433 | + | // enqueuing, and dequeuing a single-stream data frame. | |
| 434 | + | func BenchmarkSchedulerEnqueueDequeue(b *testing.B) { | |
| 435 | + | engine := NewStreamEngine() | |
| 436 | + | if err := engine.Open(Open{Ref: testRef(1)}); err != nil { | |
| 437 | + | b.Fatal(err) | |
| 438 | + | } | |
| 439 | + | if err := engine.Opened(Opened{Ref: testRef(1)}); err != nil { | |
| 440 | + | b.Fatal(err) | |
| 441 | + | } | |
| 442 | + | scheduler := NewScheduler(engine) | |
| 443 | + | payload := make([]byte, 256) | |
| 444 | + | ||
| 445 | + | b.ReportAllocs() | |
| 446 | + | b.ResetTimer() | |
| 447 | + | for i := 0; i < b.N; i++ { | |
| 448 | + | if err := scheduler.EnqueueClient(Data{Physical: 1, Data: payload}); err != nil { | |
| 449 | + | b.Fatal(err) | |
| 450 | + | } | |
| 451 | + | if _, ok := scheduler.Dequeue(); !ok { | |
| 452 | + | b.Fatal("scheduler empty") | |
| 453 | + | } | |
| 454 | + | } | |
| 455 | + | } | |
| 456 | + | ||
| 457 | + | // BenchmarkSchedulerEnqueueDequeueManyStreams measures the same path across a | |
| 458 | + | // full active ring. | |
| 459 | + | func BenchmarkSchedulerEnqueueDequeueManyStreams(b *testing.B) { | |
| 460 | + | const streams = 32 | |
| 461 | + | engine := NewStreamEngine() | |
| 462 | + | for i := 1; i <= streams; i++ { | |
| 463 | + | ref := testRef(PhysicalStreamID(i)) | |
| 464 | + | if err := engine.Open(Open{Ref: ref}); err != nil { | |
| 465 | + | b.Fatal(err) | |
| 466 | + | } | |
| 467 | + | if err := engine.Opened(Opened{Ref: ref}); err != nil { | |
| 468 | + | b.Fatal(err) | |
| 469 | + | } | |
| 470 | + | } | |
| 471 | + | scheduler := NewScheduler(engine) | |
| 472 | + | payload := make([]byte, 256) | |
| 473 | + | ||
| 474 | + | b.ReportAllocs() | |
| 475 | + | b.ResetTimer() | |
| 476 | + | for i := 0; i < b.N; i++ { | |
| 477 | + | id := PhysicalStreamID(i%streams + 1) | |
| 478 | + | if err := scheduler.EnqueueClient(Data{Physical: id, Data: payload}); err != nil { | |
| 479 | + | b.Fatal(err) | |
| 480 | + | } | |
| 481 | + | if _, ok := scheduler.Dequeue(); !ok { | |
| 482 | + | b.Fatal("scheduler empty") | |
| 483 | + | } | |
| 484 | + | } | |
| 485 | + | } | |
| 486 | + | ||
| 487 | + | // dequeueOrTimeout dequeues one frame, failing the test instead of hanging if | |
| 488 | + | // the DRR ring stalls on a frame larger than the quantum. | |
| 489 | + | func dequeueOrTimeout(t *testing.T, scheduler *Scheduler) (OutboundEnvelope, bool) { | |
| 490 | + | t.Helper() | |
| 491 | + | type result struct { | |
| 492 | + | envelope OutboundEnvelope | |
| 493 | + | ok bool | |
| 494 | + | } | |
| 495 | + | done := make(chan result, 1) | |
| 496 | + | go func() { | |
| 497 | + | envelope, ok := scheduler.Dequeue() | |
| 498 | + | done <- result{envelope: envelope, ok: ok} | |
| 499 | + | }() | |
| 500 | + | select { | |
| 501 | + | case r := <-done: | |
| 502 | + | return r.envelope, r.ok | |
| 503 | + | case <-time.After(5 * time.Second): | |
| 504 | + | t.Fatal("Dequeue made no progress: the DRR ring stalled") | |
| 505 | + | return OutboundEnvelope{}, false | |
| 506 | + | } | |
| 507 | + | } | |
| 508 | + | ||
| 509 | + | // maxEnvOpen builds one Open whose environment fills the per-request bound, so | |
| 510 | + | // its serialized envelope is far larger than the 64 KiB quantum. | |
| 511 | + | func maxEnvOpen(t *testing.T, id PhysicalStreamID) Open { | |
| 512 | + | t.Helper() | |
| 513 | + | env := make([]string, ports.BrokerMaxEnvEntries) | |
| 514 | + | for i := range env { | |
| 515 | + | prefix := fmt.Sprintf("K%d=", i) | |
| 516 | + | env[i] = prefix + strings.Repeat("v", ports.BrokerMaxEnvEntryBytes-len(prefix)) | |
| 517 | + | } | |
| 518 | + | open := testOpen() | |
| 519 | + | open.Ref = testRef(id) | |
| 520 | + | open.Env = env | |
| 521 | + | return open | |
| 522 | + | } | |
| 523 | + | ||
| 524 | + | // TestNewSchedulerWithCeilings proves the constructor validates the negotiated | |
| 525 | + | // advertisement, enforces the negotiated chunk and aggregate ceilings, and | |
| 526 | + | // copies the value so a caller cannot mutate live enforcement. | |
| 527 | + | func TestNewSchedulerWithCeilings(t *testing.T) { | |
| 528 | + | t.Run("invalid advertisement refused", func(t *testing.T) { | |
| 529 | + | bad := DefaultMuxCeilings() | |
| 530 | + | bad.StreamChunkLimit = 0 | |
| 531 | + | scheduler, err := NewSchedulerWithCeilings(nil, bad) | |
| 532 | + | require.ErrorIs(t, err, ErrInvalidCeilings) | |
| 533 | + | require.Nil(t, scheduler) | |
| 534 | + | _, err = NewSchedulerWithCeilings(nil, MuxCeilings{}) | |
| 535 | + | require.ErrorIs(t, err, ErrInvalidCeilings) | |
| 536 | + | }) | |
| 537 | + | ||
| 538 | + | t.Run("negotiated chunk and aggregate limits enforced", func(t *testing.T) { | |
| 539 | + | probeCeiling := MuxCeilings{ | |
| 540 | + | MaxReceiveEnvelopeBytes: MinMuxEnvelopeBytes, | |
| 541 | + | StreamChunkLimit: 8, | |
| 542 | + | MaxStreams: 1, | |
| 543 | + | MaxAggregateBytes: MinMuxAggregateBytes, | |
| 544 | + | } | |
| 545 | + | probe, err := NewSchedulerWithCeilings(nil, probeCeiling) | |
| 546 | + | require.NoError(t, err) | |
| 547 | + | require.NoError(t, probe.EnqueueClient(Data{Physical: 1, Data: make([]byte, 8)})) | |
| 548 | + | unit := probe.QueuedBytes(1) | |
| 549 | + | require.Positive(t, unit) | |
| 550 | + | ||
| 551 | + | // The aggregate floor admits exactly one maximum chunk plus envelope | |
| 552 | + | // overhead. | |
| 553 | + | floorCeiling := probeCeiling | |
| 554 | + | floorCeiling.StreamChunkLimit = MaxMuxChunkBytes | |
| 555 | + | floor, err := NewSchedulerWithCeilings(nil, floorCeiling) | |
| 556 | + | require.NoError(t, err) | |
| 557 | + | require.NoError(t, floor.EnqueueClient(Data{Physical: 1, Data: make([]byte, MaxMuxChunkBytes)})) | |
| 558 | + | require.Equal(t, 1, floor.Active()) | |
| 559 | + | ||
| 560 | + | // A scheduler tightened below the negotiated aggregate discards exactly | |
| 561 | + | // the offending stream and frees its budget. | |
| 562 | + | scheduler, err := NewSchedulerWithCeilings(nil, probeCeiling) | |
| 563 | + | require.NoError(t, err) | |
| 564 | + | scheduler.maxStreamFrames = 100 | |
| 565 | + | scheduler.maxStreamBytes = unit * 100 | |
| 566 | + | scheduler.maxAggregateBytes = unit * 2 | |
| 567 | + | // The negotiated chunk ceiling is refused statelessly. | |
| 568 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: 1, Data: make([]byte, 9)}), ErrTooLarge) | |
| 569 | + | require.Zero(t, scheduler.AggregateBytes()) | |
| 570 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: 1, Data: make([]byte, 8)})) | |
| 571 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: 1, Data: make([]byte, 8)})) | |
| 572 | + | require.Equal(t, unit*2, scheduler.AggregateBytes()) | |
| 573 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: 1, Data: make([]byte, 8)}), ErrSchedulerFull) | |
| 574 | + | require.Zero(t, scheduler.AggregateBytes()) | |
| 575 | + | }) | |
| 576 | + | } | |
| 577 | + | ||
| 578 | + | // TestSchedulerTerminalBypass proves exactly one terminal Close or Reset is | |
| 579 | + | // accepted for a known identity the engine has already settled, that the | |
| 580 | + | // terminal frame bypasses normal admission, that a duplicate terminal is | |
| 581 | + | // refused, and that data stays refused. | |
| 582 | + | func TestSchedulerTerminalBypass(t *testing.T) { | |
| 583 | + | t.Run("settled identity signals once", func(t *testing.T) { | |
| 584 | + | engine := testEngine() | |
| 585 | + | scheduler := NewScheduler(engine) | |
| 586 | + | openTestStream(t, engine, 1) | |
| 587 | + | _, err := engine.Reset(Reset{Physical: 1}) | |
| 588 | + | require.NoError(t, err) | |
| 589 | + | ||
| 590 | + | // Data for the settled, unadmitted identity stays refused. | |
| 591 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: 1, Data: []byte("x")}), ErrSchedulerUnadmitted) | |
| 592 | + | // Exactly one terminal frame is accepted so the peer can be signaled. | |
| 593 | + | require.NoError(t, scheduler.EnqueueClient(Reset{Physical: 1})) | |
| 594 | + | envelope, ok := dequeueOrTimeout(t, scheduler) | |
| 595 | + | require.True(t, ok) | |
| 596 | + | require.Equal(t, PhysicalStreamID(1), envelope.Physical) | |
| 597 | + | require.Equal(t, FrameControl, envelope.Class) | |
| 598 | + | _, isReset := envelope.Client.(Reset) | |
| 599 | + | require.True(t, isReset) | |
| 600 | + | ||
| 601 | + | // The duplicate terminal, and every later data frame, is refused. | |
| 602 | + | require.ErrorIs(t, scheduler.EnqueueClient(Reset{Physical: 1}), ErrSchedulerSettled) | |
| 603 | + | require.ErrorIs(t, scheduler.EnqueueClient(Close{Physical: 1}), ErrSchedulerSettled) | |
| 604 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: 1, Data: []byte("y")}), ErrSchedulerSettled) | |
| 605 | + | // An identity the engine never admitted is still refused. | |
| 606 | + | require.ErrorIs(t, scheduler.EnqueueClient(Close{Physical: 9}), ErrSchedulerUnadmitted) | |
| 607 | + | }) | |
| 608 | + | ||
| 609 | + | t.Run("queue overflow signals the peer", func(t *testing.T) { | |
| 610 | + | engine := testEngine() | |
| 611 | + | scheduler := NewScheduler(engine) | |
| 612 | + | openTestStream(t, engine, 1) | |
| 613 | + | for i := 0; i < MaxMuxStreamQueueChunks; i++ { | |
| 614 | + | _, err := engine.Data(Data{Physical: 1, Data: []byte{byte(i)}}) | |
| 615 | + | require.NoError(t, err) | |
| 616 | + | } | |
| 617 | + | _, err := engine.Data(Data{Physical: 1, Data: []byte{'x'}}) | |
| 618 | + | require.ErrorIs(t, err, ErrStreamQueueFull) | |
| 619 | + | require.Equal(t, StreamTerminal, mustStatus(t, engine, 1).State) | |
| 620 | + | ||
| 621 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: 1, Data: []byte("x")}), ErrSchedulerUnadmitted) | |
| 622 | + | require.NoError(t, scheduler.EnqueueClient(Reset{Physical: 1})) | |
| 623 | + | _, ok := dequeueOrTimeout(t, scheduler) | |
| 624 | + | require.True(t, ok) | |
| 625 | + | require.ErrorIs(t, scheduler.EnqueueClient(Reset{Physical: 1}), ErrSchedulerSettled) | |
| 626 | + | }) | |
| 627 | + | ||
| 628 | + | t.Run("nil gate trusts one terminal", func(t *testing.T) { | |
| 629 | + | scheduler := NewScheduler(nil) | |
| 630 | + | require.NoError(t, scheduler.EnqueueClient(Close{Physical: 7})) | |
| 631 | + | require.ErrorIs(t, scheduler.EnqueueClient(Close{Physical: 7}), ErrSchedulerSettled) | |
| 632 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: 7, Data: []byte("x")}), ErrSchedulerSettled) | |
| 633 | + | }) | |
| 634 | + | } | |
| 635 | + | ||
| 636 | + | // TestSchedulerLargeFrameProgress is the regression for the stalled DRR ring: | |
| 637 | + | // the maximum 64 KiB chunk and a maximal-environment Open both make progress | |
| 638 | + | // without spinning. | |
| 639 | + | func TestSchedulerLargeFrameProgress(t *testing.T) { | |
| 640 | + | t.Run("maximum chunk fits the quantum", func(t *testing.T) { | |
| 641 | + | _, scheduler, ids := testScheduler(t, 1) | |
| 642 | + | chunk := make([]byte, MaxMuxChunkBytes) | |
| 643 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: ids[0], Data: chunk})) | |
| 644 | + | envelope, ok := dequeueOrTimeout(t, scheduler) | |
| 645 | + | require.True(t, ok) | |
| 646 | + | require.Equal(t, ids[0], envelope.Physical) | |
| 647 | + | require.False(t, scheduler.Pending()) | |
| 648 | + | }) | |
| 649 | + | ||
| 650 | + | t.Run("maximum environment Open exceeds the quantum", func(t *testing.T) { | |
| 651 | + | scheduler := NewScheduler(nil) | |
| 652 | + | const streams = 5 | |
| 653 | + | ids := make([]PhysicalStreamID, streams) | |
| 654 | + | for i := range ids { | |
| 655 | + | ids[i] = PhysicalStreamID(i + 1) | |
| 656 | + | require.NoError(t, scheduler.EnqueueClient(maxEnvOpen(t, ids[i]))) | |
| 657 | + | } | |
| 658 | + | seen := make(map[PhysicalStreamID]bool, streams) | |
| 659 | + | for i := 0; i < streams; i++ { | |
| 660 | + | envelope, ok := dequeueOrTimeout(t, scheduler) | |
| 661 | + | require.True(t, ok, "dequeue %d", i) | |
| 662 | + | open, isOpen := envelope.Client.(Open) | |
| 663 | + | require.True(t, isOpen, "dequeue %d carries %T", i, envelope.Client) | |
| 664 | + | require.Len(t, open.Env, ports.BrokerMaxEnvEntries) | |
| 665 | + | seen[envelope.Physical] = true | |
| 666 | + | } | |
| 667 | + | require.Len(t, seen, streams) | |
| 668 | + | require.False(t, scheduler.Pending()) | |
| 669 | + | }) | |
| 670 | + | } | |
| 671 | + | ||
| 672 | + | // TestSchedulerBoundedRetiredRecords proves a long-lived scheduler keeps a | |
| 673 | + | // bounded record table: the oldest settled records are evicted, while the | |
| 674 | + | // newest identity still refuses a duplicate terminal. | |
| 675 | + | func TestSchedulerBoundedRetiredRecords(t *testing.T) { | |
| 676 | + | scheduler := NewScheduler(nil) | |
| 677 | + | const churn = 4000 | |
| 678 | + | for i := 1; i <= churn; i++ { | |
| 679 | + | id := PhysicalStreamID(i) | |
| 680 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte{byte(i)}})) | |
| 681 | + | require.True(t, scheduler.Reset(id)) | |
| 682 | + | } | |
| 683 | + | require.LessOrEqual(t, len(scheduler.streams), MaxRetiredSchedulerRecords) | |
| 684 | + | require.Len(t, scheduler.retired, MaxRetiredSchedulerRecords) | |
| 685 | + | require.Zero(t, scheduler.Active()) | |
| 686 | + | require.Zero(t, scheduler.AggregateBytes()) | |
| 687 | + | ||
| 688 | + | newest := PhysicalStreamID(churn) | |
| 689 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: newest, Data: []byte("x")}), ErrSchedulerSettled) | |
| 690 | + | } | |
| 691 | + | ||
| 692 | + | // FuzzSchedulerEngineSmoke drives a scheduler over an engine gate with | |
| 693 | + | // arbitrary operation bytes and proves the pair never panics, keeps its | |
| 694 | + | // accounting exact, and stays bounded. | |
| 695 | + | func FuzzSchedulerEngineSmoke(f *testing.F) { | |
| 696 | + | f.Add([]byte{0, 1, 2, 3, 4, 5, 6, 7}) | |
| 697 | + | f.Add([]byte{1, 1, 1, 1, 1, 1, 1, 1}) | |
| 698 | + | f.Fuzz(func(t *testing.T, ops []byte) { | |
| 699 | + | engine := NewStreamEngine() | |
| 700 | + | scheduler := NewScheduler(engine) | |
| 701 | + | limit := len(ops) | |
| 702 | + | if limit > 256 { | |
| 703 | + | limit = 256 | |
| 704 | + | } | |
| 705 | + | for i := 0; i+1 < limit; i += 2 { | |
| 706 | + | id := PhysicalStreamID(ops[i]%8 + 1) | |
| 707 | + | switch ops[i+1] % 6 { | |
| 708 | + | case 0: | |
| 709 | + | _ = engine.Open(Open{Ref: testRef(id)}) | |
| 710 | + | case 1: | |
| 711 | + | _ = engine.Opened(Opened{Ref: testRef(id)}) | |
| 712 | + | case 2: | |
| 713 | + | _ = scheduler.EnqueueClient(Data{Physical: id, Data: []byte{ops[i]}}) | |
| 714 | + | case 3: | |
| 715 | + | _ = scheduler.EnqueueClient(Close{Physical: id}) | |
| 716 | + | case 4: | |
| 717 | + | _, _ = engine.Data(Data{Physical: id, Data: []byte{ops[i]}}) | |
| 718 | + | case 5: | |
| 719 | + | _ = scheduler.Reset(id) | |
| 720 | + | } | |
| 721 | + | } | |
| 722 | + | _, _ = scheduler.Dequeue() | |
| 723 | + | ||
| 724 | + | total := 0 | |
| 725 | + | for id := PhysicalStreamID(1); id <= 8; id++ { | |
| 726 | + | total += scheduler.QueuedBytes(id) | |
| 727 | + | } | |
| 728 | + | require.Equal(t, total, scheduler.AggregateBytes()) | |
| 729 | + | require.GreaterOrEqual(t, scheduler.AggregateBytes(), 0) | |
| 730 | + | require.LessOrEqual(t, len(scheduler.streams), int(MaxMuxStreams+MaxRetiredSchedulerRecords)) | |
| 731 | + | }) | |
| 732 | + | } | |
| 733 | + | ||
| 734 | + | // TestSchedulerOverflowAllowsOneTerminal proves an overflowed stream discards | |
| 735 | + | // its queue and refuses data, yet still accepts exactly one terminal Close or | |
| 736 | + | // Reset, sealing only after that terminal. | |
| 737 | + | func TestSchedulerOverflowAllowsOneTerminal(t *testing.T) { | |
| 738 | + | terminals := []struct { | |
| 739 | + | name string | |
| 740 | + | enqueue func(*Scheduler, PhysicalStreamID) error | |
| 741 | + | }{ | |
| 742 | + | {"reset", func(s *Scheduler, id PhysicalStreamID) error { return s.EnqueueClient(Reset{Physical: id}) }}, | |
| 743 | + | {"close", func(s *Scheduler, id PhysicalStreamID) error { return s.EnqueueClient(Close{Physical: id}) }}, | |
| 744 | + | } | |
| 745 | + | for _, terminal := range terminals { | |
| 746 | + | t.Run(terminal.name, func(t *testing.T) { | |
| 747 | + | _, scheduler, ids := testScheduler(t, 1) | |
| 748 | + | id := ids[0] | |
| 749 | + | scheduler.maxStreamFrames = 2 | |
| 750 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte("a")})) | |
| 751 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte("b")})) | |
| 752 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte("c")}), ErrSchedulerFull) | |
| 753 | + | require.Zero(t, scheduler.QueuedFrames(id)) | |
| 754 | + | require.Zero(t, scheduler.AggregateBytes()) | |
| 755 | + | require.False(t, scheduler.Pending()) | |
| 756 | + | ||
| 757 | + | // The record is overflowed but not sealed: data stays refused with | |
| 758 | + | // the overflow sentinel. | |
| 759 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte("d")}), ErrSchedulerFull) | |
| 760 | + | ||
| 761 | + | require.NoError(t, terminal.enqueue(scheduler, id)) | |
| 762 | + | require.Equal(t, 1, scheduler.QueuedFrames(id)) | |
| 763 | + | envelope, ok := scheduler.Dequeue() | |
| 764 | + | require.True(t, ok) | |
| 765 | + | require.Equal(t, id, envelope.Physical) | |
| 766 | + | require.Equal(t, FrameControl, envelope.Class) | |
| 767 | + | ||
| 768 | + | // The one terminal sealed the record: everything afterwards, | |
| 769 | + | // including a duplicate terminal, is refused. | |
| 770 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte("e")}), ErrSchedulerSettled) | |
| 771 | + | require.ErrorIs(t, terminal.enqueue(scheduler, id), ErrSchedulerSettled) | |
| 772 | + | require.False(t, scheduler.Reset(id)) | |
| 773 | + | }) | |
| 774 | + | } | |
| 775 | + | } | |
| 776 | + | ||
| 777 | + | // TestSchedulerOverflowRecordsBounded proves overflow records that never take | |
| 778 | + | // their terminal stay bounded: they are evicted oldest-first past | |
| 779 | + | // MaxRetiredSchedulerRecords. | |
| 780 | + | func TestSchedulerOverflowRecordsBounded(t *testing.T) { | |
| 781 | + | scheduler := NewScheduler(nil) | |
| 782 | + | scheduler.maxStreamFrames = 1 | |
| 783 | + | const churn = MaxRetiredSchedulerRecords * 4 | |
| 784 | + | for i := 1; i <= churn; i++ { | |
| 785 | + | id := PhysicalStreamID(i) | |
| 786 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte("a")})) | |
| 787 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte("b")}), ErrSchedulerFull) | |
| 788 | + | } | |
| 789 | + | require.LessOrEqual(t, len(scheduler.streams), MaxRetiredSchedulerRecords) | |
| 790 | + | require.Zero(t, scheduler.AggregateBytes()) | |
| 791 | + | require.False(t, scheduler.Pending()) | |
| 792 | + | } | |
| 793 | + | ||
| 794 | + | // TestSchedulerOverflowTerminalReclaimsRetiredRecords is the regression for a | |
| 795 | + | // record that is listed retired while empty, later accepts its one terminal | |
| 796 | + | // Reset while that terminal stays queued, and is then pushed past the | |
| 797 | + | // retention bound by later streams: eviction must not permanently orphan the | |
| 798 | + | // record. Every terminal stays deliverable, and draining them returns the | |
| 799 | + | // stream table to the configured bound. | |
| 800 | + | func TestSchedulerOverflowTerminalReclaimsRetiredRecords(t *testing.T) { | |
| 801 | + | scheduler := NewScheduler(nil) | |
| 802 | + | scheduler.maxStreamFrames = 1 | |
| 803 | + | const churn = MaxRetiredSchedulerRecords + 64 | |
| 804 | + | ||
| 805 | + | for i := 1; i <= churn; i++ { | |
| 806 | + | id := PhysicalStreamID(i) | |
| 807 | + | require.NoError(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte("a")})) | |
| 808 | + | // Overflow the queue so the record is retired while still empty. | |
| 809 | + | require.ErrorIs(t, scheduler.EnqueueClient(Data{Physical: id, Data: []byte("b")}), ErrSchedulerFull) | |
| 810 | + | // The one terminal the overflow leaves open lands while the record sits | |
| 811 | + | // in the retired list; later streams force it past the bound. | |
| 812 | + | require.NoError(t, scheduler.EnqueueClient(Reset{Physical: id})) | |
| 813 | + | } | |
| 814 | + | ||
| 815 | + | delivered := make(map[PhysicalStreamID]int, churn) | |
| 816 | + | for scheduler.Pending() { | |
| 817 | + | envelope, ok := scheduler.Dequeue() | |
| 818 | + | require.True(t, ok) | |
| 819 | + | _, isReset := envelope.Client.(Reset) | |
| 820 | + | require.True(t, isReset, "terminal frame must stay deliverable, got %T", envelope.Client) | |
| 821 | + | delivered[envelope.Physical]++ | |
| 822 | + | } | |
| 823 | + | require.Len(t, delivered, churn, "every overflowed stream signals its peer exactly once") | |
| 824 | + | for id, count := range delivered { | |
| 825 | + | require.Equal(t, 1, count, "stream %d delivered %d terminals", id, count) | |
| 826 | + | } | |
| 827 | + | ||
| 828 | + | require.Zero(t, scheduler.AggregateBytes()) | |
| 829 | + | require.Zero(t, scheduler.Active()) | |
| 830 | + | require.LessOrEqual(t, len(scheduler.streams), MaxRetiredSchedulerRecords, | |
| 831 | + | "drained overflow records must not be orphaned past the retention bound") | |
| 832 | + | require.LessOrEqual(t, len(scheduler.retired), MaxRetiredSchedulerRecords) | |
| 833 | + | ||
| 834 | + | // The newest identity still refuses a duplicate terminal, so the retained | |
| 835 | + | // record set is live, not merely truncated. | |
| 836 | + | require.ErrorIs(t, scheduler.EnqueueClient(Reset{Physical: PhysicalStreamID(churn)}), ErrSchedulerSettled) | |
| 837 | + | } | |
| 838 | + | ||
| 839 | + | // TestSchedulerRefuseAdmitsFreshRefusalOnce proves the fresh-refusal exception: | |
| 840 | + | // a Refused for an identity the engine never admitted is scheduled once and | |
| 841 | + | // seals the record, so a duplicate and every later frame are refused; an | |
| 842 | + | // unencodable refusal creates no record. | |
| 843 | + | func TestSchedulerRefuseAdmitsFreshRefusalOnce(t *testing.T) { | |
| 844 | + | s := NewScheduler(nil) | |
| 845 | + | detail := testErrorDetail() | |
| 846 | + | detail.AdmissionCode = 1 | |
| 847 | + | refused := Refused{Ref: testRef(1), Error: detail} | |
| 848 | + | ||
| 849 | + | require.NoError(t, s.Refuse(refused)) | |
| 850 | + | envelope, ok := s.Dequeue() | |
| 851 | + | require.True(t, ok) | |
| 852 | + | require.Equal(t, PhysicalStreamID(1), envelope.Physical) | |
| 853 | + | require.Equal(t, DirectionServer, envelope.Direction) | |
| 854 | + | require.Equal(t, FrameControl, envelope.Class) | |
| 855 | + | require.Equal(t, refused, envelope.Server) | |
| 856 | + | ||
| 857 | + | // The fresh refusal seals the identity even though the engine never | |
| 858 | + | // admitted the stream: a duplicate and every later frame are refused. | |
| 859 | + | require.ErrorIs(t, s.Refuse(refused), ErrSchedulerSettled) | |
| 860 | + | require.ErrorIs(t, s.EnqueueServer(Refused{Ref: testRef(1), Error: detail}), ErrSchedulerSettled) | |
| 861 | + | require.ErrorIs(t, s.EnqueueServer(Data{Physical: 1, Data: []byte("x")}), ErrSchedulerSettled) | |
| 862 | + | _, ok = s.Dequeue() | |
| 863 | + | require.False(t, ok) | |
| 864 | + | ||
| 865 | + | // A refusal whose reference cannot be encoded is rejected without creating | |
| 866 | + | // a record. | |
| 867 | + | require.ErrorIs(t, s.Refuse(Refused{}), ErrInvalidMessage) | |
| 868 | + | require.Zero(t, s.QueuedFrames(0)) | |
| 869 | + | ||
| 870 | + | var nilScheduler *Scheduler | |
| 871 | + | require.ErrorIs(t, nilScheduler.Refuse(refused), ErrInvalidMessage) | |
| 872 | + | } | |
A internal/adapters/daemonmux/ssh_carriage_reap_other_test.go +6 -0
| @@ -0,0 +1,6 @@ | |||
| 1 | + | //go:build !unix | |
| 2 | + | ||
| 3 | + | package daemonmux | |
| 4 | + | ||
| 5 | + | // sshMuxProcessGone is a no-op on platforms without unix process signaling. | |
| 6 | + | func sshMuxProcessGone(int) bool { return true } | |
A internal/adapters/daemonmux/ssh_carriage_reap_unix_test.go +18 -0
| @@ -0,0 +1,18 @@ | |||
| 1 | + | //go:build unix | |
| 2 | + | ||
| 3 | + | package daemonmux | |
| 4 | + | ||
| 5 | + | import ( | |
| 6 | + | "errors" | |
| 7 | + | "syscall" | |
| 8 | + | ) | |
| 9 | + | ||
| 10 | + | // sshMuxProcessGone reports whether pid no longer names any process, including a | |
| 11 | + | // zombie. A zombie remains signalable, so kill(pid, 0) reports ESRCH only once | |
| 12 | + | // the child was reaped. | |
| 13 | + | func sshMuxProcessGone(pid int) bool { | |
| 14 | + | if pid <= 0 { | |
| 15 | + | return true | |
| 16 | + | } | |
| 17 | + | return errors.Is(syscall.Kill(pid, 0), syscall.ESRCH) | |
| 18 | + | } | |
A internal/adapters/daemonmux/ssh_carriage_test.go +640 -0
| @@ -0,0 +1,640 @@ | |||
| 1 | + | // Integration coverage for the real SSH stdio daemonmux carriage. | |
| 2 | + | // | |
| 3 | + | // These tests exercise the daemonmux multiplexer over an actual SSH stdio | |
| 4 | + | // carriage: a real owned subprocess whose stdio is the physical carriage, | |
| 5 | + | // dialed by sshstdio.DialMuxContext and served by the helper half, | |
| 6 | + | // sshstdio.NewStdioTransport, not over the in-memory carrier used by the unit | |
| 7 | + | // tests or the socket/QUIC carriages. One subprocess is one physical daemonmux | |
| 8 | + | // connection: every logical stream rides its stdio, and no logical attachment | |
| 9 | + | // is ever mapped to a second process. | |
| 10 | + | // | |
| 11 | + | // The broker half (EndpointConnector, PhysicalConnection, LogicalConnector) | |
| 12 | + | // runs in the test process exactly as it would in the client. The daemon half | |
| 13 | + | // runs in a re-executed copy of this test binary (the helper process), so the | |
| 14 | + | // process start, pipe ownership, reaping, and teardown are real; no production | |
| 15 | + | // composition is started. For the raw pump-accounting checks the daemon-side | |
| 16 | + | // pump runs in the test process over the subprocess stdio relay so the stream | |
| 17 | + | // accounting can be asserted directly, exactly as the Unix and QUIC carriage | |
| 18 | + | // tests do. | |
| 19 | + | package daemonmux | |
| 20 | + | ||
| 21 | + | import ( | |
| 22 | + | "context" | |
| 23 | + | "errors" | |
| 24 | + | "flag" | |
| 25 | + | "fmt" | |
| 26 | + | "io" | |
| 27 | + | "net" | |
| 28 | + | "os" | |
| 29 | + | "path/filepath" | |
| 30 | + | "runtime" | |
| 31 | + | "strconv" | |
| 32 | + | "strings" | |
| 33 | + | "sync" | |
| 34 | + | "sync/atomic" | |
| 35 | + | "testing" | |
| 36 | + | "time" | |
| 37 | + | ||
| 38 | + | "github.com/stretchr/testify/require" | |
| 39 | + | ||
| 40 | + | "github.com/bnema/vev/internal/adapters/sshstdio" | |
| 41 | + | "github.com/bnema/vev/internal/domain" | |
| 42 | + | "github.com/bnema/vev/internal/ports" | |
| 43 | + | "github.com/bnema/vev/internal/protocol" | |
| 44 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 45 | + | ) | |
| 46 | + | ||
| 47 | + | // sshMuxCarriageDeadline bounds every real subprocess carriage wait. | |
| 48 | + | const sshMuxCarriageDeadline = 10 * time.Second | |
| 49 | + | ||
| 50 | + | // sshMuxHelperMarker distinguishes the re-executed helper process from an | |
| 51 | + | // ordinary test run. It travels as a positional argument, so a normal test run | |
| 52 | + | // (no positional args) skips every helper entry point. | |
| 53 | + | const sshMuxHelperMarker = "vev-daemonmux-ssh-carriage-helper" | |
| 54 | + | ||
| 55 | + | // TestSSHMuxHelperProcess is the daemon (or relay) half of one SSH stdio | |
| 56 | + | // carriage. It is inert unless re-executed by a carriage test with the helper | |
| 57 | + | // marker, at which point its stdio is the physical carriage and it must never | |
| 58 | + | // write anything but framed envelopes to stdout. | |
| 59 | + | func TestSSHMuxHelperProcess(t *testing.T) { | |
| 60 | + | args := flag.Args() | |
| 61 | + | if len(args) < 2 || args[0] != sshMuxHelperMarker { | |
| 62 | + | t.Skip("ssh stdio mux helper process") | |
| 63 | + | } | |
| 64 | + | switch mode := args[1]; mode { | |
| 65 | + | case "daemon": | |
| 66 | + | runSSHMuxDaemonHelper(t, sshMuxHelperArg(args, 2)) | |
| 67 | + | case "relay": | |
| 68 | + | runSSHMuxRelayHelper(sshMuxHelperArg(args, 2)) | |
| 69 | + | case "silent": | |
| 70 | + | runSSHMuxSilentHelper() | |
| 71 | + | default: | |
| 72 | + | fmt.Fprintf(os.Stderr, "daemonmux ssh mux helper: unknown mode %q\n", mode) | |
| 73 | + | os.Exit(2) | |
| 74 | + | } | |
| 75 | + | } | |
| 76 | + | ||
| 77 | + | func sshMuxHelperArg(args []string, index int) string { | |
| 78 | + | if index < len(args) { | |
| 79 | + | return args[index] | |
| 80 | + | } | |
| 81 | + | return "" | |
| 82 | + | } | |
| 83 | + | ||
| 84 | + | // runSSHMuxDaemonHelper serves one daemonmux physical connection over this | |
| 85 | + | // process' own stdin/stdout using the helper-side stdio carriage constructor | |
| 86 | + | // and the real ServerSupervisor/AggregateListener/typed Listener path. It stops | |
| 87 | + | // when the carriage ends and never lets the test framework's report reach the | |
| 88 | + | // carriage's stdout, which would corrupt the framing stream. | |
| 89 | + | func runSSHMuxDaemonHelper(t *testing.T, pidPath string) { | |
| 90 | + | if pidPath != "" { | |
| 91 | + | if err := os.WriteFile(pidPath, []byte(strconv.Itoa(os.Getpid())), 0o600); err != nil { | |
| 92 | + | sshMuxHelperFatal(err) | |
| 93 | + | } | |
| 94 | + | } | |
| 95 | + | raw := sshstdio.NewStdioTransport() | |
| 96 | + | binding := mustServerBinding(t) | |
| 97 | + | aggregate := NewAggregateListener() | |
| 98 | + | supervisor, err := NewServerSupervisor(aggregate, binding, DefaultMuxCeilings(), 0) | |
| 99 | + | sshMuxHelperFatal(err) | |
| 100 | + | sshMuxHelperFatal(supervisor.Adopt(context.Background(), raw)) | |
| 101 | + | go func() { | |
| 102 | + | for { | |
| 103 | + | conn, err := aggregate.Accept() | |
| 104 | + | if err != nil { | |
| 105 | + | return | |
| 106 | + | } | |
| 107 | + | go func() { _ = servePongs(conn) }() | |
| 108 | + | } | |
| 109 | + | }() | |
| 110 | + | if pump := sshMuxHelperPump(supervisor); pump != nil { | |
| 111 | + | <-pump.Done() | |
| 112 | + | } | |
| 113 | + | _ = supervisor.Close() | |
| 114 | + | _ = aggregate.Close() | |
| 115 | + | sshMuxSilenceStdout() | |
| 116 | + | } | |
| 117 | + | ||
| 118 | + | // sshMuxHelperPump returns the supervisor's one owned child pump, so the helper | |
| 119 | + | // exits exactly when the physical carriage reaches its terminal outcome. | |
| 120 | + | func sshMuxHelperPump(s *ServerSupervisor) *Pump { | |
| 121 | + | s.mu.Lock() | |
| 122 | + | defer s.mu.Unlock() | |
| 123 | + | for child := range s.children { | |
| 124 | + | return child.pump | |
| 125 | + | } | |
| 126 | + | return nil | |
| 127 | + | } | |
| 128 | + | ||
| 129 | + | // runSSHMuxRelayHelper relays this process' stdio and one parent-bound Unix | |
| 130 | + | // socket transparently. It lets the raw pump tests keep both mux ends in the | |
| 131 | + | // test process while every byte still crosses a real subprocess pipe. | |
| 132 | + | func runSSHMuxRelayHelper(sockPath string) { | |
| 133 | + | if sockPath == "" { | |
| 134 | + | sshMuxHelperFatal(errors.New("relay: missing socket path")) | |
| 135 | + | } | |
| 136 | + | conn, err := net.Dial("unix", sockPath) | |
| 137 | + | sshMuxHelperFatal(err) | |
| 138 | + | go func() { | |
| 139 | + | _, _ = io.Copy(conn, os.Stdin) | |
| 140 | + | _ = conn.Close() | |
| 141 | + | }() | |
| 142 | + | _, _ = io.Copy(os.Stdout, conn) | |
| 143 | + | _ = conn.Close() | |
| 144 | + | sshMuxSilenceStdout() | |
| 145 | + | } | |
| 146 | + | ||
| 147 | + | // runSSHMuxSilentHelper reads its stdin to EOF without ever answering, so a | |
| 148 | + | // setup deadline is the only way a caller can give up. It exits cleanly once the | |
| 149 | + | // caller closes the carriage. | |
| 150 | + | func runSSHMuxSilentHelper() { | |
| 151 | + | _, _ = io.Copy(io.Discard, os.Stdin) | |
| 152 | + | sshMuxSilenceStdout() | |
| 153 | + | } | |
| 154 | + | ||
| 155 | + | // sshMuxHelperFatal reports a helper failure on stderr (the bounded diagnostic | |
| 156 | + | // channel) and exits non-cleanly. | |
| 157 | + | func sshMuxHelperFatal(err error) { | |
| 158 | + | if err == nil { | |
| 159 | + | return | |
| 160 | + | } | |
| 161 | + | fmt.Fprintln(os.Stderr, "daemonmux ssh mux helper:", err) | |
| 162 | + | os.Exit(2) | |
| 163 | + | } | |
| 164 | + | ||
| 165 | + | // sshMuxSilenceStdout points the test framework's final report at /dev/null so | |
| 166 | + | // a helper that returns normally never writes its report into the carriage. | |
| 167 | + | func sshMuxSilenceStdout() { | |
| 168 | + | devnull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0) | |
| 169 | + | if err == nil { | |
| 170 | + | os.Stdout = devnull | |
| 171 | + | } | |
| 172 | + | } | |
| 173 | + | ||
| 174 | + | // sshMuxHelperCommand builds an explicit command specification that re-executes | |
| 175 | + | // this test binary in one helper mode. The command is never the session | |
| 176 | + | // `_stdio` command: the mux carriage takes its whole command from the caller. | |
| 177 | + | func sshMuxHelperCommand(mode string, extra ...string) sshstdio.CommandSpec { | |
| 178 | + | args := append([]string{"-test.run=^TestSSHMuxHelperProcess$", sshMuxHelperMarker, mode}, extra...) | |
| 179 | + | return sshstdio.CommandSpec{Path: os.Args[0], Args: args} | |
| 180 | + | } | |
| 181 | + | ||
| 182 | + | // sshMuxDialer returns a RawCarrierDialer over one explicit command | |
| 183 | + | // specification and a counter of how many physical carriages it opened. | |
| 184 | + | func sshMuxDialer(spec sshstdio.CommandSpec) (RawCarrierDialer, *atomic.Int32) { | |
| 185 | + | dials := &atomic.Int32{} | |
| 186 | + | return func(ctx context.Context, _ string) (RawFramedTransport, error) { | |
| 187 | + | dials.Add(1) | |
| 188 | + | return sshstdio.DialMuxContext(ctx, spec, nil) | |
| 189 | + | }, dials | |
| 190 | + | } | |
| 191 | + | ||
| 192 | + | // sshMuxConnect dials one helper carriage and returns the live broker physical | |
| 193 | + | // connection, registering its teardown. | |
| 194 | + | func sshMuxConnect(t *testing.T, binding ServerBinding, spec sshstdio.CommandSpec) ports.BrokerPhysicalConnection { | |
| 195 | + | t.Helper() | |
| 196 | + | dial, _ := sshMuxDialer(spec) | |
| 197 | + | connector, err := NewEndpointConnector(dial, DefaultMuxCeilings()) | |
| 198 | + | require.NoError(t, err) | |
| 199 | + | ctx, cancel := context.WithTimeout(context.Background(), sshMuxCarriageDeadline) | |
| 200 | + | defer cancel() | |
| 201 | + | physical, err := connector.Connect(ctx, muxEndpoint(binding.Identity(), binding.Policy(), "ssh://helper")) | |
| 202 | + | require.NoError(t, err) | |
| 203 | + | t.Cleanup(func() { _ = physical.Close() }) | |
| 204 | + | return physical | |
| 205 | + | } | |
| 206 | + | ||
| 207 | + | // sshMuxReadPID waits for the helper to publish its process id. | |
| 208 | + | func sshMuxReadPID(t *testing.T, path string) int { | |
| 209 | + | t.Helper() | |
| 210 | + | pid := 0 | |
| 211 | + | require.Eventually(t, func() bool { | |
| 212 | + | data, err := os.ReadFile(path) | |
| 213 | + | if err != nil { | |
| 214 | + | return false | |
| 215 | + | } | |
| 216 | + | value, err := strconv.Atoi(strings.TrimSpace(string(data))) | |
| 217 | + | if err != nil { | |
| 218 | + | return false | |
| 219 | + | } | |
| 220 | + | pid = value | |
| 221 | + | return true | |
| 222 | + | }, sshMuxCarriageDeadline, 2*time.Millisecond) | |
| 223 | + | return pid | |
| 224 | + | } | |
| 225 | + | ||
| 226 | + | // requireNoGoroutineLeak waits for the test process to return to its baseline | |
| 227 | + | // goroutine count once every carriage was torn down. | |
| 228 | + | func requireNoGoroutineLeak(t *testing.T, baseline int) { | |
| 229 | + | t.Helper() | |
| 230 | + | require.Eventually(t, func() bool { return runtime.NumGoroutine() <= baseline+4 }, 5*time.Second, 20*time.Millisecond, | |
| 231 | + | "goroutines leaked: baseline %d, now %d", baseline, runtime.NumGoroutine()) | |
| 232 | + | } | |
| 233 | + | ||
| 234 | + | // TestSSHCarriageSupervisorTwoThenHundredTypedAdmissions drives the full daemon | |
| 235 | + | // path over one real SSH stdio physical connection: EndpointConnector dials a | |
| 236 | + | // real subprocess started by DialMuxContext, the helper process adopts it with | |
| 237 | + | // ServerSupervisor/AggregateListener, and the typed Listener delivers | |
| 238 | + | // independent daemon admissions. Only one physical connection and one process | |
| 239 | + | // exist for all 102 streams. | |
| 240 | + | func TestSSHCarriageSupervisorTwoThenHundredTypedAdmissions(t *testing.T) { | |
| 241 | + | baseline := runtime.NumGoroutine() | |
| 242 | + | binding := mustServerBinding(t) | |
| 243 | + | policy := binding.Policy() | |
| 244 | + | pidPath := filepath.Join(t.TempDir(), "helper.pid") | |
| 245 | + | spec := sshMuxHelperCommand("daemon", pidPath) | |
| 246 | + | ||
| 247 | + | dial, dials := sshMuxDialer(spec) | |
| 248 | + | connector, err := NewEndpointConnector(dial, DefaultMuxCeilings()) | |
| 249 | + | require.NoError(t, err) | |
| 250 | + | physical, err := connector.Connect(context.Background(), muxEndpoint(binding.Identity(), policy, "ssh://helper")) | |
| 251 | + | require.NoError(t, err) | |
| 252 | + | t.Cleanup(func() { _ = physical.Close() }) | |
| 253 | + | pid := sshMuxReadPID(t, pidPath) | |
| 254 | + | ||
| 255 | + | // Two independent typed admissions over the one physical carriage. | |
| 256 | + | first, err := physical.OpenStream(context.Background(), muxOpenRequest(1, policy)) | |
| 257 | + | require.NoError(t, err) | |
| 258 | + | t.Cleanup(func() { _ = first.Close() }) | |
| 259 | + | second, err := physical.OpenStream(context.Background(), muxOpenRequest(2, policy)) | |
| 260 | + | require.NoError(t, err) | |
| 261 | + | t.Cleanup(func() { _ = second.Close() }) | |
| 262 | + | for _, stream := range []ports.BrokerLogicalConnection{first, second} { | |
| 263 | + | require.NoError(t, stream.SendClient(protocol.Ping{})) | |
| 264 | + | message, err := stream.ReceiveServer() | |
| 265 | + | require.NoError(t, err) | |
| 266 | + | require.Equal(t, protocol.Pong{}, message) | |
| 267 | + | } | |
| 268 | + | ||
| 269 | + | // One hundred more independently admitted typed streams, opened | |
| 270 | + | // concurrently over the same physical carriage. | |
| 271 | + | const extra = 100 | |
| 272 | + | connections := make([]ports.BrokerLogicalConnection, extra) | |
| 273 | + | openErrs := make([]error, extra) | |
| 274 | + | var wg sync.WaitGroup | |
| 275 | + | for i := 0; i < extra; i++ { | |
| 276 | + | wg.Add(1) | |
| 277 | + | go func(i int) { | |
| 278 | + | defer wg.Done() | |
| 279 | + | connections[i], openErrs[i] = physical.OpenStream(context.Background(), muxOpenRequest(uint64(3+i), policy)) | |
| 280 | + | }(i) | |
| 281 | + | } | |
| 282 | + | wg.Wait() | |
| 283 | + | for i, openErr := range openErrs { | |
| 284 | + | require.NoError(t, openErr, "stream %d", i+3) | |
| 285 | + | } | |
| 286 | + | for i, connection := range connections { | |
| 287 | + | require.NotNil(t, connection, "stream %d", i+3) | |
| 288 | + | t.Cleanup(func() { _ = connection.Close() }) | |
| 289 | + | } | |
| 290 | + | for i, connection := range connections { | |
| 291 | + | require.NoError(t, connection.SendClient(protocol.Ping{}), "stream %d", i+3) | |
| 292 | + | message, err := connection.ReceiveServer() | |
| 293 | + | require.NoError(t, err, "stream %d", i+3) | |
| 294 | + | require.Equal(t, protocol.Pong{}, message) | |
| 295 | + | } | |
| 296 | + | ||
| 297 | + | // All 102 streams share exactly one physical connection: exactly one dial, | |
| 298 | + | // one subprocess, and one live broker carriage. | |
| 299 | + | require.Equal(t, int32(1), dials.Load(), "all streams must share one ssh carriage") | |
| 300 | + | require.False(t, channelClosed(physical.Done()), "the physical carriage must stay open while streams are live") | |
| 301 | + | ||
| 302 | + | all := make([]ports.BrokerLogicalConnection, 0, 2+extra) | |
| 303 | + | all = append(all, first, second) | |
| 304 | + | all = append(all, connections...) | |
| 305 | + | for _, connection := range all { | |
| 306 | + | require.NoError(t, connection.Close()) | |
| 307 | + | } | |
| 308 | + | require.False(t, channelClosed(physical.Done()), "closing every stream must leave the physical carriage open") | |
| 309 | + | ||
| 310 | + | // Teardown terminates and reaps the one owned subprocess, and no worker | |
| 311 | + | // outlives the carriage. | |
| 312 | + | require.NoError(t, physical.Close()) | |
| 313 | + | require.Eventually(t, func() bool { return sshMuxProcessGone(pid) }, sshMuxCarriageDeadline, 5*time.Millisecond, | |
| 314 | + | "the ssh carriage subprocess was never reaped") | |
| 315 | + | requireNoGoroutineLeak(t, baseline) | |
| 316 | + | } | |
| 317 | + | ||
| 318 | + | // TestSSHCarriagePhysicalFailureFansOutAndIsIsolated proves a lost SSH stdio | |
| 319 | + | // physical carriage publishes its terminal outcome before fanning out to every | |
| 320 | + | // logical stream it carried, while a sibling physical carriage on the same | |
| 321 | + | // supervisor is untouched and a fresh stream on the lost physical is refused. | |
| 322 | + | func TestSSHCarriagePhysicalFailureFansOutAndIsIsolated(t *testing.T) { | |
| 323 | + | binding := mustServerBinding(t) | |
| 324 | + | policy := binding.Policy() | |
| 325 | + | pidA := filepath.Join(t.TempDir(), "a.pid") | |
| 326 | + | pidB := filepath.Join(t.TempDir(), "b.pid") | |
| 327 | + | ||
| 328 | + | physicalA := sshMuxConnect(t, binding, sshMuxHelperCommand("daemon", pidA)) | |
| 329 | + | physicalB := sshMuxConnect(t, binding, sshMuxHelperCommand("daemon", pidB)) | |
| 330 | + | lostPID := sshMuxReadPID(t, pidA) | |
| 331 | + | _ = sshMuxReadPID(t, pidB) | |
| 332 | + | ||
| 333 | + | lost := make([]ports.BrokerLogicalConnection, 3) | |
| 334 | + | for i := range lost { | |
| 335 | + | connection, err := physicalA.OpenStream(context.Background(), muxOpenRequest(uint64(i+1), policy)) | |
| 336 | + | require.NoError(t, err, "stream %d", i+1) | |
| 337 | + | lost[i] = connection | |
| 338 | + | t.Cleanup(func() { _ = connection.Close() }) | |
| 339 | + | } | |
| 340 | + | survivor, err := physicalB.OpenStream(context.Background(), muxOpenRequest(1, policy)) | |
| 341 | + | require.NoError(t, err) | |
| 342 | + | t.Cleanup(func() { _ = survivor.Close() }) | |
| 343 | + | ||
| 344 | + | // The physical terminal outcome is ordered before its logical fan-out. | |
| 345 | + | ordered := make(chan bool, 1) | |
| 346 | + | go func() { | |
| 347 | + | select { | |
| 348 | + | case <-lost[0].Done(): | |
| 349 | + | case <-time.After(sshMuxCarriageDeadline): | |
| 350 | + | ordered <- false | |
| 351 | + | return | |
| 352 | + | } | |
| 353 | + | select { | |
| 354 | + | case <-physicalA.Done(): | |
| 355 | + | ordered <- true | |
| 356 | + | case <-time.After(sshMuxCarriageDeadline): | |
| 357 | + | ordered <- false | |
| 358 | + | } | |
| 359 | + | }() | |
| 360 | + | ||
| 361 | + | // Lose the ssh carriage entirely: the subprocess dies, its stdout closes, | |
| 362 | + | // and the broker observes the pipe failure. | |
| 363 | + | process, err := os.FindProcess(lostPID) | |
| 364 | + | require.NoError(t, err) | |
| 365 | + | require.NoError(t, process.Kill()) | |
| 366 | + | ||
| 367 | + | require.True(t, <-ordered, "the logical Done fired before the physical Done") | |
| 368 | + | require.True(t, channelClosed(physicalA.Done())) | |
| 369 | + | require.Error(t, physicalA.Err()) | |
| 370 | + | require.NotEqual(t, domain.RemoteFailureNone, physicalA.FailureKind()) | |
| 371 | + | for i, stream := range lost { | |
| 372 | + | require.Eventually(t, func() bool { | |
| 373 | + | return channelClosed(stream.Done()) && stream.Err() != nil | |
| 374 | + | }, sshMuxCarriageDeadline, time.Millisecond, "lost stream %d never received the physical failure", i+1) | |
| 375 | + | } | |
| 376 | + | ||
| 377 | + | // The sibling physical carriage is unaffected. | |
| 378 | + | require.False(t, channelClosed(physicalB.Done())) | |
| 379 | + | require.False(t, channelClosed(survivor.Done())) | |
| 380 | + | require.NoError(t, survivor.SendClient(protocol.Ping{})) | |
| 381 | + | message, err := survivor.ReceiveServer() | |
| 382 | + | require.NoError(t, err) | |
| 383 | + | require.Equal(t, protocol.Pong{}, message) | |
| 384 | + | ||
| 385 | + | // The lost physical refuses a fresh stream. | |
| 386 | + | _, err = physicalA.OpenStream(context.Background(), muxOpenRequest(9, policy)) | |
| 387 | + | require.Error(t, err) | |
| 388 | + | ||
| 389 | + | // The failed subprocess is reaped, not left a zombie. | |
| 390 | + | require.Eventually(t, func() bool { return sshMuxProcessGone(lostPID) }, sshMuxCarriageDeadline, 5*time.Millisecond, | |
| 391 | + | "the failed ssh carriage subprocess was never reaped") | |
| 392 | + | } | |
| 393 | + | ||
| 394 | + | // TestSSHCarriageSetupContextDetachedFromPhysicalLifetime proves the setup | |
| 395 | + | // context covers only setup: once EndpointConnector.Connect returned over a real | |
| 396 | + | // ssh carriage, cancelling the setup context never stops the pooled physical | |
| 397 | + | // connection, which Close alone owns. | |
| 398 | + | func TestSSHCarriageSetupContextDetachedFromPhysicalLifetime(t *testing.T) { | |
| 399 | + | binding := mustServerBinding(t) | |
| 400 | + | spec := sshMuxHelperCommand("daemon", filepath.Join(t.TempDir(), "helper.pid")) | |
| 401 | + | dial, _ := sshMuxDialer(spec) | |
| 402 | + | connector, err := NewEndpointConnector(dial, DefaultMuxCeilings()) | |
| 403 | + | require.NoError(t, err) | |
| 404 | + | ||
| 405 | + | setupCtx, cancel := context.WithCancel(context.Background()) | |
| 406 | + | physical, err := connector.Connect(setupCtx, muxEndpoint(binding.Identity(), binding.Policy(), "ssh://helper")) | |
| 407 | + | require.NoError(t, err) | |
| 408 | + | t.Cleanup(func() { _ = physical.Close() }) | |
| 409 | + | cancel() | |
| 410 | + | ||
| 411 | + | require.False(t, channelClosed(physical.Done())) | |
| 412 | + | logical, err := physical.OpenStream(context.Background(), muxOpenRequest(1, binding.Policy())) | |
| 413 | + | require.NoError(t, err) | |
| 414 | + | t.Cleanup(func() { _ = logical.Close() }) | |
| 415 | + | require.NoError(t, logical.SendClient(protocol.Ping{})) | |
| 416 | + | message, err := logical.ReceiveServer() | |
| 417 | + | require.NoError(t, err) | |
| 418 | + | require.Equal(t, protocol.Pong{}, message) | |
| 419 | + | require.False(t, channelClosed(physical.Done())) | |
| 420 | + | } | |
| 421 | + | ||
| 422 | + | // TestSSHCarriageCanceledSetupAdmitsNothing proves one canceled setup context | |
| 423 | + | // prevents the subprocess from starting at all: no process, no physical | |
| 424 | + | // connection, no stream. | |
| 425 | + | func TestSSHCarriageCanceledSetupAdmitsNothing(t *testing.T) { | |
| 426 | + | binding := mustServerBinding(t) | |
| 427 | + | pidPath := filepath.Join(t.TempDir(), "helper.pid") | |
| 428 | + | spec := sshMuxHelperCommand("daemon", pidPath) | |
| 429 | + | dial, _ := sshMuxDialer(spec) | |
| 430 | + | connector, err := NewEndpointConnector(dial, DefaultMuxCeilings()) | |
| 431 | + | require.NoError(t, err) | |
| 432 | + | ||
| 433 | + | setupCtx, cancel := context.WithCancel(context.Background()) | |
| 434 | + | cancel() | |
| 435 | + | physical, err := connector.Connect(setupCtx, muxEndpoint(binding.Identity(), binding.Policy(), "ssh://helper")) | |
| 436 | + | require.ErrorIs(t, err, context.Canceled) | |
| 437 | + | require.Nil(t, physical) | |
| 438 | + | ||
| 439 | + | _, statErr := os.Stat(pidPath) | |
| 440 | + | require.ErrorIs(t, statErr, os.ErrNotExist, "a canceled setup must not start the ssh subprocess") | |
| 441 | + | } | |
| 442 | + | ||
| 443 | + | // TestSSHCarriageSetupDeadlineBoundsTheMuxHandshake proves the setup deadline | |
| 444 | + | // covers the daemonmux physical handshake over a real ssh carriage: a helper | |
| 445 | + | // that never answers the preamble fails the whole Connect within the setup | |
| 446 | + | // deadline and publishes no physical connection. | |
| 447 | + | func TestSSHCarriageSetupDeadlineBoundsTheMuxHandshake(t *testing.T) { | |
| 448 | + | binding := mustServerBinding(t) | |
| 449 | + | spec := sshMuxHelperCommand("silent") | |
| 450 | + | dial, _ := sshMuxDialer(spec) | |
| 451 | + | connector, err := NewEndpointConnector(dial, DefaultMuxCeilings()) | |
| 452 | + | require.NoError(t, err) | |
| 453 | + | ||
| 454 | + | setupCtx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) | |
| 455 | + | defer cancel() | |
| 456 | + | physical, err := connector.Connect(setupCtx, muxEndpoint(binding.Identity(), binding.Policy(), "ssh://helper")) | |
| 457 | + | require.ErrorIs(t, err, context.DeadlineExceeded) | |
| 458 | + | require.Nil(t, physical) | |
| 459 | + | } | |
| 460 | + | ||
| 461 | + | // TestSSHCarriageConnectErrorBoundedAndSanitized proves a failed ssh carriage | |
| 462 | + | // surfaces a bounded, control-byte-free public error that never carries raw | |
| 463 | + | // remote stderr. | |
| 464 | + | func TestSSHCarriageConnectErrorBoundedAndSanitized(t *testing.T) { | |
| 465 | + | const marker = "remote-secret-value" | |
| 466 | + | binding := mustServerBinding(t) | |
| 467 | + | script := "sleep 0.2; printf '\\033[31m" + marker + "\\033[0m' >&2; printf 'B%.0s' $(seq 1 20000) >&2; exit 3" | |
| 468 | + | spec := sshstdio.CommandSpec{Path: "sh", Args: []string{"-c", script}} | |
| 469 | + | dial, _ := sshMuxDialer(spec) | |
| 470 | + | connector, err := NewEndpointConnector(dial, DefaultMuxCeilings()) | |
| 471 | + | require.NoError(t, err) | |
| 472 | + | ||
| 473 | + | ctx, cancel := context.WithTimeout(context.Background(), sshMuxCarriageDeadline) | |
| 474 | + | defer cancel() | |
| 475 | + | physical, err := connector.Connect(ctx, muxEndpoint(binding.Identity(), binding.Policy(), "ssh://helper")) | |
| 476 | + | require.Error(t, err) | |
| 477 | + | require.Nil(t, physical) | |
| 478 | + | ||
| 479 | + | cause := errors.Unwrap(err) | |
| 480 | + | if cause == nil { | |
| 481 | + | cause = err | |
| 482 | + | } | |
| 483 | + | require.NotContains(t, cause.Error(), marker) | |
| 484 | + | require.NotContains(t, cause.Error(), "\x1b") | |
| 485 | + | require.Less(t, len(cause.Error()), 4096) | |
| 486 | + | } | |
| 487 | + | ||
| 488 | + | // sshRawCarriagePair establishes one real SSH stdio carriage whose subprocess is | |
| 489 | + | // a transparent relay to an in-process Unix socket, and returns its two raw | |
| 490 | + | // ends: the ssh client end (the subprocess stdio DialMuxContext owns) and the | |
| 491 | + | // in-process relay end. Every byte crosses the real subprocess pipes. | |
| 492 | + | func sshRawCarriagePair(t *testing.T) (RawFramedTransport, RawFramedTransport) { | |
| 493 | + | t.Helper() | |
| 494 | + | sockPath := filepath.Join(t.TempDir(), "relay.sock") | |
| 495 | + | listener, err := net.Listen("unix", sockPath) | |
| 496 | + | require.NoError(t, err) | |
| 497 | + | t.Cleanup(func() { _ = listener.Close() }) | |
| 498 | + | ||
| 499 | + | accepted := make(chan net.Conn, 1) | |
| 500 | + | acceptErr := make(chan error, 1) | |
| 501 | + | go func() { | |
| 502 | + | conn, err := listener.Accept() | |
| 503 | + | if err != nil { | |
| 504 | + | acceptErr <- err | |
| 505 | + | return | |
| 506 | + | } | |
| 507 | + | accepted <- conn | |
| 508 | + | }() | |
| 509 | + | ||
| 510 | + | client, err := sshstdio.DialMuxContext(context.Background(), sshMuxHelperCommand("relay", sockPath), nil) | |
| 511 | + | require.NoError(t, err) | |
| 512 | + | t.Cleanup(func() { _ = client.Close() }) | |
| 513 | + | ||
| 514 | + | var conn net.Conn | |
| 515 | + | select { | |
| 516 | + | case conn = <-accepted: | |
| 517 | + | case err := <-acceptErr: | |
| 518 | + | t.Fatalf("relay accept: %v", err) | |
| 519 | + | case <-time.After(sshMuxCarriageDeadline): | |
| 520 | + | t.Fatal("relay subprocess never connected") | |
| 521 | + | } | |
| 522 | + | t.Cleanup(func() { _ = conn.Close() }) | |
| 523 | + | daemon := sshstdio.NewTransport(conn, conn, conn.Close).(wire.BoundedTransport) | |
| 524 | + | return client, daemon | |
| 525 | + | } | |
| 526 | + | ||
| 527 | + | // sshDaemonPump builds a daemon-side pump over one end of a real SSH stdio | |
| 528 | + | // carriage and returns the raw ssh client end for direct frame injection, so a | |
| 529 | + | // test can drive deterministic inbound traffic over the actual subprocess | |
| 530 | + | // pipes. | |
| 531 | + | func sshDaemonPump(t *testing.T) (*Pump, RawFramedTransport) { | |
| 532 | + | t.Helper() | |
| 533 | + | client, daemon := sshRawCarriagePair(t) | |
| 534 | + | carrier, err := NewPreambleCarrier(daemon) | |
| 535 | + | require.NoError(t, err) | |
| 536 | + | require.NoError(t, carrier.Negotiate(DefaultMuxCeilings())) | |
| 537 | + | pump, err := NewPump(carrier, DirectionClient, DefaultMuxCeilings()) | |
| 538 | + | require.NoError(t, err) | |
| 539 | + | t.Cleanup(func() { _ = pump.Close() }) | |
| 540 | + | pump.Start(context.Background()) | |
| 541 | + | return pump, client | |
| 542 | + | } | |
| 543 | + | ||
| 544 | + | // injectSSHClientFrame writes one client frame over the real ssh carriage. | |
| 545 | + | func injectSSHClientFrame(t *testing.T, client RawFramedTransport, message ClientMessage) { | |
| 546 | + | t.Helper() | |
| 547 | + | require.NoError(t, client.Send(wire.Envelope{Payload: mustEncodeClient(t, message)})) | |
| 548 | + | } | |
| 549 | + | ||
| 550 | + | // nextSSHServerFrame reads one daemon frame from the real ssh carriage. | |
| 551 | + | func nextSSHServerFrame(t *testing.T, client RawFramedTransport) []byte { | |
| 552 | + | t.Helper() | |
| 553 | + | envelope, err := client.RecvBounded(testEnvelopeCeiling) | |
| 554 | + | require.NoError(t, err) | |
| 555 | + | return envelope.Payload | |
| 556 | + | } | |
| 557 | + | ||
| 558 | + | // TestSSHCarriageBlockedConsumerSiblingProgress proves, over a real ssh | |
| 559 | + | // carriage, that a logical consumer which never drains one stream's inbound | |
| 560 | + | // queue neither blocks the reader nor a sibling: the stalled stream is reset by | |
| 561 | + | // its own bound while the sibling keeps receiving. | |
| 562 | + | func TestSSHCarriageBlockedConsumerSiblingProgress(t *testing.T) { | |
| 563 | + | pump, client := sshDaemonPump(t) | |
| 564 | + | ||
| 565 | + | const siblings = 2 | |
| 566 | + | for i := 1; i <= siblings; i++ { | |
| 567 | + | injectSSHClientFrame(t, client, openFor(PhysicalStreamID(i))) | |
| 568 | + | } | |
| 569 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == siblings }) | |
| 570 | + | for i := 1; i <= siblings; i++ { | |
| 571 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(PhysicalStreamID(i))})) | |
| 572 | + | } | |
| 573 | + | ||
| 574 | + | // The consumer never Takes stream 1: its bounded queue fills. | |
| 575 | + | for i := 0; i < MaxMuxStreamQueueChunks; i++ { | |
| 576 | + | injectSSHClientFrame(t, client, Data{Physical: 1, Data: []byte{byte(i)}}) | |
| 577 | + | } | |
| 578 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { | |
| 579 | + | status, ok := e.Status(1) | |
| 580 | + | return ok && status.QueuedChunks == MaxMuxStreamQueueChunks | |
| 581 | + | }) | |
| 582 | + | ||
| 583 | + | // One more chunk overflows stream 1's own bound; a sibling chunk still | |
| 584 | + | // lands, so neither the reader nor the sibling was blocked. | |
| 585 | + | injectSSHClientFrame(t, client, Data{Physical: 1, Data: []byte{'x'}}) | |
| 586 | + | injectSSHClientFrame(t, client, Data{Physical: 2, Data: []byte("sibling")}) | |
| 587 | + | ||
| 588 | + | taken, ok := takeEventually(t, pump, 2) | |
| 589 | + | require.True(t, ok) | |
| 590 | + | require.Equal(t, []byte("sibling"), taken) | |
| 591 | + | ||
| 592 | + | stalled := mustStatus(t, pump.Engine(), 1) | |
| 593 | + | require.Equal(t, StreamTerminal, stalled.State) | |
| 594 | + | require.ErrorIs(t, stalled.Err, ErrStreamQueueFull) | |
| 595 | + | require.Equal(t, domain.RemoteFailureTransport, stalled.FailureKind) | |
| 596 | + | require.Equal(t, StreamOpen, mustStatus(t, pump.Engine(), 2).State) | |
| 597 | + | require.False(t, pump.Engine().Closed()) | |
| 598 | + | require.False(t, channelClosed(pump.Done())) | |
| 599 | + | ||
| 600 | + | reset := decodeReset(t, nextSSHServerFrame(t, client), DirectionServer) | |
| 601 | + | require.Equal(t, PhysicalStreamID(1), reset.Physical) | |
| 602 | + | require.Equal(t, domain.RemoteFailureTransport, reset.Error.FailureKind) | |
| 603 | + | } | |
| 604 | + | ||
| 605 | + | // TestSSHCarriageResetIsolation proves, over a real ssh carriage, that one | |
| 606 | + | // stream-local refusal schedules exactly one Reset for its own stream and | |
| 607 | + | // leaves the physical connection and every sibling healthy. | |
| 608 | + | func TestSSHCarriageResetIsolation(t *testing.T) { | |
| 609 | + | pump, client := sshDaemonPump(t) | |
| 610 | + | ||
| 611 | + | for _, id := range []PhysicalStreamID{1, 2, 3} { | |
| 612 | + | injectSSHClientFrame(t, client, openFor(id)) | |
| 613 | + | } | |
| 614 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == 3 }) | |
| 615 | + | for _, id := range []PhysicalStreamID{2, 3} { | |
| 616 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(id)})) | |
| 617 | + | } | |
| 618 | + | ||
| 619 | + | // Stream 1 is still opening: inbound data is a stream-local state refusal. | |
| 620 | + | injectSSHClientFrame(t, client, Data{Physical: 1, Data: []byte("early")}) | |
| 621 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { | |
| 622 | + | status, ok := e.Status(1) | |
| 623 | + | return ok && status.State == StreamTerminal | |
| 624 | + | }) | |
| 625 | + | settled := mustStatus(t, pump.Engine(), 1) | |
| 626 | + | require.ErrorContains(t, settled.Err, "invalid stream state") | |
| 627 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, settled.FailureKind) | |
| 628 | + | ||
| 629 | + | injectSSHClientFrame(t, client, Data{Physical: 2, Data: []byte("sibling")}) | |
| 630 | + | taken, ok := takeEventually(t, pump, 2) | |
| 631 | + | require.True(t, ok) | |
| 632 | + | require.Equal(t, []byte("sibling"), taken) | |
| 633 | + | ||
| 634 | + | reset := decodeReset(t, nextSSHServerFrame(t, client), DirectionServer) | |
| 635 | + | require.Equal(t, PhysicalStreamID(1), reset.Physical) | |
| 636 | + | require.True(t, reset.HasError) | |
| 637 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, reset.Error.FailureKind) | |
| 638 | + | require.False(t, pump.Engine().Closed()) | |
| 639 | + | require.False(t, channelClosed(pump.Done())) | |
| 640 | + | } | |
A internal/adapters/daemonmux/supervisor.go +368 -0
| @@ -0,0 +1,368 @@ | |||
| 1 | + | // Server supervisor over accepted raw physical carriers (P3.2c). | |
| 2 | + | // | |
| 3 | + | // ServerSupervisor is the daemon-side owner of accepted daemonmux physical | |
| 4 | + | // connections. A daemon accept loop hands it one authenticated raw framed | |
| 5 | + | // carriage per connection; the supervisor completes the daemonmux physical | |
| 6 | + | // preamble under its immutable ServerBinding, then publishes a fully wired | |
| 7 | + | // typed listener to the daemon's AggregateListener. It performs no listener | |
| 8 | + | // policy of its own: the carriage must already be authenticated, and the | |
| 9 | + | // binding is the only authority the handshake enforces. | |
| 10 | + | // | |
| 11 | + | // Each accepted carriage becomes one owned supervisedPhysical: a Listener, its | |
| 12 | + | // Pump, and the framing bridge that carries it. The listener is always built | |
| 13 | + | // before its pump starts, because it registers the pump's single admission | |
| 14 | + | // observer and the pump refuses a late registration. The supervised child | |
| 15 | + | // implements ports.ServerListener by delegating Accept to that listener, and | |
| 16 | + | // its Close tears the listener, pump, and carrier down together, so the | |
| 17 | + | // AggregateListener never owns a half-closed child. | |
| 18 | + | // | |
| 19 | + | // The accepted physical policy is enforced on every inbound Open before fresh | |
| 20 | + | // stream admission: the supervised pump's engine is restricted to the binding | |
| 21 | + | // policy, so a stream that names a policy the daemon did not accept is refused | |
| 22 | + | // on its own stream and never reaches the daemon's accept stream. | |
| 23 | + | // | |
| 24 | + | // Admission is bounded: at most the configured number of connections may be | |
| 25 | + | // inside dial-handshake-registration at once, so a burst of raw carriers can | |
| 26 | + | // never start an unbounded number of handshakes or pump goroutines. Each child | |
| 27 | + | // is reaped on its own: when its pump reaches a terminal outcome - a lost | |
| 28 | + | // carriage, a closed listener, or a local teardown - one goroutine deregisters | |
| 29 | + | // it from the aggregate and closes it, without touching a sibling or the | |
| 30 | + | // supervisor. The supervisor's Close closes every owned child and joins the | |
| 31 | + | // reapers; it does not close the AggregateListener, which its caller owns. | |
| 32 | + | package daemonmux | |
| 33 | + | ||
| 34 | + | import ( | |
| 35 | + | "context" | |
| 36 | + | "errors" | |
| 37 | + | "fmt" | |
| 38 | + | "sync" | |
| 39 | + | ||
| 40 | + | "github.com/bnema/vev/internal/adapters/clock" | |
| 41 | + | "github.com/bnema/vev/internal/ports" | |
| 42 | + | "github.com/bnema/vev/internal/protocol" | |
| 43 | + | ) | |
| 44 | + | ||
| 45 | + | // Server supervisor sentinels. They are typed so a caller can classify a | |
| 46 | + | // configuration, admission, or lifecycle outcome without matching on message | |
| 47 | + | // text. | |
| 48 | + | var ( | |
| 49 | + | // ErrSupervisorConfig reports a supervisor constructed without an | |
| 50 | + | // aggregate listener or binding, with an invalid ceiling advertisement, or | |
| 51 | + | // an Adopt call with a nil raw carriage. | |
| 52 | + | ErrSupervisorConfig = errors.New("daemonmux: invalid server supervisor configuration") | |
| 53 | + | // ErrSupervisorClosed reports a physical carriage offered to a supervisor | |
| 54 | + | // that was closed, or a registration attempted after Close. | |
| 55 | + | ErrSupervisorClosed = errors.New("daemonmux: server supervisor is closed") | |
| 56 | + | ) | |
| 57 | + | ||
| 58 | + | // MaxSupervisedAdmissions bounds the accepted physical connections one | |
| 59 | + | // supervisor may have in handshake-or-registration at once. It mirrors the | |
| 60 | + | // per-connection stream ceiling, so a daemon never starts more handshakes than | |
| 61 | + | // one connection could multiplex. | |
| 62 | + | const MaxSupervisedAdmissions = int(MaxMuxStreams) | |
| 63 | + | ||
| 64 | + | // ServerSupervisor accepts authenticated raw physical carriage and publishes | |
| 65 | + | // owned typed listeners to one AggregateListener. It is safe for concurrent | |
| 66 | + | // use. | |
| 67 | + | type ServerSupervisor struct { | |
| 68 | + | aggregate *AggregateListener | |
| 69 | + | binding ServerBinding | |
| 70 | + | ceilings MuxCeilings | |
| 71 | + | clock ports.Clock | |
| 72 | + | ||
| 73 | + | // slots bounds concurrent handshakes and admissions: one token is held | |
| 74 | + | // from the start of Adopt until the child is registered and its pump | |
| 75 | + | // started. | |
| 76 | + | slots chan struct{} | |
| 77 | + | ||
| 78 | + | // mu guards children and closed. children holds exactly the live owned | |
| 79 | + | // children, so Close and the reapers agree on what still needs teardown. | |
| 80 | + | mu sync.Mutex | |
| 81 | + | children map[*supervisedPhysical]struct{} | |
| 82 | + | closed bool | |
| 83 | + | ||
| 84 | + | done chan struct{} | |
| 85 | + | wg sync.WaitGroup | |
| 86 | + | closeOnce sync.Once | |
| 87 | + | } | |
| 88 | + | ||
| 89 | + | // NewServerSupervisor returns a supervisor that publishes every accepted | |
| 90 | + | // physical connection to aggregate under binding, advertising ceilings in the | |
| 91 | + | // physical preamble. admissionLimit bounds concurrent handshakes and | |
| 92 | + | // admissions; a non-positive value falls back to MaxSupervisedAdmissions. A | |
| 93 | + | // nil aggregate, an invalid binding, or an invalid ceiling advertisement is | |
| 94 | + | // refused with ErrSupervisorConfig. | |
| 95 | + | func NewServerSupervisor(aggregate *AggregateListener, binding ServerBinding, ceilings MuxCeilings, admissionLimit int) (*ServerSupervisor, error) { | |
| 96 | + | return newServerSupervisor(aggregate, binding, ceilings, admissionLimit, clock.New()) | |
| 97 | + | } | |
| 98 | + | ||
| 99 | + | // newServerSupervisor builds a supervisor from an explicit time source, so a | |
| 100 | + | // test can drive the per-stream handshake deadline deterministically. The | |
| 101 | + | // exported constructor delegates with the wall clock. | |
| 102 | + | func newServerSupervisor(aggregate *AggregateListener, binding ServerBinding, ceilings MuxCeilings, admissionLimit int, timeSource ports.Clock) (*ServerSupervisor, error) { | |
| 103 | + | if aggregate == nil { | |
| 104 | + | return nil, ErrSupervisorConfig | |
| 105 | + | } | |
| 106 | + | if err := binding.Validate(); err != nil { | |
| 107 | + | return nil, fmt.Errorf("%w: %w", ErrSupervisorConfig, err) | |
| 108 | + | } | |
| 109 | + | if err := ceilings.Validate(); err != nil { | |
| 110 | + | return nil, fmt.Errorf("%w: %w", ErrSupervisorConfig, err) | |
| 111 | + | } | |
| 112 | + | if admissionLimit <= 0 { | |
| 113 | + | admissionLimit = MaxSupervisedAdmissions | |
| 114 | + | } | |
| 115 | + | if timeSource == nil { | |
| 116 | + | timeSource = clock.New() | |
| 117 | + | } | |
| 118 | + | return &ServerSupervisor{ | |
| 119 | + | aggregate: aggregate, | |
| 120 | + | binding: binding, | |
| 121 | + | ceilings: ceilings, | |
| 122 | + | clock: timeSource, | |
| 123 | + | slots: make(chan struct{}, admissionLimit), | |
| 124 | + | children: make(map[*supervisedPhysical]struct{}), | |
| 125 | + | done: make(chan struct{}), | |
| 126 | + | }, nil | |
| 127 | + | } | |
| 128 | + | ||
| 129 | + | // Adopt accepts one authenticated raw physical carriage: it bounds admission, | |
| 130 | + | // completes the daemonmux physical preamble against the supervisor's binding, | |
| 131 | + | // negotiates the effective ceilings, builds the pump and its typed listener, | |
| 132 | + | // registers the owned child with the aggregate, and starts the pump. The | |
| 133 | + | // caller's ctx bounds the handshake; the started connection is detached from it | |
| 134 | + | // and owned by the supervisor. | |
| 135 | + | // | |
| 136 | + | // A failed preamble, a refused policy, or a closed supervisor closes the | |
| 137 | + | // carriage and registers nothing. After a successful return the caller's only | |
| 138 | + | // remaining role is to stop offering carriage (or Close the supervisor); the | |
| 139 | + | // supervisor owns the carriage from then on. | |
| 140 | + | func (s *ServerSupervisor) Adopt(ctx context.Context, raw RawFramedTransport) error { | |
| 141 | + | if s == nil { | |
| 142 | + | return ErrSupervisorConfig | |
| 143 | + | } | |
| 144 | + | if raw == nil { | |
| 145 | + | return ErrSupervisorConfig | |
| 146 | + | } | |
| 147 | + | if ctx == nil { | |
| 148 | + | ctx = context.Background() | |
| 149 | + | } | |
| 150 | + | if err := s.acquire(ctx); err != nil { | |
| 151 | + | _ = raw.Close() | |
| 152 | + | return err | |
| 153 | + | } | |
| 154 | + | defer s.release() | |
| 155 | + | ||
| 156 | + | bridge, err := NewPreambleCarrier(raw) | |
| 157 | + | if err != nil { | |
| 158 | + | _ = raw.Close() | |
| 159 | + | return fmt.Errorf("%w: %w", ErrSupervisorConfig, err) | |
| 160 | + | } | |
| 161 | + | ||
| 162 | + | result, err := RunServerHandshake(ctx, bridge, s.binding, s.ceilings) | |
| 163 | + | if err != nil { | |
| 164 | + | // The handshake already closed a refused carriage; Close is idempotent. | |
| 165 | + | _ = bridge.Close() | |
| 166 | + | return err | |
| 167 | + | } | |
| 168 | + | if err := bridge.Negotiate(result.Ceilings); err != nil { | |
| 169 | + | _ = bridge.Close() | |
| 170 | + | return err | |
| 171 | + | } | |
| 172 | + | ||
| 173 | + | pump, err := NewPump(bridge, DirectionClient, result.Ceilings) | |
| 174 | + | if err != nil { | |
| 175 | + | _ = bridge.Close() | |
| 176 | + | return err | |
| 177 | + | } | |
| 178 | + | // Enforce the accepted physical policy before any fresh inbound Open can be | |
| 179 | + | // admitted: the pump's engine must be restricted before the pump starts, | |
| 180 | + | // and the listener below is built before that start too. | |
| 181 | + | if err := pump.Engine().RestrictAdmissions(result.Binding.Policy()); err != nil { | |
| 182 | + | _ = pump.Close() | |
| 183 | + | return err | |
| 184 | + | } | |
| 185 | + | ||
| 186 | + | listener, err := newListener(pump, protocol.HandshakeTimeout, MaxAcceptQueue, s.clock) | |
| 187 | + | if err != nil { | |
| 188 | + | _ = pump.Close() | |
| 189 | + | return err | |
| 190 | + | } | |
| 191 | + | ||
| 192 | + | child := &supervisedPhysical{listener: listener, pump: pump} | |
| 193 | + | child.start(context.WithoutCancel(ctx)) | |
| 194 | + | if err := s.register(child); err != nil { | |
| 195 | + | _ = child.Close() | |
| 196 | + | return err | |
| 197 | + | } | |
| 198 | + | return nil | |
| 199 | + | } | |
| 200 | + | ||
| 201 | + | // Children reports how many physical connections the supervisor currently | |
| 202 | + | // owns. It is a diagnostic accessor; a lost child disappears once its reaper | |
| 203 | + | // has closed it. | |
| 204 | + | func (s *ServerSupervisor) Children() int { | |
| 205 | + | if s == nil { | |
| 206 | + | return 0 | |
| 207 | + | } | |
| 208 | + | s.mu.Lock() | |
| 209 | + | defer s.mu.Unlock() | |
| 210 | + | return len(s.children) | |
| 211 | + | } | |
| 212 | + | ||
| 213 | + | // Close stops accepting physical carriage and tears every owned child down: | |
| 214 | + | // it deregisters and closes each child - listener, pump, and carrier together - | |
| 215 | + | // and joins every reaper. Close is idempotent and concurrent-safe. It leaves | |
| 216 | + | // the caller-owned AggregateListener open; the daemon closes that itself. | |
| 217 | + | func (s *ServerSupervisor) Close() error { | |
| 218 | + | if s == nil { | |
| 219 | + | return nil | |
| 220 | + | } | |
| 221 | + | s.closeOnce.Do(func() { | |
| 222 | + | s.mu.Lock() | |
| 223 | + | s.closed = true | |
| 224 | + | children := make([]*supervisedPhysical, 0, len(s.children)) | |
| 225 | + | for child := range s.children { | |
| 226 | + | children = append(children, child) | |
| 227 | + | } | |
| 228 | + | s.mu.Unlock() | |
| 229 | + | close(s.done) | |
| 230 | + | for _, child := range children { | |
| 231 | + | // Remove closes the child and joins its aggregate forwarder. A | |
| 232 | + | // child its own reaper already deregistered reports | |
| 233 | + | // ErrAggregateUnknownChild and is closed below regardless. | |
| 234 | + | _ = s.aggregate.Remove(child) | |
| 235 | + | _ = child.Close() | |
| 236 | + | s.forget(child) | |
| 237 | + | } | |
| 238 | + | s.wg.Wait() | |
| 239 | + | }) | |
| 240 | + | return nil | |
| 241 | + | } | |
| 242 | + | ||
| 243 | + | // acquire takes one admission slot, blocking until one is free or ctx or the | |
| 244 | + | // supervisor ends. It refuses admission outright once the supervisor closed. | |
| 245 | + | func (s *ServerSupervisor) acquire(ctx context.Context) error { | |
| 246 | + | s.mu.Lock() | |
| 247 | + | closed := s.closed | |
| 248 | + | s.mu.Unlock() | |
| 249 | + | if closed { | |
| 250 | + | return ErrSupervisorClosed | |
| 251 | + | } | |
| 252 | + | select { | |
| 253 | + | case s.slots <- struct{}{}: | |
| 254 | + | return nil | |
| 255 | + | default: | |
| 256 | + | } | |
| 257 | + | select { | |
| 258 | + | case s.slots <- struct{}{}: | |
| 259 | + | return nil | |
| 260 | + | case <-s.done: | |
| 261 | + | return ErrSupervisorClosed | |
| 262 | + | case <-ctx.Done(): | |
| 263 | + | return ctx.Err() | |
| 264 | + | } | |
| 265 | + | } | |
| 266 | + | ||
| 267 | + | // release returns one admission slot. | |
| 268 | + | func (s *ServerSupervisor) release() { <-s.slots } | |
| 269 | + | ||
| 270 | + | // register takes ownership of one fully wired, already started child and | |
| 271 | + | // publishes it to the aggregate, then starts its reaper. It registers with the | |
| 272 | + | // aggregate and takes the wait-group reference under the same lock that checks | |
| 273 | + | // closed, so Close can never observe a child it did not also own and wait for, | |
| 274 | + | // and no child is ever registered after Close saw the supervisor closed. A | |
| 275 | + | // closed supervisor, or a closed aggregate, refuses the child, which its | |
| 276 | + | // caller closes. | |
| 277 | + | func (s *ServerSupervisor) register(child *supervisedPhysical) error { | |
| 278 | + | s.mu.Lock() | |
| 279 | + | defer s.mu.Unlock() | |
| 280 | + | if s.closed { | |
| 281 | + | return ErrSupervisorClosed | |
| 282 | + | } | |
| 283 | + | if err := s.aggregate.Register(child); err != nil { | |
| 284 | + | return err | |
| 285 | + | } | |
| 286 | + | s.children[child] = struct{}{} | |
| 287 | + | s.wg.Add(1) | |
| 288 | + | go s.reap(child) | |
| 289 | + | return nil | |
| 290 | + | } | |
| 291 | + | ||
| 292 | + | // forget drops one child from the owned set. It is idempotent. | |
| 293 | + | func (s *ServerSupervisor) forget(child *supervisedPhysical) { | |
| 294 | + | s.mu.Lock() | |
| 295 | + | delete(s.children, child) | |
| 296 | + | s.mu.Unlock() | |
| 297 | + | } | |
| 298 | + | ||
| 299 | + | // reap watches one child's pump and closes the child once its carriage reaches | |
| 300 | + | // a terminal outcome, isolating the loss from every sibling and from the | |
| 301 | + | // supervisor. It deregisters the child from the aggregate first (a child whose | |
| 302 | + | // own forwarder already observed the terminal Accept reports | |
| 303 | + | // ErrAggregateUnknownChild) and then closes listener, pump, and carrier | |
| 304 | + | // together. It exits when Close wakes it through done. | |
| 305 | + | func (s *ServerSupervisor) reap(child *supervisedPhysical) { | |
| 306 | + | defer s.wg.Done() | |
| 307 | + | select { | |
| 308 | + | case <-child.pump.Done(): | |
| 309 | + | case <-s.done: | |
| 310 | + | } | |
| 311 | + | _ = s.aggregate.Remove(child) | |
| 312 | + | _ = child.Close() | |
| 313 | + | s.forget(child) | |
| 314 | + | } | |
| 315 | + | ||
| 316 | + | // supervisedPhysical is one owned accepted physical connection. It implements | |
| 317 | + | // ports.ServerListener by delegating Accept to its typed Listener, so the | |
| 318 | + | // aggregate composes it exactly like any other physical listener, while Close | |
| 319 | + | // owns the whole triple: listener, pump, and carrier. | |
| 320 | + | type supervisedPhysical struct { | |
| 321 | + | listener *Listener | |
| 322 | + | pump *Pump | |
| 323 | + | ||
| 324 | + | closeOnce sync.Once | |
| 325 | + | closeErr error | |
| 326 | + | } | |
| 327 | + | ||
| 328 | + | var _ ports.ServerListener = (*supervisedPhysical)(nil) | |
| 329 | + | ||
| 330 | + | // start launches the child's pump. The listener was already constructed and | |
| 331 | + | // registered its admission observer, so no admitted start can be missed. | |
| 332 | + | func (c *supervisedPhysical) start(ctx context.Context) { | |
| 333 | + | c.pump.Start(ctx) | |
| 334 | + | } | |
| 335 | + | ||
| 336 | + | // Addr names the child's carriage for ports.ServerListener. | |
| 337 | + | func (c *supervisedPhysical) Addr() string { | |
| 338 | + | if c == nil || c.listener == nil { | |
| 339 | + | return "daemonmux" | |
| 340 | + | } | |
| 341 | + | return c.listener.Addr() | |
| 342 | + | } | |
| 343 | + | ||
| 344 | + | // Accept returns the next typed connection from the child's listener. | |
| 345 | + | func (c *supervisedPhysical) Accept() (ports.ServerConnection, error) { | |
| 346 | + | if c == nil || c.listener == nil { | |
| 347 | + | return nil, ErrSupervisorConfig | |
| 348 | + | } | |
| 349 | + | return c.listener.Accept() | |
| 350 | + | } | |
| 351 | + | ||
| 352 | + | // Close tears the child down exactly once: it closes the listener (unblocking | |
| 353 | + | // and joining its watchdog), then the pump (cancelling the carriage goroutines | |
| 354 | + | // and closing the carrier). Every caller observes the same pump close error. | |
| 355 | + | func (c *supervisedPhysical) Close() error { | |
| 356 | + | if c == nil { | |
| 357 | + | return nil | |
| 358 | + | } | |
| 359 | + | c.closeOnce.Do(func() { | |
| 360 | + | if c.listener != nil { | |
| 361 | + | _ = c.listener.Close() | |
| 362 | + | } | |
| 363 | + | if c.pump != nil { | |
| 364 | + | c.closeErr = c.pump.Close() | |
| 365 | + | } | |
| 366 | + | }) | |
| 367 | + | return c.closeErr | |
| 368 | + | } | |
A internal/adapters/daemonmux/supervisor_test.go +237 -0
| @@ -0,0 +1,237 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "testing" | |
| 6 | + | "time" | |
| 7 | + | ||
| 8 | + | "github.com/stretchr/testify/require" | |
| 9 | + | ||
| 10 | + | "github.com/bnema/vev/internal/ports" | |
| 11 | + | "github.com/bnema/vev/internal/protocol" | |
| 12 | + | ) | |
| 13 | + | ||
| 14 | + | // connectRaw builds one broker PhysicalConnection over an already established | |
| 15 | + | // raw carriage whose daemon end was handed to a supervisor. | |
| 16 | + | func connectRaw(t *testing.T, raw RawFramedTransport, endpoint ports.BrokerResolvedEndpoint) ports.BrokerPhysicalConnection { | |
| 17 | + | t.Helper() | |
| 18 | + | connector, err := NewEndpointConnector(func(context.Context, string) (RawFramedTransport, error) { | |
| 19 | + | return raw, nil | |
| 20 | + | }, DefaultMuxCeilings()) | |
| 21 | + | require.NoError(t, err) | |
| 22 | + | physical, err := connector.Connect(context.Background(), endpoint) | |
| 23 | + | require.NoError(t, err) | |
| 24 | + | require.NotNil(t, physical) | |
| 25 | + | t.Cleanup(func() { _ = physical.Close() }) | |
| 26 | + | return physical | |
| 27 | + | } | |
| 28 | + | ||
| 29 | + | // awaitHandshakeBridge performs the broker side of the daemonmux physical | |
| 30 | + | // preamble over raw and returns the negotiated carrier, ready for a pump. | |
| 31 | + | func awaitHandshakeBridge(t *testing.T, raw RawFramedTransport, endpoint ports.BrokerResolvedEndpoint) (FramedCarrier, ClientHandshakeResult) { | |
| 32 | + | t.Helper() | |
| 33 | + | bridge, err := NewPreambleCarrier(raw) | |
| 34 | + | require.NoError(t, err) | |
| 35 | + | result, err := RunClientHandshake(context.Background(), bridge, endpoint, DefaultMuxCeilings()) | |
| 36 | + | require.NoError(t, err) | |
| 37 | + | require.NoError(t, bridge.Negotiate(result.Ceilings)) | |
| 38 | + | return bridge, result | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | // TestNewServerSupervisorValidation proves the supervisor refuses a nil | |
| 42 | + | // aggregate, an invalid binding, an invalid ceiling advertisement, and a nil | |
| 43 | + | // carriage. | |
| 44 | + | func TestNewServerSupervisorValidation(t *testing.T) { | |
| 45 | + | binding := mustServerBinding(t) | |
| 46 | + | ||
| 47 | + | _, err := NewServerSupervisor(nil, binding, DefaultMuxCeilings(), 0) | |
| 48 | + | require.ErrorIs(t, err, ErrSupervisorConfig) | |
| 49 | + | ||
| 50 | + | _, err = NewServerSupervisor(NewAggregateListener(), ServerBinding{}, DefaultMuxCeilings(), 0) | |
| 51 | + | require.ErrorIs(t, err, ErrSupervisorConfig) | |
| 52 | + | ||
| 53 | + | _, err = NewServerSupervisor(NewAggregateListener(), binding, MuxCeilings{}, 0) | |
| 54 | + | require.ErrorIs(t, err, ErrSupervisorConfig) | |
| 55 | + | ||
| 56 | + | supervisor, err := NewServerSupervisor(NewAggregateListener(), binding, DefaultMuxCeilings(), 0) | |
| 57 | + | require.NoError(t, err) | |
| 58 | + | t.Cleanup(func() { _ = supervisor.Close() }) | |
| 59 | + | require.ErrorIs(t, supervisor.Adopt(context.Background(), nil), ErrSupervisorConfig) | |
| 60 | + | } | |
| 61 | + | ||
| 62 | + | // TestServerSupervisorFailedHandshakeRegistersNoChild proves a preamble that | |
| 63 | + | // the binding refuses closes the carriage, admits no stream, and registers no | |
| 64 | + | // owned child with the aggregate. | |
| 65 | + | func TestServerSupervisorFailedHandshakeRegistersNoChild(t *testing.T) { | |
| 66 | + | binding := mustServerBinding(t) | |
| 67 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 68 | + | ||
| 69 | + | clientRaw, serverRaw := rawCarriagePair(t) | |
| 70 | + | server.adopt(context.Background(), serverRaw) | |
| 71 | + | ||
| 72 | + | bridge, err := NewPreambleCarrier(clientRaw) | |
| 73 | + | require.NoError(t, err) | |
| 74 | + | t.Cleanup(func() { _ = bridge.Close() }) | |
| 75 | + | _, err = RunClientHandshake(context.Background(), bridge, muxEndpoint(binding.Identity(), alternatePolicy(binding.Policy()), "raw://refused"), DefaultMuxCeilings()) | |
| 76 | + | require.ErrorIs(t, err, ErrHandshakeRejected) | |
| 77 | + | ||
| 78 | + | require.ErrorIs(t, server.awaitAdoption(t), ErrHandshakeRejected) | |
| 79 | + | require.Zero(t, server.supervisor.Children(), "a refused handshake admits no physical child") | |
| 80 | + | require.Zero(t, server.acceptedCount(), "a refused handshake admits no stream") | |
| 81 | + | } | |
| 82 | + | ||
| 83 | + | // TestServerSupervisorEnforcesAcceptedPolicyBeforeAdmission proves the accepted | |
| 84 | + | // physical policy is enforced on every inbound Open before fresh stream | |
| 85 | + | // admission: a broker that completed the handshake at the accepted policy but | |
| 86 | + | // names a different policy in an Open is refused on that stream alone, while a | |
| 87 | + | // matching Open is admitted to the daemon's accept stream. | |
| 88 | + | func TestServerSupervisorEnforcesAcceptedPolicyBeforeAdmission(t *testing.T) { | |
| 89 | + | binding := mustServerBinding(t) | |
| 90 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 91 | + | server.serve() | |
| 92 | + | endpoint := muxEndpoint(binding.Identity(), binding.Policy(), "raw://stream-policy") | |
| 93 | + | ||
| 94 | + | clientRaw, serverRaw := rawCarriagePair(t) | |
| 95 | + | server.adopt(context.Background(), serverRaw) | |
| 96 | + | carrier, result := awaitHandshakeBridge(t, clientRaw, endpoint) | |
| 97 | + | require.NoError(t, server.awaitAdoption(t)) | |
| 98 | + | ||
| 99 | + | pump, err := NewPump(carrier, DirectionServer, result.Ceilings) | |
| 100 | + | require.NoError(t, err) | |
| 101 | + | t.Cleanup(func() { _ = pump.Close() }) | |
| 102 | + | pump.Start(context.Background()) | |
| 103 | + | logical, err := NewLogicalConnector(pump) | |
| 104 | + | require.NoError(t, err) | |
| 105 | + | ||
| 106 | + | _, err = logical.Open(context.Background(), muxOpenRequest(1, alternatePolicy(binding.Policy()))) | |
| 107 | + | require.ErrorIs(t, err, ports.BrokerAdmissionInvalid) | |
| 108 | + | require.Zero(t, server.acceptedCount(), "a policy-mismatched open never reaches the accept stream") | |
| 109 | + | ||
| 110 | + | connection, err := logical.Open(context.Background(), muxOpenRequest(2, binding.Policy())) | |
| 111 | + | require.NoError(t, err) | |
| 112 | + | require.NotNil(t, connection) | |
| 113 | + | server.awaitAccepted(t, 1) | |
| 114 | + | require.NoError(t, connection.SendClient(protocol.Ping{})) | |
| 115 | + | message, err := connection.ReceiveServer() | |
| 116 | + | require.NoError(t, err) | |
| 117 | + | require.Equal(t, protocol.Pong{}, message) | |
| 118 | + | } | |
| 119 | + | ||
| 120 | + | // TestServerSupervisorServerLossIsolated proves one lost physical child never | |
| 121 | + | // stops the aggregate: a surviving child keeps accepting while the lost child | |
| 122 | + | // is reaped and removed. | |
| 123 | + | func TestServerSupervisorServerLossIsolated(t *testing.T) { | |
| 124 | + | binding := mustServerBinding(t) | |
| 125 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 126 | + | server.serve() | |
| 127 | + | endpoint := muxEndpoint(binding.Identity(), binding.Policy(), "raw://isolation") | |
| 128 | + | ||
| 129 | + | clientA, serverA := rawCarriagePair(t) | |
| 130 | + | server.adopt(context.Background(), serverA) | |
| 131 | + | physicalA := connectRaw(t, clientA, endpoint) | |
| 132 | + | require.NoError(t, server.awaitAdoption(t)) | |
| 133 | + | ||
| 134 | + | clientB, serverB := rawCarriagePair(t) | |
| 135 | + | server.adopt(context.Background(), serverB) | |
| 136 | + | physicalB := connectRaw(t, clientB, endpoint) | |
| 137 | + | require.NoError(t, server.awaitAdoption(t)) | |
| 138 | + | ||
| 139 | + | logicalA, err := physicalA.OpenStream(context.Background(), muxOpenRequest(1, binding.Policy())) | |
| 140 | + | require.NoError(t, err) | |
| 141 | + | logicalB, err := physicalB.OpenStream(context.Background(), muxOpenRequest(1, binding.Policy())) | |
| 142 | + | require.NoError(t, err) | |
| 143 | + | server.awaitAccepted(t, 2) | |
| 144 | + | require.Equal(t, 2, server.supervisor.Children()) | |
| 145 | + | ||
| 146 | + | // Lose child A's carriage entirely. | |
| 147 | + | _ = serverA.Close() | |
| 148 | + | require.Eventually(t, func() bool { return channelClosed(physicalA.Done()) }, p3cTestDeadline, time.Millisecond, | |
| 149 | + | "the lost physical connection never reached its terminal outcome") | |
| 150 | + | require.Eventually(t, func() bool { return server.supervisor.Children() == 1 }, p3cTestDeadline, time.Millisecond, | |
| 151 | + | "the lost child was never reaped") | |
| 152 | + | ||
| 153 | + | // The surviving child still accepts and the aggregate never failed. | |
| 154 | + | logicalB2, err := physicalB.OpenStream(context.Background(), muxOpenRequest(2, binding.Policy())) | |
| 155 | + | require.NoError(t, err) | |
| 156 | + | require.NotNil(t, logicalB2) | |
| 157 | + | server.awaitAccepted(t, 3) | |
| 158 | + | ||
| 159 | + | // The lost physical refuses a fresh stream while its published logical | |
| 160 | + | // stream reports the physical loss. | |
| 161 | + | _, err = physicalA.OpenStream(context.Background(), muxOpenRequest(2, binding.Policy())) | |
| 162 | + | require.Error(t, err) | |
| 163 | + | require.True(t, channelClosed(logicalA.Done())) | |
| 164 | + | require.False(t, channelClosed(logicalB.Done())) | |
| 165 | + | } | |
| 166 | + | ||
| 167 | + | // TestServerSupervisorCloseTearsDownOwnedChildren proves Close removes and | |
| 168 | + | // closes every owned child, refuses later carriage, and leaves the aggregate | |
| 169 | + | // open. | |
| 170 | + | func TestServerSupervisorCloseTearsDownOwnedChildren(t *testing.T) { | |
| 171 | + | binding := mustServerBinding(t) | |
| 172 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 173 | + | endpoint := muxEndpoint(binding.Identity(), binding.Policy(), "raw://close") | |
| 174 | + | ||
| 175 | + | clientA, serverA := rawCarriagePair(t) | |
| 176 | + | server.adopt(context.Background(), serverA) | |
| 177 | + | physicalA := connectRaw(t, clientA, endpoint) | |
| 178 | + | require.NoError(t, server.awaitAdoption(t)) | |
| 179 | + | ||
| 180 | + | clientB, serverB := rawCarriagePair(t) | |
| 181 | + | server.adopt(context.Background(), serverB) | |
| 182 | + | physicalB := connectRaw(t, clientB, endpoint) | |
| 183 | + | require.NoError(t, server.awaitAdoption(t)) | |
| 184 | + | require.Equal(t, 2, server.supervisor.Children()) | |
| 185 | + | ||
| 186 | + | require.NoError(t, server.supervisor.Close()) | |
| 187 | + | require.Zero(t, server.supervisor.Children()) | |
| 188 | + | require.Eventually(t, func() bool { | |
| 189 | + | return channelClosed(physicalA.Done()) && channelClosed(physicalB.Done()) | |
| 190 | + | }, p3cTestDeadline, time.Millisecond, "Close did not tear down every owned carriage") | |
| 191 | + | ||
| 192 | + | // A closed supervisor refuses later carriage and closes it. | |
| 193 | + | lateClient, lateServer := rawCarriagePair(t) | |
| 194 | + | t.Cleanup(func() { _ = lateClient.Close() }) | |
| 195 | + | require.ErrorIs(t, server.supervisor.Adopt(context.Background(), lateServer), ErrSupervisorClosed) | |
| 196 | + | ||
| 197 | + | // Close is idempotent. | |
| 198 | + | require.NoError(t, server.supervisor.Close()) | |
| 199 | + | } | |
| 200 | + | ||
| 201 | + | // TestServerSupervisorBoundsConcurrentAdmissions proves admission is bounded: a | |
| 202 | + | // second carriage waits while the first holds the only admission slot in its | |
| 203 | + | // handshake, and proceeds once the first finishes. | |
| 204 | + | func TestServerSupervisorBoundsConcurrentAdmissions(t *testing.T) { | |
| 205 | + | binding := mustServerBinding(t) | |
| 206 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 1, nil) | |
| 207 | + | endpoint := muxEndpoint(binding.Identity(), binding.Policy(), "raw://bounds") | |
| 208 | + | ||
| 209 | + | clientA, serverA := rawCarriagePair(t) | |
| 210 | + | server.adopt(context.Background(), serverA) | |
| 211 | + | require.Eventually(t, func() bool { return len(server.supervisor.slots) == 1 }, p3cTestDeadline, time.Millisecond, | |
| 212 | + | "the first admission never took the only slot") | |
| 213 | + | ||
| 214 | + | clientB, serverB := rawCarriagePair(t) | |
| 215 | + | admittedB := make(chan error, 1) | |
| 216 | + | go func() { admittedB <- server.supervisor.Adopt(context.Background(), serverB) }() | |
| 217 | + | select { | |
| 218 | + | case err := <-admittedB: | |
| 219 | + | t.Fatalf("the admission bound did not hold the second carriage: %v", err) | |
| 220 | + | case <-time.After(150 * time.Millisecond): | |
| 221 | + | } | |
| 222 | + | ||
| 223 | + | // Finish the first handshake; the slot is released and the second proceeds. | |
| 224 | + | bridgeA, _ := awaitHandshakeBridge(t, clientA, endpoint) | |
| 225 | + | t.Cleanup(func() { _ = bridgeA.Close() }) | |
| 226 | + | require.NoError(t, server.awaitAdoption(t)) | |
| 227 | + | ||
| 228 | + | bridgeB, _ := awaitHandshakeBridge(t, clientB, endpoint) | |
| 229 | + | t.Cleanup(func() { _ = bridgeB.Close() }) | |
| 230 | + | select { | |
| 231 | + | case err := <-admittedB: | |
| 232 | + | require.NoError(t, err) | |
| 233 | + | case <-time.After(p3cTestDeadline): | |
| 234 | + | t.Fatal("the second carriage never proceeded after the first released its slot") | |
| 235 | + | } | |
| 236 | + | require.Equal(t, 2, server.supervisor.Children()) | |
| 237 | + | } | |
A internal/adapters/daemonmux/terminal.go +120 -0
| @@ -0,0 +1,120 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "errors" | |
| 5 | + | "sync" | |
| 6 | + | ||
| 7 | + | "github.com/bnema/vev/internal/domain" | |
| 8 | + | ) | |
| 9 | + | ||
| 10 | + | // ErrTerminalFailure is the stable package sentinel published by Fail when it | |
| 11 | + | // is called with a nil cause. A failure always pairs a non-nil Err() with a | |
| 12 | + | // nonzero RemoteFailureKind, so Fail substitutes this sentinel for a nil cause | |
| 13 | + | // rather than publish an inconsistent terminal outcome. | |
| 14 | + | var ErrTerminalFailure = errors.New("daemonmux: terminal failure without a cause") | |
| 15 | + | ||
| 16 | + | // terminalState is the publish-once terminal authority shared by a physical | |
| 17 | + | // daemon connection and every logical stream it owns. It mirrors the | |
| 18 | + | // Done/Err/FailureKind contract of ports.BrokerPhysicalConnection: once the | |
| 19 | + | // connection reaches a terminal outcome, every getter returns the same stable | |
| 20 | + | // values and Done is closed exactly once, independent of reads. | |
| 21 | + | // | |
| 22 | + | // Exactly one publication wins. The first concurrent Fail or Close decides Err | |
| 23 | + | // and FailureKind; later publishers observe the winner and never change it. An | |
| 24 | + | // orderly local Close publishes a nil error and RemoteFailureNone. A failure | |
| 25 | + | // publishes its cause and a nonzero valid RemoteFailureKind, preserves the | |
| 26 | + | // error identity for errors.Is/As, and normalizes any kind outside the | |
| 27 | + | // transport-failure taxonomy (including RemoteFailureNone) to | |
| 28 | + | // RemoteFailureTransport. A failure never publishes a nil error: a nil cause is | |
| 29 | + | // replaced with the ErrTerminalFailure sentinel. | |
| 30 | + | type terminalState struct { | |
| 31 | + | once sync.Once | |
| 32 | + | done chan struct{} | |
| 33 | + | err error | |
| 34 | + | kind domain.RemoteFailureKind | |
| 35 | + | } | |
| 36 | + | ||
| 37 | + | // newTerminalState returns an open terminal state. Done stays open until a | |
| 38 | + | // publisher wins. | |
| 39 | + | func newTerminalState() *terminalState { | |
| 40 | + | return &terminalState{done: make(chan struct{})} | |
| 41 | + | } | |
| 42 | + | ||
| 43 | + | // Done returns a channel closed exactly once when the terminal outcome is | |
| 44 | + | // published. The channel is stable and this method is safe before or after | |
| 45 | + | // publication. | |
| 46 | + | func (s *terminalState) Done() <-chan struct{} { return s.done } | |
| 47 | + | ||
| 48 | + | // Err returns the published failure cause: nil while open and for an orderly | |
| 49 | + | // local Close. The result is stable once Done is closed. | |
| 50 | + | func (s *terminalState) Err() error { | |
| 51 | + | select { | |
| 52 | + | case <-s.done: | |
| 53 | + | return s.err | |
| 54 | + | default: | |
| 55 | + | return nil | |
| 56 | + | } | |
| 57 | + | } | |
| 58 | + | ||
| 59 | + | // FailureKind returns the published failure classification: RemoteFailureNone | |
| 60 | + | // while open and for an orderly local Close. The result is stable once Done is | |
| 61 | + | // closed. | |
| 62 | + | func (s *terminalState) FailureKind() domain.RemoteFailureKind { | |
| 63 | + | select { | |
| 64 | + | case <-s.done: | |
| 65 | + | return s.kind | |
| 66 | + | default: | |
| 67 | + | return domain.RemoteFailureNone | |
| 68 | + | } | |
| 69 | + | } | |
| 70 | + | ||
| 71 | + | // Fail publishes a failure terminal outcome. It wins only while the state is | |
| 72 | + | // open, so at most one publisher ever applies. The kind must be a nonzero | |
| 73 | + | // valid RemoteFailureKind; any other value, including RemoteFailureNone, is | |
| 74 | + | // normalized to RemoteFailureTransport. A non-nil error is stored verbatim so | |
| 75 | + | // callers keep errors.Is/As identity; a nil error is replaced with the | |
| 76 | + | // ErrTerminalFailure package sentinel so a failure never publishes a nil Err() | |
| 77 | + | // alongside a nonzero kind. | |
| 78 | + | func (s *terminalState) Fail(kind domain.RemoteFailureKind, err error) { | |
| 79 | + | s.publishFail(kind, err) | |
| 80 | + | } | |
| 81 | + | ||
| 82 | + | // publishFail applies a failure outcome exactly once and reports whether this | |
| 83 | + | // call won the publication. It is the primitive behind Fail and the callers | |
| 84 | + | // that must know they are the one publisher - for example, so a stream signals | |
| 85 | + | // its peer with exactly one Reset instead of one per observer. The first | |
| 86 | + | // publisher's values win; a later caller observes false and must not apply its | |
| 87 | + | // own outcome. | |
| 88 | + | func (s *terminalState) publishFail(kind domain.RemoteFailureKind, err error) bool { | |
| 89 | + | if kind < domain.RemoteFailureTransport || kind > domain.RemoteFailureInvalidResponse { | |
| 90 | + | kind = domain.RemoteFailureTransport | |
| 91 | + | } | |
| 92 | + | if err == nil { | |
| 93 | + | err = ErrTerminalFailure | |
| 94 | + | } | |
| 95 | + | won := false | |
| 96 | + | s.once.Do(func() { | |
| 97 | + | s.err = err | |
| 98 | + | s.kind = kind | |
| 99 | + | close(s.done) | |
| 100 | + | won = true | |
| 101 | + | }) | |
| 102 | + | return won | |
| 103 | + | } | |
| 104 | + | ||
| 105 | + | // Close publishes an orderly local terminal outcome: a nil error and | |
| 106 | + | // RemoteFailureNone. It wins only while the state is open. | |
| 107 | + | func (s *terminalState) Close() { | |
| 108 | + | s.publish(nil, domain.RemoteFailureNone) | |
| 109 | + | } | |
| 110 | + | ||
| 111 | + | // publish applies the first outcome exactly once and then closes Done. The | |
| 112 | + | // winning values are stored before close(done), so every reader that observes | |
| 113 | + | // the closed channel sees them. | |
| 114 | + | func (s *terminalState) publish(err error, kind domain.RemoteFailureKind) { | |
| 115 | + | s.once.Do(func() { | |
| 116 | + | s.err = err | |
| 117 | + | s.kind = kind | |
| 118 | + | close(s.done) | |
| 119 | + | }) | |
| 120 | + | } | |
A internal/adapters/daemonmux/terminal_test.go +283 -0
| @@ -0,0 +1,283 @@ | |||
| 1 | + | package daemonmux | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "errors" | |
| 5 | + | "fmt" | |
| 6 | + | "sync" | |
| 7 | + | "sync/atomic" | |
| 8 | + | "testing" | |
| 9 | + | ||
| 10 | + | "github.com/bnema/vev/internal/domain" | |
| 11 | + | ) | |
| 12 | + | ||
| 13 | + | // errTerminalSentinel is a stable cause used across tests so error identity | |
| 14 | + | // can be checked with errors.Is. | |
| 15 | + | var errTerminalSentinel = errors.New("terminal sentinel") | |
| 16 | + | ||
| 17 | + | func terminalDone(s *terminalState) bool { | |
| 18 | + | select { | |
| 19 | + | case <-s.Done(): | |
| 20 | + | return true | |
| 21 | + | default: | |
| 22 | + | return false | |
| 23 | + | } | |
| 24 | + | } | |
| 25 | + | ||
| 26 | + | func TestTerminalStatePreTerminalGetters(t *testing.T) { | |
| 27 | + | s := newTerminalState() | |
| 28 | + | ||
| 29 | + | if terminalDone(s) { | |
| 30 | + | t.Fatal("Done closed before any publication") | |
| 31 | + | } | |
| 32 | + | if err := s.Err(); err != nil { | |
| 33 | + | t.Fatalf("Err() = %v before publication, want nil", err) | |
| 34 | + | } | |
| 35 | + | if kind := s.FailureKind(); kind != domain.RemoteFailureNone { | |
| 36 | + | t.Fatalf("FailureKind() = %v before publication, want none", kind) | |
| 37 | + | } | |
| 38 | + | } | |
| 39 | + | ||
| 40 | + | func TestTerminalStateLocalClose(t *testing.T) { | |
| 41 | + | s := newTerminalState() | |
| 42 | + | s.Close() | |
| 43 | + | ||
| 44 | + | if !terminalDone(s) { | |
| 45 | + | t.Fatal("Done not closed after local Close") | |
| 46 | + | } | |
| 47 | + | if err := s.Err(); err != nil { | |
| 48 | + | t.Fatalf("Err() = %v after local Close, want nil", err) | |
| 49 | + | } | |
| 50 | + | if kind := s.FailureKind(); kind != domain.RemoteFailureNone { | |
| 51 | + | t.Fatalf("FailureKind() = %v after local Close, want none", kind) | |
| 52 | + | } | |
| 53 | + | ||
| 54 | + | // A second local Close stays orderly, and a later failure must not | |
| 55 | + | // overwrite the winning outcome. | |
| 56 | + | s.Close() | |
| 57 | + | s.Fail(domain.RemoteFailureTimeout, errors.New("late failure")) | |
| 58 | + | if err := s.Err(); err != nil { | |
| 59 | + | t.Fatalf("Err() = %v after late Fail, want nil", err) | |
| 60 | + | } | |
| 61 | + | if kind := s.FailureKind(); kind != domain.RemoteFailureNone { | |
| 62 | + | t.Fatalf("FailureKind() = %v after late Fail, want none", kind) | |
| 63 | + | } | |
| 64 | + | } | |
| 65 | + | ||
| 66 | + | type wrappedError struct{ err error } | |
| 67 | + | ||
| 68 | + | func (w wrappedError) Error() string { return w.err.Error() } | |
| 69 | + | func (w wrappedError) Unwrap() error { return w.err } | |
| 70 | + | ||
| 71 | + | func TestTerminalStateFailurePreservesErrorIdentity(t *testing.T) { | |
| 72 | + | s := newTerminalState() | |
| 73 | + | cause := wrappedError{err: fmt.Errorf("dial daemon: %w", errTerminalSentinel)} | |
| 74 | + | s.Fail(domain.RemoteFailureTimeout, cause) | |
| 75 | + | ||
| 76 | + | if !terminalDone(s) { | |
| 77 | + | t.Fatal("Done not closed after Fail") | |
| 78 | + | } | |
| 79 | + | if got := s.Err(); got != cause { | |
| 80 | + | t.Fatalf("Err() = %v, want the exact published error value", got) | |
| 81 | + | } | |
| 82 | + | if !errors.Is(s.Err(), errTerminalSentinel) { | |
| 83 | + | t.Fatal("errors.Is lost the wrapped sentinel") | |
| 84 | + | } | |
| 85 | + | var wrapped wrappedError | |
| 86 | + | if !errors.As(s.Err(), &wrapped) { | |
| 87 | + | t.Fatal("errors.As lost the wrapped error identity") | |
| 88 | + | } | |
| 89 | + | if kind := s.FailureKind(); kind != domain.RemoteFailureTimeout { | |
| 90 | + | t.Fatalf("FailureKind() = %v, want timeout", kind) | |
| 91 | + | } | |
| 92 | + | ||
| 93 | + | // A later orderly Close must not clear the published failure. | |
| 94 | + | s.Close() | |
| 95 | + | if s.Err() != cause || s.FailureKind() != domain.RemoteFailureTimeout { | |
| 96 | + | t.Fatalf("Close overwrote failure: Err()=%v FailureKind()=%v", s.Err(), s.FailureKind()) | |
| 97 | + | } | |
| 98 | + | } | |
| 99 | + | ||
| 100 | + | // TestTerminalStateNilCauseUsesSentinel proves a failure publisher that | |
| 101 | + | // supplies no cause still publishes a non-nil Err() carrying the package | |
| 102 | + | // sentinel, paired with a nonzero valid kind. | |
| 103 | + | func TestTerminalStateNilCauseUsesSentinel(t *testing.T) { | |
| 104 | + | s := newTerminalState() | |
| 105 | + | s.Fail(domain.RemoteFailureTimeout, nil) | |
| 106 | + | ||
| 107 | + | if !terminalDone(s) { | |
| 108 | + | t.Fatal("Done not closed after Fail with a nil cause") | |
| 109 | + | } | |
| 110 | + | if s.Err() == nil { | |
| 111 | + | t.Fatal("Err() is nil after Fail with a nil cause") | |
| 112 | + | } | |
| 113 | + | if !errors.Is(s.Err(), ErrTerminalFailure) { | |
| 114 | + | t.Fatalf("Err() = %v, want ErrTerminalFailure", s.Err()) | |
| 115 | + | } | |
| 116 | + | if kind := s.FailureKind(); kind != domain.RemoteFailureTimeout { | |
| 117 | + | t.Fatalf("FailureKind() = %v, want timeout", kind) | |
| 118 | + | } | |
| 119 | + | // A failure always pairs a non-nil error with a nonzero valid kind, so a | |
| 120 | + | // concurrent reader never observes the inconsistent (nil, nonzero) pair. | |
| 121 | + | if (s.Err() == nil) != (s.FailureKind() == domain.RemoteFailureNone) { | |
| 122 | + | t.Fatal("failure published an inconsistent (err, kind) pair") | |
| 123 | + | } | |
| 124 | + | ||
| 125 | + | // A nil cause with an out-of-range kind still normalizes the kind and | |
| 126 | + | // substitutes the sentinel. | |
| 127 | + | serialized := newTerminalState() | |
| 128 | + | serialized.Fail(domain.RemoteFailureKind(99), nil) | |
| 129 | + | if !errors.Is(serialized.Err(), ErrTerminalFailure) { | |
| 130 | + | t.Fatalf("Err() = %v, want ErrTerminalFailure", serialized.Err()) | |
| 131 | + | } | |
| 132 | + | if kind := serialized.FailureKind(); kind != domain.RemoteFailureTransport { | |
| 133 | + | t.Fatalf("FailureKind() = %v, want transport", kind) | |
| 134 | + | } | |
| 135 | + | } | |
| 136 | + | ||
| 137 | + | func TestTerminalStateFailureKindNormalization(t *testing.T) { | |
| 138 | + | tests := []struct { | |
| 139 | + | name string | |
| 140 | + | give domain.RemoteFailureKind | |
| 141 | + | want domain.RemoteFailureKind | |
| 142 | + | }{ | |
| 143 | + | {name: "none normalizes to transport", give: domain.RemoteFailureNone, want: domain.RemoteFailureTransport}, | |
| 144 | + | {name: "out of range normalizes to transport", give: domain.RemoteFailureKind(99), want: domain.RemoteFailureTransport}, | |
| 145 | + | {name: "transport stays", give: domain.RemoteFailureTransport, want: domain.RemoteFailureTransport}, | |
| 146 | + | {name: "timeout stays", give: domain.RemoteFailureTimeout, want: domain.RemoteFailureTimeout}, | |
| 147 | + | {name: "trust stays", give: domain.RemoteFailureTrust, want: domain.RemoteFailureTrust}, | |
| 148 | + | {name: "authentication stays", give: domain.RemoteFailureAuthentication, want: domain.RemoteFailureAuthentication}, | |
| 149 | + | {name: "incompatible stays", give: domain.RemoteFailureIncompatible, want: domain.RemoteFailureIncompatible}, | |
| 150 | + | {name: "invalid response stays", give: domain.RemoteFailureInvalidResponse, want: domain.RemoteFailureInvalidResponse}, | |
| 151 | + | } | |
| 152 | + | for _, tt := range tests { | |
| 153 | + | t.Run(tt.name, func(t *testing.T) { | |
| 154 | + | s := newTerminalState() | |
| 155 | + | s.Fail(tt.give, errors.New("boom")) | |
| 156 | + | if got := s.FailureKind(); got != tt.want { | |
| 157 | + | t.Fatalf("FailureKind() = %v, want %v", got, tt.want) | |
| 158 | + | } | |
| 159 | + | if s.Err() == nil { | |
| 160 | + | t.Fatal("Err() is nil after Fail") | |
| 161 | + | } | |
| 162 | + | }) | |
| 163 | + | } | |
| 164 | + | } | |
| 165 | + | ||
| 166 | + | func TestTerminalStateFirstWinsConcurrentPublishers(t *testing.T) { | |
| 167 | + | const publishers = 100 | |
| 168 | + | validKinds := []domain.RemoteFailureKind{ | |
| 169 | + | domain.RemoteFailureTransport, | |
| 170 | + | domain.RemoteFailureTimeout, | |
| 171 | + | domain.RemoteFailureTrust, | |
| 172 | + | domain.RemoteFailureAuthentication, | |
| 173 | + | domain.RemoteFailureIncompatible, | |
| 174 | + | domain.RemoteFailureInvalidResponse, | |
| 175 | + | } | |
| 176 | + | ||
| 177 | + | s := newTerminalState() | |
| 178 | + | publishedErrs := make([]error, publishers) | |
| 179 | + | publishedKinds := make([]domain.RemoteFailureKind, publishers) | |
| 180 | + | ||
| 181 | + | var wg sync.WaitGroup | |
| 182 | + | start := make(chan struct{}) | |
| 183 | + | for i := 0; i < publishers; i++ { | |
| 184 | + | publishedErrs[i] = fmt.Errorf("publisher %d: %w", i, errTerminalSentinel) | |
| 185 | + | publishedKinds[i] = validKinds[i%len(validKinds)] | |
| 186 | + | wg.Add(1) | |
| 187 | + | go func(i int) { | |
| 188 | + | defer wg.Done() | |
| 189 | + | <-start | |
| 190 | + | s.Fail(publishedKinds[i], publishedErrs[i]) | |
| 191 | + | }(i) | |
| 192 | + | } | |
| 193 | + | close(start) | |
| 194 | + | wg.Wait() | |
| 195 | + | ||
| 196 | + | if !terminalDone(s) { | |
| 197 | + | t.Fatal("Done not closed after concurrent publishers") | |
| 198 | + | } | |
| 199 | + | winnerErr := s.Err() | |
| 200 | + | winnerKind := s.FailureKind() | |
| 201 | + | if winnerErr == nil { | |
| 202 | + | t.Fatal("Err() is nil after failure publishers") | |
| 203 | + | } | |
| 204 | + | if !errors.Is(winnerErr, errTerminalSentinel) { | |
| 205 | + | t.Fatal("winner error lost the sentinel identity") | |
| 206 | + | } | |
| 207 | + | ||
| 208 | + | matches := 0 | |
| 209 | + | for i := range publishedErrs { | |
| 210 | + | if winnerErr != publishedErrs[i] { | |
| 211 | + | continue | |
| 212 | + | } | |
| 213 | + | matches++ | |
| 214 | + | if winnerKind != publishedKinds[i] { | |
| 215 | + | t.Fatalf("winner error from publisher %d paired with kind %v, want %v", i, winnerKind, publishedKinds[i]) | |
| 216 | + | } | |
| 217 | + | } | |
| 218 | + | if matches != 1 { | |
| 219 | + | t.Fatalf("exactly one publisher must win, got %d", matches) | |
| 220 | + | } | |
| 221 | + | ||
| 222 | + | // Later publications, local or failing, never change the winner. | |
| 223 | + | s.Close() | |
| 224 | + | s.Fail(domain.RemoteFailureTrust, errors.New("late failure")) | |
| 225 | + | if s.Err() != winnerErr || s.FailureKind() != winnerKind { | |
| 226 | + | t.Fatalf("terminal values changed: (%v, %v)", s.Err(), s.FailureKind()) | |
| 227 | + | } | |
| 228 | + | } | |
| 229 | + | ||
| 230 | + | func TestTerminalStateConcurrentGettersAndPublishers(t *testing.T) { | |
| 231 | + | const goroutines = 100 | |
| 232 | + | ||
| 233 | + | s := newTerminalState() | |
| 234 | + | var wg sync.WaitGroup | |
| 235 | + | var inconsistent atomic.Int32 | |
| 236 | + | start := make(chan struct{}) | |
| 237 | + | for i := 0; i < goroutines; i++ { | |
| 238 | + | wg.Add(1) | |
| 239 | + | go func(i int) { | |
| 240 | + | defer wg.Done() | |
| 241 | + | <-start | |
| 242 | + | switch i % 4 { | |
| 243 | + | case 0: | |
| 244 | + | s.Fail(domain.RemoteFailureTransport, fmt.Errorf("failure %d: %w", i, errTerminalSentinel)) | |
| 245 | + | case 1: | |
| 246 | + | s.Close() | |
| 247 | + | default: | |
| 248 | + | select { | |
| 249 | + | case <-s.Done(): | |
| 250 | + | err := s.Err() | |
| 251 | + | kind := s.FailureKind() | |
| 252 | + | // Failure pairs a non-nil error with a nonzero valid | |
| 253 | + | // kind; an orderly local Close pairs nil with none. | |
| 254 | + | if (err == nil) != (kind == domain.RemoteFailureNone) { | |
| 255 | + | inconsistent.Add(1) | |
| 256 | + | } | |
| 257 | + | default: | |
| 258 | + | } | |
| 259 | + | } | |
| 260 | + | }(i) | |
| 261 | + | } | |
| 262 | + | close(start) | |
| 263 | + | wg.Wait() | |
| 264 | + | ||
| 265 | + | if !terminalDone(s) { | |
| 266 | + | t.Fatal("Done not closed after concurrent getters and publishers") | |
| 267 | + | } | |
| 268 | + | if inconsistent.Load() != 0 { | |
| 269 | + | t.Fatalf("observed %d inconsistent terminal reads", inconsistent.Load()) | |
| 270 | + | } | |
| 271 | + | ||
| 272 | + | // Exactly-once close: repeated publication never panics and never changes | |
| 273 | + | // the terminal outcome. | |
| 274 | + | gotErr := s.Err() | |
| 275 | + | gotKind := s.FailureKind() | |
| 276 | + | for i := 0; i < 10; i++ { | |
| 277 | + | s.Close() | |
| 278 | + | s.Fail(domain.RemoteFailureTimeout, errors.New("late failure")) | |
| 279 | + | if s.Err() != gotErr || s.FailureKind() != gotKind { | |
| 280 | + | t.Fatalf("terminal values changed: (%v, %v) -> (%v, %v)", gotErr, gotKind, s.Err(), s.FailureKind()) | |
| 281 | + | } | |
| 282 | + | } | |
| 283 | + | } | |
A internal/adapters/daemonmux/unix_carriage_test.go +325 -0
| @@ -0,0 +1,325 @@ | |||
| 1 | + | // Integration coverage for the real private Unix daemonmux carriage (P3.2d). | |
| 2 | + | // | |
| 3 | + | // These tests exercise the daemonmux multiplexer over an actual AF_UNIX socket | |
| 4 | + | // bound by ipc.ListenMux and dialed by ipc.DialMuxContext, not over the | |
| 5 | + | // in-memory carrier used by the unit tests. The physical connection is the one | |
| 6 | + | // real private mux carriage; every logical stream rides it. The full daemon | |
| 7 | + | // path (EndpointConnector -> ServerSupervisor -> AggregateListener -> typed | |
| 8 | + | // Listener) is exercised for the two-then-hundred admissions test, and the raw | |
| 9 | + | // pump path is exercised for the blocked-consumer and reset-isolation checks so | |
| 10 | + | // the stream accounting is asserted directly. | |
| 11 | + | package daemonmux | |
| 12 | + | ||
| 13 | + | import ( | |
| 14 | + | "context" | |
| 15 | + | "os" | |
| 16 | + | "path/filepath" | |
| 17 | + | "sync" | |
| 18 | + | "testing" | |
| 19 | + | "time" | |
| 20 | + | ||
| 21 | + | "github.com/stretchr/testify/require" | |
| 22 | + | ||
| 23 | + | "github.com/bnema/vev/internal/adapters/ipc" | |
| 24 | + | "github.com/bnema/vev/internal/domain" | |
| 25 | + | "github.com/bnema/vev/internal/ports" | |
| 26 | + | "github.com/bnema/vev/internal/protocol" | |
| 27 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 28 | + | ) | |
| 29 | + | ||
| 30 | + | // muxCarriagePath returns an owner-only temporary directory and one mux.sock | |
| 31 | + | // path inside it. The parent is created 0700 (os.MkdirTemp's default) and the | |
| 32 | + | // path stays well inside the cross-platform AF_UNIX limit. | |
| 33 | + | func muxCarriagePath(t *testing.T) string { | |
| 34 | + | t.Helper() | |
| 35 | + | root, err := os.MkdirTemp("/tmp", "m") | |
| 36 | + | require.NoError(t, err) | |
| 37 | + | t.Cleanup(func() { _ = os.RemoveAll(root) }) | |
| 38 | + | return filepath.Join(root, "mux.sock") | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | // unixRawCarriagePair binds a real private mux carriage at a temporary path and | |
| 42 | + | // returns its two ends as raw framed transports, joined by an actual AF_UNIX | |
| 43 | + | // SOCK_STREAM connection. | |
| 44 | + | func unixRawCarriagePair(t *testing.T) (RawFramedTransport, RawFramedTransport) { | |
| 45 | + | t.Helper() | |
| 46 | + | path := muxCarriagePath(t) | |
| 47 | + | listener, err := ipc.ListenMux(path) | |
| 48 | + | require.NoError(t, err) | |
| 49 | + | t.Cleanup(func() { _ = listener.Close() }) | |
| 50 | + | ||
| 51 | + | type acceptOutcome struct { | |
| 52 | + | raw RawFramedTransport | |
| 53 | + | err error | |
| 54 | + | } | |
| 55 | + | accepted := make(chan acceptOutcome, 1) | |
| 56 | + | go func() { | |
| 57 | + | raw, acceptErr := listener.Accept() | |
| 58 | + | if acceptErr != nil { | |
| 59 | + | accepted <- acceptOutcome{err: acceptErr} | |
| 60 | + | return | |
| 61 | + | } | |
| 62 | + | accepted <- acceptOutcome{raw: raw} | |
| 63 | + | }() | |
| 64 | + | ||
| 65 | + | ctx, cancel := context.WithTimeout(context.Background(), p3cTestDeadline) | |
| 66 | + | defer cancel() | |
| 67 | + | dialed, err := ipc.DialMuxContext(ctx, path) | |
| 68 | + | require.NoError(t, err) | |
| 69 | + | ||
| 70 | + | outcome := <-accepted | |
| 71 | + | require.NoError(t, outcome.err) | |
| 72 | + | ||
| 73 | + | client, server := dialed, outcome.raw | |
| 74 | + | t.Cleanup(func() { | |
| 75 | + | _ = client.Close() | |
| 76 | + | _ = server.Close() | |
| 77 | + | }) | |
| 78 | + | return client, server | |
| 79 | + | } | |
| 80 | + | ||
| 81 | + | // unixDialer returns a RawCarrierDialer that establishes the real mux carriage | |
| 82 | + | // at path through ipc.DialMuxContext (which also verifies same-user peer | |
| 83 | + | // credentials). | |
| 84 | + | func unixDialer(path string) RawCarrierDialer { | |
| 85 | + | return func(ctx context.Context, _ string) (RawFramedTransport, error) { | |
| 86 | + | raw, err := ipc.DialMuxContext(ctx, path) | |
| 87 | + | if err != nil { | |
| 88 | + | return nil, err | |
| 89 | + | } | |
| 90 | + | return raw, nil | |
| 91 | + | } | |
| 92 | + | } | |
| 93 | + | ||
| 94 | + | // serveUnix adopts every accepted real mux carriage into the test supervisor | |
| 95 | + | // until the listener closes. | |
| 96 | + | func (s *superviseServer) serveUnix(t *testing.T, listener ipc.MuxListener) { | |
| 97 | + | t.Helper() | |
| 98 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 99 | + | t.Cleanup(cancel) | |
| 100 | + | go func() { | |
| 101 | + | for { | |
| 102 | + | raw, err := listener.Accept() | |
| 103 | + | if err != nil { | |
| 104 | + | return | |
| 105 | + | } | |
| 106 | + | s.adopt(ctx, raw) | |
| 107 | + | } | |
| 108 | + | }() | |
| 109 | + | } | |
| 110 | + | ||
| 111 | + | // soleChildPump returns the pump of the supervisor's one owned physical child, | |
| 112 | + | // so a test can assert the accepted stream accounting of the real carriage. | |
| 113 | + | func (s *superviseServer) soleChildPump(t *testing.T) *Pump { | |
| 114 | + | t.Helper() | |
| 115 | + | s.supervisor.mu.Lock() | |
| 116 | + | defer s.supervisor.mu.Unlock() | |
| 117 | + | require.Len(t, s.supervisor.children, 1, "expected exactly one physical mux child") | |
| 118 | + | for child := range s.supervisor.children { | |
| 119 | + | return child.pump | |
| 120 | + | } | |
| 121 | + | return nil | |
| 122 | + | } | |
| 123 | + | ||
| 124 | + | // TestUnixCarriageSupervisorTwoThenHundredTypedAdmissions drives the full | |
| 125 | + | // daemon path over one real Unix physical connection: EndpointConnector dials | |
| 126 | + | // the private carriage, ServerSupervisor adopts it, the AggregateListener | |
| 127 | + | // delivers independent typed daemon admissions, and every stream's traffic is | |
| 128 | + | // isolated. Only one physical connection and one physical child exist for all | |
| 129 | + | // 102 streams. | |
| 130 | + | func TestUnixCarriageSupervisorTwoThenHundredTypedAdmissions(t *testing.T) { | |
| 131 | + | binding := mustServerBinding(t) | |
| 132 | + | policy := binding.Policy() | |
| 133 | + | path := muxCarriagePath(t) | |
| 134 | + | ||
| 135 | + | listener, err := ipc.ListenMux(path) | |
| 136 | + | require.NoError(t, err) | |
| 137 | + | t.Cleanup(func() { _ = listener.Close() }) | |
| 138 | + | ||
| 139 | + | server := newSuperviseServer(t, binding, DefaultMuxCeilings(), 0, nil) | |
| 140 | + | server.serve() | |
| 141 | + | server.serveUnix(t, listener) | |
| 142 | + | ||
| 143 | + | connector, err := NewEndpointConnector(unixDialer(path), DefaultMuxCeilings()) | |
| 144 | + | require.NoError(t, err) | |
| 145 | + | physical, err := connector.Connect(context.Background(), muxEndpoint(binding.Identity(), policy, path)) | |
| 146 | + | require.NoError(t, err) | |
| 147 | + | t.Cleanup(func() { _ = physical.Close() }) | |
| 148 | + | require.NoError(t, server.awaitAdoption(t)) | |
| 149 | + | ||
| 150 | + | // Two independent typed admissions over the one physical carriage. | |
| 151 | + | first, err := physical.OpenStream(context.Background(), muxOpenRequest(1, policy)) | |
| 152 | + | require.NoError(t, err) | |
| 153 | + | t.Cleanup(func() { _ = first.Close() }) | |
| 154 | + | second, err := physical.OpenStream(context.Background(), muxOpenRequest(2, policy)) | |
| 155 | + | require.NoError(t, err) | |
| 156 | + | t.Cleanup(func() { _ = second.Close() }) | |
| 157 | + | server.awaitAccepted(t, 2) | |
| 158 | + | for _, stream := range []ports.BrokerLogicalConnection{first, second} { | |
| 159 | + | require.NoError(t, stream.SendClient(protocol.Ping{})) | |
| 160 | + | message, err := stream.ReceiveServer() | |
| 161 | + | require.NoError(t, err) | |
| 162 | + | require.Equal(t, protocol.Pong{}, message) | |
| 163 | + | } | |
| 164 | + | ||
| 165 | + | // One hundred more independently admitted typed streams, opened | |
| 166 | + | // concurrently over the same physical carriage. | |
| 167 | + | const extra = 100 | |
| 168 | + | connections := make([]ports.BrokerLogicalConnection, extra) | |
| 169 | + | openErrs := make([]error, extra) | |
| 170 | + | var wg sync.WaitGroup | |
| 171 | + | for i := 0; i < extra; i++ { | |
| 172 | + | wg.Add(1) | |
| 173 | + | go func(i int) { | |
| 174 | + | defer wg.Done() | |
| 175 | + | connections[i], openErrs[i] = physical.OpenStream(context.Background(), muxOpenRequest(uint64(3+i), policy)) | |
| 176 | + | }(i) | |
| 177 | + | } | |
| 178 | + | wg.Wait() | |
| 179 | + | for i, openErr := range openErrs { | |
| 180 | + | require.NoError(t, openErr, "stream %d", i+3) | |
| 181 | + | } | |
| 182 | + | for i, connection := range connections { | |
| 183 | + | require.NotNil(t, connection, "stream %d", i+3) | |
| 184 | + | t.Cleanup(func() { _ = connection.Close() }) | |
| 185 | + | } | |
| 186 | + | ||
| 187 | + | server.awaitAccepted(t, 2+extra) | |
| 188 | + | for i, connection := range connections { | |
| 189 | + | require.NoError(t, connection.SendClient(protocol.Ping{}), "stream %d", i+3) | |
| 190 | + | message, err := connection.ReceiveServer() | |
| 191 | + | require.NoError(t, err, "stream %d", i+3) | |
| 192 | + | require.Equal(t, protocol.Pong{}, message) | |
| 193 | + | } | |
| 194 | + | ||
| 195 | + | // All 102 streams share exactly one physical child and one physical | |
| 196 | + | // connection, and the daemon engine accounts for every admitted stream. | |
| 197 | + | require.Equal(t, 1, server.supervisor.Children(), "all streams must share one physical Unix connection") | |
| 198 | + | pump := server.soleChildPump(t) | |
| 199 | + | require.Eventually(t, func() bool { return pump.Engine().Live() == 2+extra }, p3cTestDeadline, time.Millisecond, | |
| 200 | + | "daemon engine admitted %d of %d streams", pump.Engine().Live(), 2+extra) | |
| 201 | + | ||
| 202 | + | all := make([]ports.BrokerLogicalConnection, 0, 2+extra) | |
| 203 | + | all = append(all, first, second) | |
| 204 | + | all = append(all, connections...) | |
| 205 | + | for _, connection := range all { | |
| 206 | + | require.NoError(t, connection.Close()) | |
| 207 | + | } | |
| 208 | + | require.Eventually(t, func() bool { return pump.Engine().Live() == 0 }, p3cTestDeadline, time.Millisecond, | |
| 209 | + | "daemon engine still holds %d live streams after close", pump.Engine().Live()) | |
| 210 | + | require.False(t, channelClosed(physical.Done()), "closing every stream must leave the physical carriage open") | |
| 211 | + | } | |
| 212 | + | ||
| 213 | + | // unixDaemonPump builds a daemon-side pump over the server end of a real mux | |
| 214 | + | // carriage and returns the raw client end for direct frame injection, so a test | |
| 215 | + | // can drive deterministic inbound traffic over the actual socket. | |
| 216 | + | func unixDaemonPump(t *testing.T) (*Pump, RawFramedTransport) { | |
| 217 | + | t.Helper() | |
| 218 | + | client, server := unixRawCarriagePair(t) | |
| 219 | + | carrier, err := NewPreambleCarrier(server) | |
| 220 | + | require.NoError(t, err) | |
| 221 | + | require.NoError(t, carrier.Negotiate(DefaultMuxCeilings())) | |
| 222 | + | pump, err := NewPump(carrier, DirectionClient, DefaultMuxCeilings()) | |
| 223 | + | require.NoError(t, err) | |
| 224 | + | t.Cleanup(func() { _ = pump.Close() }) | |
| 225 | + | pump.Start(context.Background()) | |
| 226 | + | return pump, client | |
| 227 | + | } | |
| 228 | + | ||
| 229 | + | // injectUnixClientFrame writes one client frame over the real Unix carriage. | |
| 230 | + | func injectUnixClientFrame(t *testing.T, client RawFramedTransport, message ClientMessage) { | |
| 231 | + | t.Helper() | |
| 232 | + | require.NoError(t, client.Send(wire.Envelope{Payload: mustEncodeClient(t, message)})) | |
| 233 | + | } | |
| 234 | + | ||
| 235 | + | // nextUnixServerFrame reads one daemon frame from the real Unix carriage. | |
| 236 | + | func nextUnixServerFrame(t *testing.T, client RawFramedTransport) []byte { | |
| 237 | + | t.Helper() | |
| 238 | + | envelope, err := client.RecvBounded(testEnvelopeCeiling) | |
| 239 | + | require.NoError(t, err) | |
| 240 | + | return envelope.Payload | |
| 241 | + | } | |
| 242 | + | ||
| 243 | + | // TestUnixCarriageBlockedConsumerSiblingProgress proves, over a real Unix | |
| 244 | + | // carriage, that a logical consumer which never drains one stream's inbound | |
| 245 | + | // queue neither blocks the reader nor a sibling: the stalled stream is reset by | |
| 246 | + | // its own bound while the sibling keeps receiving. | |
| 247 | + | func TestUnixCarriageBlockedConsumerSiblingProgress(t *testing.T) { | |
| 248 | + | pump, client := unixDaemonPump(t) | |
| 249 | + | ||
| 250 | + | const siblings = 2 | |
| 251 | + | for i := 1; i <= siblings; i++ { | |
| 252 | + | injectUnixClientFrame(t, client, openFor(PhysicalStreamID(i))) | |
| 253 | + | } | |
| 254 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == siblings }) | |
| 255 | + | for i := 1; i <= siblings; i++ { | |
| 256 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(PhysicalStreamID(i))})) | |
| 257 | + | } | |
| 258 | + | ||
| 259 | + | // The consumer never Takes stream 1: its bounded queue fills. | |
| 260 | + | for i := 0; i < MaxMuxStreamQueueChunks; i++ { | |
| 261 | + | injectUnixClientFrame(t, client, Data{Physical: 1, Data: []byte{byte(i)}}) | |
| 262 | + | } | |
| 263 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { | |
| 264 | + | status, ok := e.Status(1) | |
| 265 | + | return ok && status.QueuedChunks == MaxMuxStreamQueueChunks | |
| 266 | + | }) | |
| 267 | + | ||
| 268 | + | // One more chunk overflows stream 1's own bound; a sibling chunk still | |
| 269 | + | // lands, so neither the reader nor the sibling was blocked. | |
| 270 | + | injectUnixClientFrame(t, client, Data{Physical: 1, Data: []byte{'x'}}) | |
| 271 | + | injectUnixClientFrame(t, client, Data{Physical: 2, Data: []byte("sibling")}) | |
| 272 | + | ||
| 273 | + | taken, ok := takeEventually(t, pump, 2) | |
| 274 | + | require.True(t, ok) | |
| 275 | + | require.Equal(t, []byte("sibling"), taken) | |
| 276 | + | ||
| 277 | + | stalled := mustStatus(t, pump.Engine(), 1) | |
| 278 | + | require.Equal(t, StreamTerminal, stalled.State) | |
| 279 | + | require.ErrorIs(t, stalled.Err, ErrStreamQueueFull) | |
| 280 | + | require.Equal(t, domain.RemoteFailureTransport, stalled.FailureKind) | |
| 281 | + | require.Equal(t, StreamOpen, mustStatus(t, pump.Engine(), 2).State) | |
| 282 | + | require.False(t, pump.Engine().Closed()) | |
| 283 | + | require.False(t, channelClosed(pump.Done())) | |
| 284 | + | ||
| 285 | + | reset := decodeReset(t, nextUnixServerFrame(t, client), DirectionServer) | |
| 286 | + | require.Equal(t, PhysicalStreamID(1), reset.Physical) | |
| 287 | + | require.Equal(t, domain.RemoteFailureTransport, reset.Error.FailureKind) | |
| 288 | + | } | |
| 289 | + | ||
| 290 | + | // TestUnixCarriageResetIsolation proves, over a real Unix carriage, that one | |
| 291 | + | // stream-local refusal schedules exactly one Reset for its own stream and | |
| 292 | + | // leaves the physical connection and every sibling healthy. | |
| 293 | + | func TestUnixCarriageResetIsolation(t *testing.T) { | |
| 294 | + | pump, client := unixDaemonPump(t) | |
| 295 | + | ||
| 296 | + | for _, id := range []PhysicalStreamID{1, 2, 3} { | |
| 297 | + | injectUnixClientFrame(t, client, openFor(id)) | |
| 298 | + | } | |
| 299 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { return e.Live() == 3 }) | |
| 300 | + | for _, id := range []PhysicalStreamID{2, 3} { | |
| 301 | + | require.NoError(t, pump.Engine().Opened(Opened{Ref: testRef(id)})) | |
| 302 | + | } | |
| 303 | + | ||
| 304 | + | // Stream 1 is still opening: inbound data is a stream-local state refusal. | |
| 305 | + | injectUnixClientFrame(t, client, Data{Physical: 1, Data: []byte("early")}) | |
| 306 | + | requireEngineEventually(t, pump, func(e *StreamEngine) bool { | |
| 307 | + | status, ok := e.Status(1) | |
| 308 | + | return ok && status.State == StreamTerminal | |
| 309 | + | }) | |
| 310 | + | settled := mustStatus(t, pump.Engine(), 1) | |
| 311 | + | require.ErrorContains(t, settled.Err, "invalid stream state") | |
| 312 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, settled.FailureKind) | |
| 313 | + | ||
| 314 | + | injectUnixClientFrame(t, client, Data{Physical: 2, Data: []byte("sibling")}) | |
| 315 | + | taken, ok := takeEventually(t, pump, 2) | |
| 316 | + | require.True(t, ok) | |
| 317 | + | require.Equal(t, []byte("sibling"), taken) | |
| 318 | + | ||
| 319 | + | reset := decodeReset(t, nextUnixServerFrame(t, client), DirectionServer) | |
| 320 | + | require.Equal(t, PhysicalStreamID(1), reset.Physical) | |
| 321 | + | require.True(t, reset.HasError) | |
| 322 | + | require.Equal(t, domain.RemoteFailureInvalidResponse, reset.Error.FailureKind) | |
| 323 | + | require.False(t, pump.Engine().Closed()) | |
| 324 | + | require.False(t, channelClosed(pump.Done())) | |
| 325 | + | } | |
M internal/adapters/ipc/listener.go +69 -17
| @@ -25,6 +25,12 @@ func SocketPath(runtimeDir string) string { | |||
| 25 | 25 | // bound by a live daemon (a dial-probe against it succeeded). | |
| 26 | 26 | var ErrDaemonRunning = errors.New("ipc: a daemon is already listening on this socket") | |
| 27 | 27 | ||
| 28 | + | // staleRecoveryAttempts bounds the bind/probe/remove/rebind loop that recovers | |
| 29 | + | // from a socket file whose owner died without unlinking it. A racing owner can | |
| 30 | + | // rebind between our liveness probe and our removal, in which case we leave its | |
| 31 | + | // socket alone and retry; the bound keeps a pathological race from spinning. | |
| 32 | + | const staleRecoveryAttempts = 3 | |
| 33 | + | ||
| 28 | 34 | // unixListener implements wire.Listener over an AF_UNIX SOCK_STREAM | |
| 29 | 35 | // listener. | |
| 30 | 36 | type unixListener struct { | |
| @@ -39,8 +45,9 @@ type unixListener struct { | |||
| 39 | 45 | // If the socket path is already bound (EADDRINUSE), Listen dial-probes it: | |
| 40 | 46 | // a failed dial (connection refused, or the file having vanished) means | |
| 41 | 47 | // the previous owner died without cleaning up, so the stale socket file is | |
| 42 | - | // unlinked and bind is retried once. A successful dial means a live daemon | |
| 43 | - | // owns the socket, and Listen returns ErrDaemonRunning. | |
| 48 | + | // unlinked and bind is retried (race-safely, never unlinking a socket a | |
| 49 | + | // racing owner rebound in the probe window). A successful dial means a live | |
| 50 | + | // daemon owns the socket, and Listen returns ErrDaemonRunning. | |
| 44 | 51 | func Listen(dir string, opts ...Option) (wire.Listener, error) { | |
| 45 | 52 | if err := safedir.EnsurePrivate(dir); err != nil { | |
| 46 | 53 | return nil, fmt.Errorf("ipc: securing socket directory: %w", err) | |
| @@ -48,34 +55,79 @@ func Listen(dir string, opts ...Option) (wire.Listener, error) { | |||
| 48 | 55 | ||
| 49 | 56 | sockPath := filepath.Join(dir, socketFileName) | |
| 50 | 57 | ||
| 51 | - | ln, err := bindUnix(sockPath) | |
| 58 | + | ln, err := listenUnixStaleSafe(sockPath) | |
| 52 | 59 | if err != nil { | |
| 60 | + | return nil, fmt.Errorf("ipc: listen on %s: %w", sockPath, err) | |
| 61 | + | } | |
| 62 | + | ||
| 63 | + | if err := os.Chmod(sockPath, 0o600); err != nil { | |
| 64 | + | _ = ln.Close() | |
| 65 | + | return nil, fmt.Errorf("ipc: chmod %s: %w", sockPath, err) | |
| 66 | + | } | |
| 67 | + | ||
| 68 | + | return &unixListener{ln: ln, addr: sockPath, opts: opts}, nil | |
| 69 | + | } | |
| 70 | + | ||
| 71 | + | // listenUnixStaleSafe binds sockPath, recovering race-safely from a stale | |
| 72 | + | // socket file left behind by an owner that died without unlinking it. A socket | |
| 73 | + | // still served by a live peer is reported as ErrDaemonRunning. A path that | |
| 74 | + | // exists and is not a socket is refused without being removed, so a caller's | |
| 75 | + | // foreign file is never unlinked. | |
| 76 | + | // | |
| 77 | + | // The recovery is race-safe: the socket file's inode is observed before the | |
| 78 | + | // liveness probe and compared with the inode at removal time, so a socket a | |
| 79 | + | // racing owner rebound in that window is never unlinked. When the inode | |
| 80 | + | // changed, the bind is retried (and typically reports the racing owner live) | |
| 81 | + | // instead of removing it. | |
| 82 | + | func listenUnixStaleSafe(sockPath string) (*net.UnixListener, error) { | |
| 83 | + | for attempt := 0; attempt < staleRecoveryAttempts; attempt++ { | |
| 84 | + | ln, err := bindUnix(sockPath) | |
| 85 | + | if err == nil { | |
| 86 | + | return ln, nil | |
| 87 | + | } | |
| 53 | 88 | if !errors.Is(err, syscall.EADDRINUSE) { | |
| 54 | - | return nil, fmt.Errorf("ipc: listen on %s: %w", sockPath, err) | |
| 89 | + | return nil, err | |
| 90 | + | } | |
| 91 | + | ||
| 92 | + | before, statErr := os.Lstat(sockPath) | |
| 93 | + | if statErr != nil { | |
| 94 | + | if os.IsNotExist(statErr) { | |
| 95 | + | continue // a racing owner removed it; retry the bind | |
| 96 | + | } | |
| 97 | + | return nil, fmt.Errorf("ipc: inspecting socket %s: %w", sockPath, statErr) | |
| 98 | + | } | |
| 99 | + | if before.Mode()&os.ModeSocket == 0 { | |
| 100 | + | return nil, fmt.Errorf("ipc: %s exists and is not a socket", sockPath) | |
| 55 | 101 | } | |
| 56 | 102 | ||
| 57 | 103 | if probeErr := probeLiveDaemon(sockPath); probeErr != nil { | |
| 58 | 104 | return nil, probeErr | |
| 59 | 105 | } | |
| 60 | 106 | ||
| 61 | - | // Dial failed: the socket file is stale (its listener died | |
| 62 | - | // without unlinking). Remove it and retry the bind once. | |
| 63 | - | if rmErr := os.Remove(sockPath); rmErr != nil && !os.IsNotExist(rmErr) { | |
| 64 | - | return nil, fmt.Errorf("ipc: removing stale socket %s: %w", sockPath, rmErr) | |
| 107 | + | current, statErr := os.Lstat(sockPath) | |
| 108 | + | if statErr != nil { | |
| 109 | + | if os.IsNotExist(statErr) { | |
| 110 | + | continue // a racing owner removed it; retry the bind | |
| 111 | + | } | |
| 112 | + | return nil, fmt.Errorf("ipc: inspecting socket %s: %w", sockPath, statErr) | |
| 65 | 113 | } | |
| 66 | - | ||
| 67 | - | ln, err = bindUnix(sockPath) | |
| 68 | - | if err != nil { | |
| 69 | - | return nil, fmt.Errorf("ipc: listen on %s after removing stale socket: %w", sockPath, err) | |
| 114 | + | if !os.SameFile(before, current) { | |
| 115 | + | // A racing owner rebound the path after the probe said the old | |
| 116 | + | // socket was dead. Never unlink its live socket; retry the bind, | |
| 117 | + | // which will observe the new owner. | |
| 118 | + | continue | |
| 70 | 119 | } | |
| 71 | - | } | |
| 72 | 120 | ||
| 73 | - | if err := os.Chmod(sockPath, 0o600); err != nil { | |
| 74 | - | _ = ln.Close() | |
| 75 | - | return nil, fmt.Errorf("ipc: chmod %s: %w", sockPath, err) | |
| 121 | + | // The inode is still the stale one the probe found dead. Remove it and | |
| 122 | + | // retry the bind. A racing owner that binds between this removal and the | |
| 123 | + | // next bind simply wins; the next bind reports it live. | |
| 124 | + | if rmErr := os.Remove(sockPath); rmErr != nil && !os.IsNotExist(rmErr) { | |
| 125 | + | return nil, fmt.Errorf("ipc: removing stale socket %s: %w", sockPath, rmErr) | |
| 126 | + | } | |
| 76 | 127 | } | |
| 77 | 128 | ||
| 78 | - | return &unixListener{ln: ln, addr: sockPath, opts: opts}, nil | |
| 129 | + | // Every attempt observed a live owner (or a racing owner kept winning). | |
| 130 | + | return nil, ErrDaemonRunning | |
| 79 | 131 | } | |
| 80 | 132 | ||
| 81 | 133 | // bindUnix binds and listens on sockPath. | |
A internal/adapters/ipc/mux.go +314 -0
| @@ -0,0 +1,314 @@ | |||
| 1 | + | // Private daemonmux Unix carriage (P3.2d). | |
| 2 | + | // | |
| 3 | + | // This file exposes the explicit, mux-specific raw Unix carriage that the | |
| 4 | + | // daemonmux multiplexer (internal/adapters/daemonmux) runs over. It is | |
| 5 | + | // deliberately separate from the daemon's public Listen/DialContext pair: the | |
| 6 | + | // mux carriage is bound at a caller-supplied, isolated path, never the live | |
| 7 | + | // production daemon.sock, so the two endpoints cannot be confused and the | |
| 8 | + | // daemon's public socket is never reused or inspected here. | |
| 9 | + | // | |
| 10 | + | // Both ends of the carriage are authenticated before they are handed out. A | |
| 11 | + | // listener and a dialer verify that the connected AF_UNIX peer is the current | |
| 12 | + | // effective user from the kernel's own peer credentials (SO_PEERCRED on | |
| 13 | + | // Linux), not from filesystem permissions or an X11-style "the directory is | |
| 14 | + | // private so the peer is trusted" assumption. On a platform or build without a | |
| 15 | + | // same-user peer-credential check this file fails closed: it refuses the | |
| 16 | + | // connection rather than admitting an unverified peer (peer_other.go). | |
| 17 | + | // | |
| 18 | + | // The carriage reuses the existing IPC framing and transport (NewTransport, | |
| 19 | + | // streamframe, RecvBounded) and the shared race-safe stale-socket recovery | |
| 20 | + | // (listenUnixStaleSafe). Listener setup creates or validates an owner-only | |
| 21 | + | // (0700) parent directory with safedir.EnsurePrivate and tightens the bound | |
| 22 | + | // socket to 0600; a foreign path that is not a socket is refused without being | |
| 23 | + | // removed. Close stops Accept, unlinks only the socket inode this listener | |
| 24 | + | // created, and leaves a caller replacement alone. | |
| 25 | + | ||
| 26 | + | package ipc | |
| 27 | + | ||
| 28 | + | import ( | |
| 29 | + | "context" | |
| 30 | + | "errors" | |
| 31 | + | "fmt" | |
| 32 | + | "net" | |
| 33 | + | "os" | |
| 34 | + | "path/filepath" | |
| 35 | + | "sync" | |
| 36 | + | ||
| 37 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 38 | + | "github.com/bnema/vev/pkg/safedir" | |
| 39 | + | ) | |
| 40 | + | ||
| 41 | + | const ( | |
| 42 | + | // muxSocketMode is the owner-only permission a bound mux socket is | |
| 43 | + | // tightened to after bind. | |
| 44 | + | muxSocketMode = 0o600 | |
| 45 | + | // muxSocketPathMax is the worst-case AF_UNIX pathname length observed | |
| 46 | + | // across the supported platforms (BSD/macOS 103 < Linux 107). A mux path | |
| 47 | + | // longer than this is refused on every platform, so a path that binds on | |
| 48 | + | // Linux can never be silently truncated on macOS. | |
| 49 | + | muxSocketPathMax = 103 | |
| 50 | + | ) | |
| 51 | + | ||
| 52 | + | // Mux carriage sentinels. They are typed so a caller can classify a path, | |
| 53 | + | // platform, or peer-credential outcome without matching on message text. | |
| 54 | + | var ( | |
| 55 | + | // ErrMuxPath reports an invalid caller-supplied carriage path: empty, | |
| 56 | + | // relative, unclean, too long, containing a NUL, or naming a connection | |
| 57 | + | // that is not a Unix socket. | |
| 58 | + | ErrMuxPath = errors.New("ipc: invalid mux socket path") | |
| 59 | + | // ErrMuxForeignPath reports a mux path that already exists and is not a | |
| 60 | + | // socket. ListenMux refuses it without removing the caller's file. | |
| 61 | + | ErrMuxForeignPath = errors.New("ipc: mux socket path exists and is not a socket") | |
| 62 | + | // ErrMuxUnsupported reports a same-user peer-credential check on a | |
| 63 | + | // platform or build that has none. The carriage fails closed: it never | |
| 64 | + | // admits an unverified peer. | |
| 65 | + | ErrMuxUnsupported = errors.New("ipc: same-user peer credential verification is unsupported on this platform") | |
| 66 | + | // ErrMuxPeerRejected reports a connected AF_UNIX peer whose kernel | |
| 67 | + | // credentials are not the current effective user, or could not be read. | |
| 68 | + | ErrMuxPeerRejected = errors.New("ipc: mux peer failed same-user credential verification") | |
| 69 | + | ) | |
| 70 | + | ||
| 71 | + | // muxPeerVerifier checks one connected AF_UNIX peer's kernel credentials. It | |
| 72 | + | // returns nil only for a peer proven to be the current effective user. It is a | |
| 73 | + | // function type so a test can inject an observable refusal without root. | |
| 74 | + | type muxPeerVerifier func(conn *net.UnixConn) error | |
| 75 | + | ||
| 76 | + | // MuxListener is the listener half of a private daemonmux Unix carriage. Accept | |
| 77 | + | // returns a raw bounded transport (Send/RecvBounded/Close) ready for the | |
| 78 | + | // daemonmux FramedCarrierBridge, and verifies same-user peer credentials before | |
| 79 | + | // handing it out. Close unblocks a blocked Accept and removes only the socket | |
| 80 | + | // inode this listener created. | |
| 81 | + | type MuxListener interface { | |
| 82 | + | Accept() (wire.BoundedTransport, error) | |
| 83 | + | Close() error | |
| 84 | + | Addr() string | |
| 85 | + | } | |
| 86 | + | ||
| 87 | + | // muxListener implements MuxListener over one bound AF_UNIX SOCK_STREAM socket. | |
| 88 | + | type muxListener struct { | |
| 89 | + | ln *net.UnixListener | |
| 90 | + | path string | |
| 91 | + | file os.FileInfo | |
| 92 | + | opts []Option | |
| 93 | + | verify muxPeerVerifier | |
| 94 | + | ||
| 95 | + | closeOnce sync.Once | |
| 96 | + | closeErr error | |
| 97 | + | } | |
| 98 | + | ||
| 99 | + | var _ MuxListener = (*muxListener)(nil) | |
| 100 | + | ||
| 101 | + | // ListenMux binds the private daemonmux carriage at exactly path and returns a | |
| 102 | + | // listener that verifies same-user peer credentials before handing out a raw | |
| 103 | + | // bounded transport. | |
| 104 | + | // | |
| 105 | + | // path is caller-supplied and isolation is the caller's responsibility: it must | |
| 106 | + | // be an absolute, cleaned path inside an owner-only directory and must not be | |
| 107 | + | // the daemon's public daemon.sock. ListenMux creates or validates the parent | |
| 108 | + | // directory as 0700 (safedir.EnsurePrivate), refuses a path that already exists | |
| 109 | + | // and is not a socket without removing it, recovers race-safely from a stale | |
| 110 | + | // socket left by a dead owner, and tightens the bound socket to 0600. A live | |
| 111 | + | // owner already bound at path is reported as ErrDaemonRunning. | |
| 112 | + | func ListenMux(path string, opts ...Option) (MuxListener, error) { | |
| 113 | + | return listenMux(path, verifySameUserUnixPeer, opts...) | |
| 114 | + | } | |
| 115 | + | ||
| 116 | + | // listenMux is ListenMux with an injected peer verifier, so a test can observe | |
| 117 | + | // a credential refusal deterministically without root. A nil verifier falls | |
| 118 | + | // back to the platform default. | |
| 119 | + | func listenMux(path string, verify muxPeerVerifier, opts ...Option) (MuxListener, error) { | |
| 120 | + | if err := validateMuxSocketPath(path); err != nil { | |
| 121 | + | return nil, err | |
| 122 | + | } | |
| 123 | + | if verify == nil { | |
| 124 | + | verify = verifySameUserUnixPeer | |
| 125 | + | } | |
| 126 | + | if err := prepareMuxParent(path); err != nil { | |
| 127 | + | return nil, err | |
| 128 | + | } | |
| 129 | + | if err := inspectMuxSocketPath(path); err != nil { | |
| 130 | + | return nil, err | |
| 131 | + | } | |
| 132 | + | ||
| 133 | + | ln, err := listenUnixStaleSafe(path) | |
| 134 | + | if err != nil { | |
| 135 | + | return nil, fmt.Errorf("ipc: listen on mux socket %s: %w", path, err) | |
| 136 | + | } | |
| 137 | + | // The listener must never unlink a pathname it did not create. A caller may | |
| 138 | + | // replace the path after bind; Close preserves that replacement. | |
| 139 | + | ln.SetUnlinkOnClose(false) | |
| 140 | + | ||
| 141 | + | cleanup := func(cause error) (MuxListener, error) { | |
| 142 | + | _ = ln.Close() | |
| 143 | + | return nil, cause | |
| 144 | + | } | |
| 145 | + | if err := os.Chmod(path, muxSocketMode); err != nil { | |
| 146 | + | return cleanup(fmt.Errorf("ipc: securing mux socket %s: %w", path, err)) | |
| 147 | + | } | |
| 148 | + | info, err := os.Lstat(path) | |
| 149 | + | if err != nil { | |
| 150 | + | return cleanup(fmt.Errorf("ipc: stat mux socket %s: %w", path, err)) | |
| 151 | + | } | |
| 152 | + | if info.Mode()&os.ModeSocket == 0 { | |
| 153 | + | return cleanup(fmt.Errorf("%w: %s", ErrMuxForeignPath, path)) | |
| 154 | + | } | |
| 155 | + | return &muxListener{ln: ln, path: path, file: info, opts: opts, verify: verify}, nil | |
| 156 | + | } | |
| 157 | + | ||
| 158 | + | // DialMuxContext connects to the private daemonmux carriage at exactly path and | |
| 159 | + | // returns a raw bounded transport (Send/RecvBounded/Close) ready for the | |
| 160 | + | // daemonmux FramedCarrierBridge. The peer's kernel credentials are verified as | |
| 161 | + | // the current effective user before the transport is returned; a peer that | |
| 162 | + | // cannot be proven same-user is refused and its connection closed. | |
| 163 | + | // | |
| 164 | + | // A dial failure (no carriage bound, or a stale path) is wrapped so callers can | |
| 165 | + | // distinguish it; cancellation and deadline errors are preserved for | |
| 166 | + | // classification. | |
| 167 | + | func DialMuxContext(ctx context.Context, path string, opts ...Option) (wire.BoundedTransport, error) { | |
| 168 | + | return dialMux(ctx, path, verifySameUserUnixPeer, opts...) | |
| 169 | + | } | |
| 170 | + | ||
| 171 | + | // dialMux is DialMuxContext with an injected peer verifier, so a test can | |
| 172 | + | // observe a credential refusal deterministically without root. A nil verifier | |
| 173 | + | // falls back to the platform default. | |
| 174 | + | func dialMux(ctx context.Context, path string, verify muxPeerVerifier, opts ...Option) (wire.BoundedTransport, error) { | |
| 175 | + | if err := validateMuxSocketPath(path); err != nil { | |
| 176 | + | return nil, err | |
| 177 | + | } | |
| 178 | + | if verify == nil { | |
| 179 | + | verify = verifySameUserUnixPeer | |
| 180 | + | } | |
| 181 | + | var dialer net.Dialer | |
| 182 | + | conn, err := dialer.DialContext(ctx, "unix", path) | |
| 183 | + | if err != nil { | |
| 184 | + | return nil, fmt.Errorf("ipc: dial mux socket %s: %w", path, err) | |
| 185 | + | } | |
| 186 | + | unixConn, ok := conn.(*net.UnixConn) | |
| 187 | + | if !ok { | |
| 188 | + | _ = conn.Close() | |
| 189 | + | return nil, fmt.Errorf("%w: %s is not a Unix connection", ErrMuxPath, path) | |
| 190 | + | } | |
| 191 | + | if err := verify(unixConn); err != nil { | |
| 192 | + | _ = conn.Close() | |
| 193 | + | return nil, err | |
| 194 | + | } | |
| 195 | + | transport, ok := NewTransport(conn, opts...).(wire.BoundedTransport) | |
| 196 | + | if !ok { | |
| 197 | + | _ = conn.Close() | |
| 198 | + | return nil, fmt.Errorf("%w: %s transport is not bounded", ErrMuxPath, path) | |
| 199 | + | } | |
| 200 | + | return transport, nil | |
| 201 | + | } | |
| 202 | + | ||
| 203 | + | // Accept waits for the next connection on the bound socket. Before handing a | |
| 204 | + | // transport out it verifies the peer's same-user kernel credentials; a refused | |
| 205 | + | // peer is closed and the refusal returned, so an unverified carriage is never | |
| 206 | + | // admitted. Accept fails once the listener is closed (Close unblocks it). | |
| 207 | + | func (l *muxListener) Accept() (wire.BoundedTransport, error) { | |
| 208 | + | conn, err := l.ln.AcceptUnix() | |
| 209 | + | if err != nil { | |
| 210 | + | return nil, err | |
| 211 | + | } | |
| 212 | + | if err := l.verify(conn); err != nil { | |
| 213 | + | _ = conn.Close() | |
| 214 | + | return nil, err | |
| 215 | + | } | |
| 216 | + | transport, ok := NewTransport(conn, l.opts...).(wire.BoundedTransport) | |
| 217 | + | if !ok { | |
| 218 | + | _ = conn.Close() | |
| 219 | + | return nil, fmt.Errorf("%w: %s transport is not bounded", ErrMuxPath, l.path) | |
| 220 | + | } | |
| 221 | + | return transport, nil | |
| 222 | + | } | |
| 223 | + | ||
| 224 | + | // Addr returns the bound filesystem path. | |
| 225 | + | func (l *muxListener) Addr() string { return l.path } | |
| 226 | + | ||
| 227 | + | // Close stops accepting, unblocking a blocked Accept, and removes the socket | |
| 228 | + | // only if the pathname still names the inode this listener created. A caller | |
| 229 | + | // replacement is preserved. Close is idempotent and concurrent-safe. | |
| 230 | + | func (l *muxListener) Close() error { | |
| 231 | + | if l == nil { | |
| 232 | + | return nil | |
| 233 | + | } | |
| 234 | + | l.closeOnce.Do(func() { | |
| 235 | + | l.closeErr = l.ln.Close() | |
| 236 | + | if err := removeMuxSocket(l.path, l.file); err != nil { | |
| 237 | + | l.closeErr = errors.Join(l.closeErr, err) | |
| 238 | + | } | |
| 239 | + | }) | |
| 240 | + | return l.closeErr | |
| 241 | + | } | |
| 242 | + | ||
| 243 | + | // validateMuxSocketPath refuses a caller path that cannot be a portable private | |
| 244 | + | // carriage endpoint: it must be absolute, already cleaned, within the | |
| 245 | + | // cross-platform AF_UNIX pathname limit, free of NUL, and name a real base. | |
| 246 | + | func validateMuxSocketPath(path string) error { | |
| 247 | + | if path == "" || !filepath.IsAbs(path) { | |
| 248 | + | return fmt.Errorf("%w: path must be absolute", ErrMuxPath) | |
| 249 | + | } | |
| 250 | + | if filepath.Clean(path) != path || path == string(filepath.Separator) { | |
| 251 | + | return fmt.Errorf("%w: path must be cleaned", ErrMuxPath) | |
| 252 | + | } | |
| 253 | + | base := filepath.Base(path) | |
| 254 | + | if base == "." || base == ".." || base == string(filepath.Separator) { | |
| 255 | + | return fmt.Errorf("%w: invalid base name", ErrMuxPath) | |
| 256 | + | } | |
| 257 | + | if len(path) > muxSocketPathMax { | |
| 258 | + | return fmt.Errorf("%w: path exceeds %d bytes", ErrMuxPath, muxSocketPathMax) | |
| 259 | + | } | |
| 260 | + | for i := 0; i < len(path); i++ { | |
| 261 | + | if path[i] == 0 { | |
| 262 | + | return fmt.Errorf("%w: path contains a NUL", ErrMuxPath) | |
| 263 | + | } | |
| 264 | + | } | |
| 265 | + | return nil | |
| 266 | + | } | |
| 267 | + | ||
| 268 | + | // prepareMuxParent creates or validates the carriage's parent directory as an | |
| 269 | + | // owner-only (0700) directory owned by the current user. | |
| 270 | + | func prepareMuxParent(path string) error { | |
| 271 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { | |
| 272 | + | return fmt.Errorf("ipc: securing mux socket directory: %w", err) | |
| 273 | + | } | |
| 274 | + | return nil | |
| 275 | + | } | |
| 276 | + | ||
| 277 | + | // inspectMuxSocketPath refuses an existing path that is not a socket before any | |
| 278 | + | // bind or liveness probe, so a caller's regular file, directory, or symlink is | |
| 279 | + | // never mistaken for a stale endpoint and never removed. | |
| 280 | + | func inspectMuxSocketPath(path string) error { | |
| 281 | + | info, err := os.Lstat(path) | |
| 282 | + | if err != nil { | |
| 283 | + | if errors.Is(err, os.ErrNotExist) { | |
| 284 | + | return nil | |
| 285 | + | } | |
| 286 | + | return fmt.Errorf("ipc: inspecting mux socket %s: %w", path, err) | |
| 287 | + | } | |
| 288 | + | if info.Mode()&os.ModeSocket == 0 { | |
| 289 | + | return fmt.Errorf("%w: %s", ErrMuxForeignPath, path) | |
| 290 | + | } | |
| 291 | + | return nil | |
| 292 | + | } | |
| 293 | + | ||
| 294 | + | // removeMuxSocket removes path only when it still names expected, so a socket | |
| 295 | + | // a caller replaced during Close is preserved. | |
| 296 | + | func removeMuxSocket(path string, expected os.FileInfo) error { | |
| 297 | + | if expected == nil { | |
| 298 | + | return nil | |
| 299 | + | } | |
| 300 | + | info, err := os.Lstat(path) | |
| 301 | + | if err != nil { | |
| 302 | + | if errors.Is(err, os.ErrNotExist) { | |
| 303 | + | return nil | |
| 304 | + | } | |
| 305 | + | return err | |
| 306 | + | } | |
| 307 | + | if !os.SameFile(info, expected) { | |
| 308 | + | return nil | |
| 309 | + | } | |
| 310 | + | if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { | |
| 311 | + | return err | |
| 312 | + | } | |
| 313 | + | return nil | |
| 314 | + | } | |
A internal/adapters/ipc/mux_test.go +531 -0
| @@ -0,0 +1,531 @@ | |||
| 1 | + | package ipc | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "errors" | |
| 6 | + | "net" | |
| 7 | + | "os" | |
| 8 | + | "path/filepath" | |
| 9 | + | "sync" | |
| 10 | + | "testing" | |
| 11 | + | "time" | |
| 12 | + | ||
| 13 | + | "github.com/bnema/vev/internal/protocol/wire" | |
| 14 | + | ) | |
| 15 | + | ||
| 16 | + | // muxTestPath returns a short, owner-only parent directory and one mux.sock | |
| 17 | + | // path inside it, so a caller-supplied mux carriage path is exercised at the | |
| 18 | + | // real AF_UNIX pathname limit rather than a long temporary path. | |
| 19 | + | func muxTestPath(t *testing.T) string { | |
| 20 | + | t.Helper() | |
| 21 | + | return filepath.Join(shortSocketDir(t, "vev"), "mux.sock") | |
| 22 | + | } | |
| 23 | + | ||
| 24 | + | // mustListenMux binds one mux carriage at path and closes it at test end. | |
| 25 | + | func mustListenMux(t *testing.T, path string) MuxListener { | |
| 26 | + | t.Helper() | |
| 27 | + | listener, err := ListenMux(path) | |
| 28 | + | if err != nil { | |
| 29 | + | t.Fatalf("ListenMux(%q): %v", path, err) | |
| 30 | + | } | |
| 31 | + | t.Cleanup(func() { _ = listener.Close() }) | |
| 32 | + | return listener | |
| 33 | + | } | |
| 34 | + | ||
| 35 | + | // dialMuxAsync dials one mux carriage and delivers the outcome. | |
| 36 | + | func dialMuxAsync(t *testing.T, ctx context.Context, path string) <-chan struct { | |
| 37 | + | transport wire.BoundedTransport | |
| 38 | + | err error | |
| 39 | + | } { | |
| 40 | + | t.Helper() | |
| 41 | + | out := make(chan struct { | |
| 42 | + | transport wire.BoundedTransport | |
| 43 | + | err error | |
| 44 | + | }, 1) | |
| 45 | + | go func() { | |
| 46 | + | transport, err := DialMuxContext(ctx, path) | |
| 47 | + | out <- struct { | |
| 48 | + | transport wire.BoundedTransport | |
| 49 | + | err error | |
| 50 | + | }{transport: transport, err: err} | |
| 51 | + | }() | |
| 52 | + | return out | |
| 53 | + | } | |
| 54 | + | ||
| 55 | + | func awaitMuxDial(t *testing.T, out <-chan struct { | |
| 56 | + | transport wire.BoundedTransport | |
| 57 | + | err error | |
| 58 | + | }) wire.BoundedTransport { | |
| 59 | + | t.Helper() | |
| 60 | + | select { | |
| 61 | + | case result := <-out: | |
| 62 | + | if result.err != nil { | |
| 63 | + | t.Fatalf("DialMuxContext: %v", result.err) | |
| 64 | + | } | |
| 65 | + | if result.transport == nil { | |
| 66 | + | t.Fatal("DialMuxContext returned a nil transport") | |
| 67 | + | } | |
| 68 | + | t.Cleanup(func() { _ = result.transport.Close() }) | |
| 69 | + | return result.transport | |
| 70 | + | case <-time.After(5 * time.Second): | |
| 71 | + | t.Fatal("DialMuxContext did not return") | |
| 72 | + | return nil | |
| 73 | + | } | |
| 74 | + | } | |
| 75 | + | ||
| 76 | + | // TestListenMuxCreatesOwnerOnlyParentAndSocket proves listener setup creates the | |
| 77 | + | // caller's parent directory owner-only (0700) and tightens the bound socket to | |
| 78 | + | // 0600, so the private carriage is never exposed through filesystem modes. | |
| 79 | + | func TestListenMuxCreatesOwnerOnlyParentAndSocket(t *testing.T) { | |
| 80 | + | path := muxTestPath(t) | |
| 81 | + | parent := filepath.Dir(path) | |
| 82 | + | ||
| 83 | + | listener := mustListenMux(t, path) | |
| 84 | + | if listener.Addr() != path { | |
| 85 | + | t.Fatalf("Addr() = %q, want %q", listener.Addr(), path) | |
| 86 | + | } | |
| 87 | + | ||
| 88 | + | dirInfo, err := os.Stat(parent) | |
| 89 | + | if err != nil { | |
| 90 | + | t.Fatalf("stat parent: %v", err) | |
| 91 | + | } | |
| 92 | + | if perm := dirInfo.Mode().Perm(); perm != 0o700 { | |
| 93 | + | t.Fatalf("parent perm = %o, want 0700", perm) | |
| 94 | + | } | |
| 95 | + | ||
| 96 | + | sockInfo, err := os.Lstat(path) | |
| 97 | + | if err != nil { | |
| 98 | + | t.Fatalf("stat socket: %v", err) | |
| 99 | + | } | |
| 100 | + | if sockInfo.Mode()&os.ModeSocket == 0 { | |
| 101 | + | t.Fatalf("%s is not a socket: mode %v", path, sockInfo.Mode()) | |
| 102 | + | } | |
| 103 | + | if perm := sockInfo.Mode().Perm(); perm != 0o600 { | |
| 104 | + | t.Fatalf("socket perm = %o, want 0600", perm) | |
| 105 | + | } | |
| 106 | + | } | |
| 107 | + | ||
| 108 | + | // TestListenMuxSameUserRoundTrip proves a real AF_UNIX mux carriage carries | |
| 109 | + | // bounded framed envelopes both ways after both ends verify the peer is the | |
| 110 | + | // current user. | |
| 111 | + | func TestListenMuxSameUserRoundTrip(t *testing.T) { | |
| 112 | + | path := muxTestPath(t) | |
| 113 | + | listener := mustListenMux(t, path) | |
| 114 | + | ||
| 115 | + | accepted := make(chan struct { | |
| 116 | + | transport wire.BoundedTransport | |
| 117 | + | err error | |
| 118 | + | }, 1) | |
| 119 | + | go func() { | |
| 120 | + | transport, err := listener.Accept() | |
| 121 | + | accepted <- struct { | |
| 122 | + | transport wire.BoundedTransport | |
| 123 | + | err error | |
| 124 | + | }{transport: transport, err: err} | |
| 125 | + | }() | |
| 126 | + | ||
| 127 | + | client := awaitMuxDial(t, dialMuxAsync(t, context.Background(), path)) | |
| 128 | + | ||
| 129 | + | serverResult := <-accepted | |
| 130 | + | if serverResult.err != nil { | |
| 131 | + | t.Fatalf("Accept: %v", serverResult.err) | |
| 132 | + | } | |
| 133 | + | server := serverResult.transport | |
| 134 | + | t.Cleanup(func() { _ = server.Close() }) | |
| 135 | + | ||
| 136 | + | request := wire.Envelope{Payload: []byte("mux request stays exact")} | |
| 137 | + | if err := client.Send(request); err != nil { | |
| 138 | + | t.Fatalf("client Send: %v", err) | |
| 139 | + | } | |
| 140 | + | got, err := server.RecvBounded(uint64(len(request.Payload))) | |
| 141 | + | if err != nil { | |
| 142 | + | t.Fatalf("server RecvBounded: %v", err) | |
| 143 | + | } | |
| 144 | + | if string(got.Payload) != string(request.Payload) { | |
| 145 | + | t.Fatalf("server RecvBounded payload = %q, want %q", got.Payload, request.Payload) | |
| 146 | + | } | |
| 147 | + | ||
| 148 | + | reply := wire.Envelope{Payload: []byte("mux reply stays exact")} | |
| 149 | + | if err := server.Send(reply); err != nil { | |
| 150 | + | t.Fatalf("server Send: %v", err) | |
| 151 | + | } | |
| 152 | + | gotReply, err := client.RecvBounded(uint64(len(reply.Payload))) | |
| 153 | + | if err != nil { | |
| 154 | + | t.Fatalf("client RecvBounded: %v", err) | |
| 155 | + | } | |
| 156 | + | if string(gotReply.Payload) != string(reply.Payload) { | |
| 157 | + | t.Fatalf("client RecvBounded payload = %q, want %q", gotReply.Payload, reply.Payload) | |
| 158 | + | } | |
| 159 | + | } | |
| 160 | + | ||
| 161 | + | // TestListenMuxRejectsUnverifiedAccept proves the accept side verifies peer | |
| 162 | + | // credentials before handing out a transport: an injected refusal is observable | |
| 163 | + | // as ErrMuxPeerRejected and the connection is closed. | |
| 164 | + | func TestListenMuxRejectsUnverifiedAccept(t *testing.T) { | |
| 165 | + | path := muxTestPath(t) | |
| 166 | + | reject := func(*net.UnixConn) error { return ErrMuxPeerRejected } | |
| 167 | + | listener, err := listenMux(path, reject) | |
| 168 | + | if err != nil { | |
| 169 | + | t.Fatalf("listenMux: %v", err) | |
| 170 | + | } | |
| 171 | + | t.Cleanup(func() { _ = listener.Close() }) | |
| 172 | + | ||
| 173 | + | dialOutcome := dialMuxAsync(t, context.Background(), path) | |
| 174 | + | ||
| 175 | + | transport, err := listener.Accept() | |
| 176 | + | if transport != nil { | |
| 177 | + | _ = transport.Close() | |
| 178 | + | t.Fatal("Accept returned a transport for an unverified peer") | |
| 179 | + | } | |
| 180 | + | if !errors.Is(err, ErrMuxPeerRejected) { | |
| 181 | + | t.Fatalf("Accept error = %v, want ErrMuxPeerRejected", err) | |
| 182 | + | } | |
| 183 | + | ||
| 184 | + | result := <-dialOutcome | |
| 185 | + | if result.transport != nil { | |
| 186 | + | _ = result.transport.Close() | |
| 187 | + | } | |
| 188 | + | } | |
| 189 | + | ||
| 190 | + | // TestDialMuxRejectsUnverifiedPeer proves the dial side verifies peer | |
| 191 | + | // credentials before handing out a transport: an injected refusal fails the | |
| 192 | + | // dial with ErrMuxPeerRejected and no transport escapes. | |
| 193 | + | func TestDialMuxRejectsUnverifiedPeer(t *testing.T) { | |
| 194 | + | path := muxTestPath(t) | |
| 195 | + | permissive := func(*net.UnixConn) error { return nil } | |
| 196 | + | listener, err := listenMux(path, permissive) | |
| 197 | + | if err != nil { | |
| 198 | + | t.Fatalf("listenMux: %v", err) | |
| 199 | + | } | |
| 200 | + | t.Cleanup(func() { _ = listener.Close() }) | |
| 201 | + | ||
| 202 | + | accepted := make(chan wire.BoundedTransport, 1) | |
| 203 | + | go func() { | |
| 204 | + | transport, acceptErr := listener.Accept() | |
| 205 | + | if acceptErr == nil { | |
| 206 | + | accepted <- transport | |
| 207 | + | return | |
| 208 | + | } | |
| 209 | + | accepted <- nil | |
| 210 | + | }() | |
| 211 | + | ||
| 212 | + | reject := func(*net.UnixConn) error { return ErrMuxPeerRejected } | |
| 213 | + | transport, err := dialMux(context.Background(), path, reject) | |
| 214 | + | if transport != nil { | |
| 215 | + | _ = transport.Close() | |
| 216 | + | t.Fatal("dialMux returned a transport for an unverified peer") | |
| 217 | + | } | |
| 218 | + | if !errors.Is(err, ErrMuxPeerRejected) { | |
| 219 | + | t.Fatalf("dialMux error = %v, want ErrMuxPeerRejected", err) | |
| 220 | + | } | |
| 221 | + | ||
| 222 | + | select { | |
| 223 | + | case server := <-accepted: | |
| 224 | + | if server != nil { | |
| 225 | + | _ = server.Close() | |
| 226 | + | } | |
| 227 | + | case <-time.After(5 * time.Second): | |
| 228 | + | t.Fatal("Accept did not return") | |
| 229 | + | } | |
| 230 | + | } | |
| 231 | + | ||
| 232 | + | // createStaleMuxSocket leaves a socket file at path with nobody listening | |
| 233 | + | // behind it, simulating an owner that died without unlinking its socket. | |
| 234 | + | func createStaleMuxSocket(t *testing.T, path string) os.FileInfo { | |
| 235 | + | t.Helper() | |
| 236 | + | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 237 | + | t.Fatalf("mkdir parent: %v", err) | |
| 238 | + | } | |
| 239 | + | addr, err := net.ResolveUnixAddr("unix", path) | |
| 240 | + | if err != nil { | |
| 241 | + | t.Fatalf("ResolveUnixAddr: %v", err) | |
| 242 | + | } | |
| 243 | + | dead, err := net.ListenUnix("unix", addr) | |
| 244 | + | if err != nil { | |
| 245 | + | t.Fatalf("ListenUnix: %v", err) | |
| 246 | + | } | |
| 247 | + | dead.SetUnlinkOnClose(false) | |
| 248 | + | if err := dead.Close(); err != nil { | |
| 249 | + | t.Fatalf("closing dead listener: %v", err) | |
| 250 | + | } | |
| 251 | + | info, err := os.Lstat(path) | |
| 252 | + | if err != nil { | |
| 253 | + | t.Fatalf("expected stale socket file: %v", err) | |
| 254 | + | } | |
| 255 | + | return info | |
| 256 | + | } | |
| 257 | + | ||
| 258 | + | // TestListenMuxStaleSocketRecovered proves a socket file left by a dead owner is | |
| 259 | + | // recovered race-safely and the carriage binds and serves. | |
| 260 | + | func TestListenMuxStaleSocketRecovered(t *testing.T) { | |
| 261 | + | path := muxTestPath(t) | |
| 262 | + | stale := createStaleMuxSocket(t, path) | |
| 263 | + | ||
| 264 | + | mustListenMux(t, path) | |
| 265 | + | ||
| 266 | + | recovered, err := os.Lstat(path) | |
| 267 | + | if err != nil { | |
| 268 | + | t.Fatalf("stat recovered socket: %v", err) | |
| 269 | + | } | |
| 270 | + | if os.SameFile(stale, recovered) { | |
| 271 | + | t.Fatal("stale socket inode was not replaced by the recovered listener") | |
| 272 | + | } | |
| 273 | + | ||
| 274 | + | // The recovered carriage serves a same-user connection. | |
| 275 | + | _ = awaitMuxDial(t, dialMuxAsync(t, context.Background(), path)) | |
| 276 | + | } | |
| 277 | + | ||
| 278 | + | // TestListenMuxStaleRecoveryRacesToOneWinner proves the stale recovery is | |
| 279 | + | // race-safe: concurrent listeners racing over one stale socket produce exactly | |
| 280 | + | // one winner, and every loser observes the winner's live socket instead of | |
| 281 | + | // unlinking it. | |
| 282 | + | func TestListenMuxStaleRecoveryRacesToOneWinner(t *testing.T) { | |
| 283 | + | path := muxTestPath(t) | |
| 284 | + | createStaleMuxSocket(t, path) | |
| 285 | + | ||
| 286 | + | const contenders = 8 | |
| 287 | + | type outcome struct { | |
| 288 | + | listener MuxListener | |
| 289 | + | err error | |
| 290 | + | } | |
| 291 | + | results := make(chan outcome, contenders) | |
| 292 | + | var wg sync.WaitGroup | |
| 293 | + | for i := 0; i < contenders; i++ { | |
| 294 | + | wg.Add(1) | |
| 295 | + | go func() { | |
| 296 | + | defer wg.Done() | |
| 297 | + | listener, err := ListenMux(path) | |
| 298 | + | results <- outcome{listener: listener, err: err} | |
| 299 | + | }() | |
| 300 | + | } | |
| 301 | + | wg.Wait() | |
| 302 | + | close(results) | |
| 303 | + | ||
| 304 | + | successes, live := 0, 0 | |
| 305 | + | for result := range results { | |
| 306 | + | switch { | |
| 307 | + | case result.err == nil: | |
| 308 | + | successes++ | |
| 309 | + | t.Cleanup(func() { _ = result.listener.Close() }) | |
| 310 | + | case errors.Is(result.err, ErrDaemonRunning): | |
| 311 | + | live++ | |
| 312 | + | default: | |
| 313 | + | t.Fatalf("unexpected race outcome: %v", result.err) | |
| 314 | + | } | |
| 315 | + | } | |
| 316 | + | if successes != 1 { | |
| 317 | + | t.Fatalf("winners = %d, want exactly 1", successes) | |
| 318 | + | } | |
| 319 | + | if live != contenders-1 { | |
| 320 | + | t.Fatalf("losers observing a live owner = %d, want %d", live, contenders-1) | |
| 321 | + | } | |
| 322 | + | } | |
| 323 | + | ||
| 324 | + | // TestListenMuxRefusesLiveEndpoint proves a second listener on a live mux path | |
| 325 | + | // is refused rather than clobbering it. | |
| 326 | + | func TestListenMuxRefusesLiveEndpoint(t *testing.T) { | |
| 327 | + | path := muxTestPath(t) | |
| 328 | + | mustListenMux(t, path) | |
| 329 | + | ||
| 330 | + | second, err := ListenMux(path) | |
| 331 | + | if second != nil { | |
| 332 | + | _ = second.Close() | |
| 333 | + | } | |
| 334 | + | if !errors.Is(err, ErrDaemonRunning) { | |
| 335 | + | t.Fatalf("second ListenMux error = %v, want ErrDaemonRunning", err) | |
| 336 | + | } | |
| 337 | + | } | |
| 338 | + | ||
| 339 | + | // TestListenMuxRefusesForeignPath proves a path that exists and is not a socket | |
| 340 | + | // is refused without being removed: a regular file and a directory are left | |
| 341 | + | // intact. | |
| 342 | + | func TestListenMuxRefusesForeignPath(t *testing.T) { | |
| 343 | + | t.Run("regular file", func(t *testing.T) { | |
| 344 | + | path := muxTestPath(t) | |
| 345 | + | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 346 | + | t.Fatalf("mkdir parent: %v", err) | |
| 347 | + | } | |
| 348 | + | const contents = "caller data" | |
| 349 | + | if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { | |
| 350 | + | t.Fatalf("write file: %v", err) | |
| 351 | + | } | |
| 352 | + | ||
| 353 | + | listener, err := ListenMux(path) | |
| 354 | + | if listener != nil { | |
| 355 | + | _ = listener.Close() | |
| 356 | + | } | |
| 357 | + | if !errors.Is(err, ErrMuxForeignPath) { | |
| 358 | + | t.Fatalf("ListenMux error = %v, want ErrMuxForeignPath", err) | |
| 359 | + | } | |
| 360 | + | got, readErr := os.ReadFile(path) | |
| 361 | + | if readErr != nil { | |
| 362 | + | t.Fatalf("foreign file was removed: %v", readErr) | |
| 363 | + | } | |
| 364 | + | if string(got) != contents { | |
| 365 | + | t.Fatalf("foreign file contents = %q, want %q", got, contents) | |
| 366 | + | } | |
| 367 | + | }) | |
| 368 | + | ||
| 369 | + | t.Run("directory", func(t *testing.T) { | |
| 370 | + | path := muxTestPath(t) | |
| 371 | + | if err := os.MkdirAll(path, 0o700); err != nil { | |
| 372 | + | t.Fatalf("mkdir foreign directory: %v", err) | |
| 373 | + | } | |
| 374 | + | listener, err := ListenMux(path) | |
| 375 | + | if listener != nil { | |
| 376 | + | _ = listener.Close() | |
| 377 | + | } | |
| 378 | + | if !errors.Is(err, ErrMuxForeignPath) { | |
| 379 | + | t.Fatalf("ListenMux error = %v, want ErrMuxForeignPath", err) | |
| 380 | + | } | |
| 381 | + | if info, statErr := os.Stat(path); statErr != nil || !info.IsDir() { | |
| 382 | + | t.Fatalf("foreign directory was removed or changed: info=%v err=%v", info, statErr) | |
| 383 | + | } | |
| 384 | + | }) | |
| 385 | + | } | |
| 386 | + | ||
| 387 | + | // TestMuxPathValidation proves invalid caller paths are refused before any | |
| 388 | + | // filesystem change. | |
| 389 | + | func TestMuxPathValidation(t *testing.T) { | |
| 390 | + | tests := []struct { | |
| 391 | + | name string | |
| 392 | + | path string | |
| 393 | + | }{ | |
| 394 | + | {name: "empty", path: ""}, | |
| 395 | + | {name: "relative", path: "mux.sock"}, | |
| 396 | + | {name: "unclean", path: "/tmp/a/../mux.sock"}, | |
| 397 | + | {name: "root", path: "/"}, | |
| 398 | + | {name: "too long", path: "/" + string(make([]byte, muxSocketPathMax))}, | |
| 399 | + | } | |
| 400 | + | for _, tt := range tests { | |
| 401 | + | t.Run(tt.name, func(t *testing.T) { | |
| 402 | + | if _, err := ListenMux(tt.path); !errors.Is(err, ErrMuxPath) { | |
| 403 | + | t.Fatalf("ListenMux(%q) error = %v, want ErrMuxPath", tt.path, err) | |
| 404 | + | } | |
| 405 | + | if _, err := DialMuxContext(context.Background(), tt.path); !errors.Is(err, ErrMuxPath) { | |
| 406 | + | t.Fatalf("DialMuxContext(%q) error = %v, want ErrMuxPath", tt.path, err) | |
| 407 | + | } | |
| 408 | + | }) | |
| 409 | + | } | |
| 410 | + | } | |
| 411 | + | ||
| 412 | + | // TestListenMuxCloseUnblocksAcceptAndCleansSocket proves Close unblocks a | |
| 413 | + | // blocked Accept, unlinks the socket it created, refuses later dials, and is | |
| 414 | + | // idempotent. | |
| 415 | + | func TestListenMuxCloseUnblocksAcceptAndCleansSocket(t *testing.T) { | |
| 416 | + | path := muxTestPath(t) | |
| 417 | + | listener := mustListenMux(t, path) | |
| 418 | + | ||
| 419 | + | acceptErr := make(chan error, 1) | |
| 420 | + | go func() { | |
| 421 | + | transport, err := listener.Accept() | |
| 422 | + | if transport != nil { | |
| 423 | + | _ = transport.Close() | |
| 424 | + | } | |
| 425 | + | acceptErr <- err | |
| 426 | + | }() | |
| 427 | + | ||
| 428 | + | if err := listener.Close(); err != nil { | |
| 429 | + | t.Fatalf("Close: %v", err) | |
| 430 | + | } | |
| 431 | + | select { | |
| 432 | + | case err := <-acceptErr: | |
| 433 | + | if err == nil { | |
| 434 | + | t.Fatal("Accept returned nil error after Close") | |
| 435 | + | } | |
| 436 | + | case <-time.After(5 * time.Second): | |
| 437 | + | t.Fatal("Close did not unblock Accept") | |
| 438 | + | } | |
| 439 | + | ||
| 440 | + | if _, err := os.Lstat(path); !os.IsNotExist(err) { | |
| 441 | + | t.Fatalf("socket still present after Close: %v", err) | |
| 442 | + | } | |
| 443 | + | if transport, err := DialMuxContext(context.Background(), path); err == nil { | |
| 444 | + | _ = transport.Close() | |
| 445 | + | t.Fatal("DialMuxContext succeeded after Close") | |
| 446 | + | } | |
| 447 | + | if err := listener.Close(); err != nil { | |
| 448 | + | t.Fatalf("second Close: %v", err) | |
| 449 | + | } | |
| 450 | + | } | |
| 451 | + | ||
| 452 | + | // TestDialMuxCancellationAndCloseUnblocksIO proves a canceled dial fails | |
| 453 | + | // promptly and that closing one end unblocks a blocked read on the other. | |
| 454 | + | func TestDialMuxCancellationAndCloseUnblocksIO(t *testing.T) { | |
| 455 | + | t.Run("canceled context", func(t *testing.T) { | |
| 456 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 457 | + | cancel() | |
| 458 | + | if transport, err := DialMuxContext(ctx, muxTestPath(t)); !errors.Is(err, context.Canceled) { | |
| 459 | + | if transport != nil { | |
| 460 | + | _ = transport.Close() | |
| 461 | + | } | |
| 462 | + | t.Fatalf("DialMuxContext error = %v, want context.Canceled", err) | |
| 463 | + | } | |
| 464 | + | }) | |
| 465 | + | ||
| 466 | + | t.Run("close unblocks read", func(t *testing.T) { | |
| 467 | + | path := muxTestPath(t) | |
| 468 | + | listener := mustListenMux(t, path) | |
| 469 | + | ||
| 470 | + | accepted := make(chan wire.BoundedTransport, 1) | |
| 471 | + | go func() { | |
| 472 | + | transport, _ := listener.Accept() | |
| 473 | + | accepted <- transport | |
| 474 | + | }() | |
| 475 | + | client := awaitMuxDial(t, dialMuxAsync(t, context.Background(), path)) | |
| 476 | + | server := <-accepted | |
| 477 | + | if server == nil { | |
| 478 | + | t.Fatal("Accept did not return a transport") | |
| 479 | + | } | |
| 480 | + | ||
| 481 | + | readErr := make(chan error, 1) | |
| 482 | + | go func() { | |
| 483 | + | _, err := client.Recv() | |
| 484 | + | readErr <- err | |
| 485 | + | }() | |
| 486 | + | if err := server.Close(); err != nil { | |
| 487 | + | t.Fatalf("server Close: %v", err) | |
| 488 | + | } | |
| 489 | + | select { | |
| 490 | + | case err := <-readErr: | |
| 491 | + | if err == nil { | |
| 492 | + | t.Fatal("blocked Recv returned nil error after peer Close") | |
| 493 | + | } | |
| 494 | + | case <-time.After(5 * time.Second): | |
| 495 | + | t.Fatal("peer Close did not unblock Recv") | |
| 496 | + | } | |
| 497 | + | }) | |
| 498 | + | } | |
| 499 | + | ||
| 500 | + | // TestListenMuxConcurrentCloseAndDial proves a Close racing many dials neither | |
| 501 | + | // panics nor hangs, and every dial result is closed by the test. | |
| 502 | + | func TestListenMuxConcurrentCloseAndDial(t *testing.T) { | |
| 503 | + | path := muxTestPath(t) | |
| 504 | + | listener := mustListenMux(t, path) | |
| 505 | + | ||
| 506 | + | const dialers = 16 | |
| 507 | + | results := make(chan wire.BoundedTransport, dialers) | |
| 508 | + | var wg sync.WaitGroup | |
| 509 | + | for i := 0; i < dialers; i++ { | |
| 510 | + | wg.Add(1) | |
| 511 | + | go func() { | |
| 512 | + | defer wg.Done() | |
| 513 | + | transport, err := DialMuxContext(context.Background(), path) | |
| 514 | + | if err != nil { | |
| 515 | + | results <- nil | |
| 516 | + | return | |
| 517 | + | } | |
| 518 | + | results <- transport | |
| 519 | + | }() | |
| 520 | + | } | |
| 521 | + | if err := listener.Close(); err != nil { | |
| 522 | + | t.Fatalf("Close: %v", err) | |
| 523 | + | } | |
| 524 | + | wg.Wait() | |
| 525 | + | close(results) | |
| 526 | + | for transport := range results { | |
| 527 | + | if transport != nil { | |
| 528 | + | _ = transport.Close() | |
| 529 | + | } | |
| 530 | + | } | |
| 531 | + | } | |
A internal/adapters/ipc/mux_unsupported_test.go +60 -0
| @@ -0,0 +1,60 @@ | |||
| 1 | + | //go:build !linux | |
| 2 | + | ||
| 3 | + | package ipc | |
| 4 | + | ||
| 5 | + | import ( | |
| 6 | + | "context" | |
| 7 | + | "errors" | |
| 8 | + | "path/filepath" | |
| 9 | + | "testing" | |
| 10 | + | "time" | |
| 11 | + | ) | |
| 12 | + | ||
| 13 | + | // TestMuxFailsClosedWithoutPeerCredentialCheck proves the private mux carriage | |
| 14 | + | // fails closed on a platform or build without the Linux SO_PEERCRED same-user | |
| 15 | + | // check: a bound listener refuses every peer instead of admitting an unverified | |
| 16 | + | // one, and dialing never returns an unverified transport. | |
| 17 | + | func TestMuxFailsClosedWithoutPeerCredentialCheck(t *testing.T) { | |
| 18 | + | path := filepath.Join(shortSocketDir(t, "vev"), "mux.sock") | |
| 19 | + | listener, err := ListenMux(path) | |
| 20 | + | if err != nil { | |
| 21 | + | t.Fatalf("ListenMux: %v", err) | |
| 22 | + | } | |
| 23 | + | defer func() { _ = listener.Close() }() | |
| 24 | + | ||
| 25 | + | type dialOutcome struct { | |
| 26 | + | transport interface{ Close() error } | |
| 27 | + | err error | |
| 28 | + | } | |
| 29 | + | dialed := make(chan dialOutcome, 1) | |
| 30 | + | go func() { | |
| 31 | + | transport, dialErr := DialMuxContext(context.Background(), path) | |
| 32 | + | if transport == nil { | |
| 33 | + | dialed <- dialOutcome{err: dialErr} | |
| 34 | + | return | |
| 35 | + | } | |
| 36 | + | dialed <- dialOutcome{transport: transport, err: dialErr} | |
| 37 | + | }() | |
| 38 | + | ||
| 39 | + | transport, acceptErr := listener.Accept() | |
| 40 | + | if transport != nil { | |
| 41 | + | _ = transport.Close() | |
| 42 | + | t.Fatal("Accept returned an unverified transport") | |
| 43 | + | } | |
| 44 | + | if !errors.Is(acceptErr, ErrMuxUnsupported) { | |
| 45 | + | t.Fatalf("Accept error = %v, want ErrMuxUnsupported", acceptErr) | |
| 46 | + | } | |
| 47 | + | ||
| 48 | + | select { | |
| 49 | + | case outcome := <-dialed: | |
| 50 | + | if outcome.transport != nil { | |
| 51 | + | _ = outcome.transport.Close() | |
| 52 | + | t.Fatal("DialMuxContext returned an unverified transport") | |
| 53 | + | } | |
| 54 | + | if !errors.Is(outcome.err, ErrMuxUnsupported) { | |
| 55 | + | t.Fatalf("DialMuxContext error = %v, want ErrMuxUnsupported", outcome.err) | |
| 56 | + | } | |
| 57 | + | case <-time.After(5 * time.Second): | |
| 58 | + | t.Fatal("DialMuxContext did not return") | |
| 59 | + | } | |
| 60 | + | } | |
A internal/adapters/ipc/peer_linux.go +46 -0 collapsed to keep the page light
A internal/adapters/ipc/peer_other.go +14 -0 collapsed to keep the page light
M internal/adapters/ipc/transport.go +17 -0 collapsed to keep the page light
M internal/adapters/ipc/transport_test.go +58 -0 collapsed to keep the page light
M internal/adapters/quic/bootstrap.go +2 -2 collapsed to keep the page light
A internal/adapters/quic/mux.go +111 -0 collapsed to keep the page light
A internal/adapters/quic/mux_test.go +440 -0 collapsed to keep the page light
M internal/adapters/quic/quic.go +22 -34 collapsed to keep the page light
M internal/adapters/sessionwire/client.go +80 -2 collapsed to keep the page light
A internal/adapters/sessionwire/client_handshake_test.go +98 -0 collapsed to keep the page light
M internal/adapters/sessionwire/server.go +94 -2 collapsed to keep the page light
A internal/adapters/sessionwire/server_handshake_test.go +139 -0 collapsed to keep the page light
A internal/adapters/sshstdio/mux.go +251 -0 collapsed to keep the page light
A internal/adapters/sshstdio/mux_test.go +110 -0 collapsed to keep the page light
M internal/adapters/sshstdio/transport.go +57 -12 collapsed to keep the page light
M internal/adapters/sshstdio/transport_test.go +40 -0 collapsed to keep the page light
M internal/adapters/streamframe/streamframe.go +36 -3 collapsed to keep the page light
M internal/adapters/streamframe/streamframe_test.go +177 -0 collapsed to keep the page light
M internal/ports/mocks/session_mock.go +72 -0 collapsed to keep the page light
M internal/ports/session.go +22 -0 collapsed to keep the page light
M internal/protocol/session.go +5 -2 collapsed to keep the page light
M internal/protocol/session_test.go +1 -1
| @@ -10,7 +10,7 @@ import ( | |||
| 10 | 10 | ) | |
| 11 | 11 | ||
| 12 | 12 | func TestVersionRemainsWireCompatible(t *testing.T) { | |
| 13 | - | require.Equal(t, uint16(53), Version) | |
| 13 | + | require.Equal(t, uint16(54), Version) | |
| 14 | 14 | require.Equal(t, (16<<20)-407, MaxOutputDataLen) | |
| 15 | 15 | } | |
| 16 | 16 | ||
M internal/protocol/wire/envelope_inventory_test.go +2 -2
| @@ -15,8 +15,8 @@ import ( | |||
| 15 | 15 | // shares a payload type except route navigation failure, and the preamble | |
| 16 | 16 | // constants agree with the schema authority. | |
| 17 | 17 | func TestEnvelopeInventoryProvesDirectionAndSemanticPath(t *testing.T) { | |
| 18 | - | if protocol.Version != 53 { | |
| 19 | - | t.Fatalf("protocol.Version = %d, want 53", protocol.Version) | |
| 18 | + | if protocol.Version != 54 { | |
| 19 | + | t.Fatalf("protocol.Version = %d, want 54", protocol.Version) | |
| 20 | 20 | } | |
| 21 | 21 | if ProtocolEpoch != 1 { | |
| 22 | 22 | t.Fatalf("ProtocolEpoch = %d, want 1", ProtocolEpoch) | |