Merge pull request #65 from bnema/fix/udp-mobile-congestion-resilience
fix(remote): improve UDP mobile congestion resilience
37 files · +4546 -750
M docs/performance.md +4 -0
| @@ -24,6 +24,10 @@ strace -ff -c -o tmux-cat.strace tmux new-session -d -s perf-cat 'timeout 30s sh | |||
| 24 | 24 | ||
| 25 | 25 | Record host CPU, OS/kernel, terminal emulator, vev commit, tmux version, terminal size, workload duration, raw command lines, syscall summaries, bytes written per rendered frame, and heap profiles or allocation counts before comparing. | |
| 26 | 26 | ||
| 27 | + | ## UDP mobile congestion behavior | |
| 28 | + | ||
| 29 | + | The datagram transport uses one shared congestion-responsive byte pacer for initial sends and retransmits. Retransmissions therefore consume the same byte budget as new data instead of bypassing congestion feedback. Datagram clients use a one-state output window: the cumulative ACK releases that state, and newer repaint state coalesces while it remains unacknowledged rather than retaining an unbounded queue. | |
| 30 | + | ||
| 27 | 31 | Microbenchmarks executed locally for this review using `go test ./internal/adapters/ipc ./internal/usecase/daemon -run '^$' -bench=. -benchmem`: | |
| 28 | 32 | ||
| 29 | 33 | | benchmark | methodology | | |
M docs/remote-resilience.md +11 -5
| @@ -15,14 +15,20 @@ Resilience work uses these user-facing states: | |||
| 15 | 15 | ||
| 16 | 16 | | State | Meaning | | |
| 17 | 17 | | --- | --- | | |
| 18 | - | | `connected` | authenticated UDP traffic is flowing | | |
| 19 | - | | `degraded` | traffic is delayed or incomplete, but recovery is still expected | | |
| 20 | - | | `probing` | vev is trying alternate UDP paths before using SSH again | | |
| 18 | + | | `connected` | a complete record or meaningful ACK progress is recent | | |
| 19 | + | | `degraded` | authenticated UDP packets still arrive, but complete-record or ACK progress is delayed | | |
| 20 | + | | `probing` | authenticated UDP packet contact is absent, so vev probes for path recovery | | |
| 21 | 21 | | `resuming` | a replacement transport is attaching to the same session | | |
| 22 | - | | `offline` | no path currently works, but retries continue | | |
| 22 | + | | `offline` | authenticated UDP contact remains absent and vev resumes over SSH stdio | | |
| 23 | 23 | | `expired` | the session or recovery window is no longer available | | |
| 24 | 24 | ||
| 25 | - | When a terminal is in raw mode, recovery messages should appear as vev status text rather than as bytes written into the remote program's output stream. | |
| 25 | + | When a terminal is in raw mode, recovery messages appear as vev status text rather than as bytes written into the remote program's output stream. | |
| 26 | + | ||
| 27 | + | `VEV_REMOTE_TRANSPORT=stdio` is the SSH-stdio fallback: it keeps the session inside authenticated SSH when UDP is blocked or unavailable. It remains available while UDP recovery is attempted. | |
| 28 | + | ||
| 29 | + | ## UDP transport health diagnostics | |
| 30 | + | ||
| 31 | + | At debug logging, UDP transport-health lines report only state, elapsed progress ages, and bounded transport counters; they do not include payloads, addresses, keys, or identities. For a real remote attempt, `scripts/debug-remote-attach.sh --udp-health user@host` stores redacted transport-health lines and Linux `ss -u -m` socket-memory counters, then prints the exact collected file paths. | |
| 26 | 32 | ||
| 27 | 33 | ## Manual checks | |
| 28 | 34 | ||
A internal/adapters/dgram/congestion.go +68 -0
| @@ -0,0 +1,68 @@ | |||
| 1 | + | package dgram | |
| 2 | + | ||
| 3 | + | import "time" | |
| 4 | + | ||
| 5 | + | const ( | |
| 6 | + | initialCongestionPackets = 10 | |
| 7 | + | minimumCongestionPackets = 2 | |
| 8 | + | maximumCongestionBytes = 4 << 20 | |
| 9 | + | initialPacingRTT = 250 * time.Millisecond | |
| 10 | + | dataBurstPackets = 2 | |
| 11 | + | ) | |
| 12 | + | ||
| 13 | + | type congestionController struct { | |
| 14 | + | mtu int | |
| 15 | + | cwndBytes int | |
| 16 | + | minBytes int | |
| 17 | + | maxBytes int | |
| 18 | + | srtt time.Duration | |
| 19 | + | } | |
| 20 | + | ||
| 21 | + | func newCongestionController(mtu int) congestionController { | |
| 22 | + | minimum := minimumCongestionPackets * mtu | |
| 23 | + | maximum := maximumCongestionBytes | |
| 24 | + | if maximum < minimum { | |
| 25 | + | maximum = minimum | |
| 26 | + | } | |
| 27 | + | initial := initialCongestionPackets * mtu | |
| 28 | + | if initial > maximum { | |
| 29 | + | initial = maximum | |
| 30 | + | } | |
| 31 | + | return congestionController{ | |
| 32 | + | mtu: mtu, | |
| 33 | + | cwndBytes: initial, | |
| 34 | + | minBytes: minimum, | |
| 35 | + | maxBytes: maximum, | |
| 36 | + | srtt: initialPacingRTT, | |
| 37 | + | } | |
| 38 | + | } | |
| 39 | + | ||
| 40 | + | func (c *congestionController) onRTT(srtt time.Duration) { | |
| 41 | + | if srtt > 0 { | |
| 42 | + | c.srtt = srtt | |
| 43 | + | } | |
| 44 | + | } | |
| 45 | + | ||
| 46 | + | func (c *congestionController) onACK(ackedBytes int) { | |
| 47 | + | if ackedBytes <= 0 || c.cwndBytes >= c.maxBytes { | |
| 48 | + | return | |
| 49 | + | } | |
| 50 | + | increase := int(int64(c.mtu) * int64(ackedBytes) / int64(c.cwndBytes)) | |
| 51 | + | if increase < 1 { | |
| 52 | + | increase = 1 | |
| 53 | + | } | |
| 54 | + | c.cwndBytes = min(c.cwndBytes+increase, c.maxBytes) | |
| 55 | + | } | |
| 56 | + | ||
| 57 | + | func (c *congestionController) onLoss() { | |
| 58 | + | c.cwndBytes = max(c.cwndBytes/2, c.minBytes) | |
| 59 | + | } | |
| 60 | + | ||
| 61 | + | func (c congestionController) bytesPerSecond() int { | |
| 62 | + | rtt := max(c.srtt, time.Millisecond) | |
| 63 | + | return int(int64(c.cwndBytes) * int64(time.Second) / int64(rtt)) | |
| 64 | + | } | |
| 65 | + | ||
| 66 | + | func (c congestionController) burstBytes() int { | |
| 67 | + | return min(c.cwndBytes, dataBurstPackets*c.mtu) | |
| 68 | + | } | |
A internal/adapters/dgram/control.go +216 -0
| @@ -0,0 +1,216 @@ | |||
| 1 | + | package dgram | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "encoding/binary" | |
| 5 | + | "errors" | |
| 6 | + | "net" | |
| 7 | + | "time" | |
| 8 | + | ||
| 9 | + | "github.com/bnema/vev/internal/ports" | |
| 10 | + | pdgram "github.com/bnema/vev/pkg/dgram" | |
| 11 | + | ) | |
| 12 | + | ||
| 13 | + | const ( | |
| 14 | + | controlQueueSize = 16 | |
| 15 | + | maxACKDelay = 25 * time.Millisecond | |
| 16 | + | ) | |
| 17 | + | ||
| 18 | + | type controlRecord struct { | |
| 19 | + | kind byte | |
| 20 | + | id uint64 | |
| 21 | + | result chan<- error | |
| 22 | + | } | |
| 23 | + | ||
| 24 | + | // queueACK retains the greatest cumulative acknowledgement without ever making | |
| 25 | + | // packet receipt wait for a socket write. | |
| 26 | + | func (t *Transport) queueACK(seq uint64) { | |
| 27 | + | t.controlMu.Lock() | |
| 28 | + | if !t.ackQueued || seq > t.ackSeq { | |
| 29 | + | t.ackSeq = seq | |
| 30 | + | t.ackQueued = true | |
| 31 | + | } | |
| 32 | + | t.controlMu.Unlock() | |
| 33 | + | if t.ackWake == nil { | |
| 34 | + | // Small unit-test transports may exercise record handling without the | |
| 35 | + | // goroutines installed by NewTransportWithOptions. | |
| 36 | + | go t.sendAck(seq) | |
| 37 | + | return | |
| 38 | + | } | |
| 39 | + | select { | |
| 40 | + | case t.ackWake <- struct{}{}: | |
| 41 | + | t.mu.Lock() | |
| 42 | + | hook := t.afterACKWakeAccepted | |
| 43 | + | t.mu.Unlock() | |
| 44 | + | if hook != nil { | |
| 45 | + | hook() | |
| 46 | + | } | |
| 47 | + | default: | |
| 48 | + | } | |
| 49 | + | } | |
| 50 | + | ||
| 51 | + | // queueControl is deliberately lossy when the bounded control queue is full: | |
| 52 | + | // probes and pongs are advisory and must not stall authenticated receipt. | |
| 53 | + | func (t *Transport) queueControl(kind byte, id uint64) bool { | |
| 54 | + | return t.queueControlRecord(controlRecord{kind: kind, id: id}) | |
| 55 | + | } | |
| 56 | + | ||
| 57 | + | func (t *Transport) queueControlRecord(record controlRecord) bool { | |
| 58 | + | select { | |
| 59 | + | case t.control <- record: | |
| 60 | + | return true | |
| 61 | + | default: | |
| 62 | + | return false | |
| 63 | + | } | |
| 64 | + | } | |
| 65 | + | ||
| 66 | + | func (t *Transport) takeACK() (uint64, bool) { | |
| 67 | + | t.controlMu.Lock() | |
| 68 | + | defer t.controlMu.Unlock() | |
| 69 | + | seq, ok := t.ackSeq, t.ackQueued | |
| 70 | + | t.ackSeq = 0 | |
| 71 | + | t.ackQueued = false | |
| 72 | + | return seq, ok | |
| 73 | + | } | |
| 74 | + | ||
| 75 | + | func (t *Transport) controlLoop() { | |
| 76 | + | go t.ackScheduleLoop() | |
| 77 | + | for { | |
| 78 | + | select { | |
| 79 | + | case record := <-t.control: | |
| 80 | + | err := t.sendControl(record.kind, record.id) | |
| 81 | + | if record.result != nil { | |
| 82 | + | select { | |
| 83 | + | case record.result <- err: | |
| 84 | + | default: | |
| 85 | + | } | |
| 86 | + | } | |
| 87 | + | case <-t.done: | |
| 88 | + | return | |
| 89 | + | } | |
| 90 | + | } | |
| 91 | + | } | |
| 92 | + | ||
| 93 | + | func (t *Transport) ackScheduleLoop() { | |
| 94 | + | var ackTimer ports.Timer | |
| 95 | + | var ackTimerC <-chan time.Time | |
| 96 | + | defer func() { | |
| 97 | + | if ackTimer != nil { | |
| 98 | + | ackTimer.Stop() | |
| 99 | + | } | |
| 100 | + | }() | |
| 101 | + | scheduleACK := func() { | |
| 102 | + | if ackTimerC == nil { | |
| 103 | + | if ackTimer == nil { | |
| 104 | + | ackTimer = t.clock.NewTimer(maxACKDelay) | |
| 105 | + | } else { | |
| 106 | + | ackTimer.Reset(maxACKDelay) | |
| 107 | + | } | |
| 108 | + | ackTimerC = ackTimer.C() | |
| 109 | + | } | |
| 110 | + | t.mu.Lock() | |
| 111 | + | hook := t.afterACKScheduled | |
| 112 | + | t.mu.Unlock() | |
| 113 | + | if hook != nil { | |
| 114 | + | hook() | |
| 115 | + | } | |
| 116 | + | } | |
| 117 | + | dispatchACK := func() { | |
| 118 | + | seq, ok := t.takeACK() | |
| 119 | + | if !ok { | |
| 120 | + | return | |
| 121 | + | } | |
| 122 | + | select { | |
| 123 | + | case t.ackSend <- seq: | |
| 124 | + | t.mu.Lock() | |
| 125 | + | hook := t.afterACKDispatched | |
| 126 | + | t.mu.Unlock() | |
| 127 | + | if hook != nil { | |
| 128 | + | hook() | |
| 129 | + | } | |
| 130 | + | default: | |
| 131 | + | // Preserve the cumulative maximum until the bounded ACK sender has | |
| 132 | + | // capacity again. | |
| 133 | + | t.queueACK(seq) | |
| 134 | + | } | |
| 135 | + | } | |
| 136 | + | ||
| 137 | + | for { | |
| 138 | + | // Check ACK scheduling explicitly so a sustained stream of advisory | |
| 139 | + | // controls cannot prevent the bounded coalescing deadline from starting | |
| 140 | + | // or expiring. Dispatch itself never writes to the socket. | |
| 141 | + | select { | |
| 142 | + | case <-ackTimerC: | |
| 143 | + | ackTimerC = nil | |
| 144 | + | dispatchACK() | |
| 145 | + | continue | |
| 146 | + | default: | |
| 147 | + | } | |
| 148 | + | select { | |
| 149 | + | case <-t.ackWake: | |
| 150 | + | scheduleACK() | |
| 151 | + | continue | |
| 152 | + | default: | |
| 153 | + | } | |
| 154 | + | ||
| 155 | + | select { | |
| 156 | + | case <-t.ackWake: | |
| 157 | + | scheduleACK() | |
| 158 | + | case <-ackTimerC: | |
| 159 | + | ackTimerC = nil | |
| 160 | + | dispatchACK() | |
| 161 | + | case <-t.done: | |
| 162 | + | return | |
| 163 | + | } | |
| 164 | + | } | |
| 165 | + | } | |
| 166 | + | ||
| 167 | + | func (t *Transport) ackSendLoop() { | |
| 168 | + | for { | |
| 169 | + | select { | |
| 170 | + | case seq := <-t.ackSend: | |
| 171 | + | _ = t.sendControl(recAck, seq) | |
| 172 | + | case <-t.done: | |
| 173 | + | return | |
| 174 | + | } | |
| 175 | + | } | |
| 176 | + | } | |
| 177 | + | ||
| 178 | + | func (t *Transport) sendControl(kind byte, id uint64) error { | |
| 179 | + | var payload [9]byte | |
| 180 | + | payload[0] = kind | |
| 181 | + | binary.BigEndian.PutUint64(payload[1:], id) | |
| 182 | + | frags, err := pdgram.FragmentPayload(t.nextCounter(), payload[:], t.mtu-pdgram.HeaderSize-t.codec.Overhead()) | |
| 183 | + | if err != nil { | |
| 184 | + | return err | |
| 185 | + | } | |
| 186 | + | for _, frag := range frags { | |
| 187 | + | raw, err := pdgram.MarshalFragment(frag) | |
| 188 | + | if err != nil { | |
| 189 | + | return err | |
| 190 | + | } | |
| 191 | + | pkt := t.codec.Seal(t.sendDir, t.nextCounter(), raw, nil) | |
| 192 | + | t.mu.Lock() | |
| 193 | + | pc := t.pc | |
| 194 | + | peer := t.peer | |
| 195 | + | closed := t.closed | |
| 196 | + | closeErr := t.closeErr | |
| 197 | + | writeTimeout := t.writeTimeout | |
| 198 | + | writeDeadlines := t.writeDeadlines | |
| 199 | + | t.mu.Unlock() | |
| 200 | + | if closed { | |
| 201 | + | if closeErr != nil { | |
| 202 | + | return closeErr | |
| 203 | + | } | |
| 204 | + | return net.ErrClosed | |
| 205 | + | } | |
| 206 | + | if peer == nil { | |
| 207 | + | return errors.New("dgram: control peer unavailable") | |
| 208 | + | } | |
| 209 | + | // PacketConn permits concurrent callers. Control traffic deliberately | |
| 210 | + | // bypasses data's write lock so a retransmit cannot starve heartbeats. | |
| 211 | + | if err := t.writeDatagram(pc, peer, pkt, writeDeadlines, writeTimeout); err != nil { | |
| 212 | + | return err | |
| 213 | + | } | |
| 214 | + | } | |
| 215 | + | return nil | |
| 216 | + | } | |
A internal/adapters/dgram/control_plane_test.go +495 -0
| @@ -0,0 +1,495 @@ | |||
| 1 | + | package dgram | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "encoding/binary" | |
| 6 | + | "errors" | |
| 7 | + | "net" | |
| 8 | + | "sync" | |
| 9 | + | "testing" | |
| 10 | + | "time" | |
| 11 | + | ||
| 12 | + | "github.com/bnema/vev/internal/ports" | |
| 13 | + | pdgram "github.com/bnema/vev/pkg/dgram" | |
| 14 | + | ) | |
| 15 | + | ||
| 16 | + | type stagedTimeoutPC struct { | |
| 17 | + | mu sync.Mutex | |
| 18 | + | deadline time.Time | |
| 19 | + | calls int | |
| 20 | + | started chan struct{} | |
| 21 | + | release chan struct{} | |
| 22 | + | } | |
| 23 | + | ||
| 24 | + | func (p *stagedTimeoutPC) ReadFrom([]byte) (int, net.Addr, error) { | |
| 25 | + | return 0, nil, net.ErrClosed | |
| 26 | + | } | |
| 27 | + | func (p *stagedTimeoutPC) WriteTo(b []byte, _ net.Addr) (int, error) { | |
| 28 | + | p.mu.Lock() | |
| 29 | + | p.calls++ | |
| 30 | + | call := p.calls | |
| 31 | + | p.mu.Unlock() | |
| 32 | + | if call <= 2 { | |
| 33 | + | p.started <- struct{}{} | |
| 34 | + | <-p.release | |
| 35 | + | return 0, timeoutErr{} | |
| 36 | + | } | |
| 37 | + | return len(b), nil | |
| 38 | + | } | |
| 39 | + | func (p *stagedTimeoutPC) Close() error { return nil } | |
| 40 | + | func (p *stagedTimeoutPC) LocalAddr() net.Addr { return testAddr("staged") } | |
| 41 | + | func (p *stagedTimeoutPC) SetDeadline(time.Time) error { return nil } | |
| 42 | + | func (p *stagedTimeoutPC) SetReadDeadline(time.Time) error { return nil } | |
| 43 | + | func (p *stagedTimeoutPC) SetWriteDeadline(d time.Time) error { | |
| 44 | + | p.mu.Lock() | |
| 45 | + | p.deadline = d | |
| 46 | + | p.mu.Unlock() | |
| 47 | + | return nil | |
| 48 | + | } | |
| 49 | + | ||
| 50 | + | func TestReadLoopDoesNotWriteACKSynchronously(t *testing.T) { | |
| 51 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 52 | + | aPC, bPC := newPair() | |
| 53 | + | a, err := NewTransportWithOptions(aPC, bPC.addr, key(), 1, 2, Options{ | |
| 54 | + | Clock: clk, ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 55 | + | }) | |
| 56 | + | if err != nil { | |
| 57 | + | t.Fatal(err) | |
| 58 | + | } | |
| 59 | + | b, err := NewTransportWithOptions(bPC, aPC.addr, key(), 2, 1, Options{ | |
| 60 | + | Clock: clk, ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 61 | + | }) | |
| 62 | + | if err != nil { | |
| 63 | + | _ = a.Close() | |
| 64 | + | t.Fatal(err) | |
| 65 | + | } | |
| 66 | + | releaseACK := make(chan struct{}) | |
| 67 | + | defer close(releaseACK) | |
| 68 | + | defer closeFloodTransports(a, b) | |
| 69 | + | ||
| 70 | + | ackStarted := make(chan struct{}, 1) | |
| 71 | + | aPC.drop = func(_ []byte, _ net.Addr) bool { | |
| 72 | + | ackStarted <- struct{}{} | |
| 73 | + | <-releaseACK | |
| 74 | + | return true | |
| 75 | + | } | |
| 76 | + | contacts := make(chan struct{}, 2) | |
| 77 | + | a.mu.Lock() | |
| 78 | + | a.afterAuthenticatedPacket = func() { contacts <- struct{}{} } | |
| 79 | + | // Keep diagnostics local to this test so each snapshot has an explicit | |
| 80 | + | // causative event instead of depending on observer scheduling. | |
| 81 | + | a.diagnosticCh = make(chan Diagnostic, 1) | |
| 82 | + | a.mu.Unlock() | |
| 83 | + | ||
| 84 | + | if err := b.Send(ports.Frame{Type: ports.MsgInput, Payload: []byte("first")}); err != nil { | |
| 85 | + | t.Fatal(err) | |
| 86 | + | } | |
| 87 | + | awaitSignal(t, contacts, "first authenticated record") | |
| 88 | + | for range 7 { | |
| 89 | + | awaitSignal(t, clk.timerCreated, "transport and ACK timer creation") | |
| 90 | + | } | |
| 91 | + | clk.advance(maxACKDelay) | |
| 92 | + | awaitSignal(t, ackStarted, "blocked ACK") | |
| 93 | + | ||
| 94 | + | clk.advance(time.Second) | |
| 95 | + | if err := b.Send(ports.Frame{Type: ports.MsgInput, Payload: []byte("second")}); err != nil { | |
| 96 | + | t.Fatal(err) | |
| 97 | + | } | |
| 98 | + | awaitSignal(t, contacts, "second authenticated record while ACK write is blocked") | |
| 99 | + | wantComplete := clk.Now() | |
| 100 | + | eventually(t, time.Second, func() bool { | |
| 101 | + | a.mu.Lock() | |
| 102 | + | defer a.mu.Unlock() | |
| 103 | + | return a.lastCompleteRecord.Equal(wantComplete) | |
| 104 | + | }) | |
| 105 | + | a.emitDiagnostic() | |
| 106 | + | diagnostic := awaitResult(t, a.diagnosticCh, "second-record diagnostic") | |
| 107 | + | if diagnostic.SinceCompleteRecord != 0 { | |
| 108 | + | t.Fatalf("second record age=%v, want 0 before ACK release", diagnostic.SinceCompleteRecord) | |
| 109 | + | } | |
| 110 | + | } | |
| 111 | + | ||
| 112 | + | func TestCumulativeACKCoalescesForBoundedDelay(t *testing.T) { | |
| 113 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 114 | + | aPC, bPC := newPair() | |
| 115 | + | a, err := NewTransportWithOptions(aPC, bPC.addr, key(), 1, 2, Options{ | |
| 116 | + | Clock: clk, ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 117 | + | }) | |
| 118 | + | if err != nil { | |
| 119 | + | t.Fatal(err) | |
| 120 | + | } | |
| 121 | + | defer func() { _ = a.Close() }() | |
| 122 | + | for range 3 { | |
| 123 | + | awaitSignal(t, clk.timerCreated, "transport timer creation") | |
| 124 | + | } | |
| 125 | + | ||
| 126 | + | acks := make(chan uint64, 2) | |
| 127 | + | aPC.drop = func(pkt []byte, _ net.Addr) bool { | |
| 128 | + | _, raw, err := a.codec.Open(pkt, a.sendDir, nil, nil) | |
| 129 | + | if err != nil { | |
| 130 | + | t.Errorf("open outbound ACK: %v", err) | |
| 131 | + | return true | |
| 132 | + | } | |
| 133 | + | frag, err := pdgram.UnmarshalFragment(raw) | |
| 134 | + | if err != nil { | |
| 135 | + | t.Errorf("unmarshal outbound ACK: %v", err) | |
| 136 | + | return true | |
| 137 | + | } | |
| 138 | + | if frag.Count == 1 && len(frag.Data) == 9 && frag.Data[0] == recAck { | |
| 139 | + | acks <- binary.BigEndian.Uint64(frag.Data[1:]) | |
| 140 | + | } | |
| 141 | + | return true | |
| 142 | + | } | |
| 143 | + | ||
| 144 | + | a.queueACK(1) | |
| 145 | + | a.queueACK(2) | |
| 146 | + | awaitSignal(t, clk.timerCreated, "ACK coalescing timer creation") | |
| 147 | + | clk.advance(maxACKDelay - time.Millisecond) | |
| 148 | + | select { | |
| 149 | + | case seq := <-acks: | |
| 150 | + | t.Fatalf("ACK %d sent before coalescing delay", seq) | |
| 151 | + | default: | |
| 152 | + | } | |
| 153 | + | clk.advance(time.Millisecond) | |
| 154 | + | if seq := awaitResult(t, acks, "coalesced ACK"); seq != 2 { | |
| 155 | + | t.Fatalf("ACK=%d, want cumulative maximum 2", seq) | |
| 156 | + | } | |
| 157 | + | select { | |
| 158 | + | case seq := <-acks: | |
| 159 | + | t.Fatalf("extra ACK %d sent for coalesced records", seq) | |
| 160 | + | default: | |
| 161 | + | } | |
| 162 | + | } | |
| 163 | + | ||
| 164 | + | func TestACKDeadlineDispatchesWhileAdvisoryWriteIsBlocked(t *testing.T) { | |
| 165 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 166 | + | aPC, bPC := newPair() | |
| 167 | + | a, err := NewTransportWithOptions(aPC, bPC.addr, key(), 1, 2, Options{ | |
| 168 | + | Clock: clk, ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 169 | + | }) | |
| 170 | + | if err != nil { | |
| 171 | + | t.Fatal(err) | |
| 172 | + | } | |
| 173 | + | defer func() { _ = a.Close() }() | |
| 174 | + | for range 3 { | |
| 175 | + | awaitSignal(t, clk.timerCreated, "transport timer creation") | |
| 176 | + | } | |
| 177 | + | ||
| 178 | + | probeStarted := make(chan struct{}, 1) | |
| 179 | + | ackStarted := make(chan struct{}, 1) | |
| 180 | + | releaseProbe := make(chan struct{}) | |
| 181 | + | defer close(releaseProbe) | |
| 182 | + | aPC.drop = func(pkt []byte, _ net.Addr) bool { | |
| 183 | + | _, raw, err := a.codec.Open(pkt, a.sendDir, nil, nil) | |
| 184 | + | if err != nil { | |
| 185 | + | return true | |
| 186 | + | } | |
| 187 | + | frag, err := pdgram.UnmarshalFragment(raw) | |
| 188 | + | if err != nil || frag.Count != 1 || len(frag.Data) != 9 { | |
| 189 | + | return true | |
| 190 | + | } | |
| 191 | + | switch frag.Data[0] { | |
| 192 | + | case recProbe: | |
| 193 | + | probeStarted <- struct{}{} | |
| 194 | + | <-releaseProbe | |
| 195 | + | case recAck: | |
| 196 | + | ackStarted <- struct{}{} | |
| 197 | + | } | |
| 198 | + | return true | |
| 199 | + | } | |
| 200 | + | ||
| 201 | + | if !a.queueControl(recProbe, 1) { | |
| 202 | + | t.Fatal("probe control queue unexpectedly full") | |
| 203 | + | } | |
| 204 | + | awaitSignal(t, probeStarted, "blocked advisory probe write") | |
| 205 | + | a.queueACK(1) | |
| 206 | + | awaitSignal(t, clk.timerCreated, "ACK coalescing timer creation") | |
| 207 | + | clk.advance(maxACKDelay) | |
| 208 | + | awaitSignal(t, ackStarted, "ACK write while advisory probe remains blocked") | |
| 209 | + | } | |
| 210 | + | ||
| 211 | + | func TestProbeDoesNotBlockOnDataWriter(t *testing.T) { | |
| 212 | + | aPC, bPC := newPair() | |
| 213 | + | a, err := NewTransportWithOptions(aPC, bPC.addr, key(), 1, 2, Options{ | |
| 214 | + | ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 215 | + | }) | |
| 216 | + | if err != nil { | |
| 217 | + | t.Fatal(err) | |
| 218 | + | } | |
| 219 | + | b, err := NewTransportWithOptions(bPC, aPC.addr, key(), 2, 1, Options{ | |
| 220 | + | ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 221 | + | }) | |
| 222 | + | if err != nil { | |
| 223 | + | _ = a.Close() | |
| 224 | + | t.Fatal(err) | |
| 225 | + | } | |
| 226 | + | defer closeFloodTransports(a, b) | |
| 227 | + | ||
| 228 | + | a.writeMu.Lock() | |
| 229 | + | ctx, cancel := context.WithTimeout(context.Background(), time.Second) | |
| 230 | + | defer cancel() | |
| 231 | + | done := make(chan error, 1) | |
| 232 | + | go func() { done <- a.Probe(ctx) }() | |
| 233 | + | select { | |
| 234 | + | case err := <-done: | |
| 235 | + | a.writeMu.Unlock() | |
| 236 | + | if err != nil { | |
| 237 | + | t.Fatalf("Probe failed: %v", err) | |
| 238 | + | } | |
| 239 | + | case <-time.After(50 * time.Millisecond): | |
| 240 | + | a.writeMu.Unlock() | |
| 241 | + | <-done | |
| 242 | + | t.Fatal("Probe blocked behind data writer") | |
| 243 | + | } | |
| 244 | + | } | |
| 245 | + | ||
| 246 | + | func TestProbeDoesNotBlockBehindACKWrite(t *testing.T) { | |
| 247 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 248 | + | aPC, bPC := newPair() | |
| 249 | + | a, err := NewTransportWithOptions(aPC, bPC.addr, key(), 1, 2, Options{ | |
| 250 | + | Clock: clk, ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 251 | + | }) | |
| 252 | + | if err != nil { | |
| 253 | + | t.Fatal(err) | |
| 254 | + | } | |
| 255 | + | defer func() { _ = a.Close() }() | |
| 256 | + | for range 3 { | |
| 257 | + | awaitSignal(t, clk.timerCreated, "transport timer creation") | |
| 258 | + | } | |
| 259 | + | ||
| 260 | + | ackStarted := make(chan struct{}, 1) | |
| 261 | + | probeStarted := make(chan struct{}, 1) | |
| 262 | + | releaseACK := make(chan struct{}) | |
| 263 | + | defer close(releaseACK) | |
| 264 | + | aPC.drop = func(pkt []byte, _ net.Addr) bool { | |
| 265 | + | _, raw, err := a.codec.Open(pkt, a.sendDir, nil, nil) | |
| 266 | + | if err != nil { | |
| 267 | + | return true | |
| 268 | + | } | |
| 269 | + | frag, err := pdgram.UnmarshalFragment(raw) | |
| 270 | + | if err != nil || frag.Count != 1 || len(frag.Data) != 9 { | |
| 271 | + | return true | |
| 272 | + | } | |
| 273 | + | switch frag.Data[0] { | |
| 274 | + | case recAck: | |
| 275 | + | ackStarted <- struct{}{} | |
| 276 | + | <-releaseACK | |
| 277 | + | case recProbe: | |
| 278 | + | probeStarted <- struct{}{} | |
| 279 | + | } | |
| 280 | + | return true | |
| 281 | + | } | |
| 282 | + | ||
| 283 | + | a.queueACK(1) | |
| 284 | + | awaitSignal(t, clk.timerCreated, "ACK coalescing timer creation") | |
| 285 | + | clk.advance(maxACKDelay) | |
| 286 | + | awaitSignal(t, ackStarted, "blocked ACK") | |
| 287 | + | if !a.queueControl(recProbe, 1) { | |
| 288 | + | t.Fatal("probe control queue unexpectedly full") | |
| 289 | + | } | |
| 290 | + | awaitSignal(t, probeStarted, "probe while ACK write is blocked") | |
| 291 | + | } | |
| 292 | + | ||
| 293 | + | func TestControlWriteTimeoutBoundsBlockedPacketConn(t *testing.T) { | |
| 294 | + | pc := newBlockingWritePC() | |
| 295 | + | tr, err := NewTransportWithOptions(pc, testAddr("peer"), key(), 1, 2, Options{ | |
| 296 | + | WriteTimeout: 10 * time.Millisecond, | |
| 297 | + | ResendAfter: time.Hour, | |
| 298 | + | Heartbeat: time.Hour, | |
| 299 | + | }) | |
| 300 | + | if err != nil { | |
| 301 | + | t.Fatal(err) | |
| 302 | + | } | |
| 303 | + | defer func() { _ = tr.Close() }() | |
| 304 | + | ||
| 305 | + | done := make(chan error, 1) | |
| 306 | + | go func() { done <- tr.sendControl(recAck, 1) }() | |
| 307 | + | select { | |
| 308 | + | case err := <-done: | |
| 309 | + | var timeout interface{ Timeout() bool } | |
| 310 | + | if !errors.As(err, &timeout) || !timeout.Timeout() { | |
| 311 | + | t.Fatalf("control write err=%v, want timeout", err) | |
| 312 | + | } | |
| 313 | + | case <-time.After(250 * time.Millisecond): | |
| 314 | + | t.Fatal("control write blocked without honoring WriteTimeout") | |
| 315 | + | } | |
| 316 | + | } | |
| 317 | + | ||
| 318 | + | func TestConcurrentWriterCannotClearEarlierDeadline(t *testing.T) { | |
| 319 | + | pc := &deadlineCapturePC{addr: testAddr("a"), done: make(chan struct{})} | |
| 320 | + | state := newWriteDeadlineState(pc) | |
| 321 | + | earlier := time.Now().Add(time.Second) | |
| 322 | + | later := earlier.Add(time.Second) | |
| 323 | + | finishEarlier, err := state.begin(earlier) | |
| 324 | + | if err != nil { | |
| 325 | + | t.Fatal(err) | |
| 326 | + | } | |
| 327 | + | defer finishEarlier() | |
| 328 | + | finishLater, err := state.begin(later) | |
| 329 | + | if err != nil { | |
| 330 | + | t.Fatal(err) | |
| 331 | + | } | |
| 332 | + | finishLater() | |
| 333 | + | ||
| 334 | + | got, _ := pc.deadline.Load().(time.Time) | |
| 335 | + | if !got.Equal(earlier) { | |
| 336 | + | t.Fatalf("deadline after concurrent writer completed=%v, want earlier active deadline %v", got, earlier) | |
| 337 | + | } | |
| 338 | + | } | |
| 339 | + | ||
| 340 | + | func TestFreshControlWriteRetriesAfterInheritedDeadline(t *testing.T) { | |
| 341 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 342 | + | pc := &stagedTimeoutPC{started: make(chan struct{}, 2), release: make(chan struct{})} | |
| 343 | + | tr := &Transport{clock: clk} | |
| 344 | + | deadlines := newWriteDeadlineState(pc) | |
| 345 | + | oldDone := make(chan error, 1) | |
| 346 | + | go func() { | |
| 347 | + | oldDone <- tr.writeDatagram(pc, testAddr("peer"), []byte("old"), deadlines, 10*time.Millisecond) | |
| 348 | + | }() | |
| 349 | + | awaitSignal(t, pc.started, "older blocked write") | |
| 350 | + | clk.advance(5 * time.Millisecond) | |
| 351 | + | freshDone := make(chan error, 1) | |
| 352 | + | go func() { | |
| 353 | + | freshDone <- tr.writeDatagram(pc, testAddr("peer"), []byte("fresh"), deadlines, 100*time.Millisecond) | |
| 354 | + | }() | |
| 355 | + | awaitSignal(t, pc.started, "fresh write sharing earlier deadline") | |
| 356 | + | clk.advance(5 * time.Millisecond) | |
| 357 | + | close(pc.release) | |
| 358 | + | if err := awaitResult(t, oldDone, "older write timeout"); err == nil { | |
| 359 | + | t.Fatal("older write succeeded, want timeout") | |
| 360 | + | } | |
| 361 | + | if err := awaitResult(t, freshDone, "fresh write retry"); err != nil { | |
| 362 | + | t.Fatalf("fresh write failed at inherited deadline: %v", err) | |
| 363 | + | } | |
| 364 | + | } | |
| 365 | + | ||
| 366 | + | func TestProbeReturnsLocalWriteError(t *testing.T) { | |
| 367 | + | aPC, bPC := newPair() | |
| 368 | + | a, err := NewTransportWithOptions(aPC, bPC.addr, key(), 1, 2, Options{ | |
| 369 | + | ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 370 | + | }) | |
| 371 | + | if err != nil { | |
| 372 | + | t.Fatal(err) | |
| 373 | + | } | |
| 374 | + | defer func() { _ = a.Close() }() | |
| 375 | + | if err := bPC.Close(); err != nil { | |
| 376 | + | t.Fatal(err) | |
| 377 | + | } | |
| 378 | + | ||
| 379 | + | ctx, cancel := context.WithTimeout(context.Background(), time.Second) | |
| 380 | + | defer cancel() | |
| 381 | + | start := time.Now() | |
| 382 | + | if err := a.Probe(ctx); err == nil || errors.Is(err, context.DeadlineExceeded) { | |
| 383 | + | t.Fatalf("Probe err=%v, want immediate local write error", err) | |
| 384 | + | } | |
| 385 | + | if elapsed := time.Since(start); elapsed > 250*time.Millisecond { | |
| 386 | + | t.Fatalf("Probe returned local write error after %v", elapsed) | |
| 387 | + | } | |
| 388 | + | } | |
| 389 | + | ||
| 390 | + | func TestHeartbeatContinuesDuringRetransmitStorm(t *testing.T) { | |
| 391 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 392 | + | aPC, bPC := newPair() | |
| 393 | + | const heartbeat = 11 * time.Millisecond | |
| 394 | + | a, err := NewTransportWithOptions(aPC, bPC.addr, key(), 1, 2, Options{ | |
| 395 | + | Clock: clk, MTU: 128, ResendAfter: 10 * time.Millisecond, Heartbeat: heartbeat, | |
| 396 | + | }) | |
| 397 | + | if err != nil { | |
| 398 | + | t.Fatal(err) | |
| 399 | + | } | |
| 400 | + | defer func() { _ = a.Close() }() | |
| 401 | + | for range 3 { | |
| 402 | + | awaitSignal(t, clk.timerCreated, "retransmit, health, and heartbeat timer creation") | |
| 403 | + | } | |
| 404 | + | ||
| 405 | + | retransmitStarted := make(chan struct{}, 1) | |
| 406 | + | releaseRetransmit := make(chan struct{}) | |
| 407 | + | defer close(releaseRetransmit) | |
| 408 | + | heartbeats := make(chan struct{}, 2) | |
| 409 | + | aPC.drop = func(pkt []byte, _ net.Addr) bool { | |
| 410 | + | _, raw, err := a.codec.Open(pkt, a.sendDir, nil, nil) | |
| 411 | + | if err != nil { | |
| 412 | + | t.Errorf("open outbound packet: %v", err) | |
| 413 | + | return true | |
| 414 | + | } | |
| 415 | + | frag, err := pdgram.UnmarshalFragment(raw) | |
| 416 | + | if err != nil { | |
| 417 | + | t.Errorf("unmarshal outbound fragment: %v", err) | |
| 418 | + | return true | |
| 419 | + | } | |
| 420 | + | if frag.Count == 1 && len(frag.Data) == 9 && frag.Data[0] == recProbe { | |
| 421 | + | heartbeats <- struct{}{} | |
| 422 | + | return true | |
| 423 | + | } | |
| 424 | + | select { | |
| 425 | + | case retransmitStarted <- struct{}{}: | |
| 426 | + | <-releaseRetransmit | |
| 427 | + | default: | |
| 428 | + | } | |
| 429 | + | return true | |
| 430 | + | } | |
| 431 | + | ||
| 432 | + | now := clk.Now() | |
| 433 | + | a.mu.Lock() | |
| 434 | + | for seq := uint64(1); seq <= 64; seq++ { | |
| 435 | + | a.pending[seq] = &pending{ | |
| 436 | + | frame: ports.Frame{Type: ports.MsgInput, Payload: make([]byte, 24*a.mtu)}, | |
| 437 | + | enqueued: now.Add(-time.Second), first: now.Add(-time.Second), last: now.Add(-time.Second), | |
| 438 | + | } | |
| 439 | + | } | |
| 440 | + | a.mu.Unlock() | |
| 441 | + | ||
| 442 | + | clk.advance(10 * time.Millisecond) | |
| 443 | + | awaitSignal(t, retransmitStarted, "first retransmit in 64-record storm") | |
| 444 | + | for interval := 1; interval <= 2; interval++ { | |
| 445 | + | clk.advance(heartbeat) | |
| 446 | + | awaitSignal(t, heartbeats, "heartbeat during retransmit storm") | |
| 447 | + | } | |
| 448 | + | } | |
| 449 | + | ||
| 450 | + | func TestHealthChecksContinueDuringRetransmitStorm(t *testing.T) { | |
| 451 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 452 | + | aPC, bPC := newPair() | |
| 453 | + | a, err := NewTransportWithOptions(aPC, bPC.addr, key(), 1, 2, Options{ | |
| 454 | + | Clock: clk, MTU: 128, ResendAfter: 10 * time.Millisecond, | |
| 455 | + | Heartbeat: time.Hour, DegradedAfter: 10 * time.Millisecond, | |
| 456 | + | }) | |
| 457 | + | if err != nil { | |
| 458 | + | t.Fatal(err) | |
| 459 | + | } | |
| 460 | + | defer func() { _ = a.Close() }() | |
| 461 | + | for range 3 { | |
| 462 | + | awaitSignal(t, clk.timerCreated, "transport timer creation") | |
| 463 | + | } | |
| 464 | + | ||
| 465 | + | retransmitStarted := make(chan struct{}, 1) | |
| 466 | + | releaseRetransmit := make(chan struct{}) | |
| 467 | + | defer close(releaseRetransmit) | |
| 468 | + | aPC.drop = func(_ []byte, _ net.Addr) bool { | |
| 469 | + | select { | |
| 470 | + | case retransmitStarted <- struct{}{}: | |
| 471 | + | <-releaseRetransmit | |
| 472 | + | default: | |
| 473 | + | } | |
| 474 | + | return true | |
| 475 | + | } | |
| 476 | + | ||
| 477 | + | now := clk.Now() | |
| 478 | + | a.mu.Lock() | |
| 479 | + | a.pending[1] = &pending{ | |
| 480 | + | frame: ports.Frame{Type: ports.MsgInput, Payload: make([]byte, 24*a.mtu)}, | |
| 481 | + | enqueued: now.Add(-time.Second), first: now.Add(-time.Second), last: now.Add(-time.Second), | |
| 482 | + | } | |
| 483 | + | a.mu.Unlock() | |
| 484 | + | ||
| 485 | + | clk.advance(10 * time.Millisecond) | |
| 486 | + | awaitSignal(t, retransmitStarted, "blocked retransmit") | |
| 487 | + | select { | |
| 488 | + | case event := <-a.LinkEvents(): | |
| 489 | + | if event.State != ports.LinkStateDegraded { | |
| 490 | + | t.Fatalf("link state=%s, want degraded", event.State) | |
| 491 | + | } | |
| 492 | + | case <-time.After(time.Second): | |
| 493 | + | t.Fatal("health check did not run while retransmit sender was blocked") | |
| 494 | + | } | |
| 495 | + | } | |
A internal/adapters/dgram/diagnostics.go +47 -0
| @@ -0,0 +1,47 @@ | |||
| 1 | + | package dgram | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "log/slog" | |
| 5 | + | "time" | |
| 6 | + | ||
| 7 | + | "github.com/bnema/vev/internal/ports" | |
| 8 | + | ) | |
| 9 | + | ||
| 10 | + | const diagnosticBufferSize = 16 | |
| 11 | + | ||
| 12 | + | // Diagnostic is an immutable, privacy-safe snapshot of UDP link health. | |
| 13 | + | // It intentionally contains no packet payloads, addresses, keys, or identities. | |
| 14 | + | type Diagnostic struct { | |
| 15 | + | At time.Time | |
| 16 | + | State ports.LinkState | |
| 17 | + | SinceAuthenticatedPacket time.Duration | |
| 18 | + | SinceCompleteRecord time.Duration | |
| 19 | + | SinceACKProgress time.Duration | |
| 20 | + | PendingRecords int | |
| 21 | + | PendingBytes int | |
| 22 | + | Retransmits uint64 | |
| 23 | + | ReassemblyInflight int | |
| 24 | + | } | |
| 25 | + | ||
| 26 | + | // DiagnosticObserver receives bounded asynchronous UDP health snapshots. | |
| 27 | + | type DiagnosticObserver func(Diagnostic) | |
| 28 | + | ||
| 29 | + | // DiagnosticLogObserver returns an observer suitable for composition roots. It | |
| 30 | + | // logs only the privacy-safe diagnostic counters and ages. | |
| 31 | + | func DiagnosticLogObserver(log *slog.Logger) DiagnosticObserver { | |
| 32 | + | if log == nil { | |
| 33 | + | return nil | |
| 34 | + | } | |
| 35 | + | return func(d Diagnostic) { | |
| 36 | + | log.Debug("udp health", | |
| 37 | + | "state", d.State, | |
| 38 | + | "since_authenticated_packet", d.SinceAuthenticatedPacket, | |
| 39 | + | "since_complete_record", d.SinceCompleteRecord, | |
| 40 | + | "since_ack_progress", d.SinceACKProgress, | |
| 41 | + | "pending_records", d.PendingRecords, | |
| 42 | + | "pending_bytes", d.PendingBytes, | |
| 43 | + | "retransmits", d.Retransmits, | |
| 44 | + | "reassembly_inflight", d.ReassemblyInflight, | |
| 45 | + | ) | |
| 46 | + | } | |
| 47 | + | } | |
A internal/adapters/dgram/diagnostics_test.go +140 -0
| @@ -0,0 +1,140 @@ | |||
| 1 | + | package dgram | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "sync" | |
| 5 | + | "testing" | |
| 6 | + | "time" | |
| 7 | + | ||
| 8 | + | "github.com/bnema/vev/internal/ports" | |
| 9 | + | ) | |
| 10 | + | ||
| 11 | + | func TestDiagnosticSnapshotSeparatesProgressAges(t *testing.T) { | |
| 12 | + | now := time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC) | |
| 13 | + | got := make(chan Diagnostic, 1) | |
| 14 | + | tr := &Transport{ | |
| 15 | + | clock: fixedClock{now: now}, | |
| 16 | + | linkState: ports.LinkStateDegraded, | |
| 17 | + | lastAuthenticatedPacket: now.Add(-time.Second), | |
| 18 | + | lastCompleteRecord: now.Add(-2 * time.Second), | |
| 19 | + | lastACKProgress: now.Add(-3 * time.Second), | |
| 20 | + | retransmits: 64, | |
| 21 | + | reassemblyInflight: 2, | |
| 22 | + | pending: map[uint64]*pending{ | |
| 23 | + | 1: {frame: ports.Frame{Payload: make([]byte, 7)}}, | |
| 24 | + | 2: {frame: ports.Frame{Payload: make([]byte, 11)}}, | |
| 25 | + | }, | |
| 26 | + | diagnosticCh: make(chan Diagnostic, diagnosticBufferSize), | |
| 27 | + | done: make(chan struct{}), | |
| 28 | + | } | |
| 29 | + | tr.observe = func(d Diagnostic) { | |
| 30 | + | // This would deadlock if the observer were invoked under Transport.mu. | |
| 31 | + | _ = tr.LinkState() | |
| 32 | + | got <- d | |
| 33 | + | } | |
| 34 | + | go tr.diagnosticLoop() | |
| 35 | + | defer close(tr.done) | |
| 36 | + | ||
| 37 | + | tr.emitDiagnostic() | |
| 38 | + | select { | |
| 39 | + | case d := <-got: | |
| 40 | + | if d.At != now || d.State != ports.LinkStateDegraded { | |
| 41 | + | t.Fatalf("identity = %+v", d) | |
| 42 | + | } | |
| 43 | + | if d.SinceAuthenticatedPacket != time.Second || d.SinceCompleteRecord != 2*time.Second || d.SinceACKProgress != 3*time.Second { | |
| 44 | + | t.Fatalf("ages = packet %v, record %v, ack %v", d.SinceAuthenticatedPacket, d.SinceCompleteRecord, d.SinceACKProgress) | |
| 45 | + | } | |
| 46 | + | if d.PendingRecords != 2 || d.PendingBytes != 18 || d.Retransmits != 64 || d.ReassemblyInflight != 2 { | |
| 47 | + | t.Fatalf("counters = %+v", d) | |
| 48 | + | } | |
| 49 | + | case <-time.After(time.Second): | |
| 50 | + | t.Fatal("diagnostic observer was not called") | |
| 51 | + | } | |
| 52 | + | } | |
| 53 | + | ||
| 54 | + | func TestDiagnosticSnapshotIsLockConsistentDuringConcurrentUpdate(t *testing.T) { | |
| 55 | + | now := time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC) | |
| 56 | + | clock := &snapshotClock{now: now, sampled: make(chan struct{}), release: make(chan struct{})} | |
| 57 | + | tr := &Transport{ | |
| 58 | + | clock: clock, | |
| 59 | + | lastAuthenticatedPacket: now.Add(-time.Second), | |
| 60 | + | lastCompleteRecord: now.Add(-2 * time.Second), | |
| 61 | + | lastACKProgress: now.Add(-3 * time.Second), | |
| 62 | + | pending: make(map[uint64]*pending), | |
| 63 | + | diagnosticCh: make(chan Diagnostic, 1), | |
| 64 | + | } | |
| 65 | + | ||
| 66 | + | go tr.emitDiagnostic() | |
| 67 | + | select { | |
| 68 | + | case <-clock.sampled: | |
| 69 | + | case <-time.After(time.Second): | |
| 70 | + | t.Fatal("diagnostic did not begin its clock snapshot") | |
| 71 | + | } | |
| 72 | + | updated := make(chan struct{}) | |
| 73 | + | go func() { | |
| 74 | + | tr.mu.Lock() | |
| 75 | + | tr.lastAuthenticatedPacket = now.Add(time.Second) | |
| 76 | + | tr.lastCompleteRecord = now.Add(time.Second) | |
| 77 | + | tr.lastACKProgress = now.Add(time.Second) | |
| 78 | + | tr.retransmits = 1 | |
| 79 | + | tr.mu.Unlock() | |
| 80 | + | close(updated) | |
| 81 | + | }() | |
| 82 | + | close(clock.release) | |
| 83 | + | ||
| 84 | + | var d Diagnostic | |
| 85 | + | select { | |
| 86 | + | case d = <-tr.diagnosticCh: | |
| 87 | + | case <-time.After(time.Second): | |
| 88 | + | t.Fatal("diagnostic was not emitted") | |
| 89 | + | } | |
| 90 | + | select { | |
| 91 | + | case <-updated: | |
| 92 | + | case <-time.After(time.Second): | |
| 93 | + | t.Fatal("concurrent update was blocked") | |
| 94 | + | } | |
| 95 | + | if d.At != now || d.Retransmits != 0 { | |
| 96 | + | t.Fatalf("mixed diagnostic snapshot: %+v", d) | |
| 97 | + | } | |
| 98 | + | if d.SinceAuthenticatedPacket != time.Second || d.SinceCompleteRecord != 2*time.Second || d.SinceACKProgress != 3*time.Second { | |
| 99 | + | t.Fatalf("inconsistent diagnostic ages: %+v", d) | |
| 100 | + | } | |
| 101 | + | if d.SinceAuthenticatedPacket < 0 || d.SinceCompleteRecord < 0 || d.SinceACKProgress < 0 { | |
| 102 | + | t.Fatalf("negative diagnostic age: %+v", d) | |
| 103 | + | } | |
| 104 | + | } | |
| 105 | + | ||
| 106 | + | type snapshotClock struct { | |
| 107 | + | mu sync.Mutex | |
| 108 | + | now time.Time | |
| 109 | + | sampled chan struct{} | |
| 110 | + | release chan struct{} | |
| 111 | + | } | |
| 112 | + | ||
| 113 | + | func (c *snapshotClock) Now() time.Time { | |
| 114 | + | c.mu.Lock() | |
| 115 | + | defer c.mu.Unlock() | |
| 116 | + | select { | |
| 117 | + | case c.sampled <- struct{}{}: | |
| 118 | + | default: | |
| 119 | + | } | |
| 120 | + | <-c.release | |
| 121 | + | return c.now | |
| 122 | + | } | |
| 123 | + | ||
| 124 | + | func (c *snapshotClock) NewTimer(time.Duration) ports.Timer { panic("not used") } | |
| 125 | + | ||
| 126 | + | func TestDiagnosticObserverIsBounded(t *testing.T) { | |
| 127 | + | tr := &Transport{ | |
| 128 | + | clock: fixedClock{now: time.Now()}, | |
| 129 | + | pending: make(map[uint64]*pending), | |
| 130 | + | diagnosticCh: make(chan Diagnostic, 1), | |
| 131 | + | done: make(chan struct{}), | |
| 132 | + | } | |
| 133 | + | for range 100 { | |
| 134 | + | tr.emitDiagnostic() | |
| 135 | + | } | |
| 136 | + | if got := len(tr.diagnosticCh); got != 1 { | |
| 137 | + | t.Fatalf("queued diagnostics = %d, want bounded buffer", got) | |
| 138 | + | } | |
| 139 | + | close(tr.done) | |
| 140 | + | } | |
A internal/adapters/dgram/health.go +77 -0
| @@ -0,0 +1,77 @@ | |||
| 1 | + | package dgram | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "time" | |
| 5 | + | ||
| 6 | + | "github.com/bnema/vev/internal/ports" | |
| 7 | + | ) | |
| 8 | + | ||
| 9 | + | // healthTracker separates authenticated path contact from useful stream | |
| 10 | + | // progress. Transport.mu protects every field. | |
| 11 | + | type healthTracker struct { | |
| 12 | + | lastPacket time.Time | |
| 13 | + | lastRecord time.Time | |
| 14 | + | lastProgress time.Time | |
| 15 | + | pendingSince time.Time | |
| 16 | + | generation uint64 | |
| 17 | + | } | |
| 18 | + | ||
| 19 | + | func newHealthTracker(now time.Time) healthTracker { | |
| 20 | + | return healthTracker{lastPacket: now, lastRecord: now, lastProgress: now} | |
| 21 | + | } | |
| 22 | + | ||
| 23 | + | func (h *healthTracker) authenticatedPacket(now time.Time) { | |
| 24 | + | h.lastPacket = now | |
| 25 | + | h.generation++ | |
| 26 | + | } | |
| 27 | + | ||
| 28 | + | func (h *healthTracker) completeRecord(now time.Time) { | |
| 29 | + | h.lastRecord = now | |
| 30 | + | h.generation++ | |
| 31 | + | } | |
| 32 | + | ||
| 33 | + | func (h *healthTracker) ackProgress(now time.Time) { | |
| 34 | + | h.lastProgress = now | |
| 35 | + | h.generation++ | |
| 36 | + | } | |
| 37 | + | ||
| 38 | + | func (h *healthTracker) pendingStarted(now time.Time) { | |
| 39 | + | h.pendingSince = now | |
| 40 | + | h.generation++ | |
| 41 | + | } | |
| 42 | + | ||
| 43 | + | func (h *healthTracker) pendingCleared() { | |
| 44 | + | h.pendingSince = time.Time{} | |
| 45 | + | h.generation++ | |
| 46 | + | } | |
| 47 | + | ||
| 48 | + | func (h healthTracker) decide(now time.Time, hasPending bool, degradedAfter, probeAfter, offlineAfter, deadAfter time.Duration) (ports.LinkState, bool, bool) { | |
| 49 | + | packetAge := now.Sub(h.lastPacket) | |
| 50 | + | switch { | |
| 51 | + | case deadAfter > 0 && packetAge >= deadAfter: | |
| 52 | + | return ports.LinkStateDead, false, true | |
| 53 | + | case offlineAfter > 0 && packetAge >= offlineAfter: | |
| 54 | + | return ports.LinkStateOffline, false, false | |
| 55 | + | case probeAfter > 0 && packetAge >= probeAfter: | |
| 56 | + | return ports.LinkStateProbing, true, false | |
| 57 | + | } | |
| 58 | + | ||
| 59 | + | qualityAt := h.lastRecord | |
| 60 | + | if h.lastProgress.After(qualityAt) { | |
| 61 | + | qualityAt = h.lastProgress | |
| 62 | + | } | |
| 63 | + | qualityAge := now.Sub(qualityAt) | |
| 64 | + | if hasPending && probeAfter > 0 { | |
| 65 | + | pendingBaseline := h.pendingSince | |
| 66 | + | if pendingBaseline.IsZero() || h.lastProgress.After(pendingBaseline) { | |
| 67 | + | pendingBaseline = h.lastProgress | |
| 68 | + | } | |
| 69 | + | if now.Sub(pendingBaseline) >= probeAfter { | |
| 70 | + | return ports.LinkStateProbing, true, false | |
| 71 | + | } | |
| 72 | + | } | |
| 73 | + | if degradedAfter > 0 && qualityAge >= degradedAfter { | |
| 74 | + | return ports.LinkStateDegraded, false, false | |
| 75 | + | } | |
| 76 | + | return ports.LinkStateConnected, false, false | |
| 77 | + | } | |
A internal/adapters/dgram/network_sim_test.go +1011 -0
| @@ -0,0 +1,1011 @@ | |||
| 1 | + | package dgram | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "bytes" | |
| 5 | + | "errors" | |
| 6 | + | "net" | |
| 7 | + | "sync" | |
| 8 | + | "sync/atomic" | |
| 9 | + | "testing" | |
| 10 | + | "time" | |
| 11 | + | ||
| 12 | + | "github.com/bnema/vev/internal/ports" | |
| 13 | + | pdgram "github.com/bnema/vev/pkg/dgram" | |
| 14 | + | ) | |
| 15 | + | ||
| 16 | + | type testAddr string | |
| 17 | + | ||
| 18 | + | func (a testAddr) Network() string { return "mem" } | |
| 19 | + | func (a testAddr) String() string { return string(a) } | |
| 20 | + | ||
| 21 | + | type packet struct { | |
| 22 | + | b []byte | |
| 23 | + | addr net.Addr | |
| 24 | + | } | |
| 25 | + | ||
| 26 | + | // packetMeta is the deterministic simulator's view of a datagram before it is | |
| 27 | + | // accepted by the link. It deliberately contains only link-level information. | |
| 28 | + | type packetMeta struct { | |
| 29 | + | From, To net.Addr | |
| 30 | + | Bytes int | |
| 31 | + | At time.Time | |
| 32 | + | QueueBytes int | |
| 33 | + | } | |
| 34 | + | ||
| 35 | + | // packetPolicy makes loss and constrained links explicit in transport tests. | |
| 36 | + | // A zero value delivers immediately, preserving newPair's historical behavior. | |
| 37 | + | type packetPolicy struct { | |
| 38 | + | Drop func(packetMeta) bool | |
| 39 | + | Delay time.Duration | |
| 40 | + | MaxQueueBytes int | |
| 41 | + | BytesPerSecond int | |
| 42 | + | } | |
| 43 | + | ||
| 44 | + | type scheduledPacket struct { | |
| 45 | + | packet | |
| 46 | + | due time.Time | |
| 47 | + | } | |
| 48 | + | ||
| 49 | + | // simulatedLink has a bounded byte queue. Tests advance its clock explicitly; | |
| 50 | + | // it never starts a goroutine or relies on wall-clock time. | |
| 51 | + | type simulatedLink struct { | |
| 52 | + | mu sync.Mutex | |
| 53 | + | clock interface{ Now() time.Time } | |
| 54 | + | policy packetPolicy | |
| 55 | + | queue []scheduledPacket | |
| 56 | + | queueBytes int | |
| 57 | + | next time.Time | |
| 58 | + | dropped int | |
| 59 | + | maxDelay time.Duration | |
| 60 | + | } | |
| 61 | + | ||
| 62 | + | func newSimulatedLink(clock interface{ Now() time.Time }, policy packetPolicy) *simulatedLink { | |
| 63 | + | return &simulatedLink{clock: clock, policy: policy} | |
| 64 | + | } | |
| 65 | + | ||
| 66 | + | func (l *simulatedLink) enqueue(p *fakePC, peer *fakePC, b []byte) bool { | |
| 67 | + | l.mu.Lock() | |
| 68 | + | defer l.mu.Unlock() | |
| 69 | + | meta := packetMeta{From: p.addr, To: peer.addr, Bytes: len(b), At: l.clock.Now(), QueueBytes: l.queueBytes} | |
| 70 | + | if l.policy.Drop != nil && l.policy.Drop(meta) { | |
| 71 | + | return false | |
| 72 | + | } | |
| 73 | + | if l.policy.Delay == 0 && l.policy.BytesPerSecond == 0 { | |
| 74 | + | return l.deliverLocked(peer, packet{b: append([]byte(nil), b...), addr: p.addr}) | |
| 75 | + | } | |
| 76 | + | if l.policy.MaxQueueBytes <= 0 || l.queueBytes+len(b) > l.policy.MaxQueueBytes { | |
| 77 | + | l.dropped++ | |
| 78 | + | return false | |
| 79 | + | } | |
| 80 | + | due := meta.At.Add(l.policy.Delay) | |
| 81 | + | if l.policy.BytesPerSecond > 0 { | |
| 82 | + | if due.Before(l.next) { | |
| 83 | + | due = l.next | |
| 84 | + | } | |
| 85 | + | ns := int64(time.Second) * int64(len(b)) / int64(l.policy.BytesPerSecond) | |
| 86 | + | if ns == 0 { | |
| 87 | + | ns = 1 | |
| 88 | + | } | |
| 89 | + | l.next = due.Add(time.Duration(ns)) | |
| 90 | + | } | |
| 91 | + | l.maxDelay = max(l.maxDelay, due.Sub(meta.At)) | |
| 92 | + | l.queue = append(l.queue, scheduledPacket{packet: packet{b: append([]byte(nil), b...), addr: p.addr}, due: due}) | |
| 93 | + | l.queueBytes += len(b) | |
| 94 | + | return true | |
| 95 | + | } | |
| 96 | + | ||
| 97 | + | func (l *simulatedLink) flush(peer *fakePC) int { | |
| 98 | + | l.mu.Lock() | |
| 99 | + | defer l.mu.Unlock() | |
| 100 | + | now := l.clock.Now() | |
| 101 | + | delivered := 0 | |
| 102 | + | for len(l.queue) > 0 && !l.queue[0].due.After(now) { | |
| 103 | + | q := l.queue[0] | |
| 104 | + | l.queue = l.queue[1:] | |
| 105 | + | l.queueBytes -= len(q.b) | |
| 106 | + | if l.deliverLocked(peer, q.packet) { | |
| 107 | + | delivered++ | |
| 108 | + | } | |
| 109 | + | } | |
| 110 | + | return delivered | |
| 111 | + | } | |
| 112 | + | ||
| 113 | + | func (l *simulatedLink) queuedBytes() int { | |
| 114 | + | l.mu.Lock() | |
| 115 | + | defer l.mu.Unlock() | |
| 116 | + | return l.queueBytes | |
| 117 | + | } | |
| 118 | + | ||
| 119 | + | func (l *simulatedLink) droppedPackets() int { | |
| 120 | + | l.mu.Lock() | |
| 121 | + | defer l.mu.Unlock() | |
| 122 | + | return l.dropped | |
| 123 | + | } | |
| 124 | + | ||
| 125 | + | func (l *simulatedLink) maximumQueueDelay() time.Duration { | |
| 126 | + | l.mu.Lock() | |
| 127 | + | defer l.mu.Unlock() | |
| 128 | + | return l.maxDelay | |
| 129 | + | } | |
| 130 | + | ||
| 131 | + | func (l *simulatedLink) deliverLocked(peer *fakePC, q packet) bool { | |
| 132 | + | peer.mu.Lock() | |
| 133 | + | defer peer.mu.Unlock() | |
| 134 | + | if peer.closed { | |
| 135 | + | return false | |
| 136 | + | } | |
| 137 | + | select { | |
| 138 | + | case peer.in <- q: | |
| 139 | + | return true | |
| 140 | + | default: | |
| 141 | + | // The endpoint queue is intentionally bounded just like the link queue. | |
| 142 | + | return false | |
| 143 | + | } | |
| 144 | + | } | |
| 145 | + | ||
| 146 | + | type fakePC struct { | |
| 147 | + | addr net.Addr | |
| 148 | + | mu sync.Mutex | |
| 149 | + | closed bool | |
| 150 | + | in chan packet | |
| 151 | + | read chan struct{} | |
| 152 | + | peers map[string]*fakePC | |
| 153 | + | drop func([]byte, net.Addr) bool | |
| 154 | + | afterWrite func() | |
| 155 | + | link *simulatedLink | |
| 156 | + | } | |
| 157 | + | ||
| 158 | + | func newPair() (*fakePC, *fakePC) { return newPairWithCapacity(nil, 100) } | |
| 159 | + | ||
| 160 | + | func newFloodPair(link *simulatedLink) (*fakePC, *fakePC) { | |
| 161 | + | return newPairWithCapacity(link, floodRecordCount*floodRecordMTUs*2) | |
| 162 | + | } | |
| 163 | + | ||
| 164 | + | func newPairWithCapacity(link *simulatedLink, queuePackets int) (*fakePC, *fakePC) { | |
| 165 | + | a := &fakePC{addr: testAddr("a"), in: make(chan packet, queuePackets), read: make(chan struct{}, queuePackets), peers: map[string]*fakePC{}, link: link} | |
| 166 | + | b := &fakePC{addr: testAddr("b"), in: make(chan packet, queuePackets), read: make(chan struct{}, queuePackets), peers: map[string]*fakePC{}, link: link} | |
| 167 | + | a.peers["b"] = b | |
| 168 | + | b.peers["a"] = a | |
| 169 | + | return a, b | |
| 170 | + | } | |
| 171 | + | func (p *fakePC) ReadFrom(b []byte) (int, net.Addr, error) { | |
| 172 | + | q, ok := <-p.in | |
| 173 | + | if !ok { | |
| 174 | + | return 0, nil, errors.New("closed") | |
| 175 | + | } | |
| 176 | + | copy(b, q.b) | |
| 177 | + | select { | |
| 178 | + | case p.read <- struct{}{}: | |
| 179 | + | default: | |
| 180 | + | } | |
| 181 | + | return len(q.b), q.addr, nil | |
| 182 | + | } | |
| 183 | + | func (p *fakePC) WriteTo(b []byte, addr net.Addr) (int, error) { | |
| 184 | + | p.mu.Lock() | |
| 185 | + | closed := p.closed | |
| 186 | + | afterWrite := p.afterWrite | |
| 187 | + | p.mu.Unlock() | |
| 188 | + | if closed { | |
| 189 | + | return 0, errors.New("closed") | |
| 190 | + | } | |
| 191 | + | if p.drop != nil && p.drop(b, addr) { | |
| 192 | + | return len(b), nil | |
| 193 | + | } | |
| 194 | + | peer := p.peers[addr.String()] | |
| 195 | + | if peer == nil { | |
| 196 | + | return 0, errors.New("unknown peer") | |
| 197 | + | } | |
| 198 | + | if p.link != nil { | |
| 199 | + | p.link.enqueue(p, peer, b) | |
| 200 | + | } else { | |
| 201 | + | peer.mu.Lock() | |
| 202 | + | if peer.closed { | |
| 203 | + | peer.mu.Unlock() | |
| 204 | + | return 0, errors.New("peer closed") | |
| 205 | + | } | |
| 206 | + | select { | |
| 207 | + | case peer.in <- packet{append([]byte(nil), b...), p.addr}: | |
| 208 | + | default: | |
| 209 | + | } | |
| 210 | + | peer.mu.Unlock() | |
| 211 | + | } | |
| 212 | + | if afterWrite != nil { | |
| 213 | + | afterWrite() | |
| 214 | + | } | |
| 215 | + | return len(b), nil | |
| 216 | + | } | |
| 217 | + | func (p *fakePC) replaceAfterWrite(afterWrite func()) { | |
| 218 | + | p.mu.Lock() | |
| 219 | + | defer p.mu.Unlock() | |
| 220 | + | p.afterWrite = afterWrite | |
| 221 | + | } | |
| 222 | + | ||
| 223 | + | func (p *fakePC) addAfterWrite(afterWrite func()) { | |
| 224 | + | p.mu.Lock() | |
| 225 | + | defer p.mu.Unlock() | |
| 226 | + | previous := p.afterWrite | |
| 227 | + | p.afterWrite = func() { | |
| 228 | + | if previous != nil { | |
| 229 | + | previous() | |
| 230 | + | } | |
| 231 | + | afterWrite() | |
| 232 | + | } | |
| 233 | + | } | |
| 234 | + | ||
| 235 | + | func (p *fakePC) Close() error { | |
| 236 | + | p.mu.Lock() | |
| 237 | + | defer p.mu.Unlock() | |
| 238 | + | if !p.closed { | |
| 239 | + | p.closed = true | |
| 240 | + | close(p.in) | |
| 241 | + | } | |
| 242 | + | return nil | |
| 243 | + | } | |
| 244 | + | func (p *fakePC) LocalAddr() net.Addr { return p.addr } | |
| 245 | + | func (p *fakePC) SetDeadline(time.Time) error { return nil } | |
| 246 | + | func (p *fakePC) SetReadDeadline(time.Time) error { return nil } | |
| 247 | + | func (p *fakePC) SetWriteDeadline(time.Time) error { return nil } | |
| 248 | + | ||
| 249 | + | func key() []byte { return bytes.Repeat([]byte{1}, pdgram.KeySize) } | |
| 250 | + | ||
| 251 | + | const fixtureWaitTimeout = time.Second | |
| 252 | + | ||
| 253 | + | const ( | |
| 254 | + | floodRecordCount = 32 | |
| 255 | + | floodRecordMTUs = 24 | |
| 256 | + | ) | |
| 257 | + | ||
| 258 | + | // awaitSignal gives fixture barriers a diagnostic deadline. The deadline is a | |
| 259 | + | // hang guard only; tests advance behavior with the manual clock and barriers. | |
| 260 | + | func awaitSignal(t *testing.T, ch <-chan struct{}, context string) { | |
| 261 | + | t.Helper() | |
| 262 | + | select { | |
| 263 | + | case <-ch: | |
| 264 | + | case <-time.After(fixtureWaitTimeout): | |
| 265 | + | t.Fatalf("timed out waiting for %s", context) | |
| 266 | + | } | |
| 267 | + | } | |
| 268 | + | ||
| 269 | + | func awaitResult[T any](t *testing.T, ch <-chan T, context string) T { | |
| 270 | + | t.Helper() | |
| 271 | + | select { | |
| 272 | + | case result := <-ch: | |
| 273 | + | return result | |
| 274 | + | case <-time.After(fixtureWaitTimeout): | |
| 275 | + | t.Fatalf("timed out waiting for %s", context) | |
| 276 | + | var zero T | |
| 277 | + | return zero | |
| 278 | + | } | |
| 279 | + | } | |
| 280 | + | ||
| 281 | + | func flushAndAwait(t *testing.T, link *simulatedLink, peer *fakePC, processed <-chan struct{}) int { | |
| 282 | + | t.Helper() | |
| 283 | + | delivered := link.flush(peer) | |
| 284 | + | for range delivered { | |
| 285 | + | awaitSignal(t, processed, "flushed packet processing") | |
| 286 | + | } | |
| 287 | + | return delivered | |
| 288 | + | } | |
| 289 | + | ||
| 290 | + | // TestTransportFloodClassification drives authenticated datagrams through real | |
| 291 | + | // transports. The metrics are transport state, not simulator bookkeeping. | |
| 292 | + | func TestTransportFloodClassification(t *testing.T) { | |
| 293 | + | t.Run("one real fragment per large record loss contacts without completing", func(t *testing.T) { | |
| 294 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 295 | + | aPC, bPC := newFloodPair(nil) | |
| 296 | + | inspect, err := pdgram.NewCodec(key()) | |
| 297 | + | if err != nil { | |
| 298 | + | t.Fatal(err) | |
| 299 | + | } | |
| 300 | + | dropped := map[uint64]bool{} | |
| 301 | + | aPC.drop = func(pkt []byte, _ net.Addr) bool { | |
| 302 | + | _, raw, err := inspect.Open(pkt, 1, nil, nil) | |
| 303 | + | if err != nil { | |
| 304 | + | return false | |
| 305 | + | } | |
| 306 | + | frag, err := pdgram.UnmarshalFragment(raw) | |
| 307 | + | if err != nil || frag.Index != frag.Count-1 || dropped[frag.Seq] { | |
| 308 | + | return false | |
| 309 | + | } | |
| 310 | + | dropped[frag.Seq] = true | |
| 311 | + | return true | |
| 312 | + | } | |
| 313 | + | const mtu = 128 | |
| 314 | + | a, b := newFloodTransports(t, aPC, bPC, clk, mtu) | |
| 315 | + | defer closeFloodTransports(a, b) | |
| 316 | + | contacted := make(chan struct{}, 1) | |
| 317 | + | b.mu.Lock() | |
| 318 | + | b.afterAuthenticatedPacket = func() { | |
| 319 | + | select { | |
| 320 | + | case contacted <- struct{}{}: | |
| 321 | + | default: | |
| 322 | + | } | |
| 323 | + | } | |
| 324 | + | b.mu.Unlock() | |
| 325 | + | start := clk.Now() | |
| 326 | + | clk.advance(time.Nanosecond) | |
| 327 | + | sendFloodOutputs(t, a, floodRecordCount, mtu) | |
| 328 | + | awaitSignal(t, contacted, "first authenticated fragment") | |
| 329 | + | state := floodState(a, b, nil) | |
| 330 | + | if len(dropped) != floodRecordCount { | |
| 331 | + | t.Fatalf("flood state = %+v, dropped records=%d, want exactly one fragment from each of %d records", state, len(dropped), floodRecordCount) | |
| 332 | + | } | |
| 333 | + | if state.packetAge > initialPacingRTT || state.recordAge <= 0 || state.ackAge <= 0 { | |
| 334 | + | t.Fatalf("flood state = %+v, want fresh packet contact without record or ACK progress", state) | |
| 335 | + | } | |
| 336 | + | if !clk.Now().After(start) { | |
| 337 | + | t.Fatal("real authenticated fragments did not update contact") | |
| 338 | + | } | |
| 339 | + | }) | |
| 340 | + | ||
| 341 | + | t.Run("bandwidth_queue", func(t *testing.T) { | |
| 342 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 343 | + | const ( | |
| 344 | + | mtu = 128 | |
| 345 | + | oneWayDelay = 50 * time.Millisecond | |
| 346 | + | bandwidth = 16 * mtu | |
| 347 | + | queueLimit = 4 * mtu | |
| 348 | + | outputCount = 3 | |
| 349 | + | outputDataBytes = 6 * mtu | |
| 350 | + | ) | |
| 351 | + | var attempts, deterministicDrops atomic.Int64 | |
| 352 | + | forward := newSimulatedLink(clk, packetPolicy{ | |
| 353 | + | Delay: oneWayDelay, | |
| 354 | + | MaxQueueBytes: queueLimit, | |
| 355 | + | BytesPerSecond: bandwidth, | |
| 356 | + | Drop: func(packetMeta) bool { | |
| 357 | + | attempt := attempts.Add(1) | |
| 358 | + | if attempt%20 != 0 { | |
| 359 | + | return false | |
| 360 | + | } | |
| 361 | + | deterministicDrops.Add(1) | |
| 362 | + | return true | |
| 363 | + | }, | |
| 364 | + | }) | |
| 365 | + | reverse := newSimulatedLink(clk, packetPolicy{ | |
| 366 | + | Delay: oneWayDelay, | |
| 367 | + | MaxQueueBytes: queueLimit, | |
| 368 | + | BytesPerSecond: bandwidth, | |
| 369 | + | }) | |
| 370 | + | aPC, bPC := newFloodPair(nil) | |
| 371 | + | aPC.link = forward | |
| 372 | + | bPC.link = reverse | |
| 373 | + | const probeAfter = defaultProbe | |
| 374 | + | opts := Options{ | |
| 375 | + | Clock: clk, MTU: mtu, ResendAfter: 200 * time.Millisecond, MaxResendAfter: 800 * time.Millisecond, | |
| 376 | + | Heartbeat: 500 * time.Millisecond, DegradedAfter: defaultDegraded, | |
| 377 | + | ProbeAfter: probeAfter, OfflineAfter: defaultOffline, DeadAfter: defaultDead, | |
| 378 | + | } | |
| 379 | + | a, err := NewTransportWithOptions(aPC, bPC.addr, key(), 1, 2, opts) | |
| 380 | + | if err != nil { | |
| 381 | + | t.Fatal(err) | |
| 382 | + | } | |
| 383 | + | b, err := NewTransportWithOptions(bPC, aPC.addr, key(), 2, 1, opts) | |
| 384 | + | if err != nil { | |
| 385 | + | _ = a.Close() | |
| 386 | + | t.Fatal(err) | |
| 387 | + | } | |
| 388 | + | defer closeFloodTransports(a, b) | |
| 389 | + | waitForManualTimers(t, clk, 6) | |
| 390 | + | for range 6 { | |
| 391 | + | awaitSignal(t, clk.timerCreated, "initial transport timer creation") | |
| 392 | + | } | |
| 393 | + | var contacted atomic.Bool | |
| 394 | + | // All fixture hooks are lossy notifications: transport read/control loops | |
| 395 | + | // must never wait for the test goroutine to be scheduled. | |
| 396 | + | processed := make(chan struct{}, 256) | |
| 397 | + | primerWritten := make(chan struct{}, 1) | |
| 398 | + | primerAcknowledged := make(chan struct{}, 1) | |
| 399 | + | ackWakeAccepted := make(chan struct{}, 128) | |
| 400 | + | ackScheduled := make(chan struct{}, 128) | |
| 401 | + | ackDispatched := make(chan struct{}, 128) | |
| 402 | + | ackWritten := make(chan struct{}, 128) | |
| 403 | + | aPC.replaceAfterWrite(func() { | |
| 404 | + | select { | |
| 405 | + | case primerWritten <- struct{}{}: | |
| 406 | + | default: | |
| 407 | + | } | |
| 408 | + | }) | |
| 409 | + | bPC.replaceAfterWrite(func() { | |
| 410 | + | select { | |
| 411 | + | case primerAcknowledged <- struct{}{}: | |
| 412 | + | default: | |
| 413 | + | } | |
| 414 | + | select { | |
| 415 | + | case ackWritten <- struct{}{}: | |
| 416 | + | default: | |
| 417 | + | } | |
| 418 | + | }) | |
| 419 | + | a.mu.Lock() | |
| 420 | + | a.afterPacketProcessed = func() { | |
| 421 | + | select { | |
| 422 | + | case processed <- struct{}{}: | |
| 423 | + | default: | |
| 424 | + | } | |
| 425 | + | } | |
| 426 | + | a.mu.Unlock() | |
| 427 | + | b.mu.Lock() | |
| 428 | + | b.afterAuthenticatedPacket = func() { contacted.Store(true) } | |
| 429 | + | b.afterPacketProcessed = func() { | |
| 430 | + | select { | |
| 431 | + | case processed <- struct{}{}: | |
| 432 | + | default: | |
| 433 | + | } | |
| 434 | + | } | |
| 435 | + | b.afterACKWakeAccepted = func() { | |
| 436 | + | select { | |
| 437 | + | case ackWakeAccepted <- struct{}{}: | |
| 438 | + | default: | |
| 439 | + | } | |
| 440 | + | } | |
| 441 | + | b.afterACKScheduled = func() { | |
| 442 | + | select { | |
| 443 | + | case ackScheduled <- struct{}{}: | |
| 444 | + | default: | |
| 445 | + | } | |
| 446 | + | } | |
| 447 | + | b.afterACKDispatched = func() { | |
| 448 | + | select { | |
| 449 | + | case ackDispatched <- struct{}{}: | |
| 450 | + | default: | |
| 451 | + | } | |
| 452 | + | } | |
| 453 | + | b.mu.Unlock() | |
| 454 | + | primer := ports.MarshalOutput(ports.Output{BaseStateNum: 0, NewStateNum: 1, Data: []byte("primer")}) | |
| 455 | + | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: primer}); err != nil { | |
| 456 | + | t.Fatal(err) | |
| 457 | + | } | |
| 458 | + | awaitSignal(t, primerWritten, "primer write") | |
| 459 | + | primerDrained := false | |
| 460 | + | awaitingPrimerACK := false | |
| 461 | + | for range 40 { | |
| 462 | + | clk.advance(25 * time.Millisecond) | |
| 463 | + | if awaitingPrimerACK { | |
| 464 | + | awaitSignal(t, primerAcknowledged, "primer ACK write") | |
| 465 | + | awaitingPrimerACK = false | |
| 466 | + | } | |
| 467 | + | if flushAndAwait(t, forward, bPC, processed) > 0 { | |
| 468 | + | awaitSignal(t, ackWakeAccepted, "primer accepted ACK wake") | |
| 469 | + | awaitSignal(t, ackScheduled, "primer ACK scheduling") | |
| 470 | + | awaitingPrimerACK = true | |
| 471 | + | } | |
| 472 | + | flushAndAwait(t, reverse, aPC, processed) | |
| 473 | + | a.mu.Lock() | |
| 474 | + | primerDrained = len(a.pending) == 0 | |
| 475 | + | a.mu.Unlock() | |
| 476 | + | if primerDrained { | |
| 477 | + | break | |
| 478 | + | } | |
| 479 | + | } | |
| 480 | + | if !primerDrained { | |
| 481 | + | t.Fatal("clean RTT primer did not drain") | |
| 482 | + | } | |
| 483 | + | a.mu.Lock() | |
| 484 | + | initialCwnd := a.congestion.cwndBytes | |
| 485 | + | a.mu.Unlock() | |
| 486 | + | timerWork := make(chan struct{}, 128) | |
| 487 | + | dataPaceStarted := make(chan struct{}, 128) | |
| 488 | + | dataPaceWait := make(chan time.Time, 128) | |
| 489 | + | dataJobDone := make(chan struct{}, 128) | |
| 490 | + | dataWritten := make(chan struct{}, 128) | |
| 491 | + | notifyTimerWork := func() { | |
| 492 | + | select { | |
| 493 | + | case timerWork <- struct{}{}: | |
| 494 | + | default: | |
| 495 | + | } | |
| 496 | + | } | |
| 497 | + | for _, tr := range []*Transport{a, b} { | |
| 498 | + | tr.mu.Lock() | |
| 499 | + | tr.afterRetransmitTimer = notifyTimerWork | |
| 500 | + | tr.afterHealthTimer = notifyTimerWork | |
| 501 | + | tr.afterHeartbeatTimer = notifyTimerWork | |
| 502 | + | tr.mu.Unlock() | |
| 503 | + | } | |
| 504 | + | a.mu.Lock() | |
| 505 | + | a.beforeDataPace = func() { | |
| 506 | + | select { | |
| 507 | + | case dataPaceStarted <- struct{}{}: | |
| 508 | + | default: | |
| 509 | + | } | |
| 510 | + | } | |
| 511 | + | a.afterDataPaceWait = func(deadline time.Time) { | |
| 512 | + | select { | |
| 513 | + | case dataPaceWait <- deadline: | |
| 514 | + | default: | |
| 515 | + | } | |
| 516 | + | } | |
| 517 | + | a.afterDataJob = func() { | |
| 518 | + | select { | |
| 519 | + | case dataJobDone <- struct{}{}: | |
| 520 | + | default: | |
| 521 | + | } | |
| 522 | + | } | |
| 523 | + | a.mu.Unlock() | |
| 524 | + | aPC.addAfterWrite(func() { | |
| 525 | + | select { | |
| 526 | + | case dataWritten <- struct{}{}: | |
| 527 | + | default: | |
| 528 | + | } | |
| 529 | + | }) | |
| 530 | + | for state := 1; state < outputCount; state++ { | |
| 531 | + | payload := ports.MarshalOutput(ports.Output{ | |
| 532 | + | BaseStateNum: uint64(state), | |
| 533 | + | NewStateNum: uint64(state + 1), | |
| 534 | + | Data: bytes.Repeat([]byte{byte(state + 1)}, outputDataBytes), | |
| 535 | + | }) | |
| 536 | + | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: payload}); err != nil { | |
| 537 | + | t.Fatal(err) | |
| 538 | + | } | |
| 539 | + | } | |
| 540 | + | awaitSignal(t, dataPaceStarted, "output pacing") | |
| 541 | + | awaitSignal(t, dataWritten, "first paced output write") | |
| 542 | + | nextPace := awaitResult(t, dataPaceWait, "data pacing deadline") | |
| 543 | + | ||
| 544 | + | var maxContactAge time.Duration | |
| 545 | + | awaitingACKWrite := false | |
| 546 | + | lossReducedCwnd := false | |
| 547 | + | latestState := uint64(0) | |
| 548 | + | probed := false | |
| 549 | + | var probeAt time.Duration | |
| 550 | + | probeWho := "" | |
| 551 | + | for range 2000 { | |
| 552 | + | clk.advance(25 * time.Millisecond) | |
| 553 | + | elapsed := clk.Now().Sub(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 554 | + | if elapsed%opts.ResendAfter == 0 { | |
| 555 | + | // Both peers complete retransmit and health work before this | |
| 556 | + | // advance may be followed by another manual-clock step. | |
| 557 | + | for range 4 { | |
| 558 | + | awaitSignal(t, timerWork, "retransmit or health timer work") | |
| 559 | + | } | |
| 560 | + | } | |
| 561 | + | if elapsed%opts.Heartbeat == 0 { | |
| 562 | + | for range 2 { | |
| 563 | + | awaitSignal(t, timerWork, "heartbeat timer work") | |
| 564 | + | } | |
| 565 | + | } | |
| 566 | + | // A manual-clock advance is not a scheduler yield. When it fires the | |
| 567 | + | // data pacer, wait until the write reaches its next deterministic | |
| 568 | + | // deadline (or its job completes) before advancing again. | |
| 569 | + | if !nextPace.IsZero() && !nextPace.After(clk.Now()) { | |
| 570 | + | awaitSignal(t, dataWritten, "paced data write") | |
| 571 | + | select { | |
| 572 | + | case nextPace = <-dataPaceWait: | |
| 573 | + | case <-dataJobDone: | |
| 574 | + | // A completed job may be awaiting an ACK or retransmit timer; | |
| 575 | + | // neither has a data-pacer deadline to await yet. | |
| 576 | + | nextPace = time.Time{} | |
| 577 | + | } | |
| 578 | + | } | |
| 579 | + | if awaitingACKWrite { | |
| 580 | + | awaitSignal(t, ackDispatched, "cumulative ACK dispatch") | |
| 581 | + | awaitSignal(t, ackWritten, "cumulative ACK write") | |
| 582 | + | awaitingACKWrite = false | |
| 583 | + | } | |
| 584 | + | if flushAndAwait(t, forward, bPC, processed) > 0 { | |
| 585 | + | select { | |
| 586 | + | case <-ackWakeAccepted: | |
| 587 | + | awaitSignal(t, ackScheduled, "cumulative ACK scheduling") | |
| 588 | + | awaitingACKWrite = true | |
| 589 | + | default: | |
| 590 | + | } | |
| 591 | + | } | |
| 592 | + | flushAndAwait(t, reverse, aPC, processed) | |
| 593 | + | a.mu.Lock() | |
| 594 | + | queuedOutput := len(a.outputQueue) > 0 | |
| 595 | + | a.mu.Unlock() | |
| 596 | + | if queuedOutput { | |
| 597 | + | awaitSignal(t, dataPaceStarted, "output pacing") | |
| 598 | + | } | |
| 599 | + | ||
| 600 | + | if forward.queuedBytes() > queueLimit || reverse.queuedBytes() > queueLimit { | |
| 601 | + | t.Fatalf("queue exceeded bound: forward=%d reverse=%d limit=%d", forward.queuedBytes(), reverse.queuedBytes(), queueLimit) | |
| 602 | + | } | |
| 603 | + | if contacted.Load() { | |
| 604 | + | b.mu.Lock() | |
| 605 | + | age := clk.Now().Sub(b.lastAuthenticatedPacket) | |
| 606 | + | b.mu.Unlock() | |
| 607 | + | maxContactAge = max(maxContactAge, age) | |
| 608 | + | } | |
| 609 | + | for i, tr := range []*Transport{a, b} { | |
| 610 | + | if tr.LinkState() == ports.LinkStateProbing { | |
| 611 | + | if !probed { | |
| 612 | + | probeAt = clk.Now().Sub(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 613 | + | probeWho = []string{"sender", "receiver"}[i] | |
| 614 | + | } | |
| 615 | + | probed = true | |
| 616 | + | } | |
| 617 | + | select { | |
| 618 | + | case event := <-tr.LinkEvents(): | |
| 619 | + | if event.State == ports.LinkStateProbing && !probed { | |
| 620 | + | probeAt = event.At.Sub(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 621 | + | probeWho = []string{"sender", "receiver"}[i] | |
| 622 | + | } | |
| 623 | + | probed = probed || event.State == ports.LinkStateProbing | |
| 624 | + | default: | |
| 625 | + | } | |
| 626 | + | } | |
| 627 | + | for { | |
| 628 | + | select { | |
| 629 | + | case frame := <-b.in: | |
| 630 | + | output, err := ports.UnmarshalOutput(frame.Payload) | |
| 631 | + | if err != nil { | |
| 632 | + | t.Fatal(err) | |
| 633 | + | } | |
| 634 | + | latestState = max(latestState, output.NewStateNum) | |
| 635 | + | default: | |
| 636 | + | goto drainedOutput | |
| 637 | + | } | |
| 638 | + | } | |
| 639 | + | drainedOutput: | |
| 640 | + | a.mu.Lock() | |
| 641 | + | pending := len(a.pending) | |
| 642 | + | lossReducedCwnd = lossReducedCwnd || a.congestion.cwndBytes < initialCwnd | |
| 643 | + | a.mu.Unlock() | |
| 644 | + | if pending == 0 && latestState == outputCount && forward.queuedBytes() == 0 && reverse.queuedBytes() == 0 { | |
| 645 | + | break | |
| 646 | + | } | |
| 647 | + | } | |
| 648 | + | ||
| 649 | + | a.mu.Lock() | |
| 650 | + | pending := len(a.pending) | |
| 651 | + | srtt := a.srtt | |
| 652 | + | a.mu.Unlock() | |
| 653 | + | if got, want := deterministicDrops.Load(), attempts.Load()/20; got == 0 || got != want { | |
| 654 | + | t.Fatalf("deterministic loss=%d/%d, want exactly one in twenty attempts (%d)", got, attempts.Load(), want) | |
| 655 | + | } | |
| 656 | + | if forward.droppedPackets() == 0 { | |
| 657 | + | t.Fatal("bounded bandwidth queue never overflowed") | |
| 658 | + | } | |
| 659 | + | if delay := max(forward.maximumQueueDelay(), reverse.maximumQueueDelay()); delay < 2*oneWayDelay || delay >= probeAfter { | |
| 660 | + | t.Fatalf("maximum queue delay=%v, want in [%v,%v)", delay, 2*oneWayDelay, probeAfter) | |
| 661 | + | } | |
| 662 | + | if srtt < 2*oneWayDelay || srtt >= probeAfter { | |
| 663 | + | t.Fatalf("smoothed RTT=%v, want in [%v,%v)", srtt, 2*oneWayDelay, probeAfter) | |
| 664 | + | } | |
| 665 | + | if !contacted.Load() || maxContactAge >= probeAfter { | |
| 666 | + | t.Fatalf("authenticated contact=%v maximum age=%v, want fresh below %v", contacted.Load(), maxContactAge, probeAfter) | |
| 667 | + | } | |
| 668 | + | if probed { | |
| 669 | + | t.Fatalf("healthy lossy transfer entered probing: %s at %v pending=%d latest=%d attempts=%d loss=%d overflow=%d", probeWho, probeAt, pending, latestState, attempts.Load(), deterministicDrops.Load(), forward.droppedPackets()) | |
| 670 | + | } | |
| 671 | + | if !lossReducedCwnd { | |
| 672 | + | t.Fatal("deterministic loss did not reduce congestion window") | |
| 673 | + | } | |
| 674 | + | if pending != 0 || latestState != outputCount { | |
| 675 | + | t.Fatalf("convergence pending=%d latest state=%d, want 0 and %d", pending, latestState, outputCount) | |
| 676 | + | } | |
| 677 | + | }) | |
| 678 | + | ||
| 679 | + | t.Run("control stays responsive while fresh output and large retransmits contend", func(t *testing.T) { | |
| 680 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 681 | + | aPC, bPC := newFloodPair(nil) | |
| 682 | + | inspect, err := pdgram.NewCodec(key()) | |
| 683 | + | if err != nil { | |
| 684 | + | t.Fatal(err) | |
| 685 | + | } | |
| 686 | + | ||
| 687 | + | acksDropped := make(chan struct{}, floodRecordCount) | |
| 688 | + | bPC.drop = func(pkt []byte, _ net.Addr) bool { | |
| 689 | + | _, raw, err := inspect.Open(pkt, 2, nil, nil) | |
| 690 | + | if err != nil { | |
| 691 | + | return false | |
| 692 | + | } | |
| 693 | + | frag, err := pdgram.UnmarshalFragment(raw) | |
| 694 | + | if err != nil || frag.Count != 1 || len(frag.Data) != 9 || frag.Data[0] != recAck { | |
| 695 | + | return false | |
| 696 | + | } | |
| 697 | + | select { | |
| 698 | + | case acksDropped <- struct{}{}: | |
| 699 | + | default: | |
| 700 | + | } | |
| 701 | + | return true | |
| 702 | + | } | |
| 703 | + | ||
| 704 | + | retransmitBlocked := make(chan struct{}, 1) | |
| 705 | + | releaseRetransmit := make(chan struct{}) | |
| 706 | + | lastRetransmitted := make(chan struct{}, 1) | |
| 707 | + | probeSent := make(chan struct{}, 1) | |
| 708 | + | freshReturned := make(chan error, 1) | |
| 709 | + | writes := make(map[uint64]int) | |
| 710 | + | var writesMu sync.Mutex | |
| 711 | + | aPC.drop = func(pkt []byte, _ net.Addr) bool { | |
| 712 | + | _, raw, err := inspect.Open(pkt, 1, nil, nil) | |
| 713 | + | if err != nil { | |
| 714 | + | return false | |
| 715 | + | } | |
| 716 | + | frag, err := pdgram.UnmarshalFragment(raw) | |
| 717 | + | if err != nil { | |
| 718 | + | return false | |
| 719 | + | } | |
| 720 | + | if frag.Count == 1 && len(frag.Data) == 9 && frag.Data[0] == recProbe { | |
| 721 | + | probeSent <- struct{}{} | |
| 722 | + | return false | |
| 723 | + | } | |
| 724 | + | if frag.Index != 0 { | |
| 725 | + | return false | |
| 726 | + | } | |
| 727 | + | seq, _, _, ok := decodeData(frag.Data) | |
| 728 | + | if !ok { | |
| 729 | + | return false | |
| 730 | + | } | |
| 731 | + | writesMu.Lock() | |
| 732 | + | writes[seq]++ | |
| 733 | + | secondWrite := writes[seq] == 2 | |
| 734 | + | writesMu.Unlock() | |
| 735 | + | if secondWrite { | |
| 736 | + | select { | |
| 737 | + | case retransmitBlocked <- struct{}{}: | |
| 738 | + | default: | |
| 739 | + | } | |
| 740 | + | awaitSignal(t, releaseRetransmit, "retransmit release") | |
| 741 | + | if seq == floodRecordCount { | |
| 742 | + | lastRetransmitted <- struct{}{} | |
| 743 | + | } | |
| 744 | + | } | |
| 745 | + | return false | |
| 746 | + | } | |
| 747 | + | ||
| 748 | + | opts := Options{Clock: clk, MTU: 128, ResendAfter: 10 * time.Millisecond, Heartbeat: 11 * time.Millisecond} | |
| 749 | + | a, err := NewTransportWithOptions(aPC, bPC.addr, key(), 1, 2, opts) | |
| 750 | + | if err != nil { | |
| 751 | + | t.Fatal(err) | |
| 752 | + | } | |
| 753 | + | b, err := NewTransportWithOptions(bPC, aPC.addr, key(), 2, 1, opts) | |
| 754 | + | if err != nil { | |
| 755 | + | _ = a.Close() | |
| 756 | + | t.Fatal(err) | |
| 757 | + | } | |
| 758 | + | defer closeFloodTransports(a, b) | |
| 759 | + | for range 6 { // Both transports install retransmit, health, and heartbeat timers. | |
| 760 | + | awaitSignal(t, clk.timerCreated, "transport timer creation") | |
| 761 | + | } | |
| 762 | + | a.mu.Lock() | |
| 763 | + | a.rto = time.Hour | |
| 764 | + | a.mu.Unlock() | |
| 765 | + | ||
| 766 | + | // Fresh, fragmented output records become overdue while their real | |
| 767 | + | // cumulative ACKs are lost. Disabling pacing intentionally forces overload | |
| 768 | + | // to classify contention; it does not claim production pacing creates it. | |
| 769 | + | sendFloodOutputs(t, a, floodRecordCount, 128) | |
| 770 | + | awaitSignal(t, clk.timerCreated, "ACK coalescing timer creation") | |
| 771 | + | clk.advance(maxACKDelay) | |
| 772 | + | awaitSignal(t, acksDropped, "dropped cumulative ACK") | |
| 773 | + | a.mu.Lock() | |
| 774 | + | a.rto = 10 * time.Millisecond | |
| 775 | + | a.mu.Unlock() | |
| 776 | + | a.queueRetransmits() | |
| 777 | + | a.mu.Lock() | |
| 778 | + | a.rto = time.Hour | |
| 779 | + | a.mu.Unlock() | |
| 780 | + | clk.advance(initialPacingRTT) | |
| 781 | + | ||
| 782 | + | awaitSignal(t, retransmitBlocked, "retransmit pacing barrier") | |
| 783 | + | if state := floodState(a, b, nil); state.retransmits < floodRecordCount { | |
| 784 | + | t.Fatalf("flood state = %+v, want all %d overdue records selected", state, floodRecordCount) | |
| 785 | + | } | |
| 786 | + | select { | |
| 787 | + | case <-probeSent: | |
| 788 | + | default: | |
| 789 | + | } | |
| 790 | + | ||
| 791 | + | // A fresh record queues behind the sole data sender while control traffic | |
| 792 | + | // remains independent of congestion tokens and the blocked data write. | |
| 793 | + | go func() { | |
| 794 | + | freshReturned <- a.Send(ports.Frame{Type: ports.MsgOutput, Payload: floodOutputPayload(floodRecordCount, 128)}) | |
| 795 | + | }() | |
| 796 | + | select { | |
| 797 | + | case err := <-freshReturned: | |
| 798 | + | t.Fatalf("fresh output completed while retransmit held pacing: %v", err) | |
| 799 | + | default: | |
| 800 | + | } | |
| 801 | + | ||
| 802 | + | clk.advance(11 * time.Millisecond) | |
| 803 | + | awaitSignal(t, probeSent, "heartbeat probe during output contention") | |
| 804 | + | ||
| 805 | + | close(releaseRetransmit) | |
| 806 | + | for range 1024 { | |
| 807 | + | clk.advance(initialPacingRTT) | |
| 808 | + | time.Sleep(time.Millisecond) | |
| 809 | + | select { | |
| 810 | + | case err := <-freshReturned: | |
| 811 | + | if err != nil { | |
| 812 | + | t.Fatal(err) | |
| 813 | + | } | |
| 814 | + | awaitSignal(t, lastRetransmitted, "last overdue record retransmission") | |
| 815 | + | writesMu.Lock() | |
| 816 | + | defer writesMu.Unlock() | |
| 817 | + | if writes[1] < 2 || writes[floodRecordCount] < 2 || writes[floodRecordCount+1] == 0 { | |
| 818 | + | t.Fatalf("flood state = %+v, writes=%v, want fresh large traffic and retransmissions across %d overdue records", floodState(a, b, nil), writes, floodRecordCount) | |
| 819 | + | } | |
| 820 | + | return | |
| 821 | + | default: | |
| 822 | + | } | |
| 823 | + | } | |
| 824 | + | writesMu.Lock() | |
| 825 | + | defer writesMu.Unlock() | |
| 826 | + | t.Fatalf("fresh output did not complete after paced retransmissions: writes=%v state=%+v", writes, floodState(a, b, nil)) | |
| 827 | + | }) | |
| 828 | + | } | |
| 829 | + | ||
| 830 | + | type floodTransportState struct { | |
| 831 | + | packetAge, recordAge, ackAge time.Duration | |
| 832 | + | queuedBytes int | |
| 833 | + | retransmits uint64 | |
| 834 | + | } | |
| 835 | + | ||
| 836 | + | func floodState(sender, receiver *Transport, link *simulatedLink) floodTransportState { | |
| 837 | + | now := sender.clock.Now() | |
| 838 | + | receiver.mu.Lock() | |
| 839 | + | packetAge := now.Sub(receiver.lastAuthenticatedPacket) | |
| 840 | + | recordAge := now.Sub(receiver.lastCompleteRecord) | |
| 841 | + | receiver.mu.Unlock() | |
| 842 | + | sender.mu.Lock() | |
| 843 | + | ackAge := now.Sub(sender.lastACKProgress) | |
| 844 | + | retransmits := sender.retransmits | |
| 845 | + | sender.mu.Unlock() | |
| 846 | + | queuedBytes := 0 | |
| 847 | + | if link != nil { | |
| 848 | + | queuedBytes = link.queuedBytes() | |
| 849 | + | } | |
| 850 | + | return floodTransportState{packetAge: packetAge, recordAge: recordAge, ackAge: ackAge, queuedBytes: queuedBytes, retransmits: retransmits} | |
| 851 | + | } | |
| 852 | + | ||
| 853 | + | func floodOutputPayload(baseState, mtu int) []byte { | |
| 854 | + | return ports.MarshalOutput(ports.Output{ | |
| 855 | + | BaseStateNum: uint64(baseState), | |
| 856 | + | NewStateNum: uint64(baseState + 1), | |
| 857 | + | Data: make([]byte, floodRecordMTUs*mtu), | |
| 858 | + | }) | |
| 859 | + | } | |
| 860 | + | ||
| 861 | + | func TestFloodOutputPayloadIsIncrementalStateBearingOutput(t *testing.T) { | |
| 862 | + | const mtu = 128 | |
| 863 | + | for state := range floodRecordCount { | |
| 864 | + | payload := floodOutputPayload(state, mtu) | |
| 865 | + | output, err := ports.UnmarshalOutput(payload) | |
| 866 | + | if err != nil { | |
| 867 | + | t.Fatalf("state %d: %v", state, err) | |
| 868 | + | } | |
| 869 | + | if output.BaseStateNum != uint64(state) || output.NewStateNum != uint64(state+1) { | |
| 870 | + | t.Fatalf("output state = %d -> %d, want %d -> %d", output.BaseStateNum, output.NewStateNum, state, state+1) | |
| 871 | + | } | |
| 872 | + | minimumBytes := floodRecordMTUs * mtu | |
| 873 | + | if len(output.Data) < minimumBytes || len(payload) < minimumBytes { | |
| 874 | + | t.Fatalf("state %d: data bytes=%d encoded bytes=%d, want each at least %d", state, len(output.Data), len(payload), minimumBytes) | |
| 875 | + | } | |
| 876 | + | } | |
| 877 | + | } | |
| 878 | + | ||
| 879 | + | func sendFloodOutputs(t *testing.T, sender *Transport, count, mtu int) { | |
| 880 | + | t.Helper() | |
| 881 | + | for state := range count { | |
| 882 | + | if err := sender.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: floodOutputPayload(state, mtu)}); err != nil { | |
| 883 | + | t.Fatal(err) | |
| 884 | + | } | |
| 885 | + | } | |
| 886 | + | clk, ok := sender.clock.(*manualClock) | |
| 887 | + | if !ok { | |
| 888 | + | return | |
| 889 | + | } | |
| 890 | + | for range count * floodRecordMTUs * 2 { | |
| 891 | + | clk.advance(initialPacingRTT) | |
| 892 | + | time.Sleep(time.Millisecond) | |
| 893 | + | sender.mu.Lock() | |
| 894 | + | complete := len(sender.outputQueue) == 0 && len(sender.dataSend) == 0 | |
| 895 | + | for _, p := range sender.pending { | |
| 896 | + | complete = complete && !p.initialInFlight && !p.last.IsZero() | |
| 897 | + | } | |
| 898 | + | sender.mu.Unlock() | |
| 899 | + | if complete { | |
| 900 | + | return | |
| 901 | + | } | |
| 902 | + | } | |
| 903 | + | t.Fatal("flood output pacing did not drain") | |
| 904 | + | } | |
| 905 | + | ||
| 906 | + | func newFloodTransports(t *testing.T, aPC, bPC *fakePC, clk *manualClock, mtu int) (*Transport, *Transport) { | |
| 907 | + | t.Helper() | |
| 908 | + | opts := Options{ | |
| 909 | + | Clock: clk, MTU: mtu, ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 910 | + | DegradedAfter: time.Hour, ProbeAfter: time.Hour, OfflineAfter: time.Hour, DeadAfter: time.Hour, | |
| 911 | + | } | |
| 912 | + | a, err := NewTransportWithOptions(aPC, bPC.addr, key(), 1, 2, opts) | |
| 913 | + | if err != nil { | |
| 914 | + | t.Fatal(err) | |
| 915 | + | } | |
| 916 | + | b, err := NewTransportWithOptions(bPC, aPC.addr, key(), 2, 1, opts) | |
| 917 | + | if err != nil { | |
| 918 | + | _ = a.Close() | |
| 919 | + | t.Fatal(err) | |
| 920 | + | } | |
| 921 | + | return a, b | |
| 922 | + | } | |
| 923 | + | ||
| 924 | + | func closeFloodTransports(a, b *Transport) { | |
| 925 | + | _ = a.Close() | |
| 926 | + | _ = b.Close() | |
| 927 | + | } | |
| 928 | + | ||
| 929 | + | func TestMalformedAuthenticatedFragmentDoesNotCountAsContact(t *testing.T) { | |
| 930 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 931 | + | aPC, bPC := newPair() | |
| 932 | + | a, b := newFloodTransports(t, aPC, bPC, clk, 128) | |
| 933 | + | defer closeFloodTransports(a, b) | |
| 934 | + | b.mu.Lock() | |
| 935 | + | startPacket, startRecord := b.lastAuthenticatedPacket, b.lastCompleteRecord | |
| 936 | + | b.mu.Unlock() | |
| 937 | + | codec, err := pdgram.NewCodec(key()) | |
| 938 | + | if err != nil { | |
| 939 | + | t.Fatal(err) | |
| 940 | + | } | |
| 941 | + | malformedRejected := make(chan struct{}, 1) | |
| 942 | + | b.mu.Lock() | |
| 943 | + | b.afterMalformedFragment = func() { malformedRejected <- struct{}{} } | |
| 944 | + | b.mu.Unlock() | |
| 945 | + | clk.advance(time.Second) | |
| 946 | + | if _, err := aPC.WriteTo(codec.Seal(1, 999, []byte("not a fragment"), nil), bPC.addr); err != nil { | |
| 947 | + | t.Fatal(err) | |
| 948 | + | } | |
| 949 | + | awaitSignal(t, malformedRejected, "malformed fragment rejection") | |
| 950 | + | b.mu.Lock() | |
| 951 | + | gotPacket, gotRecord := b.lastAuthenticatedPacket, b.lastCompleteRecord | |
| 952 | + | b.mu.Unlock() | |
| 953 | + | if gotPacket != startPacket || gotRecord != startRecord { | |
| 954 | + | t.Fatal("malformed authenticated plaintext counted as packet contact") | |
| 955 | + | } | |
| 956 | + | } | |
| 957 | + | ||
| 958 | + | func TestInconsistentAuthenticatedFragmentDoesNotCountAsContact(t *testing.T) { | |
| 959 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 960 | + | aPC, bPC := newPair() | |
| 961 | + | a, b := newFloodTransports(t, aPC, bPC, clk, 128) | |
| 962 | + | defer closeFloodTransports(a, b) | |
| 963 | + | codec, err := pdgram.NewCodec(key()) | |
| 964 | + | if err != nil { | |
| 965 | + | t.Fatal(err) | |
| 966 | + | } | |
| 967 | + | accepted := make(chan struct{}, 1) | |
| 968 | + | rejected := make(chan struct{}, 1) | |
| 969 | + | b.mu.Lock() | |
| 970 | + | b.afterAuthenticatedPacket = func() { accepted <- struct{}{} } | |
| 971 | + | b.afterMalformedFragment = func() { rejected <- struct{}{} } | |
| 972 | + | b.mu.Unlock() | |
| 973 | + | ||
| 974 | + | writeFragment := func(counter uint64, frag pdgram.Fragment) { | |
| 975 | + | t.Helper() | |
| 976 | + | raw, err := pdgram.MarshalFragment(frag) | |
| 977 | + | if err != nil { | |
| 978 | + | t.Fatal(err) | |
| 979 | + | } | |
| 980 | + | if _, err := aPC.WriteTo(codec.Seal(1, counter, raw, nil), bPC.addr); err != nil { | |
| 981 | + | t.Fatal(err) | |
| 982 | + | } | |
| 983 | + | } | |
| 984 | + | writeFragment(999, pdgram.Fragment{Seq: 42, Index: 0, Count: 2, Data: []byte("first")}) | |
| 985 | + | awaitSignal(t, accepted, "accepted incomplete fragment") | |
| 986 | + | b.mu.Lock() | |
| 987 | + | lastContact := b.lastAuthenticatedPacket | |
| 988 | + | acceptedInflight := b.reassemblyInflight | |
| 989 | + | b.mu.Unlock() | |
| 990 | + | if acceptedInflight != 1 { | |
| 991 | + | t.Fatalf("reassembly inflight after accepted fragment=%d, want 1", acceptedInflight) | |
| 992 | + | } | |
| 993 | + | ||
| 994 | + | clk.advance(time.Second) | |
| 995 | + | writeFragment(1000, pdgram.Fragment{Seq: 42, Index: 1, Count: 3, Data: []byte("inconsistent")}) | |
| 996 | + | awaitSignal(t, rejected, "inconsistent fragment rejection") | |
| 997 | + | b.mu.Lock() | |
| 998 | + | gotContact := b.lastAuthenticatedPacket | |
| 999 | + | gotInflight := b.reassemblyInflight | |
| 1000 | + | b.mu.Unlock() | |
| 1001 | + | if gotContact != lastContact { | |
| 1002 | + | t.Fatalf("rejected inconsistent fragment refreshed contact from %v to %v", lastContact, gotContact) | |
| 1003 | + | } | |
| 1004 | + | if gotInflight != 0 { | |
| 1005 | + | t.Fatalf("reassembly inflight after rejected inconsistent fragment=%d, want 0", gotInflight) | |
| 1006 | + | } | |
| 1007 | + | diagnostic := b.diagnosticSnapshot() | |
| 1008 | + | if diagnostic.ReassemblyInflight != 0 { | |
| 1009 | + | t.Fatalf("diagnostic reassembly inflight after rejected inconsistent fragment=%d, want 0", diagnostic.ReassemblyInflight) | |
| 1010 | + | } | |
| 1011 | + | } | |
M internal/adapters/dgram/output_pacer.go +11 -73
| @@ -1,18 +1,16 @@ | |||
| 1 | 1 | package dgram | |
| 2 | 2 | ||
| 3 | - | import ( | |
| 4 | - | "time" | |
| 3 | + | import "github.com/bnema/vev/internal/ports" | |
| 5 | 4 | ||
| 6 | - | "github.com/bnema/vev/internal/ports" | |
| 7 | - | ) | |
| 8 | - | ||
| 9 | - | type queuedSend struct { | |
| 5 | + | type dataSendJob struct { | |
| 10 | 6 | seq uint64 | |
| 11 | 7 | reliable bool | |
| 12 | 8 | frame ports.Frame | |
| 13 | 9 | done chan error | |
| 14 | 10 | } | |
| 15 | 11 | ||
| 12 | + | type queuedSend = dataSendJob | |
| 13 | + | ||
| 16 | 14 | func (t *Transport) outputPaceLoop() { | |
| 17 | 15 | for { | |
| 18 | 16 | t.outboundMu.Lock() | |
| @@ -33,45 +31,14 @@ func (t *Transport) outputPaceLoop() { | |||
| 33 | 31 | t.outboundMu.Unlock() | |
| 34 | 32 | return | |
| 35 | 33 | } | |
| 36 | - | now := t.clock.Now() | |
| 37 | - | wait := t.outputNext.Sub(now) | |
| 38 | - | if wait > 0 { | |
| 39 | - | wake := t.outputWake | |
| 40 | - | timer := t.clock.NewTimer(wait) | |
| 41 | - | t.mu.Unlock() | |
| 42 | - | t.outboundMu.Unlock() | |
| 43 | - | select { | |
| 44 | - | case <-timer.C(): | |
| 45 | - | case <-wake: | |
| 46 | - | case <-t.done: | |
| 47 | - | timer.Stop() | |
| 48 | - | return | |
| 49 | - | } | |
| 50 | - | timer.Stop() | |
| 51 | - | continue | |
| 52 | - | } | |
| 53 | - | limit := 0 | |
| 54 | - | budget := t.mtu | |
| 55 | - | for limit < len(t.outputQueue) && limit < defaultOutputPaceBatch { | |
| 56 | - | cost := len(t.outputQueue[limit].frame.Payload) + dataRecordHeaderSize | |
| 57 | - | if limit > 0 && cost > budget { | |
| 58 | - | break | |
| 59 | - | } | |
| 60 | - | limit++ | |
| 61 | - | budget -= min(cost, budget) | |
| 62 | - | } | |
| 63 | - | batch := append([]queuedSend(nil), t.outputQueue[:limit]...) | |
| 64 | - | copy(t.outputQueue, t.outputQueue[limit:]) | |
| 65 | - | t.outputQueue = t.outputQueue[:len(t.outputQueue)-limit] | |
| 66 | - | t.outputNext = now.Add(t.outputPaceDelayLocked()) | |
| 34 | + | q := t.outputQueue[0] | |
| 35 | + | copy(t.outputQueue, t.outputQueue[1:]) | |
| 36 | + | t.outputQueue = t.outputQueue[:len(t.outputQueue)-1] | |
| 67 | 37 | t.mu.Unlock() | |
| 68 | - | for i, q := range batch { | |
| 69 | - | if err := t.sendQueuedData(q); err != nil { | |
| 70 | - | t.removeQueuedPending(batch[i+1:], err) | |
| 71 | - | t.closeWithError(err) | |
| 72 | - | t.outboundMu.Unlock() | |
| 73 | - | return | |
| 74 | - | } | |
| 38 | + | if err := t.queueDataJob(q); err != nil { | |
| 39 | + | t.removeQueuedPending([]queuedSend{q}, err) | |
| 40 | + | t.outboundMu.Unlock() | |
| 41 | + | return | |
| 75 | 42 | } | |
| 76 | 43 | t.outboundMu.Unlock() | |
| 77 | 44 | } | |
| @@ -85,39 +52,10 @@ func (t *Transport) notifyOutputPacerLocked() { | |||
| 85 | 52 | t.outputWake = make(chan struct{}) | |
| 86 | 53 | } | |
| 87 | 54 | ||
| 88 | - | func (t *Transport) outputPaceDelayLocked() time.Duration { | |
| 89 | - | if t.srtt <= 0 { | |
| 90 | - | return defaultOutputPaceMinDelay | |
| 91 | - | } | |
| 92 | - | d := t.srtt / 2 | |
| 93 | - | if d < defaultOutputPaceMinCadence { | |
| 94 | - | d = defaultOutputPaceMinCadence | |
| 95 | - | } | |
| 96 | - | if d > defaultOutputPaceMaxDelay { | |
| 97 | - | d = defaultOutputPaceMaxDelay | |
| 98 | - | } | |
| 99 | - | return d | |
| 100 | - | } | |
| 101 | - | ||
| 102 | 55 | func shouldPaceOutput(f ports.Frame) bool { | |
| 103 | 56 | return f.Type == ports.MsgOutput | |
| 104 | 57 | } | |
| 105 | 58 | ||
| 106 | - | func (t *Transport) sendQueuedData(q queuedSend) error { | |
| 107 | - | t.markPendingSent(q.seq, q.reliable) | |
| 108 | - | err := t.sendData(q.seq, q.reliable, q.frame) | |
| 109 | - | if err != nil { | |
| 110 | - | t.removePending(q.seq, q.reliable) | |
| 111 | - | } else { | |
| 112 | - | t.markPendingReady(q.seq, q.reliable) | |
| 113 | - | } | |
| 114 | - | if q.done != nil { | |
| 115 | - | q.done <- err | |
| 116 | - | close(q.done) | |
| 117 | - | } | |
| 118 | - | return err | |
| 119 | - | } | |
| 120 | - | ||
| 121 | 59 | func (t *Transport) markPendingSent(seq uint64, reliable bool) { | |
| 122 | 60 | if !reliable { | |
| 123 | 61 | return | |
A internal/adapters/dgram/pacer.go +91 -0
| @@ -0,0 +1,91 @@ | |||
| 1 | + | package dgram | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "errors" | |
| 5 | + | "math" | |
| 6 | + | "math/bits" | |
| 7 | + | "time" | |
| 8 | + | ||
| 9 | + | "github.com/bnema/vev/internal/ports" | |
| 10 | + | ) | |
| 11 | + | ||
| 12 | + | var errPacerClosed = errors.New("dgram: pacer closed") | |
| 13 | + | ||
| 14 | + | type paceLimits func() (bytesPerSecond int, burstBytes int) | |
| 15 | + | ||
| 16 | + | type bytePacer struct { | |
| 17 | + | clk ports.Clock | |
| 18 | + | tokens int64 | |
| 19 | + | last time.Time | |
| 20 | + | afterTimer func(time.Time) | |
| 21 | + | } | |
| 22 | + | ||
| 23 | + | func (p *bytePacer) wait(done <-chan struct{}, n int, limits paceLimits) error { | |
| 24 | + | if n <= 0 { | |
| 25 | + | return nil | |
| 26 | + | } | |
| 27 | + | for { | |
| 28 | + | now := p.clk.Now() | |
| 29 | + | rate, burst := limits() | |
| 30 | + | if rate < 1 { | |
| 31 | + | rate = 1 | |
| 32 | + | } | |
| 33 | + | if burst < n { | |
| 34 | + | burst = n | |
| 35 | + | } | |
| 36 | + | if p.last.IsZero() { | |
| 37 | + | p.last = now | |
| 38 | + | p.tokens = int64(burst) | |
| 39 | + | } else { | |
| 40 | + | elapsed := now.Sub(p.last) | |
| 41 | + | if elapsed > 0 { | |
| 42 | + | capacity := int64(burst) - p.tokens | |
| 43 | + | p.tokens += mulDivFloorSaturating(int64(elapsed), int64(rate), int64(time.Second), capacity) | |
| 44 | + | p.last = now | |
| 45 | + | } | |
| 46 | + | p.tokens = min(p.tokens, int64(burst)) | |
| 47 | + | } | |
| 48 | + | if p.tokens >= int64(n) { | |
| 49 | + | p.tokens -= int64(n) | |
| 50 | + | return nil | |
| 51 | + | } | |
| 52 | + | ||
| 53 | + | deficit := int64(n) - p.tokens | |
| 54 | + | wait := time.Duration(mulDivCeilSaturating(deficit, int64(time.Second), int64(rate))) | |
| 55 | + | timer := p.clk.NewTimer(wait) | |
| 56 | + | if p.afterTimer != nil { | |
| 57 | + | p.afterTimer(now.Add(wait)) | |
| 58 | + | } | |
| 59 | + | select { | |
| 60 | + | case <-timer.C(): | |
| 61 | + | timer.Stop() | |
| 62 | + | case <-done: | |
| 63 | + | timer.Stop() | |
| 64 | + | return errPacerClosed | |
| 65 | + | } | |
| 66 | + | } | |
| 67 | + | } | |
| 68 | + | ||
| 69 | + | func mulDivFloorSaturating(a, b, divisor, limit int64) int64 { | |
| 70 | + | if a <= 0 || b <= 0 || limit <= 0 { | |
| 71 | + | return 0 | |
| 72 | + | } | |
| 73 | + | hi, lo := bits.Mul64(uint64(a), uint64(b)) | |
| 74 | + | if hi >= uint64(divisor) { | |
| 75 | + | return limit | |
| 76 | + | } | |
| 77 | + | quotient, _ := bits.Div64(hi, lo, uint64(divisor)) | |
| 78 | + | return min(int64(min(quotient, uint64(math.MaxInt64))), limit) | |
| 79 | + | } | |
| 80 | + | ||
| 81 | + | func mulDivCeilSaturating(a, b, divisor int64) int64 { | |
| 82 | + | hi, lo := bits.Mul64(uint64(a), uint64(b)) | |
| 83 | + | if hi >= uint64(divisor) { | |
| 84 | + | return math.MaxInt64 | |
| 85 | + | } | |
| 86 | + | quotient, remainder := bits.Div64(hi, lo, uint64(divisor)) | |
| 87 | + | if remainder != 0 { | |
| 88 | + | quotient++ | |
| 89 | + | } | |
| 90 | + | return int64(min(quotient, uint64(math.MaxInt64))) | |
| 91 | + | } | |
M internal/adapters/dgram/proxy.go +25 -1
| @@ -51,6 +51,7 @@ type proxyCopyDirection struct { | |||
| 51 | 51 | recvKind proxyCopyErrKind | |
| 52 | 52 | sendKind proxyCopyErrKind | |
| 53 | 53 | retryRecoverable bool | |
| 54 | + | transform func(ports.Frame) ports.Frame | |
| 54 | 55 | } | |
| 55 | 56 | ||
| 56 | 57 | func (p ProxyRuntime) Run(ctx context.Context) error { | |
| @@ -76,7 +77,11 @@ func (p ProxyRuntime) Run(ctx context.Context) error { | |||
| 76 | 77 | ||
| 77 | 78 | errCh := make(chan proxyCopyErr, 2) | |
| 78 | 79 | copier := proxyCopier{ctx: copyCtx, errCh: errCh, retryBackoff: retryBackoff, clk: clk} | |
| 79 | - | clientToDaemon := proxyCopyDirection{src: p.Client, dst: p.Daemon, recvKind: proxyClientRecv, sendKind: proxyDaemonSend} | |
| 80 | + | clientToDaemon := proxyCopyDirection{ | |
| 81 | + | src: p.Client, dst: p.Daemon, | |
| 82 | + | recvKind: proxyClientRecv, sendKind: proxyDaemonSend, | |
| 83 | + | transform: clampDatagramHelloOutputWindow, | |
| 84 | + | } | |
| 80 | 85 | daemonToClient := proxyCopyDirection{src: p.Daemon, dst: p.Client, recvKind: proxyDaemonRecv, sendKind: proxyClientSend, retryRecoverable: true} | |
| 81 | 86 | go copier.copyFrames(clientToDaemon) | |
| 82 | 87 | go copier.copyFrames(daemonToClient) | |
| @@ -157,6 +162,9 @@ func (c proxyCopier) copyFrames(dir proxyCopyDirection) { | |||
| 157 | 162 | c.errCh <- proxyCopyErr{kind: dir.recvKind, err: err} | |
| 158 | 163 | return | |
| 159 | 164 | } | |
| 165 | + | if dir.transform != nil { | |
| 166 | + | f = dir.transform(f) | |
| 167 | + | } | |
| 160 | 168 | // Retry the same frame in place while the destination link error stays | |
| 161 | 169 | // recoverable (congestion, transient path loss). The frame is never | |
| 162 | 170 | // dropped — on a reliable stream a gap desynchronizes the client screen. | |
| @@ -199,3 +207,19 @@ func recoverableClientLink(t ports.Transport, err error) bool { | |||
| 199 | 207 | return false | |
| 200 | 208 | } | |
| 201 | 209 | } | |
| 210 | + | ||
| 211 | + | // clampDatagramHelloOutputWindow enforces the datagram-safe state window at | |
| 212 | + | // the adapter boundary. Invalid Hello payloads pass through unchanged so the | |
| 213 | + | // daemon remains the single authority for strict decoding and version checks. | |
| 214 | + | func clampDatagramHelloOutputWindow(f ports.Frame) ports.Frame { | |
| 215 | + | if f.Type != ports.MsgHello { | |
| 216 | + | return f | |
| 217 | + | } | |
| 218 | + | hello, err := ports.UnmarshalHello(f.Payload) | |
| 219 | + | if err != nil { | |
| 220 | + | return f | |
| 221 | + | } | |
| 222 | + | hello.MaxOutputInFlight = 1 | |
| 223 | + | f.Payload = ports.MarshalHello(hello) | |
| 224 | + | return f | |
| 225 | + | } | |
M internal/adapters/dgram/proxy_test.go +74 -22
| @@ -3,7 +3,9 @@ package dgram | |||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | 5 | "errors" | |
| 6 | + | "fmt" | |
| 6 | 7 | "io" | |
| 8 | + | "reflect" | |
| 7 | 9 | "sync" | |
| 8 | 10 | "testing" | |
| 9 | 11 | "time" | |
| @@ -78,30 +80,49 @@ func TestProxyRuntimeCopiesFramesAndDaemonEOFIsTerminal(t *testing.T) { | |||
| 78 | 80 | } | |
| 79 | 81 | } | |
| 80 | 82 | ||
| 81 | - | func TestProxyRuntimeForwardsOutputAckCapabilityInHello(t *testing.T) { | |
| 82 | - | client := newFakeTransport() | |
| 83 | - | daemon := newFakeTransport() | |
| 84 | - | errCh := make(chan error, 1) | |
| 85 | - | go func() { errCh <- ProxyRuntime{Client: client, Daemon: daemon, IdleTTL: time.Hour}.Run(t.Context()) }() | |
| 86 | - | hello := ports.Hello{Version: ports.ProtocolVersion, Intent: ports.IntentAttach, Name: "work", AckOutput: true} | |
| 87 | - | client.recv <- recvResult{frame: ports.Frame{Type: ports.MsgHello, Payload: ports.MarshalHello(hello)}} | |
| 83 | + | func TestProxyRuntimeClampsDatagramHelloOutputWindowAndPreservesControl(t *testing.T) { | |
| 84 | + | for _, requested := range []uint8{0, 1, 8} { | |
| 85 | + | t.Run(fmt.Sprintf("requested_%d", requested), func(t *testing.T) { | |
| 86 | + | client := newFakeTransport() | |
| 87 | + | daemon := newFakeTransport() | |
| 88 | + | errCh := make(chan error, 1) | |
| 89 | + | go func() { errCh <- ProxyRuntime{Client: client, Daemon: daemon, IdleTTL: time.Hour}.Run(t.Context()) }() | |
| 90 | + | hello := ports.Hello{Version: ports.ProtocolVersion, Intent: ports.IntentAttach, Name: "work", MaxOutputInFlight: requested} | |
| 91 | + | client.recv <- recvResult{frame: ports.Frame{Type: ports.MsgHello, Payload: ports.MarshalHello(hello)}} | |
| 88 | 92 | ||
| 89 | - | select { | |
| 90 | - | case frame := <-daemon.sent: | |
| 91 | - | got, err := ports.UnmarshalHello(frame.Payload) | |
| 92 | - | if err != nil { | |
| 93 | - | t.Fatal(err) | |
| 94 | - | } | |
| 95 | - | if !got.AckOutput { | |
| 96 | - | t.Fatal("proxied Hello lost AckOutput capability") | |
| 97 | - | } | |
| 98 | - | case <-time.After(time.Second): | |
| 99 | - | t.Fatal("timeout waiting for proxied Hello") | |
| 100 | - | } | |
| 93 | + | select { | |
| 94 | + | case frame := <-daemon.sent: | |
| 95 | + | got, err := ports.UnmarshalHello(frame.Payload) | |
| 96 | + | if err != nil { | |
| 97 | + | t.Fatal(err) | |
| 98 | + | } | |
| 99 | + | if got.MaxOutputInFlight != 1 { | |
| 100 | + | t.Fatalf("proxied output window=%d, want datagram-safe 1", got.MaxOutputInFlight) | |
| 101 | + | } | |
| 102 | + | case <-time.After(time.Second): | |
| 103 | + | t.Fatal("timeout waiting for proxied Hello") | |
| 104 | + | } | |
| 101 | 105 | ||
| 102 | - | daemon.recv <- recvResult{err: io.EOF} | |
| 103 | - | if err := <-errCh; err != nil { | |
| 104 | - | t.Fatalf("Run err=%v, want nil", err) | |
| 106 | + | input := ports.Frame{Type: ports.MsgInput, Payload: []byte("still flowing")} | |
| 107 | + | ack := ports.Frame{Type: ports.MsgAck, Payload: ports.MarshalAck(ports.Ack{AckedStateNum: 7})} | |
| 108 | + | client.recv <- recvResult{frame: input} | |
| 109 | + | client.recv <- recvResult{frame: ack} | |
| 110 | + | for _, want := range []ports.Frame{input, ack} { | |
| 111 | + | select { | |
| 112 | + | case got := <-daemon.sent: | |
| 113 | + | if !reflect.DeepEqual(got, want) { | |
| 114 | + | t.Fatalf("proxied control frame=%+v, want %+v", got, want) | |
| 115 | + | } | |
| 116 | + | case <-time.After(time.Second): | |
| 117 | + | t.Fatalf("timeout waiting for proxied %v", want.Type) | |
| 118 | + | } | |
| 119 | + | } | |
| 120 | + | ||
| 121 | + | daemon.recv <- recvResult{err: io.EOF} | |
| 122 | + | if err := <-errCh; err != nil { | |
| 123 | + | t.Fatalf("Run err=%v, want nil", err) | |
| 124 | + | } | |
| 125 | + | }) | |
| 105 | 126 | } | |
| 106 | 127 | } | |
| 107 | 128 | ||
| @@ -464,3 +485,34 @@ func TestProxyCopierStopsRetryingWhenContextCanceled(t *testing.T) { | |||
| 464 | 485 | t.Fatal("retrying copier did not stop after context cancellation") | |
| 465 | 486 | } | |
| 466 | 487 | } | |
| 488 | + | ||
| 489 | + | func TestClampDatagramHelloOutputWindowPreservesProtocolValidation(t *testing.T) { | |
| 490 | + | t.Run("malformed payload remains malformed", func(t *testing.T) { | |
| 491 | + | payload := append(ports.MarshalHello(ports.Hello{Version: ports.ProtocolVersion, MaxOutputInFlight: 8}), 0xff) | |
| 492 | + | frame := ports.Frame{Type: ports.MsgHello, Payload: payload} | |
| 493 | + | got := clampDatagramHelloOutputWindow(frame) | |
| 494 | + | if !reflect.DeepEqual(got, frame) { | |
| 495 | + | t.Fatalf("malformed Hello changed: got %#v, want %#v", got, frame) | |
| 496 | + | } | |
| 497 | + | if _, err := ports.UnmarshalHello(got.Payload); err == nil { | |
| 498 | + | t.Fatal("malformed Hello became decodable") | |
| 499 | + | } | |
| 500 | + | }) | |
| 501 | + | ||
| 502 | + | t.Run("mismatched version remains mismatched", func(t *testing.T) { | |
| 503 | + | wantVersion := ports.ProtocolVersion + 1 | |
| 504 | + | frame := ports.Frame{Type: ports.MsgHello, Payload: ports.MarshalHello(ports.Hello{ | |
| 505 | + | Version: wantVersion, MaxOutputInFlight: 8, | |
| 506 | + | })} | |
| 507 | + | got, err := ports.UnmarshalHello(clampDatagramHelloOutputWindow(frame).Payload) | |
| 508 | + | if err != nil { | |
| 509 | + | t.Fatal(err) | |
| 510 | + | } | |
| 511 | + | if got.Version != wantVersion { | |
| 512 | + | t.Fatalf("version=%d, want unchanged mismatch %d", got.Version, wantVersion) | |
| 513 | + | } | |
| 514 | + | if got.MaxOutputInFlight != 1 { | |
| 515 | + | t.Fatalf("output window=%d, want 1", got.MaxOutputInFlight) | |
| 516 | + | } | |
| 517 | + | }) | |
| 518 | + | } | |
M internal/adapters/dgram/ssh_dialer.go +1 -0
| @@ -176,6 +176,7 @@ func (d RemoteDialer) Dial(ctx context.Context) (ports.Transport, error) { | |||
| 176 | 176 | // fresh local UDP socket once the link is silent long enough to probe. The | |
| 177 | 177 | // bind reuses listenUDPPacket so tests can stub it. | |
| 178 | 178 | t, err := NewTransportWithOptions(pc, peer, ready.key, 1, 2, Options{ | |
| 179 | + | Observe: DiagnosticLogObserver(d.Log), | |
| 179 | 180 | RebindPacketConn: func(net.PacketConn) (net.PacketConn, error) { | |
| 180 | 181 | return listenUDPPacket(ctx) | |
| 181 | 182 | }, | |
M internal/adapters/dgram/transport.go +806 -328
| @@ -20,33 +20,29 @@ const ( | |||
| 20 | 20 | recProbe byte = 3 | |
| 21 | 21 | recPong byte = 4 | |
| 22 | 22 | ||
| 23 | - | defaultMTU = pdgram.DefaultMTU | |
| 24 | - | defaultResend = 250 * time.Millisecond | |
| 25 | - | defaultMaxResendAfter = 2 * time.Second | |
| 26 | - | defaultMaxResendPerTick = 64 | |
| 27 | - | defaultWriteTimeout = 250 * time.Millisecond | |
| 28 | - | defaultHeartbeat = 3 * time.Second | |
| 29 | - | defaultDegraded = 10 * time.Second | |
| 30 | - | defaultProbe = 20 * time.Second | |
| 31 | - | defaultOffline = 30 * time.Second | |
| 32 | - | defaultDead = 60 * time.Second | |
| 33 | - | defaultMaxPending = 1024 | |
| 34 | - | defaultMaxPendingWait = 50 * time.Millisecond | |
| 35 | - | defaultMaxRecvBuf = 1024 | |
| 36 | - | defaultOutputPaceMinDelay = 8 * time.Millisecond | |
| 37 | - | defaultOutputPaceMinCadence = 20 * time.Millisecond | |
| 38 | - | defaultOutputPaceMaxDelay = 250 * time.Millisecond | |
| 39 | - | defaultOutputPaceBatch = 2 | |
| 40 | - | defaultPacketPaceBudget = 8 | |
| 41 | - | dataRecordHeaderSize = 12 | |
| 42 | - | maxRecvBuffer = defaultMaxRecvBuf | |
| 43 | - | linkEventBufferSize = 16 | |
| 44 | - | probeReplyBufferSize = 16 | |
| 23 | + | defaultMTU = pdgram.DefaultMTU | |
| 24 | + | defaultResend = 250 * time.Millisecond | |
| 25 | + | defaultMaxResendAfter = 2 * time.Second | |
| 26 | + | defaultMaxResendPerTick = 64 | |
| 27 | + | defaultWriteTimeout = 250 * time.Millisecond | |
| 28 | + | defaultHeartbeat = 3 * time.Second | |
| 29 | + | defaultDegraded = 10 * time.Second | |
| 30 | + | defaultProbe = 20 * time.Second | |
| 31 | + | defaultOffline = 30 * time.Second | |
| 32 | + | defaultDead = 60 * time.Second | |
| 33 | + | defaultMaxPending = 1024 | |
| 34 | + | defaultMaxPendingWait = 50 * time.Millisecond | |
| 35 | + | defaultMaxRecvBuf = 1024 | |
| 36 | + | dataRecordHeaderSize = 12 | |
| 37 | + | maxRecvBuffer = defaultMaxRecvBuf | |
| 38 | + | linkEventBufferSize = 16 | |
| 39 | + | probeReplyBufferSize = controlQueueSize // legacy test burst bound | |
| 45 | 40 | ) | |
| 46 | 41 | ||
| 47 | 42 | var ( | |
| 48 | 43 | ErrPendingFull = errors.New("dgram: pending reliable queue full") | |
| 49 | 44 | ErrLinkDead = errors.New("dgram: link dead") | |
| 45 | + | errControlFull = errors.New("dgram: control queue full") | |
| 50 | 46 | ) | |
| 51 | 47 | ||
| 52 | 48 | type Options struct { | |
| @@ -64,6 +60,8 @@ type Options struct { | |||
| 64 | 60 | MaxPendingWait time.Duration | |
| 65 | 61 | MaxRecvBuffer int | |
| 66 | 62 | Clock ports.Clock | |
| 63 | + | // Observe receives bounded, asynchronous, privacy-safe link health snapshots. | |
| 64 | + | Observe DiagnosticObserver | |
| 67 | 65 | // RebindPacketConn is an optional client-side hook used to hop the local UDP | |
| 68 | 66 | // socket when the peer is offline. Server/proxy transports should leave it nil. | |
| 69 | 67 | RebindPacketConn func(net.PacketConn) (net.PacketConn, error) | |
| @@ -131,47 +129,83 @@ func normalizeOptions(opts Options) Options { | |||
| 131 | 129 | return opts | |
| 132 | 130 | } | |
| 133 | 131 | ||
| 132 | + | var _ ports.DatagramTransport = (*Transport)(nil) | |
| 133 | + | ||
| 134 | 134 | type Transport struct { | |
| 135 | 135 | pc net.PacketConn | |
| 136 | 136 | codec *pdgram.Codec | |
| 137 | 137 | sendDir, recvDir uint32 | |
| 138 | 138 | mtu int | |
| 139 | 139 | ||
| 140 | - | mu sync.Mutex | |
| 141 | - | peer net.Addr | |
| 142 | - | ctr uint64 | |
| 143 | - | seq uint64 | |
| 144 | - | probeSeq uint64 | |
| 145 | - | pending map[uint64]*pending | |
| 146 | - | closed bool | |
| 147 | - | closeErr error | |
| 148 | - | lastHeard time.Time | |
| 149 | - | heartbeat time.Duration | |
| 150 | - | resendAfter time.Duration | |
| 151 | - | maxResendAfter time.Duration | |
| 152 | - | maxResendPerTick int | |
| 153 | - | srtt time.Duration | |
| 154 | - | rttvar time.Duration | |
| 155 | - | rto time.Duration | |
| 156 | - | writeTimeout time.Duration | |
| 157 | - | degradedAfter time.Duration | |
| 158 | - | probeAfter time.Duration | |
| 159 | - | offlineAfter time.Duration | |
| 160 | - | deadAfter time.Duration | |
| 161 | - | maxPending int | |
| 162 | - | maxPendingWait time.Duration | |
| 163 | - | maxRecvBuffer int | |
| 164 | - | clock ports.Clock | |
| 165 | - | linkState ports.LinkState | |
| 166 | - | linkEvents chan ports.LinkEvent | |
| 167 | - | probeWait map[uint64]chan struct{} | |
| 168 | - | sendWake chan struct{} | |
| 169 | - | probeReply chan uint64 | |
| 170 | - | rebind func(net.PacketConn) (net.PacketConn, error) | |
| 171 | - | hoppedOffline bool | |
| 172 | - | outputQueue []queuedSend | |
| 173 | - | outputWake chan struct{} | |
| 174 | - | outputNext time.Time | |
| 140 | + | mu sync.Mutex | |
| 141 | + | peer net.Addr | |
| 142 | + | ctr uint64 | |
| 143 | + | seq uint64 | |
| 144 | + | probeSeq uint64 | |
| 145 | + | pending map[uint64]*pending | |
| 146 | + | closed bool | |
| 147 | + | closeErr error | |
| 148 | + | health healthTracker | |
| 149 | + | peerCounter uint64 | |
| 150 | + | peerCounterSet bool | |
| 151 | + | // These mirrors preserve the established diagnostic snapshot fields while | |
| 152 | + | // health owns link-state decisions. | |
| 153 | + | lastAuthenticatedPacket time.Time | |
| 154 | + | lastCompleteRecord time.Time | |
| 155 | + | lastACKProgress time.Time | |
| 156 | + | retransmits uint64 | |
| 157 | + | reassemblyInflight int | |
| 158 | + | // afterAuthenticatedPacket is a test synchronization hook. It runs after | |
| 159 | + | // authenticated-contact state has been committed. | |
| 160 | + | afterAuthenticatedPacket func() | |
| 161 | + | // afterMalformedFragment is a test synchronization hook. It runs after an | |
| 162 | + | // authenticated datagram has been rejected as an invalid fragment. | |
| 163 | + | afterMalformedFragment func() | |
| 164 | + | // afterHealthDecision is a test synchronization hook. It runs after a health | |
| 165 | + | // snapshot is taken and before that decision is conditionally committed. | |
| 166 | + | afterHealthDecision func() | |
| 167 | + | // afterLinkStateCommit is a test synchronization hook. It runs after state | |
| 168 | + | // mutation and before ordered event publication. | |
| 169 | + | afterLinkStateCommit func(ports.LinkState) | |
| 170 | + | observe DiagnosticObserver | |
| 171 | + | diagnosticCh chan Diagnostic | |
| 172 | + | heartbeat time.Duration | |
| 173 | + | resendAfter time.Duration | |
| 174 | + | maxResendAfter time.Duration | |
| 175 | + | maxResendPerTick int | |
| 176 | + | srtt time.Duration | |
| 177 | + | rttvar time.Duration | |
| 178 | + | rto time.Duration | |
| 179 | + | writeTimeout time.Duration | |
| 180 | + | degradedAfter time.Duration | |
| 181 | + | probeAfter time.Duration | |
| 182 | + | offlineAfter time.Duration | |
| 183 | + | deadAfter time.Duration | |
| 184 | + | maxPending int | |
| 185 | + | maxPendingWait time.Duration | |
| 186 | + | maxRecvBuffer int | |
| 187 | + | clock ports.Clock | |
| 188 | + | congestion congestionController | |
| 189 | + | linkState ports.LinkState | |
| 190 | + | linkStateGeneration uint64 | |
| 191 | + | linkEvents chan ports.LinkEvent | |
| 192 | + | linkEventMu sync.Mutex | |
| 193 | + | probeWait map[uint64]chan struct{} | |
| 194 | + | sendWake chan struct{} | |
| 195 | + | control chan controlRecord | |
| 196 | + | ackWake chan struct{} | |
| 197 | + | ackSend chan uint64 | |
| 198 | + | controlMu sync.Mutex | |
| 199 | + | ackSeq uint64 | |
| 200 | + | ackQueued bool | |
| 201 | + | rebind func(net.PacketConn) (net.PacketConn, error) | |
| 202 | + | hoppedOffline bool | |
| 203 | + | hopGeneration uint64 | |
| 204 | + | outputQueue []queuedSend | |
| 205 | + | outputWake chan struct{} | |
| 206 | + | dataSend chan dataSendJob | |
| 207 | + | retransmitWork chan []retransmitRecord | |
| 208 | + | writeDeadlines *writeDeadlineState | |
| 175 | 209 | ||
| 176 | 210 | recvMu sync.Mutex | |
| 177 | 211 | replay *pdgram.ReplayWindow | |
| @@ -179,10 +213,27 @@ type Transport struct { | |||
| 179 | 213 | in chan ports.Frame | |
| 180 | 214 | done chan struct{} | |
| 181 | 215 | ||
| 182 | - | writeMu sync.Mutex | |
| 183 | - | dataPaceMu sync.Mutex | |
| 184 | - | dataPaceRemaining int | |
| 185 | - | dataPaceNext time.Time | |
| 216 | + | writeMu sync.Mutex | |
| 217 | + | // beforeDataPace is a test synchronization hook. It is read under mu. | |
| 218 | + | beforeDataPace func() | |
| 219 | + | // afterDataPaceWait observes a data pacer's next manual-clock deadline. | |
| 220 | + | // Test observers must be non-blocking. | |
| 221 | + | afterDataPaceWait func(time.Time) | |
| 222 | + | // afterDataJob observes completion of a data or retransmit send job. Test | |
| 223 | + | // observers must be non-blocking. | |
| 224 | + | afterDataJob func() | |
| 225 | + | // Timer hooks are test-only completion notifications. Observers must be | |
| 226 | + | // non-blocking so timer loops never depend on test scheduling. | |
| 227 | + | afterRetransmitTimer func() | |
| 228 | + | afterHealthTimer func() | |
| 229 | + | afterHeartbeatTimer func() | |
| 230 | + | // afterPacketProcessed is a test synchronization hook called after an | |
| 231 | + | // authenticated packet has been fully processed. It is read under mu. | |
| 232 | + | afterPacketProcessed func() | |
| 233 | + | // ACK hooks are test-only and observers must be non-blocking. | |
| 234 | + | afterACKWakeAccepted func() | |
| 235 | + | afterACKScheduled func() | |
| 236 | + | afterACKDispatched func() | |
| 186 | 237 | // outboundMu orders first transmissions and paced batches. Retransmits use | |
| 187 | 238 | // writeMu directly because they do not establish new sequence order. | |
| 188 | 239 | outboundMu sync.Mutex | |
| @@ -201,6 +252,63 @@ type pending struct { | |||
| 201 | 252 | initialInFlight bool | |
| 202 | 253 | attempts int | |
| 203 | 254 | retransmitted bool | |
| 255 | + | wireBytes int | |
| 256 | + | } | |
| 257 | + | ||
| 258 | + | type retransmitRecord struct { | |
| 259 | + | seq uint64 | |
| 260 | + | f ports.Frame | |
| 261 | + | } | |
| 262 | + | ||
| 263 | + | type writeDeadlineState struct { | |
| 264 | + | mu sync.Mutex | |
| 265 | + | pc net.PacketConn | |
| 266 | + | nextID uint64 | |
| 267 | + | deadlines map[uint64]time.Time | |
| 268 | + | } | |
| 269 | + | ||
| 270 | + | func newWriteDeadlineState(pc net.PacketConn) *writeDeadlineState { | |
| 271 | + | return &writeDeadlineState{pc: pc, deadlines: make(map[uint64]time.Time)} | |
| 272 | + | } | |
| 273 | + | ||
| 274 | + | func (s *writeDeadlineState) begin(deadline time.Time) (func(), error) { | |
| 275 | + | s.mu.Lock() | |
| 276 | + | s.nextID++ | |
| 277 | + | id := s.nextID | |
| 278 | + | s.deadlines[id] = deadline | |
| 279 | + | if err := s.applyLocked(); err != nil { | |
| 280 | + | delete(s.deadlines, id) | |
| 281 | + | s.mu.Unlock() | |
| 282 | + | return nil, err | |
| 283 | + | } | |
| 284 | + | s.mu.Unlock() | |
| 285 | + | return func() { | |
| 286 | + | s.mu.Lock() | |
| 287 | + | delete(s.deadlines, id) | |
| 288 | + | _ = s.applyLocked() | |
| 289 | + | s.mu.Unlock() | |
| 290 | + | }, nil | |
| 291 | + | } | |
| 292 | + | ||
| 293 | + | func (s *writeDeadlineState) applyLocked() error { | |
| 294 | + | var earliest time.Time | |
| 295 | + | for _, deadline := range s.deadlines { | |
| 296 | + | if earliest.IsZero() || deadline.Before(earliest) { | |
| 297 | + | earliest = deadline | |
| 298 | + | } | |
| 299 | + | } | |
| 300 | + | return s.pc.SetWriteDeadline(earliest) | |
| 301 | + | } | |
| 302 | + | ||
| 303 | + | func (s *writeDeadlineState) expire(now time.Time) { | |
| 304 | + | s.mu.Lock() | |
| 305 | + | for id, deadline := range s.deadlines { | |
| 306 | + | if !deadline.After(now) { | |
| 307 | + | delete(s.deadlines, id) | |
| 308 | + | } | |
| 309 | + | } | |
| 310 | + | _ = s.applyLocked() | |
| 311 | + | s.mu.Unlock() | |
| 204 | 312 | } | |
| 205 | 313 | ||
| 206 | 314 | func NewTransport(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32) (*Transport, error) { | |
| @@ -214,48 +322,68 @@ func NewTransportWithOptions(pc net.PacketConn, peer net.Addr, key []byte, sendD | |||
| 214 | 322 | } | |
| 215 | 323 | opts = normalizeOptions(opts) | |
| 216 | 324 | t := &Transport{ | |
| 217 | - | pc: pc, | |
| 218 | - | codec: c, | |
| 219 | - | sendDir: sendDir, | |
| 220 | - | recvDir: recvDir, | |
| 221 | - | mtu: opts.MTU, | |
| 222 | - | peer: peer, | |
| 223 | - | pending: make(map[uint64]*pending), | |
| 224 | - | replay: pdgram.NewReplayWindow(), | |
| 225 | - | reasm: pdgram.NewReassembler(), | |
| 226 | - | in: make(chan ports.Frame, 32), | |
| 227 | - | done: make(chan struct{}), | |
| 228 | - | lastHeard: opts.Clock.Now(), | |
| 229 | - | heartbeat: opts.Heartbeat, | |
| 230 | - | resendAfter: opts.ResendAfter, | |
| 231 | - | maxResendAfter: opts.MaxResendAfter, | |
| 232 | - | maxResendPerTick: opts.MaxResendPerTick, | |
| 233 | - | rto: opts.ResendAfter, | |
| 234 | - | writeTimeout: opts.WriteTimeout, | |
| 235 | - | degradedAfter: opts.DegradedAfter, | |
| 236 | - | probeAfter: opts.ProbeAfter, | |
| 237 | - | offlineAfter: opts.OfflineAfter, | |
| 238 | - | deadAfter: opts.DeadAfter, | |
| 239 | - | maxPending: opts.MaxPending, | |
| 240 | - | maxPendingWait: opts.MaxPendingWait, | |
| 241 | - | maxRecvBuffer: opts.MaxRecvBuffer, | |
| 242 | - | clock: opts.Clock, | |
| 243 | - | linkState: ports.LinkStateConnected, | |
| 244 | - | linkEvents: make(chan ports.LinkEvent, linkEventBufferSize), | |
| 245 | - | probeWait: make(map[uint64]chan struct{}), | |
| 246 | - | rebind: opts.RebindPacketConn, | |
| 247 | - | nextRecvSeq: 1, | |
| 248 | - | recvBuf: make(map[uint64]ports.Frame), | |
| 325 | + | pc: pc, | |
| 326 | + | codec: c, | |
| 327 | + | sendDir: sendDir, | |
| 328 | + | recvDir: recvDir, | |
| 329 | + | mtu: opts.MTU, | |
| 330 | + | peer: peer, | |
| 331 | + | pending: make(map[uint64]*pending), | |
| 332 | + | replay: pdgram.NewReplayWindow(), | |
| 333 | + | reasm: pdgram.NewReassembler(), | |
| 334 | + | in: make(chan ports.Frame, 32), | |
| 335 | + | done: make(chan struct{}), | |
| 336 | + | health: newHealthTracker(opts.Clock.Now()), | |
| 337 | + | lastAuthenticatedPacket: opts.Clock.Now(), | |
| 338 | + | lastCompleteRecord: opts.Clock.Now(), | |
| 339 | + | lastACKProgress: opts.Clock.Now(), | |
| 340 | + | observe: opts.Observe, | |
| 341 | + | heartbeat: opts.Heartbeat, | |
| 342 | + | resendAfter: opts.ResendAfter, | |
| 343 | + | maxResendAfter: opts.MaxResendAfter, | |
| 344 | + | maxResendPerTick: opts.MaxResendPerTick, | |
| 345 | + | rto: opts.ResendAfter, | |
| 346 | + | writeTimeout: opts.WriteTimeout, | |
| 347 | + | degradedAfter: opts.DegradedAfter, | |
| 348 | + | probeAfter: opts.ProbeAfter, | |
| 349 | + | offlineAfter: opts.OfflineAfter, | |
| 350 | + | deadAfter: opts.DeadAfter, | |
| 351 | + | maxPending: opts.MaxPending, | |
| 352 | + | maxPendingWait: opts.MaxPendingWait, | |
| 353 | + | maxRecvBuffer: opts.MaxRecvBuffer, | |
| 354 | + | clock: opts.Clock, | |
| 355 | + | congestion: newCongestionController(opts.MTU), | |
| 356 | + | linkState: ports.LinkStateConnected, | |
| 357 | + | linkEvents: make(chan ports.LinkEvent, linkEventBufferSize), | |
| 358 | + | probeWait: make(map[uint64]chan struct{}), | |
| 359 | + | rebind: opts.RebindPacketConn, | |
| 360 | + | nextRecvSeq: 1, | |
| 361 | + | recvBuf: make(map[uint64]ports.Frame), | |
| 249 | 362 | } | |
| 250 | 363 | t.sendWake = make(chan struct{}) | |
| 364 | + | t.control = make(chan controlRecord, controlQueueSize) | |
| 365 | + | t.ackWake = make(chan struct{}, 1) | |
| 366 | + | t.ackSend = make(chan uint64, 1) | |
| 251 | 367 | t.outputWake = make(chan struct{}) | |
| 252 | - | t.probeReply = make(chan uint64, probeReplyBufferSize) | |
| 368 | + | t.dataSend = make(chan dataSendJob, opts.MaxPending) | |
| 369 | + | t.writeDeadlines = newWriteDeadlineState(pc) | |
| 370 | + | // One queued batch bounds retransmit work while its sole sender is paced. | |
| 371 | + | t.retransmitWork = make(chan []retransmitRecord, 1) | |
| 372 | + | if t.observe != nil { | |
| 373 | + | t.diagnosticCh = make(chan Diagnostic, diagnosticBufferSize) | |
| 374 | + | go t.diagnosticLoop() | |
| 375 | + | } | |
| 253 | 376 | t.deliverCond = sync.NewCond(&t.deliverMu) | |
| 254 | 377 | go t.readLoop(pc) | |
| 255 | - | go t.resendLoop() | |
| 378 | + | go t.retransmitLoop() | |
| 379 | + | go t.heartbeatLoop() | |
| 380 | + | go t.healthLoop() | |
| 381 | + | go t.dataSendLoop() | |
| 256 | 382 | go t.deliveryLoop() | |
| 257 | - | go t.probeReplyLoop() | |
| 383 | + | go t.controlLoop() | |
| 384 | + | go t.ackSendLoop() | |
| 258 | 385 | go t.outputPaceLoop() | |
| 386 | + | t.emitDiagnostic() | |
| 259 | 387 | return t, nil | |
| 260 | 388 | } | |
| 261 | 389 | ||
| @@ -283,60 +411,51 @@ func (t *Transport) send(f ports.Frame, async bool) error { | |||
| 283 | 411 | t.seq++ | |
| 284 | 412 | seq := t.seq | |
| 285 | 413 | if reliable { | |
| 286 | - | t.pending[seq] = &pending{frame: f, enqueued: t.clock.Now()} | |
| 287 | - | } | |
| 288 | - | if shouldPaceOutput(f) { | |
| 289 | 414 | now := t.clock.Now() | |
| 290 | - | if len(t.outputQueue) > 0 || (!t.outputNext.IsZero() && now.Before(t.outputNext)) { | |
| 291 | - | wasEmpty := len(t.outputQueue) == 0 | |
| 292 | - | var done chan error | |
| 293 | - | if !async { | |
| 294 | - | done = make(chan error, 1) | |
| 295 | - | } | |
| 296 | - | t.outputQueue = append(t.outputQueue, queuedSend{seq: seq, reliable: reliable, frame: f, done: done}) | |
| 297 | - | if wasEmpty { | |
| 298 | - | t.notifyOutputPacerLocked() | |
| 299 | - | } | |
| 300 | - | t.mu.Unlock() | |
| 301 | - | if done == nil { | |
| 302 | - | return nil | |
| 303 | - | } | |
| 304 | - | t.outboundMu.Unlock() | |
| 305 | - | unlockOutbound = false | |
| 306 | - | return t.waitQueuedSend(done) | |
| 307 | - | } | |
| 308 | - | t.outputNext = now.Add(t.outputPaceDelayLocked()) | |
| 309 | - | t.mu.Unlock() | |
| 310 | - | t.markPendingSent(seq, reliable) | |
| 311 | - | if err := t.sendData(seq, reliable, f); err != nil { | |
| 312 | - | t.removePending(seq, reliable) | |
| 313 | - | return err | |
| 415 | + | if len(t.pending) == 0 { | |
| 416 | + | t.health.pendingStarted(now) | |
| 314 | 417 | } | |
| 315 | - | t.markPendingReady(seq, reliable) | |
| 316 | - | return nil | |
| 317 | - | } | |
| 318 | - | if len(t.outputQueue) > 0 { | |
| 319 | - | var done chan error | |
| 320 | - | if !async { | |
| 321 | - | done = make(chan error, 1) | |
| 418 | + | t.pending[seq] = &pending{frame: f, enqueued: now} | |
| 419 | + | } | |
| 420 | + | var done chan error | |
| 421 | + | if !async { | |
| 422 | + | done = make(chan error, 1) | |
| 423 | + | } | |
| 424 | + | job := dataSendJob{seq: seq, reliable: reliable, frame: f, done: done} | |
| 425 | + | if shouldPaceOutput(f) || len(t.outputQueue) > 0 { | |
| 426 | + | wasEmpty := len(t.outputQueue) == 0 | |
| 427 | + | t.outputQueue = append(t.outputQueue, queuedSend(job)) | |
| 428 | + | if wasEmpty { | |
| 429 | + | t.notifyOutputPacerLocked() | |
| 322 | 430 | } | |
| 323 | - | t.outputQueue = append(t.outputQueue, queuedSend{seq: seq, reliable: reliable, frame: f, done: done}) | |
| 324 | 431 | t.mu.Unlock() | |
| 325 | - | if done == nil { | |
| 326 | - | return nil | |
| 327 | - | } | |
| 328 | 432 | t.outboundMu.Unlock() | |
| 329 | 433 | unlockOutbound = false | |
| 434 | + | if async { | |
| 435 | + | return nil | |
| 436 | + | } | |
| 330 | 437 | return t.waitQueuedSend(done) | |
| 331 | 438 | } | |
| 332 | 439 | t.mu.Unlock() | |
| 333 | - | t.markPendingSent(seq, reliable) | |
| 334 | - | if err := t.sendData(seq, reliable, f); err != nil { | |
| 440 | + | if err := t.queueDataJob(job); err != nil { | |
| 335 | 441 | t.removePending(seq, reliable) | |
| 336 | 442 | return err | |
| 337 | 443 | } | |
| 338 | - | t.markPendingReady(seq, reliable) | |
| 339 | - | return nil | |
| 444 | + | t.outboundMu.Unlock() | |
| 445 | + | unlockOutbound = false | |
| 446 | + | if async { | |
| 447 | + | return nil | |
| 448 | + | } | |
| 449 | + | return t.waitQueuedSend(done) | |
| 450 | + | } | |
| 451 | + | ||
| 452 | + | func (t *Transport) queueDataJob(job dataSendJob) error { | |
| 453 | + | select { | |
| 454 | + | case t.dataSend <- job: | |
| 455 | + | return nil | |
| 456 | + | case <-t.done: | |
| 457 | + | return t.closedError() | |
| 458 | + | } | |
| 340 | 459 | } | |
| 341 | 460 | ||
| 342 | 461 | func (t *Transport) waitQueuedSend(done <-chan error) error { | |
| @@ -442,6 +561,9 @@ func (t *Transport) removePendingLocked(seq uint64) bool { | |||
| 442 | 561 | return false | |
| 443 | 562 | } | |
| 444 | 563 | delete(t.pending, seq) | |
| 564 | + | if len(t.pending) == 0 { | |
| 565 | + | t.health.pendingCleared() | |
| 566 | + | } | |
| 445 | 567 | return true | |
| 446 | 568 | } | |
| 447 | 569 | ||
| @@ -485,22 +607,30 @@ func (t *Transport) Probe(ctx context.Context) error { | |||
| 485 | 607 | return err | |
| 486 | 608 | } | |
| 487 | 609 | defer t.unregisterProbe(id) | |
| 488 | - | if err := t.sendProbe(recProbe, id); err != nil { | |
| 489 | - | return err | |
| 610 | + | sendResult := make(chan error, 1) | |
| 611 | + | if !t.queueControlRecord(controlRecord{kind: recProbe, id: id, result: sendResult}) { | |
| 612 | + | return errControlFull | |
| 490 | 613 | } | |
| 491 | - | select { | |
| 492 | - | case <-ch: | |
| 493 | - | return nil | |
| 494 | - | case <-ctx.Done(): | |
| 495 | - | return ctx.Err() | |
| 496 | - | case <-t.done: | |
| 497 | - | t.mu.Lock() | |
| 498 | - | err := t.closeErr | |
| 499 | - | t.mu.Unlock() | |
| 500 | - | if err != nil { | |
| 501 | - | return err | |
| 614 | + | for { | |
| 615 | + | select { | |
| 616 | + | case err := <-sendResult: | |
| 617 | + | if err != nil { | |
| 618 | + | return err | |
| 619 | + | } | |
| 620 | + | sendResult = nil | |
| 621 | + | case <-ch: | |
| 622 | + | return nil | |
| 623 | + | case <-ctx.Done(): | |
| 624 | + | return ctx.Err() | |
| 625 | + | case <-t.done: | |
| 626 | + | t.mu.Lock() | |
| 627 | + | err := t.closeErr | |
| 628 | + | t.mu.Unlock() | |
| 629 | + | if err != nil { | |
| 630 | + | return err | |
| 631 | + | } | |
| 632 | + | return errors.New("dgram: closed") | |
| 502 | 633 | } | |
| 503 | - | return errors.New("dgram: closed") | |
| 504 | 634 | } | |
| 505 | 635 | } | |
| 506 | 636 | ||
| @@ -526,40 +656,18 @@ func (t *Transport) unregisterProbe(id uint64) { | |||
| 526 | 656 | t.mu.Unlock() | |
| 527 | 657 | } | |
| 528 | 658 | ||
| 529 | - | func (t *Transport) sendData(seq uint64, reliable bool, f ports.Frame) error { | |
| 530 | - | p := encodeData(seq, reliable, f) | |
| 531 | - | return t.sendPayload(p, true, seq) | |
| 532 | - | } | |
| 533 | 659 | func (t *Transport) sendAck(seq uint64) { | |
| 534 | 660 | var b [9]byte | |
| 535 | 661 | b[0] = recAck | |
| 536 | 662 | binary.BigEndian.PutUint64(b[1:], seq) | |
| 537 | - | _ = t.sendPayload(b[:], false, 0) | |
| 663 | + | _ = t.sendPayload(b[:]) | |
| 538 | 664 | } | |
| 539 | - | func (t *Transport) sendProbe(kind byte, id uint64) error { | |
| 540 | - | var b [9]byte | |
| 541 | - | b[0] = kind | |
| 542 | - | binary.BigEndian.PutUint64(b[1:], id) | |
| 543 | - | return t.sendPayload(b[:], false, 0) | |
| 544 | - | } | |
| 545 | - | func (t *Transport) sendPayload(p []byte, paced bool, pendingSeq uint64) error { | |
| 546 | - | if paced { | |
| 547 | - | t.dataPaceMu.Lock() | |
| 548 | - | defer t.dataPaceMu.Unlock() | |
| 549 | - | } | |
| 665 | + | func (t *Transport) sendPayload(p []byte) error { | |
| 550 | 666 | frags, err := pdgram.FragmentPayload(t.nextCounter(), p, t.mtu-pdgram.HeaderSize-t.codec.Overhead()) | |
| 551 | 667 | if err != nil { | |
| 552 | 668 | return err | |
| 553 | 669 | } | |
| 554 | - | for i, f := range frags { | |
| 555 | - | if pendingSeq != 0 && !t.pendingExists(pendingSeq) { | |
| 556 | - | return nil | |
| 557 | - | } | |
| 558 | - | if paced { | |
| 559 | - | if err := t.takeDataPaceSlot(); err != nil { | |
| 560 | - | return err | |
| 561 | - | } | |
| 562 | - | } | |
| 670 | + | for _, f := range frags { | |
| 563 | 671 | raw, err := pdgram.MarshalFragment(f) | |
| 564 | 672 | if err != nil { | |
| 565 | 673 | return err | |
| @@ -579,9 +687,6 @@ func (t *Transport) sendPayload(p []byte, paced bool, pendingSeq uint64) error { | |||
| 579 | 687 | if peer == nil { | |
| 580 | 688 | return errors.New("dgram: no peer") | |
| 581 | 689 | } | |
| 582 | - | if pendingSeq != 0 && i == len(frags)-1 { | |
| 583 | - | t.markPendingFinalWrite(pendingSeq) | |
| 584 | - | } | |
| 585 | 690 | if err := t.writePacket(pkt, peer); err != nil { | |
| 586 | 691 | return err | |
| 587 | 692 | } | |
| @@ -596,30 +701,6 @@ func (t *Transport) pendingExists(seq uint64) bool { | |||
| 596 | 701 | return ok | |
| 597 | 702 | } | |
| 598 | 703 | ||
| 599 | - | func (t *Transport) takeDataPaceSlot() error { | |
| 600 | - | now := t.clock.Now() | |
| 601 | - | if t.dataPaceRemaining > 0 && now.Before(t.dataPaceNext) { | |
| 602 | - | t.dataPaceRemaining-- | |
| 603 | - | return nil | |
| 604 | - | } | |
| 605 | - | if t.dataPaceNext.After(now) { | |
| 606 | - | timer := t.clock.NewTimer(t.dataPaceNext.Sub(now)) | |
| 607 | - | select { | |
| 608 | - | case <-timer.C(): | |
| 609 | - | case <-t.done: | |
| 610 | - | timer.Stop() | |
| 611 | - | return t.closedError() | |
| 612 | - | } | |
| 613 | - | timer.Stop() | |
| 614 | - | } | |
| 615 | - | t.mu.Lock() | |
| 616 | - | delay := t.outputPaceDelayLocked() | |
| 617 | - | t.mu.Unlock() | |
| 618 | - | t.dataPaceRemaining = defaultPacketPaceBudget - 1 | |
| 619 | - | t.dataPaceNext = t.clock.Now().Add(delay) | |
| 620 | - | return nil | |
| 621 | - | } | |
| 622 | - | ||
| 623 | 704 | func (t *Transport) writePacket(pkt []byte, peer net.Addr) error { | |
| 624 | 705 | t.writeMu.Lock() | |
| 625 | 706 | defer t.writeMu.Unlock() | |
| @@ -627,16 +708,35 @@ func (t *Transport) writePacket(pkt []byte, peer net.Addr) error { | |||
| 627 | 708 | t.mu.Lock() | |
| 628 | 709 | pc := t.pc | |
| 629 | 710 | writeTimeout := t.writeTimeout | |
| 711 | + | writeDeadlines := t.writeDeadlines | |
| 630 | 712 | t.mu.Unlock() | |
| 631 | 713 | ||
| 632 | - | if writeTimeout > 0 { | |
| 633 | - | if err := pc.SetWriteDeadline(t.clock.Now().Add(writeTimeout)); err != nil { | |
| 714 | + | return t.writeDatagram(pc, peer, pkt, writeDeadlines, writeTimeout) | |
| 715 | + | } | |
| 716 | + | ||
| 717 | + | func (t *Transport) writeDatagram(pc net.PacketConn, peer net.Addr, pkt []byte, deadlines *writeDeadlineState, timeout time.Duration) error { | |
| 718 | + | if timeout <= 0 { | |
| 719 | + | _, err := pc.WriteTo(pkt, peer) | |
| 720 | + | return err | |
| 721 | + | } | |
| 722 | + | ownDeadline := t.clock.Now().Add(timeout) | |
| 723 | + | finish, err := deadlines.begin(ownDeadline) | |
| 724 | + | if err != nil { | |
| 725 | + | return err | |
| 726 | + | } | |
| 727 | + | defer finish() | |
| 728 | + | for { | |
| 729 | + | _, err = pc.WriteTo(pkt, peer) | |
| 730 | + | if err == nil { | |
| 731 | + | return nil | |
| 732 | + | } | |
| 733 | + | var timeoutErr interface{ Timeout() bool } | |
| 734 | + | now := t.clock.Now() | |
| 735 | + | if !errors.As(err, &timeoutErr) || !timeoutErr.Timeout() || !now.Before(ownDeadline) { | |
| 634 | 736 | return err | |
| 635 | 737 | } | |
| 636 | - | defer func() { _ = pc.SetWriteDeadline(time.Time{}) }() | |
| 738 | + | deadlines.expire(now) | |
| 637 | 739 | } | |
| 638 | - | _, err := pc.WriteTo(pkt, peer) | |
| 639 | - | return err | |
| 640 | 740 | } | |
| 641 | 741 | ||
| 642 | 742 | func (t *Transport) nextCounter() uint64 { t.mu.Lock(); defer t.mu.Unlock(); t.ctr++; return t.ctr } | |
| @@ -656,7 +756,7 @@ func (t *Transport) readLoop(pc net.PacketConn) { | |||
| 656 | 756 | return | |
| 657 | 757 | } | |
| 658 | 758 | t.recvMu.Lock() | |
| 659 | - | _, pt, err := t.codec.Open(append([]byte(nil), buf[:n]...), t.recvDir, nil, t.replay) | |
| 759 | + | counter, pt, err := t.codec.Open(append([]byte(nil), buf[:n]...), t.recvDir, nil, t.replay) | |
| 660 | 760 | if err != nil { | |
| 661 | 761 | t.recvMu.Unlock() | |
| 662 | 762 | continue | |
| @@ -664,25 +764,72 @@ func (t *Transport) readLoop(pc net.PacketConn) { | |||
| 664 | 764 | frag, err := pdgram.UnmarshalFragment(pt) | |
| 665 | 765 | if err != nil { | |
| 666 | 766 | t.recvMu.Unlock() | |
| 767 | + | t.notifyMalformedFragment() | |
| 667 | 768 | continue | |
| 668 | 769 | } | |
| 669 | - | payload, ok, err := t.reasm.Add(frag) | |
| 770 | + | payload, complete, err := t.reasm.Add(frag) | |
| 771 | + | inflight := t.reasm.Inflight() | |
| 772 | + | // Keep the diagnostic mirror ordered with reassembly mutations. Rebind can | |
| 773 | + | // overlap old and new read loops, so recvMu must remain held through commit. | |
| 774 | + | t.mu.Lock() | |
| 775 | + | t.reassemblyInflight = inflight | |
| 776 | + | var afterAuthenticated func() | |
| 777 | + | if err == nil { | |
| 778 | + | now := t.clock.Now() | |
| 779 | + | t.health.authenticatedPacket(now) | |
| 780 | + | t.lastAuthenticatedPacket = now | |
| 781 | + | if !t.peerCounterSet || counter > t.peerCounter { | |
| 782 | + | t.peer = addr | |
| 783 | + | t.peerCounter = counter | |
| 784 | + | t.peerCounterSet = true | |
| 785 | + | } | |
| 786 | + | afterAuthenticated = t.afterAuthenticatedPacket | |
| 787 | + | } | |
| 788 | + | t.mu.Unlock() | |
| 670 | 789 | t.recvMu.Unlock() | |
| 671 | - | if err != nil || !ok { | |
| 790 | + | if afterAuthenticated != nil { | |
| 791 | + | afterAuthenticated() | |
| 792 | + | } | |
| 793 | + | if err != nil { | |
| 794 | + | t.notifyMalformedFragment() | |
| 795 | + | t.notifyPacketProcessed() | |
| 796 | + | continue | |
| 797 | + | } | |
| 798 | + | if !complete { | |
| 799 | + | t.notifyPacketProcessed() | |
| 672 | 800 | continue | |
| 673 | 801 | } | |
| 674 | - | t.updatePeerFromAuthenticated(addr) | |
| 802 | + | t.recordCompleteRecord() | |
| 675 | 803 | t.handleRecord(payload) | |
| 804 | + | t.checkSilence() | |
| 805 | + | t.notifyPacketProcessed() | |
| 676 | 806 | } | |
| 677 | 807 | } | |
| 678 | 808 | ||
| 679 | - | func (t *Transport) updatePeerFromAuthenticated(addr net.Addr) { | |
| 809 | + | func (t *Transport) notifyPacketProcessed() { | |
| 680 | 810 | t.mu.Lock() | |
| 681 | - | t.peer = addr | |
| 682 | - | t.lastHeard = t.clock.Now() | |
| 683 | - | t.hoppedOffline = false | |
| 811 | + | afterPacketProcessed := t.afterPacketProcessed | |
| 812 | + | t.mu.Unlock() | |
| 813 | + | if afterPacketProcessed != nil { | |
| 814 | + | afterPacketProcessed() | |
| 815 | + | } | |
| 816 | + | } | |
| 817 | + | ||
| 818 | + | func (t *Transport) notifyMalformedFragment() { | |
| 819 | + | t.mu.Lock() | |
| 820 | + | afterMalformed := t.afterMalformedFragment | |
| 821 | + | t.mu.Unlock() | |
| 822 | + | if afterMalformed != nil { | |
| 823 | + | afterMalformed() | |
| 824 | + | } | |
| 825 | + | } | |
| 826 | + | ||
| 827 | + | func (t *Transport) recordCompleteRecord() { | |
| 828 | + | t.mu.Lock() | |
| 829 | + | now := t.clock.Now() | |
| 830 | + | t.health.completeRecord(now) | |
| 831 | + | t.lastCompleteRecord = now | |
| 684 | 832 | t.mu.Unlock() | |
| 685 | - | t.setLinkState(ports.LinkStateConnected, nil) | |
| 686 | 833 | } | |
| 687 | 834 | ||
| 688 | 835 | func (t *Transport) handleRecord(p []byte) { | |
| @@ -697,6 +844,7 @@ func (t *Transport) handleRecord(p []byte) { | |||
| 697 | 844 | seq := binary.BigEndian.Uint64(p[1:]) | |
| 698 | 845 | t.mu.Lock() | |
| 699 | 846 | acked := false | |
| 847 | + | ackedBytes := 0 | |
| 700 | 848 | for pendingSeq, p := range t.pending { | |
| 701 | 849 | if pendingSeq > seq { | |
| 702 | 850 | continue | |
| @@ -704,21 +852,32 @@ func (t *Transport) handleRecord(p []byte) { | |||
| 704 | 852 | if pendingSeq == seq && p != nil && !p.retransmitted { | |
| 705 | 853 | t.updateRTTLocked(t.clock.Now().Sub(p.first)) | |
| 706 | 854 | } | |
| 855 | + | if p != nil { | |
| 856 | + | ackedBytes += p.wireBytes | |
| 857 | + | } | |
| 707 | 858 | delete(t.pending, pendingSeq) | |
| 708 | 859 | acked = true | |
| 709 | 860 | } | |
| 710 | 861 | if acked { | |
| 862 | + | t.congestion.onACK(ackedBytes) | |
| 863 | + | now := t.clock.Now() | |
| 864 | + | t.health.ackProgress(now) | |
| 865 | + | if len(t.pending) == 0 { | |
| 866 | + | t.health.pendingCleared() | |
| 867 | + | } | |
| 868 | + | t.lastACKProgress = now | |
| 711 | 869 | t.notifySendWaitersLocked() | |
| 712 | 870 | } | |
| 713 | 871 | t.mu.Unlock() | |
| 872 | + | if acked { | |
| 873 | + | t.setLinkState(ports.LinkStateConnected, nil) | |
| 874 | + | t.emitDiagnostic() | |
| 875 | + | } | |
| 714 | 876 | case recProbe: | |
| 715 | 877 | if len(p) != 9 { | |
| 716 | 878 | return | |
| 717 | 879 | } | |
| 718 | - | select { | |
| 719 | - | case t.probeReply <- binary.BigEndian.Uint64(p[1:]): | |
| 720 | - | default: | |
| 721 | - | } | |
| 880 | + | t.queueControl(recPong, binary.BigEndian.Uint64(p[1:])) | |
| 722 | 881 | case recPong: | |
| 723 | 882 | if len(p) != 9 { | |
| 724 | 883 | return | |
| @@ -739,7 +898,7 @@ func (t *Transport) handleRecord(p []byte) { | |||
| 739 | 898 | if reliable { | |
| 740 | 899 | ackSeq, ack, queued := t.enqueueReliable(seq, f) | |
| 741 | 900 | if ack { | |
| 742 | - | t.sendAck(ackSeq) | |
| 901 | + | t.queueACK(ackSeq) | |
| 743 | 902 | } | |
| 744 | 903 | if queued { | |
| 745 | 904 | t.signalDelivery() | |
| @@ -816,61 +975,266 @@ func (t *Transport) deliver(f ports.Frame) { | |||
| 816 | 975 | } | |
| 817 | 976 | } | |
| 818 | 977 | ||
| 819 | - | func (t *Transport) probeReplyLoop() { | |
| 978 | + | func (t *Transport) notifySendWaitersLocked() { | |
| 979 | + | if t.sendWake == nil { | |
| 980 | + | return | |
| 981 | + | } | |
| 982 | + | close(t.sendWake) | |
| 983 | + | t.sendWake = make(chan struct{}) | |
| 984 | + | } | |
| 985 | + | ||
| 986 | + | func (t *Transport) retransmitLoop() { | |
| 987 | + | resend := t.clock.NewTimer(t.resendAfter) | |
| 988 | + | defer resend.Stop() | |
| 820 | 989 | for { | |
| 821 | 990 | select { | |
| 822 | - | case id := <-t.probeReply: | |
| 823 | - | _ = t.sendProbe(recPong, id) | |
| 991 | + | case <-resend.C(): | |
| 992 | + | t.queueRetransmits() | |
| 993 | + | resend.Reset(t.resendAfter) | |
| 994 | + | t.notifyTimerHook(func() func() { return t.afterRetransmitTimer }) | |
| 824 | 995 | case <-t.done: | |
| 825 | 996 | return | |
| 826 | 997 | } | |
| 827 | 998 | } | |
| 828 | 999 | } | |
| 829 | 1000 | ||
| 830 | - | func (t *Transport) notifySendWaitersLocked() { | |
| 831 | - | if t.sendWake == nil { | |
| 1001 | + | // healthLoop owns link-health timing so bulk retransmission cannot delay state | |
| 1002 | + | // transitions or socket hopping. | |
| 1003 | + | func (t *Transport) healthLoop() { | |
| 1004 | + | health := t.clock.NewTimer(t.resendAfter) | |
| 1005 | + | defer health.Stop() | |
| 1006 | + | for { | |
| 1007 | + | select { | |
| 1008 | + | case <-health.C(): | |
| 1009 | + | t.checkSilence() | |
| 1010 | + | health.Reset(t.resendAfter) | |
| 1011 | + | t.notifyTimerHook(func() func() { return t.afterHealthTimer }) | |
| 1012 | + | case <-t.done: | |
| 1013 | + | return | |
| 1014 | + | } | |
| 1015 | + | } | |
| 1016 | + | } | |
| 1017 | + | ||
| 1018 | + | func (t *Transport) dataSendLoop() { | |
| 1019 | + | pacer := bytePacer{clk: t.clock, afterTimer: func(deadline time.Time) { | |
| 1020 | + | t.mu.Lock() | |
| 1021 | + | hook := t.afterDataPaceWait | |
| 1022 | + | t.mu.Unlock() | |
| 1023 | + | if hook != nil { | |
| 1024 | + | hook(deadline) | |
| 1025 | + | } | |
| 1026 | + | }} | |
| 1027 | + | for { | |
| 1028 | + | select { | |
| 1029 | + | case job := <-t.dataSend: | |
| 1030 | + | t.runDataSendJob(&pacer, job, true) | |
| 1031 | + | case batch := <-t.retransmitWork: | |
| 1032 | + | t.runRetransmitBatch(&pacer, batch) | |
| 1033 | + | case <-t.done: | |
| 1034 | + | return | |
| 1035 | + | } | |
| 1036 | + | } | |
| 1037 | + | } | |
| 1038 | + | ||
| 1039 | + | func (t *Transport) runDataSendJob(pacer *bytePacer, job dataSendJob, initial bool) { | |
| 1040 | + | if initial { | |
| 1041 | + | t.markPendingSent(job.seq, job.reliable) | |
| 1042 | + | } | |
| 1043 | + | _, err := t.writeDataJob(pacer, job, nil) | |
| 1044 | + | if err != nil { | |
| 1045 | + | if initial { | |
| 1046 | + | t.removePending(job.seq, job.reliable) | |
| 1047 | + | } else { | |
| 1048 | + | t.markPendingReady(job.seq, job.reliable) | |
| 1049 | + | } | |
| 1050 | + | if !errors.Is(err, errPacerClosed) { | |
| 1051 | + | t.closeWithError(err) | |
| 1052 | + | } | |
| 1053 | + | } else { | |
| 1054 | + | t.markPendingReady(job.seq, job.reliable) | |
| 1055 | + | } | |
| 1056 | + | t.mu.Lock() | |
| 1057 | + | afterDataJob := t.afterDataJob | |
| 1058 | + | t.mu.Unlock() | |
| 1059 | + | if afterDataJob != nil { | |
| 1060 | + | afterDataJob() | |
| 1061 | + | } | |
| 1062 | + | if job.done != nil { | |
| 1063 | + | if errors.Is(err, errPacerClosed) { | |
| 1064 | + | err = t.closedError() | |
| 1065 | + | } | |
| 1066 | + | job.done <- err | |
| 1067 | + | close(job.done) | |
| 1068 | + | } | |
| 1069 | + | } | |
| 1070 | + | ||
| 1071 | + | func (t *Transport) writeDataJob(pacer *bytePacer, job dataSendJob, onFirstWrite func()) (bool, error) { | |
| 1072 | + | if job.reliable && !t.pendingExists(job.seq) { | |
| 1073 | + | return false, nil | |
| 1074 | + | } | |
| 1075 | + | payload := encodeData(job.seq, job.reliable, job.frame) | |
| 1076 | + | frags, err := pdgram.FragmentPayload(t.nextCounter(), payload, t.mtu-pdgram.HeaderSize-t.codec.Overhead()) | |
| 1077 | + | if err != nil { | |
| 1078 | + | return false, err | |
| 1079 | + | } | |
| 1080 | + | t.mu.Lock() | |
| 1081 | + | beforeDataPace := t.beforeDataPace | |
| 1082 | + | t.mu.Unlock() | |
| 1083 | + | if beforeDataPace != nil { | |
| 1084 | + | beforeDataPace() | |
| 1085 | + | } | |
| 1086 | + | limits := func() (int, int) { | |
| 1087 | + | t.mu.Lock() | |
| 1088 | + | defer t.mu.Unlock() | |
| 1089 | + | return t.congestion.bytesPerSecond(), t.congestion.burstBytes() | |
| 1090 | + | } | |
| 1091 | + | emitted := false | |
| 1092 | + | wireBytes := 0 | |
| 1093 | + | for i, frag := range frags { | |
| 1094 | + | if job.reliable && !t.pendingExists(job.seq) { | |
| 1095 | + | return emitted, nil | |
| 1096 | + | } | |
| 1097 | + | raw, err := pdgram.MarshalFragment(frag) | |
| 1098 | + | if err != nil { | |
| 1099 | + | return emitted, err | |
| 1100 | + | } | |
| 1101 | + | pkt := t.codec.Seal(t.sendDir, t.nextCounter(), raw, nil) | |
| 1102 | + | wireBytes += len(pkt) | |
| 1103 | + | if job.reliable { | |
| 1104 | + | t.mu.Lock() | |
| 1105 | + | if pending := t.pending[job.seq]; pending != nil && pending.wireBytes < wireBytes { | |
| 1106 | + | pending.wireBytes = wireBytes | |
| 1107 | + | } | |
| 1108 | + | t.mu.Unlock() | |
| 1109 | + | } | |
| 1110 | + | if err := pacer.wait(t.done, len(pkt), limits); err != nil { | |
| 1111 | + | return emitted, err | |
| 1112 | + | } | |
| 1113 | + | if job.reliable && !t.pendingExists(job.seq) { | |
| 1114 | + | return emitted, nil | |
| 1115 | + | } | |
| 1116 | + | t.mu.Lock() | |
| 1117 | + | peer := t.peer | |
| 1118 | + | closed := t.closed | |
| 1119 | + | closeErr := t.closeErr | |
| 1120 | + | t.mu.Unlock() | |
| 1121 | + | if closed { | |
| 1122 | + | if closeErr != nil { | |
| 1123 | + | return emitted, closeErr | |
| 1124 | + | } | |
| 1125 | + | return emitted, net.ErrClosed | |
| 1126 | + | } | |
| 1127 | + | if peer == nil { | |
| 1128 | + | return emitted, errors.New("dgram: no peer") | |
| 1129 | + | } | |
| 1130 | + | if i == len(frags)-1 { | |
| 1131 | + | t.markPendingFinalWrite(job.seq) | |
| 1132 | + | } | |
| 1133 | + | if err := t.writePacket(pkt, peer); err != nil { | |
| 1134 | + | return emitted, err | |
| 1135 | + | } | |
| 1136 | + | if !emitted { | |
| 1137 | + | emitted = true | |
| 1138 | + | if onFirstWrite != nil { | |
| 1139 | + | onFirstWrite() | |
| 1140 | + | } | |
| 1141 | + | } | |
| 1142 | + | } | |
| 1143 | + | return emitted, nil | |
| 1144 | + | } | |
| 1145 | + | ||
| 1146 | + | func (t *Transport) runRetransmitBatch(pacer *bytePacer, batch []retransmitRecord) { | |
| 1147 | + | defer func() { | |
| 1148 | + | t.mu.Lock() | |
| 1149 | + | hook := t.afterDataJob | |
| 1150 | + | t.mu.Unlock() | |
| 1151 | + | if hook != nil { | |
| 1152 | + | hook() | |
| 1153 | + | } | |
| 1154 | + | }() | |
| 1155 | + | lossApplied := false | |
| 1156 | + | onFirstWrite := func() { | |
| 1157 | + | if lossApplied { | |
| 1158 | + | return | |
| 1159 | + | } | |
| 1160 | + | t.mu.Lock() | |
| 1161 | + | t.congestion.onLoss() | |
| 1162 | + | t.mu.Unlock() | |
| 1163 | + | lossApplied = true | |
| 1164 | + | } | |
| 1165 | + | for _, r := range batch { | |
| 1166 | + | job := dataSendJob{seq: r.seq, reliable: true, frame: r.f} | |
| 1167 | + | _, err := t.writeDataJob(pacer, job, onFirstWrite) | |
| 1168 | + | if err == nil { | |
| 1169 | + | t.markPendingReady(r.seq, true) | |
| 1170 | + | continue | |
| 1171 | + | } | |
| 1172 | + | t.markPendingReady(r.seq, true) | |
| 832 | 1173 | return | |
| 833 | 1174 | } | |
| 834 | - | close(t.sendWake) | |
| 835 | - | t.sendWake = make(chan struct{}) | |
| 836 | 1175 | } | |
| 837 | 1176 | ||
| 838 | - | func (t *Transport) resendLoop() { | |
| 839 | - | resend := t.clock.NewTimer(t.resendAfter) | |
| 840 | - | defer resend.Stop() | |
| 1177 | + | func (t *Transport) heartbeatLoop() { | |
| 841 | 1178 | heartbeat := t.clock.NewTimer(t.heartbeat) | |
| 842 | 1179 | defer heartbeat.Stop() | |
| 843 | 1180 | for { | |
| 844 | 1181 | select { | |
| 845 | - | case <-resend.C(): | |
| 846 | - | t.resendPending() | |
| 847 | - | t.checkSilence() | |
| 848 | - | t.checkSendStall() | |
| 849 | - | resend.Reset(t.resendAfter) | |
| 850 | 1182 | case <-heartbeat.C(): | |
| 851 | - | _ = t.sendProbe(recProbe, 0) | |
| 852 | - | t.checkSilence() | |
| 853 | - | t.checkSendStall() | |
| 1183 | + | t.queueControl(recProbe, 0) | |
| 854 | 1184 | heartbeat.Reset(t.heartbeat) | |
| 1185 | + | t.notifyTimerHook(func() func() { return t.afterHeartbeatTimer }) | |
| 855 | 1186 | case <-t.done: | |
| 856 | 1187 | return | |
| 857 | 1188 | } | |
| 858 | 1189 | } | |
| 859 | 1190 | } | |
| 860 | 1191 | ||
| 861 | - | func (t *Transport) resendPending() { | |
| 862 | - | now := t.clock.Now() | |
| 863 | - | var resend []struct { | |
| 864 | - | seq uint64 | |
| 865 | - | f ports.Frame | |
| 1192 | + | func (t *Transport) notifyTimerHook(get func() func()) { | |
| 1193 | + | t.mu.Lock() | |
| 1194 | + | hook := get() | |
| 1195 | + | t.mu.Unlock() | |
| 1196 | + | if hook != nil { | |
| 1197 | + | hook() | |
| 1198 | + | } | |
| 1199 | + | } | |
| 1200 | + | ||
| 1201 | + | func (t *Transport) queueRetransmits() { | |
| 1202 | + | // Do not mark records in-flight while the one-slot sender queue is full. | |
| 1203 | + | // There is one producer and the sender only consumes, so capacity cannot be | |
| 1204 | + | // lost after this check. | |
| 1205 | + | if len(t.retransmitWork) == cap(t.retransmitWork) { | |
| 1206 | + | return | |
| 866 | 1207 | } | |
| 1208 | + | ||
| 1209 | + | now := t.clock.Now() | |
| 867 | 1210 | t.mu.Lock() | |
| 1211 | + | resend, emitDiagnostic := t.selectRetransmitsLocked(now) | |
| 1212 | + | t.mu.Unlock() | |
| 1213 | + | ||
| 1214 | + | if len(resend) > 0 { | |
| 1215 | + | select { | |
| 1216 | + | case t.retransmitWork <- resend: | |
| 1217 | + | case <-t.done: | |
| 1218 | + | return | |
| 1219 | + | } | |
| 1220 | + | } | |
| 1221 | + | if emitDiagnostic { | |
| 1222 | + | t.emitDiagnostic() | |
| 1223 | + | } | |
| 1224 | + | } | |
| 1225 | + | ||
| 1226 | + | // selectRetransmitsLocked deterministically reserves one bounded retransmit | |
| 1227 | + | // turn. The caller must hold t.mu and performs all pacing and writes after | |
| 1228 | + | // releasing it. | |
| 1229 | + | func (t *Transport) selectRetransmitsLocked(now time.Time) ([]retransmitRecord, bool) { | |
| 1230 | + | resend := make([]retransmitRecord, 0, t.maxResendPerTick) | |
| 868 | 1231 | seqs := make([]uint64, 0, len(t.pending)) | |
| 869 | 1232 | for seq := range t.pending { | |
| 870 | 1233 | seqs = append(seqs, seq) | |
| 871 | 1234 | } | |
| 872 | 1235 | slices.Sort(seqs) | |
| 873 | 1236 | limit := t.maxResendPerTick | |
| 1237 | + | emitDiagnostic := false | |
| 874 | 1238 | for _, seq := range seqs { | |
| 875 | 1239 | if limit <= 0 { | |
| 876 | 1240 | break | |
| @@ -882,24 +1246,31 @@ func (t *Transport) resendPending() { | |||
| 882 | 1246 | p.initialInFlight = true | |
| 883 | 1247 | p.attempts++ | |
| 884 | 1248 | p.retransmitted = true | |
| 885 | - | resend = append(resend, struct { | |
| 886 | - | seq uint64 | |
| 887 | - | f ports.Frame | |
| 888 | - | }{seq, p.frame}) | |
| 1249 | + | t.retransmits++ | |
| 1250 | + | emitDiagnostic = emitDiagnostic || t.retransmits%64 == 0 | |
| 1251 | + | resend = append(resend, retransmitRecord{seq: seq, f: p.frame}) | |
| 889 | 1252 | limit-- | |
| 890 | 1253 | } | |
| 1254 | + | return resend, emitDiagnostic | |
| 1255 | + | } | |
| 1256 | + | ||
| 1257 | + | // resendPending remains a test hook for retransmission selection. Constructed | |
| 1258 | + | // transports transfer its batch to the sole data sender; small unit fixtures | |
| 1259 | + | // without transport goroutines execute the selected writes inline. | |
| 1260 | + | func (t *Transport) resendPending() { | |
| 1261 | + | now := t.clock.Now() | |
| 1262 | + | t.mu.Lock() | |
| 1263 | + | resend, _ := t.selectRetransmitsLocked(now) | |
| 891 | 1264 | t.mu.Unlock() | |
| 892 | - | for _, r := range resend { | |
| 893 | - | if err := t.sendData(r.seq, true, r.f); err == nil { | |
| 894 | - | t.markPendingReady(r.seq, true) | |
| 895 | - | } else { | |
| 896 | - | t.mu.Lock() | |
| 897 | - | if p := t.pending[r.seq]; p != nil { | |
| 898 | - | p.initialInFlight = false | |
| 899 | - | } | |
| 900 | - | t.mu.Unlock() | |
| 1265 | + | if t.retransmitWork != nil { | |
| 1266 | + | select { | |
| 1267 | + | case t.retransmitWork <- resend: | |
| 1268 | + | case <-t.done: | |
| 901 | 1269 | } | |
| 1270 | + | return | |
| 902 | 1271 | } | |
| 1272 | + | pacer := bytePacer{clk: t.clock} | |
| 1273 | + | t.runRetransmitBatch(&pacer, resend) | |
| 903 | 1274 | } | |
| 904 | 1275 | ||
| 905 | 1276 | func (t *Transport) resendDelayLocked(p *pending) time.Duration { | |
| @@ -943,112 +1314,134 @@ func (t *Transport) updateRTTLocked(sample time.Duration) { | |||
| 943 | 1314 | rto = t.maxResendAfter | |
| 944 | 1315 | } | |
| 945 | 1316 | t.rto = rto | |
| 1317 | + | t.congestion.onRTT(t.srtt) | |
| 946 | 1318 | } | |
| 947 | 1319 | ||
| 948 | - | func (t *Transport) hopPacketConnOnce() { | |
| 1320 | + | func (t *Transport) hopPacketConnOnce(generation uint64) { | |
| 949 | 1321 | t.mu.Lock() | |
| 950 | - | if t.rebind == nil || t.hoppedOffline || t.closed { | |
| 1322 | + | if t.rebind == nil || t.hoppedOffline || t.closed || t.linkState != ports.LinkStateProbing || t.health.generation != generation { | |
| 951 | 1323 | t.mu.Unlock() | |
| 952 | 1324 | return | |
| 953 | 1325 | } | |
| 954 | 1326 | old := t.pc | |
| 955 | 1327 | rebind := t.rebind | |
| 956 | 1328 | t.hoppedOffline = true | |
| 1329 | + | t.hopGeneration = generation | |
| 957 | 1330 | t.mu.Unlock() | |
| 958 | 1331 | pc, err := rebind(old) | |
| 1332 | + | ||
| 1333 | + | // Serialize the swap with writes. writePacket takes writeMu before mu, so | |
| 1334 | + | // preserve that order here while retiring old. | |
| 1335 | + | t.writeMu.Lock() | |
| 1336 | + | defer t.writeMu.Unlock() | |
| 1337 | + | t.mu.Lock() | |
| 959 | 1338 | if err != nil { | |
| 960 | - | t.mu.Lock() | |
| 961 | - | if t.pc == old { | |
| 962 | - | t.hoppedOffline = false | |
| 963 | - | } | |
| 1339 | + | t.rollbackHopLocked(generation) | |
| 964 | 1340 | t.mu.Unlock() | |
| 965 | 1341 | return | |
| 966 | 1342 | } | |
| 967 | - | t.mu.Lock() | |
| 968 | - | if t.closed || t.pc != old { | |
| 1343 | + | if t.closed || t.pc != old || t.health.generation != generation || t.linkState != ports.LinkStateProbing { | |
| 1344 | + | t.rollbackHopLocked(generation) | |
| 969 | 1345 | t.mu.Unlock() | |
| 970 | 1346 | _ = pc.Close() | |
| 971 | 1347 | return | |
| 972 | 1348 | } | |
| 973 | 1349 | t.pc = pc | |
| 1350 | + | t.writeDeadlines = newWriteDeadlineState(pc) | |
| 974 | 1351 | t.mu.Unlock() | |
| 975 | 1352 | go t.readLoop(pc) | |
| 976 | 1353 | _ = old.Close() | |
| 977 | 1354 | } | |
| 978 | 1355 | ||
| 979 | - | func (t *Transport) checkSilence() { | |
| 980 | - | t.mu.Lock() | |
| 981 | - | last := t.lastHeard | |
| 982 | - | closed := t.closed | |
| 983 | - | degradedAfter := t.degradedAfter | |
| 984 | - | probeAfter := t.probeAfter | |
| 985 | - | offlineAfter := t.offlineAfter | |
| 986 | - | deadAfter := t.deadAfter | |
| 987 | - | now := t.clock.Now() | |
| 988 | - | t.mu.Unlock() | |
| 989 | - | if closed { | |
| 990 | - | return | |
| 1356 | + | func (t *Transport) rollbackHopLocked(generation uint64) { | |
| 1357 | + | if t.hoppedOffline && t.hopGeneration == generation { | |
| 1358 | + | t.hoppedOffline = false | |
| 1359 | + | t.hopGeneration = 0 | |
| 991 | 1360 | } | |
| 992 | - | silentFor := now.Sub(last) | |
| 993 | - | switch { | |
| 994 | - | case deadAfter > 0 && silentFor >= deadAfter: | |
| 995 | - | t.setLinkState(ports.LinkStateDead, ErrLinkDead) | |
| 996 | - | t.closeWithError(ErrLinkDead) | |
| 997 | - | case offlineAfter > 0 && silentFor >= offlineAfter: | |
| 998 | - | t.setLinkState(ports.LinkStateOffline, nil) | |
| 999 | - | case probeAfter > 0 && silentFor >= probeAfter: | |
| 1000 | - | t.setLinkState(ports.LinkStateProbing, nil) | |
| 1001 | - | t.hopPacketConnOnce() | |
| 1002 | - | case degradedAfter > 0 && silentFor >= degradedAfter: | |
| 1003 | - | t.setLinkState(ports.LinkStateDegraded, nil) | |
| 1004 | - | } | |
| 1005 | - | } | |
| 1006 | - | ||
| 1007 | - | // checkSendStall declares the link dead when the oldest unacked reliable frame | |
| 1008 | - | // has gone unacknowledged for deadAfter despite at least one retransmit. The | |
| 1009 | - | // receive-silence check in checkSilence misses one-way-dead links whose receive | |
| 1010 | - | // side stays fresh from peer heartbeats; this catches them at the layer that | |
| 1011 | - | // owns death. enqueued is set once when a frame is accepted and never refreshed, | |
| 1012 | - | // so acks (which delete pending entries) keep the oldest age bounded on a | |
| 1013 | - | // progressing link — only a frame that is never acked crosses the threshold. | |
| 1014 | - | func (t *Transport) checkSendStall() { | |
| 1015 | - | t.mu.Lock() | |
| 1016 | - | if t.closed || t.deadAfter <= 0 { | |
| 1361 | + | } | |
| 1362 | + | ||
| 1363 | + | func (t *Transport) checkSilence() { | |
| 1364 | + | runHook := true | |
| 1365 | + | for { | |
| 1366 | + | t.mu.Lock() | |
| 1367 | + | if t.closed { | |
| 1368 | + | t.mu.Unlock() | |
| 1369 | + | return | |
| 1370 | + | } | |
| 1371 | + | now := t.clock.Now() | |
| 1372 | + | generation := t.health.generation | |
| 1373 | + | state, hop, dead := t.health.decide(now, len(t.pending) > 0, t.degradedAfter, t.probeAfter, t.offlineAfter, t.deadAfter) | |
| 1374 | + | afterDecision := t.afterHealthDecision | |
| 1017 | 1375 | t.mu.Unlock() | |
| 1018 | - | return | |
| 1019 | - | } | |
| 1020 | - | deadAfter := t.deadAfter | |
| 1021 | - | now := t.clock.Now() | |
| 1022 | - | var oldest time.Time | |
| 1023 | - | var attempts int | |
| 1024 | - | found := false | |
| 1025 | - | for _, p := range t.pending { | |
| 1026 | - | if p == nil || p.enqueued.IsZero() { | |
| 1376 | + | ||
| 1377 | + | if runHook { | |
| 1378 | + | runHook = false | |
| 1379 | + | if afterDecision != nil { | |
| 1380 | + | afterDecision() | |
| 1381 | + | } | |
| 1382 | + | } | |
| 1383 | + | ||
| 1384 | + | t.mu.Lock() | |
| 1385 | + | if t.closed { | |
| 1386 | + | t.mu.Unlock() | |
| 1387 | + | return | |
| 1388 | + | } | |
| 1389 | + | if t.health.generation != generation { | |
| 1390 | + | t.mu.Unlock() | |
| 1027 | 1391 | continue | |
| 1028 | 1392 | } | |
| 1029 | - | if !found || p.enqueued.Before(oldest) { | |
| 1030 | - | oldest = p.enqueued | |
| 1031 | - | attempts = p.attempts | |
| 1032 | - | found = true | |
| 1393 | + | changed, stateGeneration := t.setLinkStateLocked(state) | |
| 1394 | + | afterStateCommit := t.afterLinkStateCommit | |
| 1395 | + | closed := false | |
| 1396 | + | if dead { | |
| 1397 | + | closed = t.closeWithErrorLocked(ErrLinkDead) | |
| 1398 | + | } | |
| 1399 | + | t.mu.Unlock() | |
| 1400 | + | ||
| 1401 | + | if changed { | |
| 1402 | + | if afterStateCommit != nil { | |
| 1403 | + | afterStateCommit(state) | |
| 1404 | + | } | |
| 1405 | + | var linkErr error | |
| 1406 | + | if dead { | |
| 1407 | + | linkErr = ErrLinkDead | |
| 1408 | + | } | |
| 1409 | + | t.publishLinkState(state, now, linkErr, stateGeneration) | |
| 1410 | + | } | |
| 1411 | + | if closed { | |
| 1412 | + | t.broadcastClosed() | |
| 1413 | + | return | |
| 1414 | + | } | |
| 1415 | + | if hop { | |
| 1416 | + | t.hopPacketConnOnce(generation) | |
| 1033 | 1417 | } | |
| 1034 | - | } | |
| 1035 | - | t.mu.Unlock() | |
| 1036 | - | if !found || attempts < 1 || now.Sub(oldest) < deadAfter { | |
| 1037 | 1418 | return | |
| 1038 | 1419 | } | |
| 1039 | - | t.setLinkState(ports.LinkStateDead, ErrLinkDead) | |
| 1040 | - | t.closeWithError(ErrLinkDead) | |
| 1041 | 1420 | } | |
| 1042 | 1421 | ||
| 1043 | 1422 | func (t *Transport) setLinkState(state ports.LinkState, err error) { | |
| 1044 | 1423 | now := t.clock.Now() | |
| 1045 | 1424 | t.mu.Lock() | |
| 1425 | + | changed, stateGeneration := t.setLinkStateLocked(state) | |
| 1426 | + | afterStateCommit := t.afterLinkStateCommit | |
| 1427 | + | t.mu.Unlock() | |
| 1428 | + | if changed { | |
| 1429 | + | if afterStateCommit != nil { | |
| 1430 | + | afterStateCommit(state) | |
| 1431 | + | } | |
| 1432 | + | t.publishLinkState(state, now, err, stateGeneration) | |
| 1433 | + | } | |
| 1434 | + | } | |
| 1435 | + | ||
| 1436 | + | func (t *Transport) setLinkStateLocked(state ports.LinkState) (bool, uint64) { | |
| 1046 | 1437 | if t.linkState == state { | |
| 1047 | - | t.mu.Unlock() | |
| 1048 | - | return | |
| 1438 | + | return false, t.linkStateGeneration | |
| 1049 | 1439 | } | |
| 1050 | 1440 | t.linkState = state | |
| 1441 | + | t.linkStateGeneration++ | |
| 1051 | 1442 | if state == ports.LinkStateConnected { | |
| 1443 | + | t.hoppedOffline = false | |
| 1444 | + | t.hopGeneration = 0 | |
| 1052 | 1445 | for _, p := range t.pending { | |
| 1053 | 1446 | if p != nil { | |
| 1054 | 1447 | p.attempts = 0 | |
| @@ -1056,19 +1449,98 @@ func (t *Transport) setLinkState(state ports.LinkState, err error) { | |||
| 1056 | 1449 | } | |
| 1057 | 1450 | } | |
| 1058 | 1451 | t.notifySendWaitersLocked() | |
| 1059 | - | t.mu.Unlock() | |
| 1452 | + | return true, t.linkStateGeneration | |
| 1453 | + | } | |
| 1454 | + | ||
| 1455 | + | func (t *Transport) publishLinkState(state ports.LinkState, now time.Time, err error, generation uint64) { | |
| 1456 | + | t.linkEventMu.Lock() | |
| 1457 | + | defer t.linkEventMu.Unlock() | |
| 1458 | + | t.mu.Lock() | |
| 1459 | + | if t.linkState != state || t.linkStateGeneration != generation { | |
| 1460 | + | t.mu.Unlock() | |
| 1461 | + | return | |
| 1462 | + | } | |
| 1060 | 1463 | event := ports.LinkEvent{State: state, At: now, Err: err} | |
| 1061 | 1464 | select { | |
| 1062 | 1465 | case t.linkEvents <- event: | |
| 1063 | 1466 | default: | |
| 1064 | 1467 | } | |
| 1468 | + | t.mu.Unlock() | |
| 1469 | + | t.emitDiagnostic() | |
| 1470 | + | } | |
| 1471 | + | ||
| 1472 | + | func (t *Transport) diagnosticLoop() { | |
| 1473 | + | for { | |
| 1474 | + | select { | |
| 1475 | + | case d := <-t.diagnosticCh: | |
| 1476 | + | t.observe(d) | |
| 1477 | + | case <-t.done: | |
| 1478 | + | return | |
| 1479 | + | } | |
| 1480 | + | } | |
| 1481 | + | } | |
| 1482 | + | ||
| 1483 | + | func (t *Transport) emitDiagnostic() { | |
| 1484 | + | if t.diagnosticCh == nil { | |
| 1485 | + | return | |
| 1486 | + | } | |
| 1487 | + | d := t.diagnosticSnapshot() | |
| 1488 | + | select { | |
| 1489 | + | case t.diagnosticCh <- d: | |
| 1490 | + | default: | |
| 1491 | + | } | |
| 1492 | + | } | |
| 1493 | + | ||
| 1494 | + | func (t *Transport) diagnosticSnapshot() Diagnostic { | |
| 1495 | + | t.mu.Lock() | |
| 1496 | + | now := t.clock.Now() | |
| 1497 | + | lastAuthenticatedPacket := t.lastAuthenticatedPacket | |
| 1498 | + | lastCompleteRecord := t.lastCompleteRecord | |
| 1499 | + | lastACKProgress := t.lastACKProgress | |
| 1500 | + | pendingBytes := 0 | |
| 1501 | + | for _, p := range t.pending { | |
| 1502 | + | if p != nil { | |
| 1503 | + | pendingBytes += len(p.frame.Payload) | |
| 1504 | + | } | |
| 1505 | + | } | |
| 1506 | + | state := t.linkState | |
| 1507 | + | pendingRecords := len(t.pending) | |
| 1508 | + | retransmits := t.retransmits | |
| 1509 | + | reassemblyInflight := t.reassemblyInflight | |
| 1510 | + | t.mu.Unlock() | |
| 1511 | + | ||
| 1512 | + | return Diagnostic{ | |
| 1513 | + | At: now, | |
| 1514 | + | State: state, | |
| 1515 | + | SinceAuthenticatedPacket: diagnosticAge(now, lastAuthenticatedPacket), | |
| 1516 | + | SinceCompleteRecord: diagnosticAge(now, lastCompleteRecord), | |
| 1517 | + | SinceACKProgress: diagnosticAge(now, lastACKProgress), | |
| 1518 | + | PendingRecords: pendingRecords, | |
| 1519 | + | PendingBytes: pendingBytes, | |
| 1520 | + | Retransmits: retransmits, | |
| 1521 | + | ReassemblyInflight: reassemblyInflight, | |
| 1522 | + | } | |
| 1523 | + | } | |
| 1524 | + | ||
| 1525 | + | func diagnosticAge(now, then time.Time) time.Duration { | |
| 1526 | + | if then.After(now) { | |
| 1527 | + | return 0 | |
| 1528 | + | } | |
| 1529 | + | return now.Sub(then) | |
| 1065 | 1530 | } | |
| 1066 | 1531 | ||
| 1067 | 1532 | func (t *Transport) closeWithError(err error) { | |
| 1068 | 1533 | t.mu.Lock() | |
| 1534 | + | closed := t.closeWithErrorLocked(err) | |
| 1535 | + | t.mu.Unlock() | |
| 1536 | + | if closed { | |
| 1537 | + | t.broadcastClosed() | |
| 1538 | + | } | |
| 1539 | + | } | |
| 1540 | + | ||
| 1541 | + | func (t *Transport) closeWithErrorLocked(err error) bool { | |
| 1069 | 1542 | if t.closed { | |
| 1070 | - | t.mu.Unlock() | |
| 1071 | - | return | |
| 1543 | + | return false | |
| 1072 | 1544 | } | |
| 1073 | 1545 | t.closed = true | |
| 1074 | 1546 | t.closeErr = err | |
| @@ -1082,7 +1554,10 @@ func (t *Transport) closeWithError(err error) { | |||
| 1082 | 1554 | t.probeWait = make(map[uint64]chan struct{}) | |
| 1083 | 1555 | close(t.done) | |
| 1084 | 1556 | t.notifySendWaitersLocked() | |
| 1085 | - | t.mu.Unlock() | |
| 1557 | + | return true | |
| 1558 | + | } | |
| 1559 | + | ||
| 1560 | + | func (t *Transport) broadcastClosed() { | |
| 1086 | 1561 | t.deliverMu.Lock() | |
| 1087 | 1562 | t.deliverCond.Broadcast() | |
| 1088 | 1563 | t.deliverMu.Unlock() | |
| @@ -1108,3 +1583,6 @@ func decodeData(b []byte) (uint64, bool, ports.Frame, bool) { | |||
| 1108 | 1583 | } | |
| 1109 | 1584 | return binary.BigEndian.Uint64(b[1:9]), b[9] == 1, ports.Frame{Type: ports.MsgType(b[10]), Payload: append([]byte(nil), b[dataRecordHeaderSize:]...)}, true | |
| 1110 | 1585 | } | |
| 1586 | + | ||
| 1587 | + | // DatagramTransport marks Transport as a UDP-style datagram transport. | |
| 1588 | + | func (*Transport) DatagramTransport() {} | |
M internal/adapters/dgram/transport_test.go +841 -235
| @@ -5,6 +5,7 @@ import ( | |||
| 5 | 5 | "context" | |
| 6 | 6 | "encoding/binary" | |
| 7 | 7 | "errors" | |
| 8 | + | "fmt" | |
| 8 | 9 | "net" | |
| 9 | 10 | "sync" | |
| 10 | 11 | "sync/atomic" | |
| @@ -16,79 +17,6 @@ import ( | |||
| 16 | 17 | "github.com/stretchr/testify/require" | |
| 17 | 18 | ) | |
| 18 | 19 | ||
| 19 | - | type testAddr string | |
| 20 | - | ||
| 21 | - | func (a testAddr) Network() string { return "mem" } | |
| 22 | - | func (a testAddr) String() string { return string(a) } | |
| 23 | - | ||
| 24 | - | type packet struct { | |
| 25 | - | b []byte | |
| 26 | - | addr net.Addr | |
| 27 | - | } | |
| 28 | - | type fakePC struct { | |
| 29 | - | addr net.Addr | |
| 30 | - | mu sync.Mutex | |
| 31 | - | closed bool | |
| 32 | - | in chan packet | |
| 33 | - | peers map[string]*fakePC | |
| 34 | - | drop func([]byte, net.Addr) bool | |
| 35 | - | afterWrite func() | |
| 36 | - | } | |
| 37 | - | ||
| 38 | - | func newPair() (*fakePC, *fakePC) { | |
| 39 | - | a := &fakePC{addr: testAddr("a"), in: make(chan packet, 100), peers: map[string]*fakePC{}} | |
| 40 | - | b := &fakePC{addr: testAddr("b"), in: make(chan packet, 100), peers: map[string]*fakePC{}} | |
| 41 | - | a.peers["b"] = b | |
| 42 | - | b.peers["a"] = a | |
| 43 | - | return a, b | |
| 44 | - | } | |
| 45 | - | func (p *fakePC) ReadFrom(b []byte) (int, net.Addr, error) { | |
| 46 | - | q, ok := <-p.in | |
| 47 | - | if !ok { | |
| 48 | - | return 0, nil, errors.New("closed") | |
| 49 | - | } | |
| 50 | - | copy(b, q.b) | |
| 51 | - | return len(q.b), q.addr, nil | |
| 52 | - | } | |
| 53 | - | func (p *fakePC) WriteTo(b []byte, addr net.Addr) (int, error) { | |
| 54 | - | p.mu.Lock() | |
| 55 | - | closed := p.closed | |
| 56 | - | p.mu.Unlock() | |
| 57 | - | if closed { | |
| 58 | - | return 0, errors.New("closed") | |
| 59 | - | } | |
| 60 | - | if p.drop != nil && p.drop(b, addr) { | |
| 61 | - | return len(b), nil | |
| 62 | - | } | |
| 63 | - | peer := p.peers[addr.String()] | |
| 64 | - | peer.mu.Lock() | |
| 65 | - | if peer.closed { | |
| 66 | - | peer.mu.Unlock() | |
| 67 | - | return 0, errors.New("peer closed") | |
| 68 | - | } | |
| 69 | - | peer.in <- packet{append([]byte(nil), b...), p.addr} | |
| 70 | - | peer.mu.Unlock() | |
| 71 | - | if p.afterWrite != nil { | |
| 72 | - | p.afterWrite() | |
| 73 | - | } | |
| 74 | - | return len(b), nil | |
| 75 | - | } | |
| 76 | - | func (p *fakePC) Close() error { | |
| 77 | - | p.mu.Lock() | |
| 78 | - | defer p.mu.Unlock() | |
| 79 | - | if !p.closed { | |
| 80 | - | p.closed = true | |
| 81 | - | close(p.in) | |
| 82 | - | } | |
| 83 | - | return nil | |
| 84 | - | } | |
| 85 | - | func (p *fakePC) LocalAddr() net.Addr { return p.addr } | |
| 86 | - | func (p *fakePC) SetDeadline(time.Time) error { return nil } | |
| 87 | - | func (p *fakePC) SetReadDeadline(time.Time) error { return nil } | |
| 88 | - | func (p *fakePC) SetWriteDeadline(time.Time) error { return nil } | |
| 89 | - | ||
| 90 | - | func key() []byte { return bytes.Repeat([]byte{1}, pdgram.KeySize) } | |
| 91 | - | ||
| 92 | 20 | type timeoutErr struct{} | |
| 93 | 21 | ||
| 94 | 22 | func (timeoutErr) Error() string { return "i/o timeout" } | |
| @@ -175,13 +103,18 @@ func (c fixedClock) NewTimer(d time.Duration) ports.Timer { | |||
| 175 | 103 | } | |
| 176 | 104 | ||
| 177 | 105 | type manualClock struct { | |
| 178 | - | mu sync.Mutex | |
| 179 | - | now time.Time | |
| 180 | - | timers map[*manualTimer]struct{} | |
| 106 | + | mu sync.Mutex | |
| 107 | + | now time.Time | |
| 108 | + | timers map[*manualTimer]struct{} | |
| 109 | + | timerCreated chan struct{} | |
| 181 | 110 | } | |
| 182 | 111 | ||
| 183 | 112 | func newManualClock(now time.Time) *manualClock { | |
| 184 | - | return &manualClock{now: now, timers: make(map[*manualTimer]struct{})} | |
| 113 | + | return &manualClock{ | |
| 114 | + | now: now, | |
| 115 | + | timers: make(map[*manualTimer]struct{}), | |
| 116 | + | timerCreated: make(chan struct{}, 64), | |
| 117 | + | } | |
| 185 | 118 | } | |
| 186 | 119 | ||
| 187 | 120 | func (c *manualClock) Now() time.Time { | |
| @@ -195,6 +128,10 @@ func (c *manualClock) NewTimer(d time.Duration) ports.Timer { | |||
| 195 | 128 | tm := &manualTimer{clock: c, c: make(chan time.Time, 1), deadline: c.now.Add(d), active: true} | |
| 196 | 129 | c.timers[tm] = struct{}{} | |
| 197 | 130 | c.mu.Unlock() | |
| 131 | + | select { | |
| 132 | + | case c.timerCreated <- struct{}{}: | |
| 133 | + | default: | |
| 134 | + | } | |
| 198 | 135 | return tm | |
| 199 | 136 | } | |
| 200 | 137 | ||
| @@ -276,6 +213,220 @@ func waitForManualTimers(t *testing.T, clk *manualClock, n int) { | |||
| 276 | 213 | t.Fatalf("manual timers=%d, want at least %d", active, n) | |
| 277 | 214 | } | |
| 278 | 215 | ||
| 216 | + | func TestCongestionControllerBoundsAndIntegerGrowth(t *testing.T) { | |
| 217 | + | const mtu = 1200 | |
| 218 | + | c := newCongestionController(mtu) | |
| 219 | + | if got, want := c.cwndBytes, initialCongestionPackets*mtu; got != want { | |
| 220 | + | t.Fatalf("initial cwnd=%d, want %d", got, want) | |
| 221 | + | } | |
| 222 | + | if got, want := c.burstBytes(), dataBurstPackets*mtu; got != want { | |
| 223 | + | t.Fatalf("initial burst=%d, want %d", got, want) | |
| 224 | + | } | |
| 225 | + | if got, want := c.bytesPerSecond(), initialCongestionPackets*mtu*int(time.Second/initialPacingRTT); got != want { | |
| 226 | + | t.Fatalf("initial pacing rate=%d, want %d", got, want) | |
| 227 | + | } | |
| 228 | + | ||
| 229 | + | c.onLoss() | |
| 230 | + | if got, want := c.cwndBytes, initialCongestionPackets*mtu/2; got != want { | |
| 231 | + | t.Fatalf("cwnd after first loss=%d, want %d", got, want) | |
| 232 | + | } | |
| 233 | + | c.cwndBytes = 3 * mtu | |
| 234 | + | c.onLoss() | |
| 235 | + | if got, want := c.cwndBytes, minimumCongestionPackets*mtu; got != want { | |
| 236 | + | t.Fatalf("minimum cwnd=%d, want %d", got, want) | |
| 237 | + | } | |
| 238 | + | ||
| 239 | + | c.cwndBytes = 10 * mtu | |
| 240 | + | c.onACK(1) | |
| 241 | + | if got, want := c.cwndBytes, 10*mtu+1; got != want { | |
| 242 | + | t.Fatalf("minimum additive increase cwnd=%d, want %d", got, want) | |
| 243 | + | } | |
| 244 | + | c.onACK(10 * mtu) | |
| 245 | + | if got, want := c.cwndBytes, 10*mtu+1+(mtu*10*mtu)/(10*mtu+1); got != want { | |
| 246 | + | t.Fatalf("integer additive increase cwnd=%d, want %d", got, want) | |
| 247 | + | } | |
| 248 | + | c.cwndBytes = maximumCongestionBytes | |
| 249 | + | c.onACK(maximumCongestionBytes) | |
| 250 | + | if got := c.cwndBytes; got != maximumCongestionBytes { | |
| 251 | + | t.Fatalf("maximum cwnd=%d, want %d", got, maximumCongestionBytes) | |
| 252 | + | } | |
| 253 | + | ||
| 254 | + | c.onRTT(500 * time.Microsecond) | |
| 255 | + | if got, want := c.bytesPerSecond(), maximumCongestionBytes*1000; got != want { | |
| 256 | + | t.Fatalf("sub-millisecond pacing rate=%d, want %d", got, want) | |
| 257 | + | } | |
| 258 | + | } | |
| 259 | + | ||
| 260 | + | func TestBytePacerUsesBurstThenFakeClockRate(t *testing.T) { | |
| 261 | + | const mtu = 1200 | |
| 262 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 263 | + | c := newCongestionController(mtu) | |
| 264 | + | p := bytePacer{clk: clk} | |
| 265 | + | done := make(chan struct{}) | |
| 266 | + | limits := func() (int, int) { return c.bytesPerSecond(), c.burstBytes() } | |
| 267 | + | ||
| 268 | + | for range dataBurstPackets { | |
| 269 | + | if err := p.wait(done, mtu, limits); err != nil { | |
| 270 | + | t.Fatal(err) | |
| 271 | + | } | |
| 272 | + | } | |
| 273 | + | wait := make(chan error, 1) | |
| 274 | + | go func() { wait <- p.wait(done, mtu, limits) }() | |
| 275 | + | awaitSignal(t, clk.timerCreated, "byte pacer timer") | |
| 276 | + | clk.advance(24 * time.Millisecond) | |
| 277 | + | select { | |
| 278 | + | case err := <-wait: | |
| 279 | + | t.Fatalf("third MTU released early: %v", err) | |
| 280 | + | default: | |
| 281 | + | } | |
| 282 | + | clk.advance(time.Millisecond) | |
| 283 | + | if err := awaitResult(t, wait, "third paced MTU"); err != nil { | |
| 284 | + | t.Fatal(err) | |
| 285 | + | } | |
| 286 | + | ||
| 287 | + | c.onLoss() | |
| 288 | + | next := make(chan error, 1) | |
| 289 | + | go func() { next <- p.wait(done, mtu, limits) }() | |
| 290 | + | awaitSignal(t, clk.timerCreated, "reduced-cwnd byte pacer timer") | |
| 291 | + | clk.advance(49 * time.Millisecond) | |
| 292 | + | select { | |
| 293 | + | case err := <-next: | |
| 294 | + | t.Fatalf("reduced cwnd did not lengthen wait: %v", err) | |
| 295 | + | default: | |
| 296 | + | } | |
| 297 | + | clk.advance(time.Millisecond) | |
| 298 | + | if err := awaitResult(t, next, "reduced-cwnd paced MTU"); err != nil { | |
| 299 | + | t.Fatal(err) | |
| 300 | + | } | |
| 301 | + | } | |
| 302 | + | ||
| 303 | + | func TestBytePacerMaximumRateRefillsAfterMultiSecondFakeClockJump(t *testing.T) { | |
| 304 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 305 | + | p := bytePacer{clk: clk} | |
| 306 | + | done := make(chan struct{}) | |
| 307 | + | burst := int(^uint(0) >> 1) | |
| 308 | + | limits := func() (int, int) { return burst, burst } | |
| 309 | + | ||
| 310 | + | if err := p.wait(done, burst, limits); err != nil { | |
| 311 | + | t.Fatal(err) | |
| 312 | + | } | |
| 313 | + | clk.advance(3 * time.Second) | |
| 314 | + | if err := p.wait(done, burst, limits); err != nil { | |
| 315 | + | t.Fatal(err) | |
| 316 | + | } | |
| 317 | + | ||
| 318 | + | wait := make(chan error, 1) | |
| 319 | + | go func() { wait <- p.wait(done, burst, limits) }() | |
| 320 | + | awaitSignal(t, clk.timerCreated, "maximum-rate byte pacer timer") | |
| 321 | + | clk.advance(time.Second - time.Nanosecond) | |
| 322 | + | select { | |
| 323 | + | case err := <-wait: | |
| 324 | + | t.Fatalf("maximum-rate burst released early: %v", err) | |
| 325 | + | default: | |
| 326 | + | } | |
| 327 | + | clk.advance(time.Nanosecond) | |
| 328 | + | if err := awaitResult(t, wait, "maximum-rate burst"); err != nil { | |
| 329 | + | t.Fatal(err) | |
| 330 | + | } | |
| 331 | + | } | |
| 332 | + | ||
| 333 | + | func TestBytePacerCancellation(t *testing.T) { | |
| 334 | + | const mtu = 1200 | |
| 335 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 336 | + | p := bytePacer{clk: clk} | |
| 337 | + | done := make(chan struct{}) | |
| 338 | + | limits := func() (int, int) { return mtu, mtu } | |
| 339 | + | if err := p.wait(done, mtu, limits); err != nil { | |
| 340 | + | t.Fatal(err) | |
| 341 | + | } | |
| 342 | + | wait := make(chan error, 1) | |
| 343 | + | go func() { wait <- p.wait(done, mtu, limits) }() | |
| 344 | + | awaitSignal(t, clk.timerCreated, "cancelled byte pacer timer") | |
| 345 | + | close(done) | |
| 346 | + | if err := awaitResult(t, wait, "byte pacer cancellation"); !errors.Is(err, errPacerClosed) { | |
| 347 | + | t.Fatalf("cancellation error=%v, want %v", err, errPacerClosed) | |
| 348 | + | } | |
| 349 | + | } | |
| 350 | + | ||
| 351 | + | func BenchmarkCongestionControllerACK(b *testing.B) { | |
| 352 | + | c := newCongestionController(defaultMTU) | |
| 353 | + | b.ReportAllocs() | |
| 354 | + | for b.Loop() { | |
| 355 | + | c.onACK(defaultMTU) | |
| 356 | + | if c.cwndBytes == c.maxBytes { | |
| 357 | + | c = newCongestionController(defaultMTU) | |
| 358 | + | } | |
| 359 | + | } | |
| 360 | + | } | |
| 361 | + | ||
| 362 | + | func TestPendingWireBytesMatchSealedDatagrams(t *testing.T) { | |
| 363 | + | const mtu = 128 | |
| 364 | + | aPC, bPC := newPair() | |
| 365 | + | a, err := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ | |
| 366 | + | MTU: mtu, ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 367 | + | }) | |
| 368 | + | if err != nil { | |
| 369 | + | t.Fatal(err) | |
| 370 | + | } | |
| 371 | + | defer func() { _ = a.Close() }() | |
| 372 | + | ||
| 373 | + | maxFragmentData := mtu - pdgram.HeaderSize - a.codec.Overhead() - 16 | |
| 374 | + | frame := ports.Frame{Type: ports.MsgOutput, Payload: make([]byte, maxFragmentData-dataRecordHeaderSize+1)} | |
| 375 | + | if err := a.Send(frame); err != nil { | |
| 376 | + | t.Fatal(err) | |
| 377 | + | } | |
| 378 | + | ||
| 379 | + | want := 0 | |
| 380 | + | for len(bPC.in) > 0 { | |
| 381 | + | want += len((<-bPC.in).b) | |
| 382 | + | } | |
| 383 | + | a.mu.Lock() | |
| 384 | + | got := a.pending[1].wireBytes | |
| 385 | + | initialCwnd := a.congestion.cwndBytes | |
| 386 | + | a.mu.Unlock() | |
| 387 | + | if got != want { | |
| 388 | + | t.Fatalf("pending wire bytes=%d, want sealed datagram bytes=%d", got, want) | |
| 389 | + | } | |
| 390 | + | ||
| 391 | + | ack := make([]byte, 9) | |
| 392 | + | ack[0] = recAck | |
| 393 | + | binary.BigEndian.PutUint64(ack[1:], 1) | |
| 394 | + | a.handleRecord(ack) | |
| 395 | + | a.mu.Lock() | |
| 396 | + | gotCwnd := a.congestion.cwndBytes | |
| 397 | + | a.mu.Unlock() | |
| 398 | + | wantCwnd := initialCwnd + max(1, mtu*want/initialCwnd) | |
| 399 | + | if gotCwnd != wantCwnd { | |
| 400 | + | t.Fatalf("cwnd after ACK=%d, want %d from %d acknowledged sealed bytes", gotCwnd, wantCwnd, want) | |
| 401 | + | } | |
| 402 | + | } | |
| 403 | + | ||
| 404 | + | func TestSendAsyncTransfersFragmentedRecordToBytePacedSender(t *testing.T) { | |
| 405 | + | const mtu = 128 | |
| 406 | + | aPC, bPC := newPair() | |
| 407 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 408 | + | a, err := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ | |
| 409 | + | Clock: clk, MTU: mtu, ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 410 | + | }) | |
| 411 | + | if err != nil { | |
| 412 | + | t.Fatal(err) | |
| 413 | + | } | |
| 414 | + | defer func() { _ = a.Close() }() | |
| 415 | + | ||
| 416 | + | maxFragmentData := mtu - pdgram.HeaderSize - a.codec.Overhead() | |
| 417 | + | frame := ports.Frame{Type: ports.MsgOutput, Payload: make([]byte, 2*maxFragmentData)} | |
| 418 | + | if err := a.SendAsync(frame); err != nil { | |
| 419 | + | t.Fatal(err) | |
| 420 | + | } | |
| 421 | + | eventuallyPacketCount(t, bPC, dataBurstPackets) | |
| 422 | + | if got := len(bPC.in); got != dataBurstPackets { | |
| 423 | + | t.Fatalf("immediate datagrams=%d, want %d MTU burst", got, dataBurstPackets) | |
| 424 | + | } | |
| 425 | + | awaitSignal(t, clk.timerCreated, "shared data sender pacing timer") | |
| 426 | + | clk.advance(initialPacingRTT) | |
| 427 | + | eventuallyPacketCount(t, bPC, 3) | |
| 428 | + | } | |
| 429 | + | ||
| 279 | 430 | type deadlineCapturePC struct { | |
| 280 | 431 | addr net.Addr | |
| 281 | 432 | deadline atomic.Value | |
| @@ -305,7 +456,7 @@ func (p *deadlineCapturePC) SetWriteDeadline(t time.Time) error { | |||
| 305 | 456 | return nil | |
| 306 | 457 | } | |
| 307 | 458 | ||
| 308 | - | func TestOutputFirstSendIsPacedAndQueuedOutputsAreBatched(t *testing.T) { | |
| 459 | + | func TestSmallOutputsShareInitialByteBurst(t *testing.T) { | |
| 309 | 460 | aPC, bPC := newPair() | |
| 310 | 461 | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 311 | 462 | a, err := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{Clock: clk, ResendAfter: time.Hour, Heartbeat: time.Hour}) | |
| @@ -319,19 +470,10 @@ func TestOutputFirstSendIsPacedAndQueuedOutputsAreBatched(t *testing.T) { | |||
| 319 | 470 | t.Fatal(err) | |
| 320 | 471 | } | |
| 321 | 472 | } | |
| 322 | - | if got := len(bPC.in); got != 1 { | |
| 323 | - | t.Fatalf("immediate datagrams=%d, want only the first output sent before pacing timer", got) | |
| 324 | - | } | |
| 325 | - | waitForManualTimers(t, clk, 3) | |
| 326 | - | clk.advance(defaultOutputPaceMinDelay) | |
| 327 | 473 | eventuallyPacketCount(t, bPC, 3) | |
| 328 | 474 | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: []byte("next")}); err != nil { | |
| 329 | 475 | t.Fatal(err) | |
| 330 | 476 | } | |
| 331 | - | if got := len(bPC.in); got != 3 { | |
| 332 | - | t.Fatalf("datagrams before second pace tick=%d, want 3", got) | |
| 333 | - | } | |
| 334 | - | clk.advance(defaultOutputPaceMinDelay) | |
| 335 | 477 | eventuallyPacketCount(t, bPC, 4) | |
| 336 | 478 | } | |
| 337 | 479 | ||
| @@ -353,14 +495,14 @@ func TestOutputPacingSendsAtMostOneOversizedFramePerTick(t *testing.T) { | |||
| 353 | 495 | t.Fatal(err) | |
| 354 | 496 | } | |
| 355 | 497 | } | |
| 356 | - | waitForManualTimers(t, clk, 3) | |
| 498 | + | waitForManualTimers(t, clk, 4) | |
| 357 | 499 | previousPackets := len(aPC.in) | |
| 358 | 500 | for tick := 0; tick < 128; tick++ { | |
| 359 | - | clk.advance(defaultOutputPaceMaxDelay) | |
| 501 | + | clk.advance(initialPacingRTT) | |
| 360 | 502 | time.Sleep(time.Millisecond) | |
| 361 | 503 | packets := len(aPC.in) | |
| 362 | - | if delta := packets - previousPackets; delta > defaultPacketPaceBudget { | |
| 363 | - | t.Fatalf("tick %d emitted %d packets, budget %d", tick, delta, defaultPacketPaceBudget) | |
| 504 | + | if delta := packets - previousPackets; delta > initialCongestionPackets { | |
| 505 | + | t.Fatalf("tick %d emitted %d packets, cwnd packets %d", tick, delta, initialCongestionPackets) | |
| 364 | 506 | } | |
| 365 | 507 | previousPackets = packets | |
| 366 | 508 | a.mu.Lock() | |
| @@ -386,7 +528,7 @@ func TestPacedInitialSendTimestampsFinalFragmentCompletion(t *testing.T) { | |||
| 386 | 528 | defer func() { _ = a.Close() }() | |
| 387 | 529 | large := ports.Frame{Type: ports.MsgOutput, Payload: make([]byte, 24*a.mtu)} | |
| 388 | 530 | done := make(chan error, 1) | |
| 389 | - | go func() { done <- a.SendAsync(large) }() | |
| 531 | + | go func() { done <- a.Send(large) }() | |
| 390 | 532 | start := clk.Now() | |
| 391 | 533 | for tick := 0; tick < 128; tick++ { | |
| 392 | 534 | a.mu.Lock() | |
| @@ -396,7 +538,7 @@ func TestPacedInitialSendTimestampsFinalFragmentCompletion(t *testing.T) { | |||
| 396 | 538 | t.Fatal("pending last timestamp set before final fragment") | |
| 397 | 539 | } | |
| 398 | 540 | a.mu.Unlock() | |
| 399 | - | clk.advance(defaultOutputPaceMaxDelay) | |
| 541 | + | clk.advance(initialPacingRTT) | |
| 400 | 542 | time.Sleep(time.Millisecond) | |
| 401 | 543 | select { | |
| 402 | 544 | case err := <-done: | |
| @@ -406,7 +548,7 @@ func TestPacedInitialSendTimestampsFinalFragmentCompletion(t *testing.T) { | |||
| 406 | 548 | a.mu.Lock() | |
| 407 | 549 | completed := a.pending[1].last | |
| 408 | 550 | a.mu.Unlock() | |
| 409 | - | if elapsed := completed.Sub(start); elapsed <= defaultOutputPaceMaxDelay { | |
| 551 | + | if elapsed := completed.Sub(start); elapsed <= initialPacingRTT { | |
| 410 | 552 | t.Fatalf("large send completed too early after %v", elapsed) | |
| 411 | 553 | } | |
| 412 | 554 | before := len(aPC.in) | |
| @@ -459,7 +601,7 @@ func TestOwnedSynchronousMaxSideEffectCompletesThroughConcurrentPacedWork(t *tes | |||
| 459 | 601 | go func() { done <- a.SendSynchronous(sideEffect) }() | |
| 460 | 602 | ||
| 461 | 603 | for tick := 0; tick < 512; tick++ { | |
| 462 | - | clk.advance(defaultOutputPaceMaxDelay) | |
| 604 | + | clk.advance(initialPacingRTT) | |
| 463 | 605 | time.Sleep(time.Millisecond) | |
| 464 | 606 | select { | |
| 465 | 607 | case err := <-done: | |
| @@ -511,31 +653,6 @@ func TestResendSkipsOutputDuringFirstWireAttempt(t *testing.T) { | |||
| 511 | 653 | } | |
| 512 | 654 | } | |
| 513 | 655 | ||
| 514 | - | func TestOutputPaceDelayUsesSRTTHalfClamped(t *testing.T) { | |
| 515 | - | tr, _ := newResendTestTransport(t, 10) | |
| 516 | - | tests := []struct { | |
| 517 | - | name string | |
| 518 | - | srtt time.Duration | |
| 519 | - | want time.Duration | |
| 520 | - | }{ | |
| 521 | - | {name: "high rtt clamps to max", srtt: time.Second, want: defaultOutputPaceMaxDelay}, | |
| 522 | - | {name: "low rtt clamps to min cadence", srtt: 30 * time.Millisecond, want: defaultOutputPaceMinCadence}, | |
| 523 | - | {name: "mid rtt uses half", srtt: 80 * time.Millisecond, want: 40 * time.Millisecond}, | |
| 524 | - | {name: "initial rtt uses min delay", want: defaultOutputPaceMinDelay}, | |
| 525 | - | } | |
| 526 | - | for _, tt := range tests { | |
| 527 | - | t.Run(tt.name, func(t *testing.T) { | |
| 528 | - | tr.mu.Lock() | |
| 529 | - | tr.srtt = tt.srtt | |
| 530 | - | got := tr.outputPaceDelayLocked() | |
| 531 | - | tr.mu.Unlock() | |
| 532 | - | if got != tt.want { | |
| 533 | - | t.Fatalf("output pace delay=%v, want %v", got, tt.want) | |
| 534 | - | } | |
| 535 | - | }) | |
| 536 | - | } | |
| 537 | - | } | |
| 538 | - | ||
| 539 | 656 | func TestInputWaitsForDequeuedOutputBatch(t *testing.T) { | |
| 540 | 657 | aPC, bPC := newPair() | |
| 541 | 658 | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| @@ -545,29 +662,12 @@ func TestInputWaitsForDequeuedOutputBatch(t *testing.T) { | |||
| 545 | 662 | } | |
| 546 | 663 | defer func() { _ = a.Close() }() | |
| 547 | 664 | ||
| 665 | + | a.writeMu.Lock() | |
| 548 | 666 | for _, payload := range []string{"first", "queued"} { | |
| 549 | 667 | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: []byte(payload)}); err != nil { | |
| 550 | 668 | t.Fatal(err) | |
| 551 | 669 | } | |
| 552 | 670 | } | |
| 553 | - | waitForManualTimers(t, clk, 3) | |
| 554 | - | a.writeMu.Lock() | |
| 555 | - | clk.advance(defaultOutputPaceMinDelay) | |
| 556 | - | deadline := time.Now().Add(time.Second) | |
| 557 | - | for { | |
| 558 | - | a.mu.Lock() | |
| 559 | - | dequeued := len(a.outputQueue) == 0 | |
| 560 | - | a.mu.Unlock() | |
| 561 | - | if dequeued { | |
| 562 | - | break | |
| 563 | - | } | |
| 564 | - | if time.Now().After(deadline) { | |
| 565 | - | a.writeMu.Unlock() | |
| 566 | - | t.Fatal("pacer did not dequeue output batch") | |
| 567 | - | } | |
| 568 | - | time.Sleep(time.Millisecond) | |
| 569 | - | } | |
| 570 | - | ||
| 571 | 671 | done := make(chan error, 1) | |
| 572 | 672 | go func() { done <- a.Send(ports.Frame{Type: ports.MsgInput, Payload: []byte("typed")}) }() | |
| 573 | 673 | select { | |
| @@ -599,29 +699,13 @@ func TestPacedControlQueueFailureClosesTransport(t *testing.T) { | |||
| 599 | 699 | } | |
| 600 | 700 | defer func() { _ = a.Close() }() | |
| 601 | 701 | ||
| 602 | - | for i := range 3 { | |
| 603 | - | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: []byte{byte(i)}}); err != nil { | |
| 604 | - | t.Fatal(err) | |
| 605 | - | } | |
| 606 | - | } | |
| 607 | - | if got := len(bPC.in); got != 1 { | |
| 608 | - | t.Fatalf("immediate output datagrams=%d, want 1", got) | |
| 609 | - | } | |
| 610 | 702 | if err := bPC.Close(); err != nil { | |
| 611 | 703 | t.Fatal(err) | |
| 612 | 704 | } | |
| 613 | - | ||
| 614 | 705 | inputDone := make(chan error, 1) | |
| 615 | 706 | go func() { inputDone <- a.Send(ports.Frame{Type: ports.MsgInput, Payload: []byte("typed")}) }() | |
| 616 | - | eventually(t, time.Second, func() bool { | |
| 617 | - | a.mu.Lock() | |
| 618 | - | defer a.mu.Unlock() | |
| 619 | - | return len(a.outputQueue) == 2 | |
| 620 | - | }) | |
| 621 | - | waitForManualTimers(t, clk, 3) | |
| 622 | - | clk.advance(defaultOutputPaceMinDelay) | |
| 623 | 707 | if err := <-inputDone; err == nil { | |
| 624 | - | t.Fatal("queued control send succeeded after paced wire failure") | |
| 708 | + | t.Fatal("data send succeeded after paced wire failure") | |
| 625 | 709 | } | |
| 626 | 710 | select { | |
| 627 | 711 | case <-a.done: | |
| @@ -665,10 +749,7 @@ func TestCloseUnblocksQueuedSynchronousFrames(t *testing.T) { | |||
| 665 | 749 | if err != nil { | |
| 666 | 750 | t.Fatal(err) | |
| 667 | 751 | } | |
| 668 | - | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: []byte("prime")}); err != nil { | |
| 669 | - | t.Fatal(err) | |
| 670 | - | } | |
| 671 | - | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: []byte("queued")}); err != nil { | |
| 752 | + | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: make([]byte, 3*a.mtu)}); err != nil { | |
| 672 | 753 | t.Fatal(err) | |
| 673 | 754 | } | |
| 674 | 755 | done := make(chan error, 1) | |
| @@ -676,7 +757,7 @@ func TestCloseUnblocksQueuedSynchronousFrames(t *testing.T) { | |||
| 676 | 757 | eventually(t, time.Second, func() bool { | |
| 677 | 758 | a.mu.Lock() | |
| 678 | 759 | defer a.mu.Unlock() | |
| 679 | - | return len(a.outputQueue) == 2 | |
| 760 | + | return len(a.pending) == 2 | |
| 680 | 761 | }) | |
| 681 | 762 | _ = a.Close() | |
| 682 | 763 | if err := <-done; err == nil { | |
| @@ -695,16 +776,12 @@ func TestOutputPaceFailureClosesTransportWithWriteError(t *testing.T) { | |||
| 695 | 776 | } | |
| 696 | 777 | defer func() { _ = a.Close() }() | |
| 697 | 778 | ||
| 698 | - | for i := range 2 { | |
| 699 | - | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: []byte{byte(i)}}); err != nil { | |
| 700 | - | t.Fatal(err) | |
| 701 | - | } | |
| 702 | - | } | |
| 703 | - | waitForManualTimers(t, clk, 3) | |
| 704 | 779 | if err := bPC.Close(); err != nil { | |
| 705 | 780 | t.Fatal(err) | |
| 706 | 781 | } | |
| 707 | - | clk.advance(defaultOutputPaceMinDelay) | |
| 782 | + | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: []byte("fails")}); err != nil { | |
| 783 | + | t.Fatal(err) | |
| 784 | + | } | |
| 708 | 785 | ||
| 709 | 786 | select { | |
| 710 | 787 | case <-a.done: | |
| @@ -731,16 +808,9 @@ func TestInputAndControlRemainOrderedInBoundedPacedQueue(t *testing.T) { | |||
| 731 | 808 | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: []byte("queued")}); err != nil { | |
| 732 | 809 | t.Fatal(err) | |
| 733 | 810 | } | |
| 734 | - | if got := len(bPC.in); got != 1 { | |
| 735 | - | t.Fatalf("immediate output datagrams=%d, want 1", got) | |
| 736 | - | } | |
| 811 | + | eventuallyPacketCount(t, bPC, 2) | |
| 737 | 812 | inputDone := make(chan error, 1) | |
| 738 | 813 | go func() { inputDone <- a.Send(ports.Frame{Type: ports.MsgInput, Payload: []byte("typed")}) }() | |
| 739 | - | if got := len(bPC.in); got != 1 { | |
| 740 | - | t.Fatalf("datagrams after input=%d, want prerequisites and input still paced", got) | |
| 741 | - | } | |
| 742 | - | waitForManualTimers(t, clk, 3) | |
| 743 | - | clk.advance(defaultOutputPaceMinDelay) | |
| 744 | 814 | eventuallyPacketCount(t, bPC, 3) | |
| 745 | 815 | if err := <-inputDone; err != nil { | |
| 746 | 816 | t.Fatal(err) | |
| @@ -748,20 +818,8 @@ func TestInputAndControlRemainOrderedInBoundedPacedQueue(t *testing.T) { | |||
| 748 | 818 | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: []byte("queued-before-control")}); err != nil { | |
| 749 | 819 | t.Fatal(err) | |
| 750 | 820 | } | |
| 751 | - | if got := len(bPC.in); got != 3 { | |
| 752 | - | t.Fatalf("datagrams before control=%d, want queued output still paced", got) | |
| 753 | - | } | |
| 754 | 821 | controlDone := make(chan error, 1) | |
| 755 | 822 | go func() { controlDone <- a.Send(ports.Frame{Type: ports.MsgPing}) }() | |
| 756 | - | eventually(t, time.Second, func() bool { | |
| 757 | - | a.mu.Lock() | |
| 758 | - | defer a.mu.Unlock() | |
| 759 | - | return len(a.outputQueue) == 2 | |
| 760 | - | }) | |
| 761 | - | if got := len(bPC.in); got != 3 { | |
| 762 | - | t.Fatalf("datagrams after control=%d, want ordered paced queue", got) | |
| 763 | - | } | |
| 764 | - | clk.advance(defaultOutputPaceMinDelay) | |
| 765 | 823 | eventuallyPacketCount(t, bPC, 5) | |
| 766 | 824 | if err := <-controlDone; err != nil { | |
| 767 | 825 | t.Fatal(err) | |
| @@ -854,9 +912,10 @@ func TestReliableDuplicateAckedButNotDeliveredTwice(t *testing.T) { | |||
| 854 | 912 | t.Fatal(err) | |
| 855 | 913 | } | |
| 856 | 914 | got := recvWithin(t, b, time.Second) | |
| 857 | - | if got.Type != ports.MsgInput || string(got.Payload) != "typed" || !droppedAck.Load() { | |
| 858 | - | t.Fatalf("got=%+v droppedAck=%v", got, droppedAck.Load()) | |
| 915 | + | if got.Type != ports.MsgInput || string(got.Payload) != "typed" { | |
| 916 | + | t.Fatalf("got=%+v", got) | |
| 859 | 917 | } | |
| 918 | + | eventually(t, time.Second, droppedAck.Load) | |
| 860 | 919 | if got, ok := recvMaybe(b, 3*defaultResend); ok { | |
| 861 | 920 | t.Fatalf("duplicate reliable frame delivered: %+v", got) | |
| 862 | 921 | } | |
| @@ -940,6 +999,126 @@ func TestAuthenticatedPeerChangeUpdatesPeerAndEmitsConnected(t *testing.T) { | |||
| 940 | 999 | } | |
| 941 | 1000 | } | |
| 942 | 1001 | ||
| 1002 | + | func TestAuthenticatedFragmentSeparatesContactProgressAndPeerEligibility(t *testing.T) { | |
| 1003 | + | aPC, bPC := newPair() | |
| 1004 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 1005 | + | a, err := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ | |
| 1006 | + | Clock: clk, ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 1007 | + | }) | |
| 1008 | + | if err != nil { | |
| 1009 | + | t.Fatal(err) | |
| 1010 | + | } | |
| 1011 | + | defer func() { _ = a.Close() }() | |
| 1012 | + | defer func() { _ = bPC.Close() }() | |
| 1013 | + | ||
| 1014 | + | a.setLinkState(ports.LinkStateDegraded, nil) | |
| 1015 | + | <-a.LinkEvents() | |
| 1016 | + | codec, err := pdgram.NewCodec(key()) | |
| 1017 | + | if err != nil { | |
| 1018 | + | t.Fatal(err) | |
| 1019 | + | } | |
| 1020 | + | authenticated := make(chan struct{}, 8) | |
| 1021 | + | malformed := make(chan struct{}, 8) | |
| 1022 | + | a.mu.Lock() | |
| 1023 | + | a.afterAuthenticatedPacket = func() { authenticated <- struct{}{} } | |
| 1024 | + | a.afterMalformedFragment = func() { malformed <- struct{}{} } | |
| 1025 | + | initialPacket := a.health.lastPacket | |
| 1026 | + | initialRecord := a.health.lastRecord | |
| 1027 | + | a.mu.Unlock() | |
| 1028 | + | ||
| 1029 | + | sealFragment := func(counter uint64, frag pdgram.Fragment) []byte { | |
| 1030 | + | t.Helper() | |
| 1031 | + | raw, marshalErr := pdgram.MarshalFragment(frag) | |
| 1032 | + | if marshalErr != nil { | |
| 1033 | + | t.Fatal(marshalErr) | |
| 1034 | + | } | |
| 1035 | + | return codec.Seal(2, counter, raw, nil) | |
| 1036 | + | } | |
| 1037 | + | waitSignal := func(ch <-chan struct{}, what string) { | |
| 1038 | + | t.Helper() | |
| 1039 | + | select { | |
| 1040 | + | case <-ch: | |
| 1041 | + | case <-time.After(time.Second): | |
| 1042 | + | t.Fatalf("timeout waiting for %s", what) | |
| 1043 | + | } | |
| 1044 | + | } | |
| 1045 | + | assertHealth := func(wantPacket, wantRecord time.Time, wantPeer string, wantState ports.LinkState) { | |
| 1046 | + | t.Helper() | |
| 1047 | + | a.mu.Lock() | |
| 1048 | + | gotPacket := a.health.lastPacket | |
| 1049 | + | gotRecord := a.health.lastRecord | |
| 1050 | + | a.mu.Unlock() | |
| 1051 | + | if !gotPacket.Equal(wantPacket) || !gotRecord.Equal(wantRecord) { | |
| 1052 | + | t.Fatalf("health packet/record=(%v,%v), want (%v,%v)", gotPacket, gotRecord, wantPacket, wantRecord) | |
| 1053 | + | } | |
| 1054 | + | if got := a.Peer().String(); got != wantPeer { | |
| 1055 | + | t.Fatalf("peer=%s, want %s", got, wantPeer) | |
| 1056 | + | } | |
| 1057 | + | if got := a.LinkState(); got != wantState { | |
| 1058 | + | t.Fatalf("state=%v, want %v", got, wantState) | |
| 1059 | + | } | |
| 1060 | + | } | |
| 1061 | + | ||
| 1062 | + | // Neither failed authentication nor an authenticated but invalid fragment | |
| 1063 | + | // crosses the contact or roaming trust boundary. | |
| 1064 | + | badAEAD := codec.Seal(2, 1, []byte("not a fragment"), nil) | |
| 1065 | + | badAEAD[len(badAEAD)-1] ^= 0xff | |
| 1066 | + | aPC.in <- packet{b: badAEAD, addr: testAddr("roam1")} | |
| 1067 | + | aPC.in <- packet{b: codec.Seal(2, 2, []byte("not a fragment"), nil), addr: testAddr("roam1")} | |
| 1068 | + | waitSignal(malformed, "malformed fragment") | |
| 1069 | + | assertHealth(initialPacket, initialRecord, "b", ports.LinkStateDegraded) | |
| 1070 | + | ||
| 1071 | + | clk.advance(time.Second) | |
| 1072 | + | partial := pdgram.Fragment{Seq: 100, Index: 0, Count: 2, Data: []byte("half")} | |
| 1073 | + | acceptedPacket := clk.Now() | |
| 1074 | + | acceptedRaw := sealFragment(10, partial) | |
| 1075 | + | aPC.in <- packet{b: acceptedRaw, addr: testAddr("roam1")} | |
| 1076 | + | waitSignal(authenticated, "accepted incomplete fragment") | |
| 1077 | + | assertHealth(acceptedPacket, initialRecord, "roam1", ports.LinkStateDegraded) | |
| 1078 | + | ||
| 1079 | + | // A replay and an Add-time fragment inconsistency cannot refresh contact or | |
| 1080 | + | // migrate the peer. Reassembly diagnostics still update on the Add error. | |
| 1081 | + | aPC.in <- packet{b: acceptedRaw, addr: testAddr("roam2")} | |
| 1082 | + | aPC.in <- packet{b: codec.Seal(2, 11, []byte("still not a fragment"), nil), addr: testAddr("roam2")} | |
| 1083 | + | waitSignal(malformed, "replay barrier") | |
| 1084 | + | assertHealth(acceptedPacket, initialRecord, "roam1", ports.LinkStateDegraded) | |
| 1085 | + | ||
| 1086 | + | aPC.in <- packet{b: sealFragment(12, pdgram.Fragment{Seq: 100, Index: 1, Count: 3, Data: []byte("mismatch")}), addr: testAddr("roam2")} | |
| 1087 | + | waitSignal(malformed, "reassembly rejection") | |
| 1088 | + | assertHealth(acceptedPacket, initialRecord, "roam1", ports.LinkStateDegraded) | |
| 1089 | + | ||
| 1090 | + | // Fresh reordered packets refresh contact, but only the greatest accepted | |
| 1091 | + | // AEAD counter is eligible to move the roaming peer. Contact alone must not | |
| 1092 | + | // re-arm socket hopping while stream progress remains stalled. | |
| 1093 | + | a.mu.Lock() | |
| 1094 | + | a.hoppedOffline = true | |
| 1095 | + | a.mu.Unlock() | |
| 1096 | + | clk.advance(time.Second) | |
| 1097 | + | reorderedPacket := clk.Now() | |
| 1098 | + | aPC.in <- packet{b: sealFragment(9, pdgram.Fragment{Seq: 101, Index: 0, Count: 2, Data: []byte("older")}), addr: testAddr("roam2")} | |
| 1099 | + | waitSignal(authenticated, "fresh reordered fragment") | |
| 1100 | + | assertHealth(reorderedPacket, initialRecord, "roam1", ports.LinkStateDegraded) | |
| 1101 | + | a.mu.Lock() | |
| 1102 | + | if !a.hoppedOffline { | |
| 1103 | + | a.mu.Unlock() | |
| 1104 | + | t.Fatal("fragment-only contact re-armed socket hopping") | |
| 1105 | + | } | |
| 1106 | + | a.mu.Unlock() | |
| 1107 | + | ||
| 1108 | + | clk.advance(time.Second) | |
| 1109 | + | newestPacket := clk.Now() | |
| 1110 | + | aPC.in <- packet{b: sealFragment(13, pdgram.Fragment{Seq: 102, Index: 0, Count: 2, Data: []byte("newer")}), addr: testAddr("roam2")} | |
| 1111 | + | waitSignal(authenticated, "new greatest fragment") | |
| 1112 | + | assertHealth(newestPacket, initialRecord, "roam2", ports.LinkStateDegraded) | |
| 1113 | + | ||
| 1114 | + | clk.advance(time.Second) | |
| 1115 | + | completeAt := clk.Now() | |
| 1116 | + | aPC.in <- packet{b: sealFragment(14, pdgram.Fragment{Seq: 103, Index: 0, Count: 1, Data: probeRecord(recPong, 999)}), addr: testAddr("roam2")} | |
| 1117 | + | waitSignal(authenticated, "complete record") | |
| 1118 | + | eventually(t, time.Second, func() bool { return a.LinkState() == ports.LinkStateConnected }) | |
| 1119 | + | assertHealth(completeAt, completeAt, "roam2", ports.LinkStateConnected) | |
| 1120 | + | } | |
| 1121 | + | ||
| 943 | 1122 | func TestPortHopPreservesPendingReliableMessages(t *testing.T) { | |
| 944 | 1123 | aPC, bPC := newPair() | |
| 945 | 1124 | var hopped atomic.Bool | |
| @@ -963,7 +1142,7 @@ func TestPortHopPreservesPendingReliableMessages(t *testing.T) { | |||
| 963 | 1142 | t.Fatal(err) | |
| 964 | 1143 | } | |
| 965 | 1144 | a.mu.Lock() | |
| 966 | - | a.lastHeard = time.Now().Add(-time.Second) | |
| 1145 | + | a.health.lastPacket = time.Now().Add(-time.Second) | |
| 967 | 1146 | a.mu.Unlock() | |
| 968 | 1147 | a.checkSilence() | |
| 969 | 1148 | if !hopped.Load() { | |
| @@ -1003,7 +1182,7 @@ func TestPortHopRetriesAfterRebindFailure(t *testing.T) { | |||
| 1003 | 1182 | defer func() { _ = b.Close() }() | |
| 1004 | 1183 | ||
| 1005 | 1184 | a.mu.Lock() | |
| 1006 | - | a.lastHeard = time.Now().Add(-time.Second) | |
| 1185 | + | a.health.lastPacket = time.Now().Add(-time.Second) | |
| 1007 | 1186 | a.mu.Unlock() | |
| 1008 | 1187 | a.checkSilence() | |
| 1009 | 1188 | a.mu.Lock() | |
| @@ -1029,7 +1208,7 @@ func TestServerWithoutRebindDoesNotHopPorts(t *testing.T) { | |||
| 1029 | 1208 | defer func() { _ = a.Close() }() | |
| 1030 | 1209 | defer func() { _ = b.Close() }() | |
| 1031 | 1210 | a.mu.Lock() | |
| 1032 | - | a.lastHeard = time.Now().Add(-time.Second) | |
| 1211 | + | a.health.lastPacket = time.Now().Add(-time.Second) | |
| 1033 | 1212 | a.mu.Unlock() | |
| 1034 | 1213 | a.checkSilence() | |
| 1035 | 1214 | if a.pc != aPC { | |
| @@ -1121,6 +1300,11 @@ func TestImmediateLoopbackAckSamplesStampedFinalWrite(t *testing.T) { | |||
| 1121 | 1300 | if err := a.SendAsync(ports.Frame{Type: ports.MsgOutput, Payload: []byte("state")}); err != nil { | |
| 1122 | 1301 | t.Fatal(err) | |
| 1123 | 1302 | } | |
| 1303 | + | eventually(t, time.Second, func() bool { | |
| 1304 | + | a.mu.Lock() | |
| 1305 | + | defer a.mu.Unlock() | |
| 1306 | + | return a.srtt > 0 | |
| 1307 | + | }) | |
| 1124 | 1308 | a.mu.Lock() | |
| 1125 | 1309 | srtt, rto := a.srtt, a.rto | |
| 1126 | 1310 | a.mu.Unlock() | |
| @@ -1235,6 +1419,8 @@ func newResendTestTransport(t *testing.T, maxResendPerTick int) (*Transport, *at | |||
| 1235 | 1419 | maxResendPerTick: maxResendPerTick, | |
| 1236 | 1420 | rto: 100 * time.Millisecond, | |
| 1237 | 1421 | clock: realClock{}, | |
| 1422 | + | congestion: newCongestionController(pdgram.DefaultMTU), | |
| 1423 | + | done: make(chan struct{}), | |
| 1238 | 1424 | }, &writes | |
| 1239 | 1425 | } | |
| 1240 | 1426 | ||
| @@ -1251,6 +1437,60 @@ func TestResendPendingCapsBurst(t *testing.T) { | |||
| 1251 | 1437 | } | |
| 1252 | 1438 | } | |
| 1253 | 1439 | ||
| 1440 | + | func TestRetransmitBatchReducesCongestionWindowOnce(t *testing.T) { | |
| 1441 | + | tr, writes := newResendTestTransport(t, 10) | |
| 1442 | + | now := tr.clock.Now() | |
| 1443 | + | for seq := uint64(1); seq <= 3; seq++ { | |
| 1444 | + | tr.pending[seq] = &pending{ | |
| 1445 | + | frame: ports.Frame{Type: ports.MsgOutput, Payload: []byte{byte(seq)}}, | |
| 1446 | + | last: now.Add(-time.Second), | |
| 1447 | + | } | |
| 1448 | + | } | |
| 1449 | + | initialCwnd := tr.congestion.cwndBytes | |
| 1450 | + | ||
| 1451 | + | tr.resendPending() | |
| 1452 | + | ||
| 1453 | + | if got := writes.Load(); got != 3 { | |
| 1454 | + | t.Fatalf("retransmit writes=%d, want 3", got) | |
| 1455 | + | } | |
| 1456 | + | if got, want := tr.congestion.cwndBytes, initialCwnd/2; got != want { | |
| 1457 | + | t.Fatalf("cwnd after one retransmit batch=%d, want %d", got, want) | |
| 1458 | + | } | |
| 1459 | + | } | |
| 1460 | + | ||
| 1461 | + | func TestACKBeforeQueuedRetransmitDoesNotReduceCongestionWindow(t *testing.T) { | |
| 1462 | + | tr, writes := newResendTestTransport(t, 1) | |
| 1463 | + | now := tr.clock.Now() | |
| 1464 | + | tr.pending[1] = &pending{ | |
| 1465 | + | frame: ports.Frame{Type: ports.MsgOutput, Payload: []byte("acked")}, | |
| 1466 | + | first: now.Add(-time.Second), | |
| 1467 | + | last: now.Add(-time.Second), | |
| 1468 | + | wireBytes: tr.mtu, | |
| 1469 | + | } | |
| 1470 | + | initialCwnd := tr.congestion.cwndBytes | |
| 1471 | + | tr.mu.Lock() | |
| 1472 | + | batch, _ := tr.selectRetransmitsLocked(now) | |
| 1473 | + | tr.mu.Unlock() | |
| 1474 | + | if len(batch) != 1 { | |
| 1475 | + | t.Fatalf("selected retransmits=%d, want 1", len(batch)) | |
| 1476 | + | } | |
| 1477 | + | tr.handleRecord(ackRecord(1)) | |
| 1478 | + | cwndAfterACK := initialCwnd + max(1, tr.mtu*tr.mtu/initialCwnd) | |
| 1479 | + | counterAfterACK := tr.ctr | |
| 1480 | + | ||
| 1481 | + | tr.runRetransmitBatch(&bytePacer{clk: tr.clock}, batch) | |
| 1482 | + | ||
| 1483 | + | if got := tr.ctr; got != counterAfterACK { | |
| 1484 | + | t.Fatalf("packet counter after ACK=%d, want unchanged %d", got, counterAfterACK) | |
| 1485 | + | } | |
| 1486 | + | if got := writes.Load(); got != 0 { | |
| 1487 | + | t.Fatalf("writes after ACK=%d, want 0", got) | |
| 1488 | + | } | |
| 1489 | + | if got := tr.congestion.cwndBytes; got != cwndAfterACK { | |
| 1490 | + | t.Fatalf("cwnd after ACK race=%d, want ACK-only growth %d", got, cwndAfterACK) | |
| 1491 | + | } | |
| 1492 | + | } | |
| 1493 | + | ||
| 1254 | 1494 | func TestOversizedRetransmitHonorsPacketBudget(t *testing.T) { | |
| 1255 | 1495 | tr, writes := newResendTestTransport(t, 1) | |
| 1256 | 1496 | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| @@ -1268,11 +1508,11 @@ func TestOversizedRetransmitHonorsPacketBudget(t *testing.T) { | |||
| 1268 | 1508 | ||
| 1269 | 1509 | previous := int32(0) | |
| 1270 | 1510 | for tick := 0; tick < 128; tick++ { | |
| 1271 | - | clk.advance(defaultOutputPaceMaxDelay) | |
| 1511 | + | clk.advance(initialPacingRTT) | |
| 1272 | 1512 | time.Sleep(time.Millisecond) | |
| 1273 | 1513 | current := writes.Load() | |
| 1274 | - | if delta := current - previous; delta > defaultPacketPaceBudget { | |
| 1275 | - | t.Fatalf("tick %d retransmitted %d packets, budget %d", tick, delta, defaultPacketPaceBudget) | |
| 1514 | + | if delta := current - previous; delta > dataBurstPackets { | |
| 1515 | + | t.Fatalf("tick %d retransmitted %d packets, burst %d", tick, delta, dataBurstPackets) | |
| 1276 | 1516 | } | |
| 1277 | 1517 | previous = current | |
| 1278 | 1518 | select { | |
| @@ -1335,7 +1575,7 @@ func TestRecvUnblocksWhenPeerDeadAfter(t *testing.T) { | |||
| 1335 | 1575 | defer func() { _ = a.Close() }() | |
| 1336 | 1576 | defer func() { _ = b.Close() }() | |
| 1337 | 1577 | a.mu.Lock() | |
| 1338 | - | a.lastHeard = time.Now().Add(-time.Second) | |
| 1578 | + | a.health.lastPacket = time.Now().Add(-time.Second) | |
| 1339 | 1579 | a.mu.Unlock() | |
| 1340 | 1580 | ||
| 1341 | 1581 | _, err := recvErrWithin(t, a, time.Second) | |
| @@ -1482,7 +1722,7 @@ func TestPendingReliableQueueReturnsErrPendingFullWhileConnected(t *testing.T) { | |||
| 1482 | 1722 | } | |
| 1483 | 1723 | done := make(chan error, 1) | |
| 1484 | 1724 | go func() { done <- a.Send(ports.Frame{Type: ports.MsgInput, Payload: []byte("second")}) }() | |
| 1485 | - | waitForManualTimers(t, clk, 3) | |
| 1725 | + | waitForManualTimers(t, clk, 4) | |
| 1486 | 1726 | ||
| 1487 | 1727 | clk.advance(50 * time.Millisecond) | |
| 1488 | 1728 | select { | |
| @@ -1747,7 +1987,8 @@ func TestLinkStateTransitionsFromConfigurableSilence(t *testing.T) { | |||
| 1747 | 1987 | for _, tt := range tests { | |
| 1748 | 1988 | t.Run(tt.name, func(t *testing.T) { | |
| 1749 | 1989 | a.mu.Lock() | |
| 1750 | - | a.lastHeard = time.Now().Add(-tt.age) | |
| 1990 | + | staleAt := time.Now().Add(-tt.age) | |
| 1991 | + | a.health = healthTracker{lastPacket: staleAt, lastRecord: staleAt, lastProgress: staleAt} | |
| 1751 | 1992 | a.mu.Unlock() | |
| 1752 | 1993 | a.checkSilence() | |
| 1753 | 1994 | if got := a.LinkState(); got != tt.want { | |
| @@ -1765,59 +2006,422 @@ func TestLinkStateTransitionsFromConfigurableSilence(t *testing.T) { | |||
| 1765 | 2006 | } | |
| 1766 | 2007 | } | |
| 1767 | 2008 | ||
| 1768 | - | func TestSendStallDeathRequiresOldRetransmittedFrame(t *testing.T) { | |
| 2009 | + | func TestLinkHealthSeparatesPacketContactFromUsefulProgress(t *testing.T) { | |
| 2010 | + | now := time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC) | |
| 2011 | + | threshold := func(seconds int) time.Duration { return time.Duration(seconds) * time.Second } | |
| 2012 | + | tests := []struct { | |
| 2013 | + | name string | |
| 2014 | + | packetAge time.Duration | |
| 2015 | + | recordAge time.Duration | |
| 2016 | + | ackAge time.Duration | |
| 2017 | + | hasPending bool | |
| 2018 | + | wantState ports.LinkState | |
| 2019 | + | wantHop bool | |
| 2020 | + | wantDead bool | |
| 2021 | + | }{ | |
| 2022 | + | {name: "fresh contact without useful progress degrades", packetAge: 0, recordAge: threshold(10), ackAge: threshold(10), wantState: ports.LinkStateDegraded}, | |
| 2023 | + | {name: "pending round trip stall probes and hops", packetAge: 0, recordAge: threshold(20), ackAge: threshold(20), hasPending: true, wantState: ports.LinkStateProbing, wantHop: true}, | |
| 2024 | + | {name: "fragment-only contact never becomes offline", packetAge: 0, recordAge: threshold(30), ackAge: threshold(30), wantState: ports.LinkStateDegraded}, | |
| 2025 | + | {name: "packet silence probes and hops", packetAge: threshold(20), recordAge: threshold(20), ackAge: threshold(20), wantState: ports.LinkStateProbing, wantHop: true}, | |
| 2026 | + | {name: "packet silence becomes offline", packetAge: threshold(30), recordAge: threshold(30), ackAge: threshold(30), wantState: ports.LinkStateOffline}, | |
| 2027 | + | {name: "packet silence becomes dead", packetAge: threshold(60), recordAge: threshold(60), ackAge: threshold(60), wantState: ports.LinkStateDead, wantDead: true}, | |
| 2028 | + | {name: "complete record cannot mask pending stall", packetAge: 0, recordAge: 0, ackAge: threshold(30), hasPending: true, wantState: ports.LinkStateProbing, wantHop: true}, | |
| 2029 | + | {name: "complete record restores connected without pending data", packetAge: 0, recordAge: 0, ackAge: threshold(30), wantState: ports.LinkStateConnected}, | |
| 2030 | + | {name: "ack progress restores connected", packetAge: 0, recordAge: threshold(30), ackAge: 0, hasPending: true, wantState: ports.LinkStateConnected}, | |
| 2031 | + | } | |
| 2032 | + | ||
| 2033 | + | for _, tt := range tests { | |
| 2034 | + | t.Run(tt.name, func(t *testing.T) { | |
| 2035 | + | health := healthTracker{ | |
| 2036 | + | lastPacket: now.Add(-tt.packetAge), | |
| 2037 | + | lastRecord: now.Add(-tt.recordAge), | |
| 2038 | + | lastProgress: now.Add(-tt.ackAge), | |
| 2039 | + | } | |
| 2040 | + | state, hop, dead := health.decide(now, tt.hasPending, threshold(10), threshold(20), threshold(30), threshold(60)) | |
| 2041 | + | if state != tt.wantState || hop != tt.wantHop || dead != tt.wantDead { | |
| 2042 | + | t.Fatalf("decide()=(%v,%v,%v), want (%v,%v,%v)", state, hop, dead, tt.wantState, tt.wantHop, tt.wantDead) | |
| 2043 | + | } | |
| 2044 | + | }) | |
| 2045 | + | } | |
| 2046 | + | } | |
| 2047 | + | ||
| 2048 | + | func TestLinkHealthPendingStallWithFreshContactProbesOnceWithoutDying(t *testing.T) { | |
| 2049 | + | aPC, bPC := newPair() | |
| 2050 | + | var hops atomic.Int32 | |
| 2051 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ | |
| 2052 | + | ResendAfter: time.Hour, Heartbeat: time.Hour, ProbeAfter: 20 * time.Second, DeadAfter: 60 * time.Second, | |
| 2053 | + | RebindPacketConn: func(net.PacketConn) (net.PacketConn, error) { | |
| 2054 | + | hops.Add(1) | |
| 2055 | + | return &fakePC{addr: testAddr("a2"), in: make(chan packet, 100), peers: map[string]*fakePC{"b": bPC}}, nil | |
| 2056 | + | }, | |
| 2057 | + | }) | |
| 2058 | + | defer func() { _ = a.Close() }() | |
| 2059 | + | defer func() { _ = bPC.Close() }() | |
| 2060 | + | ||
| 2061 | + | now := time.Now() | |
| 2062 | + | a.mu.Lock() | |
| 2063 | + | a.health = healthTracker{lastPacket: now, lastRecord: now.Add(-time.Minute), lastProgress: now.Add(-time.Minute)} | |
| 2064 | + | a.pending[1] = &pending{frame: ports.Frame{Type: ports.MsgOutput}, enqueued: now.Add(-time.Minute), attempts: 3} | |
| 2065 | + | a.mu.Unlock() | |
| 2066 | + | ||
| 2067 | + | a.checkSilence() | |
| 2068 | + | a.checkSilence() | |
| 2069 | + | if got := a.LinkState(); got != ports.LinkStateProbing { | |
| 2070 | + | t.Fatalf("state=%v, want probing", got) | |
| 2071 | + | } | |
| 2072 | + | if got := hops.Load(); got != 1 { | |
| 2073 | + | t.Fatalf("socket hops=%d, want one", got) | |
| 2074 | + | } | |
| 2075 | + | select { | |
| 2076 | + | case <-a.done: | |
| 2077 | + | t.Fatal("fresh authenticated contact must prevent pending-stall death") | |
| 2078 | + | default: | |
| 2079 | + | } | |
| 2080 | + | } | |
| 2081 | + | ||
| 2082 | + | func TestLinkHealthHeartbeatControlCannotMaskPendingACKStall(t *testing.T) { | |
| 2083 | + | aPC, bPC := newPair() | |
| 2084 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 2085 | + | var hops atomic.Int32 | |
| 2086 | + | tr, err := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ | |
| 2087 | + | Clock: clk, ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 2088 | + | RebindPacketConn: func(net.PacketConn) (net.PacketConn, error) { | |
| 2089 | + | attempt := hops.Add(1) | |
| 2090 | + | return &fakePC{addr: testAddr(fmt.Sprintf("a%d", attempt+1)), in: make(chan packet, 100), peers: map[string]*fakePC{"b": bPC}}, nil | |
| 2091 | + | }, | |
| 2092 | + | }) | |
| 2093 | + | if err != nil { | |
| 2094 | + | t.Fatal(err) | |
| 2095 | + | } | |
| 2096 | + | defer func() { _ = tr.Close() }() | |
| 2097 | + | defer func() { _ = bPC.Close() }() | |
| 2098 | + | if err := tr.Send(ports.Frame{Type: ports.MsgInput, Payload: []byte("unacked")}); err != nil { | |
| 2099 | + | t.Fatal(err) | |
| 2100 | + | } | |
| 2101 | + | ||
| 2102 | + | codec, err := pdgram.NewCodec(key()) | |
| 2103 | + | if err != nil { | |
| 2104 | + | t.Fatal(err) | |
| 2105 | + | } | |
| 2106 | + | authenticated := make(chan struct{}, 32) | |
| 2107 | + | tr.mu.Lock() | |
| 2108 | + | tr.afterAuthenticatedPacket = func() { authenticated <- struct{}{} } | |
| 2109 | + | tr.mu.Unlock() | |
| 2110 | + | var counter uint64 | |
| 2111 | + | sendHeartbeatPong := func() { | |
| 2112 | + | t.Helper() | |
| 2113 | + | counter++ | |
| 2114 | + | frag := pdgram.Fragment{Seq: counter, Index: 0, Count: 1, Data: probeRecord(recPong, counter)} | |
| 2115 | + | raw, marshalErr := pdgram.MarshalFragment(frag) | |
| 2116 | + | if marshalErr != nil { | |
| 2117 | + | t.Fatal(marshalErr) | |
| 2118 | + | } | |
| 2119 | + | tr.mu.Lock() | |
| 2120 | + | actual := tr.pc | |
| 2121 | + | pc, ok := actual.(*fakePC) | |
| 2122 | + | tr.mu.Unlock() | |
| 2123 | + | if !ok { | |
| 2124 | + | t.Fatalf("transport packet conn = %T, want *fakePC", actual) | |
| 2125 | + | } | |
| 2126 | + | pc.in <- packet{b: codec.Seal(2, counter, raw, nil), addr: testAddr("b")} | |
| 2127 | + | select { | |
| 2128 | + | case <-authenticated: | |
| 2129 | + | case <-time.After(time.Second): | |
| 2130 | + | t.Fatal("heartbeat pong was not authenticated") | |
| 2131 | + | } | |
| 2132 | + | } | |
| 2133 | + | ||
| 2134 | + | for elapsed := 3 * time.Second; elapsed <= 66*time.Second; elapsed += 3 * time.Second { | |
| 2135 | + | clk.advance(3 * time.Second) | |
| 2136 | + | sendHeartbeatPong() | |
| 2137 | + | eventually(t, time.Second, func() bool { | |
| 2138 | + | tr.mu.Lock() | |
| 2139 | + | defer tr.mu.Unlock() | |
| 2140 | + | return tr.health.lastRecord.Equal(clk.Now()) | |
| 2141 | + | }) | |
| 2142 | + | if elapsed >= 20*time.Second { | |
| 2143 | + | eventually(t, time.Second, func() bool { | |
| 2144 | + | tr.mu.Lock() | |
| 2145 | + | defer tr.mu.Unlock() | |
| 2146 | + | return tr.pc != aPC | |
| 2147 | + | }) | |
| 2148 | + | } | |
| 2149 | + | } | |
| 2150 | + | if got := tr.LinkState(); got != ports.LinkStateProbing { | |
| 2151 | + | t.Fatalf("state=%v, want probing despite fresh heartbeat control traffic", got) | |
| 2152 | + | } | |
| 2153 | + | if got := hops.Load(); got != 1 { | |
| 2154 | + | t.Fatalf("socket hops=%d, want one pending round-trip recovery hop", got) | |
| 2155 | + | } | |
| 2156 | + | select { | |
| 2157 | + | case <-tr.done: | |
| 2158 | + | t.Fatal("fresh authenticated control contact must prevent offline/dead closure") | |
| 2159 | + | default: | |
| 2160 | + | } | |
| 2161 | + | } | |
| 2162 | + | ||
| 2163 | + | func TestLinkHealthOnlyAckProgressThatRemovesPendingRestoresConnected(t *testing.T) { | |
| 2164 | + | tr, _ := newResendTestTransport(t, 10) | |
| 2165 | + | now := tr.clock.Now() | |
| 2166 | + | tr.linkState = ports.LinkStateDegraded | |
| 2167 | + | tr.health = healthTracker{lastPacket: now, lastRecord: now.Add(-time.Minute), lastProgress: now.Add(-time.Minute)} | |
| 2168 | + | tr.pending[1] = &pending{frame: ports.Frame{Type: ports.MsgOutput}, first: now, last: now} | |
| 2169 | + | ||
| 2170 | + | tr.handleRecord(ackRecord(1)) | |
| 2171 | + | if got := tr.LinkState(); got != ports.LinkStateConnected { | |
| 2172 | + | t.Fatalf("state after advancing ACK=%v, want connected", got) | |
| 2173 | + | } | |
| 2174 | + | tr.mu.Lock() | |
| 2175 | + | if tr.health.lastProgress.Before(now) { | |
| 2176 | + | tr.mu.Unlock() | |
| 2177 | + | t.Fatalf("last progress=%v, want at or after %v", tr.health.lastProgress, now) | |
| 2178 | + | } | |
| 2179 | + | tr.mu.Unlock() | |
| 2180 | + | ||
| 2181 | + | tr.setLinkState(ports.LinkStateDegraded, nil) | |
| 2182 | + | tr.handleRecord(ackRecord(1)) | |
| 2183 | + | if got := tr.LinkState(); got != ports.LinkStateDegraded { | |
| 2184 | + | t.Fatalf("duplicate ACK state=%v, want degraded", got) | |
| 2185 | + | } | |
| 2186 | + | } | |
| 2187 | + | ||
| 2188 | + | func TestLinkHealthFreshProgressInvalidatesStaleDecision(t *testing.T) { | |
| 1769 | 2189 | tests := []struct { | |
| 1770 | - | name string | |
| 1771 | - | age time.Duration | |
| 1772 | - | attempts int | |
| 1773 | - | wantDead bool | |
| 2190 | + | name string | |
| 2191 | + | staleAge time.Duration | |
| 2192 | + | freshen func(*Transport) | |
| 2193 | + | wantState ports.LinkState | |
| 1774 | 2194 | }{ | |
| 1775 | - | {name: "recent frame stays alive", age: time.Second, attempts: 5, wantDead: false}, | |
| 1776 | - | {name: "old but never retransmitted stays alive", age: 61 * time.Second, attempts: 0, wantDead: false}, | |
| 1777 | - | {name: "old retransmitted frame dies", age: 61 * time.Second, attempts: 3, wantDead: true}, | |
| 2195 | + | { | |
| 2196 | + | name: "fresh packet contact prevents stale dead close", | |
| 2197 | + | staleAge: time.Minute, | |
| 2198 | + | freshen: func(tr *Transport) { | |
| 2199 | + | tr.mu.Lock() | |
| 2200 | + | now := tr.clock.Now() | |
| 2201 | + | tr.health.authenticatedPacket(now) | |
| 2202 | + | tr.lastAuthenticatedPacket = now | |
| 2203 | + | tr.mu.Unlock() | |
| 2204 | + | }, | |
| 2205 | + | wantState: ports.LinkStateDegraded, | |
| 2206 | + | }, | |
| 2207 | + | { | |
| 2208 | + | name: "complete record prevents stale probing state and hop", | |
| 2209 | + | staleAge: 20 * time.Second, | |
| 2210 | + | freshen: func(tr *Transport) { | |
| 2211 | + | tr.mu.Lock() | |
| 2212 | + | now := tr.clock.Now() | |
| 2213 | + | tr.health.authenticatedPacket(now) | |
| 2214 | + | tr.health.completeRecord(now) | |
| 2215 | + | tr.lastAuthenticatedPacket = now | |
| 2216 | + | tr.lastCompleteRecord = now | |
| 2217 | + | tr.mu.Unlock() | |
| 2218 | + | tr.setLinkState(ports.LinkStateConnected, nil) | |
| 2219 | + | }, | |
| 2220 | + | wantState: ports.LinkStateConnected, | |
| 2221 | + | }, | |
| 1778 | 2222 | } | |
| 2223 | + | ||
| 1779 | 2224 | for _, tt := range tests { | |
| 1780 | 2225 | t.Run(tt.name, func(t *testing.T) { | |
| 1781 | 2226 | aPC, bPC := newPair() | |
| 1782 | - | // Drop everything a sends so the frame never gets acked, and keep the | |
| 1783 | - | // receive side fresh so only the send-stall check can declare death. | |
| 1784 | - | aPC.drop = func(_ []byte, addr net.Addr) bool { return addr.String() == "b" } | |
| 1785 | - | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ | |
| 1786 | - | ResendAfter: time.Hour, Heartbeat: time.Hour, DeadAfter: 60 * time.Second, | |
| 2227 | + | var hops atomic.Int32 | |
| 2228 | + | tr, err := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ | |
| 2229 | + | ResendAfter: time.Hour, | |
| 2230 | + | Heartbeat: time.Hour, | |
| 2231 | + | RebindPacketConn: func(net.PacketConn) (net.PacketConn, error) { | |
| 2232 | + | hops.Add(1) | |
| 2233 | + | return &fakePC{addr: testAddr("a2"), in: make(chan packet, 100), peers: map[string]*fakePC{"b": bPC}}, nil | |
| 2234 | + | }, | |
| 1787 | 2235 | }) | |
| 1788 | - | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | |
| 1789 | - | defer func() { _ = a.Close() }() | |
| 1790 | - | defer func() { _ = b.Close() }() | |
| 2236 | + | if err != nil { | |
| 2237 | + | t.Fatal(err) | |
| 2238 | + | } | |
| 2239 | + | defer func() { _ = tr.Close() }() | |
| 2240 | + | defer func() { _ = bPC.Close() }() | |
| 1791 | 2241 | ||
| 1792 | - | now := time.Now() | |
| 1793 | - | a.mu.Lock() | |
| 1794 | - | a.lastHeard = now | |
| 1795 | - | a.pending[1] = &pending{ | |
| 1796 | - | frame: ports.Frame{Type: ports.MsgOutput, Payload: []byte("stuck")}, | |
| 1797 | - | enqueued: now.Add(-tt.age), | |
| 1798 | - | last: now, | |
| 1799 | - | attempts: tt.attempts, | |
| 2242 | + | now := tr.clock.Now() | |
| 2243 | + | tr.mu.Lock() | |
| 2244 | + | staleAt := now.Add(-tt.staleAge) | |
| 2245 | + | tr.health = healthTracker{lastPacket: staleAt, lastRecord: staleAt, lastProgress: staleAt} | |
| 2246 | + | decisionReady := make(chan struct{}) | |
| 2247 | + | resume := make(chan struct{}) | |
| 2248 | + | tr.afterHealthDecision = func() { | |
| 2249 | + | close(decisionReady) | |
| 2250 | + | <-resume | |
| 1800 | 2251 | } | |
| 1801 | - | a.mu.Unlock() | |
| 2252 | + | tr.mu.Unlock() | |
| 1802 | 2253 | ||
| 1803 | - | a.checkSendStall() | |
| 1804 | - | if dead := a.LinkState() == ports.LinkStateDead; dead != tt.wantDead { | |
| 1805 | - | t.Fatalf("dead=%v, want %v", dead, tt.wantDead) | |
| 2254 | + | done := make(chan struct{}) | |
| 2255 | + | go func() { | |
| 2256 | + | tr.checkSilence() | |
| 2257 | + | close(done) | |
| 2258 | + | }() | |
| 2259 | + | select { | |
| 2260 | + | case <-decisionReady: | |
| 2261 | + | case <-time.After(time.Second): | |
| 2262 | + | t.Fatal("health decision hook was not reached") | |
| 1806 | 2263 | } | |
| 2264 | + | tt.freshen(tr) | |
| 2265 | + | close(resume) | |
| 1807 | 2266 | select { | |
| 1808 | - | case <-a.done: | |
| 1809 | - | if !tt.wantDead { | |
| 1810 | - | t.Fatal("transport closed without a send stall") | |
| 2267 | + | case <-done: | |
| 2268 | + | case <-time.After(time.Second): | |
| 2269 | + | t.Fatal("health check did not finish") | |
| 2270 | + | } | |
| 2271 | + | ||
| 2272 | + | if got := tr.LinkState(); got != tt.wantState { | |
| 2273 | + | t.Fatalf("state=%v, want %v after fresh health evidence", got, tt.wantState) | |
| 2274 | + | } | |
| 2275 | + | if got := hops.Load(); got != 0 { | |
| 2276 | + | t.Fatalf("socket hops=%d, want none after recovery", got) | |
| 2277 | + | } | |
| 2278 | + | for len(tr.linkEvents) > 0 { | |
| 2279 | + | ev := <-tr.linkEvents | |
| 2280 | + | if ev.State == ports.LinkStateDead || ev.State == ports.LinkStateProbing { | |
| 2281 | + | t.Fatalf("stale link event emitted after recovery: %v", ev.State) | |
| 1811 | 2282 | } | |
| 2283 | + | } | |
| 2284 | + | select { | |
| 2285 | + | case <-tr.done: | |
| 2286 | + | t.Fatal("transport closed from stale health decision") | |
| 1812 | 2287 | default: | |
| 1813 | - | if tt.wantDead { | |
| 1814 | - | t.Fatal("transport not closed after send stall") | |
| 1815 | - | } | |
| 1816 | 2288 | } | |
| 1817 | 2289 | }) | |
| 1818 | 2290 | } | |
| 1819 | 2291 | } | |
| 1820 | 2292 | ||
| 2293 | + | func TestLinkHealthStaleRebindRestoresFutureHopAllowance(t *testing.T) { | |
| 2294 | + | aPC, bPC := newPair() | |
| 2295 | + | clk := newManualClock(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) | |
| 2296 | + | rebindStarted := make(chan struct{}) | |
| 2297 | + | resumeRebind := make(chan struct{}) | |
| 2298 | + | var rebinds atomic.Int32 | |
| 2299 | + | tr, err := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ | |
| 2300 | + | Clock: clk, ResendAfter: time.Hour, Heartbeat: time.Hour, | |
| 2301 | + | RebindPacketConn: func(net.PacketConn) (net.PacketConn, error) { | |
| 2302 | + | attempt := rebinds.Add(1) | |
| 2303 | + | pc := &fakePC{addr: testAddr(fmt.Sprintf("a%d", attempt+1)), in: make(chan packet, 100), peers: map[string]*fakePC{"b": bPC}} | |
| 2304 | + | if attempt == 1 { | |
| 2305 | + | close(rebindStarted) | |
| 2306 | + | <-resumeRebind | |
| 2307 | + | } | |
| 2308 | + | return pc, nil | |
| 2309 | + | }, | |
| 2310 | + | }) | |
| 2311 | + | if err != nil { | |
| 2312 | + | t.Fatal(err) | |
| 2313 | + | } | |
| 2314 | + | defer func() { _ = tr.Close() }() | |
| 2315 | + | defer func() { _ = bPC.Close() }() | |
| 2316 | + | ||
| 2317 | + | tr.mu.Lock() | |
| 2318 | + | staleAt := clk.Now().Add(-20 * time.Second) | |
| 2319 | + | tr.health = healthTracker{lastPacket: staleAt, lastRecord: staleAt, lastProgress: staleAt} | |
| 2320 | + | tr.mu.Unlock() | |
| 2321 | + | firstDone := make(chan struct{}) | |
| 2322 | + | go func() { | |
| 2323 | + | tr.checkSilence() | |
| 2324 | + | close(firstDone) | |
| 2325 | + | }() | |
| 2326 | + | select { | |
| 2327 | + | case <-rebindStarted: | |
| 2328 | + | case <-time.After(time.Second): | |
| 2329 | + | t.Fatal("first rebind did not start") | |
| 2330 | + | } | |
| 2331 | + | ||
| 2332 | + | tr.mu.Lock() | |
| 2333 | + | now := clk.Now() | |
| 2334 | + | tr.health.authenticatedPacket(now) | |
| 2335 | + | tr.lastAuthenticatedPacket = now | |
| 2336 | + | tr.mu.Unlock() | |
| 2337 | + | close(resumeRebind) | |
| 2338 | + | select { | |
| 2339 | + | case <-firstDone: | |
| 2340 | + | case <-time.After(time.Second): | |
| 2341 | + | t.Fatal("stale rebind did not finish") | |
| 2342 | + | } | |
| 2343 | + | tr.mu.Lock() | |
| 2344 | + | latched := tr.hoppedOffline | |
| 2345 | + | pcAfterStale := tr.pc | |
| 2346 | + | tr.mu.Unlock() | |
| 2347 | + | if latched { | |
| 2348 | + | t.Fatal("stale rebind consumed the future hop allowance") | |
| 2349 | + | } | |
| 2350 | + | if pcAfterStale != aPC { | |
| 2351 | + | t.Fatal("stale rebind replaced the active packet conn") | |
| 2352 | + | } | |
| 2353 | + | ||
| 2354 | + | clk.advance(20 * time.Second) | |
| 2355 | + | tr.checkSilence() | |
| 2356 | + | tr.mu.Lock() | |
| 2357 | + | pcAfterRetry := tr.pc | |
| 2358 | + | tr.mu.Unlock() | |
| 2359 | + | if got := rebinds.Load(); got != 2 { | |
| 2360 | + | t.Fatalf("rebind attempts=%d, want a later eligible retry", got) | |
| 2361 | + | } | |
| 2362 | + | if pcAfterRetry == aPC { | |
| 2363 | + | t.Fatal("later stale interval did not hop the packet conn") | |
| 2364 | + | } | |
| 2365 | + | } | |
| 2366 | + | ||
| 2367 | + | func TestLinkHealthStateEventsCannotPublishAfterNewerRecovery(t *testing.T) { | |
| 2368 | + | aPC, bPC := newPair() | |
| 2369 | + | tr, err := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ResendAfter: time.Hour, Heartbeat: time.Hour}) | |
| 2370 | + | if err != nil { | |
| 2371 | + | t.Fatal(err) | |
| 2372 | + | } | |
| 2373 | + | defer func() { _ = tr.Close() }() | |
| 2374 | + | defer func() { _ = bPC.Close() }() | |
| 2375 | + | ||
| 2376 | + | probingCommitted := make(chan struct{}) | |
| 2377 | + | resumeProbing := make(chan struct{}) | |
| 2378 | + | tr.mu.Lock() | |
| 2379 | + | tr.linkState = ports.LinkStateDegraded | |
| 2380 | + | tr.afterLinkStateCommit = func(state ports.LinkState) { | |
| 2381 | + | if state != ports.LinkStateProbing { | |
| 2382 | + | return | |
| 2383 | + | } | |
| 2384 | + | close(probingCommitted) | |
| 2385 | + | <-resumeProbing | |
| 2386 | + | } | |
| 2387 | + | tr.mu.Unlock() | |
| 2388 | + | ||
| 2389 | + | probingDone := make(chan struct{}) | |
| 2390 | + | go func() { | |
| 2391 | + | tr.setLinkState(ports.LinkStateProbing, nil) | |
| 2392 | + | close(probingDone) | |
| 2393 | + | }() | |
| 2394 | + | select { | |
| 2395 | + | case <-probingCommitted: | |
| 2396 | + | case <-time.After(time.Second): | |
| 2397 | + | t.Fatal("probing commit hook was not reached") | |
| 2398 | + | } | |
| 2399 | + | tr.setLinkState(ports.LinkStateConnected, nil) | |
| 2400 | + | close(resumeProbing) | |
| 2401 | + | select { | |
| 2402 | + | case <-probingDone: | |
| 2403 | + | case <-time.After(time.Second): | |
| 2404 | + | t.Fatal("probing publisher did not finish") | |
| 2405 | + | } | |
| 2406 | + | ||
| 2407 | + | if got := tr.LinkState(); got != ports.LinkStateConnected { | |
| 2408 | + | t.Fatalf("state=%v, want connected", got) | |
| 2409 | + | } | |
| 2410 | + | select { | |
| 2411 | + | case ev := <-tr.LinkEvents(): | |
| 2412 | + | if ev.State != ports.LinkStateConnected { | |
| 2413 | + | t.Fatalf("first event=%v, want connected", ev.State) | |
| 2414 | + | } | |
| 2415 | + | default: | |
| 2416 | + | t.Fatal("missing connected recovery event") | |
| 2417 | + | } | |
| 2418 | + | select { | |
| 2419 | + | case ev := <-tr.LinkEvents(): | |
| 2420 | + | t.Fatalf("stale event published after recovery: %v", ev.State) | |
| 2421 | + | default: | |
| 2422 | + | } | |
| 2423 | + | } | |
| 2424 | + | ||
| 1821 | 2425 | func TestHopFiresAtProbingNotDegraded(t *testing.T) { | |
| 1822 | 2426 | aPC, bPC := newPair() | |
| 1823 | 2427 | var hopped atomic.Bool | |
| @@ -1836,7 +2440,8 @@ func TestHopFiresAtProbingNotDegraded(t *testing.T) { | |||
| 1836 | 2440 | defer func() { _ = b.Close() }() | |
| 1837 | 2441 | ||
| 1838 | 2442 | a.mu.Lock() | |
| 1839 | - | a.lastHeard = time.Now().Add(-time.Second) | |
| 2443 | + | staleAt := time.Now().Add(-time.Second) | |
| 2444 | + | a.health = healthTracker{lastPacket: staleAt, lastRecord: staleAt, lastProgress: staleAt} | |
| 1840 | 2445 | a.mu.Unlock() | |
| 1841 | 2446 | a.checkSilence() | |
| 1842 | 2447 | if hopped.Load() { | |
| @@ -1844,7 +2449,7 @@ func TestHopFiresAtProbingNotDegraded(t *testing.T) { | |||
| 1844 | 2449 | } | |
| 1845 | 2450 | ||
| 1846 | 2451 | a.mu.Lock() | |
| 1847 | - | a.lastHeard = time.Now().Add(-2 * time.Second) | |
| 2452 | + | a.health.lastPacket = time.Now().Add(-2 * time.Second) | |
| 1848 | 2453 | a.mu.Unlock() | |
| 1849 | 2454 | a.checkSilence() | |
| 1850 | 2455 | if !hopped.Load() { | |
| @@ -1859,7 +2464,8 @@ func TestShortSilenceDoesNotKillTransport(t *testing.T) { | |||
| 1859 | 2464 | defer func() { _ = a.Close() }() | |
| 1860 | 2465 | defer func() { _ = b.Close() }() | |
| 1861 | 2466 | a.mu.Lock() | |
| 1862 | - | a.lastHeard = time.Now().Add(-time.Second) | |
| 2467 | + | staleAt := time.Now().Add(-time.Second) | |
| 2468 | + | a.health = healthTracker{lastPacket: staleAt, lastRecord: staleAt, lastProgress: staleAt} | |
| 1863 | 2469 | a.mu.Unlock() | |
| 1864 | 2470 | a.checkSilence() | |
| 1865 | 2471 | if got := a.LinkState(); got != ports.LinkStateDegraded { | |
M internal/app/run.go +3 -1
| @@ -745,7 +745,9 @@ func runUDPProxy(ctx context.Context, session string, ready io.Writer) error { | |||
| 745 | 745 | return err | |
| 746 | 746 | } | |
| 747 | 747 | defer func() { _ = daemonTr.Close() }() | |
| 748 | - | dg, err := dgram.NewTransportWithOptions(conn, nil, key, 2, 1, udpProxyClientTransportOptions) | |
| 748 | + | udpOptions := udpProxyClientTransportOptions | |
| 749 | + | udpOptions.Observe = dgram.DiagnosticLogObserver(log) | |
| 750 | + | dg, err := dgram.NewTransportWithOptions(conn, nil, key, 2, 1, udpOptions) | |
| 749 | 751 | if err != nil { | |
| 750 | 752 | return err | |
| 751 | 753 | } | |
M internal/ports/frame.go +1 -1
| @@ -1,7 +1,7 @@ | |||
| 1 | 1 | package ports | |
| 2 | 2 | ||
| 3 | 3 | // ProtocolVersion is the current vev IPC wire protocol version. | |
| 4 | - | const ProtocolVersion uint16 = 13 | |
| 4 | + | const ProtocolVersion uint16 = 14 | |
| 5 | 5 | ||
| 6 | 6 | // MsgType identifies the kind of payload carried by a Frame. | |
| 7 | 7 | type MsgType uint8 | |
M internal/ports/frame_test.go +2 -2
| @@ -3,8 +3,8 @@ package ports | |||
| 3 | 3 | import "testing" | |
| 4 | 4 | ||
| 5 | 5 | func TestProtocolVersion(t *testing.T) { | |
| 6 | - | if ProtocolVersion != 13 { | |
| 7 | - | t.Fatalf("ProtocolVersion = %d, want 13", ProtocolVersion) | |
| 6 | + | if ProtocolVersion != 14 { | |
| 7 | + | t.Fatalf("ProtocolVersion = %d, want 14", ProtocolVersion) | |
| 8 | 8 | } | |
| 9 | 9 | } | |
| 10 | 10 | ||
M internal/ports/link.go +12 -0
| @@ -3,13 +3,25 @@ package ports | |||
| 3 | 3 | import "time" | |
| 4 | 4 | ||
| 5 | 5 | // LinkState reports best-effort transport health without changing Transport. | |
| 6 | + | // Implementations may distinguish authenticated path contact from useful | |
| 7 | + | // stream progress when classifying the link. | |
| 6 | 8 | type LinkState int | |
| 7 | 9 | ||
| 8 | 10 | const ( | |
| 11 | + | // LinkStateConnected means a complete record or meaningful acknowledgement | |
| 12 | + | // progress was observed recently. | |
| 9 | 13 | LinkStateConnected LinkState = iota | |
| 14 | + | // LinkStateDegraded means authenticated packets still reach the path, but | |
| 15 | + | // complete records and acknowledgement progress have stalled. | |
| 10 | 16 | LinkStateDegraded | |
| 17 | + | // LinkStateProbing means packet contact is silent or pending data has made no | |
| 18 | + | // round-trip progress long enough for path recovery to begin. | |
| 11 | 19 | LinkStateProbing | |
| 20 | + | // LinkStateOffline means authenticated packet contact has been absent long | |
| 21 | + | // enough for the client to resume over its fallback transport. | |
| 12 | 22 | LinkStateOffline | |
| 23 | + | // LinkStateDead means prolonged authenticated-packet silence has made the | |
| 24 | + | // transport terminal. | |
| 13 | 25 | LinkStateDead | |
| 14 | 26 | ) | |
| 15 | 27 | ||
M internal/ports/mocks/mocks.go +208 -0
| @@ -1120,6 +1120,214 @@ func (_c *MockTransport_Send_Call) RunAndReturn(run func(frame ports.Frame) erro | |||
| 1120 | 1120 | return _c | |
| 1121 | 1121 | } | |
| 1122 | 1122 | ||
| 1123 | + | // NewMockDatagramTransport creates a new instance of MockDatagramTransport. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | |
| 1124 | + | // The first argument is typically a *testing.T value. | |
| 1125 | + | func NewMockDatagramTransport(t interface { | |
| 1126 | + | mock.TestingT | |
| 1127 | + | Cleanup(func()) | |
| 1128 | + | }) *MockDatagramTransport { | |
| 1129 | + | mock := &MockDatagramTransport{} | |
| 1130 | + | mock.Mock.Test(t) | |
| 1131 | + | ||
| 1132 | + | t.Cleanup(func() { mock.AssertExpectations(t) }) | |
| 1133 | + | ||
| 1134 | + | return mock | |
| 1135 | + | } | |
| 1136 | + | ||
| 1137 | + | // MockDatagramTransport is an autogenerated mock type for the DatagramTransport type | |
| 1138 | + | type MockDatagramTransport struct { | |
| 1139 | + | mock.Mock | |
| 1140 | + | } | |
| 1141 | + | ||
| 1142 | + | type MockDatagramTransport_Expecter struct { | |
| 1143 | + | mock *mock.Mock | |
| 1144 | + | } | |
| 1145 | + | ||
| 1146 | + | func (_m *MockDatagramTransport) EXPECT() *MockDatagramTransport_Expecter { | |
| 1147 | + | return &MockDatagramTransport_Expecter{mock: &_m.Mock} | |
| 1148 | + | } | |
| 1149 | + | ||
| 1150 | + | // Close provides a mock function for the type MockDatagramTransport | |
| 1151 | + | func (_mock *MockDatagramTransport) Close() error { | |
| 1152 | + | ret := _mock.Called() | |
| 1153 | + | ||
| 1154 | + | if len(ret) == 0 { | |
| 1155 | + | panic("no return value specified for Close") | |
| 1156 | + | } | |
| 1157 | + | ||
| 1158 | + | var r0 error | |
| 1159 | + | if returnFunc, ok := ret.Get(0).(func() error); ok { | |
| 1160 | + | r0 = returnFunc() | |
| 1161 | + | } else { | |
| 1162 | + | r0 = ret.Error(0) | |
| 1163 | + | } | |
| 1164 | + | return r0 | |
| 1165 | + | } | |
| 1166 | + | ||
| 1167 | + | // MockDatagramTransport_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' | |
| 1168 | + | type MockDatagramTransport_Close_Call struct { | |
| 1169 | + | *mock.Call | |
| 1170 | + | } | |
| 1171 | + | ||
| 1172 | + | // Close is a helper method to define mock.On call | |
| 1173 | + | func (_e *MockDatagramTransport_Expecter) Close() *MockDatagramTransport_Close_Call { | |
| 1174 | + | return &MockDatagramTransport_Close_Call{Call: _e.mock.On("Close")} | |
| 1175 | + | } | |
| 1176 | + | ||
| 1177 | + | func (_c *MockDatagramTransport_Close_Call) Run(run func()) *MockDatagramTransport_Close_Call { | |
| 1178 | + | _c.Call.Run(func(args mock.Arguments) { | |
| 1179 | + | run() | |
| 1180 | + | }) | |
| 1181 | + | return _c | |
| 1182 | + | } | |
| 1183 | + | ||
| 1184 | + | func (_c *MockDatagramTransport_Close_Call) Return(err error) *MockDatagramTransport_Close_Call { | |
| 1185 | + | _c.Call.Return(err) | |
| 1186 | + | return _c | |
| 1187 | + | } | |
| 1188 | + | ||
| 1189 | + | func (_c *MockDatagramTransport_Close_Call) RunAndReturn(run func() error) *MockDatagramTransport_Close_Call { | |
| 1190 | + | _c.Call.Return(run) | |
| 1191 | + | return _c | |
| 1192 | + | } | |
| 1193 | + | ||
| 1194 | + | // DatagramTransport provides a mock function for the type MockDatagramTransport | |
| 1195 | + | func (_mock *MockDatagramTransport) DatagramTransport() { | |
| 1196 | + | _mock.Called() | |
| 1197 | + | return | |
| 1198 | + | } | |
| 1199 | + | ||
| 1200 | + | // MockDatagramTransport_DatagramTransport_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DatagramTransport' | |
| 1201 | + | type MockDatagramTransport_DatagramTransport_Call struct { | |
| 1202 | + | *mock.Call | |
| 1203 | + | } | |
| 1204 | + | ||
| 1205 | + | // DatagramTransport is a helper method to define mock.On call | |
| 1206 | + | func (_e *MockDatagramTransport_Expecter) DatagramTransport() *MockDatagramTransport_DatagramTransport_Call { | |
| 1207 | + | return &MockDatagramTransport_DatagramTransport_Call{Call: _e.mock.On("DatagramTransport")} | |
| 1208 | + | } | |
| 1209 | + | ||
| 1210 | + | func (_c *MockDatagramTransport_DatagramTransport_Call) Run(run func()) *MockDatagramTransport_DatagramTransport_Call { | |
| 1211 | + | _c.Call.Run(func(args mock.Arguments) { | |
| 1212 | + | run() | |
| 1213 | + | }) | |
| 1214 | + | return _c | |
| 1215 | + | } | |
| 1216 | + | ||
| 1217 | + | func (_c *MockDatagramTransport_DatagramTransport_Call) Return() *MockDatagramTransport_DatagramTransport_Call { | |
| 1218 | + | _c.Call.Return() | |
| 1219 | + | return _c | |
| 1220 | + | } | |
| 1221 | + | ||
| 1222 | + | func (_c *MockDatagramTransport_DatagramTransport_Call) RunAndReturn(run func()) *MockDatagramTransport_DatagramTransport_Call { | |
| 1223 | + | _c.Run(run) | |
| 1224 | + | return _c | |
| 1225 | + | } | |
| 1226 | + | ||
| 1227 | + | // Recv provides a mock function for the type MockDatagramTransport | |
| 1228 | + | func (_mock *MockDatagramTransport) Recv() (ports.Frame, error) { | |
| 1229 | + | ret := _mock.Called() | |
| 1230 | + | ||
| 1231 | + | if len(ret) == 0 { | |
| 1232 | + | panic("no return value specified for Recv") | |
| 1233 | + | } | |
| 1234 | + | ||
| 1235 | + | var r0 ports.Frame | |
| 1236 | + | var r1 error | |
| 1237 | + | if returnFunc, ok := ret.Get(0).(func() (ports.Frame, error)); ok { | |
| 1238 | + | return returnFunc() | |
| 1239 | + | } | |
| 1240 | + | if returnFunc, ok := ret.Get(0).(func() ports.Frame); ok { | |
| 1241 | + | r0 = returnFunc() | |
| 1242 | + | } else { | |
| 1243 | + | r0 = ret.Get(0).(ports.Frame) | |
| 1244 | + | } | |
| 1245 | + | if returnFunc, ok := ret.Get(1).(func() error); ok { | |
| 1246 | + | r1 = returnFunc() | |
| 1247 | + | } else { | |
| 1248 | + | r1 = ret.Error(1) | |
| 1249 | + | } | |
| 1250 | + | return r0, r1 | |
| 1251 | + | } | |
| 1252 | + | ||
| 1253 | + | // MockDatagramTransport_Recv_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Recv' | |
| 1254 | + | type MockDatagramTransport_Recv_Call struct { | |
| 1255 | + | *mock.Call | |
| 1256 | + | } | |
| 1257 | + | ||
| 1258 | + | // Recv is a helper method to define mock.On call | |
| 1259 | + | func (_e *MockDatagramTransport_Expecter) Recv() *MockDatagramTransport_Recv_Call { | |
| 1260 | + | return &MockDatagramTransport_Recv_Call{Call: _e.mock.On("Recv")} | |
| 1261 | + | } | |
| 1262 | + | ||
| 1263 | + | func (_c *MockDatagramTransport_Recv_Call) Run(run func()) *MockDatagramTransport_Recv_Call { | |
| 1264 | + | _c.Call.Run(func(args mock.Arguments) { | |
| 1265 | + | run() | |
| 1266 | + | }) | |
| 1267 | + | return _c | |
| 1268 | + | } | |
| 1269 | + | ||
| 1270 | + | func (_c *MockDatagramTransport_Recv_Call) Return(frame ports.Frame, err error) *MockDatagramTransport_Recv_Call { | |
| 1271 | + | _c.Call.Return(frame, err) | |
| 1272 | + | return _c | |
| 1273 | + | } | |
| 1274 | + | ||
| 1275 | + | func (_c *MockDatagramTransport_Recv_Call) RunAndReturn(run func() (ports.Frame, error)) *MockDatagramTransport_Recv_Call { | |
| 1276 | + | _c.Call.Return(run) | |
| 1277 | + | return _c | |
| 1278 | + | } | |
| 1279 | + | ||
| 1280 | + | // Send provides a mock function for the type MockDatagramTransport | |
| 1281 | + | func (_mock *MockDatagramTransport) Send(frame ports.Frame) error { | |
| 1282 | + | ret := _mock.Called(frame) | |
| 1283 | + | ||
| 1284 | + | if len(ret) == 0 { | |
| 1285 | + | panic("no return value specified for Send") | |
| 1286 | + | } | |
| 1287 | + | ||
| 1288 | + | var r0 error | |
| 1289 | + | if returnFunc, ok := ret.Get(0).(func(ports.Frame) error); ok { | |
| 1290 | + | r0 = returnFunc(frame) | |
| 1291 | + | } else { | |
| 1292 | + | r0 = ret.Error(0) | |
| 1293 | + | } | |
| 1294 | + | return r0 | |
| 1295 | + | } | |
| 1296 | + | ||
| 1297 | + | // MockDatagramTransport_Send_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Send' | |
| 1298 | + | type MockDatagramTransport_Send_Call struct { | |
| 1299 | + | *mock.Call | |
| 1300 | + | } | |
| 1301 | + | ||
| 1302 | + | // Send is a helper method to define mock.On call | |
| 1303 | + | // - frame ports.Frame | |
| 1304 | + | func (_e *MockDatagramTransport_Expecter) Send(frame any) *MockDatagramTransport_Send_Call { | |
| 1305 | + | return &MockDatagramTransport_Send_Call{Call: _e.mock.On("Send", frame)} | |
| 1306 | + | } | |
| 1307 | + | ||
| 1308 | + | func (_c *MockDatagramTransport_Send_Call) Run(run func(frame ports.Frame)) *MockDatagramTransport_Send_Call { | |
| 1309 | + | _c.Call.Run(func(args mock.Arguments) { | |
| 1310 | + | var arg0 ports.Frame | |
| 1311 | + | if args[0] != nil { | |
| 1312 | + | arg0 = args[0].(ports.Frame) | |
| 1313 | + | } | |
| 1314 | + | run( | |
| 1315 | + | arg0, | |
| 1316 | + | ) | |
| 1317 | + | }) | |
| 1318 | + | return _c | |
| 1319 | + | } | |
| 1320 | + | ||
| 1321 | + | func (_c *MockDatagramTransport_Send_Call) Return(err error) *MockDatagramTransport_Send_Call { | |
| 1322 | + | _c.Call.Return(err) | |
| 1323 | + | return _c | |
| 1324 | + | } | |
| 1325 | + | ||
| 1326 | + | func (_c *MockDatagramTransport_Send_Call) RunAndReturn(run func(frame ports.Frame) error) *MockDatagramTransport_Send_Call { | |
| 1327 | + | _c.Call.Return(run) | |
| 1328 | + | return _c | |
| 1329 | + | } | |
| 1330 | + | ||
| 1123 | 1331 | // NewMockAsyncTransport creates a new instance of MockAsyncTransport. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | |
| 1124 | 1332 | // The first argument is typically a *testing.T value. | |
| 1125 | 1333 | func NewMockAsyncTransport(t interface { | |
M internal/ports/ports.go +7 -0
| @@ -45,6 +45,13 @@ type Transport interface { | |||
| 45 | 45 | Close() error | |
| 46 | 46 | } | |
| 47 | 47 | ||
| 48 | + | // DatagramTransport marks transports backed by a datagram link. It lets | |
| 49 | + | // usecases negotiate a conservative output window without importing adapters. | |
| 50 | + | type DatagramTransport interface { | |
| 51 | + | Transport | |
| 52 | + | DatagramTransport() | |
| 53 | + | } | |
| 54 | + | ||
| 48 | 55 | // AsyncTransport accepts frames for ordered background transmission. SendAsync | |
| 49 | 56 | // returns once the adapter owns the frame; Send retains its synchronous wire- | |
| 50 | 57 | // attempt contract. Daemon paint output may use this capability to pipeline. | |
M internal/ports/wire.go +5 -11
| @@ -66,9 +66,9 @@ type Hello struct { | |||
| 66 | 66 | TermEnv string | |
| 67 | 67 | Cwd string | |
| 68 | 68 | TrueColor bool | |
| 69 | - | // AckOutput is reserved for wire compatibility in protocol v13. The daemon | |
| 70 | - | // ignores it because v13 clients cumulatively ACK output on every transport. | |
| 71 | - | AckOutput bool | |
| 69 | + | // MaxOutputInFlight is the requested maximum number of unacknowledged | |
| 70 | + | // state-bearing output frames. | |
| 71 | + | MaxOutputInFlight uint8 | |
| 72 | 72 | } | |
| 73 | 73 | ||
| 74 | 74 | // Input carries raw bytes typed/pasted by the client, destined for the PTY. | |
| @@ -312,11 +312,7 @@ func MarshalHello(h Hello) []byte { | |||
| 312 | 312 | } else { | |
| 313 | 313 | w.putUint8(0) | |
| 314 | 314 | } | |
| 315 | - | if h.AckOutput { | |
| 316 | - | w.putUint8(1) | |
| 317 | - | } else { | |
| 318 | - | w.putUint8(0) | |
| 319 | - | } | |
| 315 | + | w.putUint8(h.MaxOutputInFlight) | |
| 320 | 316 | return w.b | |
| 321 | 317 | } | |
| 322 | 318 | ||
| @@ -363,11 +359,9 @@ func UnmarshalHello(b []byte) (Hello, error) { | |||
| 363 | 359 | return Hello{}, err | |
| 364 | 360 | } | |
| 365 | 361 | h.TrueColor = trueColor != 0 | |
| 366 | - | ackOutput, err := r.getUint8() | |
| 367 | - | if err != nil { | |
| 362 | + | if h.MaxOutputInFlight, err = r.getUint8(); err != nil { | |
| 368 | 363 | return Hello{}, err | |
| 369 | 364 | } | |
| 370 | - | h.AckOutput = ackOutput != 0 | |
| 371 | 365 | if err := r.done(); err != nil { | |
| 372 | 366 | return Hello{}, err | |
| 373 | 367 | } | |
M internal/ports/wire_test.go +38 -11
| @@ -8,6 +8,7 @@ import ( | |||
| 8 | 8 | ||
| 9 | 9 | "github.com/bnema/vev/internal/domain" | |
| 10 | 10 | "github.com/bnema/vev/pkg/renderer" | |
| 11 | + | "github.com/stretchr/testify/require" | |
| 11 | 12 | ) | |
| 12 | 13 | ||
| 13 | 14 | // mustNotPanic runs f and fails the test (instead of crashing the test | |
| @@ -60,18 +61,18 @@ func TestHelloGoldenAndRoundTrip(t *testing.T) { | |||
| 60 | 61 | { | |
| 61 | 62 | name: "typical", | |
| 62 | 63 | msg: Hello{ | |
| 63 | - | Version: 1, | |
| 64 | - | Intent: IntentNew, | |
| 65 | - | ClientID: [16]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10}, | |
| 66 | - | ResumeToken: 0x0102030405060708, | |
| 67 | - | Name: "w0", | |
| 68 | - | Size: domain.Size{Cols: 80, Rows: 24}, | |
| 69 | - | TermEnv: "xterm-256color", | |
| 70 | - | Cwd: "/tmp/project", | |
| 71 | - | TrueColor: true, | |
| 72 | - | AckOutput: true, | |
| 64 | + | Version: 1, | |
| 65 | + | Intent: IntentNew, | |
| 66 | + | ClientID: [16]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10}, | |
| 67 | + | ResumeToken: 0x0102030405060708, | |
| 68 | + | Name: "w0", | |
| 69 | + | Size: domain.Size{Cols: 80, Rows: 24}, | |
| 70 | + | TermEnv: "xterm-256color", | |
| 71 | + | Cwd: "/tmp/project", | |
| 72 | + | TrueColor: true, | |
| 73 | + | MaxOutputInFlight: 8, | |
| 73 | 74 | }, | |
| 74 | - | want: []byte{0x00, 0x01, 0x01, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x02, 0x77, 0x30, 0x00, 0x50, 0x00, 0x18, 0x00, 0x0e, 0x78, 0x74, 0x65, 0x72, 0x6d, 0x2d, 0x32, 0x35, 0x36, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x00, 0x0c, 0x2f, 0x74, 0x6d, 0x70, 0x2f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x01, 0x01}, | |
| 75 | + | want: []byte{0x00, 0x01, 0x01, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x02, 0x77, 0x30, 0x00, 0x50, 0x00, 0x18, 0x00, 0x0e, 0x78, 0x74, 0x65, 0x72, 0x6d, 0x2d, 0x32, 0x35, 0x36, 0x63, 0x6f, 0x6c, 0x6f, 0x72, 0x00, 0x0c, 0x2f, 0x74, 0x6d, 0x70, 0x2f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x01, 0x08}, | |
| 75 | 76 | }, | |
| 76 | 77 | { | |
| 77 | 78 | name: "empty strings", | |
| @@ -568,3 +569,29 @@ func TestMsgTypeConstantsDistinct(t *testing.T) { | |||
| 568 | 569 | } | |
| 569 | 570 | }) | |
| 570 | 571 | } | |
| 572 | + | ||
| 573 | + | func TestHelloOutputWindowByteExactValues(t *testing.T) { | |
| 574 | + | for _, window := range []uint8{0, 1, 8} { | |
| 575 | + | t.Run(fmt.Sprintf("window_%d", window), func(t *testing.T) { | |
| 576 | + | hello := Hello{Version: 14, MaxOutputInFlight: window} | |
| 577 | + | // Every preceding field in this deliberately empty Hello has a fixed, | |
| 578 | + | // byte-exact zero encoding; the final byte is the negotiated window. | |
| 579 | + | want := append(make([]byte, 38), window) | |
| 580 | + | want[1] = 14 | |
| 581 | + | got := MarshalHello(hello) | |
| 582 | + | requireBytesEqual(t, want, got) | |
| 583 | + | back, err := UnmarshalHello(got) | |
| 584 | + | require.NoError(t, err) | |
| 585 | + | require.Equal(t, window, back.MaxOutputInFlight) | |
| 586 | + | assertAllPrefixesFail(t, got, UnmarshalHello) | |
| 587 | + | assertTrailingGarbageFails(t, got, UnmarshalHello) | |
| 588 | + | }) | |
| 589 | + | } | |
| 590 | + | } | |
| 591 | + | ||
| 592 | + | func requireBytesEqual(t *testing.T, want, got []byte) { | |
| 593 | + | t.Helper() | |
| 594 | + | if !bytes.Equal(want, got) { | |
| 595 | + | t.Fatalf("bytes = %#v, want %#v", got, want) | |
| 596 | + | } | |
| 597 | + | } | |
M internal/usecase/client/client.go +22 -12
| @@ -339,6 +339,13 @@ func DetectTrueColor(termEnv, colorTerm string) bool { | |||
| 339 | 339 | return termEnv == "xterm-direct" || strings.HasSuffix(termEnv, "-direct") | |
| 340 | 340 | } | |
| 341 | 341 | ||
| 342 | + | func requestedOutputWindow(transport ports.Transport) uint8 { | |
| 343 | + | if _, ok := transport.(ports.DatagramTransport); ok { | |
| 344 | + | return 1 | |
| 345 | + | } | |
| 346 | + | return 8 | |
| 347 | + | } | |
| 348 | + | ||
| 342 | 349 | func attachOnce(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string, resumeToken uint64, clientID [16]byte, ms *milestones, enterRaw func() error, clearStatus func(), drawStatusStage func(reconnectStage), linkEvents <-chan ports.LinkEvent, remote bool, clipboard ports.ClipboardReader, log *slog.Logger) attachResult { | |
| 343 | 350 | // 1. Handshake: send Hello with our size and TERM. | |
| 344 | 351 | size, err := term.Size() | |
| @@ -353,18 +360,16 @@ func attachOnce(ctx context.Context, transport ports.Transport, term ports.Termi | |||
| 353 | 360 | colorTerm := os.Getenv("COLORTERM") | |
| 354 | 361 | trueColor := DetectTrueColor(termEnv, colorTerm) | |
| 355 | 362 | hello := ports.Hello{ | |
| 356 | - | Version: ports.ProtocolVersion, | |
| 357 | - | Intent: intent, | |
| 358 | - | ClientID: clientID, | |
| 359 | - | ResumeToken: resumeToken, | |
| 360 | - | Name: name, | |
| 361 | - | Size: size, | |
| 362 | - | TermEnv: termEnv, | |
| 363 | - | Cwd: cwd, | |
| 364 | - | TrueColor: trueColor, | |
| 365 | - | // Protocol v13 requires cumulative output ACKs on every transport. Keep | |
| 366 | - | // the field asserted while it remains in the wire layout. | |
| 367 | - | AckOutput: true, | |
| 363 | + | Version: ports.ProtocolVersion, | |
| 364 | + | Intent: intent, | |
| 365 | + | ClientID: clientID, | |
| 366 | + | ResumeToken: resumeToken, | |
| 367 | + | Name: name, | |
| 368 | + | Size: size, | |
| 369 | + | TermEnv: termEnv, | |
| 370 | + | Cwd: cwd, | |
| 371 | + | TrueColor: trueColor, | |
| 372 | + | MaxOutputInFlight: requestedOutputWindow(transport), | |
| 368 | 373 | } | |
| 369 | 374 | if err := transport.Send(ports.Frame{Type: ports.MsgHello, Payload: ports.MarshalHello(hello)}); err != nil { | |
| 370 | 375 | return attachResult{err: fmt.Errorf("vev: sending hello: %w", err)} | |
| @@ -461,6 +466,11 @@ func attachOnce(ctx context.Context, transport ports.Transport, term ports.Termi | |||
| 461 | 466 | clearStatus() | |
| 462 | 467 | continue | |
| 463 | 468 | } | |
| 469 | + | if ev.State == ports.LinkStateDegraded { | |
| 470 | + | log.Warn("UDP link degraded") | |
| 471 | + | clearStatus() | |
| 472 | + | continue | |
| 473 | + | } | |
| 464 | 474 | drawStatusStage(stageForLinkState(ev.State)) | |
| 465 | 475 | if ev.State == ports.LinkStateOffline { | |
| 466 | 476 | // Stop waiting for the transport to reach Dead at 60s: exit with | |
M internal/usecase/client/client_test.go +29 -1
| @@ -46,6 +46,10 @@ type recvItem struct { | |||
| 46 | 46 | err error | |
| 47 | 47 | } | |
| 48 | 48 | ||
| 49 | + | type markedDatagramTransport struct{ ports.Transport } | |
| 50 | + | ||
| 51 | + | func (markedDatagramTransport) DatagramTransport() {} | |
| 52 | + | ||
| 49 | 53 | // scriptRecv wires a MockTransport's Recv to yield the given items in order, | |
| 50 | 54 | // then block forever (simulating a live but idle connection). It returns a | |
| 51 | 55 | // cleanup that unblocks the parked reader. | |
| @@ -183,12 +187,36 @@ func TestAttachHelloIncludesTrueColor(t *testing.T) { | |||
| 183 | 187 | case hello := <-gotHello: | |
| 184 | 188 | require.Equal(t, "xterm-256color", hello.TermEnv) | |
| 185 | 189 | require.True(t, hello.TrueColor) | |
| 186 | - | require.True(t, hello.AckOutput) | |
| 190 | + | require.Equal(t, uint8(8), hello.MaxOutputInFlight) | |
| 187 | 191 | case <-time.After(2 * time.Second): | |
| 188 | 192 | t.Fatal("hello frame was not sent") | |
| 189 | 193 | } | |
| 190 | 194 | } | |
| 191 | 195 | ||
| 196 | + | func TestAttachHelloRequestsSingleOutputForDatagramTransport(t *testing.T) { | |
| 197 | + | var out bytes.Buffer | |
| 198 | + | var restoreCount atomic.Int32 | |
| 199 | + | resizeCh := make(chan domain.Size) | |
| 200 | + | tm, in := newHappyTerminal(t, &out, &restoreCount, resizeCh) | |
| 201 | + | defer in.unblock() | |
| 202 | + | ||
| 203 | + | tr := portsmocks.NewMockTransport(t) | |
| 204 | + | tr.EXPECT().Send(isType(ports.MsgHello)).RunAndReturn(func(f ports.Frame) error { | |
| 205 | + | hello, err := ports.UnmarshalHello(f.Payload) | |
| 206 | + | require.NoError(t, err) | |
| 207 | + | require.Equal(t, uint8(1), hello.MaxOutputInFlight) | |
| 208 | + | return nil | |
| 209 | + | }).Once() | |
| 210 | + | unblock := scriptRecv(tr, | |
| 211 | + | recvItem{f: frameOf(ports.MsgWelcome, ports.MarshalWelcome(ports.Welcome{SessionID: "s1"}))}, | |
| 212 | + | recvItem{f: frameOf(ports.MsgDetached, ports.MarshalDetached(ports.Detached{Reason: ports.ReasonDetach}))}, | |
| 213 | + | ) | |
| 214 | + | defer unblock() | |
| 215 | + | tr.EXPECT().Close().Return(nil).Once() | |
| 216 | + | ||
| 217 | + | require.NoError(t, client.Attach(context.Background(), markedDatagramTransport{tr}, tm, realClock{}, ports.IntentEphemeral, "")) | |
| 218 | + | } | |
| 219 | + | ||
| 192 | 220 | func TestAttachHappyPath(t *testing.T) { | |
| 193 | 221 | var out bytes.Buffer | |
| 194 | 222 | var restoreCount atomic.Int32 | |
M internal/usecase/client/reconnect_toast_test.go +15 -3
| @@ -179,26 +179,38 @@ func reconnectToastHelloFromSend(t *testing.T, tr *reconnectToastRecordingTransp | |||
| 179 | 179 | return hello | |
| 180 | 180 | } | |
| 181 | 181 | ||
| 182 | - | func TestAttachOnceLinkStateEventsDriveReconnectUIStage(t *testing.T) { | |
| 182 | + | func TestReconnectToastDegradedClearsModalAndProbingIsVisible(t *testing.T) { | |
| 183 | 183 | term := newReconnectToastTerminalHarness(t) | |
| 184 | 184 | defer term.closeInput() | |
| 185 | 185 | tr := newReconnectToastLinkTransport() | |
| 186 | 186 | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |
| 187 | 187 | ||
| 188 | 188 | stages := make(chan reconnectStage, 4) | |
| 189 | + | clears := make(chan struct{}, 4) | |
| 189 | 190 | resultCh := make(chan attachResult, 1) | |
| 190 | 191 | ms := milestones{} | |
| 191 | 192 | go func() { | |
| 192 | 193 | resultCh <- attachOnce(context.Background(), tr, term.term, portsmocks.NewMockClock(t), ports.IntentAttach, "main", 0, [16]byte{1}, &ms, func() error { | |
| 193 | 194 | _, err := term.term.EnterRaw() | |
| 194 | 195 | return err | |
| 195 | - | }, func() {}, func(stage reconnectStage) { | |
| 196 | + | }, func() { clears <- struct{}{} }, func(stage reconnectStage) { | |
| 196 | 197 | stages <- stage | |
| 197 | 198 | }, tr.LinkEvents(), true, nil, slog.New(slog.DiscardHandler)) | |
| 198 | 199 | }() | |
| 199 | 200 | ||
| 201 | + | require.Eventually(t, func() bool { return len(clears) == 1 }, time.Second, time.Millisecond) | |
| 202 | + | <-clears // Welcome clears any stale status from an earlier attempt. | |
| 200 | 203 | tr.events <- ports.LinkEvent{State: ports.LinkStateDegraded} | |
| 201 | - | require.Equal(t, reconnectStageDegraded, <-stages) | |
| 204 | + | select { | |
| 205 | + | case <-clears: | |
| 206 | + | case <-time.After(time.Second): | |
| 207 | + | t.Fatal("degraded link did not clear stale reconnect modal") | |
| 208 | + | } | |
| 209 | + | select { | |
| 210 | + | case stage := <-stages: | |
| 211 | + | t.Fatalf("degraded link drew modal stage %v", stage) | |
| 212 | + | case <-time.After(25 * time.Millisecond): | |
| 213 | + | } | |
| 202 | 214 | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |
| 203 | 215 | require.Equal(t, reconnectStageProbingUDP, <-stages) | |
| 204 | 216 | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | |
M internal/usecase/daemon/client.go +4 -3
| @@ -370,8 +370,9 @@ func (d *Daemon) notifiesSnapshot() []chan struct{} { | |||
| 370 | 370 | } | |
| 371 | 371 | ||
| 372 | 372 | type attachClientOptions struct { | |
| 373 | - | clientID [16]byte | |
| 374 | - | resumeCapable bool | |
| 373 | + | clientID [16]byte | |
| 374 | + | resumeCapable bool | |
| 375 | + | maxOutputInFlight uint8 | |
| 375 | 376 | } | |
| 376 | 377 | ||
| 377 | 378 | func (d *Daemon) attachClient(sess *session, tr ports.Transport, sz domain.Size, opts attachClientOptions) (*attachedClient, *attachedClient) { | |
| @@ -381,7 +382,7 @@ func (d *Daemon) attachClient(sess *session, tr ports.Transport, sz domain.Size, | |||
| 381 | 382 | } | |
| 382 | 383 | ac := &attachedClient{ | |
| 383 | 384 | tr: tr, | |
| 384 | - | output: newOutputStateStream(), | |
| 385 | + | output: newOutputStateStream(opts.maxOutputInFlight), | |
| 385 | 386 | size: sz, | |
| 386 | 387 | clientID: opts.clientID, | |
| 387 | 388 | resumeCapable: opts.resumeCapable, | |
M internal/usecase/daemon/daemon.go +13 -4
| @@ -74,6 +74,15 @@ const detachNotifyTimeout = time.Second | |||
| 74 | 74 | // the proxy's reliable queue never fills from painting alone. | |
| 75 | 75 | const maxUnackedOutputStates = 8 | |
| 76 | 76 | ||
| 77 | + | // normalizeOutputWindow bounds the untrusted Hello value. Zero deliberately | |
| 78 | + | // means the legacy/default window, so malformed or absent values remain safe. | |
| 79 | + | func normalizeOutputWindow(window uint8) uint8 { | |
| 80 | + | if window == 0 || window > maxUnackedOutputStates { | |
| 81 | + | return maxUnackedOutputStates | |
| 82 | + | } | |
| 83 | + | return window | |
| 84 | + | } | |
| 85 | + | ||
| 77 | 86 | const defaultResumeParkGrace = 15 * time.Minute | |
| 78 | 87 | ||
| 79 | 88 | // defaultSize is used when a client's Hello carries no valid dimensions. | |
| @@ -667,7 +676,7 @@ func (d *Daemon) route(h ports.Hello, tr ports.Transport) (*session, *attachedCl | |||
| 667 | 676 | } | |
| 668 | 677 | } | |
| 669 | 678 | d.purgeParkedForSessionLocked(sess) | |
| 670 | - | ac, old := d.attachClient(sess, tr, sz, attachClientOptions{clientID: h.ClientID, resumeCapable: true}) | |
| 679 | + | ac, old := d.attachClient(sess, tr, sz, attachClientOptions{clientID: h.ClientID, resumeCapable: true, maxOutputInFlight: normalizeOutputWindow(h.MaxOutputInFlight)}) | |
| 671 | 680 | sess.mu.Lock() | |
| 672 | 681 | sess.terminal = term | |
| 673 | 682 | sess.mu.Unlock() | |
| @@ -683,7 +692,7 @@ func (d *Daemon) route(h ports.Hello, tr ports.Transport) (*session, *attachedCl | |||
| 683 | 692 | return nil, nil, err | |
| 684 | 693 | } | |
| 685 | 694 | d.purgeParkedForSessionLocked(sess) | |
| 686 | - | ac, old := d.attachClient(sess, tr, sz, attachClientOptions{clientID: h.ClientID, resumeCapable: true}) | |
| 695 | + | ac, old := d.attachClient(sess, tr, sz, attachClientOptions{clientID: h.ClientID, resumeCapable: true, maxOutputInFlight: normalizeOutputWindow(h.MaxOutputInFlight)}) | |
| 687 | 696 | sess.mu.Lock() | |
| 688 | 697 | sess.terminal = term | |
| 689 | 698 | sess.mu.Unlock() | |
| @@ -710,7 +719,7 @@ func (d *Daemon) route(h ports.Hello, tr ports.Transport) (*session, *attachedCl | |||
| 710 | 719 | return nil, nil, err | |
| 711 | 720 | } | |
| 712 | 721 | d.purgeParkedForSessionLocked(sess) | |
| 713 | - | ac, old := d.attachClient(sess, tr, sz, attachClientOptions{clientID: h.ClientID, resumeCapable: true}) | |
| 722 | + | ac, old := d.attachClient(sess, tr, sz, attachClientOptions{clientID: h.ClientID, resumeCapable: true, maxOutputInFlight: normalizeOutputWindow(h.MaxOutputInFlight)}) | |
| 714 | 723 | sess.mu.Lock() | |
| 715 | 724 | sess.terminal = term | |
| 716 | 725 | sess.mu.Unlock() | |
| @@ -735,7 +744,7 @@ func (d *Daemon) route(h ports.Hello, tr ports.Transport) (*session, *attachedCl | |||
| 735 | 744 | } | |
| 736 | 745 | } | |
| 737 | 746 | d.purgeParkedForSessionLocked(sess) | |
| 738 | - | ac, old := d.attachClient(sess, tr, sz, attachClientOptions{clientID: h.ClientID, resumeCapable: true}) | |
| 747 | + | ac, old := d.attachClient(sess, tr, sz, attachClientOptions{clientID: h.ClientID, resumeCapable: true, maxOutputInFlight: normalizeOutputWindow(h.MaxOutputInFlight)}) | |
| 739 | 748 | sess.mu.Lock() | |
| 740 | 749 | sess.terminal = term | |
| 741 | 750 | sess.mu.Unlock() | |
M internal/usecase/daemon/output_state_stream.go +14 -8
| @@ -14,16 +14,22 @@ import ( | |||
| 14 | 14 | // | |
| 15 | 15 | // Callers serialize access with attachedClient.sendMu. | |
| 16 | 16 | type outputStateStream struct { | |
| 17 | - | renderer *renderer.Renderer | |
| 18 | - | next uint64 | |
| 19 | - | acked uint64 | |
| 20 | - | deferred bool | |
| 21 | - | deferredReset bool | |
| 17 | + | renderer *renderer.Renderer | |
| 18 | + | next uint64 | |
| 19 | + | acked uint64 | |
| 20 | + | deferred bool | |
| 21 | + | deferredReset bool | |
| 22 | + | maxOutstanding uint64 | |
| 22 | 23 | } | |
| 23 | 24 | ||
| 24 | - | func newOutputStateStream() *outputStateStream { | |
| 25 | + | func newOutputStateStream(windowSize ...uint8) *outputStateStream { | |
| 26 | + | window := uint8(maxUnackedOutputStates) | |
| 27 | + | if len(windowSize) > 0 { | |
| 28 | + | window = normalizeOutputWindow(windowSize[0]) | |
| 29 | + | } | |
| 25 | 30 | return &outputStateStream{ | |
| 26 | - | renderer: renderer.New(renderer.Capabilities{}), | |
| 31 | + | renderer: renderer.New(renderer.Capabilities{}), | |
| 32 | + | maxOutstanding: uint64(window), | |
| 27 | 33 | } | |
| 28 | 34 | } | |
| 29 | 35 | ||
| @@ -87,7 +93,7 @@ func (s *outputStateStream) rebase() { | |||
| 87 | 93 | } | |
| 88 | 94 | ||
| 89 | 95 | func (s *outputStateStream) atCapacity() bool { | |
| 90 | - | return s.outstanding() >= maxUnackedOutputStates | |
| 96 | + | return s.outstanding() >= s.maxOutstanding | |
| 91 | 97 | } | |
| 92 | 98 | ||
| 93 | 99 | func (s *outputStateStream) deferIfAtCapacity(reset bool) bool { | |
M internal/usecase/daemon/output_state_stream_test.go +43 -0
| @@ -123,3 +123,46 @@ func drawOutputState(t *testing.T, stream *outputStateStream, frame renderer.Fra | |||
| 123 | 123 | } | |
| 124 | 124 | return stream.frame(data, reset, echoAck), true, nil | |
| 125 | 125 | } | |
| 126 | + | ||
| 127 | + | func TestOutputStateStreamWindowOneCoalescesUntilCumulativeAck(t *testing.T) { | |
| 128 | + | stream := newOutputStateStream(1) | |
| 129 | + | stream.frame([]byte("first"), true, 0) | |
| 130 | + | for range 100 { | |
| 131 | + | require.True(t, stream.deferIfAtCapacity(false)) | |
| 132 | + | } | |
| 133 | + | require.Equal(t, uint64(1), stream.outstanding()) | |
| 134 | + | ||
| 135 | + | // A stale ACK cannot release the coalesced paint; the cumulative ACK for | |
| 136 | + | // the actually sent state does. | |
| 137 | + | stream.ack(0) | |
| 138 | + | _, ok := stream.takeDeferred() | |
| 139 | + | require.False(t, ok) | |
| 140 | + | stream.ack(1) | |
| 141 | + | reset, ok := stream.takeDeferred() | |
| 142 | + | require.True(t, ok) | |
| 143 | + | require.False(t, reset) | |
| 144 | + | stream.frame([]byte("latest"), reset, 0) | |
| 145 | + | require.Equal(t, uint64(1), stream.outstanding()) | |
| 146 | + | ||
| 147 | + | sideEffect := stream.sideEffect([]byte("reliable"), 0) | |
| 148 | + | output, err := ports.UnmarshalOutput(sideEffect.Payload) | |
| 149 | + | require.NoError(t, err) | |
| 150 | + | require.Zero(t, output.BaseStateNum) | |
| 151 | + | require.Zero(t, output.NewStateNum) | |
| 152 | + | require.Equal(t, uint64(2), stream.next) | |
| 153 | + | } | |
| 154 | + | ||
| 155 | + | func TestOutputStateStreamDefaultsAndNormalizesWindow(t *testing.T) { | |
| 156 | + | require.Equal(t, uint64(8), newOutputStateStream().maxOutstanding) | |
| 157 | + | require.Equal(t, uint64(8), newOutputStateStream(0).maxOutstanding) | |
| 158 | + | require.Equal(t, uint64(8), newOutputStateStream(9).maxOutstanding) | |
| 159 | + | require.Equal(t, uint64(1), newOutputStateStream(1).maxOutstanding) | |
| 160 | + | } | |
| 161 | + | ||
| 162 | + | func TestNormalizeOutputWindow(t *testing.T) { | |
| 163 | + | for _, tt := range []struct { | |
| 164 | + | input, want uint8 | |
| 165 | + | }{{0, 8}, {1, 1}, {8, 8}, {9, 8}, {255, 8}} { | |
| 166 | + | require.Equal(t, tt.want, normalizeOutputWindow(tt.input)) | |
| 167 | + | } | |
| 168 | + | } | |
M internal/usecase/daemon/output_state_sync_test.go +62 -10
| @@ -71,11 +71,11 @@ func TestDatagramAttachPipelinesRendererBeforeAck(t *testing.T) { | |||
| 71 | 71 | d, sess, _, _ := newManualSessionWithPTYs(t, p) | |
| 72 | 72 | tr := newDatagramTestTransport() | |
| 73 | 73 | routed, ac, err := d.route(ports.Hello{ | |
| 74 | - | Version: ports.ProtocolVersion, | |
| 75 | - | Intent: ports.IntentAttach, | |
| 76 | - | Name: sess.name, | |
| 77 | - | Size: domain.Size{Cols: 80, Rows: 24}, | |
| 78 | - | AckOutput: true, | |
| 74 | + | Version: ports.ProtocolVersion, | |
| 75 | + | Intent: ports.IntentAttach, | |
| 76 | + | Name: sess.name, | |
| 77 | + | Size: domain.Size{Cols: 80, Rows: 24}, | |
| 78 | + | MaxOutputInFlight: 8, | |
| 79 | 79 | }, tr) | |
| 80 | 80 | require.NoError(t, err) | |
| 81 | 81 | require.Same(t, sess, routed) | |
| @@ -126,11 +126,11 @@ func TestDatagramMultipleUnackedScrollPaintsMatchLatestFrame(t *testing.T) { | |||
| 126 | 126 | d, sess, _, _ := newManualSessionWithPTYs(t, p) | |
| 127 | 127 | tr := newDatagramTestTransport() | |
| 128 | 128 | _, ac, err := d.route(ports.Hello{ | |
| 129 | - | Version: ports.ProtocolVersion, | |
| 130 | - | Intent: ports.IntentAttach, | |
| 131 | - | Name: sess.name, | |
| 132 | - | Size: domain.Size{Cols: 80, Rows: 24}, | |
| 133 | - | AckOutput: true, | |
| 129 | + | Version: ports.ProtocolVersion, | |
| 130 | + | Intent: ports.IntentAttach, | |
| 131 | + | Name: sess.name, | |
| 132 | + | Size: domain.Size{Cols: 80, Rows: 24}, | |
| 133 | + | MaxOutputInFlight: 8, | |
| 134 | 134 | }, tr) | |
| 135 | 135 | require.NoError(t, err) | |
| 136 | 136 | ||
| @@ -268,3 +268,55 @@ func TestFailedPaintSendRollsBackOutputState(t *testing.T) { | |||
| 268 | 268 | require.Zero(t, ac.output.next) | |
| 269 | 269 | require.Zero(t, ac.output.acked) | |
| 270 | 270 | } | |
| 271 | + | ||
| 272 | + | func TestDatagramWindowOneCoalescesPaintsUntilMsgAck(t *testing.T) { | |
| 273 | + | p, releasePTY := newBlockingPTY(t) | |
| 274 | + | defer releasePTY() | |
| 275 | + | d, sess, _, _ := newManualSessionWithPTYs(t, p) | |
| 276 | + | tr := newDatagramTestTransport() | |
| 277 | + | _, ac, err := d.route(ports.Hello{ | |
| 278 | + | Version: ports.ProtocolVersion, | |
| 279 | + | Intent: ports.IntentAttach, | |
| 280 | + | Name: sess.name, | |
| 281 | + | Size: domain.Size{Cols: 80, Rows: 24}, | |
| 282 | + | MaxOutputInFlight: 1, | |
| 283 | + | }, tr) | |
| 284 | + | require.NoError(t, err) | |
| 285 | + | ||
| 286 | + | pane := sess.tabs[0].focusedPane() | |
| 287 | + | pane.screen.Write([]byte("A")) | |
| 288 | + | d.paint(sess, ac, false) | |
| 289 | + | first := awaitFrame(t, tr.sends, ports.MsgOutput) | |
| 290 | + | firstOutput, err := ports.UnmarshalOutput(first.Payload) | |
| 291 | + | require.NoError(t, err) | |
| 292 | + | require.Equal(t, uint64(1), firstOutput.NewStateNum) | |
| 293 | + | for range 100 { | |
| 294 | + | pane.screen.Write([]byte("x")) | |
| 295 | + | d.paint(sess, ac, false) | |
| 296 | + | } | |
| 297 | + | pane.screen.Write([]byte("LATEST")) | |
| 298 | + | d.paint(sess, ac, false) | |
| 299 | + | requireNoOutputFrame(t, tr.sends) | |
| 300 | + | require.Equal(t, uint64(1), ac.output.outstanding(), "only one state-bearing datagram output may be unacknowledged") | |
| 301 | + | ||
| 302 | + | // Side-effect output is control-like: it must pass while the state window is | |
| 303 | + | // full without consuming another state number. | |
| 304 | + | require.NoError(t, d.boundedSendOutputErr(ac, []byte("side-effect"))) | |
| 305 | + | sideEffect := awaitFrame(t, tr.sends, ports.MsgOutput) | |
| 306 | + | sideEffectOutput, err := ports.UnmarshalOutput(sideEffect.Payload) | |
| 307 | + | require.NoError(t, err) | |
| 308 | + | require.Zero(t, sideEffectOutput.BaseStateNum) | |
| 309 | + | require.Zero(t, sideEffectOutput.NewStateNum) | |
| 310 | + | require.Equal(t, uint64(1), ac.output.outstanding()) | |
| 311 | + | ||
| 312 | + | tr.recv <- ports.Frame{Type: ports.MsgAck, Payload: ports.MarshalAck(ports.Ack{AckedStateNum: firstOutput.NewStateNum})} | |
| 313 | + | require.NoError(t, tr.Close()) | |
| 314 | + | d.runConnLoop(ac) | |
| 315 | + | second := awaitFrame(t, tr.sends, ports.MsgOutput) | |
| 316 | + | secondOutput, err := ports.UnmarshalOutput(second.Payload) | |
| 317 | + | require.NoError(t, err) | |
| 318 | + | require.Equal(t, firstOutput.NewStateNum, secondOutput.BaseStateNum) | |
| 319 | + | require.Equal(t, uint64(2), secondOutput.NewStateNum) | |
| 320 | + | require.Contains(t, string(secondOutput.Data), "LATEST", "cumulative ACK must release the latest coalesced state") | |
| 321 | + | require.Equal(t, uint64(1), ac.output.outstanding()) | |
| 322 | + | } | |
M internal/usecase/daemon/resume.go +1 -0
| @@ -175,6 +175,7 @@ func (d *Daemon) resumeParkedLocked(h ports.Hello, tr ports.Transport, sz domain | |||
| 175 | 175 | // abandoned transport's output chain before binding the replacement so the | |
| 176 | 176 | // mandatory first paint cannot be blocked by ACKs that died with the link. | |
| 177 | 177 | ac.output.rebase() | |
| 178 | + | ac.output.maxOutstanding = uint64(normalizeOutputWindow(h.MaxOutputInFlight)) | |
| 178 | 179 | ac.replaceTransport(tr) | |
| 179 | 180 | ac.size = sz | |
| 180 | 181 | ac.resumeToken = d.nextResumeTokenLocked() | |
M internal/usecase/daemon/resume_test.go +32 -0
| @@ -471,3 +471,35 @@ func TestResumeParkedUpdatesTerminalEnv(t *testing.T) { | |||
| 471 | 471 | defer sess.mu.Unlock() | |
| 472 | 472 | require.True(t, sess.terminal.TrueColor) | |
| 473 | 473 | } | |
| 474 | + | ||
| 475 | + | func TestResumeRenegotiatesOutputWindowOnReusedStream(t *testing.T) { | |
| 476 | + | pty, release := newBlockingPTY(t) | |
| 477 | + | defer release() | |
| 478 | + | d := newTestDaemon(t, newFactorySeq(t, pty), stubClock{}) | |
| 479 | + | firstHello := helloResumeCapable(ports.IntentNew, "work", 0) | |
| 480 | + | firstHello.MaxOutputInFlight = 8 | |
| 481 | + | sess, ac, err := d.route(firstHello, &closeTrackingTransport{}) | |
| 482 | + | require.NoError(t, err) | |
| 483 | + | require.Equal(t, uint64(8), ac.output.maxOutstanding) | |
| 484 | + | stream := ac.output | |
| 485 | + | token := ac.resumeToken | |
| 486 | + | require.True(t, sess.detachIfCurrent(ac)) | |
| 487 | + | require.True(t, d.parkAttachment(sess, ac)) | |
| 488 | + | ||
| 489 | + | resumeOne := helloResumeCapable(ports.IntentResume, "work", token) | |
| 490 | + | resumeOne.MaxOutputInFlight = 1 | |
| 491 | + | _, resumed, err := d.route(resumeOne, &closeTrackingTransport{}) | |
| 492 | + | require.NoError(t, err) | |
| 493 | + | require.Same(t, stream, resumed.output) | |
| 494 | + | require.Equal(t, uint64(1), resumed.output.maxOutstanding) | |
| 495 | + | ||
| 496 | + | token = resumed.resumeToken | |
| 497 | + | require.True(t, sess.detachIfCurrent(resumed)) | |
| 498 | + | require.True(t, d.parkAttachment(sess, resumed)) | |
| 499 | + | resumeEight := helloResumeCapable(ports.IntentResume, "work", token) | |
| 500 | + | resumeEight.MaxOutputInFlight = 8 | |
| 501 | + | _, resumed, err = d.route(resumeEight, &closeTrackingTransport{}) | |
| 502 | + | require.NoError(t, err) | |
| 503 | + | require.Same(t, stream, resumed.output) | |
| 504 | + | require.Equal(t, uint64(8), resumed.output.maxOutstanding) | |
| 505 | + | } | |
M pkg/dgram/dgram.go +4 -0
| @@ -191,6 +191,10 @@ type assembly struct { | |||
| 191 | 191 | } | |
| 192 | 192 | ||
| 193 | 193 | func NewReassembler() *Reassembler { return &Reassembler{inflight: make(map[uint64]*assembly)} } | |
| 194 | + | ||
| 195 | + | // Inflight returns the number of incomplete records currently being reassembled. | |
| 196 | + | func (r *Reassembler) Inflight() int { return len(r.inflight) } | |
| 197 | + | ||
| 194 | 198 | func (r *Reassembler) Add(f Fragment) ([]byte, bool, error) { | |
| 195 | 199 | if f.Count == 0 || int(f.Count) > maxFragmentCount || f.Index >= f.Count { | |
| 196 | 200 | return nil, false, ErrFragment | |
M scripts/debug-remote-attach.sh +83 -19
| @@ -3,7 +3,7 @@ set -euo pipefail | |||
| 3 | 3 | ||
| 4 | 4 | usage() { | |
| 5 | 5 | cat <<'USAGE' >&2 | |
| 6 | - | Usage: scripts/debug-remote-attach.sh [--mode stdio|udp|default] [--session NAME] [--duration 8s] [--vev-bin PATH] user@host | |
| 6 | + | Usage: scripts/debug-remote-attach.sh [--mode stdio|udp|default] [--session NAME] [--duration 8s] [--vev-bin PATH] [--udp-health] user@host | |
| 7 | 7 | ||
| 8 | 8 | Runs a real remote attach smoke/debug attempt under a PTY, captures local and | |
| 9 | 9 | remote vev versions, and stores local/remote log tails around the attempt. | |
| @@ -19,6 +19,8 @@ Notes: | |||
| 19 | 19 | - Exit 124 from timeout is treated as a likely successful attach if no early | |
| 20 | 20 | vev/ssh error is captured; inspect the bundle for confirmation. | |
| 21 | 21 | - The remote host must be reachable by SSH for log collection. | |
| 22 | + | - --udp-health collects redacted UDP transport-health log lines and Linux | |
| 23 | + | ss -u -m socket-memory counters when available. | |
| 22 | 24 | - The attach attempt unsets VEV so the smoke test can run from inside vev. | |
| 23 | 25 | USAGE | |
| 24 | 26 | } | |
| @@ -27,6 +29,7 @@ mode="default" | |||
| 27 | 29 | session="" | |
| 28 | 30 | duration="8s" | |
| 29 | 31 | vev_bin="${VEV_BIN:-vev}" | |
| 32 | + | collect_udp_health=false | |
| 30 | 33 | while [[ $# -gt 0 ]]; do | |
| 31 | 34 | case "$1" in | |
| 32 | 35 | --mode) | |
| @@ -37,6 +40,8 @@ while [[ $# -gt 0 ]]; do | |||
| 37 | 40 | duration="${2:-}"; shift 2 ;; | |
| 38 | 41 | --vev-bin) | |
| 39 | 42 | vev_bin="${2:-}"; shift 2 ;; | |
| 43 | + | --udp-health) | |
| 44 | + | collect_udp_health=true; shift ;; | |
| 40 | 45 | -h|--help) | |
| 41 | 46 | usage; exit 0 ;; | |
| 42 | 47 | --) | |
| @@ -64,59 +69,110 @@ if [[ -n "$session" ]]; then | |||
| 64 | 69 | fi | |
| 65 | 70 | ||
| 66 | 71 | repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" | |
| 67 | - | out_dir="${TMPDIR:-/tmp}/vev-remote-attach-$target-$(date +%Y%m%d-%H%M%S)" | |
| 68 | - | out_dir="${out_dir//@/_}" | |
| 69 | - | out_dir="${out_dir//:/_}" | |
| 72 | + | out_dir="${TMPDIR:-/tmp}/vev-remote-attach-$(date +%Y%m%d-%H%M%S)" | |
| 70 | 73 | mkdir -p "$out_dir" | |
| 71 | 74 | ||
| 72 | 75 | local_state="${XDG_STATE_HOME:-$HOME/.local/state}/vev" | |
| 76 | + | diagnostic_files=() | |
| 77 | + | ||
| 78 | + | # All persisted and displayed command output passes through this filter. Keep | |
| 79 | + | # diagnostic counters while removing the target, local home, network addresses, | |
| 80 | + | # and common key material labels from bundles that may be shared. | |
| 81 | + | redact() { | |
| 82 | + | sed -E \ | |
| 83 | + | -e "s|${target//\/\\/\\}|[remote]|g" \ | |
| 84 | + | -e "s|${HOME//\/\\/\\}|[home]|g" \ | |
| 85 | + | -e 's/([[:alnum:]_.-]+@)?([[:alnum:]_.-]+\.)+[[:alpha:]]{2,}/[host]/g' \ | |
| 86 | + | -e 's/([0-9]{1,3}\.){3}[0-9]{1,3}/[address]/g' \ | |
| 87 | + | -e 's/([[:xdigit:]]{1,4}:){2,}[[:xdigit:]:]*/[address]/g' \ | |
| 88 | + | -e 's/(private[ _-]?key|ssh-rsa|ssh-ed25519|ecdsa-sha2-[^[:space:]]*)/[redacted-key]/Ig' | |
| 89 | + | } | |
| 90 | + | ||
| 91 | + | redact_file() { | |
| 92 | + | local raw="$1" dest="$2" | |
| 93 | + | redact <"$raw" >"$dest" || true | |
| 94 | + | rm -f "$raw" | |
| 95 | + | } | |
| 73 | 96 | ||
| 74 | 97 | run_capture() { | |
| 75 | - | local name="$1"; shift | |
| 98 | + | local name="$1" raw_out raw_err status | |
| 99 | + | shift | |
| 100 | + | raw_out="$out_dir/.$name.out.raw" | |
| 101 | + | raw_err="$out_dir/.$name.err.raw" | |
| 102 | + | status=0 | |
| 76 | 103 | { | |
| 77 | 104 | printf '$' | |
| 78 | 105 | printf ' %q' "$@" | |
| 79 | 106 | printf '\n' | |
| 80 | 107 | "$@" | |
| 81 | - | } >"$out_dir/$name.out" 2>"$out_dir/$name.err" || printf '%s\n' "$?" >"$out_dir/$name.exit" | |
| 108 | + | } >"$raw_out" 2>"$raw_err" || status=$? | |
| 109 | + | redact_file "$raw_out" "$out_dir/$name.out" | |
| 110 | + | redact_file "$raw_err" "$out_dir/$name.err" | |
| 111 | + | if [[ "$status" -ne 0 ]]; then printf '%s\n' "$status" >"$out_dir/$name.exit"; fi | |
| 82 | 112 | } | |
| 83 | 113 | ||
| 84 | 114 | copy_local_logs() { | |
| 85 | - | local label="$1" | |
| 115 | + | local label="$1" raw | |
| 86 | 116 | mkdir -p "$out_dir/local-$label" | |
| 87 | 117 | if [[ -d "$local_state" ]]; then | |
| 88 | 118 | find "$local_state" -maxdepth 1 -type f -name 'vev*.log' -print0 2>/dev/null | while IFS= read -r -d '' f; do | |
| 89 | - | tail -n 200 "$f" >"$out_dir/local-$label/$(basename "$f")" || true | |
| 119 | + | raw="$out_dir/local-$label/.$(basename "$f").raw" | |
| 120 | + | tail -n 200 "$f" >"$raw" || true | |
| 121 | + | redact_file "$raw" "$out_dir/local-$label/$(basename "$f")" | |
| 90 | 122 | done | |
| 91 | 123 | fi | |
| 92 | 124 | } | |
| 93 | 125 | ||
| 94 | 126 | copy_remote_logs() { | |
| 95 | - | local label="$1" | |
| 127 | + | local label="$1" raw_out="$out_dir/remote-$1/.logs.out.raw" raw_err="$out_dir/remote-$1/.logs.err.raw" | |
| 96 | 128 | mkdir -p "$out_dir/remote-$label" | |
| 97 | 129 | ssh -o BatchMode=yes -o ConnectTimeout=8 "$target" 'state="${XDG_STATE_HOME:-$HOME/.local/state}/vev"; if [ -d "$state" ]; then for f in "$state"/vev*.log; do [ -e "$f" ] || continue; echo "===== $f ====="; tail -n 200 "$f"; done; fi' \ | |
| 98 | - | >"$out_dir/remote-$label/logs.out" 2>"$out_dir/remote-$label/logs.err" || true | |
| 130 | + | >"$raw_out" 2>"$raw_err" || true | |
| 131 | + | redact_file "$raw_out" "$out_dir/remote-$label/logs.out" | |
| 132 | + | redact_file "$raw_err" "$out_dir/remote-$label/logs.err" | |
| 133 | + | } | |
| 134 | + | ||
| 135 | + | collect_udp_diagnostics() { | |
| 136 | + | local label="$1" local_dir="$out_dir/local-$1" remote_dir="$out_dir/remote-$1" | |
| 137 | + | if ! "$collect_udp_health"; then return; fi | |
| 138 | + | ||
| 139 | + | grep -hE 'udp( transport)? health' "$local_dir"/*.log 2>/dev/null | redact >"$local_dir/udp-transport-health.out" || true | |
| 140 | + | (ss -u -m 2>/dev/null || true) | awk '/skmem:/' | redact >"$local_dir/udp-socket-memory.out" | |
| 141 | + | diagnostic_files+=("$local_dir/udp-transport-health.out" "$local_dir/udp-socket-memory.out") | |
| 142 | + | ||
| 143 | + | ssh -o BatchMode=yes -o ConnectTimeout=8 "$target" 'state="${XDG_STATE_HOME:-$HOME/.local/state}/vev"; [ -d "$state" ] && grep -hE "udp( transport)? health" "$state"/vev*.log 2>/dev/null || true' \ | |
| 144 | + | 2>&1 | redact >"$remote_dir/udp-transport-health.out" || true | |
| 145 | + | ssh -o BatchMode=yes -o ConnectTimeout=8 "$target" 'ss -u -m 2>/dev/null | awk "/skmem:/" || true' \ | |
| 146 | + | 2>&1 | redact >"$remote_dir/udp-socket-memory.out" || true | |
| 147 | + | diagnostic_files+=("$remote_dir/udp-transport-health.out" "$remote_dir/udp-socket-memory.out") | |
| 99 | 148 | } | |
| 100 | 149 | ||
| 101 | 150 | printf 'debug bundle: %s\n' "$out_dir" | |
| 102 | - | printf 'target: %s\nattach_target: %s\nmode: %s\nduration: %s\n' "$target" "$attach_target" "$mode" "$duration" >"$out_dir/summary.txt" | |
| 151 | + | printf 'mode: %s\nduration: %s\n' "$mode" "$duration" >"$out_dir/summary.txt" | |
| 103 | 152 | ||
| 153 | + | raw_git="$out_dir/.git.raw" | |
| 104 | 154 | ( | |
| 105 | 155 | cd "$repo_root" | |
| 106 | 156 | git rev-parse --show-toplevel | |
| 107 | 157 | git rev-parse --short HEAD | |
| 108 | 158 | git status --short | |
| 109 | - | ) >"$out_dir/git.txt" 2>&1 || true | |
| 159 | + | ) >"$raw_git" 2>&1 || true | |
| 160 | + | redact_file "$raw_git" "$out_dir/git.txt" | |
| 110 | 161 | ||
| 111 | 162 | run_capture local-version "$vev_bin" --version | |
| 163 | + | raw_remote_version_out="$out_dir/.remote-version.out.raw" | |
| 164 | + | raw_remote_version_err="$out_dir/.remote-version.err.raw" | |
| 112 | 165 | ssh -o BatchMode=yes -o ConnectTimeout=8 "$target" 'set -e; command -v vev; vev --version; pgrep -a vev || true; state="${XDG_STATE_HOME:-$HOME/.local/state}/vev"; echo "state=$state"' \ | |
| 113 | - | >"$out_dir/remote-version.out" 2>"$out_dir/remote-version.err" || echo "$?" >"$out_dir/remote-version.exit" | |
| 166 | + | >"$raw_remote_version_out" 2>"$raw_remote_version_err" || echo "$?" >"$out_dir/remote-version.exit" | |
| 167 | + | redact_file "$raw_remote_version_out" "$out_dir/remote-version.out" | |
| 168 | + | redact_file "$raw_remote_version_err" "$out_dir/remote-version.err" | |
| 114 | 169 | ||
| 115 | 170 | copy_local_logs before | |
| 116 | 171 | copy_remote_logs before | |
| 117 | 172 | ||
| 118 | - | attach_log="$out_dir/attach.typescript" | |
| 119 | - | attach_err="$out_dir/attach.err" | |
| 173 | + | attach_log_raw="$out_dir/.attach.typescript.raw" | |
| 174 | + | attach_err_raw="$out_dir/.attach.err.raw" | |
| 175 | + | attach_stdout_raw="$out_dir/.attach.stdout.raw" | |
| 120 | 176 | cmd=("$vev_bin" attach "$attach_target") | |
| 121 | 177 | if [[ "$mode" == "default" ]]; then | |
| 122 | 178 | env_cmd=(env -u VEV -u VEV_REMOTE_TRANSPORT) | |
| @@ -127,16 +183,20 @@ fi | |||
| 127 | 183 | set +e | |
| 128 | 184 | if command -v script >/dev/null 2>&1; then | |
| 129 | 185 | printf -v attach_cmd '%q ' "${cmd[@]}" | |
| 130 | - | "${env_cmd[@]}" timeout "$duration" script -q -e -c "$attach_cmd" "$attach_log" >"$out_dir/attach.stdout" 2>"$attach_err" | |
| 186 | + | "${env_cmd[@]}" timeout "$duration" script -q -e -c "$attach_cmd" "$attach_log_raw" >"$attach_stdout_raw" 2>"$attach_err_raw" | |
| 131 | 187 | else | |
| 132 | - | "${env_cmd[@]}" timeout "$duration" "${cmd[@]}" >"$out_dir/attach.stdout" 2>"$attach_err" | |
| 188 | + | "${env_cmd[@]}" timeout "$duration" "${cmd[@]}" >"$attach_stdout_raw" 2>"$attach_err_raw" | |
| 133 | 189 | fi | |
| 134 | 190 | attach_status=$? | |
| 135 | 191 | set -e | |
| 192 | + | redact_file "$attach_stdout_raw" "$out_dir/attach.stdout" | |
| 193 | + | redact_file "$attach_err_raw" "$out_dir/attach.err" | |
| 194 | + | if [[ -e "$attach_log_raw" ]]; then redact_file "$attach_log_raw" "$out_dir/attach.typescript"; fi | |
| 136 | 195 | printf '%s\n' "$attach_status" >"$out_dir/attach.exit" | |
| 137 | 196 | ||
| 138 | 197 | copy_local_logs after | |
| 139 | 198 | copy_remote_logs after | |
| 199 | + | collect_udp_diagnostics after | |
| 140 | 200 | ||
| 141 | 201 | { | |
| 142 | 202 | echo "debug bundle: $out_dir" | |
| @@ -145,9 +205,13 @@ copy_remote_logs after | |||
| 145 | 205 | sed 's/^/ /' "$out_dir/local-version.out" 2>/dev/null || true | |
| 146 | 206 | echo "remote version:" | |
| 147 | 207 | sed 's/^/ /' "$out_dir/remote-version.out" 2>/dev/null || true | |
| 148 | - | if [[ -s "$attach_err" ]]; then | |
| 208 | + | if [[ -s "$out_dir/attach.err" ]]; then | |
| 149 | 209 | echo "attach stderr:" | |
| 150 | - | sed 's/^/ /' "$attach_err" | tail -40 | |
| 210 | + | sed 's/^/ /' "$out_dir/attach.err" | tail -40 | |
| 211 | + | fi | |
| 212 | + | if ((${#diagnostic_files[@]})); then | |
| 213 | + | echo "collected diagnostic files:" | |
| 214 | + | printf ' %s\n' "${diagnostic_files[@]}" | |
| 151 | 215 | fi | |
| 152 | 216 | echo "recent local after logs:" | |
| 153 | 217 | find "$out_dir/local-after" -maxdepth 1 -type f -print0 2>/dev/null | while IFS= read -r -d '' f; do | |
A scripts/debug-remote-attach_test.sh +30 -0
| @@ -0,0 +1,30 @@ | |||
| 1 | + | #!/usr/bin/env bash | |
| 2 | + | set -euo pipefail | |
| 3 | + | ||
| 4 | + | repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" | |
| 5 | + | help="$("$repo_root/scripts/debug-remote-attach.sh" --help 2>&1)" | |
| 6 | + | grep -Fq -- '--udp-health' <<<"$help" | |
| 7 | + | ||
| 8 | + | work="$(mktemp -d)" | |
| 9 | + | trap 'rm -rf "$work"' EXIT | |
| 10 | + | mkdir -p "$work/bin" "$work/state/vev" | |
| 11 | + | printf 'udp health peer=192.0.2.1 home=%s/local\n' "$HOME" >"$work/state/vev/vev-test.log" | |
| 12 | + | cat >"$work/bin/ssh" <<'SSH' | |
| 13 | + | #!/usr/bin/env bash | |
| 14 | + | printf 'udp transport health peer=198.51.100.2 key=ssh-ed25519\nskmem:(r0,rb1,t0,tb1,f0,w0,o0,bl0,d0)\n' | |
| 15 | + | SSH | |
| 16 | + | chmod +x "$work/bin/ssh" | |
| 17 | + | ||
| 18 | + | TMPDIR="$work" XDG_STATE_HOME="$work/state" PATH="$work/bin:$PATH" \ | |
| 19 | + | "$repo_root/scripts/debug-remote-attach.sh" --udp-health --duration 0s --vev-bin /bin/true alice@mobile.example.com >"$work/run.out" 2>&1 | |
| 20 | + | bundle="$(find "$work" -maxdepth 1 -type d -name 'vev-remote-attach-*' -print -quit)" | |
| 21 | + | test -n "$bundle" | |
| 22 | + | test -f "$bundle/local-after/udp-transport-health.out" | |
| 23 | + | test -f "$bundle/remote-after/udp-socket-memory.out" | |
| 24 | + | grep -Fq "$bundle/remote-after/udp-socket-memory.out" "$work/run.out" | |
| 25 | + | if grep -R --exclude='*.raw' -F -e 'alice@mobile.example.com' -e '192.0.2.1' -e '198.51.100.2' -e 'ssh-ed25519' -e "$HOME/local" "$bundle"; then | |
| 26 | + | echo 'unredacted diagnostic value found' >&2 | |
| 27 | + | exit 1 | |
| 28 | + | fi | |
| 29 | + | ||
| 30 | + | echo 'debug-remote-attach diagnostics/redaction: ok' | |