Merge pull request #36 from bnema/feat/ultra-resilient-remote-attach
feat(remote): make SSH UDP attach resilient
26 files · +1656 -170
M README.md +4 -2
| @@ -122,9 +122,11 @@ vev attach user@host:session | |||
| 122 | 122 | ||
| 123 | 123 | Remote attach uses SSH only to start a UDP proxy on the remote host; the session itself then talks directly to the remote host over UDP. `vev` must be installed on the remote host, and the host name you pass to `vev attach` must be reachable both by SSH for bootstrap and by UDP for the session transport. Omitting `:session` opens an ephemeral remote session. Remote sessions can resume after temporary network disconnects, such as sleep or Wi-Fi changes, while the daemon still has the session. Named sessions additionally persist across daemon restarts. | |
| 124 | 124 | ||
| 125 | - | The UDP proxy binds a port in a fixed range on the remote (default `61000-61023`), so a host firewall that only allows SSH will drop the session datagrams and attach fails with `remote UDP transport unavailable: probe UDP transport`. Allow inbound UDP for that range on the remote to fix it — for example `sudo ufw allow 61000:61023/udp`, or on a trusted Tailscale network `sudo ufw allow in on tailscale0`. Override the range with `VEV_UDP_PORT_RANGE` on the remote (e.g. `VEV_UDP_PORT_RANGE=51000-51009`, a single `VEV_UDP_PORT_RANGE=51000`, or `VEV_UDP_PORT_RANGE=0` for a random ephemeral port). | |
| 125 | + | vev's resilient remote attach work is inspired by [mosh](https://mosh.org/)'s mobile-shell approach to UDP-based recovery across flaky networks. vev uses its own wire protocol and implementation; it is not compatible with mosh clients or servers and does not copy mosh protocol code. | |
| 126 | 126 | ||
| 127 | - | Set `VEV_REMOTE_TRANSPORT=stdio` to use SSH stdio compatibility mode on networks where direct UDP is blocked or the SSH target is only reachable through jump-host or SSH-config-only routing. Stdio mode keeps all traffic inside SSH, but it may not notice sleep or Wi-Fi loss promptly. | |
| 127 | + | The default remote transport uses UDP. Its proxy binds a port in a fixed range on the remote (default `61000-61023`), so a host firewall that only allows SSH will drop the session datagrams and attach fails with `remote UDP transport unavailable: probe UDP transport`. Allow inbound UDP for that range on the remote to fix it — for example `sudo ufw allow 61000:61023/udp`, or on a trusted Tailscale network `sudo ufw allow in on tailscale0`. Override the range with `VEV_UDP_PORT_RANGE` on the remote (e.g. `VEV_UDP_PORT_RANGE=51000-51009`, a single `VEV_UDP_PORT_RANGE=51000`, or `VEV_UDP_PORT_RANGE=0` for a random ephemeral port). | |
| 128 | + | ||
| 129 | + | Set `VEV_REMOTE_TRANSPORT=stdio` to use SSH stdio compatibility mode on networks where direct UDP is blocked or the SSH target is only reachable through jump-host or SSH-config-only routing. Stdio mode keeps all traffic inside SSH and does not require inbound UDP, but it may not notice sleep or Wi-Fi loss promptly. See `docs/remote-resilience.md` for concise operational notes. | |
| 128 | 130 | ||
| 129 | 131 | ## Terminal color | |
| 130 | 132 | ||
| @@ -122,9 +122,11 @@ vev attach user@host:session | |||||
| 122 | 122 | ||||
| 123 | Remote attach uses SSH only to start a UDP proxy on the remote host; the session itself then talks directly to the remote host over UDP. `vev` must be installed on the remote host, and the host name you pass to `vev attach` must be reachable both by SSH for bootstrap and by UDP for the session transport. Omitting `:session` opens an ephemeral remote session. Remote sessions can resume after temporary network disconnects, such as sleep or Wi-Fi changes, while the daemon still has the session. Named sessions additionally persist across daemon restarts. | 123 | Remote attach uses SSH only to start a UDP proxy on the remote host; the session itself then talks directly to the remote host over UDP. `vev` must be installed on the remote host, and the host name you pass to `vev attach` must be reachable both by SSH for bootstrap and by UDP for the session transport. Omitting `:session` opens an ephemeral remote session. Remote sessions can resume after temporary network disconnects, such as sleep or Wi-Fi changes, while the daemon still has the session. Named sessions additionally persist across daemon restarts. | ||
| 124 | 124 | ||||
| 125 | - | The UDP proxy binds a port in a fixed range on the remote (default `61000-61023`), so a host firewall that only allows SSH will drop the session datagrams and attach fails with `remote UDP transport unavailable: probe UDP transport`. Allow inbound UDP for that range on the remote to fix it — for example `sudo ufw allow 61000:61023/udp`, or on a trusted Tailscale network `sudo ufw allow in on tailscale0`. Override the range with `VEV_UDP_PORT_RANGE` on the remote (e.g. `VEV_UDP_PORT_RANGE=51000-51009`, a single `VEV_UDP_PORT_RANGE=51000`, or `VEV_UDP_PORT_RANGE=0` for a random ephemeral port). | 125 | + | vev's resilient remote attach work is inspired by [mosh](https://mosh.org/)'s mobile-shell approach to UDP-based recovery across flaky networks. vev uses its own wire protocol and implementation; it is not compatible with mosh clients or servers and does not copy mosh protocol code. |
| 126 | 126 | ||||
| 127 | - | Set `VEV_REMOTE_TRANSPORT=stdio` to use SSH stdio compatibility mode on networks where direct UDP is blocked or the SSH target is only reachable through jump-host or SSH-config-only routing. Stdio mode keeps all traffic inside SSH, but it may not notice sleep or Wi-Fi loss promptly. | 127 | + | The default remote transport uses UDP. Its proxy binds a port in a fixed range on the remote (default `61000-61023`), so a host firewall that only allows SSH will drop the session datagrams and attach fails with `remote UDP transport unavailable: probe UDP transport`. Allow inbound UDP for that range on the remote to fix it — for example `sudo ufw allow 61000:61023/udp`, or on a trusted Tailscale network `sudo ufw allow in on tailscale0`. Override the range with `VEV_UDP_PORT_RANGE` on the remote (e.g. `VEV_UDP_PORT_RANGE=51000-51009`, a single `VEV_UDP_PORT_RANGE=51000`, or `VEV_UDP_PORT_RANGE=0` for a random ephemeral port). |
| 128 | + | ||||
| 129 | + | Set `VEV_REMOTE_TRANSPORT=stdio` to use SSH stdio compatibility mode on networks where direct UDP is blocked or the SSH target is only reachable through jump-host or SSH-config-only routing. Stdio mode keeps all traffic inside SSH and does not require inbound UDP, but it may not notice sleep or Wi-Fi loss promptly. See `docs/remote-resilience.md` for concise operational notes. | |||
| 128 | 130 | ||||
| 129 | ## Terminal color | 131 | ## Terminal color | ||
| 130 | 132 | ||||
A docs/remote-resilience.md +37 -0
| @@ -0,0 +1,37 @@ | |||
| 1 | + | # Remote resilience | |
| 2 | + | ||
| 3 | + | Remote attach starts with SSH bootstrap and then carries the vev session over authenticated UDP. The design is inspired by mosh's mobile-shell resilience, but vev uses its own protocol and is not compatible with mosh clients or servers. | |
| 4 | + | ||
| 5 | + | ## Operational model | |
| 6 | + | ||
| 7 | + | - SSH bootstrap starts or finds the remote UDP endpoint. | |
| 8 | + | - UDP carries normal session traffic after bootstrap. | |
| 9 | + | - Named sessions remain recoverable across daemon restarts; ephemeral sessions remain recoverable only while the daemon retains them. | |
| 10 | + | - `VEV_REMOTE_TRANSPORT=stdio` keeps all traffic inside SSH when direct UDP is blocked. | |
| 11 | + | ||
| 12 | + | ## Recovery states | |
| 13 | + | ||
| 14 | + | Resilience work uses these user-facing states: | |
| 15 | + | ||
| 16 | + | | State | Meaning | | |
| 17 | + | | --- | --- | | |
| 18 | + | | `connected` | authenticated UDP traffic is flowing | | |
| 19 | + | | `degraded` | traffic is delayed or incomplete, but recovery is still expected | | |
| 20 | + | | `probing` | vev is trying alternate UDP paths before using SSH again | | |
| 21 | + | | `resuming` | a replacement transport is attaching to the same session | | |
| 22 | + | | `offline` | no path currently works, but retries continue | | |
| 23 | + | | `expired` | the session or recovery window is no longer available | | |
| 24 | + | ||
| 25 | + | When a terminal is in raw mode, recovery messages should appear as vev status text rather than as bytes written into the remote program's output stream. | |
| 26 | + | ||
| 27 | + | ## Manual checks | |
| 28 | + | ||
| 29 | + | Useful scenarios for validating resilience behavior: | |
| 30 | + | ||
| 31 | + | - block and unblock the configured UDP port range; | |
| 32 | + | - move between networks or change VPN state; | |
| 33 | + | - suspend and resume the client machine; | |
| 34 | + | - retry `vev attach user@host:session` for the same named session; | |
| 35 | + | - compare logs for UDP silence, probing, SSH bootstrap failure, proxy expiry, and missing sessions. | |
| 36 | + | ||
| 37 | + | Keep hostnames, usernames, and local paths out of shared logs and screenshots. | |
| @@ -0,0 +1,37 @@ | |||||
| 1 | + | # Remote resilience | |||
| 2 | + | ||||
| 3 | + | Remote attach starts with SSH bootstrap and then carries the vev session over authenticated UDP. The design is inspired by mosh's mobile-shell resilience, but vev uses its own protocol and is not compatible with mosh clients or servers. | |||
| 4 | + | ||||
| 5 | + | ## Operational model | |||
| 6 | + | ||||
| 7 | + | - SSH bootstrap starts or finds the remote UDP endpoint. | |||
| 8 | + | - UDP carries normal session traffic after bootstrap. | |||
| 9 | + | - Named sessions remain recoverable across daemon restarts; ephemeral sessions remain recoverable only while the daemon retains them. | |||
| 10 | + | - `VEV_REMOTE_TRANSPORT=stdio` keeps all traffic inside SSH when direct UDP is blocked. | |||
| 11 | + | ||||
| 12 | + | ## Recovery states | |||
| 13 | + | ||||
| 14 | + | Resilience work uses these user-facing states: | |||
| 15 | + | ||||
| 16 | + | | State | Meaning | | |||
| 17 | + | | --- | --- | | |||
| 18 | + | | `connected` | authenticated UDP traffic is flowing | | |||
| 19 | + | | `degraded` | traffic is delayed or incomplete, but recovery is still expected | | |||
| 20 | + | | `probing` | vev is trying alternate UDP paths before using SSH again | | |||
| 21 | + | | `resuming` | a replacement transport is attaching to the same session | | |||
| 22 | + | | `offline` | no path currently works, but retries continue | | |||
| 23 | + | | `expired` | the session or recovery window is no longer available | | |||
| 24 | + | ||||
| 25 | + | When a terminal is in raw mode, recovery messages should appear as vev status text rather than as bytes written into the remote program's output stream. | |||
| 26 | + | ||||
| 27 | + | ## Manual checks | |||
| 28 | + | ||||
| 29 | + | Useful scenarios for validating resilience behavior: | |||
| 30 | + | ||||
| 31 | + | - block and unblock the configured UDP port range; | |||
| 32 | + | - move between networks or change VPN state; | |||
| 33 | + | - suspend and resume the client machine; | |||
| 34 | + | - retry `vev attach user@host:session` for the same named session; | |||
| 35 | + | - compare logs for UDP silence, probing, SSH bootstrap failure, proxy expiry, and missing sessions. | |||
| 36 | + | ||||
| 37 | + | Keep hostnames, usernames, and local paths out of shared logs and screenshots. | |||
A internal/adapters/dgram/proxy.go +190 -0
| @@ -0,0 +1,190 @@ | |||
| 1 | + | package dgram | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "errors" | |
| 6 | + | "io" | |
| 7 | + | "log/slog" | |
| 8 | + | "time" | |
| 9 | + | ||
| 10 | + | "github.com/bnema/vev/internal/ports" | |
| 11 | + | ) | |
| 12 | + | ||
| 13 | + | const ( | |
| 14 | + | DefaultProxyIdleTTL = 30 * time.Second | |
| 15 | + | DefaultProxyRetryBackoff = 100 * time.Millisecond | |
| 16 | + | ) | |
| 17 | + | ||
| 18 | + | type ProxyRuntime struct { | |
| 19 | + | Client ports.Transport | |
| 20 | + | Daemon ports.Transport | |
| 21 | + | Log *slog.Logger | |
| 22 | + | IdleTTL time.Duration | |
| 23 | + | RetryBackoff time.Duration | |
| 24 | + | Clock ports.Clock | |
| 25 | + | } | |
| 26 | + | ||
| 27 | + | type proxyCopyErrKind uint8 | |
| 28 | + | ||
| 29 | + | const ( | |
| 30 | + | proxyClientRecv proxyCopyErrKind = iota | |
| 31 | + | proxyDaemonRecv | |
| 32 | + | proxyClientSend | |
| 33 | + | proxyDaemonSend | |
| 34 | + | ) | |
| 35 | + | ||
| 36 | + | type proxyCopyErr struct { | |
| 37 | + | kind proxyCopyErrKind | |
| 38 | + | err error | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | func (p ProxyRuntime) Run(ctx context.Context) error { | |
| 42 | + | defer func() { | |
| 43 | + | _ = p.Client.Close() | |
| 44 | + | _ = p.Daemon.Close() | |
| 45 | + | }() | |
| 46 | + | ||
| 47 | + | idleTTL := p.IdleTTL | |
| 48 | + | if idleTTL <= 0 { | |
| 49 | + | idleTTL = DefaultProxyIdleTTL | |
| 50 | + | } | |
| 51 | + | retryBackoff := p.RetryBackoff | |
| 52 | + | if retryBackoff <= 0 { | |
| 53 | + | retryBackoff = DefaultProxyRetryBackoff | |
| 54 | + | } | |
| 55 | + | clk := p.Clock | |
| 56 | + | if clk == nil { | |
| 57 | + | clk = realClock{} | |
| 58 | + | } | |
| 59 | + | ||
| 60 | + | errCh := make(chan proxyCopyErr, 2) | |
| 61 | + | startClientCopy := func() { | |
| 62 | + | go copyProxyFrames(p.Daemon, p.Client, proxyClientRecv, proxyDaemonSend, errCh) | |
| 63 | + | } | |
| 64 | + | startDaemonCopy := func() { | |
| 65 | + | go copyProxyFrames(p.Client, p.Daemon, proxyDaemonRecv, proxyClientSend, errCh) | |
| 66 | + | } | |
| 67 | + | startClientCopy() | |
| 68 | + | startDaemonCopy() | |
| 69 | + | ||
| 70 | + | var linkEvents <-chan ports.LinkEvent | |
| 71 | + | if reporter, ok := p.Client.(ports.LinkStateReporter); ok { | |
| 72 | + | linkEvents = reporter.LinkEvents() | |
| 73 | + | } | |
| 74 | + | var idle <-chan time.Time | |
| 75 | + | var timer ports.Timer | |
| 76 | + | defer func() { | |
| 77 | + | if timer != nil { | |
| 78 | + | timer.Stop() | |
| 79 | + | } | |
| 80 | + | }() | |
| 81 | + | startIdle := func() { | |
| 82 | + | if timer == nil { | |
| 83 | + | timer = clk.NewTimer(idleTTL) | |
| 84 | + | } else { | |
| 85 | + | timer.Reset(idleTTL) | |
| 86 | + | } | |
| 87 | + | idle = timer.C() | |
| 88 | + | } | |
| 89 | + | stopIdle := func() { | |
| 90 | + | if timer != nil { | |
| 91 | + | timer.Stop() | |
| 92 | + | } | |
| 93 | + | idle = nil | |
| 94 | + | } | |
| 95 | + | waitRetry := func() error { | |
| 96 | + | t := clk.NewTimer(retryBackoff) | |
| 97 | + | defer t.Stop() | |
| 98 | + | select { | |
| 99 | + | case <-t.C(): | |
| 100 | + | return nil | |
| 101 | + | case <-ctx.Done(): | |
| 102 | + | return ctx.Err() | |
| 103 | + | } | |
| 104 | + | } | |
| 105 | + | for { | |
| 106 | + | select { | |
| 107 | + | case <-ctx.Done(): | |
| 108 | + | return ctx.Err() | |
| 109 | + | case ev, ok := <-linkEvents: | |
| 110 | + | if !ok { | |
| 111 | + | linkEvents = nil | |
| 112 | + | continue | |
| 113 | + | } | |
| 114 | + | switch ev.State { | |
| 115 | + | case ports.LinkStateDegraded, ports.LinkStateProbing, ports.LinkStateOffline: | |
| 116 | + | startIdle() | |
| 117 | + | case ports.LinkStateConnected: | |
| 118 | + | stopIdle() | |
| 119 | + | case ports.LinkStateDead: | |
| 120 | + | return ErrLinkDead | |
| 121 | + | } | |
| 122 | + | case <-idle: | |
| 123 | + | return ErrLinkDead | |
| 124 | + | case ce := <-errCh: | |
| 125 | + | switch ce.kind { | |
| 126 | + | case proxyDaemonRecv: | |
| 127 | + | if errors.Is(ce.err, io.EOF) { | |
| 128 | + | return nil | |
| 129 | + | } | |
| 130 | + | return ce.err | |
| 131 | + | case proxyDaemonSend: | |
| 132 | + | return ce.err | |
| 133 | + | case proxyClientSend: | |
| 134 | + | if !recoverableClientLink(p.Client, ce.err) { | |
| 135 | + | return ce.err | |
| 136 | + | } | |
| 137 | + | if p.Log != nil { | |
| 138 | + | p.Log.Info("udp proxy restarting recoverable client send path", "err", ce.err) | |
| 139 | + | } | |
| 140 | + | startIdle() | |
| 141 | + | if err := waitRetry(); err != nil { | |
| 142 | + | return err | |
| 143 | + | } | |
| 144 | + | startDaemonCopy() | |
| 145 | + | case proxyClientRecv: | |
| 146 | + | if !recoverableClientLink(p.Client, ce.err) { | |
| 147 | + | return ce.err | |
| 148 | + | } | |
| 149 | + | if p.Log != nil { | |
| 150 | + | p.Log.Info("udp proxy restarting recoverable client receive path", "err", ce.err) | |
| 151 | + | } | |
| 152 | + | startIdle() | |
| 153 | + | if err := waitRetry(); err != nil { | |
| 154 | + | return err | |
| 155 | + | } | |
| 156 | + | startClientCopy() | |
| 157 | + | } | |
| 158 | + | } | |
| 159 | + | } | |
| 160 | + | } | |
| 161 | + | ||
| 162 | + | func copyProxyFrames(dst, src ports.Transport, recvKind, sendKind proxyCopyErrKind, errCh chan<- proxyCopyErr) { | |
| 163 | + | for { | |
| 164 | + | f, err := src.Recv() | |
| 165 | + | if err != nil { | |
| 166 | + | errCh <- proxyCopyErr{kind: recvKind, err: err} | |
| 167 | + | return | |
| 168 | + | } | |
| 169 | + | if err := dst.Send(f); err != nil { | |
| 170 | + | errCh <- proxyCopyErr{kind: sendKind, err: err} | |
| 171 | + | return | |
| 172 | + | } | |
| 173 | + | } | |
| 174 | + | } | |
| 175 | + | ||
| 176 | + | func recoverableClientLink(t ports.Transport, err error) bool { | |
| 177 | + | if errors.Is(err, ErrLinkDead) { | |
| 178 | + | return false | |
| 179 | + | } | |
| 180 | + | reporter, ok := t.(ports.LinkStateReporter) | |
| 181 | + | if !ok { | |
| 182 | + | return false | |
| 183 | + | } | |
| 184 | + | switch reporter.LinkState() { | |
| 185 | + | case ports.LinkStateDegraded, ports.LinkStateProbing, ports.LinkStateOffline: | |
| 186 | + | return true | |
| 187 | + | default: | |
| 188 | + | return false | |
| 189 | + | } | |
| 190 | + | } | |
| @@ -0,0 +1,190 @@ | |||||
| 1 | + | package dgram | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "errors" | |||
| 6 | + | "io" | |||
| 7 | + | "log/slog" | |||
| 8 | + | "time" | |||
| 9 | + | ||||
| 10 | + | "github.com/bnema/vev/internal/ports" | |||
| 11 | + | ) | |||
| 12 | + | ||||
| 13 | + | const ( | |||
| 14 | + | DefaultProxyIdleTTL = 30 * time.Second | |||
| 15 | + | DefaultProxyRetryBackoff = 100 * time.Millisecond | |||
| 16 | + | ) | |||
| 17 | + | ||||
| 18 | + | type ProxyRuntime struct { | |||
| 19 | + | Client ports.Transport | |||
| 20 | + | Daemon ports.Transport | |||
| 21 | + | Log *slog.Logger | |||
| 22 | + | IdleTTL time.Duration | |||
| 23 | + | RetryBackoff time.Duration | |||
| 24 | + | Clock ports.Clock | |||
| 25 | + | } | |||
| 26 | + | ||||
| 27 | + | type proxyCopyErrKind uint8 | |||
| 28 | + | ||||
| 29 | + | const ( | |||
| 30 | + | proxyClientRecv proxyCopyErrKind = iota | |||
| 31 | + | proxyDaemonRecv | |||
| 32 | + | proxyClientSend | |||
| 33 | + | proxyDaemonSend | |||
| 34 | + | ) | |||
| 35 | + | ||||
| 36 | + | type proxyCopyErr struct { | |||
| 37 | + | kind proxyCopyErrKind | |||
| 38 | + | err error | |||
| 39 | + | } | |||
| 40 | + | ||||
| 41 | + | func (p ProxyRuntime) Run(ctx context.Context) error { | |||
| 42 | + | defer func() { | |||
| 43 | + | _ = p.Client.Close() | |||
| 44 | + | _ = p.Daemon.Close() | |||
| 45 | + | }() | |||
| 46 | + | ||||
| 47 | + | idleTTL := p.IdleTTL | |||
| 48 | + | if idleTTL <= 0 { | |||
| 49 | + | idleTTL = DefaultProxyIdleTTL | |||
| 50 | + | } | |||
| 51 | + | retryBackoff := p.RetryBackoff | |||
| 52 | + | if retryBackoff <= 0 { | |||
| 53 | + | retryBackoff = DefaultProxyRetryBackoff | |||
| 54 | + | } | |||
| 55 | + | clk := p.Clock | |||
| 56 | + | if clk == nil { | |||
| 57 | + | clk = realClock{} | |||
| 58 | + | } | |||
| 59 | + | ||||
| 60 | + | errCh := make(chan proxyCopyErr, 2) | |||
| 61 | + | startClientCopy := func() { | |||
| 62 | + | go copyProxyFrames(p.Daemon, p.Client, proxyClientRecv, proxyDaemonSend, errCh) | |||
| 63 | + | } | |||
| 64 | + | startDaemonCopy := func() { | |||
| 65 | + | go copyProxyFrames(p.Client, p.Daemon, proxyDaemonRecv, proxyClientSend, errCh) | |||
| 66 | + | } | |||
| 67 | + | startClientCopy() | |||
| 68 | + | startDaemonCopy() | |||
| 69 | + | ||||
| 70 | + | var linkEvents <-chan ports.LinkEvent | |||
| 71 | + | if reporter, ok := p.Client.(ports.LinkStateReporter); ok { | |||
| 72 | + | linkEvents = reporter.LinkEvents() | |||
| 73 | + | } | |||
| 74 | + | var idle <-chan time.Time | |||
| 75 | + | var timer ports.Timer | |||
| 76 | + | defer func() { | |||
| 77 | + | if timer != nil { | |||
| 78 | + | timer.Stop() | |||
| 79 | + | } | |||
| 80 | + | }() | |||
| 81 | + | startIdle := func() { | |||
| 82 | + | if timer == nil { | |||
| 83 | + | timer = clk.NewTimer(idleTTL) | |||
| 84 | + | } else { | |||
| 85 | + | timer.Reset(idleTTL) | |||
| 86 | + | } | |||
| 87 | + | idle = timer.C() | |||
| 88 | + | } | |||
| 89 | + | stopIdle := func() { | |||
| 90 | + | if timer != nil { | |||
| 91 | + | timer.Stop() | |||
| 92 | + | } | |||
| 93 | + | idle = nil | |||
| 94 | + | } | |||
| 95 | + | waitRetry := func() error { | |||
| 96 | + | t := clk.NewTimer(retryBackoff) | |||
| 97 | + | defer t.Stop() | |||
| 98 | + | select { | |||
| 99 | + | case <-t.C(): | |||
| 100 | + | return nil | |||
| 101 | + | case <-ctx.Done(): | |||
| 102 | + | return ctx.Err() | |||
| 103 | + | } | |||
| 104 | + | } | |||
| 105 | + | for { | |||
| 106 | + | select { | |||
| 107 | + | case <-ctx.Done(): | |||
| 108 | + | return ctx.Err() | |||
| 109 | + | case ev, ok := <-linkEvents: | |||
| 110 | + | if !ok { | |||
| 111 | + | linkEvents = nil | |||
| 112 | + | continue | |||
| 113 | + | } | |||
| 114 | + | switch ev.State { | |||
| 115 | + | case ports.LinkStateDegraded, ports.LinkStateProbing, ports.LinkStateOffline: | |||
| 116 | + | startIdle() | |||
| 117 | + | case ports.LinkStateConnected: | |||
| 118 | + | stopIdle() | |||
| 119 | + | case ports.LinkStateDead: | |||
| 120 | + | return ErrLinkDead | |||
| 121 | + | } | |||
| 122 | + | case <-idle: | |||
| 123 | + | return ErrLinkDead | |||
| 124 | + | case ce := <-errCh: | |||
| 125 | + | switch ce.kind { | |||
| 126 | + | case proxyDaemonRecv: | |||
| 127 | + | if errors.Is(ce.err, io.EOF) { | |||
| 128 | + | return nil | |||
| 129 | + | } | |||
| 130 | + | return ce.err | |||
| 131 | + | case proxyDaemonSend: | |||
| 132 | + | return ce.err | |||
| 133 | + | case proxyClientSend: | |||
| 134 | + | if !recoverableClientLink(p.Client, ce.err) { | |||
| 135 | + | return ce.err | |||
| 136 | + | } | |||
| 137 | + | if p.Log != nil { | |||
| 138 | + | p.Log.Info("udp proxy restarting recoverable client send path", "err", ce.err) | |||
| 139 | + | } | |||
| 140 | + | startIdle() | |||
| 141 | + | if err := waitRetry(); err != nil { | |||
| 142 | + | return err | |||
| 143 | + | } | |||
| 144 | + | startDaemonCopy() | |||
| 145 | + | case proxyClientRecv: | |||
| 146 | + | if !recoverableClientLink(p.Client, ce.err) { | |||
| 147 | + | return ce.err | |||
| 148 | + | } | |||
| 149 | + | if p.Log != nil { | |||
| 150 | + | p.Log.Info("udp proxy restarting recoverable client receive path", "err", ce.err) | |||
| 151 | + | } | |||
| 152 | + | startIdle() | |||
| 153 | + | if err := waitRetry(); err != nil { | |||
| 154 | + | return err | |||
| 155 | + | } | |||
| 156 | + | startClientCopy() | |||
| 157 | + | } | |||
| 158 | + | } | |||
| 159 | + | } | |||
| 160 | + | } | |||
| 161 | + | ||||
| 162 | + | func copyProxyFrames(dst, src ports.Transport, recvKind, sendKind proxyCopyErrKind, errCh chan<- proxyCopyErr) { | |||
| 163 | + | for { | |||
| 164 | + | f, err := src.Recv() | |||
| 165 | + | if err != nil { | |||
| 166 | + | errCh <- proxyCopyErr{kind: recvKind, err: err} | |||
| 167 | + | return | |||
| 168 | + | } | |||
| 169 | + | if err := dst.Send(f); err != nil { | |||
| 170 | + | errCh <- proxyCopyErr{kind: sendKind, err: err} | |||
| 171 | + | return | |||
| 172 | + | } | |||
| 173 | + | } | |||
| 174 | + | } | |||
| 175 | + | ||||
| 176 | + | func recoverableClientLink(t ports.Transport, err error) bool { | |||
| 177 | + | if errors.Is(err, ErrLinkDead) { | |||
| 178 | + | return false | |||
| 179 | + | } | |||
| 180 | + | reporter, ok := t.(ports.LinkStateReporter) | |||
| 181 | + | if !ok { | |||
| 182 | + | return false | |||
| 183 | + | } | |||
| 184 | + | switch reporter.LinkState() { | |||
| 185 | + | case ports.LinkStateDegraded, ports.LinkStateProbing, ports.LinkStateOffline: | |||
| 186 | + | return true | |||
| 187 | + | default: | |||
| 188 | + | return false | |||
| 189 | + | } | |||
| 190 | + | } | |||
A internal/adapters/dgram/proxy_registry.go +150 -0
| @@ -0,0 +1,150 @@ | |||
| 1 | + | package dgram | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "encoding/base64" | |
| 5 | + | "encoding/json" | |
| 6 | + | "errors" | |
| 7 | + | "fmt" | |
| 8 | + | "os" | |
| 9 | + | "path/filepath" | |
| 10 | + | "syscall" | |
| 11 | + | "time" | |
| 12 | + | ) | |
| 13 | + | ||
| 14 | + | type ProxyRecord struct { | |
| 15 | + | Session string `json:"session"` | |
| 16 | + | PID int `json:"pid"` | |
| 17 | + | Port int `json:"port"` | |
| 18 | + | Key string `json:"key"` | |
| 19 | + | Created time.Time `json:"created"` | |
| 20 | + | } | |
| 21 | + | ||
| 22 | + | func (r ProxyRecord) ReadyLine() string { return fmt.Sprintf("VEV-UDP %d %s\n", r.Port, r.Key) } | |
| 23 | + | ||
| 24 | + | type ProxyRegistry struct{ dir string } | |
| 25 | + | ||
| 26 | + | func NewProxyRegistry(dir string) *ProxyRegistry { return &ProxyRegistry{dir: dir} } | |
| 27 | + | ||
| 28 | + | func (r *ProxyRegistry) Lookup(session string) (ProxyRecord, bool) { | |
| 29 | + | var rec ProxyRecord | |
| 30 | + | ok := false | |
| 31 | + | _ = r.withLock(func() error { | |
| 32 | + | var err error | |
| 33 | + | rec, err = r.read(session) | |
| 34 | + | if err != nil { | |
| 35 | + | return nil | |
| 36 | + | } | |
| 37 | + | if !processAlive(rec.PID) { | |
| 38 | + | _ = r.removeLocked(session) | |
| 39 | + | return nil | |
| 40 | + | } | |
| 41 | + | if rec.PID != os.Getpid() { | |
| 42 | + | return nil | |
| 43 | + | } | |
| 44 | + | ok = true | |
| 45 | + | return nil | |
| 46 | + | }) | |
| 47 | + | return rec, ok | |
| 48 | + | } | |
| 49 | + | ||
| 50 | + | func (r *ProxyRegistry) Publish(rec ProxyRecord) error { | |
| 51 | + | if rec.Created.IsZero() { | |
| 52 | + | rec.Created = time.Now() | |
| 53 | + | } | |
| 54 | + | return r.withLock(func() error { | |
| 55 | + | b, err := json.Marshal(rec) | |
| 56 | + | if err != nil { | |
| 57 | + | return err | |
| 58 | + | } | |
| 59 | + | tmp, err := os.CreateTemp(r.dir, ".udp-proxy-*.tmp") | |
| 60 | + | if err != nil { | |
| 61 | + | return err | |
| 62 | + | } | |
| 63 | + | tmpName := tmp.Name() | |
| 64 | + | defer func() { _ = os.Remove(tmpName) }() | |
| 65 | + | if _, err := tmp.Write(b); err != nil { | |
| 66 | + | _ = tmp.Close() | |
| 67 | + | return err | |
| 68 | + | } | |
| 69 | + | if err := tmp.Close(); err != nil { | |
| 70 | + | return err | |
| 71 | + | } | |
| 72 | + | return os.Rename(tmpName, r.path(rec.Session)) | |
| 73 | + | }) | |
| 74 | + | } | |
| 75 | + | ||
| 76 | + | func (r *ProxyRegistry) Remove(session string) error { | |
| 77 | + | return r.withLock(func() error { return r.removeLocked(session) }) | |
| 78 | + | } | |
| 79 | + | ||
| 80 | + | func (r *ProxyRegistry) RemoveOwned(rec ProxyRecord) error { | |
| 81 | + | return r.withLock(func() error { | |
| 82 | + | cur, err := r.read(rec.Session) | |
| 83 | + | if errors.Is(err, os.ErrNotExist) { | |
| 84 | + | return nil | |
| 85 | + | } | |
| 86 | + | if err != nil { | |
| 87 | + | return err | |
| 88 | + | } | |
| 89 | + | if cur.PID != rec.PID || cur.Port != rec.Port || cur.Key != rec.Key { | |
| 90 | + | return nil | |
| 91 | + | } | |
| 92 | + | return r.removeLocked(rec.Session) | |
| 93 | + | }) | |
| 94 | + | } | |
| 95 | + | ||
| 96 | + | func (r *ProxyRegistry) withLock(fn func() error) error { | |
| 97 | + | if err := os.MkdirAll(r.dir, 0o700); err != nil { | |
| 98 | + | return err | |
| 99 | + | } | |
| 100 | + | lock, err := os.OpenFile(filepath.Join(r.dir, ".lock"), os.O_CREATE|os.O_RDWR, 0o600) | |
| 101 | + | if err != nil { | |
| 102 | + | return err | |
| 103 | + | } | |
| 104 | + | defer func() { _ = lock.Close() }() | |
| 105 | + | if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil { | |
| 106 | + | return err | |
| 107 | + | } | |
| 108 | + | defer func() { _ = syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) }() | |
| 109 | + | return fn() | |
| 110 | + | } | |
| 111 | + | ||
| 112 | + | func (r *ProxyRegistry) removeLocked(session string) error { | |
| 113 | + | err := os.Remove(r.path(session)) | |
| 114 | + | if errors.Is(err, os.ErrNotExist) { | |
| 115 | + | return nil | |
| 116 | + | } | |
| 117 | + | return err | |
| 118 | + | } | |
| 119 | + | ||
| 120 | + | func (r *ProxyRegistry) read(session string) (ProxyRecord, error) { | |
| 121 | + | b, err := os.ReadFile(r.path(session)) | |
| 122 | + | if err != nil { | |
| 123 | + | return ProxyRecord{}, err | |
| 124 | + | } | |
| 125 | + | var rec ProxyRecord | |
| 126 | + | if err := json.Unmarshal(b, &rec); err != nil { | |
| 127 | + | return ProxyRecord{}, err | |
| 128 | + | } | |
| 129 | + | return rec, nil | |
| 130 | + | } | |
| 131 | + | ||
| 132 | + | func (r *ProxyRegistry) path(session string) string { | |
| 133 | + | if session == "" { | |
| 134 | + | return filepath.Join(r.dir, "udp-proxy-empty.json") | |
| 135 | + | } | |
| 136 | + | name := base64.RawURLEncoding.EncodeToString([]byte(session)) | |
| 137 | + | return filepath.Join(r.dir, "udp-proxy-"+name+".json") | |
| 138 | + | } | |
| 139 | + | ||
| 140 | + | func processAlive(pid int) bool { | |
| 141 | + | if pid <= 0 { | |
| 142 | + | return false | |
| 143 | + | } | |
| 144 | + | p, err := os.FindProcess(pid) | |
| 145 | + | if err != nil { | |
| 146 | + | return false | |
| 147 | + | } | |
| 148 | + | err = p.Signal(syscall.Signal(0)) | |
| 149 | + | return err == nil || errors.Is(err, syscall.EPERM) | |
| 150 | + | } | |
| @@ -0,0 +1,150 @@ | |||||
| 1 | + | package dgram | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "encoding/base64" | |||
| 5 | + | "encoding/json" | |||
| 6 | + | "errors" | |||
| 7 | + | "fmt" | |||
| 8 | + | "os" | |||
| 9 | + | "path/filepath" | |||
| 10 | + | "syscall" | |||
| 11 | + | "time" | |||
| 12 | + | ) | |||
| 13 | + | ||||
| 14 | + | type ProxyRecord struct { | |||
| 15 | + | Session string `json:"session"` | |||
| 16 | + | PID int `json:"pid"` | |||
| 17 | + | Port int `json:"port"` | |||
| 18 | + | Key string `json:"key"` | |||
| 19 | + | Created time.Time `json:"created"` | |||
| 20 | + | } | |||
| 21 | + | ||||
| 22 | + | func (r ProxyRecord) ReadyLine() string { return fmt.Sprintf("VEV-UDP %d %s\n", r.Port, r.Key) } | |||
| 23 | + | ||||
| 24 | + | type ProxyRegistry struct{ dir string } | |||
| 25 | + | ||||
| 26 | + | func NewProxyRegistry(dir string) *ProxyRegistry { return &ProxyRegistry{dir: dir} } | |||
| 27 | + | ||||
| 28 | + | func (r *ProxyRegistry) Lookup(session string) (ProxyRecord, bool) { | |||
| 29 | + | var rec ProxyRecord | |||
| 30 | + | ok := false | |||
| 31 | + | _ = r.withLock(func() error { | |||
| 32 | + | var err error | |||
| 33 | + | rec, err = r.read(session) | |||
| 34 | + | if err != nil { | |||
| 35 | + | return nil | |||
| 36 | + | } | |||
| 37 | + | if !processAlive(rec.PID) { | |||
| 38 | + | _ = r.removeLocked(session) | |||
| 39 | + | return nil | |||
| 40 | + | } | |||
| 41 | + | if rec.PID != os.Getpid() { | |||
| 42 | + | return nil | |||
| 43 | + | } | |||
| 44 | + | ok = true | |||
| 45 | + | return nil | |||
| 46 | + | }) | |||
| 47 | + | return rec, ok | |||
| 48 | + | } | |||
| 49 | + | ||||
| 50 | + | func (r *ProxyRegistry) Publish(rec ProxyRecord) error { | |||
| 51 | + | if rec.Created.IsZero() { | |||
| 52 | + | rec.Created = time.Now() | |||
| 53 | + | } | |||
| 54 | + | return r.withLock(func() error { | |||
| 55 | + | b, err := json.Marshal(rec) | |||
| 56 | + | if err != nil { | |||
| 57 | + | return err | |||
| 58 | + | } | |||
| 59 | + | tmp, err := os.CreateTemp(r.dir, ".udp-proxy-*.tmp") | |||
| 60 | + | if err != nil { | |||
| 61 | + | return err | |||
| 62 | + | } | |||
| 63 | + | tmpName := tmp.Name() | |||
| 64 | + | defer func() { _ = os.Remove(tmpName) }() | |||
| 65 | + | if _, err := tmp.Write(b); err != nil { | |||
| 66 | + | _ = tmp.Close() | |||
| 67 | + | return err | |||
| 68 | + | } | |||
| 69 | + | if err := tmp.Close(); err != nil { | |||
| 70 | + | return err | |||
| 71 | + | } | |||
| 72 | + | return os.Rename(tmpName, r.path(rec.Session)) | |||
| 73 | + | }) | |||
| 74 | + | } | |||
| 75 | + | ||||
| 76 | + | func (r *ProxyRegistry) Remove(session string) error { | |||
| 77 | + | return r.withLock(func() error { return r.removeLocked(session) }) | |||
| 78 | + | } | |||
| 79 | + | ||||
| 80 | + | func (r *ProxyRegistry) RemoveOwned(rec ProxyRecord) error { | |||
| 81 | + | return r.withLock(func() error { | |||
| 82 | + | cur, err := r.read(rec.Session) | |||
| 83 | + | if errors.Is(err, os.ErrNotExist) { | |||
| 84 | + | return nil | |||
| 85 | + | } | |||
| 86 | + | if err != nil { | |||
| 87 | + | return err | |||
| 88 | + | } | |||
| 89 | + | if cur.PID != rec.PID || cur.Port != rec.Port || cur.Key != rec.Key { | |||
| 90 | + | return nil | |||
| 91 | + | } | |||
| 92 | + | return r.removeLocked(rec.Session) | |||
| 93 | + | }) | |||
| 94 | + | } | |||
| 95 | + | ||||
| 96 | + | func (r *ProxyRegistry) withLock(fn func() error) error { | |||
| 97 | + | if err := os.MkdirAll(r.dir, 0o700); err != nil { | |||
| 98 | + | return err | |||
| 99 | + | } | |||
| 100 | + | lock, err := os.OpenFile(filepath.Join(r.dir, ".lock"), os.O_CREATE|os.O_RDWR, 0o600) | |||
| 101 | + | if err != nil { | |||
| 102 | + | return err | |||
| 103 | + | } | |||
| 104 | + | defer func() { _ = lock.Close() }() | |||
| 105 | + | if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil { | |||
| 106 | + | return err | |||
| 107 | + | } | |||
| 108 | + | defer func() { _ = syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) }() | |||
| 109 | + | return fn() | |||
| 110 | + | } | |||
| 111 | + | ||||
| 112 | + | func (r *ProxyRegistry) removeLocked(session string) error { | |||
| 113 | + | err := os.Remove(r.path(session)) | |||
| 114 | + | if errors.Is(err, os.ErrNotExist) { | |||
| 115 | + | return nil | |||
| 116 | + | } | |||
| 117 | + | return err | |||
| 118 | + | } | |||
| 119 | + | ||||
| 120 | + | func (r *ProxyRegistry) read(session string) (ProxyRecord, error) { | |||
| 121 | + | b, err := os.ReadFile(r.path(session)) | |||
| 122 | + | if err != nil { | |||
| 123 | + | return ProxyRecord{}, err | |||
| 124 | + | } | |||
| 125 | + | var rec ProxyRecord | |||
| 126 | + | if err := json.Unmarshal(b, &rec); err != nil { | |||
| 127 | + | return ProxyRecord{}, err | |||
| 128 | + | } | |||
| 129 | + | return rec, nil | |||
| 130 | + | } | |||
| 131 | + | ||||
| 132 | + | func (r *ProxyRegistry) path(session string) string { | |||
| 133 | + | if session == "" { | |||
| 134 | + | return filepath.Join(r.dir, "udp-proxy-empty.json") | |||
| 135 | + | } | |||
| 136 | + | name := base64.RawURLEncoding.EncodeToString([]byte(session)) | |||
| 137 | + | return filepath.Join(r.dir, "udp-proxy-"+name+".json") | |||
| 138 | + | } | |||
| 139 | + | ||||
| 140 | + | func processAlive(pid int) bool { | |||
| 141 | + | if pid <= 0 { | |||
| 142 | + | return false | |||
| 143 | + | } | |||
| 144 | + | p, err := os.FindProcess(pid) | |||
| 145 | + | if err != nil { | |||
| 146 | + | return false | |||
| 147 | + | } | |||
| 148 | + | err = p.Signal(syscall.Signal(0)) | |||
| 149 | + | return err == nil || errors.Is(err, syscall.EPERM) | |||
| 150 | + | } | |||
A internal/adapters/dgram/proxy_registry_test.go +98 -0
| @@ -0,0 +1,98 @@ | |||
| 1 | + | package dgram | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "os" | |
| 5 | + | "testing" | |
| 6 | + | ) | |
| 7 | + | ||
| 8 | + | func TestProxyRegistryReuseStaleCleanupAndSupersede(t *testing.T) { | |
| 9 | + | t.Run("reuses live matching record", func(t *testing.T) { | |
| 10 | + | r := NewProxyRegistry(t.TempDir()) | |
| 11 | + | want := ProxyRecord{Session: "work", PID: os.Getpid(), Port: 61000, Key: "abc"} | |
| 12 | + | if err := r.Publish(want); err != nil { | |
| 13 | + | t.Fatal(err) | |
| 14 | + | } | |
| 15 | + | got, ok := r.Lookup("work") | |
| 16 | + | if !ok { | |
| 17 | + | t.Fatal("Lookup ok=false, want true") | |
| 18 | + | } | |
| 19 | + | if got.Port != want.Port || got.Key != want.Key || got.PID != want.PID { | |
| 20 | + | t.Fatalf("Lookup()=%+v, want %+v", got, want) | |
| 21 | + | } | |
| 22 | + | }) | |
| 23 | + | t.Run("removes stale record", func(t *testing.T) { | |
| 24 | + | r := NewProxyRegistry(t.TempDir()) | |
| 25 | + | if err := r.Publish(ProxyRecord{Session: "work", PID: os.Getpid(), Port: 61000, Key: "abc"}); err != nil { | |
| 26 | + | t.Fatal(err) | |
| 27 | + | } | |
| 28 | + | if err := os.WriteFile(r.path("work"), []byte(`{"session":"work","pid":-1,"port":61000,"key":"abc"}`), 0o600); err != nil { | |
| 29 | + | t.Fatal(err) | |
| 30 | + | } | |
| 31 | + | if _, ok := r.Lookup("work"); ok { | |
| 32 | + | t.Fatal("Lookup ok=true for stale pid") | |
| 33 | + | } | |
| 34 | + | if _, err := os.Stat(r.path("work")); !os.IsNotExist(err) { | |
| 35 | + | t.Fatalf("stale record stat err=%v, want not exist", err) | |
| 36 | + | } | |
| 37 | + | }) | |
| 38 | + | t.Run("supersedes without pid-based kill and cleanup is owned", func(t *testing.T) { | |
| 39 | + | r := NewProxyRegistry(t.TempDir()) | |
| 40 | + | old := ProxyRecord{Session: "work", PID: os.Getpid(), Port: 61000, Key: "old"} | |
| 41 | + | if err := r.Publish(old); err != nil { | |
| 42 | + | t.Fatal(err) | |
| 43 | + | } | |
| 44 | + | newRec := ProxyRecord{Session: "work", PID: os.Getpid(), Port: 61001, Key: "new"} | |
| 45 | + | if err := r.Publish(newRec); err != nil { | |
| 46 | + | t.Fatal(err) | |
| 47 | + | } | |
| 48 | + | if err := r.RemoveOwned(old); err != nil { | |
| 49 | + | t.Fatal(err) | |
| 50 | + | } | |
| 51 | + | got, ok := r.Lookup("work") | |
| 52 | + | if !ok { | |
| 53 | + | t.Fatal("new live record removed by stale owner") | |
| 54 | + | } | |
| 55 | + | if got.Port != newRec.Port || got.Key != newRec.Key { | |
| 56 | + | t.Fatalf("Lookup()=%+v, want %+v", got, newRec) | |
| 57 | + | } | |
| 58 | + | if err := r.RemoveOwned(newRec); err != nil { | |
| 59 | + | t.Fatal(err) | |
| 60 | + | } | |
| 61 | + | if _, ok := r.Lookup("work"); ok { | |
| 62 | + | t.Fatal("owned remove left registry record") | |
| 63 | + | } | |
| 64 | + | }) | |
| 65 | + | } | |
| 66 | + | ||
| 67 | + | func TestProxyRegistrySessionPathIsCollisionFree(t *testing.T) { | |
| 68 | + | r := NewProxyRegistry(t.TempDir()) | |
| 69 | + | tests := []struct { | |
| 70 | + | a string | |
| 71 | + | b string | |
| 72 | + | }{ | |
| 73 | + | {a: "", b: "default"}, | |
| 74 | + | {a: "a/b", b: "a_b"}, | |
| 75 | + | {a: `a\b`, b: "a_b"}, | |
| 76 | + | } | |
| 77 | + | for _, tt := range tests { | |
| 78 | + | if r.path(tt.a) == r.path(tt.b) { | |
| 79 | + | t.Fatalf("path(%q) collides with path(%q): %s", tt.a, tt.b, r.path(tt.a)) | |
| 80 | + | } | |
| 81 | + | first := ProxyRecord{Session: tt.a, PID: os.Getpid(), Port: 61000, Key: "first"} | |
| 82 | + | second := ProxyRecord{Session: tt.b, PID: os.Getpid(), Port: 61001, Key: "second"} | |
| 83 | + | if err := r.Publish(first); err != nil { | |
| 84 | + | t.Fatal(err) | |
| 85 | + | } | |
| 86 | + | if err := r.Publish(second); err != nil { | |
| 87 | + | t.Fatal(err) | |
| 88 | + | } | |
| 89 | + | gotFirst, ok := r.Lookup(tt.a) | |
| 90 | + | if !ok || gotFirst.Key != first.Key { | |
| 91 | + | t.Fatalf("Lookup(%q)=%+v ok=%v, want first record", tt.a, gotFirst, ok) | |
| 92 | + | } | |
| 93 | + | gotSecond, ok := r.Lookup(tt.b) | |
| 94 | + | if !ok || gotSecond.Key != second.Key { | |
| 95 | + | t.Fatalf("Lookup(%q)=%+v ok=%v, want second record", tt.b, gotSecond, ok) | |
| 96 | + | } | |
| 97 | + | } | |
| 98 | + | } | |
| @@ -0,0 +1,98 @@ | |||||
| 1 | + | package dgram | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "os" | |||
| 5 | + | "testing" | |||
| 6 | + | ) | |||
| 7 | + | ||||
| 8 | + | func TestProxyRegistryReuseStaleCleanupAndSupersede(t *testing.T) { | |||
| 9 | + | t.Run("reuses live matching record", func(t *testing.T) { | |||
| 10 | + | r := NewProxyRegistry(t.TempDir()) | |||
| 11 | + | want := ProxyRecord{Session: "work", PID: os.Getpid(), Port: 61000, Key: "abc"} | |||
| 12 | + | if err := r.Publish(want); err != nil { | |||
| 13 | + | t.Fatal(err) | |||
| 14 | + | } | |||
| 15 | + | got, ok := r.Lookup("work") | |||
| 16 | + | if !ok { | |||
| 17 | + | t.Fatal("Lookup ok=false, want true") | |||
| 18 | + | } | |||
| 19 | + | if got.Port != want.Port || got.Key != want.Key || got.PID != want.PID { | |||
| 20 | + | t.Fatalf("Lookup()=%+v, want %+v", got, want) | |||
| 21 | + | } | |||
| 22 | + | }) | |||
| 23 | + | t.Run("removes stale record", func(t *testing.T) { | |||
| 24 | + | r := NewProxyRegistry(t.TempDir()) | |||
| 25 | + | if err := r.Publish(ProxyRecord{Session: "work", PID: os.Getpid(), Port: 61000, Key: "abc"}); err != nil { | |||
| 26 | + | t.Fatal(err) | |||
| 27 | + | } | |||
| 28 | + | if err := os.WriteFile(r.path("work"), []byte(`{"session":"work","pid":-1,"port":61000,"key":"abc"}`), 0o600); err != nil { | |||
| 29 | + | t.Fatal(err) | |||
| 30 | + | } | |||
| 31 | + | if _, ok := r.Lookup("work"); ok { | |||
| 32 | + | t.Fatal("Lookup ok=true for stale pid") | |||
| 33 | + | } | |||
| 34 | + | if _, err := os.Stat(r.path("work")); !os.IsNotExist(err) { | |||
| 35 | + | t.Fatalf("stale record stat err=%v, want not exist", err) | |||
| 36 | + | } | |||
| 37 | + | }) | |||
| 38 | + | t.Run("supersedes without pid-based kill and cleanup is owned", func(t *testing.T) { | |||
| 39 | + | r := NewProxyRegistry(t.TempDir()) | |||
| 40 | + | old := ProxyRecord{Session: "work", PID: os.Getpid(), Port: 61000, Key: "old"} | |||
| 41 | + | if err := r.Publish(old); err != nil { | |||
| 42 | + | t.Fatal(err) | |||
| 43 | + | } | |||
| 44 | + | newRec := ProxyRecord{Session: "work", PID: os.Getpid(), Port: 61001, Key: "new"} | |||
| 45 | + | if err := r.Publish(newRec); err != nil { | |||
| 46 | + | t.Fatal(err) | |||
| 47 | + | } | |||
| 48 | + | if err := r.RemoveOwned(old); err != nil { | |||
| 49 | + | t.Fatal(err) | |||
| 50 | + | } | |||
| 51 | + | got, ok := r.Lookup("work") | |||
| 52 | + | if !ok { | |||
| 53 | + | t.Fatal("new live record removed by stale owner") | |||
| 54 | + | } | |||
| 55 | + | if got.Port != newRec.Port || got.Key != newRec.Key { | |||
| 56 | + | t.Fatalf("Lookup()=%+v, want %+v", got, newRec) | |||
| 57 | + | } | |||
| 58 | + | if err := r.RemoveOwned(newRec); err != nil { | |||
| 59 | + | t.Fatal(err) | |||
| 60 | + | } | |||
| 61 | + | if _, ok := r.Lookup("work"); ok { | |||
| 62 | + | t.Fatal("owned remove left registry record") | |||
| 63 | + | } | |||
| 64 | + | }) | |||
| 65 | + | } | |||
| 66 | + | ||||
| 67 | + | func TestProxyRegistrySessionPathIsCollisionFree(t *testing.T) { | |||
| 68 | + | r := NewProxyRegistry(t.TempDir()) | |||
| 69 | + | tests := []struct { | |||
| 70 | + | a string | |||
| 71 | + | b string | |||
| 72 | + | }{ | |||
| 73 | + | {a: "", b: "default"}, | |||
| 74 | + | {a: "a/b", b: "a_b"}, | |||
| 75 | + | {a: `a\b`, b: "a_b"}, | |||
| 76 | + | } | |||
| 77 | + | for _, tt := range tests { | |||
| 78 | + | if r.path(tt.a) == r.path(tt.b) { | |||
| 79 | + | t.Fatalf("path(%q) collides with path(%q): %s", tt.a, tt.b, r.path(tt.a)) | |||
| 80 | + | } | |||
| 81 | + | first := ProxyRecord{Session: tt.a, PID: os.Getpid(), Port: 61000, Key: "first"} | |||
| 82 | + | second := ProxyRecord{Session: tt.b, PID: os.Getpid(), Port: 61001, Key: "second"} | |||
| 83 | + | if err := r.Publish(first); err != nil { | |||
| 84 | + | t.Fatal(err) | |||
| 85 | + | } | |||
| 86 | + | if err := r.Publish(second); err != nil { | |||
| 87 | + | t.Fatal(err) | |||
| 88 | + | } | |||
| 89 | + | gotFirst, ok := r.Lookup(tt.a) | |||
| 90 | + | if !ok || gotFirst.Key != first.Key { | |||
| 91 | + | t.Fatalf("Lookup(%q)=%+v ok=%v, want first record", tt.a, gotFirst, ok) | |||
| 92 | + | } | |||
| 93 | + | gotSecond, ok := r.Lookup(tt.b) | |||
| 94 | + | if !ok || gotSecond.Key != second.Key { | |||
| 95 | + | t.Fatalf("Lookup(%q)=%+v ok=%v, want second record", tt.b, gotSecond, ok) | |||
| 96 | + | } | |||
| 97 | + | } | |||
| 98 | + | } | |||
A internal/adapters/dgram/proxy_test.go +181 -0
| @@ -0,0 +1,181 @@ | |||
| 1 | + | package dgram | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "errors" | |
| 5 | + | "io" | |
| 6 | + | "sync" | |
| 7 | + | "testing" | |
| 8 | + | "time" | |
| 9 | + | ||
| 10 | + | "github.com/bnema/vev/internal/ports" | |
| 11 | + | ) | |
| 12 | + | ||
| 13 | + | type fakeTransport struct { | |
| 14 | + | recv chan recvResult | |
| 15 | + | sent chan ports.Frame | |
| 16 | + | sendErr chan error | |
| 17 | + | linkState ports.LinkState | |
| 18 | + | linkEvents chan ports.LinkEvent | |
| 19 | + | closed chan struct{} | |
| 20 | + | closeOnce sync.Once | |
| 21 | + | } | |
| 22 | + | ||
| 23 | + | type recvResult struct { | |
| 24 | + | frame ports.Frame | |
| 25 | + | err error | |
| 26 | + | } | |
| 27 | + | ||
| 28 | + | func newFakeTransport() *fakeTransport { | |
| 29 | + | return &fakeTransport{recv: make(chan recvResult, 4), sent: make(chan ports.Frame, 4), sendErr: make(chan error, 4), linkEvents: make(chan ports.LinkEvent, 4), linkState: ports.LinkStateConnected, closed: make(chan struct{})} | |
| 30 | + | } | |
| 31 | + | func (f *fakeTransport) Send(fr ports.Frame) error { | |
| 32 | + | select { | |
| 33 | + | case err := <-f.sendErr: | |
| 34 | + | return err | |
| 35 | + | default: | |
| 36 | + | } | |
| 37 | + | f.sent <- fr | |
| 38 | + | return nil | |
| 39 | + | } | |
| 40 | + | func (f *fakeTransport) Recv() (ports.Frame, error) { | |
| 41 | + | select { | |
| 42 | + | case r := <-f.recv: | |
| 43 | + | return r.frame, r.err | |
| 44 | + | case <-f.closed: | |
| 45 | + | return ports.Frame{}, io.EOF | |
| 46 | + | } | |
| 47 | + | } | |
| 48 | + | func (f *fakeTransport) Close() error { | |
| 49 | + | f.closeOnce.Do(func() { close(f.closed) }) | |
| 50 | + | return nil | |
| 51 | + | } | |
| 52 | + | func (f *fakeTransport) LinkState() ports.LinkState { return f.linkState } | |
| 53 | + | func (f *fakeTransport) LinkEvents() <-chan ports.LinkEvent { return f.linkEvents } | |
| 54 | + | ||
| 55 | + | func TestProxyRuntimeCopiesFramesAndDaemonEOFIsTerminal(t *testing.T) { | |
| 56 | + | client := newFakeTransport() | |
| 57 | + | daemon := newFakeTransport() | |
| 58 | + | errCh := make(chan error, 1) | |
| 59 | + | go func() { errCh <- ProxyRuntime{Client: client, Daemon: daemon, IdleTTL: time.Hour}.Run(t.Context()) }() | |
| 60 | + | client.recv <- recvResult{frame: ports.Frame{Type: ports.MsgInput, Payload: []byte("x")}} | |
| 61 | + | select { | |
| 62 | + | case got := <-daemon.sent: | |
| 63 | + | if got.Type != ports.MsgInput { | |
| 64 | + | t.Fatalf("daemon got type %v", got.Type) | |
| 65 | + | } | |
| 66 | + | case <-time.After(time.Second): | |
| 67 | + | t.Fatal("timeout waiting for copied frame") | |
| 68 | + | } | |
| 69 | + | daemon.recv <- recvResult{err: io.EOF} | |
| 70 | + | select { | |
| 71 | + | case err := <-errCh: | |
| 72 | + | if err != nil { | |
| 73 | + | t.Fatalf("Run err=%v, want nil", err) | |
| 74 | + | } | |
| 75 | + | case <-time.After(time.Second): | |
| 76 | + | t.Fatal("timeout waiting for terminal EOF") | |
| 77 | + | } | |
| 78 | + | } | |
| 79 | + | ||
| 80 | + | func TestProxyRuntimeRestartsRecoverableClientReceiveUntilIdleTTL(t *testing.T) { | |
| 81 | + | client := newFakeTransport() | |
| 82 | + | daemon := newFakeTransport() | |
| 83 | + | client.linkState = ports.LinkStateOffline | |
| 84 | + | client.linkEvents <- ports.LinkEvent{State: ports.LinkStateOffline} | |
| 85 | + | errCh := make(chan error, 1) | |
| 86 | + | go func() { | |
| 87 | + | errCh <- ProxyRuntime{Client: client, Daemon: daemon, IdleTTL: 50 * time.Millisecond, RetryBackoff: time.Nanosecond}.Run(t.Context()) | |
| 88 | + | }() | |
| 89 | + | client.recv <- recvResult{err: errors.New("temporary receive failure")} | |
| 90 | + | client.recv <- recvResult{frame: ports.Frame{Type: ports.MsgInput, Payload: []byte("after recovery")}} | |
| 91 | + | select { | |
| 92 | + | case got := <-daemon.sent: | |
| 93 | + | if got.Type != ports.MsgInput || string(got.Payload) != "after recovery" { | |
| 94 | + | t.Fatalf("daemon got %+v, want recovered input", got) | |
| 95 | + | } | |
| 96 | + | case <-time.After(time.Second): | |
| 97 | + | t.Fatal("timeout waiting for recovered frame") | |
| 98 | + | } | |
| 99 | + | daemon.recv <- recvResult{err: io.EOF} | |
| 100 | + | select { | |
| 101 | + | case err := <-errCh: | |
| 102 | + | if err != nil { | |
| 103 | + | t.Fatalf("Run err=%v, want nil", err) | |
| 104 | + | } | |
| 105 | + | case <-time.After(time.Second): | |
| 106 | + | t.Fatal("timeout waiting for daemon EOF") | |
| 107 | + | } | |
| 108 | + | } | |
| 109 | + | ||
| 110 | + | func TestProxyRuntimeRestartsRecoverableClientSendUntilIdleTTL(t *testing.T) { | |
| 111 | + | client := newFakeTransport() | |
| 112 | + | daemon := newFakeTransport() | |
| 113 | + | client.linkState = ports.LinkStateOffline | |
| 114 | + | client.linkEvents <- ports.LinkEvent{State: ports.LinkStateOffline} | |
| 115 | + | errCh := make(chan error, 1) | |
| 116 | + | go func() { | |
| 117 | + | errCh <- ProxyRuntime{Client: client, Daemon: daemon, IdleTTL: 50 * time.Millisecond, RetryBackoff: time.Nanosecond}.Run(t.Context()) | |
| 118 | + | }() | |
| 119 | + | client.sendErr <- errors.New("temporary send failure") | |
| 120 | + | daemon.recv <- recvResult{frame: ports.Frame{Type: ports.MsgOutput, Payload: []byte("lost during outage")}} | |
| 121 | + | daemon.recv <- recvResult{frame: ports.Frame{Type: ports.MsgOutput, Payload: []byte("after recovery")}} | |
| 122 | + | select { | |
| 123 | + | case got := <-client.sent: | |
| 124 | + | if got.Type != ports.MsgOutput || string(got.Payload) != "after recovery" { | |
| 125 | + | t.Fatalf("client got %+v, want recovered output", got) | |
| 126 | + | } | |
| 127 | + | case <-time.After(time.Second): | |
| 128 | + | t.Fatal("timeout waiting for recovered daemon output") | |
| 129 | + | } | |
| 130 | + | daemon.recv <- recvResult{err: io.EOF} | |
| 131 | + | select { | |
| 132 | + | case err := <-errCh: | |
| 133 | + | if err != nil { | |
| 134 | + | t.Fatalf("Run err=%v, want nil", err) | |
| 135 | + | } | |
| 136 | + | case <-time.After(time.Second): | |
| 137 | + | t.Fatal("timeout waiting for daemon EOF") | |
| 138 | + | } | |
| 139 | + | } | |
| 140 | + | ||
| 141 | + | func TestProxyRuntimeKeepsRecoverableLinkUntilIdleTTL(t *testing.T) { | |
| 142 | + | client := newFakeTransport() | |
| 143 | + | daemon := newFakeTransport() | |
| 144 | + | client.linkState = ports.LinkStateOffline | |
| 145 | + | client.linkEvents <- ports.LinkEvent{State: ports.LinkStateOffline} | |
| 146 | + | errCh := make(chan error, 1) | |
| 147 | + | go func() { | |
| 148 | + | errCh <- ProxyRuntime{Client: client, Daemon: daemon, IdleTTL: 20 * time.Millisecond}.Run(t.Context()) | |
| 149 | + | }() | |
| 150 | + | client.recv <- recvResult{err: errors.New("temporary receive failure")} | |
| 151 | + | select { | |
| 152 | + | case err := <-errCh: | |
| 153 | + | t.Fatalf("Run returned before idle TTL: %v", err) | |
| 154 | + | case <-time.After(5 * time.Millisecond): | |
| 155 | + | } | |
| 156 | + | select { | |
| 157 | + | case err := <-errCh: | |
| 158 | + | if !errors.Is(err, ErrLinkDead) { | |
| 159 | + | t.Fatalf("Run err=%v, want ErrLinkDead", err) | |
| 160 | + | } | |
| 161 | + | case <-time.After(time.Second): | |
| 162 | + | t.Fatal("timeout waiting for idle expiry") | |
| 163 | + | } | |
| 164 | + | } | |
| 165 | + | ||
| 166 | + | func TestProxyRuntimeClosedLinkEventsDoNotSpin(t *testing.T) { | |
| 167 | + | client := newFakeTransport() | |
| 168 | + | daemon := newFakeTransport() | |
| 169 | + | close(client.linkEvents) | |
| 170 | + | errCh := make(chan error, 1) | |
| 171 | + | go func() { errCh <- ProxyRuntime{Client: client, Daemon: daemon, IdleTTL: time.Hour}.Run(t.Context()) }() | |
| 172 | + | daemon.recv <- recvResult{err: io.EOF} | |
| 173 | + | select { | |
| 174 | + | case err := <-errCh: | |
| 175 | + | if err != nil { | |
| 176 | + | t.Fatalf("Run err=%v, want nil", err) | |
| 177 | + | } | |
| 178 | + | case <-time.After(time.Second): | |
| 179 | + | t.Fatal("timeout waiting for daemon EOF") | |
| 180 | + | } | |
| 181 | + | } | |
| @@ -0,0 +1,181 @@ | |||||
| 1 | + | package dgram | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "errors" | |||
| 5 | + | "io" | |||
| 6 | + | "sync" | |||
| 7 | + | "testing" | |||
| 8 | + | "time" | |||
| 9 | + | ||||
| 10 | + | "github.com/bnema/vev/internal/ports" | |||
| 11 | + | ) | |||
| 12 | + | ||||
| 13 | + | type fakeTransport struct { | |||
| 14 | + | recv chan recvResult | |||
| 15 | + | sent chan ports.Frame | |||
| 16 | + | sendErr chan error | |||
| 17 | + | linkState ports.LinkState | |||
| 18 | + | linkEvents chan ports.LinkEvent | |||
| 19 | + | closed chan struct{} | |||
| 20 | + | closeOnce sync.Once | |||
| 21 | + | } | |||
| 22 | + | ||||
| 23 | + | type recvResult struct { | |||
| 24 | + | frame ports.Frame | |||
| 25 | + | err error | |||
| 26 | + | } | |||
| 27 | + | ||||
| 28 | + | func newFakeTransport() *fakeTransport { | |||
| 29 | + | return &fakeTransport{recv: make(chan recvResult, 4), sent: make(chan ports.Frame, 4), sendErr: make(chan error, 4), linkEvents: make(chan ports.LinkEvent, 4), linkState: ports.LinkStateConnected, closed: make(chan struct{})} | |||
| 30 | + | } | |||
| 31 | + | func (f *fakeTransport) Send(fr ports.Frame) error { | |||
| 32 | + | select { | |||
| 33 | + | case err := <-f.sendErr: | |||
| 34 | + | return err | |||
| 35 | + | default: | |||
| 36 | + | } | |||
| 37 | + | f.sent <- fr | |||
| 38 | + | return nil | |||
| 39 | + | } | |||
| 40 | + | func (f *fakeTransport) Recv() (ports.Frame, error) { | |||
| 41 | + | select { | |||
| 42 | + | case r := <-f.recv: | |||
| 43 | + | return r.frame, r.err | |||
| 44 | + | case <-f.closed: | |||
| 45 | + | return ports.Frame{}, io.EOF | |||
| 46 | + | } | |||
| 47 | + | } | |||
| 48 | + | func (f *fakeTransport) Close() error { | |||
| 49 | + | f.closeOnce.Do(func() { close(f.closed) }) | |||
| 50 | + | return nil | |||
| 51 | + | } | |||
| 52 | + | func (f *fakeTransport) LinkState() ports.LinkState { return f.linkState } | |||
| 53 | + | func (f *fakeTransport) LinkEvents() <-chan ports.LinkEvent { return f.linkEvents } | |||
| 54 | + | ||||
| 55 | + | func TestProxyRuntimeCopiesFramesAndDaemonEOFIsTerminal(t *testing.T) { | |||
| 56 | + | client := newFakeTransport() | |||
| 57 | + | daemon := newFakeTransport() | |||
| 58 | + | errCh := make(chan error, 1) | |||
| 59 | + | go func() { errCh <- ProxyRuntime{Client: client, Daemon: daemon, IdleTTL: time.Hour}.Run(t.Context()) }() | |||
| 60 | + | client.recv <- recvResult{frame: ports.Frame{Type: ports.MsgInput, Payload: []byte("x")}} | |||
| 61 | + | select { | |||
| 62 | + | case got := <-daemon.sent: | |||
| 63 | + | if got.Type != ports.MsgInput { | |||
| 64 | + | t.Fatalf("daemon got type %v", got.Type) | |||
| 65 | + | } | |||
| 66 | + | case <-time.After(time.Second): | |||
| 67 | + | t.Fatal("timeout waiting for copied frame") | |||
| 68 | + | } | |||
| 69 | + | daemon.recv <- recvResult{err: io.EOF} | |||
| 70 | + | select { | |||
| 71 | + | case err := <-errCh: | |||
| 72 | + | if err != nil { | |||
| 73 | + | t.Fatalf("Run err=%v, want nil", err) | |||
| 74 | + | } | |||
| 75 | + | case <-time.After(time.Second): | |||
| 76 | + | t.Fatal("timeout waiting for terminal EOF") | |||
| 77 | + | } | |||
| 78 | + | } | |||
| 79 | + | ||||
| 80 | + | func TestProxyRuntimeRestartsRecoverableClientReceiveUntilIdleTTL(t *testing.T) { | |||
| 81 | + | client := newFakeTransport() | |||
| 82 | + | daemon := newFakeTransport() | |||
| 83 | + | client.linkState = ports.LinkStateOffline | |||
| 84 | + | client.linkEvents <- ports.LinkEvent{State: ports.LinkStateOffline} | |||
| 85 | + | errCh := make(chan error, 1) | |||
| 86 | + | go func() { | |||
| 87 | + | errCh <- ProxyRuntime{Client: client, Daemon: daemon, IdleTTL: 50 * time.Millisecond, RetryBackoff: time.Nanosecond}.Run(t.Context()) | |||
| 88 | + | }() | |||
| 89 | + | client.recv <- recvResult{err: errors.New("temporary receive failure")} | |||
| 90 | + | client.recv <- recvResult{frame: ports.Frame{Type: ports.MsgInput, Payload: []byte("after recovery")}} | |||
| 91 | + | select { | |||
| 92 | + | case got := <-daemon.sent: | |||
| 93 | + | if got.Type != ports.MsgInput || string(got.Payload) != "after recovery" { | |||
| 94 | + | t.Fatalf("daemon got %+v, want recovered input", got) | |||
| 95 | + | } | |||
| 96 | + | case <-time.After(time.Second): | |||
| 97 | + | t.Fatal("timeout waiting for recovered frame") | |||
| 98 | + | } | |||
| 99 | + | daemon.recv <- recvResult{err: io.EOF} | |||
| 100 | + | select { | |||
| 101 | + | case err := <-errCh: | |||
| 102 | + | if err != nil { | |||
| 103 | + | t.Fatalf("Run err=%v, want nil", err) | |||
| 104 | + | } | |||
| 105 | + | case <-time.After(time.Second): | |||
| 106 | + | t.Fatal("timeout waiting for daemon EOF") | |||
| 107 | + | } | |||
| 108 | + | } | |||
| 109 | + | ||||
| 110 | + | func TestProxyRuntimeRestartsRecoverableClientSendUntilIdleTTL(t *testing.T) { | |||
| 111 | + | client := newFakeTransport() | |||
| 112 | + | daemon := newFakeTransport() | |||
| 113 | + | client.linkState = ports.LinkStateOffline | |||
| 114 | + | client.linkEvents <- ports.LinkEvent{State: ports.LinkStateOffline} | |||
| 115 | + | errCh := make(chan error, 1) | |||
| 116 | + | go func() { | |||
| 117 | + | errCh <- ProxyRuntime{Client: client, Daemon: daemon, IdleTTL: 50 * time.Millisecond, RetryBackoff: time.Nanosecond}.Run(t.Context()) | |||
| 118 | + | }() | |||
| 119 | + | client.sendErr <- errors.New("temporary send failure") | |||
| 120 | + | daemon.recv <- recvResult{frame: ports.Frame{Type: ports.MsgOutput, Payload: []byte("lost during outage")}} | |||
| 121 | + | daemon.recv <- recvResult{frame: ports.Frame{Type: ports.MsgOutput, Payload: []byte("after recovery")}} | |||
| 122 | + | select { | |||
| 123 | + | case got := <-client.sent: | |||
| 124 | + | if got.Type != ports.MsgOutput || string(got.Payload) != "after recovery" { | |||
| 125 | + | t.Fatalf("client got %+v, want recovered output", got) | |||
| 126 | + | } | |||
| 127 | + | case <-time.After(time.Second): | |||
| 128 | + | t.Fatal("timeout waiting for recovered daemon output") | |||
| 129 | + | } | |||
| 130 | + | daemon.recv <- recvResult{err: io.EOF} | |||
| 131 | + | select { | |||
| 132 | + | case err := <-errCh: | |||
| 133 | + | if err != nil { | |||
| 134 | + | t.Fatalf("Run err=%v, want nil", err) | |||
| 135 | + | } | |||
| 136 | + | case <-time.After(time.Second): | |||
| 137 | + | t.Fatal("timeout waiting for daemon EOF") | |||
| 138 | + | } | |||
| 139 | + | } | |||
| 140 | + | ||||
| 141 | + | func TestProxyRuntimeKeepsRecoverableLinkUntilIdleTTL(t *testing.T) { | |||
| 142 | + | client := newFakeTransport() | |||
| 143 | + | daemon := newFakeTransport() | |||
| 144 | + | client.linkState = ports.LinkStateOffline | |||
| 145 | + | client.linkEvents <- ports.LinkEvent{State: ports.LinkStateOffline} | |||
| 146 | + | errCh := make(chan error, 1) | |||
| 147 | + | go func() { | |||
| 148 | + | errCh <- ProxyRuntime{Client: client, Daemon: daemon, IdleTTL: 20 * time.Millisecond}.Run(t.Context()) | |||
| 149 | + | }() | |||
| 150 | + | client.recv <- recvResult{err: errors.New("temporary receive failure")} | |||
| 151 | + | select { | |||
| 152 | + | case err := <-errCh: | |||
| 153 | + | t.Fatalf("Run returned before idle TTL: %v", err) | |||
| 154 | + | case <-time.After(5 * time.Millisecond): | |||
| 155 | + | } | |||
| 156 | + | select { | |||
| 157 | + | case err := <-errCh: | |||
| 158 | + | if !errors.Is(err, ErrLinkDead) { | |||
| 159 | + | t.Fatalf("Run err=%v, want ErrLinkDead", err) | |||
| 160 | + | } | |||
| 161 | + | case <-time.After(time.Second): | |||
| 162 | + | t.Fatal("timeout waiting for idle expiry") | |||
| 163 | + | } | |||
| 164 | + | } | |||
| 165 | + | ||||
| 166 | + | func TestProxyRuntimeClosedLinkEventsDoNotSpin(t *testing.T) { | |||
| 167 | + | client := newFakeTransport() | |||
| 168 | + | daemon := newFakeTransport() | |||
| 169 | + | close(client.linkEvents) | |||
| 170 | + | errCh := make(chan error, 1) | |||
| 171 | + | go func() { errCh <- ProxyRuntime{Client: client, Daemon: daemon, IdleTTL: time.Hour}.Run(t.Context()) }() | |||
| 172 | + | daemon.recv <- recvResult{err: io.EOF} | |||
| 173 | + | select { | |||
| 174 | + | case err := <-errCh: | |||
| 175 | + | if err != nil { | |||
| 176 | + | t.Fatalf("Run err=%v, want nil", err) | |||
| 177 | + | } | |||
| 178 | + | case <-time.After(time.Second): | |||
| 179 | + | t.Fatal("timeout waiting for daemon EOF") | |||
| 180 | + | } | |||
| 181 | + | } | |||
M internal/adapters/dgram/ssh_dialer.go +37 -7
| @@ -52,6 +52,34 @@ var ( | |||
| 52 | 52 | probeUDPTransport = func(ctx context.Context, t *Transport) error { return t.Probe(ctx) } | |
| 53 | 53 | ) | |
| 54 | 54 | ||
| 55 | + | // RemoteDialFailureKind classifies which part of UDP remote bootstrap failed | |
| 56 | + | // so client-side recovery policy and tests do not have to parse human text. | |
| 57 | + | type RemoteDialFailureKind uint8 | |
| 58 | + | ||
| 59 | + | const ( | |
| 60 | + | RemoteDialBootstrapUnavailable RemoteDialFailureKind = iota + 1 | |
| 61 | + | RemoteDialProbeUnreachable | |
| 62 | + | ) | |
| 63 | + | ||
| 64 | + | // RemoteDialError wraps a UDP remote bootstrap failure with a stable machine | |
| 65 | + | // classification while preserving the existing human-facing error text. | |
| 66 | + | type RemoteDialError struct { | |
| 67 | + | Kind RemoteDialFailureKind | |
| 68 | + | Action string | |
| 69 | + | Err error | |
| 70 | + | Hint string | |
| 71 | + | } | |
| 72 | + | ||
| 73 | + | func (e *RemoteDialError) Error() string { | |
| 74 | + | base := fmt.Sprintf("remote UDP transport unavailable: %s: %v", e.Action, e.Err) | |
| 75 | + | if e.Hint != "" { | |
| 76 | + | return base + e.Hint | |
| 77 | + | } | |
| 78 | + | return base | |
| 79 | + | } | |
| 80 | + | ||
| 81 | + | func (e *RemoteDialError) Unwrap() error { return e.Err } | |
| 82 | + | ||
| 55 | 83 | type limitedBuffer struct { | |
| 56 | 84 | buf []byte | |
| 57 | 85 | } | |
| @@ -233,11 +261,14 @@ func readUDPReady(r io.Reader) (udpReady, error) { | |||
| 233 | 261 | } | |
| 234 | 262 | ||
| 235 | 263 | func udpUnavailable(action string, err error, stderr fmt.Stringer) error { | |
| 236 | - | msg := strings.TrimSpace(stderr.String()) | |
| 237 | - | if msg != "" { | |
| 238 | - | return fmt.Errorf("remote UDP transport unavailable: %s: %w (stderr: %s)", action, err, msg) | |
| 264 | + | return udpDialFailure(RemoteDialBootstrapUnavailable, action, err, stderr, "") | |
| 265 | + | } | |
| 266 | + | ||
| 267 | + | func udpDialFailure(kind RemoteDialFailureKind, action string, err error, stderr fmt.Stringer, hint string) error { | |
| 268 | + | if msg := strings.TrimSpace(stderr.String()); msg != "" { | |
| 269 | + | hint = fmt.Sprintf(" (stderr: %s)%s", msg, hint) | |
| 239 | 270 | } | |
| 240 | - | return fmt.Errorf("remote UDP transport unavailable: %s: %w", action, err) | |
| 271 | + | return &RemoteDialError{Kind: kind, Action: action, Err: err, Hint: hint} | |
| 241 | 272 | } | |
| 242 | 273 | ||
| 243 | 274 | // udpProbeUnreachable wraps a probe failure with actionable guidance. A failed | |
| @@ -245,10 +276,9 @@ func udpUnavailable(action string, err error, stderr fmt.Stringer) error { | |||
| 245 | 276 | // vev's UDP proxy rather than a vev bug, so name the two real fixes instead of | |
| 246 | 277 | // surfacing a bare deadline error. | |
| 247 | 278 | func udpProbeUnreachable(target string, err error, stderr fmt.Stringer) error { | |
| 248 | - | return fmt.Errorf("%w\n"+ | |
| 279 | + | return udpDialFailure(RemoteDialProbeUnreachable, "probe UDP transport", err, stderr, fmt.Sprintf("\n"+ | |
| 249 | 280 | " the remote host is not reachable over UDP; a firewall is likely dropping the datagrams.\n"+ | |
| 250 | - | " open UDP on the remote (e.g. `sudo ufw allow in on tailscale0`) or attach over SSH with `VEV_REMOTE_TRANSPORT=stdio vev attach %s`", | |
| 251 | - | udpUnavailable("probe UDP transport", err, stderr), target) | |
| 281 | + | " open UDP on the remote (e.g. `sudo ufw allow in on tailscale0`) or attach over SSH with `VEV_REMOTE_TRANSPORT=stdio vev attach %s`", target)) | |
| 252 | 282 | } | |
| 253 | 283 | ||
| 254 | 284 | func sshTargetHost(target string) string { | |
| @@ -52,6 +52,34 @@ var ( | |||||
| 52 | probeUDPTransport = func(ctx context.Context, t *Transport) error { return t.Probe(ctx) } | 52 | probeUDPTransport = func(ctx context.Context, t *Transport) error { return t.Probe(ctx) } | ||
| 53 | ) | 53 | ) | ||
| 54 | 54 | ||||
| 55 | + | // RemoteDialFailureKind classifies which part of UDP remote bootstrap failed | |||
| 56 | + | // so client-side recovery policy and tests do not have to parse human text. | |||
| 57 | + | type RemoteDialFailureKind uint8 | |||
| 58 | + | ||||
| 59 | + | const ( | |||
| 60 | + | RemoteDialBootstrapUnavailable RemoteDialFailureKind = iota + 1 | |||
| 61 | + | RemoteDialProbeUnreachable | |||
| 62 | + | ) | |||
| 63 | + | ||||
| 64 | + | // RemoteDialError wraps a UDP remote bootstrap failure with a stable machine | |||
| 65 | + | // classification while preserving the existing human-facing error text. | |||
| 66 | + | type RemoteDialError struct { | |||
| 67 | + | Kind RemoteDialFailureKind | |||
| 68 | + | Action string | |||
| 69 | + | Err error | |||
| 70 | + | Hint string | |||
| 71 | + | } | |||
| 72 | + | ||||
| 73 | + | func (e *RemoteDialError) Error() string { | |||
| 74 | + | base := fmt.Sprintf("remote UDP transport unavailable: %s: %v", e.Action, e.Err) | |||
| 75 | + | if e.Hint != "" { | |||
| 76 | + | return base + e.Hint | |||
| 77 | + | } | |||
| 78 | + | return base | |||
| 79 | + | } | |||
| 80 | + | ||||
| 81 | + | func (e *RemoteDialError) Unwrap() error { return e.Err } | |||
| 82 | + | ||||
| 55 | type limitedBuffer struct { | 83 | type limitedBuffer struct { | ||
| 56 | buf []byte | 84 | buf []byte | ||
| 57 | } | 85 | } | ||
| @@ -233,11 +261,14 @@ func readUDPReady(r io.Reader) (udpReady, error) { | |||||
| 233 | } | 261 | } | ||
| 234 | 262 | ||||
| 235 | func udpUnavailable(action string, err error, stderr fmt.Stringer) error { | 263 | func udpUnavailable(action string, err error, stderr fmt.Stringer) error { | ||
| 236 | - | msg := strings.TrimSpace(stderr.String()) | 264 | + | return udpDialFailure(RemoteDialBootstrapUnavailable, action, err, stderr, "") |
| 237 | - | if msg != "" { | 265 | + | } |
| 238 | - | return fmt.Errorf("remote UDP transport unavailable: %s: %w (stderr: %s)", action, err, msg) | 266 | + | |
| 267 | + | func udpDialFailure(kind RemoteDialFailureKind, action string, err error, stderr fmt.Stringer, hint string) error { | |||
| 268 | + | if msg := strings.TrimSpace(stderr.String()); msg != "" { | |||
| 269 | + | hint = fmt.Sprintf(" (stderr: %s)%s", msg, hint) | |||
| 239 | } | 270 | } | ||
| 240 | - | return fmt.Errorf("remote UDP transport unavailable: %s: %w", action, err) | 271 | + | return &RemoteDialError{Kind: kind, Action: action, Err: err, Hint: hint} |
| 241 | } | 272 | } | ||
| 242 | 273 | ||||
| 243 | // udpProbeUnreachable wraps a probe failure with actionable guidance. A failed | 274 | // udpProbeUnreachable wraps a probe failure with actionable guidance. A failed | ||
| @@ -245,10 +276,9 @@ func udpUnavailable(action string, err error, stderr fmt.Stringer) error { | |||||
| 245 | // vev's UDP proxy rather than a vev bug, so name the two real fixes instead of | 276 | // vev's UDP proxy rather than a vev bug, so name the two real fixes instead of | ||
| 246 | // surfacing a bare deadline error. | 277 | // surfacing a bare deadline error. | ||
| 247 | func udpProbeUnreachable(target string, err error, stderr fmt.Stringer) error { | 278 | func udpProbeUnreachable(target string, err error, stderr fmt.Stringer) error { | ||
| 248 | - | return fmt.Errorf("%w\n"+ | 279 | + | return udpDialFailure(RemoteDialProbeUnreachable, "probe UDP transport", err, stderr, fmt.Sprintf("\n"+ |
| 249 | " the remote host is not reachable over UDP; a firewall is likely dropping the datagrams.\n"+ | 280 | " the remote host is not reachable over UDP; a firewall is likely dropping the datagrams.\n"+ | ||
| 250 | - | " open UDP on the remote (e.g. `sudo ufw allow in on tailscale0`) or attach over SSH with `VEV_REMOTE_TRANSPORT=stdio vev attach %s`", | 281 | + | " open UDP on the remote (e.g. `sudo ufw allow in on tailscale0`) or attach over SSH with `VEV_REMOTE_TRANSPORT=stdio vev attach %s`", target)) |
| 251 | - | udpUnavailable("probe UDP transport", err, stderr), target) | |||
| 252 | } | 282 | } | ||
| 253 | 283 | ||||
| 254 | func sshTargetHost(target string) string { | 284 | func sshTargetHost(target string) string { | ||
M internal/adapters/dgram/ssh_dialer_test.go +14 -0
| @@ -79,6 +79,13 @@ func TestRemoteDialerUDPBootstrapFailures(t *testing.T) { | |||
| 79 | 79 | if err == nil || !strings.Contains(err.Error(), tt.want) { | |
| 80 | 80 | t.Fatalf("Dial() error = %v, want containing %q", err, tt.want) | |
| 81 | 81 | } | |
| 82 | + | var dialErr *RemoteDialError | |
| 83 | + | if !errors.As(err, &dialErr) { | |
| 84 | + | t.Fatalf("Dial() error = %T, want *RemoteDialError", err) | |
| 85 | + | } | |
| 86 | + | if dialErr.Kind != RemoteDialBootstrapUnavailable { | |
| 87 | + | t.Fatalf("Dial() error kind = %v, want bootstrap unavailable", dialErr.Kind) | |
| 88 | + | } | |
| 82 | 89 | if tr != nil { | |
| 83 | 90 | t.Fatalf("Dial() transport = %T, want nil", tr) | |
| 84 | 91 | } | |
| @@ -134,6 +141,13 @@ func TestRemoteDialerProbeFailureCleansUpWithoutStdioFallback(t *testing.T) { | |||
| 134 | 141 | if err == nil || !strings.Contains(err.Error(), "remote UDP transport unavailable: probe UDP transport") { | |
| 135 | 142 | t.Fatalf("Dial() error = %v, want probe unavailable", err) | |
| 136 | 143 | } | |
| 144 | + | var dialErr *RemoteDialError | |
| 145 | + | if !errors.As(err, &dialErr) { | |
| 146 | + | t.Fatalf("Dial() error = %T, want *RemoteDialError", err) | |
| 147 | + | } | |
| 148 | + | if dialErr.Kind != RemoteDialProbeUnreachable { | |
| 149 | + | t.Fatalf("Dial() error kind = %v, want probe unreachable", dialErr.Kind) | |
| 150 | + | } | |
| 137 | 151 | for _, want := range []string{"firewall", "VEV_REMOTE_TRANSPORT=stdio vev attach 127.0.0.1"} { | |
| 138 | 152 | if !strings.Contains(err.Error(), want) { | |
| 139 | 153 | t.Fatalf("Dial() error = %q, want actionable hint %q", err, want) | |
| @@ -79,6 +79,13 @@ func TestRemoteDialerUDPBootstrapFailures(t *testing.T) { | |||||
| 79 | if err == nil || !strings.Contains(err.Error(), tt.want) { | 79 | if err == nil || !strings.Contains(err.Error(), tt.want) { | ||
| 80 | t.Fatalf("Dial() error = %v, want containing %q", err, tt.want) | 80 | t.Fatalf("Dial() error = %v, want containing %q", err, tt.want) | ||
| 81 | } | 81 | } | ||
| 82 | + | var dialErr *RemoteDialError | |||
| 83 | + | if !errors.As(err, &dialErr) { | |||
| 84 | + | t.Fatalf("Dial() error = %T, want *RemoteDialError", err) | |||
| 85 | + | } | |||
| 86 | + | if dialErr.Kind != RemoteDialBootstrapUnavailable { | |||
| 87 | + | t.Fatalf("Dial() error kind = %v, want bootstrap unavailable", dialErr.Kind) | |||
| 88 | + | } | |||
| 82 | if tr != nil { | 89 | if tr != nil { | ||
| 83 | t.Fatalf("Dial() transport = %T, want nil", tr) | 90 | t.Fatalf("Dial() transport = %T, want nil", tr) | ||
| 84 | } | 91 | } | ||
| @@ -134,6 +141,13 @@ func TestRemoteDialerProbeFailureCleansUpWithoutStdioFallback(t *testing.T) { | |||||
| 134 | if err == nil || !strings.Contains(err.Error(), "remote UDP transport unavailable: probe UDP transport") { | 141 | if err == nil || !strings.Contains(err.Error(), "remote UDP transport unavailable: probe UDP transport") { | ||
| 135 | t.Fatalf("Dial() error = %v, want probe unavailable", err) | 142 | t.Fatalf("Dial() error = %v, want probe unavailable", err) | ||
| 136 | } | 143 | } | ||
| 144 | + | var dialErr *RemoteDialError | |||
| 145 | + | if !errors.As(err, &dialErr) { | |||
| 146 | + | t.Fatalf("Dial() error = %T, want *RemoteDialError", err) | |||
| 147 | + | } | |||
| 148 | + | if dialErr.Kind != RemoteDialProbeUnreachable { | |||
| 149 | + | t.Fatalf("Dial() error kind = %v, want probe unreachable", dialErr.Kind) | |||
| 150 | + | } | |||
| 137 | for _, want := range []string{"firewall", "VEV_REMOTE_TRANSPORT=stdio vev attach 127.0.0.1"} { | 151 | for _, want := range []string{"firewall", "VEV_REMOTE_TRANSPORT=stdio vev attach 127.0.0.1"} { | ||
| 138 | if !strings.Contains(err.Error(), want) { | 152 | if !strings.Contains(err.Error(), want) { | ||
| 139 | t.Fatalf("Dial() error = %q, want actionable hint %q", err, want) | 153 | t.Fatalf("Dial() error = %q, want actionable hint %q", err, want) | ||
M internal/adapters/dgram/transport.go +222 -45
| @@ -19,11 +19,18 @@ const ( | |||
| 19 | 19 | recProbe byte = 3 | |
| 20 | 20 | recPong byte = 4 | |
| 21 | 21 | ||
| 22 | - | defaultResend = 40 * time.Millisecond | |
| 23 | - | defaultHeartbeat = 3 * time.Second | |
| 24 | - | defaultSilenceTimeout = 10 * time.Second | |
| 25 | - | maxPendingReliable = 1024 | |
| 26 | - | maxRecvBuffer = 1024 | |
| 22 | + | defaultMTU = pdgram.DefaultMTU | |
| 23 | + | defaultResend = 40 * time.Millisecond | |
| 24 | + | defaultHeartbeat = 3 * time.Second | |
| 25 | + | defaultDegraded = 10 * time.Second | |
| 26 | + | defaultProbe = 20 * time.Second | |
| 27 | + | defaultOffline = 30 * time.Second | |
| 28 | + | defaultDead = 60 * time.Second | |
| 29 | + | defaultMaxPending = 1024 | |
| 30 | + | defaultMaxRecvBuf = 1024 | |
| 31 | + | maxPendingReliable = defaultMaxPending | |
| 32 | + | maxRecvBuffer = defaultMaxRecvBuf | |
| 33 | + | linkEventBufferSize = 16 | |
| 27 | 34 | ) | |
| 28 | 35 | ||
| 29 | 36 | var ( | |
| @@ -31,25 +38,100 @@ var ( | |||
| 31 | 38 | ErrLinkDead = errors.New("dgram: link dead") | |
| 32 | 39 | ) | |
| 33 | 40 | ||
| 41 | + | type Options struct { | |
| 42 | + | MTU int | |
| 43 | + | ResendAfter time.Duration | |
| 44 | + | Heartbeat time.Duration | |
| 45 | + | DegradedAfter time.Duration | |
| 46 | + | ProbeAfter time.Duration | |
| 47 | + | OfflineAfter time.Duration | |
| 48 | + | DeadAfter time.Duration | |
| 49 | + | MaxPending int | |
| 50 | + | MaxRecvBuffer int | |
| 51 | + | Clock ports.Clock | |
| 52 | + | // RebindPacketConn is an optional client-side hook used to hop the local UDP | |
| 53 | + | // socket when the peer is offline. Server/proxy transports should leave it nil. | |
| 54 | + | RebindPacketConn func(net.PacketConn) (net.PacketConn, error) | |
| 55 | + | } | |
| 56 | + | ||
| 57 | + | type realClock struct{} | |
| 58 | + | ||
| 59 | + | func (realClock) Now() time.Time { return time.Now() } | |
| 60 | + | func (realClock) NewTimer(d time.Duration) ports.Timer { | |
| 61 | + | return realTimer{t: time.NewTimer(d)} | |
| 62 | + | } | |
| 63 | + | ||
| 64 | + | type realTimer struct{ t *time.Timer } | |
| 65 | + | ||
| 66 | + | func (r realTimer) C() <-chan time.Time { return r.t.C } | |
| 67 | + | func (r realTimer) Reset(d time.Duration) bool { return r.t.Reset(d) } | |
| 68 | + | func (r realTimer) Stop() bool { return r.t.Stop() } | |
| 69 | + | ||
| 70 | + | func normalizeOptions(opts Options) Options { | |
| 71 | + | if opts.MTU <= 0 { | |
| 72 | + | opts.MTU = defaultMTU | |
| 73 | + | } | |
| 74 | + | if opts.ResendAfter <= 0 { | |
| 75 | + | opts.ResendAfter = defaultResend | |
| 76 | + | } | |
| 77 | + | if opts.Heartbeat <= 0 { | |
| 78 | + | opts.Heartbeat = defaultHeartbeat | |
| 79 | + | } | |
| 80 | + | if opts.DegradedAfter <= 0 { | |
| 81 | + | opts.DegradedAfter = defaultDegraded | |
| 82 | + | } | |
| 83 | + | if opts.ProbeAfter <= 0 { | |
| 84 | + | opts.ProbeAfter = defaultProbe | |
| 85 | + | } | |
| 86 | + | if opts.OfflineAfter <= 0 { | |
| 87 | + | opts.OfflineAfter = defaultOffline | |
| 88 | + | } | |
| 89 | + | if opts.DeadAfter <= 0 { | |
| 90 | + | opts.DeadAfter = defaultDead | |
| 91 | + | } | |
| 92 | + | if opts.MaxPending <= 0 { | |
| 93 | + | opts.MaxPending = defaultMaxPending | |
| 94 | + | } | |
| 95 | + | if opts.MaxRecvBuffer <= 0 { | |
| 96 | + | opts.MaxRecvBuffer = defaultMaxRecvBuf | |
| 97 | + | } | |
| 98 | + | if opts.Clock == nil { | |
| 99 | + | opts.Clock = realClock{} | |
| 100 | + | } | |
| 101 | + | return opts | |
| 102 | + | } | |
| 103 | + | ||
| 34 | 104 | type Transport struct { | |
| 35 | 105 | pc net.PacketConn | |
| 36 | 106 | codec *pdgram.Codec | |
| 37 | 107 | sendDir, recvDir uint32 | |
| 38 | 108 | mtu int | |
| 39 | 109 | ||
| 40 | - | mu sync.Mutex | |
| 41 | - | peer net.Addr | |
| 42 | - | ctr uint64 | |
| 43 | - | seq uint64 | |
| 44 | - | probeSeq uint64 | |
| 45 | - | pending map[uint64]*pending | |
| 46 | - | closed bool | |
| 47 | - | closeErr error | |
| 48 | - | lastHeard time.Time | |
| 49 | - | heartbeat time.Duration | |
| 50 | - | silence time.Duration | |
| 51 | - | probeWait map[uint64]chan struct{} | |
| 110 | + | mu sync.Mutex | |
| 111 | + | peer net.Addr | |
| 112 | + | ctr uint64 | |
| 113 | + | seq uint64 | |
| 114 | + | probeSeq uint64 | |
| 115 | + | pending map[uint64]*pending | |
| 116 | + | closed bool | |
| 117 | + | closeErr error | |
| 118 | + | lastHeard time.Time | |
| 119 | + | heartbeat time.Duration | |
| 120 | + | resendAfter time.Duration | |
| 121 | + | degradedAfter time.Duration | |
| 122 | + | probeAfter time.Duration | |
| 123 | + | offlineAfter time.Duration | |
| 124 | + | deadAfter time.Duration | |
| 125 | + | maxPending int | |
| 126 | + | maxRecvBuffer int | |
| 127 | + | clock ports.Clock | |
| 128 | + | linkState ports.LinkState | |
| 129 | + | linkEvents chan ports.LinkEvent | |
| 130 | + | probeWait map[uint64]chan struct{} | |
| 131 | + | rebind func(net.PacketConn) (net.PacketConn, error) | |
| 132 | + | hoppedOffline bool | |
| 52 | 133 | ||
| 134 | + | recvMu sync.Mutex | |
| 53 | 135 | replay *pdgram.ReplayWindow | |
| 54 | 136 | reasm *pdgram.Reassembler | |
| 55 | 137 | in chan ports.Frame | |
| @@ -67,13 +149,18 @@ type pending struct { | |||
| 67 | 149 | } | |
| 68 | 150 | ||
| 69 | 151 | func NewTransport(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32) (*Transport, error) { | |
| 152 | + | return NewTransportWithOptions(pc, peer, key, sendDir, recvDir, Options{}) | |
| 153 | + | } | |
| 154 | + | ||
| 155 | + | func NewTransportWithOptions(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32, opts Options) (*Transport, error) { | |
| 70 | 156 | c, err := pdgram.NewCodec(key) | |
| 71 | 157 | if err != nil { | |
| 72 | 158 | return nil, err | |
| 73 | 159 | } | |
| 74 | - | t := &Transport{pc: pc, codec: c, sendDir: sendDir, recvDir: recvDir, mtu: pdgram.DefaultMTU, peer: peer, pending: make(map[uint64]*pending), replay: pdgram.NewReplayWindow(), reasm: pdgram.NewReassembler(), in: make(chan ports.Frame, 32), done: make(chan struct{}), lastHeard: time.Now(), heartbeat: defaultHeartbeat, silence: defaultSilenceTimeout, probeWait: make(map[uint64]chan struct{}), nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame)} | |
| 160 | + | opts = normalizeOptions(opts) | |
| 161 | + | t := &Transport{pc: pc, codec: c, sendDir: sendDir, recvDir: recvDir, mtu: opts.MTU, peer: peer, pending: make(map[uint64]*pending), replay: pdgram.NewReplayWindow(), reasm: pdgram.NewReassembler(), in: make(chan ports.Frame, 32), done: make(chan struct{}), lastHeard: opts.Clock.Now(), heartbeat: opts.Heartbeat, resendAfter: opts.ResendAfter, degradedAfter: opts.DegradedAfter, probeAfter: opts.ProbeAfter, offlineAfter: opts.OfflineAfter, deadAfter: opts.DeadAfter, maxPending: opts.MaxPending, maxRecvBuffer: opts.MaxRecvBuffer, clock: opts.Clock, linkState: ports.LinkStateConnected, linkEvents: make(chan ports.LinkEvent, linkEventBufferSize), probeWait: make(map[uint64]chan struct{}), rebind: opts.RebindPacketConn, nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame)} | |
| 75 | 162 | t.deliverCond = sync.NewCond(&t.deliverMu) | |
| 76 | - | go t.readLoop() | |
| 163 | + | go t.readLoop(pc) | |
| 77 | 164 | go t.resendLoop() | |
| 78 | 165 | go t.deliveryLoop() | |
| 79 | 166 | return t, nil | |
| @@ -92,7 +179,7 @@ func (t *Transport) Send(f ports.Frame) error { | |||
| 92 | 179 | } | |
| 93 | 180 | return errors.New("dgram: closed") | |
| 94 | 181 | } | |
| 95 | - | if reliable && len(t.pending) >= maxPendingReliable { | |
| 182 | + | if reliable && len(t.pending) >= t.maxPending { | |
| 96 | 183 | t.mu.Unlock() | |
| 97 | 184 | return ErrPendingFull | |
| 98 | 185 | } | |
| @@ -121,11 +208,22 @@ func (t *Transport) Recv() (ports.Frame, error) { | |||
| 121 | 208 | } | |
| 122 | 209 | func (t *Transport) Close() error { | |
| 123 | 210 | t.closeWithError(errors.New("dgram: closed")) | |
| 124 | - | return t.pc.Close() | |
| 211 | + | t.mu.Lock() | |
| 212 | + | pc := t.pc | |
| 213 | + | t.mu.Unlock() | |
| 214 | + | return pc.Close() | |
| 125 | 215 | } | |
| 126 | 216 | ||
| 127 | 217 | func (t *Transport) Peer() net.Addr { t.mu.Lock(); defer t.mu.Unlock(); return t.peer } | |
| 128 | 218 | ||
| 219 | + | func (t *Transport) LinkState() ports.LinkState { | |
| 220 | + | t.mu.Lock() | |
| 221 | + | defer t.mu.Unlock() | |
| 222 | + | return t.linkState | |
| 223 | + | } | |
| 224 | + | ||
| 225 | + | func (t *Transport) LinkEvents() <-chan ports.LinkEvent { return t.linkEvents } | |
| 226 | + | ||
| 129 | 227 | // Probe sends an authenticated datagram and waits for the peer to authenticate a | |
| 130 | 228 | // response or for ctx to expire. It is intended for UDP bootstrap reachability checks. | |
| 131 | 229 | func (t *Transport) Probe(ctx context.Context) error { | |
| @@ -216,7 +314,10 @@ func (t *Transport) sendPayload(p []byte) error { | |||
| 216 | 314 | if peer == nil { | |
| 217 | 315 | return errors.New("dgram: no peer") | |
| 218 | 316 | } | |
| 219 | - | if _, err := t.pc.WriteTo(pkt, peer); err != nil { | |
| 317 | + | t.mu.Lock() | |
| 318 | + | pc := t.pc | |
| 319 | + | t.mu.Unlock() | |
| 320 | + | if _, err := pc.WriteTo(pkt, peer); err != nil { | |
| 220 | 321 | return err | |
| 221 | 322 | } | |
| 222 | 323 | } | |
| @@ -224,34 +325,50 @@ func (t *Transport) sendPayload(p []byte) error { | |||
| 224 | 325 | } | |
| 225 | 326 | func (t *Transport) nextCounter() uint64 { t.mu.Lock(); defer t.mu.Unlock(); t.ctr++; return t.ctr } | |
| 226 | 327 | ||
| 227 | - | func (t *Transport) readLoop() { | |
| 328 | + | func (t *Transport) readLoop(pc net.PacketConn) { | |
| 228 | 329 | buf := make([]byte, 64*1024) | |
| 229 | 330 | for { | |
| 230 | - | n, addr, err := t.pc.ReadFrom(buf) | |
| 331 | + | n, addr, err := pc.ReadFrom(buf) | |
| 231 | 332 | if err != nil { | |
| 232 | - | t.closeWithError(err) | |
| 333 | + | t.mu.Lock() | |
| 334 | + | current := t.pc == pc | |
| 335 | + | closed := t.closed | |
| 336 | + | t.mu.Unlock() | |
| 337 | + | if current && !closed { | |
| 338 | + | t.closeWithError(err) | |
| 339 | + | } | |
| 233 | 340 | return | |
| 234 | 341 | } | |
| 342 | + | t.recvMu.Lock() | |
| 235 | 343 | _, pt, err := t.codec.Open(append([]byte(nil), buf[:n]...), t.recvDir, nil, t.replay) | |
| 236 | 344 | if err != nil { | |
| 345 | + | t.recvMu.Unlock() | |
| 237 | 346 | continue | |
| 238 | 347 | } | |
| 239 | - | t.mu.Lock() | |
| 240 | - | t.peer = addr | |
| 241 | - | t.lastHeard = time.Now() | |
| 242 | - | t.mu.Unlock() | |
| 243 | 348 | frag, err := pdgram.UnmarshalFragment(pt) | |
| 244 | 349 | if err != nil { | |
| 350 | + | t.recvMu.Unlock() | |
| 245 | 351 | continue | |
| 246 | 352 | } | |
| 247 | 353 | payload, ok, err := t.reasm.Add(frag) | |
| 354 | + | t.recvMu.Unlock() | |
| 248 | 355 | if err != nil || !ok { | |
| 249 | 356 | continue | |
| 250 | 357 | } | |
| 358 | + | t.updatePeerFromAuthenticated(addr) | |
| 251 | 359 | t.handleRecord(payload) | |
| 252 | 360 | } | |
| 253 | 361 | } | |
| 254 | 362 | ||
| 363 | + | func (t *Transport) updatePeerFromAuthenticated(addr net.Addr) { | |
| 364 | + | t.mu.Lock() | |
| 365 | + | t.peer = addr | |
| 366 | + | t.lastHeard = t.clock.Now() | |
| 367 | + | t.hoppedOffline = false | |
| 368 | + | t.mu.Unlock() | |
| 369 | + | t.setLinkState(ports.LinkStateConnected, nil) | |
| 370 | + | } | |
| 371 | + | ||
| 255 | 372 | func (t *Transport) handleRecord(p []byte) { | |
| 256 | 373 | if len(p) < 1 { | |
| 257 | 374 | return | |
| @@ -310,14 +427,8 @@ func (t *Transport) enqueueReliable(seq uint64, f ports.Frame) (ack bool, queued | |||
| 310 | 427 | if _, exists := t.recvBuf[seq]; exists { | |
| 311 | 428 | return true, false | |
| 312 | 429 | } | |
| 313 | - | if len(t.recvBuf) >= maxRecvBuffer { | |
| 314 | - | if seq != t.nextRecvSeq { | |
| 315 | - | return false, false | |
| 316 | - | } | |
| 317 | - | for bufferedSeq := range t.recvBuf { | |
| 318 | - | delete(t.recvBuf, bufferedSeq) | |
| 319 | - | break | |
| 320 | - | } | |
| 430 | + | if len(t.recvBuf) >= t.maxRecvBuffer && seq != t.nextRecvSeq { | |
| 431 | + | return false, false | |
| 321 | 432 | } | |
| 322 | 433 | t.recvBuf[seq] = f | |
| 323 | 434 | return true, true | |
| @@ -362,18 +473,20 @@ func (t *Transport) deliver(f ports.Frame) { | |||
| 362 | 473 | } | |
| 363 | 474 | ||
| 364 | 475 | func (t *Transport) resendLoop() { | |
| 365 | - | resend := time.NewTicker(defaultResend) | |
| 476 | + | resend := t.clock.NewTimer(t.resendAfter) | |
| 366 | 477 | defer resend.Stop() | |
| 367 | - | heartbeat := time.NewTicker(t.heartbeat) | |
| 478 | + | heartbeat := t.clock.NewTimer(t.heartbeat) | |
| 368 | 479 | defer heartbeat.Stop() | |
| 369 | 480 | for { | |
| 370 | 481 | select { | |
| 371 | - | case <-resend.C: | |
| 482 | + | case <-resend.C(): | |
| 372 | 483 | t.resendPending() | |
| 373 | 484 | t.checkSilence() | |
| 374 | - | case <-heartbeat.C: | |
| 485 | + | resend.Reset(t.resendAfter) | |
| 486 | + | case <-heartbeat.C(): | |
| 375 | 487 | _ = t.sendProbe(recProbe, 0) | |
| 376 | 488 | t.checkSilence() | |
| 489 | + | heartbeat.Reset(t.heartbeat) | |
| 377 | 490 | case <-t.done: | |
| 378 | 491 | return | |
| 379 | 492 | } | |
| @@ -381,14 +494,14 @@ func (t *Transport) resendLoop() { | |||
| 381 | 494 | } | |
| 382 | 495 | ||
| 383 | 496 | func (t *Transport) resendPending() { | |
| 384 | - | now := time.Now() | |
| 497 | + | now := t.clock.Now() | |
| 385 | 498 | var resend []struct { | |
| 386 | 499 | seq uint64 | |
| 387 | 500 | f ports.Frame | |
| 388 | 501 | } | |
| 389 | 502 | t.mu.Lock() | |
| 390 | 503 | for seq, p := range t.pending { | |
| 391 | - | if p.last.IsZero() || now.Sub(p.last) >= defaultResend { | |
| 504 | + | if p.last.IsZero() || now.Sub(p.last) >= t.resendAfter { | |
| 392 | 505 | p.last = now | |
| 393 | 506 | resend = append(resend, struct { | |
| 394 | 507 | seq uint64 | |
| @@ -402,14 +515,78 @@ func (t *Transport) resendPending() { | |||
| 402 | 515 | } | |
| 403 | 516 | } | |
| 404 | 517 | ||
| 518 | + | func (t *Transport) hopPacketConnOnce() { | |
| 519 | + | t.mu.Lock() | |
| 520 | + | if t.rebind == nil || t.hoppedOffline || t.closed { | |
| 521 | + | t.mu.Unlock() | |
| 522 | + | return | |
| 523 | + | } | |
| 524 | + | old := t.pc | |
| 525 | + | rebind := t.rebind | |
| 526 | + | t.hoppedOffline = true | |
| 527 | + | t.mu.Unlock() | |
| 528 | + | pc, err := rebind(old) | |
| 529 | + | if err != nil { | |
| 530 | + | t.mu.Lock() | |
| 531 | + | if t.pc == old { | |
| 532 | + | t.hoppedOffline = false | |
| 533 | + | } | |
| 534 | + | t.mu.Unlock() | |
| 535 | + | return | |
| 536 | + | } | |
| 537 | + | t.mu.Lock() | |
| 538 | + | if t.closed || t.pc != old { | |
| 539 | + | t.mu.Unlock() | |
| 540 | + | _ = pc.Close() | |
| 541 | + | return | |
| 542 | + | } | |
| 543 | + | t.pc = pc | |
| 544 | + | t.mu.Unlock() | |
| 545 | + | go t.readLoop(pc) | |
| 546 | + | _ = old.Close() | |
| 547 | + | } | |
| 548 | + | ||
| 405 | 549 | func (t *Transport) checkSilence() { | |
| 406 | 550 | t.mu.Lock() | |
| 407 | 551 | last := t.lastHeard | |
| 408 | - | silence := t.silence | |
| 409 | 552 | closed := t.closed | |
| 553 | + | degradedAfter := t.degradedAfter | |
| 554 | + | probeAfter := t.probeAfter | |
| 555 | + | offlineAfter := t.offlineAfter | |
| 556 | + | deadAfter := t.deadAfter | |
| 557 | + | now := t.clock.Now() | |
| 410 | 558 | t.mu.Unlock() | |
| 411 | - | if !closed && silence > 0 && time.Since(last) > silence { | |
| 559 | + | if closed { | |
| 560 | + | return | |
| 561 | + | } | |
| 562 | + | silentFor := now.Sub(last) | |
| 563 | + | switch { | |
| 564 | + | case deadAfter > 0 && silentFor >= deadAfter: | |
| 565 | + | t.setLinkState(ports.LinkStateDead, ErrLinkDead) | |
| 412 | 566 | t.closeWithError(ErrLinkDead) | |
| 567 | + | case offlineAfter > 0 && silentFor >= offlineAfter: | |
| 568 | + | t.setLinkState(ports.LinkStateOffline, nil) | |
| 569 | + | t.hopPacketConnOnce() | |
| 570 | + | case probeAfter > 0 && silentFor >= probeAfter: | |
| 571 | + | t.setLinkState(ports.LinkStateProbing, nil) | |
| 572 | + | case degradedAfter > 0 && silentFor >= degradedAfter: | |
| 573 | + | t.setLinkState(ports.LinkStateDegraded, nil) | |
| 574 | + | } | |
| 575 | + | } | |
| 576 | + | ||
| 577 | + | func (t *Transport) setLinkState(state ports.LinkState, err error) { | |
| 578 | + | now := t.clock.Now() | |
| 579 | + | t.mu.Lock() | |
| 580 | + | if t.linkState == state { | |
| 581 | + | t.mu.Unlock() | |
| 582 | + | return | |
| 583 | + | } | |
| 584 | + | t.linkState = state | |
| 585 | + | t.mu.Unlock() | |
| 586 | + | event := ports.LinkEvent{State: state, At: now, Err: err} | |
| 587 | + | select { | |
| 588 | + | case t.linkEvents <- event: | |
| 589 | + | default: | |
| 413 | 590 | } | |
| 414 | 591 | } | |
| 415 | 592 | ||
| @@ -19,11 +19,18 @@ const ( | |||||
| 19 | recProbe byte = 3 | 19 | recProbe byte = 3 | ||
| 20 | recPong byte = 4 | 20 | recPong byte = 4 | ||
| 21 | 21 | ||||
| 22 | - | defaultResend = 40 * time.Millisecond | 22 | + | defaultMTU = pdgram.DefaultMTU |
| 23 | - | defaultHeartbeat = 3 * time.Second | 23 | + | defaultResend = 40 * time.Millisecond |
| 24 | - | defaultSilenceTimeout = 10 * time.Second | 24 | + | defaultHeartbeat = 3 * time.Second |
| 25 | - | maxPendingReliable = 1024 | 25 | + | defaultDegraded = 10 * time.Second |
| 26 | - | maxRecvBuffer = 1024 | 26 | + | defaultProbe = 20 * time.Second |
| 27 | + | defaultOffline = 30 * time.Second | |||
| 28 | + | defaultDead = 60 * time.Second | |||
| 29 | + | defaultMaxPending = 1024 | |||
| 30 | + | defaultMaxRecvBuf = 1024 | |||
| 31 | + | maxPendingReliable = defaultMaxPending | |||
| 32 | + | maxRecvBuffer = defaultMaxRecvBuf | |||
| 33 | + | linkEventBufferSize = 16 | |||
| 27 | ) | 34 | ) | ||
| 28 | 35 | ||||
| 29 | var ( | 36 | var ( | ||
| @@ -31,25 +38,100 @@ var ( | |||||
| 31 | ErrLinkDead = errors.New("dgram: link dead") | 38 | ErrLinkDead = errors.New("dgram: link dead") | ||
| 32 | ) | 39 | ) | ||
| 33 | 40 | ||||
| 41 | + | type Options struct { | |||
| 42 | + | MTU int | |||
| 43 | + | ResendAfter time.Duration | |||
| 44 | + | Heartbeat time.Duration | |||
| 45 | + | DegradedAfter time.Duration | |||
| 46 | + | ProbeAfter time.Duration | |||
| 47 | + | OfflineAfter time.Duration | |||
| 48 | + | DeadAfter time.Duration | |||
| 49 | + | MaxPending int | |||
| 50 | + | MaxRecvBuffer int | |||
| 51 | + | Clock ports.Clock | |||
| 52 | + | // RebindPacketConn is an optional client-side hook used to hop the local UDP | |||
| 53 | + | // socket when the peer is offline. Server/proxy transports should leave it nil. | |||
| 54 | + | RebindPacketConn func(net.PacketConn) (net.PacketConn, error) | |||
| 55 | + | } | |||
| 56 | + | ||||
| 57 | + | type realClock struct{} | |||
| 58 | + | ||||
| 59 | + | func (realClock) Now() time.Time { return time.Now() } | |||
| 60 | + | func (realClock) NewTimer(d time.Duration) ports.Timer { | |||
| 61 | + | return realTimer{t: time.NewTimer(d)} | |||
| 62 | + | } | |||
| 63 | + | ||||
| 64 | + | type realTimer struct{ t *time.Timer } | |||
| 65 | + | ||||
| 66 | + | func (r realTimer) C() <-chan time.Time { return r.t.C } | |||
| 67 | + | func (r realTimer) Reset(d time.Duration) bool { return r.t.Reset(d) } | |||
| 68 | + | func (r realTimer) Stop() bool { return r.t.Stop() } | |||
| 69 | + | ||||
| 70 | + | func normalizeOptions(opts Options) Options { | |||
| 71 | + | if opts.MTU <= 0 { | |||
| 72 | + | opts.MTU = defaultMTU | |||
| 73 | + | } | |||
| 74 | + | if opts.ResendAfter <= 0 { | |||
| 75 | + | opts.ResendAfter = defaultResend | |||
| 76 | + | } | |||
| 77 | + | if opts.Heartbeat <= 0 { | |||
| 78 | + | opts.Heartbeat = defaultHeartbeat | |||
| 79 | + | } | |||
| 80 | + | if opts.DegradedAfter <= 0 { | |||
| 81 | + | opts.DegradedAfter = defaultDegraded | |||
| 82 | + | } | |||
| 83 | + | if opts.ProbeAfter <= 0 { | |||
| 84 | + | opts.ProbeAfter = defaultProbe | |||
| 85 | + | } | |||
| 86 | + | if opts.OfflineAfter <= 0 { | |||
| 87 | + | opts.OfflineAfter = defaultOffline | |||
| 88 | + | } | |||
| 89 | + | if opts.DeadAfter <= 0 { | |||
| 90 | + | opts.DeadAfter = defaultDead | |||
| 91 | + | } | |||
| 92 | + | if opts.MaxPending <= 0 { | |||
| 93 | + | opts.MaxPending = defaultMaxPending | |||
| 94 | + | } | |||
| 95 | + | if opts.MaxRecvBuffer <= 0 { | |||
| 96 | + | opts.MaxRecvBuffer = defaultMaxRecvBuf | |||
| 97 | + | } | |||
| 98 | + | if opts.Clock == nil { | |||
| 99 | + | opts.Clock = realClock{} | |||
| 100 | + | } | |||
| 101 | + | return opts | |||
| 102 | + | } | |||
| 103 | + | ||||
| 34 | type Transport struct { | 104 | type Transport struct { | ||
| 35 | pc net.PacketConn | 105 | pc net.PacketConn | ||
| 36 | codec *pdgram.Codec | 106 | codec *pdgram.Codec | ||
| 37 | sendDir, recvDir uint32 | 107 | sendDir, recvDir uint32 | ||
| 38 | mtu int | 108 | mtu int | ||
| 39 | 109 | ||||
| 40 | - | mu sync.Mutex | 110 | + | mu sync.Mutex |
| 41 | - | peer net.Addr | 111 | + | peer net.Addr |
| 42 | - | ctr uint64 | 112 | + | ctr uint64 |
| 43 | - | seq uint64 | 113 | + | seq uint64 |
| 44 | - | probeSeq uint64 | 114 | + | probeSeq uint64 |
| 45 | - | pending map[uint64]*pending | 115 | + | pending map[uint64]*pending |
| 46 | - | closed bool | 116 | + | closed bool |
| 47 | - | closeErr error | 117 | + | closeErr error |
| 48 | - | lastHeard time.Time | 118 | + | lastHeard time.Time |
| 49 | - | heartbeat time.Duration | 119 | + | heartbeat time.Duration |
| 50 | - | silence time.Duration | 120 | + | resendAfter time.Duration |
| 51 | - | probeWait map[uint64]chan struct{} | 121 | + | degradedAfter time.Duration |
| 122 | + | probeAfter time.Duration | |||
| 123 | + | offlineAfter time.Duration | |||
| 124 | + | deadAfter time.Duration | |||
| 125 | + | maxPending int | |||
| 126 | + | maxRecvBuffer int | |||
| 127 | + | clock ports.Clock | |||
| 128 | + | linkState ports.LinkState | |||
| 129 | + | linkEvents chan ports.LinkEvent | |||
| 130 | + | probeWait map[uint64]chan struct{} | |||
| 131 | + | rebind func(net.PacketConn) (net.PacketConn, error) | |||
| 132 | + | hoppedOffline bool | |||
| 52 | 133 | ||||
| 134 | + | recvMu sync.Mutex | |||
| 53 | replay *pdgram.ReplayWindow | 135 | replay *pdgram.ReplayWindow | ||
| 54 | reasm *pdgram.Reassembler | 136 | reasm *pdgram.Reassembler | ||
| 55 | in chan ports.Frame | 137 | in chan ports.Frame | ||
| @@ -67,13 +149,18 @@ type pending struct { | |||||
| 67 | } | 149 | } | ||
| 68 | 150 | ||||
| 69 | func NewTransport(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32) (*Transport, error) { | 151 | func NewTransport(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32) (*Transport, error) { | ||
| 152 | + | return NewTransportWithOptions(pc, peer, key, sendDir, recvDir, Options{}) | |||
| 153 | + | } | |||
| 154 | + | ||||
| 155 | + | func NewTransportWithOptions(pc net.PacketConn, peer net.Addr, key []byte, sendDir, recvDir uint32, opts Options) (*Transport, error) { | |||
| 70 | c, err := pdgram.NewCodec(key) | 156 | c, err := pdgram.NewCodec(key) | ||
| 71 | if err != nil { | 157 | if err != nil { | ||
| 72 | return nil, err | 158 | return nil, err | ||
| 73 | } | 159 | } | ||
| 74 | - | t := &Transport{pc: pc, codec: c, sendDir: sendDir, recvDir: recvDir, mtu: pdgram.DefaultMTU, peer: peer, pending: make(map[uint64]*pending), replay: pdgram.NewReplayWindow(), reasm: pdgram.NewReassembler(), in: make(chan ports.Frame, 32), done: make(chan struct{}), lastHeard: time.Now(), heartbeat: defaultHeartbeat, silence: defaultSilenceTimeout, probeWait: make(map[uint64]chan struct{}), nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame)} | 160 | + | opts = normalizeOptions(opts) |
| 161 | + | t := &Transport{pc: pc, codec: c, sendDir: sendDir, recvDir: recvDir, mtu: opts.MTU, peer: peer, pending: make(map[uint64]*pending), replay: pdgram.NewReplayWindow(), reasm: pdgram.NewReassembler(), in: make(chan ports.Frame, 32), done: make(chan struct{}), lastHeard: opts.Clock.Now(), heartbeat: opts.Heartbeat, resendAfter: opts.ResendAfter, degradedAfter: opts.DegradedAfter, probeAfter: opts.ProbeAfter, offlineAfter: opts.OfflineAfter, deadAfter: opts.DeadAfter, maxPending: opts.MaxPending, maxRecvBuffer: opts.MaxRecvBuffer, clock: opts.Clock, linkState: ports.LinkStateConnected, linkEvents: make(chan ports.LinkEvent, linkEventBufferSize), probeWait: make(map[uint64]chan struct{}), rebind: opts.RebindPacketConn, nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame)} | |||
| 75 | t.deliverCond = sync.NewCond(&t.deliverMu) | 162 | t.deliverCond = sync.NewCond(&t.deliverMu) | ||
| 76 | - | go t.readLoop() | 163 | + | go t.readLoop(pc) |
| 77 | go t.resendLoop() | 164 | go t.resendLoop() | ||
| 78 | go t.deliveryLoop() | 165 | go t.deliveryLoop() | ||
| 79 | return t, nil | 166 | return t, nil | ||
| @@ -92,7 +179,7 @@ func (t *Transport) Send(f ports.Frame) error { | |||||
| 92 | } | 179 | } | ||
| 93 | return errors.New("dgram: closed") | 180 | return errors.New("dgram: closed") | ||
| 94 | } | 181 | } | ||
| 95 | - | if reliable && len(t.pending) >= maxPendingReliable { | 182 | + | if reliable && len(t.pending) >= t.maxPending { |
| 96 | t.mu.Unlock() | 183 | t.mu.Unlock() | ||
| 97 | return ErrPendingFull | 184 | return ErrPendingFull | ||
| 98 | } | 185 | } | ||
| @@ -121,11 +208,22 @@ func (t *Transport) Recv() (ports.Frame, error) { | |||||
| 121 | } | 208 | } | ||
| 122 | func (t *Transport) Close() error { | 209 | func (t *Transport) Close() error { | ||
| 123 | t.closeWithError(errors.New("dgram: closed")) | 210 | t.closeWithError(errors.New("dgram: closed")) | ||
| 124 | - | return t.pc.Close() | 211 | + | t.mu.Lock() |
| 212 | + | pc := t.pc | |||
| 213 | + | t.mu.Unlock() | |||
| 214 | + | return pc.Close() | |||
| 125 | } | 215 | } | ||
| 126 | 216 | ||||
| 127 | func (t *Transport) Peer() net.Addr { t.mu.Lock(); defer t.mu.Unlock(); return t.peer } | 217 | func (t *Transport) Peer() net.Addr { t.mu.Lock(); defer t.mu.Unlock(); return t.peer } | ||
| 128 | 218 | ||||
| 219 | + | func (t *Transport) LinkState() ports.LinkState { | |||
| 220 | + | t.mu.Lock() | |||
| 221 | + | defer t.mu.Unlock() | |||
| 222 | + | return t.linkState | |||
| 223 | + | } | |||
| 224 | + | ||||
| 225 | + | func (t *Transport) LinkEvents() <-chan ports.LinkEvent { return t.linkEvents } | |||
| 226 | + | ||||
| 129 | // Probe sends an authenticated datagram and waits for the peer to authenticate a | 227 | // Probe sends an authenticated datagram and waits for the peer to authenticate a | ||
| 130 | // response or for ctx to expire. It is intended for UDP bootstrap reachability checks. | 228 | // response or for ctx to expire. It is intended for UDP bootstrap reachability checks. | ||
| 131 | func (t *Transport) Probe(ctx context.Context) error { | 229 | func (t *Transport) Probe(ctx context.Context) error { | ||
| @@ -216,7 +314,10 @@ func (t *Transport) sendPayload(p []byte) error { | |||||
| 216 | if peer == nil { | 314 | if peer == nil { | ||
| 217 | return errors.New("dgram: no peer") | 315 | return errors.New("dgram: no peer") | ||
| 218 | } | 316 | } | ||
| 219 | - | if _, err := t.pc.WriteTo(pkt, peer); err != nil { | 317 | + | t.mu.Lock() |
| 318 | + | pc := t.pc | |||
| 319 | + | t.mu.Unlock() | |||
| 320 | + | if _, err := pc.WriteTo(pkt, peer); err != nil { | |||
| 220 | return err | 321 | return err | ||
| 221 | } | 322 | } | ||
| 222 | } | 323 | } | ||
| @@ -224,34 +325,50 @@ func (t *Transport) sendPayload(p []byte) error { | |||||
| 224 | } | 325 | } | ||
| 225 | func (t *Transport) nextCounter() uint64 { t.mu.Lock(); defer t.mu.Unlock(); t.ctr++; return t.ctr } | 326 | func (t *Transport) nextCounter() uint64 { t.mu.Lock(); defer t.mu.Unlock(); t.ctr++; return t.ctr } | ||
| 226 | 327 | ||||
| 227 | - | func (t *Transport) readLoop() { | 328 | + | func (t *Transport) readLoop(pc net.PacketConn) { |
| 228 | buf := make([]byte, 64*1024) | 329 | buf := make([]byte, 64*1024) | ||
| 229 | for { | 330 | for { | ||
| 230 | - | n, addr, err := t.pc.ReadFrom(buf) | 331 | + | n, addr, err := pc.ReadFrom(buf) |
| 231 | if err != nil { | 332 | if err != nil { | ||
| 232 | - | t.closeWithError(err) | 333 | + | t.mu.Lock() |
| 334 | + | current := t.pc == pc | |||
| 335 | + | closed := t.closed | |||
| 336 | + | t.mu.Unlock() | |||
| 337 | + | if current && !closed { | |||
| 338 | + | t.closeWithError(err) | |||
| 339 | + | } | |||
| 233 | return | 340 | return | ||
| 234 | } | 341 | } | ||
| 342 | + | t.recvMu.Lock() | |||
| 235 | _, pt, err := t.codec.Open(append([]byte(nil), buf[:n]...), t.recvDir, nil, t.replay) | 343 | _, pt, err := t.codec.Open(append([]byte(nil), buf[:n]...), t.recvDir, nil, t.replay) | ||
| 236 | if err != nil { | 344 | if err != nil { | ||
| 345 | + | t.recvMu.Unlock() | |||
| 237 | continue | 346 | continue | ||
| 238 | } | 347 | } | ||
| 239 | - | t.mu.Lock() | |||
| 240 | - | t.peer = addr | |||
| 241 | - | t.lastHeard = time.Now() | |||
| 242 | - | t.mu.Unlock() | |||
| 243 | frag, err := pdgram.UnmarshalFragment(pt) | 348 | frag, err := pdgram.UnmarshalFragment(pt) | ||
| 244 | if err != nil { | 349 | if err != nil { | ||
| 350 | + | t.recvMu.Unlock() | |||
| 245 | continue | 351 | continue | ||
| 246 | } | 352 | } | ||
| 247 | payload, ok, err := t.reasm.Add(frag) | 353 | payload, ok, err := t.reasm.Add(frag) | ||
| 354 | + | t.recvMu.Unlock() | |||
| 248 | if err != nil || !ok { | 355 | if err != nil || !ok { | ||
| 249 | continue | 356 | continue | ||
| 250 | } | 357 | } | ||
| 358 | + | t.updatePeerFromAuthenticated(addr) | |||
| 251 | t.handleRecord(payload) | 359 | t.handleRecord(payload) | ||
| 252 | } | 360 | } | ||
| 253 | } | 361 | } | ||
| 254 | 362 | ||||
| 363 | + | func (t *Transport) updatePeerFromAuthenticated(addr net.Addr) { | |||
| 364 | + | t.mu.Lock() | |||
| 365 | + | t.peer = addr | |||
| 366 | + | t.lastHeard = t.clock.Now() | |||
| 367 | + | t.hoppedOffline = false | |||
| 368 | + | t.mu.Unlock() | |||
| 369 | + | t.setLinkState(ports.LinkStateConnected, nil) | |||
| 370 | + | } | |||
| 371 | + | ||||
| 255 | func (t *Transport) handleRecord(p []byte) { | 372 | func (t *Transport) handleRecord(p []byte) { | ||
| 256 | if len(p) < 1 { | 373 | if len(p) < 1 { | ||
| 257 | return | 374 | return | ||
| @@ -310,14 +427,8 @@ func (t *Transport) enqueueReliable(seq uint64, f ports.Frame) (ack bool, queued | |||||
| 310 | if _, exists := t.recvBuf[seq]; exists { | 427 | if _, exists := t.recvBuf[seq]; exists { | ||
| 311 | return true, false | 428 | return true, false | ||
| 312 | } | 429 | } | ||
| 313 | - | if len(t.recvBuf) >= maxRecvBuffer { | 430 | + | if len(t.recvBuf) >= t.maxRecvBuffer && seq != t.nextRecvSeq { |
| 314 | - | if seq != t.nextRecvSeq { | 431 | + | return false, false |
| 315 | - | return false, false | |||
| 316 | - | } | |||
| 317 | - | for bufferedSeq := range t.recvBuf { | |||
| 318 | - | delete(t.recvBuf, bufferedSeq) | |||
| 319 | - | break | |||
| 320 | - | } | |||
| 321 | } | 432 | } | ||
| 322 | t.recvBuf[seq] = f | 433 | t.recvBuf[seq] = f | ||
| 323 | return true, true | 434 | return true, true | ||
| @@ -362,18 +473,20 @@ func (t *Transport) deliver(f ports.Frame) { | |||||
| 362 | } | 473 | } | ||
| 363 | 474 | ||||
| 364 | func (t *Transport) resendLoop() { | 475 | func (t *Transport) resendLoop() { | ||
| 365 | - | resend := time.NewTicker(defaultResend) | 476 | + | resend := t.clock.NewTimer(t.resendAfter) |
| 366 | defer resend.Stop() | 477 | defer resend.Stop() | ||
| 367 | - | heartbeat := time.NewTicker(t.heartbeat) | 478 | + | heartbeat := t.clock.NewTimer(t.heartbeat) |
| 368 | defer heartbeat.Stop() | 479 | defer heartbeat.Stop() | ||
| 369 | for { | 480 | for { | ||
| 370 | select { | 481 | select { | ||
| 371 | - | case <-resend.C: | 482 | + | case <-resend.C(): |
| 372 | t.resendPending() | 483 | t.resendPending() | ||
| 373 | t.checkSilence() | 484 | t.checkSilence() | ||
| 374 | - | case <-heartbeat.C: | 485 | + | resend.Reset(t.resendAfter) |
| 486 | + | case <-heartbeat.C(): | |||
| 375 | _ = t.sendProbe(recProbe, 0) | 487 | _ = t.sendProbe(recProbe, 0) | ||
| 376 | t.checkSilence() | 488 | t.checkSilence() | ||
| 489 | + | heartbeat.Reset(t.heartbeat) | |||
| 377 | case <-t.done: | 490 | case <-t.done: | ||
| 378 | return | 491 | return | ||
| 379 | } | 492 | } | ||
| @@ -381,14 +494,14 @@ func (t *Transport) resendLoop() { | |||||
| 381 | } | 494 | } | ||
| 382 | 495 | ||||
| 383 | func (t *Transport) resendPending() { | 496 | func (t *Transport) resendPending() { | ||
| 384 | - | now := time.Now() | 497 | + | now := t.clock.Now() |
| 385 | var resend []struct { | 498 | var resend []struct { | ||
| 386 | seq uint64 | 499 | seq uint64 | ||
| 387 | f ports.Frame | 500 | f ports.Frame | ||
| 388 | } | 501 | } | ||
| 389 | t.mu.Lock() | 502 | t.mu.Lock() | ||
| 390 | for seq, p := range t.pending { | 503 | for seq, p := range t.pending { | ||
| 391 | - | if p.last.IsZero() || now.Sub(p.last) >= defaultResend { | 504 | + | if p.last.IsZero() || now.Sub(p.last) >= t.resendAfter { |
| 392 | p.last = now | 505 | p.last = now | ||
| 393 | resend = append(resend, struct { | 506 | resend = append(resend, struct { | ||
| 394 | seq uint64 | 507 | seq uint64 | ||
| @@ -402,14 +515,78 @@ func (t *Transport) resendPending() { | |||||
| 402 | } | 515 | } | ||
| 403 | } | 516 | } | ||
| 404 | 517 | ||||
| 518 | + | func (t *Transport) hopPacketConnOnce() { | |||
| 519 | + | t.mu.Lock() | |||
| 520 | + | if t.rebind == nil || t.hoppedOffline || t.closed { | |||
| 521 | + | t.mu.Unlock() | |||
| 522 | + | return | |||
| 523 | + | } | |||
| 524 | + | old := t.pc | |||
| 525 | + | rebind := t.rebind | |||
| 526 | + | t.hoppedOffline = true | |||
| 527 | + | t.mu.Unlock() | |||
| 528 | + | pc, err := rebind(old) | |||
| 529 | + | if err != nil { | |||
| 530 | + | t.mu.Lock() | |||
| 531 | + | if t.pc == old { | |||
| 532 | + | t.hoppedOffline = false | |||
| 533 | + | } | |||
| 534 | + | t.mu.Unlock() | |||
| 535 | + | return | |||
| 536 | + | } | |||
| 537 | + | t.mu.Lock() | |||
| 538 | + | if t.closed || t.pc != old { | |||
| 539 | + | t.mu.Unlock() | |||
| 540 | + | _ = pc.Close() | |||
| 541 | + | return | |||
| 542 | + | } | |||
| 543 | + | t.pc = pc | |||
| 544 | + | t.mu.Unlock() | |||
| 545 | + | go t.readLoop(pc) | |||
| 546 | + | _ = old.Close() | |||
| 547 | + | } | |||
| 548 | + | ||||
| 405 | func (t *Transport) checkSilence() { | 549 | func (t *Transport) checkSilence() { | ||
| 406 | t.mu.Lock() | 550 | t.mu.Lock() | ||
| 407 | last := t.lastHeard | 551 | last := t.lastHeard | ||
| 408 | - | silence := t.silence | |||
| 409 | closed := t.closed | 552 | closed := t.closed | ||
| 553 | + | degradedAfter := t.degradedAfter | |||
| 554 | + | probeAfter := t.probeAfter | |||
| 555 | + | offlineAfter := t.offlineAfter | |||
| 556 | + | deadAfter := t.deadAfter | |||
| 557 | + | now := t.clock.Now() | |||
| 410 | t.mu.Unlock() | 558 | t.mu.Unlock() | ||
| 411 | - | if !closed && silence > 0 && time.Since(last) > silence { | 559 | + | if closed { |
| 560 | + | return | |||
| 561 | + | } | |||
| 562 | + | silentFor := now.Sub(last) | |||
| 563 | + | switch { | |||
| 564 | + | case deadAfter > 0 && silentFor >= deadAfter: | |||
| 565 | + | t.setLinkState(ports.LinkStateDead, ErrLinkDead) | |||
| 412 | t.closeWithError(ErrLinkDead) | 566 | t.closeWithError(ErrLinkDead) | ||
| 567 | + | case offlineAfter > 0 && silentFor >= offlineAfter: | |||
| 568 | + | t.setLinkState(ports.LinkStateOffline, nil) | |||
| 569 | + | t.hopPacketConnOnce() | |||
| 570 | + | case probeAfter > 0 && silentFor >= probeAfter: | |||
| 571 | + | t.setLinkState(ports.LinkStateProbing, nil) | |||
| 572 | + | case degradedAfter > 0 && silentFor >= degradedAfter: | |||
| 573 | + | t.setLinkState(ports.LinkStateDegraded, nil) | |||
| 574 | + | } | |||
| 575 | + | } | |||
| 576 | + | ||||
| 577 | + | func (t *Transport) setLinkState(state ports.LinkState, err error) { | |||
| 578 | + | now := t.clock.Now() | |||
| 579 | + | t.mu.Lock() | |||
| 580 | + | if t.linkState == state { | |||
| 581 | + | t.mu.Unlock() | |||
| 582 | + | return | |||
| 583 | + | } | |||
| 584 | + | t.linkState = state | |||
| 585 | + | t.mu.Unlock() | |||
| 586 | + | event := ports.LinkEvent{State: state, At: now, Err: err} | |||
| 587 | + | select { | |||
| 588 | + | case t.linkEvents <- event: | |||
| 589 | + | default: | |||
| 413 | } | 590 | } | ||
| 414 | } | 591 | } | ||
| 415 | 592 | ||||
M internal/adapters/dgram/transport_test.go +209 -13
| @@ -151,7 +151,7 @@ func TestReliableInputRetransmitsUntilAck(t *testing.T) { | |||
| 151 | 151 | } | |
| 152 | 152 | } | |
| 153 | 153 | ||
| 154 | - | func TestAuthenticatedOnlyRehome(t *testing.T) { | |
| 154 | + | func TestAuthenticatedPeerChangeUpdatesPeerAndEmitsConnected(t *testing.T) { | |
| 155 | 155 | aPC, bPC := newPair() | |
| 156 | 156 | a, err := NewTransport(aPC, testAddr("b"), key(), 1, 2) | |
| 157 | 157 | if err != nil { | |
| @@ -162,21 +162,149 @@ func TestAuthenticatedOnlyRehome(t *testing.T) { | |||
| 162 | 162 | t.Errorf("close a: %v", err) | |
| 163 | 163 | } | |
| 164 | 164 | }() | |
| 165 | + | evilPC := &fakePC{addr: testAddr("evil"), in: make(chan packet, 100), peers: map[string]*fakePC{}} | |
| 166 | + | aPC.peers["evil"] = evilPC | |
| 165 | 167 | _ = bPC | |
| 166 | 168 | old := a.Peer().String() | |
| 169 | + | a.setLinkState(ports.LinkStateOffline, nil) | |
| 170 | + | <-a.LinkEvents() | |
| 167 | 171 | aPC.in <- packet{[]byte("not authenticated"), testAddr("evil")} | |
| 168 | - | time.Sleep(30 * time.Millisecond) | |
| 169 | 172 | if a.Peer().String() != old { | |
| 170 | 173 | t.Fatalf("unauthenticated packet changed peer to %v", a.Peer()) | |
| 171 | 174 | } | |
| 172 | 175 | c, _ := pdgram.NewCodec(key()) | |
| 173 | - | rec := encodeData(7, false, ports.Frame{Type: ports.MsgPing}) | |
| 176 | + | rec := encodeData(7, true, ports.Frame{Type: ports.MsgPing}) | |
| 174 | 177 | frags, _ := pdgram.FragmentPayload(9, rec, pdgram.DefaultMTU) | |
| 175 | 178 | raw, _ := pdgram.MarshalFragment(frags[0]) | |
| 176 | 179 | aPC.in <- packet{c.Seal(2, 99, raw, nil), testAddr("evil")} | |
| 177 | - | time.Sleep(30 * time.Millisecond) | |
| 178 | - | if a.Peer().String() != "evil" { | |
| 179 | - | t.Fatalf("authenticated packet did not rehome: %v", a.Peer()) | |
| 180 | + | waitPeer(t, a, "evil") | |
| 181 | + | select { | |
| 182 | + | case ev := <-a.LinkEvents(): | |
| 183 | + | if ev.State != ports.LinkStateConnected { | |
| 184 | + | t.Fatalf("event state=%v, want connected", ev.State) | |
| 185 | + | } | |
| 186 | + | case <-time.After(time.Second): | |
| 187 | + | t.Fatal("missing connected event after authenticated rehome") | |
| 188 | + | } | |
| 189 | + | select { | |
| 190 | + | case pkt := <-evilPC.in: | |
| 191 | + | if pkt.addr.String() != "a" { | |
| 192 | + | t.Fatalf("ack source=%v, want a", pkt.addr) | |
| 193 | + | } | |
| 194 | + | case <-time.After(time.Second): | |
| 195 | + | t.Fatal("missing ACK to authenticated new peer") | |
| 196 | + | } | |
| 197 | + | } | |
| 198 | + | ||
| 199 | + | func TestPortHopPreservesPendingReliableMessages(t *testing.T) { | |
| 200 | + | aPC, bPC := newPair() | |
| 201 | + | var hopped atomic.Bool | |
| 202 | + | aPC.drop = func(_ []byte, addr net.Addr) bool { return addr.String() == "b" } | |
| 203 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ | |
| 204 | + | ResendAfter: 20 * time.Millisecond, | |
| 205 | + | OfflineAfter: time.Millisecond, | |
| 206 | + | DeadAfter: time.Hour, | |
| 207 | + | RebindPacketConn: func(old net.PacketConn) (net.PacketConn, error) { | |
| 208 | + | newPC := &fakePC{addr: testAddr("a2"), in: make(chan packet, 100), peers: map[string]*fakePC{"b": bPC}} | |
| 209 | + | bPC.peers["a2"] = newPC | |
| 210 | + | hopped.Store(true) | |
| 211 | + | return newPC, nil | |
| 212 | + | }, | |
| 213 | + | }) | |
| 214 | + | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | |
| 215 | + | defer func() { _ = a.Close() }() | |
| 216 | + | defer func() { _ = b.Close() }() | |
| 217 | + | ||
| 218 | + | if err := a.Send(ports.Frame{Type: ports.MsgInput, Payload: []byte("survive")}); err != nil { | |
| 219 | + | t.Fatal(err) | |
| 220 | + | } | |
| 221 | + | a.mu.Lock() | |
| 222 | + | a.lastHeard = time.Now().Add(-time.Second) | |
| 223 | + | a.mu.Unlock() | |
| 224 | + | a.checkSilence() | |
| 225 | + | if !hopped.Load() { | |
| 226 | + | t.Fatal("expected packet conn rebind") | |
| 227 | + | } | |
| 228 | + | aPC.drop = nil | |
| 229 | + | got := recvWithin(t, b, time.Second) | |
| 230 | + | if got.Type != ports.MsgInput || string(got.Payload) != "survive" { | |
| 231 | + | t.Fatalf("got=%+v, want pending reliable message", got) | |
| 232 | + | } | |
| 233 | + | select { | |
| 234 | + | case <-a.done: | |
| 235 | + | t.Fatal("transport closed during port hop") | |
| 236 | + | default: | |
| 237 | + | } | |
| 238 | + | } | |
| 239 | + | ||
| 240 | + | func TestPortHopRetriesAfterRebindFailure(t *testing.T) { | |
| 241 | + | aPC, bPC := newPair() | |
| 242 | + | var attempts atomic.Int32 | |
| 243 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ | |
| 244 | + | ResendAfter: time.Hour, | |
| 245 | + | Heartbeat: time.Hour, | |
| 246 | + | OfflineAfter: time.Millisecond, | |
| 247 | + | DeadAfter: time.Hour, | |
| 248 | + | RebindPacketConn: func(old net.PacketConn) (net.PacketConn, error) { | |
| 249 | + | if attempts.Add(1) == 1 { | |
| 250 | + | return nil, errors.New("temporary rebind failure") | |
| 251 | + | } | |
| 252 | + | newPC := &fakePC{addr: testAddr("a2"), in: make(chan packet, 100), peers: map[string]*fakePC{"b": bPC}} | |
| 253 | + | bPC.peers["a2"] = newPC | |
| 254 | + | return newPC, nil | |
| 255 | + | }, | |
| 256 | + | }) | |
| 257 | + | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | |
| 258 | + | defer func() { _ = a.Close() }() | |
| 259 | + | defer func() { _ = b.Close() }() | |
| 260 | + | ||
| 261 | + | a.mu.Lock() | |
| 262 | + | a.lastHeard = time.Now().Add(-time.Second) | |
| 263 | + | a.mu.Unlock() | |
| 264 | + | a.checkSilence() | |
| 265 | + | a.mu.Lock() | |
| 266 | + | firstPC := a.pc | |
| 267 | + | a.mu.Unlock() | |
| 268 | + | if attempts.Load() != 1 || firstPC != aPC { | |
| 269 | + | t.Fatalf("first failed hop attempts=%d pc=%v, want one attempt and original pc", attempts.Load(), firstPC.LocalAddr()) | |
| 270 | + | } | |
| 271 | + | ||
| 272 | + | a.checkSilence() | |
| 273 | + | a.mu.Lock() | |
| 274 | + | secondPC := a.pc | |
| 275 | + | a.mu.Unlock() | |
| 276 | + | if attempts.Load() != 2 || secondPC == aPC { | |
| 277 | + | t.Fatalf("second hop attempts=%d pc=%v, want retry with new pc", attempts.Load(), secondPC.LocalAddr()) | |
| 278 | + | } | |
| 279 | + | } | |
| 280 | + | ||
| 281 | + | func TestServerWithoutRebindDoesNotHopPorts(t *testing.T) { | |
| 282 | + | aPC, bPC := newPair() | |
| 283 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{OfflineAfter: time.Millisecond, DeadAfter: time.Hour}) | |
| 284 | + | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | |
| 285 | + | defer func() { _ = a.Close() }() | |
| 286 | + | defer func() { _ = b.Close() }() | |
| 287 | + | a.mu.Lock() | |
| 288 | + | a.lastHeard = time.Now().Add(-time.Second) | |
| 289 | + | a.mu.Unlock() | |
| 290 | + | a.checkSilence() | |
| 291 | + | if a.pc != aPC { | |
| 292 | + | t.Fatal("server/proxy transport hopped without rebind hook") | |
| 293 | + | } | |
| 294 | + | } | |
| 295 | + | ||
| 296 | + | func TestReliableReceiveBufferDoesNotDropAckedFutureFrame(t *testing.T) { | |
| 297 | + | tr := &Transport{maxRecvBuffer: 1, nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame)} | |
| 298 | + | ack, queued := tr.enqueueReliable(2, ports.Frame{Type: ports.MsgOutput, Payload: []byte("future")}) | |
| 299 | + | if !ack || !queued { | |
| 300 | + | t.Fatalf("future enqueue ack=%v queued=%v, want true/true", ack, queued) | |
| 301 | + | } | |
| 302 | + | ack, queued = tr.enqueueReliable(1, ports.Frame{Type: ports.MsgOutput, Payload: []byte("next")}) | |
| 303 | + | if !ack || !queued { | |
| 304 | + | t.Fatalf("next enqueue ack=%v queued=%v, want true/true", ack, queued) | |
| 305 | + | } | |
| 306 | + | if _, ok := tr.recvBuf[2]; !ok { | |
| 307 | + | t.Fatal("acked future frame was dropped when next frame arrived") | |
| 180 | 308 | } | |
| 181 | 309 | } | |
| 182 | 310 | ||
| @@ -277,14 +405,13 @@ func TestProbeTimeoutAndSuccess(t *testing.T) { | |||
| 277 | 405 | } | |
| 278 | 406 | } | |
| 279 | 407 | ||
| 280 | - | func TestRecvUnblocksWhenPeerSilent(t *testing.T) { | |
| 408 | + | func TestRecvUnblocksWhenPeerDeadAfter(t *testing.T) { | |
| 281 | 409 | aPC, bPC := newPair() | |
| 282 | - | a, _ := NewTransport(aPC, testAddr("b"), key(), 1, 2) | |
| 410 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ResendAfter: 25 * time.Millisecond, DeadAfter: 75 * time.Millisecond}) | |
| 283 | 411 | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | |
| 284 | 412 | defer func() { _ = a.Close() }() | |
| 285 | 413 | defer func() { _ = b.Close() }() | |
| 286 | 414 | a.mu.Lock() | |
| 287 | - | a.silence = 75 * time.Millisecond | |
| 288 | 415 | a.lastHeard = time.Now().Add(-time.Second) | |
| 289 | 416 | a.mu.Unlock() | |
| 290 | 417 | ||
| @@ -319,7 +446,7 @@ func TestPendingReliableQueueBounded(t *testing.T) { | |||
| 319 | 446 | } | |
| 320 | 447 | ||
| 321 | 448 | func TestReliableRecvBufferBoundedForFarFutureSequences(t *testing.T) { | |
| 322 | - | tr := &Transport{nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame), done: make(chan struct{})} | |
| 449 | + | tr := &Transport{nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame), maxRecvBuffer: maxRecvBuffer, done: make(chan struct{})} | |
| 323 | 450 | tr.deliverCond = sync.NewCond(&tr.deliverMu) | |
| 324 | 451 | for i := 0; i < maxRecvBuffer+100; i++ { | |
| 325 | 452 | tr.enqueueReliable(uint64(1000+i), ports.Frame{Type: ports.MsgOutput, Payload: []byte{byte(i)}}) | |
| @@ -331,8 +458,8 @@ func TestReliableRecvBufferBoundedForFarFutureSequences(t *testing.T) { | |||
| 331 | 458 | if _, ok := tr.recvBuf[1]; !ok { | |
| 332 | 459 | t.Fatalf("next expected sequence was dropped when far-future buffer was full") | |
| 333 | 460 | } | |
| 334 | - | if got := len(tr.recvBuf); got > maxRecvBuffer { | |
| 335 | - | t.Fatalf("recvBuf len=%d, want <= %d", got, maxRecvBuffer) | |
| 461 | + | if got := len(tr.recvBuf); got > maxRecvBuffer+1 { | |
| 462 | + | t.Fatalf("recvBuf len=%d, want <= %d", got, maxRecvBuffer+1) | |
| 336 | 463 | } | |
| 337 | 464 | } | |
| 338 | 465 | ||
| @@ -342,7 +469,7 @@ func TestReliableFullRecvBufferDoesNotAckDroppedFarFutureFrame(t *testing.T) { | |||
| 342 | 469 | if err != nil { | |
| 343 | 470 | t.Fatal(err) | |
| 344 | 471 | } | |
| 345 | - | tr := &Transport{pc: aPC, peer: testAddr("b"), codec: codec, sendDir: 1, mtu: pdgram.DefaultMTU, nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame), done: make(chan struct{})} | |
| 472 | + | tr := &Transport{pc: aPC, peer: testAddr("b"), codec: codec, sendDir: 1, mtu: pdgram.DefaultMTU, nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame), maxRecvBuffer: maxRecvBuffer, clock: realClock{}, done: make(chan struct{})} | |
| 346 | 473 | tr.deliverCond = sync.NewCond(&tr.deliverMu) | |
| 347 | 474 | defer func() { _ = aPC.Close() }() | |
| 348 | 475 | defer func() { _ = bPC.Close() }() | |
| @@ -389,6 +516,18 @@ func recvWithin(t *testing.T, tr *Transport, d time.Duration) ports.Frame { | |||
| 389 | 516 | } | |
| 390 | 517 | } | |
| 391 | 518 | ||
| 519 | + | func waitPeer(t *testing.T, tr *Transport, want string) { | |
| 520 | + | t.Helper() | |
| 521 | + | deadline := time.Now().Add(time.Second) | |
| 522 | + | for time.Now().Before(deadline) { | |
| 523 | + | if tr.Peer().String() == want { | |
| 524 | + | return | |
| 525 | + | } | |
| 526 | + | time.Sleep(time.Millisecond) | |
| 527 | + | } | |
| 528 | + | t.Fatalf("peer=%v, want %s", tr.Peer(), want) | |
| 529 | + | } | |
| 530 | + | ||
| 392 | 531 | func recvErrWithin(t *testing.T, tr *Transport, d time.Duration) (ports.Frame, error) { | |
| 393 | 532 | t.Helper() | |
| 394 | 533 | type result struct { | |
| @@ -408,3 +547,60 @@ func recvErrWithin(t *testing.T, tr *Transport, d time.Duration) (ports.Frame, e | |||
| 408 | 547 | return ports.Frame{}, nil | |
| 409 | 548 | } | |
| 410 | 549 | } | |
| 550 | + | ||
| 551 | + | func TestLinkStateTransitionsFromConfigurableSilence(t *testing.T) { | |
| 552 | + | aPC, bPC := newPair() | |
| 553 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{DegradedAfter: time.Second, ProbeAfter: 2 * time.Second, OfflineAfter: 3 * time.Second, DeadAfter: 4 * time.Second}) | |
| 554 | + | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | |
| 555 | + | defer func() { _ = a.Close() }() | |
| 556 | + | defer func() { _ = b.Close() }() | |
| 557 | + | ||
| 558 | + | tests := []struct { | |
| 559 | + | name string | |
| 560 | + | age time.Duration | |
| 561 | + | want ports.LinkState | |
| 562 | + | }{ | |
| 563 | + | {"degraded", time.Second, ports.LinkStateDegraded}, | |
| 564 | + | {"probing", 2 * time.Second, ports.LinkStateProbing}, | |
| 565 | + | {"offline", 3 * time.Second, ports.LinkStateOffline}, | |
| 566 | + | } | |
| 567 | + | for _, tt := range tests { | |
| 568 | + | t.Run(tt.name, func(t *testing.T) { | |
| 569 | + | a.mu.Lock() | |
| 570 | + | a.lastHeard = time.Now().Add(-tt.age) | |
| 571 | + | a.mu.Unlock() | |
| 572 | + | a.checkSilence() | |
| 573 | + | if got := a.LinkState(); got != tt.want { | |
| 574 | + | t.Fatalf("LinkState()=%v, want %v", got, tt.want) | |
| 575 | + | } | |
| 576 | + | select { | |
| 577 | + | case ev := <-a.LinkEvents(): | |
| 578 | + | if ev.State != tt.want { | |
| 579 | + | t.Fatalf("event state=%v, want %v", ev.State, tt.want) | |
| 580 | + | } | |
| 581 | + | default: | |
| 582 | + | t.Fatalf("missing link event for %v", tt.want) | |
| 583 | + | } | |
| 584 | + | }) | |
| 585 | + | } | |
| 586 | + | } | |
| 587 | + | ||
| 588 | + | func TestShortSilenceDoesNotKillTransport(t *testing.T) { | |
| 589 | + | aPC, bPC := newPair() | |
| 590 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{DegradedAfter: 50 * time.Millisecond, DeadAfter: time.Hour}) | |
| 591 | + | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | |
| 592 | + | defer func() { _ = a.Close() }() | |
| 593 | + | defer func() { _ = b.Close() }() | |
| 594 | + | a.mu.Lock() | |
| 595 | + | a.lastHeard = time.Now().Add(-time.Second) | |
| 596 | + | a.mu.Unlock() | |
| 597 | + | a.checkSilence() | |
| 598 | + | if got := a.LinkState(); got != ports.LinkStateDegraded { | |
| 599 | + | t.Fatalf("LinkState()=%v, want degraded", got) | |
| 600 | + | } | |
| 601 | + | select { | |
| 602 | + | case <-a.done: | |
| 603 | + | t.Fatal("transport closed before DeadAfter") | |
| 604 | + | default: | |
| 605 | + | } | |
| 606 | + | } | |
| @@ -151,7 +151,7 @@ func TestReliableInputRetransmitsUntilAck(t *testing.T) { | |||||
| 151 | } | 151 | } | ||
| 152 | } | 152 | } | ||
| 153 | 153 | ||||
| 154 | - | func TestAuthenticatedOnlyRehome(t *testing.T) { | 154 | + | func TestAuthenticatedPeerChangeUpdatesPeerAndEmitsConnected(t *testing.T) { |
| 155 | aPC, bPC := newPair() | 155 | aPC, bPC := newPair() | ||
| 156 | a, err := NewTransport(aPC, testAddr("b"), key(), 1, 2) | 156 | a, err := NewTransport(aPC, testAddr("b"), key(), 1, 2) | ||
| 157 | if err != nil { | 157 | if err != nil { | ||
| @@ -162,21 +162,149 @@ func TestAuthenticatedOnlyRehome(t *testing.T) { | |||||
| 162 | t.Errorf("close a: %v", err) | 162 | t.Errorf("close a: %v", err) | ||
| 163 | } | 163 | } | ||
| 164 | }() | 164 | }() | ||
| 165 | + | evilPC := &fakePC{addr: testAddr("evil"), in: make(chan packet, 100), peers: map[string]*fakePC{}} | |||
| 166 | + | aPC.peers["evil"] = evilPC | |||
| 165 | _ = bPC | 167 | _ = bPC | ||
| 166 | old := a.Peer().String() | 168 | old := a.Peer().String() | ||
| 169 | + | a.setLinkState(ports.LinkStateOffline, nil) | |||
| 170 | + | <-a.LinkEvents() | |||
| 167 | aPC.in <- packet{[]byte("not authenticated"), testAddr("evil")} | 171 | aPC.in <- packet{[]byte("not authenticated"), testAddr("evil")} | ||
| 168 | - | time.Sleep(30 * time.Millisecond) | |||
| 169 | if a.Peer().String() != old { | 172 | if a.Peer().String() != old { | ||
| 170 | t.Fatalf("unauthenticated packet changed peer to %v", a.Peer()) | 173 | t.Fatalf("unauthenticated packet changed peer to %v", a.Peer()) | ||
| 171 | } | 174 | } | ||
| 172 | c, _ := pdgram.NewCodec(key()) | 175 | c, _ := pdgram.NewCodec(key()) | ||
| 173 | - | rec := encodeData(7, false, ports.Frame{Type: ports.MsgPing}) | 176 | + | rec := encodeData(7, true, ports.Frame{Type: ports.MsgPing}) |
| 174 | frags, _ := pdgram.FragmentPayload(9, rec, pdgram.DefaultMTU) | 177 | frags, _ := pdgram.FragmentPayload(9, rec, pdgram.DefaultMTU) | ||
| 175 | raw, _ := pdgram.MarshalFragment(frags[0]) | 178 | raw, _ := pdgram.MarshalFragment(frags[0]) | ||
| 176 | aPC.in <- packet{c.Seal(2, 99, raw, nil), testAddr("evil")} | 179 | aPC.in <- packet{c.Seal(2, 99, raw, nil), testAddr("evil")} | ||
| 177 | - | time.Sleep(30 * time.Millisecond) | 180 | + | waitPeer(t, a, "evil") |
| 178 | - | if a.Peer().String() != "evil" { | 181 | + | select { |
| 179 | - | t.Fatalf("authenticated packet did not rehome: %v", a.Peer()) | 182 | + | case ev := <-a.LinkEvents(): |
| 183 | + | if ev.State != ports.LinkStateConnected { | |||
| 184 | + | t.Fatalf("event state=%v, want connected", ev.State) | |||
| 185 | + | } | |||
| 186 | + | case <-time.After(time.Second): | |||
| 187 | + | t.Fatal("missing connected event after authenticated rehome") | |||
| 188 | + | } | |||
| 189 | + | select { | |||
| 190 | + | case pkt := <-evilPC.in: | |||
| 191 | + | if pkt.addr.String() != "a" { | |||
| 192 | + | t.Fatalf("ack source=%v, want a", pkt.addr) | |||
| 193 | + | } | |||
| 194 | + | case <-time.After(time.Second): | |||
| 195 | + | t.Fatal("missing ACK to authenticated new peer") | |||
| 196 | + | } | |||
| 197 | + | } | |||
| 198 | + | ||||
| 199 | + | func TestPortHopPreservesPendingReliableMessages(t *testing.T) { | |||
| 200 | + | aPC, bPC := newPair() | |||
| 201 | + | var hopped atomic.Bool | |||
| 202 | + | aPC.drop = func(_ []byte, addr net.Addr) bool { return addr.String() == "b" } | |||
| 203 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ | |||
| 204 | + | ResendAfter: 20 * time.Millisecond, | |||
| 205 | + | OfflineAfter: time.Millisecond, | |||
| 206 | + | DeadAfter: time.Hour, | |||
| 207 | + | RebindPacketConn: func(old net.PacketConn) (net.PacketConn, error) { | |||
| 208 | + | newPC := &fakePC{addr: testAddr("a2"), in: make(chan packet, 100), peers: map[string]*fakePC{"b": bPC}} | |||
| 209 | + | bPC.peers["a2"] = newPC | |||
| 210 | + | hopped.Store(true) | |||
| 211 | + | return newPC, nil | |||
| 212 | + | }, | |||
| 213 | + | }) | |||
| 214 | + | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | |||
| 215 | + | defer func() { _ = a.Close() }() | |||
| 216 | + | defer func() { _ = b.Close() }() | |||
| 217 | + | ||||
| 218 | + | if err := a.Send(ports.Frame{Type: ports.MsgInput, Payload: []byte("survive")}); err != nil { | |||
| 219 | + | t.Fatal(err) | |||
| 220 | + | } | |||
| 221 | + | a.mu.Lock() | |||
| 222 | + | a.lastHeard = time.Now().Add(-time.Second) | |||
| 223 | + | a.mu.Unlock() | |||
| 224 | + | a.checkSilence() | |||
| 225 | + | if !hopped.Load() { | |||
| 226 | + | t.Fatal("expected packet conn rebind") | |||
| 227 | + | } | |||
| 228 | + | aPC.drop = nil | |||
| 229 | + | got := recvWithin(t, b, time.Second) | |||
| 230 | + | if got.Type != ports.MsgInput || string(got.Payload) != "survive" { | |||
| 231 | + | t.Fatalf("got=%+v, want pending reliable message", got) | |||
| 232 | + | } | |||
| 233 | + | select { | |||
| 234 | + | case <-a.done: | |||
| 235 | + | t.Fatal("transport closed during port hop") | |||
| 236 | + | default: | |||
| 237 | + | } | |||
| 238 | + | } | |||
| 239 | + | ||||
| 240 | + | func TestPortHopRetriesAfterRebindFailure(t *testing.T) { | |||
| 241 | + | aPC, bPC := newPair() | |||
| 242 | + | var attempts atomic.Int32 | |||
| 243 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ | |||
| 244 | + | ResendAfter: time.Hour, | |||
| 245 | + | Heartbeat: time.Hour, | |||
| 246 | + | OfflineAfter: time.Millisecond, | |||
| 247 | + | DeadAfter: time.Hour, | |||
| 248 | + | RebindPacketConn: func(old net.PacketConn) (net.PacketConn, error) { | |||
| 249 | + | if attempts.Add(1) == 1 { | |||
| 250 | + | return nil, errors.New("temporary rebind failure") | |||
| 251 | + | } | |||
| 252 | + | newPC := &fakePC{addr: testAddr("a2"), in: make(chan packet, 100), peers: map[string]*fakePC{"b": bPC}} | |||
| 253 | + | bPC.peers["a2"] = newPC | |||
| 254 | + | return newPC, nil | |||
| 255 | + | }, | |||
| 256 | + | }) | |||
| 257 | + | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | |||
| 258 | + | defer func() { _ = a.Close() }() | |||
| 259 | + | defer func() { _ = b.Close() }() | |||
| 260 | + | ||||
| 261 | + | a.mu.Lock() | |||
| 262 | + | a.lastHeard = time.Now().Add(-time.Second) | |||
| 263 | + | a.mu.Unlock() | |||
| 264 | + | a.checkSilence() | |||
| 265 | + | a.mu.Lock() | |||
| 266 | + | firstPC := a.pc | |||
| 267 | + | a.mu.Unlock() | |||
| 268 | + | if attempts.Load() != 1 || firstPC != aPC { | |||
| 269 | + | t.Fatalf("first failed hop attempts=%d pc=%v, want one attempt and original pc", attempts.Load(), firstPC.LocalAddr()) | |||
| 270 | + | } | |||
| 271 | + | ||||
| 272 | + | a.checkSilence() | |||
| 273 | + | a.mu.Lock() | |||
| 274 | + | secondPC := a.pc | |||
| 275 | + | a.mu.Unlock() | |||
| 276 | + | if attempts.Load() != 2 || secondPC == aPC { | |||
| 277 | + | t.Fatalf("second hop attempts=%d pc=%v, want retry with new pc", attempts.Load(), secondPC.LocalAddr()) | |||
| 278 | + | } | |||
| 279 | + | } | |||
| 280 | + | ||||
| 281 | + | func TestServerWithoutRebindDoesNotHopPorts(t *testing.T) { | |||
| 282 | + | aPC, bPC := newPair() | |||
| 283 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{OfflineAfter: time.Millisecond, DeadAfter: time.Hour}) | |||
| 284 | + | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | |||
| 285 | + | defer func() { _ = a.Close() }() | |||
| 286 | + | defer func() { _ = b.Close() }() | |||
| 287 | + | a.mu.Lock() | |||
| 288 | + | a.lastHeard = time.Now().Add(-time.Second) | |||
| 289 | + | a.mu.Unlock() | |||
| 290 | + | a.checkSilence() | |||
| 291 | + | if a.pc != aPC { | |||
| 292 | + | t.Fatal("server/proxy transport hopped without rebind hook") | |||
| 293 | + | } | |||
| 294 | + | } | |||
| 295 | + | ||||
| 296 | + | func TestReliableReceiveBufferDoesNotDropAckedFutureFrame(t *testing.T) { | |||
| 297 | + | tr := &Transport{maxRecvBuffer: 1, nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame)} | |||
| 298 | + | ack, queued := tr.enqueueReliable(2, ports.Frame{Type: ports.MsgOutput, Payload: []byte("future")}) | |||
| 299 | + | if !ack || !queued { | |||
| 300 | + | t.Fatalf("future enqueue ack=%v queued=%v, want true/true", ack, queued) | |||
| 301 | + | } | |||
| 302 | + | ack, queued = tr.enqueueReliable(1, ports.Frame{Type: ports.MsgOutput, Payload: []byte("next")}) | |||
| 303 | + | if !ack || !queued { | |||
| 304 | + | t.Fatalf("next enqueue ack=%v queued=%v, want true/true", ack, queued) | |||
| 305 | + | } | |||
| 306 | + | if _, ok := tr.recvBuf[2]; !ok { | |||
| 307 | + | t.Fatal("acked future frame was dropped when next frame arrived") | |||
| 180 | } | 308 | } | ||
| 181 | } | 309 | } | ||
| 182 | 310 | ||||
| @@ -277,14 +405,13 @@ func TestProbeTimeoutAndSuccess(t *testing.T) { | |||||
| 277 | } | 405 | } | ||
| 278 | } | 406 | } | ||
| 279 | 407 | ||||
| 280 | - | func TestRecvUnblocksWhenPeerSilent(t *testing.T) { | 408 | + | func TestRecvUnblocksWhenPeerDeadAfter(t *testing.T) { |
| 281 | aPC, bPC := newPair() | 409 | aPC, bPC := newPair() | ||
| 282 | - | a, _ := NewTransport(aPC, testAddr("b"), key(), 1, 2) | 410 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{ResendAfter: 25 * time.Millisecond, DeadAfter: 75 * time.Millisecond}) |
| 283 | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | 411 | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | ||
| 284 | defer func() { _ = a.Close() }() | 412 | defer func() { _ = a.Close() }() | ||
| 285 | defer func() { _ = b.Close() }() | 413 | defer func() { _ = b.Close() }() | ||
| 286 | a.mu.Lock() | 414 | a.mu.Lock() | ||
| 287 | - | a.silence = 75 * time.Millisecond | |||
| 288 | a.lastHeard = time.Now().Add(-time.Second) | 415 | a.lastHeard = time.Now().Add(-time.Second) | ||
| 289 | a.mu.Unlock() | 416 | a.mu.Unlock() | ||
| 290 | 417 | ||||
| @@ -319,7 +446,7 @@ func TestPendingReliableQueueBounded(t *testing.T) { | |||||
| 319 | } | 446 | } | ||
| 320 | 447 | ||||
| 321 | func TestReliableRecvBufferBoundedForFarFutureSequences(t *testing.T) { | 448 | func TestReliableRecvBufferBoundedForFarFutureSequences(t *testing.T) { | ||
| 322 | - | tr := &Transport{nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame), done: make(chan struct{})} | 449 | + | tr := &Transport{nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame), maxRecvBuffer: maxRecvBuffer, done: make(chan struct{})} |
| 323 | tr.deliverCond = sync.NewCond(&tr.deliverMu) | 450 | tr.deliverCond = sync.NewCond(&tr.deliverMu) | ||
| 324 | for i := 0; i < maxRecvBuffer+100; i++ { | 451 | for i := 0; i < maxRecvBuffer+100; i++ { | ||
| 325 | tr.enqueueReliable(uint64(1000+i), ports.Frame{Type: ports.MsgOutput, Payload: []byte{byte(i)}}) | 452 | tr.enqueueReliable(uint64(1000+i), ports.Frame{Type: ports.MsgOutput, Payload: []byte{byte(i)}}) | ||
| @@ -331,8 +458,8 @@ func TestReliableRecvBufferBoundedForFarFutureSequences(t *testing.T) { | |||||
| 331 | if _, ok := tr.recvBuf[1]; !ok { | 458 | if _, ok := tr.recvBuf[1]; !ok { | ||
| 332 | t.Fatalf("next expected sequence was dropped when far-future buffer was full") | 459 | t.Fatalf("next expected sequence was dropped when far-future buffer was full") | ||
| 333 | } | 460 | } | ||
| 334 | - | if got := len(tr.recvBuf); got > maxRecvBuffer { | 461 | + | if got := len(tr.recvBuf); got > maxRecvBuffer+1 { |
| 335 | - | t.Fatalf("recvBuf len=%d, want <= %d", got, maxRecvBuffer) | 462 | + | t.Fatalf("recvBuf len=%d, want <= %d", got, maxRecvBuffer+1) |
| 336 | } | 463 | } | ||
| 337 | } | 464 | } | ||
| 338 | 465 | ||||
| @@ -342,7 +469,7 @@ func TestReliableFullRecvBufferDoesNotAckDroppedFarFutureFrame(t *testing.T) { | |||||
| 342 | if err != nil { | 469 | if err != nil { | ||
| 343 | t.Fatal(err) | 470 | t.Fatal(err) | ||
| 344 | } | 471 | } | ||
| 345 | - | tr := &Transport{pc: aPC, peer: testAddr("b"), codec: codec, sendDir: 1, mtu: pdgram.DefaultMTU, nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame), done: make(chan struct{})} | 472 | + | tr := &Transport{pc: aPC, peer: testAddr("b"), codec: codec, sendDir: 1, mtu: pdgram.DefaultMTU, nextRecvSeq: 1, recvBuf: make(map[uint64]ports.Frame), maxRecvBuffer: maxRecvBuffer, clock: realClock{}, done: make(chan struct{})} |
| 346 | tr.deliverCond = sync.NewCond(&tr.deliverMu) | 473 | tr.deliverCond = sync.NewCond(&tr.deliverMu) | ||
| 347 | defer func() { _ = aPC.Close() }() | 474 | defer func() { _ = aPC.Close() }() | ||
| 348 | defer func() { _ = bPC.Close() }() | 475 | defer func() { _ = bPC.Close() }() | ||
| @@ -389,6 +516,18 @@ func recvWithin(t *testing.T, tr *Transport, d time.Duration) ports.Frame { | |||||
| 389 | } | 516 | } | ||
| 390 | } | 517 | } | ||
| 391 | 518 | ||||
| 519 | + | func waitPeer(t *testing.T, tr *Transport, want string) { | |||
| 520 | + | t.Helper() | |||
| 521 | + | deadline := time.Now().Add(time.Second) | |||
| 522 | + | for time.Now().Before(deadline) { | |||
| 523 | + | if tr.Peer().String() == want { | |||
| 524 | + | return | |||
| 525 | + | } | |||
| 526 | + | time.Sleep(time.Millisecond) | |||
| 527 | + | } | |||
| 528 | + | t.Fatalf("peer=%v, want %s", tr.Peer(), want) | |||
| 529 | + | } | |||
| 530 | + | ||||
| 392 | func recvErrWithin(t *testing.T, tr *Transport, d time.Duration) (ports.Frame, error) { | 531 | func recvErrWithin(t *testing.T, tr *Transport, d time.Duration) (ports.Frame, error) { | ||
| 393 | t.Helper() | 532 | t.Helper() | ||
| 394 | type result struct { | 533 | type result struct { | ||
| @@ -408,3 +547,60 @@ func recvErrWithin(t *testing.T, tr *Transport, d time.Duration) (ports.Frame, e | |||||
| 408 | return ports.Frame{}, nil | 547 | return ports.Frame{}, nil | ||
| 409 | } | 548 | } | ||
| 410 | } | 549 | } | ||
| 550 | + | ||||
| 551 | + | func TestLinkStateTransitionsFromConfigurableSilence(t *testing.T) { | |||
| 552 | + | aPC, bPC := newPair() | |||
| 553 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{DegradedAfter: time.Second, ProbeAfter: 2 * time.Second, OfflineAfter: 3 * time.Second, DeadAfter: 4 * time.Second}) | |||
| 554 | + | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | |||
| 555 | + | defer func() { _ = a.Close() }() | |||
| 556 | + | defer func() { _ = b.Close() }() | |||
| 557 | + | ||||
| 558 | + | tests := []struct { | |||
| 559 | + | name string | |||
| 560 | + | age time.Duration | |||
| 561 | + | want ports.LinkState | |||
| 562 | + | }{ | |||
| 563 | + | {"degraded", time.Second, ports.LinkStateDegraded}, | |||
| 564 | + | {"probing", 2 * time.Second, ports.LinkStateProbing}, | |||
| 565 | + | {"offline", 3 * time.Second, ports.LinkStateOffline}, | |||
| 566 | + | } | |||
| 567 | + | for _, tt := range tests { | |||
| 568 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 569 | + | a.mu.Lock() | |||
| 570 | + | a.lastHeard = time.Now().Add(-tt.age) | |||
| 571 | + | a.mu.Unlock() | |||
| 572 | + | a.checkSilence() | |||
| 573 | + | if got := a.LinkState(); got != tt.want { | |||
| 574 | + | t.Fatalf("LinkState()=%v, want %v", got, tt.want) | |||
| 575 | + | } | |||
| 576 | + | select { | |||
| 577 | + | case ev := <-a.LinkEvents(): | |||
| 578 | + | if ev.State != tt.want { | |||
| 579 | + | t.Fatalf("event state=%v, want %v", ev.State, tt.want) | |||
| 580 | + | } | |||
| 581 | + | default: | |||
| 582 | + | t.Fatalf("missing link event for %v", tt.want) | |||
| 583 | + | } | |||
| 584 | + | }) | |||
| 585 | + | } | |||
| 586 | + | } | |||
| 587 | + | ||||
| 588 | + | func TestShortSilenceDoesNotKillTransport(t *testing.T) { | |||
| 589 | + | aPC, bPC := newPair() | |||
| 590 | + | a, _ := NewTransportWithOptions(aPC, testAddr("b"), key(), 1, 2, Options{DegradedAfter: 50 * time.Millisecond, DeadAfter: time.Hour}) | |||
| 591 | + | b, _ := NewTransport(bPC, testAddr("a"), key(), 2, 1) | |||
| 592 | + | defer func() { _ = a.Close() }() | |||
| 593 | + | defer func() { _ = b.Close() }() | |||
| 594 | + | a.mu.Lock() | |||
| 595 | + | a.lastHeard = time.Now().Add(-time.Second) | |||
| 596 | + | a.mu.Unlock() | |||
| 597 | + | a.checkSilence() | |||
| 598 | + | if got := a.LinkState(); got != ports.LinkStateDegraded { | |||
| 599 | + | t.Fatalf("LinkState()=%v, want degraded", got) | |||
| 600 | + | } | |||
| 601 | + | select { | |||
| 602 | + | case <-a.done: | |||
| 603 | + | t.Fatal("transport closed before DeadAfter") | |||
| 604 | + | default: | |||
| 605 | + | } | |||
| 606 | + | } | |||
M internal/app/run.go +15 -37
| @@ -634,6 +634,11 @@ var ( | |||
| 634 | 634 | // runUDPBootstrap starts a detached _udp-proxy, forwards its single readiness | |
| 635 | 635 | // line, and exits so SSH stdio can close without owning the UDP proxy lifetime. | |
| 636 | 636 | func runUDPBootstrap(ctx context.Context, session string) error { | |
| 637 | + | // Always start a fresh proxy for a bootstrap request. A client attach begins | |
| 638 | + | // with MsgHello, which must reach the daemon handshake path; reusing an | |
| 639 | + | // already-running proxy would forward that Hello into the daemon connection's | |
| 640 | + | // post-handshake runConnLoop, where it is intentionally ignored. The new | |
| 641 | + | // proxy publishes itself and retires any older registry entry once ready. | |
| 637 | 642 | r, w, err := os.Pipe() | |
| 638 | 643 | if err != nil { | |
| 639 | 644 | return err | |
| @@ -735,7 +740,14 @@ func runUDPProxy(ctx context.Context, session string, ready io.Writer) error { | |||
| 735 | 740 | if !ok { | |
| 736 | 741 | return errors.New("vev: udp proxy did not get a UDP address") | |
| 737 | 742 | } | |
| 738 | - | if _, err := fmt.Fprintf(ready, "VEV-UDP %d %s\n", addr.Port, base64.StdEncoding.EncodeToString(key)); err != nil { | |
| 743 | + | keyText := base64.StdEncoding.EncodeToString(key) | |
| 744 | + | registry := dgram.NewProxyRegistry(filepath.Join(ipc.SocketDir(), "udp-proxies")) | |
| 745 | + | record := dgram.ProxyRecord{Session: session, PID: os.Getpid(), Port: addr.Port, Key: keyText} | |
| 746 | + | if err := registry.Publish(record); err != nil { | |
| 747 | + | return err | |
| 748 | + | } | |
| 749 | + | defer func() { _ = registry.RemoveOwned(record) }() | |
| 750 | + | if _, err := fmt.Fprintf(ready, "VEV-UDP %d %s\n", addr.Port, keyText); err != nil { | |
| 739 | 751 | return err | |
| 740 | 752 | } | |
| 741 | 753 | ||
| @@ -743,7 +755,7 @@ func runUDPProxy(ctx context.Context, session string, ready io.Writer) error { | |||
| 743 | 755 | if session != "" { | |
| 744 | 756 | dgramTr = preHelloNameTransport{Transport: dg, session: session} | |
| 745 | 757 | } | |
| 746 | - | return proxyTransports(ctx, dgramTr, daemonTr, log) | |
| 758 | + | return dgram.ProxyRuntime{Client: dgramTr, Daemon: daemonTr, Log: log}.Run(ctx) | |
| 747 | 759 | } | |
| 748 | 760 | ||
| 749 | 761 | const ( | |
| @@ -849,41 +861,7 @@ func (t preHelloNameTransport) Recv() (ports.Frame, error) { | |||
| 849 | 861 | } | |
| 850 | 862 | ||
| 851 | 863 | func proxyTransports(ctx context.Context, a, b ports.Transport, log *slog.Logger) error { | |
| 852 | - | errCh := make(chan error, 2) | |
| 853 | - | copyFrames := func(dst, src ports.Transport) { | |
| 854 | - | for { | |
| 855 | - | f, err := src.Recv() | |
| 856 | - | if err != nil { | |
| 857 | - | errCh <- err | |
| 858 | - | return | |
| 859 | - | } | |
| 860 | - | if err := dst.Send(f); err != nil { | |
| 861 | - | errCh <- err | |
| 862 | - | return | |
| 863 | - | } | |
| 864 | - | } | |
| 865 | - | } | |
| 866 | - | go copyFrames(b, a) | |
| 867 | - | go copyFrames(a, b) | |
| 868 | - | ||
| 869 | - | select { | |
| 870 | - | case <-ctx.Done(): | |
| 871 | - | if log != nil { | |
| 872 | - | log.Info("stdio proxy stopped by context", "err", ctx.Err()) | |
| 873 | - | } | |
| 874 | - | return ctx.Err() | |
| 875 | - | case err := <-errCh: | |
| 876 | - | if errors.Is(err, io.EOF) { | |
| 877 | - | if log != nil { | |
| 878 | - | log.Info("stdio proxy reached clean EOF") | |
| 879 | - | } | |
| 880 | - | return nil | |
| 881 | - | } | |
| 882 | - | if log != nil { | |
| 883 | - | log.Error("stdio proxy copy failed", "err", err) | |
| 884 | - | } | |
| 885 | - | return err | |
| 886 | - | } | |
| 864 | + | return dgram.ProxyRuntime{Client: a, Daemon: b, Log: log}.Run(ctx) | |
| 887 | 865 | } | |
| 888 | 866 | ||
| 889 | 867 | // runList prints the daemon's session listing. With no daemon running, it | |
| @@ -634,6 +634,11 @@ var ( | |||||
| 634 | // runUDPBootstrap starts a detached _udp-proxy, forwards its single readiness | 634 | // runUDPBootstrap starts a detached _udp-proxy, forwards its single readiness | ||
| 635 | // line, and exits so SSH stdio can close without owning the UDP proxy lifetime. | 635 | // line, and exits so SSH stdio can close without owning the UDP proxy lifetime. | ||
| 636 | func runUDPBootstrap(ctx context.Context, session string) error { | 636 | func runUDPBootstrap(ctx context.Context, session string) error { | ||
| 637 | + | // Always start a fresh proxy for a bootstrap request. A client attach begins | |||
| 638 | + | // with MsgHello, which must reach the daemon handshake path; reusing an | |||
| 639 | + | // already-running proxy would forward that Hello into the daemon connection's | |||
| 640 | + | // post-handshake runConnLoop, where it is intentionally ignored. The new | |||
| 641 | + | // proxy publishes itself and retires any older registry entry once ready. | |||
| 637 | r, w, err := os.Pipe() | 642 | r, w, err := os.Pipe() | ||
| 638 | if err != nil { | 643 | if err != nil { | ||
| 639 | return err | 644 | return err | ||
| @@ -735,7 +740,14 @@ func runUDPProxy(ctx context.Context, session string, ready io.Writer) error { | |||||
| 735 | if !ok { | 740 | if !ok { | ||
| 736 | return errors.New("vev: udp proxy did not get a UDP address") | 741 | return errors.New("vev: udp proxy did not get a UDP address") | ||
| 737 | } | 742 | } | ||
| 738 | - | if _, err := fmt.Fprintf(ready, "VEV-UDP %d %s\n", addr.Port, base64.StdEncoding.EncodeToString(key)); err != nil { | 743 | + | keyText := base64.StdEncoding.EncodeToString(key) |
| 744 | + | registry := dgram.NewProxyRegistry(filepath.Join(ipc.SocketDir(), "udp-proxies")) | |||
| 745 | + | record := dgram.ProxyRecord{Session: session, PID: os.Getpid(), Port: addr.Port, Key: keyText} | |||
| 746 | + | if err := registry.Publish(record); err != nil { | |||
| 747 | + | return err | |||
| 748 | + | } | |||
| 749 | + | defer func() { _ = registry.RemoveOwned(record) }() | |||
| 750 | + | if _, err := fmt.Fprintf(ready, "VEV-UDP %d %s\n", addr.Port, keyText); err != nil { | |||
| 739 | return err | 751 | return err | ||
| 740 | } | 752 | } | ||
| 741 | 753 | ||||
| @@ -743,7 +755,7 @@ func runUDPProxy(ctx context.Context, session string, ready io.Writer) error { | |||||
| 743 | if session != "" { | 755 | if session != "" { | ||
| 744 | dgramTr = preHelloNameTransport{Transport: dg, session: session} | 756 | dgramTr = preHelloNameTransport{Transport: dg, session: session} | ||
| 745 | } | 757 | } | ||
| 746 | - | return proxyTransports(ctx, dgramTr, daemonTr, log) | 758 | + | return dgram.ProxyRuntime{Client: dgramTr, Daemon: daemonTr, Log: log}.Run(ctx) |
| 747 | } | 759 | } | ||
| 748 | 760 | ||||
| 749 | const ( | 761 | const ( | ||
| @@ -849,41 +861,7 @@ func (t preHelloNameTransport) Recv() (ports.Frame, error) { | |||||
| 849 | } | 861 | } | ||
| 850 | 862 | ||||
| 851 | func proxyTransports(ctx context.Context, a, b ports.Transport, log *slog.Logger) error { | 863 | func proxyTransports(ctx context.Context, a, b ports.Transport, log *slog.Logger) error { | ||
| 852 | - | errCh := make(chan error, 2) | 864 | + | return dgram.ProxyRuntime{Client: a, Daemon: b, Log: log}.Run(ctx) |
| 853 | - | copyFrames := func(dst, src ports.Transport) { | |||
| 854 | - | for { | |||
| 855 | - | f, err := src.Recv() | |||
| 856 | - | if err != nil { | |||
| 857 | - | errCh <- err | |||
| 858 | - | return | |||
| 859 | - | } | |||
| 860 | - | if err := dst.Send(f); err != nil { | |||
| 861 | - | errCh <- err | |||
| 862 | - | return | |||
| 863 | - | } | |||
| 864 | - | } | |||
| 865 | - | } | |||
| 866 | - | go copyFrames(b, a) | |||
| 867 | - | go copyFrames(a, b) | |||
| 868 | - | ||||
| 869 | - | select { | |||
| 870 | - | case <-ctx.Done(): | |||
| 871 | - | if log != nil { | |||
| 872 | - | log.Info("stdio proxy stopped by context", "err", ctx.Err()) | |||
| 873 | - | } | |||
| 874 | - | return ctx.Err() | |||
| 875 | - | case err := <-errCh: | |||
| 876 | - | if errors.Is(err, io.EOF) { | |||
| 877 | - | if log != nil { | |||
| 878 | - | log.Info("stdio proxy reached clean EOF") | |||
| 879 | - | } | |||
| 880 | - | return nil | |||
| 881 | - | } | |||
| 882 | - | if log != nil { | |||
| 883 | - | log.Error("stdio proxy copy failed", "err", err) | |||
| 884 | - | } | |||
| 885 | - | return err | |||
| 886 | - | } | |||
| 887 | } | 865 | } | ||
| 888 | 866 | ||||
| 889 | // runList prints the daemon's session listing. With no daemon running, it | 867 | // runList prints the daemon's session listing. With no daemon running, it | ||
M internal/ports/frame.go +1 -1
| @@ -1,7 +1,7 @@ | |||
| 1 | 1 | package ports | |
| 2 | 2 | ||
| 3 | 3 | // ProtocolVersion is the current vev IPC wire protocol version. | |
| 4 | - | const ProtocolVersion uint16 = 9 | |
| 4 | + | const ProtocolVersion uint16 = 10 | |
| 5 | 5 | ||
| 6 | 6 | // MsgType identifies the kind of payload carried by a Frame. | |
| 7 | 7 | type MsgType uint8 | |
| @@ -1,7 +1,7 @@ | |||||
| 1 | package ports | 1 | package ports | ||
| 2 | 2 | ||||
| 3 | // ProtocolVersion is the current vev IPC wire protocol version. | 3 | // ProtocolVersion is the current vev IPC wire protocol version. | ||
| 4 | - | const ProtocolVersion uint16 = 9 | 4 | + | const ProtocolVersion uint16 = 10 |
| 5 | 5 | ||||
| 6 | // MsgType identifies the kind of payload carried by a Frame. | 6 | // MsgType identifies the kind of payload carried by a Frame. | ||
| 7 | type MsgType uint8 | 7 | type MsgType uint8 | ||
M internal/ports/frame_test.go +2 -2
| @@ -3,8 +3,8 @@ package ports | |||
| 3 | 3 | import "testing" | |
| 4 | 4 | ||
| 5 | 5 | func TestProtocolVersion(t *testing.T) { | |
| 6 | - | if ProtocolVersion != 9 { | |
| 7 | - | t.Fatalf("ProtocolVersion = %d, want 9", ProtocolVersion) | |
| 6 | + | if ProtocolVersion != 10 { | |
| 7 | + | t.Fatalf("ProtocolVersion = %d, want 10", ProtocolVersion) | |
| 8 | 8 | } | |
| 9 | 9 | } | |
| 10 | 10 | ||
| @@ -3,8 +3,8 @@ package ports | |||||
| 3 | import "testing" | 3 | import "testing" | ||
| 4 | 4 | ||||
| 5 | func TestProtocolVersion(t *testing.T) { | 5 | func TestProtocolVersion(t *testing.T) { | ||
| 6 | - | if ProtocolVersion != 9 { | 6 | + | if ProtocolVersion != 10 { |
| 7 | - | t.Fatalf("ProtocolVersion = %d, want 9", ProtocolVersion) | 7 | + | t.Fatalf("ProtocolVersion = %d, want 10", ProtocolVersion) |
| 8 | } | 8 | } | ||
| 9 | } | 9 | } | ||
| 10 | 10 | ||||
A internal/ports/link.go +51 -0
| @@ -0,0 +1,51 @@ | |||
| 1 | + | package ports | |
| 2 | + | ||
| 3 | + | import "time" | |
| 4 | + | ||
| 5 | + | // LinkState reports best-effort transport health without changing Transport. | |
| 6 | + | type LinkState int | |
| 7 | + | ||
| 8 | + | const ( | |
| 9 | + | LinkStateConnected LinkState = iota | |
| 10 | + | LinkStateDegraded | |
| 11 | + | LinkStateProbing | |
| 12 | + | LinkStateOffline | |
| 13 | + | LinkStateDead | |
| 14 | + | ) | |
| 15 | + | ||
| 16 | + | func (s LinkState) String() string { | |
| 17 | + | switch s { | |
| 18 | + | case LinkStateConnected: | |
| 19 | + | return "connected" | |
| 20 | + | case LinkStateDegraded: | |
| 21 | + | return "degraded" | |
| 22 | + | case LinkStateProbing: | |
| 23 | + | return "probing" | |
| 24 | + | case LinkStateOffline: | |
| 25 | + | return "offline" | |
| 26 | + | case LinkStateDead: | |
| 27 | + | return "dead" | |
| 28 | + | default: | |
| 29 | + | return "unknown" | |
| 30 | + | } | |
| 31 | + | } | |
| 32 | + | ||
| 33 | + | // LinkEvent is emitted when a transport's observed link state changes. | |
| 34 | + | type LinkEvent struct { | |
| 35 | + | State LinkState | |
| 36 | + | At time.Time | |
| 37 | + | Err error | |
| 38 | + | } | |
| 39 | + | ||
| 40 | + | // LinkStateReporter is an optional boundary-safe port implemented by transports | |
| 41 | + | // that can report link degradation before Transport fails terminally. | |
| 42 | + | // | |
| 43 | + | // LinkEvents returns a best-effort event stream. Implementations must not block | |
| 44 | + | // transport progress while publishing events; use a buffered channel and drop | |
| 45 | + | // events when the consumer is slow. The channel is not required to close on | |
| 46 | + | // transport shutdown, so consumers should also watch their own context or the | |
| 47 | + | // underlying Transport result. | |
| 48 | + | type LinkStateReporter interface { | |
| 49 | + | LinkState() LinkState | |
| 50 | + | LinkEvents() <-chan LinkEvent | |
| 51 | + | } | |
| @@ -0,0 +1,51 @@ | |||||
| 1 | + | package ports | |||
| 2 | + | ||||
| 3 | + | import "time" | |||
| 4 | + | ||||
| 5 | + | // LinkState reports best-effort transport health without changing Transport. | |||
| 6 | + | type LinkState int | |||
| 7 | + | ||||
| 8 | + | const ( | |||
| 9 | + | LinkStateConnected LinkState = iota | |||
| 10 | + | LinkStateDegraded | |||
| 11 | + | LinkStateProbing | |||
| 12 | + | LinkStateOffline | |||
| 13 | + | LinkStateDead | |||
| 14 | + | ) | |||
| 15 | + | ||||
| 16 | + | func (s LinkState) String() string { | |||
| 17 | + | switch s { | |||
| 18 | + | case LinkStateConnected: | |||
| 19 | + | return "connected" | |||
| 20 | + | case LinkStateDegraded: | |||
| 21 | + | return "degraded" | |||
| 22 | + | case LinkStateProbing: | |||
| 23 | + | return "probing" | |||
| 24 | + | case LinkStateOffline: | |||
| 25 | + | return "offline" | |||
| 26 | + | case LinkStateDead: | |||
| 27 | + | return "dead" | |||
| 28 | + | default: | |||
| 29 | + | return "unknown" | |||
| 30 | + | } | |||
| 31 | + | } | |||
| 32 | + | ||||
| 33 | + | // LinkEvent is emitted when a transport's observed link state changes. | |||
| 34 | + | type LinkEvent struct { | |||
| 35 | + | State LinkState | |||
| 36 | + | At time.Time | |||
| 37 | + | Err error | |||
| 38 | + | } | |||
| 39 | + | ||||
| 40 | + | // LinkStateReporter is an optional boundary-safe port implemented by transports | |||
| 41 | + | // that can report link degradation before Transport fails terminally. | |||
| 42 | + | // | |||
| 43 | + | // LinkEvents returns a best-effort event stream. Implementations must not block | |||
| 44 | + | // transport progress while publishing events; use a buffered channel and drop | |||
| 45 | + | // events when the consumer is slow. The channel is not required to close on | |||
| 46 | + | // transport shutdown, so consumers should also watch their own context or the | |||
| 47 | + | // underlying Transport result. | |||
| 48 | + | type LinkStateReporter interface { | |||
| 49 | + | LinkState() LinkState | |||
| 50 | + | LinkEvents() <-chan LinkEvent | |||
| 51 | + | } | |||
M internal/ports/mocks/mocks.go +117 -0
| @@ -15,6 +15,123 @@ import ( | |||
| 15 | 15 | mock "github.com/stretchr/testify/mock" | |
| 16 | 16 | ) | |
| 17 | 17 | ||
| 18 | + | // NewMockLinkStateReporter creates a new instance of MockLinkStateReporter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | |
| 19 | + | // The first argument is typically a *testing.T value. | |
| 20 | + | func NewMockLinkStateReporter(t interface { | |
| 21 | + | mock.TestingT | |
| 22 | + | Cleanup(func()) | |
| 23 | + | }) *MockLinkStateReporter { | |
| 24 | + | mock := &MockLinkStateReporter{} | |
| 25 | + | mock.Mock.Test(t) | |
| 26 | + | ||
| 27 | + | t.Cleanup(func() { mock.AssertExpectations(t) }) | |
| 28 | + | ||
| 29 | + | return mock | |
| 30 | + | } | |
| 31 | + | ||
| 32 | + | // MockLinkStateReporter is an autogenerated mock type for the LinkStateReporter type | |
| 33 | + | type MockLinkStateReporter struct { | |
| 34 | + | mock.Mock | |
| 35 | + | } | |
| 36 | + | ||
| 37 | + | type MockLinkStateReporter_Expecter struct { | |
| 38 | + | mock *mock.Mock | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | func (_m *MockLinkStateReporter) EXPECT() *MockLinkStateReporter_Expecter { | |
| 42 | + | return &MockLinkStateReporter_Expecter{mock: &_m.Mock} | |
| 43 | + | } | |
| 44 | + | ||
| 45 | + | // LinkEvents provides a mock function for the type MockLinkStateReporter | |
| 46 | + | func (_mock *MockLinkStateReporter) LinkEvents() <-chan ports.LinkEvent { | |
| 47 | + | ret := _mock.Called() | |
| 48 | + | ||
| 49 | + | if len(ret) == 0 { | |
| 50 | + | panic("no return value specified for LinkEvents") | |
| 51 | + | } | |
| 52 | + | ||
| 53 | + | var r0 <-chan ports.LinkEvent | |
| 54 | + | if returnFunc, ok := ret.Get(0).(func() <-chan ports.LinkEvent); ok { | |
| 55 | + | r0 = returnFunc() | |
| 56 | + | } else { | |
| 57 | + | if ret.Get(0) != nil { | |
| 58 | + | r0 = ret.Get(0).(<-chan ports.LinkEvent) | |
| 59 | + | } | |
| 60 | + | } | |
| 61 | + | return r0 | |
| 62 | + | } | |
| 63 | + | ||
| 64 | + | // MockLinkStateReporter_LinkEvents_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LinkEvents' | |
| 65 | + | type MockLinkStateReporter_LinkEvents_Call struct { | |
| 66 | + | *mock.Call | |
| 67 | + | } | |
| 68 | + | ||
| 69 | + | // LinkEvents is a helper method to define mock.On call | |
| 70 | + | func (_e *MockLinkStateReporter_Expecter) LinkEvents() *MockLinkStateReporter_LinkEvents_Call { | |
| 71 | + | return &MockLinkStateReporter_LinkEvents_Call{Call: _e.mock.On("LinkEvents")} | |
| 72 | + | } | |
| 73 | + | ||
| 74 | + | func (_c *MockLinkStateReporter_LinkEvents_Call) Run(run func()) *MockLinkStateReporter_LinkEvents_Call { | |
| 75 | + | _c.Call.Run(func(args mock.Arguments) { | |
| 76 | + | run() | |
| 77 | + | }) | |
| 78 | + | return _c | |
| 79 | + | } | |
| 80 | + | ||
| 81 | + | func (_c *MockLinkStateReporter_LinkEvents_Call) Return(linkEventCh <-chan ports.LinkEvent) *MockLinkStateReporter_LinkEvents_Call { | |
| 82 | + | _c.Call.Return(linkEventCh) | |
| 83 | + | return _c | |
| 84 | + | } | |
| 85 | + | ||
| 86 | + | func (_c *MockLinkStateReporter_LinkEvents_Call) RunAndReturn(run func() <-chan ports.LinkEvent) *MockLinkStateReporter_LinkEvents_Call { | |
| 87 | + | _c.Call.Return(run) | |
| 88 | + | return _c | |
| 89 | + | } | |
| 90 | + | ||
| 91 | + | // LinkState provides a mock function for the type MockLinkStateReporter | |
| 92 | + | func (_mock *MockLinkStateReporter) LinkState() ports.LinkState { | |
| 93 | + | ret := _mock.Called() | |
| 94 | + | ||
| 95 | + | if len(ret) == 0 { | |
| 96 | + | panic("no return value specified for LinkState") | |
| 97 | + | } | |
| 98 | + | ||
| 99 | + | var r0 ports.LinkState | |
| 100 | + | if returnFunc, ok := ret.Get(0).(func() ports.LinkState); ok { | |
| 101 | + | r0 = returnFunc() | |
| 102 | + | } else { | |
| 103 | + | r0 = ret.Get(0).(ports.LinkState) | |
| 104 | + | } | |
| 105 | + | return r0 | |
| 106 | + | } | |
| 107 | + | ||
| 108 | + | // MockLinkStateReporter_LinkState_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LinkState' | |
| 109 | + | type MockLinkStateReporter_LinkState_Call struct { | |
| 110 | + | *mock.Call | |
| 111 | + | } | |
| 112 | + | ||
| 113 | + | // LinkState is a helper method to define mock.On call | |
| 114 | + | func (_e *MockLinkStateReporter_Expecter) LinkState() *MockLinkStateReporter_LinkState_Call { | |
| 115 | + | return &MockLinkStateReporter_LinkState_Call{Call: _e.mock.On("LinkState")} | |
| 116 | + | } | |
| 117 | + | ||
| 118 | + | func (_c *MockLinkStateReporter_LinkState_Call) Run(run func()) *MockLinkStateReporter_LinkState_Call { | |
| 119 | + | _c.Call.Run(func(args mock.Arguments) { | |
| 120 | + | run() | |
| 121 | + | }) | |
| 122 | + | return _c | |
| 123 | + | } | |
| 124 | + | ||
| 125 | + | func (_c *MockLinkStateReporter_LinkState_Call) Return(linkState ports.LinkState) *MockLinkStateReporter_LinkState_Call { | |
| 126 | + | _c.Call.Return(linkState) | |
| 127 | + | return _c | |
| 128 | + | } | |
| 129 | + | ||
| 130 | + | func (_c *MockLinkStateReporter_LinkState_Call) RunAndReturn(run func() ports.LinkState) *MockLinkStateReporter_LinkState_Call { | |
| 131 | + | _c.Call.Return(run) | |
| 132 | + | return _c | |
| 133 | + | } | |
| 134 | + | ||
| 18 | 135 | // NewMockPTY creates a new instance of MockPTY. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | |
| 19 | 136 | // The first argument is typically a *testing.T value. | |
| 20 | 137 | func NewMockPTY(t interface { | |
| @@ -15,6 +15,123 @@ import ( | |||||
| 15 | mock "github.com/stretchr/testify/mock" | 15 | mock "github.com/stretchr/testify/mock" | ||
| 16 | ) | 16 | ) | ||
| 17 | 17 | ||||
| 18 | + | // NewMockLinkStateReporter creates a new instance of MockLinkStateReporter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | |||
| 19 | + | // The first argument is typically a *testing.T value. | |||
| 20 | + | func NewMockLinkStateReporter(t interface { | |||
| 21 | + | mock.TestingT | |||
| 22 | + | Cleanup(func()) | |||
| 23 | + | }) *MockLinkStateReporter { | |||
| 24 | + | mock := &MockLinkStateReporter{} | |||
| 25 | + | mock.Mock.Test(t) | |||
| 26 | + | ||||
| 27 | + | t.Cleanup(func() { mock.AssertExpectations(t) }) | |||
| 28 | + | ||||
| 29 | + | return mock | |||
| 30 | + | } | |||
| 31 | + | ||||
| 32 | + | // MockLinkStateReporter is an autogenerated mock type for the LinkStateReporter type | |||
| 33 | + | type MockLinkStateReporter struct { | |||
| 34 | + | mock.Mock | |||
| 35 | + | } | |||
| 36 | + | ||||
| 37 | + | type MockLinkStateReporter_Expecter struct { | |||
| 38 | + | mock *mock.Mock | |||
| 39 | + | } | |||
| 40 | + | ||||
| 41 | + | func (_m *MockLinkStateReporter) EXPECT() *MockLinkStateReporter_Expecter { | |||
| 42 | + | return &MockLinkStateReporter_Expecter{mock: &_m.Mock} | |||
| 43 | + | } | |||
| 44 | + | ||||
| 45 | + | // LinkEvents provides a mock function for the type MockLinkStateReporter | |||
| 46 | + | func (_mock *MockLinkStateReporter) LinkEvents() <-chan ports.LinkEvent { | |||
| 47 | + | ret := _mock.Called() | |||
| 48 | + | ||||
| 49 | + | if len(ret) == 0 { | |||
| 50 | + | panic("no return value specified for LinkEvents") | |||
| 51 | + | } | |||
| 52 | + | ||||
| 53 | + | var r0 <-chan ports.LinkEvent | |||
| 54 | + | if returnFunc, ok := ret.Get(0).(func() <-chan ports.LinkEvent); ok { | |||
| 55 | + | r0 = returnFunc() | |||
| 56 | + | } else { | |||
| 57 | + | if ret.Get(0) != nil { | |||
| 58 | + | r0 = ret.Get(0).(<-chan ports.LinkEvent) | |||
| 59 | + | } | |||
| 60 | + | } | |||
| 61 | + | return r0 | |||
| 62 | + | } | |||
| 63 | + | ||||
| 64 | + | // MockLinkStateReporter_LinkEvents_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LinkEvents' | |||
| 65 | + | type MockLinkStateReporter_LinkEvents_Call struct { | |||
| 66 | + | *mock.Call | |||
| 67 | + | } | |||
| 68 | + | ||||
| 69 | + | // LinkEvents is a helper method to define mock.On call | |||
| 70 | + | func (_e *MockLinkStateReporter_Expecter) LinkEvents() *MockLinkStateReporter_LinkEvents_Call { | |||
| 71 | + | return &MockLinkStateReporter_LinkEvents_Call{Call: _e.mock.On("LinkEvents")} | |||
| 72 | + | } | |||
| 73 | + | ||||
| 74 | + | func (_c *MockLinkStateReporter_LinkEvents_Call) Run(run func()) *MockLinkStateReporter_LinkEvents_Call { | |||
| 75 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 76 | + | run() | |||
| 77 | + | }) | |||
| 78 | + | return _c | |||
| 79 | + | } | |||
| 80 | + | ||||
| 81 | + | func (_c *MockLinkStateReporter_LinkEvents_Call) Return(linkEventCh <-chan ports.LinkEvent) *MockLinkStateReporter_LinkEvents_Call { | |||
| 82 | + | _c.Call.Return(linkEventCh) | |||
| 83 | + | return _c | |||
| 84 | + | } | |||
| 85 | + | ||||
| 86 | + | func (_c *MockLinkStateReporter_LinkEvents_Call) RunAndReturn(run func() <-chan ports.LinkEvent) *MockLinkStateReporter_LinkEvents_Call { | |||
| 87 | + | _c.Call.Return(run) | |||
| 88 | + | return _c | |||
| 89 | + | } | |||
| 90 | + | ||||
| 91 | + | // LinkState provides a mock function for the type MockLinkStateReporter | |||
| 92 | + | func (_mock *MockLinkStateReporter) LinkState() ports.LinkState { | |||
| 93 | + | ret := _mock.Called() | |||
| 94 | + | ||||
| 95 | + | if len(ret) == 0 { | |||
| 96 | + | panic("no return value specified for LinkState") | |||
| 97 | + | } | |||
| 98 | + | ||||
| 99 | + | var r0 ports.LinkState | |||
| 100 | + | if returnFunc, ok := ret.Get(0).(func() ports.LinkState); ok { | |||
| 101 | + | r0 = returnFunc() | |||
| 102 | + | } else { | |||
| 103 | + | r0 = ret.Get(0).(ports.LinkState) | |||
| 104 | + | } | |||
| 105 | + | return r0 | |||
| 106 | + | } | |||
| 107 | + | ||||
| 108 | + | // MockLinkStateReporter_LinkState_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LinkState' | |||
| 109 | + | type MockLinkStateReporter_LinkState_Call struct { | |||
| 110 | + | *mock.Call | |||
| 111 | + | } | |||
| 112 | + | ||||
| 113 | + | // LinkState is a helper method to define mock.On call | |||
| 114 | + | func (_e *MockLinkStateReporter_Expecter) LinkState() *MockLinkStateReporter_LinkState_Call { | |||
| 115 | + | return &MockLinkStateReporter_LinkState_Call{Call: _e.mock.On("LinkState")} | |||
| 116 | + | } | |||
| 117 | + | ||||
| 118 | + | func (_c *MockLinkStateReporter_LinkState_Call) Run(run func()) *MockLinkStateReporter_LinkState_Call { | |||
| 119 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 120 | + | run() | |||
| 121 | + | }) | |||
| 122 | + | return _c | |||
| 123 | + | } | |||
| 124 | + | ||||
| 125 | + | func (_c *MockLinkStateReporter_LinkState_Call) Return(linkState ports.LinkState) *MockLinkStateReporter_LinkState_Call { | |||
| 126 | + | _c.Call.Return(linkState) | |||
| 127 | + | return _c | |||
| 128 | + | } | |||
| 129 | + | ||||
| 130 | + | func (_c *MockLinkStateReporter_LinkState_Call) RunAndReturn(run func() ports.LinkState) *MockLinkStateReporter_LinkState_Call { | |||
| 131 | + | _c.Call.Return(run) | |||
| 132 | + | return _c | |||
| 133 | + | } | |||
| 134 | + | ||||
| 18 | // NewMockPTY creates a new instance of MockPTY. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | 135 | // NewMockPTY creates a new instance of MockPTY. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | ||
| 19 | // The first argument is typically a *testing.T value. | 136 | // The first argument is typically a *testing.T value. | ||
| 20 | func NewMockPTY(t interface { | 137 | func NewMockPTY(t interface { | ||
M internal/usecase/client/client.go +42 -31
| @@ -102,13 +102,6 @@ const stdinBufSize = 4096 | |||
| 102 | 102 | // goroutine, decoupling the input/resize pumps from transport back-pressure. | |
| 103 | 103 | const sendQueueDepth = 64 | |
| 104 | 104 | ||
| 105 | - | type reconnectBackoff struct { | |
| 106 | - | initial time.Duration | |
| 107 | - | max time.Duration | |
| 108 | - | } | |
| 109 | - | ||
| 110 | - | var defaultReconnectBackoff = reconnectBackoff{initial: 100 * time.Millisecond, max: 2 * time.Second} | |
| 111 | - | ||
| 112 | 105 | var reconnectSleep = sleepReconnect | |
| 113 | 106 | var reconnectSleepWithResize = sleepReconnectWithResizeEvents | |
| 114 | 107 | ||
| @@ -168,6 +161,7 @@ func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk port | |||
| 168 | 161 | backoff := defaultReconnectBackoff.initial | |
| 169 | 162 | showingStatus := false | |
| 170 | 163 | var reconnectToastSize domain.Size | |
| 164 | + | statusStage := reconnectStageOfflineRetrying | |
| 171 | 165 | redrawRemoteStatus := func(size domain.Size) { | |
| 172 | 166 | if !rawEntered || !remote { | |
| 173 | 167 | return | |
| @@ -175,12 +169,24 @@ func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk port | |||
| 175 | 169 | if showingStatus { | |
| 176 | 170 | _ = clearReconnectToast(term.Out(), reconnectToastSize) | |
| 177 | 171 | } | |
| 178 | - | _ = drawReconnectToast(term.Out(), size) | |
| 172 | + | _ = drawReconnectToastStage(term.Out(), size, statusStage) | |
| 179 | 173 | reconnectToastSize = size | |
| 180 | 174 | _ = term.Flush() | |
| 181 | 175 | showingStatus = true | |
| 182 | 176 | } | |
| 183 | - | drawStatus := func() { | |
| 177 | + | var drawStatus func() | |
| 178 | + | drawStatusStage := func(stage reconnectStage) { | |
| 179 | + | statusStage = stage | |
| 180 | + | if showingStatus && remote { | |
| 181 | + | size, err := term.Size() | |
| 182 | + | if err == nil { | |
| 183 | + | redrawRemoteStatus(size) | |
| 184 | + | } | |
| 185 | + | return | |
| 186 | + | } | |
| 187 | + | drawStatus() | |
| 188 | + | } | |
| 189 | + | drawStatus = func() { | |
| 184 | 190 | if !rawEntered || showingStatus { | |
| 185 | 191 | return | |
| 186 | 192 | } | |
| @@ -234,7 +240,11 @@ func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk port | |||
| 234 | 240 | } | |
| 235 | 241 | ms.dialed = true | |
| 236 | 242 | ||
| 237 | - | result := attachOnce(ctx, transport, term, clk, attemptIntent, attemptName, resumeToken, processClientID, &ms, enterRaw, clearStatus, remote, clipboard, log) | |
| 243 | + | var linkEvents <-chan ports.LinkEvent | |
| 244 | + | if reporter, ok := transport.(ports.LinkStateReporter); ok { | |
| 245 | + | linkEvents = reporter.LinkEvents() | |
| 246 | + | } | |
| 247 | + | result := attachOnce(ctx, transport, term, clk, attemptIntent, attemptName, resumeToken, processClientID, &ms, enterRaw, clearStatus, drawStatusStage, linkEvents, remote, clipboard, log) | |
| 238 | 248 | if result.welcomed { | |
| 239 | 249 | backoff = defaultReconnectBackoff.initial | |
| 240 | 250 | } | |
| @@ -254,7 +264,7 @@ func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk port | |||
| 254 | 264 | return result.err | |
| 255 | 265 | } | |
| 256 | 266 | log.Warn("reconnecting after attach error", "err", result.err, "backoff", backoff) | |
| 257 | - | drawStatus() | |
| 267 | + | drawStatusStage(reconnectStageSSH) | |
| 258 | 268 | if !sleepWhileReconnecting(backoff) { | |
| 259 | 269 | clearStatus() | |
| 260 | 270 | return ctx.Err() | |
| @@ -296,25 +306,6 @@ func sleepReconnectWithResizeEvents(ctx context.Context, clk ports.Clock, d time | |||
| 296 | 306 | } | |
| 297 | 307 | } | |
| 298 | 308 | ||
| 299 | - | func nextReconnectBackoff(cur, limit time.Duration) time.Duration { | |
| 300 | - | cur *= 2 | |
| 301 | - | if cur > limit { | |
| 302 | - | return limit | |
| 303 | - | } | |
| 304 | - | return cur | |
| 305 | - | } | |
| 306 | - | ||
| 307 | - | func shouldReconnect(err error) bool { | |
| 308 | - | if err == nil { | |
| 309 | - | return false | |
| 310 | - | } | |
| 311 | - | if _, ok := errors.AsType[*ProtocolError](err); ok { | |
| 312 | - | return false | |
| 313 | - | } | |
| 314 | - | _, ok := errors.AsType[*DetachedError](err) | |
| 315 | - | return !ok | |
| 316 | - | } | |
| 317 | - | ||
| 318 | 309 | // Attach connects an already-dialed transport to the controlling terminal | |
| 319 | 310 | // and runs the attach loop until the session detaches, the daemon | |
| 320 | 311 | // disappears, or the context is cancelled. | |
| @@ -348,7 +339,7 @@ func DetectTrueColor(termEnv, colorTerm string) bool { | |||
| 348 | 339 | return termEnv == "xterm-direct" || strings.HasSuffix(termEnv, "-direct") | |
| 349 | 340 | } | |
| 350 | 341 | ||
| 351 | - | func attachOnce(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string, resumeToken uint64, clientID [16]byte, ms *milestones, enterRaw func() error, clearStatus func(), remote bool, clipboard ports.ClipboardReader, log *slog.Logger) attachResult { | |
| 342 | + | func attachOnce(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string, resumeToken uint64, clientID [16]byte, ms *milestones, enterRaw func() error, clearStatus func(), drawStatusStage func(reconnectStage), linkEvents <-chan ports.LinkEvent, remote bool, clipboard ports.ClipboardReader, log *slog.Logger) attachResult { | |
| 352 | 343 | // 1. Handshake: send Hello with our size and TERM. | |
| 353 | 344 | size, err := term.Size() | |
| 354 | 345 | if err != nil { | |
| @@ -457,6 +448,16 @@ func attachOnce(ctx context.Context, transport ports.Transport, term ports.Termi | |||
| 457 | 448 | default: | |
| 458 | 449 | return welcomedResult(nil) | |
| 459 | 450 | } | |
| 451 | + | case ev, ok := <-linkEvents: | |
| 452 | + | if !ok { | |
| 453 | + | linkEvents = nil | |
| 454 | + | continue | |
| 455 | + | } | |
| 456 | + | if ev.State == ports.LinkStateConnected { | |
| 457 | + | clearStatus() | |
| 458 | + | continue | |
| 459 | + | } | |
| 460 | + | drawStatusStage(stageForLinkState(ev.State)) | |
| 460 | 461 | case <-colorQueryCh: | |
| 461 | 462 | if err := term.QueryColors(); err != nil { | |
| 462 | 463 | log.Warn("querying terminal colors", "err", err) | |
| @@ -480,6 +481,16 @@ func attachOnce(ctx context.Context, transport ports.Transport, term ports.Termi | |||
| 480 | 481 | if ferr := term.Flush(); ferr != nil { | |
| 481 | 482 | return welcomedResult(fmt.Errorf("vev: flushing terminal: %w", ferr)) | |
| 482 | 483 | } | |
| 484 | + | if o.NewStateNum != 0 { | |
| 485 | + | ack := ports.Frame{Type: ports.MsgAck, Payload: ports.MarshalAck(ports.Ack{AckedStateNum: o.NewStateNum})} | |
| 486 | + | select { | |
| 487 | + | case sendCh <- ack: | |
| 488 | + | case <-loopCtx.Done(): | |
| 489 | + | return welcomedResult(nil) | |
| 490 | + | default: | |
| 491 | + | log.Debug("dropping output ack because send queue is full", "state_num", o.NewStateNum) | |
| 492 | + | } | |
| 493 | + | } | |
| 483 | 494 | if !ms.firstOutput { | |
| 484 | 495 | ms.firstOutput = true | |
| 485 | 496 | log.Debug("received first output") | |
| @@ -102,13 +102,6 @@ const stdinBufSize = 4096 | |||||
| 102 | // goroutine, decoupling the input/resize pumps from transport back-pressure. | 102 | // goroutine, decoupling the input/resize pumps from transport back-pressure. | ||
| 103 | const sendQueueDepth = 64 | 103 | const sendQueueDepth = 64 | ||
| 104 | 104 | ||||
| 105 | - | type reconnectBackoff struct { | |||
| 106 | - | initial time.Duration | |||
| 107 | - | max time.Duration | |||
| 108 | - | } | |||
| 109 | - | ||||
| 110 | - | var defaultReconnectBackoff = reconnectBackoff{initial: 100 * time.Millisecond, max: 2 * time.Second} | |||
| 111 | - | ||||
| 112 | var reconnectSleep = sleepReconnect | 105 | var reconnectSleep = sleepReconnect | ||
| 113 | var reconnectSleepWithResize = sleepReconnectWithResizeEvents | 106 | var reconnectSleepWithResize = sleepReconnectWithResizeEvents | ||
| 114 | 107 | ||||
| @@ -168,6 +161,7 @@ func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk port | |||||
| 168 | backoff := defaultReconnectBackoff.initial | 161 | backoff := defaultReconnectBackoff.initial | ||
| 169 | showingStatus := false | 162 | showingStatus := false | ||
| 170 | var reconnectToastSize domain.Size | 163 | var reconnectToastSize domain.Size | ||
| 164 | + | statusStage := reconnectStageOfflineRetrying | |||
| 171 | redrawRemoteStatus := func(size domain.Size) { | 165 | redrawRemoteStatus := func(size domain.Size) { | ||
| 172 | if !rawEntered || !remote { | 166 | if !rawEntered || !remote { | ||
| 173 | return | 167 | return | ||
| @@ -175,12 +169,24 @@ func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk port | |||||
| 175 | if showingStatus { | 169 | if showingStatus { | ||
| 176 | _ = clearReconnectToast(term.Out(), reconnectToastSize) | 170 | _ = clearReconnectToast(term.Out(), reconnectToastSize) | ||
| 177 | } | 171 | } | ||
| 178 | - | _ = drawReconnectToast(term.Out(), size) | 172 | + | _ = drawReconnectToastStage(term.Out(), size, statusStage) |
| 179 | reconnectToastSize = size | 173 | reconnectToastSize = size | ||
| 180 | _ = term.Flush() | 174 | _ = term.Flush() | ||
| 181 | showingStatus = true | 175 | showingStatus = true | ||
| 182 | } | 176 | } | ||
| 183 | - | drawStatus := func() { | 177 | + | var drawStatus func() |
| 178 | + | drawStatusStage := func(stage reconnectStage) { | |||
| 179 | + | statusStage = stage | |||
| 180 | + | if showingStatus && remote { | |||
| 181 | + | size, err := term.Size() | |||
| 182 | + | if err == nil { | |||
| 183 | + | redrawRemoteStatus(size) | |||
| 184 | + | } | |||
| 185 | + | return | |||
| 186 | + | } | |||
| 187 | + | drawStatus() | |||
| 188 | + | } | |||
| 189 | + | drawStatus = func() { | |||
| 184 | if !rawEntered || showingStatus { | 190 | if !rawEntered || showingStatus { | ||
| 185 | return | 191 | return | ||
| 186 | } | 192 | } | ||
| @@ -234,7 +240,11 @@ func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk port | |||||
| 234 | } | 240 | } | ||
| 235 | ms.dialed = true | 241 | ms.dialed = true | ||
| 236 | 242 | ||||
| 237 | - | result := attachOnce(ctx, transport, term, clk, attemptIntent, attemptName, resumeToken, processClientID, &ms, enterRaw, clearStatus, remote, clipboard, log) | 243 | + | var linkEvents <-chan ports.LinkEvent |
| 244 | + | if reporter, ok := transport.(ports.LinkStateReporter); ok { | |||
| 245 | + | linkEvents = reporter.LinkEvents() | |||
| 246 | + | } | |||
| 247 | + | result := attachOnce(ctx, transport, term, clk, attemptIntent, attemptName, resumeToken, processClientID, &ms, enterRaw, clearStatus, drawStatusStage, linkEvents, remote, clipboard, log) | |||
| 238 | if result.welcomed { | 248 | if result.welcomed { | ||
| 239 | backoff = defaultReconnectBackoff.initial | 249 | backoff = defaultReconnectBackoff.initial | ||
| 240 | } | 250 | } | ||
| @@ -254,7 +264,7 @@ func Run(ctx context.Context, dialer ports.Dialer, term ports.Terminal, clk port | |||||
| 254 | return result.err | 264 | return result.err | ||
| 255 | } | 265 | } | ||
| 256 | log.Warn("reconnecting after attach error", "err", result.err, "backoff", backoff) | 266 | log.Warn("reconnecting after attach error", "err", result.err, "backoff", backoff) | ||
| 257 | - | drawStatus() | 267 | + | drawStatusStage(reconnectStageSSH) |
| 258 | if !sleepWhileReconnecting(backoff) { | 268 | if !sleepWhileReconnecting(backoff) { | ||
| 259 | clearStatus() | 269 | clearStatus() | ||
| 260 | return ctx.Err() | 270 | return ctx.Err() | ||
| @@ -296,25 +306,6 @@ func sleepReconnectWithResizeEvents(ctx context.Context, clk ports.Clock, d time | |||||
| 296 | } | 306 | } | ||
| 297 | } | 307 | } | ||
| 298 | 308 | ||||
| 299 | - | func nextReconnectBackoff(cur, limit time.Duration) time.Duration { | |||
| 300 | - | cur *= 2 | |||
| 301 | - | if cur > limit { | |||
| 302 | - | return limit | |||
| 303 | - | } | |||
| 304 | - | return cur | |||
| 305 | - | } | |||
| 306 | - | ||||
| 307 | - | func shouldReconnect(err error) bool { | |||
| 308 | - | if err == nil { | |||
| 309 | - | return false | |||
| 310 | - | } | |||
| 311 | - | if _, ok := errors.AsType[*ProtocolError](err); ok { | |||
| 312 | - | return false | |||
| 313 | - | } | |||
| 314 | - | _, ok := errors.AsType[*DetachedError](err) | |||
| 315 | - | return !ok | |||
| 316 | - | } | |||
| 317 | - | ||||
| 318 | // Attach connects an already-dialed transport to the controlling terminal | 309 | // Attach connects an already-dialed transport to the controlling terminal | ||
| 319 | // and runs the attach loop until the session detaches, the daemon | 310 | // and runs the attach loop until the session detaches, the daemon | ||
| 320 | // disappears, or the context is cancelled. | 311 | // disappears, or the context is cancelled. | ||
| @@ -348,7 +339,7 @@ func DetectTrueColor(termEnv, colorTerm string) bool { | |||||
| 348 | return termEnv == "xterm-direct" || strings.HasSuffix(termEnv, "-direct") | 339 | return termEnv == "xterm-direct" || strings.HasSuffix(termEnv, "-direct") | ||
| 349 | } | 340 | } | ||
| 350 | 341 | ||||
| 351 | - | func attachOnce(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string, resumeToken uint64, clientID [16]byte, ms *milestones, enterRaw func() error, clearStatus func(), remote bool, clipboard ports.ClipboardReader, log *slog.Logger) attachResult { | 342 | + | func attachOnce(ctx context.Context, transport ports.Transport, term ports.Terminal, clk ports.Clock, intent uint8, name string, resumeToken uint64, clientID [16]byte, ms *milestones, enterRaw func() error, clearStatus func(), drawStatusStage func(reconnectStage), linkEvents <-chan ports.LinkEvent, remote bool, clipboard ports.ClipboardReader, log *slog.Logger) attachResult { |
| 352 | // 1. Handshake: send Hello with our size and TERM. | 343 | // 1. Handshake: send Hello with our size and TERM. | ||
| 353 | size, err := term.Size() | 344 | size, err := term.Size() | ||
| 354 | if err != nil { | 345 | if err != nil { | ||
| @@ -457,6 +448,16 @@ func attachOnce(ctx context.Context, transport ports.Transport, term ports.Termi | |||||
| 457 | default: | 448 | default: | ||
| 458 | return welcomedResult(nil) | 449 | return welcomedResult(nil) | ||
| 459 | } | 450 | } | ||
| 451 | + | case ev, ok := <-linkEvents: | |||
| 452 | + | if !ok { | |||
| 453 | + | linkEvents = nil | |||
| 454 | + | continue | |||
| 455 | + | } | |||
| 456 | + | if ev.State == ports.LinkStateConnected { | |||
| 457 | + | clearStatus() | |||
| 458 | + | continue | |||
| 459 | + | } | |||
| 460 | + | drawStatusStage(stageForLinkState(ev.State)) | |||
| 460 | case <-colorQueryCh: | 461 | case <-colorQueryCh: | ||
| 461 | if err := term.QueryColors(); err != nil { | 462 | if err := term.QueryColors(); err != nil { | ||
| 462 | log.Warn("querying terminal colors", "err", err) | 463 | log.Warn("querying terminal colors", "err", err) | ||
| @@ -480,6 +481,16 @@ func attachOnce(ctx context.Context, transport ports.Transport, term ports.Termi | |||||
| 480 | if ferr := term.Flush(); ferr != nil { | 481 | if ferr := term.Flush(); ferr != nil { | ||
| 481 | return welcomedResult(fmt.Errorf("vev: flushing terminal: %w", ferr)) | 482 | return welcomedResult(fmt.Errorf("vev: flushing terminal: %w", ferr)) | ||
| 482 | } | 483 | } | ||
| 484 | + | if o.NewStateNum != 0 { | |||
| 485 | + | ack := ports.Frame{Type: ports.MsgAck, Payload: ports.MarshalAck(ports.Ack{AckedStateNum: o.NewStateNum})} | |||
| 486 | + | select { | |||
| 487 | + | case sendCh <- ack: | |||
| 488 | + | case <-loopCtx.Done(): | |||
| 489 | + | return welcomedResult(nil) | |||
| 490 | + | default: | |||
| 491 | + | log.Debug("dropping output ack because send queue is full", "state_num", o.NewStateNum) | |||
| 492 | + | } | |||
| 493 | + | } | |||
| 483 | if !ms.firstOutput { | 494 | if !ms.firstOutput { | ||
| 484 | ms.firstOutput = true | 495 | ms.firstOutput = true | ||
| 485 | log.Debug("received first output") | 496 | log.Debug("received first output") | ||
A internal/usecase/client/reconnect.go +56 -0
| @@ -0,0 +1,56 @@ | |||
| 1 | + | package client | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "errors" | |
| 5 | + | "time" | |
| 6 | + | ||
| 7 | + | "github.com/bnema/vev/internal/ports" | |
| 8 | + | ) | |
| 9 | + | ||
| 10 | + | type reconnectBackoff struct { | |
| 11 | + | initial time.Duration | |
| 12 | + | max time.Duration | |
| 13 | + | } | |
| 14 | + | ||
| 15 | + | var defaultReconnectBackoff = reconnectBackoff{initial: 100 * time.Millisecond, max: 2 * time.Second} | |
| 16 | + | ||
| 17 | + | type reconnectStage int | |
| 18 | + | ||
| 19 | + | const ( | |
| 20 | + | reconnectStageDegraded reconnectStage = iota | |
| 21 | + | reconnectStageProbingUDP | |
| 22 | + | reconnectStageSSH | |
| 23 | + | reconnectStageOfflineRetrying | |
| 24 | + | ) | |
| 25 | + | ||
| 26 | + | func stageForLinkState(state ports.LinkState) reconnectStage { | |
| 27 | + | switch state { | |
| 28 | + | case ports.LinkStateDegraded: | |
| 29 | + | return reconnectStageDegraded | |
| 30 | + | case ports.LinkStateProbing: | |
| 31 | + | return reconnectStageProbingUDP | |
| 32 | + | case ports.LinkStateOffline, ports.LinkStateDead: | |
| 33 | + | return reconnectStageOfflineRetrying | |
| 34 | + | default: | |
| 35 | + | return reconnectStageSSH | |
| 36 | + | } | |
| 37 | + | } | |
| 38 | + | ||
| 39 | + | func nextReconnectBackoff(cur, limit time.Duration) time.Duration { | |
| 40 | + | cur *= 2 | |
| 41 | + | if cur > limit { | |
| 42 | + | return limit | |
| 43 | + | } | |
| 44 | + | return cur | |
| 45 | + | } | |
| 46 | + | ||
| 47 | + | func shouldReconnect(err error) bool { | |
| 48 | + | if err == nil { | |
| 49 | + | return false | |
| 50 | + | } | |
| 51 | + | if _, ok := errors.AsType[*ProtocolError](err); ok { | |
| 52 | + | return false | |
| 53 | + | } | |
| 54 | + | _, ok := errors.AsType[*DetachedError](err) | |
| 55 | + | return !ok | |
| 56 | + | } | |
| @@ -0,0 +1,56 @@ | |||||
| 1 | + | package client | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "errors" | |||
| 5 | + | "time" | |||
| 6 | + | ||||
| 7 | + | "github.com/bnema/vev/internal/ports" | |||
| 8 | + | ) | |||
| 9 | + | ||||
| 10 | + | type reconnectBackoff struct { | |||
| 11 | + | initial time.Duration | |||
| 12 | + | max time.Duration | |||
| 13 | + | } | |||
| 14 | + | ||||
| 15 | + | var defaultReconnectBackoff = reconnectBackoff{initial: 100 * time.Millisecond, max: 2 * time.Second} | |||
| 16 | + | ||||
| 17 | + | type reconnectStage int | |||
| 18 | + | ||||
| 19 | + | const ( | |||
| 20 | + | reconnectStageDegraded reconnectStage = iota | |||
| 21 | + | reconnectStageProbingUDP | |||
| 22 | + | reconnectStageSSH | |||
| 23 | + | reconnectStageOfflineRetrying | |||
| 24 | + | ) | |||
| 25 | + | ||||
| 26 | + | func stageForLinkState(state ports.LinkState) reconnectStage { | |||
| 27 | + | switch state { | |||
| 28 | + | case ports.LinkStateDegraded: | |||
| 29 | + | return reconnectStageDegraded | |||
| 30 | + | case ports.LinkStateProbing: | |||
| 31 | + | return reconnectStageProbingUDP | |||
| 32 | + | case ports.LinkStateOffline, ports.LinkStateDead: | |||
| 33 | + | return reconnectStageOfflineRetrying | |||
| 34 | + | default: | |||
| 35 | + | return reconnectStageSSH | |||
| 36 | + | } | |||
| 37 | + | } | |||
| 38 | + | ||||
| 39 | + | func nextReconnectBackoff(cur, limit time.Duration) time.Duration { | |||
| 40 | + | cur *= 2 | |||
| 41 | + | if cur > limit { | |||
| 42 | + | return limit | |||
| 43 | + | } | |||
| 44 | + | return cur | |||
| 45 | + | } | |||
| 46 | + | ||||
| 47 | + | func shouldReconnect(err error) bool { | |||
| 48 | + | if err == nil { | |||
| 49 | + | return false | |||
| 50 | + | } | |||
| 51 | + | if _, ok := errors.AsType[*ProtocolError](err); ok { | |||
| 52 | + | return false | |||
| 53 | + | } | |||
| 54 | + | _, ok := errors.AsType[*DetachedError](err) | |||
| 55 | + | return !ok | |||
| 56 | + | } | |||
M internal/usecase/client/reconnect_toast.go +33 -5
| @@ -10,14 +10,34 @@ import ( | |||
| 10 | 10 | "github.com/bnema/vev/pkg/renderer" | |
| 11 | 11 | ) | |
| 12 | 12 | ||
| 13 | - | const reconnectToastMessage = "Reconnection attempts…" | |
| 13 | + | const reconnectToastMessage = "offline; retrying…" | |
| 14 | + | ||
| 15 | + | func reconnectStageMessage(stage reconnectStage) string { | |
| 16 | + | switch stage { | |
| 17 | + | case reconnectStageDegraded: | |
| 18 | + | return "connection degraded" | |
| 19 | + | case reconnectStageProbingUDP: | |
| 20 | + | return "probing UDP path" | |
| 21 | + | case reconnectStageSSH: | |
| 22 | + | return "offline; retrying… reconnecting through SSH" | |
| 23 | + | case reconnectStageOfflineRetrying: | |
| 24 | + | return reconnectToastMessage | |
| 25 | + | default: | |
| 26 | + | return reconnectToastMessage | |
| 27 | + | } | |
| 28 | + | } | |
| 14 | 29 | ||
| 15 | 30 | func drawReconnectToast(out io.Writer, size domain.Size) error { | |
| 16 | - | bounds := reconnectToastBounds(size) | |
| 31 | + | return drawReconnectToastStage(out, size, reconnectStageOfflineRetrying) | |
| 32 | + | } | |
| 33 | + | ||
| 34 | + | func drawReconnectToastStage(out io.Writer, size domain.Size, stage reconnectStage) error { | |
| 35 | + | message := reconnectStageMessage(stage) | |
| 36 | + | bounds := reconnectToastBoundsFor(size, message) | |
| 17 | 37 | if bounds.Width <= 0 || bounds.Height <= 0 { | |
| 18 | 38 | return nil | |
| 19 | 39 | } | |
| 20 | - | return writeReconnectToast(out, bounds, reconnectToastLines(bounds)) | |
| 40 | + | return writeReconnectToast(out, bounds, reconnectToastLinesFor(bounds, message)) | |
| 21 | 41 | } | |
| 22 | 42 | ||
| 23 | 43 | func clearReconnectToast(out io.Writer, size domain.Size) error { | |
| @@ -34,15 +54,23 @@ func clearReconnectToast(out io.Writer, size domain.Size) error { | |||
| 34 | 54 | } | |
| 35 | 55 | ||
| 36 | 56 | func reconnectToastBounds(size domain.Size) domain.Rect { | |
| 37 | - | return ui.ToastBounds(size, ui.Toast{Message: reconnectToastMessage, Anchor: ui.ToastCenter}) | |
| 57 | + | return reconnectToastBoundsFor(size, reconnectToastMessage) | |
| 58 | + | } | |
| 59 | + | ||
| 60 | + | func reconnectToastBoundsFor(size domain.Size, message string) domain.Rect { | |
| 61 | + | return ui.ToastBounds(size, ui.Toast{Message: message, Anchor: ui.ToastCenter}) | |
| 38 | 62 | } | |
| 39 | 63 | ||
| 40 | 64 | func reconnectToastLines(bounds domain.Rect) []string { | |
| 65 | + | return reconnectToastLinesFor(bounds, reconnectToastMessage) | |
| 66 | + | } | |
| 67 | + | ||
| 68 | + | func reconnectToastLinesFor(bounds domain.Rect, message string) []string { | |
| 41 | 69 | if bounds.Width <= 0 || bounds.Height <= 0 { | |
| 42 | 70 | return nil | |
| 43 | 71 | } | |
| 44 | 72 | frame := renderer.NewFrame(bounds.Width, bounds.Height) | |
| 45 | - | ui.CompositeToasts(frame, []ui.ActiveToast{{Toast: ui.Toast{Message: reconnectToastMessage, Anchor: ui.ToastCenter}}}, ui.ToastStyles{ | |
| 73 | + | ui.CompositeToasts(frame, []ui.ActiveToast{{Toast: ui.Toast{Message: message, Anchor: ui.ToastCenter}}}, ui.ToastStyles{ | |
| 46 | 74 | Text: renderer.DefaultStyle(), | |
| 47 | 75 | Box: renderer.DefaultStyle(), | |
| 48 | 76 | }, nil) | |
| @@ -10,14 +10,34 @@ import ( | |||||
| 10 | "github.com/bnema/vev/pkg/renderer" | 10 | "github.com/bnema/vev/pkg/renderer" | ||
| 11 | ) | 11 | ) | ||
| 12 | 12 | ||||
| 13 | - | const reconnectToastMessage = "Reconnection attempts…" | 13 | + | const reconnectToastMessage = "offline; retrying…" |
| 14 | + | ||||
| 15 | + | func reconnectStageMessage(stage reconnectStage) string { | |||
| 16 | + | switch stage { | |||
| 17 | + | case reconnectStageDegraded: | |||
| 18 | + | return "connection degraded" | |||
| 19 | + | case reconnectStageProbingUDP: | |||
| 20 | + | return "probing UDP path" | |||
| 21 | + | case reconnectStageSSH: | |||
| 22 | + | return "offline; retrying… reconnecting through SSH" | |||
| 23 | + | case reconnectStageOfflineRetrying: | |||
| 24 | + | return reconnectToastMessage | |||
| 25 | + | default: | |||
| 26 | + | return reconnectToastMessage | |||
| 27 | + | } | |||
| 28 | + | } | |||
| 14 | 29 | ||||
| 15 | func drawReconnectToast(out io.Writer, size domain.Size) error { | 30 | func drawReconnectToast(out io.Writer, size domain.Size) error { | ||
| 16 | - | bounds := reconnectToastBounds(size) | 31 | + | return drawReconnectToastStage(out, size, reconnectStageOfflineRetrying) |
| 32 | + | } | |||
| 33 | + | ||||
| 34 | + | func drawReconnectToastStage(out io.Writer, size domain.Size, stage reconnectStage) error { | |||
| 35 | + | message := reconnectStageMessage(stage) | |||
| 36 | + | bounds := reconnectToastBoundsFor(size, message) | |||
| 17 | if bounds.Width <= 0 || bounds.Height <= 0 { | 37 | if bounds.Width <= 0 || bounds.Height <= 0 { | ||
| 18 | return nil | 38 | return nil | ||
| 19 | } | 39 | } | ||
| 20 | - | return writeReconnectToast(out, bounds, reconnectToastLines(bounds)) | 40 | + | return writeReconnectToast(out, bounds, reconnectToastLinesFor(bounds, message)) |
| 21 | } | 41 | } | ||
| 22 | 42 | ||||
| 23 | func clearReconnectToast(out io.Writer, size domain.Size) error { | 43 | func clearReconnectToast(out io.Writer, size domain.Size) error { | ||
| @@ -34,15 +54,23 @@ func clearReconnectToast(out io.Writer, size domain.Size) error { | |||||
| 34 | } | 54 | } | ||
| 35 | 55 | ||||
| 36 | func reconnectToastBounds(size domain.Size) domain.Rect { | 56 | func reconnectToastBounds(size domain.Size) domain.Rect { | ||
| 37 | - | return ui.ToastBounds(size, ui.Toast{Message: reconnectToastMessage, Anchor: ui.ToastCenter}) | 57 | + | return reconnectToastBoundsFor(size, reconnectToastMessage) |
| 58 | + | } | |||
| 59 | + | ||||
| 60 | + | func reconnectToastBoundsFor(size domain.Size, message string) domain.Rect { | |||
| 61 | + | return ui.ToastBounds(size, ui.Toast{Message: message, Anchor: ui.ToastCenter}) | |||
| 38 | } | 62 | } | ||
| 39 | 63 | ||||
| 40 | func reconnectToastLines(bounds domain.Rect) []string { | 64 | func reconnectToastLines(bounds domain.Rect) []string { | ||
| 65 | + | return reconnectToastLinesFor(bounds, reconnectToastMessage) | |||
| 66 | + | } | |||
| 67 | + | ||||
| 68 | + | func reconnectToastLinesFor(bounds domain.Rect, message string) []string { | |||
| 41 | if bounds.Width <= 0 || bounds.Height <= 0 { | 69 | if bounds.Width <= 0 || bounds.Height <= 0 { | ||
| 42 | return nil | 70 | return nil | ||
| 43 | } | 71 | } | ||
| 44 | frame := renderer.NewFrame(bounds.Width, bounds.Height) | 72 | frame := renderer.NewFrame(bounds.Width, bounds.Height) | ||
| 45 | - | ui.CompositeToasts(frame, []ui.ActiveToast{{Toast: ui.Toast{Message: reconnectToastMessage, Anchor: ui.ToastCenter}}}, ui.ToastStyles{ | 73 | + | ui.CompositeToasts(frame, []ui.ActiveToast{{Toast: ui.Toast{Message: message, Anchor: ui.ToastCenter}}}, ui.ToastStyles{ |
| 46 | Text: renderer.DefaultStyle(), | 74 | Text: renderer.DefaultStyle(), | ||
| 47 | Box: renderer.DefaultStyle(), | 75 | Box: renderer.DefaultStyle(), | ||
| 48 | }, nil) | 76 | }, nil) | ||
M internal/usecase/client/reconnect_toast_test.go +59 -0
| @@ -100,6 +100,37 @@ type reconnectToastRecordingTransport struct { | |||
| 100 | 100 | closed bool | |
| 101 | 101 | } | |
| 102 | 102 | ||
| 103 | + | type reconnectToastLinkTransport struct { | |
| 104 | + | recvCh chan reconnectToastRecv | |
| 105 | + | sends []ports.Frame | |
| 106 | + | events chan ports.LinkEvent | |
| 107 | + | state ports.LinkState | |
| 108 | + | } | |
| 109 | + | ||
| 110 | + | func newReconnectToastLinkTransport() *reconnectToastLinkTransport { | |
| 111 | + | return &reconnectToastLinkTransport{ | |
| 112 | + | recvCh: make(chan reconnectToastRecv, 4), | |
| 113 | + | events: make(chan ports.LinkEvent, 4), | |
| 114 | + | state: ports.LinkStateConnected, | |
| 115 | + | } | |
| 116 | + | } | |
| 117 | + | ||
| 118 | + | func (t *reconnectToastLinkTransport) Send(f ports.Frame) error { | |
| 119 | + | t.sends = append(t.sends, f) | |
| 120 | + | return nil | |
| 121 | + | } | |
| 122 | + | ||
| 123 | + | func (t *reconnectToastLinkTransport) Recv() (ports.Frame, error) { | |
| 124 | + | recv := <-t.recvCh | |
| 125 | + | return recv.frame, recv.err | |
| 126 | + | } | |
| 127 | + | ||
| 128 | + | func (t *reconnectToastLinkTransport) Close() error { return nil } | |
| 129 | + | ||
| 130 | + | func (t *reconnectToastLinkTransport) LinkEvents() <-chan ports.LinkEvent { return t.events } | |
| 131 | + | ||
| 132 | + | func (t *reconnectToastLinkTransport) LinkState() ports.LinkState { return t.state } | |
| 133 | + | ||
| 103 | 134 | func (t *reconnectToastRecordingTransport) Send(f ports.Frame) error { | |
| 104 | 135 | t.sends = append(t.sends, f) | |
| 105 | 136 | return nil | |
| @@ -128,6 +159,34 @@ func reconnectToastHelloFromSend(t *testing.T, tr *reconnectToastRecordingTransp | |||
| 128 | 159 | return hello | |
| 129 | 160 | } | |
| 130 | 161 | ||
| 162 | + | func TestAttachOnceLinkStateEventsDriveReconnectUIStage(t *testing.T) { | |
| 163 | + | term := newReconnectToastTerminalHarness(t) | |
| 164 | + | defer term.closeInput() | |
| 165 | + | tr := newReconnectToastLinkTransport() | |
| 166 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |
| 167 | + | ||
| 168 | + | stages := make(chan reconnectStage, 4) | |
| 169 | + | resultCh := make(chan attachResult, 1) | |
| 170 | + | ms := milestones{} | |
| 171 | + | go func() { | |
| 172 | + | resultCh <- attachOnce(context.Background(), tr, term.term, portsmocks.NewMockClock(t), ports.IntentAttach, "main", 0, [16]byte{1}, &ms, func() error { | |
| 173 | + | _, err := term.term.EnterRaw() | |
| 174 | + | return err | |
| 175 | + | }, func() {}, func(stage reconnectStage) { | |
| 176 | + | stages <- stage | |
| 177 | + | }, tr.LinkEvents(), true, nil, slog.New(slog.DiscardHandler)) | |
| 178 | + | }() | |
| 179 | + | ||
| 180 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateDegraded} | |
| 181 | + | require.Equal(t, reconnectStageDegraded, <-stages) | |
| 182 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |
| 183 | + | require.Equal(t, reconnectStageProbingUDP, <-stages) | |
| 184 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | |
| 185 | + | result := <-resultCh | |
| 186 | + | require.NoError(t, result.err) | |
| 187 | + | require.True(t, result.welcomed) | |
| 188 | + | } | |
| 189 | + | ||
| 131 | 190 | func TestReconnectToastDrawAndClearHelpers(t *testing.T) { | |
| 132 | 191 | var out bytes.Buffer | |
| 133 | 192 | size := domain.Size{Cols: 80, Rows: 24} | |
| @@ -100,6 +100,37 @@ type reconnectToastRecordingTransport struct { | |||||
| 100 | closed bool | 100 | closed bool | ||
| 101 | } | 101 | } | ||
| 102 | 102 | ||||
| 103 | + | type reconnectToastLinkTransport struct { | |||
| 104 | + | recvCh chan reconnectToastRecv | |||
| 105 | + | sends []ports.Frame | |||
| 106 | + | events chan ports.LinkEvent | |||
| 107 | + | state ports.LinkState | |||
| 108 | + | } | |||
| 109 | + | ||||
| 110 | + | func newReconnectToastLinkTransport() *reconnectToastLinkTransport { | |||
| 111 | + | return &reconnectToastLinkTransport{ | |||
| 112 | + | recvCh: make(chan reconnectToastRecv, 4), | |||
| 113 | + | events: make(chan ports.LinkEvent, 4), | |||
| 114 | + | state: ports.LinkStateConnected, | |||
| 115 | + | } | |||
| 116 | + | } | |||
| 117 | + | ||||
| 118 | + | func (t *reconnectToastLinkTransport) Send(f ports.Frame) error { | |||
| 119 | + | t.sends = append(t.sends, f) | |||
| 120 | + | return nil | |||
| 121 | + | } | |||
| 122 | + | ||||
| 123 | + | func (t *reconnectToastLinkTransport) Recv() (ports.Frame, error) { | |||
| 124 | + | recv := <-t.recvCh | |||
| 125 | + | return recv.frame, recv.err | |||
| 126 | + | } | |||
| 127 | + | ||||
| 128 | + | func (t *reconnectToastLinkTransport) Close() error { return nil } | |||
| 129 | + | ||||
| 130 | + | func (t *reconnectToastLinkTransport) LinkEvents() <-chan ports.LinkEvent { return t.events } | |||
| 131 | + | ||||
| 132 | + | func (t *reconnectToastLinkTransport) LinkState() ports.LinkState { return t.state } | |||
| 133 | + | ||||
| 103 | func (t *reconnectToastRecordingTransport) Send(f ports.Frame) error { | 134 | func (t *reconnectToastRecordingTransport) Send(f ports.Frame) error { | ||
| 104 | t.sends = append(t.sends, f) | 135 | t.sends = append(t.sends, f) | ||
| 105 | return nil | 136 | return nil | ||
| @@ -128,6 +159,34 @@ func reconnectToastHelloFromSend(t *testing.T, tr *reconnectToastRecordingTransp | |||||
| 128 | return hello | 159 | return hello | ||
| 129 | } | 160 | } | ||
| 130 | 161 | ||||
| 162 | + | func TestAttachOnceLinkStateEventsDriveReconnectUIStage(t *testing.T) { | |||
| 163 | + | term := newReconnectToastTerminalHarness(t) | |||
| 164 | + | defer term.closeInput() | |||
| 165 | + | tr := newReconnectToastLinkTransport() | |||
| 166 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |||
| 167 | + | ||||
| 168 | + | stages := make(chan reconnectStage, 4) | |||
| 169 | + | resultCh := make(chan attachResult, 1) | |||
| 170 | + | ms := milestones{} | |||
| 171 | + | go func() { | |||
| 172 | + | resultCh <- attachOnce(context.Background(), tr, term.term, portsmocks.NewMockClock(t), ports.IntentAttach, "main", 0, [16]byte{1}, &ms, func() error { | |||
| 173 | + | _, err := term.term.EnterRaw() | |||
| 174 | + | return err | |||
| 175 | + | }, func() {}, func(stage reconnectStage) { | |||
| 176 | + | stages <- stage | |||
| 177 | + | }, tr.LinkEvents(), true, nil, slog.New(slog.DiscardHandler)) | |||
| 178 | + | }() | |||
| 179 | + | ||||
| 180 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateDegraded} | |||
| 181 | + | require.Equal(t, reconnectStageDegraded, <-stages) | |||
| 182 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |||
| 183 | + | require.Equal(t, reconnectStageProbingUDP, <-stages) | |||
| 184 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | |||
| 185 | + | result := <-resultCh | |||
| 186 | + | require.NoError(t, result.err) | |||
| 187 | + | require.True(t, result.welcomed) | |||
| 188 | + | } | |||
| 189 | + | ||||
| 131 | func TestReconnectToastDrawAndClearHelpers(t *testing.T) { | 190 | func TestReconnectToastDrawAndClearHelpers(t *testing.T) { | ||
| 132 | var out bytes.Buffer | 191 | var out bytes.Buffer | ||
| 133 | size := domain.Size{Cols: 80, Rows: 24} | 192 | size := domain.Size{Cols: 80, Rows: 24} | ||
M internal/usecase/daemon/client.go +42 -5
| @@ -48,6 +48,7 @@ type attachedClient struct { | |||
| 48 | 48 | parked bool | |
| 49 | 49 | nextStateNum uint64 | |
| 50 | 50 | echoAck atomic.Uint64 | |
| 51 | + | outputAck atomic.Uint64 | |
| 51 | 52 | bars barCache // only touched while sendMu is held | |
| 52 | 53 | size domain.Size | |
| 53 | 54 | keys *keys.Router | |
| @@ -192,9 +193,44 @@ func (ac *attachedClient) currentTransportIs(tr ports.Transport) bool { | |||
| 192 | 193 | return tr != nil && ac.transportIs(tr) | |
| 193 | 194 | } | |
| 194 | 195 | ||
| 195 | - | func (ac *attachedClient) nextOutputFrameLocked(b []byte) ports.Frame { | |
| 196 | + | func (ac *attachedClient) nextOutputFrameLocked(b []byte, reset bool) ports.Frame { | |
| 196 | 197 | ac.nextStateNum++ | |
| 197 | - | return frameOutputState(b, ac.nextStateNum, ac.echoAck.Load()) | |
| 198 | + | baseStateNum := ac.nextStateNum - 1 | |
| 199 | + | if reset { | |
| 200 | + | baseStateNum = 0 | |
| 201 | + | } | |
| 202 | + | return frameOutputState(b, baseStateNum, ac.nextStateNum, ac.echoAck.Load()) | |
| 203 | + | } | |
| 204 | + | ||
| 205 | + | // shouldResetOutputState reads nextStateNum and must only be called while | |
| 206 | + | // sendMu is held, matching nextOutputFrameLocked. | |
| 207 | + | func (ac *attachedClient) shouldResetOutputState(reset bool) bool { | |
| 208 | + | if reset { | |
| 209 | + | return true | |
| 210 | + | } | |
| 211 | + | latest := ac.nextStateNum | |
| 212 | + | if latest == 0 { | |
| 213 | + | return false | |
| 214 | + | } | |
| 215 | + | acked := ac.outputAck.Load() | |
| 216 | + | return acked+1 < latest | |
| 217 | + | } | |
| 218 | + | ||
| 219 | + | func (ac *attachedClient) advanceOutputAck(state uint64) { | |
| 220 | + | ac.sendMu.Lock() | |
| 221 | + | defer ac.sendMu.Unlock() | |
| 222 | + | if state > ac.nextStateNum { | |
| 223 | + | return | |
| 224 | + | } | |
| 225 | + | for { | |
| 226 | + | cur := ac.outputAck.Load() | |
| 227 | + | if state <= cur { | |
| 228 | + | return | |
| 229 | + | } | |
| 230 | + | if ac.outputAck.CompareAndSwap(cur, state) { | |
| 231 | + | return | |
| 232 | + | } | |
| 233 | + | } | |
| 198 | 234 | } | |
| 199 | 235 | ||
| 200 | 236 | // send serialises a frame onto the client's transport. | |
| @@ -248,7 +284,7 @@ func (d *Daemon) boundedSendOutputErrTransport(ac *attachedClient, b []byte) (po | |||
| 248 | 284 | if tr == nil { | |
| 249 | 285 | return errors.New("client transport is nil") | |
| 250 | 286 | } | |
| 251 | - | return tr.Send(ac.nextOutputFrameLocked(b)) | |
| 287 | + | return tr.Send(ac.nextOutputFrameLocked(b, false)) | |
| 252 | 288 | }) | |
| 253 | 289 | } | |
| 254 | 290 | ||
| @@ -462,8 +498,9 @@ func (d *Daemon) runConnLoop(ac *attachedClient) { | |||
| 462 | 498 | d.clientGone(sess, ac, tr, true) | |
| 463 | 499 | return | |
| 464 | 500 | case ports.MsgAck: | |
| 465 | - | // Output state ACKs are reserved for the datagram renderer path; the | |
| 466 | - | // reliable transport daemon has no state reader today. | |
| 501 | + | if ack, derr := ports.UnmarshalAck(f.Payload); derr == nil { | |
| 502 | + | ac.advanceOutputAck(ack.AckedStateNum) | |
| 503 | + | } | |
| 467 | 504 | case ports.MsgPing: | |
| 468 | 505 | _ = ac.send(framePong()) | |
| 469 | 506 | default: | |
| @@ -48,6 +48,7 @@ type attachedClient struct { | |||||
| 48 | parked bool | 48 | parked bool | ||
| 49 | nextStateNum uint64 | 49 | nextStateNum uint64 | ||
| 50 | echoAck atomic.Uint64 | 50 | echoAck atomic.Uint64 | ||
| 51 | + | outputAck atomic.Uint64 | |||
| 51 | bars barCache // only touched while sendMu is held | 52 | bars barCache // only touched while sendMu is held | ||
| 52 | size domain.Size | 53 | size domain.Size | ||
| 53 | keys *keys.Router | 54 | keys *keys.Router | ||
| @@ -192,9 +193,44 @@ func (ac *attachedClient) currentTransportIs(tr ports.Transport) bool { | |||||
| 192 | return tr != nil && ac.transportIs(tr) | 193 | return tr != nil && ac.transportIs(tr) | ||
| 193 | } | 194 | } | ||
| 194 | 195 | ||||
| 195 | - | func (ac *attachedClient) nextOutputFrameLocked(b []byte) ports.Frame { | 196 | + | func (ac *attachedClient) nextOutputFrameLocked(b []byte, reset bool) ports.Frame { |
| 196 | ac.nextStateNum++ | 197 | ac.nextStateNum++ | ||
| 197 | - | return frameOutputState(b, ac.nextStateNum, ac.echoAck.Load()) | 198 | + | baseStateNum := ac.nextStateNum - 1 |
| 199 | + | if reset { | |||
| 200 | + | baseStateNum = 0 | |||
| 201 | + | } | |||
| 202 | + | return frameOutputState(b, baseStateNum, ac.nextStateNum, ac.echoAck.Load()) | |||
| 203 | + | } | |||
| 204 | + | ||||
| 205 | + | // shouldResetOutputState reads nextStateNum and must only be called while | |||
| 206 | + | // sendMu is held, matching nextOutputFrameLocked. | |||
| 207 | + | func (ac *attachedClient) shouldResetOutputState(reset bool) bool { | |||
| 208 | + | if reset { | |||
| 209 | + | return true | |||
| 210 | + | } | |||
| 211 | + | latest := ac.nextStateNum | |||
| 212 | + | if latest == 0 { | |||
| 213 | + | return false | |||
| 214 | + | } | |||
| 215 | + | acked := ac.outputAck.Load() | |||
| 216 | + | return acked+1 < latest | |||
| 217 | + | } | |||
| 218 | + | ||||
| 219 | + | func (ac *attachedClient) advanceOutputAck(state uint64) { | |||
| 220 | + | ac.sendMu.Lock() | |||
| 221 | + | defer ac.sendMu.Unlock() | |||
| 222 | + | if state > ac.nextStateNum { | |||
| 223 | + | return | |||
| 224 | + | } | |||
| 225 | + | for { | |||
| 226 | + | cur := ac.outputAck.Load() | |||
| 227 | + | if state <= cur { | |||
| 228 | + | return | |||
| 229 | + | } | |||
| 230 | + | if ac.outputAck.CompareAndSwap(cur, state) { | |||
| 231 | + | return | |||
| 232 | + | } | |||
| 233 | + | } | |||
| 198 | } | 234 | } | ||
| 199 | 235 | ||||
| 200 | // send serialises a frame onto the client's transport. | 236 | // send serialises a frame onto the client's transport. | ||
| @@ -248,7 +284,7 @@ func (d *Daemon) boundedSendOutputErrTransport(ac *attachedClient, b []byte) (po | |||||
| 248 | if tr == nil { | 284 | if tr == nil { | ||
| 249 | return errors.New("client transport is nil") | 285 | return errors.New("client transport is nil") | ||
| 250 | } | 286 | } | ||
| 251 | - | return tr.Send(ac.nextOutputFrameLocked(b)) | 287 | + | return tr.Send(ac.nextOutputFrameLocked(b, false)) |
| 252 | }) | 288 | }) | ||
| 253 | } | 289 | } | ||
| 254 | 290 | ||||
| @@ -462,8 +498,9 @@ func (d *Daemon) runConnLoop(ac *attachedClient) { | |||||
| 462 | d.clientGone(sess, ac, tr, true) | 498 | d.clientGone(sess, ac, tr, true) | ||
| 463 | return | 499 | return | ||
| 464 | case ports.MsgAck: | 500 | case ports.MsgAck: | ||
| 465 | - | // Output state ACKs are reserved for the datagram renderer path; the | 501 | + | if ack, derr := ports.UnmarshalAck(f.Payload); derr == nil { |
| 466 | - | // reliable transport daemon has no state reader today. | 502 | + | ac.advanceOutputAck(ack.AckedStateNum) |
| 503 | + | } | |||
| 467 | case ports.MsgPing: | 504 | case ports.MsgPing: | ||
| 468 | _ = ac.send(framePong()) | 505 | _ = ac.send(framePong()) | ||
| 469 | default: | 506 | default: | ||
M internal/usecase/daemon/daemon.go +26 -14
| @@ -65,7 +65,7 @@ const defaultScrollbackRows = 10_000 | |||
| 65 | 65 | // the in-flight send. Teardown is never gated on a client draining its socket. | |
| 66 | 66 | const detachNotifyTimeout = time.Second | |
| 67 | 67 | ||
| 68 | - | const resumeParkGrace = 60 * time.Second | |
| 68 | + | const defaultResumeParkGrace = 15 * time.Minute | |
| 69 | 69 | ||
| 70 | 70 | // defaultSize is used when a client's Hello carries no valid dimensions. | |
| 71 | 71 | var defaultSize = domain.Size{Cols: 80, Rows: 24} | |
| @@ -119,6 +119,7 @@ type Daemon struct { | |||
| 119 | 119 | restoreProcessAllowlist atomic.Pointer[map[string]struct{}] | |
| 120 | 120 | themeMode atomic.Uint32 | |
| 121 | 121 | barScripts *barScriptState | |
| 122 | + | resumeParkGrace time.Duration | |
| 122 | 123 | // tempDir overrides os.TempDir() for clipboard-image-transfer writes | |
| 123 | 124 | // (see clipboard.go); empty means use os.TempDir(). | |
| 124 | 125 | tempDir string | |
| @@ -236,6 +237,16 @@ func WithBarScriptCommandRunner(runner ports.ShellCommandRunner) Option { | |||
| 236 | 237 | } | |
| 237 | 238 | } | |
| 238 | 239 | ||
| 240 | + | // WithResumeParkGrace overrides how long detached resume-capable clients stay | |
| 241 | + | // parked for reconnection. Non-positive durations keep the default. | |
| 242 | + | func WithResumeParkGrace(grace time.Duration) Option { | |
| 243 | + | return func(d *Daemon) { | |
| 244 | + | if grace > 0 { | |
| 245 | + | d.resumeParkGrace = grace | |
| 246 | + | } | |
| 247 | + | } | |
| 248 | + | } | |
| 249 | + | ||
| 239 | 250 | // WithConfig applies the initial user configuration. | |
| 240 | 251 | func WithConfig(cfg domain.Config) Option { | |
| 241 | 252 | return func(d *Daemon) { | |
| @@ -254,19 +265,20 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt | |||
| 254 | 265 | shell = "/bin/sh" | |
| 255 | 266 | } | |
| 256 | 267 | d := &Daemon{ | |
| 257 | - | sessions: make(map[domain.SessionID]*session), | |
| 258 | - | stopped: make(map[string]stoppedSession), | |
| 259 | - | parked: make(map[uint64]*parkedAttachment), | |
| 260 | - | ptys: ptys, | |
| 261 | - | clock: clock, | |
| 262 | - | log: log, | |
| 263 | - | baseEnv: os.Environ(), | |
| 264 | - | shell: shell, | |
| 265 | - | persist: persist.New(nil), | |
| 266 | - | dirOrHome: dirOrHome, | |
| 267 | - | done: make(chan struct{}), | |
| 268 | - | restoreDone: make(chan struct{}), | |
| 269 | - | animWake: make(chan struct{}, 1), | |
| 268 | + | sessions: make(map[domain.SessionID]*session), | |
| 269 | + | stopped: make(map[string]stoppedSession), | |
| 270 | + | parked: make(map[uint64]*parkedAttachment), | |
| 271 | + | ptys: ptys, | |
| 272 | + | clock: clock, | |
| 273 | + | log: log, | |
| 274 | + | baseEnv: os.Environ(), | |
| 275 | + | shell: shell, | |
| 276 | + | persist: persist.New(nil), | |
| 277 | + | dirOrHome: dirOrHome, | |
| 278 | + | done: make(chan struct{}), | |
| 279 | + | restoreDone: make(chan struct{}), | |
| 280 | + | animWake: make(chan struct{}, 1), | |
| 281 | + | resumeParkGrace: defaultResumeParkGrace, | |
| 270 | 282 | barScripts: &barScriptState{ | |
| 271 | 283 | cfg: barConfigFromDomain(domain.Defaults().Bar), | |
| 272 | 284 | outputs: make(map[domain.SessionID]barScriptOutputs), | |
| @@ -65,7 +65,7 @@ const defaultScrollbackRows = 10_000 | |||||
| 65 | // the in-flight send. Teardown is never gated on a client draining its socket. | 65 | // the in-flight send. Teardown is never gated on a client draining its socket. | ||
| 66 | const detachNotifyTimeout = time.Second | 66 | const detachNotifyTimeout = time.Second | ||
| 67 | 67 | ||||
| 68 | - | const resumeParkGrace = 60 * time.Second | 68 | + | const defaultResumeParkGrace = 15 * time.Minute |
| 69 | 69 | ||||
| 70 | // defaultSize is used when a client's Hello carries no valid dimensions. | 70 | // defaultSize is used when a client's Hello carries no valid dimensions. | ||
| 71 | var defaultSize = domain.Size{Cols: 80, Rows: 24} | 71 | var defaultSize = domain.Size{Cols: 80, Rows: 24} | ||
| @@ -119,6 +119,7 @@ type Daemon struct { | |||||
| 119 | restoreProcessAllowlist atomic.Pointer[map[string]struct{}] | 119 | restoreProcessAllowlist atomic.Pointer[map[string]struct{}] | ||
| 120 | themeMode atomic.Uint32 | 120 | themeMode atomic.Uint32 | ||
| 121 | barScripts *barScriptState | 121 | barScripts *barScriptState | ||
| 122 | + | resumeParkGrace time.Duration | |||
| 122 | // tempDir overrides os.TempDir() for clipboard-image-transfer writes | 123 | // tempDir overrides os.TempDir() for clipboard-image-transfer writes | ||
| 123 | // (see clipboard.go); empty means use os.TempDir(). | 124 | // (see clipboard.go); empty means use os.TempDir(). | ||
| 124 | tempDir string | 125 | tempDir string | ||
| @@ -236,6 +237,16 @@ func WithBarScriptCommandRunner(runner ports.ShellCommandRunner) Option { | |||||
| 236 | } | 237 | } | ||
| 237 | } | 238 | } | ||
| 238 | 239 | ||||
| 240 | + | // WithResumeParkGrace overrides how long detached resume-capable clients stay | |||
| 241 | + | // parked for reconnection. Non-positive durations keep the default. | |||
| 242 | + | func WithResumeParkGrace(grace time.Duration) Option { | |||
| 243 | + | return func(d *Daemon) { | |||
| 244 | + | if grace > 0 { | |||
| 245 | + | d.resumeParkGrace = grace | |||
| 246 | + | } | |||
| 247 | + | } | |||
| 248 | + | } | |||
| 249 | + | ||||
| 239 | // WithConfig applies the initial user configuration. | 250 | // WithConfig applies the initial user configuration. | ||
| 240 | func WithConfig(cfg domain.Config) Option { | 251 | func WithConfig(cfg domain.Config) Option { | ||
| 241 | return func(d *Daemon) { | 252 | return func(d *Daemon) { | ||
| @@ -254,19 +265,20 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt | |||||
| 254 | shell = "/bin/sh" | 265 | shell = "/bin/sh" | ||
| 255 | } | 266 | } | ||
| 256 | d := &Daemon{ | 267 | d := &Daemon{ | ||
| 257 | - | sessions: make(map[domain.SessionID]*session), | 268 | + | sessions: make(map[domain.SessionID]*session), |
| 258 | - | stopped: make(map[string]stoppedSession), | 269 | + | stopped: make(map[string]stoppedSession), |
| 259 | - | parked: make(map[uint64]*parkedAttachment), | 270 | + | parked: make(map[uint64]*parkedAttachment), |
| 260 | - | ptys: ptys, | 271 | + | ptys: ptys, |
| 261 | - | clock: clock, | 272 | + | clock: clock, |
| 262 | - | log: log, | 273 | + | log: log, |
| 263 | - | baseEnv: os.Environ(), | 274 | + | baseEnv: os.Environ(), |
| 264 | - | shell: shell, | 275 | + | shell: shell, |
| 265 | - | persist: persist.New(nil), | 276 | + | persist: persist.New(nil), |
| 266 | - | dirOrHome: dirOrHome, | 277 | + | dirOrHome: dirOrHome, |
| 267 | - | done: make(chan struct{}), | 278 | + | done: make(chan struct{}), |
| 268 | - | restoreDone: make(chan struct{}), | 279 | + | restoreDone: make(chan struct{}), |
| 269 | - | animWake: make(chan struct{}, 1), | 280 | + | animWake: make(chan struct{}, 1), |
| 281 | + | resumeParkGrace: defaultResumeParkGrace, | |||
| 270 | barScripts: &barScriptState{ | 282 | barScripts: &barScriptState{ | ||
| 271 | cfg: barConfigFromDomain(domain.Defaults().Bar), | 283 | cfg: barConfigFromDomain(domain.Defaults().Bar), | ||
| 272 | outputs: make(map[domain.SessionID]barScriptOutputs), | 284 | outputs: make(map[domain.SessionID]barScriptOutputs), | ||
M internal/usecase/daemon/daemon_test.go +4 -3
| @@ -42,20 +42,21 @@ type signalClock struct { | |||
| 42 | 42 | } | |
| 43 | 43 | ||
| 44 | 44 | func (c *signalClock) Now() time.Time { return time.Time{} } | |
| 45 | - | func (c *signalClock) NewTimer(time.Duration) ports.Timer { | |
| 45 | + | func (c *signalClock) NewTimer(d time.Duration) ports.Timer { | |
| 46 | 46 | if c.called != nil { | |
| 47 | 47 | c.once.Do(func() { close(c.called) }) | |
| 48 | 48 | } | |
| 49 | 49 | if c.timers == nil { | |
| 50 | 50 | return stubTimer{} | |
| 51 | 51 | } | |
| 52 | - | t := &signalTimer{ch: make(chan time.Time, 1)} | |
| 52 | + | t := &signalTimer{ch: make(chan time.Time, 1), duration: d} | |
| 53 | 53 | c.timers <- t | |
| 54 | 54 | return t | |
| 55 | 55 | } | |
| 56 | 56 | ||
| 57 | 57 | type signalTimer struct { | |
| 58 | - | ch chan time.Time | |
| 58 | + | ch chan time.Time | |
| 59 | + | duration time.Duration | |
| 59 | 60 | } | |
| 60 | 61 | ||
| 61 | 62 | func (t *signalTimer) C() <-chan time.Time { return t.ch } | |
| @@ -42,20 +42,21 @@ type signalClock struct { | |||||
| 42 | } | 42 | } | ||
| 43 | 43 | ||||
| 44 | func (c *signalClock) Now() time.Time { return time.Time{} } | 44 | func (c *signalClock) Now() time.Time { return time.Time{} } | ||
| 45 | - | func (c *signalClock) NewTimer(time.Duration) ports.Timer { | 45 | + | func (c *signalClock) NewTimer(d time.Duration) ports.Timer { |
| 46 | if c.called != nil { | 46 | if c.called != nil { | ||
| 47 | c.once.Do(func() { close(c.called) }) | 47 | c.once.Do(func() { close(c.called) }) | ||
| 48 | } | 48 | } | ||
| 49 | if c.timers == nil { | 49 | if c.timers == nil { | ||
| 50 | return stubTimer{} | 50 | return stubTimer{} | ||
| 51 | } | 51 | } | ||
| 52 | - | t := &signalTimer{ch: make(chan time.Time, 1)} | 52 | + | t := &signalTimer{ch: make(chan time.Time, 1), duration: d} |
| 53 | c.timers <- t | 53 | c.timers <- t | ||
| 54 | return t | 54 | return t | ||
| 55 | } | 55 | } | ||
| 56 | 56 | ||||
| 57 | type signalTimer struct { | 57 | type signalTimer struct { | ||
| 58 | - | ch chan time.Time | 58 | + | ch chan time.Time |
| 59 | + | duration time.Duration | |||
| 59 | } | 60 | } | ||
| 60 | 61 | ||||
| 61 | func (t *signalTimer) C() <-chan time.Time { return t.ch } | 62 | func (t *signalTimer) C() <-chan time.Time { return t.ch } | ||
M internal/usecase/daemon/render.go +2 -1
| @@ -315,6 +315,7 @@ func (d *Daemon) paint(sess *session, ac *attachedClient, reset bool) { | |||
| 315 | 315 | ac.sendMu.Unlock() | |
| 316 | 316 | return | |
| 317 | 317 | } | |
| 318 | + | reset = ac.shouldResetOutputState(reset) | |
| 318 | 319 | if reset || overlays.copyActive || overlays.pickerActive || overlays.paletteActive || overlays.promptActive { | |
| 319 | 320 | ac.rend.Reset() | |
| 320 | 321 | ac.bars.Reset() | |
| @@ -372,7 +373,7 @@ func (d *Daemon) paint(sess *session, ac *attachedClient, reset bool) { | |||
| 372 | 373 | if sendTr == nil { | |
| 373 | 374 | serr = errors.New("client transport is nil") | |
| 374 | 375 | } else { | |
| 375 | - | serr = sendTr.Send(ac.nextOutputFrameLocked(data)) | |
| 376 | + | serr = sendTr.Send(ac.nextOutputFrameLocked(data, reset)) | |
| 376 | 377 | } | |
| 377 | 378 | } | |
| 378 | 379 | } | |
| @@ -315,6 +315,7 @@ func (d *Daemon) paint(sess *session, ac *attachedClient, reset bool) { | |||||
| 315 | ac.sendMu.Unlock() | 315 | ac.sendMu.Unlock() | ||
| 316 | return | 316 | return | ||
| 317 | } | 317 | } | ||
| 318 | + | reset = ac.shouldResetOutputState(reset) | |||
| 318 | if reset || overlays.copyActive || overlays.pickerActive || overlays.paletteActive || overlays.promptActive { | 319 | if reset || overlays.copyActive || overlays.pickerActive || overlays.paletteActive || overlays.promptActive { | ||
| 319 | ac.rend.Reset() | 320 | ac.rend.Reset() | ||
| 320 | ac.bars.Reset() | 321 | ac.bars.Reset() | ||
| @@ -372,7 +373,7 @@ func (d *Daemon) paint(sess *session, ac *attachedClient, reset bool) { | |||||
| 372 | if sendTr == nil { | 373 | if sendTr == nil { | ||
| 373 | serr = errors.New("client transport is nil") | 374 | serr = errors.New("client transport is nil") | ||
| 374 | } else { | 375 | } else { | ||
| 375 | - | serr = sendTr.Send(ac.nextOutputFrameLocked(data)) | 376 | + | serr = sendTr.Send(ac.nextOutputFrameLocked(data, reset)) |
| 376 | } | 377 | } | ||
| 377 | } | 378 | } | ||
| 378 | } | 379 | } | ||
M internal/usecase/daemon/resume.go +3 -2
| @@ -58,12 +58,13 @@ func (d *Daemon) parkAttachment(sess *session, ac *attachedClient) bool { | |||
| 58 | 58 | old.timer.Stop() | |
| 59 | 59 | old.closeDone() | |
| 60 | 60 | } | |
| 61 | - | timer := d.clock.NewTimer(resumeParkGrace) | |
| 61 | + | grace := d.resumeParkGrace | |
| 62 | + | timer := d.clock.NewTimer(grace) | |
| 62 | 63 | parked := &parkedAttachment{sess: sess, ac: ac, timer: timer, done: make(chan struct{})} | |
| 63 | 64 | ac.parked = true | |
| 64 | 65 | d.parked[token] = parked | |
| 65 | 66 | d.mu.Unlock() | |
| 66 | - | d.log.Info("client parked for resume", "session", sess.name, "grace", resumeParkGrace) | |
| 67 | + | d.log.Info("client parked for resume", "session", sess.name, "grace", grace) | |
| 67 | 68 | ||
| 68 | 69 | go func(token uint64, parked *parkedAttachment) { | |
| 69 | 70 | select { | |
| @@ -58,12 +58,13 @@ func (d *Daemon) parkAttachment(sess *session, ac *attachedClient) bool { | |||||
| 58 | old.timer.Stop() | 58 | old.timer.Stop() | ||
| 59 | old.closeDone() | 59 | old.closeDone() | ||
| 60 | } | 60 | } | ||
| 61 | - | timer := d.clock.NewTimer(resumeParkGrace) | 61 | + | grace := d.resumeParkGrace |
| 62 | + | timer := d.clock.NewTimer(grace) | |||
| 62 | parked := &parkedAttachment{sess: sess, ac: ac, timer: timer, done: make(chan struct{})} | 63 | parked := &parkedAttachment{sess: sess, ac: ac, timer: timer, done: make(chan struct{})} | ||
| 63 | ac.parked = true | 64 | ac.parked = true | ||
| 64 | d.parked[token] = parked | 65 | d.parked[token] = parked | ||
| 65 | d.mu.Unlock() | 66 | d.mu.Unlock() | ||
| 66 | - | d.log.Info("client parked for resume", "session", sess.name, "grace", resumeParkGrace) | 67 | + | d.log.Info("client parked for resume", "session", sess.name, "grace", grace) |
| 67 | 68 | ||||
| 68 | go func(token uint64, parked *parkedAttachment) { | 69 | go func(token uint64, parked *parkedAttachment) { | ||
| 69 | select { | 70 | select { | ||
M internal/usecase/daemon/resume_test.go +59 -0
| @@ -126,6 +126,37 @@ func TestResumeRebindsRotatesAndDoesNotOpenPTY(t *testing.T) { | |||
| 126 | 126 | require.Same(t, resumedAC, sess.client) | |
| 127 | 127 | } | |
| 128 | 128 | ||
| 129 | + | func TestOutputAckGapForcesFullStateRepaint(t *testing.T) { | |
| 130 | + | ac := &attachedClient{} | |
| 131 | + | ac.nextStateNum = 3 | |
| 132 | + | ac.advanceOutputAck(3) | |
| 133 | + | ac.advanceOutputAck(2) | |
| 134 | + | ac.advanceOutputAck(4) | |
| 135 | + | require.Equal(t, uint64(3), ac.outputAck.Load(), "stale or future ACKs must not move output state incorrectly") | |
| 136 | + | ac.sendMu.Lock() | |
| 137 | + | ac.nextStateNum = 5 | |
| 138 | + | reset := ac.shouldResetOutputState(false) | |
| 139 | + | require.True(t, reset, "client ack gap should force dependency-free full repaint") | |
| 140 | + | ||
| 141 | + | f := ac.nextOutputFrameLocked([]byte("full repaint"), reset) | |
| 142 | + | ac.sendMu.Unlock() | |
| 143 | + | out, err := ports.UnmarshalOutput(f.Payload) | |
| 144 | + | require.NoError(t, err) | |
| 145 | + | require.Equal(t, uint64(0), out.BaseStateNum, "full repaint must not depend on dropped prior output states") | |
| 146 | + | require.Equal(t, uint64(6), out.NewStateNum) | |
| 147 | + | ||
| 148 | + | ac.outputAck.Store(out.NewStateNum) | |
| 149 | + | ac.sendMu.Lock() | |
| 150 | + | reset = ac.shouldResetOutputState(false) | |
| 151 | + | require.False(t, reset, "acked current state can resume incremental output") | |
| 152 | + | incremental := ac.nextOutputFrameLocked([]byte("incremental"), reset) | |
| 153 | + | ac.sendMu.Unlock() | |
| 154 | + | incrementalOut, err := ports.UnmarshalOutput(incremental.Payload) | |
| 155 | + | require.NoError(t, err) | |
| 156 | + | require.Equal(t, uint64(6), incrementalOut.BaseStateNum) | |
| 157 | + | require.Equal(t, uint64(7), incrementalOut.NewStateNum) | |
| 158 | + | } | |
| 159 | + | ||
| 129 | 160 | func TestResumeClientIDMismatchDoesNotConsumeParkedToken(t *testing.T) { | |
| 130 | 161 | pty, release := newBlockingPTY(t) | |
| 131 | 162 | defer release() | |
| @@ -197,6 +228,34 @@ func TestExplicitDetachDoesNotPark(t *testing.T) { | |||
| 197 | 228 | require.Zero(t, parked) | |
| 198 | 229 | } | |
| 199 | 230 | ||
| 231 | + | func TestResumeParkUsesConfiguredGraceAndExpiresOnlyAfterGrace(t *testing.T) { | |
| 232 | + | clk := &signalClock{timers: make(chan *signalTimer, 8)} | |
| 233 | + | pty, release := newBlockingPTY(t) | |
| 234 | + | defer release() | |
| 235 | + | d := newTestDaemon(t, newFactory(t, pty), clk) | |
| 236 | + | WithResumeParkGrace(20 * time.Minute)(d) | |
| 237 | + | tr, _, _ := newConn(t, mustHello(ports.IntentAttach, "unused", domain.Size{})) | |
| 238 | + | sess, ac, err := d.route(helloResumeCapable(ports.IntentNew, "work", 0), tr) | |
| 239 | + | require.NoError(t, err) | |
| 240 | + | token := ac.resumeToken | |
| 241 | + | ||
| 242 | + | d.clientGone(sess, ac, ac.transport(), false) | |
| 243 | + | timer := <-clk.timers | |
| 244 | + | require.Equal(t, 20*time.Minute, timer.duration) | |
| 245 | + | d.mu.Lock() | |
| 246 | + | _, parkedBeforeGrace := d.parked[token] | |
| 247 | + | d.mu.Unlock() | |
| 248 | + | require.True(t, parkedBeforeGrace, "parked attachment remains before configured grace timer fires") | |
| 249 | + | ||
| 250 | + | timer.ch <- clk.Now() | |
| 251 | + | require.Eventually(t, func() bool { | |
| 252 | + | d.mu.Lock() | |
| 253 | + | defer d.mu.Unlock() | |
| 254 | + | _, ok := d.parked[token] | |
| 255 | + | return !ok | |
| 256 | + | }, 2*time.Second, 10*time.Millisecond) | |
| 257 | + | } | |
| 258 | + | ||
| 200 | 259 | func TestParkExpiryAndShutdownCleanup(t *testing.T) { | |
| 201 | 260 | clk := &signalClock{timers: make(chan *signalTimer, 8)} | |
| 202 | 261 | pty, release := newBlockingPTY(t) | |
| @@ -126,6 +126,37 @@ func TestResumeRebindsRotatesAndDoesNotOpenPTY(t *testing.T) { | |||||
| 126 | require.Same(t, resumedAC, sess.client) | 126 | require.Same(t, resumedAC, sess.client) | ||
| 127 | } | 127 | } | ||
| 128 | 128 | ||||
| 129 | + | func TestOutputAckGapForcesFullStateRepaint(t *testing.T) { | |||
| 130 | + | ac := &attachedClient{} | |||
| 131 | + | ac.nextStateNum = 3 | |||
| 132 | + | ac.advanceOutputAck(3) | |||
| 133 | + | ac.advanceOutputAck(2) | |||
| 134 | + | ac.advanceOutputAck(4) | |||
| 135 | + | require.Equal(t, uint64(3), ac.outputAck.Load(), "stale or future ACKs must not move output state incorrectly") | |||
| 136 | + | ac.sendMu.Lock() | |||
| 137 | + | ac.nextStateNum = 5 | |||
| 138 | + | reset := ac.shouldResetOutputState(false) | |||
| 139 | + | require.True(t, reset, "client ack gap should force dependency-free full repaint") | |||
| 140 | + | ||||
| 141 | + | f := ac.nextOutputFrameLocked([]byte("full repaint"), reset) | |||
| 142 | + | ac.sendMu.Unlock() | |||
| 143 | + | out, err := ports.UnmarshalOutput(f.Payload) | |||
| 144 | + | require.NoError(t, err) | |||
| 145 | + | require.Equal(t, uint64(0), out.BaseStateNum, "full repaint must not depend on dropped prior output states") | |||
| 146 | + | require.Equal(t, uint64(6), out.NewStateNum) | |||
| 147 | + | ||||
| 148 | + | ac.outputAck.Store(out.NewStateNum) | |||
| 149 | + | ac.sendMu.Lock() | |||
| 150 | + | reset = ac.shouldResetOutputState(false) | |||
| 151 | + | require.False(t, reset, "acked current state can resume incremental output") | |||
| 152 | + | incremental := ac.nextOutputFrameLocked([]byte("incremental"), reset) | |||
| 153 | + | ac.sendMu.Unlock() | |||
| 154 | + | incrementalOut, err := ports.UnmarshalOutput(incremental.Payload) | |||
| 155 | + | require.NoError(t, err) | |||
| 156 | + | require.Equal(t, uint64(6), incrementalOut.BaseStateNum) | |||
| 157 | + | require.Equal(t, uint64(7), incrementalOut.NewStateNum) | |||
| 158 | + | } | |||
| 159 | + | ||||
| 129 | func TestResumeClientIDMismatchDoesNotConsumeParkedToken(t *testing.T) { | 160 | func TestResumeClientIDMismatchDoesNotConsumeParkedToken(t *testing.T) { | ||
| 130 | pty, release := newBlockingPTY(t) | 161 | pty, release := newBlockingPTY(t) | ||
| 131 | defer release() | 162 | defer release() | ||
| @@ -197,6 +228,34 @@ func TestExplicitDetachDoesNotPark(t *testing.T) { | |||||
| 197 | require.Zero(t, parked) | 228 | require.Zero(t, parked) | ||
| 198 | } | 229 | } | ||
| 199 | 230 | ||||
| 231 | + | func TestResumeParkUsesConfiguredGraceAndExpiresOnlyAfterGrace(t *testing.T) { | |||
| 232 | + | clk := &signalClock{timers: make(chan *signalTimer, 8)} | |||
| 233 | + | pty, release := newBlockingPTY(t) | |||
| 234 | + | defer release() | |||
| 235 | + | d := newTestDaemon(t, newFactory(t, pty), clk) | |||
| 236 | + | WithResumeParkGrace(20 * time.Minute)(d) | |||
| 237 | + | tr, _, _ := newConn(t, mustHello(ports.IntentAttach, "unused", domain.Size{})) | |||
| 238 | + | sess, ac, err := d.route(helloResumeCapable(ports.IntentNew, "work", 0), tr) | |||
| 239 | + | require.NoError(t, err) | |||
| 240 | + | token := ac.resumeToken | |||
| 241 | + | ||||
| 242 | + | d.clientGone(sess, ac, ac.transport(), false) | |||
| 243 | + | timer := <-clk.timers | |||
| 244 | + | require.Equal(t, 20*time.Minute, timer.duration) | |||
| 245 | + | d.mu.Lock() | |||
| 246 | + | _, parkedBeforeGrace := d.parked[token] | |||
| 247 | + | d.mu.Unlock() | |||
| 248 | + | require.True(t, parkedBeforeGrace, "parked attachment remains before configured grace timer fires") | |||
| 249 | + | ||||
| 250 | + | timer.ch <- clk.Now() | |||
| 251 | + | require.Eventually(t, func() bool { | |||
| 252 | + | d.mu.Lock() | |||
| 253 | + | defer d.mu.Unlock() | |||
| 254 | + | _, ok := d.parked[token] | |||
| 255 | + | return !ok | |||
| 256 | + | }, 2*time.Second, 10*time.Millisecond) | |||
| 257 | + | } | |||
| 258 | + | ||||
| 200 | func TestParkExpiryAndShutdownCleanup(t *testing.T) { | 259 | func TestParkExpiryAndShutdownCleanup(t *testing.T) { | ||
| 201 | clk := &signalClock{timers: make(chan *signalTimer, 8)} | 260 | clk := &signalClock{timers: make(chan *signalTimer, 8)} | ||
| 202 | pty, release := newBlockingPTY(t) | 261 | pty, release := newBlockingPTY(t) | ||
M internal/usecase/daemon/wire.go +2 -2
| @@ -43,8 +43,8 @@ func frameError(code uint16, text string) ports.Frame { | |||
| 43 | 43 | return ports.Frame{Type: ports.MsgError, Payload: ports.MarshalErrorMsg(ports.ErrorMsg{Code: code, Text: text})} | |
| 44 | 44 | } | |
| 45 | 45 | ||
| 46 | - | func frameOutputState(b []byte, state uint64, echoAck uint64) ports.Frame { | |
| 47 | - | return ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{NewStateNum: state, EchoAck: echoAck, Data: b})} | |
| 46 | + | func frameOutputState(b []byte, baseState uint64, state uint64, echoAck uint64) ports.Frame { | |
| 47 | + | return ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{BaseStateNum: baseState, NewStateNum: state, EchoAck: echoAck, Data: b})} | |
| 48 | 48 | } | |
| 49 | 49 | ||
| 50 | 50 | func frameDetached(reason uint8) ports.Frame { | |
| @@ -43,8 +43,8 @@ func frameError(code uint16, text string) ports.Frame { | |||||
| 43 | return ports.Frame{Type: ports.MsgError, Payload: ports.MarshalErrorMsg(ports.ErrorMsg{Code: code, Text: text})} | 43 | return ports.Frame{Type: ports.MsgError, Payload: ports.MarshalErrorMsg(ports.ErrorMsg{Code: code, Text: text})} | ||
| 44 | } | 44 | } | ||
| 45 | 45 | ||||
| 46 | - | func frameOutputState(b []byte, state uint64, echoAck uint64) ports.Frame { | 46 | + | func frameOutputState(b []byte, baseState uint64, state uint64, echoAck uint64) ports.Frame { |
| 47 | - | return ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{NewStateNum: state, EchoAck: echoAck, Data: b})} | 47 | + | return ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{BaseStateNum: baseState, NewStateNum: state, EchoAck: echoAck, Data: b})} |
| 48 | } | 48 | } | ||
| 49 | 49 | ||||
| 50 | func frameDetached(reason uint8) ports.Frame { | 50 | func frameDetached(reason uint8) ports.Frame { | ||