Merge pull request #138 from bnema/fix/remote-probing-toast
fix(client): reconcile remote probing toast
3 files · +1218 -170
M internal/usecase/client/client.go +178 -54
| @@ -269,13 +269,23 @@ func (r *Runner) Run(ctx context.Context, request AttachRequest) (retErr error) | |||
| 269 | 269 | transport, err := r.dialer.Dial(ctx) | |
| 270 | 270 | if err != nil { | |
| 271 | 271 | if resumeToken == 0 || ctx.Err() != nil { | |
| 272 | - | reconnect.clear() | |
| 272 | + | if clearErr := reconnect.clear(); clearErr != nil { | |
| 273 | + | return errors.Join(err, clearErr) | |
| 274 | + | } | |
| 273 | 275 | return err | |
| 274 | 276 | } | |
| 275 | 277 | r.logger.Warn("reconnect dial failed", "err", err, "backoff", backoff) | |
| 276 | - | reconnect.draw() | |
| 277 | - | if !reconnect.sleep(ctx, r.clock, backoff) { | |
| 278 | - | reconnect.clear() | |
| 278 | + | if drawErr := reconnect.draw(); drawErr != nil { | |
| 279 | + | return errors.Join(err, drawErr) | |
| 280 | + | } | |
| 281 | + | slept, sleepErr := reconnect.sleep(ctx, r.clock, backoff) | |
| 282 | + | if sleepErr != nil { | |
| 283 | + | return errors.Join(err, sleepErr) | |
| 284 | + | } | |
| 285 | + | if !slept { | |
| 286 | + | if clearErr := reconnect.clear(); clearErr != nil { | |
| 287 | + | return errors.Join(ctx.Err(), clearErr) | |
| 288 | + | } | |
| 279 | 289 | return ctx.Err() | |
| 280 | 290 | } | |
| 281 | 291 | backoff = nextReconnectBackoff(backoff, defaultReconnectBackoff.max) | |
| @@ -316,17 +326,29 @@ func (r *Runner) Run(ctx context.Context, request AttachRequest) (retErr error) | |||
| 316 | 326 | attemptRequest.SessionName = result.sessionName | |
| 317 | 327 | } | |
| 318 | 328 | if result.err == nil { | |
| 319 | - | reconnect.clear() | |
| 329 | + | if clearErr := reconnect.clear(); clearErr != nil { | |
| 330 | + | return clearErr | |
| 331 | + | } | |
| 320 | 332 | return nil | |
| 321 | 333 | } | |
| 322 | 334 | if !shouldReconnect(result.err) || resumeToken == 0 || ctx.Err() != nil { | |
| 323 | - | reconnect.clear() | |
| 335 | + | if clearErr := reconnect.clear(); clearErr != nil { | |
| 336 | + | return errors.Join(result.err, clearErr) | |
| 337 | + | } | |
| 324 | 338 | return result.err | |
| 325 | 339 | } | |
| 326 | 340 | r.logger.Warn("reconnecting after attach error", "err", result.err, "backoff", backoff) | |
| 327 | - | reconnect.drawStage(reconnectStageSSH) | |
| 328 | - | if !reconnect.sleep(ctx, r.clock, backoff) { | |
| 329 | - | reconnect.clear() | |
| 341 | + | if drawErr := reconnect.drawStage(reconnectStageSSH); drawErr != nil { | |
| 342 | + | return errors.Join(result.err, drawErr) | |
| 343 | + | } | |
| 344 | + | slept, sleepErr := reconnect.sleep(ctx, r.clock, backoff) | |
| 345 | + | if sleepErr != nil { | |
| 346 | + | return errors.Join(result.err, sleepErr) | |
| 347 | + | } | |
| 348 | + | if !slept { | |
| 349 | + | if clearErr := reconnect.clear(); clearErr != nil { | |
| 350 | + | return errors.Join(ctx.Err(), clearErr) | |
| 351 | + | } | |
| 330 | 352 | return ctx.Err() | |
| 331 | 353 | } | |
| 332 | 354 | backoff = nextReconnectBackoff(backoff, defaultReconnectBackoff.max) | |
| @@ -342,70 +364,78 @@ type reconnectUI struct { | |||
| 342 | 364 | remote bool | |
| 343 | 365 | rawEntered *bool | |
| 344 | 366 | showing bool | |
| 345 | - | rect domain.Rect | |
| 346 | 367 | stage reconnectStage | |
| 347 | 368 | } | |
| 348 | 369 | ||
| 349 | - | func (u *reconnectUI) redraw(size domain.Size) { | |
| 370 | + | func (u *reconnectUI) redraw(size domain.Size) error { | |
| 350 | 371 | if !*u.rawEntered || !u.remote { | |
| 351 | - | return | |
| 372 | + | return nil | |
| 352 | 373 | } | |
| 353 | - | if u.showing { | |
| 354 | - | _ = clearReconnectToast(u.term.Out(), u.rect) | |
| 374 | + | _, err := drawReconnectToastStage(u.term.Out(), size, u.stage) | |
| 375 | + | if err != nil { | |
| 376 | + | return fmt.Errorf("drawing reconnect toast: %w", err) | |
| 377 | + | } | |
| 378 | + | if err := u.term.Flush(); err != nil { | |
| 379 | + | return fmt.Errorf("flushing reconnect toast: %w", err) | |
| 355 | 380 | } | |
| 356 | - | drawnRect, _ := drawReconnectToastStage(u.term.Out(), size, u.stage) | |
| 357 | - | u.rect = drawnRect | |
| 358 | - | _ = u.term.Flush() | |
| 359 | 381 | u.showing = true | |
| 382 | + | return nil | |
| 360 | 383 | } | |
| 361 | 384 | ||
| 362 | - | func (u *reconnectUI) drawStage(stage reconnectStage) { | |
| 385 | + | func (u *reconnectUI) drawStage(stage reconnectStage) error { | |
| 363 | 386 | u.stage = stage | |
| 364 | 387 | if u.showing && u.remote { | |
| 365 | 388 | size, err := u.term.Size() | |
| 366 | - | if err == nil { | |
| 367 | - | u.redraw(size) | |
| 389 | + | if err != nil { | |
| 390 | + | return fmt.Errorf("reading terminal size for reconnect toast: %w", err) | |
| 368 | 391 | } | |
| 369 | - | return | |
| 392 | + | return u.redraw(size) | |
| 370 | 393 | } | |
| 371 | - | u.draw() | |
| 394 | + | return u.draw() | |
| 372 | 395 | } | |
| 373 | 396 | ||
| 374 | - | func (u *reconnectUI) draw() { | |
| 397 | + | func (u *reconnectUI) draw() error { | |
| 375 | 398 | if !*u.rawEntered || u.showing { | |
| 376 | - | return | |
| 399 | + | return nil | |
| 377 | 400 | } | |
| 378 | 401 | if u.remote { | |
| 379 | 402 | size, err := u.term.Size() | |
| 380 | 403 | if err != nil { | |
| 381 | - | return | |
| 404 | + | return fmt.Errorf("reading terminal size for reconnect toast: %w", err) | |
| 382 | 405 | } | |
| 383 | - | u.redraw(size) | |
| 384 | - | return | |
| 406 | + | return u.redraw(size) | |
| 407 | + | } | |
| 408 | + | if _, err := u.term.Out().Write([]byte(statusReconnect)); err != nil { | |
| 409 | + | return fmt.Errorf("writing reconnect status: %w", err) | |
| 410 | + | } | |
| 411 | + | if err := u.term.Flush(); err != nil { | |
| 412 | + | return fmt.Errorf("flushing reconnect status: %w", err) | |
| 385 | 413 | } | |
| 386 | - | _, _ = u.term.Out().Write([]byte(statusReconnect)) | |
| 387 | - | _ = u.term.Flush() | |
| 388 | 414 | u.showing = true | |
| 415 | + | return nil | |
| 389 | 416 | } | |
| 390 | 417 | ||
| 391 | - | func (u *reconnectUI) clear() { | |
| 418 | + | func (u *reconnectUI) clear() error { | |
| 392 | 419 | if !u.showing { | |
| 393 | - | return | |
| 420 | + | return nil | |
| 394 | 421 | } | |
| 395 | - | if u.remote { | |
| 396 | - | _ = clearReconnectToast(u.term.Out(), u.rect) | |
| 397 | - | } else { | |
| 398 | - | _, _ = u.term.Out().Write([]byte(statusClear)) | |
| 422 | + | if !u.remote { | |
| 423 | + | if _, err := u.term.Out().Write([]byte(statusClear)); err != nil { | |
| 424 | + | return fmt.Errorf("clearing reconnect status: %w", err) | |
| 425 | + | } | |
| 426 | + | if err := u.term.Flush(); err != nil { | |
| 427 | + | return fmt.Errorf("flushing reconnect status clear: %w", err) | |
| 428 | + | } | |
| 399 | 429 | } | |
| 400 | - | _ = u.term.Flush() | |
| 401 | 430 | u.showing = false | |
| 431 | + | return nil | |
| 402 | 432 | } | |
| 403 | 433 | ||
| 404 | - | func (u *reconnectUI) sleep(ctx context.Context, clk ports.Clock, d time.Duration) bool { | |
| 434 | + | func (u *reconnectUI) sleep(ctx context.Context, clk ports.Clock, d time.Duration) (bool, error) { | |
| 405 | 435 | if u.remote && u.showing { | |
| 406 | 436 | return reconnectSleepWithResize(ctx, clk, d, u.term.ResizeEvents(), u.redraw) | |
| 407 | 437 | } | |
| 408 | - | return reconnectSleep(ctx, clk, d) | |
| 438 | + | return reconnectSleep(ctx, clk, d), nil | |
| 409 | 439 | } | |
| 410 | 440 | ||
| 411 | 441 | func sleepReconnect(ctx context.Context, clk ports.Clock, d time.Duration) bool { | |
| @@ -419,22 +449,24 @@ func sleepReconnect(ctx context.Context, clk ports.Clock, d time.Duration) bool | |||
| 419 | 449 | } | |
| 420 | 450 | } | |
| 421 | 451 | ||
| 422 | - | func sleepReconnectWithResizeEvents(ctx context.Context, clk ports.Clock, d time.Duration, resizeEvents <-chan domain.Size, onResize func(domain.Size)) bool { | |
| 452 | + | func sleepReconnectWithResizeEvents(ctx context.Context, clk ports.Clock, d time.Duration, resizeEvents <-chan domain.Size, onResize func(domain.Size) error) (bool, error) { | |
| 423 | 453 | t := clk.NewTimer(d) | |
| 424 | 454 | defer t.Stop() | |
| 425 | 455 | for { | |
| 426 | 456 | select { | |
| 427 | 457 | case <-t.C(): | |
| 428 | - | return true | |
| 458 | + | return true, nil | |
| 429 | 459 | case <-ctx.Done(): | |
| 430 | - | return false | |
| 460 | + | return false, nil | |
| 431 | 461 | case size, ok := <-resizeEvents: | |
| 432 | 462 | if !ok { | |
| 433 | 463 | resizeEvents = nil | |
| 434 | 464 | continue | |
| 435 | 465 | } | |
| 436 | 466 | if onResize != nil { | |
| 437 | - | onResize(size) | |
| 467 | + | if err := onResize(size); err != nil { | |
| 468 | + | return false, err | |
| 469 | + | } | |
| 438 | 470 | } | |
| 439 | 471 | } | |
| 440 | 472 | } | |
| @@ -619,7 +651,6 @@ func (a *attachAttempt) run(ctx context.Context) attachResult { | |||
| 619 | 651 | if err := enterRaw(); err != nil { | |
| 620 | 652 | return welcomedResult(err) | |
| 621 | 653 | } | |
| 622 | - | reconnect.clear() | |
| 623 | 654 | ||
| 624 | 655 | // 4. Derive a cancellable context so the pumps always unwind when the | |
| 625 | 656 | // loop returns. The attach loop is the sole terminal writer and palette | |
| @@ -757,6 +788,25 @@ func (a *attachAttempt) run(ctx context.Context) attachResult { | |||
| 757 | 788 | if remote { | |
| 758 | 789 | clip = clipboard | |
| 759 | 790 | } | |
| 791 | + | awaitingReconnectReset := false | |
| 792 | + | if reconnect.showing { | |
| 793 | + | if reconnect.remote { | |
| 794 | + | // Before the asynchronous sender starts, preserve transport ordering by | |
| 795 | + | // requesting the reset synchronously after the initial Theme publication. | |
| 796 | + | closed, resetErr := boundedPreWelcome(ctx, clk, transport, func() error { | |
| 797 | + | return transport.Send(ports.Frame{Type: ports.MsgResize, Payload: ports.MarshalResize(ports.Resize{Size: size})}) | |
| 798 | + | }) | |
| 799 | + | if resetErr != nil { | |
| 800 | + | resetResult := welcomedResult(fmt.Errorf("requesting reconnect reconciliation: %w", resetErr)) | |
| 801 | + | resetResult.transportClosed = closed | |
| 802 | + | return resetResult | |
| 803 | + | } | |
| 804 | + | awaitingReconnectReset = true | |
| 805 | + | } | |
| 806 | + | if err := reconnect.clear(); err != nil { | |
| 807 | + | return welcomedResult(err) | |
| 808 | + | } | |
| 809 | + | } | |
| 760 | 810 | senderStarted = true | |
| 761 | 811 | go runSender(loopCtx, cancel, transport, sendCh, ackQueue, sendErrCh, log) | |
| 762 | 812 | stdinDone := make(chan struct{}) | |
| @@ -780,24 +830,60 @@ func (a *attachAttempt) run(ctx context.Context) attachResult { | |||
| 780 | 830 | recvCh := make(chan recvResult, 1) | |
| 781 | 831 | go runRecv(loopCtx, transport, recvCh, log) | |
| 782 | 832 | ||
| 833 | + | requestReconnectReset := func() error { | |
| 834 | + | if awaitingReconnectReset { | |
| 835 | + | return nil | |
| 836 | + | } | |
| 837 | + | size, serr := term.Size() | |
| 838 | + | if serr != nil { | |
| 839 | + | return fmt.Errorf("reading terminal size for reconnect reconciliation: %w", serr) | |
| 840 | + | } | |
| 841 | + | select { | |
| 842 | + | case sendCh <- ports.Frame{Type: ports.MsgResize, Payload: ports.MarshalResize(ports.Resize{Size: size})}: | |
| 843 | + | awaitingReconnectReset = true | |
| 844 | + | return nil | |
| 845 | + | case <-loopCtx.Done(): | |
| 846 | + | return context.Canceled | |
| 847 | + | } | |
| 848 | + | } | |
| 849 | + | dismissReconnect := func() error { | |
| 850 | + | if !reconnect.showing { | |
| 851 | + | return nil | |
| 852 | + | } | |
| 853 | + | if err := reconnect.clear(); err != nil { | |
| 854 | + | return err | |
| 855 | + | } | |
| 856 | + | if !reconnect.remote { | |
| 857 | + | return nil | |
| 858 | + | } | |
| 859 | + | return requestReconnectReset() | |
| 860 | + | } | |
| 861 | + | loopCanceledResult := func() attachResult { | |
| 862 | + | // A pump initiated shutdown (stdin EOF/detach) or the parent context | |
| 863 | + | // was cancelled. A queued sender error takes priority. | |
| 864 | + | select { | |
| 865 | + | case serr := <-sendErrCh: | |
| 866 | + | return welcomedResult(fmt.Errorf("vev: sending to daemon: %w", serr)) | |
| 867 | + | default: | |
| 868 | + | return welcomedResult(nil) | |
| 869 | + | } | |
| 870 | + | } | |
| 783 | 871 | for { | |
| 784 | 872 | select { | |
| 785 | 873 | case <-loopCtx.Done(): | |
| 786 | - | // A pump initiated shutdown (stdin EOF/detach) or the parent | |
| 787 | - | // context was cancelled. A queued sender error takes priority. | |
| 788 | - | select { | |
| 789 | - | case serr := <-sendErrCh: | |
| 790 | - | return welcomedResult(fmt.Errorf("vev: sending to daemon: %w", serr)) | |
| 791 | - | default: | |
| 792 | - | return welcomedResult(nil) | |
| 793 | - | } | |
| 874 | + | return loopCanceledResult() | |
| 794 | 875 | case ev, ok := <-linkEvents: | |
| 795 | 876 | if !ok { | |
| 796 | 877 | linkEvents = nil | |
| 797 | 878 | continue | |
| 798 | 879 | } | |
| 799 | 880 | if ev.State == ports.LinkStateConnected { | |
| 800 | - | reconnect.clear() | |
| 881 | + | if err := dismissReconnect(); err != nil { | |
| 882 | + | if errors.Is(err, context.Canceled) { | |
| 883 | + | return loopCanceledResult() | |
| 884 | + | } | |
| 885 | + | return welcomedResult(fmt.Errorf("dismissing reconnect toast: %w", err)) | |
| 886 | + | } | |
| 801 | 887 | select { | |
| 802 | 888 | case sendCh <- ports.Frame{Type: ports.MsgClientNotice, Payload: ports.MarshalClientNotice(ports.ClientNotice{Action: ports.ClientNoticeLinkConnected})}: | |
| 803 | 889 | case <-loopCtx.Done(): | |
| @@ -814,7 +900,18 @@ func (a *attachAttempt) run(ctx context.Context) attachResult { | |||
| 814 | 900 | } | |
| 815 | 901 | continue | |
| 816 | 902 | } | |
| 817 | - | reconnect.drawStage(stageForLinkState(ev.State)) | |
| 903 | + | stage := stageForLinkState(ev.State) | |
| 904 | + | if reconnect.remote && reconnect.showing && reconnect.stage != stage { | |
| 905 | + | reconnect.stage = stage | |
| 906 | + | if err := requestReconnectReset(); err != nil { | |
| 907 | + | if errors.Is(err, context.Canceled) { | |
| 908 | + | return loopCanceledResult() | |
| 909 | + | } | |
| 910 | + | return welcomedResult(fmt.Errorf("requesting reconnect reconciliation: %w", err)) | |
| 911 | + | } | |
| 912 | + | } else if err := reconnect.drawStage(stage); err != nil { | |
| 913 | + | return welcomedResult(err) | |
| 914 | + | } | |
| 818 | 915 | if ev.State == ports.LinkStateOffline { | |
| 819 | 916 | // Stop waiting for the transport to reach Dead at 60s: exit with | |
| 820 | 917 | // a retryable error so Run re-dials over ssh with the resume | |
| @@ -850,10 +947,30 @@ func (a *attachAttempt) run(ctx context.Context) attachResult { | |||
| 850 | 947 | if derr != nil { | |
| 851 | 948 | return welcomedResult(fmt.Errorf("vev: decoding output: %w", derr)) | |
| 852 | 949 | } | |
| 950 | + | // Dismissing or changing a local overlay requests a same-size resize, | |
| 951 | + | // whose stateful base-zero Output is the daemon's authoritative reset. | |
| 952 | + | // ACK intervening increments so the output window keeps moving, but do | |
| 953 | + | // not apply diffs based on cells hidden by the client-local overlay. | |
| 954 | + | resetFrame := isReconnectResetFrame(o) | |
| 955 | + | if awaitingReconnectReset && !resetFrame && o.NewStateNum != 0 { | |
| 956 | + | ackQueue.offer(o.NewStateNum) | |
| 957 | + | continue | |
| 958 | + | } | |
| 853 | 959 | if _, werr := term.Out().Write(o.Data); werr != nil { | |
| 854 | 960 | return welcomedResult(fmt.Errorf("vev: writing terminal output: %w", werr)) | |
| 855 | 961 | } | |
| 856 | - | if ferr := term.Flush(); ferr != nil { | |
| 962 | + | if awaitingReconnectReset && resetFrame { | |
| 963 | + | awaitingReconnectReset = false | |
| 964 | + | } | |
| 965 | + | if reconnect.showing && reconnect.remote { | |
| 966 | + | size, serr := term.Size() | |
| 967 | + | if serr != nil { | |
| 968 | + | return welcomedResult(fmt.Errorf("vev: reading terminal size for reconnect redraw: %w", serr)) | |
| 969 | + | } | |
| 970 | + | if rerr := reconnect.redraw(size); rerr != nil { | |
| 971 | + | return welcomedResult(fmt.Errorf("vev: redrawing reconnect toast: %w", rerr)) | |
| 972 | + | } | |
| 973 | + | } else if ferr := term.Flush(); ferr != nil { | |
| 857 | 974 | return welcomedResult(fmt.Errorf("vev: flushing terminal: %w", ferr)) | |
| 858 | 975 | } | |
| 859 | 976 | // The terminal boundary is after a successful flush and before ACK. | |
| @@ -889,6 +1006,13 @@ func (a *attachAttempt) run(ctx context.Context) attachResult { | |||
| 889 | 1006 | } | |
| 890 | 1007 | } | |
| 891 | 1008 | ||
| 1009 | + | // isReconnectResetFrame identifies the authoritative stateful reset emitted | |
| 1010 | + | // by the daemon's existing resize path. Stateless side effects can also have a | |
| 1011 | + | // zero base, so they must not dismiss or reconcile a client-local overlay. | |
| 1012 | + | func isReconnectResetFrame(o ports.Output) bool { | |
| 1013 | + | return o.BaseStateNum == 0 && o.NewStateNum != 0 | |
| 1014 | + | } | |
| 1015 | + | ||
| 892 | 1016 | // recvResult carries one framed message (or a read error) from the receive | |
| 893 | 1017 | // pump to the main loop. | |
| 894 | 1018 | type recvResult struct { | |
| @@ -269,13 +269,23 @@ func (r *Runner) Run(ctx context.Context, request AttachRequest) (retErr error) | |||||
| 269 | transport, err := r.dialer.Dial(ctx) | 269 | transport, err := r.dialer.Dial(ctx) | ||
| 270 | if err != nil { | 270 | if err != nil { | ||
| 271 | if resumeToken == 0 || ctx.Err() != nil { | 271 | if resumeToken == 0 || ctx.Err() != nil { | ||
| 272 | - | reconnect.clear() | 272 | + | if clearErr := reconnect.clear(); clearErr != nil { |
| 273 | + | return errors.Join(err, clearErr) | |||
| 274 | + | } | |||
| 273 | return err | 275 | return err | ||
| 274 | } | 276 | } | ||
| 275 | r.logger.Warn("reconnect dial failed", "err", err, "backoff", backoff) | 277 | r.logger.Warn("reconnect dial failed", "err", err, "backoff", backoff) | ||
| 276 | - | reconnect.draw() | 278 | + | if drawErr := reconnect.draw(); drawErr != nil { |
| 277 | - | if !reconnect.sleep(ctx, r.clock, backoff) { | 279 | + | return errors.Join(err, drawErr) |
| 278 | - | reconnect.clear() | 280 | + | } |
| 281 | + | slept, sleepErr := reconnect.sleep(ctx, r.clock, backoff) | |||
| 282 | + | if sleepErr != nil { | |||
| 283 | + | return errors.Join(err, sleepErr) | |||
| 284 | + | } | |||
| 285 | + | if !slept { | |||
| 286 | + | if clearErr := reconnect.clear(); clearErr != nil { | |||
| 287 | + | return errors.Join(ctx.Err(), clearErr) | |||
| 288 | + | } | |||
| 279 | return ctx.Err() | 289 | return ctx.Err() | ||
| 280 | } | 290 | } | ||
| 281 | backoff = nextReconnectBackoff(backoff, defaultReconnectBackoff.max) | 291 | backoff = nextReconnectBackoff(backoff, defaultReconnectBackoff.max) | ||
| @@ -316,17 +326,29 @@ func (r *Runner) Run(ctx context.Context, request AttachRequest) (retErr error) | |||||
| 316 | attemptRequest.SessionName = result.sessionName | 326 | attemptRequest.SessionName = result.sessionName | ||
| 317 | } | 327 | } | ||
| 318 | if result.err == nil { | 328 | if result.err == nil { | ||
| 319 | - | reconnect.clear() | 329 | + | if clearErr := reconnect.clear(); clearErr != nil { |
| 330 | + | return clearErr | |||
| 331 | + | } | |||
| 320 | return nil | 332 | return nil | ||
| 321 | } | 333 | } | ||
| 322 | if !shouldReconnect(result.err) || resumeToken == 0 || ctx.Err() != nil { | 334 | if !shouldReconnect(result.err) || resumeToken == 0 || ctx.Err() != nil { | ||
| 323 | - | reconnect.clear() | 335 | + | if clearErr := reconnect.clear(); clearErr != nil { |
| 336 | + | return errors.Join(result.err, clearErr) | |||
| 337 | + | } | |||
| 324 | return result.err | 338 | return result.err | ||
| 325 | } | 339 | } | ||
| 326 | r.logger.Warn("reconnecting after attach error", "err", result.err, "backoff", backoff) | 340 | r.logger.Warn("reconnecting after attach error", "err", result.err, "backoff", backoff) | ||
| 327 | - | reconnect.drawStage(reconnectStageSSH) | 341 | + | if drawErr := reconnect.drawStage(reconnectStageSSH); drawErr != nil { |
| 328 | - | if !reconnect.sleep(ctx, r.clock, backoff) { | 342 | + | return errors.Join(result.err, drawErr) |
| 329 | - | reconnect.clear() | 343 | + | } |
| 344 | + | slept, sleepErr := reconnect.sleep(ctx, r.clock, backoff) | |||
| 345 | + | if sleepErr != nil { | |||
| 346 | + | return errors.Join(result.err, sleepErr) | |||
| 347 | + | } | |||
| 348 | + | if !slept { | |||
| 349 | + | if clearErr := reconnect.clear(); clearErr != nil { | |||
| 350 | + | return errors.Join(ctx.Err(), clearErr) | |||
| 351 | + | } | |||
| 330 | return ctx.Err() | 352 | return ctx.Err() | ||
| 331 | } | 353 | } | ||
| 332 | backoff = nextReconnectBackoff(backoff, defaultReconnectBackoff.max) | 354 | backoff = nextReconnectBackoff(backoff, defaultReconnectBackoff.max) | ||
| @@ -342,70 +364,78 @@ type reconnectUI struct { | |||||
| 342 | remote bool | 364 | remote bool | ||
| 343 | rawEntered *bool | 365 | rawEntered *bool | ||
| 344 | showing bool | 366 | showing bool | ||
| 345 | - | rect domain.Rect | |||
| 346 | stage reconnectStage | 367 | stage reconnectStage | ||
| 347 | } | 368 | } | ||
| 348 | 369 | ||||
| 349 | - | func (u *reconnectUI) redraw(size domain.Size) { | 370 | + | func (u *reconnectUI) redraw(size domain.Size) error { |
| 350 | if !*u.rawEntered || !u.remote { | 371 | if !*u.rawEntered || !u.remote { | ||
| 351 | - | return | 372 | + | return nil |
| 352 | } | 373 | } | ||
| 353 | - | if u.showing { | 374 | + | _, err := drawReconnectToastStage(u.term.Out(), size, u.stage) |
| 354 | - | _ = clearReconnectToast(u.term.Out(), u.rect) | 375 | + | if err != nil { |
| 376 | + | return fmt.Errorf("drawing reconnect toast: %w", err) | |||
| 377 | + | } | |||
| 378 | + | if err := u.term.Flush(); err != nil { | |||
| 379 | + | return fmt.Errorf("flushing reconnect toast: %w", err) | |||
| 355 | } | 380 | } | ||
| 356 | - | drawnRect, _ := drawReconnectToastStage(u.term.Out(), size, u.stage) | |||
| 357 | - | u.rect = drawnRect | |||
| 358 | - | _ = u.term.Flush() | |||
| 359 | u.showing = true | 381 | u.showing = true | ||
| 382 | + | return nil | |||
| 360 | } | 383 | } | ||
| 361 | 384 | ||||
| 362 | - | func (u *reconnectUI) drawStage(stage reconnectStage) { | 385 | + | func (u *reconnectUI) drawStage(stage reconnectStage) error { |
| 363 | u.stage = stage | 386 | u.stage = stage | ||
| 364 | if u.showing && u.remote { | 387 | if u.showing && u.remote { | ||
| 365 | size, err := u.term.Size() | 388 | size, err := u.term.Size() | ||
| 366 | - | if err == nil { | 389 | + | if err != nil { |
| 367 | - | u.redraw(size) | 390 | + | return fmt.Errorf("reading terminal size for reconnect toast: %w", err) |
| 368 | } | 391 | } | ||
| 369 | - | return | 392 | + | return u.redraw(size) |
| 370 | } | 393 | } | ||
| 371 | - | u.draw() | 394 | + | return u.draw() |
| 372 | } | 395 | } | ||
| 373 | 396 | ||||
| 374 | - | func (u *reconnectUI) draw() { | 397 | + | func (u *reconnectUI) draw() error { |
| 375 | if !*u.rawEntered || u.showing { | 398 | if !*u.rawEntered || u.showing { | ||
| 376 | - | return | 399 | + | return nil |
| 377 | } | 400 | } | ||
| 378 | if u.remote { | 401 | if u.remote { | ||
| 379 | size, err := u.term.Size() | 402 | size, err := u.term.Size() | ||
| 380 | if err != nil { | 403 | if err != nil { | ||
| 381 | - | return | 404 | + | return fmt.Errorf("reading terminal size for reconnect toast: %w", err) |
| 382 | } | 405 | } | ||
| 383 | - | u.redraw(size) | 406 | + | return u.redraw(size) |
| 384 | - | return | 407 | + | } |
| 408 | + | if _, err := u.term.Out().Write([]byte(statusReconnect)); err != nil { | |||
| 409 | + | return fmt.Errorf("writing reconnect status: %w", err) | |||
| 410 | + | } | |||
| 411 | + | if err := u.term.Flush(); err != nil { | |||
| 412 | + | return fmt.Errorf("flushing reconnect status: %w", err) | |||
| 385 | } | 413 | } | ||
| 386 | - | _, _ = u.term.Out().Write([]byte(statusReconnect)) | |||
| 387 | - | _ = u.term.Flush() | |||
| 388 | u.showing = true | 414 | u.showing = true | ||
| 415 | + | return nil | |||
| 389 | } | 416 | } | ||
| 390 | 417 | ||||
| 391 | - | func (u *reconnectUI) clear() { | 418 | + | func (u *reconnectUI) clear() error { |
| 392 | if !u.showing { | 419 | if !u.showing { | ||
| 393 | - | return | 420 | + | return nil |
| 394 | } | 421 | } | ||
| 395 | - | if u.remote { | 422 | + | if !u.remote { |
| 396 | - | _ = clearReconnectToast(u.term.Out(), u.rect) | 423 | + | if _, err := u.term.Out().Write([]byte(statusClear)); err != nil { |
| 397 | - | } else { | 424 | + | return fmt.Errorf("clearing reconnect status: %w", err) |
| 398 | - | _, _ = u.term.Out().Write([]byte(statusClear)) | 425 | + | } |
| 426 | + | if err := u.term.Flush(); err != nil { | |||
| 427 | + | return fmt.Errorf("flushing reconnect status clear: %w", err) | |||
| 428 | + | } | |||
| 399 | } | 429 | } | ||
| 400 | - | _ = u.term.Flush() | |||
| 401 | u.showing = false | 430 | u.showing = false | ||
| 431 | + | return nil | |||
| 402 | } | 432 | } | ||
| 403 | 433 | ||||
| 404 | - | func (u *reconnectUI) sleep(ctx context.Context, clk ports.Clock, d time.Duration) bool { | 434 | + | func (u *reconnectUI) sleep(ctx context.Context, clk ports.Clock, d time.Duration) (bool, error) { |
| 405 | if u.remote && u.showing { | 435 | if u.remote && u.showing { | ||
| 406 | return reconnectSleepWithResize(ctx, clk, d, u.term.ResizeEvents(), u.redraw) | 436 | return reconnectSleepWithResize(ctx, clk, d, u.term.ResizeEvents(), u.redraw) | ||
| 407 | } | 437 | } | ||
| 408 | - | return reconnectSleep(ctx, clk, d) | 438 | + | return reconnectSleep(ctx, clk, d), nil |
| 409 | } | 439 | } | ||
| 410 | 440 | ||||
| 411 | func sleepReconnect(ctx context.Context, clk ports.Clock, d time.Duration) bool { | 441 | func sleepReconnect(ctx context.Context, clk ports.Clock, d time.Duration) bool { | ||
| @@ -419,22 +449,24 @@ func sleepReconnect(ctx context.Context, clk ports.Clock, d time.Duration) bool | |||||
| 419 | } | 449 | } | ||
| 420 | } | 450 | } | ||
| 421 | 451 | ||||
| 422 | - | func sleepReconnectWithResizeEvents(ctx context.Context, clk ports.Clock, d time.Duration, resizeEvents <-chan domain.Size, onResize func(domain.Size)) bool { | 452 | + | func sleepReconnectWithResizeEvents(ctx context.Context, clk ports.Clock, d time.Duration, resizeEvents <-chan domain.Size, onResize func(domain.Size) error) (bool, error) { |
| 423 | t := clk.NewTimer(d) | 453 | t := clk.NewTimer(d) | ||
| 424 | defer t.Stop() | 454 | defer t.Stop() | ||
| 425 | for { | 455 | for { | ||
| 426 | select { | 456 | select { | ||
| 427 | case <-t.C(): | 457 | case <-t.C(): | ||
| 428 | - | return true | 458 | + | return true, nil |
| 429 | case <-ctx.Done(): | 459 | case <-ctx.Done(): | ||
| 430 | - | return false | 460 | + | return false, nil |
| 431 | case size, ok := <-resizeEvents: | 461 | case size, ok := <-resizeEvents: | ||
| 432 | if !ok { | 462 | if !ok { | ||
| 433 | resizeEvents = nil | 463 | resizeEvents = nil | ||
| 434 | continue | 464 | continue | ||
| 435 | } | 465 | } | ||
| 436 | if onResize != nil { | 466 | if onResize != nil { | ||
| 437 | - | onResize(size) | 467 | + | if err := onResize(size); err != nil { |
| 468 | + | return false, err | |||
| 469 | + | } | |||
| 438 | } | 470 | } | ||
| 439 | } | 471 | } | ||
| 440 | } | 472 | } | ||
| @@ -619,7 +651,6 @@ func (a *attachAttempt) run(ctx context.Context) attachResult { | |||||
| 619 | if err := enterRaw(); err != nil { | 651 | if err := enterRaw(); err != nil { | ||
| 620 | return welcomedResult(err) | 652 | return welcomedResult(err) | ||
| 621 | } | 653 | } | ||
| 622 | - | reconnect.clear() | |||
| 623 | 654 | ||||
| 624 | // 4. Derive a cancellable context so the pumps always unwind when the | 655 | // 4. Derive a cancellable context so the pumps always unwind when the | ||
| 625 | // loop returns. The attach loop is the sole terminal writer and palette | 656 | // loop returns. The attach loop is the sole terminal writer and palette | ||
| @@ -757,6 +788,25 @@ func (a *attachAttempt) run(ctx context.Context) attachResult { | |||||
| 757 | if remote { | 788 | if remote { | ||
| 758 | clip = clipboard | 789 | clip = clipboard | ||
| 759 | } | 790 | } | ||
| 791 | + | awaitingReconnectReset := false | |||
| 792 | + | if reconnect.showing { | |||
| 793 | + | if reconnect.remote { | |||
| 794 | + | // Before the asynchronous sender starts, preserve transport ordering by | |||
| 795 | + | // requesting the reset synchronously after the initial Theme publication. | |||
| 796 | + | closed, resetErr := boundedPreWelcome(ctx, clk, transport, func() error { | |||
| 797 | + | return transport.Send(ports.Frame{Type: ports.MsgResize, Payload: ports.MarshalResize(ports.Resize{Size: size})}) | |||
| 798 | + | }) | |||
| 799 | + | if resetErr != nil { | |||
| 800 | + | resetResult := welcomedResult(fmt.Errorf("requesting reconnect reconciliation: %w", resetErr)) | |||
| 801 | + | resetResult.transportClosed = closed | |||
| 802 | + | return resetResult | |||
| 803 | + | } | |||
| 804 | + | awaitingReconnectReset = true | |||
| 805 | + | } | |||
| 806 | + | if err := reconnect.clear(); err != nil { | |||
| 807 | + | return welcomedResult(err) | |||
| 808 | + | } | |||
| 809 | + | } | |||
| 760 | senderStarted = true | 810 | senderStarted = true | ||
| 761 | go runSender(loopCtx, cancel, transport, sendCh, ackQueue, sendErrCh, log) | 811 | go runSender(loopCtx, cancel, transport, sendCh, ackQueue, sendErrCh, log) | ||
| 762 | stdinDone := make(chan struct{}) | 812 | stdinDone := make(chan struct{}) | ||
| @@ -780,24 +830,60 @@ func (a *attachAttempt) run(ctx context.Context) attachResult { | |||||
| 780 | recvCh := make(chan recvResult, 1) | 830 | recvCh := make(chan recvResult, 1) | ||
| 781 | go runRecv(loopCtx, transport, recvCh, log) | 831 | go runRecv(loopCtx, transport, recvCh, log) | ||
| 782 | 832 | ||||
| 833 | + | requestReconnectReset := func() error { | |||
| 834 | + | if awaitingReconnectReset { | |||
| 835 | + | return nil | |||
| 836 | + | } | |||
| 837 | + | size, serr := term.Size() | |||
| 838 | + | if serr != nil { | |||
| 839 | + | return fmt.Errorf("reading terminal size for reconnect reconciliation: %w", serr) | |||
| 840 | + | } | |||
| 841 | + | select { | |||
| 842 | + | case sendCh <- ports.Frame{Type: ports.MsgResize, Payload: ports.MarshalResize(ports.Resize{Size: size})}: | |||
| 843 | + | awaitingReconnectReset = true | |||
| 844 | + | return nil | |||
| 845 | + | case <-loopCtx.Done(): | |||
| 846 | + | return context.Canceled | |||
| 847 | + | } | |||
| 848 | + | } | |||
| 849 | + | dismissReconnect := func() error { | |||
| 850 | + | if !reconnect.showing { | |||
| 851 | + | return nil | |||
| 852 | + | } | |||
| 853 | + | if err := reconnect.clear(); err != nil { | |||
| 854 | + | return err | |||
| 855 | + | } | |||
| 856 | + | if !reconnect.remote { | |||
| 857 | + | return nil | |||
| 858 | + | } | |||
| 859 | + | return requestReconnectReset() | |||
| 860 | + | } | |||
| 861 | + | loopCanceledResult := func() attachResult { | |||
| 862 | + | // A pump initiated shutdown (stdin EOF/detach) or the parent context | |||
| 863 | + | // was cancelled. A queued sender error takes priority. | |||
| 864 | + | select { | |||
| 865 | + | case serr := <-sendErrCh: | |||
| 866 | + | return welcomedResult(fmt.Errorf("vev: sending to daemon: %w", serr)) | |||
| 867 | + | default: | |||
| 868 | + | return welcomedResult(nil) | |||
| 869 | + | } | |||
| 870 | + | } | |||
| 783 | for { | 871 | for { | ||
| 784 | select { | 872 | select { | ||
| 785 | case <-loopCtx.Done(): | 873 | case <-loopCtx.Done(): | ||
| 786 | - | // A pump initiated shutdown (stdin EOF/detach) or the parent | 874 | + | return loopCanceledResult() |
| 787 | - | // context was cancelled. A queued sender error takes priority. | |||
| 788 | - | select { | |||
| 789 | - | case serr := <-sendErrCh: | |||
| 790 | - | return welcomedResult(fmt.Errorf("vev: sending to daemon: %w", serr)) | |||
| 791 | - | default: | |||
| 792 | - | return welcomedResult(nil) | |||
| 793 | - | } | |||
| 794 | case ev, ok := <-linkEvents: | 875 | case ev, ok := <-linkEvents: | ||
| 795 | if !ok { | 876 | if !ok { | ||
| 796 | linkEvents = nil | 877 | linkEvents = nil | ||
| 797 | continue | 878 | continue | ||
| 798 | } | 879 | } | ||
| 799 | if ev.State == ports.LinkStateConnected { | 880 | if ev.State == ports.LinkStateConnected { | ||
| 800 | - | reconnect.clear() | 881 | + | if err := dismissReconnect(); err != nil { |
| 882 | + | if errors.Is(err, context.Canceled) { | |||
| 883 | + | return loopCanceledResult() | |||
| 884 | + | } | |||
| 885 | + | return welcomedResult(fmt.Errorf("dismissing reconnect toast: %w", err)) | |||
| 886 | + | } | |||
| 801 | select { | 887 | select { | ||
| 802 | case sendCh <- ports.Frame{Type: ports.MsgClientNotice, Payload: ports.MarshalClientNotice(ports.ClientNotice{Action: ports.ClientNoticeLinkConnected})}: | 888 | case sendCh <- ports.Frame{Type: ports.MsgClientNotice, Payload: ports.MarshalClientNotice(ports.ClientNotice{Action: ports.ClientNoticeLinkConnected})}: | ||
| 803 | case <-loopCtx.Done(): | 889 | case <-loopCtx.Done(): | ||
| @@ -814,7 +900,18 @@ func (a *attachAttempt) run(ctx context.Context) attachResult { | |||||
| 814 | } | 900 | } | ||
| 815 | continue | 901 | continue | ||
| 816 | } | 902 | } | ||
| 817 | - | reconnect.drawStage(stageForLinkState(ev.State)) | 903 | + | stage := stageForLinkState(ev.State) |
| 904 | + | if reconnect.remote && reconnect.showing && reconnect.stage != stage { | |||
| 905 | + | reconnect.stage = stage | |||
| 906 | + | if err := requestReconnectReset(); err != nil { | |||
| 907 | + | if errors.Is(err, context.Canceled) { | |||
| 908 | + | return loopCanceledResult() | |||
| 909 | + | } | |||
| 910 | + | return welcomedResult(fmt.Errorf("requesting reconnect reconciliation: %w", err)) | |||
| 911 | + | } | |||
| 912 | + | } else if err := reconnect.drawStage(stage); err != nil { | |||
| 913 | + | return welcomedResult(err) | |||
| 914 | + | } | |||
| 818 | if ev.State == ports.LinkStateOffline { | 915 | if ev.State == ports.LinkStateOffline { | ||
| 819 | // Stop waiting for the transport to reach Dead at 60s: exit with | 916 | // Stop waiting for the transport to reach Dead at 60s: exit with | ||
| 820 | // a retryable error so Run re-dials over ssh with the resume | 917 | // a retryable error so Run re-dials over ssh with the resume | ||
| @@ -850,10 +947,30 @@ func (a *attachAttempt) run(ctx context.Context) attachResult { | |||||
| 850 | if derr != nil { | 947 | if derr != nil { | ||
| 851 | return welcomedResult(fmt.Errorf("vev: decoding output: %w", derr)) | 948 | return welcomedResult(fmt.Errorf("vev: decoding output: %w", derr)) | ||
| 852 | } | 949 | } | ||
| 950 | + | // Dismissing or changing a local overlay requests a same-size resize, | |||
| 951 | + | // whose stateful base-zero Output is the daemon's authoritative reset. | |||
| 952 | + | // ACK intervening increments so the output window keeps moving, but do | |||
| 953 | + | // not apply diffs based on cells hidden by the client-local overlay. | |||
| 954 | + | resetFrame := isReconnectResetFrame(o) | |||
| 955 | + | if awaitingReconnectReset && !resetFrame && o.NewStateNum != 0 { | |||
| 956 | + | ackQueue.offer(o.NewStateNum) | |||
| 957 | + | continue | |||
| 958 | + | } | |||
| 853 | if _, werr := term.Out().Write(o.Data); werr != nil { | 959 | if _, werr := term.Out().Write(o.Data); werr != nil { | ||
| 854 | return welcomedResult(fmt.Errorf("vev: writing terminal output: %w", werr)) | 960 | return welcomedResult(fmt.Errorf("vev: writing terminal output: %w", werr)) | ||
| 855 | } | 961 | } | ||
| 856 | - | if ferr := term.Flush(); ferr != nil { | 962 | + | if awaitingReconnectReset && resetFrame { |
| 963 | + | awaitingReconnectReset = false | |||
| 964 | + | } | |||
| 965 | + | if reconnect.showing && reconnect.remote { | |||
| 966 | + | size, serr := term.Size() | |||
| 967 | + | if serr != nil { | |||
| 968 | + | return welcomedResult(fmt.Errorf("vev: reading terminal size for reconnect redraw: %w", serr)) | |||
| 969 | + | } | |||
| 970 | + | if rerr := reconnect.redraw(size); rerr != nil { | |||
| 971 | + | return welcomedResult(fmt.Errorf("vev: redrawing reconnect toast: %w", rerr)) | |||
| 972 | + | } | |||
| 973 | + | } else if ferr := term.Flush(); ferr != nil { | |||
| 857 | return welcomedResult(fmt.Errorf("vev: flushing terminal: %w", ferr)) | 974 | return welcomedResult(fmt.Errorf("vev: flushing terminal: %w", ferr)) | ||
| 858 | } | 975 | } | ||
| 859 | // The terminal boundary is after a successful flush and before ACK. | 976 | // The terminal boundary is after a successful flush and before ACK. | ||
| @@ -889,6 +1006,13 @@ func (a *attachAttempt) run(ctx context.Context) attachResult { | |||||
| 889 | } | 1006 | } | ||
| 890 | } | 1007 | } | ||
| 891 | 1008 | ||||
| 1009 | + | // isReconnectResetFrame identifies the authoritative stateful reset emitted | |||
| 1010 | + | // by the daemon's existing resize path. Stateless side effects can also have a | |||
| 1011 | + | // zero base, so they must not dismiss or reconcile a client-local overlay. | |||
| 1012 | + | func isReconnectResetFrame(o ports.Output) bool { | |||
| 1013 | + | return o.BaseStateNum == 0 && o.NewStateNum != 0 | |||
| 1014 | + | } | |||
| 1015 | + | ||||
| 892 | // recvResult carries one framed message (or a read error) from the receive | 1016 | // recvResult carries one framed message (or a read error) from the receive | ||
| 893 | // pump to the main loop. | 1017 | // pump to the main loop. | ||
| 894 | type recvResult struct { | 1018 | type recvResult struct { | ||
M internal/usecase/client/reconnect_toast.go +0 -12
| @@ -47,18 +47,6 @@ func drawClientToast(out io.Writer, size domain.Size, message string) (domain.Re | |||
| 47 | 47 | return bounds, writeReconnectToast(out, bounds, reconnectToastLinesFor(bounds, message)) | |
| 48 | 48 | } | |
| 49 | 49 | ||
| 50 | - | func clearReconnectToast(out io.Writer, bounds domain.Rect) error { | |
| 51 | - | if bounds.Width <= 0 || bounds.Height <= 0 { | |
| 52 | - | return nil | |
| 53 | - | } | |
| 54 | - | blank := strings.Repeat(" ", bounds.Width) | |
| 55 | - | lines := make([]string, bounds.Height) | |
| 56 | - | for i := range lines { | |
| 57 | - | lines[i] = blank | |
| 58 | - | } | |
| 59 | - | return writeReconnectToast(out, bounds, lines) | |
| 60 | - | } | |
| 61 | - | ||
| 62 | 50 | func reconnectToastBounds(size domain.Size) domain.Rect { | |
| 63 | 51 | return reconnectToastBoundsFor(size, reconnectToastMessage) | |
| 64 | 52 | } | |
| @@ -47,18 +47,6 @@ func drawClientToast(out io.Writer, size domain.Size, message string) (domain.Re | |||||
| 47 | return bounds, writeReconnectToast(out, bounds, reconnectToastLinesFor(bounds, message)) | 47 | return bounds, writeReconnectToast(out, bounds, reconnectToastLinesFor(bounds, message)) | ||
| 48 | } | 48 | } | ||
| 49 | 49 | ||||
| 50 | - | func clearReconnectToast(out io.Writer, bounds domain.Rect) error { | |||
| 51 | - | if bounds.Width <= 0 || bounds.Height <= 0 { | |||
| 52 | - | return nil | |||
| 53 | - | } | |||
| 54 | - | blank := strings.Repeat(" ", bounds.Width) | |||
| 55 | - | lines := make([]string, bounds.Height) | |||
| 56 | - | for i := range lines { | |||
| 57 | - | lines[i] = blank | |||
| 58 | - | } | |||
| 59 | - | return writeReconnectToast(out, bounds, lines) | |||
| 60 | - | } | |||
| 61 | - | ||||
| 62 | func reconnectToastBounds(size domain.Size) domain.Rect { | 50 | func reconnectToastBounds(size domain.Size) domain.Rect { | ||
| 63 | return reconnectToastBoundsFor(size, reconnectToastMessage) | 51 | return reconnectToastBoundsFor(size, reconnectToastMessage) | ||
| 64 | } | 52 | } | ||
M internal/usecase/client/reconnect_toast_test.go +1040 -104
| @@ -9,6 +9,7 @@ import ( | |||
| 9 | 9 | "log/slog" | |
| 10 | 10 | "strings" | |
| 11 | 11 | "sync" | |
| 12 | + | "sync/atomic" | |
| 12 | 13 | "testing" | |
| 13 | 14 | "time" | |
| 14 | 15 | ||
| @@ -22,26 +23,35 @@ import ( | |||
| 22 | 23 | ) | |
| 23 | 24 | ||
| 24 | 25 | type reconnectToastTerminalHarness struct { | |
| 25 | - | term *portsmocks.MockTerminal | |
| 26 | - | in *io.PipeReader | |
| 27 | - | inWriter *io.PipeWriter | |
| 28 | - | out bytes.Buffer | |
| 29 | - | size domain.Size | |
| 30 | - | resizeCh chan domain.Size | |
| 26 | + | term *portsmocks.MockTerminal | |
| 27 | + | in *io.PipeReader | |
| 28 | + | inWriter *io.PipeWriter | |
| 29 | + | out bytes.Buffer | |
| 30 | + | size domain.Size | |
| 31 | + | resizeCh chan domain.Size | |
| 32 | + | sizeCalls atomic.Int32 | |
| 33 | + | sizeCalled chan int32 | |
| 31 | 34 | } | |
| 32 | 35 | ||
| 33 | 36 | func newReconnectToastTerminalHarness(t *testing.T) *reconnectToastTerminalHarness { | |
| 34 | 37 | t.Helper() | |
| 35 | 38 | in, inWriter := io.Pipe() | |
| 36 | 39 | h := &reconnectToastTerminalHarness{ | |
| 37 | - | term: portsmocks.NewMockTerminal(t), | |
| 38 | - | in: in, | |
| 39 | - | inWriter: inWriter, | |
| 40 | - | size: domain.Size{Cols: 80, Rows: 24}, | |
| 41 | - | resizeCh: make(chan domain.Size), | |
| 40 | + | term: portsmocks.NewMockTerminal(t), | |
| 41 | + | in: in, | |
| 42 | + | inWriter: inWriter, | |
| 43 | + | size: domain.Size{Cols: 80, Rows: 24}, | |
| 44 | + | resizeCh: make(chan domain.Size, sendQueueDepth), | |
| 45 | + | sizeCalled: make(chan int32, 16), | |
| 42 | 46 | } | |
| 43 | 47 | h.term.EXPECT().EnterRaw().Return(func() error { return nil }, nil).Maybe() | |
| 44 | - | h.term.EXPECT().Size().Return(h.size, nil).Maybe() | |
| 48 | + | h.term.EXPECT().Size().Run(func() { | |
| 49 | + | call := h.sizeCalls.Add(1) | |
| 50 | + | select { | |
| 51 | + | case h.sizeCalled <- call: | |
| 52 | + | default: | |
| 53 | + | } | |
| 54 | + | }).Return(h.size, nil).Maybe() | |
| 45 | 55 | h.term.EXPECT().ResizeEvents().Return((<-chan domain.Size)(h.resizeCh)).Maybe() | |
| 46 | 56 | h.term.EXPECT().In().Return(h.in).Maybe() | |
| 47 | 57 | h.term.EXPECT().Out().Return(&h.out).Maybe() | |
| @@ -50,21 +60,33 @@ func newReconnectToastTerminalHarness(t *testing.T) *reconnectToastTerminalHarne | |||
| 50 | 60 | } | |
| 51 | 61 | ||
| 52 | 62 | func newReconnectToastTerminalHarnessWithOutput(t *testing.T, out io.Writer) *reconnectToastTerminalHarness { | |
| 63 | + | t.Helper() | |
| 64 | + | return newReconnectToastTerminalHarnessWithOutputAndFlush(t, out, func() error { return nil }) | |
| 65 | + | } | |
| 66 | + | ||
| 67 | + | func newReconnectToastTerminalHarnessWithOutputAndFlush(t *testing.T, out io.Writer, flush func() error) *reconnectToastTerminalHarness { | |
| 53 | 68 | t.Helper() | |
| 54 | 69 | in, inWriter := io.Pipe() | |
| 55 | 70 | h := &reconnectToastTerminalHarness{ | |
| 56 | - | term: portsmocks.NewMockTerminal(t), | |
| 57 | - | in: in, | |
| 58 | - | inWriter: inWriter, | |
| 59 | - | size: domain.Size{Cols: 80, Rows: 24}, | |
| 60 | - | resizeCh: make(chan domain.Size), | |
| 71 | + | term: portsmocks.NewMockTerminal(t), | |
| 72 | + | in: in, | |
| 73 | + | inWriter: inWriter, | |
| 74 | + | size: domain.Size{Cols: 80, Rows: 24}, | |
| 75 | + | resizeCh: make(chan domain.Size, sendQueueDepth), | |
| 76 | + | sizeCalled: make(chan int32, 16), | |
| 61 | 77 | } | |
| 62 | 78 | h.term.EXPECT().EnterRaw().Return(func() error { return nil }, nil).Maybe() | |
| 63 | - | h.term.EXPECT().Size().Return(h.size, nil).Maybe() | |
| 79 | + | h.term.EXPECT().Size().Run(func() { | |
| 80 | + | call := h.sizeCalls.Add(1) | |
| 81 | + | select { | |
| 82 | + | case h.sizeCalled <- call: | |
| 83 | + | default: | |
| 84 | + | } | |
| 85 | + | }).Return(h.size, nil).Maybe() | |
| 64 | 86 | h.term.EXPECT().ResizeEvents().Return((<-chan domain.Size)(h.resizeCh)).Maybe() | |
| 65 | 87 | h.term.EXPECT().In().Return(h.in).Maybe() | |
| 66 | 88 | h.term.EXPECT().Out().Return(out).Maybe() | |
| 67 | - | h.term.EXPECT().Flush().Return(nil).Maybe() | |
| 89 | + | h.term.EXPECT().Flush().RunAndReturn(flush).Maybe() | |
| 68 | 90 | return h | |
| 69 | 91 | } | |
| 70 | 92 | ||
| @@ -73,6 +95,7 @@ func (h *reconnectToastTerminalHarness) closeInput() { _ = h.inWriter.Close() } | |||
| 73 | 95 | type reconnectToastOutputRecorder struct { | |
| 74 | 96 | mu sync.Mutex | |
| 75 | 97 | pending strings.Builder | |
| 98 | + | all strings.Builder | |
| 76 | 99 | completed chan string | |
| 77 | 100 | } | |
| 78 | 101 | ||
| @@ -85,6 +108,7 @@ func (r *reconnectToastOutputRecorder) Write(p []byte) (int, error) { | |||
| 85 | 108 | ||
| 86 | 109 | r.mu.Lock() | |
| 87 | 110 | r.pending.Write(p) | |
| 111 | + | r.all.Write(p) | |
| 88 | 112 | var output string | |
| 89 | 113 | if string(p) == restoreCursor { | |
| 90 | 114 | output = r.pending.String() | |
| @@ -98,6 +122,12 @@ func (r *reconnectToastOutputRecorder) Write(p []byte) (int, error) { | |||
| 98 | 122 | return len(p), nil | |
| 99 | 123 | } | |
| 100 | 124 | ||
| 125 | + | func (r *reconnectToastOutputRecorder) String() string { | |
| 126 | + | r.mu.Lock() | |
| 127 | + | defer r.mu.Unlock() | |
| 128 | + | return r.all.String() | |
| 129 | + | } | |
| 130 | + | ||
| 101 | 131 | func reconnectToastWelcome(token uint64) ports.Frame { | |
| 102 | 132 | return reconnectToastWelcomeNamed("", token) | |
| 103 | 133 | } | |
| @@ -140,8 +170,40 @@ func newReconnectToastRecordingTransport(recvs ...reconnectToastRecv) *reconnect | |||
| 140 | 170 | return &reconnectToastRecordingTransport{recvs: append([]reconnectToastRecv(nil), recvs...)} | |
| 141 | 171 | } | |
| 142 | 172 | ||
| 173 | + | type reconnectToastSentFrames struct { | |
| 174 | + | mu sync.Mutex | |
| 175 | + | frames []ports.Frame | |
| 176 | + | changed chan struct{} | |
| 177 | + | } | |
| 178 | + | ||
| 179 | + | func newReconnectToastSentFrames() *reconnectToastSentFrames { | |
| 180 | + | return &reconnectToastSentFrames{changed: make(chan struct{}, 1)} | |
| 181 | + | } | |
| 182 | + | ||
| 183 | + | func (s *reconnectToastSentFrames) record(frame ports.Frame) { | |
| 184 | + | s.mu.Lock() | |
| 185 | + | s.frames = append(s.frames, frame) | |
| 186 | + | s.mu.Unlock() | |
| 187 | + | select { | |
| 188 | + | case s.changed <- struct{}{}: | |
| 189 | + | default: | |
| 190 | + | } | |
| 191 | + | } | |
| 192 | + | ||
| 193 | + | func (s *reconnectToastSentFrames) find(match func(ports.Frame) bool) (ports.Frame, bool) { | |
| 194 | + | s.mu.Lock() | |
| 195 | + | defer s.mu.Unlock() | |
| 196 | + | for _, frame := range s.frames { | |
| 197 | + | if match(frame) { | |
| 198 | + | return frame, true | |
| 199 | + | } | |
| 200 | + | } | |
| 201 | + | return ports.Frame{}, false | |
| 202 | + | } | |
| 203 | + | ||
| 143 | 204 | type reconnectToastLinkTransport struct { | |
| 144 | 205 | recvCh chan reconnectToastRecv | |
| 206 | + | sends *reconnectToastSentFrames | |
| 145 | 207 | clientNotices chan ports.ClientNotice | |
| 146 | 208 | events chan ports.LinkEvent | |
| 147 | 209 | state ports.LinkState | |
| @@ -150,6 +212,7 @@ type reconnectToastLinkTransport struct { | |||
| 150 | 212 | func newReconnectToastLinkTransport() *reconnectToastLinkTransport { | |
| 151 | 213 | return &reconnectToastLinkTransport{ | |
| 152 | 214 | recvCh: make(chan reconnectToastRecv, 4), | |
| 215 | + | sends: newReconnectToastSentFrames(), | |
| 153 | 216 | clientNotices: make(chan ports.ClientNotice, 2), | |
| 154 | 217 | events: make(chan ports.LinkEvent, 4), | |
| 155 | 218 | state: ports.LinkStateConnected, | |
| @@ -157,6 +220,7 @@ func newReconnectToastLinkTransport() *reconnectToastLinkTransport { | |||
| 157 | 220 | } | |
| 158 | 221 | ||
| 159 | 222 | func (t *reconnectToastLinkTransport) Send(f ports.Frame) error { | |
| 223 | + | t.sends.record(f) | |
| 160 | 224 | if f.Type != ports.MsgClientNotice { | |
| 161 | 225 | return nil | |
| 162 | 226 | } | |
| @@ -222,7 +286,7 @@ func (t *reconnectToastRecordingTransport) wasClosed() bool { | |||
| 222 | 286 | func assertReconnectToastAttemptPublishesOnlyClearedTheme(t *testing.T, tr *reconnectToastRecordingTransport) { | |
| 223 | 287 | t.Helper() | |
| 224 | 288 | frames := tr.sentFrames() | |
| 225 | - | require.Len(t, frames, 2) | |
| 289 | + | require.GreaterOrEqual(t, len(frames), 2) | |
| 226 | 290 | require.Equal(t, ports.MsgHello, frames[0].Type) | |
| 227 | 291 | require.Equal(t, ports.MsgTheme, frames[1].Type) | |
| 228 | 292 | theme, err := ports.UnmarshalTheme(frames[1].Payload) | |
| @@ -290,6 +354,431 @@ func newReconnectAttachAttempt(term ports.Terminal, transport ports.Transport, c | |||
| 290 | 354 | } | |
| 291 | 355 | } | |
| 292 | 356 | ||
| 357 | + | func TestProbingToastReconcilesDaemonOutputAndDismissal(t *testing.T) { | |
| 358 | + | out := newReconnectToastOutputRecorder() | |
| 359 | + | var flushes atomic.Int32 | |
| 360 | + | term := newReconnectToastTerminalHarnessWithOutputAndFlush(t, out, func() error { | |
| 361 | + | flushes.Add(1) | |
| 362 | + | return nil | |
| 363 | + | }) | |
| 364 | + | defer term.closeInput() | |
| 365 | + | tr := newReconnectToastLinkTransport() | |
| 366 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |
| 367 | + | ||
| 368 | + | resultCh := make(chan attachResult, 1) | |
| 369 | + | ms := milestones{} | |
| 370 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |
| 371 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |
| 372 | + | ||
| 373 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |
| 374 | + | firstToast := requireReconnectToastOutput(t, out.completed) | |
| 375 | + | require.Contains(t, firstToast, "┌") | |
| 376 | + | require.Contains(t, firstToast, "probing UDP path") | |
| 377 | + | ||
| 378 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |
| 379 | + | BaseStateNum: 1, | |
| 380 | + | NewStateNum: 2, | |
| 381 | + | Data: []byte("daemon incremental"), | |
| 382 | + | })}} | |
| 383 | + | redrawn := requireReconnectToastOutput(t, out.completed) | |
| 384 | + | require.Contains(t, redrawn, "daemon incremental") | |
| 385 | + | require.Contains(t, redrawn, "┌") | |
| 386 | + | require.Contains(t, redrawn, "probing UDP path") | |
| 387 | + | ||
| 388 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateConnected} | |
| 389 | + | require.Equal(t, ports.ClientNoticeLinkConnected, (<-tr.clientNotices).Action) | |
| 390 | + | require.Equal(t, term.size, requireResize(t, tr.sends).Size) | |
| 391 | + | ||
| 392 | + | beforeAwaitingReset := out.String() | |
| 393 | + | beforeStatelessFlushes := flushes.Load() | |
| 394 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |
| 395 | + | BaseStateNum: 0, | |
| 396 | + | NewStateNum: 0, | |
| 397 | + | Data: []byte("stateless side effect"), | |
| 398 | + | })}} | |
| 399 | + | require.Eventually(t, func() bool { | |
| 400 | + | return strings.Contains(out.String(), "stateless side effect") && flushes.Load() > beforeStatelessFlushes | |
| 401 | + | }, time.Second, time.Millisecond) | |
| 402 | + | ||
| 403 | + | afterStateless := out.String() | |
| 404 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |
| 405 | + | BaseStateNum: 2, | |
| 406 | + | NewStateNum: 3, | |
| 407 | + | Data: []byte("intervening incremental"), | |
| 408 | + | })}} | |
| 409 | + | requireAckedState(t, tr.sends, 3) | |
| 410 | + | require.Equal(t, afterStateless, out.String(), "stateless output must not satisfy pending reset reconciliation") | |
| 411 | + | require.Contains(t, afterStateless[len(beforeAwaitingReset):], "stateless side effect") | |
| 412 | + | ||
| 413 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |
| 414 | + | BaseStateNum: 0, | |
| 415 | + | NewStateNum: 4, | |
| 416 | + | Data: []byte("authoritative reset"), | |
| 417 | + | })}} | |
| 418 | + | requireAckedState(t, tr.sends, 4) | |
| 419 | + | require.Eventually(t, func() bool { return strings.Contains(out.String(), "authoritative reset") }, time.Second, time.Millisecond) | |
| 420 | + | require.NotContains(t, out.String()[len(beforeAwaitingReset):], strings.Repeat(" ", reconnectToastBoundsFor(term.size, "probing UDP path").Width)) | |
| 421 | + | ||
| 422 | + | beforeIncrementFlushes := flushes.Load() | |
| 423 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |
| 424 | + | BaseStateNum: 4, | |
| 425 | + | NewStateNum: 5, | |
| 426 | + | Data: []byte("increment after reset"), | |
| 427 | + | })}} | |
| 428 | + | require.Eventually(t, func() bool { | |
| 429 | + | return strings.Contains(out.String(), "increment after reset") && flushes.Load() > beforeIncrementFlushes | |
| 430 | + | }, time.Second, time.Millisecond) | |
| 431 | + | requireAckedState(t, tr.sends, 5) | |
| 432 | + | ||
| 433 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | |
| 434 | + | result := requireAttachResult(t, resultCh) | |
| 435 | + | require.NoError(t, result.err) | |
| 436 | + | } | |
| 437 | + | ||
| 438 | + | func TestActiveReconnectToastStageTransitionReconcilesBeforeRedraw(t *testing.T) { | |
| 439 | + | out := newReconnectToastOutputRecorder() | |
| 440 | + | var flushes atomic.Int32 | |
| 441 | + | term := newReconnectToastTerminalHarnessWithOutputAndFlush(t, out, func() error { | |
| 442 | + | flushes.Add(1) | |
| 443 | + | return nil | |
| 444 | + | }) | |
| 445 | + | defer term.closeInput() | |
| 446 | + | tr := newReconnectToastLinkTransport() | |
| 447 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |
| 448 | + | ||
| 449 | + | resultCh := make(chan attachResult, 1) | |
| 450 | + | ms := milestones{} | |
| 451 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |
| 452 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |
| 453 | + | ||
| 454 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |
| 455 | + | probingToast := requireReconnectToastOutput(t, out.completed) | |
| 456 | + | require.Contains(t, probingToast, "probing UDP path") | |
| 457 | + | ||
| 458 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateDead} | |
| 459 | + | require.Equal(t, term.size, requireResize(t, tr.sends).Size) | |
| 460 | + | offlineMessage := reconnectStageMessage(reconnectStageOfflineRetrying) | |
| 461 | + | require.NotContains(t, out.String(), offlineMessage) | |
| 462 | + | ||
| 463 | + | beforeReset := out.String() | |
| 464 | + | for _, output := range []ports.Output{ | |
| 465 | + | {BaseStateNum: 1, NewStateNum: 2, Data: []byte("first skipped increment")}, | |
| 466 | + | {BaseStateNum: 2, NewStateNum: 3, Data: []byte("second skipped increment")}, | |
| 467 | + | } { | |
| 468 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(output)}} | |
| 469 | + | requireAckedState(t, tr.sends, output.NewStateNum) | |
| 470 | + | require.Equal(t, beforeReset, out.String()) | |
| 471 | + | } | |
| 472 | + | ||
| 473 | + | beforeResetFlushes := flushes.Load() | |
| 474 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |
| 475 | + | BaseStateNum: 0, | |
| 476 | + | NewStateNum: 4, | |
| 477 | + | Data: []byte("stage transition reset"), | |
| 478 | + | })}} | |
| 479 | + | redrawn := requireReconnectToastOutput(t, out.completed) | |
| 480 | + | require.Contains(t, redrawn, "stage transition reset") | |
| 481 | + | require.Contains(t, redrawn, offlineMessage) | |
| 482 | + | require.Eventually(t, func() bool { return flushes.Load() > beforeResetFlushes }, time.Second, time.Millisecond) | |
| 483 | + | requireAckedState(t, tr.sends, 4) | |
| 484 | + | ||
| 485 | + | beforeIncrementFlushes := flushes.Load() | |
| 486 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |
| 487 | + | BaseStateNum: 4, | |
| 488 | + | NewStateNum: 5, | |
| 489 | + | Data: []byte("stage increment after reset"), | |
| 490 | + | })}} | |
| 491 | + | incremental := requireReconnectToastOutput(t, out.completed) | |
| 492 | + | require.Contains(t, incremental, "stage increment after reset") | |
| 493 | + | require.Contains(t, incremental, offlineMessage) | |
| 494 | + | require.Eventually(t, func() bool { return flushes.Load() > beforeIncrementFlushes }, time.Second, time.Millisecond) | |
| 495 | + | requireAckedState(t, tr.sends, 5) | |
| 496 | + | ||
| 497 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | |
| 498 | + | result := requireAttachResult(t, resultCh) | |
| 499 | + | require.NoError(t, result.err) | |
| 500 | + | } | |
| 501 | + | ||
| 502 | + | func requireReconnectToastOutput(t *testing.T, completed <-chan string) string { | |
| 503 | + | t.Helper() | |
| 504 | + | select { | |
| 505 | + | case output := <-completed: | |
| 506 | + | return output | |
| 507 | + | case <-time.After(time.Second): | |
| 508 | + | t.Fatal("timed out waiting for reconnect toast terminal output") | |
| 509 | + | return "" | |
| 510 | + | } | |
| 511 | + | } | |
| 512 | + | ||
| 513 | + | func requireSentFrame(t *testing.T, sends *reconnectToastSentFrames, description string, match func(ports.Frame) bool) ports.Frame { | |
| 514 | + | t.Helper() | |
| 515 | + | timer := time.NewTimer(time.Second) | |
| 516 | + | defer timer.Stop() | |
| 517 | + | for { | |
| 518 | + | if frame, ok := sends.find(match); ok { | |
| 519 | + | return frame | |
| 520 | + | } | |
| 521 | + | select { | |
| 522 | + | case <-sends.changed: | |
| 523 | + | case <-timer.C: | |
| 524 | + | t.Fatalf("timed out waiting for %s", description) | |
| 525 | + | return ports.Frame{} | |
| 526 | + | } | |
| 527 | + | } | |
| 528 | + | } | |
| 529 | + | ||
| 530 | + | func requireResize(t *testing.T, sends *reconnectToastSentFrames) ports.Resize { | |
| 531 | + | t.Helper() | |
| 532 | + | frame := requireSentFrame(t, sends, "Resize frame", func(frame ports.Frame) bool { | |
| 533 | + | return frame.Type == ports.MsgResize | |
| 534 | + | }) | |
| 535 | + | resize, err := ports.UnmarshalResize(frame.Payload) | |
| 536 | + | require.NoError(t, err) | |
| 537 | + | return resize | |
| 538 | + | } | |
| 539 | + | ||
| 540 | + | func requireAckedState(t *testing.T, sends *reconnectToastSentFrames, state uint64) { | |
| 541 | + | t.Helper() | |
| 542 | + | frame := requireSentFrame(t, sends, fmt.Sprintf("ACK for state %d", state), func(frame ports.Frame) bool { | |
| 543 | + | if frame.Type != ports.MsgAck { | |
| 544 | + | return false | |
| 545 | + | } | |
| 546 | + | ack, err := ports.UnmarshalAck(frame.Payload) | |
| 547 | + | return err == nil && ack.AckedStateNum >= state | |
| 548 | + | }) | |
| 549 | + | _, err := ports.UnmarshalAck(frame.Payload) | |
| 550 | + | require.NoError(t, err) | |
| 551 | + | } | |
| 552 | + | ||
| 553 | + | func requireAttachResult(t *testing.T, results <-chan attachResult) attachResult { | |
| 554 | + | t.Helper() | |
| 555 | + | select { | |
| 556 | + | case result := <-results: | |
| 557 | + | return result | |
| 558 | + | case <-time.After(time.Second): | |
| 559 | + | t.Fatal("timed out waiting for attach attempt result") | |
| 560 | + | return attachResult{} | |
| 561 | + | } | |
| 562 | + | } | |
| 563 | + | ||
| 564 | + | type armedReconnectToastWriter struct { | |
| 565 | + | next io.Writer | |
| 566 | + | needle string | |
| 567 | + | armed atomic.Bool | |
| 568 | + | failure error | |
| 569 | + | } | |
| 570 | + | ||
| 571 | + | func (w *armedReconnectToastWriter) Write(p []byte) (int, error) { | |
| 572 | + | if w.armed.Load() && strings.Contains(string(p), w.needle) { | |
| 573 | + | return 0, w.failure | |
| 574 | + | } | |
| 575 | + | return w.next.Write(p) | |
| 576 | + | } | |
| 577 | + | ||
| 578 | + | type reconnectRuntimeObserver struct { | |
| 579 | + | mu sync.Mutex | |
| 580 | + | marks []ports.RuntimeMark | |
| 581 | + | } | |
| 582 | + | ||
| 583 | + | func (o *reconnectRuntimeObserver) ObserveRuntime(mark ports.RuntimeMark) { | |
| 584 | + | o.mu.Lock() | |
| 585 | + | o.marks = append(o.marks, mark) | |
| 586 | + | o.mu.Unlock() | |
| 587 | + | } | |
| 588 | + | ||
| 589 | + | func (o *reconnectRuntimeObserver) Flush() {} | |
| 590 | + | func (o *reconnectRuntimeObserver) Close() {} | |
| 591 | + | ||
| 592 | + | func (o *reconnectRuntimeObserver) count() int { | |
| 593 | + | o.mu.Lock() | |
| 594 | + | defer o.mu.Unlock() | |
| 595 | + | return len(o.marks) | |
| 596 | + | } | |
| 597 | + | ||
| 598 | + | type reconnectFlushObserver struct { | |
| 599 | + | flushes *atomic.Int32 | |
| 600 | + | seen chan int32 | |
| 601 | + | } | |
| 602 | + | ||
| 603 | + | func (o *reconnectFlushObserver) ObserveRuntime(mark ports.RuntimeMark) { | |
| 604 | + | if mark.Kind != ports.RuntimeTerminalFlushed { | |
| 605 | + | return | |
| 606 | + | } | |
| 607 | + | select { | |
| 608 | + | case o.seen <- o.flushes.Load(): | |
| 609 | + | default: | |
| 610 | + | } | |
| 611 | + | } | |
| 612 | + | func (o *reconnectFlushObserver) Flush() {} | |
| 613 | + | func (o *reconnectFlushObserver) Close() {} | |
| 614 | + | ||
| 615 | + | type reconnectStatusSignalWriter struct { | |
| 616 | + | written chan struct{} | |
| 617 | + | once sync.Once | |
| 618 | + | } | |
| 619 | + | ||
| 620 | + | func (w *reconnectStatusSignalWriter) Write(p []byte) (int, error) { | |
| 621 | + | if strings.Contains(string(p), statusReconnect) { | |
| 622 | + | w.once.Do(func() { close(w.written) }) | |
| 623 | + | } | |
| 624 | + | return len(p), nil | |
| 625 | + | } | |
| 626 | + | ||
| 627 | + | func TestLocalReconnectStatusFlushesDaemonOutputBeforeObservationAndAck(t *testing.T) { | |
| 628 | + | var flushes atomic.Int32 | |
| 629 | + | writer := &reconnectStatusSignalWriter{written: make(chan struct{})} | |
| 630 | + | term := newReconnectToastTerminalHarnessWithOutputAndFlush(t, writer, func() error { | |
| 631 | + | flushes.Add(1) | |
| 632 | + | return nil | |
| 633 | + | }) | |
| 634 | + | defer term.closeInput() | |
| 635 | + | tr := newReconnectToastLinkTransport() | |
| 636 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |
| 637 | + | observer := &reconnectFlushObserver{flushes: &flushes, seen: make(chan int32, 1)} | |
| 638 | + | ||
| 639 | + | resultCh := make(chan attachResult, 1) | |
| 640 | + | ms := milestones{} | |
| 641 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main"}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |
| 642 | + | attempt.runner.runtimeObserver = observer | |
| 643 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |
| 644 | + | ||
| 645 | + | beforeStatus := flushes.Load() | |
| 646 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |
| 647 | + | select { | |
| 648 | + | case <-writer.written: | |
| 649 | + | case <-time.After(time.Second): | |
| 650 | + | t.Fatal("timed out waiting for local reconnect status write") | |
| 651 | + | } | |
| 652 | + | require.Eventually(t, func() bool { return flushes.Load() > beforeStatus }, time.Second, time.Millisecond) | |
| 653 | + | beforeOutput := flushes.Load() | |
| 654 | + | ||
| 655 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |
| 656 | + | BaseStateNum: 1, | |
| 657 | + | NewStateNum: 2, | |
| 658 | + | Data: []byte("daemon output behind local reconnect status"), | |
| 659 | + | })}} | |
| 660 | + | ||
| 661 | + | select { | |
| 662 | + | case observedAtFlush := <-observer.seen: | |
| 663 | + | require.Greater(t, observedAtFlush, beforeOutput, "terminal output must be flushed before publishing RuntimeTerminalFlushed") | |
| 664 | + | case <-time.After(time.Second): | |
| 665 | + | t.Fatal("timed out waiting for terminal-flushed observation") | |
| 666 | + | } | |
| 667 | + | requireAckedState(t, tr.sends, 2) | |
| 668 | + | require.Greater(t, flushes.Load(), beforeOutput, "daemon output must be flushed before ACK") | |
| 669 | + | ||
| 670 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | |
| 671 | + | require.NoError(t, requireAttachResult(t, resultCh).err) | |
| 672 | + | } | |
| 673 | + | ||
| 674 | + | func TestLocalReconnectStatusTransitionsDoNotRequestAuthoritativeReset(t *testing.T) { | |
| 675 | + | tests := []struct { | |
| 676 | + | name string | |
| 677 | + | transition ports.LinkState | |
| 678 | + | }{ | |
| 679 | + | {name: "dismissal", transition: ports.LinkStateConnected}, | |
| 680 | + | {name: "stage change", transition: ports.LinkStateDead}, | |
| 681 | + | } | |
| 682 | + | ||
| 683 | + | for _, tt := range tests { | |
| 684 | + | t.Run(tt.name, func(t *testing.T) { | |
| 685 | + | out := newReconnectToastOutputRecorder() | |
| 686 | + | term := newReconnectToastTerminalHarnessWithOutput(t, out) | |
| 687 | + | defer term.closeInput() | |
| 688 | + | tr := newReconnectToastLinkTransport() | |
| 689 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |
| 690 | + | ||
| 691 | + | resultCh := make(chan attachResult, 1) | |
| 692 | + | ms := milestones{} | |
| 693 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main"}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |
| 694 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |
| 695 | + | ||
| 696 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |
| 697 | + | require.Eventually(t, func() bool { return strings.Contains(out.String(), statusReconnect) }, time.Second, time.Millisecond) | |
| 698 | + | tr.events <- ports.LinkEvent{State: tt.transition} | |
| 699 | + | if tt.transition == ports.LinkStateConnected { | |
| 700 | + | require.Equal(t, ports.ClientNoticeLinkConnected, (<-tr.clientNotices).Action) | |
| 701 | + | } | |
| 702 | + | ||
| 703 | + | const output = "local status output remains visible" | |
| 704 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |
| 705 | + | BaseStateNum: 1, | |
| 706 | + | NewStateNum: 2, | |
| 707 | + | Data: []byte(output), | |
| 708 | + | })}} | |
| 709 | + | require.Eventually(t, func() bool { return strings.Contains(out.String(), output) }, time.Second, time.Millisecond) | |
| 710 | + | requireAckedState(t, tr.sends, 2) | |
| 711 | + | _, sentResize := tr.sends.find(func(frame ports.Frame) bool { return frame.Type == ports.MsgResize }) | |
| 712 | + | require.False(t, sentResize, "local reconnect status must not request a daemon reset") | |
| 713 | + | ||
| 714 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | |
| 715 | + | require.NoError(t, requireAttachResult(t, resultCh).err) | |
| 716 | + | }) | |
| 717 | + | } | |
| 718 | + | } | |
| 719 | + | ||
| 720 | + | func TestReconnectOverlayRedrawFailureDoesNotObserveOrAckOutput(t *testing.T) { | |
| 721 | + | tests := []struct { | |
| 722 | + | name string | |
| 723 | + | configure func(*testing.T, *reconnectToastOutputRecorder) (*reconnectToastTerminalHarness, func()) | |
| 724 | + | wantError string | |
| 725 | + | }{ | |
| 726 | + | { | |
| 727 | + | name: "write", | |
| 728 | + | configure: func(t *testing.T, out *reconnectToastOutputRecorder) (*reconnectToastTerminalHarness, func()) { | |
| 729 | + | failure := errors.New("redraw write failed") | |
| 730 | + | writer := &armedReconnectToastWriter{next: out, needle: "probing UDP path", failure: failure} | |
| 731 | + | return newReconnectToastTerminalHarnessWithOutput(t, writer), func() { writer.armed.Store(true) } | |
| 732 | + | }, | |
| 733 | + | wantError: "redraw write failed", | |
| 734 | + | }, | |
| 735 | + | { | |
| 736 | + | name: "flush", | |
| 737 | + | configure: func(t *testing.T, out *reconnectToastOutputRecorder) (*reconnectToastTerminalHarness, func()) { | |
| 738 | + | var armed atomic.Bool | |
| 739 | + | term := newReconnectToastTerminalHarnessWithOutputAndFlush(t, out, func() error { | |
| 740 | + | if armed.Load() { | |
| 741 | + | return errors.New("redraw flush failed") | |
| 742 | + | } | |
| 743 | + | return nil | |
| 744 | + | }) | |
| 745 | + | return term, func() { armed.Store(true) } | |
| 746 | + | }, | |
| 747 | + | wantError: "redraw flush failed", | |
| 748 | + | }, | |
| 749 | + | } | |
| 750 | + | ||
| 751 | + | for _, tt := range tests { | |
| 752 | + | t.Run(tt.name, func(t *testing.T) { | |
| 753 | + | out := newReconnectToastOutputRecorder() | |
| 754 | + | term, arm := tt.configure(t, out) | |
| 755 | + | defer term.closeInput() | |
| 756 | + | tr := newReconnectToastLinkTransport() | |
| 757 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |
| 758 | + | observer := &reconnectRuntimeObserver{} | |
| 759 | + | ||
| 760 | + | resultCh := make(chan attachResult, 1) | |
| 761 | + | ms := milestones{} | |
| 762 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |
| 763 | + | attempt.runner.runtimeObserver = observer | |
| 764 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |
| 765 | + | ||
| 766 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |
| 767 | + | requireReconnectToastOutput(t, out.completed) | |
| 768 | + | arm() | |
| 769 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |
| 770 | + | BaseStateNum: 1, | |
| 771 | + | NewStateNum: 2, | |
| 772 | + | Data: []byte("daemon output before failed redraw"), | |
| 773 | + | })}} | |
| 774 | + | ||
| 775 | + | result := requireAttachResult(t, resultCh) | |
| 776 | + | require.ErrorContains(t, result.err, tt.wantError) | |
| 777 | + | require.Zero(t, observer.count(), "failed terminal redraw must not publish a terminal-flushed mark") | |
| 778 | + | }) | |
| 779 | + | } | |
| 780 | + | } | |
| 781 | + | ||
| 293 | 782 | func TestReconnectLinkEventsNotifyDaemonWithoutLocalTerminalWrites(t *testing.T) { | |
| 294 | 783 | out := newReconnectToastOutputRecorder() | |
| 295 | 784 | term := newReconnectToastTerminalHarnessWithOutput(t, out) | |
| @@ -313,7 +802,7 @@ func TestReconnectLinkEventsNotifyDaemonWithoutLocalTerminalWrites(t *testing.T) | |||
| 313 | 802 | } | |
| 314 | 803 | ||
| 315 | 804 | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | |
| 316 | - | result := <-resultCh | |
| 805 | + | result := requireAttachResult(t, resultCh) | |
| 317 | 806 | require.NoError(t, result.err) | |
| 318 | 807 | require.True(t, result.welcomed) | |
| 319 | 808 | } | |
| @@ -330,12 +819,305 @@ func TestAttachAttemptOfflineLinkEventReturnsReconnectableError(t *testing.T) { | |||
| 330 | 819 | go func() { resultCh <- attempt.run(context.Background()) }() | |
| 331 | 820 | ||
| 332 | 821 | tr.events <- ports.LinkEvent{State: ports.LinkStateOffline} | |
| 333 | - | result := <-resultCh | |
| 822 | + | result := requireAttachResult(t, resultCh) | |
| 334 | 823 | require.ErrorIs(t, result.err, errLinkOffline) | |
| 335 | 824 | require.True(t, result.welcomed) | |
| 336 | 825 | require.True(t, shouldReconnect(result.err), "offline exit must be reconnectable") | |
| 337 | 826 | } | |
| 338 | 827 | ||
| 828 | + | type reconnectResetTimer struct { | |
| 829 | + | ch chan time.Time | |
| 830 | + | } | |
| 831 | + | ||
| 832 | + | func (t *reconnectResetTimer) C() <-chan time.Time { return t.ch } | |
| 833 | + | func (t *reconnectResetTimer) Reset(time.Duration) bool { return true } | |
| 834 | + | func (t *reconnectResetTimer) Stop() bool { return true } | |
| 835 | + | ||
| 836 | + | type reconnectResetClock struct { | |
| 837 | + | preWelcome chan *reconnectResetTimer | |
| 838 | + | } | |
| 839 | + | ||
| 840 | + | func newReconnectResetClock() *reconnectResetClock { | |
| 841 | + | return &reconnectResetClock{preWelcome: make(chan *reconnectResetTimer, 4)} | |
| 842 | + | } | |
| 843 | + | ||
| 844 | + | func (c *reconnectResetClock) Now() time.Time { return time.Time{} } | |
| 845 | + | ||
| 846 | + | func (c *reconnectResetClock) NewTimer(d time.Duration) ports.Timer { | |
| 847 | + | timer := &reconnectResetTimer{ch: make(chan time.Time, 1)} | |
| 848 | + | if d == preWelcomeTimeout { | |
| 849 | + | c.preWelcome <- timer | |
| 850 | + | } | |
| 851 | + | return timer | |
| 852 | + | } | |
| 853 | + | ||
| 854 | + | type reconnectResetBlockingTransport struct { | |
| 855 | + | mu sync.Mutex | |
| 856 | + | recvs []reconnectToastRecv | |
| 857 | + | entered chan struct{} | |
| 858 | + | closed chan struct{} | |
| 859 | + | enterOnce sync.Once | |
| 860 | + | closeOnce sync.Once | |
| 861 | + | } | |
| 862 | + | ||
| 863 | + | func newReconnectResetBlockingTransport() *reconnectResetBlockingTransport { | |
| 864 | + | return &reconnectResetBlockingTransport{ | |
| 865 | + | recvs: []reconnectToastRecv{{frame: reconnectToastWelcomeNamed("assigned", 44)}}, | |
| 866 | + | entered: make(chan struct{}), | |
| 867 | + | closed: make(chan struct{}), | |
| 868 | + | } | |
| 869 | + | } | |
| 870 | + | ||
| 871 | + | func (t *reconnectResetBlockingTransport) Send(frame ports.Frame) error { | |
| 872 | + | if frame.Type != ports.MsgResize { | |
| 873 | + | return nil | |
| 874 | + | } | |
| 875 | + | t.enterOnce.Do(func() { close(t.entered) }) | |
| 876 | + | <-t.closed | |
| 877 | + | return io.ErrClosedPipe | |
| 878 | + | } | |
| 879 | + | ||
| 880 | + | func (t *reconnectResetBlockingTransport) Recv() (ports.Frame, error) { | |
| 881 | + | t.mu.Lock() | |
| 882 | + | defer t.mu.Unlock() | |
| 883 | + | if len(t.recvs) == 0 { | |
| 884 | + | return ports.Frame{}, io.EOF | |
| 885 | + | } | |
| 886 | + | recv := t.recvs[0] | |
| 887 | + | t.recvs = t.recvs[1:] | |
| 888 | + | return recv.frame, recv.err | |
| 889 | + | } | |
| 890 | + | ||
| 891 | + | func (t *reconnectResetBlockingTransport) Close() error { | |
| 892 | + | t.closeOnce.Do(func() { close(t.closed) }) | |
| 893 | + | return nil | |
| 894 | + | } | |
| 895 | + | ||
| 896 | + | func TestPreSenderReconnectResetIsBounded(t *testing.T) { | |
| 897 | + | tests := []struct { | |
| 898 | + | name string | |
| 899 | + | trigger func(context.CancelFunc, *reconnectResetTimer) | |
| 900 | + | wantErr error | |
| 901 | + | }{ | |
| 902 | + | { | |
| 903 | + | name: "timeout", | |
| 904 | + | trigger: func(_ context.CancelFunc, timer *reconnectResetTimer) { | |
| 905 | + | timer.ch <- time.Time{} | |
| 906 | + | }, | |
| 907 | + | wantErr: context.DeadlineExceeded, | |
| 908 | + | }, | |
| 909 | + | { | |
| 910 | + | name: "cancellation", | |
| 911 | + | trigger: func(cancel context.CancelFunc, _ *reconnectResetTimer) { | |
| 912 | + | cancel() | |
| 913 | + | }, | |
| 914 | + | wantErr: context.Canceled, | |
| 915 | + | }, | |
| 916 | + | } | |
| 917 | + | ||
| 918 | + | for _, tt := range tests { | |
| 919 | + | t.Run(tt.name, func(t *testing.T) { | |
| 920 | + | term := newReconnectToastTerminalHarness(t) | |
| 921 | + | defer term.closeInput() | |
| 922 | + | tr := newReconnectResetBlockingTransport() | |
| 923 | + | clock := newReconnectResetClock() | |
| 924 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 925 | + | defer cancel() | |
| 926 | + | ms := milestones{} | |
| 927 | + | attempt := newReconnectAttachAttempt(term.term, tr, clock, AttachRequest{Intent: ports.IntentResume, SessionName: "requested", Remote: true}, 12, &terminalThemeState{}, nil, &ms) | |
| 928 | + | attempt.reconnect.showing = true | |
| 929 | + | resultCh := make(chan attachResult, 1) | |
| 930 | + | go func() { resultCh <- attempt.run(ctx) }() | |
| 931 | + | ||
| 932 | + | <-clock.preWelcome // Hello Send | |
| 933 | + | <-clock.preWelcome // Welcome Recv | |
| 934 | + | resetTimer := <-clock.preWelcome | |
| 935 | + | select { | |
| 936 | + | case <-tr.entered: | |
| 937 | + | case <-time.After(time.Second): | |
| 938 | + | t.Fatal("timed out waiting for synchronous reconnect reset Send") | |
| 939 | + | } | |
| 940 | + | tt.trigger(cancel, resetTimer) | |
| 941 | + | ||
| 942 | + | result := requireAttachResult(t, resultCh) | |
| 943 | + | require.ErrorIs(t, result.err, tt.wantErr) | |
| 944 | + | require.True(t, result.transportClosed) | |
| 945 | + | require.True(t, result.welcomed) | |
| 946 | + | require.Equal(t, uint64(44), result.resumeToken) | |
| 947 | + | require.Equal(t, "assigned", result.sessionName) | |
| 948 | + | }) | |
| 949 | + | } | |
| 950 | + | } | |
| 951 | + | ||
| 952 | + | type reconnectToastBlockingSendTransport struct { | |
| 953 | + | *reconnectToastLinkTransport | |
| 954 | + | armed atomic.Bool | |
| 955 | + | blockType ports.MsgType | |
| 956 | + | sendErr error | |
| 957 | + | entered chan struct{} | |
| 958 | + | release chan struct{} | |
| 959 | + | enteredOnce sync.Once | |
| 960 | + | releaseOnce sync.Once | |
| 961 | + | } | |
| 962 | + | ||
| 963 | + | func newReconnectToastBlockingSendTransport() *reconnectToastBlockingSendTransport { | |
| 964 | + | return &reconnectToastBlockingSendTransport{ | |
| 965 | + | reconnectToastLinkTransport: newReconnectToastLinkTransport(), | |
| 966 | + | blockType: ports.MsgClientNotice, | |
| 967 | + | entered: make(chan struct{}), | |
| 968 | + | release: make(chan struct{}), | |
| 969 | + | } | |
| 970 | + | } | |
| 971 | + | ||
| 972 | + | func (t *reconnectToastBlockingSendTransport) Send(frame ports.Frame) error { | |
| 973 | + | if err := t.reconnectToastLinkTransport.Send(frame); err != nil { | |
| 974 | + | return err | |
| 975 | + | } | |
| 976 | + | if t.armed.Load() && frame.Type == t.blockType { | |
| 977 | + | t.enteredOnce.Do(func() { close(t.entered) }) | |
| 978 | + | <-t.release | |
| 979 | + | return t.sendErr | |
| 980 | + | } | |
| 981 | + | return nil | |
| 982 | + | } | |
| 983 | + | ||
| 984 | + | func requireTerminalSizeCalls(t *testing.T, term *reconnectToastTerminalHarness, want int32) { | |
| 985 | + | t.Helper() | |
| 986 | + | timer := time.NewTimer(time.Second) | |
| 987 | + | defer timer.Stop() | |
| 988 | + | for term.sizeCalls.Load() < want { | |
| 989 | + | select { | |
| 990 | + | case <-term.sizeCalled: | |
| 991 | + | case <-timer.C: | |
| 992 | + | t.Fatalf("timed out waiting for terminal Size call %d; got %d", want, term.sizeCalls.Load()) | |
| 993 | + | } | |
| 994 | + | } | |
| 995 | + | } | |
| 996 | + | ||
| 997 | + | func TestReconnectResetEnqueueCancellationExitsCleanly(t *testing.T) { | |
| 998 | + | tests := []struct { | |
| 999 | + | name string | |
| 1000 | + | transition ports.LinkState | |
| 1001 | + | }{ | |
| 1002 | + | {name: "dismissal", transition: ports.LinkStateConnected}, | |
| 1003 | + | {name: "stage change", transition: ports.LinkStateDead}, | |
| 1004 | + | } | |
| 1005 | + | ||
| 1006 | + | for _, tt := range tests { | |
| 1007 | + | t.Run(tt.name, func(t *testing.T) { | |
| 1008 | + | term := newReconnectToastTerminalHarness(t) | |
| 1009 | + | defer term.closeInput() | |
| 1010 | + | tr := newReconnectToastBlockingSendTransport() | |
| 1011 | + | releaseSender := func() { tr.releaseOnce.Do(func() { close(tr.release) }) } | |
| 1012 | + | defer releaseSender() | |
| 1013 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |
| 1014 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 1015 | + | defer cancel() | |
| 1016 | + | ||
| 1017 | + | resultCh := make(chan attachResult, 1) | |
| 1018 | + | ms := milestones{} | |
| 1019 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |
| 1020 | + | go func() { resultCh <- attempt.run(ctx) }() | |
| 1021 | + | ||
| 1022 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |
| 1023 | + | requireTerminalSizeCalls(t, term, 2) | |
| 1024 | + | tr.blockType = ports.MsgResize | |
| 1025 | + | tr.armed.Store(true) | |
| 1026 | + | term.resizeCh <- term.size | |
| 1027 | + | select { | |
| 1028 | + | case <-tr.entered: | |
| 1029 | + | case <-time.After(time.Second): | |
| 1030 | + | t.Fatal("timed out waiting for sender to block") | |
| 1031 | + | } | |
| 1032 | + | for range sendQueueDepth { | |
| 1033 | + | term.resizeCh <- term.size | |
| 1034 | + | } | |
| 1035 | + | require.Eventually(t, func() bool { return len(term.resizeCh) == 0 }, time.Second, time.Millisecond) | |
| 1036 | + | ||
| 1037 | + | tr.events <- ports.LinkEvent{State: tt.transition} | |
| 1038 | + | requireTerminalSizeCalls(t, term, 3) | |
| 1039 | + | cancel() | |
| 1040 | + | ||
| 1041 | + | result := requireAttachResult(t, resultCh) | |
| 1042 | + | require.NoError(t, result.err) | |
| 1043 | + | require.True(t, result.welcomed) | |
| 1044 | + | releaseSender() | |
| 1045 | + | }) | |
| 1046 | + | } | |
| 1047 | + | } | |
| 1048 | + | ||
| 1049 | + | func TestReconnectResetCancellationPreservesQueuedSenderError(t *testing.T) { | |
| 1050 | + | term := newReconnectToastTerminalHarness(t) | |
| 1051 | + | defer term.closeInput() | |
| 1052 | + | tr := newReconnectToastBlockingSendTransport() | |
| 1053 | + | releaseSender := func() { tr.releaseOnce.Do(func() { close(tr.release) }) } | |
| 1054 | + | defer releaseSender() | |
| 1055 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |
| 1056 | + | ||
| 1057 | + | resultCh := make(chan attachResult, 1) | |
| 1058 | + | ms := milestones{} | |
| 1059 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |
| 1060 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |
| 1061 | + | ||
| 1062 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |
| 1063 | + | requireTerminalSizeCalls(t, term, 2) | |
| 1064 | + | tr.blockType = ports.MsgResize | |
| 1065 | + | tr.armed.Store(true) | |
| 1066 | + | term.resizeCh <- term.size | |
| 1067 | + | select { | |
| 1068 | + | case <-tr.entered: | |
| 1069 | + | case <-time.After(time.Second): | |
| 1070 | + | t.Fatal("timed out waiting for sender to block") | |
| 1071 | + | } | |
| 1072 | + | for range sendQueueDepth { | |
| 1073 | + | term.resizeCh <- term.size | |
| 1074 | + | } | |
| 1075 | + | require.Eventually(t, func() bool { return len(term.resizeCh) == 0 }, time.Second, time.Millisecond) | |
| 1076 | + | ||
| 1077 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateConnected} | |
| 1078 | + | requireTerminalSizeCalls(t, term, 3) | |
| 1079 | + | sendFailure := errors.New("queued sender failure") | |
| 1080 | + | tr.sendErr = sendFailure | |
| 1081 | + | releaseSender() | |
| 1082 | + | ||
| 1083 | + | result := requireAttachResult(t, resultCh) | |
| 1084 | + | require.ErrorIs(t, result.err, sendFailure) | |
| 1085 | + | require.ErrorContains(t, result.err, "sending to daemon") | |
| 1086 | + | } | |
| 1087 | + | ||
| 1088 | + | func TestAttachAttemptReturnsWhileSenderSendIsBlockedAfterCancellation(t *testing.T) { | |
| 1089 | + | term := newReconnectToastTerminalHarness(t) | |
| 1090 | + | defer term.closeInput() | |
| 1091 | + | tr := newReconnectToastBlockingSendTransport() | |
| 1092 | + | releaseSender := func() { tr.releaseOnce.Do(func() { close(tr.release) }) } | |
| 1093 | + | defer releaseSender() | |
| 1094 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |
| 1095 | + | ||
| 1096 | + | resultCh := make(chan attachResult, 1) | |
| 1097 | + | ms := milestones{} | |
| 1098 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |
| 1099 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |
| 1100 | + | ||
| 1101 | + | tr.armed.Store(true) | |
| 1102 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateDegraded} | |
| 1103 | + | select { | |
| 1104 | + | case <-tr.entered: | |
| 1105 | + | case <-time.After(time.Second): | |
| 1106 | + | t.Fatal("timed out waiting for sender to block in transport Send") | |
| 1107 | + | } | |
| 1108 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateOffline} | |
| 1109 | + | ||
| 1110 | + | select { | |
| 1111 | + | case result := <-resultCh: | |
| 1112 | + | require.ErrorIs(t, result.err, errLinkOffline) | |
| 1113 | + | case <-time.After(time.Second): | |
| 1114 | + | releaseSender() | |
| 1115 | + | requireAttachResult(t, resultCh) | |
| 1116 | + | t.Fatal("attach attempt waited for the blocked sender before returning control to Runner") | |
| 1117 | + | } | |
| 1118 | + | releaseSender() | |
| 1119 | + | } | |
| 1120 | + | ||
| 339 | 1121 | func TestDetachedResultReplacementAndCleanDetach(t *testing.T) { | |
| 340 | 1122 | for _, tt := range []struct { | |
| 341 | 1123 | name string | |
| @@ -363,18 +1145,12 @@ func TestReconnectToastBoundsUseCenterAnchor(t *testing.T) { | |||
| 363 | 1145 | require.Equal(t, domain.Rect{X: 29, Y: 10, Width: 22, Height: 3}, bounds) | |
| 364 | 1146 | } | |
| 365 | 1147 | ||
| 366 | - | func TestReconnectToastDrawAndClearHelpers(t *testing.T) { | |
| 1148 | + | func TestReconnectToastDrawHelper(t *testing.T) { | |
| 367 | 1149 | var out bytes.Buffer | |
| 368 | - | size := domain.Size{Cols: 80, Rows: 24} | |
| 369 | - | ||
| 370 | - | require.NoError(t, drawReconnectToast(&out, size)) | |
| 1150 | + | require.NoError(t, drawReconnectToast(&out, domain.Size{Cols: 80, Rows: 24})) | |
| 371 | 1151 | require.Contains(t, out.String(), "┌") | |
| 372 | 1152 | require.Contains(t, out.String(), reconnectToastMessage) | |
| 373 | 1153 | require.Contains(t, out.String(), "\x1b[0m") | |
| 374 | - | ||
| 375 | - | bounds := reconnectToastBounds(size) | |
| 376 | - | require.NoError(t, clearReconnectToast(&out, bounds)) | |
| 377 | - | require.Contains(t, out.String(), strings.Repeat(" ", bounds.Width)) | |
| 378 | 1154 | } | |
| 379 | 1155 | ||
| 380 | 1156 | func TestReconnectToastLinesClampToBounds(t *testing.T) { | |
| @@ -409,61 +1185,17 @@ func legacyReconnectToastStage(out io.Writer, size domain.Size, stage reconnectS | |||
| 409 | 1185 | return bounds, writeReconnectToast(out, bounds, reconnectToastLinesFor(bounds, message)) | |
| 410 | 1186 | } | |
| 411 | 1187 | ||
| 412 | - | func TestReconnectToastStageTransitionsClearDrawnBounds(t *testing.T) { | |
| 413 | - | size := domain.Size{Cols: 80, Rows: 24} | |
| 414 | - | tests := []struct { | |
| 415 | - | name string | |
| 416 | - | stages []reconnectStage | |
| 417 | - | }{ | |
| 418 | - | { | |
| 419 | - | name: "degraded to probing to ssh to offline", | |
| 420 | - | stages: []reconnectStage{reconnectStageDegraded, reconnectStageProbingUDP, reconnectStageSSH, reconnectStageOfflineRetrying}, | |
| 421 | - | }, | |
| 422 | - | { | |
| 423 | - | name: "widest ssh to offline", | |
| 424 | - | stages: []reconnectStage{reconnectStageSSH, reconnectStageOfflineRetrying}, | |
| 425 | - | }, | |
| 426 | - | } | |
| 427 | - | ||
| 428 | - | for _, tt := range tests { | |
| 429 | - | t.Run(tt.name, func(t *testing.T) { | |
| 1188 | + | func TestReconnectToastStagesDrawTheirCurrentMessage(t *testing.T) { | |
| 1189 | + | for _, stage := range []reconnectStage{reconnectStageDegraded, reconnectStageProbingUDP, reconnectStageSSH, reconnectStageOfflineRetrying} { | |
| 1190 | + | t.Run(reconnectStageMessage(stage), func(t *testing.T) { | |
| 430 | 1191 | var out bytes.Buffer | |
| 431 | - | previousBounds, err := drawReconnectToastStage(&out, size, tt.stages[0]) | |
| 1192 | + | _, err := drawReconnectToastStage(&out, domain.Size{Cols: 80, Rows: 24}, stage) | |
| 432 | 1193 | require.NoError(t, err) | |
| 433 | - | ||
| 434 | - | for _, stage := range tt.stages[1:] { | |
| 435 | - | out.Reset() | |
| 436 | - | require.NoError(t, clearReconnectToast(&out, previousBounds)) | |
| 437 | - | assertReconnectToastClearCoversBounds(t, out.String(), previousBounds) | |
| 438 | - | ||
| 439 | - | out.Reset() | |
| 440 | - | previousBounds, err = drawReconnectToastStage(&out, size, stage) | |
| 441 | - | require.NoError(t, err) | |
| 442 | - | } | |
| 1194 | + | require.Contains(t, out.String(), reconnectStageMessage(stage)) | |
| 443 | 1195 | }) | |
| 444 | 1196 | } | |
| 445 | 1197 | } | |
| 446 | 1198 | ||
| 447 | - | func TestReconnectToastClearAfterWidestStageErasesFullBorder(t *testing.T) { | |
| 448 | - | size := domain.Size{Cols: 80, Rows: 24} | |
| 449 | - | var out bytes.Buffer | |
| 450 | - | bounds, err := drawReconnectToastStage(&out, size, reconnectStageSSH) | |
| 451 | - | require.NoError(t, err) | |
| 452 | - | require.Greater(t, bounds.Width, reconnectToastBounds(size).Width) | |
| 453 | - | ||
| 454 | - | out.Reset() | |
| 455 | - | require.NoError(t, clearReconnectToast(&out, bounds)) | |
| 456 | - | assertReconnectToastClearCoversBounds(t, out.String(), bounds) | |
| 457 | - | } | |
| 458 | - | ||
| 459 | - | func assertReconnectToastClearCoversBounds(t *testing.T, output string, bounds domain.Rect) { | |
| 460 | - | t.Helper() | |
| 461 | - | blank := strings.Repeat(" ", bounds.Width) | |
| 462 | - | for row := range bounds.Height { | |
| 463 | - | require.Contains(t, output, fmt.Sprintf("\x1b[%d;%dH\x1b[0m%s", bounds.Y+row+1, bounds.X+1, blank)) | |
| 464 | - | } | |
| 465 | - | } | |
| 466 | - | ||
| 467 | 1199 | func displayWidth(s string) int { | |
| 468 | 1200 | width := 0 | |
| 469 | 1201 | for _, r := range s { | |
| @@ -483,18 +1215,44 @@ func TestReconnectSleepWithResizeEventsRedrawsUntilTimerFires(t *testing.T) { | |||
| 483 | 1215 | resizeCh <- domain.Size{Cols: 100, Rows: 30} | |
| 484 | 1216 | resizeCh <- domain.Size{Cols: 120, Rows: 40} | |
| 485 | 1217 | got := make(chan domain.Size, 2) | |
| 486 | - | done := make(chan bool, 1) | |
| 1218 | + | type sleepResult struct { | |
| 1219 | + | slept bool | |
| 1220 | + | err error | |
| 1221 | + | } | |
| 1222 | + | done := make(chan sleepResult, 1) | |
| 487 | 1223 | ||
| 488 | 1224 | go func() { | |
| 489 | - | done <- sleepReconnectWithResizeEvents(context.Background(), clk, time.Hour, resizeCh, func(size domain.Size) { | |
| 1225 | + | slept, err := sleepReconnectWithResizeEvents(context.Background(), clk, time.Hour, resizeCh, func(size domain.Size) error { | |
| 490 | 1226 | got <- size | |
| 1227 | + | return nil | |
| 491 | 1228 | }) | |
| 1229 | + | done <- sleepResult{slept: slept, err: err} | |
| 492 | 1230 | }() | |
| 493 | 1231 | ||
| 494 | 1232 | require.Equal(t, domain.Size{Cols: 100, Rows: 30}, <-got) | |
| 495 | 1233 | require.Equal(t, domain.Size{Cols: 120, Rows: 40}, <-got) | |
| 496 | 1234 | timerCh <- time.Time{} | |
| 497 | - | require.True(t, <-done) | |
| 1235 | + | result := <-done | |
| 1236 | + | require.NoError(t, result.err) | |
| 1237 | + | require.True(t, result.slept) | |
| 1238 | + | } | |
| 1239 | + | ||
| 1240 | + | func TestReconnectSleepWithResizeEventsReturnsRedrawError(t *testing.T) { | |
| 1241 | + | clk := portsmocks.NewMockClock(t) | |
| 1242 | + | timer := portsmocks.NewMockTimer(t) | |
| 1243 | + | clk.EXPECT().NewTimer(time.Hour).Return(timer).Once() | |
| 1244 | + | timer.EXPECT().C().Return((<-chan time.Time)(make(chan time.Time))).Maybe() | |
| 1245 | + | timer.EXPECT().Stop().Return(true).Once() | |
| 1246 | + | resizeCh := make(chan domain.Size, 1) | |
| 1247 | + | resizeCh <- domain.Size{Cols: 100, Rows: 30} | |
| 1248 | + | redrawErr := errors.New("redraw failed") | |
| 1249 | + | ||
| 1250 | + | slept, err := sleepReconnectWithResizeEvents(context.Background(), clk, time.Hour, resizeCh, func(domain.Size) error { | |
| 1251 | + | return redrawErr | |
| 1252 | + | }) | |
| 1253 | + | ||
| 1254 | + | require.False(t, slept) | |
| 1255 | + | require.ErrorIs(t, err, redrawErr) | |
| 498 | 1256 | } | |
| 499 | 1257 | ||
| 500 | 1258 | type reconnectToastFailOnceWriter struct { | |
| @@ -512,14 +1270,120 @@ func (w *reconnectToastFailOnceWriter) Write(p []byte) (int, error) { | |||
| 512 | 1270 | return n, nil | |
| 513 | 1271 | } | |
| 514 | 1272 | ||
| 515 | - | func TestRemoteReconnectToastClearsFailedDrawBounds(t *testing.T) { | |
| 1273 | + | func TestReconnectRetryFailuresPreserveAttachError(t *testing.T) { | |
| 1274 | + | attachErr := errors.New("original attach failure") | |
| 1275 | + | retryErr := errors.New("retry presentation failure") | |
| 1276 | + | tests := []struct { | |
| 1277 | + | name string | |
| 1278 | + | output func() io.Writer | |
| 1279 | + | retrySleep func(func(domain.Size) error) (bool, error) | |
| 1280 | + | }{ | |
| 1281 | + | { | |
| 1282 | + | name: "draw stage", | |
| 1283 | + | output: func() io.Writer { | |
| 1284 | + | writer := &armedReconnectToastWriter{next: io.Discard, needle: reconnectStageMessage(reconnectStageSSH), failure: retryErr} | |
| 1285 | + | writer.armed.Store(true) | |
| 1286 | + | return writer | |
| 1287 | + | }, | |
| 1288 | + | retrySleep: func(func(domain.Size) error) (bool, error) { return true, nil }, | |
| 1289 | + | }, | |
| 1290 | + | { | |
| 1291 | + | name: "sleep", | |
| 1292 | + | output: func() io.Writer { return io.Discard }, | |
| 1293 | + | retrySleep: func(func(domain.Size) error) (bool, error) { | |
| 1294 | + | return false, retryErr | |
| 1295 | + | }, | |
| 1296 | + | }, | |
| 1297 | + | } | |
| 1298 | + | ||
| 1299 | + | for _, tt := range tests { | |
| 1300 | + | t.Run(tt.name, func(t *testing.T) { | |
| 1301 | + | oldSleepWithResize := reconnectSleepWithResize | |
| 1302 | + | reconnectSleepWithResize = func(_ context.Context, _ ports.Clock, _ time.Duration, _ <-chan domain.Size, redraw func(domain.Size) error) (bool, error) { | |
| 1303 | + | return tt.retrySleep(redraw) | |
| 1304 | + | } | |
| 1305 | + | defer func() { reconnectSleepWithResize = oldSleepWithResize }() | |
| 1306 | + | ||
| 1307 | + | term := newReconnectToastTerminalHarnessWithOutput(t, tt.output()) | |
| 1308 | + | defer term.closeInput() | |
| 1309 | + | tr := newReconnectToastRecordingTransport( | |
| 1310 | + | reconnectToastRecv{frame: reconnectToastWelcome(44)}, | |
| 1311 | + | reconnectToastRecv{err: attachErr}, | |
| 1312 | + | ) | |
| 1313 | + | dialer := &reconnectToastSequenceDialer{transports: []ports.Transport{tr}} | |
| 1314 | + | ||
| 1315 | + | err := NewRunner(Dependencies{Dialer: dialer, Terminal: term.term, Clock: newReconnectHandshakeClock(t), Logger: slog.New(slog.DiscardHandler)}).Run(context.Background(), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}) | |
| 1316 | + | require.ErrorIs(t, err, attachErr) | |
| 1317 | + | require.ErrorIs(t, err, retryErr) | |
| 1318 | + | }) | |
| 1319 | + | } | |
| 1320 | + | } | |
| 1321 | + | ||
| 1322 | + | func TestReconnectRetrySleepFailuresPreserveDialError(t *testing.T) { | |
| 1323 | + | dialErr := errors.New("original reconnect dial failure") | |
| 1324 | + | retryErr := errors.New("retry presentation failure") | |
| 1325 | + | tests := []struct { | |
| 1326 | + | name string | |
| 1327 | + | failSecond func(func(domain.Size) error) (bool, error) | |
| 1328 | + | }{ | |
| 1329 | + | { | |
| 1330 | + | name: "resize redraw", | |
| 1331 | + | failSecond: func(redraw func(domain.Size) error) (bool, error) { | |
| 1332 | + | return false, redraw(domain.Size{Cols: 100, Rows: 30}) | |
| 1333 | + | }, | |
| 1334 | + | }, | |
| 1335 | + | { | |
| 1336 | + | name: "sleep", | |
| 1337 | + | failSecond: func(func(domain.Size) error) (bool, error) { | |
| 1338 | + | return false, retryErr | |
| 1339 | + | }, | |
| 1340 | + | }, | |
| 1341 | + | } | |
| 1342 | + | ||
| 1343 | + | for _, tt := range tests { | |
| 1344 | + | t.Run(tt.name, func(t *testing.T) { | |
| 1345 | + | oldSleepWithResize := reconnectSleepWithResize | |
| 1346 | + | calls := 0 | |
| 1347 | + | reconnectSleepWithResize = func(_ context.Context, _ ports.Clock, _ time.Duration, _ <-chan domain.Size, redraw func(domain.Size) error) (bool, error) { | |
| 1348 | + | calls++ | |
| 1349 | + | if calls == 1 { | |
| 1350 | + | return true, nil | |
| 1351 | + | } | |
| 1352 | + | return tt.failSecond(redraw) | |
| 1353 | + | } | |
| 1354 | + | defer func() { reconnectSleepWithResize = oldSleepWithResize }() | |
| 1355 | + | ||
| 1356 | + | var armed atomic.Bool | |
| 1357 | + | term := newReconnectToastTerminalHarnessWithOutputAndFlush(t, io.Discard, func() error { | |
| 1358 | + | if armed.Load() { | |
| 1359 | + | return retryErr | |
| 1360 | + | } | |
| 1361 | + | return nil | |
| 1362 | + | }) | |
| 1363 | + | defer term.closeInput() | |
| 1364 | + | tr := newReconnectToastRecordingTransport( | |
| 1365 | + | reconnectToastRecv{frame: reconnectToastWelcome(44)}, | |
| 1366 | + | reconnectToastRecv{err: io.EOF}, | |
| 1367 | + | ) | |
| 1368 | + | dialer := portsmocks.NewMockDialer(t) | |
| 1369 | + | dialer.EXPECT().Dial(mock.Anything).Return(tr, nil).Once() | |
| 1370 | + | dialer.EXPECT().Dial(mock.Anything).Run(func(context.Context) { armed.Store(true) }).Return(nil, dialErr).Once() | |
| 1371 | + | ||
| 1372 | + | err := NewRunner(Dependencies{Dialer: dialer, Terminal: term.term, Clock: newReconnectHandshakeClock(t), Logger: slog.New(slog.DiscardHandler)}).Run(context.Background(), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}) | |
| 1373 | + | require.ErrorIs(t, err, dialErr) | |
| 1374 | + | require.ErrorIs(t, err, retryErr) | |
| 1375 | + | }) | |
| 1376 | + | } | |
| 1377 | + | } | |
| 1378 | + | ||
| 1379 | + | func TestRemoteReconnectToastFailedDrawDoesNotBlankBounds(t *testing.T) { | |
| 516 | 1380 | oldSleep := reconnectSleep | |
| 517 | 1381 | oldSleepWithResize := reconnectSleepWithResize | |
| 518 | 1382 | ctx, cancel := context.WithCancel(context.Background()) | |
| 519 | 1383 | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return false } | |
| 520 | - | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size)) bool { | |
| 1384 | + | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size) error) (bool, error) { | |
| 521 | 1385 | cancel() | |
| 522 | - | return false | |
| 1386 | + | return false, nil | |
| 523 | 1387 | } | |
| 524 | 1388 | defer func() { | |
| 525 | 1389 | reconnectSleep = oldSleep | |
| @@ -536,9 +1400,87 @@ func TestRemoteReconnectToastClearsFailedDrawBounds(t *testing.T) { | |||
| 536 | 1400 | dialer := &reconnectToastSequenceDialer{transports: []ports.Transport{tr}} | |
| 537 | 1401 | ||
| 538 | 1402 | err := NewRunner(Dependencies{Dialer: dialer, Terminal: term.term, Clock: newReconnectHandshakeClock(t), Logger: slog.New(slog.DiscardHandler)}).Run(ctx, AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}) | |
| 539 | - | require.ErrorIs(t, err, context.Canceled) | |
| 1403 | + | require.ErrorContains(t, err, "reconnect toast write failed") | |
| 540 | 1404 | require.True(t, out.failed) | |
| 541 | - | assertReconnectToastClearCoversBounds(t, out.String(), reconnectToastBoundsFor(term.size, reconnectStageMessage(reconnectStageSSH))) | |
| 1405 | + | require.NotContains(t, out.String(), strings.Repeat(" ", reconnectToastBoundsFor(term.size, reconnectStageMessage(reconnectStageSSH)).Width)) | |
| 1406 | + | } | |
| 1407 | + | ||
| 1408 | + | type reconnectStatusClearFailWriter struct { | |
| 1409 | + | clearErr error | |
| 1410 | + | } | |
| 1411 | + | ||
| 1412 | + | func (w *reconnectStatusClearFailWriter) Write(p []byte) (int, error) { | |
| 1413 | + | if string(p) == statusClear { | |
| 1414 | + | return 0, w.clearErr | |
| 1415 | + | } | |
| 1416 | + | return len(p), nil | |
| 1417 | + | } | |
| 1418 | + | ||
| 1419 | + | func TestReconnectCancellationPreservesContextWhenStatusClearFails(t *testing.T) { | |
| 1420 | + | tests := []struct { | |
| 1421 | + | name string | |
| 1422 | + | configure func(*portsmocks.MockDialer, ports.Transport) | |
| 1423 | + | sleep func(context.CancelFunc) bool | |
| 1424 | + | }{ | |
| 1425 | + | { | |
| 1426 | + | name: "after attach failure", | |
| 1427 | + | configure: func(dialer *portsmocks.MockDialer, tr ports.Transport) { | |
| 1428 | + | dialer.EXPECT().Dial(mock.Anything).Return(tr, nil).Once() | |
| 1429 | + | }, | |
| 1430 | + | sleep: func(cancel context.CancelFunc) bool { | |
| 1431 | + | cancel() | |
| 1432 | + | return false | |
| 1433 | + | }, | |
| 1434 | + | }, | |
| 1435 | + | { | |
| 1436 | + | name: "after reconnect dial failure", | |
| 1437 | + | configure: func(dialer *portsmocks.MockDialer, tr ports.Transport) { | |
| 1438 | + | dialer.EXPECT().Dial(mock.Anything).Return(tr, nil).Once() | |
| 1439 | + | dialer.EXPECT().Dial(mock.Anything).Return(nil, errors.New("redial failed")).Once() | |
| 1440 | + | }, | |
| 1441 | + | sleep: func() func(context.CancelFunc) bool { | |
| 1442 | + | calls := 0 | |
| 1443 | + | return func(cancel context.CancelFunc) bool { | |
| 1444 | + | calls++ | |
| 1445 | + | if calls == 1 { | |
| 1446 | + | return true | |
| 1447 | + | } | |
| 1448 | + | cancel() | |
| 1449 | + | return false | |
| 1450 | + | } | |
| 1451 | + | }(), | |
| 1452 | + | }, | |
| 1453 | + | } | |
| 1454 | + | ||
| 1455 | + | for _, tt := range tests { | |
| 1456 | + | t.Run(tt.name, func(t *testing.T) { | |
| 1457 | + | oldSleep := reconnectSleep | |
| 1458 | + | oldSleepWithResize := reconnectSleepWithResize | |
| 1459 | + | ctx, cancel := context.WithCancel(context.Background()) | |
| 1460 | + | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return tt.sleep(cancel) } | |
| 1461 | + | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size) error) (bool, error) { | |
| 1462 | + | return tt.sleep(cancel), nil | |
| 1463 | + | } | |
| 1464 | + | defer func() { | |
| 1465 | + | reconnectSleep = oldSleep | |
| 1466 | + | reconnectSleepWithResize = oldSleepWithResize | |
| 1467 | + | }() | |
| 1468 | + | ||
| 1469 | + | clearErr := errors.New("status clear failed") | |
| 1470 | + | term := newReconnectToastTerminalHarnessWithOutput(t, &reconnectStatusClearFailWriter{clearErr: clearErr}) | |
| 1471 | + | defer term.closeInput() | |
| 1472 | + | tr := newReconnectToastRecordingTransport( | |
| 1473 | + | reconnectToastRecv{frame: reconnectToastWelcome(11)}, | |
| 1474 | + | reconnectToastRecv{err: io.EOF}, | |
| 1475 | + | ) | |
| 1476 | + | dialer := portsmocks.NewMockDialer(t) | |
| 1477 | + | tt.configure(dialer, tr) | |
| 1478 | + | ||
| 1479 | + | err := NewRunner(Dependencies{Dialer: dialer, Terminal: term.term, Clock: newReconnectHandshakeClock(t), Logger: slog.New(slog.DiscardHandler)}).Run(ctx, AttachRequest{Intent: ports.IntentAttach, SessionName: "main"}) | |
| 1480 | + | require.ErrorIs(t, err, context.Canceled) | |
| 1481 | + | require.ErrorIs(t, err, clearErr) | |
| 1482 | + | }) | |
| 1483 | + | } | |
| 542 | 1484 | } | |
| 543 | 1485 | ||
| 544 | 1486 | func TestRemoteReconnectToastLifecycleWithWrappedTransportError(t *testing.T) { | |
| @@ -550,8 +1492,8 @@ func TestRemoteReconnectToastLifecycleWithWrappedTransportError(t *testing.T) { | |||
| 550 | 1492 | oldSleep := reconnectSleep | |
| 551 | 1493 | oldSleepWithResize := reconnectSleepWithResize | |
| 552 | 1494 | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return true } | |
| 553 | - | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size)) bool { | |
| 554 | - | return true | |
| 1495 | + | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size) error) (bool, error) { | |
| 1496 | + | return true, nil | |
| 555 | 1497 | } | |
| 556 | 1498 | defer func() { | |
| 557 | 1499 | reconnectSleep = oldSleep | |
| @@ -586,7 +1528,7 @@ func TestRemoteReconnectToastLifecycleWithWrappedTransportError(t *testing.T) { | |||
| 586 | 1528 | ||
| 587 | 1529 | out := term.out.String() | |
| 588 | 1530 | require.Contains(t, out, reconnectToastMessage) | |
| 589 | - | require.Contains(t, out, strings.Repeat(" ", reconnectToastBounds(term.size).Width)) | |
| 1531 | + | require.NotContains(t, out, strings.Repeat(" ", reconnectToastBounds(term.size).Width)) | |
| 590 | 1532 | } | |
| 591 | 1533 | ||
| 592 | 1534 | func TestRemoteEphemeralReconnectUsesAssignedSessionName(t *testing.T) { | |
| @@ -594,8 +1536,8 @@ func TestRemoteEphemeralReconnectUsesAssignedSessionName(t *testing.T) { | |||
| 594 | 1536 | oldSleep := reconnectSleep | |
| 595 | 1537 | oldSleepWithResize := reconnectSleepWithResize | |
| 596 | 1538 | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return true } | |
| 597 | - | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size)) bool { | |
| 598 | - | return true | |
| 1539 | + | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size) error) (bool, error) { | |
| 1540 | + | return true, nil | |
| 599 | 1541 | } | |
| 600 | 1542 | defer func() { | |
| 601 | 1543 | reconnectSleep = oldSleep | |
| @@ -636,7 +1578,6 @@ func TestRemoteReconnectToastLifecycle(t *testing.T) { | |||
| 636 | 1578 | configure func(t *testing.T, dialer *portsmocks.MockDialer) []*reconnectToastRecordingTransport | |
| 637 | 1579 | sleep func(cancel context.CancelFunc) bool | |
| 638 | 1580 | wantErr func(t *testing.T, err error) | |
| 639 | - | wantBlank bool | |
| 640 | 1581 | }{ | |
| 641 | 1582 | { | |
| 642 | 1583 | name: "clears on successful reconnect", | |
| @@ -647,9 +1588,8 @@ func TestRemoteReconnectToastLifecycle(t *testing.T) { | |||
| 647 | 1588 | dialer.EXPECT().Dial(mock.Anything).Return(tr2, nil).Once() | |
| 648 | 1589 | return []*reconnectToastRecordingTransport{tr1, tr2} | |
| 649 | 1590 | }, | |
| 650 | - | sleep: func(context.CancelFunc) bool { return true }, | |
| 651 | - | wantErr: func(t *testing.T, err error) { require.NoError(t, err) }, | |
| 652 | - | wantBlank: true, | |
| 1591 | + | sleep: func(context.CancelFunc) bool { return true }, | |
| 1592 | + | wantErr: func(t *testing.T, err error) { require.NoError(t, err) }, | |
| 653 | 1593 | }, | |
| 654 | 1594 | { | |
| 655 | 1595 | name: "clears on cancellation", | |
| @@ -662,8 +1602,7 @@ func TestRemoteReconnectToastLifecycle(t *testing.T) { | |||
| 662 | 1602 | cancel() | |
| 663 | 1603 | return false | |
| 664 | 1604 | }, | |
| 665 | - | wantErr: func(t *testing.T, err error) { require.ErrorIs(t, err, context.Canceled) }, | |
| 666 | - | wantBlank: true, | |
| 1605 | + | wantErr: func(t *testing.T, err error) { require.ErrorIs(t, err, context.Canceled) }, | |
| 667 | 1606 | }, | |
| 668 | 1607 | { | |
| 669 | 1608 | name: "clears on final exit", | |
| @@ -679,7 +1618,6 @@ func TestRemoteReconnectToastLifecycle(t *testing.T) { | |||
| 679 | 1618 | var detached *DetachedError | |
| 680 | 1619 | require.True(t, errors.As(err, &detached)) | |
| 681 | 1620 | }, | |
| 682 | - | wantBlank: true, | |
| 683 | 1621 | }, | |
| 684 | 1622 | } | |
| 685 | 1623 | ||
| @@ -689,8 +1627,8 @@ func TestRemoteReconnectToastLifecycle(t *testing.T) { | |||
| 689 | 1627 | oldSleepWithResize := reconnectSleepWithResize | |
| 690 | 1628 | ctx, cancel := context.WithCancel(context.Background()) | |
| 691 | 1629 | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return tt.sleep(cancel) } | |
| 692 | - | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size)) bool { | |
| 693 | - | return tt.sleep(cancel) | |
| 1630 | + | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size) error) (bool, error) { | |
| 1631 | + | return tt.sleep(cancel), nil | |
| 694 | 1632 | } | |
| 695 | 1633 | defer func() { | |
| 696 | 1634 | reconnectSleep = oldSleep | |
| @@ -709,9 +1647,7 @@ func TestRemoteReconnectToastLifecycle(t *testing.T) { | |||
| 709 | 1647 | } | |
| 710 | 1648 | out := term.out.String() | |
| 711 | 1649 | require.Contains(t, out, reconnectToastMessage) | |
| 712 | - | if tt.wantBlank { | |
| 713 | - | require.Contains(t, out, strings.Repeat(" ", reconnectToastBounds(term.size).Width)) | |
| 714 | - | } | |
| 1650 | + | require.NotContains(t, out, strings.Repeat(" ", reconnectToastBounds(term.size).Width)) | |
| 715 | 1651 | }) | |
| 716 | 1652 | } | |
| 717 | 1653 | } | |
| @@ -9,6 +9,7 @@ import ( | |||||
| 9 | "log/slog" | 9 | "log/slog" | ||
| 10 | "strings" | 10 | "strings" | ||
| 11 | "sync" | 11 | "sync" | ||
| 12 | + | "sync/atomic" | |||
| 12 | "testing" | 13 | "testing" | ||
| 13 | "time" | 14 | "time" | ||
| 14 | 15 | ||||
| @@ -22,26 +23,35 @@ import ( | |||||
| 22 | ) | 23 | ) | ||
| 23 | 24 | ||||
| 24 | type reconnectToastTerminalHarness struct { | 25 | type reconnectToastTerminalHarness struct { | ||
| 25 | - | term *portsmocks.MockTerminal | 26 | + | term *portsmocks.MockTerminal |
| 26 | - | in *io.PipeReader | 27 | + | in *io.PipeReader |
| 27 | - | inWriter *io.PipeWriter | 28 | + | inWriter *io.PipeWriter |
| 28 | - | out bytes.Buffer | 29 | + | out bytes.Buffer |
| 29 | - | size domain.Size | 30 | + | size domain.Size |
| 30 | - | resizeCh chan domain.Size | 31 | + | resizeCh chan domain.Size |
| 32 | + | sizeCalls atomic.Int32 | |||
| 33 | + | sizeCalled chan int32 | |||
| 31 | } | 34 | } | ||
| 32 | 35 | ||||
| 33 | func newReconnectToastTerminalHarness(t *testing.T) *reconnectToastTerminalHarness { | 36 | func newReconnectToastTerminalHarness(t *testing.T) *reconnectToastTerminalHarness { | ||
| 34 | t.Helper() | 37 | t.Helper() | ||
| 35 | in, inWriter := io.Pipe() | 38 | in, inWriter := io.Pipe() | ||
| 36 | h := &reconnectToastTerminalHarness{ | 39 | h := &reconnectToastTerminalHarness{ | ||
| 37 | - | term: portsmocks.NewMockTerminal(t), | 40 | + | term: portsmocks.NewMockTerminal(t), |
| 38 | - | in: in, | 41 | + | in: in, |
| 39 | - | inWriter: inWriter, | 42 | + | inWriter: inWriter, |
| 40 | - | size: domain.Size{Cols: 80, Rows: 24}, | 43 | + | size: domain.Size{Cols: 80, Rows: 24}, |
| 41 | - | resizeCh: make(chan domain.Size), | 44 | + | resizeCh: make(chan domain.Size, sendQueueDepth), |
| 45 | + | sizeCalled: make(chan int32, 16), | |||
| 42 | } | 46 | } | ||
| 43 | h.term.EXPECT().EnterRaw().Return(func() error { return nil }, nil).Maybe() | 47 | h.term.EXPECT().EnterRaw().Return(func() error { return nil }, nil).Maybe() | ||
| 44 | - | h.term.EXPECT().Size().Return(h.size, nil).Maybe() | 48 | + | h.term.EXPECT().Size().Run(func() { |
| 49 | + | call := h.sizeCalls.Add(1) | |||
| 50 | + | select { | |||
| 51 | + | case h.sizeCalled <- call: | |||
| 52 | + | default: | |||
| 53 | + | } | |||
| 54 | + | }).Return(h.size, nil).Maybe() | |||
| 45 | h.term.EXPECT().ResizeEvents().Return((<-chan domain.Size)(h.resizeCh)).Maybe() | 55 | h.term.EXPECT().ResizeEvents().Return((<-chan domain.Size)(h.resizeCh)).Maybe() | ||
| 46 | h.term.EXPECT().In().Return(h.in).Maybe() | 56 | h.term.EXPECT().In().Return(h.in).Maybe() | ||
| 47 | h.term.EXPECT().Out().Return(&h.out).Maybe() | 57 | h.term.EXPECT().Out().Return(&h.out).Maybe() | ||
| @@ -50,21 +60,33 @@ func newReconnectToastTerminalHarness(t *testing.T) *reconnectToastTerminalHarne | |||||
| 50 | } | 60 | } | ||
| 51 | 61 | ||||
| 52 | func newReconnectToastTerminalHarnessWithOutput(t *testing.T, out io.Writer) *reconnectToastTerminalHarness { | 62 | func newReconnectToastTerminalHarnessWithOutput(t *testing.T, out io.Writer) *reconnectToastTerminalHarness { | ||
| 63 | + | t.Helper() | |||
| 64 | + | return newReconnectToastTerminalHarnessWithOutputAndFlush(t, out, func() error { return nil }) | |||
| 65 | + | } | |||
| 66 | + | ||||
| 67 | + | func newReconnectToastTerminalHarnessWithOutputAndFlush(t *testing.T, out io.Writer, flush func() error) *reconnectToastTerminalHarness { | |||
| 53 | t.Helper() | 68 | t.Helper() | ||
| 54 | in, inWriter := io.Pipe() | 69 | in, inWriter := io.Pipe() | ||
| 55 | h := &reconnectToastTerminalHarness{ | 70 | h := &reconnectToastTerminalHarness{ | ||
| 56 | - | term: portsmocks.NewMockTerminal(t), | 71 | + | term: portsmocks.NewMockTerminal(t), |
| 57 | - | in: in, | 72 | + | in: in, |
| 58 | - | inWriter: inWriter, | 73 | + | inWriter: inWriter, |
| 59 | - | size: domain.Size{Cols: 80, Rows: 24}, | 74 | + | size: domain.Size{Cols: 80, Rows: 24}, |
| 60 | - | resizeCh: make(chan domain.Size), | 75 | + | resizeCh: make(chan domain.Size, sendQueueDepth), |
| 76 | + | sizeCalled: make(chan int32, 16), | |||
| 61 | } | 77 | } | ||
| 62 | h.term.EXPECT().EnterRaw().Return(func() error { return nil }, nil).Maybe() | 78 | h.term.EXPECT().EnterRaw().Return(func() error { return nil }, nil).Maybe() | ||
| 63 | - | h.term.EXPECT().Size().Return(h.size, nil).Maybe() | 79 | + | h.term.EXPECT().Size().Run(func() { |
| 80 | + | call := h.sizeCalls.Add(1) | |||
| 81 | + | select { | |||
| 82 | + | case h.sizeCalled <- call: | |||
| 83 | + | default: | |||
| 84 | + | } | |||
| 85 | + | }).Return(h.size, nil).Maybe() | |||
| 64 | h.term.EXPECT().ResizeEvents().Return((<-chan domain.Size)(h.resizeCh)).Maybe() | 86 | h.term.EXPECT().ResizeEvents().Return((<-chan domain.Size)(h.resizeCh)).Maybe() | ||
| 65 | h.term.EXPECT().In().Return(h.in).Maybe() | 87 | h.term.EXPECT().In().Return(h.in).Maybe() | ||
| 66 | h.term.EXPECT().Out().Return(out).Maybe() | 88 | h.term.EXPECT().Out().Return(out).Maybe() | ||
| 67 | - | h.term.EXPECT().Flush().Return(nil).Maybe() | 89 | + | h.term.EXPECT().Flush().RunAndReturn(flush).Maybe() |
| 68 | return h | 90 | return h | ||
| 69 | } | 91 | } | ||
| 70 | 92 | ||||
| @@ -73,6 +95,7 @@ func (h *reconnectToastTerminalHarness) closeInput() { _ = h.inWriter.Close() } | |||||
| 73 | type reconnectToastOutputRecorder struct { | 95 | type reconnectToastOutputRecorder struct { | ||
| 74 | mu sync.Mutex | 96 | mu sync.Mutex | ||
| 75 | pending strings.Builder | 97 | pending strings.Builder | ||
| 98 | + | all strings.Builder | |||
| 76 | completed chan string | 99 | completed chan string | ||
| 77 | } | 100 | } | ||
| 78 | 101 | ||||
| @@ -85,6 +108,7 @@ func (r *reconnectToastOutputRecorder) Write(p []byte) (int, error) { | |||||
| 85 | 108 | ||||
| 86 | r.mu.Lock() | 109 | r.mu.Lock() | ||
| 87 | r.pending.Write(p) | 110 | r.pending.Write(p) | ||
| 111 | + | r.all.Write(p) | |||
| 88 | var output string | 112 | var output string | ||
| 89 | if string(p) == restoreCursor { | 113 | if string(p) == restoreCursor { | ||
| 90 | output = r.pending.String() | 114 | output = r.pending.String() | ||
| @@ -98,6 +122,12 @@ func (r *reconnectToastOutputRecorder) Write(p []byte) (int, error) { | |||||
| 98 | return len(p), nil | 122 | return len(p), nil | ||
| 99 | } | 123 | } | ||
| 100 | 124 | ||||
| 125 | + | func (r *reconnectToastOutputRecorder) String() string { | |||
| 126 | + | r.mu.Lock() | |||
| 127 | + | defer r.mu.Unlock() | |||
| 128 | + | return r.all.String() | |||
| 129 | + | } | |||
| 130 | + | ||||
| 101 | func reconnectToastWelcome(token uint64) ports.Frame { | 131 | func reconnectToastWelcome(token uint64) ports.Frame { | ||
| 102 | return reconnectToastWelcomeNamed("", token) | 132 | return reconnectToastWelcomeNamed("", token) | ||
| 103 | } | 133 | } | ||
| @@ -140,8 +170,40 @@ func newReconnectToastRecordingTransport(recvs ...reconnectToastRecv) *reconnect | |||||
| 140 | return &reconnectToastRecordingTransport{recvs: append([]reconnectToastRecv(nil), recvs...)} | 170 | return &reconnectToastRecordingTransport{recvs: append([]reconnectToastRecv(nil), recvs...)} | ||
| 141 | } | 171 | } | ||
| 142 | 172 | ||||
| 173 | + | type reconnectToastSentFrames struct { | |||
| 174 | + | mu sync.Mutex | |||
| 175 | + | frames []ports.Frame | |||
| 176 | + | changed chan struct{} | |||
| 177 | + | } | |||
| 178 | + | ||||
| 179 | + | func newReconnectToastSentFrames() *reconnectToastSentFrames { | |||
| 180 | + | return &reconnectToastSentFrames{changed: make(chan struct{}, 1)} | |||
| 181 | + | } | |||
| 182 | + | ||||
| 183 | + | func (s *reconnectToastSentFrames) record(frame ports.Frame) { | |||
| 184 | + | s.mu.Lock() | |||
| 185 | + | s.frames = append(s.frames, frame) | |||
| 186 | + | s.mu.Unlock() | |||
| 187 | + | select { | |||
| 188 | + | case s.changed <- struct{}{}: | |||
| 189 | + | default: | |||
| 190 | + | } | |||
| 191 | + | } | |||
| 192 | + | ||||
| 193 | + | func (s *reconnectToastSentFrames) find(match func(ports.Frame) bool) (ports.Frame, bool) { | |||
| 194 | + | s.mu.Lock() | |||
| 195 | + | defer s.mu.Unlock() | |||
| 196 | + | for _, frame := range s.frames { | |||
| 197 | + | if match(frame) { | |||
| 198 | + | return frame, true | |||
| 199 | + | } | |||
| 200 | + | } | |||
| 201 | + | return ports.Frame{}, false | |||
| 202 | + | } | |||
| 203 | + | ||||
| 143 | type reconnectToastLinkTransport struct { | 204 | type reconnectToastLinkTransport struct { | ||
| 144 | recvCh chan reconnectToastRecv | 205 | recvCh chan reconnectToastRecv | ||
| 206 | + | sends *reconnectToastSentFrames | |||
| 145 | clientNotices chan ports.ClientNotice | 207 | clientNotices chan ports.ClientNotice | ||
| 146 | events chan ports.LinkEvent | 208 | events chan ports.LinkEvent | ||
| 147 | state ports.LinkState | 209 | state ports.LinkState | ||
| @@ -150,6 +212,7 @@ type reconnectToastLinkTransport struct { | |||||
| 150 | func newReconnectToastLinkTransport() *reconnectToastLinkTransport { | 212 | func newReconnectToastLinkTransport() *reconnectToastLinkTransport { | ||
| 151 | return &reconnectToastLinkTransport{ | 213 | return &reconnectToastLinkTransport{ | ||
| 152 | recvCh: make(chan reconnectToastRecv, 4), | 214 | recvCh: make(chan reconnectToastRecv, 4), | ||
| 215 | + | sends: newReconnectToastSentFrames(), | |||
| 153 | clientNotices: make(chan ports.ClientNotice, 2), | 216 | clientNotices: make(chan ports.ClientNotice, 2), | ||
| 154 | events: make(chan ports.LinkEvent, 4), | 217 | events: make(chan ports.LinkEvent, 4), | ||
| 155 | state: ports.LinkStateConnected, | 218 | state: ports.LinkStateConnected, | ||
| @@ -157,6 +220,7 @@ func newReconnectToastLinkTransport() *reconnectToastLinkTransport { | |||||
| 157 | } | 220 | } | ||
| 158 | 221 | ||||
| 159 | func (t *reconnectToastLinkTransport) Send(f ports.Frame) error { | 222 | func (t *reconnectToastLinkTransport) Send(f ports.Frame) error { | ||
| 223 | + | t.sends.record(f) | |||
| 160 | if f.Type != ports.MsgClientNotice { | 224 | if f.Type != ports.MsgClientNotice { | ||
| 161 | return nil | 225 | return nil | ||
| 162 | } | 226 | } | ||
| @@ -222,7 +286,7 @@ func (t *reconnectToastRecordingTransport) wasClosed() bool { | |||||
| 222 | func assertReconnectToastAttemptPublishesOnlyClearedTheme(t *testing.T, tr *reconnectToastRecordingTransport) { | 286 | func assertReconnectToastAttemptPublishesOnlyClearedTheme(t *testing.T, tr *reconnectToastRecordingTransport) { | ||
| 223 | t.Helper() | 287 | t.Helper() | ||
| 224 | frames := tr.sentFrames() | 288 | frames := tr.sentFrames() | ||
| 225 | - | require.Len(t, frames, 2) | 289 | + | require.GreaterOrEqual(t, len(frames), 2) |
| 226 | require.Equal(t, ports.MsgHello, frames[0].Type) | 290 | require.Equal(t, ports.MsgHello, frames[0].Type) | ||
| 227 | require.Equal(t, ports.MsgTheme, frames[1].Type) | 291 | require.Equal(t, ports.MsgTheme, frames[1].Type) | ||
| 228 | theme, err := ports.UnmarshalTheme(frames[1].Payload) | 292 | theme, err := ports.UnmarshalTheme(frames[1].Payload) | ||
| @@ -290,6 +354,431 @@ func newReconnectAttachAttempt(term ports.Terminal, transport ports.Transport, c | |||||
| 290 | } | 354 | } | ||
| 291 | } | 355 | } | ||
| 292 | 356 | ||||
| 357 | + | func TestProbingToastReconcilesDaemonOutputAndDismissal(t *testing.T) { | |||
| 358 | + | out := newReconnectToastOutputRecorder() | |||
| 359 | + | var flushes atomic.Int32 | |||
| 360 | + | term := newReconnectToastTerminalHarnessWithOutputAndFlush(t, out, func() error { | |||
| 361 | + | flushes.Add(1) | |||
| 362 | + | return nil | |||
| 363 | + | }) | |||
| 364 | + | defer term.closeInput() | |||
| 365 | + | tr := newReconnectToastLinkTransport() | |||
| 366 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |||
| 367 | + | ||||
| 368 | + | resultCh := make(chan attachResult, 1) | |||
| 369 | + | ms := milestones{} | |||
| 370 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |||
| 371 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |||
| 372 | + | ||||
| 373 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |||
| 374 | + | firstToast := requireReconnectToastOutput(t, out.completed) | |||
| 375 | + | require.Contains(t, firstToast, "┌") | |||
| 376 | + | require.Contains(t, firstToast, "probing UDP path") | |||
| 377 | + | ||||
| 378 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |||
| 379 | + | BaseStateNum: 1, | |||
| 380 | + | NewStateNum: 2, | |||
| 381 | + | Data: []byte("daemon incremental"), | |||
| 382 | + | })}} | |||
| 383 | + | redrawn := requireReconnectToastOutput(t, out.completed) | |||
| 384 | + | require.Contains(t, redrawn, "daemon incremental") | |||
| 385 | + | require.Contains(t, redrawn, "┌") | |||
| 386 | + | require.Contains(t, redrawn, "probing UDP path") | |||
| 387 | + | ||||
| 388 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateConnected} | |||
| 389 | + | require.Equal(t, ports.ClientNoticeLinkConnected, (<-tr.clientNotices).Action) | |||
| 390 | + | require.Equal(t, term.size, requireResize(t, tr.sends).Size) | |||
| 391 | + | ||||
| 392 | + | beforeAwaitingReset := out.String() | |||
| 393 | + | beforeStatelessFlushes := flushes.Load() | |||
| 394 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |||
| 395 | + | BaseStateNum: 0, | |||
| 396 | + | NewStateNum: 0, | |||
| 397 | + | Data: []byte("stateless side effect"), | |||
| 398 | + | })}} | |||
| 399 | + | require.Eventually(t, func() bool { | |||
| 400 | + | return strings.Contains(out.String(), "stateless side effect") && flushes.Load() > beforeStatelessFlushes | |||
| 401 | + | }, time.Second, time.Millisecond) | |||
| 402 | + | ||||
| 403 | + | afterStateless := out.String() | |||
| 404 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |||
| 405 | + | BaseStateNum: 2, | |||
| 406 | + | NewStateNum: 3, | |||
| 407 | + | Data: []byte("intervening incremental"), | |||
| 408 | + | })}} | |||
| 409 | + | requireAckedState(t, tr.sends, 3) | |||
| 410 | + | require.Equal(t, afterStateless, out.String(), "stateless output must not satisfy pending reset reconciliation") | |||
| 411 | + | require.Contains(t, afterStateless[len(beforeAwaitingReset):], "stateless side effect") | |||
| 412 | + | ||||
| 413 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |||
| 414 | + | BaseStateNum: 0, | |||
| 415 | + | NewStateNum: 4, | |||
| 416 | + | Data: []byte("authoritative reset"), | |||
| 417 | + | })}} | |||
| 418 | + | requireAckedState(t, tr.sends, 4) | |||
| 419 | + | require.Eventually(t, func() bool { return strings.Contains(out.String(), "authoritative reset") }, time.Second, time.Millisecond) | |||
| 420 | + | require.NotContains(t, out.String()[len(beforeAwaitingReset):], strings.Repeat(" ", reconnectToastBoundsFor(term.size, "probing UDP path").Width)) | |||
| 421 | + | ||||
| 422 | + | beforeIncrementFlushes := flushes.Load() | |||
| 423 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |||
| 424 | + | BaseStateNum: 4, | |||
| 425 | + | NewStateNum: 5, | |||
| 426 | + | Data: []byte("increment after reset"), | |||
| 427 | + | })}} | |||
| 428 | + | require.Eventually(t, func() bool { | |||
| 429 | + | return strings.Contains(out.String(), "increment after reset") && flushes.Load() > beforeIncrementFlushes | |||
| 430 | + | }, time.Second, time.Millisecond) | |||
| 431 | + | requireAckedState(t, tr.sends, 5) | |||
| 432 | + | ||||
| 433 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | |||
| 434 | + | result := requireAttachResult(t, resultCh) | |||
| 435 | + | require.NoError(t, result.err) | |||
| 436 | + | } | |||
| 437 | + | ||||
| 438 | + | func TestActiveReconnectToastStageTransitionReconcilesBeforeRedraw(t *testing.T) { | |||
| 439 | + | out := newReconnectToastOutputRecorder() | |||
| 440 | + | var flushes atomic.Int32 | |||
| 441 | + | term := newReconnectToastTerminalHarnessWithOutputAndFlush(t, out, func() error { | |||
| 442 | + | flushes.Add(1) | |||
| 443 | + | return nil | |||
| 444 | + | }) | |||
| 445 | + | defer term.closeInput() | |||
| 446 | + | tr := newReconnectToastLinkTransport() | |||
| 447 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |||
| 448 | + | ||||
| 449 | + | resultCh := make(chan attachResult, 1) | |||
| 450 | + | ms := milestones{} | |||
| 451 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |||
| 452 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |||
| 453 | + | ||||
| 454 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |||
| 455 | + | probingToast := requireReconnectToastOutput(t, out.completed) | |||
| 456 | + | require.Contains(t, probingToast, "probing UDP path") | |||
| 457 | + | ||||
| 458 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateDead} | |||
| 459 | + | require.Equal(t, term.size, requireResize(t, tr.sends).Size) | |||
| 460 | + | offlineMessage := reconnectStageMessage(reconnectStageOfflineRetrying) | |||
| 461 | + | require.NotContains(t, out.String(), offlineMessage) | |||
| 462 | + | ||||
| 463 | + | beforeReset := out.String() | |||
| 464 | + | for _, output := range []ports.Output{ | |||
| 465 | + | {BaseStateNum: 1, NewStateNum: 2, Data: []byte("first skipped increment")}, | |||
| 466 | + | {BaseStateNum: 2, NewStateNum: 3, Data: []byte("second skipped increment")}, | |||
| 467 | + | } { | |||
| 468 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(output)}} | |||
| 469 | + | requireAckedState(t, tr.sends, output.NewStateNum) | |||
| 470 | + | require.Equal(t, beforeReset, out.String()) | |||
| 471 | + | } | |||
| 472 | + | ||||
| 473 | + | beforeResetFlushes := flushes.Load() | |||
| 474 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |||
| 475 | + | BaseStateNum: 0, | |||
| 476 | + | NewStateNum: 4, | |||
| 477 | + | Data: []byte("stage transition reset"), | |||
| 478 | + | })}} | |||
| 479 | + | redrawn := requireReconnectToastOutput(t, out.completed) | |||
| 480 | + | require.Contains(t, redrawn, "stage transition reset") | |||
| 481 | + | require.Contains(t, redrawn, offlineMessage) | |||
| 482 | + | require.Eventually(t, func() bool { return flushes.Load() > beforeResetFlushes }, time.Second, time.Millisecond) | |||
| 483 | + | requireAckedState(t, tr.sends, 4) | |||
| 484 | + | ||||
| 485 | + | beforeIncrementFlushes := flushes.Load() | |||
| 486 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |||
| 487 | + | BaseStateNum: 4, | |||
| 488 | + | NewStateNum: 5, | |||
| 489 | + | Data: []byte("stage increment after reset"), | |||
| 490 | + | })}} | |||
| 491 | + | incremental := requireReconnectToastOutput(t, out.completed) | |||
| 492 | + | require.Contains(t, incremental, "stage increment after reset") | |||
| 493 | + | require.Contains(t, incremental, offlineMessage) | |||
| 494 | + | require.Eventually(t, func() bool { return flushes.Load() > beforeIncrementFlushes }, time.Second, time.Millisecond) | |||
| 495 | + | requireAckedState(t, tr.sends, 5) | |||
| 496 | + | ||||
| 497 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | |||
| 498 | + | result := requireAttachResult(t, resultCh) | |||
| 499 | + | require.NoError(t, result.err) | |||
| 500 | + | } | |||
| 501 | + | ||||
| 502 | + | func requireReconnectToastOutput(t *testing.T, completed <-chan string) string { | |||
| 503 | + | t.Helper() | |||
| 504 | + | select { | |||
| 505 | + | case output := <-completed: | |||
| 506 | + | return output | |||
| 507 | + | case <-time.After(time.Second): | |||
| 508 | + | t.Fatal("timed out waiting for reconnect toast terminal output") | |||
| 509 | + | return "" | |||
| 510 | + | } | |||
| 511 | + | } | |||
| 512 | + | ||||
| 513 | + | func requireSentFrame(t *testing.T, sends *reconnectToastSentFrames, description string, match func(ports.Frame) bool) ports.Frame { | |||
| 514 | + | t.Helper() | |||
| 515 | + | timer := time.NewTimer(time.Second) | |||
| 516 | + | defer timer.Stop() | |||
| 517 | + | for { | |||
| 518 | + | if frame, ok := sends.find(match); ok { | |||
| 519 | + | return frame | |||
| 520 | + | } | |||
| 521 | + | select { | |||
| 522 | + | case <-sends.changed: | |||
| 523 | + | case <-timer.C: | |||
| 524 | + | t.Fatalf("timed out waiting for %s", description) | |||
| 525 | + | return ports.Frame{} | |||
| 526 | + | } | |||
| 527 | + | } | |||
| 528 | + | } | |||
| 529 | + | ||||
| 530 | + | func requireResize(t *testing.T, sends *reconnectToastSentFrames) ports.Resize { | |||
| 531 | + | t.Helper() | |||
| 532 | + | frame := requireSentFrame(t, sends, "Resize frame", func(frame ports.Frame) bool { | |||
| 533 | + | return frame.Type == ports.MsgResize | |||
| 534 | + | }) | |||
| 535 | + | resize, err := ports.UnmarshalResize(frame.Payload) | |||
| 536 | + | require.NoError(t, err) | |||
| 537 | + | return resize | |||
| 538 | + | } | |||
| 539 | + | ||||
| 540 | + | func requireAckedState(t *testing.T, sends *reconnectToastSentFrames, state uint64) { | |||
| 541 | + | t.Helper() | |||
| 542 | + | frame := requireSentFrame(t, sends, fmt.Sprintf("ACK for state %d", state), func(frame ports.Frame) bool { | |||
| 543 | + | if frame.Type != ports.MsgAck { | |||
| 544 | + | return false | |||
| 545 | + | } | |||
| 546 | + | ack, err := ports.UnmarshalAck(frame.Payload) | |||
| 547 | + | return err == nil && ack.AckedStateNum >= state | |||
| 548 | + | }) | |||
| 549 | + | _, err := ports.UnmarshalAck(frame.Payload) | |||
| 550 | + | require.NoError(t, err) | |||
| 551 | + | } | |||
| 552 | + | ||||
| 553 | + | func requireAttachResult(t *testing.T, results <-chan attachResult) attachResult { | |||
| 554 | + | t.Helper() | |||
| 555 | + | select { | |||
| 556 | + | case result := <-results: | |||
| 557 | + | return result | |||
| 558 | + | case <-time.After(time.Second): | |||
| 559 | + | t.Fatal("timed out waiting for attach attempt result") | |||
| 560 | + | return attachResult{} | |||
| 561 | + | } | |||
| 562 | + | } | |||
| 563 | + | ||||
| 564 | + | type armedReconnectToastWriter struct { | |||
| 565 | + | next io.Writer | |||
| 566 | + | needle string | |||
| 567 | + | armed atomic.Bool | |||
| 568 | + | failure error | |||
| 569 | + | } | |||
| 570 | + | ||||
| 571 | + | func (w *armedReconnectToastWriter) Write(p []byte) (int, error) { | |||
| 572 | + | if w.armed.Load() && strings.Contains(string(p), w.needle) { | |||
| 573 | + | return 0, w.failure | |||
| 574 | + | } | |||
| 575 | + | return w.next.Write(p) | |||
| 576 | + | } | |||
| 577 | + | ||||
| 578 | + | type reconnectRuntimeObserver struct { | |||
| 579 | + | mu sync.Mutex | |||
| 580 | + | marks []ports.RuntimeMark | |||
| 581 | + | } | |||
| 582 | + | ||||
| 583 | + | func (o *reconnectRuntimeObserver) ObserveRuntime(mark ports.RuntimeMark) { | |||
| 584 | + | o.mu.Lock() | |||
| 585 | + | o.marks = append(o.marks, mark) | |||
| 586 | + | o.mu.Unlock() | |||
| 587 | + | } | |||
| 588 | + | ||||
| 589 | + | func (o *reconnectRuntimeObserver) Flush() {} | |||
| 590 | + | func (o *reconnectRuntimeObserver) Close() {} | |||
| 591 | + | ||||
| 592 | + | func (o *reconnectRuntimeObserver) count() int { | |||
| 593 | + | o.mu.Lock() | |||
| 594 | + | defer o.mu.Unlock() | |||
| 595 | + | return len(o.marks) | |||
| 596 | + | } | |||
| 597 | + | ||||
| 598 | + | type reconnectFlushObserver struct { | |||
| 599 | + | flushes *atomic.Int32 | |||
| 600 | + | seen chan int32 | |||
| 601 | + | } | |||
| 602 | + | ||||
| 603 | + | func (o *reconnectFlushObserver) ObserveRuntime(mark ports.RuntimeMark) { | |||
| 604 | + | if mark.Kind != ports.RuntimeTerminalFlushed { | |||
| 605 | + | return | |||
| 606 | + | } | |||
| 607 | + | select { | |||
| 608 | + | case o.seen <- o.flushes.Load(): | |||
| 609 | + | default: | |||
| 610 | + | } | |||
| 611 | + | } | |||
| 612 | + | func (o *reconnectFlushObserver) Flush() {} | |||
| 613 | + | func (o *reconnectFlushObserver) Close() {} | |||
| 614 | + | ||||
| 615 | + | type reconnectStatusSignalWriter struct { | |||
| 616 | + | written chan struct{} | |||
| 617 | + | once sync.Once | |||
| 618 | + | } | |||
| 619 | + | ||||
| 620 | + | func (w *reconnectStatusSignalWriter) Write(p []byte) (int, error) { | |||
| 621 | + | if strings.Contains(string(p), statusReconnect) { | |||
| 622 | + | w.once.Do(func() { close(w.written) }) | |||
| 623 | + | } | |||
| 624 | + | return len(p), nil | |||
| 625 | + | } | |||
| 626 | + | ||||
| 627 | + | func TestLocalReconnectStatusFlushesDaemonOutputBeforeObservationAndAck(t *testing.T) { | |||
| 628 | + | var flushes atomic.Int32 | |||
| 629 | + | writer := &reconnectStatusSignalWriter{written: make(chan struct{})} | |||
| 630 | + | term := newReconnectToastTerminalHarnessWithOutputAndFlush(t, writer, func() error { | |||
| 631 | + | flushes.Add(1) | |||
| 632 | + | return nil | |||
| 633 | + | }) | |||
| 634 | + | defer term.closeInput() | |||
| 635 | + | tr := newReconnectToastLinkTransport() | |||
| 636 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |||
| 637 | + | observer := &reconnectFlushObserver{flushes: &flushes, seen: make(chan int32, 1)} | |||
| 638 | + | ||||
| 639 | + | resultCh := make(chan attachResult, 1) | |||
| 640 | + | ms := milestones{} | |||
| 641 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main"}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |||
| 642 | + | attempt.runner.runtimeObserver = observer | |||
| 643 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |||
| 644 | + | ||||
| 645 | + | beforeStatus := flushes.Load() | |||
| 646 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |||
| 647 | + | select { | |||
| 648 | + | case <-writer.written: | |||
| 649 | + | case <-time.After(time.Second): | |||
| 650 | + | t.Fatal("timed out waiting for local reconnect status write") | |||
| 651 | + | } | |||
| 652 | + | require.Eventually(t, func() bool { return flushes.Load() > beforeStatus }, time.Second, time.Millisecond) | |||
| 653 | + | beforeOutput := flushes.Load() | |||
| 654 | + | ||||
| 655 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |||
| 656 | + | BaseStateNum: 1, | |||
| 657 | + | NewStateNum: 2, | |||
| 658 | + | Data: []byte("daemon output behind local reconnect status"), | |||
| 659 | + | })}} | |||
| 660 | + | ||||
| 661 | + | select { | |||
| 662 | + | case observedAtFlush := <-observer.seen: | |||
| 663 | + | require.Greater(t, observedAtFlush, beforeOutput, "terminal output must be flushed before publishing RuntimeTerminalFlushed") | |||
| 664 | + | case <-time.After(time.Second): | |||
| 665 | + | t.Fatal("timed out waiting for terminal-flushed observation") | |||
| 666 | + | } | |||
| 667 | + | requireAckedState(t, tr.sends, 2) | |||
| 668 | + | require.Greater(t, flushes.Load(), beforeOutput, "daemon output must be flushed before ACK") | |||
| 669 | + | ||||
| 670 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | |||
| 671 | + | require.NoError(t, requireAttachResult(t, resultCh).err) | |||
| 672 | + | } | |||
| 673 | + | ||||
| 674 | + | func TestLocalReconnectStatusTransitionsDoNotRequestAuthoritativeReset(t *testing.T) { | |||
| 675 | + | tests := []struct { | |||
| 676 | + | name string | |||
| 677 | + | transition ports.LinkState | |||
| 678 | + | }{ | |||
| 679 | + | {name: "dismissal", transition: ports.LinkStateConnected}, | |||
| 680 | + | {name: "stage change", transition: ports.LinkStateDead}, | |||
| 681 | + | } | |||
| 682 | + | ||||
| 683 | + | for _, tt := range tests { | |||
| 684 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 685 | + | out := newReconnectToastOutputRecorder() | |||
| 686 | + | term := newReconnectToastTerminalHarnessWithOutput(t, out) | |||
| 687 | + | defer term.closeInput() | |||
| 688 | + | tr := newReconnectToastLinkTransport() | |||
| 689 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |||
| 690 | + | ||||
| 691 | + | resultCh := make(chan attachResult, 1) | |||
| 692 | + | ms := milestones{} | |||
| 693 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main"}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |||
| 694 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |||
| 695 | + | ||||
| 696 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |||
| 697 | + | require.Eventually(t, func() bool { return strings.Contains(out.String(), statusReconnect) }, time.Second, time.Millisecond) | |||
| 698 | + | tr.events <- ports.LinkEvent{State: tt.transition} | |||
| 699 | + | if tt.transition == ports.LinkStateConnected { | |||
| 700 | + | require.Equal(t, ports.ClientNoticeLinkConnected, (<-tr.clientNotices).Action) | |||
| 701 | + | } | |||
| 702 | + | ||||
| 703 | + | const output = "local status output remains visible" | |||
| 704 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |||
| 705 | + | BaseStateNum: 1, | |||
| 706 | + | NewStateNum: 2, | |||
| 707 | + | Data: []byte(output), | |||
| 708 | + | })}} | |||
| 709 | + | require.Eventually(t, func() bool { return strings.Contains(out.String(), output) }, time.Second, time.Millisecond) | |||
| 710 | + | requireAckedState(t, tr.sends, 2) | |||
| 711 | + | _, sentResize := tr.sends.find(func(frame ports.Frame) bool { return frame.Type == ports.MsgResize }) | |||
| 712 | + | require.False(t, sentResize, "local reconnect status must not request a daemon reset") | |||
| 713 | + | ||||
| 714 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | |||
| 715 | + | require.NoError(t, requireAttachResult(t, resultCh).err) | |||
| 716 | + | }) | |||
| 717 | + | } | |||
| 718 | + | } | |||
| 719 | + | ||||
| 720 | + | func TestReconnectOverlayRedrawFailureDoesNotObserveOrAckOutput(t *testing.T) { | |||
| 721 | + | tests := []struct { | |||
| 722 | + | name string | |||
| 723 | + | configure func(*testing.T, *reconnectToastOutputRecorder) (*reconnectToastTerminalHarness, func()) | |||
| 724 | + | wantError string | |||
| 725 | + | }{ | |||
| 726 | + | { | |||
| 727 | + | name: "write", | |||
| 728 | + | configure: func(t *testing.T, out *reconnectToastOutputRecorder) (*reconnectToastTerminalHarness, func()) { | |||
| 729 | + | failure := errors.New("redraw write failed") | |||
| 730 | + | writer := &armedReconnectToastWriter{next: out, needle: "probing UDP path", failure: failure} | |||
| 731 | + | return newReconnectToastTerminalHarnessWithOutput(t, writer), func() { writer.armed.Store(true) } | |||
| 732 | + | }, | |||
| 733 | + | wantError: "redraw write failed", | |||
| 734 | + | }, | |||
| 735 | + | { | |||
| 736 | + | name: "flush", | |||
| 737 | + | configure: func(t *testing.T, out *reconnectToastOutputRecorder) (*reconnectToastTerminalHarness, func()) { | |||
| 738 | + | var armed atomic.Bool | |||
| 739 | + | term := newReconnectToastTerminalHarnessWithOutputAndFlush(t, out, func() error { | |||
| 740 | + | if armed.Load() { | |||
| 741 | + | return errors.New("redraw flush failed") | |||
| 742 | + | } | |||
| 743 | + | return nil | |||
| 744 | + | }) | |||
| 745 | + | return term, func() { armed.Store(true) } | |||
| 746 | + | }, | |||
| 747 | + | wantError: "redraw flush failed", | |||
| 748 | + | }, | |||
| 749 | + | } | |||
| 750 | + | ||||
| 751 | + | for _, tt := range tests { | |||
| 752 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 753 | + | out := newReconnectToastOutputRecorder() | |||
| 754 | + | term, arm := tt.configure(t, out) | |||
| 755 | + | defer term.closeInput() | |||
| 756 | + | tr := newReconnectToastLinkTransport() | |||
| 757 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |||
| 758 | + | observer := &reconnectRuntimeObserver{} | |||
| 759 | + | ||||
| 760 | + | resultCh := make(chan attachResult, 1) | |||
| 761 | + | ms := milestones{} | |||
| 762 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |||
| 763 | + | attempt.runner.runtimeObserver = observer | |||
| 764 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |||
| 765 | + | ||||
| 766 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |||
| 767 | + | requireReconnectToastOutput(t, out.completed) | |||
| 768 | + | arm() | |||
| 769 | + | tr.recvCh <- reconnectToastRecv{frame: ports.Frame{Type: ports.MsgOutput, Payload: ports.MarshalOutput(ports.Output{ | |||
| 770 | + | BaseStateNum: 1, | |||
| 771 | + | NewStateNum: 2, | |||
| 772 | + | Data: []byte("daemon output before failed redraw"), | |||
| 773 | + | })}} | |||
| 774 | + | ||||
| 775 | + | result := requireAttachResult(t, resultCh) | |||
| 776 | + | require.ErrorContains(t, result.err, tt.wantError) | |||
| 777 | + | require.Zero(t, observer.count(), "failed terminal redraw must not publish a terminal-flushed mark") | |||
| 778 | + | }) | |||
| 779 | + | } | |||
| 780 | + | } | |||
| 781 | + | ||||
| 293 | func TestReconnectLinkEventsNotifyDaemonWithoutLocalTerminalWrites(t *testing.T) { | 782 | func TestReconnectLinkEventsNotifyDaemonWithoutLocalTerminalWrites(t *testing.T) { | ||
| 294 | out := newReconnectToastOutputRecorder() | 783 | out := newReconnectToastOutputRecorder() | ||
| 295 | term := newReconnectToastTerminalHarnessWithOutput(t, out) | 784 | term := newReconnectToastTerminalHarnessWithOutput(t, out) | ||
| @@ -313,7 +802,7 @@ func TestReconnectLinkEventsNotifyDaemonWithoutLocalTerminalWrites(t *testing.T) | |||||
| 313 | } | 802 | } | ||
| 314 | 803 | ||||
| 315 | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | 804 | tr.recvCh <- reconnectToastRecv{frame: reconnectToastDetach(ports.ReasonDetach)} | ||
| 316 | - | result := <-resultCh | 805 | + | result := requireAttachResult(t, resultCh) |
| 317 | require.NoError(t, result.err) | 806 | require.NoError(t, result.err) | ||
| 318 | require.True(t, result.welcomed) | 807 | require.True(t, result.welcomed) | ||
| 319 | } | 808 | } | ||
| @@ -330,12 +819,305 @@ func TestAttachAttemptOfflineLinkEventReturnsReconnectableError(t *testing.T) { | |||||
| 330 | go func() { resultCh <- attempt.run(context.Background()) }() | 819 | go func() { resultCh <- attempt.run(context.Background()) }() | ||
| 331 | 820 | ||||
| 332 | tr.events <- ports.LinkEvent{State: ports.LinkStateOffline} | 821 | tr.events <- ports.LinkEvent{State: ports.LinkStateOffline} | ||
| 333 | - | result := <-resultCh | 822 | + | result := requireAttachResult(t, resultCh) |
| 334 | require.ErrorIs(t, result.err, errLinkOffline) | 823 | require.ErrorIs(t, result.err, errLinkOffline) | ||
| 335 | require.True(t, result.welcomed) | 824 | require.True(t, result.welcomed) | ||
| 336 | require.True(t, shouldReconnect(result.err), "offline exit must be reconnectable") | 825 | require.True(t, shouldReconnect(result.err), "offline exit must be reconnectable") | ||
| 337 | } | 826 | } | ||
| 338 | 827 | ||||
| 828 | + | type reconnectResetTimer struct { | |||
| 829 | + | ch chan time.Time | |||
| 830 | + | } | |||
| 831 | + | ||||
| 832 | + | func (t *reconnectResetTimer) C() <-chan time.Time { return t.ch } | |||
| 833 | + | func (t *reconnectResetTimer) Reset(time.Duration) bool { return true } | |||
| 834 | + | func (t *reconnectResetTimer) Stop() bool { return true } | |||
| 835 | + | ||||
| 836 | + | type reconnectResetClock struct { | |||
| 837 | + | preWelcome chan *reconnectResetTimer | |||
| 838 | + | } | |||
| 839 | + | ||||
| 840 | + | func newReconnectResetClock() *reconnectResetClock { | |||
| 841 | + | return &reconnectResetClock{preWelcome: make(chan *reconnectResetTimer, 4)} | |||
| 842 | + | } | |||
| 843 | + | ||||
| 844 | + | func (c *reconnectResetClock) Now() time.Time { return time.Time{} } | |||
| 845 | + | ||||
| 846 | + | func (c *reconnectResetClock) NewTimer(d time.Duration) ports.Timer { | |||
| 847 | + | timer := &reconnectResetTimer{ch: make(chan time.Time, 1)} | |||
| 848 | + | if d == preWelcomeTimeout { | |||
| 849 | + | c.preWelcome <- timer | |||
| 850 | + | } | |||
| 851 | + | return timer | |||
| 852 | + | } | |||
| 853 | + | ||||
| 854 | + | type reconnectResetBlockingTransport struct { | |||
| 855 | + | mu sync.Mutex | |||
| 856 | + | recvs []reconnectToastRecv | |||
| 857 | + | entered chan struct{} | |||
| 858 | + | closed chan struct{} | |||
| 859 | + | enterOnce sync.Once | |||
| 860 | + | closeOnce sync.Once | |||
| 861 | + | } | |||
| 862 | + | ||||
| 863 | + | func newReconnectResetBlockingTransport() *reconnectResetBlockingTransport { | |||
| 864 | + | return &reconnectResetBlockingTransport{ | |||
| 865 | + | recvs: []reconnectToastRecv{{frame: reconnectToastWelcomeNamed("assigned", 44)}}, | |||
| 866 | + | entered: make(chan struct{}), | |||
| 867 | + | closed: make(chan struct{}), | |||
| 868 | + | } | |||
| 869 | + | } | |||
| 870 | + | ||||
| 871 | + | func (t *reconnectResetBlockingTransport) Send(frame ports.Frame) error { | |||
| 872 | + | if frame.Type != ports.MsgResize { | |||
| 873 | + | return nil | |||
| 874 | + | } | |||
| 875 | + | t.enterOnce.Do(func() { close(t.entered) }) | |||
| 876 | + | <-t.closed | |||
| 877 | + | return io.ErrClosedPipe | |||
| 878 | + | } | |||
| 879 | + | ||||
| 880 | + | func (t *reconnectResetBlockingTransport) Recv() (ports.Frame, error) { | |||
| 881 | + | t.mu.Lock() | |||
| 882 | + | defer t.mu.Unlock() | |||
| 883 | + | if len(t.recvs) == 0 { | |||
| 884 | + | return ports.Frame{}, io.EOF | |||
| 885 | + | } | |||
| 886 | + | recv := t.recvs[0] | |||
| 887 | + | t.recvs = t.recvs[1:] | |||
| 888 | + | return recv.frame, recv.err | |||
| 889 | + | } | |||
| 890 | + | ||||
| 891 | + | func (t *reconnectResetBlockingTransport) Close() error { | |||
| 892 | + | t.closeOnce.Do(func() { close(t.closed) }) | |||
| 893 | + | return nil | |||
| 894 | + | } | |||
| 895 | + | ||||
| 896 | + | func TestPreSenderReconnectResetIsBounded(t *testing.T) { | |||
| 897 | + | tests := []struct { | |||
| 898 | + | name string | |||
| 899 | + | trigger func(context.CancelFunc, *reconnectResetTimer) | |||
| 900 | + | wantErr error | |||
| 901 | + | }{ | |||
| 902 | + | { | |||
| 903 | + | name: "timeout", | |||
| 904 | + | trigger: func(_ context.CancelFunc, timer *reconnectResetTimer) { | |||
| 905 | + | timer.ch <- time.Time{} | |||
| 906 | + | }, | |||
| 907 | + | wantErr: context.DeadlineExceeded, | |||
| 908 | + | }, | |||
| 909 | + | { | |||
| 910 | + | name: "cancellation", | |||
| 911 | + | trigger: func(cancel context.CancelFunc, _ *reconnectResetTimer) { | |||
| 912 | + | cancel() | |||
| 913 | + | }, | |||
| 914 | + | wantErr: context.Canceled, | |||
| 915 | + | }, | |||
| 916 | + | } | |||
| 917 | + | ||||
| 918 | + | for _, tt := range tests { | |||
| 919 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 920 | + | term := newReconnectToastTerminalHarness(t) | |||
| 921 | + | defer term.closeInput() | |||
| 922 | + | tr := newReconnectResetBlockingTransport() | |||
| 923 | + | clock := newReconnectResetClock() | |||
| 924 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 925 | + | defer cancel() | |||
| 926 | + | ms := milestones{} | |||
| 927 | + | attempt := newReconnectAttachAttempt(term.term, tr, clock, AttachRequest{Intent: ports.IntentResume, SessionName: "requested", Remote: true}, 12, &terminalThemeState{}, nil, &ms) | |||
| 928 | + | attempt.reconnect.showing = true | |||
| 929 | + | resultCh := make(chan attachResult, 1) | |||
| 930 | + | go func() { resultCh <- attempt.run(ctx) }() | |||
| 931 | + | ||||
| 932 | + | <-clock.preWelcome // Hello Send | |||
| 933 | + | <-clock.preWelcome // Welcome Recv | |||
| 934 | + | resetTimer := <-clock.preWelcome | |||
| 935 | + | select { | |||
| 936 | + | case <-tr.entered: | |||
| 937 | + | case <-time.After(time.Second): | |||
| 938 | + | t.Fatal("timed out waiting for synchronous reconnect reset Send") | |||
| 939 | + | } | |||
| 940 | + | tt.trigger(cancel, resetTimer) | |||
| 941 | + | ||||
| 942 | + | result := requireAttachResult(t, resultCh) | |||
| 943 | + | require.ErrorIs(t, result.err, tt.wantErr) | |||
| 944 | + | require.True(t, result.transportClosed) | |||
| 945 | + | require.True(t, result.welcomed) | |||
| 946 | + | require.Equal(t, uint64(44), result.resumeToken) | |||
| 947 | + | require.Equal(t, "assigned", result.sessionName) | |||
| 948 | + | }) | |||
| 949 | + | } | |||
| 950 | + | } | |||
| 951 | + | ||||
| 952 | + | type reconnectToastBlockingSendTransport struct { | |||
| 953 | + | *reconnectToastLinkTransport | |||
| 954 | + | armed atomic.Bool | |||
| 955 | + | blockType ports.MsgType | |||
| 956 | + | sendErr error | |||
| 957 | + | entered chan struct{} | |||
| 958 | + | release chan struct{} | |||
| 959 | + | enteredOnce sync.Once | |||
| 960 | + | releaseOnce sync.Once | |||
| 961 | + | } | |||
| 962 | + | ||||
| 963 | + | func newReconnectToastBlockingSendTransport() *reconnectToastBlockingSendTransport { | |||
| 964 | + | return &reconnectToastBlockingSendTransport{ | |||
| 965 | + | reconnectToastLinkTransport: newReconnectToastLinkTransport(), | |||
| 966 | + | blockType: ports.MsgClientNotice, | |||
| 967 | + | entered: make(chan struct{}), | |||
| 968 | + | release: make(chan struct{}), | |||
| 969 | + | } | |||
| 970 | + | } | |||
| 971 | + | ||||
| 972 | + | func (t *reconnectToastBlockingSendTransport) Send(frame ports.Frame) error { | |||
| 973 | + | if err := t.reconnectToastLinkTransport.Send(frame); err != nil { | |||
| 974 | + | return err | |||
| 975 | + | } | |||
| 976 | + | if t.armed.Load() && frame.Type == t.blockType { | |||
| 977 | + | t.enteredOnce.Do(func() { close(t.entered) }) | |||
| 978 | + | <-t.release | |||
| 979 | + | return t.sendErr | |||
| 980 | + | } | |||
| 981 | + | return nil | |||
| 982 | + | } | |||
| 983 | + | ||||
| 984 | + | func requireTerminalSizeCalls(t *testing.T, term *reconnectToastTerminalHarness, want int32) { | |||
| 985 | + | t.Helper() | |||
| 986 | + | timer := time.NewTimer(time.Second) | |||
| 987 | + | defer timer.Stop() | |||
| 988 | + | for term.sizeCalls.Load() < want { | |||
| 989 | + | select { | |||
| 990 | + | case <-term.sizeCalled: | |||
| 991 | + | case <-timer.C: | |||
| 992 | + | t.Fatalf("timed out waiting for terminal Size call %d; got %d", want, term.sizeCalls.Load()) | |||
| 993 | + | } | |||
| 994 | + | } | |||
| 995 | + | } | |||
| 996 | + | ||||
| 997 | + | func TestReconnectResetEnqueueCancellationExitsCleanly(t *testing.T) { | |||
| 998 | + | tests := []struct { | |||
| 999 | + | name string | |||
| 1000 | + | transition ports.LinkState | |||
| 1001 | + | }{ | |||
| 1002 | + | {name: "dismissal", transition: ports.LinkStateConnected}, | |||
| 1003 | + | {name: "stage change", transition: ports.LinkStateDead}, | |||
| 1004 | + | } | |||
| 1005 | + | ||||
| 1006 | + | for _, tt := range tests { | |||
| 1007 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 1008 | + | term := newReconnectToastTerminalHarness(t) | |||
| 1009 | + | defer term.closeInput() | |||
| 1010 | + | tr := newReconnectToastBlockingSendTransport() | |||
| 1011 | + | releaseSender := func() { tr.releaseOnce.Do(func() { close(tr.release) }) } | |||
| 1012 | + | defer releaseSender() | |||
| 1013 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |||
| 1014 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 1015 | + | defer cancel() | |||
| 1016 | + | ||||
| 1017 | + | resultCh := make(chan attachResult, 1) | |||
| 1018 | + | ms := milestones{} | |||
| 1019 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |||
| 1020 | + | go func() { resultCh <- attempt.run(ctx) }() | |||
| 1021 | + | ||||
| 1022 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |||
| 1023 | + | requireTerminalSizeCalls(t, term, 2) | |||
| 1024 | + | tr.blockType = ports.MsgResize | |||
| 1025 | + | tr.armed.Store(true) | |||
| 1026 | + | term.resizeCh <- term.size | |||
| 1027 | + | select { | |||
| 1028 | + | case <-tr.entered: | |||
| 1029 | + | case <-time.After(time.Second): | |||
| 1030 | + | t.Fatal("timed out waiting for sender to block") | |||
| 1031 | + | } | |||
| 1032 | + | for range sendQueueDepth { | |||
| 1033 | + | term.resizeCh <- term.size | |||
| 1034 | + | } | |||
| 1035 | + | require.Eventually(t, func() bool { return len(term.resizeCh) == 0 }, time.Second, time.Millisecond) | |||
| 1036 | + | ||||
| 1037 | + | tr.events <- ports.LinkEvent{State: tt.transition} | |||
| 1038 | + | requireTerminalSizeCalls(t, term, 3) | |||
| 1039 | + | cancel() | |||
| 1040 | + | ||||
| 1041 | + | result := requireAttachResult(t, resultCh) | |||
| 1042 | + | require.NoError(t, result.err) | |||
| 1043 | + | require.True(t, result.welcomed) | |||
| 1044 | + | releaseSender() | |||
| 1045 | + | }) | |||
| 1046 | + | } | |||
| 1047 | + | } | |||
| 1048 | + | ||||
| 1049 | + | func TestReconnectResetCancellationPreservesQueuedSenderError(t *testing.T) { | |||
| 1050 | + | term := newReconnectToastTerminalHarness(t) | |||
| 1051 | + | defer term.closeInput() | |||
| 1052 | + | tr := newReconnectToastBlockingSendTransport() | |||
| 1053 | + | releaseSender := func() { tr.releaseOnce.Do(func() { close(tr.release) }) } | |||
| 1054 | + | defer releaseSender() | |||
| 1055 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |||
| 1056 | + | ||||
| 1057 | + | resultCh := make(chan attachResult, 1) | |||
| 1058 | + | ms := milestones{} | |||
| 1059 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |||
| 1060 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |||
| 1061 | + | ||||
| 1062 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateProbing} | |||
| 1063 | + | requireTerminalSizeCalls(t, term, 2) | |||
| 1064 | + | tr.blockType = ports.MsgResize | |||
| 1065 | + | tr.armed.Store(true) | |||
| 1066 | + | term.resizeCh <- term.size | |||
| 1067 | + | select { | |||
| 1068 | + | case <-tr.entered: | |||
| 1069 | + | case <-time.After(time.Second): | |||
| 1070 | + | t.Fatal("timed out waiting for sender to block") | |||
| 1071 | + | } | |||
| 1072 | + | for range sendQueueDepth { | |||
| 1073 | + | term.resizeCh <- term.size | |||
| 1074 | + | } | |||
| 1075 | + | require.Eventually(t, func() bool { return len(term.resizeCh) == 0 }, time.Second, time.Millisecond) | |||
| 1076 | + | ||||
| 1077 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateConnected} | |||
| 1078 | + | requireTerminalSizeCalls(t, term, 3) | |||
| 1079 | + | sendFailure := errors.New("queued sender failure") | |||
| 1080 | + | tr.sendErr = sendFailure | |||
| 1081 | + | releaseSender() | |||
| 1082 | + | ||||
| 1083 | + | result := requireAttachResult(t, resultCh) | |||
| 1084 | + | require.ErrorIs(t, result.err, sendFailure) | |||
| 1085 | + | require.ErrorContains(t, result.err, "sending to daemon") | |||
| 1086 | + | } | |||
| 1087 | + | ||||
| 1088 | + | func TestAttachAttemptReturnsWhileSenderSendIsBlockedAfterCancellation(t *testing.T) { | |||
| 1089 | + | term := newReconnectToastTerminalHarness(t) | |||
| 1090 | + | defer term.closeInput() | |||
| 1091 | + | tr := newReconnectToastBlockingSendTransport() | |||
| 1092 | + | releaseSender := func() { tr.releaseOnce.Do(func() { close(tr.release) }) } | |||
| 1093 | + | defer releaseSender() | |||
| 1094 | + | tr.recvCh <- reconnectToastRecv{frame: reconnectToastWelcome(44)} | |||
| 1095 | + | ||||
| 1096 | + | resultCh := make(chan attachResult, 1) | |||
| 1097 | + | ms := milestones{} | |||
| 1098 | + | attempt := newReconnectAttachAttempt(term.term, tr, newReconnectHandshakeClock(t), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}, 0, &terminalThemeState{}, tr.LinkEvents(), &ms) | |||
| 1099 | + | go func() { resultCh <- attempt.run(context.Background()) }() | |||
| 1100 | + | ||||
| 1101 | + | tr.armed.Store(true) | |||
| 1102 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateDegraded} | |||
| 1103 | + | select { | |||
| 1104 | + | case <-tr.entered: | |||
| 1105 | + | case <-time.After(time.Second): | |||
| 1106 | + | t.Fatal("timed out waiting for sender to block in transport Send") | |||
| 1107 | + | } | |||
| 1108 | + | tr.events <- ports.LinkEvent{State: ports.LinkStateOffline} | |||
| 1109 | + | ||||
| 1110 | + | select { | |||
| 1111 | + | case result := <-resultCh: | |||
| 1112 | + | require.ErrorIs(t, result.err, errLinkOffline) | |||
| 1113 | + | case <-time.After(time.Second): | |||
| 1114 | + | releaseSender() | |||
| 1115 | + | requireAttachResult(t, resultCh) | |||
| 1116 | + | t.Fatal("attach attempt waited for the blocked sender before returning control to Runner") | |||
| 1117 | + | } | |||
| 1118 | + | releaseSender() | |||
| 1119 | + | } | |||
| 1120 | + | ||||
| 339 | func TestDetachedResultReplacementAndCleanDetach(t *testing.T) { | 1121 | func TestDetachedResultReplacementAndCleanDetach(t *testing.T) { | ||
| 340 | for _, tt := range []struct { | 1122 | for _, tt := range []struct { | ||
| 341 | name string | 1123 | name string | ||
| @@ -363,18 +1145,12 @@ func TestReconnectToastBoundsUseCenterAnchor(t *testing.T) { | |||||
| 363 | require.Equal(t, domain.Rect{X: 29, Y: 10, Width: 22, Height: 3}, bounds) | 1145 | require.Equal(t, domain.Rect{X: 29, Y: 10, Width: 22, Height: 3}, bounds) | ||
| 364 | } | 1146 | } | ||
| 365 | 1147 | ||||
| 366 | - | func TestReconnectToastDrawAndClearHelpers(t *testing.T) { | 1148 | + | func TestReconnectToastDrawHelper(t *testing.T) { |
| 367 | var out bytes.Buffer | 1149 | var out bytes.Buffer | ||
| 368 | - | size := domain.Size{Cols: 80, Rows: 24} | 1150 | + | require.NoError(t, drawReconnectToast(&out, domain.Size{Cols: 80, Rows: 24})) |
| 369 | - | ||||
| 370 | - | require.NoError(t, drawReconnectToast(&out, size)) | |||
| 371 | require.Contains(t, out.String(), "┌") | 1151 | require.Contains(t, out.String(), "┌") | ||
| 372 | require.Contains(t, out.String(), reconnectToastMessage) | 1152 | require.Contains(t, out.String(), reconnectToastMessage) | ||
| 373 | require.Contains(t, out.String(), "\x1b[0m") | 1153 | require.Contains(t, out.String(), "\x1b[0m") | ||
| 374 | - | ||||
| 375 | - | bounds := reconnectToastBounds(size) | |||
| 376 | - | require.NoError(t, clearReconnectToast(&out, bounds)) | |||
| 377 | - | require.Contains(t, out.String(), strings.Repeat(" ", bounds.Width)) | |||
| 378 | } | 1154 | } | ||
| 379 | 1155 | ||||
| 380 | func TestReconnectToastLinesClampToBounds(t *testing.T) { | 1156 | func TestReconnectToastLinesClampToBounds(t *testing.T) { | ||
| @@ -409,61 +1185,17 @@ func legacyReconnectToastStage(out io.Writer, size domain.Size, stage reconnectS | |||||
| 409 | return bounds, writeReconnectToast(out, bounds, reconnectToastLinesFor(bounds, message)) | 1185 | return bounds, writeReconnectToast(out, bounds, reconnectToastLinesFor(bounds, message)) | ||
| 410 | } | 1186 | } | ||
| 411 | 1187 | ||||
| 412 | - | func TestReconnectToastStageTransitionsClearDrawnBounds(t *testing.T) { | 1188 | + | func TestReconnectToastStagesDrawTheirCurrentMessage(t *testing.T) { |
| 413 | - | size := domain.Size{Cols: 80, Rows: 24} | 1189 | + | for _, stage := range []reconnectStage{reconnectStageDegraded, reconnectStageProbingUDP, reconnectStageSSH, reconnectStageOfflineRetrying} { |
| 414 | - | tests := []struct { | 1190 | + | t.Run(reconnectStageMessage(stage), func(t *testing.T) { |
| 415 | - | name string | |||
| 416 | - | stages []reconnectStage | |||
| 417 | - | }{ | |||
| 418 | - | { | |||
| 419 | - | name: "degraded to probing to ssh to offline", | |||
| 420 | - | stages: []reconnectStage{reconnectStageDegraded, reconnectStageProbingUDP, reconnectStageSSH, reconnectStageOfflineRetrying}, | |||
| 421 | - | }, | |||
| 422 | - | { | |||
| 423 | - | name: "widest ssh to offline", | |||
| 424 | - | stages: []reconnectStage{reconnectStageSSH, reconnectStageOfflineRetrying}, | |||
| 425 | - | }, | |||
| 426 | - | } | |||
| 427 | - | ||||
| 428 | - | for _, tt := range tests { | |||
| 429 | - | t.Run(tt.name, func(t *testing.T) { | |||
| 430 | var out bytes.Buffer | 1191 | var out bytes.Buffer | ||
| 431 | - | previousBounds, err := drawReconnectToastStage(&out, size, tt.stages[0]) | 1192 | + | _, err := drawReconnectToastStage(&out, domain.Size{Cols: 80, Rows: 24}, stage) |
| 432 | require.NoError(t, err) | 1193 | require.NoError(t, err) | ||
| 433 | - | 1194 | + | require.Contains(t, out.String(), reconnectStageMessage(stage)) | |
| 434 | - | for _, stage := range tt.stages[1:] { | |||
| 435 | - | out.Reset() | |||
| 436 | - | require.NoError(t, clearReconnectToast(&out, previousBounds)) | |||
| 437 | - | assertReconnectToastClearCoversBounds(t, out.String(), previousBounds) | |||
| 438 | - | ||||
| 439 | - | out.Reset() | |||
| 440 | - | previousBounds, err = drawReconnectToastStage(&out, size, stage) | |||
| 441 | - | require.NoError(t, err) | |||
| 442 | - | } | |||
| 443 | }) | 1195 | }) | ||
| 444 | } | 1196 | } | ||
| 445 | } | 1197 | } | ||
| 446 | 1198 | ||||
| 447 | - | func TestReconnectToastClearAfterWidestStageErasesFullBorder(t *testing.T) { | |||
| 448 | - | size := domain.Size{Cols: 80, Rows: 24} | |||
| 449 | - | var out bytes.Buffer | |||
| 450 | - | bounds, err := drawReconnectToastStage(&out, size, reconnectStageSSH) | |||
| 451 | - | require.NoError(t, err) | |||
| 452 | - | require.Greater(t, bounds.Width, reconnectToastBounds(size).Width) | |||
| 453 | - | ||||
| 454 | - | out.Reset() | |||
| 455 | - | require.NoError(t, clearReconnectToast(&out, bounds)) | |||
| 456 | - | assertReconnectToastClearCoversBounds(t, out.String(), bounds) | |||
| 457 | - | } | |||
| 458 | - | ||||
| 459 | - | func assertReconnectToastClearCoversBounds(t *testing.T, output string, bounds domain.Rect) { | |||
| 460 | - | t.Helper() | |||
| 461 | - | blank := strings.Repeat(" ", bounds.Width) | |||
| 462 | - | for row := range bounds.Height { | |||
| 463 | - | require.Contains(t, output, fmt.Sprintf("\x1b[%d;%dH\x1b[0m%s", bounds.Y+row+1, bounds.X+1, blank)) | |||
| 464 | - | } | |||
| 465 | - | } | |||
| 466 | - | ||||
| 467 | func displayWidth(s string) int { | 1199 | func displayWidth(s string) int { | ||
| 468 | width := 0 | 1200 | width := 0 | ||
| 469 | for _, r := range s { | 1201 | for _, r := range s { | ||
| @@ -483,18 +1215,44 @@ func TestReconnectSleepWithResizeEventsRedrawsUntilTimerFires(t *testing.T) { | |||||
| 483 | resizeCh <- domain.Size{Cols: 100, Rows: 30} | 1215 | resizeCh <- domain.Size{Cols: 100, Rows: 30} | ||
| 484 | resizeCh <- domain.Size{Cols: 120, Rows: 40} | 1216 | resizeCh <- domain.Size{Cols: 120, Rows: 40} | ||
| 485 | got := make(chan domain.Size, 2) | 1217 | got := make(chan domain.Size, 2) | ||
| 486 | - | done := make(chan bool, 1) | 1218 | + | type sleepResult struct { |
| 1219 | + | slept bool | |||
| 1220 | + | err error | |||
| 1221 | + | } | |||
| 1222 | + | done := make(chan sleepResult, 1) | |||
| 487 | 1223 | ||||
| 488 | go func() { | 1224 | go func() { | ||
| 489 | - | done <- sleepReconnectWithResizeEvents(context.Background(), clk, time.Hour, resizeCh, func(size domain.Size) { | 1225 | + | slept, err := sleepReconnectWithResizeEvents(context.Background(), clk, time.Hour, resizeCh, func(size domain.Size) error { |
| 490 | got <- size | 1226 | got <- size | ||
| 1227 | + | return nil | |||
| 491 | }) | 1228 | }) | ||
| 1229 | + | done <- sleepResult{slept: slept, err: err} | |||
| 492 | }() | 1230 | }() | ||
| 493 | 1231 | ||||
| 494 | require.Equal(t, domain.Size{Cols: 100, Rows: 30}, <-got) | 1232 | require.Equal(t, domain.Size{Cols: 100, Rows: 30}, <-got) | ||
| 495 | require.Equal(t, domain.Size{Cols: 120, Rows: 40}, <-got) | 1233 | require.Equal(t, domain.Size{Cols: 120, Rows: 40}, <-got) | ||
| 496 | timerCh <- time.Time{} | 1234 | timerCh <- time.Time{} | ||
| 497 | - | require.True(t, <-done) | 1235 | + | result := <-done |
| 1236 | + | require.NoError(t, result.err) | |||
| 1237 | + | require.True(t, result.slept) | |||
| 1238 | + | } | |||
| 1239 | + | ||||
| 1240 | + | func TestReconnectSleepWithResizeEventsReturnsRedrawError(t *testing.T) { | |||
| 1241 | + | clk := portsmocks.NewMockClock(t) | |||
| 1242 | + | timer := portsmocks.NewMockTimer(t) | |||
| 1243 | + | clk.EXPECT().NewTimer(time.Hour).Return(timer).Once() | |||
| 1244 | + | timer.EXPECT().C().Return((<-chan time.Time)(make(chan time.Time))).Maybe() | |||
| 1245 | + | timer.EXPECT().Stop().Return(true).Once() | |||
| 1246 | + | resizeCh := make(chan domain.Size, 1) | |||
| 1247 | + | resizeCh <- domain.Size{Cols: 100, Rows: 30} | |||
| 1248 | + | redrawErr := errors.New("redraw failed") | |||
| 1249 | + | ||||
| 1250 | + | slept, err := sleepReconnectWithResizeEvents(context.Background(), clk, time.Hour, resizeCh, func(domain.Size) error { | |||
| 1251 | + | return redrawErr | |||
| 1252 | + | }) | |||
| 1253 | + | ||||
| 1254 | + | require.False(t, slept) | |||
| 1255 | + | require.ErrorIs(t, err, redrawErr) | |||
| 498 | } | 1256 | } | ||
| 499 | 1257 | ||||
| 500 | type reconnectToastFailOnceWriter struct { | 1258 | type reconnectToastFailOnceWriter struct { | ||
| @@ -512,14 +1270,120 @@ func (w *reconnectToastFailOnceWriter) Write(p []byte) (int, error) { | |||||
| 512 | return n, nil | 1270 | return n, nil | ||
| 513 | } | 1271 | } | ||
| 514 | 1272 | ||||
| 515 | - | func TestRemoteReconnectToastClearsFailedDrawBounds(t *testing.T) { | 1273 | + | func TestReconnectRetryFailuresPreserveAttachError(t *testing.T) { |
| 1274 | + | attachErr := errors.New("original attach failure") | |||
| 1275 | + | retryErr := errors.New("retry presentation failure") | |||
| 1276 | + | tests := []struct { | |||
| 1277 | + | name string | |||
| 1278 | + | output func() io.Writer | |||
| 1279 | + | retrySleep func(func(domain.Size) error) (bool, error) | |||
| 1280 | + | }{ | |||
| 1281 | + | { | |||
| 1282 | + | name: "draw stage", | |||
| 1283 | + | output: func() io.Writer { | |||
| 1284 | + | writer := &armedReconnectToastWriter{next: io.Discard, needle: reconnectStageMessage(reconnectStageSSH), failure: retryErr} | |||
| 1285 | + | writer.armed.Store(true) | |||
| 1286 | + | return writer | |||
| 1287 | + | }, | |||
| 1288 | + | retrySleep: func(func(domain.Size) error) (bool, error) { return true, nil }, | |||
| 1289 | + | }, | |||
| 1290 | + | { | |||
| 1291 | + | name: "sleep", | |||
| 1292 | + | output: func() io.Writer { return io.Discard }, | |||
| 1293 | + | retrySleep: func(func(domain.Size) error) (bool, error) { | |||
| 1294 | + | return false, retryErr | |||
| 1295 | + | }, | |||
| 1296 | + | }, | |||
| 1297 | + | } | |||
| 1298 | + | ||||
| 1299 | + | for _, tt := range tests { | |||
| 1300 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 1301 | + | oldSleepWithResize := reconnectSleepWithResize | |||
| 1302 | + | reconnectSleepWithResize = func(_ context.Context, _ ports.Clock, _ time.Duration, _ <-chan domain.Size, redraw func(domain.Size) error) (bool, error) { | |||
| 1303 | + | return tt.retrySleep(redraw) | |||
| 1304 | + | } | |||
| 1305 | + | defer func() { reconnectSleepWithResize = oldSleepWithResize }() | |||
| 1306 | + | ||||
| 1307 | + | term := newReconnectToastTerminalHarnessWithOutput(t, tt.output()) | |||
| 1308 | + | defer term.closeInput() | |||
| 1309 | + | tr := newReconnectToastRecordingTransport( | |||
| 1310 | + | reconnectToastRecv{frame: reconnectToastWelcome(44)}, | |||
| 1311 | + | reconnectToastRecv{err: attachErr}, | |||
| 1312 | + | ) | |||
| 1313 | + | dialer := &reconnectToastSequenceDialer{transports: []ports.Transport{tr}} | |||
| 1314 | + | ||||
| 1315 | + | err := NewRunner(Dependencies{Dialer: dialer, Terminal: term.term, Clock: newReconnectHandshakeClock(t), Logger: slog.New(slog.DiscardHandler)}).Run(context.Background(), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}) | |||
| 1316 | + | require.ErrorIs(t, err, attachErr) | |||
| 1317 | + | require.ErrorIs(t, err, retryErr) | |||
| 1318 | + | }) | |||
| 1319 | + | } | |||
| 1320 | + | } | |||
| 1321 | + | ||||
| 1322 | + | func TestReconnectRetrySleepFailuresPreserveDialError(t *testing.T) { | |||
| 1323 | + | dialErr := errors.New("original reconnect dial failure") | |||
| 1324 | + | retryErr := errors.New("retry presentation failure") | |||
| 1325 | + | tests := []struct { | |||
| 1326 | + | name string | |||
| 1327 | + | failSecond func(func(domain.Size) error) (bool, error) | |||
| 1328 | + | }{ | |||
| 1329 | + | { | |||
| 1330 | + | name: "resize redraw", | |||
| 1331 | + | failSecond: func(redraw func(domain.Size) error) (bool, error) { | |||
| 1332 | + | return false, redraw(domain.Size{Cols: 100, Rows: 30}) | |||
| 1333 | + | }, | |||
| 1334 | + | }, | |||
| 1335 | + | { | |||
| 1336 | + | name: "sleep", | |||
| 1337 | + | failSecond: func(func(domain.Size) error) (bool, error) { | |||
| 1338 | + | return false, retryErr | |||
| 1339 | + | }, | |||
| 1340 | + | }, | |||
| 1341 | + | } | |||
| 1342 | + | ||||
| 1343 | + | for _, tt := range tests { | |||
| 1344 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 1345 | + | oldSleepWithResize := reconnectSleepWithResize | |||
| 1346 | + | calls := 0 | |||
| 1347 | + | reconnectSleepWithResize = func(_ context.Context, _ ports.Clock, _ time.Duration, _ <-chan domain.Size, redraw func(domain.Size) error) (bool, error) { | |||
| 1348 | + | calls++ | |||
| 1349 | + | if calls == 1 { | |||
| 1350 | + | return true, nil | |||
| 1351 | + | } | |||
| 1352 | + | return tt.failSecond(redraw) | |||
| 1353 | + | } | |||
| 1354 | + | defer func() { reconnectSleepWithResize = oldSleepWithResize }() | |||
| 1355 | + | ||||
| 1356 | + | var armed atomic.Bool | |||
| 1357 | + | term := newReconnectToastTerminalHarnessWithOutputAndFlush(t, io.Discard, func() error { | |||
| 1358 | + | if armed.Load() { | |||
| 1359 | + | return retryErr | |||
| 1360 | + | } | |||
| 1361 | + | return nil | |||
| 1362 | + | }) | |||
| 1363 | + | defer term.closeInput() | |||
| 1364 | + | tr := newReconnectToastRecordingTransport( | |||
| 1365 | + | reconnectToastRecv{frame: reconnectToastWelcome(44)}, | |||
| 1366 | + | reconnectToastRecv{err: io.EOF}, | |||
| 1367 | + | ) | |||
| 1368 | + | dialer := portsmocks.NewMockDialer(t) | |||
| 1369 | + | dialer.EXPECT().Dial(mock.Anything).Return(tr, nil).Once() | |||
| 1370 | + | dialer.EXPECT().Dial(mock.Anything).Run(func(context.Context) { armed.Store(true) }).Return(nil, dialErr).Once() | |||
| 1371 | + | ||||
| 1372 | + | err := NewRunner(Dependencies{Dialer: dialer, Terminal: term.term, Clock: newReconnectHandshakeClock(t), Logger: slog.New(slog.DiscardHandler)}).Run(context.Background(), AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}) | |||
| 1373 | + | require.ErrorIs(t, err, dialErr) | |||
| 1374 | + | require.ErrorIs(t, err, retryErr) | |||
| 1375 | + | }) | |||
| 1376 | + | } | |||
| 1377 | + | } | |||
| 1378 | + | ||||
| 1379 | + | func TestRemoteReconnectToastFailedDrawDoesNotBlankBounds(t *testing.T) { | |||
| 516 | oldSleep := reconnectSleep | 1380 | oldSleep := reconnectSleep | ||
| 517 | oldSleepWithResize := reconnectSleepWithResize | 1381 | oldSleepWithResize := reconnectSleepWithResize | ||
| 518 | ctx, cancel := context.WithCancel(context.Background()) | 1382 | ctx, cancel := context.WithCancel(context.Background()) | ||
| 519 | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return false } | 1383 | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return false } | ||
| 520 | - | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size)) bool { | 1384 | + | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size) error) (bool, error) { |
| 521 | cancel() | 1385 | cancel() | ||
| 522 | - | return false | 1386 | + | return false, nil |
| 523 | } | 1387 | } | ||
| 524 | defer func() { | 1388 | defer func() { | ||
| 525 | reconnectSleep = oldSleep | 1389 | reconnectSleep = oldSleep | ||
| @@ -536,9 +1400,87 @@ func TestRemoteReconnectToastClearsFailedDrawBounds(t *testing.T) { | |||||
| 536 | dialer := &reconnectToastSequenceDialer{transports: []ports.Transport{tr}} | 1400 | dialer := &reconnectToastSequenceDialer{transports: []ports.Transport{tr}} | ||
| 537 | 1401 | ||||
| 538 | err := NewRunner(Dependencies{Dialer: dialer, Terminal: term.term, Clock: newReconnectHandshakeClock(t), Logger: slog.New(slog.DiscardHandler)}).Run(ctx, AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}) | 1402 | err := NewRunner(Dependencies{Dialer: dialer, Terminal: term.term, Clock: newReconnectHandshakeClock(t), Logger: slog.New(slog.DiscardHandler)}).Run(ctx, AttachRequest{Intent: ports.IntentAttach, SessionName: "main", Remote: true}) | ||
| 539 | - | require.ErrorIs(t, err, context.Canceled) | 1403 | + | require.ErrorContains(t, err, "reconnect toast write failed") |
| 540 | require.True(t, out.failed) | 1404 | require.True(t, out.failed) | ||
| 541 | - | assertReconnectToastClearCoversBounds(t, out.String(), reconnectToastBoundsFor(term.size, reconnectStageMessage(reconnectStageSSH))) | 1405 | + | require.NotContains(t, out.String(), strings.Repeat(" ", reconnectToastBoundsFor(term.size, reconnectStageMessage(reconnectStageSSH)).Width)) |
| 1406 | + | } | |||
| 1407 | + | ||||
| 1408 | + | type reconnectStatusClearFailWriter struct { | |||
| 1409 | + | clearErr error | |||
| 1410 | + | } | |||
| 1411 | + | ||||
| 1412 | + | func (w *reconnectStatusClearFailWriter) Write(p []byte) (int, error) { | |||
| 1413 | + | if string(p) == statusClear { | |||
| 1414 | + | return 0, w.clearErr | |||
| 1415 | + | } | |||
| 1416 | + | return len(p), nil | |||
| 1417 | + | } | |||
| 1418 | + | ||||
| 1419 | + | func TestReconnectCancellationPreservesContextWhenStatusClearFails(t *testing.T) { | |||
| 1420 | + | tests := []struct { | |||
| 1421 | + | name string | |||
| 1422 | + | configure func(*portsmocks.MockDialer, ports.Transport) | |||
| 1423 | + | sleep func(context.CancelFunc) bool | |||
| 1424 | + | }{ | |||
| 1425 | + | { | |||
| 1426 | + | name: "after attach failure", | |||
| 1427 | + | configure: func(dialer *portsmocks.MockDialer, tr ports.Transport) { | |||
| 1428 | + | dialer.EXPECT().Dial(mock.Anything).Return(tr, nil).Once() | |||
| 1429 | + | }, | |||
| 1430 | + | sleep: func(cancel context.CancelFunc) bool { | |||
| 1431 | + | cancel() | |||
| 1432 | + | return false | |||
| 1433 | + | }, | |||
| 1434 | + | }, | |||
| 1435 | + | { | |||
| 1436 | + | name: "after reconnect dial failure", | |||
| 1437 | + | configure: func(dialer *portsmocks.MockDialer, tr ports.Transport) { | |||
| 1438 | + | dialer.EXPECT().Dial(mock.Anything).Return(tr, nil).Once() | |||
| 1439 | + | dialer.EXPECT().Dial(mock.Anything).Return(nil, errors.New("redial failed")).Once() | |||
| 1440 | + | }, | |||
| 1441 | + | sleep: func() func(context.CancelFunc) bool { | |||
| 1442 | + | calls := 0 | |||
| 1443 | + | return func(cancel context.CancelFunc) bool { | |||
| 1444 | + | calls++ | |||
| 1445 | + | if calls == 1 { | |||
| 1446 | + | return true | |||
| 1447 | + | } | |||
| 1448 | + | cancel() | |||
| 1449 | + | return false | |||
| 1450 | + | } | |||
| 1451 | + | }(), | |||
| 1452 | + | }, | |||
| 1453 | + | } | |||
| 1454 | + | ||||
| 1455 | + | for _, tt := range tests { | |||
| 1456 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 1457 | + | oldSleep := reconnectSleep | |||
| 1458 | + | oldSleepWithResize := reconnectSleepWithResize | |||
| 1459 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 1460 | + | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return tt.sleep(cancel) } | |||
| 1461 | + | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size) error) (bool, error) { | |||
| 1462 | + | return tt.sleep(cancel), nil | |||
| 1463 | + | } | |||
| 1464 | + | defer func() { | |||
| 1465 | + | reconnectSleep = oldSleep | |||
| 1466 | + | reconnectSleepWithResize = oldSleepWithResize | |||
| 1467 | + | }() | |||
| 1468 | + | ||||
| 1469 | + | clearErr := errors.New("status clear failed") | |||
| 1470 | + | term := newReconnectToastTerminalHarnessWithOutput(t, &reconnectStatusClearFailWriter{clearErr: clearErr}) | |||
| 1471 | + | defer term.closeInput() | |||
| 1472 | + | tr := newReconnectToastRecordingTransport( | |||
| 1473 | + | reconnectToastRecv{frame: reconnectToastWelcome(11)}, | |||
| 1474 | + | reconnectToastRecv{err: io.EOF}, | |||
| 1475 | + | ) | |||
| 1476 | + | dialer := portsmocks.NewMockDialer(t) | |||
| 1477 | + | tt.configure(dialer, tr) | |||
| 1478 | + | ||||
| 1479 | + | err := NewRunner(Dependencies{Dialer: dialer, Terminal: term.term, Clock: newReconnectHandshakeClock(t), Logger: slog.New(slog.DiscardHandler)}).Run(ctx, AttachRequest{Intent: ports.IntentAttach, SessionName: "main"}) | |||
| 1480 | + | require.ErrorIs(t, err, context.Canceled) | |||
| 1481 | + | require.ErrorIs(t, err, clearErr) | |||
| 1482 | + | }) | |||
| 1483 | + | } | |||
| 542 | } | 1484 | } | ||
| 543 | 1485 | ||||
| 544 | func TestRemoteReconnectToastLifecycleWithWrappedTransportError(t *testing.T) { | 1486 | func TestRemoteReconnectToastLifecycleWithWrappedTransportError(t *testing.T) { | ||
| @@ -550,8 +1492,8 @@ func TestRemoteReconnectToastLifecycleWithWrappedTransportError(t *testing.T) { | |||||
| 550 | oldSleep := reconnectSleep | 1492 | oldSleep := reconnectSleep | ||
| 551 | oldSleepWithResize := reconnectSleepWithResize | 1493 | oldSleepWithResize := reconnectSleepWithResize | ||
| 552 | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return true } | 1494 | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return true } | ||
| 553 | - | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size)) bool { | 1495 | + | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size) error) (bool, error) { |
| 554 | - | return true | 1496 | + | return true, nil |
| 555 | } | 1497 | } | ||
| 556 | defer func() { | 1498 | defer func() { | ||
| 557 | reconnectSleep = oldSleep | 1499 | reconnectSleep = oldSleep | ||
| @@ -586,7 +1528,7 @@ func TestRemoteReconnectToastLifecycleWithWrappedTransportError(t *testing.T) { | |||||
| 586 | 1528 | ||||
| 587 | out := term.out.String() | 1529 | out := term.out.String() | ||
| 588 | require.Contains(t, out, reconnectToastMessage) | 1530 | require.Contains(t, out, reconnectToastMessage) | ||
| 589 | - | require.Contains(t, out, strings.Repeat(" ", reconnectToastBounds(term.size).Width)) | 1531 | + | require.NotContains(t, out, strings.Repeat(" ", reconnectToastBounds(term.size).Width)) |
| 590 | } | 1532 | } | ||
| 591 | 1533 | ||||
| 592 | func TestRemoteEphemeralReconnectUsesAssignedSessionName(t *testing.T) { | 1534 | func TestRemoteEphemeralReconnectUsesAssignedSessionName(t *testing.T) { | ||
| @@ -594,8 +1536,8 @@ func TestRemoteEphemeralReconnectUsesAssignedSessionName(t *testing.T) { | |||||
| 594 | oldSleep := reconnectSleep | 1536 | oldSleep := reconnectSleep | ||
| 595 | oldSleepWithResize := reconnectSleepWithResize | 1537 | oldSleepWithResize := reconnectSleepWithResize | ||
| 596 | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return true } | 1538 | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return true } | ||
| 597 | - | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size)) bool { | 1539 | + | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size) error) (bool, error) { |
| 598 | - | return true | 1540 | + | return true, nil |
| 599 | } | 1541 | } | ||
| 600 | defer func() { | 1542 | defer func() { | ||
| 601 | reconnectSleep = oldSleep | 1543 | reconnectSleep = oldSleep | ||
| @@ -636,7 +1578,6 @@ func TestRemoteReconnectToastLifecycle(t *testing.T) { | |||||
| 636 | configure func(t *testing.T, dialer *portsmocks.MockDialer) []*reconnectToastRecordingTransport | 1578 | configure func(t *testing.T, dialer *portsmocks.MockDialer) []*reconnectToastRecordingTransport | ||
| 637 | sleep func(cancel context.CancelFunc) bool | 1579 | sleep func(cancel context.CancelFunc) bool | ||
| 638 | wantErr func(t *testing.T, err error) | 1580 | wantErr func(t *testing.T, err error) | ||
| 639 | - | wantBlank bool | |||
| 640 | }{ | 1581 | }{ | ||
| 641 | { | 1582 | { | ||
| 642 | name: "clears on successful reconnect", | 1583 | name: "clears on successful reconnect", | ||
| @@ -647,9 +1588,8 @@ func TestRemoteReconnectToastLifecycle(t *testing.T) { | |||||
| 647 | dialer.EXPECT().Dial(mock.Anything).Return(tr2, nil).Once() | 1588 | dialer.EXPECT().Dial(mock.Anything).Return(tr2, nil).Once() | ||
| 648 | return []*reconnectToastRecordingTransport{tr1, tr2} | 1589 | return []*reconnectToastRecordingTransport{tr1, tr2} | ||
| 649 | }, | 1590 | }, | ||
| 650 | - | sleep: func(context.CancelFunc) bool { return true }, | 1591 | + | sleep: func(context.CancelFunc) bool { return true }, |
| 651 | - | wantErr: func(t *testing.T, err error) { require.NoError(t, err) }, | 1592 | + | wantErr: func(t *testing.T, err error) { require.NoError(t, err) }, |
| 652 | - | wantBlank: true, | |||
| 653 | }, | 1593 | }, | ||
| 654 | { | 1594 | { | ||
| 655 | name: "clears on cancellation", | 1595 | name: "clears on cancellation", | ||
| @@ -662,8 +1602,7 @@ func TestRemoteReconnectToastLifecycle(t *testing.T) { | |||||
| 662 | cancel() | 1602 | cancel() | ||
| 663 | return false | 1603 | return false | ||
| 664 | }, | 1604 | }, | ||
| 665 | - | wantErr: func(t *testing.T, err error) { require.ErrorIs(t, err, context.Canceled) }, | 1605 | + | wantErr: func(t *testing.T, err error) { require.ErrorIs(t, err, context.Canceled) }, |
| 666 | - | wantBlank: true, | |||
| 667 | }, | 1606 | }, | ||
| 668 | { | 1607 | { | ||
| 669 | name: "clears on final exit", | 1608 | name: "clears on final exit", | ||
| @@ -679,7 +1618,6 @@ func TestRemoteReconnectToastLifecycle(t *testing.T) { | |||||
| 679 | var detached *DetachedError | 1618 | var detached *DetachedError | ||
| 680 | require.True(t, errors.As(err, &detached)) | 1619 | require.True(t, errors.As(err, &detached)) | ||
| 681 | }, | 1620 | }, | ||
| 682 | - | wantBlank: true, | |||
| 683 | }, | 1621 | }, | ||
| 684 | } | 1622 | } | ||
| 685 | 1623 | ||||
| @@ -689,8 +1627,8 @@ func TestRemoteReconnectToastLifecycle(t *testing.T) { | |||||
| 689 | oldSleepWithResize := reconnectSleepWithResize | 1627 | oldSleepWithResize := reconnectSleepWithResize | ||
| 690 | ctx, cancel := context.WithCancel(context.Background()) | 1628 | ctx, cancel := context.WithCancel(context.Background()) | ||
| 691 | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return tt.sleep(cancel) } | 1629 | reconnectSleep = func(context.Context, ports.Clock, time.Duration) bool { return tt.sleep(cancel) } | ||
| 692 | - | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size)) bool { | 1630 | + | reconnectSleepWithResize = func(context.Context, ports.Clock, time.Duration, <-chan domain.Size, func(domain.Size) error) (bool, error) { |
| 693 | - | return tt.sleep(cancel) | 1631 | + | return tt.sleep(cancel), nil |
| 694 | } | 1632 | } | ||
| 695 | defer func() { | 1633 | defer func() { | ||
| 696 | reconnectSleep = oldSleep | 1634 | reconnectSleep = oldSleep | ||
| @@ -709,9 +1647,7 @@ func TestRemoteReconnectToastLifecycle(t *testing.T) { | |||||
| 709 | } | 1647 | } | ||
| 710 | out := term.out.String() | 1648 | out := term.out.String() | ||
| 711 | require.Contains(t, out, reconnectToastMessage) | 1649 | require.Contains(t, out, reconnectToastMessage) | ||
| 712 | - | if tt.wantBlank { | 1650 | + | require.NotContains(t, out, strings.Repeat(" ", reconnectToastBounds(term.size).Width)) |
| 713 | - | require.Contains(t, out, strings.Repeat(" ", reconnectToastBounds(term.size).Width)) | |||
| 714 | - | } | |||
| 715 | }) | 1651 | }) | ||
| 716 | } | 1652 | } | ||
| 717 | } | 1653 | } | ||