refactor(daemon): harden snapshot coordination
- commit
- 476768f2fe489ada008ce4165f9cee6e066b70b3
- author
- brice · · 2026-07-22 01:22 UTC
- parent
- d4589c7
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
16 files · +1340 -1292
M internal/ports/mocks/store_mock.go +114 -141
| @@ -420,6 +420,63 @@ func (_c *MockSnapshotRepository_Delete_Call) RunAndReturn(run func(context1 con | |||
| 420 | 420 | return _c | |
| 421 | 421 | } | |
| 422 | 422 | ||
| 423 | + | // DeleteTombstone provides a mock function for the type MockSnapshotRepository | |
| 424 | + | func (_mock *MockSnapshotRepository) DeleteTombstone(context1 context.Context, s string) error { | |
| 425 | + | ret := _mock.Called(context1, s) | |
| 426 | + | ||
| 427 | + | if len(ret) == 0 { | |
| 428 | + | panic("no return value specified for DeleteTombstone") | |
| 429 | + | } | |
| 430 | + | ||
| 431 | + | var r0 error | |
| 432 | + | if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok { | |
| 433 | + | r0 = returnFunc(context1, s) | |
| 434 | + | } else { | |
| 435 | + | r0 = ret.Error(0) | |
| 436 | + | } | |
| 437 | + | return r0 | |
| 438 | + | } | |
| 439 | + | ||
| 440 | + | // MockSnapshotRepository_DeleteTombstone_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteTombstone' | |
| 441 | + | type MockSnapshotRepository_DeleteTombstone_Call struct { | |
| 442 | + | *mock.Call | |
| 443 | + | } | |
| 444 | + | ||
| 445 | + | // DeleteTombstone is a helper method to define mock.On call | |
| 446 | + | // - context1 context.Context | |
| 447 | + | // - s string | |
| 448 | + | func (_e *MockSnapshotRepository_Expecter) DeleteTombstone(context1 any, s any) *MockSnapshotRepository_DeleteTombstone_Call { | |
| 449 | + | return &MockSnapshotRepository_DeleteTombstone_Call{Call: _e.mock.On("DeleteTombstone", context1, s)} | |
| 450 | + | } | |
| 451 | + | ||
| 452 | + | func (_c *MockSnapshotRepository_DeleteTombstone_Call) Run(run func(context1 context.Context, s string)) *MockSnapshotRepository_DeleteTombstone_Call { | |
| 453 | + | _c.Call.Run(func(args mock.Arguments) { | |
| 454 | + | var arg0 context.Context | |
| 455 | + | if args[0] != nil { | |
| 456 | + | arg0 = args[0].(context.Context) | |
| 457 | + | } | |
| 458 | + | var arg1 string | |
| 459 | + | if args[1] != nil { | |
| 460 | + | arg1 = args[1].(string) | |
| 461 | + | } | |
| 462 | + | run( | |
| 463 | + | arg0, | |
| 464 | + | arg1, | |
| 465 | + | ) | |
| 466 | + | }) | |
| 467 | + | return _c | |
| 468 | + | } | |
| 469 | + | ||
| 470 | + | func (_c *MockSnapshotRepository_DeleteTombstone_Call) Return(err error) *MockSnapshotRepository_DeleteTombstone_Call { | |
| 471 | + | _c.Call.Return(err) | |
| 472 | + | return _c | |
| 473 | + | } | |
| 474 | + | ||
| 475 | + | func (_c *MockSnapshotRepository_DeleteTombstone_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockSnapshotRepository_DeleteTombstone_Call { | |
| 476 | + | _c.Call.Return(run) | |
| 477 | + | return _c | |
| 478 | + | } | |
| 479 | + | ||
| 423 | 480 | // List provides a mock function for the type MockSnapshotRepository | |
| 424 | 481 | func (_mock *MockSnapshotRepository) List(context1 context.Context) ([]string, error) { | |
| 425 | 482 | ret := _mock.Called(context1) | |
| @@ -656,6 +713,63 @@ func (_c *MockSnapshotRepository_Publish_Call) RunAndReturn(run func(context1 co | |||
| 656 | 713 | return _c | |
| 657 | 714 | } | |
| 658 | 715 | ||
| 716 | + | // Tombstone provides a mock function for the type MockSnapshotRepository | |
| 717 | + | func (_mock *MockSnapshotRepository) Tombstone(context1 context.Context, s string) error { | |
| 718 | + | ret := _mock.Called(context1, s) | |
| 719 | + | ||
| 720 | + | if len(ret) == 0 { | |
| 721 | + | panic("no return value specified for Tombstone") | |
| 722 | + | } | |
| 723 | + | ||
| 724 | + | var r0 error | |
| 725 | + | if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok { | |
| 726 | + | r0 = returnFunc(context1, s) | |
| 727 | + | } else { | |
| 728 | + | r0 = ret.Error(0) | |
| 729 | + | } | |
| 730 | + | return r0 | |
| 731 | + | } | |
| 732 | + | ||
| 733 | + | // MockSnapshotRepository_Tombstone_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Tombstone' | |
| 734 | + | type MockSnapshotRepository_Tombstone_Call struct { | |
| 735 | + | *mock.Call | |
| 736 | + | } | |
| 737 | + | ||
| 738 | + | // Tombstone is a helper method to define mock.On call | |
| 739 | + | // - context1 context.Context | |
| 740 | + | // - s string | |
| 741 | + | func (_e *MockSnapshotRepository_Expecter) Tombstone(context1 any, s any) *MockSnapshotRepository_Tombstone_Call { | |
| 742 | + | return &MockSnapshotRepository_Tombstone_Call{Call: _e.mock.On("Tombstone", context1, s)} | |
| 743 | + | } | |
| 744 | + | ||
| 745 | + | func (_c *MockSnapshotRepository_Tombstone_Call) Run(run func(context1 context.Context, s string)) *MockSnapshotRepository_Tombstone_Call { | |
| 746 | + | _c.Call.Run(func(args mock.Arguments) { | |
| 747 | + | var arg0 context.Context | |
| 748 | + | if args[0] != nil { | |
| 749 | + | arg0 = args[0].(context.Context) | |
| 750 | + | } | |
| 751 | + | var arg1 string | |
| 752 | + | if args[1] != nil { | |
| 753 | + | arg1 = args[1].(string) | |
| 754 | + | } | |
| 755 | + | run( | |
| 756 | + | arg0, | |
| 757 | + | arg1, | |
| 758 | + | ) | |
| 759 | + | }) | |
| 760 | + | return _c | |
| 761 | + | } | |
| 762 | + | ||
| 763 | + | func (_c *MockSnapshotRepository_Tombstone_Call) Return(err error) *MockSnapshotRepository_Tombstone_Call { | |
| 764 | + | _c.Call.Return(err) | |
| 765 | + | return _c | |
| 766 | + | } | |
| 767 | + | ||
| 768 | + | func (_c *MockSnapshotRepository_Tombstone_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockSnapshotRepository_Tombstone_Call { | |
| 769 | + | _c.Call.Return(run) | |
| 770 | + | return _c | |
| 771 | + | } | |
| 772 | + | ||
| 659 | 773 | // NewMockLegacySnapshotSource creates a new instance of MockLegacySnapshotSource. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | |
| 660 | 774 | // The first argument is typically a *testing.T value. | |
| 661 | 775 | func NewMockLegacySnapshotSource(t interface { | |
| @@ -801,144 +915,3 @@ func (_c *MockLegacySnapshotSource_LoadLegacy_Call) RunAndReturn(run func(contex | |||
| 801 | 915 | _c.Call.Return(run) | |
| 802 | 916 | return _c | |
| 803 | 917 | } | |
| 804 | - | ||
| 805 | - | // NewMockSnapshotDeletionTombstone creates a new instance of MockSnapshotDeletionTombstone. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | |
| 806 | - | // The first argument is typically a *testing.T value. | |
| 807 | - | func NewMockSnapshotDeletionTombstone(t interface { | |
| 808 | - | mock.TestingT | |
| 809 | - | Cleanup(func()) | |
| 810 | - | }) *MockSnapshotDeletionTombstone { | |
| 811 | - | mock := &MockSnapshotDeletionTombstone{} | |
| 812 | - | mock.Mock.Test(t) | |
| 813 | - | ||
| 814 | - | t.Cleanup(func() { mock.AssertExpectations(t) }) | |
| 815 | - | ||
| 816 | - | return mock | |
| 817 | - | } | |
| 818 | - | ||
| 819 | - | // MockSnapshotDeletionTombstone is an autogenerated mock type for the SnapshotDeletionTombstone type | |
| 820 | - | type MockSnapshotDeletionTombstone struct { | |
| 821 | - | mock.Mock | |
| 822 | - | } | |
| 823 | - | ||
| 824 | - | type MockSnapshotDeletionTombstone_Expecter struct { | |
| 825 | - | mock *mock.Mock | |
| 826 | - | } | |
| 827 | - | ||
| 828 | - | func (_m *MockSnapshotDeletionTombstone) EXPECT() *MockSnapshotDeletionTombstone_Expecter { | |
| 829 | - | return &MockSnapshotDeletionTombstone_Expecter{mock: &_m.Mock} | |
| 830 | - | } | |
| 831 | - | ||
| 832 | - | // DeleteTombstone provides a mock function for the type MockSnapshotDeletionTombstone | |
| 833 | - | func (_mock *MockSnapshotDeletionTombstone) DeleteTombstone(context1 context.Context, s string) error { | |
| 834 | - | ret := _mock.Called(context1, s) | |
| 835 | - | ||
| 836 | - | if len(ret) == 0 { | |
| 837 | - | panic("no return value specified for DeleteTombstone") | |
| 838 | - | } | |
| 839 | - | ||
| 840 | - | var r0 error | |
| 841 | - | if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok { | |
| 842 | - | r0 = returnFunc(context1, s) | |
| 843 | - | } else { | |
| 844 | - | r0 = ret.Error(0) | |
| 845 | - | } | |
| 846 | - | return r0 | |
| 847 | - | } | |
| 848 | - | ||
| 849 | - | // MockSnapshotDeletionTombstone_DeleteTombstone_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteTombstone' | |
| 850 | - | type MockSnapshotDeletionTombstone_DeleteTombstone_Call struct { | |
| 851 | - | *mock.Call | |
| 852 | - | } | |
| 853 | - | ||
| 854 | - | // DeleteTombstone is a helper method to define mock.On call | |
| 855 | - | // - context1 context.Context | |
| 856 | - | // - s string | |
| 857 | - | func (_e *MockSnapshotDeletionTombstone_Expecter) DeleteTombstone(context1 any, s any) *MockSnapshotDeletionTombstone_DeleteTombstone_Call { | |
| 858 | - | return &MockSnapshotDeletionTombstone_DeleteTombstone_Call{Call: _e.mock.On("DeleteTombstone", context1, s)} | |
| 859 | - | } | |
| 860 | - | ||
| 861 | - | func (_c *MockSnapshotDeletionTombstone_DeleteTombstone_Call) Run(run func(context1 context.Context, s string)) *MockSnapshotDeletionTombstone_DeleteTombstone_Call { | |
| 862 | - | _c.Call.Run(func(args mock.Arguments) { | |
| 863 | - | var arg0 context.Context | |
| 864 | - | if args[0] != nil { | |
| 865 | - | arg0 = args[0].(context.Context) | |
| 866 | - | } | |
| 867 | - | var arg1 string | |
| 868 | - | if args[1] != nil { | |
| 869 | - | arg1 = args[1].(string) | |
| 870 | - | } | |
| 871 | - | run( | |
| 872 | - | arg0, | |
| 873 | - | arg1, | |
| 874 | - | ) | |
| 875 | - | }) | |
| 876 | - | return _c | |
| 877 | - | } | |
| 878 | - | ||
| 879 | - | func (_c *MockSnapshotDeletionTombstone_DeleteTombstone_Call) Return(err error) *MockSnapshotDeletionTombstone_DeleteTombstone_Call { | |
| 880 | - | _c.Call.Return(err) | |
| 881 | - | return _c | |
| 882 | - | } | |
| 883 | - | ||
| 884 | - | func (_c *MockSnapshotDeletionTombstone_DeleteTombstone_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockSnapshotDeletionTombstone_DeleteTombstone_Call { | |
| 885 | - | _c.Call.Return(run) | |
| 886 | - | return _c | |
| 887 | - | } | |
| 888 | - | ||
| 889 | - | // Tombstone provides a mock function for the type MockSnapshotDeletionTombstone | |
| 890 | - | func (_mock *MockSnapshotDeletionTombstone) Tombstone(context1 context.Context, s string) error { | |
| 891 | - | ret := _mock.Called(context1, s) | |
| 892 | - | ||
| 893 | - | if len(ret) == 0 { | |
| 894 | - | panic("no return value specified for Tombstone") | |
| 895 | - | } | |
| 896 | - | ||
| 897 | - | var r0 error | |
| 898 | - | if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok { | |
| 899 | - | r0 = returnFunc(context1, s) | |
| 900 | - | } else { | |
| 901 | - | r0 = ret.Error(0) | |
| 902 | - | } | |
| 903 | - | return r0 | |
| 904 | - | } | |
| 905 | - | ||
| 906 | - | // MockSnapshotDeletionTombstone_Tombstone_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Tombstone' | |
| 907 | - | type MockSnapshotDeletionTombstone_Tombstone_Call struct { | |
| 908 | - | *mock.Call | |
| 909 | - | } | |
| 910 | - | ||
| 911 | - | // Tombstone is a helper method to define mock.On call | |
| 912 | - | // - context1 context.Context | |
| 913 | - | // - s string | |
| 914 | - | func (_e *MockSnapshotDeletionTombstone_Expecter) Tombstone(context1 any, s any) *MockSnapshotDeletionTombstone_Tombstone_Call { | |
| 915 | - | return &MockSnapshotDeletionTombstone_Tombstone_Call{Call: _e.mock.On("Tombstone", context1, s)} | |
| 916 | - | } | |
| 917 | - | ||
| 918 | - | func (_c *MockSnapshotDeletionTombstone_Tombstone_Call) Run(run func(context1 context.Context, s string)) *MockSnapshotDeletionTombstone_Tombstone_Call { | |
| 919 | - | _c.Call.Run(func(args mock.Arguments) { | |
| 920 | - | var arg0 context.Context | |
| 921 | - | if args[0] != nil { | |
| 922 | - | arg0 = args[0].(context.Context) | |
| 923 | - | } | |
| 924 | - | var arg1 string | |
| 925 | - | if args[1] != nil { | |
| 926 | - | arg1 = args[1].(string) | |
| 927 | - | } | |
| 928 | - | run( | |
| 929 | - | arg0, | |
| 930 | - | arg1, | |
| 931 | - | ) | |
| 932 | - | }) | |
| 933 | - | return _c | |
| 934 | - | } | |
| 935 | - | ||
| 936 | - | func (_c *MockSnapshotDeletionTombstone_Tombstone_Call) Return(err error) *MockSnapshotDeletionTombstone_Tombstone_Call { | |
| 937 | - | _c.Call.Return(err) | |
| 938 | - | return _c | |
| 939 | - | } | |
| 940 | - | ||
| 941 | - | func (_c *MockSnapshotDeletionTombstone_Tombstone_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockSnapshotDeletionTombstone_Tombstone_Call { | |
| 942 | - | _c.Call.Return(run) | |
| 943 | - | return _c | |
| 944 | - | } | |
| @@ -420,6 +420,63 @@ func (_c *MockSnapshotRepository_Delete_Call) RunAndReturn(run func(context1 con | |||||
| 420 | return _c | 420 | return _c | ||
| 421 | } | 421 | } | ||
| 422 | 422 | ||||
| 423 | + | // DeleteTombstone provides a mock function for the type MockSnapshotRepository | |||
| 424 | + | func (_mock *MockSnapshotRepository) DeleteTombstone(context1 context.Context, s string) error { | |||
| 425 | + | ret := _mock.Called(context1, s) | |||
| 426 | + | ||||
| 427 | + | if len(ret) == 0 { | |||
| 428 | + | panic("no return value specified for DeleteTombstone") | |||
| 429 | + | } | |||
| 430 | + | ||||
| 431 | + | var r0 error | |||
| 432 | + | if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok { | |||
| 433 | + | r0 = returnFunc(context1, s) | |||
| 434 | + | } else { | |||
| 435 | + | r0 = ret.Error(0) | |||
| 436 | + | } | |||
| 437 | + | return r0 | |||
| 438 | + | } | |||
| 439 | + | ||||
| 440 | + | // MockSnapshotRepository_DeleteTombstone_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteTombstone' | |||
| 441 | + | type MockSnapshotRepository_DeleteTombstone_Call struct { | |||
| 442 | + | *mock.Call | |||
| 443 | + | } | |||
| 444 | + | ||||
| 445 | + | // DeleteTombstone is a helper method to define mock.On call | |||
| 446 | + | // - context1 context.Context | |||
| 447 | + | // - s string | |||
| 448 | + | func (_e *MockSnapshotRepository_Expecter) DeleteTombstone(context1 any, s any) *MockSnapshotRepository_DeleteTombstone_Call { | |||
| 449 | + | return &MockSnapshotRepository_DeleteTombstone_Call{Call: _e.mock.On("DeleteTombstone", context1, s)} | |||
| 450 | + | } | |||
| 451 | + | ||||
| 452 | + | func (_c *MockSnapshotRepository_DeleteTombstone_Call) Run(run func(context1 context.Context, s string)) *MockSnapshotRepository_DeleteTombstone_Call { | |||
| 453 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 454 | + | var arg0 context.Context | |||
| 455 | + | if args[0] != nil { | |||
| 456 | + | arg0 = args[0].(context.Context) | |||
| 457 | + | } | |||
| 458 | + | var arg1 string | |||
| 459 | + | if args[1] != nil { | |||
| 460 | + | arg1 = args[1].(string) | |||
| 461 | + | } | |||
| 462 | + | run( | |||
| 463 | + | arg0, | |||
| 464 | + | arg1, | |||
| 465 | + | ) | |||
| 466 | + | }) | |||
| 467 | + | return _c | |||
| 468 | + | } | |||
| 469 | + | ||||
| 470 | + | func (_c *MockSnapshotRepository_DeleteTombstone_Call) Return(err error) *MockSnapshotRepository_DeleteTombstone_Call { | |||
| 471 | + | _c.Call.Return(err) | |||
| 472 | + | return _c | |||
| 473 | + | } | |||
| 474 | + | ||||
| 475 | + | func (_c *MockSnapshotRepository_DeleteTombstone_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockSnapshotRepository_DeleteTombstone_Call { | |||
| 476 | + | _c.Call.Return(run) | |||
| 477 | + | return _c | |||
| 478 | + | } | |||
| 479 | + | ||||
| 423 | // List provides a mock function for the type MockSnapshotRepository | 480 | // List provides a mock function for the type MockSnapshotRepository | ||
| 424 | func (_mock *MockSnapshotRepository) List(context1 context.Context) ([]string, error) { | 481 | func (_mock *MockSnapshotRepository) List(context1 context.Context) ([]string, error) { | ||
| 425 | ret := _mock.Called(context1) | 482 | ret := _mock.Called(context1) | ||
| @@ -656,6 +713,63 @@ func (_c *MockSnapshotRepository_Publish_Call) RunAndReturn(run func(context1 co | |||||
| 656 | return _c | 713 | return _c | ||
| 657 | } | 714 | } | ||
| 658 | 715 | ||||
| 716 | + | // Tombstone provides a mock function for the type MockSnapshotRepository | |||
| 717 | + | func (_mock *MockSnapshotRepository) Tombstone(context1 context.Context, s string) error { | |||
| 718 | + | ret := _mock.Called(context1, s) | |||
| 719 | + | ||||
| 720 | + | if len(ret) == 0 { | |||
| 721 | + | panic("no return value specified for Tombstone") | |||
| 722 | + | } | |||
| 723 | + | ||||
| 724 | + | var r0 error | |||
| 725 | + | if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok { | |||
| 726 | + | r0 = returnFunc(context1, s) | |||
| 727 | + | } else { | |||
| 728 | + | r0 = ret.Error(0) | |||
| 729 | + | } | |||
| 730 | + | return r0 | |||
| 731 | + | } | |||
| 732 | + | ||||
| 733 | + | // MockSnapshotRepository_Tombstone_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Tombstone' | |||
| 734 | + | type MockSnapshotRepository_Tombstone_Call struct { | |||
| 735 | + | *mock.Call | |||
| 736 | + | } | |||
| 737 | + | ||||
| 738 | + | // Tombstone is a helper method to define mock.On call | |||
| 739 | + | // - context1 context.Context | |||
| 740 | + | // - s string | |||
| 741 | + | func (_e *MockSnapshotRepository_Expecter) Tombstone(context1 any, s any) *MockSnapshotRepository_Tombstone_Call { | |||
| 742 | + | return &MockSnapshotRepository_Tombstone_Call{Call: _e.mock.On("Tombstone", context1, s)} | |||
| 743 | + | } | |||
| 744 | + | ||||
| 745 | + | func (_c *MockSnapshotRepository_Tombstone_Call) Run(run func(context1 context.Context, s string)) *MockSnapshotRepository_Tombstone_Call { | |||
| 746 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 747 | + | var arg0 context.Context | |||
| 748 | + | if args[0] != nil { | |||
| 749 | + | arg0 = args[0].(context.Context) | |||
| 750 | + | } | |||
| 751 | + | var arg1 string | |||
| 752 | + | if args[1] != nil { | |||
| 753 | + | arg1 = args[1].(string) | |||
| 754 | + | } | |||
| 755 | + | run( | |||
| 756 | + | arg0, | |||
| 757 | + | arg1, | |||
| 758 | + | ) | |||
| 759 | + | }) | |||
| 760 | + | return _c | |||
| 761 | + | } | |||
| 762 | + | ||||
| 763 | + | func (_c *MockSnapshotRepository_Tombstone_Call) Return(err error) *MockSnapshotRepository_Tombstone_Call { | |||
| 764 | + | _c.Call.Return(err) | |||
| 765 | + | return _c | |||
| 766 | + | } | |||
| 767 | + | ||||
| 768 | + | func (_c *MockSnapshotRepository_Tombstone_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockSnapshotRepository_Tombstone_Call { | |||
| 769 | + | _c.Call.Return(run) | |||
| 770 | + | return _c | |||
| 771 | + | } | |||
| 772 | + | ||||
| 659 | // NewMockLegacySnapshotSource creates a new instance of MockLegacySnapshotSource. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | 773 | // NewMockLegacySnapshotSource creates a new instance of MockLegacySnapshotSource. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | ||
| 660 | // The first argument is typically a *testing.T value. | 774 | // The first argument is typically a *testing.T value. | ||
| 661 | func NewMockLegacySnapshotSource(t interface { | 775 | func NewMockLegacySnapshotSource(t interface { | ||
| @@ -801,144 +915,3 @@ func (_c *MockLegacySnapshotSource_LoadLegacy_Call) RunAndReturn(run func(contex | |||||
| 801 | _c.Call.Return(run) | 915 | _c.Call.Return(run) | ||
| 802 | return _c | 916 | return _c | ||
| 803 | } | 917 | } | ||
| 804 | - | ||||
| 805 | - | // NewMockSnapshotDeletionTombstone creates a new instance of MockSnapshotDeletionTombstone. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | |||
| 806 | - | // The first argument is typically a *testing.T value. | |||
| 807 | - | func NewMockSnapshotDeletionTombstone(t interface { | |||
| 808 | - | mock.TestingT | |||
| 809 | - | Cleanup(func()) | |||
| 810 | - | }) *MockSnapshotDeletionTombstone { | |||
| 811 | - | mock := &MockSnapshotDeletionTombstone{} | |||
| 812 | - | mock.Mock.Test(t) | |||
| 813 | - | ||||
| 814 | - | t.Cleanup(func() { mock.AssertExpectations(t) }) | |||
| 815 | - | ||||
| 816 | - | return mock | |||
| 817 | - | } | |||
| 818 | - | ||||
| 819 | - | // MockSnapshotDeletionTombstone is an autogenerated mock type for the SnapshotDeletionTombstone type | |||
| 820 | - | type MockSnapshotDeletionTombstone struct { | |||
| 821 | - | mock.Mock | |||
| 822 | - | } | |||
| 823 | - | ||||
| 824 | - | type MockSnapshotDeletionTombstone_Expecter struct { | |||
| 825 | - | mock *mock.Mock | |||
| 826 | - | } | |||
| 827 | - | ||||
| 828 | - | func (_m *MockSnapshotDeletionTombstone) EXPECT() *MockSnapshotDeletionTombstone_Expecter { | |||
| 829 | - | return &MockSnapshotDeletionTombstone_Expecter{mock: &_m.Mock} | |||
| 830 | - | } | |||
| 831 | - | ||||
| 832 | - | // DeleteTombstone provides a mock function for the type MockSnapshotDeletionTombstone | |||
| 833 | - | func (_mock *MockSnapshotDeletionTombstone) DeleteTombstone(context1 context.Context, s string) error { | |||
| 834 | - | ret := _mock.Called(context1, s) | |||
| 835 | - | ||||
| 836 | - | if len(ret) == 0 { | |||
| 837 | - | panic("no return value specified for DeleteTombstone") | |||
| 838 | - | } | |||
| 839 | - | ||||
| 840 | - | var r0 error | |||
| 841 | - | if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok { | |||
| 842 | - | r0 = returnFunc(context1, s) | |||
| 843 | - | } else { | |||
| 844 | - | r0 = ret.Error(0) | |||
| 845 | - | } | |||
| 846 | - | return r0 | |||
| 847 | - | } | |||
| 848 | - | ||||
| 849 | - | // MockSnapshotDeletionTombstone_DeleteTombstone_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteTombstone' | |||
| 850 | - | type MockSnapshotDeletionTombstone_DeleteTombstone_Call struct { | |||
| 851 | - | *mock.Call | |||
| 852 | - | } | |||
| 853 | - | ||||
| 854 | - | // DeleteTombstone is a helper method to define mock.On call | |||
| 855 | - | // - context1 context.Context | |||
| 856 | - | // - s string | |||
| 857 | - | func (_e *MockSnapshotDeletionTombstone_Expecter) DeleteTombstone(context1 any, s any) *MockSnapshotDeletionTombstone_DeleteTombstone_Call { | |||
| 858 | - | return &MockSnapshotDeletionTombstone_DeleteTombstone_Call{Call: _e.mock.On("DeleteTombstone", context1, s)} | |||
| 859 | - | } | |||
| 860 | - | ||||
| 861 | - | func (_c *MockSnapshotDeletionTombstone_DeleteTombstone_Call) Run(run func(context1 context.Context, s string)) *MockSnapshotDeletionTombstone_DeleteTombstone_Call { | |||
| 862 | - | _c.Call.Run(func(args mock.Arguments) { | |||
| 863 | - | var arg0 context.Context | |||
| 864 | - | if args[0] != nil { | |||
| 865 | - | arg0 = args[0].(context.Context) | |||
| 866 | - | } | |||
| 867 | - | var arg1 string | |||
| 868 | - | if args[1] != nil { | |||
| 869 | - | arg1 = args[1].(string) | |||
| 870 | - | } | |||
| 871 | - | run( | |||
| 872 | - | arg0, | |||
| 873 | - | arg1, | |||
| 874 | - | ) | |||
| 875 | - | }) | |||
| 876 | - | return _c | |||
| 877 | - | } | |||
| 878 | - | ||||
| 879 | - | func (_c *MockSnapshotDeletionTombstone_DeleteTombstone_Call) Return(err error) *MockSnapshotDeletionTombstone_DeleteTombstone_Call { | |||
| 880 | - | _c.Call.Return(err) | |||
| 881 | - | return _c | |||
| 882 | - | } | |||
| 883 | - | ||||
| 884 | - | func (_c *MockSnapshotDeletionTombstone_DeleteTombstone_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockSnapshotDeletionTombstone_DeleteTombstone_Call { | |||
| 885 | - | _c.Call.Return(run) | |||
| 886 | - | return _c | |||
| 887 | - | } | |||
| 888 | - | ||||
| 889 | - | // Tombstone provides a mock function for the type MockSnapshotDeletionTombstone | |||
| 890 | - | func (_mock *MockSnapshotDeletionTombstone) Tombstone(context1 context.Context, s string) error { | |||
| 891 | - | ret := _mock.Called(context1, s) | |||
| 892 | - | ||||
| 893 | - | if len(ret) == 0 { | |||
| 894 | - | panic("no return value specified for Tombstone") | |||
| 895 | - | } | |||
| 896 | - | ||||
| 897 | - | var r0 error | |||
| 898 | - | if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok { | |||
| 899 | - | r0 = returnFunc(context1, s) | |||
| 900 | - | } else { | |||
| 901 | - | r0 = ret.Error(0) | |||
| 902 | - | } | |||
| 903 | - | return r0 | |||
| 904 | - | } | |||
| 905 | - | ||||
| 906 | - | // MockSnapshotDeletionTombstone_Tombstone_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Tombstone' | |||
| 907 | - | type MockSnapshotDeletionTombstone_Tombstone_Call struct { | |||
| 908 | - | *mock.Call | |||
| 909 | - | } | |||
| 910 | - | ||||
| 911 | - | // Tombstone is a helper method to define mock.On call | |||
| 912 | - | // - context1 context.Context | |||
| 913 | - | // - s string | |||
| 914 | - | func (_e *MockSnapshotDeletionTombstone_Expecter) Tombstone(context1 any, s any) *MockSnapshotDeletionTombstone_Tombstone_Call { | |||
| 915 | - | return &MockSnapshotDeletionTombstone_Tombstone_Call{Call: _e.mock.On("Tombstone", context1, s)} | |||
| 916 | - | } | |||
| 917 | - | ||||
| 918 | - | func (_c *MockSnapshotDeletionTombstone_Tombstone_Call) Run(run func(context1 context.Context, s string)) *MockSnapshotDeletionTombstone_Tombstone_Call { | |||
| 919 | - | _c.Call.Run(func(args mock.Arguments) { | |||
| 920 | - | var arg0 context.Context | |||
| 921 | - | if args[0] != nil { | |||
| 922 | - | arg0 = args[0].(context.Context) | |||
| 923 | - | } | |||
| 924 | - | var arg1 string | |||
| 925 | - | if args[1] != nil { | |||
| 926 | - | arg1 = args[1].(string) | |||
| 927 | - | } | |||
| 928 | - | run( | |||
| 929 | - | arg0, | |||
| 930 | - | arg1, | |||
| 931 | - | ) | |||
| 932 | - | }) | |||
| 933 | - | return _c | |||
| 934 | - | } | |||
| 935 | - | ||||
| 936 | - | func (_c *MockSnapshotDeletionTombstone_Tombstone_Call) Return(err error) *MockSnapshotDeletionTombstone_Tombstone_Call { | |||
| 937 | - | _c.Call.Return(err) | |||
| 938 | - | return _c | |||
| 939 | - | } | |||
| 940 | - | ||||
| 941 | - | func (_c *MockSnapshotDeletionTombstone_Tombstone_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockSnapshotDeletionTombstone_Tombstone_Call { | |||
| 942 | - | _c.Call.Return(run) | |||
| 943 | - | return _c | |||
| 944 | - | } | |||
M internal/ports/store.go +4 -9
| @@ -57,6 +57,10 @@ type SnapshotRepository interface { | |||
| 57 | 57 | List(context.Context) ([]string, error) | |
| 58 | 58 | Load(context.Context, string) (SnapshotGeneration, error) | |
| 59 | 59 | Delete(context.Context, string) error | |
| 60 | + | // Tombstone fences a named-session purge before either incremental or | |
| 61 | + | // legacy data is deleted. It must remain durable until DeleteTombstone. | |
| 62 | + | Tombstone(context.Context, string) error | |
| 63 | + | DeleteTombstone(context.Context, string) error | |
| 60 | 64 | Maintain(context.Context) error | |
| 61 | 65 | } | |
| 62 | 66 | ||
| @@ -73,12 +77,3 @@ type LegacySnapshotSource interface { | |||
| 73 | 77 | LoadLegacy(context.Context) ([]LegacySnapshot, error) | |
| 74 | 78 | DeleteLegacy(context.Context, string) error | |
| 75 | 79 | } | |
| 76 | - | ||
| 77 | - | // SnapshotDeletionTombstone makes a named-session purge durable across the | |
| 78 | - | // independent incremental and legacy snapshot sources. A tombstoned name must | |
| 79 | - | // not be restored or imported until both source deletions and metadata removal | |
| 80 | - | // have completed. | |
| 81 | - | type SnapshotDeletionTombstone interface { | |
| 82 | - | Tombstone(context.Context, string) error | |
| 83 | - | DeleteTombstone(context.Context, string) error | |
| 84 | - | } | |
| @@ -57,6 +57,10 @@ type SnapshotRepository interface { | |||||
| 57 | List(context.Context) ([]string, error) | 57 | List(context.Context) ([]string, error) | ||
| 58 | Load(context.Context, string) (SnapshotGeneration, error) | 58 | Load(context.Context, string) (SnapshotGeneration, error) | ||
| 59 | Delete(context.Context, string) error | 59 | Delete(context.Context, string) error | ||
| 60 | + | // Tombstone fences a named-session purge before either incremental or | |||
| 61 | + | // legacy data is deleted. It must remain durable until DeleteTombstone. | |||
| 62 | + | Tombstone(context.Context, string) error | |||
| 63 | + | DeleteTombstone(context.Context, string) error | |||
| 60 | Maintain(context.Context) error | 64 | Maintain(context.Context) error | ||
| 61 | } | 65 | } | ||
| 62 | 66 | ||||
| @@ -73,12 +77,3 @@ type LegacySnapshotSource interface { | |||||
| 73 | LoadLegacy(context.Context) ([]LegacySnapshot, error) | 77 | LoadLegacy(context.Context) ([]LegacySnapshot, error) | ||
| 74 | DeleteLegacy(context.Context, string) error | 78 | DeleteLegacy(context.Context, string) error | ||
| 75 | } | 79 | } | ||
| 76 | - | ||||
| 77 | - | // SnapshotDeletionTombstone makes a named-session purge durable across the | |||
| 78 | - | // independent incremental and legacy snapshot sources. A tombstoned name must | |||
| 79 | - | // not be restored or imported until both source deletions and metadata removal | |||
| 80 | - | // have completed. | |||
| 81 | - | type SnapshotDeletionTombstone interface { | |||
| 82 | - | Tombstone(context.Context, string) error | |||
| 83 | - | DeleteTombstone(context.Context, string) error | |||
| 84 | - | } | |||
M internal/usecase/daemon/daemon.go +1 -10
| @@ -119,11 +119,7 @@ type Daemon struct { | |||
| 119 | 119 | // read-only migration input and is never used for new writes. | |
| 120 | 120 | snapshotRepository ports.SnapshotRepository | |
| 121 | 121 | legacySnapshots ports.LegacySnapshotSource | |
| 122 | - | // snapshotDeletion durably fences a live purge across both snapshot | |
| 123 | - | // sources. Production repositories must provide it; nil preserves the | |
| 124 | - | // narrower checkpoint-only test seam. | |
| 125 | - | snapshotDeletion ports.SnapshotDeletionTombstone | |
| 126 | - | snapsEnabled bool | |
| 122 | + | snapsEnabled bool | |
| 127 | 123 | // legacyImportPending holds verified legacy blobs whose source deletion | |
| 128 | 124 | // failed, so a later import retries only deletion rather than publication. | |
| 129 | 125 | legacyImportMu sync.Mutex | |
| @@ -251,11 +247,6 @@ func WithSnapshotRepository(repository ports.SnapshotRepository, legacy ports.Le | |||
| 251 | 247 | return func(d *Daemon) { | |
| 252 | 248 | d.snapshotRepository = repository | |
| 253 | 249 | d.legacySnapshots = legacy | |
| 254 | - | if deletion, ok := repository.(ports.SnapshotDeletionTombstone); ok { | |
| 255 | - | d.snapshotDeletion = deletion | |
| 256 | - | } else if deletion, ok := legacy.(ports.SnapshotDeletionTombstone); ok { | |
| 257 | - | d.snapshotDeletion = deletion | |
| 258 | - | } | |
| 259 | 250 | d.snapsEnabled = repository != nil | |
| 260 | 251 | } | |
| 261 | 252 | } | |
| @@ -119,11 +119,7 @@ type Daemon struct { | |||||
| 119 | // read-only migration input and is never used for new writes. | 119 | // read-only migration input and is never used for new writes. | ||
| 120 | snapshotRepository ports.SnapshotRepository | 120 | snapshotRepository ports.SnapshotRepository | ||
| 121 | legacySnapshots ports.LegacySnapshotSource | 121 | legacySnapshots ports.LegacySnapshotSource | ||
| 122 | - | // snapshotDeletion durably fences a live purge across both snapshot | 122 | + | snapsEnabled bool |
| 123 | - | // sources. Production repositories must provide it; nil preserves the | |||
| 124 | - | // narrower checkpoint-only test seam. | |||
| 125 | - | snapshotDeletion ports.SnapshotDeletionTombstone | |||
| 126 | - | snapsEnabled bool | |||
| 127 | // legacyImportPending holds verified legacy blobs whose source deletion | 123 | // legacyImportPending holds verified legacy blobs whose source deletion | ||
| 128 | // failed, so a later import retries only deletion rather than publication. | 124 | // failed, so a later import retries only deletion rather than publication. | ||
| 129 | legacyImportMu sync.Mutex | 125 | legacyImportMu sync.Mutex | ||
| @@ -251,11 +247,6 @@ func WithSnapshotRepository(repository ports.SnapshotRepository, legacy ports.Le | |||||
| 251 | return func(d *Daemon) { | 247 | return func(d *Daemon) { | ||
| 252 | d.snapshotRepository = repository | 248 | d.snapshotRepository = repository | ||
| 253 | d.legacySnapshots = legacy | 249 | d.legacySnapshots = legacy | ||
| 254 | - | if deletion, ok := repository.(ports.SnapshotDeletionTombstone); ok { | |||
| 255 | - | d.snapshotDeletion = deletion | |||
| 256 | - | } else if deletion, ok := legacy.(ports.SnapshotDeletionTombstone); ok { | |||
| 257 | - | d.snapshotDeletion = deletion | |||
| 258 | - | } | |||
| 259 | d.snapsEnabled = repository != nil | 250 | d.snapsEnabled = repository != nil | ||
| 260 | } | 251 | } | ||
| 261 | } | 252 | } | ||
M internal/usecase/daemon/performance_benchmark_test.go +4 -2
| @@ -1061,8 +1061,10 @@ func (*countingSnapshotRepository) List(context.Context) ([]string, error) { ret | |||
| 1061 | 1061 | func (*countingSnapshotRepository) Load(context.Context, string) (ports.SnapshotGeneration, error) { | |
| 1062 | 1062 | return ports.SnapshotGeneration{}, nil | |
| 1063 | 1063 | } | |
| 1064 | - | func (*countingSnapshotRepository) Delete(context.Context, string) error { return nil } | |
| 1065 | - | func (*countingSnapshotRepository) Maintain(context.Context) error { return nil } | |
| 1064 | + | func (*countingSnapshotRepository) Delete(context.Context, string) error { return nil } | |
| 1065 | + | func (*countingSnapshotRepository) Tombstone(context.Context, string) error { return nil } | |
| 1066 | + | func (*countingSnapshotRepository) DeleteTombstone(context.Context, string) error { return nil } | |
| 1067 | + | func (*countingSnapshotRepository) Maintain(context.Context) error { return nil } | |
| 1066 | 1068 | func (s *countingSnapshotRepository) reset() { | |
| 1067 | 1069 | s.mu.Lock() | |
| 1068 | 1070 | s.countingSnapshotMetrics = countingSnapshotMetrics{} | |
| @@ -1061,8 +1061,10 @@ func (*countingSnapshotRepository) List(context.Context) ([]string, error) { ret | |||||
| 1061 | func (*countingSnapshotRepository) Load(context.Context, string) (ports.SnapshotGeneration, error) { | 1061 | func (*countingSnapshotRepository) Load(context.Context, string) (ports.SnapshotGeneration, error) { | ||
| 1062 | return ports.SnapshotGeneration{}, nil | 1062 | return ports.SnapshotGeneration{}, nil | ||
| 1063 | } | 1063 | } | ||
| 1064 | - | func (*countingSnapshotRepository) Delete(context.Context, string) error { return nil } | 1064 | + | func (*countingSnapshotRepository) Delete(context.Context, string) error { return nil } |
| 1065 | - | func (*countingSnapshotRepository) Maintain(context.Context) error { return nil } | 1065 | + | func (*countingSnapshotRepository) Tombstone(context.Context, string) error { return nil } |
| 1066 | + | func (*countingSnapshotRepository) DeleteTombstone(context.Context, string) error { return nil } | |||
| 1067 | + | func (*countingSnapshotRepository) Maintain(context.Context) error { return nil } | |||
| 1066 | func (s *countingSnapshotRepository) reset() { | 1068 | func (s *countingSnapshotRepository) reset() { | ||
| 1067 | s.mu.Lock() | 1069 | s.mu.Lock() | ||
| 1068 | s.countingSnapshotMetrics = countingSnapshotMetrics{} | 1070 | s.countingSnapshotMetrics = countingSnapshotMetrics{} | ||
M internal/usecase/daemon/picker_test.go +6 -2
| @@ -44,8 +44,12 @@ func (refusingSnapshotDeleteRepository) List(context.Context) ([]string, error) | |||
| 44 | 44 | func (refusingSnapshotDeleteRepository) Load(context.Context, string) (ports.SnapshotGeneration, error) { | |
| 45 | 45 | return ports.SnapshotGeneration{}, nil | |
| 46 | 46 | } | |
| 47 | - | func (s refusingSnapshotDeleteRepository) Delete(context.Context, string) error { return s.err } | |
| 48 | - | func (refusingSnapshotDeleteRepository) Maintain(context.Context) error { return nil } | |
| 47 | + | func (s refusingSnapshotDeleteRepository) Delete(context.Context, string) error { return s.err } | |
| 48 | + | func (refusingSnapshotDeleteRepository) Tombstone(context.Context, string) error { return nil } | |
| 49 | + | func (refusingSnapshotDeleteRepository) DeleteTombstone(context.Context, string) error { | |
| 50 | + | return nil | |
| 51 | + | } | |
| 52 | + | func (refusingSnapshotDeleteRepository) Maintain(context.Context) error { return nil } | |
| 49 | 53 | ||
| 50 | 54 | func newTestTabWithContext(p ports.PTY, ctx context.Context, cancel context.CancelFunc) *tab { | |
| 51 | 55 | tb := newTab(p, domain.Size{Cols: 80, Rows: 23}) | |
| @@ -44,8 +44,12 @@ func (refusingSnapshotDeleteRepository) List(context.Context) ([]string, error) | |||||
| 44 | func (refusingSnapshotDeleteRepository) Load(context.Context, string) (ports.SnapshotGeneration, error) { | 44 | func (refusingSnapshotDeleteRepository) Load(context.Context, string) (ports.SnapshotGeneration, error) { | ||
| 45 | return ports.SnapshotGeneration{}, nil | 45 | return ports.SnapshotGeneration{}, nil | ||
| 46 | } | 46 | } | ||
| 47 | - | func (s refusingSnapshotDeleteRepository) Delete(context.Context, string) error { return s.err } | 47 | + | func (s refusingSnapshotDeleteRepository) Delete(context.Context, string) error { return s.err } |
| 48 | - | func (refusingSnapshotDeleteRepository) Maintain(context.Context) error { return nil } | 48 | + | func (refusingSnapshotDeleteRepository) Tombstone(context.Context, string) error { return nil } |
| 49 | + | func (refusingSnapshotDeleteRepository) DeleteTombstone(context.Context, string) error { | |||
| 50 | + | return nil | |||
| 51 | + | } | |||
| 52 | + | func (refusingSnapshotDeleteRepository) Maintain(context.Context) error { return nil } | |||
| 49 | 53 | ||||
| 50 | func newTestTabWithContext(p ports.PTY, ctx context.Context, cancel context.CancelFunc) *tab { | 54 | func newTestTabWithContext(p ports.PTY, ctx context.Context, cancel context.CancelFunc) *tab { | ||
| 51 | tb := newTab(p, domain.Size{Cols: 80, Rows: 23}) | 55 | tb := newTab(p, domain.Size{Cols: 80, Rows: 23}) | ||
M internal/usecase/daemon/session.go +10 -12
| @@ -800,10 +800,10 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) { | |||
| 800 | 800 | // identity or persisted metadata is removed. It never runs under daemon or | |
| 801 | 801 | // session locks. | |
| 802 | 802 | func (d *Daemon) beginSnapshotPurge(name string) error { | |
| 803 | - | if d.snapshotDeletion == nil || name == "" { | |
| 803 | + | if d.snapshotRepository == nil || name == "" { | |
| 804 | 804 | return nil | |
| 805 | 805 | } | |
| 806 | - | return d.snapshotDeletion.Tombstone(context.Background(), name) | |
| 806 | + | return d.snapshotRepository.Tombstone(context.Background(), name) | |
| 807 | 807 | } | |
| 808 | 808 | ||
| 809 | 809 | // finishSnapshotPurge deletes both independently durable snapshot sources, | |
| @@ -811,10 +811,11 @@ func (d *Daemon) beginSnapshotPurge(name string) error { | |||
| 811 | 811 | // leaves the marker in place so startup cannot restore or import the name and | |
| 812 | 812 | // a later live/offline kill can retry the idempotent source deletes. | |
| 813 | 813 | func (d *Daemon) finishSnapshotPurge(name string) error { | |
| 814 | - | var incrementalErr, legacyErr error | |
| 815 | - | if d.snapshotRepository != nil { | |
| 816 | - | incrementalErr = d.snapshotRepository.Delete(context.Background(), name) | |
| 814 | + | if d.snapshotRepository == nil { | |
| 815 | + | return d.persist.Delete(name) | |
| 817 | 816 | } | |
| 817 | + | incrementalErr := d.snapshotRepository.Delete(context.Background(), name) | |
| 818 | + | var legacyErr error | |
| 818 | 819 | if d.legacySnapshots != nil { | |
| 819 | 820 | legacyErr = d.legacySnapshots.DeleteLegacy(context.Background(), name) | |
| 820 | 821 | } | |
| @@ -824,10 +825,7 @@ func (d *Daemon) finishSnapshotPurge(name string) error { | |||
| 824 | 825 | if err := d.persist.Delete(name); err != nil { | |
| 825 | 826 | return err | |
| 826 | 827 | } | |
| 827 | - | if d.snapshotDeletion != nil { | |
| 828 | - | return d.snapshotDeletion.DeleteTombstone(context.Background(), name) | |
| 829 | - | } | |
| 830 | - | return nil | |
| 828 | + | return d.snapshotRepository.DeleteTombstone(context.Background(), name) | |
| 831 | 829 | } | |
| 832 | 830 | ||
| 833 | 831 | // retryStoppedPurge completes a previously closed session's durable purge. | |
| @@ -840,9 +838,9 @@ func (d *Daemon) retryStoppedPurge(name string) error { | |||
| 840 | 838 | d.mu.Unlock() | |
| 841 | 839 | return nil | |
| 842 | 840 | } | |
| 843 | - | // A stopped session without a durable snapshot source retains the existing | |
| 844 | - | // metadata-only delete behavior. There is no restore/import source to fence. | |
| 845 | - | if d.snapshotDeletion == nil { | |
| 841 | + | // Without a repository there is no restore/import source to fence, so the | |
| 842 | + | // stopped record retains the metadata-only deletion behavior. | |
| 843 | + | if d.snapshotRepository == nil { | |
| 846 | 844 | d.mu.Unlock() | |
| 847 | 845 | if err := d.persist.Delete(name); err != nil { | |
| 848 | 846 | return err | |
| @@ -800,10 +800,10 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) { | |||||
| 800 | // identity or persisted metadata is removed. It never runs under daemon or | 800 | // identity or persisted metadata is removed. It never runs under daemon or | ||
| 801 | // session locks. | 801 | // session locks. | ||
| 802 | func (d *Daemon) beginSnapshotPurge(name string) error { | 802 | func (d *Daemon) beginSnapshotPurge(name string) error { | ||
| 803 | - | if d.snapshotDeletion == nil || name == "" { | 803 | + | if d.snapshotRepository == nil || name == "" { |
| 804 | return nil | 804 | return nil | ||
| 805 | } | 805 | } | ||
| 806 | - | return d.snapshotDeletion.Tombstone(context.Background(), name) | 806 | + | return d.snapshotRepository.Tombstone(context.Background(), name) |
| 807 | } | 807 | } | ||
| 808 | 808 | ||||
| 809 | // finishSnapshotPurge deletes both independently durable snapshot sources, | 809 | // finishSnapshotPurge deletes both independently durable snapshot sources, | ||
| @@ -811,10 +811,11 @@ func (d *Daemon) beginSnapshotPurge(name string) error { | |||||
| 811 | // leaves the marker in place so startup cannot restore or import the name and | 811 | // leaves the marker in place so startup cannot restore or import the name and | ||
| 812 | // a later live/offline kill can retry the idempotent source deletes. | 812 | // a later live/offline kill can retry the idempotent source deletes. | ||
| 813 | func (d *Daemon) finishSnapshotPurge(name string) error { | 813 | func (d *Daemon) finishSnapshotPurge(name string) error { | ||
| 814 | - | var incrementalErr, legacyErr error | 814 | + | if d.snapshotRepository == nil { |
| 815 | - | if d.snapshotRepository != nil { | 815 | + | return d.persist.Delete(name) |
| 816 | - | incrementalErr = d.snapshotRepository.Delete(context.Background(), name) | |||
| 817 | } | 816 | } | ||
| 817 | + | incrementalErr := d.snapshotRepository.Delete(context.Background(), name) | |||
| 818 | + | var legacyErr error | |||
| 818 | if d.legacySnapshots != nil { | 819 | if d.legacySnapshots != nil { | ||
| 819 | legacyErr = d.legacySnapshots.DeleteLegacy(context.Background(), name) | 820 | legacyErr = d.legacySnapshots.DeleteLegacy(context.Background(), name) | ||
| 820 | } | 821 | } | ||
| @@ -824,10 +825,7 @@ func (d *Daemon) finishSnapshotPurge(name string) error { | |||||
| 824 | if err := d.persist.Delete(name); err != nil { | 825 | if err := d.persist.Delete(name); err != nil { | ||
| 825 | return err | 826 | return err | ||
| 826 | } | 827 | } | ||
| 827 | - | if d.snapshotDeletion != nil { | 828 | + | return d.snapshotRepository.DeleteTombstone(context.Background(), name) |
| 828 | - | return d.snapshotDeletion.DeleteTombstone(context.Background(), name) | |||
| 829 | - | } | |||
| 830 | - | return nil | |||
| 831 | } | 829 | } | ||
| 832 | 830 | ||||
| 833 | // retryStoppedPurge completes a previously closed session's durable purge. | 831 | // retryStoppedPurge completes a previously closed session's durable purge. | ||
| @@ -840,9 +838,9 @@ func (d *Daemon) retryStoppedPurge(name string) error { | |||||
| 840 | d.mu.Unlock() | 838 | d.mu.Unlock() | ||
| 841 | return nil | 839 | return nil | ||
| 842 | } | 840 | } | ||
| 843 | - | // A stopped session without a durable snapshot source retains the existing | 841 | + | // Without a repository there is no restore/import source to fence, so the |
| 844 | - | // metadata-only delete behavior. There is no restore/import source to fence. | 842 | + | // stopped record retains the metadata-only deletion behavior. |
| 845 | - | if d.snapshotDeletion == nil { | 843 | + | if d.snapshotRepository == nil { |
| 846 | d.mu.Unlock() | 844 | d.mu.Unlock() | ||
| 847 | if err := d.persist.Delete(name); err != nil { | 845 | if err := d.persist.Delete(name); err != nil { | ||
| 848 | return err | 846 | return err | ||
M internal/usecase/daemon/snapshot.go +0 -1078
| @@ -4,11 +4,9 @@ import ( | |||
| 4 | 4 | "context" | |
| 5 | 5 | "errors" | |
| 6 | 6 | "fmt" | |
| 7 | - | "math" | |
| 8 | 7 | "sync" | |
| 9 | 8 | "time" | |
| 10 | 9 | ||
| 11 | - | "github.com/bnema/vev/internal/domain" | |
| 12 | 10 | "github.com/bnema/vev/internal/usecase/layout" | |
| 13 | 11 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |
| 14 | 12 | "github.com/bnema/vev/pkg/vt" | |
| @@ -94,1079 +92,3 @@ const ( | |||
| 94 | 92 | snapshotInterval = 2 * time.Minute | |
| 95 | 93 | snapshotFinalFlushTimeout = time.Second | |
| 96 | 94 | ) | |
| 97 | - | ||
| 98 | - | func (d *Daemon) closeRestoreDone() { | |
| 99 | - | d.restoreOnce.Do(func() { close(d.restoreDone) }) | |
| 100 | - | } | |
| 101 | - | ||
| 102 | - | func (d *Daemon) restoreSnapshots(ctx context.Context) { | |
| 103 | - | if d == nil { | |
| 104 | - | return | |
| 105 | - | } | |
| 106 | - | defer d.closeRestoreDone() | |
| 107 | - | if ns := d.noticeStore; ns != nil { | |
| 108 | - | claimed, err := ns.Claim() | |
| 109 | - | if err != nil { | |
| 110 | - | d.log.Warn("claiming pending notices failed", "err", err) | |
| 111 | - | } else { | |
| 112 | - | for _, n := range claimed { | |
| 113 | - | d.notices.record(n) | |
| 114 | - | d.notices.queueGlobal(n) | |
| 115 | - | } | |
| 116 | - | // Do not discard the claim until every notice has been recorded and | |
| 117 | - | // queued. An Ack failure is safe: the claim is replayed at startup. | |
| 118 | - | if err := ns.Ack(); err != nil { | |
| 119 | - | d.log.Warn("acknowledging pending notices failed", "err", err) | |
| 120 | - | } | |
| 121 | - | } | |
| 122 | - | } | |
| 123 | - | if !d.snapsEnabled { | |
| 124 | - | return | |
| 125 | - | } | |
| 126 | - | if d.snapshotRepository != nil { | |
| 127 | - | d.restoreIncrementalSnapshots(ctx) | |
| 128 | - | } | |
| 129 | - | ||
| 130 | - | } | |
| 131 | - | ||
| 132 | - | func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session) error { | |
| 133 | - | if err := ctx.Err(); err != nil { | |
| 134 | - | return err | |
| 135 | - | } | |
| 136 | - | if len(snap.Tabs) == 0 { | |
| 137 | - | if snap.Active != 0 { | |
| 138 | - | return fmt.Errorf("snapshot: active tab out of range") | |
| 139 | - | } | |
| 140 | - | } else if int(snap.Active) >= len(snap.Tabs) { | |
| 141 | - | return fmt.Errorf("snapshot: active tab out of range") | |
| 142 | - | } | |
| 143 | - | if snap.Name == "" { | |
| 144 | - | return fmt.Errorf("snapshot: empty session name") | |
| 145 | - | } | |
| 146 | - | if snap.CreatedAt > math.MaxInt64 { | |
| 147 | - | return fmt.Errorf("snapshot: created_at overflows int64") | |
| 148 | - | } | |
| 149 | - | d.mu.Lock() | |
| 150 | - | if d.closing || d.findByNameLocked(snap.Name) != nil { | |
| 151 | - | d.mu.Unlock() | |
| 152 | - | return nil | |
| 153 | - | } | |
| 154 | - | d.mu.Unlock() | |
| 155 | - | ||
| 156 | - | parent := d.serveCtx | |
| 157 | - | if parent == nil { | |
| 158 | - | parent = context.Background() | |
| 159 | - | } | |
| 160 | - | sctx, cancel := context.WithCancel(parent) | |
| 161 | - | opened := make([]*tab, 0, len(snap.Tabs)) | |
| 162 | - | // Snapshot restore runs before any client Hello is available. Start panes | |
| 163 | - | // from the daemon environment captured at startup; the next attach replaces | |
| 164 | - | // this snapshot for future panes from the client's capability. | |
| 165 | - | restoreTerm := terminalEnv{} | |
| 166 | - | restoreEnv := copyEnvironment(d.baseEnv) | |
| 167 | - | closeOpened := func() { | |
| 168 | - | for _, tb := range opened { | |
| 169 | - | tb.closeAllPanes() | |
| 170 | - | } | |
| 171 | - | cancel() | |
| 172 | - | } | |
| 173 | - | allowlist := d.restoreProcessAllowlistSnapshot() | |
| 174 | - | d.mu.Lock() | |
| 175 | - | persisted := d.stopped[snap.Name] | |
| 176 | - | d.mu.Unlock() | |
| 177 | - | for tabIndex, tabSnap := range snap.Tabs { | |
| 178 | - | if err := ctx.Err(); err != nil { | |
| 179 | - | closeOpened() | |
| 180 | - | return err | |
| 181 | - | } | |
| 182 | - | tbSize := domain.Size{Cols: int(tabSnap.Cols), Rows: int(tabSnap.Rows)} | |
| 183 | - | if !tbSize.Valid() { | |
| 184 | - | closeOpened() | |
| 185 | - | return fmt.Errorf("snapshot: invalid tab size") | |
| 186 | - | } | |
| 187 | - | placements, ok := layout.Solve(tabSnap.Tree.Root, domain.Rect{Width: tbSize.Cols, Height: tbSize.Rows}) | |
| 188 | - | if !ok { | |
| 189 | - | closeOpened() | |
| 190 | - | return fmt.Errorf("snapshot: unsolvable layout") | |
| 191 | - | } | |
| 192 | - | placementByPane := make(map[layout.PaneID]domain.Rect, len(placements)) | |
| 193 | - | for _, pl := range placements { | |
| 194 | - | placementByPane[pl.ID] = pl.Content | |
| 195 | - | } | |
| 196 | - | tabStableID := tabSnap.StableID | |
| 197 | - | if tabStableID == "" { | |
| 198 | - | var err error | |
| 199 | - | tabStableID, err = newStableID("t") | |
| 200 | - | if err != nil { | |
| 201 | - | closeOpened() | |
| 202 | - | return fmt.Errorf("snapshot: generating tab identity: %w", err) | |
| 203 | - | } | |
| 204 | - | } | |
| 205 | - | tb := &tab{stableID: tabStableID, tree: tabSnap.Tree.Clone(), panes: make(map[layout.PaneID]*pane, len(tabSnap.Panes)), nextPaneID: int(tabSnap.NextPaneID), size: tbSize} | |
| 206 | - | if tabIndex < len(persisted.tabNames) { | |
| 207 | - | tb.name = persisted.tabNames[tabIndex] | |
| 208 | - | } | |
| 209 | - | if tb.nextPaneID <= 0 { | |
| 210 | - | tb.nextPaneID = 1 | |
| 211 | - | } | |
| 212 | - | tb.ctx, tb.cancel = context.WithCancel(sctx) | |
| 213 | - | opened = append(opened, tb) | |
| 214 | - | for _, paneSnap := range tabSnap.Panes { | |
| 215 | - | restoreCommand := "" | |
| 216 | - | if decision := planProcessRestore(paneSnap.Process, allowlist); decision.Restore { | |
| 217 | - | restoreCommand = decision.Command | |
| 218 | - | } | |
| 219 | - | if err := ctx.Err(); err != nil { | |
| 220 | - | closeOpened() | |
| 221 | - | return err | |
| 222 | - | } | |
| 223 | - | contentRect, ok := placementByPane[paneSnap.ID] | |
| 224 | - | if !ok { | |
| 225 | - | closeOpened() | |
| 226 | - | return fmt.Errorf("snapshot: missing pane placement") | |
| 227 | - | } | |
| 228 | - | contentSize := restorePTYSize(contentRect, tbSize) | |
| 229 | - | paneStableID := paneSnap.StableID | |
| 230 | - | if paneStableID == "" { | |
| 231 | - | var err error | |
| 232 | - | paneStableID, err = newStableID("p") | |
| 233 | - | if err != nil { | |
| 234 | - | closeOpened() | |
| 235 | - | return fmt.Errorf("snapshot: generating pane identity: %w", err) | |
| 236 | - | } | |
| 237 | - | } | |
| 238 | - | command, args := d.ptyCommand(restoreEnv) | |
| 239 | - | pty, err := d.ptys.Open(ctx, command, args, childEnvFrom(restoreEnv, snap.Name, tabStableID, paneStableID, restoreTerm), paneSnap.Cwd, contentSize) | |
| 240 | - | if err != nil { | |
| 241 | - | closeOpened() | |
| 242 | - | return err | |
| 243 | - | } | |
| 244 | - | p := newPaneWithStableID(paneSnap.ID, paneStableID, pty, contentSize) | |
| 245 | - | p.ctx, p.cancel = context.WithCancel(tb.ctx) | |
| 246 | - | p.rect = contentRect | |
| 247 | - | if err := restorePaneTerminal(p, paneSnap); err != nil { | |
| 248 | - | // p is not in tb.panes yet, so closeOpened cannot reach it. | |
| 249 | - | p.cancel() | |
| 250 | - | _ = pty.Close() | |
| 251 | - | closeOpened() | |
| 252 | - | return err | |
| 253 | - | } | |
| 254 | - | if restoreCommand != "" { | |
| 255 | - | if _, err := pty.Write([]byte(restoreCommand + "\n")); err != nil { | |
| 256 | - | d.log.Warn("writing snapshot restore command failed", "err", err, "session", snap.Name, "pane", paneSnap.ID) | |
| 257 | - | d.NotifyGlobal(domain.NoticeWarn, domain.NoticeAutoResume, | |
| 258 | - | "couldn't restore the running program in session "+snap.Name, err) | |
| 259 | - | } | |
| 260 | - | } | |
| 261 | - | tb.panes[paneSnap.ID] = p | |
| 262 | - | } | |
| 263 | - | } | |
| 264 | - | if len(opened) == 0 { | |
| 265 | - | cancel() | |
| 266 | - | return nil | |
| 267 | - | } | |
| 268 | - | createdAt := int64(snap.CreatedAt) | |
| 269 | - | sess := &session{name: snap.Name, ctx: sctx, cancel: cancel, tabs: opened, active: int(snap.Active), terminal: restoreTerm, env: restoreEnv, createdAt: createdAt, snapshotWake: d.snapshotWake, snapshotChunkCache: newSnapshotChunkCache(snapshotChunkCacheLimit)} | |
| 270 | - | sess.snapEligible.Store(true) | |
| 271 | - | if len(snap.Tabs) > 0 && len(snap.Tabs[0].Panes) > 0 { | |
| 272 | - | sess.cwd = snap.Tabs[0].Panes[0].Cwd | |
| 273 | - | } | |
| 274 | - | d.mu.Lock() | |
| 275 | - | if stopped, ok := d.stopped[snap.Name]; ok { | |
| 276 | - | sess.mruAt.Store(stopped.lastUsedSeq) | |
| 277 | - | } | |
| 278 | - | d.mu.Unlock() | |
| 279 | - | record := sess.persistRecordLocked(createdAt) | |
| 280 | - | if err := d.persist.Save(record); err != nil { | |
| 281 | - | closeOpened() | |
| 282 | - | return err | |
| 283 | - | } | |
| 284 | - | d.mu.Lock() | |
| 285 | - | if d.closing || d.findByNameLocked(snap.Name) != nil || ctx.Err() != nil { | |
| 286 | - | d.mu.Unlock() | |
| 287 | - | closeOpened() | |
| 288 | - | return nil | |
| 289 | - | } | |
| 290 | - | id := domain.SessionID(fmt.Sprintf("sess-%d", d.nextID)) | |
| 291 | - | d.nextID++ | |
| 292 | - | sess.id = id | |
| 293 | - | delete(d.stopped, snap.Name) | |
| 294 | - | d.sessions[id] = sess | |
| 295 | - | d.mu.Unlock() | |
| 296 | - | sess.snapDirty.Store(false) | |
| 297 | - | for _, tb := range opened { | |
| 298 | - | d.startTabGoroutines(sess, tb) | |
| 299 | - | } | |
| 300 | - | return nil | |
| 301 | - | } | |
| 302 | - | ||
| 303 | - | // snapshotCoordinatorContext returns the cancellation context shared by this | |
| 304 | - | // session's queued and in-flight captures. snapshotMu serializes creation with | |
| 305 | - | // quarantine, so a publisher can never obtain a live context after quarantine. | |
| 306 | - | func snapshotCoordinatorContext(sess *session) (context.Context, context.CancelFunc) { | |
| 307 | - | sess.snapshotMu.Lock() | |
| 308 | - | defer sess.snapshotMu.Unlock() | |
| 309 | - | if sess.snapshotPublicationContext == nil { | |
| 310 | - | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |
| 311 | - | } | |
| 312 | - | return sess.snapshotPublicationContext, sess.snapshotPublicationCancel | |
| 313 | - | } | |
| 314 | - | ||
| 315 | - | // snapshotCoordinatorQuarantine identifies one stop request. A later teardown | |
| 316 | - | // can supersede it, in which case only that later owner may resume publication. | |
| 317 | - | type snapshotCoordinatorQuarantine struct { | |
| 318 | - | done <-chan struct{} | |
| 319 | - | epoch uint64 | |
| 320 | - | } | |
| 321 | - | ||
| 322 | - | // quarantineSnapshotCoordinator prevents a deleted session from starting any | |
| 323 | - | // further publication. The returned join channel is closed after an already | |
| 324 | - | // started repository call returns; callers must wait without daemon or session | |
| 325 | - | // locks before deleting repository state. | |
| 326 | - | func quarantineSnapshotCoordinator(sess *session) <-chan struct{} { | |
| 327 | - | return quarantineSnapshotCoordinatorWithEpoch(sess).done | |
| 328 | - | } | |
| 329 | - | ||
| 330 | - | // quarantineSnapshotCoordinatorRetainingQueuedCapture is used after a forced | |
| 331 | - | // shutdown checkpoint times out. Its queued capture is logical retry state, not | |
| 332 | - | // a repository operation: startSnapshotPublication rejects it while | |
| 333 | - | // quarantined, and the worker eventually discards it after its blocked call | |
| 334 | - | // returns. | |
| 335 | - | func quarantineSnapshotCoordinatorRetainingQueuedCapture(sess *session) <-chan struct{} { | |
| 336 | - | return quarantineSnapshotCoordinatorWithOptions(sess, true).done | |
| 337 | - | } | |
| 338 | - | ||
| 339 | - | func quarantineSnapshotCoordinatorWithEpoch(sess *session) snapshotCoordinatorQuarantine { | |
| 340 | - | return quarantineSnapshotCoordinatorWithOptions(sess, false) | |
| 341 | - | } | |
| 342 | - | ||
| 343 | - | func quarantineSnapshotCoordinatorWithOptions(sess *session, retainQueuedCapture bool) snapshotCoordinatorQuarantine { | |
| 344 | - | if sess == nil { | |
| 345 | - | done := make(chan struct{}) | |
| 346 | - | close(done) | |
| 347 | - | return snapshotCoordinatorQuarantine{done: done} | |
| 348 | - | } | |
| 349 | - | sess.snapshotMu.Lock() | |
| 350 | - | sess.snapshotQuarantineEpoch++ | |
| 351 | - | epoch := sess.snapshotQuarantineEpoch | |
| 352 | - | sess.snapshotQuarantined = true | |
| 353 | - | sess.snapEligible.Store(false) | |
| 354 | - | if sess.snapshotPublicationCancel != nil { | |
| 355 | - | sess.snapshotPublicationCancel() | |
| 356 | - | } | |
| 357 | - | if sess.snapshotQueuedCapture != nil && !retainQueuedCapture { | |
| 358 | - | // A rename or purge must discard a capture under the old identity. The | |
| 359 | - | // global queue may still contain it, but startSnapshotPublication | |
| 360 | - | // compares identity and will discard it even if this session is renamed. | |
| 361 | - | sess.snapshotQueuedCapture.coordinatorDiscarded = true | |
| 362 | - | sess.snapshotQueuedCapture = nil | |
| 363 | - | if sess.snapshotPendingCaptures > 0 { | |
| 364 | - | sess.snapshotPendingCaptures-- | |
| 365 | - | } | |
| 366 | - | sess.snapshotPending = sess.snapshotPendingCaptures > 0 || sess.snapshotInFlightCapture != nil | |
| 367 | - | } | |
| 368 | - | done := sess.snapshotPublicationDone | |
| 369 | - | if done == nil { | |
| 370 | - | done = make(chan struct{}) | |
| 371 | - | close(done) | |
| 372 | - | } | |
| 373 | - | sess.signalSnapshotChangedLocked() | |
| 374 | - | sess.snapshotMu.Unlock() | |
| 375 | - | return snapshotCoordinatorQuarantine{done: done, epoch: epoch} | |
| 376 | - | } | |
| 377 | - | ||
| 378 | - | // resumeSnapshotCoordinatorForNewIdentity starts a fresh repository lineage | |
| 379 | - | // only after rename committed its new in-memory name. The epoch prevents a | |
| 380 | - | // concurrent teardown from being undone by a late rename completion. | |
| 381 | - | func resumeSnapshotCoordinatorForNewIdentity(sess *session, quarantine snapshotCoordinatorQuarantine) bool { | |
| 382 | - | if sess == nil { | |
| 383 | - | return false | |
| 384 | - | } | |
| 385 | - | sess.snapshotMu.Lock() | |
| 386 | - | if !sess.snapshotQuarantined || sess.snapshotQuarantineEpoch != quarantine.epoch { | |
| 387 | - | sess.snapshotMu.Unlock() | |
| 388 | - | return false | |
| 389 | - | } | |
| 390 | - | if sess.snapshotPublicationCancel != nil { | |
| 391 | - | sess.snapshotPublicationCancel() | |
| 392 | - | } | |
| 393 | - | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |
| 394 | - | sess.snapshotQuarantined = false | |
| 395 | - | // A repository name owns its own generation stream. Reset all scheduler | |
| 396 | - | // state while eligibility remains false, then atomically make generation 1 | |
| 397 | - | // dirty and wake the scheduler. | |
| 398 | - | sess.snapshotGeneration = 1 | |
| 399 | - | sess.snapshotCapturedGeneration = 0 | |
| 400 | - | sess.snapshotPublishedGeneration = 0 | |
| 401 | - | sess.snapshotForcedGeneration = 0 | |
| 402 | - | sess.snapshotNextEligibleAt = time.Time{} | |
| 403 | - | sess.snapshotAttempted = false | |
| 404 | - | sess.snapshotAttemptKind = snapshotAttemptRoutine | |
| 405 | - | sess.snapshotFailureSig = "" | |
| 406 | - | sess.snapshotChunkCache = newSnapshotChunkCache(snapshotChunkCacheLimit) | |
| 407 | - | sess.snapDirty.Store(true) | |
| 408 | - | sess.snapEligible.Store(true) | |
| 409 | - | sess.signalSnapshotChangedLocked() | |
| 410 | - | wake := sess.snapshotWake | |
| 411 | - | sess.snapshotMu.Unlock() | |
| 412 | - | if wake != nil { | |
| 413 | - | select { | |
| 414 | - | case wake <- struct{}{}: | |
| 415 | - | default: | |
| 416 | - | } | |
| 417 | - | } | |
| 418 | - | return true | |
| 419 | - | } | |
| 420 | - | ||
| 421 | - | func markSnapshotDirty(sess *session) { | |
| 422 | - | if sess == nil || !sess.snapEligible.Load() { | |
| 423 | - | return | |
| 424 | - | } | |
| 425 | - | sess.snapshotMu.Lock() | |
| 426 | - | sess.snapshotGeneration++ | |
| 427 | - | sess.snapDirty.Store(true) | |
| 428 | - | sess.signalSnapshotChangedLocked() | |
| 429 | - | wake := sess.snapshotWake | |
| 430 | - | sess.snapshotMu.Unlock() | |
| 431 | - | if wake != nil { | |
| 432 | - | select { | |
| 433 | - | case wake <- struct{}{}: | |
| 434 | - | default: | |
| 435 | - | } | |
| 436 | - | } | |
| 437 | - | } | |
| 438 | - | ||
| 439 | - | // snapshotChangeLocked returns a channel closed by the next snapshot state | |
| 440 | - | // transition. snapshotMu must be held. | |
| 441 | - | func (sess *session) snapshotChangeLocked() chan struct{} { | |
| 442 | - | if sess.snapshotChanged == nil { | |
| 443 | - | sess.snapshotChanged = make(chan struct{}) | |
| 444 | - | } | |
| 445 | - | return sess.snapshotChanged | |
| 446 | - | } | |
| 447 | - | ||
| 448 | - | // signalSnapshotChangedLocked wakes waiters after a snapshot state transition. | |
| 449 | - | // snapshotMu must be held. | |
| 450 | - | func (sess *session) signalSnapshotChangedLocked() { | |
| 451 | - | close(sess.snapshotChangeLocked()) | |
| 452 | - | sess.snapshotChanged = make(chan struct{}) | |
| 453 | - | } | |
| 454 | - | ||
| 455 | - | func restorePaneTerminal(p *pane, snap snapcodec.Pane) error { | |
| 456 | - | if len(snap.Tail) == 0 { | |
| 457 | - | return fmt.Errorf("snapshot history: missing tail blob") | |
| 458 | - | } | |
| 459 | - | if len(snap.Visible) == 0 { | |
| 460 | - | return fmt.Errorf("snapshot visible: missing visible blob") | |
| 461 | - | } | |
| 462 | - | p.mu.Lock() | |
| 463 | - | defer p.mu.Unlock() | |
| 464 | - | screen, err := vt.NewScreenWithRestoredHistory(p.screen.Frame.Width, p.screen.Frame.Height, vt.HistoryConfig{MaxRows: defaultScrollbackRows, MaxCells: defaultScrollbackCells}, snap.SealedChunks, snap.Tail) | |
| 465 | - | if err != nil { | |
| 466 | - | return fmt.Errorf("snapshot history: %w", err) | |
| 467 | - | } | |
| 468 | - | if err := screen.RestorePrimaryVisible(snap.Visible); err != nil { | |
| 469 | - | return fmt.Errorf("snapshot visible: %w", err) | |
| 470 | - | } | |
| 471 | - | p.screen = screen | |
| 472 | - | p.history = screen.History() | |
| 473 | - | return nil | |
| 474 | - | } | |
| 475 | - | ||
| 476 | - | func restorePTYSize(contentRect domain.Rect, tabSize domain.Size) domain.Size { | |
| 477 | - | cols := contentRect.Width | |
| 478 | - | if cols <= 0 { | |
| 479 | - | cols = tabSize.Cols | |
| 480 | - | } | |
| 481 | - | if cols < layout.MinPaneCols { | |
| 482 | - | cols = layout.MinPaneCols | |
| 483 | - | } | |
| 484 | - | rows := contentRect.Height | |
| 485 | - | if rows <= 0 { | |
| 486 | - | rows = min(max(tabSize.Rows, layout.MinPaneRows), 24) | |
| 487 | - | } | |
| 488 | - | if rows < layout.MinPaneRows { | |
| 489 | - | rows = layout.MinPaneRows | |
| 490 | - | } | |
| 491 | - | return domain.Size{Cols: cols, Rows: rows} | |
| 492 | - | } | |
| 493 | - | ||
| 494 | - | // scheduleSnapshot captures a session only once while an immutable capture is | |
| 495 | - | // queued or being written. It never waits for the bounded worker queue. | |
| 496 | - | func (d *Daemon) scheduleSnapshot(sess *session) bool { | |
| 497 | - | return d.scheduleSnapshotWithFinalFallback(sess, false) | |
| 498 | - | } | |
| 499 | - | ||
| 500 | - | // scheduleFinalSnapshot captures the terminal state even when an older capture | |
| 501 | - | // is pending. A removed session cannot be retried by the repository scheduler, so the | |
| 502 | - | // worker drains its retained terminal state before it stops. | |
| 503 | - | func (d *Daemon) scheduleFinalSnapshot(sess *session) bool { | |
| 504 | - | return d.scheduleSnapshotWithFinalFallback(sess, true) | |
| 505 | - | } | |
| 506 | - | ||
| 507 | - | func (d *Daemon) scheduleSnapshotWithFinalFallback(sess *session, final bool) bool { | |
| 508 | - | if d == nil || sess == nil || !d.snapsEnabled || d.snapshotRepository == nil || !sess.snapEligible.Load() { | |
| 509 | - | return true | |
| 510 | - | } | |
| 511 | - | kind := snapshotAttemptRoutine | |
| 512 | - | if final { | |
| 513 | - | kind = snapshotAttemptForced | |
| 514 | - | } | |
| 515 | - | sess.snapshotMu.Lock() | |
| 516 | - | routineRateLimited := d.snapshotRepository != nil | |
| 517 | - | if sess.snapshotQuarantined || !sess.snapDirty.Load() { | |
| 518 | - | available := !sess.snapshotQuarantined | |
| 519 | - | sess.snapshotMu.Unlock() | |
| 520 | - | return available | |
| 521 | - | } | |
| 522 | - | if final && sess.snapshotGeneration > sess.snapshotForcedGeneration { | |
| 523 | - | // Do not let an older queued or in-flight routine publication satisfy a | |
| 524 | - | // terminal checkpoint. The intent is retained until a publication at | |
| 525 | - | // this generation (or newer) succeeds. | |
| 526 | - | sess.snapshotForcedGeneration = sess.snapshotGeneration | |
| 527 | - | } | |
| 528 | - | if sess.snapshotPending || | |
| 529 | - | (!final && routineRateLimited && !sess.snapshotNextEligibleAt.IsZero() && d.clock.Now().Before(sess.snapshotNextEligibleAt)) { | |
| 530 | - | sess.snapshotMu.Unlock() | |
| 531 | - | return true | |
| 532 | - | } | |
| 533 | - | if sess.snapshotPublicationContext == nil { | |
| 534 | - | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |
| 535 | - | } | |
| 536 | - | generation := sess.snapshotGeneration | |
| 537 | - | publicationContext := sess.snapshotPublicationContext | |
| 538 | - | sess.snapshotCapturedGeneration = generation | |
| 539 | - | sess.snapshotAttempted = true | |
| 540 | - | sess.snapshotAttemptKind = kind | |
| 541 | - | sess.snapshotPendingCaptures++ | |
| 542 | - | sess.snapshotPending = true | |
| 543 | - | sess.signalSnapshotChangedLocked() | |
| 544 | - | sess.snapshotMu.Unlock() | |
| 545 | - | ||
| 546 | - | capture, ok := d.captureSnapshotState(sess, generation) | |
| 547 | - | if capture != nil { | |
| 548 | - | capture.attemptKind = kind | |
| 549 | - | capture.publicationContext = publicationContext | |
| 550 | - | } | |
| 551 | - | if !ok { | |
| 552 | - | d.finishSnapshotCapture(capture, false) | |
| 553 | - | return false | |
| 554 | - | } | |
| 555 | - | sess.snapshotMu.Lock() | |
| 556 | - | quarantined := sess.snapshotQuarantined | |
| 557 | - | if !quarantined { | |
| 558 | - | sess.snapshotQueuedCapture = capture | |
| 559 | - | } | |
| 560 | - | sess.snapshotMu.Unlock() | |
| 561 | - | if quarantined { | |
| 562 | - | d.finishSnapshotCapture(capture, false) | |
| 563 | - | return false | |
| 564 | - | } | |
| 565 | - | if d.enqueueSnapshotCapture(capture) || (final && d.enqueueFinalSnapshotCapture(capture)) { | |
| 566 | - | return true | |
| 567 | - | } | |
| 568 | - | // Coalesce under saturation or shutdown: the latest state stays dirty and | |
| 569 | - | // will be captured on a later tick once an active worker has room. | |
| 570 | - | d.finishSnapshotCapture(capture, false) | |
| 571 | - | return false | |
| 572 | - | } | |
| 573 | - | ||
| 574 | - | // waitForSnapshotGeneration waits only for the bounded terminal checkpoint | |
| 575 | - | // deadline. It observes successful publication, rather than dirty state, so a | |
| 576 | - | // concurrent later mutation cannot turn a completed forced checkpoint into a | |
| 577 | - | // false timeout. | |
| 578 | - | func (d *Daemon) waitForSnapshotGeneration(sess *session, generation uint64) bool { | |
| 579 | - | if d == nil || sess == nil || generation == 0 { | |
| 580 | - | return true | |
| 581 | - | } | |
| 582 | - | deadline := newSnapshotShutdownDeadline(d.clock) | |
| 583 | - | defer deadline.stop() | |
| 584 | - | return d.waitForSnapshotGenerationWithDeadline(sess, generation, deadline) | |
| 585 | - | } | |
| 586 | - | ||
| 587 | - | // waitForSnapshotGenerationWithDeadline waits for a forced checkpoint without | |
| 588 | - | // allocating another shutdown interval. Serve shares one deadline across every | |
| 589 | - | // session and the subsequent worker drain. | |
| 590 | - | func (d *Daemon) waitForSnapshotGenerationWithDeadline(sess *session, generation uint64, deadline *snapshotShutdownDeadline) bool { | |
| 591 | - | if d == nil || sess == nil || generation == 0 { | |
| 592 | - | return true | |
| 593 | - | } | |
| 594 | - | if deadline == nil { | |
| 595 | - | return d.waitForSnapshotGeneration(sess, generation) | |
| 596 | - | } | |
| 597 | - | for { | |
| 598 | - | sess.snapshotMu.Lock() | |
| 599 | - | published := sess.snapshotPublishedGeneration >= generation | |
| 600 | - | changed := sess.snapshotChangeLocked() | |
| 601 | - | sess.snapshotMu.Unlock() | |
| 602 | - | if published { | |
| 603 | - | return true | |
| 604 | - | } | |
| 605 | - | select { | |
| 606 | - | case <-changed: | |
| 607 | - | case <-deadline.Done(): | |
| 608 | - | return false | |
| 609 | - | } | |
| 610 | - | } | |
| 611 | - | } | |
| 612 | - | ||
| 613 | - | // enqueueSnapshotCapture serializes worker shutdown with the non-blocking | |
| 614 | - | // queue send. The queue is deliberately never closed: producers can race | |
| 615 | - | // shutdown without risking a send-on-closed panic. | |
| 616 | - | func (d *Daemon) enqueueSnapshotCapture(capture *snapshotCapture) bool { | |
| 617 | - | d.snapshotWorkerMu.Lock() | |
| 618 | - | defer d.snapshotWorkerMu.Unlock() | |
| 619 | - | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |
| 620 | - | return false | |
| 621 | - | } | |
| 622 | - | select { | |
| 623 | - | case d.snapshotJobs <- capture: | |
| 624 | - | return true | |
| 625 | - | default: | |
| 626 | - | return false | |
| 627 | - | } | |
| 628 | - | } | |
| 629 | - | ||
| 630 | - | // enqueueFinalSnapshotCapture hands terminal captures to the existing worker | |
| 631 | - | // without making session teardown wait behind its normal bounded queue. A | |
| 632 | - | // blocked worker retains at most one terminal capture per session, replacing | |
| 633 | - | // stale state with the newest immutable capture. | |
| 634 | - | func (d *Daemon) enqueueFinalSnapshotCapture(capture *snapshotCapture) bool { | |
| 635 | - | if capture == nil || capture.session == nil { | |
| 636 | - | return false | |
| 637 | - | } | |
| 638 | - | d.snapshotWorkerMu.Lock() | |
| 639 | - | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |
| 640 | - | d.snapshotWorkerMu.Unlock() | |
| 641 | - | return false | |
| 642 | - | } | |
| 643 | - | if d.snapshotFinalJobs == nil { | |
| 644 | - | d.snapshotFinalJobs = make(map[*session]*snapshotCapture) | |
| 645 | - | } | |
| 646 | - | replaced, exists := d.snapshotFinalJobs[capture.session] | |
| 647 | - | if !exists && len(d.snapshotFinalJobs) >= snapshotFinalQueueCapacity { | |
| 648 | - | d.snapshotWorkerMu.Unlock() | |
| 649 | - | d.log.Warn("terminal snapshot retention saturated; capture rejected", "session", capture.name, "capacity", snapshotFinalQueueCapacity) | |
| 650 | - | return false | |
| 651 | - | } | |
| 652 | - | // Finish the stale capture before publishing its replacement. Otherwise a | |
| 653 | - | // fast worker could persist the replacement and then have this stale failure | |
| 654 | - | // mark the session dirty again. | |
| 655 | - | d.finishSnapshotCapture(replaced, false) | |
| 656 | - | if !exists { | |
| 657 | - | d.snapshotFinalOrder = append(d.snapshotFinalOrder, capture.session) | |
| 658 | - | } | |
| 659 | - | d.snapshotFinalJobs[capture.session] = capture | |
| 660 | - | select { | |
| 661 | - | case d.snapshotWorkerFinalWake <- struct{}{}: | |
| 662 | - | default: | |
| 663 | - | } | |
| 664 | - | d.snapshotWorkerMu.Unlock() | |
| 665 | - | return true | |
| 666 | - | } | |
| 667 | - | ||
| 668 | - | // captureSession rotates history tails and clones visible frames while holding | |
| 669 | - | // each pane lock. The returned capture contains only immutable state; encoding | |
| 670 | - | // and persistence are deliberately deferred to snapshotEncodeWorker. | |
| 671 | - | func (d *Daemon) captureSnapshotState(sess *session, generation uint64) (*snapshotCapture, bool) { | |
| 672 | - | sess.mu.Lock() | |
| 673 | - | capture := &snapshotCapture{ | |
| 674 | - | session: sess, | |
| 675 | - | generation: generation, | |
| 676 | - | name: sess.name, | |
| 677 | - | createdAt: uint64(sess.createdAt), | |
| 678 | - | active: uint16(max(sess.active, 0)), | |
| 679 | - | } | |
| 680 | - | ephemeral := sess.ephemeral | |
| 681 | - | fallbackCwd := sess.cwd | |
| 682 | - | tabs := append([]*tab(nil), sess.tabs...) | |
| 683 | - | sess.mu.Unlock() | |
| 684 | - | if ephemeral || capture.name == "" { | |
| 685 | - | return capture, false | |
| 686 | - | } | |
| 687 | - | ||
| 688 | - | capture.tabs = make([]snapshotCaptureTab, 0, len(tabs)) | |
| 689 | - | for _, tb := range tabs { | |
| 690 | - | tb.mu.Lock() | |
| 691 | - | tabCapture := snapshotCaptureTab{ | |
| 692 | - | stableID: tb.stableID, | |
| 693 | - | cols: uint16(max(tb.size.Cols, 0)), | |
| 694 | - | rows: uint16(max(tb.size.Rows, 0)), | |
| 695 | - | nextPaneID: uint64(max(tb.nextPaneID, 0)), | |
| 696 | - | tree: tb.tree.Clone(), | |
| 697 | - | } | |
| 698 | - | if tb.tree != nil { | |
| 699 | - | tabCapture.focus = tb.tree.Focus | |
| 700 | - | } | |
| 701 | - | panes := make([]*pane, 0, len(tb.panes)) | |
| 702 | - | for _, p := range tb.panes { | |
| 703 | - | panes = append(panes, p) | |
| 704 | - | } | |
| 705 | - | tb.mu.Unlock() | |
| 706 | - | ||
| 707 | - | tabCapture.panes = make([]snapshotCapturePane, 0, len(panes)) | |
| 708 | - | for _, p := range panes { | |
| 709 | - | p.mu.Lock() | |
| 710 | - | pty := p.pty | |
| 711 | - | pid := 0 | |
| 712 | - | if pty != nil { | |
| 713 | - | pid = pty.Pid() | |
| 714 | - | } | |
| 715 | - | paneCapture := snapshotCapturePane{ | |
| 716 | - | id: p.id, | |
| 717 | - | stableID: p.stableID, | |
| 718 | - | visible: p.screen.PrimaryVisibleSnapshot(), | |
| 719 | - | } | |
| 720 | - | paneCapture.sealed = p.history.SnapshotView() | |
| 721 | - | paneCapture.tail = paneCapture.sealed.Tail() | |
| 722 | - | p.mu.Unlock() | |
| 723 | - | paneCapture.cwd = fallbackCwd | |
| 724 | - | if d.procCwd != nil && pid > 0 { | |
| 725 | - | if cwd, err := d.procCwd(pid); err == nil && cwd != "" { | |
| 726 | - | paneCapture.cwd = cwd | |
| 727 | - | } | |
| 728 | - | } | |
| 729 | - | paneCapture.process = d.capturePaneProcess(pty, pid) | |
| 730 | - | tabCapture.panes = append(tabCapture.panes, paneCapture) | |
| 731 | - | } | |
| 732 | - | capture.tabs = append(capture.tabs, tabCapture) | |
| 733 | - | } | |
| 734 | - | return capture, true | |
| 735 | - | } | |
| 736 | - | ||
| 737 | - | // captureSession remains the synchronous producer-facing trigger for callers | |
| 738 | - | // such as teardown and benchmarks; the actual encoding and Write stay async. | |
| 739 | - | func (d *Daemon) captureSession(sess *session) bool { | |
| 740 | - | markSnapshotDirty(sess) | |
| 741 | - | return d.scheduleSnapshot(sess) | |
| 742 | - | } | |
| 743 | - | ||
| 744 | - | // startSnapshotPublication moves a retained capture from the producer queue to | |
| 745 | - | // the repository worker. Quarantine and this transition share snapshotMu, so a | |
| 746 | - | // repository call cannot begin after destructive teardown has quarantined the | |
| 747 | - | // session. | |
| 748 | - | func startSnapshotPublication(capture *snapshotCapture) bool { | |
| 749 | - | if capture == nil || capture.session == nil { | |
| 750 | - | return false | |
| 751 | - | } | |
| 752 | - | sess := capture.session | |
| 753 | - | sess.snapshotMu.Lock() | |
| 754 | - | defer sess.snapshotMu.Unlock() | |
| 755 | - | if sess.snapshotQuarantined || sess.snapshotQueuedCapture != capture { | |
| 756 | - | return false | |
| 757 | - | } | |
| 758 | - | sess.snapshotQueuedCapture = nil | |
| 759 | - | sess.snapshotInFlightCapture = capture | |
| 760 | - | sess.snapshotPublicationDone = make(chan struct{}) | |
| 761 | - | sess.snapshotPending = true | |
| 762 | - | sess.signalSnapshotChangedLocked() | |
| 763 | - | return true | |
| 764 | - | } | |
| 765 | - | ||
| 766 | - | func (d *Daemon) finishSnapshotCapture(capture *snapshotCapture, succeeded bool) { | |
| 767 | - | if capture == nil { | |
| 768 | - | return | |
| 769 | - | } | |
| 770 | - | capture.finishOnce.Do(func() { | |
| 771 | - | shouldScheduleForcedSuccessor := false | |
| 772 | - | capture.session.snapshotMu.Lock() | |
| 773 | - | if capture.session.snapshotPendingCaptures > 0 { | |
| 774 | - | capture.session.snapshotPendingCaptures-- | |
| 775 | - | } | |
| 776 | - | if capture.session.snapshotQueuedCapture == capture { | |
| 777 | - | capture.session.snapshotQueuedCapture = nil | |
| 778 | - | } | |
| 779 | - | if capture.session.snapshotInFlightCapture == capture { | |
| 780 | - | capture.session.snapshotInFlightCapture = nil | |
| 781 | - | if capture.session.snapshotPublicationDone != nil { | |
| 782 | - | close(capture.session.snapshotPublicationDone) | |
| 783 | - | capture.session.snapshotPublicationDone = nil | |
| 784 | - | } | |
| 785 | - | } | |
| 786 | - | capture.session.snapshotPending = capture.session.snapshotPendingCaptures > 0 || | |
| 787 | - | capture.session.snapshotQueuedCapture != nil || capture.session.snapshotInFlightCapture != nil | |
| 788 | - | if !capture.coordinatorDiscarded && capture.attemptKind == snapshotAttemptRoutine && d.snapshotRepository != nil { | |
| 789 | - | // The interval is measured from completion, including a failed | |
| 790 | - | // attempt. A forced teardown checkpoint deliberately leaves it alone. | |
| 791 | - | capture.session.snapshotNextEligibleAt = d.clock.Now().Add(snapshotInterval) | |
| 792 | - | } | |
| 793 | - | if succeeded { | |
| 794 | - | capture.session.snapshotPublishedGeneration = max(capture.session.snapshotPublishedGeneration, capture.generation) | |
| 795 | - | } | |
| 796 | - | if succeeded && capture.session.snapshotGeneration == capture.generation { | |
| 797 | - | capture.session.snapDirty.Store(false) | |
| 798 | - | capture.session.snapshotFailureSig = "" | |
| 799 | - | } else if !succeeded { | |
| 800 | - | capture.session.snapDirty.Store(true) | |
| 801 | - | } | |
| 802 | - | if succeeded && capture.session.snapshotForcedGeneration != 0 && | |
| 803 | - | capture.generation >= capture.session.snapshotForcedGeneration { | |
| 804 | - | capture.session.snapshotForcedGeneration = 0 | |
| 805 | - | } | |
| 806 | - | // A failed forced publication remains dirty for ordinary retry rather | |
| 807 | - | // than recursively creating forced jobs. A later forced request records | |
| 808 | - | // a newer intent and can create its one bounded successor. | |
| 809 | - | shouldScheduleForcedSuccessor = capture.session.snapshotForcedGeneration != 0 && | |
| 810 | - | capture.generation < capture.session.snapshotForcedGeneration && | |
| 811 | - | capture.session.snapshotQueuedCapture == nil && capture.session.snapshotInFlightCapture == nil && | |
| 812 | - | (succeeded || capture.attemptKind != snapshotAttemptForced) | |
| 813 | - | capture.session.signalSnapshotChangedLocked() | |
| 814 | - | wake := capture.session.snapshotWake | |
| 815 | - | capture.session.snapshotMu.Unlock() | |
| 816 | - | if wake != nil { | |
| 817 | - | select { | |
| 818 | - | case wake <- struct{}{}: | |
| 819 | - | default: | |
| 820 | - | } | |
| 821 | - | } | |
| 822 | - | if shouldScheduleForcedSuccessor { | |
| 823 | - | // This runs after publication completion and outside snapshotMu. It | |
| 824 | - | // bypasses routine eligibility while preserving the single queued and | |
| 825 | - | // single in-flight capture bound. | |
| 826 | - | _ = d.scheduleFinalSnapshot(capture.session) | |
| 827 | - | } | |
| 828 | - | if succeeded { | |
| 829 | - | d.clearSnapshotFailure() | |
| 830 | - | } | |
| 831 | - | }) | |
| 832 | - | } | |
| 833 | - | ||
| 834 | - | // reportSnapshotFailure records every failed persistence attempt but only | |
| 835 | - | // displays a toast when its stable phase/error class changes. This keeps a | |
| 836 | - | // blocked disk from repeatedly repainting every attached client while retaining | |
| 837 | - | // the count in notification history. | |
| 838 | - | func (d *Daemon) reportSnapshotFailure(capture *snapshotCapture, phase string, cause error) { | |
| 839 | - | if d == nil || capture == nil || capture.session == nil || cause == nil { | |
| 840 | - | return | |
| 841 | - | } | |
| 842 | - | signature := snapshotFailureSignature(phase, cause) | |
| 843 | - | capture.session.snapshotMu.Lock() | |
| 844 | - | capture.session.snapshotFailureSig = signature | |
| 845 | - | capture.session.snapshotMu.Unlock() | |
| 846 | - | ||
| 847 | - | d.snapshotNoticeMu.Lock() | |
| 848 | - | changed := d.snapshotActiveFailureSignature != signature | |
| 849 | - | d.snapshotActiveFailureSignature = signature | |
| 850 | - | d.snapshotNoticeMu.Unlock() | |
| 851 | - | ||
| 852 | - | n := domain.Notification{ | |
| 853 | - | Code: domain.NoticeSnapshotWrite, | |
| 854 | - | Severity: domain.NoticeError, | |
| 855 | - | Message: "couldn't save session state; recent state may be lost on restart", | |
| 856 | - | // The history is intentionally diagnostic only at the error-class level. | |
| 857 | - | // Raw causes can include user paths and terminal content. | |
| 858 | - | Details: signature, | |
| 859 | - | Time: d.clock.Now(), | |
| 860 | - | } | |
| 861 | - | n, _ = d.notices.recordSnapshotFailure(n) | |
| 862 | - | d.log.Warn("writing session snapshot failed", "phase", phase, "class", signature, "session", capture.name) | |
| 863 | - | if changed { | |
| 864 | - | d.deliverGlobal(n) | |
| 865 | - | } | |
| 866 | - | } | |
| 867 | - | ||
| 868 | - | func (d *Daemon) clearSnapshotFailure() { | |
| 869 | - | if d == nil { | |
| 870 | - | return | |
| 871 | - | } | |
| 872 | - | d.snapshotNoticeMu.Lock() | |
| 873 | - | d.snapshotActiveFailureSignature = "" | |
| 874 | - | d.snapshotNoticeMu.Unlock() | |
| 875 | - | } | |
| 876 | - | ||
| 877 | - | func (d *Daemon) startSnapshotEncodeWorker() { | |
| 878 | - | if d == nil { | |
| 879 | - | return | |
| 880 | - | } | |
| 881 | - | d.snapshotWorkerMu.Lock() | |
| 882 | - | if d.snapshotWorkerCancel != nil { | |
| 883 | - | d.snapshotWorkerMu.Unlock() | |
| 884 | - | return | |
| 885 | - | } | |
| 886 | - | // Shutdown owns the worker lifetime so it can flush captures after Serve | |
| 887 | - | // cancels its parent context. The worker is always stopped explicitly. | |
| 888 | - | workerCtx, cancel := context.WithCancel(context.Background()) | |
| 889 | - | d.snapshotWorkerID++ | |
| 890 | - | workerID := d.snapshotWorkerID | |
| 891 | - | d.snapshotWorkerCtx = workerCtx | |
| 892 | - | d.snapshotWorkerCancel = cancel | |
| 893 | - | d.snapshotWorkerDone = make(chan struct{}) | |
| 894 | - | d.snapshotWorkerFlush = make(chan struct{}) | |
| 895 | - | d.snapshotWorkerFinalWake = make(chan struct{}, 1) | |
| 896 | - | d.snapshotFinalJobs = make(map[*session]*snapshotCapture) | |
| 897 | - | d.snapshotFinalOrder = nil | |
| 898 | - | d.snapshotWorkerClosing = false | |
| 899 | - | d.snapshotWorkerInFlight = nil | |
| 900 | - | done := d.snapshotWorkerDone | |
| 901 | - | flush := d.snapshotWorkerFlush | |
| 902 | - | finalWake := d.snapshotWorkerFinalWake | |
| 903 | - | d.snapshotWorkerMu.Unlock() | |
| 904 | - | go func() { | |
| 905 | - | defer close(done) | |
| 906 | - | write := func(capture *snapshotCapture) bool { | |
| 907 | - | if workerCtx.Err() != nil || !startSnapshotPublication(capture) || !d.setSnapshotWorkerInFlight(workerID, capture) { | |
| 908 | - | d.finishSnapshotCapture(capture, false) | |
| 909 | - | return workerCtx.Err() == nil | |
| 910 | - | } | |
| 911 | - | publication, err := d.incrementalPublication(capture) | |
| 912 | - | publicationContext := capture.publicationContext | |
| 913 | - | if publicationContext == nil { | |
| 914 | - | publicationContext = workerCtx | |
| 915 | - | } | |
| 916 | - | if err == nil && workerCtx.Err() == nil { | |
| 917 | - | if err = publicationContext.Err(); err == nil { | |
| 918 | - | err = d.snapshotRepository.Publish(publicationContext, publication) | |
| 919 | - | } | |
| 920 | - | } | |
| 921 | - | d.clearSnapshotWorkerInFlight(workerID, capture) | |
| 922 | - | if err != nil && workerCtx.Err() == nil && publicationContext.Err() == nil { | |
| 923 | - | // Global, not session-scoped: by now the session may already be | |
| 924 | - | // torn down, so a session notice would be dead-on-arrival. No lock | |
| 925 | - | // is held here (clearSnapshotWorkerInFlight released its own). | |
| 926 | - | d.reportSnapshotFailure(capture, "publish", err) | |
| 927 | - | } | |
| 928 | - | d.finishSnapshotCapture(capture, err == nil && workerCtx.Err() == nil) | |
| 929 | - | return workerCtx.Err() == nil | |
| 930 | - | } | |
| 931 | - | drainFinal := func() bool { | |
| 932 | - | for { | |
| 933 | - | capture := d.takeFinalSnapshotCapture() | |
| 934 | - | if capture == nil { | |
| 935 | - | return true | |
| 936 | - | } | |
| 937 | - | if !write(capture) { | |
| 938 | - | return false | |
| 939 | - | } | |
| 940 | - | } | |
| 941 | - | } | |
| 942 | - | flushQueued := func() { | |
| 943 | - | for { | |
| 944 | - | select { | |
| 945 | - | case capture := <-d.snapshotJobs: | |
| 946 | - | if !write(capture) { | |
| 947 | - | return | |
| 948 | - | } | |
| 949 | - | default: | |
| 950 | - | _ = drainFinal() | |
| 951 | - | return | |
| 952 | - | } | |
| 953 | - | } | |
| 954 | - | } | |
| 955 | - | for { | |
| 956 | - | select { | |
| 957 | - | case <-workerCtx.Done(): | |
| 958 | - | return | |
| 959 | - | case <-flush: | |
| 960 | - | flushQueued() | |
| 961 | - | return | |
| 962 | - | case <-finalWake: | |
| 963 | - | if !drainFinal() { | |
| 964 | - | return | |
| 965 | - | } | |
| 966 | - | case capture := <-d.snapshotJobs: | |
| 967 | - | if !write(capture) { | |
| 968 | - | return | |
| 969 | - | } | |
| 970 | - | } | |
| 971 | - | } | |
| 972 | - | }() | |
| 973 | - | } | |
| 974 | - | ||
| 975 | - | func (d *Daemon) takeFinalSnapshotCapture() *snapshotCapture { | |
| 976 | - | d.snapshotWorkerMu.Lock() | |
| 977 | - | defer d.snapshotWorkerMu.Unlock() | |
| 978 | - | for len(d.snapshotFinalOrder) > 0 { | |
| 979 | - | sess := d.snapshotFinalOrder[0] | |
| 980 | - | d.snapshotFinalOrder[0] = nil | |
| 981 | - | d.snapshotFinalOrder = d.snapshotFinalOrder[1:] | |
| 982 | - | capture := d.snapshotFinalJobs[sess] | |
| 983 | - | delete(d.snapshotFinalJobs, sess) | |
| 984 | - | if capture != nil { | |
| 985 | - | return capture | |
| 986 | - | } | |
| 987 | - | } | |
| 988 | - | return nil | |
| 989 | - | } | |
| 990 | - | ||
| 991 | - | func (d *Daemon) setSnapshotWorkerInFlight(workerID uint64, capture *snapshotCapture) bool { | |
| 992 | - | d.snapshotWorkerMu.Lock() | |
| 993 | - | defer d.snapshotWorkerMu.Unlock() | |
| 994 | - | if d.snapshotWorkerID != workerID || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |
| 995 | - | return false | |
| 996 | - | } | |
| 997 | - | d.snapshotWorkerInFlight = capture | |
| 998 | - | return true | |
| 999 | - | } | |
| 1000 | - | ||
| 1001 | - | func (d *Daemon) clearSnapshotWorkerInFlight(workerID uint64, capture *snapshotCapture) { | |
| 1002 | - | d.snapshotWorkerMu.Lock() | |
| 1003 | - | defer d.snapshotWorkerMu.Unlock() | |
| 1004 | - | if d.snapshotWorkerID == workerID && d.snapshotWorkerInFlight == capture { | |
| 1005 | - | d.snapshotWorkerInFlight = nil | |
| 1006 | - | } | |
| 1007 | - | } | |
| 1008 | - | ||
| 1009 | - | func (d *Daemon) stopSnapshotEncodeWorker() { | |
| 1010 | - | if d == nil { | |
| 1011 | - | return | |
| 1012 | - | } | |
| 1013 | - | deadline := newSnapshotShutdownDeadline(d.clock) | |
| 1014 | - | defer deadline.stop() | |
| 1015 | - | d.stopSnapshotEncodeWorkerWithDeadline(deadline) | |
| 1016 | - | } | |
| 1017 | - | ||
| 1018 | - | // stopSnapshotEncodeWorkerWithDeadline performs the one deadline-aware join | |
| 1019 | - | // for a Serve shutdown. Queues are never closed, and detached writers only | |
| 1020 | - | // retain immutable captures plus contexts/channels that remain live, so an | |
| 1021 | - | // uncooperative repository call cannot race teardown or block process exit. | |
| 1022 | - | func (d *Daemon) stopSnapshotEncodeWorkerWithDeadline(deadline *snapshotShutdownDeadline) { | |
| 1023 | - | if d == nil { | |
| 1024 | - | return | |
| 1025 | - | } | |
| 1026 | - | d.snapshotWorkerMu.Lock() | |
| 1027 | - | cancel := d.snapshotWorkerCancel | |
| 1028 | - | if cancel == nil || d.snapshotWorkerClosing { | |
| 1029 | - | d.snapshotWorkerMu.Unlock() | |
| 1030 | - | return | |
| 1031 | - | } | |
| 1032 | - | // Stop accepting producers, then let the single worker drain its bounded | |
| 1033 | - | // queue. A non-context-aware store may still block indefinitely, so the | |
| 1034 | - | // shared shutdown deadline bounds the one and only join. | |
| 1035 | - | d.snapshotWorkerClosing = true | |
| 1036 | - | flush := d.snapshotWorkerFlush | |
| 1037 | - | done := d.snapshotWorkerDone | |
| 1038 | - | close(flush) | |
| 1039 | - | d.snapshotWorkerMu.Unlock() | |
| 1040 | - | ||
| 1041 | - | if deadline == nil { | |
| 1042 | - | deadline = newSnapshotShutdownDeadline(d.clock) | |
| 1043 | - | defer deadline.stop() | |
| 1044 | - | } | |
| 1045 | - | select { | |
| 1046 | - | case <-done: | |
| 1047 | - | d.finishStoppedSnapshotWorker(false) | |
| 1048 | - | case <-deadline.Done(): | |
| 1049 | - | cancel() | |
| 1050 | - | d.finishStoppedSnapshotWorker(true) | |
| 1051 | - | } | |
| 1052 | - | } | |
| 1053 | - | ||
| 1054 | - | func (d *Daemon) finishStoppedSnapshotWorker(abandoned bool) { | |
| 1055 | - | d.snapshotWorkerMu.Lock() | |
| 1056 | - | inFlight := d.snapshotWorkerInFlight | |
| 1057 | - | d.snapshotWorkerCtx = nil | |
| 1058 | - | d.snapshotWorkerCancel = nil | |
| 1059 | - | d.snapshotWorkerDone = nil | |
| 1060 | - | d.snapshotWorkerFlush = nil | |
| 1061 | - | d.snapshotWorkerFinalWake = nil | |
| 1062 | - | d.snapshotWorkerClosing = false | |
| 1063 | - | d.snapshotWorkerInFlight = nil | |
| 1064 | - | queued := make([]*snapshotCapture, 0, len(d.snapshotJobs)+len(d.snapshotFinalJobs)) | |
| 1065 | - | for _, capture := range d.snapshotFinalJobs { | |
| 1066 | - | queued = append(queued, capture) | |
| 1067 | - | } | |
| 1068 | - | d.snapshotFinalJobs = nil | |
| 1069 | - | d.snapshotFinalOrder = nil | |
| 1070 | - | for { | |
| 1071 | - | select { | |
| 1072 | - | case capture := <-d.snapshotJobs: | |
| 1073 | - | queued = append(queued, capture) | |
| 1074 | - | default: | |
| 1075 | - | d.snapshotWorkerMu.Unlock() | |
| 1076 | - | if abandoned { | |
| 1077 | - | d.finishSnapshotCapture(inFlight, false) | |
| 1078 | - | } | |
| 1079 | - | for _, capture := range queued { | |
| 1080 | - | d.finishSnapshotCapture(capture, false) | |
| 1081 | - | } | |
| 1082 | - | return | |
| 1083 | - | } | |
| 1084 | - | } | |
| 1085 | - | } | |
| 1086 | - | ||
| 1087 | - | func (d *Daemon) capturePaneProcess(pty interface{ ForegroundPgid() (int, error) }, shellPid int) *snapcodec.Process { | |
| 1088 | - | if d == nil || pty == nil || shellPid <= 0 || d.procGroupArgv == nil { | |
| 1089 | - | return nil | |
| 1090 | - | } | |
| 1091 | - | pgid, err := pty.ForegroundPgid() | |
| 1092 | - | if err != nil || pgid <= 0 || pgid == shellPid { | |
| 1093 | - | return nil | |
| 1094 | - | } | |
| 1095 | - | argv, err := d.procGroupArgv(pgid, shellPid) | |
| 1096 | - | if err != nil || len(argv) == 0 || argv[0] == "" { | |
| 1097 | - | return nil | |
| 1098 | - | } | |
| 1099 | - | strategy := detectProcessStrategy(argv) | |
| 1100 | - | return &snapcodec.Process{ | |
| 1101 | - | Argv: append([]string(nil), argv...), | |
| 1102 | - | Strategy: strategy, | |
| 1103 | - | Opts: snapcodec.ProcessOpts{ | |
| 1104 | - | AgentSessionID: extractAgentSessionID(strategy, argv), | |
| 1105 | - | }, | |
| 1106 | - | } | |
| 1107 | - | } | |
| 1108 | - | ||
| 1109 | - | // snapshotRepositorySaver waits until the earliest completion-derived routine | |
| 1110 | - | // eligibility. State changes wake it immediately, so a newly dirty session is | |
| 1111 | - | // captured without waiting for an unrelated session's interval. | |
| 1112 | - | func (d *Daemon) snapshotRepositorySaver(ctx context.Context) { | |
| 1113 | - | if d == nil { | |
| 1114 | - | return | |
| 1115 | - | } | |
| 1116 | - | timer := d.clock.NewTimer(24 * time.Hour) | |
| 1117 | - | defer timer.Stop() | |
| 1118 | - | for { | |
| 1119 | - | delay := d.scheduleEligibleRepositorySnapshots() | |
| 1120 | - | timer.Reset(delay) | |
| 1121 | - | select { | |
| 1122 | - | case <-ctx.Done(): | |
| 1123 | - | return | |
| 1124 | - | case <-d.snapshotWake: | |
| 1125 | - | case <-timer.C(): | |
| 1126 | - | } | |
| 1127 | - | } | |
| 1128 | - | } | |
| 1129 | - | ||
| 1130 | - | func (d *Daemon) scheduleEligibleRepositorySnapshots() time.Duration { | |
| 1131 | - | d.mu.Lock() | |
| 1132 | - | sessions := d.sessionsSnapshotLocked() | |
| 1133 | - | d.mu.Unlock() | |
| 1134 | - | now := d.clock.Now() | |
| 1135 | - | for _, sess := range sessions { | |
| 1136 | - | if !sess.snapEligible.Load() || !sess.snapDirty.Load() { | |
| 1137 | - | continue | |
| 1138 | - | } | |
| 1139 | - | sess.snapshotMu.Lock() | |
| 1140 | - | due := !sess.snapshotPending && (sess.snapshotNextEligibleAt.IsZero() || !now.Before(sess.snapshotNextEligibleAt)) | |
| 1141 | - | sess.snapshotMu.Unlock() | |
| 1142 | - | if due { | |
| 1143 | - | d.scheduleSnapshot(sess) | |
| 1144 | - | } | |
| 1145 | - | } | |
| 1146 | - | ||
| 1147 | - | // Scheduling can synchronously fail or mark a capture pending, so inspect | |
| 1148 | - | // the final state to select the actual earliest timer deadline. | |
| 1149 | - | var next time.Time | |
| 1150 | - | for _, sess := range sessions { | |
| 1151 | - | if !sess.snapEligible.Load() || !sess.snapDirty.Load() { | |
| 1152 | - | continue | |
| 1153 | - | } | |
| 1154 | - | sess.snapshotMu.Lock() | |
| 1155 | - | eligibleAt := sess.snapshotNextEligibleAt | |
| 1156 | - | pending := sess.snapshotPending | |
| 1157 | - | sess.snapshotMu.Unlock() | |
| 1158 | - | if pending || eligibleAt.IsZero() { | |
| 1159 | - | continue | |
| 1160 | - | } | |
| 1161 | - | if next.IsZero() || eligibleAt.Before(next) { | |
| 1162 | - | next = eligibleAt | |
| 1163 | - | } | |
| 1164 | - | } | |
| 1165 | - | if next.IsZero() { | |
| 1166 | - | return 24 * time.Hour | |
| 1167 | - | } | |
| 1168 | - | if delay := next.Sub(now); delay > 0 { | |
| 1169 | - | return delay | |
| 1170 | - | } | |
| 1171 | - | return 0 | |
| 1172 | - | } | |
| @@ -4,11 +4,9 @@ import ( | |||||
| 4 | "context" | 4 | "context" | ||
| 5 | "errors" | 5 | "errors" | ||
| 6 | "fmt" | 6 | "fmt" | ||
| 7 | - | "math" | |||
| 8 | "sync" | 7 | "sync" | ||
| 9 | "time" | 8 | "time" | ||
| 10 | 9 | ||||
| 11 | - | "github.com/bnema/vev/internal/domain" | |||
| 12 | "github.com/bnema/vev/internal/usecase/layout" | 10 | "github.com/bnema/vev/internal/usecase/layout" | ||
| 13 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | 11 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 14 | "github.com/bnema/vev/pkg/vt" | 12 | "github.com/bnema/vev/pkg/vt" | ||
| @@ -94,1079 +92,3 @@ const ( | |||||
| 94 | snapshotInterval = 2 * time.Minute | 92 | snapshotInterval = 2 * time.Minute | ||
| 95 | snapshotFinalFlushTimeout = time.Second | 93 | snapshotFinalFlushTimeout = time.Second | ||
| 96 | ) | 94 | ) | ||
| 97 | - | ||||
| 98 | - | func (d *Daemon) closeRestoreDone() { | |||
| 99 | - | d.restoreOnce.Do(func() { close(d.restoreDone) }) | |||
| 100 | - | } | |||
| 101 | - | ||||
| 102 | - | func (d *Daemon) restoreSnapshots(ctx context.Context) { | |||
| 103 | - | if d == nil { | |||
| 104 | - | return | |||
| 105 | - | } | |||
| 106 | - | defer d.closeRestoreDone() | |||
| 107 | - | if ns := d.noticeStore; ns != nil { | |||
| 108 | - | claimed, err := ns.Claim() | |||
| 109 | - | if err != nil { | |||
| 110 | - | d.log.Warn("claiming pending notices failed", "err", err) | |||
| 111 | - | } else { | |||
| 112 | - | for _, n := range claimed { | |||
| 113 | - | d.notices.record(n) | |||
| 114 | - | d.notices.queueGlobal(n) | |||
| 115 | - | } | |||
| 116 | - | // Do not discard the claim until every notice has been recorded and | |||
| 117 | - | // queued. An Ack failure is safe: the claim is replayed at startup. | |||
| 118 | - | if err := ns.Ack(); err != nil { | |||
| 119 | - | d.log.Warn("acknowledging pending notices failed", "err", err) | |||
| 120 | - | } | |||
| 121 | - | } | |||
| 122 | - | } | |||
| 123 | - | if !d.snapsEnabled { | |||
| 124 | - | return | |||
| 125 | - | } | |||
| 126 | - | if d.snapshotRepository != nil { | |||
| 127 | - | d.restoreIncrementalSnapshots(ctx) | |||
| 128 | - | } | |||
| 129 | - | ||||
| 130 | - | } | |||
| 131 | - | ||||
| 132 | - | func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session) error { | |||
| 133 | - | if err := ctx.Err(); err != nil { | |||
| 134 | - | return err | |||
| 135 | - | } | |||
| 136 | - | if len(snap.Tabs) == 0 { | |||
| 137 | - | if snap.Active != 0 { | |||
| 138 | - | return fmt.Errorf("snapshot: active tab out of range") | |||
| 139 | - | } | |||
| 140 | - | } else if int(snap.Active) >= len(snap.Tabs) { | |||
| 141 | - | return fmt.Errorf("snapshot: active tab out of range") | |||
| 142 | - | } | |||
| 143 | - | if snap.Name == "" { | |||
| 144 | - | return fmt.Errorf("snapshot: empty session name") | |||
| 145 | - | } | |||
| 146 | - | if snap.CreatedAt > math.MaxInt64 { | |||
| 147 | - | return fmt.Errorf("snapshot: created_at overflows int64") | |||
| 148 | - | } | |||
| 149 | - | d.mu.Lock() | |||
| 150 | - | if d.closing || d.findByNameLocked(snap.Name) != nil { | |||
| 151 | - | d.mu.Unlock() | |||
| 152 | - | return nil | |||
| 153 | - | } | |||
| 154 | - | d.mu.Unlock() | |||
| 155 | - | ||||
| 156 | - | parent := d.serveCtx | |||
| 157 | - | if parent == nil { | |||
| 158 | - | parent = context.Background() | |||
| 159 | - | } | |||
| 160 | - | sctx, cancel := context.WithCancel(parent) | |||
| 161 | - | opened := make([]*tab, 0, len(snap.Tabs)) | |||
| 162 | - | // Snapshot restore runs before any client Hello is available. Start panes | |||
| 163 | - | // from the daemon environment captured at startup; the next attach replaces | |||
| 164 | - | // this snapshot for future panes from the client's capability. | |||
| 165 | - | restoreTerm := terminalEnv{} | |||
| 166 | - | restoreEnv := copyEnvironment(d.baseEnv) | |||
| 167 | - | closeOpened := func() { | |||
| 168 | - | for _, tb := range opened { | |||
| 169 | - | tb.closeAllPanes() | |||
| 170 | - | } | |||
| 171 | - | cancel() | |||
| 172 | - | } | |||
| 173 | - | allowlist := d.restoreProcessAllowlistSnapshot() | |||
| 174 | - | d.mu.Lock() | |||
| 175 | - | persisted := d.stopped[snap.Name] | |||
| 176 | - | d.mu.Unlock() | |||
| 177 | - | for tabIndex, tabSnap := range snap.Tabs { | |||
| 178 | - | if err := ctx.Err(); err != nil { | |||
| 179 | - | closeOpened() | |||
| 180 | - | return err | |||
| 181 | - | } | |||
| 182 | - | tbSize := domain.Size{Cols: int(tabSnap.Cols), Rows: int(tabSnap.Rows)} | |||
| 183 | - | if !tbSize.Valid() { | |||
| 184 | - | closeOpened() | |||
| 185 | - | return fmt.Errorf("snapshot: invalid tab size") | |||
| 186 | - | } | |||
| 187 | - | placements, ok := layout.Solve(tabSnap.Tree.Root, domain.Rect{Width: tbSize.Cols, Height: tbSize.Rows}) | |||
| 188 | - | if !ok { | |||
| 189 | - | closeOpened() | |||
| 190 | - | return fmt.Errorf("snapshot: unsolvable layout") | |||
| 191 | - | } | |||
| 192 | - | placementByPane := make(map[layout.PaneID]domain.Rect, len(placements)) | |||
| 193 | - | for _, pl := range placements { | |||
| 194 | - | placementByPane[pl.ID] = pl.Content | |||
| 195 | - | } | |||
| 196 | - | tabStableID := tabSnap.StableID | |||
| 197 | - | if tabStableID == "" { | |||
| 198 | - | var err error | |||
| 199 | - | tabStableID, err = newStableID("t") | |||
| 200 | - | if err != nil { | |||
| 201 | - | closeOpened() | |||
| 202 | - | return fmt.Errorf("snapshot: generating tab identity: %w", err) | |||
| 203 | - | } | |||
| 204 | - | } | |||
| 205 | - | tb := &tab{stableID: tabStableID, tree: tabSnap.Tree.Clone(), panes: make(map[layout.PaneID]*pane, len(tabSnap.Panes)), nextPaneID: int(tabSnap.NextPaneID), size: tbSize} | |||
| 206 | - | if tabIndex < len(persisted.tabNames) { | |||
| 207 | - | tb.name = persisted.tabNames[tabIndex] | |||
| 208 | - | } | |||
| 209 | - | if tb.nextPaneID <= 0 { | |||
| 210 | - | tb.nextPaneID = 1 | |||
| 211 | - | } | |||
| 212 | - | tb.ctx, tb.cancel = context.WithCancel(sctx) | |||
| 213 | - | opened = append(opened, tb) | |||
| 214 | - | for _, paneSnap := range tabSnap.Panes { | |||
| 215 | - | restoreCommand := "" | |||
| 216 | - | if decision := planProcessRestore(paneSnap.Process, allowlist); decision.Restore { | |||
| 217 | - | restoreCommand = decision.Command | |||
| 218 | - | } | |||
| 219 | - | if err := ctx.Err(); err != nil { | |||
| 220 | - | closeOpened() | |||
| 221 | - | return err | |||
| 222 | - | } | |||
| 223 | - | contentRect, ok := placementByPane[paneSnap.ID] | |||
| 224 | - | if !ok { | |||
| 225 | - | closeOpened() | |||
| 226 | - | return fmt.Errorf("snapshot: missing pane placement") | |||
| 227 | - | } | |||
| 228 | - | contentSize := restorePTYSize(contentRect, tbSize) | |||
| 229 | - | paneStableID := paneSnap.StableID | |||
| 230 | - | if paneStableID == "" { | |||
| 231 | - | var err error | |||
| 232 | - | paneStableID, err = newStableID("p") | |||
| 233 | - | if err != nil { | |||
| 234 | - | closeOpened() | |||
| 235 | - | return fmt.Errorf("snapshot: generating pane identity: %w", err) | |||
| 236 | - | } | |||
| 237 | - | } | |||
| 238 | - | command, args := d.ptyCommand(restoreEnv) | |||
| 239 | - | pty, err := d.ptys.Open(ctx, command, args, childEnvFrom(restoreEnv, snap.Name, tabStableID, paneStableID, restoreTerm), paneSnap.Cwd, contentSize) | |||
| 240 | - | if err != nil { | |||
| 241 | - | closeOpened() | |||
| 242 | - | return err | |||
| 243 | - | } | |||
| 244 | - | p := newPaneWithStableID(paneSnap.ID, paneStableID, pty, contentSize) | |||
| 245 | - | p.ctx, p.cancel = context.WithCancel(tb.ctx) | |||
| 246 | - | p.rect = contentRect | |||
| 247 | - | if err := restorePaneTerminal(p, paneSnap); err != nil { | |||
| 248 | - | // p is not in tb.panes yet, so closeOpened cannot reach it. | |||
| 249 | - | p.cancel() | |||
| 250 | - | _ = pty.Close() | |||
| 251 | - | closeOpened() | |||
| 252 | - | return err | |||
| 253 | - | } | |||
| 254 | - | if restoreCommand != "" { | |||
| 255 | - | if _, err := pty.Write([]byte(restoreCommand + "\n")); err != nil { | |||
| 256 | - | d.log.Warn("writing snapshot restore command failed", "err", err, "session", snap.Name, "pane", paneSnap.ID) | |||
| 257 | - | d.NotifyGlobal(domain.NoticeWarn, domain.NoticeAutoResume, | |||
| 258 | - | "couldn't restore the running program in session "+snap.Name, err) | |||
| 259 | - | } | |||
| 260 | - | } | |||
| 261 | - | tb.panes[paneSnap.ID] = p | |||
| 262 | - | } | |||
| 263 | - | } | |||
| 264 | - | if len(opened) == 0 { | |||
| 265 | - | cancel() | |||
| 266 | - | return nil | |||
| 267 | - | } | |||
| 268 | - | createdAt := int64(snap.CreatedAt) | |||
| 269 | - | sess := &session{name: snap.Name, ctx: sctx, cancel: cancel, tabs: opened, active: int(snap.Active), terminal: restoreTerm, env: restoreEnv, createdAt: createdAt, snapshotWake: d.snapshotWake, snapshotChunkCache: newSnapshotChunkCache(snapshotChunkCacheLimit)} | |||
| 270 | - | sess.snapEligible.Store(true) | |||
| 271 | - | if len(snap.Tabs) > 0 && len(snap.Tabs[0].Panes) > 0 { | |||
| 272 | - | sess.cwd = snap.Tabs[0].Panes[0].Cwd | |||
| 273 | - | } | |||
| 274 | - | d.mu.Lock() | |||
| 275 | - | if stopped, ok := d.stopped[snap.Name]; ok { | |||
| 276 | - | sess.mruAt.Store(stopped.lastUsedSeq) | |||
| 277 | - | } | |||
| 278 | - | d.mu.Unlock() | |||
| 279 | - | record := sess.persistRecordLocked(createdAt) | |||
| 280 | - | if err := d.persist.Save(record); err != nil { | |||
| 281 | - | closeOpened() | |||
| 282 | - | return err | |||
| 283 | - | } | |||
| 284 | - | d.mu.Lock() | |||
| 285 | - | if d.closing || d.findByNameLocked(snap.Name) != nil || ctx.Err() != nil { | |||
| 286 | - | d.mu.Unlock() | |||
| 287 | - | closeOpened() | |||
| 288 | - | return nil | |||
| 289 | - | } | |||
| 290 | - | id := domain.SessionID(fmt.Sprintf("sess-%d", d.nextID)) | |||
| 291 | - | d.nextID++ | |||
| 292 | - | sess.id = id | |||
| 293 | - | delete(d.stopped, snap.Name) | |||
| 294 | - | d.sessions[id] = sess | |||
| 295 | - | d.mu.Unlock() | |||
| 296 | - | sess.snapDirty.Store(false) | |||
| 297 | - | for _, tb := range opened { | |||
| 298 | - | d.startTabGoroutines(sess, tb) | |||
| 299 | - | } | |||
| 300 | - | return nil | |||
| 301 | - | } | |||
| 302 | - | ||||
| 303 | - | // snapshotCoordinatorContext returns the cancellation context shared by this | |||
| 304 | - | // session's queued and in-flight captures. snapshotMu serializes creation with | |||
| 305 | - | // quarantine, so a publisher can never obtain a live context after quarantine. | |||
| 306 | - | func snapshotCoordinatorContext(sess *session) (context.Context, context.CancelFunc) { | |||
| 307 | - | sess.snapshotMu.Lock() | |||
| 308 | - | defer sess.snapshotMu.Unlock() | |||
| 309 | - | if sess.snapshotPublicationContext == nil { | |||
| 310 | - | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |||
| 311 | - | } | |||
| 312 | - | return sess.snapshotPublicationContext, sess.snapshotPublicationCancel | |||
| 313 | - | } | |||
| 314 | - | ||||
| 315 | - | // snapshotCoordinatorQuarantine identifies one stop request. A later teardown | |||
| 316 | - | // can supersede it, in which case only that later owner may resume publication. | |||
| 317 | - | type snapshotCoordinatorQuarantine struct { | |||
| 318 | - | done <-chan struct{} | |||
| 319 | - | epoch uint64 | |||
| 320 | - | } | |||
| 321 | - | ||||
| 322 | - | // quarantineSnapshotCoordinator prevents a deleted session from starting any | |||
| 323 | - | // further publication. The returned join channel is closed after an already | |||
| 324 | - | // started repository call returns; callers must wait without daemon or session | |||
| 325 | - | // locks before deleting repository state. | |||
| 326 | - | func quarantineSnapshotCoordinator(sess *session) <-chan struct{} { | |||
| 327 | - | return quarantineSnapshotCoordinatorWithEpoch(sess).done | |||
| 328 | - | } | |||
| 329 | - | ||||
| 330 | - | // quarantineSnapshotCoordinatorRetainingQueuedCapture is used after a forced | |||
| 331 | - | // shutdown checkpoint times out. Its queued capture is logical retry state, not | |||
| 332 | - | // a repository operation: startSnapshotPublication rejects it while | |||
| 333 | - | // quarantined, and the worker eventually discards it after its blocked call | |||
| 334 | - | // returns. | |||
| 335 | - | func quarantineSnapshotCoordinatorRetainingQueuedCapture(sess *session) <-chan struct{} { | |||
| 336 | - | return quarantineSnapshotCoordinatorWithOptions(sess, true).done | |||
| 337 | - | } | |||
| 338 | - | ||||
| 339 | - | func quarantineSnapshotCoordinatorWithEpoch(sess *session) snapshotCoordinatorQuarantine { | |||
| 340 | - | return quarantineSnapshotCoordinatorWithOptions(sess, false) | |||
| 341 | - | } | |||
| 342 | - | ||||
| 343 | - | func quarantineSnapshotCoordinatorWithOptions(sess *session, retainQueuedCapture bool) snapshotCoordinatorQuarantine { | |||
| 344 | - | if sess == nil { | |||
| 345 | - | done := make(chan struct{}) | |||
| 346 | - | close(done) | |||
| 347 | - | return snapshotCoordinatorQuarantine{done: done} | |||
| 348 | - | } | |||
| 349 | - | sess.snapshotMu.Lock() | |||
| 350 | - | sess.snapshotQuarantineEpoch++ | |||
| 351 | - | epoch := sess.snapshotQuarantineEpoch | |||
| 352 | - | sess.snapshotQuarantined = true | |||
| 353 | - | sess.snapEligible.Store(false) | |||
| 354 | - | if sess.snapshotPublicationCancel != nil { | |||
| 355 | - | sess.snapshotPublicationCancel() | |||
| 356 | - | } | |||
| 357 | - | if sess.snapshotQueuedCapture != nil && !retainQueuedCapture { | |||
| 358 | - | // A rename or purge must discard a capture under the old identity. The | |||
| 359 | - | // global queue may still contain it, but startSnapshotPublication | |||
| 360 | - | // compares identity and will discard it even if this session is renamed. | |||
| 361 | - | sess.snapshotQueuedCapture.coordinatorDiscarded = true | |||
| 362 | - | sess.snapshotQueuedCapture = nil | |||
| 363 | - | if sess.snapshotPendingCaptures > 0 { | |||
| 364 | - | sess.snapshotPendingCaptures-- | |||
| 365 | - | } | |||
| 366 | - | sess.snapshotPending = sess.snapshotPendingCaptures > 0 || sess.snapshotInFlightCapture != nil | |||
| 367 | - | } | |||
| 368 | - | done := sess.snapshotPublicationDone | |||
| 369 | - | if done == nil { | |||
| 370 | - | done = make(chan struct{}) | |||
| 371 | - | close(done) | |||
| 372 | - | } | |||
| 373 | - | sess.signalSnapshotChangedLocked() | |||
| 374 | - | sess.snapshotMu.Unlock() | |||
| 375 | - | return snapshotCoordinatorQuarantine{done: done, epoch: epoch} | |||
| 376 | - | } | |||
| 377 | - | ||||
| 378 | - | // resumeSnapshotCoordinatorForNewIdentity starts a fresh repository lineage | |||
| 379 | - | // only after rename committed its new in-memory name. The epoch prevents a | |||
| 380 | - | // concurrent teardown from being undone by a late rename completion. | |||
| 381 | - | func resumeSnapshotCoordinatorForNewIdentity(sess *session, quarantine snapshotCoordinatorQuarantine) bool { | |||
| 382 | - | if sess == nil { | |||
| 383 | - | return false | |||
| 384 | - | } | |||
| 385 | - | sess.snapshotMu.Lock() | |||
| 386 | - | if !sess.snapshotQuarantined || sess.snapshotQuarantineEpoch != quarantine.epoch { | |||
| 387 | - | sess.snapshotMu.Unlock() | |||
| 388 | - | return false | |||
| 389 | - | } | |||
| 390 | - | if sess.snapshotPublicationCancel != nil { | |||
| 391 | - | sess.snapshotPublicationCancel() | |||
| 392 | - | } | |||
| 393 | - | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |||
| 394 | - | sess.snapshotQuarantined = false | |||
| 395 | - | // A repository name owns its own generation stream. Reset all scheduler | |||
| 396 | - | // state while eligibility remains false, then atomically make generation 1 | |||
| 397 | - | // dirty and wake the scheduler. | |||
| 398 | - | sess.snapshotGeneration = 1 | |||
| 399 | - | sess.snapshotCapturedGeneration = 0 | |||
| 400 | - | sess.snapshotPublishedGeneration = 0 | |||
| 401 | - | sess.snapshotForcedGeneration = 0 | |||
| 402 | - | sess.snapshotNextEligibleAt = time.Time{} | |||
| 403 | - | sess.snapshotAttempted = false | |||
| 404 | - | sess.snapshotAttemptKind = snapshotAttemptRoutine | |||
| 405 | - | sess.snapshotFailureSig = "" | |||
| 406 | - | sess.snapshotChunkCache = newSnapshotChunkCache(snapshotChunkCacheLimit) | |||
| 407 | - | sess.snapDirty.Store(true) | |||
| 408 | - | sess.snapEligible.Store(true) | |||
| 409 | - | sess.signalSnapshotChangedLocked() | |||
| 410 | - | wake := sess.snapshotWake | |||
| 411 | - | sess.snapshotMu.Unlock() | |||
| 412 | - | if wake != nil { | |||
| 413 | - | select { | |||
| 414 | - | case wake <- struct{}{}: | |||
| 415 | - | default: | |||
| 416 | - | } | |||
| 417 | - | } | |||
| 418 | - | return true | |||
| 419 | - | } | |||
| 420 | - | ||||
| 421 | - | func markSnapshotDirty(sess *session) { | |||
| 422 | - | if sess == nil || !sess.snapEligible.Load() { | |||
| 423 | - | return | |||
| 424 | - | } | |||
| 425 | - | sess.snapshotMu.Lock() | |||
| 426 | - | sess.snapshotGeneration++ | |||
| 427 | - | sess.snapDirty.Store(true) | |||
| 428 | - | sess.signalSnapshotChangedLocked() | |||
| 429 | - | wake := sess.snapshotWake | |||
| 430 | - | sess.snapshotMu.Unlock() | |||
| 431 | - | if wake != nil { | |||
| 432 | - | select { | |||
| 433 | - | case wake <- struct{}{}: | |||
| 434 | - | default: | |||
| 435 | - | } | |||
| 436 | - | } | |||
| 437 | - | } | |||
| 438 | - | ||||
| 439 | - | // snapshotChangeLocked returns a channel closed by the next snapshot state | |||
| 440 | - | // transition. snapshotMu must be held. | |||
| 441 | - | func (sess *session) snapshotChangeLocked() chan struct{} { | |||
| 442 | - | if sess.snapshotChanged == nil { | |||
| 443 | - | sess.snapshotChanged = make(chan struct{}) | |||
| 444 | - | } | |||
| 445 | - | return sess.snapshotChanged | |||
| 446 | - | } | |||
| 447 | - | ||||
| 448 | - | // signalSnapshotChangedLocked wakes waiters after a snapshot state transition. | |||
| 449 | - | // snapshotMu must be held. | |||
| 450 | - | func (sess *session) signalSnapshotChangedLocked() { | |||
| 451 | - | close(sess.snapshotChangeLocked()) | |||
| 452 | - | sess.snapshotChanged = make(chan struct{}) | |||
| 453 | - | } | |||
| 454 | - | ||||
| 455 | - | func restorePaneTerminal(p *pane, snap snapcodec.Pane) error { | |||
| 456 | - | if len(snap.Tail) == 0 { | |||
| 457 | - | return fmt.Errorf("snapshot history: missing tail blob") | |||
| 458 | - | } | |||
| 459 | - | if len(snap.Visible) == 0 { | |||
| 460 | - | return fmt.Errorf("snapshot visible: missing visible blob") | |||
| 461 | - | } | |||
| 462 | - | p.mu.Lock() | |||
| 463 | - | defer p.mu.Unlock() | |||
| 464 | - | screen, err := vt.NewScreenWithRestoredHistory(p.screen.Frame.Width, p.screen.Frame.Height, vt.HistoryConfig{MaxRows: defaultScrollbackRows, MaxCells: defaultScrollbackCells}, snap.SealedChunks, snap.Tail) | |||
| 465 | - | if err != nil { | |||
| 466 | - | return fmt.Errorf("snapshot history: %w", err) | |||
| 467 | - | } | |||
| 468 | - | if err := screen.RestorePrimaryVisible(snap.Visible); err != nil { | |||
| 469 | - | return fmt.Errorf("snapshot visible: %w", err) | |||
| 470 | - | } | |||
| 471 | - | p.screen = screen | |||
| 472 | - | p.history = screen.History() | |||
| 473 | - | return nil | |||
| 474 | - | } | |||
| 475 | - | ||||
| 476 | - | func restorePTYSize(contentRect domain.Rect, tabSize domain.Size) domain.Size { | |||
| 477 | - | cols := contentRect.Width | |||
| 478 | - | if cols <= 0 { | |||
| 479 | - | cols = tabSize.Cols | |||
| 480 | - | } | |||
| 481 | - | if cols < layout.MinPaneCols { | |||
| 482 | - | cols = layout.MinPaneCols | |||
| 483 | - | } | |||
| 484 | - | rows := contentRect.Height | |||
| 485 | - | if rows <= 0 { | |||
| 486 | - | rows = min(max(tabSize.Rows, layout.MinPaneRows), 24) | |||
| 487 | - | } | |||
| 488 | - | if rows < layout.MinPaneRows { | |||
| 489 | - | rows = layout.MinPaneRows | |||
| 490 | - | } | |||
| 491 | - | return domain.Size{Cols: cols, Rows: rows} | |||
| 492 | - | } | |||
| 493 | - | ||||
| 494 | - | // scheduleSnapshot captures a session only once while an immutable capture is | |||
| 495 | - | // queued or being written. It never waits for the bounded worker queue. | |||
| 496 | - | func (d *Daemon) scheduleSnapshot(sess *session) bool { | |||
| 497 | - | return d.scheduleSnapshotWithFinalFallback(sess, false) | |||
| 498 | - | } | |||
| 499 | - | ||||
| 500 | - | // scheduleFinalSnapshot captures the terminal state even when an older capture | |||
| 501 | - | // is pending. A removed session cannot be retried by the repository scheduler, so the | |||
| 502 | - | // worker drains its retained terminal state before it stops. | |||
| 503 | - | func (d *Daemon) scheduleFinalSnapshot(sess *session) bool { | |||
| 504 | - | return d.scheduleSnapshotWithFinalFallback(sess, true) | |||
| 505 | - | } | |||
| 506 | - | ||||
| 507 | - | func (d *Daemon) scheduleSnapshotWithFinalFallback(sess *session, final bool) bool { | |||
| 508 | - | if d == nil || sess == nil || !d.snapsEnabled || d.snapshotRepository == nil || !sess.snapEligible.Load() { | |||
| 509 | - | return true | |||
| 510 | - | } | |||
| 511 | - | kind := snapshotAttemptRoutine | |||
| 512 | - | if final { | |||
| 513 | - | kind = snapshotAttemptForced | |||
| 514 | - | } | |||
| 515 | - | sess.snapshotMu.Lock() | |||
| 516 | - | routineRateLimited := d.snapshotRepository != nil | |||
| 517 | - | if sess.snapshotQuarantined || !sess.snapDirty.Load() { | |||
| 518 | - | available := !sess.snapshotQuarantined | |||
| 519 | - | sess.snapshotMu.Unlock() | |||
| 520 | - | return available | |||
| 521 | - | } | |||
| 522 | - | if final && sess.snapshotGeneration > sess.snapshotForcedGeneration { | |||
| 523 | - | // Do not let an older queued or in-flight routine publication satisfy a | |||
| 524 | - | // terminal checkpoint. The intent is retained until a publication at | |||
| 525 | - | // this generation (or newer) succeeds. | |||
| 526 | - | sess.snapshotForcedGeneration = sess.snapshotGeneration | |||
| 527 | - | } | |||
| 528 | - | if sess.snapshotPending || | |||
| 529 | - | (!final && routineRateLimited && !sess.snapshotNextEligibleAt.IsZero() && d.clock.Now().Before(sess.snapshotNextEligibleAt)) { | |||
| 530 | - | sess.snapshotMu.Unlock() | |||
| 531 | - | return true | |||
| 532 | - | } | |||
| 533 | - | if sess.snapshotPublicationContext == nil { | |||
| 534 | - | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |||
| 535 | - | } | |||
| 536 | - | generation := sess.snapshotGeneration | |||
| 537 | - | publicationContext := sess.snapshotPublicationContext | |||
| 538 | - | sess.snapshotCapturedGeneration = generation | |||
| 539 | - | sess.snapshotAttempted = true | |||
| 540 | - | sess.snapshotAttemptKind = kind | |||
| 541 | - | sess.snapshotPendingCaptures++ | |||
| 542 | - | sess.snapshotPending = true | |||
| 543 | - | sess.signalSnapshotChangedLocked() | |||
| 544 | - | sess.snapshotMu.Unlock() | |||
| 545 | - | ||||
| 546 | - | capture, ok := d.captureSnapshotState(sess, generation) | |||
| 547 | - | if capture != nil { | |||
| 548 | - | capture.attemptKind = kind | |||
| 549 | - | capture.publicationContext = publicationContext | |||
| 550 | - | } | |||
| 551 | - | if !ok { | |||
| 552 | - | d.finishSnapshotCapture(capture, false) | |||
| 553 | - | return false | |||
| 554 | - | } | |||
| 555 | - | sess.snapshotMu.Lock() | |||
| 556 | - | quarantined := sess.snapshotQuarantined | |||
| 557 | - | if !quarantined { | |||
| 558 | - | sess.snapshotQueuedCapture = capture | |||
| 559 | - | } | |||
| 560 | - | sess.snapshotMu.Unlock() | |||
| 561 | - | if quarantined { | |||
| 562 | - | d.finishSnapshotCapture(capture, false) | |||
| 563 | - | return false | |||
| 564 | - | } | |||
| 565 | - | if d.enqueueSnapshotCapture(capture) || (final && d.enqueueFinalSnapshotCapture(capture)) { | |||
| 566 | - | return true | |||
| 567 | - | } | |||
| 568 | - | // Coalesce under saturation or shutdown: the latest state stays dirty and | |||
| 569 | - | // will be captured on a later tick once an active worker has room. | |||
| 570 | - | d.finishSnapshotCapture(capture, false) | |||
| 571 | - | return false | |||
| 572 | - | } | |||
| 573 | - | ||||
| 574 | - | // waitForSnapshotGeneration waits only for the bounded terminal checkpoint | |||
| 575 | - | // deadline. It observes successful publication, rather than dirty state, so a | |||
| 576 | - | // concurrent later mutation cannot turn a completed forced checkpoint into a | |||
| 577 | - | // false timeout. | |||
| 578 | - | func (d *Daemon) waitForSnapshotGeneration(sess *session, generation uint64) bool { | |||
| 579 | - | if d == nil || sess == nil || generation == 0 { | |||
| 580 | - | return true | |||
| 581 | - | } | |||
| 582 | - | deadline := newSnapshotShutdownDeadline(d.clock) | |||
| 583 | - | defer deadline.stop() | |||
| 584 | - | return d.waitForSnapshotGenerationWithDeadline(sess, generation, deadline) | |||
| 585 | - | } | |||
| 586 | - | ||||
| 587 | - | // waitForSnapshotGenerationWithDeadline waits for a forced checkpoint without | |||
| 588 | - | // allocating another shutdown interval. Serve shares one deadline across every | |||
| 589 | - | // session and the subsequent worker drain. | |||
| 590 | - | func (d *Daemon) waitForSnapshotGenerationWithDeadline(sess *session, generation uint64, deadline *snapshotShutdownDeadline) bool { | |||
| 591 | - | if d == nil || sess == nil || generation == 0 { | |||
| 592 | - | return true | |||
| 593 | - | } | |||
| 594 | - | if deadline == nil { | |||
| 595 | - | return d.waitForSnapshotGeneration(sess, generation) | |||
| 596 | - | } | |||
| 597 | - | for { | |||
| 598 | - | sess.snapshotMu.Lock() | |||
| 599 | - | published := sess.snapshotPublishedGeneration >= generation | |||
| 600 | - | changed := sess.snapshotChangeLocked() | |||
| 601 | - | sess.snapshotMu.Unlock() | |||
| 602 | - | if published { | |||
| 603 | - | return true | |||
| 604 | - | } | |||
| 605 | - | select { | |||
| 606 | - | case <-changed: | |||
| 607 | - | case <-deadline.Done(): | |||
| 608 | - | return false | |||
| 609 | - | } | |||
| 610 | - | } | |||
| 611 | - | } | |||
| 612 | - | ||||
| 613 | - | // enqueueSnapshotCapture serializes worker shutdown with the non-blocking | |||
| 614 | - | // queue send. The queue is deliberately never closed: producers can race | |||
| 615 | - | // shutdown without risking a send-on-closed panic. | |||
| 616 | - | func (d *Daemon) enqueueSnapshotCapture(capture *snapshotCapture) bool { | |||
| 617 | - | d.snapshotWorkerMu.Lock() | |||
| 618 | - | defer d.snapshotWorkerMu.Unlock() | |||
| 619 | - | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |||
| 620 | - | return false | |||
| 621 | - | } | |||
| 622 | - | select { | |||
| 623 | - | case d.snapshotJobs <- capture: | |||
| 624 | - | return true | |||
| 625 | - | default: | |||
| 626 | - | return false | |||
| 627 | - | } | |||
| 628 | - | } | |||
| 629 | - | ||||
| 630 | - | // enqueueFinalSnapshotCapture hands terminal captures to the existing worker | |||
| 631 | - | // without making session teardown wait behind its normal bounded queue. A | |||
| 632 | - | // blocked worker retains at most one terminal capture per session, replacing | |||
| 633 | - | // stale state with the newest immutable capture. | |||
| 634 | - | func (d *Daemon) enqueueFinalSnapshotCapture(capture *snapshotCapture) bool { | |||
| 635 | - | if capture == nil || capture.session == nil { | |||
| 636 | - | return false | |||
| 637 | - | } | |||
| 638 | - | d.snapshotWorkerMu.Lock() | |||
| 639 | - | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |||
| 640 | - | d.snapshotWorkerMu.Unlock() | |||
| 641 | - | return false | |||
| 642 | - | } | |||
| 643 | - | if d.snapshotFinalJobs == nil { | |||
| 644 | - | d.snapshotFinalJobs = make(map[*session]*snapshotCapture) | |||
| 645 | - | } | |||
| 646 | - | replaced, exists := d.snapshotFinalJobs[capture.session] | |||
| 647 | - | if !exists && len(d.snapshotFinalJobs) >= snapshotFinalQueueCapacity { | |||
| 648 | - | d.snapshotWorkerMu.Unlock() | |||
| 649 | - | d.log.Warn("terminal snapshot retention saturated; capture rejected", "session", capture.name, "capacity", snapshotFinalQueueCapacity) | |||
| 650 | - | return false | |||
| 651 | - | } | |||
| 652 | - | // Finish the stale capture before publishing its replacement. Otherwise a | |||
| 653 | - | // fast worker could persist the replacement and then have this stale failure | |||
| 654 | - | // mark the session dirty again. | |||
| 655 | - | d.finishSnapshotCapture(replaced, false) | |||
| 656 | - | if !exists { | |||
| 657 | - | d.snapshotFinalOrder = append(d.snapshotFinalOrder, capture.session) | |||
| 658 | - | } | |||
| 659 | - | d.snapshotFinalJobs[capture.session] = capture | |||
| 660 | - | select { | |||
| 661 | - | case d.snapshotWorkerFinalWake <- struct{}{}: | |||
| 662 | - | default: | |||
| 663 | - | } | |||
| 664 | - | d.snapshotWorkerMu.Unlock() | |||
| 665 | - | return true | |||
| 666 | - | } | |||
| 667 | - | ||||
| 668 | - | // captureSession rotates history tails and clones visible frames while holding | |||
| 669 | - | // each pane lock. The returned capture contains only immutable state; encoding | |||
| 670 | - | // and persistence are deliberately deferred to snapshotEncodeWorker. | |||
| 671 | - | func (d *Daemon) captureSnapshotState(sess *session, generation uint64) (*snapshotCapture, bool) { | |||
| 672 | - | sess.mu.Lock() | |||
| 673 | - | capture := &snapshotCapture{ | |||
| 674 | - | session: sess, | |||
| 675 | - | generation: generation, | |||
| 676 | - | name: sess.name, | |||
| 677 | - | createdAt: uint64(sess.createdAt), | |||
| 678 | - | active: uint16(max(sess.active, 0)), | |||
| 679 | - | } | |||
| 680 | - | ephemeral := sess.ephemeral | |||
| 681 | - | fallbackCwd := sess.cwd | |||
| 682 | - | tabs := append([]*tab(nil), sess.tabs...) | |||
| 683 | - | sess.mu.Unlock() | |||
| 684 | - | if ephemeral || capture.name == "" { | |||
| 685 | - | return capture, false | |||
| 686 | - | } | |||
| 687 | - | ||||
| 688 | - | capture.tabs = make([]snapshotCaptureTab, 0, len(tabs)) | |||
| 689 | - | for _, tb := range tabs { | |||
| 690 | - | tb.mu.Lock() | |||
| 691 | - | tabCapture := snapshotCaptureTab{ | |||
| 692 | - | stableID: tb.stableID, | |||
| 693 | - | cols: uint16(max(tb.size.Cols, 0)), | |||
| 694 | - | rows: uint16(max(tb.size.Rows, 0)), | |||
| 695 | - | nextPaneID: uint64(max(tb.nextPaneID, 0)), | |||
| 696 | - | tree: tb.tree.Clone(), | |||
| 697 | - | } | |||
| 698 | - | if tb.tree != nil { | |||
| 699 | - | tabCapture.focus = tb.tree.Focus | |||
| 700 | - | } | |||
| 701 | - | panes := make([]*pane, 0, len(tb.panes)) | |||
| 702 | - | for _, p := range tb.panes { | |||
| 703 | - | panes = append(panes, p) | |||
| 704 | - | } | |||
| 705 | - | tb.mu.Unlock() | |||
| 706 | - | ||||
| 707 | - | tabCapture.panes = make([]snapshotCapturePane, 0, len(panes)) | |||
| 708 | - | for _, p := range panes { | |||
| 709 | - | p.mu.Lock() | |||
| 710 | - | pty := p.pty | |||
| 711 | - | pid := 0 | |||
| 712 | - | if pty != nil { | |||
| 713 | - | pid = pty.Pid() | |||
| 714 | - | } | |||
| 715 | - | paneCapture := snapshotCapturePane{ | |||
| 716 | - | id: p.id, | |||
| 717 | - | stableID: p.stableID, | |||
| 718 | - | visible: p.screen.PrimaryVisibleSnapshot(), | |||
| 719 | - | } | |||
| 720 | - | paneCapture.sealed = p.history.SnapshotView() | |||
| 721 | - | paneCapture.tail = paneCapture.sealed.Tail() | |||
| 722 | - | p.mu.Unlock() | |||
| 723 | - | paneCapture.cwd = fallbackCwd | |||
| 724 | - | if d.procCwd != nil && pid > 0 { | |||
| 725 | - | if cwd, err := d.procCwd(pid); err == nil && cwd != "" { | |||
| 726 | - | paneCapture.cwd = cwd | |||
| 727 | - | } | |||
| 728 | - | } | |||
| 729 | - | paneCapture.process = d.capturePaneProcess(pty, pid) | |||
| 730 | - | tabCapture.panes = append(tabCapture.panes, paneCapture) | |||
| 731 | - | } | |||
| 732 | - | capture.tabs = append(capture.tabs, tabCapture) | |||
| 733 | - | } | |||
| 734 | - | return capture, true | |||
| 735 | - | } | |||
| 736 | - | ||||
| 737 | - | // captureSession remains the synchronous producer-facing trigger for callers | |||
| 738 | - | // such as teardown and benchmarks; the actual encoding and Write stay async. | |||
| 739 | - | func (d *Daemon) captureSession(sess *session) bool { | |||
| 740 | - | markSnapshotDirty(sess) | |||
| 741 | - | return d.scheduleSnapshot(sess) | |||
| 742 | - | } | |||
| 743 | - | ||||
| 744 | - | // startSnapshotPublication moves a retained capture from the producer queue to | |||
| 745 | - | // the repository worker. Quarantine and this transition share snapshotMu, so a | |||
| 746 | - | // repository call cannot begin after destructive teardown has quarantined the | |||
| 747 | - | // session. | |||
| 748 | - | func startSnapshotPublication(capture *snapshotCapture) bool { | |||
| 749 | - | if capture == nil || capture.session == nil { | |||
| 750 | - | return false | |||
| 751 | - | } | |||
| 752 | - | sess := capture.session | |||
| 753 | - | sess.snapshotMu.Lock() | |||
| 754 | - | defer sess.snapshotMu.Unlock() | |||
| 755 | - | if sess.snapshotQuarantined || sess.snapshotQueuedCapture != capture { | |||
| 756 | - | return false | |||
| 757 | - | } | |||
| 758 | - | sess.snapshotQueuedCapture = nil | |||
| 759 | - | sess.snapshotInFlightCapture = capture | |||
| 760 | - | sess.snapshotPublicationDone = make(chan struct{}) | |||
| 761 | - | sess.snapshotPending = true | |||
| 762 | - | sess.signalSnapshotChangedLocked() | |||
| 763 | - | return true | |||
| 764 | - | } | |||
| 765 | - | ||||
| 766 | - | func (d *Daemon) finishSnapshotCapture(capture *snapshotCapture, succeeded bool) { | |||
| 767 | - | if capture == nil { | |||
| 768 | - | return | |||
| 769 | - | } | |||
| 770 | - | capture.finishOnce.Do(func() { | |||
| 771 | - | shouldScheduleForcedSuccessor := false | |||
| 772 | - | capture.session.snapshotMu.Lock() | |||
| 773 | - | if capture.session.snapshotPendingCaptures > 0 { | |||
| 774 | - | capture.session.snapshotPendingCaptures-- | |||
| 775 | - | } | |||
| 776 | - | if capture.session.snapshotQueuedCapture == capture { | |||
| 777 | - | capture.session.snapshotQueuedCapture = nil | |||
| 778 | - | } | |||
| 779 | - | if capture.session.snapshotInFlightCapture == capture { | |||
| 780 | - | capture.session.snapshotInFlightCapture = nil | |||
| 781 | - | if capture.session.snapshotPublicationDone != nil { | |||
| 782 | - | close(capture.session.snapshotPublicationDone) | |||
| 783 | - | capture.session.snapshotPublicationDone = nil | |||
| 784 | - | } | |||
| 785 | - | } | |||
| 786 | - | capture.session.snapshotPending = capture.session.snapshotPendingCaptures > 0 || | |||
| 787 | - | capture.session.snapshotQueuedCapture != nil || capture.session.snapshotInFlightCapture != nil | |||
| 788 | - | if !capture.coordinatorDiscarded && capture.attemptKind == snapshotAttemptRoutine && d.snapshotRepository != nil { | |||
| 789 | - | // The interval is measured from completion, including a failed | |||
| 790 | - | // attempt. A forced teardown checkpoint deliberately leaves it alone. | |||
| 791 | - | capture.session.snapshotNextEligibleAt = d.clock.Now().Add(snapshotInterval) | |||
| 792 | - | } | |||
| 793 | - | if succeeded { | |||
| 794 | - | capture.session.snapshotPublishedGeneration = max(capture.session.snapshotPublishedGeneration, capture.generation) | |||
| 795 | - | } | |||
| 796 | - | if succeeded && capture.session.snapshotGeneration == capture.generation { | |||
| 797 | - | capture.session.snapDirty.Store(false) | |||
| 798 | - | capture.session.snapshotFailureSig = "" | |||
| 799 | - | } else if !succeeded { | |||
| 800 | - | capture.session.snapDirty.Store(true) | |||
| 801 | - | } | |||
| 802 | - | if succeeded && capture.session.snapshotForcedGeneration != 0 && | |||
| 803 | - | capture.generation >= capture.session.snapshotForcedGeneration { | |||
| 804 | - | capture.session.snapshotForcedGeneration = 0 | |||
| 805 | - | } | |||
| 806 | - | // A failed forced publication remains dirty for ordinary retry rather | |||
| 807 | - | // than recursively creating forced jobs. A later forced request records | |||
| 808 | - | // a newer intent and can create its one bounded successor. | |||
| 809 | - | shouldScheduleForcedSuccessor = capture.session.snapshotForcedGeneration != 0 && | |||
| 810 | - | capture.generation < capture.session.snapshotForcedGeneration && | |||
| 811 | - | capture.session.snapshotQueuedCapture == nil && capture.session.snapshotInFlightCapture == nil && | |||
| 812 | - | (succeeded || capture.attemptKind != snapshotAttemptForced) | |||
| 813 | - | capture.session.signalSnapshotChangedLocked() | |||
| 814 | - | wake := capture.session.snapshotWake | |||
| 815 | - | capture.session.snapshotMu.Unlock() | |||
| 816 | - | if wake != nil { | |||
| 817 | - | select { | |||
| 818 | - | case wake <- struct{}{}: | |||
| 819 | - | default: | |||
| 820 | - | } | |||
| 821 | - | } | |||
| 822 | - | if shouldScheduleForcedSuccessor { | |||
| 823 | - | // This runs after publication completion and outside snapshotMu. It | |||
| 824 | - | // bypasses routine eligibility while preserving the single queued and | |||
| 825 | - | // single in-flight capture bound. | |||
| 826 | - | _ = d.scheduleFinalSnapshot(capture.session) | |||
| 827 | - | } | |||
| 828 | - | if succeeded { | |||
| 829 | - | d.clearSnapshotFailure() | |||
| 830 | - | } | |||
| 831 | - | }) | |||
| 832 | - | } | |||
| 833 | - | ||||
| 834 | - | // reportSnapshotFailure records every failed persistence attempt but only | |||
| 835 | - | // displays a toast when its stable phase/error class changes. This keeps a | |||
| 836 | - | // blocked disk from repeatedly repainting every attached client while retaining | |||
| 837 | - | // the count in notification history. | |||
| 838 | - | func (d *Daemon) reportSnapshotFailure(capture *snapshotCapture, phase string, cause error) { | |||
| 839 | - | if d == nil || capture == nil || capture.session == nil || cause == nil { | |||
| 840 | - | return | |||
| 841 | - | } | |||
| 842 | - | signature := snapshotFailureSignature(phase, cause) | |||
| 843 | - | capture.session.snapshotMu.Lock() | |||
| 844 | - | capture.session.snapshotFailureSig = signature | |||
| 845 | - | capture.session.snapshotMu.Unlock() | |||
| 846 | - | ||||
| 847 | - | d.snapshotNoticeMu.Lock() | |||
| 848 | - | changed := d.snapshotActiveFailureSignature != signature | |||
| 849 | - | d.snapshotActiveFailureSignature = signature | |||
| 850 | - | d.snapshotNoticeMu.Unlock() | |||
| 851 | - | ||||
| 852 | - | n := domain.Notification{ | |||
| 853 | - | Code: domain.NoticeSnapshotWrite, | |||
| 854 | - | Severity: domain.NoticeError, | |||
| 855 | - | Message: "couldn't save session state; recent state may be lost on restart", | |||
| 856 | - | // The history is intentionally diagnostic only at the error-class level. | |||
| 857 | - | // Raw causes can include user paths and terminal content. | |||
| 858 | - | Details: signature, | |||
| 859 | - | Time: d.clock.Now(), | |||
| 860 | - | } | |||
| 861 | - | n, _ = d.notices.recordSnapshotFailure(n) | |||
| 862 | - | d.log.Warn("writing session snapshot failed", "phase", phase, "class", signature, "session", capture.name) | |||
| 863 | - | if changed { | |||
| 864 | - | d.deliverGlobal(n) | |||
| 865 | - | } | |||
| 866 | - | } | |||
| 867 | - | ||||
| 868 | - | func (d *Daemon) clearSnapshotFailure() { | |||
| 869 | - | if d == nil { | |||
| 870 | - | return | |||
| 871 | - | } | |||
| 872 | - | d.snapshotNoticeMu.Lock() | |||
| 873 | - | d.snapshotActiveFailureSignature = "" | |||
| 874 | - | d.snapshotNoticeMu.Unlock() | |||
| 875 | - | } | |||
| 876 | - | ||||
| 877 | - | func (d *Daemon) startSnapshotEncodeWorker() { | |||
| 878 | - | if d == nil { | |||
| 879 | - | return | |||
| 880 | - | } | |||
| 881 | - | d.snapshotWorkerMu.Lock() | |||
| 882 | - | if d.snapshotWorkerCancel != nil { | |||
| 883 | - | d.snapshotWorkerMu.Unlock() | |||
| 884 | - | return | |||
| 885 | - | } | |||
| 886 | - | // Shutdown owns the worker lifetime so it can flush captures after Serve | |||
| 887 | - | // cancels its parent context. The worker is always stopped explicitly. | |||
| 888 | - | workerCtx, cancel := context.WithCancel(context.Background()) | |||
| 889 | - | d.snapshotWorkerID++ | |||
| 890 | - | workerID := d.snapshotWorkerID | |||
| 891 | - | d.snapshotWorkerCtx = workerCtx | |||
| 892 | - | d.snapshotWorkerCancel = cancel | |||
| 893 | - | d.snapshotWorkerDone = make(chan struct{}) | |||
| 894 | - | d.snapshotWorkerFlush = make(chan struct{}) | |||
| 895 | - | d.snapshotWorkerFinalWake = make(chan struct{}, 1) | |||
| 896 | - | d.snapshotFinalJobs = make(map[*session]*snapshotCapture) | |||
| 897 | - | d.snapshotFinalOrder = nil | |||
| 898 | - | d.snapshotWorkerClosing = false | |||
| 899 | - | d.snapshotWorkerInFlight = nil | |||
| 900 | - | done := d.snapshotWorkerDone | |||
| 901 | - | flush := d.snapshotWorkerFlush | |||
| 902 | - | finalWake := d.snapshotWorkerFinalWake | |||
| 903 | - | d.snapshotWorkerMu.Unlock() | |||
| 904 | - | go func() { | |||
| 905 | - | defer close(done) | |||
| 906 | - | write := func(capture *snapshotCapture) bool { | |||
| 907 | - | if workerCtx.Err() != nil || !startSnapshotPublication(capture) || !d.setSnapshotWorkerInFlight(workerID, capture) { | |||
| 908 | - | d.finishSnapshotCapture(capture, false) | |||
| 909 | - | return workerCtx.Err() == nil | |||
| 910 | - | } | |||
| 911 | - | publication, err := d.incrementalPublication(capture) | |||
| 912 | - | publicationContext := capture.publicationContext | |||
| 913 | - | if publicationContext == nil { | |||
| 914 | - | publicationContext = workerCtx | |||
| 915 | - | } | |||
| 916 | - | if err == nil && workerCtx.Err() == nil { | |||
| 917 | - | if err = publicationContext.Err(); err == nil { | |||
| 918 | - | err = d.snapshotRepository.Publish(publicationContext, publication) | |||
| 919 | - | } | |||
| 920 | - | } | |||
| 921 | - | d.clearSnapshotWorkerInFlight(workerID, capture) | |||
| 922 | - | if err != nil && workerCtx.Err() == nil && publicationContext.Err() == nil { | |||
| 923 | - | // Global, not session-scoped: by now the session may already be | |||
| 924 | - | // torn down, so a session notice would be dead-on-arrival. No lock | |||
| 925 | - | // is held here (clearSnapshotWorkerInFlight released its own). | |||
| 926 | - | d.reportSnapshotFailure(capture, "publish", err) | |||
| 927 | - | } | |||
| 928 | - | d.finishSnapshotCapture(capture, err == nil && workerCtx.Err() == nil) | |||
| 929 | - | return workerCtx.Err() == nil | |||
| 930 | - | } | |||
| 931 | - | drainFinal := func() bool { | |||
| 932 | - | for { | |||
| 933 | - | capture := d.takeFinalSnapshotCapture() | |||
| 934 | - | if capture == nil { | |||
| 935 | - | return true | |||
| 936 | - | } | |||
| 937 | - | if !write(capture) { | |||
| 938 | - | return false | |||
| 939 | - | } | |||
| 940 | - | } | |||
| 941 | - | } | |||
| 942 | - | flushQueued := func() { | |||
| 943 | - | for { | |||
| 944 | - | select { | |||
| 945 | - | case capture := <-d.snapshotJobs: | |||
| 946 | - | if !write(capture) { | |||
| 947 | - | return | |||
| 948 | - | } | |||
| 949 | - | default: | |||
| 950 | - | _ = drainFinal() | |||
| 951 | - | return | |||
| 952 | - | } | |||
| 953 | - | } | |||
| 954 | - | } | |||
| 955 | - | for { | |||
| 956 | - | select { | |||
| 957 | - | case <-workerCtx.Done(): | |||
| 958 | - | return | |||
| 959 | - | case <-flush: | |||
| 960 | - | flushQueued() | |||
| 961 | - | return | |||
| 962 | - | case <-finalWake: | |||
| 963 | - | if !drainFinal() { | |||
| 964 | - | return | |||
| 965 | - | } | |||
| 966 | - | case capture := <-d.snapshotJobs: | |||
| 967 | - | if !write(capture) { | |||
| 968 | - | return | |||
| 969 | - | } | |||
| 970 | - | } | |||
| 971 | - | } | |||
| 972 | - | }() | |||
| 973 | - | } | |||
| 974 | - | ||||
| 975 | - | func (d *Daemon) takeFinalSnapshotCapture() *snapshotCapture { | |||
| 976 | - | d.snapshotWorkerMu.Lock() | |||
| 977 | - | defer d.snapshotWorkerMu.Unlock() | |||
| 978 | - | for len(d.snapshotFinalOrder) > 0 { | |||
| 979 | - | sess := d.snapshotFinalOrder[0] | |||
| 980 | - | d.snapshotFinalOrder[0] = nil | |||
| 981 | - | d.snapshotFinalOrder = d.snapshotFinalOrder[1:] | |||
| 982 | - | capture := d.snapshotFinalJobs[sess] | |||
| 983 | - | delete(d.snapshotFinalJobs, sess) | |||
| 984 | - | if capture != nil { | |||
| 985 | - | return capture | |||
| 986 | - | } | |||
| 987 | - | } | |||
| 988 | - | return nil | |||
| 989 | - | } | |||
| 990 | - | ||||
| 991 | - | func (d *Daemon) setSnapshotWorkerInFlight(workerID uint64, capture *snapshotCapture) bool { | |||
| 992 | - | d.snapshotWorkerMu.Lock() | |||
| 993 | - | defer d.snapshotWorkerMu.Unlock() | |||
| 994 | - | if d.snapshotWorkerID != workerID || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |||
| 995 | - | return false | |||
| 996 | - | } | |||
| 997 | - | d.snapshotWorkerInFlight = capture | |||
| 998 | - | return true | |||
| 999 | - | } | |||
| 1000 | - | ||||
| 1001 | - | func (d *Daemon) clearSnapshotWorkerInFlight(workerID uint64, capture *snapshotCapture) { | |||
| 1002 | - | d.snapshotWorkerMu.Lock() | |||
| 1003 | - | defer d.snapshotWorkerMu.Unlock() | |||
| 1004 | - | if d.snapshotWorkerID == workerID && d.snapshotWorkerInFlight == capture { | |||
| 1005 | - | d.snapshotWorkerInFlight = nil | |||
| 1006 | - | } | |||
| 1007 | - | } | |||
| 1008 | - | ||||
| 1009 | - | func (d *Daemon) stopSnapshotEncodeWorker() { | |||
| 1010 | - | if d == nil { | |||
| 1011 | - | return | |||
| 1012 | - | } | |||
| 1013 | - | deadline := newSnapshotShutdownDeadline(d.clock) | |||
| 1014 | - | defer deadline.stop() | |||
| 1015 | - | d.stopSnapshotEncodeWorkerWithDeadline(deadline) | |||
| 1016 | - | } | |||
| 1017 | - | ||||
| 1018 | - | // stopSnapshotEncodeWorkerWithDeadline performs the one deadline-aware join | |||
| 1019 | - | // for a Serve shutdown. Queues are never closed, and detached writers only | |||
| 1020 | - | // retain immutable captures plus contexts/channels that remain live, so an | |||
| 1021 | - | // uncooperative repository call cannot race teardown or block process exit. | |||
| 1022 | - | func (d *Daemon) stopSnapshotEncodeWorkerWithDeadline(deadline *snapshotShutdownDeadline) { | |||
| 1023 | - | if d == nil { | |||
| 1024 | - | return | |||
| 1025 | - | } | |||
| 1026 | - | d.snapshotWorkerMu.Lock() | |||
| 1027 | - | cancel := d.snapshotWorkerCancel | |||
| 1028 | - | if cancel == nil || d.snapshotWorkerClosing { | |||
| 1029 | - | d.snapshotWorkerMu.Unlock() | |||
| 1030 | - | return | |||
| 1031 | - | } | |||
| 1032 | - | // Stop accepting producers, then let the single worker drain its bounded | |||
| 1033 | - | // queue. A non-context-aware store may still block indefinitely, so the | |||
| 1034 | - | // shared shutdown deadline bounds the one and only join. | |||
| 1035 | - | d.snapshotWorkerClosing = true | |||
| 1036 | - | flush := d.snapshotWorkerFlush | |||
| 1037 | - | done := d.snapshotWorkerDone | |||
| 1038 | - | close(flush) | |||
| 1039 | - | d.snapshotWorkerMu.Unlock() | |||
| 1040 | - | ||||
| 1041 | - | if deadline == nil { | |||
| 1042 | - | deadline = newSnapshotShutdownDeadline(d.clock) | |||
| 1043 | - | defer deadline.stop() | |||
| 1044 | - | } | |||
| 1045 | - | select { | |||
| 1046 | - | case <-done: | |||
| 1047 | - | d.finishStoppedSnapshotWorker(false) | |||
| 1048 | - | case <-deadline.Done(): | |||
| 1049 | - | cancel() | |||
| 1050 | - | d.finishStoppedSnapshotWorker(true) | |||
| 1051 | - | } | |||
| 1052 | - | } | |||
| 1053 | - | ||||
| 1054 | - | func (d *Daemon) finishStoppedSnapshotWorker(abandoned bool) { | |||
| 1055 | - | d.snapshotWorkerMu.Lock() | |||
| 1056 | - | inFlight := d.snapshotWorkerInFlight | |||
| 1057 | - | d.snapshotWorkerCtx = nil | |||
| 1058 | - | d.snapshotWorkerCancel = nil | |||
| 1059 | - | d.snapshotWorkerDone = nil | |||
| 1060 | - | d.snapshotWorkerFlush = nil | |||
| 1061 | - | d.snapshotWorkerFinalWake = nil | |||
| 1062 | - | d.snapshotWorkerClosing = false | |||
| 1063 | - | d.snapshotWorkerInFlight = nil | |||
| 1064 | - | queued := make([]*snapshotCapture, 0, len(d.snapshotJobs)+len(d.snapshotFinalJobs)) | |||
| 1065 | - | for _, capture := range d.snapshotFinalJobs { | |||
| 1066 | - | queued = append(queued, capture) | |||
| 1067 | - | } | |||
| 1068 | - | d.snapshotFinalJobs = nil | |||
| 1069 | - | d.snapshotFinalOrder = nil | |||
| 1070 | - | for { | |||
| 1071 | - | select { | |||
| 1072 | - | case capture := <-d.snapshotJobs: | |||
| 1073 | - | queued = append(queued, capture) | |||
| 1074 | - | default: | |||
| 1075 | - | d.snapshotWorkerMu.Unlock() | |||
| 1076 | - | if abandoned { | |||
| 1077 | - | d.finishSnapshotCapture(inFlight, false) | |||
| 1078 | - | } | |||
| 1079 | - | for _, capture := range queued { | |||
| 1080 | - | d.finishSnapshotCapture(capture, false) | |||
| 1081 | - | } | |||
| 1082 | - | return | |||
| 1083 | - | } | |||
| 1084 | - | } | |||
| 1085 | - | } | |||
| 1086 | - | ||||
| 1087 | - | func (d *Daemon) capturePaneProcess(pty interface{ ForegroundPgid() (int, error) }, shellPid int) *snapcodec.Process { | |||
| 1088 | - | if d == nil || pty == nil || shellPid <= 0 || d.procGroupArgv == nil { | |||
| 1089 | - | return nil | |||
| 1090 | - | } | |||
| 1091 | - | pgid, err := pty.ForegroundPgid() | |||
| 1092 | - | if err != nil || pgid <= 0 || pgid == shellPid { | |||
| 1093 | - | return nil | |||
| 1094 | - | } | |||
| 1095 | - | argv, err := d.procGroupArgv(pgid, shellPid) | |||
| 1096 | - | if err != nil || len(argv) == 0 || argv[0] == "" { | |||
| 1097 | - | return nil | |||
| 1098 | - | } | |||
| 1099 | - | strategy := detectProcessStrategy(argv) | |||
| 1100 | - | return &snapcodec.Process{ | |||
| 1101 | - | Argv: append([]string(nil), argv...), | |||
| 1102 | - | Strategy: strategy, | |||
| 1103 | - | Opts: snapcodec.ProcessOpts{ | |||
| 1104 | - | AgentSessionID: extractAgentSessionID(strategy, argv), | |||
| 1105 | - | }, | |||
| 1106 | - | } | |||
| 1107 | - | } | |||
| 1108 | - | ||||
| 1109 | - | // snapshotRepositorySaver waits until the earliest completion-derived routine | |||
| 1110 | - | // eligibility. State changes wake it immediately, so a newly dirty session is | |||
| 1111 | - | // captured without waiting for an unrelated session's interval. | |||
| 1112 | - | func (d *Daemon) snapshotRepositorySaver(ctx context.Context) { | |||
| 1113 | - | if d == nil { | |||
| 1114 | - | return | |||
| 1115 | - | } | |||
| 1116 | - | timer := d.clock.NewTimer(24 * time.Hour) | |||
| 1117 | - | defer timer.Stop() | |||
| 1118 | - | for { | |||
| 1119 | - | delay := d.scheduleEligibleRepositorySnapshots() | |||
| 1120 | - | timer.Reset(delay) | |||
| 1121 | - | select { | |||
| 1122 | - | case <-ctx.Done(): | |||
| 1123 | - | return | |||
| 1124 | - | case <-d.snapshotWake: | |||
| 1125 | - | case <-timer.C(): | |||
| 1126 | - | } | |||
| 1127 | - | } | |||
| 1128 | - | } | |||
| 1129 | - | ||||
| 1130 | - | func (d *Daemon) scheduleEligibleRepositorySnapshots() time.Duration { | |||
| 1131 | - | d.mu.Lock() | |||
| 1132 | - | sessions := d.sessionsSnapshotLocked() | |||
| 1133 | - | d.mu.Unlock() | |||
| 1134 | - | now := d.clock.Now() | |||
| 1135 | - | for _, sess := range sessions { | |||
| 1136 | - | if !sess.snapEligible.Load() || !sess.snapDirty.Load() { | |||
| 1137 | - | continue | |||
| 1138 | - | } | |||
| 1139 | - | sess.snapshotMu.Lock() | |||
| 1140 | - | due := !sess.snapshotPending && (sess.snapshotNextEligibleAt.IsZero() || !now.Before(sess.snapshotNextEligibleAt)) | |||
| 1141 | - | sess.snapshotMu.Unlock() | |||
| 1142 | - | if due { | |||
| 1143 | - | d.scheduleSnapshot(sess) | |||
| 1144 | - | } | |||
| 1145 | - | } | |||
| 1146 | - | ||||
| 1147 | - | // Scheduling can synchronously fail or mark a capture pending, so inspect | |||
| 1148 | - | // the final state to select the actual earliest timer deadline. | |||
| 1149 | - | var next time.Time | |||
| 1150 | - | for _, sess := range sessions { | |||
| 1151 | - | if !sess.snapEligible.Load() || !sess.snapDirty.Load() { | |||
| 1152 | - | continue | |||
| 1153 | - | } | |||
| 1154 | - | sess.snapshotMu.Lock() | |||
| 1155 | - | eligibleAt := sess.snapshotNextEligibleAt | |||
| 1156 | - | pending := sess.snapshotPending | |||
| 1157 | - | sess.snapshotMu.Unlock() | |||
| 1158 | - | if pending || eligibleAt.IsZero() { | |||
| 1159 | - | continue | |||
| 1160 | - | } | |||
| 1161 | - | if next.IsZero() || eligibleAt.Before(next) { | |||
| 1162 | - | next = eligibleAt | |||
| 1163 | - | } | |||
| 1164 | - | } | |||
| 1165 | - | if next.IsZero() { | |||
| 1166 | - | return 24 * time.Hour | |||
| 1167 | - | } | |||
| 1168 | - | if delay := next.Sub(now); delay > 0 { | |||
| 1169 | - | return delay | |||
| 1170 | - | } | |||
| 1171 | - | return 0 | |||
| 1172 | - | } | |||
M internal/usecase/daemon/snapshot_acceptance_test.go +4 -4
| @@ -18,8 +18,8 @@ func TestSnapshotChunkCacheIsScopedToNamedSession(t *testing.T) { | |||
| 18 | 18 | b.snapshotChunkCache = newSnapshotChunkCache(snapshotChunkCacheLimit) | |
| 19 | 19 | ||
| 20 | 20 | // Eleven maximum-size chunks exceed the per-session 16 MiB cache budget. | |
| 21 | - | // The capture retains independently owned publication objects while its LRU | |
| 22 | - | // sheds the oldest entry. | |
| 21 | + | // The cache keeps its warmed working set rather than admitting every | |
| 22 | + | // oldest-to-newest scan miss. | |
| 23 | 23 | aHistory := snapshotCacheHistory(t, 11, 'a') | |
| 24 | 24 | aCapture := snapshotCacheCapture(a, aHistory, 1) | |
| 25 | 25 | aFirst, err := d.incrementalPublication(aCapture) | |
| @@ -27,7 +27,7 @@ func TestSnapshotChunkCacheIsScopedToNamedSession(t *testing.T) { | |||
| 27 | 27 | require.NotEmpty(t, aFirst.Objects) | |
| 28 | 28 | require.LessOrEqual(t, a.snapshotChunkCache.used, a.snapshotChunkCache.limit) | |
| 29 | 29 | require.LessOrEqual(t, a.snapshotChunkCache.limit, snapshotChunkCacheLimit) | |
| 30 | - | require.NotContains(t, a.snapshotChunkCache.byPtr, aHistory.Chunk(0), "session A should evict its oldest entry") | |
| 30 | + | require.Contains(t, a.snapshotChunkCache.byPtr, aHistory.Chunk(0), "session A should retain its warmed entry") | |
| 31 | 31 | ||
| 32 | 32 | bHistory := snapshotCacheHistory(t, 1, 'b') | |
| 33 | 33 | bCapture := snapshotCacheCapture(b, bHistory, 1) | |
| @@ -39,7 +39,7 @@ func TestSnapshotChunkCacheIsScopedToNamedSession(t *testing.T) { | |||
| 39 | 39 | aSecond := snapshotCacheCapture(a, snapshotCacheHistory(t, 1, 'z'), 2) | |
| 40 | 40 | _, err = d.incrementalPublication(aSecond) | |
| 41 | 41 | require.NoError(t, err) | |
| 42 | - | require.True(t, bytes.Equal(inFlightBytes, aFirst.Objects[0].Data), "LRU eviction must not mutate queued publication bytes") | |
| 42 | + | require.True(t, bytes.Equal(inFlightBytes, aFirst.Objects[0].Data), "cache pruning must not mutate queued publication bytes") | |
| 43 | 43 | require.LessOrEqual(t, a.snapshotChunkCache.used, snapshotChunkCacheLimit) | |
| 44 | 44 | ||
| 45 | 45 | _, err = d.incrementalPublication(snapshotCacheCapture(b, bHistory, 2)) | |
| @@ -18,8 +18,8 @@ func TestSnapshotChunkCacheIsScopedToNamedSession(t *testing.T) { | |||||
| 18 | b.snapshotChunkCache = newSnapshotChunkCache(snapshotChunkCacheLimit) | 18 | b.snapshotChunkCache = newSnapshotChunkCache(snapshotChunkCacheLimit) | ||
| 19 | 19 | ||||
| 20 | // Eleven maximum-size chunks exceed the per-session 16 MiB cache budget. | 20 | // Eleven maximum-size chunks exceed the per-session 16 MiB cache budget. | ||
| 21 | - | // The capture retains independently owned publication objects while its LRU | 21 | + | // The cache keeps its warmed working set rather than admitting every |
| 22 | - | // sheds the oldest entry. | 22 | + | // oldest-to-newest scan miss. |
| 23 | aHistory := snapshotCacheHistory(t, 11, 'a') | 23 | aHistory := snapshotCacheHistory(t, 11, 'a') | ||
| 24 | aCapture := snapshotCacheCapture(a, aHistory, 1) | 24 | aCapture := snapshotCacheCapture(a, aHistory, 1) | ||
| 25 | aFirst, err := d.incrementalPublication(aCapture) | 25 | aFirst, err := d.incrementalPublication(aCapture) | ||
| @@ -27,7 +27,7 @@ func TestSnapshotChunkCacheIsScopedToNamedSession(t *testing.T) { | |||||
| 27 | require.NotEmpty(t, aFirst.Objects) | 27 | require.NotEmpty(t, aFirst.Objects) | ||
| 28 | require.LessOrEqual(t, a.snapshotChunkCache.used, a.snapshotChunkCache.limit) | 28 | require.LessOrEqual(t, a.snapshotChunkCache.used, a.snapshotChunkCache.limit) | ||
| 29 | require.LessOrEqual(t, a.snapshotChunkCache.limit, snapshotChunkCacheLimit) | 29 | require.LessOrEqual(t, a.snapshotChunkCache.limit, snapshotChunkCacheLimit) | ||
| 30 | - | require.NotContains(t, a.snapshotChunkCache.byPtr, aHistory.Chunk(0), "session A should evict its oldest entry") | 30 | + | require.Contains(t, a.snapshotChunkCache.byPtr, aHistory.Chunk(0), "session A should retain its warmed entry") |
| 31 | 31 | ||||
| 32 | bHistory := snapshotCacheHistory(t, 1, 'b') | 32 | bHistory := snapshotCacheHistory(t, 1, 'b') | ||
| 33 | bCapture := snapshotCacheCapture(b, bHistory, 1) | 33 | bCapture := snapshotCacheCapture(b, bHistory, 1) | ||
| @@ -39,7 +39,7 @@ func TestSnapshotChunkCacheIsScopedToNamedSession(t *testing.T) { | |||||
| 39 | aSecond := snapshotCacheCapture(a, snapshotCacheHistory(t, 1, 'z'), 2) | 39 | aSecond := snapshotCacheCapture(a, snapshotCacheHistory(t, 1, 'z'), 2) | ||
| 40 | _, err = d.incrementalPublication(aSecond) | 40 | _, err = d.incrementalPublication(aSecond) | ||
| 41 | require.NoError(t, err) | 41 | require.NoError(t, err) | ||
| 42 | - | require.True(t, bytes.Equal(inFlightBytes, aFirst.Objects[0].Data), "LRU eviction must not mutate queued publication bytes") | 42 | + | require.True(t, bytes.Equal(inFlightBytes, aFirst.Objects[0].Data), "cache pruning must not mutate queued publication bytes") |
| 43 | require.LessOrEqual(t, a.snapshotChunkCache.used, snapshotChunkCacheLimit) | 43 | require.LessOrEqual(t, a.snapshotChunkCache.used, snapshotChunkCacheLimit) | ||
| 44 | 44 | ||||
| 45 | _, err = d.incrementalPublication(snapshotCacheCapture(b, bHistory, 2)) | 45 | _, err = d.incrementalPublication(snapshotCacheCapture(b, bHistory, 2)) | ||
A internal/usecase/daemon/snapshot_coordinator.go +525 -0
| @@ -0,0 +1,525 @@ | |||
| 1 | + | package daemon | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "time" | |
| 6 | + | ||
| 7 | + | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |
| 8 | + | ) | |
| 9 | + | ||
| 10 | + | func snapshotCoordinatorContext(sess *session) (context.Context, context.CancelFunc) { | |
| 11 | + | sess.snapshotMu.Lock() | |
| 12 | + | defer sess.snapshotMu.Unlock() | |
| 13 | + | if sess.snapshotPublicationContext == nil { | |
| 14 | + | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |
| 15 | + | } | |
| 16 | + | return sess.snapshotPublicationContext, sess.snapshotPublicationCancel | |
| 17 | + | } | |
| 18 | + | ||
| 19 | + | // snapshotCoordinatorQuarantine identifies one stop request. A later teardown | |
| 20 | + | // can supersede it, in which case only that later owner may resume publication. | |
| 21 | + | type snapshotCoordinatorQuarantine struct { | |
| 22 | + | done <-chan struct{} | |
| 23 | + | epoch uint64 | |
| 24 | + | } | |
| 25 | + | ||
| 26 | + | // quarantineSnapshotCoordinator prevents a deleted session from starting any | |
| 27 | + | // further publication. The returned join channel is closed after an already | |
| 28 | + | // started repository call returns; callers must wait without daemon or session | |
| 29 | + | // locks before deleting repository state. | |
| 30 | + | func quarantineSnapshotCoordinator(sess *session) <-chan struct{} { | |
| 31 | + | return quarantineSnapshotCoordinatorWithEpoch(sess).done | |
| 32 | + | } | |
| 33 | + | ||
| 34 | + | // quarantineSnapshotCoordinatorRetainingQueuedCapture is used after a forced | |
| 35 | + | // shutdown checkpoint times out. Its queued capture is logical retry state, not | |
| 36 | + | // a repository operation: startSnapshotPublication rejects it while | |
| 37 | + | // quarantined, and the worker eventually discards it after its blocked call | |
| 38 | + | // returns. | |
| 39 | + | func quarantineSnapshotCoordinatorRetainingQueuedCapture(sess *session) <-chan struct{} { | |
| 40 | + | return quarantineSnapshotCoordinatorWithOptions(sess, true).done | |
| 41 | + | } | |
| 42 | + | ||
| 43 | + | func quarantineSnapshotCoordinatorWithEpoch(sess *session) snapshotCoordinatorQuarantine { | |
| 44 | + | return quarantineSnapshotCoordinatorWithOptions(sess, false) | |
| 45 | + | } | |
| 46 | + | ||
| 47 | + | func quarantineSnapshotCoordinatorWithOptions(sess *session, retainQueuedCapture bool) snapshotCoordinatorQuarantine { | |
| 48 | + | if sess == nil { | |
| 49 | + | done := make(chan struct{}) | |
| 50 | + | close(done) | |
| 51 | + | return snapshotCoordinatorQuarantine{done: done} | |
| 52 | + | } | |
| 53 | + | sess.snapshotMu.Lock() | |
| 54 | + | sess.snapshotQuarantineEpoch++ | |
| 55 | + | epoch := sess.snapshotQuarantineEpoch | |
| 56 | + | sess.snapshotQuarantined = true | |
| 57 | + | sess.snapEligible.Store(false) | |
| 58 | + | if sess.snapshotPublicationCancel != nil { | |
| 59 | + | sess.snapshotPublicationCancel() | |
| 60 | + | } | |
| 61 | + | if sess.snapshotQueuedCapture != nil && !retainQueuedCapture { | |
| 62 | + | // A rename or purge must discard a capture under the old identity. The | |
| 63 | + | // global queue may still contain it, but startSnapshotPublication | |
| 64 | + | // compares identity and will discard it even if this session is renamed. | |
| 65 | + | sess.snapshotQueuedCapture.coordinatorDiscarded = true | |
| 66 | + | sess.snapshotQueuedCapture = nil | |
| 67 | + | if sess.snapshotPendingCaptures > 0 { | |
| 68 | + | sess.snapshotPendingCaptures-- | |
| 69 | + | } | |
| 70 | + | sess.snapshotPending = sess.snapshotPendingCaptures > 0 || sess.snapshotInFlightCapture != nil | |
| 71 | + | } | |
| 72 | + | done := sess.snapshotPublicationDone | |
| 73 | + | if done == nil { | |
| 74 | + | done = make(chan struct{}) | |
| 75 | + | close(done) | |
| 76 | + | } | |
| 77 | + | sess.signalSnapshotChangedLocked() | |
| 78 | + | sess.snapshotMu.Unlock() | |
| 79 | + | return snapshotCoordinatorQuarantine{done: done, epoch: epoch} | |
| 80 | + | } | |
| 81 | + | ||
| 82 | + | // resumeSnapshotCoordinatorForNewIdentity starts a fresh repository lineage | |
| 83 | + | // only after rename committed its new in-memory name. The epoch prevents a | |
| 84 | + | // concurrent teardown from being undone by a late rename completion. | |
| 85 | + | func resumeSnapshotCoordinatorForNewIdentity(sess *session, quarantine snapshotCoordinatorQuarantine) bool { | |
| 86 | + | if sess == nil { | |
| 87 | + | return false | |
| 88 | + | } | |
| 89 | + | sess.snapshotMu.Lock() | |
| 90 | + | if !sess.snapshotQuarantined || sess.snapshotQuarantineEpoch != quarantine.epoch { | |
| 91 | + | sess.snapshotMu.Unlock() | |
| 92 | + | return false | |
| 93 | + | } | |
| 94 | + | if sess.snapshotPublicationCancel != nil { | |
| 95 | + | sess.snapshotPublicationCancel() | |
| 96 | + | } | |
| 97 | + | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |
| 98 | + | sess.snapshotQuarantined = false | |
| 99 | + | // A repository name owns its own generation stream. Reset all scheduler | |
| 100 | + | // state while eligibility remains false, then atomically make generation 1 | |
| 101 | + | // dirty and wake the scheduler. | |
| 102 | + | sess.snapshotGeneration = 1 | |
| 103 | + | sess.snapshotCapturedGeneration = 0 | |
| 104 | + | sess.snapshotPublishedGeneration = 0 | |
| 105 | + | sess.snapshotForcedGeneration = 0 | |
| 106 | + | sess.snapshotNextEligibleAt = time.Time{} | |
| 107 | + | sess.snapshotAttempted = false | |
| 108 | + | sess.snapshotAttemptKind = snapshotAttemptRoutine | |
| 109 | + | sess.snapshotFailureSig = "" | |
| 110 | + | sess.snapshotChunkCache = newSnapshotChunkCache(snapshotChunkCacheLimit) | |
| 111 | + | sess.snapDirty.Store(true) | |
| 112 | + | sess.snapEligible.Store(true) | |
| 113 | + | sess.signalSnapshotChangedLocked() | |
| 114 | + | wake := sess.snapshotWake | |
| 115 | + | sess.snapshotMu.Unlock() | |
| 116 | + | if wake != nil { | |
| 117 | + | select { | |
| 118 | + | case wake <- struct{}{}: | |
| 119 | + | default: | |
| 120 | + | } | |
| 121 | + | } | |
| 122 | + | return true | |
| 123 | + | } | |
| 124 | + | ||
| 125 | + | func markSnapshotDirty(sess *session) { | |
| 126 | + | if sess == nil || !sess.snapEligible.Load() { | |
| 127 | + | return | |
| 128 | + | } | |
| 129 | + | sess.snapshotMu.Lock() | |
| 130 | + | sess.snapshotGeneration++ | |
| 131 | + | sess.snapDirty.Store(true) | |
| 132 | + | sess.signalSnapshotChangedLocked() | |
| 133 | + | wake := sess.snapshotWake | |
| 134 | + | sess.snapshotMu.Unlock() | |
| 135 | + | if wake != nil { | |
| 136 | + | select { | |
| 137 | + | case wake <- struct{}{}: | |
| 138 | + | default: | |
| 139 | + | } | |
| 140 | + | } | |
| 141 | + | } | |
| 142 | + | ||
| 143 | + | // snapshotChangeLocked returns a channel closed by the next snapshot state | |
| 144 | + | // transition. snapshotMu must be held. | |
| 145 | + | func (sess *session) snapshotChangeLocked() chan struct{} { | |
| 146 | + | if sess.snapshotChanged == nil { | |
| 147 | + | sess.snapshotChanged = make(chan struct{}) | |
| 148 | + | } | |
| 149 | + | return sess.snapshotChanged | |
| 150 | + | } | |
| 151 | + | ||
| 152 | + | // signalSnapshotChangedLocked wakes waiters after a snapshot state transition. | |
| 153 | + | // snapshotMu must be held. | |
| 154 | + | func (sess *session) signalSnapshotChangedLocked() { | |
| 155 | + | close(sess.snapshotChangeLocked()) | |
| 156 | + | sess.snapshotChanged = make(chan struct{}) | |
| 157 | + | } | |
| 158 | + | ||
| 159 | + | func (d *Daemon) scheduleSnapshot(sess *session) bool { | |
| 160 | + | return d.scheduleSnapshotWithFinalFallback(sess, false) | |
| 161 | + | } | |
| 162 | + | ||
| 163 | + | // scheduleFinalSnapshot captures the terminal state even when an older capture | |
| 164 | + | // is pending. A removed session cannot be retried by the repository scheduler, so the | |
| 165 | + | // worker drains its retained terminal state before it stops. | |
| 166 | + | func (d *Daemon) scheduleFinalSnapshot(sess *session) bool { | |
| 167 | + | return d.scheduleSnapshotWithFinalFallback(sess, true) | |
| 168 | + | } | |
| 169 | + | ||
| 170 | + | func (d *Daemon) scheduleSnapshotWithFinalFallback(sess *session, final bool) bool { | |
| 171 | + | if d == nil || sess == nil || !d.snapsEnabled || d.snapshotRepository == nil || !sess.snapEligible.Load() { | |
| 172 | + | return true | |
| 173 | + | } | |
| 174 | + | kind := snapshotAttemptRoutine | |
| 175 | + | if final { | |
| 176 | + | kind = snapshotAttemptForced | |
| 177 | + | } | |
| 178 | + | sess.snapshotMu.Lock() | |
| 179 | + | if sess.snapshotQuarantined || !sess.snapDirty.Load() { | |
| 180 | + | available := !sess.snapshotQuarantined | |
| 181 | + | sess.snapshotMu.Unlock() | |
| 182 | + | return available | |
| 183 | + | } | |
| 184 | + | if final && sess.snapshotGeneration > sess.snapshotForcedGeneration { | |
| 185 | + | // Do not let an older queued or in-flight routine publication satisfy a | |
| 186 | + | // terminal checkpoint. The intent is retained until a publication at | |
| 187 | + | // this generation (or newer) succeeds. | |
| 188 | + | sess.snapshotForcedGeneration = sess.snapshotGeneration | |
| 189 | + | } | |
| 190 | + | if sess.snapshotPending || | |
| 191 | + | (!final && !sess.snapshotNextEligibleAt.IsZero() && d.clock.Now().Before(sess.snapshotNextEligibleAt)) { | |
| 192 | + | sess.snapshotMu.Unlock() | |
| 193 | + | return true | |
| 194 | + | } | |
| 195 | + | if sess.snapshotPublicationContext == nil { | |
| 196 | + | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |
| 197 | + | } | |
| 198 | + | generation := sess.snapshotGeneration | |
| 199 | + | publicationContext := sess.snapshotPublicationContext | |
| 200 | + | sess.snapshotCapturedGeneration = generation | |
| 201 | + | sess.snapshotAttempted = true | |
| 202 | + | sess.snapshotAttemptKind = kind | |
| 203 | + | sess.snapshotPendingCaptures++ | |
| 204 | + | sess.snapshotPending = true | |
| 205 | + | sess.signalSnapshotChangedLocked() | |
| 206 | + | sess.snapshotMu.Unlock() | |
| 207 | + | ||
| 208 | + | capture, ok := d.captureSnapshotState(sess, generation) | |
| 209 | + | if capture != nil { | |
| 210 | + | capture.attemptKind = kind | |
| 211 | + | capture.publicationContext = publicationContext | |
| 212 | + | } | |
| 213 | + | if !ok { | |
| 214 | + | d.finishSnapshotCapture(capture, false) | |
| 215 | + | return false | |
| 216 | + | } | |
| 217 | + | sess.snapshotMu.Lock() | |
| 218 | + | quarantined := sess.snapshotQuarantined | |
| 219 | + | if !quarantined { | |
| 220 | + | sess.snapshotQueuedCapture = capture | |
| 221 | + | } | |
| 222 | + | sess.snapshotMu.Unlock() | |
| 223 | + | if quarantined { | |
| 224 | + | d.finishSnapshotCapture(capture, false) | |
| 225 | + | return false | |
| 226 | + | } | |
| 227 | + | if d.enqueueSnapshotCapture(capture) || (final && d.enqueueFinalSnapshotCapture(capture)) { | |
| 228 | + | return true | |
| 229 | + | } | |
| 230 | + | // Coalesce under saturation or shutdown: the latest state stays dirty and | |
| 231 | + | // will be captured on a later tick once an active worker has room. | |
| 232 | + | d.finishSnapshotCapture(capture, false) | |
| 233 | + | return false | |
| 234 | + | } | |
| 235 | + | ||
| 236 | + | // waitForSnapshotGeneration waits only for the bounded terminal checkpoint | |
| 237 | + | // deadline. It observes successful publication, rather than dirty state, so a | |
| 238 | + | // concurrent later mutation cannot turn a completed forced checkpoint into a | |
| 239 | + | // false timeout. | |
| 240 | + | func (d *Daemon) waitForSnapshotGeneration(sess *session, generation uint64) bool { | |
| 241 | + | if d == nil || sess == nil || generation == 0 { | |
| 242 | + | return true | |
| 243 | + | } | |
| 244 | + | deadline := newSnapshotShutdownDeadline(d.clock) | |
| 245 | + | defer deadline.stop() | |
| 246 | + | return d.waitForSnapshotGenerationWithDeadline(sess, generation, deadline) | |
| 247 | + | } | |
| 248 | + | ||
| 249 | + | // waitForSnapshotGenerationWithDeadline waits for a forced checkpoint without | |
| 250 | + | // allocating another shutdown interval. Serve shares one deadline across every | |
| 251 | + | // session and the subsequent worker drain. | |
| 252 | + | func (d *Daemon) waitForSnapshotGenerationWithDeadline(sess *session, generation uint64, deadline *snapshotShutdownDeadline) bool { | |
| 253 | + | if d == nil || sess == nil || generation == 0 { | |
| 254 | + | return true | |
| 255 | + | } | |
| 256 | + | if deadline == nil { | |
| 257 | + | return d.waitForSnapshotGeneration(sess, generation) | |
| 258 | + | } | |
| 259 | + | for { | |
| 260 | + | sess.snapshotMu.Lock() | |
| 261 | + | published := sess.snapshotPublishedGeneration >= generation | |
| 262 | + | changed := sess.snapshotChangeLocked() | |
| 263 | + | sess.snapshotMu.Unlock() | |
| 264 | + | if published { | |
| 265 | + | return true | |
| 266 | + | } | |
| 267 | + | select { | |
| 268 | + | case <-changed: | |
| 269 | + | case <-deadline.Done(): | |
| 270 | + | return false | |
| 271 | + | } | |
| 272 | + | } | |
| 273 | + | } | |
| 274 | + | ||
| 275 | + | // enqueueSnapshotCapture serializes worker shutdown with the non-blocking | |
| 276 | + | // queue send. The queue is deliberately never closed: producers can race | |
| 277 | + | // shutdown without risking a send-on-closed panic. | |
| 278 | + | func (d *Daemon) enqueueSnapshotCapture(capture *snapshotCapture) bool { | |
| 279 | + | d.snapshotWorkerMu.Lock() | |
| 280 | + | defer d.snapshotWorkerMu.Unlock() | |
| 281 | + | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |
| 282 | + | return false | |
| 283 | + | } | |
| 284 | + | select { | |
| 285 | + | case d.snapshotJobs <- capture: | |
| 286 | + | return true | |
| 287 | + | default: | |
| 288 | + | return false | |
| 289 | + | } | |
| 290 | + | } | |
| 291 | + | ||
| 292 | + | // enqueueFinalSnapshotCapture hands terminal captures to the existing worker | |
| 293 | + | // without making session teardown wait behind its normal bounded queue. A | |
| 294 | + | // blocked worker retains at most one terminal capture per session, replacing | |
| 295 | + | // stale state with the newest immutable capture. | |
| 296 | + | func (d *Daemon) enqueueFinalSnapshotCapture(capture *snapshotCapture) bool { | |
| 297 | + | if capture == nil || capture.session == nil { | |
| 298 | + | return false | |
| 299 | + | } | |
| 300 | + | d.snapshotWorkerMu.Lock() | |
| 301 | + | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |
| 302 | + | d.snapshotWorkerMu.Unlock() | |
| 303 | + | return false | |
| 304 | + | } | |
| 305 | + | if d.snapshotFinalJobs == nil { | |
| 306 | + | d.snapshotFinalJobs = make(map[*session]*snapshotCapture) | |
| 307 | + | } | |
| 308 | + | replaced, exists := d.snapshotFinalJobs[capture.session] | |
| 309 | + | if !exists && len(d.snapshotFinalJobs) >= snapshotFinalQueueCapacity { | |
| 310 | + | d.snapshotWorkerMu.Unlock() | |
| 311 | + | d.log.Warn("terminal snapshot retention saturated; capture rejected", "session", capture.name, "capacity", snapshotFinalQueueCapacity) | |
| 312 | + | return false | |
| 313 | + | } | |
| 314 | + | // Finish the stale capture before publishing its replacement. Otherwise a | |
| 315 | + | // fast worker could persist the replacement and then have this stale failure | |
| 316 | + | // mark the session dirty again. | |
| 317 | + | d.finishSnapshotCapture(replaced, false) | |
| 318 | + | if !exists { | |
| 319 | + | d.snapshotFinalOrder = append(d.snapshotFinalOrder, capture.session) | |
| 320 | + | } | |
| 321 | + | d.snapshotFinalJobs[capture.session] = capture | |
| 322 | + | select { | |
| 323 | + | case d.snapshotWorkerFinalWake <- struct{}{}: | |
| 324 | + | default: | |
| 325 | + | } | |
| 326 | + | d.snapshotWorkerMu.Unlock() | |
| 327 | + | return true | |
| 328 | + | } | |
| 329 | + | ||
| 330 | + | // captureSession rotates history tails and clones visible frames while holding | |
| 331 | + | // each pane lock. The returned capture contains only immutable state; encoding | |
| 332 | + | // and persistence are deliberately deferred to snapshotEncodeWorker. | |
| 333 | + | func (d *Daemon) captureSnapshotState(sess *session, generation uint64) (*snapshotCapture, bool) { | |
| 334 | + | sess.mu.Lock() | |
| 335 | + | capture := &snapshotCapture{ | |
| 336 | + | session: sess, | |
| 337 | + | generation: generation, | |
| 338 | + | name: sess.name, | |
| 339 | + | createdAt: uint64(sess.createdAt), | |
| 340 | + | active: uint16(max(sess.active, 0)), | |
| 341 | + | } | |
| 342 | + | ephemeral := sess.ephemeral | |
| 343 | + | fallbackCwd := sess.cwd | |
| 344 | + | tabs := append([]*tab(nil), sess.tabs...) | |
| 345 | + | sess.mu.Unlock() | |
| 346 | + | if ephemeral || capture.name == "" { | |
| 347 | + | return capture, false | |
| 348 | + | } | |
| 349 | + | ||
| 350 | + | capture.tabs = make([]snapshotCaptureTab, 0, len(tabs)) | |
| 351 | + | for _, tb := range tabs { | |
| 352 | + | tb.mu.Lock() | |
| 353 | + | tabCapture := snapshotCaptureTab{ | |
| 354 | + | stableID: tb.stableID, | |
| 355 | + | cols: uint16(max(tb.size.Cols, 0)), | |
| 356 | + | rows: uint16(max(tb.size.Rows, 0)), | |
| 357 | + | nextPaneID: uint64(max(tb.nextPaneID, 0)), | |
| 358 | + | tree: tb.tree.Clone(), | |
| 359 | + | } | |
| 360 | + | if tb.tree != nil { | |
| 361 | + | tabCapture.focus = tb.tree.Focus | |
| 362 | + | } | |
| 363 | + | panes := make([]*pane, 0, len(tb.panes)) | |
| 364 | + | for _, p := range tb.panes { | |
| 365 | + | panes = append(panes, p) | |
| 366 | + | } | |
| 367 | + | tb.mu.Unlock() | |
| 368 | + | ||
| 369 | + | tabCapture.panes = make([]snapshotCapturePane, 0, len(panes)) | |
| 370 | + | for _, p := range panes { | |
| 371 | + | p.mu.Lock() | |
| 372 | + | pty := p.pty | |
| 373 | + | pid := 0 | |
| 374 | + | if pty != nil { | |
| 375 | + | pid = pty.Pid() | |
| 376 | + | } | |
| 377 | + | paneCapture := snapshotCapturePane{ | |
| 378 | + | id: p.id, | |
| 379 | + | stableID: p.stableID, | |
| 380 | + | visible: p.screen.PrimaryVisibleSnapshot(), | |
| 381 | + | } | |
| 382 | + | paneCapture.sealed = p.history.SnapshotView() | |
| 383 | + | paneCapture.tail = paneCapture.sealed.Tail() | |
| 384 | + | p.mu.Unlock() | |
| 385 | + | paneCapture.cwd = fallbackCwd | |
| 386 | + | if d.procCwd != nil && pid > 0 { | |
| 387 | + | if cwd, err := d.procCwd(pid); err == nil && cwd != "" { | |
| 388 | + | paneCapture.cwd = cwd | |
| 389 | + | } | |
| 390 | + | } | |
| 391 | + | paneCapture.process = d.capturePaneProcess(pty, pid) | |
| 392 | + | tabCapture.panes = append(tabCapture.panes, paneCapture) | |
| 393 | + | } | |
| 394 | + | capture.tabs = append(capture.tabs, tabCapture) | |
| 395 | + | } | |
| 396 | + | return capture, true | |
| 397 | + | } | |
| 398 | + | ||
| 399 | + | // captureSession remains the synchronous producer-facing trigger for callers | |
| 400 | + | // such as teardown and benchmarks; the actual encoding and Write stay async. | |
| 401 | + | func (d *Daemon) captureSession(sess *session) bool { | |
| 402 | + | markSnapshotDirty(sess) | |
| 403 | + | return d.scheduleSnapshot(sess) | |
| 404 | + | } | |
| 405 | + | ||
| 406 | + | // startSnapshotPublication moves a retained capture from the producer queue to | |
| 407 | + | // the repository worker. Quarantine and this transition share snapshotMu, so a | |
| 408 | + | // repository call cannot begin after destructive teardown has quarantined the | |
| 409 | + | // session. | |
| 410 | + | func startSnapshotPublication(capture *snapshotCapture) bool { | |
| 411 | + | if capture == nil || capture.session == nil { | |
| 412 | + | return false | |
| 413 | + | } | |
| 414 | + | sess := capture.session | |
| 415 | + | sess.snapshotMu.Lock() | |
| 416 | + | defer sess.snapshotMu.Unlock() | |
| 417 | + | if sess.snapshotQuarantined || sess.snapshotQueuedCapture != capture { | |
| 418 | + | return false | |
| 419 | + | } | |
| 420 | + | sess.snapshotQueuedCapture = nil | |
| 421 | + | sess.snapshotInFlightCapture = capture | |
| 422 | + | sess.snapshotPublicationDone = make(chan struct{}) | |
| 423 | + | sess.snapshotPending = true | |
| 424 | + | sess.signalSnapshotChangedLocked() | |
| 425 | + | return true | |
| 426 | + | } | |
| 427 | + | ||
| 428 | + | func (d *Daemon) finishSnapshotCapture(capture *snapshotCapture, succeeded bool) { | |
| 429 | + | if capture == nil { | |
| 430 | + | return | |
| 431 | + | } | |
| 432 | + | capture.finishOnce.Do(func() { | |
| 433 | + | shouldScheduleForcedSuccessor := false | |
| 434 | + | capture.session.snapshotMu.Lock() | |
| 435 | + | if capture.session.snapshotPendingCaptures > 0 { | |
| 436 | + | capture.session.snapshotPendingCaptures-- | |
| 437 | + | } | |
| 438 | + | if capture.session.snapshotQueuedCapture == capture { | |
| 439 | + | capture.session.snapshotQueuedCapture = nil | |
| 440 | + | } | |
| 441 | + | if capture.session.snapshotInFlightCapture == capture { | |
| 442 | + | capture.session.snapshotInFlightCapture = nil | |
| 443 | + | if capture.session.snapshotPublicationDone != nil { | |
| 444 | + | close(capture.session.snapshotPublicationDone) | |
| 445 | + | capture.session.snapshotPublicationDone = nil | |
| 446 | + | } | |
| 447 | + | } | |
| 448 | + | capture.session.snapshotPending = capture.session.snapshotPendingCaptures > 0 || | |
| 449 | + | capture.session.snapshotQueuedCapture != nil || capture.session.snapshotInFlightCapture != nil | |
| 450 | + | if !capture.coordinatorDiscarded && capture.attemptKind == snapshotAttemptRoutine && d.snapshotRepository != nil { | |
| 451 | + | // The interval is measured from completion, including a failed | |
| 452 | + | // attempt. A forced teardown checkpoint deliberately leaves it alone. | |
| 453 | + | capture.session.snapshotNextEligibleAt = d.clock.Now().Add(snapshotInterval) | |
| 454 | + | } | |
| 455 | + | if succeeded { | |
| 456 | + | capture.session.snapshotPublishedGeneration = max(capture.session.snapshotPublishedGeneration, capture.generation) | |
| 457 | + | } | |
| 458 | + | if succeeded && capture.session.snapshotGeneration == capture.generation { | |
| 459 | + | capture.session.snapDirty.Store(false) | |
| 460 | + | capture.session.snapshotFailureSig = "" | |
| 461 | + | } else if !succeeded { | |
| 462 | + | capture.session.snapDirty.Store(true) | |
| 463 | + | } | |
| 464 | + | if succeeded && capture.session.snapshotForcedGeneration != 0 && | |
| 465 | + | capture.generation >= capture.session.snapshotForcedGeneration { | |
| 466 | + | capture.session.snapshotForcedGeneration = 0 | |
| 467 | + | } | |
| 468 | + | // A failed forced publication remains dirty for ordinary retry rather | |
| 469 | + | // than recursively creating forced jobs. A later forced request records | |
| 470 | + | // a newer intent and can create its one bounded successor. | |
| 471 | + | shouldScheduleForcedSuccessor = capture.session.snapshotForcedGeneration != 0 && | |
| 472 | + | capture.generation < capture.session.snapshotForcedGeneration && | |
| 473 | + | capture.session.snapshotQueuedCapture == nil && capture.session.snapshotInFlightCapture == nil && | |
| 474 | + | (succeeded || capture.attemptKind != snapshotAttemptForced) | |
| 475 | + | capture.session.signalSnapshotChangedLocked() | |
| 476 | + | wake := capture.session.snapshotWake | |
| 477 | + | capture.session.snapshotMu.Unlock() | |
| 478 | + | if wake != nil { | |
| 479 | + | select { | |
| 480 | + | case wake <- struct{}{}: | |
| 481 | + | default: | |
| 482 | + | } | |
| 483 | + | } | |
| 484 | + | if shouldScheduleForcedSuccessor { | |
| 485 | + | // This runs after publication completion and outside snapshotMu. It | |
| 486 | + | // bypasses routine eligibility while preserving the single queued and | |
| 487 | + | // single in-flight capture bound. | |
| 488 | + | _ = d.scheduleFinalSnapshot(capture.session) | |
| 489 | + | } | |
| 490 | + | if succeeded { | |
| 491 | + | d.clearSnapshotFailure() | |
| 492 | + | } | |
| 493 | + | }) | |
| 494 | + | } | |
| 495 | + | ||
| 496 | + | // reportSnapshotFailure records every failed persistence attempt but only | |
| 497 | + | // displays a toast when its stable phase/error class changes. This keeps a | |
| 498 | + | // blocked disk from repeatedly repainting every attached client while retaining | |
| 499 | + | // the count in notification history. | |
| 500 | + | ||
| 501 | + | func (d *Daemon) capturePaneProcess(pty interface{ ForegroundPgid() (int, error) }, shellPid int) *snapcodec.Process { | |
| 502 | + | if d == nil || pty == nil || shellPid <= 0 || d.procGroupArgv == nil { | |
| 503 | + | return nil | |
| 504 | + | } | |
| 505 | + | pgid, err := pty.ForegroundPgid() | |
| 506 | + | if err != nil || pgid <= 0 || pgid == shellPid { | |
| 507 | + | return nil | |
| 508 | + | } | |
| 509 | + | argv, err := d.procGroupArgv(pgid, shellPid) | |
| 510 | + | if err != nil || len(argv) == 0 || argv[0] == "" { | |
| 511 | + | return nil | |
| 512 | + | } | |
| 513 | + | strategy := detectProcessStrategy(argv) | |
| 514 | + | return &snapcodec.Process{ | |
| 515 | + | Argv: append([]string(nil), argv...), | |
| 516 | + | Strategy: strategy, | |
| 517 | + | Opts: snapcodec.ProcessOpts{ | |
| 518 | + | AgentSessionID: extractAgentSessionID(strategy, argv), | |
| 519 | + | }, | |
| 520 | + | } | |
| 521 | + | } | |
| 522 | + | ||
| 523 | + | // snapshotRepositorySaver waits until the earliest completion-derived routine | |
| 524 | + | // eligibility. State changes wake it immediately, so a newly dirty session is | |
| 525 | + | // captured without waiting for an unrelated session's interval. | |
| @@ -0,0 +1,525 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "time" | |||
| 6 | + | ||||
| 7 | + | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |||
| 8 | + | ) | |||
| 9 | + | ||||
| 10 | + | func snapshotCoordinatorContext(sess *session) (context.Context, context.CancelFunc) { | |||
| 11 | + | sess.snapshotMu.Lock() | |||
| 12 | + | defer sess.snapshotMu.Unlock() | |||
| 13 | + | if sess.snapshotPublicationContext == nil { | |||
| 14 | + | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |||
| 15 | + | } | |||
| 16 | + | return sess.snapshotPublicationContext, sess.snapshotPublicationCancel | |||
| 17 | + | } | |||
| 18 | + | ||||
| 19 | + | // snapshotCoordinatorQuarantine identifies one stop request. A later teardown | |||
| 20 | + | // can supersede it, in which case only that later owner may resume publication. | |||
| 21 | + | type snapshotCoordinatorQuarantine struct { | |||
| 22 | + | done <-chan struct{} | |||
| 23 | + | epoch uint64 | |||
| 24 | + | } | |||
| 25 | + | ||||
| 26 | + | // quarantineSnapshotCoordinator prevents a deleted session from starting any | |||
| 27 | + | // further publication. The returned join channel is closed after an already | |||
| 28 | + | // started repository call returns; callers must wait without daemon or session | |||
| 29 | + | // locks before deleting repository state. | |||
| 30 | + | func quarantineSnapshotCoordinator(sess *session) <-chan struct{} { | |||
| 31 | + | return quarantineSnapshotCoordinatorWithEpoch(sess).done | |||
| 32 | + | } | |||
| 33 | + | ||||
| 34 | + | // quarantineSnapshotCoordinatorRetainingQueuedCapture is used after a forced | |||
| 35 | + | // shutdown checkpoint times out. Its queued capture is logical retry state, not | |||
| 36 | + | // a repository operation: startSnapshotPublication rejects it while | |||
| 37 | + | // quarantined, and the worker eventually discards it after its blocked call | |||
| 38 | + | // returns. | |||
| 39 | + | func quarantineSnapshotCoordinatorRetainingQueuedCapture(sess *session) <-chan struct{} { | |||
| 40 | + | return quarantineSnapshotCoordinatorWithOptions(sess, true).done | |||
| 41 | + | } | |||
| 42 | + | ||||
| 43 | + | func quarantineSnapshotCoordinatorWithEpoch(sess *session) snapshotCoordinatorQuarantine { | |||
| 44 | + | return quarantineSnapshotCoordinatorWithOptions(sess, false) | |||
| 45 | + | } | |||
| 46 | + | ||||
| 47 | + | func quarantineSnapshotCoordinatorWithOptions(sess *session, retainQueuedCapture bool) snapshotCoordinatorQuarantine { | |||
| 48 | + | if sess == nil { | |||
| 49 | + | done := make(chan struct{}) | |||
| 50 | + | close(done) | |||
| 51 | + | return snapshotCoordinatorQuarantine{done: done} | |||
| 52 | + | } | |||
| 53 | + | sess.snapshotMu.Lock() | |||
| 54 | + | sess.snapshotQuarantineEpoch++ | |||
| 55 | + | epoch := sess.snapshotQuarantineEpoch | |||
| 56 | + | sess.snapshotQuarantined = true | |||
| 57 | + | sess.snapEligible.Store(false) | |||
| 58 | + | if sess.snapshotPublicationCancel != nil { | |||
| 59 | + | sess.snapshotPublicationCancel() | |||
| 60 | + | } | |||
| 61 | + | if sess.snapshotQueuedCapture != nil && !retainQueuedCapture { | |||
| 62 | + | // A rename or purge must discard a capture under the old identity. The | |||
| 63 | + | // global queue may still contain it, but startSnapshotPublication | |||
| 64 | + | // compares identity and will discard it even if this session is renamed. | |||
| 65 | + | sess.snapshotQueuedCapture.coordinatorDiscarded = true | |||
| 66 | + | sess.snapshotQueuedCapture = nil | |||
| 67 | + | if sess.snapshotPendingCaptures > 0 { | |||
| 68 | + | sess.snapshotPendingCaptures-- | |||
| 69 | + | } | |||
| 70 | + | sess.snapshotPending = sess.snapshotPendingCaptures > 0 || sess.snapshotInFlightCapture != nil | |||
| 71 | + | } | |||
| 72 | + | done := sess.snapshotPublicationDone | |||
| 73 | + | if done == nil { | |||
| 74 | + | done = make(chan struct{}) | |||
| 75 | + | close(done) | |||
| 76 | + | } | |||
| 77 | + | sess.signalSnapshotChangedLocked() | |||
| 78 | + | sess.snapshotMu.Unlock() | |||
| 79 | + | return snapshotCoordinatorQuarantine{done: done, epoch: epoch} | |||
| 80 | + | } | |||
| 81 | + | ||||
| 82 | + | // resumeSnapshotCoordinatorForNewIdentity starts a fresh repository lineage | |||
| 83 | + | // only after rename committed its new in-memory name. The epoch prevents a | |||
| 84 | + | // concurrent teardown from being undone by a late rename completion. | |||
| 85 | + | func resumeSnapshotCoordinatorForNewIdentity(sess *session, quarantine snapshotCoordinatorQuarantine) bool { | |||
| 86 | + | if sess == nil { | |||
| 87 | + | return false | |||
| 88 | + | } | |||
| 89 | + | sess.snapshotMu.Lock() | |||
| 90 | + | if !sess.snapshotQuarantined || sess.snapshotQuarantineEpoch != quarantine.epoch { | |||
| 91 | + | sess.snapshotMu.Unlock() | |||
| 92 | + | return false | |||
| 93 | + | } | |||
| 94 | + | if sess.snapshotPublicationCancel != nil { | |||
| 95 | + | sess.snapshotPublicationCancel() | |||
| 96 | + | } | |||
| 97 | + | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |||
| 98 | + | sess.snapshotQuarantined = false | |||
| 99 | + | // A repository name owns its own generation stream. Reset all scheduler | |||
| 100 | + | // state while eligibility remains false, then atomically make generation 1 | |||
| 101 | + | // dirty and wake the scheduler. | |||
| 102 | + | sess.snapshotGeneration = 1 | |||
| 103 | + | sess.snapshotCapturedGeneration = 0 | |||
| 104 | + | sess.snapshotPublishedGeneration = 0 | |||
| 105 | + | sess.snapshotForcedGeneration = 0 | |||
| 106 | + | sess.snapshotNextEligibleAt = time.Time{} | |||
| 107 | + | sess.snapshotAttempted = false | |||
| 108 | + | sess.snapshotAttemptKind = snapshotAttemptRoutine | |||
| 109 | + | sess.snapshotFailureSig = "" | |||
| 110 | + | sess.snapshotChunkCache = newSnapshotChunkCache(snapshotChunkCacheLimit) | |||
| 111 | + | sess.snapDirty.Store(true) | |||
| 112 | + | sess.snapEligible.Store(true) | |||
| 113 | + | sess.signalSnapshotChangedLocked() | |||
| 114 | + | wake := sess.snapshotWake | |||
| 115 | + | sess.snapshotMu.Unlock() | |||
| 116 | + | if wake != nil { | |||
| 117 | + | select { | |||
| 118 | + | case wake <- struct{}{}: | |||
| 119 | + | default: | |||
| 120 | + | } | |||
| 121 | + | } | |||
| 122 | + | return true | |||
| 123 | + | } | |||
| 124 | + | ||||
| 125 | + | func markSnapshotDirty(sess *session) { | |||
| 126 | + | if sess == nil || !sess.snapEligible.Load() { | |||
| 127 | + | return | |||
| 128 | + | } | |||
| 129 | + | sess.snapshotMu.Lock() | |||
| 130 | + | sess.snapshotGeneration++ | |||
| 131 | + | sess.snapDirty.Store(true) | |||
| 132 | + | sess.signalSnapshotChangedLocked() | |||
| 133 | + | wake := sess.snapshotWake | |||
| 134 | + | sess.snapshotMu.Unlock() | |||
| 135 | + | if wake != nil { | |||
| 136 | + | select { | |||
| 137 | + | case wake <- struct{}{}: | |||
| 138 | + | default: | |||
| 139 | + | } | |||
| 140 | + | } | |||
| 141 | + | } | |||
| 142 | + | ||||
| 143 | + | // snapshotChangeLocked returns a channel closed by the next snapshot state | |||
| 144 | + | // transition. snapshotMu must be held. | |||
| 145 | + | func (sess *session) snapshotChangeLocked() chan struct{} { | |||
| 146 | + | if sess.snapshotChanged == nil { | |||
| 147 | + | sess.snapshotChanged = make(chan struct{}) | |||
| 148 | + | } | |||
| 149 | + | return sess.snapshotChanged | |||
| 150 | + | } | |||
| 151 | + | ||||
| 152 | + | // signalSnapshotChangedLocked wakes waiters after a snapshot state transition. | |||
| 153 | + | // snapshotMu must be held. | |||
| 154 | + | func (sess *session) signalSnapshotChangedLocked() { | |||
| 155 | + | close(sess.snapshotChangeLocked()) | |||
| 156 | + | sess.snapshotChanged = make(chan struct{}) | |||
| 157 | + | } | |||
| 158 | + | ||||
| 159 | + | func (d *Daemon) scheduleSnapshot(sess *session) bool { | |||
| 160 | + | return d.scheduleSnapshotWithFinalFallback(sess, false) | |||
| 161 | + | } | |||
| 162 | + | ||||
| 163 | + | // scheduleFinalSnapshot captures the terminal state even when an older capture | |||
| 164 | + | // is pending. A removed session cannot be retried by the repository scheduler, so the | |||
| 165 | + | // worker drains its retained terminal state before it stops. | |||
| 166 | + | func (d *Daemon) scheduleFinalSnapshot(sess *session) bool { | |||
| 167 | + | return d.scheduleSnapshotWithFinalFallback(sess, true) | |||
| 168 | + | } | |||
| 169 | + | ||||
| 170 | + | func (d *Daemon) scheduleSnapshotWithFinalFallback(sess *session, final bool) bool { | |||
| 171 | + | if d == nil || sess == nil || !d.snapsEnabled || d.snapshotRepository == nil || !sess.snapEligible.Load() { | |||
| 172 | + | return true | |||
| 173 | + | } | |||
| 174 | + | kind := snapshotAttemptRoutine | |||
| 175 | + | if final { | |||
| 176 | + | kind = snapshotAttemptForced | |||
| 177 | + | } | |||
| 178 | + | sess.snapshotMu.Lock() | |||
| 179 | + | if sess.snapshotQuarantined || !sess.snapDirty.Load() { | |||
| 180 | + | available := !sess.snapshotQuarantined | |||
| 181 | + | sess.snapshotMu.Unlock() | |||
| 182 | + | return available | |||
| 183 | + | } | |||
| 184 | + | if final && sess.snapshotGeneration > sess.snapshotForcedGeneration { | |||
| 185 | + | // Do not let an older queued or in-flight routine publication satisfy a | |||
| 186 | + | // terminal checkpoint. The intent is retained until a publication at | |||
| 187 | + | // this generation (or newer) succeeds. | |||
| 188 | + | sess.snapshotForcedGeneration = sess.snapshotGeneration | |||
| 189 | + | } | |||
| 190 | + | if sess.snapshotPending || | |||
| 191 | + | (!final && !sess.snapshotNextEligibleAt.IsZero() && d.clock.Now().Before(sess.snapshotNextEligibleAt)) { | |||
| 192 | + | sess.snapshotMu.Unlock() | |||
| 193 | + | return true | |||
| 194 | + | } | |||
| 195 | + | if sess.snapshotPublicationContext == nil { | |||
| 196 | + | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |||
| 197 | + | } | |||
| 198 | + | generation := sess.snapshotGeneration | |||
| 199 | + | publicationContext := sess.snapshotPublicationContext | |||
| 200 | + | sess.snapshotCapturedGeneration = generation | |||
| 201 | + | sess.snapshotAttempted = true | |||
| 202 | + | sess.snapshotAttemptKind = kind | |||
| 203 | + | sess.snapshotPendingCaptures++ | |||
| 204 | + | sess.snapshotPending = true | |||
| 205 | + | sess.signalSnapshotChangedLocked() | |||
| 206 | + | sess.snapshotMu.Unlock() | |||
| 207 | + | ||||
| 208 | + | capture, ok := d.captureSnapshotState(sess, generation) | |||
| 209 | + | if capture != nil { | |||
| 210 | + | capture.attemptKind = kind | |||
| 211 | + | capture.publicationContext = publicationContext | |||
| 212 | + | } | |||
| 213 | + | if !ok { | |||
| 214 | + | d.finishSnapshotCapture(capture, false) | |||
| 215 | + | return false | |||
| 216 | + | } | |||
| 217 | + | sess.snapshotMu.Lock() | |||
| 218 | + | quarantined := sess.snapshotQuarantined | |||
| 219 | + | if !quarantined { | |||
| 220 | + | sess.snapshotQueuedCapture = capture | |||
| 221 | + | } | |||
| 222 | + | sess.snapshotMu.Unlock() | |||
| 223 | + | if quarantined { | |||
| 224 | + | d.finishSnapshotCapture(capture, false) | |||
| 225 | + | return false | |||
| 226 | + | } | |||
| 227 | + | if d.enqueueSnapshotCapture(capture) || (final && d.enqueueFinalSnapshotCapture(capture)) { | |||
| 228 | + | return true | |||
| 229 | + | } | |||
| 230 | + | // Coalesce under saturation or shutdown: the latest state stays dirty and | |||
| 231 | + | // will be captured on a later tick once an active worker has room. | |||
| 232 | + | d.finishSnapshotCapture(capture, false) | |||
| 233 | + | return false | |||
| 234 | + | } | |||
| 235 | + | ||||
| 236 | + | // waitForSnapshotGeneration waits only for the bounded terminal checkpoint | |||
| 237 | + | // deadline. It observes successful publication, rather than dirty state, so a | |||
| 238 | + | // concurrent later mutation cannot turn a completed forced checkpoint into a | |||
| 239 | + | // false timeout. | |||
| 240 | + | func (d *Daemon) waitForSnapshotGeneration(sess *session, generation uint64) bool { | |||
| 241 | + | if d == nil || sess == nil || generation == 0 { | |||
| 242 | + | return true | |||
| 243 | + | } | |||
| 244 | + | deadline := newSnapshotShutdownDeadline(d.clock) | |||
| 245 | + | defer deadline.stop() | |||
| 246 | + | return d.waitForSnapshotGenerationWithDeadline(sess, generation, deadline) | |||
| 247 | + | } | |||
| 248 | + | ||||
| 249 | + | // waitForSnapshotGenerationWithDeadline waits for a forced checkpoint without | |||
| 250 | + | // allocating another shutdown interval. Serve shares one deadline across every | |||
| 251 | + | // session and the subsequent worker drain. | |||
| 252 | + | func (d *Daemon) waitForSnapshotGenerationWithDeadline(sess *session, generation uint64, deadline *snapshotShutdownDeadline) bool { | |||
| 253 | + | if d == nil || sess == nil || generation == 0 { | |||
| 254 | + | return true | |||
| 255 | + | } | |||
| 256 | + | if deadline == nil { | |||
| 257 | + | return d.waitForSnapshotGeneration(sess, generation) | |||
| 258 | + | } | |||
| 259 | + | for { | |||
| 260 | + | sess.snapshotMu.Lock() | |||
| 261 | + | published := sess.snapshotPublishedGeneration >= generation | |||
| 262 | + | changed := sess.snapshotChangeLocked() | |||
| 263 | + | sess.snapshotMu.Unlock() | |||
| 264 | + | if published { | |||
| 265 | + | return true | |||
| 266 | + | } | |||
| 267 | + | select { | |||
| 268 | + | case <-changed: | |||
| 269 | + | case <-deadline.Done(): | |||
| 270 | + | return false | |||
| 271 | + | } | |||
| 272 | + | } | |||
| 273 | + | } | |||
| 274 | + | ||||
| 275 | + | // enqueueSnapshotCapture serializes worker shutdown with the non-blocking | |||
| 276 | + | // queue send. The queue is deliberately never closed: producers can race | |||
| 277 | + | // shutdown without risking a send-on-closed panic. | |||
| 278 | + | func (d *Daemon) enqueueSnapshotCapture(capture *snapshotCapture) bool { | |||
| 279 | + | d.snapshotWorkerMu.Lock() | |||
| 280 | + | defer d.snapshotWorkerMu.Unlock() | |||
| 281 | + | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |||
| 282 | + | return false | |||
| 283 | + | } | |||
| 284 | + | select { | |||
| 285 | + | case d.snapshotJobs <- capture: | |||
| 286 | + | return true | |||
| 287 | + | default: | |||
| 288 | + | return false | |||
| 289 | + | } | |||
| 290 | + | } | |||
| 291 | + | ||||
| 292 | + | // enqueueFinalSnapshotCapture hands terminal captures to the existing worker | |||
| 293 | + | // without making session teardown wait behind its normal bounded queue. A | |||
| 294 | + | // blocked worker retains at most one terminal capture per session, replacing | |||
| 295 | + | // stale state with the newest immutable capture. | |||
| 296 | + | func (d *Daemon) enqueueFinalSnapshotCapture(capture *snapshotCapture) bool { | |||
| 297 | + | if capture == nil || capture.session == nil { | |||
| 298 | + | return false | |||
| 299 | + | } | |||
| 300 | + | d.snapshotWorkerMu.Lock() | |||
| 301 | + | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |||
| 302 | + | d.snapshotWorkerMu.Unlock() | |||
| 303 | + | return false | |||
| 304 | + | } | |||
| 305 | + | if d.snapshotFinalJobs == nil { | |||
| 306 | + | d.snapshotFinalJobs = make(map[*session]*snapshotCapture) | |||
| 307 | + | } | |||
| 308 | + | replaced, exists := d.snapshotFinalJobs[capture.session] | |||
| 309 | + | if !exists && len(d.snapshotFinalJobs) >= snapshotFinalQueueCapacity { | |||
| 310 | + | d.snapshotWorkerMu.Unlock() | |||
| 311 | + | d.log.Warn("terminal snapshot retention saturated; capture rejected", "session", capture.name, "capacity", snapshotFinalQueueCapacity) | |||
| 312 | + | return false | |||
| 313 | + | } | |||
| 314 | + | // Finish the stale capture before publishing its replacement. Otherwise a | |||
| 315 | + | // fast worker could persist the replacement and then have this stale failure | |||
| 316 | + | // mark the session dirty again. | |||
| 317 | + | d.finishSnapshotCapture(replaced, false) | |||
| 318 | + | if !exists { | |||
| 319 | + | d.snapshotFinalOrder = append(d.snapshotFinalOrder, capture.session) | |||
| 320 | + | } | |||
| 321 | + | d.snapshotFinalJobs[capture.session] = capture | |||
| 322 | + | select { | |||
| 323 | + | case d.snapshotWorkerFinalWake <- struct{}{}: | |||
| 324 | + | default: | |||
| 325 | + | } | |||
| 326 | + | d.snapshotWorkerMu.Unlock() | |||
| 327 | + | return true | |||
| 328 | + | } | |||
| 329 | + | ||||
| 330 | + | // captureSession rotates history tails and clones visible frames while holding | |||
| 331 | + | // each pane lock. The returned capture contains only immutable state; encoding | |||
| 332 | + | // and persistence are deliberately deferred to snapshotEncodeWorker. | |||
| 333 | + | func (d *Daemon) captureSnapshotState(sess *session, generation uint64) (*snapshotCapture, bool) { | |||
| 334 | + | sess.mu.Lock() | |||
| 335 | + | capture := &snapshotCapture{ | |||
| 336 | + | session: sess, | |||
| 337 | + | generation: generation, | |||
| 338 | + | name: sess.name, | |||
| 339 | + | createdAt: uint64(sess.createdAt), | |||
| 340 | + | active: uint16(max(sess.active, 0)), | |||
| 341 | + | } | |||
| 342 | + | ephemeral := sess.ephemeral | |||
| 343 | + | fallbackCwd := sess.cwd | |||
| 344 | + | tabs := append([]*tab(nil), sess.tabs...) | |||
| 345 | + | sess.mu.Unlock() | |||
| 346 | + | if ephemeral || capture.name == "" { | |||
| 347 | + | return capture, false | |||
| 348 | + | } | |||
| 349 | + | ||||
| 350 | + | capture.tabs = make([]snapshotCaptureTab, 0, len(tabs)) | |||
| 351 | + | for _, tb := range tabs { | |||
| 352 | + | tb.mu.Lock() | |||
| 353 | + | tabCapture := snapshotCaptureTab{ | |||
| 354 | + | stableID: tb.stableID, | |||
| 355 | + | cols: uint16(max(tb.size.Cols, 0)), | |||
| 356 | + | rows: uint16(max(tb.size.Rows, 0)), | |||
| 357 | + | nextPaneID: uint64(max(tb.nextPaneID, 0)), | |||
| 358 | + | tree: tb.tree.Clone(), | |||
| 359 | + | } | |||
| 360 | + | if tb.tree != nil { | |||
| 361 | + | tabCapture.focus = tb.tree.Focus | |||
| 362 | + | } | |||
| 363 | + | panes := make([]*pane, 0, len(tb.panes)) | |||
| 364 | + | for _, p := range tb.panes { | |||
| 365 | + | panes = append(panes, p) | |||
| 366 | + | } | |||
| 367 | + | tb.mu.Unlock() | |||
| 368 | + | ||||
| 369 | + | tabCapture.panes = make([]snapshotCapturePane, 0, len(panes)) | |||
| 370 | + | for _, p := range panes { | |||
| 371 | + | p.mu.Lock() | |||
| 372 | + | pty := p.pty | |||
| 373 | + | pid := 0 | |||
| 374 | + | if pty != nil { | |||
| 375 | + | pid = pty.Pid() | |||
| 376 | + | } | |||
| 377 | + | paneCapture := snapshotCapturePane{ | |||
| 378 | + | id: p.id, | |||
| 379 | + | stableID: p.stableID, | |||
| 380 | + | visible: p.screen.PrimaryVisibleSnapshot(), | |||
| 381 | + | } | |||
| 382 | + | paneCapture.sealed = p.history.SnapshotView() | |||
| 383 | + | paneCapture.tail = paneCapture.sealed.Tail() | |||
| 384 | + | p.mu.Unlock() | |||
| 385 | + | paneCapture.cwd = fallbackCwd | |||
| 386 | + | if d.procCwd != nil && pid > 0 { | |||
| 387 | + | if cwd, err := d.procCwd(pid); err == nil && cwd != "" { | |||
| 388 | + | paneCapture.cwd = cwd | |||
| 389 | + | } | |||
| 390 | + | } | |||
| 391 | + | paneCapture.process = d.capturePaneProcess(pty, pid) | |||
| 392 | + | tabCapture.panes = append(tabCapture.panes, paneCapture) | |||
| 393 | + | } | |||
| 394 | + | capture.tabs = append(capture.tabs, tabCapture) | |||
| 395 | + | } | |||
| 396 | + | return capture, true | |||
| 397 | + | } | |||
| 398 | + | ||||
| 399 | + | // captureSession remains the synchronous producer-facing trigger for callers | |||
| 400 | + | // such as teardown and benchmarks; the actual encoding and Write stay async. | |||
| 401 | + | func (d *Daemon) captureSession(sess *session) bool { | |||
| 402 | + | markSnapshotDirty(sess) | |||
| 403 | + | return d.scheduleSnapshot(sess) | |||
| 404 | + | } | |||
| 405 | + | ||||
| 406 | + | // startSnapshotPublication moves a retained capture from the producer queue to | |||
| 407 | + | // the repository worker. Quarantine and this transition share snapshotMu, so a | |||
| 408 | + | // repository call cannot begin after destructive teardown has quarantined the | |||
| 409 | + | // session. | |||
| 410 | + | func startSnapshotPublication(capture *snapshotCapture) bool { | |||
| 411 | + | if capture == nil || capture.session == nil { | |||
| 412 | + | return false | |||
| 413 | + | } | |||
| 414 | + | sess := capture.session | |||
| 415 | + | sess.snapshotMu.Lock() | |||
| 416 | + | defer sess.snapshotMu.Unlock() | |||
| 417 | + | if sess.snapshotQuarantined || sess.snapshotQueuedCapture != capture { | |||
| 418 | + | return false | |||
| 419 | + | } | |||
| 420 | + | sess.snapshotQueuedCapture = nil | |||
| 421 | + | sess.snapshotInFlightCapture = capture | |||
| 422 | + | sess.snapshotPublicationDone = make(chan struct{}) | |||
| 423 | + | sess.snapshotPending = true | |||
| 424 | + | sess.signalSnapshotChangedLocked() | |||
| 425 | + | return true | |||
| 426 | + | } | |||
| 427 | + | ||||
| 428 | + | func (d *Daemon) finishSnapshotCapture(capture *snapshotCapture, succeeded bool) { | |||
| 429 | + | if capture == nil { | |||
| 430 | + | return | |||
| 431 | + | } | |||
| 432 | + | capture.finishOnce.Do(func() { | |||
| 433 | + | shouldScheduleForcedSuccessor := false | |||
| 434 | + | capture.session.snapshotMu.Lock() | |||
| 435 | + | if capture.session.snapshotPendingCaptures > 0 { | |||
| 436 | + | capture.session.snapshotPendingCaptures-- | |||
| 437 | + | } | |||
| 438 | + | if capture.session.snapshotQueuedCapture == capture { | |||
| 439 | + | capture.session.snapshotQueuedCapture = nil | |||
| 440 | + | } | |||
| 441 | + | if capture.session.snapshotInFlightCapture == capture { | |||
| 442 | + | capture.session.snapshotInFlightCapture = nil | |||
| 443 | + | if capture.session.snapshotPublicationDone != nil { | |||
| 444 | + | close(capture.session.snapshotPublicationDone) | |||
| 445 | + | capture.session.snapshotPublicationDone = nil | |||
| 446 | + | } | |||
| 447 | + | } | |||
| 448 | + | capture.session.snapshotPending = capture.session.snapshotPendingCaptures > 0 || | |||
| 449 | + | capture.session.snapshotQueuedCapture != nil || capture.session.snapshotInFlightCapture != nil | |||
| 450 | + | if !capture.coordinatorDiscarded && capture.attemptKind == snapshotAttemptRoutine && d.snapshotRepository != nil { | |||
| 451 | + | // The interval is measured from completion, including a failed | |||
| 452 | + | // attempt. A forced teardown checkpoint deliberately leaves it alone. | |||
| 453 | + | capture.session.snapshotNextEligibleAt = d.clock.Now().Add(snapshotInterval) | |||
| 454 | + | } | |||
| 455 | + | if succeeded { | |||
| 456 | + | capture.session.snapshotPublishedGeneration = max(capture.session.snapshotPublishedGeneration, capture.generation) | |||
| 457 | + | } | |||
| 458 | + | if succeeded && capture.session.snapshotGeneration == capture.generation { | |||
| 459 | + | capture.session.snapDirty.Store(false) | |||
| 460 | + | capture.session.snapshotFailureSig = "" | |||
| 461 | + | } else if !succeeded { | |||
| 462 | + | capture.session.snapDirty.Store(true) | |||
| 463 | + | } | |||
| 464 | + | if succeeded && capture.session.snapshotForcedGeneration != 0 && | |||
| 465 | + | capture.generation >= capture.session.snapshotForcedGeneration { | |||
| 466 | + | capture.session.snapshotForcedGeneration = 0 | |||
| 467 | + | } | |||
| 468 | + | // A failed forced publication remains dirty for ordinary retry rather | |||
| 469 | + | // than recursively creating forced jobs. A later forced request records | |||
| 470 | + | // a newer intent and can create its one bounded successor. | |||
| 471 | + | shouldScheduleForcedSuccessor = capture.session.snapshotForcedGeneration != 0 && | |||
| 472 | + | capture.generation < capture.session.snapshotForcedGeneration && | |||
| 473 | + | capture.session.snapshotQueuedCapture == nil && capture.session.snapshotInFlightCapture == nil && | |||
| 474 | + | (succeeded || capture.attemptKind != snapshotAttemptForced) | |||
| 475 | + | capture.session.signalSnapshotChangedLocked() | |||
| 476 | + | wake := capture.session.snapshotWake | |||
| 477 | + | capture.session.snapshotMu.Unlock() | |||
| 478 | + | if wake != nil { | |||
| 479 | + | select { | |||
| 480 | + | case wake <- struct{}{}: | |||
| 481 | + | default: | |||
| 482 | + | } | |||
| 483 | + | } | |||
| 484 | + | if shouldScheduleForcedSuccessor { | |||
| 485 | + | // This runs after publication completion and outside snapshotMu. It | |||
| 486 | + | // bypasses routine eligibility while preserving the single queued and | |||
| 487 | + | // single in-flight capture bound. | |||
| 488 | + | _ = d.scheduleFinalSnapshot(capture.session) | |||
| 489 | + | } | |||
| 490 | + | if succeeded { | |||
| 491 | + | d.clearSnapshotFailure() | |||
| 492 | + | } | |||
| 493 | + | }) | |||
| 494 | + | } | |||
| 495 | + | ||||
| 496 | + | // reportSnapshotFailure records every failed persistence attempt but only | |||
| 497 | + | // displays a toast when its stable phase/error class changes. This keeps a | |||
| 498 | + | // blocked disk from repeatedly repainting every attached client while retaining | |||
| 499 | + | // the count in notification history. | |||
| 500 | + | ||||
| 501 | + | func (d *Daemon) capturePaneProcess(pty interface{ ForegroundPgid() (int, error) }, shellPid int) *snapcodec.Process { | |||
| 502 | + | if d == nil || pty == nil || shellPid <= 0 || d.procGroupArgv == nil { | |||
| 503 | + | return nil | |||
| 504 | + | } | |||
| 505 | + | pgid, err := pty.ForegroundPgid() | |||
| 506 | + | if err != nil || pgid <= 0 || pgid == shellPid { | |||
| 507 | + | return nil | |||
| 508 | + | } | |||
| 509 | + | argv, err := d.procGroupArgv(pgid, shellPid) | |||
| 510 | + | if err != nil || len(argv) == 0 || argv[0] == "" { | |||
| 511 | + | return nil | |||
| 512 | + | } | |||
| 513 | + | strategy := detectProcessStrategy(argv) | |||
| 514 | + | return &snapcodec.Process{ | |||
| 515 | + | Argv: append([]string(nil), argv...), | |||
| 516 | + | Strategy: strategy, | |||
| 517 | + | Opts: snapcodec.ProcessOpts{ | |||
| 518 | + | AgentSessionID: extractAgentSessionID(strategy, argv), | |||
| 519 | + | }, | |||
| 520 | + | } | |||
| 521 | + | } | |||
| 522 | + | ||||
| 523 | + | // snapshotRepositorySaver waits until the earliest completion-derived routine | |||
| 524 | + | // eligibility. State changes wake it immediately, so a newly dirty session is | |||
| 525 | + | // captured without waiting for an unrelated session's interval. | |||
M internal/usecase/daemon/snapshot_incremental.go +22 -32
| @@ -1,7 +1,6 @@ | |||
| 1 | 1 | package daemon | |
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | - | "container/list" | |
| 5 | 4 | "fmt" | |
| 6 | 5 | ||
| 7 | 6 | "github.com/bnema/vev/internal/ports" | |
| @@ -14,7 +13,6 @@ import ( | |||
| 14 | 13 | const snapshotChunkCacheLimit = 16 << 20 | |
| 15 | 14 | ||
| 16 | 15 | type snapshotChunkCacheEntry struct { | |
| 17 | - | chunk *vt.HistoryChunk | |
| 18 | 16 | object ports.SnapshotObject | |
| 19 | 17 | } | |
| 20 | 18 | ||
| @@ -25,12 +23,18 @@ type snapshotChunkCacheEntry struct { | |||
| 25 | 23 | type snapshotChunkCache struct { | |
| 26 | 24 | limit int | |
| 27 | 25 | used int | |
| 28 | - | byPtr map[*vt.HistoryChunk]*list.Element | |
| 29 | - | lru *list.List // front is most recently used; values are snapshotChunkCacheEntry | |
| 26 | + | byPtr map[*vt.HistoryChunk]snapshotChunkCacheEntry | |
| 27 | + | // marshalChunk makes cache admission measurable without coupling the cache | |
| 28 | + | // policy test to VT's wire encoding implementation. | |
| 29 | + | marshalChunk func(*vt.HistoryChunk) ([]byte, error) | |
| 30 | 30 | } | |
| 31 | 31 | ||
| 32 | 32 | func newSnapshotChunkCache(limit int) *snapshotChunkCache { | |
| 33 | - | return &snapshotChunkCache{limit: limit, byPtr: make(map[*vt.HistoryChunk]*list.Element), lru: list.New()} | |
| 33 | + | return &snapshotChunkCache{ | |
| 34 | + | limit: limit, | |
| 35 | + | byPtr: make(map[*vt.HistoryChunk]snapshotChunkCacheEntry), | |
| 36 | + | marshalChunk: vt.MarshalHistoryChunk, | |
| 37 | + | } | |
| 34 | 38 | } | |
| 35 | 39 | ||
| 36 | 40 | // objectLocked returns an independently owned object. Caller holds the owning | |
| @@ -39,12 +43,11 @@ func (c *snapshotChunkCache) objectLocked(chunk *vt.HistoryChunk) (ports.Snapsho | |||
| 39 | 43 | if chunk == nil { | |
| 40 | 44 | return ports.SnapshotObject{}, fmt.Errorf("snapshot: nil history chunk") | |
| 41 | 45 | } | |
| 42 | - | if elem := c.byPtr[chunk]; elem != nil { | |
| 43 | - | c.lru.MoveToFront(elem) | |
| 44 | - | return copySnapshotObject(elem.Value.(snapshotChunkCacheEntry).object), nil | |
| 46 | + | if entry, ok := c.byPtr[chunk]; ok { | |
| 47 | + | return copySnapshotObject(entry.object), nil | |
| 45 | 48 | } | |
| 46 | 49 | ||
| 47 | - | payload, err := vt.MarshalHistoryChunk(chunk) | |
| 50 | + | payload, err := c.marshalChunk(chunk) | |
| 48 | 51 | if err != nil { | |
| 49 | 52 | return ports.SnapshotObject{}, err | |
| 50 | 53 | } | |
| @@ -53,41 +56,28 @@ func (c *snapshotChunkCache) objectLocked(chunk *vt.HistoryChunk) (ports.Snapsho | |||
| 53 | 56 | return ports.SnapshotObject{}, err | |
| 54 | 57 | } | |
| 55 | 58 | ||
| 56 | - | // A single chunk over the ceiling must not flush useful smaller entries. | |
| 57 | - | // It is already independently owned by this publication, so skipping the | |
| 58 | - | // cache cannot affect its lifetime. | |
| 59 | - | if len(object.Data) > c.limit { | |
| 59 | + | // An oldest-to-newest snapshot scan can exceed the cache. LRU would admit | |
| 60 | + | // every early miss and evict the later working set, causing the next scan to | |
| 61 | + | // encode every immutable chunk again. Once full, retain the warmed set and | |
| 62 | + | // return later misses uncached. Pruning makes space when a retained chunk is | |
| 63 | + | // no longer reachable by a queued or in-flight capture. | |
| 64 | + | if len(object.Data) > c.limit || c.used > c.limit-len(object.Data) { | |
| 60 | 65 | return object, nil | |
| 61 | 66 | } | |
| 62 | 67 | stored := copySnapshotObject(object) | |
| 63 | - | elem := c.lru.PushFront(snapshotChunkCacheEntry{chunk: chunk, object: stored}) | |
| 64 | - | c.byPtr[chunk] = elem | |
| 68 | + | c.byPtr[chunk] = snapshotChunkCacheEntry{object: stored} | |
| 65 | 69 | c.used += len(stored.Data) | |
| 66 | - | for c.used > c.limit { | |
| 67 | - | oldest := c.lru.Back() | |
| 68 | - | if oldest == nil { | |
| 69 | - | break | |
| 70 | - | } | |
| 71 | - | entry := oldest.Value.(snapshotChunkCacheEntry) | |
| 72 | - | delete(c.byPtr, entry.chunk) | |
| 73 | - | c.used -= len(entry.object.Data) | |
| 74 | - | c.lru.Remove(oldest) | |
| 75 | - | } | |
| 76 | 70 | return object, nil | |
| 77 | 71 | } | |
| 78 | 72 | ||
| 79 | 73 | // pruneLocked releases cached encodings which no currently queued or in-flight | |
| 80 | 74 | // capture can use. Caller holds the owning session's snapshotMu. | |
| 81 | 75 | func (c *snapshotChunkCache) pruneLocked(referenced map[*vt.HistoryChunk]struct{}) { | |
| 82 | - | for elem := c.lru.Back(); elem != nil; { | |
| 83 | - | previous := elem.Prev() | |
| 84 | - | entry := elem.Value.(snapshotChunkCacheEntry) | |
| 85 | - | if _, keep := referenced[entry.chunk]; !keep { | |
| 86 | - | delete(c.byPtr, entry.chunk) | |
| 76 | + | for chunk, entry := range c.byPtr { | |
| 77 | + | if _, keep := referenced[chunk]; !keep { | |
| 78 | + | delete(c.byPtr, chunk) | |
| 87 | 79 | c.used -= len(entry.object.Data) | |
| 88 | - | c.lru.Remove(elem) | |
| 89 | 80 | } | |
| 90 | - | elem = previous | |
| 91 | 81 | } | |
| 92 | 82 | } | |
| 93 | 83 | ||
| @@ -1,7 +1,6 @@ | |||||
| 1 | package daemon | 1 | package daemon | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | - | "container/list" | |||
| 5 | "fmt" | 4 | "fmt" | ||
| 6 | 5 | ||||
| 7 | "github.com/bnema/vev/internal/ports" | 6 | "github.com/bnema/vev/internal/ports" | ||
| @@ -14,7 +13,6 @@ import ( | |||||
| 14 | const snapshotChunkCacheLimit = 16 << 20 | 13 | const snapshotChunkCacheLimit = 16 << 20 | ||
| 15 | 14 | ||||
| 16 | type snapshotChunkCacheEntry struct { | 15 | type snapshotChunkCacheEntry struct { | ||
| 17 | - | chunk *vt.HistoryChunk | |||
| 18 | object ports.SnapshotObject | 16 | object ports.SnapshotObject | ||
| 19 | } | 17 | } | ||
| 20 | 18 | ||||
| @@ -25,12 +23,18 @@ type snapshotChunkCacheEntry struct { | |||||
| 25 | type snapshotChunkCache struct { | 23 | type snapshotChunkCache struct { | ||
| 26 | limit int | 24 | limit int | ||
| 27 | used int | 25 | used int | ||
| 28 | - | byPtr map[*vt.HistoryChunk]*list.Element | 26 | + | byPtr map[*vt.HistoryChunk]snapshotChunkCacheEntry |
| 29 | - | lru *list.List // front is most recently used; values are snapshotChunkCacheEntry | 27 | + | // marshalChunk makes cache admission measurable without coupling the cache |
| 28 | + | // policy test to VT's wire encoding implementation. | |||
| 29 | + | marshalChunk func(*vt.HistoryChunk) ([]byte, error) | |||
| 30 | } | 30 | } | ||
| 31 | 31 | ||||
| 32 | func newSnapshotChunkCache(limit int) *snapshotChunkCache { | 32 | func newSnapshotChunkCache(limit int) *snapshotChunkCache { | ||
| 33 | - | return &snapshotChunkCache{limit: limit, byPtr: make(map[*vt.HistoryChunk]*list.Element), lru: list.New()} | 33 | + | return &snapshotChunkCache{ |
| 34 | + | limit: limit, | |||
| 35 | + | byPtr: make(map[*vt.HistoryChunk]snapshotChunkCacheEntry), | |||
| 36 | + | marshalChunk: vt.MarshalHistoryChunk, | |||
| 37 | + | } | |||
| 34 | } | 38 | } | ||
| 35 | 39 | ||||
| 36 | // objectLocked returns an independently owned object. Caller holds the owning | 40 | // objectLocked returns an independently owned object. Caller holds the owning | ||
| @@ -39,12 +43,11 @@ func (c *snapshotChunkCache) objectLocked(chunk *vt.HistoryChunk) (ports.Snapsho | |||||
| 39 | if chunk == nil { | 43 | if chunk == nil { | ||
| 40 | return ports.SnapshotObject{}, fmt.Errorf("snapshot: nil history chunk") | 44 | return ports.SnapshotObject{}, fmt.Errorf("snapshot: nil history chunk") | ||
| 41 | } | 45 | } | ||
| 42 | - | if elem := c.byPtr[chunk]; elem != nil { | 46 | + | if entry, ok := c.byPtr[chunk]; ok { |
| 43 | - | c.lru.MoveToFront(elem) | 47 | + | return copySnapshotObject(entry.object), nil |
| 44 | - | return copySnapshotObject(elem.Value.(snapshotChunkCacheEntry).object), nil | |||
| 45 | } | 48 | } | ||
| 46 | 49 | ||||
| 47 | - | payload, err := vt.MarshalHistoryChunk(chunk) | 50 | + | payload, err := c.marshalChunk(chunk) |
| 48 | if err != nil { | 51 | if err != nil { | ||
| 49 | return ports.SnapshotObject{}, err | 52 | return ports.SnapshotObject{}, err | ||
| 50 | } | 53 | } | ||
| @@ -53,41 +56,28 @@ func (c *snapshotChunkCache) objectLocked(chunk *vt.HistoryChunk) (ports.Snapsho | |||||
| 53 | return ports.SnapshotObject{}, err | 56 | return ports.SnapshotObject{}, err | ||
| 54 | } | 57 | } | ||
| 55 | 58 | ||||
| 56 | - | // A single chunk over the ceiling must not flush useful smaller entries. | 59 | + | // An oldest-to-newest snapshot scan can exceed the cache. LRU would admit |
| 57 | - | // It is already independently owned by this publication, so skipping the | 60 | + | // every early miss and evict the later working set, causing the next scan to |
| 58 | - | // cache cannot affect its lifetime. | 61 | + | // encode every immutable chunk again. Once full, retain the warmed set and |
| 59 | - | if len(object.Data) > c.limit { | 62 | + | // return later misses uncached. Pruning makes space when a retained chunk is |
| 63 | + | // no longer reachable by a queued or in-flight capture. | |||
| 64 | + | if len(object.Data) > c.limit || c.used > c.limit-len(object.Data) { | |||
| 60 | return object, nil | 65 | return object, nil | ||
| 61 | } | 66 | } | ||
| 62 | stored := copySnapshotObject(object) | 67 | stored := copySnapshotObject(object) | ||
| 63 | - | elem := c.lru.PushFront(snapshotChunkCacheEntry{chunk: chunk, object: stored}) | 68 | + | c.byPtr[chunk] = snapshotChunkCacheEntry{object: stored} |
| 64 | - | c.byPtr[chunk] = elem | |||
| 65 | c.used += len(stored.Data) | 69 | c.used += len(stored.Data) | ||
| 66 | - | for c.used > c.limit { | |||
| 67 | - | oldest := c.lru.Back() | |||
| 68 | - | if oldest == nil { | |||
| 69 | - | break | |||
| 70 | - | } | |||
| 71 | - | entry := oldest.Value.(snapshotChunkCacheEntry) | |||
| 72 | - | delete(c.byPtr, entry.chunk) | |||
| 73 | - | c.used -= len(entry.object.Data) | |||
| 74 | - | c.lru.Remove(oldest) | |||
| 75 | - | } | |||
| 76 | return object, nil | 70 | return object, nil | ||
| 77 | } | 71 | } | ||
| 78 | 72 | ||||
| 79 | // pruneLocked releases cached encodings which no currently queued or in-flight | 73 | // pruneLocked releases cached encodings which no currently queued or in-flight | ||
| 80 | // capture can use. Caller holds the owning session's snapshotMu. | 74 | // capture can use. Caller holds the owning session's snapshotMu. | ||
| 81 | func (c *snapshotChunkCache) pruneLocked(referenced map[*vt.HistoryChunk]struct{}) { | 75 | func (c *snapshotChunkCache) pruneLocked(referenced map[*vt.HistoryChunk]struct{}) { | ||
| 82 | - | for elem := c.lru.Back(); elem != nil; { | 76 | + | for chunk, entry := range c.byPtr { |
| 83 | - | previous := elem.Prev() | 77 | + | if _, keep := referenced[chunk]; !keep { |
| 84 | - | entry := elem.Value.(snapshotChunkCacheEntry) | 78 | + | delete(c.byPtr, chunk) |
| 85 | - | if _, keep := referenced[entry.chunk]; !keep { | |||
| 86 | - | delete(c.byPtr, entry.chunk) | |||
| 87 | c.used -= len(entry.object.Data) | 79 | c.used -= len(entry.object.Data) | ||
| 88 | - | c.lru.Remove(elem) | |||
| 89 | } | 80 | } | ||
| 90 | - | elem = previous | |||
| 91 | } | 81 | } | ||
| 92 | } | 82 | } | ||
| 93 | 83 | ||||
M internal/usecase/daemon/snapshot_incremental_test.go +52 -0
| @@ -4,6 +4,9 @@ import ( | |||
| 4 | 4 | "testing" | |
| 5 | 5 | "time" | |
| 6 | 6 | ||
| 7 | + | "github.com/stretchr/testify/require" | |
| 8 | + | ||
| 9 | + | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |
| 7 | 10 | "github.com/bnema/vev/pkg/renderer" | |
| 8 | 11 | "github.com/bnema/vev/pkg/vt" | |
| 9 | 12 | ) | |
| @@ -35,6 +38,55 @@ func TestIncrementalPublicationEncodesAfterPaneUnlock(t *testing.T) { | |||
| 35 | 38 | pane.mu.Unlock() | |
| 36 | 39 | } | |
| 37 | 40 | ||
| 41 | + | func TestSnapshotChunkCacheResistsWarmedOverCapacityHistoryScans(t *testing.T) { | |
| 42 | + | history := vt.NewHistory(vt.HistoryConfig{MaxRows: 6, ChunkRows: 1}) | |
| 43 | + | for i := range 6 { | |
| 44 | + | require.NoError(t, history.Append([]renderer.Cell{{Rune: rune('a' + i)}})) | |
| 45 | + | } | |
| 46 | + | view := history.SnapshotView() | |
| 47 | + | require.Equal(t, 6, view.ChunkCount()) | |
| 48 | + | ||
| 49 | + | firstPayload, err := vt.MarshalHistoryChunk(view.Chunk(0)) | |
| 50 | + | require.NoError(t, err) | |
| 51 | + | firstObject, err := snapcodec.MarshalObject(snapcodec.HistoryChunk, firstPayload) | |
| 52 | + | require.NoError(t, err) | |
| 53 | + | cache := newSnapshotChunkCache(2 * len(firstObject.Data)) | |
| 54 | + | encodes := make(map[*vt.HistoryChunk]int) | |
| 55 | + | cache.marshalChunk = func(chunk *vt.HistoryChunk) ([]byte, error) { | |
| 56 | + | encodes[chunk]++ | |
| 57 | + | return vt.MarshalHistoryChunk(chunk) | |
| 58 | + | } | |
| 59 | + | ||
| 60 | + | for pass := range 3 { | |
| 61 | + | for i := range view.ChunkCount() { | |
| 62 | + | _, err := cache.objectLocked(view.Chunk(i)) | |
| 63 | + | require.NoError(t, err) | |
| 64 | + | } | |
| 65 | + | require.LessOrEqual(t, cache.used, cache.limit) | |
| 66 | + | if pass == 0 { | |
| 67 | + | require.Len(t, cache.byPtr, 2) | |
| 68 | + | } | |
| 69 | + | } | |
| 70 | + | ||
| 71 | + | // The cache admits the first bounded working set then rejects later scan | |
| 72 | + | // misses. Consequently, each warmed retained chunk is encoded only once | |
| 73 | + | // even though the history is repeatedly scanned oldest-to-newest. | |
| 74 | + | require.Equal(t, 1, encodes[view.Chunk(0)]) | |
| 75 | + | require.Equal(t, 1, encodes[view.Chunk(1)]) | |
| 76 | + | require.Equal(t, 3, encodes[view.Chunk(2)]) | |
| 77 | + | require.Equal(t, 3, encodes[view.Chunk(5)]) | |
| 78 | + | ||
| 79 | + | isolated := newSnapshotChunkCache(cache.limit) | |
| 80 | + | isolatedEncodes := 0 | |
| 81 | + | isolated.marshalChunk = func(chunk *vt.HistoryChunk) ([]byte, error) { | |
| 82 | + | isolatedEncodes++ | |
| 83 | + | return vt.MarshalHistoryChunk(chunk) | |
| 84 | + | } | |
| 85 | + | _, err = isolated.objectLocked(view.Chunk(0)) | |
| 86 | + | require.NoError(t, err) | |
| 87 | + | require.Equal(t, 1, isolatedEncodes, "a session cache must not share encoded history with another session") | |
| 88 | + | } | |
| 89 | + | ||
| 38 | 90 | func TestIncrementalPublicationReusesSealedChunkObject(t *testing.T) { | |
| 39 | 91 | history := vt.NewHistory(vt.HistoryConfig{MaxRows: 2, ChunkRows: 1}) | |
| 40 | 92 | history.Append([]renderer.Cell{renderer.BlankCell()}) | |
| @@ -4,6 +4,9 @@ import ( | |||||
| 4 | "testing" | 4 | "testing" | ||
| 5 | "time" | 5 | "time" | ||
| 6 | 6 | ||||
| 7 | + | "github.com/stretchr/testify/require" | |||
| 8 | + | ||||
| 9 | + | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |||
| 7 | "github.com/bnema/vev/pkg/renderer" | 10 | "github.com/bnema/vev/pkg/renderer" | ||
| 8 | "github.com/bnema/vev/pkg/vt" | 11 | "github.com/bnema/vev/pkg/vt" | ||
| 9 | ) | 12 | ) | ||
| @@ -35,6 +38,55 @@ func TestIncrementalPublicationEncodesAfterPaneUnlock(t *testing.T) { | |||||
| 35 | pane.mu.Unlock() | 38 | pane.mu.Unlock() | ||
| 36 | } | 39 | } | ||
| 37 | 40 | ||||
| 41 | + | func TestSnapshotChunkCacheResistsWarmedOverCapacityHistoryScans(t *testing.T) { | |||
| 42 | + | history := vt.NewHistory(vt.HistoryConfig{MaxRows: 6, ChunkRows: 1}) | |||
| 43 | + | for i := range 6 { | |||
| 44 | + | require.NoError(t, history.Append([]renderer.Cell{{Rune: rune('a' + i)}})) | |||
| 45 | + | } | |||
| 46 | + | view := history.SnapshotView() | |||
| 47 | + | require.Equal(t, 6, view.ChunkCount()) | |||
| 48 | + | ||||
| 49 | + | firstPayload, err := vt.MarshalHistoryChunk(view.Chunk(0)) | |||
| 50 | + | require.NoError(t, err) | |||
| 51 | + | firstObject, err := snapcodec.MarshalObject(snapcodec.HistoryChunk, firstPayload) | |||
| 52 | + | require.NoError(t, err) | |||
| 53 | + | cache := newSnapshotChunkCache(2 * len(firstObject.Data)) | |||
| 54 | + | encodes := make(map[*vt.HistoryChunk]int) | |||
| 55 | + | cache.marshalChunk = func(chunk *vt.HistoryChunk) ([]byte, error) { | |||
| 56 | + | encodes[chunk]++ | |||
| 57 | + | return vt.MarshalHistoryChunk(chunk) | |||
| 58 | + | } | |||
| 59 | + | ||||
| 60 | + | for pass := range 3 { | |||
| 61 | + | for i := range view.ChunkCount() { | |||
| 62 | + | _, err := cache.objectLocked(view.Chunk(i)) | |||
| 63 | + | require.NoError(t, err) | |||
| 64 | + | } | |||
| 65 | + | require.LessOrEqual(t, cache.used, cache.limit) | |||
| 66 | + | if pass == 0 { | |||
| 67 | + | require.Len(t, cache.byPtr, 2) | |||
| 68 | + | } | |||
| 69 | + | } | |||
| 70 | + | ||||
| 71 | + | // The cache admits the first bounded working set then rejects later scan | |||
| 72 | + | // misses. Consequently, each warmed retained chunk is encoded only once | |||
| 73 | + | // even though the history is repeatedly scanned oldest-to-newest. | |||
| 74 | + | require.Equal(t, 1, encodes[view.Chunk(0)]) | |||
| 75 | + | require.Equal(t, 1, encodes[view.Chunk(1)]) | |||
| 76 | + | require.Equal(t, 3, encodes[view.Chunk(2)]) | |||
| 77 | + | require.Equal(t, 3, encodes[view.Chunk(5)]) | |||
| 78 | + | ||||
| 79 | + | isolated := newSnapshotChunkCache(cache.limit) | |||
| 80 | + | isolatedEncodes := 0 | |||
| 81 | + | isolated.marshalChunk = func(chunk *vt.HistoryChunk) ([]byte, error) { | |||
| 82 | + | isolatedEncodes++ | |||
| 83 | + | return vt.MarshalHistoryChunk(chunk) | |||
| 84 | + | } | |||
| 85 | + | _, err = isolated.objectLocked(view.Chunk(0)) | |||
| 86 | + | require.NoError(t, err) | |||
| 87 | + | require.Equal(t, 1, isolatedEncodes, "a session cache must not share encoded history with another session") | |||
| 88 | + | } | |||
| 89 | + | ||||
| 38 | func TestIncrementalPublicationReusesSealedChunkObject(t *testing.T) { | 90 | func TestIncrementalPublicationReusesSealedChunkObject(t *testing.T) { | ||
| 39 | history := vt.NewHistory(vt.HistoryConfig{MaxRows: 2, ChunkRows: 1}) | 91 | history := vt.NewHistory(vt.HistoryConfig{MaxRows: 2, ChunkRows: 1}) | ||
| 40 | history.Append([]renderer.Cell{renderer.BlankCell()}) | 92 | history.Append([]renderer.Cell{renderer.BlankCell()}) | ||
A internal/usecase/daemon/snapshot_restore.go +262 -0
| @@ -0,0 +1,262 @@ | |||
| 1 | + | package daemon | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "fmt" | |
| 6 | + | "math" | |
| 7 | + | ||
| 8 | + | "github.com/bnema/vev/internal/domain" | |
| 9 | + | "github.com/bnema/vev/internal/usecase/layout" | |
| 10 | + | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |
| 11 | + | "github.com/bnema/vev/pkg/vt" | |
| 12 | + | ) | |
| 13 | + | ||
| 14 | + | func (d *Daemon) closeRestoreDone() { | |
| 15 | + | d.restoreOnce.Do(func() { close(d.restoreDone) }) | |
| 16 | + | } | |
| 17 | + | ||
| 18 | + | func (d *Daemon) restoreSnapshots(ctx context.Context) { | |
| 19 | + | if d == nil { | |
| 20 | + | return | |
| 21 | + | } | |
| 22 | + | defer d.closeRestoreDone() | |
| 23 | + | if ns := d.noticeStore; ns != nil { | |
| 24 | + | claimed, err := ns.Claim() | |
| 25 | + | if err != nil { | |
| 26 | + | d.log.Warn("claiming pending notices failed", "err", err) | |
| 27 | + | } else { | |
| 28 | + | for _, n := range claimed { | |
| 29 | + | d.notices.record(n) | |
| 30 | + | d.notices.queueGlobal(n) | |
| 31 | + | } | |
| 32 | + | // Do not discard the claim until every notice has been recorded and | |
| 33 | + | // queued. An Ack failure is safe: the claim is replayed at startup. | |
| 34 | + | if err := ns.Ack(); err != nil { | |
| 35 | + | d.log.Warn("acknowledging pending notices failed", "err", err) | |
| 36 | + | } | |
| 37 | + | } | |
| 38 | + | } | |
| 39 | + | if !d.snapsEnabled { | |
| 40 | + | return | |
| 41 | + | } | |
| 42 | + | if d.snapshotRepository != nil { | |
| 43 | + | d.restoreIncrementalSnapshots(ctx) | |
| 44 | + | } | |
| 45 | + | ||
| 46 | + | } | |
| 47 | + | ||
| 48 | + | func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session) error { | |
| 49 | + | if err := ctx.Err(); err != nil { | |
| 50 | + | return err | |
| 51 | + | } | |
| 52 | + | if len(snap.Tabs) == 0 { | |
| 53 | + | if snap.Active != 0 { | |
| 54 | + | return fmt.Errorf("snapshot: active tab out of range") | |
| 55 | + | } | |
| 56 | + | } else if int(snap.Active) >= len(snap.Tabs) { | |
| 57 | + | return fmt.Errorf("snapshot: active tab out of range") | |
| 58 | + | } | |
| 59 | + | if snap.Name == "" { | |
| 60 | + | return fmt.Errorf("snapshot: empty session name") | |
| 61 | + | } | |
| 62 | + | if snap.CreatedAt > math.MaxInt64 { | |
| 63 | + | return fmt.Errorf("snapshot: created_at overflows int64") | |
| 64 | + | } | |
| 65 | + | d.mu.Lock() | |
| 66 | + | if d.closing || d.findByNameLocked(snap.Name) != nil { | |
| 67 | + | d.mu.Unlock() | |
| 68 | + | return nil | |
| 69 | + | } | |
| 70 | + | d.mu.Unlock() | |
| 71 | + | ||
| 72 | + | parent := d.serveCtx | |
| 73 | + | if parent == nil { | |
| 74 | + | parent = context.Background() | |
| 75 | + | } | |
| 76 | + | sctx, cancel := context.WithCancel(parent) | |
| 77 | + | opened := make([]*tab, 0, len(snap.Tabs)) | |
| 78 | + | // Snapshot restore runs before any client Hello is available. Start panes | |
| 79 | + | // from the daemon environment captured at startup; the next attach replaces | |
| 80 | + | // this snapshot for future panes from the client's capability. | |
| 81 | + | restoreTerm := terminalEnv{} | |
| 82 | + | restoreEnv := copyEnvironment(d.baseEnv) | |
| 83 | + | closeOpened := func() { | |
| 84 | + | for _, tb := range opened { | |
| 85 | + | tb.closeAllPanes() | |
| 86 | + | } | |
| 87 | + | cancel() | |
| 88 | + | } | |
| 89 | + | allowlist := d.restoreProcessAllowlistSnapshot() | |
| 90 | + | d.mu.Lock() | |
| 91 | + | persisted := d.stopped[snap.Name] | |
| 92 | + | d.mu.Unlock() | |
| 93 | + | for tabIndex, tabSnap := range snap.Tabs { | |
| 94 | + | if err := ctx.Err(); err != nil { | |
| 95 | + | closeOpened() | |
| 96 | + | return err | |
| 97 | + | } | |
| 98 | + | tbSize := domain.Size{Cols: int(tabSnap.Cols), Rows: int(tabSnap.Rows)} | |
| 99 | + | if !tbSize.Valid() { | |
| 100 | + | closeOpened() | |
| 101 | + | return fmt.Errorf("snapshot: invalid tab size") | |
| 102 | + | } | |
| 103 | + | placements, ok := layout.Solve(tabSnap.Tree.Root, domain.Rect{Width: tbSize.Cols, Height: tbSize.Rows}) | |
| 104 | + | if !ok { | |
| 105 | + | closeOpened() | |
| 106 | + | return fmt.Errorf("snapshot: unsolvable layout") | |
| 107 | + | } | |
| 108 | + | placementByPane := make(map[layout.PaneID]domain.Rect, len(placements)) | |
| 109 | + | for _, pl := range placements { | |
| 110 | + | placementByPane[pl.ID] = pl.Content | |
| 111 | + | } | |
| 112 | + | tabStableID := tabSnap.StableID | |
| 113 | + | if tabStableID == "" { | |
| 114 | + | var err error | |
| 115 | + | tabStableID, err = newStableID("t") | |
| 116 | + | if err != nil { | |
| 117 | + | closeOpened() | |
| 118 | + | return fmt.Errorf("snapshot: generating tab identity: %w", err) | |
| 119 | + | } | |
| 120 | + | } | |
| 121 | + | tb := &tab{stableID: tabStableID, tree: tabSnap.Tree.Clone(), panes: make(map[layout.PaneID]*pane, len(tabSnap.Panes)), nextPaneID: int(tabSnap.NextPaneID), size: tbSize} | |
| 122 | + | if tabIndex < len(persisted.tabNames) { | |
| 123 | + | tb.name = persisted.tabNames[tabIndex] | |
| 124 | + | } | |
| 125 | + | if tb.nextPaneID <= 0 { | |
| 126 | + | tb.nextPaneID = 1 | |
| 127 | + | } | |
| 128 | + | tb.ctx, tb.cancel = context.WithCancel(sctx) | |
| 129 | + | opened = append(opened, tb) | |
| 130 | + | for _, paneSnap := range tabSnap.Panes { | |
| 131 | + | restoreCommand := "" | |
| 132 | + | if decision := planProcessRestore(paneSnap.Process, allowlist); decision.Restore { | |
| 133 | + | restoreCommand = decision.Command | |
| 134 | + | } | |
| 135 | + | if err := ctx.Err(); err != nil { | |
| 136 | + | closeOpened() | |
| 137 | + | return err | |
| 138 | + | } | |
| 139 | + | contentRect, ok := placementByPane[paneSnap.ID] | |
| 140 | + | if !ok { | |
| 141 | + | closeOpened() | |
| 142 | + | return fmt.Errorf("snapshot: missing pane placement") | |
| 143 | + | } | |
| 144 | + | contentSize := restorePTYSize(contentRect, tbSize) | |
| 145 | + | paneStableID := paneSnap.StableID | |
| 146 | + | if paneStableID == "" { | |
| 147 | + | var err error | |
| 148 | + | paneStableID, err = newStableID("p") | |
| 149 | + | if err != nil { | |
| 150 | + | closeOpened() | |
| 151 | + | return fmt.Errorf("snapshot: generating pane identity: %w", err) | |
| 152 | + | } | |
| 153 | + | } | |
| 154 | + | command, args := d.ptyCommand(restoreEnv) | |
| 155 | + | pty, err := d.ptys.Open(ctx, command, args, childEnvFrom(restoreEnv, snap.Name, tabStableID, paneStableID, restoreTerm), paneSnap.Cwd, contentSize) | |
| 156 | + | if err != nil { | |
| 157 | + | closeOpened() | |
| 158 | + | return err | |
| 159 | + | } | |
| 160 | + | p := newPaneWithStableID(paneSnap.ID, paneStableID, pty, contentSize) | |
| 161 | + | p.ctx, p.cancel = context.WithCancel(tb.ctx) | |
| 162 | + | p.rect = contentRect | |
| 163 | + | if err := restorePaneTerminal(p, paneSnap); err != nil { | |
| 164 | + | // p is not in tb.panes yet, so closeOpened cannot reach it. | |
| 165 | + | p.cancel() | |
| 166 | + | _ = pty.Close() | |
| 167 | + | closeOpened() | |
| 168 | + | return err | |
| 169 | + | } | |
| 170 | + | if restoreCommand != "" { | |
| 171 | + | if _, err := pty.Write([]byte(restoreCommand + "\n")); err != nil { | |
| 172 | + | d.log.Warn("writing snapshot restore command failed", "err", err, "session", snap.Name, "pane", paneSnap.ID) | |
| 173 | + | d.NotifyGlobal(domain.NoticeWarn, domain.NoticeAutoResume, | |
| 174 | + | "couldn't restore the running program in session "+snap.Name, err) | |
| 175 | + | } | |
| 176 | + | } | |
| 177 | + | tb.panes[paneSnap.ID] = p | |
| 178 | + | } | |
| 179 | + | } | |
| 180 | + | if len(opened) == 0 { | |
| 181 | + | cancel() | |
| 182 | + | return nil | |
| 183 | + | } | |
| 184 | + | createdAt := int64(snap.CreatedAt) | |
| 185 | + | sess := &session{name: snap.Name, ctx: sctx, cancel: cancel, tabs: opened, active: int(snap.Active), terminal: restoreTerm, env: restoreEnv, createdAt: createdAt, snapshotWake: d.snapshotWake, snapshotChunkCache: newSnapshotChunkCache(snapshotChunkCacheLimit)} | |
| 186 | + | sess.snapEligible.Store(true) | |
| 187 | + | if len(snap.Tabs) > 0 && len(snap.Tabs[0].Panes) > 0 { | |
| 188 | + | sess.cwd = snap.Tabs[0].Panes[0].Cwd | |
| 189 | + | } | |
| 190 | + | d.mu.Lock() | |
| 191 | + | if stopped, ok := d.stopped[snap.Name]; ok { | |
| 192 | + | sess.mruAt.Store(stopped.lastUsedSeq) | |
| 193 | + | } | |
| 194 | + | d.mu.Unlock() | |
| 195 | + | record := sess.persistRecordLocked(createdAt) | |
| 196 | + | if err := d.persist.Save(record); err != nil { | |
| 197 | + | closeOpened() | |
| 198 | + | return err | |
| 199 | + | } | |
| 200 | + | d.mu.Lock() | |
| 201 | + | if d.closing || d.findByNameLocked(snap.Name) != nil || ctx.Err() != nil { | |
| 202 | + | d.mu.Unlock() | |
| 203 | + | closeOpened() | |
| 204 | + | return nil | |
| 205 | + | } | |
| 206 | + | id := domain.SessionID(fmt.Sprintf("sess-%d", d.nextID)) | |
| 207 | + | d.nextID++ | |
| 208 | + | sess.id = id | |
| 209 | + | delete(d.stopped, snap.Name) | |
| 210 | + | d.sessions[id] = sess | |
| 211 | + | d.mu.Unlock() | |
| 212 | + | sess.snapDirty.Store(false) | |
| 213 | + | for _, tb := range opened { | |
| 214 | + | d.startTabGoroutines(sess, tb) | |
| 215 | + | } | |
| 216 | + | return nil | |
| 217 | + | } | |
| 218 | + | ||
| 219 | + | // snapshotCoordinatorContext returns the cancellation context shared by this | |
| 220 | + | // session's queued and in-flight captures. snapshotMu serializes creation with | |
| 221 | + | ||
| 222 | + | func restorePaneTerminal(p *pane, snap snapcodec.Pane) error { | |
| 223 | + | if len(snap.Tail) == 0 { | |
| 224 | + | return fmt.Errorf("snapshot history: missing tail blob") | |
| 225 | + | } | |
| 226 | + | if len(snap.Visible) == 0 { | |
| 227 | + | return fmt.Errorf("snapshot visible: missing visible blob") | |
| 228 | + | } | |
| 229 | + | p.mu.Lock() | |
| 230 | + | defer p.mu.Unlock() | |
| 231 | + | screen, err := vt.NewScreenWithRestoredHistory(p.screen.Frame.Width, p.screen.Frame.Height, vt.HistoryConfig{MaxRows: defaultScrollbackRows, MaxCells: defaultScrollbackCells}, snap.SealedChunks, snap.Tail) | |
| 232 | + | if err != nil { | |
| 233 | + | return fmt.Errorf("snapshot history: %w", err) | |
| 234 | + | } | |
| 235 | + | if err := screen.RestorePrimaryVisible(snap.Visible); err != nil { | |
| 236 | + | return fmt.Errorf("snapshot visible: %w", err) | |
| 237 | + | } | |
| 238 | + | p.screen = screen | |
| 239 | + | p.history = screen.History() | |
| 240 | + | return nil | |
| 241 | + | } | |
| 242 | + | ||
| 243 | + | func restorePTYSize(contentRect domain.Rect, tabSize domain.Size) domain.Size { | |
| 244 | + | cols := contentRect.Width | |
| 245 | + | if cols <= 0 { | |
| 246 | + | cols = tabSize.Cols | |
| 247 | + | } | |
| 248 | + | if cols < layout.MinPaneCols { | |
| 249 | + | cols = layout.MinPaneCols | |
| 250 | + | } | |
| 251 | + | rows := contentRect.Height | |
| 252 | + | if rows <= 0 { | |
| 253 | + | rows = min(max(tabSize.Rows, layout.MinPaneRows), 24) | |
| 254 | + | } | |
| 255 | + | if rows < layout.MinPaneRows { | |
| 256 | + | rows = layout.MinPaneRows | |
| 257 | + | } | |
| 258 | + | return domain.Size{Cols: cols, Rows: rows} | |
| 259 | + | } | |
| 260 | + | ||
| 261 | + | // scheduleSnapshot captures a session only once while an immutable capture is | |
| 262 | + | // queued or being written. It never waits for the bounded worker queue. | |
| @@ -0,0 +1,262 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "fmt" | |||
| 6 | + | "math" | |||
| 7 | + | ||||
| 8 | + | "github.com/bnema/vev/internal/domain" | |||
| 9 | + | "github.com/bnema/vev/internal/usecase/layout" | |||
| 10 | + | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |||
| 11 | + | "github.com/bnema/vev/pkg/vt" | |||
| 12 | + | ) | |||
| 13 | + | ||||
| 14 | + | func (d *Daemon) closeRestoreDone() { | |||
| 15 | + | d.restoreOnce.Do(func() { close(d.restoreDone) }) | |||
| 16 | + | } | |||
| 17 | + | ||||
| 18 | + | func (d *Daemon) restoreSnapshots(ctx context.Context) { | |||
| 19 | + | if d == nil { | |||
| 20 | + | return | |||
| 21 | + | } | |||
| 22 | + | defer d.closeRestoreDone() | |||
| 23 | + | if ns := d.noticeStore; ns != nil { | |||
| 24 | + | claimed, err := ns.Claim() | |||
| 25 | + | if err != nil { | |||
| 26 | + | d.log.Warn("claiming pending notices failed", "err", err) | |||
| 27 | + | } else { | |||
| 28 | + | for _, n := range claimed { | |||
| 29 | + | d.notices.record(n) | |||
| 30 | + | d.notices.queueGlobal(n) | |||
| 31 | + | } | |||
| 32 | + | // Do not discard the claim until every notice has been recorded and | |||
| 33 | + | // queued. An Ack failure is safe: the claim is replayed at startup. | |||
| 34 | + | if err := ns.Ack(); err != nil { | |||
| 35 | + | d.log.Warn("acknowledging pending notices failed", "err", err) | |||
| 36 | + | } | |||
| 37 | + | } | |||
| 38 | + | } | |||
| 39 | + | if !d.snapsEnabled { | |||
| 40 | + | return | |||
| 41 | + | } | |||
| 42 | + | if d.snapshotRepository != nil { | |||
| 43 | + | d.restoreIncrementalSnapshots(ctx) | |||
| 44 | + | } | |||
| 45 | + | ||||
| 46 | + | } | |||
| 47 | + | ||||
| 48 | + | func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session) error { | |||
| 49 | + | if err := ctx.Err(); err != nil { | |||
| 50 | + | return err | |||
| 51 | + | } | |||
| 52 | + | if len(snap.Tabs) == 0 { | |||
| 53 | + | if snap.Active != 0 { | |||
| 54 | + | return fmt.Errorf("snapshot: active tab out of range") | |||
| 55 | + | } | |||
| 56 | + | } else if int(snap.Active) >= len(snap.Tabs) { | |||
| 57 | + | return fmt.Errorf("snapshot: active tab out of range") | |||
| 58 | + | } | |||
| 59 | + | if snap.Name == "" { | |||
| 60 | + | return fmt.Errorf("snapshot: empty session name") | |||
| 61 | + | } | |||
| 62 | + | if snap.CreatedAt > math.MaxInt64 { | |||
| 63 | + | return fmt.Errorf("snapshot: created_at overflows int64") | |||
| 64 | + | } | |||
| 65 | + | d.mu.Lock() | |||
| 66 | + | if d.closing || d.findByNameLocked(snap.Name) != nil { | |||
| 67 | + | d.mu.Unlock() | |||
| 68 | + | return nil | |||
| 69 | + | } | |||
| 70 | + | d.mu.Unlock() | |||
| 71 | + | ||||
| 72 | + | parent := d.serveCtx | |||
| 73 | + | if parent == nil { | |||
| 74 | + | parent = context.Background() | |||
| 75 | + | } | |||
| 76 | + | sctx, cancel := context.WithCancel(parent) | |||
| 77 | + | opened := make([]*tab, 0, len(snap.Tabs)) | |||
| 78 | + | // Snapshot restore runs before any client Hello is available. Start panes | |||
| 79 | + | // from the daemon environment captured at startup; the next attach replaces | |||
| 80 | + | // this snapshot for future panes from the client's capability. | |||
| 81 | + | restoreTerm := terminalEnv{} | |||
| 82 | + | restoreEnv := copyEnvironment(d.baseEnv) | |||
| 83 | + | closeOpened := func() { | |||
| 84 | + | for _, tb := range opened { | |||
| 85 | + | tb.closeAllPanes() | |||
| 86 | + | } | |||
| 87 | + | cancel() | |||
| 88 | + | } | |||
| 89 | + | allowlist := d.restoreProcessAllowlistSnapshot() | |||
| 90 | + | d.mu.Lock() | |||
| 91 | + | persisted := d.stopped[snap.Name] | |||
| 92 | + | d.mu.Unlock() | |||
| 93 | + | for tabIndex, tabSnap := range snap.Tabs { | |||
| 94 | + | if err := ctx.Err(); err != nil { | |||
| 95 | + | closeOpened() | |||
| 96 | + | return err | |||
| 97 | + | } | |||
| 98 | + | tbSize := domain.Size{Cols: int(tabSnap.Cols), Rows: int(tabSnap.Rows)} | |||
| 99 | + | if !tbSize.Valid() { | |||
| 100 | + | closeOpened() | |||
| 101 | + | return fmt.Errorf("snapshot: invalid tab size") | |||
| 102 | + | } | |||
| 103 | + | placements, ok := layout.Solve(tabSnap.Tree.Root, domain.Rect{Width: tbSize.Cols, Height: tbSize.Rows}) | |||
| 104 | + | if !ok { | |||
| 105 | + | closeOpened() | |||
| 106 | + | return fmt.Errorf("snapshot: unsolvable layout") | |||
| 107 | + | } | |||
| 108 | + | placementByPane := make(map[layout.PaneID]domain.Rect, len(placements)) | |||
| 109 | + | for _, pl := range placements { | |||
| 110 | + | placementByPane[pl.ID] = pl.Content | |||
| 111 | + | } | |||
| 112 | + | tabStableID := tabSnap.StableID | |||
| 113 | + | if tabStableID == "" { | |||
| 114 | + | var err error | |||
| 115 | + | tabStableID, err = newStableID("t") | |||
| 116 | + | if err != nil { | |||
| 117 | + | closeOpened() | |||
| 118 | + | return fmt.Errorf("snapshot: generating tab identity: %w", err) | |||
| 119 | + | } | |||
| 120 | + | } | |||
| 121 | + | tb := &tab{stableID: tabStableID, tree: tabSnap.Tree.Clone(), panes: make(map[layout.PaneID]*pane, len(tabSnap.Panes)), nextPaneID: int(tabSnap.NextPaneID), size: tbSize} | |||
| 122 | + | if tabIndex < len(persisted.tabNames) { | |||
| 123 | + | tb.name = persisted.tabNames[tabIndex] | |||
| 124 | + | } | |||
| 125 | + | if tb.nextPaneID <= 0 { | |||
| 126 | + | tb.nextPaneID = 1 | |||
| 127 | + | } | |||
| 128 | + | tb.ctx, tb.cancel = context.WithCancel(sctx) | |||
| 129 | + | opened = append(opened, tb) | |||
| 130 | + | for _, paneSnap := range tabSnap.Panes { | |||
| 131 | + | restoreCommand := "" | |||
| 132 | + | if decision := planProcessRestore(paneSnap.Process, allowlist); decision.Restore { | |||
| 133 | + | restoreCommand = decision.Command | |||
| 134 | + | } | |||
| 135 | + | if err := ctx.Err(); err != nil { | |||
| 136 | + | closeOpened() | |||
| 137 | + | return err | |||
| 138 | + | } | |||
| 139 | + | contentRect, ok := placementByPane[paneSnap.ID] | |||
| 140 | + | if !ok { | |||
| 141 | + | closeOpened() | |||
| 142 | + | return fmt.Errorf("snapshot: missing pane placement") | |||
| 143 | + | } | |||
| 144 | + | contentSize := restorePTYSize(contentRect, tbSize) | |||
| 145 | + | paneStableID := paneSnap.StableID | |||
| 146 | + | if paneStableID == "" { | |||
| 147 | + | var err error | |||
| 148 | + | paneStableID, err = newStableID("p") | |||
| 149 | + | if err != nil { | |||
| 150 | + | closeOpened() | |||
| 151 | + | return fmt.Errorf("snapshot: generating pane identity: %w", err) | |||
| 152 | + | } | |||
| 153 | + | } | |||
| 154 | + | command, args := d.ptyCommand(restoreEnv) | |||
| 155 | + | pty, err := d.ptys.Open(ctx, command, args, childEnvFrom(restoreEnv, snap.Name, tabStableID, paneStableID, restoreTerm), paneSnap.Cwd, contentSize) | |||
| 156 | + | if err != nil { | |||
| 157 | + | closeOpened() | |||
| 158 | + | return err | |||
| 159 | + | } | |||
| 160 | + | p := newPaneWithStableID(paneSnap.ID, paneStableID, pty, contentSize) | |||
| 161 | + | p.ctx, p.cancel = context.WithCancel(tb.ctx) | |||
| 162 | + | p.rect = contentRect | |||
| 163 | + | if err := restorePaneTerminal(p, paneSnap); err != nil { | |||
| 164 | + | // p is not in tb.panes yet, so closeOpened cannot reach it. | |||
| 165 | + | p.cancel() | |||
| 166 | + | _ = pty.Close() | |||
| 167 | + | closeOpened() | |||
| 168 | + | return err | |||
| 169 | + | } | |||
| 170 | + | if restoreCommand != "" { | |||
| 171 | + | if _, err := pty.Write([]byte(restoreCommand + "\n")); err != nil { | |||
| 172 | + | d.log.Warn("writing snapshot restore command failed", "err", err, "session", snap.Name, "pane", paneSnap.ID) | |||
| 173 | + | d.NotifyGlobal(domain.NoticeWarn, domain.NoticeAutoResume, | |||
| 174 | + | "couldn't restore the running program in session "+snap.Name, err) | |||
| 175 | + | } | |||
| 176 | + | } | |||
| 177 | + | tb.panes[paneSnap.ID] = p | |||
| 178 | + | } | |||
| 179 | + | } | |||
| 180 | + | if len(opened) == 0 { | |||
| 181 | + | cancel() | |||
| 182 | + | return nil | |||
| 183 | + | } | |||
| 184 | + | createdAt := int64(snap.CreatedAt) | |||
| 185 | + | sess := &session{name: snap.Name, ctx: sctx, cancel: cancel, tabs: opened, active: int(snap.Active), terminal: restoreTerm, env: restoreEnv, createdAt: createdAt, snapshotWake: d.snapshotWake, snapshotChunkCache: newSnapshotChunkCache(snapshotChunkCacheLimit)} | |||
| 186 | + | sess.snapEligible.Store(true) | |||
| 187 | + | if len(snap.Tabs) > 0 && len(snap.Tabs[0].Panes) > 0 { | |||
| 188 | + | sess.cwd = snap.Tabs[0].Panes[0].Cwd | |||
| 189 | + | } | |||
| 190 | + | d.mu.Lock() | |||
| 191 | + | if stopped, ok := d.stopped[snap.Name]; ok { | |||
| 192 | + | sess.mruAt.Store(stopped.lastUsedSeq) | |||
| 193 | + | } | |||
| 194 | + | d.mu.Unlock() | |||
| 195 | + | record := sess.persistRecordLocked(createdAt) | |||
| 196 | + | if err := d.persist.Save(record); err != nil { | |||
| 197 | + | closeOpened() | |||
| 198 | + | return err | |||
| 199 | + | } | |||
| 200 | + | d.mu.Lock() | |||
| 201 | + | if d.closing || d.findByNameLocked(snap.Name) != nil || ctx.Err() != nil { | |||
| 202 | + | d.mu.Unlock() | |||
| 203 | + | closeOpened() | |||
| 204 | + | return nil | |||
| 205 | + | } | |||
| 206 | + | id := domain.SessionID(fmt.Sprintf("sess-%d", d.nextID)) | |||
| 207 | + | d.nextID++ | |||
| 208 | + | sess.id = id | |||
| 209 | + | delete(d.stopped, snap.Name) | |||
| 210 | + | d.sessions[id] = sess | |||
| 211 | + | d.mu.Unlock() | |||
| 212 | + | sess.snapDirty.Store(false) | |||
| 213 | + | for _, tb := range opened { | |||
| 214 | + | d.startTabGoroutines(sess, tb) | |||
| 215 | + | } | |||
| 216 | + | return nil | |||
| 217 | + | } | |||
| 218 | + | ||||
| 219 | + | // snapshotCoordinatorContext returns the cancellation context shared by this | |||
| 220 | + | // session's queued and in-flight captures. snapshotMu serializes creation with | |||
| 221 | + | ||||
| 222 | + | func restorePaneTerminal(p *pane, snap snapcodec.Pane) error { | |||
| 223 | + | if len(snap.Tail) == 0 { | |||
| 224 | + | return fmt.Errorf("snapshot history: missing tail blob") | |||
| 225 | + | } | |||
| 226 | + | if len(snap.Visible) == 0 { | |||
| 227 | + | return fmt.Errorf("snapshot visible: missing visible blob") | |||
| 228 | + | } | |||
| 229 | + | p.mu.Lock() | |||
| 230 | + | defer p.mu.Unlock() | |||
| 231 | + | screen, err := vt.NewScreenWithRestoredHistory(p.screen.Frame.Width, p.screen.Frame.Height, vt.HistoryConfig{MaxRows: defaultScrollbackRows, MaxCells: defaultScrollbackCells}, snap.SealedChunks, snap.Tail) | |||
| 232 | + | if err != nil { | |||
| 233 | + | return fmt.Errorf("snapshot history: %w", err) | |||
| 234 | + | } | |||
| 235 | + | if err := screen.RestorePrimaryVisible(snap.Visible); err != nil { | |||
| 236 | + | return fmt.Errorf("snapshot visible: %w", err) | |||
| 237 | + | } | |||
| 238 | + | p.screen = screen | |||
| 239 | + | p.history = screen.History() | |||
| 240 | + | return nil | |||
| 241 | + | } | |||
| 242 | + | ||||
| 243 | + | func restorePTYSize(contentRect domain.Rect, tabSize domain.Size) domain.Size { | |||
| 244 | + | cols := contentRect.Width | |||
| 245 | + | if cols <= 0 { | |||
| 246 | + | cols = tabSize.Cols | |||
| 247 | + | } | |||
| 248 | + | if cols < layout.MinPaneCols { | |||
| 249 | + | cols = layout.MinPaneCols | |||
| 250 | + | } | |||
| 251 | + | rows := contentRect.Height | |||
| 252 | + | if rows <= 0 { | |||
| 253 | + | rows = min(max(tabSize.Rows, layout.MinPaneRows), 24) | |||
| 254 | + | } | |||
| 255 | + | if rows < layout.MinPaneRows { | |||
| 256 | + | rows = layout.MinPaneRows | |||
| 257 | + | } | |||
| 258 | + | return domain.Size{Cols: cols, Rows: rows} | |||
| 259 | + | } | |||
| 260 | + | ||||
| 261 | + | // scheduleSnapshot captures a session only once while an immutable capture is | |||
| 262 | + | // queued or being written. It never waits for the bounded worker queue. | |||
M internal/usecase/daemon/snapshot_restore_test.go +4 -2
| @@ -83,8 +83,10 @@ func (r *snapshotAcceptanceRepository) Load(ctx context.Context, name string) (p | |||
| 83 | 83 | return cloneAcceptanceGeneration(generation), nil | |
| 84 | 84 | } | |
| 85 | 85 | ||
| 86 | - | func (*snapshotAcceptanceRepository) Delete(context.Context, string) error { return nil } | |
| 87 | - | func (*snapshotAcceptanceRepository) Maintain(context.Context) error { return nil } | |
| 86 | + | func (*snapshotAcceptanceRepository) Delete(context.Context, string) error { return nil } | |
| 87 | + | func (*snapshotAcceptanceRepository) Tombstone(context.Context, string) error { return nil } | |
| 88 | + | func (*snapshotAcceptanceRepository) DeleteTombstone(context.Context, string) error { return nil } | |
| 89 | + | func (*snapshotAcceptanceRepository) Maintain(context.Context) error { return nil } | |
| 88 | 90 | ||
| 89 | 91 | func cloneAcceptanceGeneration(g ports.SnapshotGeneration) ports.SnapshotGeneration { | |
| 90 | 92 | out := g | |
| @@ -83,8 +83,10 @@ func (r *snapshotAcceptanceRepository) Load(ctx context.Context, name string) (p | |||||
| 83 | return cloneAcceptanceGeneration(generation), nil | 83 | return cloneAcceptanceGeneration(generation), nil | ||
| 84 | } | 84 | } | ||
| 85 | 85 | ||||
| 86 | - | func (*snapshotAcceptanceRepository) Delete(context.Context, string) error { return nil } | 86 | + | func (*snapshotAcceptanceRepository) Delete(context.Context, string) error { return nil } |
| 87 | - | func (*snapshotAcceptanceRepository) Maintain(context.Context) error { return nil } | 87 | + | func (*snapshotAcceptanceRepository) Tombstone(context.Context, string) error { return nil } |
| 88 | + | func (*snapshotAcceptanceRepository) DeleteTombstone(context.Context, string) error { return nil } | |||
| 89 | + | func (*snapshotAcceptanceRepository) Maintain(context.Context) error { return nil } | |||
| 88 | 90 | ||||
| 89 | func cloneAcceptanceGeneration(g ports.SnapshotGeneration) ports.SnapshotGeneration { | 91 | func cloneAcceptanceGeneration(g ports.SnapshotGeneration) ports.SnapshotGeneration { | ||
| 90 | out := g | 92 | out := g | ||
A internal/usecase/daemon/snapshot_scheduler.go +68 -0
| @@ -0,0 +1,68 @@ | |||
| 1 | + | package daemon | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "time" | |
| 6 | + | ) | |
| 7 | + | ||
| 8 | + | func (d *Daemon) snapshotRepositorySaver(ctx context.Context) { | |
| 9 | + | if d == nil { | |
| 10 | + | return | |
| 11 | + | } | |
| 12 | + | timer := d.clock.NewTimer(24 * time.Hour) | |
| 13 | + | defer timer.Stop() | |
| 14 | + | for { | |
| 15 | + | delay := d.scheduleEligibleRepositorySnapshots() | |
| 16 | + | timer.Reset(delay) | |
| 17 | + | select { | |
| 18 | + | case <-ctx.Done(): | |
| 19 | + | return | |
| 20 | + | case <-d.snapshotWake: | |
| 21 | + | case <-timer.C(): | |
| 22 | + | } | |
| 23 | + | } | |
| 24 | + | } | |
| 25 | + | ||
| 26 | + | func (d *Daemon) scheduleEligibleRepositorySnapshots() time.Duration { | |
| 27 | + | d.mu.Lock() | |
| 28 | + | sessions := d.sessionsSnapshotLocked() | |
| 29 | + | d.mu.Unlock() | |
| 30 | + | now := d.clock.Now() | |
| 31 | + | for _, sess := range sessions { | |
| 32 | + | if !sess.snapEligible.Load() || !sess.snapDirty.Load() { | |
| 33 | + | continue | |
| 34 | + | } | |
| 35 | + | sess.snapshotMu.Lock() | |
| 36 | + | due := !sess.snapshotPending && (sess.snapshotNextEligibleAt.IsZero() || !now.Before(sess.snapshotNextEligibleAt)) | |
| 37 | + | sess.snapshotMu.Unlock() | |
| 38 | + | if due { | |
| 39 | + | d.scheduleSnapshot(sess) | |
| 40 | + | } | |
| 41 | + | } | |
| 42 | + | ||
| 43 | + | // Scheduling can synchronously fail or mark a capture pending, so inspect | |
| 44 | + | // the final state to select the actual earliest timer deadline. | |
| 45 | + | var next time.Time | |
| 46 | + | for _, sess := range sessions { | |
| 47 | + | if !sess.snapEligible.Load() || !sess.snapDirty.Load() { | |
| 48 | + | continue | |
| 49 | + | } | |
| 50 | + | sess.snapshotMu.Lock() | |
| 51 | + | eligibleAt := sess.snapshotNextEligibleAt | |
| 52 | + | pending := sess.snapshotPending | |
| 53 | + | sess.snapshotMu.Unlock() | |
| 54 | + | if pending || eligibleAt.IsZero() { | |
| 55 | + | continue | |
| 56 | + | } | |
| 57 | + | if next.IsZero() || eligibleAt.Before(next) { | |
| 58 | + | next = eligibleAt | |
| 59 | + | } | |
| 60 | + | } | |
| 61 | + | if next.IsZero() { | |
| 62 | + | return 24 * time.Hour | |
| 63 | + | } | |
| 64 | + | if delay := next.Sub(now); delay > 0 { | |
| 65 | + | return delay | |
| 66 | + | } | |
| 67 | + | return 0 | |
| 68 | + | } | |
| @@ -0,0 +1,68 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "time" | |||
| 6 | + | ) | |||
| 7 | + | ||||
| 8 | + | func (d *Daemon) snapshotRepositorySaver(ctx context.Context) { | |||
| 9 | + | if d == nil { | |||
| 10 | + | return | |||
| 11 | + | } | |||
| 12 | + | timer := d.clock.NewTimer(24 * time.Hour) | |||
| 13 | + | defer timer.Stop() | |||
| 14 | + | for { | |||
| 15 | + | delay := d.scheduleEligibleRepositorySnapshots() | |||
| 16 | + | timer.Reset(delay) | |||
| 17 | + | select { | |||
| 18 | + | case <-ctx.Done(): | |||
| 19 | + | return | |||
| 20 | + | case <-d.snapshotWake: | |||
| 21 | + | case <-timer.C(): | |||
| 22 | + | } | |||
| 23 | + | } | |||
| 24 | + | } | |||
| 25 | + | ||||
| 26 | + | func (d *Daemon) scheduleEligibleRepositorySnapshots() time.Duration { | |||
| 27 | + | d.mu.Lock() | |||
| 28 | + | sessions := d.sessionsSnapshotLocked() | |||
| 29 | + | d.mu.Unlock() | |||
| 30 | + | now := d.clock.Now() | |||
| 31 | + | for _, sess := range sessions { | |||
| 32 | + | if !sess.snapEligible.Load() || !sess.snapDirty.Load() { | |||
| 33 | + | continue | |||
| 34 | + | } | |||
| 35 | + | sess.snapshotMu.Lock() | |||
| 36 | + | due := !sess.snapshotPending && (sess.snapshotNextEligibleAt.IsZero() || !now.Before(sess.snapshotNextEligibleAt)) | |||
| 37 | + | sess.snapshotMu.Unlock() | |||
| 38 | + | if due { | |||
| 39 | + | d.scheduleSnapshot(sess) | |||
| 40 | + | } | |||
| 41 | + | } | |||
| 42 | + | ||||
| 43 | + | // Scheduling can synchronously fail or mark a capture pending, so inspect | |||
| 44 | + | // the final state to select the actual earliest timer deadline. | |||
| 45 | + | var next time.Time | |||
| 46 | + | for _, sess := range sessions { | |||
| 47 | + | if !sess.snapEligible.Load() || !sess.snapDirty.Load() { | |||
| 48 | + | continue | |||
| 49 | + | } | |||
| 50 | + | sess.snapshotMu.Lock() | |||
| 51 | + | eligibleAt := sess.snapshotNextEligibleAt | |||
| 52 | + | pending := sess.snapshotPending | |||
| 53 | + | sess.snapshotMu.Unlock() | |||
| 54 | + | if pending || eligibleAt.IsZero() { | |||
| 55 | + | continue | |||
| 56 | + | } | |||
| 57 | + | if next.IsZero() || eligibleAt.Before(next) { | |||
| 58 | + | next = eligibleAt | |||
| 59 | + | } | |||
| 60 | + | } | |||
| 61 | + | if next.IsZero() { | |||
| 62 | + | return 24 * time.Hour | |||
| 63 | + | } | |||
| 64 | + | if delay := next.Sub(now); delay > 0 { | |||
| 65 | + | return delay | |||
| 66 | + | } | |||
| 67 | + | return 0 | |||
| 68 | + | } | |||
M internal/usecase/daemon/snapshot_scheduler_test.go +3 -0
| @@ -82,11 +82,13 @@ func TestRenameKeepsSnapshotCoordinatorQuarantinedUntilNewIdentityCommits(t *tes | |||
| 82 | 82 | deleteEntered := make(chan struct{}) | |
| 83 | 83 | allowDelete := make(chan struct{}) | |
| 84 | 84 | published := make(chan ports.SnapshotPublication, 2) | |
| 85 | + | repository.EXPECT().Tombstone(mock.Anything, "work").Return(nil).Once() | |
| 85 | 86 | repository.EXPECT().Delete(mock.Anything, "work").RunAndReturn(func(context.Context, string) error { | |
| 86 | 87 | close(deleteEntered) | |
| 87 | 88 | <-allowDelete | |
| 88 | 89 | return nil | |
| 89 | 90 | }).Once() | |
| 91 | + | repository.EXPECT().DeleteTombstone(mock.Anything, "work").Return(nil).Once() | |
| 90 | 92 | repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error { | |
| 91 | 93 | published <- publication | |
| 92 | 94 | return nil | |
| @@ -159,6 +161,7 @@ func TestRenameRollbackAfterOldDeleteLeavesOldCoordinatorStopped(t *testing.T) { | |||
| 159 | 161 | return nil | |
| 160 | 162 | } | |
| 161 | 163 | state.mu.Unlock() | |
| 164 | + | repository.EXPECT().Tombstone(mock.Anything, "work").Return(nil).Once() | |
| 162 | 165 | repository.EXPECT().Delete(mock.Anything, "work").Return(nil).Once() | |
| 163 | 166 | repository.EXPECT().Publish(mock.Anything, mock.Anything).Run(func(context.Context, ports.SnapshotPublication) { | |
| 164 | 167 | t.Fatal("rollback resurrected the deleted old snapshot identity") | |
| @@ -82,11 +82,13 @@ func TestRenameKeepsSnapshotCoordinatorQuarantinedUntilNewIdentityCommits(t *tes | |||||
| 82 | deleteEntered := make(chan struct{}) | 82 | deleteEntered := make(chan struct{}) | ||
| 83 | allowDelete := make(chan struct{}) | 83 | allowDelete := make(chan struct{}) | ||
| 84 | published := make(chan ports.SnapshotPublication, 2) | 84 | published := make(chan ports.SnapshotPublication, 2) | ||
| 85 | + | repository.EXPECT().Tombstone(mock.Anything, "work").Return(nil).Once() | |||
| 85 | repository.EXPECT().Delete(mock.Anything, "work").RunAndReturn(func(context.Context, string) error { | 86 | repository.EXPECT().Delete(mock.Anything, "work").RunAndReturn(func(context.Context, string) error { | ||
| 86 | close(deleteEntered) | 87 | close(deleteEntered) | ||
| 87 | <-allowDelete | 88 | <-allowDelete | ||
| 88 | return nil | 89 | return nil | ||
| 89 | }).Once() | 90 | }).Once() | ||
| 91 | + | repository.EXPECT().DeleteTombstone(mock.Anything, "work").Return(nil).Once() | |||
| 90 | repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error { | 92 | repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error { | ||
| 91 | published <- publication | 93 | published <- publication | ||
| 92 | return nil | 94 | return nil | ||
| @@ -159,6 +161,7 @@ func TestRenameRollbackAfterOldDeleteLeavesOldCoordinatorStopped(t *testing.T) { | |||||
| 159 | return nil | 161 | return nil | ||
| 160 | } | 162 | } | ||
| 161 | state.mu.Unlock() | 163 | state.mu.Unlock() | ||
| 164 | + | repository.EXPECT().Tombstone(mock.Anything, "work").Return(nil).Once() | |||
| 162 | repository.EXPECT().Delete(mock.Anything, "work").Return(nil).Once() | 165 | repository.EXPECT().Delete(mock.Anything, "work").Return(nil).Once() | ||
| 163 | repository.EXPECT().Publish(mock.Anything, mock.Anything).Run(func(context.Context, ports.SnapshotPublication) { | 166 | repository.EXPECT().Publish(mock.Anything, mock.Anything).Run(func(context.Context, ports.SnapshotPublication) { | ||
| 164 | t.Fatal("rollback resurrected the deleted old snapshot identity") | 167 | t.Fatal("rollback resurrected the deleted old snapshot identity") | ||
A internal/usecase/daemon/snapshot_worker.go +261 -0
| @@ -0,0 +1,261 @@ | |||
| 1 | + | package daemon | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | ||
| 6 | + | "github.com/bnema/vev/internal/domain" | |
| 7 | + | ) | |
| 8 | + | ||
| 9 | + | func (d *Daemon) reportSnapshotFailure(capture *snapshotCapture, phase string, cause error) { | |
| 10 | + | if d == nil || capture == nil || capture.session == nil || cause == nil { | |
| 11 | + | return | |
| 12 | + | } | |
| 13 | + | signature := snapshotFailureSignature(phase, cause) | |
| 14 | + | capture.session.snapshotMu.Lock() | |
| 15 | + | capture.session.snapshotFailureSig = signature | |
| 16 | + | capture.session.snapshotMu.Unlock() | |
| 17 | + | ||
| 18 | + | d.snapshotNoticeMu.Lock() | |
| 19 | + | changed := d.snapshotActiveFailureSignature != signature | |
| 20 | + | d.snapshotActiveFailureSignature = signature | |
| 21 | + | d.snapshotNoticeMu.Unlock() | |
| 22 | + | ||
| 23 | + | n := domain.Notification{ | |
| 24 | + | Code: domain.NoticeSnapshotWrite, | |
| 25 | + | Severity: domain.NoticeError, | |
| 26 | + | Message: "couldn't save session state; recent state may be lost on restart", | |
| 27 | + | // The history is intentionally diagnostic only at the error-class level. | |
| 28 | + | // Raw causes can include user paths and terminal content. | |
| 29 | + | Details: signature, | |
| 30 | + | Time: d.clock.Now(), | |
| 31 | + | } | |
| 32 | + | n, _ = d.notices.recordSnapshotFailure(n) | |
| 33 | + | d.log.Warn("writing session snapshot failed", "phase", phase, "class", signature, "session", capture.name) | |
| 34 | + | if changed { | |
| 35 | + | d.deliverGlobal(n) | |
| 36 | + | } | |
| 37 | + | } | |
| 38 | + | ||
| 39 | + | func (d *Daemon) clearSnapshotFailure() { | |
| 40 | + | if d == nil { | |
| 41 | + | return | |
| 42 | + | } | |
| 43 | + | d.snapshotNoticeMu.Lock() | |
| 44 | + | d.snapshotActiveFailureSignature = "" | |
| 45 | + | d.snapshotNoticeMu.Unlock() | |
| 46 | + | } | |
| 47 | + | ||
| 48 | + | func (d *Daemon) startSnapshotEncodeWorker() { | |
| 49 | + | if d == nil { | |
| 50 | + | return | |
| 51 | + | } | |
| 52 | + | d.snapshotWorkerMu.Lock() | |
| 53 | + | if d.snapshotWorkerCancel != nil { | |
| 54 | + | d.snapshotWorkerMu.Unlock() | |
| 55 | + | return | |
| 56 | + | } | |
| 57 | + | // Shutdown owns the worker lifetime so it can flush captures after Serve | |
| 58 | + | // cancels its parent context. The worker is always stopped explicitly. | |
| 59 | + | workerCtx, cancel := context.WithCancel(context.Background()) | |
| 60 | + | d.snapshotWorkerID++ | |
| 61 | + | workerID := d.snapshotWorkerID | |
| 62 | + | d.snapshotWorkerCtx = workerCtx | |
| 63 | + | d.snapshotWorkerCancel = cancel | |
| 64 | + | d.snapshotWorkerDone = make(chan struct{}) | |
| 65 | + | d.snapshotWorkerFlush = make(chan struct{}) | |
| 66 | + | d.snapshotWorkerFinalWake = make(chan struct{}, 1) | |
| 67 | + | d.snapshotFinalJobs = make(map[*session]*snapshotCapture) | |
| 68 | + | d.snapshotFinalOrder = nil | |
| 69 | + | d.snapshotWorkerClosing = false | |
| 70 | + | d.snapshotWorkerInFlight = nil | |
| 71 | + | done := d.snapshotWorkerDone | |
| 72 | + | flush := d.snapshotWorkerFlush | |
| 73 | + | finalWake := d.snapshotWorkerFinalWake | |
| 74 | + | d.snapshotWorkerMu.Unlock() | |
| 75 | + | go d.runSnapshotEncodeWorker(workerCtx, workerID, done, flush, finalWake) | |
| 76 | + | } | |
| 77 | + | ||
| 78 | + | // runSnapshotEncodeWorker owns the worker state machine. Each event either | |
| 79 | + | // continues processing, drains the terminal queues for shutdown, or stops; | |
| 80 | + | // publication details are kept out of the select loop. | |
| 81 | + | func (d *Daemon) runSnapshotEncodeWorker(workerCtx context.Context, workerID uint64, done chan<- struct{}, flush, finalWake <-chan struct{}) { | |
| 82 | + | defer close(done) | |
| 83 | + | for { | |
| 84 | + | select { | |
| 85 | + | case <-workerCtx.Done(): | |
| 86 | + | return | |
| 87 | + | case <-flush: | |
| 88 | + | d.flushSnapshotCaptures(workerCtx, workerID) | |
| 89 | + | return | |
| 90 | + | case <-finalWake: | |
| 91 | + | if !d.drainFinalSnapshotCaptures(workerCtx, workerID) { | |
| 92 | + | return | |
| 93 | + | } | |
| 94 | + | case capture := <-d.snapshotJobs: | |
| 95 | + | if !d.publishSnapshotCapture(workerCtx, workerID, capture) { | |
| 96 | + | return | |
| 97 | + | } | |
| 98 | + | } | |
| 99 | + | } | |
| 100 | + | } | |
| 101 | + | ||
| 102 | + | func (d *Daemon) flushSnapshotCaptures(workerCtx context.Context, workerID uint64) { | |
| 103 | + | for { | |
| 104 | + | select { | |
| 105 | + | case capture := <-d.snapshotJobs: | |
| 106 | + | if !d.publishSnapshotCapture(workerCtx, workerID, capture) { | |
| 107 | + | return | |
| 108 | + | } | |
| 109 | + | default: | |
| 110 | + | _ = d.drainFinalSnapshotCaptures(workerCtx, workerID) | |
| 111 | + | return | |
| 112 | + | } | |
| 113 | + | } | |
| 114 | + | } | |
| 115 | + | ||
| 116 | + | func (d *Daemon) drainFinalSnapshotCaptures(workerCtx context.Context, workerID uint64) bool { | |
| 117 | + | for capture := d.takeFinalSnapshotCapture(); capture != nil; capture = d.takeFinalSnapshotCapture() { | |
| 118 | + | if !d.publishSnapshotCapture(workerCtx, workerID, capture) { | |
| 119 | + | return false | |
| 120 | + | } | |
| 121 | + | } | |
| 122 | + | return true | |
| 123 | + | } | |
| 124 | + | ||
| 125 | + | func (d *Daemon) publishSnapshotCapture(workerCtx context.Context, workerID uint64, capture *snapshotCapture) bool { | |
| 126 | + | if workerCtx.Err() != nil || !startSnapshotPublication(capture) || !d.setSnapshotWorkerInFlight(workerID, capture) { | |
| 127 | + | d.finishSnapshotCapture(capture, false) | |
| 128 | + | return workerCtx.Err() == nil | |
| 129 | + | } | |
| 130 | + | publication, err := d.incrementalPublication(capture) | |
| 131 | + | publicationContext := capture.publicationContext | |
| 132 | + | if publicationContext == nil { | |
| 133 | + | publicationContext = workerCtx | |
| 134 | + | } | |
| 135 | + | if err == nil && workerCtx.Err() == nil { | |
| 136 | + | if err = publicationContext.Err(); err == nil { | |
| 137 | + | err = d.snapshotRepository.Publish(publicationContext, publication) | |
| 138 | + | } | |
| 139 | + | } | |
| 140 | + | d.clearSnapshotWorkerInFlight(workerID, capture) | |
| 141 | + | if err != nil && workerCtx.Err() == nil && publicationContext.Err() == nil { | |
| 142 | + | // Global, not session-scoped: by now the session may already be torn | |
| 143 | + | // down, so a session notice would be dead-on-arrival. No lock is held | |
| 144 | + | // here (clearSnapshotWorkerInFlight released its own). | |
| 145 | + | d.reportSnapshotFailure(capture, "publish", err) | |
| 146 | + | } | |
| 147 | + | d.finishSnapshotCapture(capture, err == nil && workerCtx.Err() == nil) | |
| 148 | + | return workerCtx.Err() == nil | |
| 149 | + | } | |
| 150 | + | ||
| 151 | + | func (d *Daemon) takeFinalSnapshotCapture() *snapshotCapture { | |
| 152 | + | d.snapshotWorkerMu.Lock() | |
| 153 | + | defer d.snapshotWorkerMu.Unlock() | |
| 154 | + | for len(d.snapshotFinalOrder) > 0 { | |
| 155 | + | sess := d.snapshotFinalOrder[0] | |
| 156 | + | d.snapshotFinalOrder[0] = nil | |
| 157 | + | d.snapshotFinalOrder = d.snapshotFinalOrder[1:] | |
| 158 | + | capture := d.snapshotFinalJobs[sess] | |
| 159 | + | delete(d.snapshotFinalJobs, sess) | |
| 160 | + | if capture != nil { | |
| 161 | + | return capture | |
| 162 | + | } | |
| 163 | + | } | |
| 164 | + | return nil | |
| 165 | + | } | |
| 166 | + | ||
| 167 | + | func (d *Daemon) setSnapshotWorkerInFlight(workerID uint64, capture *snapshotCapture) bool { | |
| 168 | + | d.snapshotWorkerMu.Lock() | |
| 169 | + | defer d.snapshotWorkerMu.Unlock() | |
| 170 | + | if d.snapshotWorkerID != workerID || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |
| 171 | + | return false | |
| 172 | + | } | |
| 173 | + | d.snapshotWorkerInFlight = capture | |
| 174 | + | return true | |
| 175 | + | } | |
| 176 | + | ||
| 177 | + | func (d *Daemon) clearSnapshotWorkerInFlight(workerID uint64, capture *snapshotCapture) { | |
| 178 | + | d.snapshotWorkerMu.Lock() | |
| 179 | + | defer d.snapshotWorkerMu.Unlock() | |
| 180 | + | if d.snapshotWorkerID == workerID && d.snapshotWorkerInFlight == capture { | |
| 181 | + | d.snapshotWorkerInFlight = nil | |
| 182 | + | } | |
| 183 | + | } | |
| 184 | + | ||
| 185 | + | func (d *Daemon) stopSnapshotEncodeWorker() { | |
| 186 | + | if d == nil { | |
| 187 | + | return | |
| 188 | + | } | |
| 189 | + | deadline := newSnapshotShutdownDeadline(d.clock) | |
| 190 | + | defer deadline.stop() | |
| 191 | + | d.stopSnapshotEncodeWorkerWithDeadline(deadline) | |
| 192 | + | } | |
| 193 | + | ||
| 194 | + | // stopSnapshotEncodeWorkerWithDeadline performs the one deadline-aware join | |
| 195 | + | // for a Serve shutdown. Queues are never closed, and detached writers only | |
| 196 | + | // retain immutable captures plus contexts/channels that remain live, so an | |
| 197 | + | // uncooperative repository call cannot race teardown or block process exit. | |
| 198 | + | func (d *Daemon) stopSnapshotEncodeWorkerWithDeadline(deadline *snapshotShutdownDeadline) { | |
| 199 | + | if d == nil { | |
| 200 | + | return | |
| 201 | + | } | |
| 202 | + | d.snapshotWorkerMu.Lock() | |
| 203 | + | cancel := d.snapshotWorkerCancel | |
| 204 | + | if cancel == nil || d.snapshotWorkerClosing { | |
| 205 | + | d.snapshotWorkerMu.Unlock() | |
| 206 | + | return | |
| 207 | + | } | |
| 208 | + | // Stop accepting producers, then let the single worker drain its bounded | |
| 209 | + | // queue. A non-context-aware store may still block indefinitely, so the | |
| 210 | + | // shared shutdown deadline bounds the one and only join. | |
| 211 | + | d.snapshotWorkerClosing = true | |
| 212 | + | flush := d.snapshotWorkerFlush | |
| 213 | + | done := d.snapshotWorkerDone | |
| 214 | + | close(flush) | |
| 215 | + | d.snapshotWorkerMu.Unlock() | |
| 216 | + | ||
| 217 | + | if deadline == nil { | |
| 218 | + | deadline = newSnapshotShutdownDeadline(d.clock) | |
| 219 | + | defer deadline.stop() | |
| 220 | + | } | |
| 221 | + | select { | |
| 222 | + | case <-done: | |
| 223 | + | d.finishStoppedSnapshotWorker(false) | |
| 224 | + | case <-deadline.Done(): | |
| 225 | + | cancel() | |
| 226 | + | d.finishStoppedSnapshotWorker(true) | |
| 227 | + | } | |
| 228 | + | } | |
| 229 | + | ||
| 230 | + | func (d *Daemon) finishStoppedSnapshotWorker(abandoned bool) { | |
| 231 | + | d.snapshotWorkerMu.Lock() | |
| 232 | + | inFlight := d.snapshotWorkerInFlight | |
| 233 | + | d.snapshotWorkerCtx = nil | |
| 234 | + | d.snapshotWorkerCancel = nil | |
| 235 | + | d.snapshotWorkerDone = nil | |
| 236 | + | d.snapshotWorkerFlush = nil | |
| 237 | + | d.snapshotWorkerFinalWake = nil | |
| 238 | + | d.snapshotWorkerClosing = false | |
| 239 | + | d.snapshotWorkerInFlight = nil | |
| 240 | + | queued := make([]*snapshotCapture, 0, len(d.snapshotJobs)+len(d.snapshotFinalJobs)) | |
| 241 | + | for _, capture := range d.snapshotFinalJobs { | |
| 242 | + | queued = append(queued, capture) | |
| 243 | + | } | |
| 244 | + | d.snapshotFinalJobs = nil | |
| 245 | + | d.snapshotFinalOrder = nil | |
| 246 | + | for { | |
| 247 | + | select { | |
| 248 | + | case capture := <-d.snapshotJobs: | |
| 249 | + | queued = append(queued, capture) | |
| 250 | + | default: | |
| 251 | + | d.snapshotWorkerMu.Unlock() | |
| 252 | + | if abandoned { | |
| 253 | + | d.finishSnapshotCapture(inFlight, false) | |
| 254 | + | } | |
| 255 | + | for _, capture := range queued { | |
| 256 | + | d.finishSnapshotCapture(capture, false) | |
| 257 | + | } | |
| 258 | + | return | |
| 259 | + | } | |
| 260 | + | } | |
| 261 | + | } | |
| @@ -0,0 +1,261 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | ||||
| 6 | + | "github.com/bnema/vev/internal/domain" | |||
| 7 | + | ) | |||
| 8 | + | ||||
| 9 | + | func (d *Daemon) reportSnapshotFailure(capture *snapshotCapture, phase string, cause error) { | |||
| 10 | + | if d == nil || capture == nil || capture.session == nil || cause == nil { | |||
| 11 | + | return | |||
| 12 | + | } | |||
| 13 | + | signature := snapshotFailureSignature(phase, cause) | |||
| 14 | + | capture.session.snapshotMu.Lock() | |||
| 15 | + | capture.session.snapshotFailureSig = signature | |||
| 16 | + | capture.session.snapshotMu.Unlock() | |||
| 17 | + | ||||
| 18 | + | d.snapshotNoticeMu.Lock() | |||
| 19 | + | changed := d.snapshotActiveFailureSignature != signature | |||
| 20 | + | d.snapshotActiveFailureSignature = signature | |||
| 21 | + | d.snapshotNoticeMu.Unlock() | |||
| 22 | + | ||||
| 23 | + | n := domain.Notification{ | |||
| 24 | + | Code: domain.NoticeSnapshotWrite, | |||
| 25 | + | Severity: domain.NoticeError, | |||
| 26 | + | Message: "couldn't save session state; recent state may be lost on restart", | |||
| 27 | + | // The history is intentionally diagnostic only at the error-class level. | |||
| 28 | + | // Raw causes can include user paths and terminal content. | |||
| 29 | + | Details: signature, | |||
| 30 | + | Time: d.clock.Now(), | |||
| 31 | + | } | |||
| 32 | + | n, _ = d.notices.recordSnapshotFailure(n) | |||
| 33 | + | d.log.Warn("writing session snapshot failed", "phase", phase, "class", signature, "session", capture.name) | |||
| 34 | + | if changed { | |||
| 35 | + | d.deliverGlobal(n) | |||
| 36 | + | } | |||
| 37 | + | } | |||
| 38 | + | ||||
| 39 | + | func (d *Daemon) clearSnapshotFailure() { | |||
| 40 | + | if d == nil { | |||
| 41 | + | return | |||
| 42 | + | } | |||
| 43 | + | d.snapshotNoticeMu.Lock() | |||
| 44 | + | d.snapshotActiveFailureSignature = "" | |||
| 45 | + | d.snapshotNoticeMu.Unlock() | |||
| 46 | + | } | |||
| 47 | + | ||||
| 48 | + | func (d *Daemon) startSnapshotEncodeWorker() { | |||
| 49 | + | if d == nil { | |||
| 50 | + | return | |||
| 51 | + | } | |||
| 52 | + | d.snapshotWorkerMu.Lock() | |||
| 53 | + | if d.snapshotWorkerCancel != nil { | |||
| 54 | + | d.snapshotWorkerMu.Unlock() | |||
| 55 | + | return | |||
| 56 | + | } | |||
| 57 | + | // Shutdown owns the worker lifetime so it can flush captures after Serve | |||
| 58 | + | // cancels its parent context. The worker is always stopped explicitly. | |||
| 59 | + | workerCtx, cancel := context.WithCancel(context.Background()) | |||
| 60 | + | d.snapshotWorkerID++ | |||
| 61 | + | workerID := d.snapshotWorkerID | |||
| 62 | + | d.snapshotWorkerCtx = workerCtx | |||
| 63 | + | d.snapshotWorkerCancel = cancel | |||
| 64 | + | d.snapshotWorkerDone = make(chan struct{}) | |||
| 65 | + | d.snapshotWorkerFlush = make(chan struct{}) | |||
| 66 | + | d.snapshotWorkerFinalWake = make(chan struct{}, 1) | |||
| 67 | + | d.snapshotFinalJobs = make(map[*session]*snapshotCapture) | |||
| 68 | + | d.snapshotFinalOrder = nil | |||
| 69 | + | d.snapshotWorkerClosing = false | |||
| 70 | + | d.snapshotWorkerInFlight = nil | |||
| 71 | + | done := d.snapshotWorkerDone | |||
| 72 | + | flush := d.snapshotWorkerFlush | |||
| 73 | + | finalWake := d.snapshotWorkerFinalWake | |||
| 74 | + | d.snapshotWorkerMu.Unlock() | |||
| 75 | + | go d.runSnapshotEncodeWorker(workerCtx, workerID, done, flush, finalWake) | |||
| 76 | + | } | |||
| 77 | + | ||||
| 78 | + | // runSnapshotEncodeWorker owns the worker state machine. Each event either | |||
| 79 | + | // continues processing, drains the terminal queues for shutdown, or stops; | |||
| 80 | + | // publication details are kept out of the select loop. | |||
| 81 | + | func (d *Daemon) runSnapshotEncodeWorker(workerCtx context.Context, workerID uint64, done chan<- struct{}, flush, finalWake <-chan struct{}) { | |||
| 82 | + | defer close(done) | |||
| 83 | + | for { | |||
| 84 | + | select { | |||
| 85 | + | case <-workerCtx.Done(): | |||
| 86 | + | return | |||
| 87 | + | case <-flush: | |||
| 88 | + | d.flushSnapshotCaptures(workerCtx, workerID) | |||
| 89 | + | return | |||
| 90 | + | case <-finalWake: | |||
| 91 | + | if !d.drainFinalSnapshotCaptures(workerCtx, workerID) { | |||
| 92 | + | return | |||
| 93 | + | } | |||
| 94 | + | case capture := <-d.snapshotJobs: | |||
| 95 | + | if !d.publishSnapshotCapture(workerCtx, workerID, capture) { | |||
| 96 | + | return | |||
| 97 | + | } | |||
| 98 | + | } | |||
| 99 | + | } | |||
| 100 | + | } | |||
| 101 | + | ||||
| 102 | + | func (d *Daemon) flushSnapshotCaptures(workerCtx context.Context, workerID uint64) { | |||
| 103 | + | for { | |||
| 104 | + | select { | |||
| 105 | + | case capture := <-d.snapshotJobs: | |||
| 106 | + | if !d.publishSnapshotCapture(workerCtx, workerID, capture) { | |||
| 107 | + | return | |||
| 108 | + | } | |||
| 109 | + | default: | |||
| 110 | + | _ = d.drainFinalSnapshotCaptures(workerCtx, workerID) | |||
| 111 | + | return | |||
| 112 | + | } | |||
| 113 | + | } | |||
| 114 | + | } | |||
| 115 | + | ||||
| 116 | + | func (d *Daemon) drainFinalSnapshotCaptures(workerCtx context.Context, workerID uint64) bool { | |||
| 117 | + | for capture := d.takeFinalSnapshotCapture(); capture != nil; capture = d.takeFinalSnapshotCapture() { | |||
| 118 | + | if !d.publishSnapshotCapture(workerCtx, workerID, capture) { | |||
| 119 | + | return false | |||
| 120 | + | } | |||
| 121 | + | } | |||
| 122 | + | return true | |||
| 123 | + | } | |||
| 124 | + | ||||
| 125 | + | func (d *Daemon) publishSnapshotCapture(workerCtx context.Context, workerID uint64, capture *snapshotCapture) bool { | |||
| 126 | + | if workerCtx.Err() != nil || !startSnapshotPublication(capture) || !d.setSnapshotWorkerInFlight(workerID, capture) { | |||
| 127 | + | d.finishSnapshotCapture(capture, false) | |||
| 128 | + | return workerCtx.Err() == nil | |||
| 129 | + | } | |||
| 130 | + | publication, err := d.incrementalPublication(capture) | |||
| 131 | + | publicationContext := capture.publicationContext | |||
| 132 | + | if publicationContext == nil { | |||
| 133 | + | publicationContext = workerCtx | |||
| 134 | + | } | |||
| 135 | + | if err == nil && workerCtx.Err() == nil { | |||
| 136 | + | if err = publicationContext.Err(); err == nil { | |||
| 137 | + | err = d.snapshotRepository.Publish(publicationContext, publication) | |||
| 138 | + | } | |||
| 139 | + | } | |||
| 140 | + | d.clearSnapshotWorkerInFlight(workerID, capture) | |||
| 141 | + | if err != nil && workerCtx.Err() == nil && publicationContext.Err() == nil { | |||
| 142 | + | // Global, not session-scoped: by now the session may already be torn | |||
| 143 | + | // down, so a session notice would be dead-on-arrival. No lock is held | |||
| 144 | + | // here (clearSnapshotWorkerInFlight released its own). | |||
| 145 | + | d.reportSnapshotFailure(capture, "publish", err) | |||
| 146 | + | } | |||
| 147 | + | d.finishSnapshotCapture(capture, err == nil && workerCtx.Err() == nil) | |||
| 148 | + | return workerCtx.Err() == nil | |||
| 149 | + | } | |||
| 150 | + | ||||
| 151 | + | func (d *Daemon) takeFinalSnapshotCapture() *snapshotCapture { | |||
| 152 | + | d.snapshotWorkerMu.Lock() | |||
| 153 | + | defer d.snapshotWorkerMu.Unlock() | |||
| 154 | + | for len(d.snapshotFinalOrder) > 0 { | |||
| 155 | + | sess := d.snapshotFinalOrder[0] | |||
| 156 | + | d.snapshotFinalOrder[0] = nil | |||
| 157 | + | d.snapshotFinalOrder = d.snapshotFinalOrder[1:] | |||
| 158 | + | capture := d.snapshotFinalJobs[sess] | |||
| 159 | + | delete(d.snapshotFinalJobs, sess) | |||
| 160 | + | if capture != nil { | |||
| 161 | + | return capture | |||
| 162 | + | } | |||
| 163 | + | } | |||
| 164 | + | return nil | |||
| 165 | + | } | |||
| 166 | + | ||||
| 167 | + | func (d *Daemon) setSnapshotWorkerInFlight(workerID uint64, capture *snapshotCapture) bool { | |||
| 168 | + | d.snapshotWorkerMu.Lock() | |||
| 169 | + | defer d.snapshotWorkerMu.Unlock() | |||
| 170 | + | if d.snapshotWorkerID != workerID || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |||
| 171 | + | return false | |||
| 172 | + | } | |||
| 173 | + | d.snapshotWorkerInFlight = capture | |||
| 174 | + | return true | |||
| 175 | + | } | |||
| 176 | + | ||||
| 177 | + | func (d *Daemon) clearSnapshotWorkerInFlight(workerID uint64, capture *snapshotCapture) { | |||
| 178 | + | d.snapshotWorkerMu.Lock() | |||
| 179 | + | defer d.snapshotWorkerMu.Unlock() | |||
| 180 | + | if d.snapshotWorkerID == workerID && d.snapshotWorkerInFlight == capture { | |||
| 181 | + | d.snapshotWorkerInFlight = nil | |||
| 182 | + | } | |||
| 183 | + | } | |||
| 184 | + | ||||
| 185 | + | func (d *Daemon) stopSnapshotEncodeWorker() { | |||
| 186 | + | if d == nil { | |||
| 187 | + | return | |||
| 188 | + | } | |||
| 189 | + | deadline := newSnapshotShutdownDeadline(d.clock) | |||
| 190 | + | defer deadline.stop() | |||
| 191 | + | d.stopSnapshotEncodeWorkerWithDeadline(deadline) | |||
| 192 | + | } | |||
| 193 | + | ||||
| 194 | + | // stopSnapshotEncodeWorkerWithDeadline performs the one deadline-aware join | |||
| 195 | + | // for a Serve shutdown. Queues are never closed, and detached writers only | |||
| 196 | + | // retain immutable captures plus contexts/channels that remain live, so an | |||
| 197 | + | // uncooperative repository call cannot race teardown or block process exit. | |||
| 198 | + | func (d *Daemon) stopSnapshotEncodeWorkerWithDeadline(deadline *snapshotShutdownDeadline) { | |||
| 199 | + | if d == nil { | |||
| 200 | + | return | |||
| 201 | + | } | |||
| 202 | + | d.snapshotWorkerMu.Lock() | |||
| 203 | + | cancel := d.snapshotWorkerCancel | |||
| 204 | + | if cancel == nil || d.snapshotWorkerClosing { | |||
| 205 | + | d.snapshotWorkerMu.Unlock() | |||
| 206 | + | return | |||
| 207 | + | } | |||
| 208 | + | // Stop accepting producers, then let the single worker drain its bounded | |||
| 209 | + | // queue. A non-context-aware store may still block indefinitely, so the | |||
| 210 | + | // shared shutdown deadline bounds the one and only join. | |||
| 211 | + | d.snapshotWorkerClosing = true | |||
| 212 | + | flush := d.snapshotWorkerFlush | |||
| 213 | + | done := d.snapshotWorkerDone | |||
| 214 | + | close(flush) | |||
| 215 | + | d.snapshotWorkerMu.Unlock() | |||
| 216 | + | ||||
| 217 | + | if deadline == nil { | |||
| 218 | + | deadline = newSnapshotShutdownDeadline(d.clock) | |||
| 219 | + | defer deadline.stop() | |||
| 220 | + | } | |||
| 221 | + | select { | |||
| 222 | + | case <-done: | |||
| 223 | + | d.finishStoppedSnapshotWorker(false) | |||
| 224 | + | case <-deadline.Done(): | |||
| 225 | + | cancel() | |||
| 226 | + | d.finishStoppedSnapshotWorker(true) | |||
| 227 | + | } | |||
| 228 | + | } | |||
| 229 | + | ||||
| 230 | + | func (d *Daemon) finishStoppedSnapshotWorker(abandoned bool) { | |||
| 231 | + | d.snapshotWorkerMu.Lock() | |||
| 232 | + | inFlight := d.snapshotWorkerInFlight | |||
| 233 | + | d.snapshotWorkerCtx = nil | |||
| 234 | + | d.snapshotWorkerCancel = nil | |||
| 235 | + | d.snapshotWorkerDone = nil | |||
| 236 | + | d.snapshotWorkerFlush = nil | |||
| 237 | + | d.snapshotWorkerFinalWake = nil | |||
| 238 | + | d.snapshotWorkerClosing = false | |||
| 239 | + | d.snapshotWorkerInFlight = nil | |||
| 240 | + | queued := make([]*snapshotCapture, 0, len(d.snapshotJobs)+len(d.snapshotFinalJobs)) | |||
| 241 | + | for _, capture := range d.snapshotFinalJobs { | |||
| 242 | + | queued = append(queued, capture) | |||
| 243 | + | } | |||
| 244 | + | d.snapshotFinalJobs = nil | |||
| 245 | + | d.snapshotFinalOrder = nil | |||
| 246 | + | for { | |||
| 247 | + | select { | |||
| 248 | + | case capture := <-d.snapshotJobs: | |||
| 249 | + | queued = append(queued, capture) | |||
| 250 | + | default: | |||
| 251 | + | d.snapshotWorkerMu.Unlock() | |||
| 252 | + | if abandoned { | |||
| 253 | + | d.finishSnapshotCapture(inFlight, false) | |||
| 254 | + | } | |||
| 255 | + | for _, capture := range queued { | |||
| 256 | + | d.finishSnapshotCapture(capture, false) | |||
| 257 | + | } | |||
| 258 | + | return | |||
| 259 | + | } | |||
| 260 | + | } | |||
| 261 | + | } | |||