detached e82f1b8

fix(recovery): retain healthy checkpoint window

commit
e82f1b85ffea532d11cbbeb57ce8de7211467e16
author
brice · · 2026-07-26 02:44 UTC
parent
2a1a401
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
4 files · +126 -4
Unified
UnifiedSplit
M internal/ports/recovery.go +3 -0
@@ -99,6 +99,9 @@ type ReconcileDecision struct {
9999 Name string
100100 Candidate *domain.CheckpointRef
101101 ReasonCode string
102+ // RetentionResolved permits collection after a conclusive adoption or
103+ // not-forward decision. Its zero value conservatively pins known sessions.
104+ RetentionResolved bool
102105 }
103106
104107 var (
@@ -99,6 +99,9 @@ type ReconcileDecision struct {
99 Name string99 Name string
100 Candidate *domain.CheckpointRef100 Candidate *domain.CheckpointRef
101 ReasonCode string101 ReasonCode string
102+ // RetentionResolved permits collection after a conclusive adoption or
103+ // not-forward decision. Its zero value conservatively pins known sessions.
104+ RetentionResolved bool
102 }105 }
103 106
104 var (107 var (
M internal/usecase/daemon/maintenance.go +1 -1
@@ -116,7 +116,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) {
116116 deps.cursor = next
117117 for _, decision := range decisions {
118118 record, ok := deps.catalogue.Record(decision.Name)
119- if ok && decision.Kind != ports.ReconcileAdopt {
119+ if ok && !decision.RetentionResolved {
120120 deps.repairUncertain[record.IncarnationID] = struct{}{}
121121 }
122122 d.log.Info("durable_reconciliation_decision",
@@ -116,7 +116,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) {
116 deps.cursor = next116 deps.cursor = next
117 for _, decision := range decisions {117 for _, decision := range decisions {
118 record, ok := deps.catalogue.Record(decision.Name)118 record, ok := deps.catalogue.Record(decision.Name)
119- if ok && decision.Kind != ports.ReconcileAdopt {119+ if ok && !decision.RetentionResolved {
120 deps.repairUncertain[record.IncarnationID] = struct{}{}120 deps.repairUncertain[record.IncarnationID] = struct{}{}
121 }121 }
122 d.log.Info("durable_reconciliation_decision",122 d.log.Info("durable_reconciliation_decision",
M internal/usecase/daemon/maintenance_test.go +114 -2
@@ -2,12 +2,15 @@ package daemon
22
33 import (
44 "context"
5+ "errors"
56 "testing"
67
78 "github.com/bnema/vev/internal/domain"
89 "github.com/bnema/vev/internal/ports"
910 portsmocks "github.com/bnema/vev/internal/ports/mocks"
11+ recoveryusecase "github.com/bnema/vev/internal/usecase/recovery"
1012 "github.com/stretchr/testify/mock"
13+ "github.com/stretchr/testify/require"
1114 )
1215
1316 type cursorMaintenanceReconciler struct {
@@ -36,8 +39,117 @@ func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) {
3639
3740 d.runDurableMaintenanceTick(context.Background())
3841 d.runDurableMaintenanceTick(context.Background())
39- if len(reconciler.seen) != 2 || reconciler.seen[0].DirectoryCookie != 0 || reconciler.seen[1].DirectoryCookie != 7 {
40- t.Fatalf("reconciliation cursors = %v, want [0 7]", reconciler.seen)
42+ d.runDurableMaintenanceTick(context.Background())
43+ if len(reconciler.seen) != 3 || reconciler.seen[0].DirectoryCookie != 0 || reconciler.seen[1].DirectoryCookie != 7 || reconciler.seen[2].DirectoryCookie != 0 {
44+ t.Fatalf("reconciliation cursors = %v, want [0 7 0]", reconciler.seen)
45+ }
46+}
47+
48+type reconciliationMaintenanceRepository struct {
49+ ports.SnapshotRepository
50+ findings []ports.ReconcileFinding
51+ err error
52+ plans []ports.RetentionPlan
53+}
54+
55+func (r *reconciliationMaintenanceRepository) Reconcile(context.Context, []domain.CatalogueRecord, ports.ReconcileCursor, ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error) {
56+ return ports.ReconcileCursor{}, r.findings, r.err
57+}
58+
59+func (r *reconciliationMaintenanceRepository) MaintainSession(_ context.Context, plan ports.RetentionPlan, _ ports.MaintenanceBudget) (bool, error) {
60+ r.plans = append(r.plans, plan)
61+ return true, nil
62+}
63+
64+func TestMaintenanceRetentionUsesReconciliationOutcome(t *testing.T) {
65+ committed := domain.CheckpointRef{Generation: 3, ManifestDigest: [32]byte{3}}
66+ fallback1 := domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}}
67+ fallback2 := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}
68+ record := domain.CatalogueRecord{
69+ Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryHealthy,
70+ Committed: &committed, Fallbacks: [2]*domain.CheckpointRef{&fallback1, &fallback2},
71+ }
72+ wrongParent := domain.CheckpointRef{Generation: 3, ManifestDigest: [32]byte{9}}
73+
74+ for _, tc := range []struct {
75+ name string
76+ finding ports.ReconcileFinding
77+ reconcileErr error
78+ wantPlan *ports.RetentionPlan
79+ }{
80+ {
81+ name: "current HEAD is conclusively not forward",
82+ finding: ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
83+ Name: record.Name, IncarnationID: record.IncarnationID, Ref: committed, Parent: &fallback1,
84+ }},
85+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}},
86+ },
87+ {
88+ name: "known predecessor is conclusively not forward",
89+ finding: ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
90+ Name: record.Name, IncarnationID: record.IncarnationID, Ref: fallback1, Parent: &fallback2,
91+ }},
92+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}},
93+ },
94+ {
95+ name: "non-forward candidate kind does not pin",
96+ finding: ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
97+ Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}},
98+ }},
99+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}},
100+ },
101+ {
102+ name: "unreadable candidate remains pinned",
103+ finding: ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileQuarantined, Candidate: ports.ReconcileCandidate{
104+ Name: record.Name, IncarnationID: record.IncarnationID,
105+ }},
106+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
107+ },
108+ {
109+ name: "quarantined forward candidate remains pinned",
110+ finding: ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileQuarantined, Candidate: ports.ReconcileCandidate{
111+ Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}}, Parent: &wrongParent,
112+ }},
113+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
114+ },
115+ {
116+ name: "incarnation mismatch remains pinned",
117+ finding: ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
118+ Name: record.Name, IncarnationID: domain.IncarnationID{9}, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}}, Parent: &committed,
119+ }},
120+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
121+ },
122+ {
123+ name: "incomplete decision remains pinned",
124+ finding: ports.ReconcileFinding{Status: ports.ReconcileBudgetExhausted, Candidate: ports.ReconcileCandidate{
125+ Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}},
126+ }},
127+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
128+ },
129+ {
130+ name: "reconciliation error withholds maintenance",
131+ reconcileErr: errors.New("scan failed"),
132+ },
133+ } {
134+ t.Run(tc.name, func(t *testing.T) {
135+ catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record})
136+ repository := &reconciliationMaintenanceRepository{err: tc.reconcileErr}
137+ if tc.finding.Candidate.Name != "" {
138+ repository.findings = []ports.ReconcileFinding{tc.finding}
139+ }
140+ coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
141+ reconciler := recoveryusecase.NewReconciler(coordinator, catalogue, repository, ports.MaintenanceBudget{Entries: 8, Bytes: 1024})
142+ d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
143+ WithDurableMaintenance(catalogue, repository, reconciler, nil)(d)
144+
145+ d.runDurableMaintenanceTick(context.Background())
146+
147+ if tc.wantPlan == nil {
148+ require.Empty(t, repository.plans)
149+ return
150+ }
151+ require.Equal(t, []ports.RetentionPlan{*tc.wantPlan}, repository.plans)
152+ })
41153 }
42154 }
43155
@@ -2,12 +2,15 @@ package daemon
2 2
3 import (3 import (
4 "context"4 "context"
5+ "errors"
5 "testing"6 "testing"
6 7
7 "github.com/bnema/vev/internal/domain"8 "github.com/bnema/vev/internal/domain"
8 "github.com/bnema/vev/internal/ports"9 "github.com/bnema/vev/internal/ports"
9 portsmocks "github.com/bnema/vev/internal/ports/mocks"10 portsmocks "github.com/bnema/vev/internal/ports/mocks"
11+ recoveryusecase "github.com/bnema/vev/internal/usecase/recovery"
10 "github.com/stretchr/testify/mock"12 "github.com/stretchr/testify/mock"
13+ "github.com/stretchr/testify/require"
11 )14 )
12 15
13 type cursorMaintenanceReconciler struct {16 type cursorMaintenanceReconciler struct {
@@ -36,8 +39,117 @@ func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) {
36 39
37 d.runDurableMaintenanceTick(context.Background())40 d.runDurableMaintenanceTick(context.Background())
38 d.runDurableMaintenanceTick(context.Background())41 d.runDurableMaintenanceTick(context.Background())
39- if len(reconciler.seen) != 2 || reconciler.seen[0].DirectoryCookie != 0 || reconciler.seen[1].DirectoryCookie != 7 {42+ d.runDurableMaintenanceTick(context.Background())
40- t.Fatalf("reconciliation cursors = %v, want [0 7]", reconciler.seen)43+ if len(reconciler.seen) != 3 || reconciler.seen[0].DirectoryCookie != 0 || reconciler.seen[1].DirectoryCookie != 7 || reconciler.seen[2].DirectoryCookie != 0 {
44+ t.Fatalf("reconciliation cursors = %v, want [0 7 0]", reconciler.seen)
45+ }
46+}
47+
48+type reconciliationMaintenanceRepository struct {
49+ ports.SnapshotRepository
50+ findings []ports.ReconcileFinding
51+ err error
52+ plans []ports.RetentionPlan
53+}
54+
55+func (r *reconciliationMaintenanceRepository) Reconcile(context.Context, []domain.CatalogueRecord, ports.ReconcileCursor, ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error) {
56+ return ports.ReconcileCursor{}, r.findings, r.err
57+}
58+
59+func (r *reconciliationMaintenanceRepository) MaintainSession(_ context.Context, plan ports.RetentionPlan, _ ports.MaintenanceBudget) (bool, error) {
60+ r.plans = append(r.plans, plan)
61+ return true, nil
62+}
63+
64+func TestMaintenanceRetentionUsesReconciliationOutcome(t *testing.T) {
65+ committed := domain.CheckpointRef{Generation: 3, ManifestDigest: [32]byte{3}}
66+ fallback1 := domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}}
67+ fallback2 := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}
68+ record := domain.CatalogueRecord{
69+ Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryHealthy,
70+ Committed: &committed, Fallbacks: [2]*domain.CheckpointRef{&fallback1, &fallback2},
71+ }
72+ wrongParent := domain.CheckpointRef{Generation: 3, ManifestDigest: [32]byte{9}}
73+
74+ for _, tc := range []struct {
75+ name string
76+ finding ports.ReconcileFinding
77+ reconcileErr error
78+ wantPlan *ports.RetentionPlan
79+ }{
80+ {
81+ name: "current HEAD is conclusively not forward",
82+ finding: ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
83+ Name: record.Name, IncarnationID: record.IncarnationID, Ref: committed, Parent: &fallback1,
84+ }},
85+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}},
86+ },
87+ {
88+ name: "known predecessor is conclusively not forward",
89+ finding: ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
90+ Name: record.Name, IncarnationID: record.IncarnationID, Ref: fallback1, Parent: &fallback2,
91+ }},
92+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}},
93+ },
94+ {
95+ name: "non-forward candidate kind does not pin",
96+ finding: ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
97+ Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}},
98+ }},
99+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}},
100+ },
101+ {
102+ name: "unreadable candidate remains pinned",
103+ finding: ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileQuarantined, Candidate: ports.ReconcileCandidate{
104+ Name: record.Name, IncarnationID: record.IncarnationID,
105+ }},
106+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
107+ },
108+ {
109+ name: "quarantined forward candidate remains pinned",
110+ finding: ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileQuarantined, Candidate: ports.ReconcileCandidate{
111+ Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}}, Parent: &wrongParent,
112+ }},
113+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
114+ },
115+ {
116+ name: "incarnation mismatch remains pinned",
117+ finding: ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
118+ Name: record.Name, IncarnationID: domain.IncarnationID{9}, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}}, Parent: &committed,
119+ }},
120+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
121+ },
122+ {
123+ name: "incomplete decision remains pinned",
124+ finding: ports.ReconcileFinding{Status: ports.ReconcileBudgetExhausted, Candidate: ports.ReconcileCandidate{
125+ Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}},
126+ }},
127+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
128+ },
129+ {
130+ name: "reconciliation error withholds maintenance",
131+ reconcileErr: errors.New("scan failed"),
132+ },
133+ } {
134+ t.Run(tc.name, func(t *testing.T) {
135+ catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record})
136+ repository := &reconciliationMaintenanceRepository{err: tc.reconcileErr}
137+ if tc.finding.Candidate.Name != "" {
138+ repository.findings = []ports.ReconcileFinding{tc.finding}
139+ }
140+ coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
141+ reconciler := recoveryusecase.NewReconciler(coordinator, catalogue, repository, ports.MaintenanceBudget{Entries: 8, Bytes: 1024})
142+ d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
143+ WithDurableMaintenance(catalogue, repository, reconciler, nil)(d)
144+
145+ d.runDurableMaintenanceTick(context.Background())
146+
147+ if tc.wantPlan == nil {
148+ require.Empty(t, repository.plans)
149+ return
150+ }
151+ require.Equal(t, []ports.RetentionPlan{*tc.wantPlan}, repository.plans)
152+ })
41 }153 }
42 }154 }
43 155
M internal/usecase/recovery/reconcile.go +8 -1
@@ -72,9 +72,15 @@ func classifyFinding(record domain.CatalogueRecord, finding ports.ReconcileFindi
7272 decision.ReasonCode = "incarnation-mismatch"
7373 return decision
7474 }
75- if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil || finding.Candidate.Ref.Generation <= record.Committed.Generation {
75+ if finding.Status == ports.ReconcileValidated && record.Committed != nil && (finding.Kind != ports.ReconcileForwardOrphan || finding.Candidate.Ref.Generation <= record.Committed.Generation) {
7676 decision.Kind = ports.ReconcileKeepQuarantined
7777 decision.ReasonCode = "not-forward"
78+ decision.RetentionResolved = true
79+ return decision
80+ }
81+ if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil {
82+ decision.Kind = ports.ReconcileKeepQuarantined
83+ decision.ReasonCode = "invalid-candidate"
7884 return decision
7985 }
8086 if finding.Status != ports.ReconcileValidated || !validForwardChain(*record.Committed, finding.Candidate, finding.AncestorChain) {
@@ -83,6 +89,7 @@ func classifyFinding(record domain.CatalogueRecord, finding ports.ReconcileFindi
8389 return decision
8490 }
8591 decision.Kind = ports.ReconcileAdopt
92+ decision.RetentionResolved = true
8693 return decision
8794 }
8895
@@ -72,9 +72,15 @@ func classifyFinding(record domain.CatalogueRecord, finding ports.ReconcileFindi
72 decision.ReasonCode = "incarnation-mismatch"72 decision.ReasonCode = "incarnation-mismatch"
73 return decision73 return decision
74 }74 }
75- if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil || finding.Candidate.Ref.Generation <= record.Committed.Generation {75+ if finding.Status == ports.ReconcileValidated && record.Committed != nil && (finding.Kind != ports.ReconcileForwardOrphan || finding.Candidate.Ref.Generation <= record.Committed.Generation) {
76 decision.Kind = ports.ReconcileKeepQuarantined76 decision.Kind = ports.ReconcileKeepQuarantined
77 decision.ReasonCode = "not-forward"77 decision.ReasonCode = "not-forward"
78+ decision.RetentionResolved = true
79+ return decision
80+ }
81+ if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil {
82+ decision.Kind = ports.ReconcileKeepQuarantined
83+ decision.ReasonCode = "invalid-candidate"
78 return decision84 return decision
79 }85 }
80 if finding.Status != ports.ReconcileValidated || !validForwardChain(*record.Committed, finding.Candidate, finding.AncestorChain) {86 if finding.Status != ports.ReconcileValidated || !validForwardChain(*record.Committed, finding.Candidate, finding.AncestorChain) {
@@ -83,6 +89,7 @@ func classifyFinding(record domain.CatalogueRecord, finding ports.ReconcileFindi
83 return decision89 return decision
84 }90 }
85 decision.Kind = ports.ReconcileAdopt91 decision.Kind = ports.ReconcileAdopt
92+ decision.RetentionResolved = true
86 return decision93 return decision
87 }94 }
88 95

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 12

Tags 11

↑↓ moveenter openesc close