detached 1f12e31

fix(recovery): tighten maintenance and restore safety

commit
1f12e310ab6edacfbd5708008b4a9003292c2ba3
author
brice · · 2026-07-26 08:09 UTC
parent
979dbfb
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
71 files · +1400 -567
M docs/durable-session-recovery.md +2 -2
@@ -10,10 +10,10 @@ vev opens named-session state only while holding `$XDG_RUNTIME_DIR/vev/lifecycle
1010
1111 - `vev cmd -s NAME session-recovery retry`
1212 - `vev cmd -s NAME session-recovery restore GENERATION`
13-- `vev cmd -s NAME session-recovery export PATH`
13+- `vev cmd -s NAME session-recovery export /ABSOLUTE/PATH`
1414 - `vev cmd -s NAME session-recovery discard`
1515
16-Retry validates persisted state without replacing it. Restore validates and promotes the selected catalogue fallback. Export writes recoverable data without mutation. Discard creates a new incarnation and retains the old record and snapshots under `quarantine/` until an explicit later purge.
16+Retry validates persisted state without replacing it. Restore validates and promotes the selected catalogue fallback. Export writes recoverable data without mutation and requires an absolute destination path. Discard creates a new incarnation and retains the old record and snapshots under `quarantine/` until an explicit later purge.
1717
1818 ## Migration and retention
1919
@@ -10,10 +10,10 @@ vev opens named-session state only while holding `$XDG_RUNTIME_DIR/vev/lifecycle
10 10
11 - `vev cmd -s NAME session-recovery retry`11 - `vev cmd -s NAME session-recovery retry`
12 - `vev cmd -s NAME session-recovery restore GENERATION`12 - `vev cmd -s NAME session-recovery restore GENERATION`
13-- `vev cmd -s NAME session-recovery export PATH`13+- `vev cmd -s NAME session-recovery export /ABSOLUTE/PATH`
14 - `vev cmd -s NAME session-recovery discard`14 - `vev cmd -s NAME session-recovery discard`
15 15
16-Retry validates persisted state without replacing it. Restore validates and promotes the selected catalogue fallback. Export writes recoverable data without mutation. Discard creates a new incarnation and retains the old record and snapshots under `quarantine/` until an explicit later purge.16+Retry validates persisted state without replacing it. Restore validates and promotes the selected catalogue fallback. Export writes recoverable data without mutation and requires an absolute destination path. Discard creates a new incarnation and retains the old record and snapshots under `quarantine/` until an explicit later purge.
17 17
18 ## Migration and retention18 ## Migration and retention
19 19
M internal/adapters/lifecycle/flock.go +2 -0
@@ -1,3 +1,5 @@
1+//go:build unix
2+
13 // Package lifecycle provides exclusive process ownership of vev's durable
24 // state for the complete daemon lifetime.
35 package lifecycle
@@ -1,3 +1,5 @@
1+//go:build unix
2+
1 // Package lifecycle provides exclusive process ownership of vev's durable3 // Package lifecycle provides exclusive process ownership of vev's durable
2 // state for the complete daemon lifetime.4 // state for the complete daemon lifetime.
3 package lifecycle5 package lifecycle
M internal/adapters/lifecycle/flock_test.go +4 -0
@@ -49,6 +49,10 @@ func testLifecycleLockProcessContentionAndReacquire(t *testing.T) {
4949 require.NoError(t, err)
5050 cmd.Stderr = os.Stderr
5151 require.NoError(t, cmd.Start())
52+ defer func() {
53+ _ = stdin.Close()
54+ _ = cmd.Wait()
55+ }()
5256
5357 line, err := bufio.NewReader(stdout).ReadString('\n')
5458 require.NoError(t, err)
@@ -49,6 +49,10 @@ func testLifecycleLockProcessContentionAndReacquire(t *testing.T) {
49 require.NoError(t, err)49 require.NoError(t, err)
50 cmd.Stderr = os.Stderr50 cmd.Stderr = os.Stderr
51 require.NoError(t, cmd.Start())51 require.NoError(t, cmd.Start())
52+ defer func() {
53+ _ = stdin.Close()
54+ _ = cmd.Wait()
55+ }()
52 56
53 line, err := bufio.NewReader(stdout).ReadString('\n')57 line, err := bufio.NewReader(stdout).ReadString('\n')
54 require.NoError(t, err)58 require.NoError(t, err)
M internal/adapters/recoveryfs/journal.go +3 -0
@@ -62,6 +62,9 @@ func decodeDiscard(data []byte) (domain.DiscardIntent, error) {
6262 return domain.DiscardIntent{}, err
6363 }
6464 if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
65+ if err != nil {
66+ return domain.DiscardIntent{}, err
67+ }
6568 return domain.DiscardIntent{}, errors.New("recovery journal: trailing data")
6669 }
6770 if file.Version != discardVersion {
@@ -62,6 +62,9 @@ func decodeDiscard(data []byte) (domain.DiscardIntent, error) {
62 return domain.DiscardIntent{}, err62 return domain.DiscardIntent{}, err
63 }63 }
64 if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {64 if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
65+ if err != nil {
66+ return domain.DiscardIntent{}, err
67+ }
65 return domain.DiscardIntent{}, errors.New("recovery journal: trailing data")68 return domain.DiscardIntent{}, errors.New("recovery journal: trailing data")
66 }69 }
67 if file.Version != discardVersion {70 if file.Version != discardVersion {
M internal/adapters/recoveryfs/journal_test.go +8 -5
@@ -3,6 +3,7 @@ package recoveryfs
33 import (
44 "context"
55 "errors"
6+ "fmt"
67 "os"
78 "path/filepath"
89 "testing"
@@ -45,7 +46,6 @@ func TestDiscardCrashMatrix(t *testing.T) {
4546 _, statErr := os.Stat(path)
4647 require.NoError(t, statErr)
4748 require.Error(t, j.SaveDiscard(ctx, intent))
48- require.NoError(t, os.WriteFile(path, encoded, 0o600))
4949 })
5050
5151 t.Run("delete sync failure rolls forward", func(t *testing.T) {
@@ -81,8 +81,8 @@ func TestRecoveryJournalRoundTripAndIdempotentDelete(t *testing.T) {
8181 func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T) {
8282 j := New(t.TempDir())
8383 require.NoError(t, os.MkdirAll(j.dir, 0o700))
84- for i := 0; i < maxDiscardIntents+1; i++ {
85- require.NoError(t, os.WriteFile(filepath.Join(j.dir, ".intent-leftover-"+string(rune(0x1000+i))), nil, 0o600))
84+ for i := range maxDiscardIntents + 1 {
85+ require.NoError(t, os.WriteFile(filepath.Join(j.dir, fmt.Sprintf(".intent-leftover-%04d", i)), nil, 0o600))
8686 }
8787
8888 got, err := j.ListDiscards(context.Background())
@@ -93,14 +93,17 @@ func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T)
9393 func TestRecoveryJournalRejectsMalformedIntent(t *testing.T) {
9494 j := New(t.TempDir())
9595 require.Error(t, j.SaveDiscard(context.Background(), domain.DiscardIntent{}))
96+
97+ _, err := decodeDiscard([]byte(`{"version":1,"intent":{}} !`))
98+ require.ErrorContains(t, err, "invalid character")
9699 }
97100
98101 func TestRecoveryJournalBoundsEnumerationAndIntentReads(t *testing.T) {
99102 state := t.TempDir()
100103 j := New(state)
101104 require.NoError(t, os.MkdirAll(j.dir, 0o700))
102- for i := 0; i <= maxDiscardIntents; i++ {
103- require.NoError(t, os.WriteFile(filepath.Join(j.dir, "discard-extra-"+string(rune(0x1000+i))), nil, 0o600))
105+ for i := range maxDiscardIntents + 1 {
106+ require.NoError(t, os.WriteFile(filepath.Join(j.dir, fmt.Sprintf("discard-extra-%04d", i)), nil, 0o600))
104107 }
105108 _, err := j.ListDiscards(context.Background())
106109 require.ErrorContains(t, err, "entry limit exceeded")
@@ -3,6 +3,7 @@ package recoveryfs
3 import (3 import (
4 "context"4 "context"
5 "errors"5 "errors"
6+ "fmt"
6 "os"7 "os"
7 "path/filepath"8 "path/filepath"
8 "testing"9 "testing"
@@ -45,7 +46,6 @@ func TestDiscardCrashMatrix(t *testing.T) {
45 _, statErr := os.Stat(path)46 _, statErr := os.Stat(path)
46 require.NoError(t, statErr)47 require.NoError(t, statErr)
47 require.Error(t, j.SaveDiscard(ctx, intent))48 require.Error(t, j.SaveDiscard(ctx, intent))
48- require.NoError(t, os.WriteFile(path, encoded, 0o600))
49 })49 })
50 50
51 t.Run("delete sync failure rolls forward", func(t *testing.T) {51 t.Run("delete sync failure rolls forward", func(t *testing.T) {
@@ -81,8 +81,8 @@ func TestRecoveryJournalRoundTripAndIdempotentDelete(t *testing.T) {
81 func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T) {81 func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T) {
82 j := New(t.TempDir())82 j := New(t.TempDir())
83 require.NoError(t, os.MkdirAll(j.dir, 0o700))83 require.NoError(t, os.MkdirAll(j.dir, 0o700))
84- for i := 0; i < maxDiscardIntents+1; i++ {84+ for i := range maxDiscardIntents + 1 {
85- require.NoError(t, os.WriteFile(filepath.Join(j.dir, ".intent-leftover-"+string(rune(0x1000+i))), nil, 0o600))85+ require.NoError(t, os.WriteFile(filepath.Join(j.dir, fmt.Sprintf(".intent-leftover-%04d", i)), nil, 0o600))
86 }86 }
87 87
88 got, err := j.ListDiscards(context.Background())88 got, err := j.ListDiscards(context.Background())
@@ -93,14 +93,17 @@ func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T)
93 func TestRecoveryJournalRejectsMalformedIntent(t *testing.T) {93 func TestRecoveryJournalRejectsMalformedIntent(t *testing.T) {
94 j := New(t.TempDir())94 j := New(t.TempDir())
95 require.Error(t, j.SaveDiscard(context.Background(), domain.DiscardIntent{}))95 require.Error(t, j.SaveDiscard(context.Background(), domain.DiscardIntent{}))
96+
97+ _, err := decodeDiscard([]byte(`{"version":1,"intent":{}} !`))
98+ require.ErrorContains(t, err, "invalid character")
96 }99 }
97 100
98 func TestRecoveryJournalBoundsEnumerationAndIntentReads(t *testing.T) {101 func TestRecoveryJournalBoundsEnumerationAndIntentReads(t *testing.T) {
99 state := t.TempDir()102 state := t.TempDir()
100 j := New(state)103 j := New(state)
101 require.NoError(t, os.MkdirAll(j.dir, 0o700))104 require.NoError(t, os.MkdirAll(j.dir, 0o700))
102- for i := 0; i <= maxDiscardIntents; i++ {105+ for i := range maxDiscardIntents + 1 {
103- require.NoError(t, os.WriteFile(filepath.Join(j.dir, "discard-extra-"+string(rune(0x1000+i))), nil, 0o600))106+ require.NoError(t, os.WriteFile(filepath.Join(j.dir, fmt.Sprintf("discard-extra-%04d", i)), nil, 0o600))
104 }107 }
105 _, err := j.ListDiscards(context.Background())108 _, err := j.ListDiscards(context.Background())
106 require.ErrorContains(t, err, "entry limit exceeded")109 require.ErrorContains(t, err, "entry limit exceeded")
M internal/adapters/snapshot/direct_load_test.go +9 -0
@@ -76,9 +76,18 @@ func TestRepairHEAD(t *testing.T) {
7676 t.Fatal(err)
7777 }
7878 valid := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(pub.Manifest)}
79+ manifestReads := 0
80+ repo.hooks.beforePayloadRead = func(path string) {
81+ if path == repo.manifestPath(id, valid.Generation) {
82+ manifestReads++
83+ }
84+ }
7985 if err := repo.RepairHEAD(context.Background(), id, valid); err != nil {
8086 t.Fatal(err)
8187 }
88+ if manifestReads != 1 {
89+ t.Fatalf("RepairHEAD manifest reads = %d, want 1", manifestReads)
90+ }
8291 before, err := os.ReadFile(repo.headPath(id))
8392 if err != nil {
8493 t.Fatal(err)
@@ -76,9 +76,18 @@ func TestRepairHEAD(t *testing.T) {
76 t.Fatal(err)76 t.Fatal(err)
77 }77 }
78 valid := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(pub.Manifest)}78 valid := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(pub.Manifest)}
79+ manifestReads := 0
80+ repo.hooks.beforePayloadRead = func(path string) {
81+ if path == repo.manifestPath(id, valid.Generation) {
82+ manifestReads++
83+ }
84+ }
79 if err := repo.RepairHEAD(context.Background(), id, valid); err != nil {85 if err := repo.RepairHEAD(context.Background(), id, valid); err != nil {
80 t.Fatal(err)86 t.Fatal(err)
81 }87 }
88+ if manifestReads != 1 {
89+ t.Fatalf("RepairHEAD manifest reads = %d, want 1", manifestReads)
90+ }
82 before, err := os.ReadFile(repo.headPath(id))91 before, err := os.ReadFile(repo.headPath(id))
83 if err != nil {92 if err != nil {
84 t.Fatal(err)93 t.Fatal(err)
M internal/adapters/snapshot/legacy_load_test.go +6 -2
@@ -47,8 +47,7 @@ func (r *Repository) Load(ctx context.Context, name string) (ports.SnapshotGener
4747 if err := ctx.Err(); err != nil {
4848 return ports.SnapshotGeneration{}, err
4949 }
50- id := legacyIncarnationID(name)
51- key, err := incarnationKey(id)
50+ key, err := r.legacyRepositoryKey(name)
5251 if err != nil {
5352 return ports.SnapshotGeneration{}, err
5453 }
@@ -84,13 +83,18 @@ func (r *Repository) load(ctx context.Context, name, key string) (ports.Snapshot
8483 if err != nil {
8584 return ports.SnapshotGeneration{}, err
8685 }
86+ var lastLoadErr error
8787 for _, generation := range candidates {
8888 got, err := r.loadGeneration(ctx, name, key, generation)
8989 if err != nil {
90+ lastLoadErr = err
9091 continue
9192 }
9293 return got, nil
9394 }
95+ if lastLoadErr != nil {
96+ return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q: %w", name, lastLoadErr)
97+ }
9498 return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name)
9599 }
96100
@@ -47,8 +47,7 @@ func (r *Repository) Load(ctx context.Context, name string) (ports.SnapshotGener
47 if err := ctx.Err(); err != nil {47 if err := ctx.Err(); err != nil {
48 return ports.SnapshotGeneration{}, err48 return ports.SnapshotGeneration{}, err
49 }49 }
50- id := legacyIncarnationID(name)50+ key, err := r.legacyRepositoryKey(name)
51- key, err := incarnationKey(id)
52 if err != nil {51 if err != nil {
53 return ports.SnapshotGeneration{}, err52 return ports.SnapshotGeneration{}, err
54 }53 }
@@ -84,13 +83,18 @@ func (r *Repository) load(ctx context.Context, name, key string) (ports.Snapshot
84 if err != nil {83 if err != nil {
85 return ports.SnapshotGeneration{}, err84 return ports.SnapshotGeneration{}, err
86 }85 }
86+ var lastLoadErr error
87 for _, generation := range candidates {87 for _, generation := range candidates {
88 got, err := r.loadGeneration(ctx, name, key, generation)88 got, err := r.loadGeneration(ctx, name, key, generation)
89 if err != nil {89 if err != nil {
90+ lastLoadErr = err
90 continue91 continue
91 }92 }
92 return got, nil93 return got, nil
93 }94 }
95+ if lastLoadErr != nil {
96+ return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q: %w", name, lastLoadErr)
97+ }
94 return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name)98 return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name)
95 }99 }
96 100
M internal/adapters/snapshot/load.go +15 -21
@@ -76,43 +76,44 @@ func (r *Repository) LoadCheckpoint(ctx context.Context, id domain.IncarnationID
7676 }
7777 lock := r.lockSession(key)
7878 defer r.unlockSession(lock)
79- return r.loadCheckpointLocked(ctx, id, name, ref)
79+ generation, _, err := r.loadCheckpointLocked(ctx, id, name, ref)
80+ return generation, err
8081 }
8182
82-func (r *Repository) loadCheckpointLocked(ctx context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) {
83+func (r *Repository) loadCheckpointLocked(ctx context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, codec.Manifest, error) {
8384 if ref.Generation == 0 || ref.ManifestDigest == ([32]byte{}) {
84- return ports.SnapshotGeneration{}, fmt.Errorf("invalid checkpoint reference")
85+ return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid checkpoint reference")
8586 }
8687 data, err := r.readBounded(r.manifestPath(id, ref.Generation))
8788 if err != nil {
88- return ports.SnapshotGeneration{}, err
89+ return ports.SnapshotGeneration{}, codec.Manifest{}, err
8990 }
9091 if sha256.Sum256(data) != ref.ManifestDigest {
91- return ports.SnapshotGeneration{}, fmt.Errorf("manifest digest mismatch")
92+ return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("manifest digest mismatch")
9293 }
9394 manifest, err := codec.UnmarshalManifest(data)
9495 if err != nil || manifest.IncarnationID != id || manifest.Generation != ref.Generation {
95- return ports.SnapshotGeneration{}, fmt.Errorf("invalid manifest")
96+ return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid manifest")
9697 }
9798 refs := manifestRefs(manifest)
9899 if refs == nil || !withinGenerationBudget(len(data), refs) {
99- return ports.SnapshotGeneration{}, fmt.Errorf("snapshot generation too large")
100+ return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("snapshot generation too large")
100101 }
101102 objects := make(map[ports.SnapshotDigest][]byte, len(refs))
102103 for digest, objectRef := range refs {
103104 if err := ctx.Err(); err != nil {
104- return ports.SnapshotGeneration{}, err
105+ return ports.SnapshotGeneration{}, codec.Manifest{}, err
105106 }
106107 object, err := r.readBounded(r.objectPath(id, digest))
107108 if err != nil {
108- return ports.SnapshotGeneration{}, err
109+ return ports.SnapshotGeneration{}, codec.Manifest{}, err
109110 }
110111 if sha256.Sum256(object) != digest || !validObject(object, objectRef) {
111- return ports.SnapshotGeneration{}, fmt.Errorf("invalid object")
112+ return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid object")
112113 }
113114 objects[digest] = object
114115 }
115- return ports.SnapshotGeneration{IncarnationID: id, Name: name, Generation: ref.Generation, ParentCheckpoint: manifest.ParentCheckpoint, Manifest: data, Objects: objects}, nil
116+ return ports.SnapshotGeneration{IncarnationID: id, Name: name, Generation: ref.Generation, ParentCheckpoint: manifest.ParentCheckpoint, Manifest: data, Objects: objects}, manifest, nil
116117 }
117118
118119 func (r *Repository) RepairHEAD(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef) error {
@@ -125,19 +126,12 @@ func (r *Repository) RepairHEAD(ctx context.Context, id domain.IncarnationID, re
125126 }
126127 lock := r.lockSession(key)
127128 defer r.unlockSession(lock)
128- data, err := r.readBounded(r.manifestPath(id, ref.Generation))
129+ _, manifest, err := r.loadCheckpointLocked(ctx, id, "", ref)
129130 if err != nil {
130131 return fmt.Errorf("validate checkpoint before HEAD repair: %w", err)
131132 }
132- if sha256.Sum256(data) != ref.ManifestDigest {
133- return fmt.Errorf("validate checkpoint before HEAD repair: manifest digest mismatch")
134- }
135- manifest, err := codec.UnmarshalManifest(data)
136- if err != nil || manifest.IncarnationID != id || manifest.Generation != ref.Generation {
137- return fmt.Errorf("validate checkpoint before HEAD repair: invalid manifest")
138- }
139- if _, err := r.loadCheckpointLocked(ctx, id, manifest.Name, ref); err != nil {
140- return err
133+ if manifest.Name == "" {
134+ return fmt.Errorf("validate checkpoint before HEAD repair: invalid manifest name")
141135 }
142136 return r.writeMutable(r.headPath(id), marshalHead(ref.Generation, ref.ManifestDigest))
143137 }
@@ -76,43 +76,44 @@ func (r *Repository) LoadCheckpoint(ctx context.Context, id domain.IncarnationID
76 }76 }
77 lock := r.lockSession(key)77 lock := r.lockSession(key)
78 defer r.unlockSession(lock)78 defer r.unlockSession(lock)
79- return r.loadCheckpointLocked(ctx, id, name, ref)79+ generation, _, err := r.loadCheckpointLocked(ctx, id, name, ref)
80+ return generation, err
80 }81 }
81 82
82-func (r *Repository) loadCheckpointLocked(ctx context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) {83+func (r *Repository) loadCheckpointLocked(ctx context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, codec.Manifest, error) {
83 if ref.Generation == 0 || ref.ManifestDigest == ([32]byte{}) {84 if ref.Generation == 0 || ref.ManifestDigest == ([32]byte{}) {
84- return ports.SnapshotGeneration{}, fmt.Errorf("invalid checkpoint reference")85+ return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid checkpoint reference")
85 }86 }
86 data, err := r.readBounded(r.manifestPath(id, ref.Generation))87 data, err := r.readBounded(r.manifestPath(id, ref.Generation))
87 if err != nil {88 if err != nil {
88- return ports.SnapshotGeneration{}, err89+ return ports.SnapshotGeneration{}, codec.Manifest{}, err
89 }90 }
90 if sha256.Sum256(data) != ref.ManifestDigest {91 if sha256.Sum256(data) != ref.ManifestDigest {
91- return ports.SnapshotGeneration{}, fmt.Errorf("manifest digest mismatch")92+ return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("manifest digest mismatch")
92 }93 }
93 manifest, err := codec.UnmarshalManifest(data)94 manifest, err := codec.UnmarshalManifest(data)
94 if err != nil || manifest.IncarnationID != id || manifest.Generation != ref.Generation {95 if err != nil || manifest.IncarnationID != id || manifest.Generation != ref.Generation {
95- return ports.SnapshotGeneration{}, fmt.Errorf("invalid manifest")96+ return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid manifest")
96 }97 }
97 refs := manifestRefs(manifest)98 refs := manifestRefs(manifest)
98 if refs == nil || !withinGenerationBudget(len(data), refs) {99 if refs == nil || !withinGenerationBudget(len(data), refs) {
99- return ports.SnapshotGeneration{}, fmt.Errorf("snapshot generation too large")100+ return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("snapshot generation too large")
100 }101 }
101 objects := make(map[ports.SnapshotDigest][]byte, len(refs))102 objects := make(map[ports.SnapshotDigest][]byte, len(refs))
102 for digest, objectRef := range refs {103 for digest, objectRef := range refs {
103 if err := ctx.Err(); err != nil {104 if err := ctx.Err(); err != nil {
104- return ports.SnapshotGeneration{}, err105+ return ports.SnapshotGeneration{}, codec.Manifest{}, err
105 }106 }
106 object, err := r.readBounded(r.objectPath(id, digest))107 object, err := r.readBounded(r.objectPath(id, digest))
107 if err != nil {108 if err != nil {
108- return ports.SnapshotGeneration{}, err109+ return ports.SnapshotGeneration{}, codec.Manifest{}, err
109 }110 }
110 if sha256.Sum256(object) != digest || !validObject(object, objectRef) {111 if sha256.Sum256(object) != digest || !validObject(object, objectRef) {
111- return ports.SnapshotGeneration{}, fmt.Errorf("invalid object")112+ return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid object")
112 }113 }
113 objects[digest] = object114 objects[digest] = object
114 }115 }
115- return ports.SnapshotGeneration{IncarnationID: id, Name: name, Generation: ref.Generation, ParentCheckpoint: manifest.ParentCheckpoint, Manifest: data, Objects: objects}, nil116+ return ports.SnapshotGeneration{IncarnationID: id, Name: name, Generation: ref.Generation, ParentCheckpoint: manifest.ParentCheckpoint, Manifest: data, Objects: objects}, manifest, nil
116 }117 }
117 118
118 func (r *Repository) RepairHEAD(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef) error {119 func (r *Repository) RepairHEAD(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef) error {
@@ -125,19 +126,12 @@ func (r *Repository) RepairHEAD(ctx context.Context, id domain.IncarnationID, re
125 }126 }
126 lock := r.lockSession(key)127 lock := r.lockSession(key)
127 defer r.unlockSession(lock)128 defer r.unlockSession(lock)
128- data, err := r.readBounded(r.manifestPath(id, ref.Generation))129+ _, manifest, err := r.loadCheckpointLocked(ctx, id, "", ref)
129 if err != nil {130 if err != nil {
130 return fmt.Errorf("validate checkpoint before HEAD repair: %w", err)131 return fmt.Errorf("validate checkpoint before HEAD repair: %w", err)
131 }132 }
132- if sha256.Sum256(data) != ref.ManifestDigest {133+ if manifest.Name == "" {
133- return fmt.Errorf("validate checkpoint before HEAD repair: manifest digest mismatch")134+ return fmt.Errorf("validate checkpoint before HEAD repair: invalid manifest name")
134- }
135- manifest, err := codec.UnmarshalManifest(data)
136- if err != nil || manifest.IncarnationID != id || manifest.Generation != ref.Generation {
137- return fmt.Errorf("validate checkpoint before HEAD repair: invalid manifest")
138- }
139- if _, err := r.loadCheckpointLocked(ctx, id, manifest.Name, ref); err != nil {
140- return err
141 }135 }
142 return r.writeMutable(r.headPath(id), marshalHead(ref.Generation, ref.ManifestDigest))136 return r.writeMutable(r.headPath(id), marshalHead(ref.Generation, ref.ManifestDigest))
143 }137 }
M internal/adapters/snapshot/load_major_test.go +17 -1
@@ -3,6 +3,7 @@ package snapshot
33 import (
44 "context"
55 "os"
6+ "path/filepath"
67 "syscall"
78 "testing"
89
@@ -29,14 +30,27 @@ func TestRepositoryCurrentGenerationPropagatesOperationalHeadErrors(t *testing.T
2930 require.ErrorIs(t, err, syscall.EIO)
3031 }
3132
33+func TestRepositoryLoadUsesEstablishedLegacySessionKey(t *testing.T) {
34+ repo := NewRepository(privateDir(t))
35+ publication := repositoryPublication(t, "named", 1, []byte("legacy-state"))
36+ require.NoError(t, repo.Publish(context.Background(), publication))
37+ require.NoError(t, os.Rename(repo.sessionPath(publication.IncarnationID), filepath.Join(repo.dir, repositorySessionsDir, sessionKey(publication.Name))))
38+
39+ got, err := repo.Load(context.Background(), publication.Name)
40+ require.NoError(t, err)
41+ require.Equal(t, publication.Objects[0].Data, got.Objects[publication.Objects[0].Digest])
42+}
43+
3244 func TestRepositoryLoadFallsBackFromInvalidHead(t *testing.T) {
3345 repo := NewRepository(privateDir(t))
34- require.NoError(t, repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state"))))
46+ publication := repositoryPublication(t, "named", 1, []byte("state"))
47+ require.NoError(t, repo.Publish(context.Background(), publication))
3548 require.NoError(t, os.WriteFile(repo.legacyHeadPath(legacyIncarnationID("named").String()), []byte("invalid"), 0o600))
3649
3750 got, err := repo.Load(context.Background(), "named")
3851 require.NoError(t, err)
3952 require.Equal(t, uint64(1), got.Generation)
53+ require.Equal(t, publication.Objects[0].Data, got.Objects[publication.Objects[0].Digest])
4054 }
4155
4256 func TestRepositoryLoadFallsBackAcrossMultipleCorruptCandidates(t *testing.T) {
@@ -53,6 +67,8 @@ func TestRepositoryLoadFallsBackAcrossMultipleCorruptCandidates(t *testing.T) {
5367 got, err := repo.Load(context.Background(), "named")
5468 require.NoError(t, err)
5569 require.Equal(t, uint64(2), got.Generation)
70+ fallback := repositoryPublication(t, "named", 2, []byte{2})
71+ require.Equal(t, fallback.Objects[0].Data, got.Objects[fallback.Objects[0].Digest])
5672 }
5773
5874 func TestRepositoryLoadHoldsSessionLockAcrossTombstoneCheck(t *testing.T) {
@@ -3,6 +3,7 @@ package snapshot
3 import (3 import (
4 "context"4 "context"
5 "os"5 "os"
6+ "path/filepath"
6 "syscall"7 "syscall"
7 "testing"8 "testing"
8 9
@@ -29,14 +30,27 @@ func TestRepositoryCurrentGenerationPropagatesOperationalHeadErrors(t *testing.T
29 require.ErrorIs(t, err, syscall.EIO)30 require.ErrorIs(t, err, syscall.EIO)
30 }31 }
31 32
33+func TestRepositoryLoadUsesEstablishedLegacySessionKey(t *testing.T) {
34+ repo := NewRepository(privateDir(t))
35+ publication := repositoryPublication(t, "named", 1, []byte("legacy-state"))
36+ require.NoError(t, repo.Publish(context.Background(), publication))
37+ require.NoError(t, os.Rename(repo.sessionPath(publication.IncarnationID), filepath.Join(repo.dir, repositorySessionsDir, sessionKey(publication.Name))))
38+
39+ got, err := repo.Load(context.Background(), publication.Name)
40+ require.NoError(t, err)
41+ require.Equal(t, publication.Objects[0].Data, got.Objects[publication.Objects[0].Digest])
42+}
43+
32 func TestRepositoryLoadFallsBackFromInvalidHead(t *testing.T) {44 func TestRepositoryLoadFallsBackFromInvalidHead(t *testing.T) {
33 repo := NewRepository(privateDir(t))45 repo := NewRepository(privateDir(t))
34- require.NoError(t, repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state"))))46+ publication := repositoryPublication(t, "named", 1, []byte("state"))
47+ require.NoError(t, repo.Publish(context.Background(), publication))
35 require.NoError(t, os.WriteFile(repo.legacyHeadPath(legacyIncarnationID("named").String()), []byte("invalid"), 0o600))48 require.NoError(t, os.WriteFile(repo.legacyHeadPath(legacyIncarnationID("named").String()), []byte("invalid"), 0o600))
36 49
37 got, err := repo.Load(context.Background(), "named")50 got, err := repo.Load(context.Background(), "named")
38 require.NoError(t, err)51 require.NoError(t, err)
39 require.Equal(t, uint64(1), got.Generation)52 require.Equal(t, uint64(1), got.Generation)
53+ require.Equal(t, publication.Objects[0].Data, got.Objects[publication.Objects[0].Digest])
40 }54 }
41 55
42 func TestRepositoryLoadFallsBackAcrossMultipleCorruptCandidates(t *testing.T) {56 func TestRepositoryLoadFallsBackAcrossMultipleCorruptCandidates(t *testing.T) {
@@ -53,6 +67,8 @@ func TestRepositoryLoadFallsBackAcrossMultipleCorruptCandidates(t *testing.T) {
53 got, err := repo.Load(context.Background(), "named")67 got, err := repo.Load(context.Background(), "named")
54 require.NoError(t, err)68 require.NoError(t, err)
55 require.Equal(t, uint64(2), got.Generation)69 require.Equal(t, uint64(2), got.Generation)
70+ fallback := repositoryPublication(t, "named", 2, []byte{2})
71+ require.Equal(t, fallback.Objects[0].Data, got.Objects[fallback.Objects[0].Digest])
56 }72 }
57 73
58 func TestRepositoryLoadHoldsSessionLockAcrossTombstoneCheck(t *testing.T) {74 func TestRepositoryLoadHoldsSessionLockAcrossTombstoneCheck(t *testing.T) {
M internal/adapters/snapshot/maintenance_delete.go +110 -66
@@ -11,6 +11,7 @@ import (
1111 "io"
1212 "os"
1313 "path/filepath"
14+ "sort"
1415 "strings"
1516
1617 "github.com/bnema/vev/internal/domain"
@@ -94,13 +95,14 @@ const (
9495 )
9596
9697 type deletionListingState struct {
97- After string `json:"after,omitempty"`
98- Offset int64 `json:"offset,omitempty"`
99- Pending []maintenanceDirEntry `json:"pending,omitempty"`
100- PendingAt int `json:"pending_at,omitempty"`
101- ScanDone bool `json:"scan_done,omitempty"`
102- Candidate string `json:"candidate,omitempty"`
103- Found uint8 `json:"found,omitempty"`
98+ After string `json:"after,omitempty"`
99+ Offset int64 `json:"offset,omitempty"`
100+ Pending []maintenanceDirEntry `json:"pending,omitempty"`
101+ PendingAt int `json:"pending_at,omitempty"`
102+ ScanDone bool `json:"scan_done,omitempty"`
103+ Candidates []string `json:"candidates,omitempty"`
104+ More bool `json:"more,omitempty"`
105+ BatchLimit int `json:"batch_limit,omitempty"`
104106 }
105107
106108 // MarshalJSON keeps maintenanceDirEntry private while making a continuation
@@ -165,14 +167,16 @@ func validateDeletionListingState(state deletionListingState) error {
165167 if state.After != "" && !canonicalDeletionTombstoneFilename(state.After) {
166168 return fmt.Errorf("snapshot: non-canonical deletion tombstone continuation key")
167169 }
168- if state.Offset < 0 || state.PendingAt < 0 || state.PendingAt > len(state.Pending) || state.Found > 2 {
170+ if state.Offset < 0 || state.PendingAt < 0 || state.PendingAt > len(state.Pending) || state.BatchLimit < 0 || state.BatchLimit > maintenanceBatch || len(state.Candidates) > state.BatchLimit {
169171 return fmt.Errorf("snapshot: malformed deletion tombstone continuation")
170172 }
171- if state.After == "" && state.Offset == 0 && len(state.Pending) == 0 && !state.ScanDone && state.Candidate == "" {
173+ if state.After == "" && state.Offset == 0 && len(state.Pending) == 0 && !state.ScanDone && len(state.Candidates) == 0 {
172174 return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation")
173175 }
174- if (state.Candidate == "") != (state.Found == 0) || (state.Candidate != "" && (!canonicalDeletionTombstoneFilename(state.Candidate) || state.Candidate <= state.After)) {
175- return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation")
176+ for i, candidate := range state.Candidates {
177+ if !canonicalDeletionTombstoneFilename(candidate) || candidate <= state.After || (i > 0 && candidate <= state.Candidates[i-1]) {
178+ return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation")
179+ }
176180 }
177181 if len(state.Pending) > maintenanceDirentBufferSize {
178182 return fmt.Errorf("snapshot: malformed deletion tombstone continuation")
@@ -228,31 +232,56 @@ func (r *Repository) ListDeletionTombstones(ctx context.Context, cursor ports.De
228232 r.maintenanceMu.Lock()
229233 defer r.maintenanceMu.Unlock()
230234
231- entryLimit := budget.Entries
232- if maxInt := uint64(^uint(0) >> 1); entryLimit > maxInt {
233- entryLimit = maxInt
234- }
235- entries, done, err := r.readDeletionListingEntries(dir, int(entryLimit), &state)
236- if errors.Is(err, os.ErrNotExist) {
237- return ports.DeletionTombstonePage{Done: true}, nil
238- }
239- if err != nil {
240- return ports.DeletionTombstonePage{}, err
235+ batchLimit := int(min(budget.Entries, uint64(maintenanceBatch)))
236+ if state.BatchLimit == 0 {
237+ state.BatchLimit = batchLimit
238+ } else if state.BatchLimit > batchLimit {
239+ state.BatchLimit = batchLimit
240+ if len(state.Candidates) > state.BatchLimit {
241+ state.Candidates = state.Candidates[:state.BatchLimit]
242+ state.More = true
243+ }
241244 }
242- for _, entry := range entries {
243- if err := ctx.Err(); err != nil {
244- return ports.DeletionTombstonePage{}, err
245+ remainingEntries := budget.Entries
246+ done := state.ScanDone
247+ for !done && remainingEntries > 0 {
248+ limit := remainingEntries
249+ if maxInt := uint64(^uint(0) >> 1); limit > maxInt {
250+ limit = maxInt
245251 }
246- if entry.isDir || !canonicalDeletionTombstoneFilename(entry.name) {
247- return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: invalid deletion tombstone entry")
252+ entries, batchDone, err := r.readDeletionListingEntries(dir, int(limit), &state)
253+ if errors.Is(err, os.ErrNotExist) {
254+ return ports.DeletionTombstonePage{Done: true}, nil
248255 }
249- if entry.name > state.After {
250- if state.Found < 2 {
251- state.Found++
256+ if err != nil {
257+ return ports.DeletionTombstonePage{}, err
258+ }
259+ remainingEntries -= uint64(len(entries))
260+ for _, entry := range entries {
261+ if err := ctx.Err(); err != nil {
262+ return ports.DeletionTombstonePage{}, err
263+ }
264+ if entry.isDir || !canonicalDeletionTombstoneFilename(entry.name) {
265+ return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: invalid deletion tombstone entry")
266+ }
267+ if entry.name <= state.After {
268+ continue
252269 }
253- if state.Candidate == "" || entry.name < state.Candidate {
254- state.Candidate = entry.name
270+ at := sort.SearchStrings(state.Candidates, entry.name)
271+ if at < len(state.Candidates) && state.Candidates[at] == entry.name {
272+ continue
255273 }
274+ state.Candidates = append(state.Candidates, "")
275+ copy(state.Candidates[at+1:], state.Candidates[at:])
276+ state.Candidates[at] = entry.name
277+ if len(state.Candidates) > state.BatchLimit {
278+ state.Candidates = state.Candidates[:state.BatchLimit]
279+ state.More = true
280+ }
281+ }
282+ done = batchDone
283+ if len(entries) == 0 && !batchDone {
284+ break
256285 }
257286 }
258287 if !done {
@@ -265,41 +294,50 @@ func (r *Repository) ListDeletionTombstones(ctx context.Context, cursor ports.De
265294 }
266295 return ports.DeletionTombstonePage{Next: next}, nil
267296 }
268- if state.Candidate == "" {
297+ if len(state.Candidates) == 0 {
269298 return ports.DeletionTombstonePage{Done: true}, nil
270299 }
271- path := filepath.Join(dir, state.Candidate)
272- info, err := r.stat(path)
273- if err != nil {
274- return ports.DeletionTombstonePage{}, err
275- }
276- charge := uint64(len(state.Candidate))
277- if info.Size() < 0 || uint64(info.Size()) > budget.Bytes || charge > budget.Bytes-uint64(info.Size()) {
278- return ports.DeletionTombstonePage{}, ErrMaintenanceBudgetTooSmall
279- }
280- if hook := r.hooks.beforeDeletionTombstoneRead; hook != nil {
281- hook(path)
282- }
283- data, err := r.readBounded(path)
284- if err != nil {
285- return ports.DeletionTombstonePage{}, err
286- }
287- decoded, err := decodeDeletionTombstone(data)
288- if err != nil {
289- return ports.DeletionTombstonePage{}, err
290- }
291- if state.Candidate != decoded.IncarnationID.String()+".tombstone" {
292- return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: non-canonical deletion tombstone filename")
300+
301+ remainingBytes := budget.Bytes
302+ tombstones := make([]domain.DeletionTombstone, 0, len(state.Candidates))
303+ last := ""
304+ for _, candidate := range state.Candidates {
305+ path := filepath.Join(dir, candidate)
306+ info, err := r.stat(path)
307+ if err != nil {
308+ return ports.DeletionTombstonePage{}, err
309+ }
310+ charge := uint64(len(candidate))
311+ if info.Size() < 0 || uint64(info.Size()) > remainingBytes || charge > remainingBytes-uint64(info.Size()) {
312+ if len(tombstones) == 0 {
313+ return ports.DeletionTombstonePage{}, ErrMaintenanceBudgetTooSmall
314+ }
315+ state.More = true
316+ break
317+ }
318+ remainingBytes -= charge + uint64(info.Size())
319+ if hook := r.hooks.beforeDeletionTombstoneRead; hook != nil {
320+ hook(path)
321+ }
322+ data, err := r.readBounded(path)
323+ if err != nil {
324+ return ports.DeletionTombstonePage{}, err
325+ }
326+ decoded, err := decodeDeletionTombstone(data)
327+ if err != nil {
328+ return ports.DeletionTombstonePage{}, err
329+ }
330+ if candidate != decoded.IncarnationID.String()+".tombstone" {
331+ return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: non-canonical deletion tombstone filename")
332+ }
333+ tombstones = append(tombstones, decoded)
334+ last = candidate
293335 }
294- next, err := encodeDeletionListingCursor(deletionListingState{After: state.Candidate})
336+ next, err := encodeDeletionListingCursor(deletionListingState{After: last})
295337 if err != nil {
296338 return ports.DeletionTombstonePage{}, err
297339 }
298- return ports.DeletionTombstonePage{
299- Tombstones: []domain.DeletionTombstone{decoded},
300- Next: next,
301- Done: state.Found == 1,
302- }, nil
340+ return ports.DeletionTombstonePage{Tombstones: tombstones, Next: next, Done: !state.More && len(tombstones) == len(state.Candidates)}, nil
303341 }
304342
305343 func (r *Repository) DeleteDeletionTombstone(ctx context.Context, id domain.IncarnationID) error {
@@ -329,8 +367,7 @@ func (r *Repository) Delete(ctx context.Context, name string) error {
329367 if err := ctx.Err(); err != nil {
330368 return err
331369 }
332- id := legacyIncarnationID(name)
333- key, err := incarnationKey(id)
370+ key, err := r.legacyRepositoryKey(name)
334371 if err != nil {
335372 return err
336373 }
@@ -340,7 +377,7 @@ func (r *Repository) Delete(ctx context.Context, name string) error {
340377 return err
341378 }
342379
343- canonical := r.sessionPath(id)
380+ canonical := r.legacySessionPath(key)
344381 sessions := filepath.Dir(canonical)
345382 // A prior rename may have succeeded while its parent sync failed. Complete
346383 // that durability boundary before considering a canonical directory: this
@@ -520,6 +557,13 @@ func (r *Repository) QuarantineDeletionSources(ctx context.Context, tombstone do
520557 if err := ctx.Err(); err != nil {
521558 return err
522559 }
560+ key, err := incarnationKey(tombstone.IncarnationID)
561+ if err != nil {
562+ return err
563+ }
564+ lock := r.lockSession(key)
565+ defer r.unlockSession(lock)
566+ r.invalidateStorageEpoch(key)
523567 quarantineDir := filepath.Join(r.dir, "quarantine", tombstone.IncarnationID.String())
524568 if err := r.ensurePrivateDirectory(quarantineDir); err != nil {
525569 return err
@@ -546,8 +590,8 @@ func (r *Repository) QuarantineDeletionSources(ctx context.Context, tombstone do
546590 }
547591
548592 func (r *Repository) quarantineSource(source, target string) error {
549- if _, err := os.Lstat(target); err == nil {
550- if _, sourceErr := os.Lstat(source); errors.Is(sourceErr, os.ErrNotExist) {
593+ if _, err := r.stat(target); err == nil {
594+ if _, sourceErr := r.stat(source); errors.Is(sourceErr, os.ErrNotExist) {
551595 if err := r.syncDirectory(filepath.Dir(source)); err != nil {
552596 return err
553597 }
@@ -560,7 +604,7 @@ func (r *Repository) quarantineSource(source, target string) error {
560604 } else if !errors.Is(err, os.ErrNotExist) {
561605 return err
562606 }
563- if _, err := os.Lstat(source); errors.Is(err, os.ErrNotExist) {
607+ if _, err := r.stat(source); errors.Is(err, os.ErrNotExist) {
564608 return nil
565609 } else if err != nil {
566610 return err
@@ -11,6 +11,7 @@ import (
11 "io"11 "io"
12 "os"12 "os"
13 "path/filepath"13 "path/filepath"
14+ "sort"
14 "strings"15 "strings"
15 16
16 "github.com/bnema/vev/internal/domain"17 "github.com/bnema/vev/internal/domain"
@@ -94,13 +95,14 @@ const (
94 )95 )
95 96
96 type deletionListingState struct {97 type deletionListingState struct {
97- After string `json:"after,omitempty"`98+ After string `json:"after,omitempty"`
98- Offset int64 `json:"offset,omitempty"`99+ Offset int64 `json:"offset,omitempty"`
99- Pending []maintenanceDirEntry `json:"pending,omitempty"`100+ Pending []maintenanceDirEntry `json:"pending,omitempty"`
100- PendingAt int `json:"pending_at,omitempty"`101+ PendingAt int `json:"pending_at,omitempty"`
101- ScanDone bool `json:"scan_done,omitempty"`102+ ScanDone bool `json:"scan_done,omitempty"`
102- Candidate string `json:"candidate,omitempty"`103+ Candidates []string `json:"candidates,omitempty"`
103- Found uint8 `json:"found,omitempty"`104+ More bool `json:"more,omitempty"`
105+ BatchLimit int `json:"batch_limit,omitempty"`
104 }106 }
105 107
106 // MarshalJSON keeps maintenanceDirEntry private while making a continuation108 // MarshalJSON keeps maintenanceDirEntry private while making a continuation
@@ -165,14 +167,16 @@ func validateDeletionListingState(state deletionListingState) error {
165 if state.After != "" && !canonicalDeletionTombstoneFilename(state.After) {167 if state.After != "" && !canonicalDeletionTombstoneFilename(state.After) {
166 return fmt.Errorf("snapshot: non-canonical deletion tombstone continuation key")168 return fmt.Errorf("snapshot: non-canonical deletion tombstone continuation key")
167 }169 }
168- if state.Offset < 0 || state.PendingAt < 0 || state.PendingAt > len(state.Pending) || state.Found > 2 {170+ if state.Offset < 0 || state.PendingAt < 0 || state.PendingAt > len(state.Pending) || state.BatchLimit < 0 || state.BatchLimit > maintenanceBatch || len(state.Candidates) > state.BatchLimit {
169 return fmt.Errorf("snapshot: malformed deletion tombstone continuation")171 return fmt.Errorf("snapshot: malformed deletion tombstone continuation")
170 }172 }
171- if state.After == "" && state.Offset == 0 && len(state.Pending) == 0 && !state.ScanDone && state.Candidate == "" {173+ if state.After == "" && state.Offset == 0 && len(state.Pending) == 0 && !state.ScanDone && len(state.Candidates) == 0 {
172 return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation")174 return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation")
173 }175 }
174- if (state.Candidate == "") != (state.Found == 0) || (state.Candidate != "" && (!canonicalDeletionTombstoneFilename(state.Candidate) || state.Candidate <= state.After)) {176+ for i, candidate := range state.Candidates {
175- return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation")177+ if !canonicalDeletionTombstoneFilename(candidate) || candidate <= state.After || (i > 0 && candidate <= state.Candidates[i-1]) {
178+ return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation")
179+ }
176 }180 }
177 if len(state.Pending) > maintenanceDirentBufferSize {181 if len(state.Pending) > maintenanceDirentBufferSize {
178 return fmt.Errorf("snapshot: malformed deletion tombstone continuation")182 return fmt.Errorf("snapshot: malformed deletion tombstone continuation")
@@ -228,31 +232,56 @@ func (r *Repository) ListDeletionTombstones(ctx context.Context, cursor ports.De
228 r.maintenanceMu.Lock()232 r.maintenanceMu.Lock()
229 defer r.maintenanceMu.Unlock()233 defer r.maintenanceMu.Unlock()
230 234
231- entryLimit := budget.Entries235+ batchLimit := int(min(budget.Entries, uint64(maintenanceBatch)))
232- if maxInt := uint64(^uint(0) >> 1); entryLimit > maxInt {236+ if state.BatchLimit == 0 {
233- entryLimit = maxInt237+ state.BatchLimit = batchLimit
234- }238+ } else if state.BatchLimit > batchLimit {
235- entries, done, err := r.readDeletionListingEntries(dir, int(entryLimit), &state)239+ state.BatchLimit = batchLimit
236- if errors.Is(err, os.ErrNotExist) {240+ if len(state.Candidates) > state.BatchLimit {
237- return ports.DeletionTombstonePage{Done: true}, nil241+ state.Candidates = state.Candidates[:state.BatchLimit]
238- }242+ state.More = true
239- if err != nil {243+ }
240- return ports.DeletionTombstonePage{}, err
241 }244 }
242- for _, entry := range entries {245+ remainingEntries := budget.Entries
243- if err := ctx.Err(); err != nil {246+ done := state.ScanDone
244- return ports.DeletionTombstonePage{}, err247+ for !done && remainingEntries > 0 {
248+ limit := remainingEntries
249+ if maxInt := uint64(^uint(0) >> 1); limit > maxInt {
250+ limit = maxInt
245 }251 }
246- if entry.isDir || !canonicalDeletionTombstoneFilename(entry.name) {252+ entries, batchDone, err := r.readDeletionListingEntries(dir, int(limit), &state)
247- return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: invalid deletion tombstone entry")253+ if errors.Is(err, os.ErrNotExist) {
254+ return ports.DeletionTombstonePage{Done: true}, nil
248 }255 }
249- if entry.name > state.After {256+ if err != nil {
250- if state.Found < 2 {257+ return ports.DeletionTombstonePage{}, err
251- state.Found++258+ }
259+ remainingEntries -= uint64(len(entries))
260+ for _, entry := range entries {
261+ if err := ctx.Err(); err != nil {
262+ return ports.DeletionTombstonePage{}, err
263+ }
264+ if entry.isDir || !canonicalDeletionTombstoneFilename(entry.name) {
265+ return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: invalid deletion tombstone entry")
266+ }
267+ if entry.name <= state.After {
268+ continue
252 }269 }
253- if state.Candidate == "" || entry.name < state.Candidate {270+ at := sort.SearchStrings(state.Candidates, entry.name)
254- state.Candidate = entry.name271+ if at < len(state.Candidates) && state.Candidates[at] == entry.name {
272+ continue
255 }273 }
274+ state.Candidates = append(state.Candidates, "")
275+ copy(state.Candidates[at+1:], state.Candidates[at:])
276+ state.Candidates[at] = entry.name
277+ if len(state.Candidates) > state.BatchLimit {
278+ state.Candidates = state.Candidates[:state.BatchLimit]
279+ state.More = true
280+ }
281+ }
282+ done = batchDone
283+ if len(entries) == 0 && !batchDone {
284+ break
256 }285 }
257 }286 }
258 if !done {287 if !done {
@@ -265,41 +294,50 @@ func (r *Repository) ListDeletionTombstones(ctx context.Context, cursor ports.De
265 }294 }
266 return ports.DeletionTombstonePage{Next: next}, nil295 return ports.DeletionTombstonePage{Next: next}, nil
267 }296 }
268- if state.Candidate == "" {297+ if len(state.Candidates) == 0 {
269 return ports.DeletionTombstonePage{Done: true}, nil298 return ports.DeletionTombstonePage{Done: true}, nil
270 }299 }
271- path := filepath.Join(dir, state.Candidate)300+
272- info, err := r.stat(path)301+ remainingBytes := budget.Bytes
273- if err != nil {302+ tombstones := make([]domain.DeletionTombstone, 0, len(state.Candidates))
274- return ports.DeletionTombstonePage{}, err303+ last := ""
275- }304+ for _, candidate := range state.Candidates {
276- charge := uint64(len(state.Candidate))305+ path := filepath.Join(dir, candidate)
277- if info.Size() < 0 || uint64(info.Size()) > budget.Bytes || charge > budget.Bytes-uint64(info.Size()) {306+ info, err := r.stat(path)
278- return ports.DeletionTombstonePage{}, ErrMaintenanceBudgetTooSmall307+ if err != nil {
279- }308+ return ports.DeletionTombstonePage{}, err
280- if hook := r.hooks.beforeDeletionTombstoneRead; hook != nil {309+ }
281- hook(path)310+ charge := uint64(len(candidate))
282- }311+ if info.Size() < 0 || uint64(info.Size()) > remainingBytes || charge > remainingBytes-uint64(info.Size()) {
283- data, err := r.readBounded(path)312+ if len(tombstones) == 0 {
284- if err != nil {313+ return ports.DeletionTombstonePage{}, ErrMaintenanceBudgetTooSmall
285- return ports.DeletionTombstonePage{}, err314+ }
286- }315+ state.More = true
287- decoded, err := decodeDeletionTombstone(data)316+ break
288- if err != nil {317+ }
289- return ports.DeletionTombstonePage{}, err318+ remainingBytes -= charge + uint64(info.Size())
290- }319+ if hook := r.hooks.beforeDeletionTombstoneRead; hook != nil {
291- if state.Candidate != decoded.IncarnationID.String()+".tombstone" {320+ hook(path)
292- return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: non-canonical deletion tombstone filename")321+ }
322+ data, err := r.readBounded(path)
323+ if err != nil {
324+ return ports.DeletionTombstonePage{}, err
325+ }
326+ decoded, err := decodeDeletionTombstone(data)
327+ if err != nil {
328+ return ports.DeletionTombstonePage{}, err
329+ }
330+ if candidate != decoded.IncarnationID.String()+".tombstone" {
331+ return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: non-canonical deletion tombstone filename")
332+ }
333+ tombstones = append(tombstones, decoded)
334+ last = candidate
293 }335 }
294- next, err := encodeDeletionListingCursor(deletionListingState{After: state.Candidate})336+ next, err := encodeDeletionListingCursor(deletionListingState{After: last})
295 if err != nil {337 if err != nil {
296 return ports.DeletionTombstonePage{}, err338 return ports.DeletionTombstonePage{}, err
297 }339 }
298- return ports.DeletionTombstonePage{340+ return ports.DeletionTombstonePage{Tombstones: tombstones, Next: next, Done: !state.More && len(tombstones) == len(state.Candidates)}, nil
299- Tombstones: []domain.DeletionTombstone{decoded},
300- Next: next,
301- Done: state.Found == 1,
302- }, nil
303 }341 }
304 342
305 func (r *Repository) DeleteDeletionTombstone(ctx context.Context, id domain.IncarnationID) error {343 func (r *Repository) DeleteDeletionTombstone(ctx context.Context, id domain.IncarnationID) error {
@@ -329,8 +367,7 @@ func (r *Repository) Delete(ctx context.Context, name string) error {
329 if err := ctx.Err(); err != nil {367 if err := ctx.Err(); err != nil {
330 return err368 return err
331 }369 }
332- id := legacyIncarnationID(name)370+ key, err := r.legacyRepositoryKey(name)
333- key, err := incarnationKey(id)
334 if err != nil {371 if err != nil {
335 return err372 return err
336 }373 }
@@ -340,7 +377,7 @@ func (r *Repository) Delete(ctx context.Context, name string) error {
340 return err377 return err
341 }378 }
342 379
343- canonical := r.sessionPath(id)380+ canonical := r.legacySessionPath(key)
344 sessions := filepath.Dir(canonical)381 sessions := filepath.Dir(canonical)
345 // A prior rename may have succeeded while its parent sync failed. Complete382 // A prior rename may have succeeded while its parent sync failed. Complete
346 // that durability boundary before considering a canonical directory: this383 // that durability boundary before considering a canonical directory: this
@@ -520,6 +557,13 @@ func (r *Repository) QuarantineDeletionSources(ctx context.Context, tombstone do
520 if err := ctx.Err(); err != nil {557 if err := ctx.Err(); err != nil {
521 return err558 return err
522 }559 }
560+ key, err := incarnationKey(tombstone.IncarnationID)
561+ if err != nil {
562+ return err
563+ }
564+ lock := r.lockSession(key)
565+ defer r.unlockSession(lock)
566+ r.invalidateStorageEpoch(key)
523 quarantineDir := filepath.Join(r.dir, "quarantine", tombstone.IncarnationID.String())567 quarantineDir := filepath.Join(r.dir, "quarantine", tombstone.IncarnationID.String())
524 if err := r.ensurePrivateDirectory(quarantineDir); err != nil {568 if err := r.ensurePrivateDirectory(quarantineDir); err != nil {
525 return err569 return err
@@ -546,8 +590,8 @@ func (r *Repository) QuarantineDeletionSources(ctx context.Context, tombstone do
546 }590 }
547 591
548 func (r *Repository) quarantineSource(source, target string) error {592 func (r *Repository) quarantineSource(source, target string) error {
549- if _, err := os.Lstat(target); err == nil {593+ if _, err := r.stat(target); err == nil {
550- if _, sourceErr := os.Lstat(source); errors.Is(sourceErr, os.ErrNotExist) {594+ if _, sourceErr := r.stat(source); errors.Is(sourceErr, os.ErrNotExist) {
551 if err := r.syncDirectory(filepath.Dir(source)); err != nil {595 if err := r.syncDirectory(filepath.Dir(source)); err != nil {
552 return err596 return err
553 }597 }
@@ -560,7 +604,7 @@ func (r *Repository) quarantineSource(source, target string) error {
560 } else if !errors.Is(err, os.ErrNotExist) {604 } else if !errors.Is(err, os.ErrNotExist) {
561 return err605 return err
562 }606 }
563- if _, err := os.Lstat(source); errors.Is(err, os.ErrNotExist) {607+ if _, err := r.stat(source); errors.Is(err, os.ErrNotExist) {
564 return nil608 return nil
565 } else if err != nil {609 } else if err != nil {
566 return err610 return err
M internal/adapters/snapshot/maintenance_delete_test.go +92 -11
@@ -5,6 +5,7 @@ import (
55 "encoding/base64"
66 "encoding/hex"
77 "errors"
8+ "fmt"
89 "os"
910 "path/filepath"
1011 "reflect"
@@ -94,7 +95,7 @@ func TestDiscardCrashMatrix(t *testing.T) {
9495 repo := NewRepository(privateDir(t))
9596 source := repo.sessionPath(old)
9697 destinationDir := filepath.Join(repo.dir, "quarantine", old.String())
97- if err := os.MkdirAll(source, 0o700); err != nil {
98+ if err := repo.ensurePrivateDirectory(source); err != nil {
9899 t.Fatal(err)
99100 }
100101 if err := repo.ensurePrivateDirectory(destinationDir); err != nil {
@@ -139,7 +140,7 @@ func TestDiscardCrashMatrix(t *testing.T) {
139140 t.Run("quarantine never removes source data", func(t *testing.T) {
140141 repo := NewRepository(privateDir(t))
141142 source := repo.sessionPath(old)
142- if err := os.MkdirAll(source, 0o700); err != nil {
143+ if err := repo.ensurePrivateDirectory(source); err != nil {
143144 t.Fatal(err)
144145 }
145146 payload := filepath.Join(source, "payload")
@@ -163,6 +164,48 @@ func TestDiscardCrashMatrix(t *testing.T) {
163164 })
164165 }
165166
167+func TestQuarantineDeletionSourcesSerializesConcurrentPublication(t *testing.T) {
168+ repo := NewRepository(privateDir(t))
169+ id := domain.IncarnationID{9}
170+ tombstone := domain.DeletionTombstone{Name: "work", IncarnationID: id}
171+ publication := incarnationPublication(t, id, "work", 1, nil)
172+ if err := repo.Publish(context.Background(), publication); err != nil {
173+ t.Fatal(err)
174+ }
175+
176+ quarantined := make(chan struct{})
177+ release := make(chan struct{})
178+ repo.hooks.afterDeletionIncarnationQuarantine = func() error {
179+ close(quarantined)
180+ <-release
181+ return nil
182+ }
183+ quarantineDone := make(chan error, 1)
184+ go func() { quarantineDone <- repo.QuarantineDeletionSources(context.Background(), tombstone, false) }()
185+ <-quarantined
186+
187+ publishDone := make(chan error, 1)
188+ go func() { publishDone <- repo.Publish(context.Background(), publication) }()
189+ select {
190+ case err := <-publishDone:
191+ t.Fatalf("Publish completed while quarantine held the incarnation lock: %v", err)
192+ default:
193+ }
194+ close(release)
195+ if err := <-quarantineDone; err != nil {
196+ t.Fatal(err)
197+ }
198+ if err := <-publishDone; err != nil {
199+ t.Fatal(err)
200+ }
201+ if _, err := os.Stat(repo.headPath(id)); err != nil {
202+ t.Fatalf("republished HEAD: %v", err)
203+ }
204+ if _, err := os.Stat(filepath.Join(repo.dir, "quarantine", id.String(), "snapshot", repositoryHead)); err != nil {
205+ t.Fatalf("quarantined HEAD: %v", err)
206+ }
207+}
208+
166209 func TestDeletionTombstoneCodec(t *testing.T) {
167210 want := domain.DeletionTombstone{Name: "work", IncarnationID: domain.IncarnationID{1}}
168211 encoded, err := encodeDeletionTombstone(want)
@@ -283,6 +326,51 @@ func TestDeletionTombstoneListingResumesAcrossRepositoryRecreation(t *testing.T)
283326 }
284327 }
285328
329+func TestDeletionTombstoneListingBatchesSortedCandidatesWithinScanBudget(t *testing.T) {
330+ dir := privateDir(t)
331+ repo := NewRepository(dir)
332+ ctx := context.Background()
333+ want := make([]domain.DeletionTombstone, 5)
334+ for i := range want {
335+ want[i] = domain.DeletionTombstone{Name: fmt.Sprintf("session-%d", i), IncarnationID: domain.IncarnationID{15: byte(i + 1)}}
336+ }
337+ for _, i := range []int{4, 1, 3, 0, 2} {
338+ if err := repo.WriteDeletionTombstone(ctx, want[i]); err != nil {
339+ t.Fatal(err)
340+ }
341+ }
342+
343+ cursor := ports.DeletionTombstoneCursor{}
344+ var got []domain.DeletionTombstone
345+ reads := 0
346+ for calls := range 64 {
347+ repo = NewRepository(dir)
348+ repo.hooks.beforeDeletionTombstoneRead = func(string) { reads++ }
349+ entryBudget := uint64(2)
350+ if calls > 0 {
351+ entryBudget = 1 // A resumed caller may reduce its per-pass budget.
352+ }
353+ page, err := repo.ListDeletionTombstones(ctx, cursor, ports.MaintenanceBudget{Entries: entryBudget, Bytes: 4096})
354+ if err != nil {
355+ t.Fatal(err)
356+ }
357+ if uint64(len(page.Tombstones)) > entryBudget {
358+ t.Fatalf("page %d returned %d tombstones over entry budget %d", calls, len(page.Tombstones), entryBudget)
359+ }
360+ got = append(got, page.Tombstones...)
361+ if page.Done {
362+ break
363+ }
364+ cursor = page.Next
365+ }
366+ if !slices.Equal(got, want) {
367+ t.Fatalf("tombstones = %#v, want %#v", got, want)
368+ }
369+ if reads != len(want) {
370+ t.Fatalf("payload reads = %d, want %d", reads, len(want))
371+ }
372+}
373+
286374 func TestMalformedDeletionTombstone(t *testing.T) {
287375 dir := privateDir(t)
288376 repo := NewRepository(dir)
@@ -346,15 +434,8 @@ func TestDeletionTombstoneListing(t *testing.T) {
346434 if err != nil {
347435 t.Fatal(err)
348436 }
349- if page.Done || len(page.Tombstones) != 1 || page.Tombstones[0] != first || page.Next.After == "" {
350- t.Fatalf("first page = %#v", page)
351- }
352- page, err = repo.ListDeletionTombstones(ctx, page.Next, ports.MaintenanceBudget{Entries: 3, Bytes: 1024})
353- if err != nil {
354- t.Fatal(err)
355- }
356- if !page.Done || len(page.Tombstones) != 1 || page.Tombstones[0] != second || page.Next.After == "" {
357- t.Fatalf("second page = %#v", page)
437+ if !page.Done || !slices.Equal(page.Tombstones, []domain.DeletionTombstone{first, second}) || page.Next.After == "" {
438+ t.Fatalf("batched page = %#v", page)
358439 }
359440 if _, err := repo.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 3, Bytes: 1}); err == nil {
360441 t.Fatal("non-advancing budget accepted")
@@ -5,6 +5,7 @@ import (
5 "encoding/base64"5 "encoding/base64"
6 "encoding/hex"6 "encoding/hex"
7 "errors"7 "errors"
8+ "fmt"
8 "os"9 "os"
9 "path/filepath"10 "path/filepath"
10 "reflect"11 "reflect"
@@ -94,7 +95,7 @@ func TestDiscardCrashMatrix(t *testing.T) {
94 repo := NewRepository(privateDir(t))95 repo := NewRepository(privateDir(t))
95 source := repo.sessionPath(old)96 source := repo.sessionPath(old)
96 destinationDir := filepath.Join(repo.dir, "quarantine", old.String())97 destinationDir := filepath.Join(repo.dir, "quarantine", old.String())
97- if err := os.MkdirAll(source, 0o700); err != nil {98+ if err := repo.ensurePrivateDirectory(source); err != nil {
98 t.Fatal(err)99 t.Fatal(err)
99 }100 }
100 if err := repo.ensurePrivateDirectory(destinationDir); err != nil {101 if err := repo.ensurePrivateDirectory(destinationDir); err != nil {
@@ -139,7 +140,7 @@ func TestDiscardCrashMatrix(t *testing.T) {
139 t.Run("quarantine never removes source data", func(t *testing.T) {140 t.Run("quarantine never removes source data", func(t *testing.T) {
140 repo := NewRepository(privateDir(t))141 repo := NewRepository(privateDir(t))
141 source := repo.sessionPath(old)142 source := repo.sessionPath(old)
142- if err := os.MkdirAll(source, 0o700); err != nil {143+ if err := repo.ensurePrivateDirectory(source); err != nil {
143 t.Fatal(err)144 t.Fatal(err)
144 }145 }
145 payload := filepath.Join(source, "payload")146 payload := filepath.Join(source, "payload")
@@ -163,6 +164,48 @@ func TestDiscardCrashMatrix(t *testing.T) {
163 })164 })
164 }165 }
165 166
167+func TestQuarantineDeletionSourcesSerializesConcurrentPublication(t *testing.T) {
168+ repo := NewRepository(privateDir(t))
169+ id := domain.IncarnationID{9}
170+ tombstone := domain.DeletionTombstone{Name: "work", IncarnationID: id}
171+ publication := incarnationPublication(t, id, "work", 1, nil)
172+ if err := repo.Publish(context.Background(), publication); err != nil {
173+ t.Fatal(err)
174+ }
175+
176+ quarantined := make(chan struct{})
177+ release := make(chan struct{})
178+ repo.hooks.afterDeletionIncarnationQuarantine = func() error {
179+ close(quarantined)
180+ <-release
181+ return nil
182+ }
183+ quarantineDone := make(chan error, 1)
184+ go func() { quarantineDone <- repo.QuarantineDeletionSources(context.Background(), tombstone, false) }()
185+ <-quarantined
186+
187+ publishDone := make(chan error, 1)
188+ go func() { publishDone <- repo.Publish(context.Background(), publication) }()
189+ select {
190+ case err := <-publishDone:
191+ t.Fatalf("Publish completed while quarantine held the incarnation lock: %v", err)
192+ default:
193+ }
194+ close(release)
195+ if err := <-quarantineDone; err != nil {
196+ t.Fatal(err)
197+ }
198+ if err := <-publishDone; err != nil {
199+ t.Fatal(err)
200+ }
201+ if _, err := os.Stat(repo.headPath(id)); err != nil {
202+ t.Fatalf("republished HEAD: %v", err)
203+ }
204+ if _, err := os.Stat(filepath.Join(repo.dir, "quarantine", id.String(), "snapshot", repositoryHead)); err != nil {
205+ t.Fatalf("quarantined HEAD: %v", err)
206+ }
207+}
208+
166 func TestDeletionTombstoneCodec(t *testing.T) {209 func TestDeletionTombstoneCodec(t *testing.T) {
167 want := domain.DeletionTombstone{Name: "work", IncarnationID: domain.IncarnationID{1}}210 want := domain.DeletionTombstone{Name: "work", IncarnationID: domain.IncarnationID{1}}
168 encoded, err := encodeDeletionTombstone(want)211 encoded, err := encodeDeletionTombstone(want)
@@ -283,6 +326,51 @@ func TestDeletionTombstoneListingResumesAcrossRepositoryRecreation(t *testing.T)
283 }326 }
284 }327 }
285 328
329+func TestDeletionTombstoneListingBatchesSortedCandidatesWithinScanBudget(t *testing.T) {
330+ dir := privateDir(t)
331+ repo := NewRepository(dir)
332+ ctx := context.Background()
333+ want := make([]domain.DeletionTombstone, 5)
334+ for i := range want {
335+ want[i] = domain.DeletionTombstone{Name: fmt.Sprintf("session-%d", i), IncarnationID: domain.IncarnationID{15: byte(i + 1)}}
336+ }
337+ for _, i := range []int{4, 1, 3, 0, 2} {
338+ if err := repo.WriteDeletionTombstone(ctx, want[i]); err != nil {
339+ t.Fatal(err)
340+ }
341+ }
342+
343+ cursor := ports.DeletionTombstoneCursor{}
344+ var got []domain.DeletionTombstone
345+ reads := 0
346+ for calls := range 64 {
347+ repo = NewRepository(dir)
348+ repo.hooks.beforeDeletionTombstoneRead = func(string) { reads++ }
349+ entryBudget := uint64(2)
350+ if calls > 0 {
351+ entryBudget = 1 // A resumed caller may reduce its per-pass budget.
352+ }
353+ page, err := repo.ListDeletionTombstones(ctx, cursor, ports.MaintenanceBudget{Entries: entryBudget, Bytes: 4096})
354+ if err != nil {
355+ t.Fatal(err)
356+ }
357+ if uint64(len(page.Tombstones)) > entryBudget {
358+ t.Fatalf("page %d returned %d tombstones over entry budget %d", calls, len(page.Tombstones), entryBudget)
359+ }
360+ got = append(got, page.Tombstones...)
361+ if page.Done {
362+ break
363+ }
364+ cursor = page.Next
365+ }
366+ if !slices.Equal(got, want) {
367+ t.Fatalf("tombstones = %#v, want %#v", got, want)
368+ }
369+ if reads != len(want) {
370+ t.Fatalf("payload reads = %d, want %d", reads, len(want))
371+ }
372+}
373+
286 func TestMalformedDeletionTombstone(t *testing.T) {374 func TestMalformedDeletionTombstone(t *testing.T) {
287 dir := privateDir(t)375 dir := privateDir(t)
288 repo := NewRepository(dir)376 repo := NewRepository(dir)
@@ -346,15 +434,8 @@ func TestDeletionTombstoneListing(t *testing.T) {
346 if err != nil {434 if err != nil {
347 t.Fatal(err)435 t.Fatal(err)
348 }436 }
349- if page.Done || len(page.Tombstones) != 1 || page.Tombstones[0] != first || page.Next.After == "" {437+ if !page.Done || !slices.Equal(page.Tombstones, []domain.DeletionTombstone{first, second}) || page.Next.After == "" {
350- t.Fatalf("first page = %#v", page)438+ t.Fatalf("batched page = %#v", page)
351- }
352- page, err = repo.ListDeletionTombstones(ctx, page.Next, ports.MaintenanceBudget{Entries: 3, Bytes: 1024})
353- if err != nil {
354- t.Fatal(err)
355- }
356- if !page.Done || len(page.Tombstones) != 1 || page.Tombstones[0] != second || page.Next.After == "" {
357- t.Fatalf("second page = %#v", page)
358 }439 }
359 if _, err := repo.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 3, Bytes: 1}); err == nil {440 if _, err := repo.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 3, Bytes: 1}); err == nil {
360 t.Fatal("non-advancing budget accepted")441 t.Fatal("non-advancing budget accepted")
M internal/adapters/snapshot/maintenance_gc.go +79 -21
@@ -17,6 +17,7 @@ import (
1717
1818 type retentionMaintenance struct {
1919 token string
20+ cursors map[string]*maintenanceCursor
2021 references map[ports.SnapshotDigest]struct{}
2122 validateQueue []ports.CheckpointRef
2223 validateIndex int
@@ -403,7 +404,7 @@ func keepSet(plan ports.RetentionPlan) map[uint64]ports.SnapshotDigest {
403404 // checkpoints. Every incarnation owns its budget and continuation state.
404405 // processRetentionEntries charges stat-reported size for every entry, requeues
405406 // the unadmitted suffix, and removes only canonical entries rejected by policy.
406-func (r *Repository) processRetentionEntries(ctx context.Context, dir, purpose string, entries []maintenanceDirEntry, budget *ports.MaintenanceBudget, consumedBefore bool, policy retentionEntryPolicy) (bool, error) {
407+func (r *Repository) processRetentionEntries(ctx context.Context, state *retentionMaintenance, dir, purpose string, entries []maintenanceDirEntry, budget *ports.MaintenanceBudget, consumedBefore bool, policy retentionEntryPolicy) (bool, error) {
407408 for i, entry := range entries {
408409 if err := ctx.Err(); err != nil {
409410 return false, err
@@ -418,7 +419,7 @@ func (r *Repository) processRetentionEntries(ctx context.Context, dir, purpose s
418419 size = uint64(info.Size())
419420 }
420421 if budget.Entries == 0 || size > budget.Bytes {
421- r.requeueMaintenanceEntries(dir, purpose, entries[i:])
422+ r.requeueRetentionEntries(state, dir, purpose, entries[i:])
422423 if !consumedBefore && i == 0 {
423424 return false, ErrMaintenanceBudgetTooSmall
424425 }
@@ -462,8 +463,13 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
462463 )
463464 }()
464465 if plan.PinAll {
466+ // Lock order is maintenanceMu -> session mutex. Pinning the same
467+ // incarnation waits for its active retention pass, while unrelated
468+ // incarnations never wait for that pass's payload I/O.
465469 r.maintenanceMu.Lock()
470+ lock := r.lockSession(key)
466471 r.clearRetentionMaintenance(key)
472+ r.unlockSession(lock)
467473 r.maintenanceMu.Unlock()
468474 return true, nil
469475 }
@@ -471,13 +477,11 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
471477 return false, nil
472478 }
473479
480+ // Establish per-incarnation state under maintenanceMu, then retain only the
481+ // session mutex across payload I/O. Never reacquire maintenanceMu while the
482+ // session mutex is held; PinAll relies on maintenanceMu -> session ordering.
474483 r.maintenanceMu.Lock()
475- defer r.maintenanceMu.Unlock()
476484 lock := r.lockSession(key)
477- defer r.unlockSession(lock)
478- if r.maintenanceCursors == nil {
479- r.maintenanceCursors = make(map[string]*maintenanceCursor)
480- }
481485 if r.retentionSessions == nil {
482486 r.retentionSessions = make(map[string]*retentionMaintenance)
483487 }
@@ -488,11 +492,23 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
488492 r.clearRetentionMaintenance(key)
489493 state = &retentionMaintenance{
490494 token: token,
495+ cursors: make(map[string]*maintenanceCursor),
491496 references: make(map[ports.SnapshotDigest]struct{}),
492497 validateQueue: append([]ports.CheckpointRef(nil), plan.Keep...),
493498 }
494499 r.retentionSessions[key] = state
495500 }
501+ r.maintenanceMu.Unlock()
502+ defer func() {
503+ r.unlockSession(lock)
504+ if done {
505+ r.maintenanceMu.Lock()
506+ if r.retentionSessions[key] == state {
507+ delete(r.retentionSessions, key)
508+ }
509+ r.maintenanceMu.Unlock()
510+ }
511+ }()
496512 keep := keepSet(plan)
497513 if !state.validated {
498514 for state.validateIndex < len(state.validateQueue) {
@@ -505,6 +521,9 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
505521 return false, fmt.Errorf("validate retained manifest %d: %w", ref.Generation, err)
506522 }
507523 if !admitted {
524+ if requested.Entries == budget.Entries && requested.Bytes == budget.Bytes {
525+ return false, ErrMaintenanceBudgetTooSmall
526+ }
508527 return false, nil
509528 }
510529 budget.Entries--
@@ -520,7 +539,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
520539 purpose := "retention-manifests:" + key + ":" + token
521540 for !state.manifestsDone && budget.Entries > 0 && budget.Bytes > 0 {
522541 limit := int(min(budget.Entries, uint64(maintenanceBatch)))
523- entries, done, err := r.readMaintenanceDir(filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations), limit, purpose)
542+ entries, done, err := r.readRetentionDir(state, filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations), limit, purpose)
524543 if errors.Is(err, os.ErrNotExist) {
525544 state.manifestsDone = true
526545 break
@@ -530,7 +549,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
530549 }
531550 dir := filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations)
532551 consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes
533- admitted, err := r.processRetentionEntries(ctx, dir, purpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionManifest, generations: keep})
552+ admitted, err := r.processRetentionEntries(ctx, state, dir, purpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionManifest, generations: keep})
534553 if err != nil {
535554 return false, err
536555 }
@@ -552,7 +571,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
552571 shard := state.objectShard
553572 dir := filepath.Join(objectsRoot, shard)
554573 objectPurpose := "retention-objects:" + key + ":" + shard + ":" + token
555- entries, done, err := r.readMaintenanceDir(dir, int(min(budget.Entries, uint64(maintenanceBatch))), objectPurpose)
574+ entries, done, err := r.readRetentionDir(state, dir, int(min(budget.Entries, uint64(maintenanceBatch))), objectPurpose)
556575 if errors.Is(err, os.ErrNotExist) {
557576 state.objectShard = ""
558577 continue
@@ -561,7 +580,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
561580 return false, err
562581 }
563582 consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes
564- admitted, err := r.processRetentionEntries(ctx, dir, objectPurpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionObject, objects: state.references})
583+ admitted, err := r.processRetentionEntries(ctx, state, dir, objectPurpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionObject, objects: state.references})
565584 if err != nil {
566585 return false, err
567586 }
@@ -576,23 +595,25 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
576595 continue
577596 }
578597 if state.objectRootDone {
579- delete(r.retentionSessions, key)
580598 return true, nil
581599 }
582- entries, done, err := r.readMaintenanceDir(objectsRoot, 1, "retention-shards:"+key+":"+token)
600+ entries, done, err := r.readRetentionDir(state, objectsRoot, 1, "retention-shards:"+key+":"+token)
583601 if errors.Is(err, os.ErrNotExist) {
584- delete(r.retentionSessions, key)
585602 return true, nil
586603 }
587604 if err != nil {
588605 return false, err
589606 }
590607 state.objectRootDone = done
591- if len(entries) != 0 {
592- budget.Entries--
593- if entries[0].isDir {
594- state.objectShard = entries[0].name
608+ if len(entries) == 0 {
609+ if !done {
610+ return false, nil
595611 }
612+ continue
613+ }
614+ budget.Entries--
615+ if entries[0].isDir {
616+ state.objectShard = entries[0].name
596617 }
597618 }
598619 return false, nil
@@ -600,11 +621,48 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
600621
601622 func (r *Repository) clearRetentionMaintenance(key string) {
602623 delete(r.retentionSessions, key)
603- for id := range r.maintenanceCursors {
604- if strings.Contains(id, "retention-") && strings.Contains(id, key) {
605- delete(r.maintenanceCursors, id)
624+}
625+
626+func (r *Repository) readRetentionDir(state *retentionMaintenance, dir string, n int, purpose string) ([]maintenanceDirEntry, bool, error) {
627+ id := maintenanceCursorID(dir, purpose)
628+ cursor := state.cursors[id]
629+ if cursor != nil && cursor.pending != nil {
630+ count := min(n, len(cursor.pending))
631+ entries := cursor.pending[:count]
632+ cursor.pending = cursor.pending[count:]
633+ done := cursor.done && len(cursor.pending) == 0
634+ if done {
635+ delete(state.cursors, id)
636+ } else if count > 0 && len(cursor.pending) == 0 {
637+ cursor.pending = nil
638+ }
639+ return entries, done, nil
640+ }
641+ if cursor == nil {
642+ if len(state.cursors) >= maxMaintenanceCursors {
643+ return nil, false, errMaintenanceCursorLimit
606644 }
645+ cursor = &maintenanceCursor{}
646+ state.cursors[id] = cursor
647+ }
648+ entries, done, err := r.readMaintenanceDirent(dir, n, cursor)
649+ if err != nil || done {
650+ delete(state.cursors, id)
651+ }
652+ return entries, done, err
653+}
654+
655+func (r *Repository) requeueRetentionEntries(state *retentionMaintenance, dir, purpose string, entries []maintenanceDirEntry) {
656+ if len(entries) == 0 {
657+ return
658+ }
659+ id := maintenanceCursorID(dir, purpose)
660+ cursor := state.cursors[id]
661+ if cursor == nil {
662+ cursor = &maintenanceCursor{done: true}
663+ state.cursors[id] = cursor
607664 }
665+ cursor.pending = append(entries, cursor.pending...)
608666 }
609667
610668 func retentionToken(plan ports.RetentionPlan) string {
@@ -17,6 +17,7 @@ import (
17 17
18 type retentionMaintenance struct {18 type retentionMaintenance struct {
19 token string19 token string
20+ cursors map[string]*maintenanceCursor
20 references map[ports.SnapshotDigest]struct{}21 references map[ports.SnapshotDigest]struct{}
21 validateQueue []ports.CheckpointRef22 validateQueue []ports.CheckpointRef
22 validateIndex int23 validateIndex int
@@ -403,7 +404,7 @@ func keepSet(plan ports.RetentionPlan) map[uint64]ports.SnapshotDigest {
403 // checkpoints. Every incarnation owns its budget and continuation state.404 // checkpoints. Every incarnation owns its budget and continuation state.
404 // processRetentionEntries charges stat-reported size for every entry, requeues405 // processRetentionEntries charges stat-reported size for every entry, requeues
405 // the unadmitted suffix, and removes only canonical entries rejected by policy.406 // the unadmitted suffix, and removes only canonical entries rejected by policy.
406-func (r *Repository) processRetentionEntries(ctx context.Context, dir, purpose string, entries []maintenanceDirEntry, budget *ports.MaintenanceBudget, consumedBefore bool, policy retentionEntryPolicy) (bool, error) {407+func (r *Repository) processRetentionEntries(ctx context.Context, state *retentionMaintenance, dir, purpose string, entries []maintenanceDirEntry, budget *ports.MaintenanceBudget, consumedBefore bool, policy retentionEntryPolicy) (bool, error) {
407 for i, entry := range entries {408 for i, entry := range entries {
408 if err := ctx.Err(); err != nil {409 if err := ctx.Err(); err != nil {
409 return false, err410 return false, err
@@ -418,7 +419,7 @@ func (r *Repository) processRetentionEntries(ctx context.Context, dir, purpose s
418 size = uint64(info.Size())419 size = uint64(info.Size())
419 }420 }
420 if budget.Entries == 0 || size > budget.Bytes {421 if budget.Entries == 0 || size > budget.Bytes {
421- r.requeueMaintenanceEntries(dir, purpose, entries[i:])422+ r.requeueRetentionEntries(state, dir, purpose, entries[i:])
422 if !consumedBefore && i == 0 {423 if !consumedBefore && i == 0 {
423 return false, ErrMaintenanceBudgetTooSmall424 return false, ErrMaintenanceBudgetTooSmall
424 }425 }
@@ -462,8 +463,13 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
462 )463 )
463 }()464 }()
464 if plan.PinAll {465 if plan.PinAll {
466+ // Lock order is maintenanceMu -> session mutex. Pinning the same
467+ // incarnation waits for its active retention pass, while unrelated
468+ // incarnations never wait for that pass's payload I/O.
465 r.maintenanceMu.Lock()469 r.maintenanceMu.Lock()
470+ lock := r.lockSession(key)
466 r.clearRetentionMaintenance(key)471 r.clearRetentionMaintenance(key)
472+ r.unlockSession(lock)
467 r.maintenanceMu.Unlock()473 r.maintenanceMu.Unlock()
468 return true, nil474 return true, nil
469 }475 }
@@ -471,13 +477,11 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
471 return false, nil477 return false, nil
472 }478 }
473 479
480+ // Establish per-incarnation state under maintenanceMu, then retain only the
481+ // session mutex across payload I/O. Never reacquire maintenanceMu while the
482+ // session mutex is held; PinAll relies on maintenanceMu -> session ordering.
474 r.maintenanceMu.Lock()483 r.maintenanceMu.Lock()
475- defer r.maintenanceMu.Unlock()
476 lock := r.lockSession(key)484 lock := r.lockSession(key)
477- defer r.unlockSession(lock)
478- if r.maintenanceCursors == nil {
479- r.maintenanceCursors = make(map[string]*maintenanceCursor)
480- }
481 if r.retentionSessions == nil {485 if r.retentionSessions == nil {
482 r.retentionSessions = make(map[string]*retentionMaintenance)486 r.retentionSessions = make(map[string]*retentionMaintenance)
483 }487 }
@@ -488,11 +492,23 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
488 r.clearRetentionMaintenance(key)492 r.clearRetentionMaintenance(key)
489 state = &retentionMaintenance{493 state = &retentionMaintenance{
490 token: token,494 token: token,
495+ cursors: make(map[string]*maintenanceCursor),
491 references: make(map[ports.SnapshotDigest]struct{}),496 references: make(map[ports.SnapshotDigest]struct{}),
492 validateQueue: append([]ports.CheckpointRef(nil), plan.Keep...),497 validateQueue: append([]ports.CheckpointRef(nil), plan.Keep...),
493 }498 }
494 r.retentionSessions[key] = state499 r.retentionSessions[key] = state
495 }500 }
501+ r.maintenanceMu.Unlock()
502+ defer func() {
503+ r.unlockSession(lock)
504+ if done {
505+ r.maintenanceMu.Lock()
506+ if r.retentionSessions[key] == state {
507+ delete(r.retentionSessions, key)
508+ }
509+ r.maintenanceMu.Unlock()
510+ }
511+ }()
496 keep := keepSet(plan)512 keep := keepSet(plan)
497 if !state.validated {513 if !state.validated {
498 for state.validateIndex < len(state.validateQueue) {514 for state.validateIndex < len(state.validateQueue) {
@@ -505,6 +521,9 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
505 return false, fmt.Errorf("validate retained manifest %d: %w", ref.Generation, err)521 return false, fmt.Errorf("validate retained manifest %d: %w", ref.Generation, err)
506 }522 }
507 if !admitted {523 if !admitted {
524+ if requested.Entries == budget.Entries && requested.Bytes == budget.Bytes {
525+ return false, ErrMaintenanceBudgetTooSmall
526+ }
508 return false, nil527 return false, nil
509 }528 }
510 budget.Entries--529 budget.Entries--
@@ -520,7 +539,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
520 purpose := "retention-manifests:" + key + ":" + token539 purpose := "retention-manifests:" + key + ":" + token
521 for !state.manifestsDone && budget.Entries > 0 && budget.Bytes > 0 {540 for !state.manifestsDone && budget.Entries > 0 && budget.Bytes > 0 {
522 limit := int(min(budget.Entries, uint64(maintenanceBatch)))541 limit := int(min(budget.Entries, uint64(maintenanceBatch)))
523- entries, done, err := r.readMaintenanceDir(filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations), limit, purpose)542+ entries, done, err := r.readRetentionDir(state, filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations), limit, purpose)
524 if errors.Is(err, os.ErrNotExist) {543 if errors.Is(err, os.ErrNotExist) {
525 state.manifestsDone = true544 state.manifestsDone = true
526 break545 break
@@ -530,7 +549,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
530 }549 }
531 dir := filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations)550 dir := filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations)
532 consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes551 consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes
533- admitted, err := r.processRetentionEntries(ctx, dir, purpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionManifest, generations: keep})552+ admitted, err := r.processRetentionEntries(ctx, state, dir, purpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionManifest, generations: keep})
534 if err != nil {553 if err != nil {
535 return false, err554 return false, err
536 }555 }
@@ -552,7 +571,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
552 shard := state.objectShard571 shard := state.objectShard
553 dir := filepath.Join(objectsRoot, shard)572 dir := filepath.Join(objectsRoot, shard)
554 objectPurpose := "retention-objects:" + key + ":" + shard + ":" + token573 objectPurpose := "retention-objects:" + key + ":" + shard + ":" + token
555- entries, done, err := r.readMaintenanceDir(dir, int(min(budget.Entries, uint64(maintenanceBatch))), objectPurpose)574+ entries, done, err := r.readRetentionDir(state, dir, int(min(budget.Entries, uint64(maintenanceBatch))), objectPurpose)
556 if errors.Is(err, os.ErrNotExist) {575 if errors.Is(err, os.ErrNotExist) {
557 state.objectShard = ""576 state.objectShard = ""
558 continue577 continue
@@ -561,7 +580,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
561 return false, err580 return false, err
562 }581 }
563 consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes582 consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes
564- admitted, err := r.processRetentionEntries(ctx, dir, objectPurpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionObject, objects: state.references})583+ admitted, err := r.processRetentionEntries(ctx, state, dir, objectPurpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionObject, objects: state.references})
565 if err != nil {584 if err != nil {
566 return false, err585 return false, err
567 }586 }
@@ -576,23 +595,25 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
576 continue595 continue
577 }596 }
578 if state.objectRootDone {597 if state.objectRootDone {
579- delete(r.retentionSessions, key)
580 return true, nil598 return true, nil
581 }599 }
582- entries, done, err := r.readMaintenanceDir(objectsRoot, 1, "retention-shards:"+key+":"+token)600+ entries, done, err := r.readRetentionDir(state, objectsRoot, 1, "retention-shards:"+key+":"+token)
583 if errors.Is(err, os.ErrNotExist) {601 if errors.Is(err, os.ErrNotExist) {
584- delete(r.retentionSessions, key)
585 return true, nil602 return true, nil
586 }603 }
587 if err != nil {604 if err != nil {
588 return false, err605 return false, err
589 }606 }
590 state.objectRootDone = done607 state.objectRootDone = done
591- if len(entries) != 0 {608+ if len(entries) == 0 {
592- budget.Entries--609+ if !done {
593- if entries[0].isDir {610+ return false, nil
594- state.objectShard = entries[0].name
595 }611 }
612+ continue
613+ }
614+ budget.Entries--
615+ if entries[0].isDir {
616+ state.objectShard = entries[0].name
596 }617 }
597 }618 }
598 return false, nil619 return false, nil
@@ -600,11 +621,48 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
600 621
601 func (r *Repository) clearRetentionMaintenance(key string) {622 func (r *Repository) clearRetentionMaintenance(key string) {
602 delete(r.retentionSessions, key)623 delete(r.retentionSessions, key)
603- for id := range r.maintenanceCursors {624+}
604- if strings.Contains(id, "retention-") && strings.Contains(id, key) {625+
605- delete(r.maintenanceCursors, id)626+func (r *Repository) readRetentionDir(state *retentionMaintenance, dir string, n int, purpose string) ([]maintenanceDirEntry, bool, error) {
627+ id := maintenanceCursorID(dir, purpose)
628+ cursor := state.cursors[id]
629+ if cursor != nil && cursor.pending != nil {
630+ count := min(n, len(cursor.pending))
631+ entries := cursor.pending[:count]
632+ cursor.pending = cursor.pending[count:]
633+ done := cursor.done && len(cursor.pending) == 0
634+ if done {
635+ delete(state.cursors, id)
636+ } else if count > 0 && len(cursor.pending) == 0 {
637+ cursor.pending = nil
638+ }
639+ return entries, done, nil
640+ }
641+ if cursor == nil {
642+ if len(state.cursors) >= maxMaintenanceCursors {
643+ return nil, false, errMaintenanceCursorLimit
606 }644 }
645+ cursor = &maintenanceCursor{}
646+ state.cursors[id] = cursor
647+ }
648+ entries, done, err := r.readMaintenanceDirent(dir, n, cursor)
649+ if err != nil || done {
650+ delete(state.cursors, id)
651+ }
652+ return entries, done, err
653+}
654+
655+func (r *Repository) requeueRetentionEntries(state *retentionMaintenance, dir, purpose string, entries []maintenanceDirEntry) {
656+ if len(entries) == 0 {
657+ return
658+ }
659+ id := maintenanceCursorID(dir, purpose)
660+ cursor := state.cursors[id]
661+ if cursor == nil {
662+ cursor = &maintenanceCursor{done: true}
663+ state.cursors[id] = cursor
607 }664 }
665+ cursor.pending = append(entries, cursor.pending...)
608 }666 }
609 667
610 func retentionToken(plan ports.RetentionPlan) string {668 func retentionToken(plan ports.RetentionPlan) string {
M internal/adapters/snapshot/maintenance_test.go +107 -33
@@ -383,7 +383,7 @@ func TestRepositoryMaintainRetainsNewestTwoCompleteGenerations(t *testing.T) {
383383 }
384384 }
385385 key := legacyIncarnationID("named").String()
386- for pass := 0; pass < maintenanceBatch; pass++ {
386+ for pass := range maintenanceBatch {
387387 if err := repo.Maintain(context.Background()); err != nil {
388388 t.Fatal(err)
389389 }
@@ -551,7 +551,7 @@ func TestRepositoryDeleteRetriesPendingQuarantineSyncWithoutDeletingRecreatedSes
551551 func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) {
552552 repo := NewRepository(privateDir(t))
553553 quarantine := filepath.Join(repo.dir, repositorySessionsDir, ".deleting-named-test")
554- for i := 0; i < maintenanceBatch+1; i++ {
554+ for i := range maintenanceBatch + 1 {
555555 path := filepath.Join(quarantine, "nested", string(rune('a'+i)))
556556 if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
557557 t.Fatal(err)
@@ -566,7 +566,7 @@ func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) {
566566 if _, err := os.Lstat(quarantine); errors.Is(err, os.ErrNotExist) {
567567 t.Fatalf("quarantine after one bounded call = %v, want remaining tree", err)
568568 }
569- for i := 0; i < 12; i++ {
569+ for range 12 {
570570 if err := repo.Maintain(context.Background()); err != nil {
571571 t.Fatal(err)
572572 }
@@ -582,12 +582,12 @@ func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) {
582582 func TestRepositoryMaintainClosesContinuationDirectories(t *testing.T) {
583583 repo := NewRepository(privateDir(t))
584584 sessions := filepath.Join(repo.dir, repositorySessionsDir)
585- for i := 0; i < maintenanceBatch+1; i++ {
585+ for i := range maintenanceBatch + 1 {
586586 generations := filepath.Join(sessions, fmt.Sprintf("named-%03d", i), repositoryGenerations)
587587 if err := os.MkdirAll(generations, 0o700); err != nil {
588588 t.Fatal(err)
589589 }
590- for j := 0; j < maintenanceBatch+1; j++ {
590+ for j := range maintenanceBatch + 1 {
591591 if err := os.WriteFile(filepath.Join(generations, fmt.Sprintf("unclassified-%03d", j)), []byte("state"), 0o600); err != nil {
592592 t.Fatal(err)
593593 }
@@ -606,14 +606,14 @@ func TestRepositoryMaintainBoundsQueuedShardNames(t *testing.T) {
606606 repo := NewRepository(privateDir(t))
607607 key := legacyIncarnationID("named").String()
608608 root := filepath.Join(repo.legacySessionPath(key), repositoryObjectsDir)
609- for i := 0; i < maintenanceBatch*2; i++ {
609+ for i := range maintenanceBatch * 2 {
610610 if err := os.MkdirAll(filepath.Join(root, fmt.Sprintf("%02x", i)), 0o700); err != nil {
611611 t.Fatal(err)
612612 }
613613 }
614614 repo.maintenanceCursors = make(map[string]*maintenanceCursor)
615615 state := &sessionMaintenance{marked: make(map[uint64]manifestMaintenance), markDone: true}
616- for i := 0; i < maintenanceBatch+2; i++ {
616+ for range maintenanceBatch + 2 {
617617 budget := maintenanceBatch
618618 if err := repo.sweepSession(context.Background(), key, state, &budget); err != nil {
619619 t.Fatal(err)
@@ -629,7 +629,7 @@ func TestRepositoryMaintainResumesCursorAfterDirectoryMutation(t *testing.T) {
629629 if err := os.Mkdir(repo.dir, 0o700); err != nil {
630630 t.Fatal(err)
631631 }
632- for i := 0; i < maintenanceBatch+1; i++ {
632+ for i := range maintenanceBatch + 1 {
633633 if err := os.WriteFile(filepath.Join(repo.dir, fmt.Sprintf("live-%03d", i)), []byte("live"), 0o600); err != nil {
634634 t.Fatal(err)
635635 }
@@ -646,7 +646,7 @@ func TestRepositoryMaintainResumesCursorAfterDirectoryMutation(t *testing.T) {
646646 if err := os.WriteFile(stale, []byte("stale"), 0o600); err != nil {
647647 t.Fatal(err)
648648 }
649- for i := 0; i < 4; i++ {
649+ for range 4 {
650650 if err := repo.Maintain(context.Background()); err != nil {
651651 t.Fatal(err)
652652 }
@@ -664,17 +664,17 @@ func TestRepositoryMaintainDoesNotStarveLaterTemporaryEntries(t *testing.T) {
664664 }
665665 // The live entries precede the stale entries in directory insertion order.
666666 // A fresh ReadDir from the start on every call would never reach the latter.
667- for i := 0; i < maintenanceBatch+1; i++ {
667+ for i := range maintenanceBatch + 1 {
668668 if err := os.WriteFile(filepath.Join(repo.dir, "live-"+string(rune('a'+i))), []byte("live"), 0o600); err != nil {
669669 t.Fatal(err)
670670 }
671671 }
672- for i := 0; i < maintenanceBatch+1; i++ {
672+ for i := range maintenanceBatch + 1 {
673673 if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-later-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil {
674674 t.Fatal(err)
675675 }
676676 }
677- for i := 0; i < 4; i++ {
677+ for range 4 {
678678 if err := repo.Maintain(context.Background()); err != nil {
679679 t.Fatal(err)
680680 }
@@ -694,7 +694,7 @@ func TestRepositoryMaintainMarksLargeGenerationSetBeforeSweeping(t *testing.T) {
694694 repo := NewRepository(privateDir(t))
695695 var newest ports.SnapshotPublication
696696 for generation := uint64(1); generation <= maintenanceBatch+2; generation++ {
697- publication := repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation)))
697+ publication := repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))
698698 if err := repo.Publish(context.Background(), publication); err != nil {
699699 t.Fatal(err)
700700 }
@@ -724,7 +724,7 @@ func TestRepositoryMaintainMarksLargeGenerationSetBeforeSweeping(t *testing.T) {
724724 }
725725 }
726726
727- for i := 0; i < 600; i++ {
727+ for i := range 600 {
728728 if err := repo.Maintain(context.Background()); err != nil {
729729 t.Fatal(err)
730730 }
@@ -747,7 +747,7 @@ func TestRepositoryMaintainUsesFixedRemovalBatch(t *testing.T) {
747747 if err := os.Mkdir(repo.dir, 0o700); err != nil {
748748 t.Fatal(err)
749749 }
750- for i := 0; i < maintenanceBatch+1; i++ {
750+ for i := range maintenanceBatch + 1 {
751751 if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil {
752752 t.Fatal(err)
753753 }
@@ -774,7 +774,7 @@ func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) {
774774 }
775775
776776 removed := make(map[string]int)
777- for i := 0; i < maintenanceBatch+1; i++ {
777+ for i := range maintenanceBatch + 1 {
778778 path := filepath.Join(repo.dir, repositorySessionsDir, fmt.Sprintf(".deleting-%03d", i))
779779 if err := os.Mkdir(path, 0o700); err != nil {
780780 t.Fatal(err)
@@ -804,7 +804,7 @@ func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) {
804804 t.Fatal(err)
805805 }
806806 queued := fetched[len(fetched)-1].Name()
807- for i := 0; i < maintenanceBatch+4; i++ {
807+ for i := range maintenanceBatch + 4 {
808808 if err := repo.Maintain(context.Background()); err != nil {
809809 t.Fatal(err)
810810 }
@@ -818,7 +818,7 @@ func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) {
818818 t.Fatalf("unread session %q = %v, want queued entry to run first", unread[0].Name(), err)
819819 }
820820
821- for i := 0; i < maintenanceBatch+4; i++ {
821+ for range maintenanceBatch + 4 {
822822 if err := repo.Maintain(context.Background()); err != nil {
823823 t.Fatal(err)
824824 }
@@ -876,7 +876,7 @@ func TestRepositoryMaintainClassifiesUnpublishedGenerationsBeforeSweep(t *testin
876876 key := legacyIncarnationID("named").String()
877877 publications := make([]ports.SnapshotPublication, 0, maintenanceBatch+2)
878878 for generation := uint64(1); generation <= maintenanceBatch+2; generation++ {
879- publication := repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation)))
879+ publication := repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))
880880 if err := repo.Publish(context.Background(), publication); err != nil {
881881 t.Fatal(err)
882882 }
@@ -904,7 +904,7 @@ func TestRepositoryMaintainClassifiesUnpublishedGenerationsBeforeSweep(t *testin
904904 }
905905 }
906906
907- for i := 0; i < 600; i++ {
907+ for i := range 600 {
908908 if err := repo.Maintain(context.Background()); err != nil {
909909 t.Fatal(err)
910910 }
@@ -928,7 +928,7 @@ func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) {
928928 key := legacyIncarnationID("named").String()
929929 first := publicationWithTailShard(t, "named", 1, "ff")
930930 stalePaths := make([]string, 0, maintenanceBatch+1)
931- for i := 0; i < maintenanceBatch+1; i++ {
931+ for i := range maintenanceBatch + 1 {
932932 digest := digestInShard(t, "00", i)
933933 path := repo.legacyObjectPath(key, digest)
934934 if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
@@ -967,7 +967,7 @@ func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) {
967967 }
968968 repo.hooks.beforeHeadWrite = nil
969969
970- for i := 0; i < 20; i++ {
970+ for i := range 20 {
971971 if err := repo.Maintain(context.Background()); err != nil {
972972 t.Fatal(err)
973973 }
@@ -991,8 +991,8 @@ func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) {
991991
992992 func publicationWithTailShard(t *testing.T, name string, generation uint64, shard string) ports.SnapshotPublication {
993993 t.Helper()
994- for i := 0; i < 4096; i++ {
995- publication := repositoryPublication(t, name, generation, []byte(fmt.Sprintf("state-%d-%d", generation, i)))
994+ for i := range 4096 {
995+ publication := repositoryPublication(t, name, generation, fmt.Appendf(nil, "state-%d-%d", generation, i))
996996 if fmt.Sprintf("%02x", publication.Objects[0].Digest[0]) == shard {
997997 return publication
998998 }
@@ -1003,8 +1003,8 @@ func publicationWithTailShard(t *testing.T, name string, generation uint64, shar
10031003
10041004 func digestInShard(t *testing.T, shard string, n int) ports.SnapshotDigest {
10051005 t.Helper()
1006- for i := 0; i < 4096; i++ {
1007- digest := sha256.Sum256([]byte(fmt.Sprintf("stale-%d-%d", n, i)))
1006+ for i := range 4096 {
1007+ digest := sha256.Sum256(fmt.Appendf(nil, "stale-%d-%d", n, i))
10081008 if fmt.Sprintf("%02x", digest[0]) == shard {
10091009 return digest
10101010 }
@@ -1068,8 +1068,8 @@ func TestMaintenanceBudgetAdmitsPayloadsBeforeRead(t *testing.T) {
10681068 var reads []string
10691069 repo.hooks.beforeMaintenancePayloadRead = func(path string) { reads = append(reads, path) }
10701070 done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: 1})
1071- if err != nil {
1072- t.Fatal(err)
1071+ if !errors.Is(err, ErrMaintenanceBudgetTooSmall) {
1072+ t.Fatalf("undersized manifest budget error = %v, want ErrMaintenanceBudgetTooSmall", err)
10731073 }
10741074 if done || len(reads) != 0 {
10751075 t.Fatalf("undersized manifest budget: done=%v payload reads=%v", done, reads)
@@ -1082,8 +1082,8 @@ func TestMaintenanceBudgetAdmitsPayloadsBeforeRead(t *testing.T) {
10821082 }
10831083 reads = nil
10841084 done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: uint64(info.Size())})
1085- if err != nil {
1086- t.Fatal(err)
1085+ if !errors.Is(err, ErrMaintenanceBudgetTooSmall) {
1086+ t.Fatalf("manifest-only budget error = %v, want ErrMaintenanceBudgetTooSmall", err)
10871087 }
10881088 if done {
10891089 t.Fatal("manifest-only budget unexpectedly admitted the generation objects")
@@ -1157,7 +1157,7 @@ func TestReconcileDoesNotHoldMaintenanceLockDuringPayloadRead(t *testing.T) {
11571157 if err != nil {
11581158 t.Fatal(err)
11591159 }
1160- case <-time.After(time.Second):
1160+ case <-time.After(remainingTestTime(t) / 2):
11611161 t.Fatal("maintenance lock remained held during reconciliation payload I/O")
11621162 }
11631163 releaseOnce.Do(func() { close(release) })
@@ -1166,6 +1166,80 @@ func TestReconcileDoesNotHoldMaintenanceLockDuringPayloadRead(t *testing.T) {
11661166 }
11671167 }
11681168
1169+func TestMaintainSessionYieldsOnEmptyNonDoneObjectShardBatch(t *testing.T) {
1170+ repo := NewRepository(privateDir(t))
1171+ publication := publishMaintenanceGenerations(t, repo, "empty-shard-batch", 1)[0]
1172+ plan := ports.RetentionPlan{IncarnationID: publication.IncarnationID}
1173+ key := publication.IncarnationID.String()
1174+ token := retentionToken(plan)
1175+ root := filepath.Join(repo.sessionPath(plan.IncarnationID), repositoryObjectsDir)
1176+ purpose := "retention-shards:" + key + ":" + token
1177+ state := &retentionMaintenance{
1178+ token: token,
1179+ cursors: map[string]*maintenanceCursor{maintenanceCursorID(root, purpose): {pending: []maintenanceDirEntry{}}},
1180+ references: make(map[ports.SnapshotDigest]struct{}),
1181+ validated: true,
1182+ manifestsDone: true,
1183+ }
1184+ repo.retentionSessions[key] = state
1185+
1186+ done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: 1024})
1187+ if err != nil || done {
1188+ t.Fatalf("empty non-done shard batch: done=%v err=%v", done, err)
1189+ }
1190+ if state.objectRootDone {
1191+ t.Fatal("empty non-done shard batch advanced object-root completion")
1192+ }
1193+}
1194+
1195+func TestMaintainSessionDoesNotHoldMaintenanceLockDuringPayloadIO(t *testing.T) {
1196+ repo := NewRepository(privateDir(t))
1197+ active := publishMaintenanceGenerations(t, repo, "active-lock", 1)
1198+ other := publishMaintenanceGenerations(t, repo, "other-lock", 1)
1199+ plan := ports.RetentionPlan{IncarnationID: active[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(active[0])}}
1200+
1201+ entered := make(chan struct{})
1202+ release := make(chan struct{})
1203+ var once sync.Once
1204+ repo.hooks.beforeMaintenancePayloadRead = func(string) {
1205+ once.Do(func() { close(entered) })
1206+ <-release
1207+ }
1208+ activeDone := make(chan error, 1)
1209+ go func() {
1210+ _, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20})
1211+ activeDone <- err
1212+ }()
1213+ <-entered
1214+
1215+ pinDone := make(chan error, 1)
1216+ go func() {
1217+ _, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: other[0].IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1})
1218+ pinDone <- err
1219+ }()
1220+ select {
1221+ case err := <-pinDone:
1222+ if err != nil {
1223+ t.Fatal(err)
1224+ }
1225+ case <-time.After(remainingTestTime(t) / 2):
1226+ close(release)
1227+ t.Fatal("unrelated PinAll was blocked by retention payload I/O")
1228+ }
1229+ close(release)
1230+ if err := <-activeDone; err != nil {
1231+ t.Fatal(err)
1232+ }
1233+}
1234+
1235+func remainingTestTime(t *testing.T) time.Duration {
1236+ t.Helper()
1237+ if deadline, ok := t.Deadline(); ok {
1238+ return time.Until(deadline)
1239+ }
1240+ return 30 * time.Second
1241+}
1242+
11691243 func TestPerSessionBudgetIsolation(t *testing.T) {
11701244 repo := NewRepository(privateDir(t))
11711245 large := publishMaintenanceGenerations(t, repo, "large", 4)
@@ -1174,8 +1248,8 @@ func TestPerSessionBudgetIsolation(t *testing.T) {
11741248 smallPlan := ports.RetentionPlan{IncarnationID: small[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(small[1])}}
11751249
11761250 done, err := repo.MaintainSession(context.Background(), largePlan, ports.MaintenanceBudget{Entries: 1, Bytes: 1})
1177- if err != nil {
1178- t.Fatal(err)
1251+ if !errors.Is(err, ErrMaintenanceBudgetTooSmall) {
1252+ t.Fatalf("large session error = %v, want ErrMaintenanceBudgetTooSmall", err)
11791253 }
11801254 if done {
11811255 t.Fatal("large session unexpectedly exhausted no budget")
@@ -383,7 +383,7 @@ func TestRepositoryMaintainRetainsNewestTwoCompleteGenerations(t *testing.T) {
383 }383 }
384 }384 }
385 key := legacyIncarnationID("named").String()385 key := legacyIncarnationID("named").String()
386- for pass := 0; pass < maintenanceBatch; pass++ {386+ for pass := range maintenanceBatch {
387 if err := repo.Maintain(context.Background()); err != nil {387 if err := repo.Maintain(context.Background()); err != nil {
388 t.Fatal(err)388 t.Fatal(err)
389 }389 }
@@ -551,7 +551,7 @@ func TestRepositoryDeleteRetriesPendingQuarantineSyncWithoutDeletingRecreatedSes
551 func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) {551 func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) {
552 repo := NewRepository(privateDir(t))552 repo := NewRepository(privateDir(t))
553 quarantine := filepath.Join(repo.dir, repositorySessionsDir, ".deleting-named-test")553 quarantine := filepath.Join(repo.dir, repositorySessionsDir, ".deleting-named-test")
554- for i := 0; i < maintenanceBatch+1; i++ {554+ for i := range maintenanceBatch + 1 {
555 path := filepath.Join(quarantine, "nested", string(rune('a'+i)))555 path := filepath.Join(quarantine, "nested", string(rune('a'+i)))
556 if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {556 if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
557 t.Fatal(err)557 t.Fatal(err)
@@ -566,7 +566,7 @@ func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) {
566 if _, err := os.Lstat(quarantine); errors.Is(err, os.ErrNotExist) {566 if _, err := os.Lstat(quarantine); errors.Is(err, os.ErrNotExist) {
567 t.Fatalf("quarantine after one bounded call = %v, want remaining tree", err)567 t.Fatalf("quarantine after one bounded call = %v, want remaining tree", err)
568 }568 }
569- for i := 0; i < 12; i++ {569+ for range 12 {
570 if err := repo.Maintain(context.Background()); err != nil {570 if err := repo.Maintain(context.Background()); err != nil {
571 t.Fatal(err)571 t.Fatal(err)
572 }572 }
@@ -582,12 +582,12 @@ func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) {
582 func TestRepositoryMaintainClosesContinuationDirectories(t *testing.T) {582 func TestRepositoryMaintainClosesContinuationDirectories(t *testing.T) {
583 repo := NewRepository(privateDir(t))583 repo := NewRepository(privateDir(t))
584 sessions := filepath.Join(repo.dir, repositorySessionsDir)584 sessions := filepath.Join(repo.dir, repositorySessionsDir)
585- for i := 0; i < maintenanceBatch+1; i++ {585+ for i := range maintenanceBatch + 1 {
586 generations := filepath.Join(sessions, fmt.Sprintf("named-%03d", i), repositoryGenerations)586 generations := filepath.Join(sessions, fmt.Sprintf("named-%03d", i), repositoryGenerations)
587 if err := os.MkdirAll(generations, 0o700); err != nil {587 if err := os.MkdirAll(generations, 0o700); err != nil {
588 t.Fatal(err)588 t.Fatal(err)
589 }589 }
590- for j := 0; j < maintenanceBatch+1; j++ {590+ for j := range maintenanceBatch + 1 {
591 if err := os.WriteFile(filepath.Join(generations, fmt.Sprintf("unclassified-%03d", j)), []byte("state"), 0o600); err != nil {591 if err := os.WriteFile(filepath.Join(generations, fmt.Sprintf("unclassified-%03d", j)), []byte("state"), 0o600); err != nil {
592 t.Fatal(err)592 t.Fatal(err)
593 }593 }
@@ -606,14 +606,14 @@ func TestRepositoryMaintainBoundsQueuedShardNames(t *testing.T) {
606 repo := NewRepository(privateDir(t))606 repo := NewRepository(privateDir(t))
607 key := legacyIncarnationID("named").String()607 key := legacyIncarnationID("named").String()
608 root := filepath.Join(repo.legacySessionPath(key), repositoryObjectsDir)608 root := filepath.Join(repo.legacySessionPath(key), repositoryObjectsDir)
609- for i := 0; i < maintenanceBatch*2; i++ {609+ for i := range maintenanceBatch * 2 {
610 if err := os.MkdirAll(filepath.Join(root, fmt.Sprintf("%02x", i)), 0o700); err != nil {610 if err := os.MkdirAll(filepath.Join(root, fmt.Sprintf("%02x", i)), 0o700); err != nil {
611 t.Fatal(err)611 t.Fatal(err)
612 }612 }
613 }613 }
614 repo.maintenanceCursors = make(map[string]*maintenanceCursor)614 repo.maintenanceCursors = make(map[string]*maintenanceCursor)
615 state := &sessionMaintenance{marked: make(map[uint64]manifestMaintenance), markDone: true}615 state := &sessionMaintenance{marked: make(map[uint64]manifestMaintenance), markDone: true}
616- for i := 0; i < maintenanceBatch+2; i++ {616+ for range maintenanceBatch + 2 {
617 budget := maintenanceBatch617 budget := maintenanceBatch
618 if err := repo.sweepSession(context.Background(), key, state, &budget); err != nil {618 if err := repo.sweepSession(context.Background(), key, state, &budget); err != nil {
619 t.Fatal(err)619 t.Fatal(err)
@@ -629,7 +629,7 @@ func TestRepositoryMaintainResumesCursorAfterDirectoryMutation(t *testing.T) {
629 if err := os.Mkdir(repo.dir, 0o700); err != nil {629 if err := os.Mkdir(repo.dir, 0o700); err != nil {
630 t.Fatal(err)630 t.Fatal(err)
631 }631 }
632- for i := 0; i < maintenanceBatch+1; i++ {632+ for i := range maintenanceBatch + 1 {
633 if err := os.WriteFile(filepath.Join(repo.dir, fmt.Sprintf("live-%03d", i)), []byte("live"), 0o600); err != nil {633 if err := os.WriteFile(filepath.Join(repo.dir, fmt.Sprintf("live-%03d", i)), []byte("live"), 0o600); err != nil {
634 t.Fatal(err)634 t.Fatal(err)
635 }635 }
@@ -646,7 +646,7 @@ func TestRepositoryMaintainResumesCursorAfterDirectoryMutation(t *testing.T) {
646 if err := os.WriteFile(stale, []byte("stale"), 0o600); err != nil {646 if err := os.WriteFile(stale, []byte("stale"), 0o600); err != nil {
647 t.Fatal(err)647 t.Fatal(err)
648 }648 }
649- for i := 0; i < 4; i++ {649+ for range 4 {
650 if err := repo.Maintain(context.Background()); err != nil {650 if err := repo.Maintain(context.Background()); err != nil {
651 t.Fatal(err)651 t.Fatal(err)
652 }652 }
@@ -664,17 +664,17 @@ func TestRepositoryMaintainDoesNotStarveLaterTemporaryEntries(t *testing.T) {
664 }664 }
665 // The live entries precede the stale entries in directory insertion order.665 // The live entries precede the stale entries in directory insertion order.
666 // A fresh ReadDir from the start on every call would never reach the latter.666 // A fresh ReadDir from the start on every call would never reach the latter.
667- for i := 0; i < maintenanceBatch+1; i++ {667+ for i := range maintenanceBatch + 1 {
668 if err := os.WriteFile(filepath.Join(repo.dir, "live-"+string(rune('a'+i))), []byte("live"), 0o600); err != nil {668 if err := os.WriteFile(filepath.Join(repo.dir, "live-"+string(rune('a'+i))), []byte("live"), 0o600); err != nil {
669 t.Fatal(err)669 t.Fatal(err)
670 }670 }
671 }671 }
672- for i := 0; i < maintenanceBatch+1; i++ {672+ for i := range maintenanceBatch + 1 {
673 if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-later-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil {673 if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-later-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil {
674 t.Fatal(err)674 t.Fatal(err)
675 }675 }
676 }676 }
677- for i := 0; i < 4; i++ {677+ for range 4 {
678 if err := repo.Maintain(context.Background()); err != nil {678 if err := repo.Maintain(context.Background()); err != nil {
679 t.Fatal(err)679 t.Fatal(err)
680 }680 }
@@ -694,7 +694,7 @@ func TestRepositoryMaintainMarksLargeGenerationSetBeforeSweeping(t *testing.T) {
694 repo := NewRepository(privateDir(t))694 repo := NewRepository(privateDir(t))
695 var newest ports.SnapshotPublication695 var newest ports.SnapshotPublication
696 for generation := uint64(1); generation <= maintenanceBatch+2; generation++ {696 for generation := uint64(1); generation <= maintenanceBatch+2; generation++ {
697- publication := repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation)))697+ publication := repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))
698 if err := repo.Publish(context.Background(), publication); err != nil {698 if err := repo.Publish(context.Background(), publication); err != nil {
699 t.Fatal(err)699 t.Fatal(err)
700 }700 }
@@ -724,7 +724,7 @@ func TestRepositoryMaintainMarksLargeGenerationSetBeforeSweeping(t *testing.T) {
724 }724 }
725 }725 }
726 726
727- for i := 0; i < 600; i++ {727+ for i := range 600 {
728 if err := repo.Maintain(context.Background()); err != nil {728 if err := repo.Maintain(context.Background()); err != nil {
729 t.Fatal(err)729 t.Fatal(err)
730 }730 }
@@ -747,7 +747,7 @@ func TestRepositoryMaintainUsesFixedRemovalBatch(t *testing.T) {
747 if err := os.Mkdir(repo.dir, 0o700); err != nil {747 if err := os.Mkdir(repo.dir, 0o700); err != nil {
748 t.Fatal(err)748 t.Fatal(err)
749 }749 }
750- for i := 0; i < maintenanceBatch+1; i++ {750+ for i := range maintenanceBatch + 1 {
751 if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil {751 if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil {
752 t.Fatal(err)752 t.Fatal(err)
753 }753 }
@@ -774,7 +774,7 @@ func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) {
774 }774 }
775 775
776 removed := make(map[string]int)776 removed := make(map[string]int)
777- for i := 0; i < maintenanceBatch+1; i++ {777+ for i := range maintenanceBatch + 1 {
778 path := filepath.Join(repo.dir, repositorySessionsDir, fmt.Sprintf(".deleting-%03d", i))778 path := filepath.Join(repo.dir, repositorySessionsDir, fmt.Sprintf(".deleting-%03d", i))
779 if err := os.Mkdir(path, 0o700); err != nil {779 if err := os.Mkdir(path, 0o700); err != nil {
780 t.Fatal(err)780 t.Fatal(err)
@@ -804,7 +804,7 @@ func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) {
804 t.Fatal(err)804 t.Fatal(err)
805 }805 }
806 queued := fetched[len(fetched)-1].Name()806 queued := fetched[len(fetched)-1].Name()
807- for i := 0; i < maintenanceBatch+4; i++ {807+ for i := range maintenanceBatch + 4 {
808 if err := repo.Maintain(context.Background()); err != nil {808 if err := repo.Maintain(context.Background()); err != nil {
809 t.Fatal(err)809 t.Fatal(err)
810 }810 }
@@ -818,7 +818,7 @@ func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) {
818 t.Fatalf("unread session %q = %v, want queued entry to run first", unread[0].Name(), err)818 t.Fatalf("unread session %q = %v, want queued entry to run first", unread[0].Name(), err)
819 }819 }
820 820
821- for i := 0; i < maintenanceBatch+4; i++ {821+ for range maintenanceBatch + 4 {
822 if err := repo.Maintain(context.Background()); err != nil {822 if err := repo.Maintain(context.Background()); err != nil {
823 t.Fatal(err)823 t.Fatal(err)
824 }824 }
@@ -876,7 +876,7 @@ func TestRepositoryMaintainClassifiesUnpublishedGenerationsBeforeSweep(t *testin
876 key := legacyIncarnationID("named").String()876 key := legacyIncarnationID("named").String()
877 publications := make([]ports.SnapshotPublication, 0, maintenanceBatch+2)877 publications := make([]ports.SnapshotPublication, 0, maintenanceBatch+2)
878 for generation := uint64(1); generation <= maintenanceBatch+2; generation++ {878 for generation := uint64(1); generation <= maintenanceBatch+2; generation++ {
879- publication := repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation)))879+ publication := repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))
880 if err := repo.Publish(context.Background(), publication); err != nil {880 if err := repo.Publish(context.Background(), publication); err != nil {
881 t.Fatal(err)881 t.Fatal(err)
882 }882 }
@@ -904,7 +904,7 @@ func TestRepositoryMaintainClassifiesUnpublishedGenerationsBeforeSweep(t *testin
904 }904 }
905 }905 }
906 906
907- for i := 0; i < 600; i++ {907+ for i := range 600 {
908 if err := repo.Maintain(context.Background()); err != nil {908 if err := repo.Maintain(context.Background()); err != nil {
909 t.Fatal(err)909 t.Fatal(err)
910 }910 }
@@ -928,7 +928,7 @@ func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) {
928 key := legacyIncarnationID("named").String()928 key := legacyIncarnationID("named").String()
929 first := publicationWithTailShard(t, "named", 1, "ff")929 first := publicationWithTailShard(t, "named", 1, "ff")
930 stalePaths := make([]string, 0, maintenanceBatch+1)930 stalePaths := make([]string, 0, maintenanceBatch+1)
931- for i := 0; i < maintenanceBatch+1; i++ {931+ for i := range maintenanceBatch + 1 {
932 digest := digestInShard(t, "00", i)932 digest := digestInShard(t, "00", i)
933 path := repo.legacyObjectPath(key, digest)933 path := repo.legacyObjectPath(key, digest)
934 if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {934 if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
@@ -967,7 +967,7 @@ func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) {
967 }967 }
968 repo.hooks.beforeHeadWrite = nil968 repo.hooks.beforeHeadWrite = nil
969 969
970- for i := 0; i < 20; i++ {970+ for i := range 20 {
971 if err := repo.Maintain(context.Background()); err != nil {971 if err := repo.Maintain(context.Background()); err != nil {
972 t.Fatal(err)972 t.Fatal(err)
973 }973 }
@@ -991,8 +991,8 @@ func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) {
991 991
992 func publicationWithTailShard(t *testing.T, name string, generation uint64, shard string) ports.SnapshotPublication {992 func publicationWithTailShard(t *testing.T, name string, generation uint64, shard string) ports.SnapshotPublication {
993 t.Helper()993 t.Helper()
994- for i := 0; i < 4096; i++ {994+ for i := range 4096 {
995- publication := repositoryPublication(t, name, generation, []byte(fmt.Sprintf("state-%d-%d", generation, i)))995+ publication := repositoryPublication(t, name, generation, fmt.Appendf(nil, "state-%d-%d", generation, i))
996 if fmt.Sprintf("%02x", publication.Objects[0].Digest[0]) == shard {996 if fmt.Sprintf("%02x", publication.Objects[0].Digest[0]) == shard {
997 return publication997 return publication
998 }998 }
@@ -1003,8 +1003,8 @@ func publicationWithTailShard(t *testing.T, name string, generation uint64, shar
1003 1003
1004 func digestInShard(t *testing.T, shard string, n int) ports.SnapshotDigest {1004 func digestInShard(t *testing.T, shard string, n int) ports.SnapshotDigest {
1005 t.Helper()1005 t.Helper()
1006- for i := 0; i < 4096; i++ {1006+ for i := range 4096 {
1007- digest := sha256.Sum256([]byte(fmt.Sprintf("stale-%d-%d", n, i)))1007+ digest := sha256.Sum256(fmt.Appendf(nil, "stale-%d-%d", n, i))
1008 if fmt.Sprintf("%02x", digest[0]) == shard {1008 if fmt.Sprintf("%02x", digest[0]) == shard {
1009 return digest1009 return digest
1010 }1010 }
@@ -1068,8 +1068,8 @@ func TestMaintenanceBudgetAdmitsPayloadsBeforeRead(t *testing.T) {
1068 var reads []string1068 var reads []string
1069 repo.hooks.beforeMaintenancePayloadRead = func(path string) { reads = append(reads, path) }1069 repo.hooks.beforeMaintenancePayloadRead = func(path string) { reads = append(reads, path) }
1070 done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: 1})1070 done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: 1})
1071- if err != nil {1071+ if !errors.Is(err, ErrMaintenanceBudgetTooSmall) {
1072- t.Fatal(err)1072+ t.Fatalf("undersized manifest budget error = %v, want ErrMaintenanceBudgetTooSmall", err)
1073 }1073 }
1074 if done || len(reads) != 0 {1074 if done || len(reads) != 0 {
1075 t.Fatalf("undersized manifest budget: done=%v payload reads=%v", done, reads)1075 t.Fatalf("undersized manifest budget: done=%v payload reads=%v", done, reads)
@@ -1082,8 +1082,8 @@ func TestMaintenanceBudgetAdmitsPayloadsBeforeRead(t *testing.T) {
1082 }1082 }
1083 reads = nil1083 reads = nil
1084 done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: uint64(info.Size())})1084 done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: uint64(info.Size())})
1085- if err != nil {1085+ if !errors.Is(err, ErrMaintenanceBudgetTooSmall) {
1086- t.Fatal(err)1086+ t.Fatalf("manifest-only budget error = %v, want ErrMaintenanceBudgetTooSmall", err)
1087 }1087 }
1088 if done {1088 if done {
1089 t.Fatal("manifest-only budget unexpectedly admitted the generation objects")1089 t.Fatal("manifest-only budget unexpectedly admitted the generation objects")
@@ -1157,7 +1157,7 @@ func TestReconcileDoesNotHoldMaintenanceLockDuringPayloadRead(t *testing.T) {
1157 if err != nil {1157 if err != nil {
1158 t.Fatal(err)1158 t.Fatal(err)
1159 }1159 }
1160- case <-time.After(time.Second):1160+ case <-time.After(remainingTestTime(t) / 2):
1161 t.Fatal("maintenance lock remained held during reconciliation payload I/O")1161 t.Fatal("maintenance lock remained held during reconciliation payload I/O")
1162 }1162 }
1163 releaseOnce.Do(func() { close(release) })1163 releaseOnce.Do(func() { close(release) })
@@ -1166,6 +1166,80 @@ func TestReconcileDoesNotHoldMaintenanceLockDuringPayloadRead(t *testing.T) {
1166 }1166 }
1167 }1167 }
1168 1168
1169+func TestMaintainSessionYieldsOnEmptyNonDoneObjectShardBatch(t *testing.T) {
1170+ repo := NewRepository(privateDir(t))
1171+ publication := publishMaintenanceGenerations(t, repo, "empty-shard-batch", 1)[0]
1172+ plan := ports.RetentionPlan{IncarnationID: publication.IncarnationID}
1173+ key := publication.IncarnationID.String()
1174+ token := retentionToken(plan)
1175+ root := filepath.Join(repo.sessionPath(plan.IncarnationID), repositoryObjectsDir)
1176+ purpose := "retention-shards:" + key + ":" + token
1177+ state := &retentionMaintenance{
1178+ token: token,
1179+ cursors: map[string]*maintenanceCursor{maintenanceCursorID(root, purpose): {pending: []maintenanceDirEntry{}}},
1180+ references: make(map[ports.SnapshotDigest]struct{}),
1181+ validated: true,
1182+ manifestsDone: true,
1183+ }
1184+ repo.retentionSessions[key] = state
1185+
1186+ done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: 1024})
1187+ if err != nil || done {
1188+ t.Fatalf("empty non-done shard batch: done=%v err=%v", done, err)
1189+ }
1190+ if state.objectRootDone {
1191+ t.Fatal("empty non-done shard batch advanced object-root completion")
1192+ }
1193+}
1194+
1195+func TestMaintainSessionDoesNotHoldMaintenanceLockDuringPayloadIO(t *testing.T) {
1196+ repo := NewRepository(privateDir(t))
1197+ active := publishMaintenanceGenerations(t, repo, "active-lock", 1)
1198+ other := publishMaintenanceGenerations(t, repo, "other-lock", 1)
1199+ plan := ports.RetentionPlan{IncarnationID: active[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(active[0])}}
1200+
1201+ entered := make(chan struct{})
1202+ release := make(chan struct{})
1203+ var once sync.Once
1204+ repo.hooks.beforeMaintenancePayloadRead = func(string) {
1205+ once.Do(func() { close(entered) })
1206+ <-release
1207+ }
1208+ activeDone := make(chan error, 1)
1209+ go func() {
1210+ _, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20})
1211+ activeDone <- err
1212+ }()
1213+ <-entered
1214+
1215+ pinDone := make(chan error, 1)
1216+ go func() {
1217+ _, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: other[0].IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1})
1218+ pinDone <- err
1219+ }()
1220+ select {
1221+ case err := <-pinDone:
1222+ if err != nil {
1223+ t.Fatal(err)
1224+ }
1225+ case <-time.After(remainingTestTime(t) / 2):
1226+ close(release)
1227+ t.Fatal("unrelated PinAll was blocked by retention payload I/O")
1228+ }
1229+ close(release)
1230+ if err := <-activeDone; err != nil {
1231+ t.Fatal(err)
1232+ }
1233+}
1234+
1235+func remainingTestTime(t *testing.T) time.Duration {
1236+ t.Helper()
1237+ if deadline, ok := t.Deadline(); ok {
1238+ return time.Until(deadline)
1239+ }
1240+ return 30 * time.Second
1241+}
1242+
1169 func TestPerSessionBudgetIsolation(t *testing.T) {1243 func TestPerSessionBudgetIsolation(t *testing.T) {
1170 repo := NewRepository(privateDir(t))1244 repo := NewRepository(privateDir(t))
1171 large := publishMaintenanceGenerations(t, repo, "large", 4)1245 large := publishMaintenanceGenerations(t, repo, "large", 4)
@@ -1174,8 +1248,8 @@ func TestPerSessionBudgetIsolation(t *testing.T) {
1174 smallPlan := ports.RetentionPlan{IncarnationID: small[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(small[1])}}1248 smallPlan := ports.RetentionPlan{IncarnationID: small[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(small[1])}}
1175 1249
1176 done, err := repo.MaintainSession(context.Background(), largePlan, ports.MaintenanceBudget{Entries: 1, Bytes: 1})1250 done, err := repo.MaintainSession(context.Background(), largePlan, ports.MaintenanceBudget{Entries: 1, Bytes: 1})
1177- if err != nil {1251+ if !errors.Is(err, ErrMaintenanceBudgetTooSmall) {
1178- t.Fatal(err)1252+ t.Fatalf("large session error = %v, want ErrMaintenanceBudgetTooSmall", err)
1179 }1253 }
1180 if done {1254 if done {
1181 t.Fatal("large session unexpectedly exhausted no budget")1255 t.Fatal("large session unexpectedly exhausted no budget")
M internal/adapters/snapshot/migration_v0.go +12 -18
@@ -45,32 +45,29 @@ func decodeManifestV1(data []byte) (legacyManifestV1, error) {
4545 if err != nil {
4646 return legacyManifestV1{}, err
4747 }
48+ return legacyObjectsOf(manifest), nil
49+}
50+
51+func legacyObjectsOf(manifest codec.Manifest) legacyManifestV1 {
4852 seen := make(map[ports.SnapshotDigest]struct{})
4953 objects := make([]legacyObjectV1, 0)
50- add := func(ref codec.ObjectRef) error {
54+ add := func(ref codec.ObjectRef) {
5155 if _, ok := seen[ref.Digest]; ok {
52- return nil
56+ return
5357 }
5458 seen[ref.Digest] = struct{}{}
5559 objects = append(objects, legacyObjectV1{Kind: uint8(ref.Kind), Digest: ref.Digest, Size: uint64(ref.Size)})
56- return nil
5760 }
5861 for _, tab := range manifest.Tabs {
5962 for _, pane := range tab.Panes {
6063 for _, ref := range pane.Sealed {
61- if err := add(ref); err != nil {
62- return legacyManifestV1{}, err
63- }
64- }
65- if err := add(pane.Tail); err != nil {
66- return legacyManifestV1{}, err
67- }
68- if err := add(pane.Visible); err != nil {
69- return legacyManifestV1{}, err
64+ add(ref)
7065 }
66+ add(pane.Tail)
67+ add(pane.Visible)
7168 }
7269 }
73- return legacyManifestV1{Name: manifest.Name, Generation: manifest.Generation, Objects: objects}, nil
70+ return legacyManifestV1{Name: manifest.Name, Generation: manifest.Generation, Objects: objects}
7471 }
7572
7673 func upgradeManifestV1(data []byte, id domain.IncarnationID) (codec.Manifest, []byte, error) {
@@ -237,10 +234,7 @@ func (r *Repository) MigrateV1Checkpoint(ctx context.Context, req ports.Snapshot
237234 if err != nil {
238235 return domain.CheckpointRef{}, uncertainLegacyError("decode manifest", err)
239236 }
240- decoded, err := decodeManifestV1(data)
241- if err != nil {
242- return domain.CheckpointRef{}, uncertainLegacyError("decode manifest objects", err)
243- }
237+ legacy := legacyObjectsOf(manifest)
244238 if manifest.Name != req.LegacyName || manifest.Generation != req.LegacyRef.Generation {
245239 return domain.CheckpointRef{}, uncertainLegacyError("manifest identity mismatch", nil)
246240 }
@@ -253,7 +247,7 @@ func (r *Repository) MigrateV1Checkpoint(ctx context.Context, req ports.Snapshot
253247 if err := r.ensureSession(req.IncarnationID); err != nil {
254248 return domain.CheckpointRef{}, err
255249 }
256- for _, ref := range decoded.Objects {
250+ for _, ref := range legacy.Objects {
257251 if err := ctx.Err(); err != nil {
258252 return domain.CheckpointRef{}, err
259253 }
@@ -45,32 +45,29 @@ func decodeManifestV1(data []byte) (legacyManifestV1, error) {
45 if err != nil {45 if err != nil {
46 return legacyManifestV1{}, err46 return legacyManifestV1{}, err
47 }47 }
48+ return legacyObjectsOf(manifest), nil
49+}
50+
51+func legacyObjectsOf(manifest codec.Manifest) legacyManifestV1 {
48 seen := make(map[ports.SnapshotDigest]struct{})52 seen := make(map[ports.SnapshotDigest]struct{})
49 objects := make([]legacyObjectV1, 0)53 objects := make([]legacyObjectV1, 0)
50- add := func(ref codec.ObjectRef) error {54+ add := func(ref codec.ObjectRef) {
51 if _, ok := seen[ref.Digest]; ok {55 if _, ok := seen[ref.Digest]; ok {
52- return nil56+ return
53 }57 }
54 seen[ref.Digest] = struct{}{}58 seen[ref.Digest] = struct{}{}
55 objects = append(objects, legacyObjectV1{Kind: uint8(ref.Kind), Digest: ref.Digest, Size: uint64(ref.Size)})59 objects = append(objects, legacyObjectV1{Kind: uint8(ref.Kind), Digest: ref.Digest, Size: uint64(ref.Size)})
56- return nil
57 }60 }
58 for _, tab := range manifest.Tabs {61 for _, tab := range manifest.Tabs {
59 for _, pane := range tab.Panes {62 for _, pane := range tab.Panes {
60 for _, ref := range pane.Sealed {63 for _, ref := range pane.Sealed {
61- if err := add(ref); err != nil {64+ add(ref)
62- return legacyManifestV1{}, err
63- }
64- }
65- if err := add(pane.Tail); err != nil {
66- return legacyManifestV1{}, err
67- }
68- if err := add(pane.Visible); err != nil {
69- return legacyManifestV1{}, err
70 }65 }
66+ add(pane.Tail)
67+ add(pane.Visible)
71 }68 }
72 }69 }
73- return legacyManifestV1{Name: manifest.Name, Generation: manifest.Generation, Objects: objects}, nil70+ return legacyManifestV1{Name: manifest.Name, Generation: manifest.Generation, Objects: objects}
74 }71 }
75 72
76 func upgradeManifestV1(data []byte, id domain.IncarnationID) (codec.Manifest, []byte, error) {73 func upgradeManifestV1(data []byte, id domain.IncarnationID) (codec.Manifest, []byte, error) {
@@ -237,10 +234,7 @@ func (r *Repository) MigrateV1Checkpoint(ctx context.Context, req ports.Snapshot
237 if err != nil {234 if err != nil {
238 return domain.CheckpointRef{}, uncertainLegacyError("decode manifest", err)235 return domain.CheckpointRef{}, uncertainLegacyError("decode manifest", err)
239 }236 }
240- decoded, err := decodeManifestV1(data)237+ legacy := legacyObjectsOf(manifest)
241- if err != nil {
242- return domain.CheckpointRef{}, uncertainLegacyError("decode manifest objects", err)
243- }
244 if manifest.Name != req.LegacyName || manifest.Generation != req.LegacyRef.Generation {238 if manifest.Name != req.LegacyName || manifest.Generation != req.LegacyRef.Generation {
245 return domain.CheckpointRef{}, uncertainLegacyError("manifest identity mismatch", nil)239 return domain.CheckpointRef{}, uncertainLegacyError("manifest identity mismatch", nil)
246 }240 }
@@ -253,7 +247,7 @@ func (r *Repository) MigrateV1Checkpoint(ctx context.Context, req ports.Snapshot
253 if err := r.ensureSession(req.IncarnationID); err != nil {247 if err := r.ensureSession(req.IncarnationID); err != nil {
254 return domain.CheckpointRef{}, err248 return domain.CheckpointRef{}, err
255 }249 }
256- for _, ref := range decoded.Objects {250+ for _, ref := range legacy.Objects {
257 if err := ctx.Err(); err != nil {251 if err := ctx.Err(); err != nil {
258 return domain.CheckpointRef{}, err252 return domain.CheckpointRef{}, err
259 }253 }
M internal/adapters/snapshot/migration_v0_test.go +26 -27
@@ -24,7 +24,18 @@ func legacyManifestBytes(name string, generation uint64) []byte {
2424 body = binary.BigEndian.AppendUint64(body, 1)
2525 body = binary.BigEndian.AppendUint16(body, 0)
2626 body = binary.BigEndian.AppendUint16(body, 0)
27- out := make([]byte, 16, 16+len(body))
27+ return legacyEnvelope(body)
28+}
29+
30+func legacyBodyFromModern(modern []byte) []byte {
31+ body := append([]byte(nil), modern[legacyManifestHeaderSize:]...)
32+ const generationSize = 8
33+ modernOnly := generationSize + len(domain.IncarnationID{}) + 1 // incarnation and parent flag follow generation in v2.
34+ return append(body[:generationSize], body[modernOnly:]...)
35+}
36+
37+func legacyEnvelope(body []byte) []byte {
38+ out := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body))
2839 copy(out, "VEVM")
2940 binary.BigEndian.PutUint16(out[4:6], 1)
3041 binary.BigEndian.PutUint32(out[8:12], uint32(len(body)))
@@ -57,14 +68,7 @@ func TestLegacyManifestV1DeduplicatesObjectsPreservingFirst(t *testing.T) {
5768 manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Sealed: []codec.ObjectRef{chunkRef, chunkRef}, Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}}
5869 modern, err := codec.MarshalManifest(manifest)
5970 require.NoError(t, err)
60- body := append([]byte(nil), modern[legacyManifestHeaderSize:]...)
61- body = append(body[:8], body[25:]...)
62- legacy := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body))
63- copy(legacy, "VEVM")
64- binary.BigEndian.PutUint16(legacy[4:6], 1)
65- binary.BigEndian.PutUint32(legacy[8:12], uint32(len(body)))
66- binary.BigEndian.PutUint32(legacy[12:16], crc32.ChecksumIEEE(body))
67- legacy = append(legacy, body...)
71+ legacy := legacyEnvelope(legacyBodyFromModern(modern))
6872
6973 decoded, err := decodeManifestV1(legacy)
7074 require.NoError(t, err)
@@ -98,8 +102,8 @@ func TestHasLegacyState(t *testing.T) {
98102 }, false, true},
99103 } {
100104 t.Run(tc.name, func(t *testing.T) {
101- dir := t.TempDir()
102- require.NoError(t, os.Chmod(dir, 0o700))
105+ dir := privateDir(t)
106+ require.NoError(t, NewRepository(dir).ensurePrivateDirectory(dir))
103107 tc.setup(dir)
104108 got, err := NewRepository(dir).HasLegacyState(context.Background())
105109 if tc.budget {
@@ -139,14 +143,7 @@ func TestMigrateV1CheckpointValidatesObjectEnvelopeAndKind(t *testing.T) {
139143 manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: object.Digest, Size: uint32(len(object.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}}
140144 modern, err := codec.MarshalManifest(manifest)
141145 require.NoError(t, err)
142- body := append([]byte(nil), modern[legacyManifestHeaderSize:]...)
143- body = append(body[:8], body[25:]...)
144- legacyManifest := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body))
145- copy(legacyManifest, "VEVM")
146- binary.BigEndian.PutUint16(legacyManifest[4:6], 1)
147- binary.BigEndian.PutUint32(legacyManifest[8:12], uint32(len(body)))
148- binary.BigEndian.PutUint32(legacyManifest[12:16], crc32.ChecksumIEEE(body))
149- legacyManifest = append(legacyManifest, body...)
146+ legacyManifest := legacyEnvelope(legacyBodyFromModern(modern))
150147 key := sessionKey("work")
151148 for _, stored := range []ports.SnapshotObject{object, visible} {
152149 require.NoError(t, os.MkdirAll(filepath.Dir(repo.legacyObjectPath(key, stored.Digest)), 0o700))
@@ -165,23 +162,25 @@ func TestMigrateV1CheckpointValidatesObjectEnvelopeAndKind(t *testing.T) {
165162 }
166163
167164 func TestMigrateV1CheckpointAcceptsLegacyNilParentBeyondGenerationOne(t *testing.T) {
168- dir := t.TempDir()
169- require.NoError(t, os.Chmod(dir, 0o700))
165+ dir := privateDir(t)
170166 repo := NewRepository(dir)
167+ require.NoError(t, repo.ensurePrivateDirectory(dir))
171168 name := "work"
172169 key := sessionKey(name)
173170 generations := filepath.Join(dir, "sessions", key, "generations")
174171 require.NoError(t, os.MkdirAll(generations, 0o700))
175- for i := 1; i <= 5000; i++ {
172+ // Place the authoritative generation just beyond the bounded legacy scan.
173+ highestGeneration := maxDirectoryTraversalEntries + 1
174+ for i := 1; i < highestGeneration; i++ {
176175 require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(uint64(i))), []byte("stale"), 0o600))
177176 }
178- legacy := legacyManifestBytes(name, 5001)
177+ legacy := legacyManifestBytes(name, uint64(highestGeneration))
179178 legacyDigest := sha256.Sum256(legacy)
180- require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(5001)), legacy, 0o600))
181- require.NoError(t, os.WriteFile(filepath.Join(dir, "sessions", key, "HEAD"), marshalHead(5001, legacyDigest), 0o600))
179+ require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(uint64(highestGeneration))), legacy, 0o600))
180+ require.NoError(t, os.WriteFile(filepath.Join(dir, "sessions", key, "HEAD"), marshalHead(uint64(highestGeneration), legacyDigest), 0o600))
182181 ref, err := repo.ReadLegacyHEAD(context.Background(), name)
183182 require.NoError(t, err)
184- require.Equal(t, uint64(5001), ref.Generation)
183+ require.Equal(t, uint64(highestGeneration), ref.Generation)
185184 id := domain.IncarnationID{1}
186185 migrated, err := repo.MigrateV1Checkpoint(context.Background(), ports.SnapshotMigrationRequest{LegacyName: name, IncarnationID: id, LegacyRef: ref})
187186 require.NoError(t, err)
@@ -191,7 +190,7 @@ func TestMigrateV1CheckpointAcceptsLegacyNilParentBeyondGenerationOne(t *testing
191190 require.NoError(t, err)
192191 require.Equal(t, id, manifest.IncarnationID)
193192 require.Nil(t, manifest.ParentCheckpoint)
194- for i := 1; i <= 5000; i++ {
193+ for i := 1; i < highestGeneration; i++ {
195194 require.FileExists(t, filepath.Join(generations, generationFilename(uint64(i))))
196195 }
197196 }
@@ -24,7 +24,18 @@ func legacyManifestBytes(name string, generation uint64) []byte {
24 body = binary.BigEndian.AppendUint64(body, 1)24 body = binary.BigEndian.AppendUint64(body, 1)
25 body = binary.BigEndian.AppendUint16(body, 0)25 body = binary.BigEndian.AppendUint16(body, 0)
26 body = binary.BigEndian.AppendUint16(body, 0)26 body = binary.BigEndian.AppendUint16(body, 0)
27- out := make([]byte, 16, 16+len(body))27+ return legacyEnvelope(body)
28+}
29+
30+func legacyBodyFromModern(modern []byte) []byte {
31+ body := append([]byte(nil), modern[legacyManifestHeaderSize:]...)
32+ const generationSize = 8
33+ modernOnly := generationSize + len(domain.IncarnationID{}) + 1 // incarnation and parent flag follow generation in v2.
34+ return append(body[:generationSize], body[modernOnly:]...)
35+}
36+
37+func legacyEnvelope(body []byte) []byte {
38+ out := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body))
28 copy(out, "VEVM")39 copy(out, "VEVM")
29 binary.BigEndian.PutUint16(out[4:6], 1)40 binary.BigEndian.PutUint16(out[4:6], 1)
30 binary.BigEndian.PutUint32(out[8:12], uint32(len(body)))41 binary.BigEndian.PutUint32(out[8:12], uint32(len(body)))
@@ -57,14 +68,7 @@ func TestLegacyManifestV1DeduplicatesObjectsPreservingFirst(t *testing.T) {
57 manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Sealed: []codec.ObjectRef{chunkRef, chunkRef}, Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}}68 manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Sealed: []codec.ObjectRef{chunkRef, chunkRef}, Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}}
58 modern, err := codec.MarshalManifest(manifest)69 modern, err := codec.MarshalManifest(manifest)
59 require.NoError(t, err)70 require.NoError(t, err)
60- body := append([]byte(nil), modern[legacyManifestHeaderSize:]...)71+ legacy := legacyEnvelope(legacyBodyFromModern(modern))
61- body = append(body[:8], body[25:]...)
62- legacy := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body))
63- copy(legacy, "VEVM")
64- binary.BigEndian.PutUint16(legacy[4:6], 1)
65- binary.BigEndian.PutUint32(legacy[8:12], uint32(len(body)))
66- binary.BigEndian.PutUint32(legacy[12:16], crc32.ChecksumIEEE(body))
67- legacy = append(legacy, body...)
68 72
69 decoded, err := decodeManifestV1(legacy)73 decoded, err := decodeManifestV1(legacy)
70 require.NoError(t, err)74 require.NoError(t, err)
@@ -98,8 +102,8 @@ func TestHasLegacyState(t *testing.T) {
98 }, false, true},102 }, false, true},
99 } {103 } {
100 t.Run(tc.name, func(t *testing.T) {104 t.Run(tc.name, func(t *testing.T) {
101- dir := t.TempDir()105+ dir := privateDir(t)
102- require.NoError(t, os.Chmod(dir, 0o700))106+ require.NoError(t, NewRepository(dir).ensurePrivateDirectory(dir))
103 tc.setup(dir)107 tc.setup(dir)
104 got, err := NewRepository(dir).HasLegacyState(context.Background())108 got, err := NewRepository(dir).HasLegacyState(context.Background())
105 if tc.budget {109 if tc.budget {
@@ -139,14 +143,7 @@ func TestMigrateV1CheckpointValidatesObjectEnvelopeAndKind(t *testing.T) {
139 manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: object.Digest, Size: uint32(len(object.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}}143 manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: object.Digest, Size: uint32(len(object.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}}
140 modern, err := codec.MarshalManifest(manifest)144 modern, err := codec.MarshalManifest(manifest)
141 require.NoError(t, err)145 require.NoError(t, err)
142- body := append([]byte(nil), modern[legacyManifestHeaderSize:]...)146+ legacyManifest := legacyEnvelope(legacyBodyFromModern(modern))
143- body = append(body[:8], body[25:]...)
144- legacyManifest := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body))
145- copy(legacyManifest, "VEVM")
146- binary.BigEndian.PutUint16(legacyManifest[4:6], 1)
147- binary.BigEndian.PutUint32(legacyManifest[8:12], uint32(len(body)))
148- binary.BigEndian.PutUint32(legacyManifest[12:16], crc32.ChecksumIEEE(body))
149- legacyManifest = append(legacyManifest, body...)
150 key := sessionKey("work")147 key := sessionKey("work")
151 for _, stored := range []ports.SnapshotObject{object, visible} {148 for _, stored := range []ports.SnapshotObject{object, visible} {
152 require.NoError(t, os.MkdirAll(filepath.Dir(repo.legacyObjectPath(key, stored.Digest)), 0o700))149 require.NoError(t, os.MkdirAll(filepath.Dir(repo.legacyObjectPath(key, stored.Digest)), 0o700))
@@ -165,23 +162,25 @@ func TestMigrateV1CheckpointValidatesObjectEnvelopeAndKind(t *testing.T) {
165 }162 }
166 163
167 func TestMigrateV1CheckpointAcceptsLegacyNilParentBeyondGenerationOne(t *testing.T) {164 func TestMigrateV1CheckpointAcceptsLegacyNilParentBeyondGenerationOne(t *testing.T) {
168- dir := t.TempDir()165+ dir := privateDir(t)
169- require.NoError(t, os.Chmod(dir, 0o700))
170 repo := NewRepository(dir)166 repo := NewRepository(dir)
167+ require.NoError(t, repo.ensurePrivateDirectory(dir))
171 name := "work"168 name := "work"
172 key := sessionKey(name)169 key := sessionKey(name)
173 generations := filepath.Join(dir, "sessions", key, "generations")170 generations := filepath.Join(dir, "sessions", key, "generations")
174 require.NoError(t, os.MkdirAll(generations, 0o700))171 require.NoError(t, os.MkdirAll(generations, 0o700))
175- for i := 1; i <= 5000; i++ {172+ // Place the authoritative generation just beyond the bounded legacy scan.
173+ highestGeneration := maxDirectoryTraversalEntries + 1
174+ for i := 1; i < highestGeneration; i++ {
176 require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(uint64(i))), []byte("stale"), 0o600))175 require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(uint64(i))), []byte("stale"), 0o600))
177 }176 }
178- legacy := legacyManifestBytes(name, 5001)177+ legacy := legacyManifestBytes(name, uint64(highestGeneration))
179 legacyDigest := sha256.Sum256(legacy)178 legacyDigest := sha256.Sum256(legacy)
180- require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(5001)), legacy, 0o600))179+ require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(uint64(highestGeneration))), legacy, 0o600))
181- require.NoError(t, os.WriteFile(filepath.Join(dir, "sessions", key, "HEAD"), marshalHead(5001, legacyDigest), 0o600))180+ require.NoError(t, os.WriteFile(filepath.Join(dir, "sessions", key, "HEAD"), marshalHead(uint64(highestGeneration), legacyDigest), 0o600))
182 ref, err := repo.ReadLegacyHEAD(context.Background(), name)181 ref, err := repo.ReadLegacyHEAD(context.Background(), name)
183 require.NoError(t, err)182 require.NoError(t, err)
184- require.Equal(t, uint64(5001), ref.Generation)183+ require.Equal(t, uint64(highestGeneration), ref.Generation)
185 id := domain.IncarnationID{1}184 id := domain.IncarnationID{1}
186 migrated, err := repo.MigrateV1Checkpoint(context.Background(), ports.SnapshotMigrationRequest{LegacyName: name, IncarnationID: id, LegacyRef: ref})185 migrated, err := repo.MigrateV1Checkpoint(context.Background(), ports.SnapshotMigrationRequest{LegacyName: name, IncarnationID: id, LegacyRef: ref})
187 require.NoError(t, err)186 require.NoError(t, err)
@@ -191,7 +190,7 @@ func TestMigrateV1CheckpointAcceptsLegacyNilParentBeyondGenerationOne(t *testing
191 require.NoError(t, err)190 require.NoError(t, err)
192 require.Equal(t, id, manifest.IncarnationID)191 require.Equal(t, id, manifest.IncarnationID)
193 require.Nil(t, manifest.ParentCheckpoint)192 require.Nil(t, manifest.ParentCheckpoint)
194- for i := 1; i <= 5000; i++ {193+ for i := 1; i < highestGeneration; i++ {
195 require.FileExists(t, filepath.Join(generations, generationFilename(uint64(i))))194 require.FileExists(t, filepath.Join(generations, generationFilename(uint64(i))))
196 }195 }
197 }196 }
M internal/adapters/snapshot/path.go +18 -2
@@ -3,7 +3,9 @@ package snapshot
33 import (
44 "crypto/sha256"
55 "encoding/hex"
6+ "errors"
67 "fmt"
8+ "os"
79 "path/filepath"
810 "regexp"
911 "strconv"
@@ -27,8 +29,9 @@ func filenameForName(name string) string {
2729 return "@" + hex.EncodeToString(sum[:])[:40] + ".snap"
2830 }
2931
30-// legacyIncarnationID is used only by retired name-keyed compatibility
31-// operations. New callers must supply the durable catalogue incarnation ID.
32+// legacyIncarnationID identifies snapshots produced by the short-lived
33+// incarnation bridge. Established name-keyed repositories use sessionKey.
34+// New callers must supply the durable catalogue incarnation ID.
3235 func legacyIncarnationID(name string) domain.IncarnationID {
3336 id := domain.IncarnationID{}
3437 digest := sha256.Sum256([]byte(name))
@@ -61,6 +64,19 @@ func (r *Repository) headPath(id domain.IncarnationID) string {
6164 func (r *Repository) legacySessionPath(key string) string {
6265 return filepath.Join(r.dir, repositorySessionsDir, key)
6366 }
67+
68+// legacyRepositoryKey resolves the established name-keyed namespace first.
69+// The digest fallback keeps the retired bridge readable without changing any
70+// durable catalogue incarnation identity.
71+func (r *Repository) legacyRepositoryKey(name string) (string, error) {
72+ key := sessionKey(name)
73+ if _, err := r.stat(r.legacySessionPath(key)); err == nil {
74+ return key, nil
75+ } else if !errors.Is(err, os.ErrNotExist) {
76+ return "", err
77+ }
78+ return legacyIncarnationID(name).String(), nil
79+}
6480 func (r *Repository) legacyObjectPath(key string, digest ports.SnapshotDigest) string {
6581 hexDigest := hex.EncodeToString(digest[:])
6682 return filepath.Join(r.legacySessionPath(key), repositoryObjectsDir, hexDigest[:2], hexDigest)
@@ -3,7 +3,9 @@ package snapshot
3 import (3 import (
4 "crypto/sha256"4 "crypto/sha256"
5 "encoding/hex"5 "encoding/hex"
6+ "errors"
6 "fmt"7 "fmt"
8+ "os"
7 "path/filepath"9 "path/filepath"
8 "regexp"10 "regexp"
9 "strconv"11 "strconv"
@@ -27,8 +29,9 @@ func filenameForName(name string) string {
27 return "@" + hex.EncodeToString(sum[:])[:40] + ".snap"29 return "@" + hex.EncodeToString(sum[:])[:40] + ".snap"
28 }30 }
29 31
30-// legacyIncarnationID is used only by retired name-keyed compatibility32+// legacyIncarnationID identifies snapshots produced by the short-lived
31-// operations. New callers must supply the durable catalogue incarnation ID.33+// incarnation bridge. Established name-keyed repositories use sessionKey.
34+// New callers must supply the durable catalogue incarnation ID.
32 func legacyIncarnationID(name string) domain.IncarnationID {35 func legacyIncarnationID(name string) domain.IncarnationID {
33 id := domain.IncarnationID{}36 id := domain.IncarnationID{}
34 digest := sha256.Sum256([]byte(name))37 digest := sha256.Sum256([]byte(name))
@@ -61,6 +64,19 @@ func (r *Repository) headPath(id domain.IncarnationID) string {
61 func (r *Repository) legacySessionPath(key string) string {64 func (r *Repository) legacySessionPath(key string) string {
62 return filepath.Join(r.dir, repositorySessionsDir, key)65 return filepath.Join(r.dir, repositorySessionsDir, key)
63 }66 }
67+
68+// legacyRepositoryKey resolves the established name-keyed namespace first.
69+// The digest fallback keeps the retired bridge readable without changing any
70+// durable catalogue incarnation identity.
71+func (r *Repository) legacyRepositoryKey(name string) (string, error) {
72+ key := sessionKey(name)
73+ if _, err := r.stat(r.legacySessionPath(key)); err == nil {
74+ return key, nil
75+ } else if !errors.Is(err, os.ErrNotExist) {
76+ return "", err
77+ }
78+ return legacyIncarnationID(name).String(), nil
79+}
64 func (r *Repository) legacyObjectPath(key string, digest ports.SnapshotDigest) string {80 func (r *Repository) legacyObjectPath(key string, digest ports.SnapshotDigest) string {
65 hexDigest := hex.EncodeToString(digest[:])81 hexDigest := hex.EncodeToString(digest[:])
66 return filepath.Join(r.legacySessionPath(key), repositoryObjectsDir, hexDigest[:2], hexDigest)82 return filepath.Join(r.legacySessionPath(key), repositoryObjectsDir, hexDigest[:2], hexDigest)
M internal/adapters/snapshot/publish_efficiency_test.go +1 -1
@@ -148,7 +148,7 @@ func largeIncrementalPublications(t testing.TB, name string, count int) (ports.S
148148 sealed := make([]codec.ObjectRef, 0, count)
149149 objects := make([]ports.SnapshotObject, 0, count+2)
150150 for i := range count {
151- object, err := codec.MarshalObject(codec.HistoryChunk, []byte(fmt.Sprintf("history-%d", i)))
151+ object, err := codec.MarshalObject(codec.HistoryChunk, fmt.Appendf(nil, "history-%d", i))
152152 if err != nil {
153153 t.Fatal(err)
154154 }
@@ -148,7 +148,7 @@ func largeIncrementalPublications(t testing.TB, name string, count int) (ports.S
148 sealed := make([]codec.ObjectRef, 0, count)148 sealed := make([]codec.ObjectRef, 0, count)
149 objects := make([]ports.SnapshotObject, 0, count+2)149 objects := make([]ports.SnapshotObject, 0, count+2)
150 for i := range count {150 for i := range count {
151- object, err := codec.MarshalObject(codec.HistoryChunk, []byte(fmt.Sprintf("history-%d", i)))151+ object, err := codec.MarshalObject(codec.HistoryChunk, fmt.Appendf(nil, "history-%d", i))
152 if err != nil {152 if err != nil {
153 t.Fatal(err)153 t.Fatal(err)
154 }154 }
M internal/adapters/snapshot/repository_test.go +4 -6
@@ -2,7 +2,6 @@ package snapshot
22
33 import (
44 "context"
5- "crypto/sha256"
65 "testing"
76
87 "github.com/bnema/vev/internal/domain"
@@ -44,7 +43,7 @@ func TestRepositoryDoesNotRewriteVerifiedImmutableBlob(t *testing.T) {
4443 t.Fatal(err)
4544 }
4645 manifest.Generation = 2
47- parent := &domain.CheckpointRef{Generation: 1, ManifestDigest: codecDigestForTest(pub.Manifest)}
46+ parent := &domain.CheckpointRef{Generation: 1, ManifestDigest: codec.ManifestDigest(pub.Manifest)}
4847 manifest.ParentCheckpoint = parent
4948 pub.ParentCheckpoint = parent
5049 pub.Manifest, err = codec.MarshalManifest(manifest)
@@ -59,15 +58,14 @@ func TestRepositoryDoesNotRewriteVerifiedImmutableBlob(t *testing.T) {
5958 }
6059 }
6160
62-func codecDigestForTest(data []byte) [32]byte {
63- return sha256.Sum256(data)
64-}
65-
6661 func repositoryPublicationAfter(t *testing.T, repo *Repository, name string, generation uint64, payload []byte) ports.SnapshotPublication {
6762 t.Helper()
6863 return publicationWithCurrentParent(t, repo, repositoryPublication(t, name, generation, payload))
6964 }
7065
66+// publicationWithCurrentParent requires repositories to contain HEAD for every
67+// publication after generation one. Callers such as prepareHeadStage and
68+// seedCompletePublication must seed or publish the parent generation first.
7169 func publicationWithCurrentParent(t *testing.T, repo *Repository, publication ports.SnapshotPublication) ports.SnapshotPublication {
7270 t.Helper()
7371 if publication.Generation == 1 {
@@ -2,7 +2,6 @@ package snapshot
2 2
3 import (3 import (
4 "context"4 "context"
5- "crypto/sha256"
6 "testing"5 "testing"
7 6
8 "github.com/bnema/vev/internal/domain"7 "github.com/bnema/vev/internal/domain"
@@ -44,7 +43,7 @@ func TestRepositoryDoesNotRewriteVerifiedImmutableBlob(t *testing.T) {
44 t.Fatal(err)43 t.Fatal(err)
45 }44 }
46 manifest.Generation = 245 manifest.Generation = 2
47- parent := &domain.CheckpointRef{Generation: 1, ManifestDigest: codecDigestForTest(pub.Manifest)}46+ parent := &domain.CheckpointRef{Generation: 1, ManifestDigest: codec.ManifestDigest(pub.Manifest)}
48 manifest.ParentCheckpoint = parent47 manifest.ParentCheckpoint = parent
49 pub.ParentCheckpoint = parent48 pub.ParentCheckpoint = parent
50 pub.Manifest, err = codec.MarshalManifest(manifest)49 pub.Manifest, err = codec.MarshalManifest(manifest)
@@ -59,15 +58,14 @@ func TestRepositoryDoesNotRewriteVerifiedImmutableBlob(t *testing.T) {
59 }58 }
60 }59 }
61 60
62-func codecDigestForTest(data []byte) [32]byte {
63- return sha256.Sum256(data)
64-}
65-
66 func repositoryPublicationAfter(t *testing.T, repo *Repository, name string, generation uint64, payload []byte) ports.SnapshotPublication {61 func repositoryPublicationAfter(t *testing.T, repo *Repository, name string, generation uint64, payload []byte) ports.SnapshotPublication {
67 t.Helper()62 t.Helper()
68 return publicationWithCurrentParent(t, repo, repositoryPublication(t, name, generation, payload))63 return publicationWithCurrentParent(t, repo, repositoryPublication(t, name, generation, payload))
69 }64 }
70 65
66+// publicationWithCurrentParent requires repositories to contain HEAD for every
67+// publication after generation one. Callers such as prepareHeadStage and
68+// seedCompletePublication must seed or publish the parent generation first.
71 func publicationWithCurrentParent(t *testing.T, repo *Repository, publication ports.SnapshotPublication) ports.SnapshotPublication {69 func publicationWithCurrentParent(t *testing.T, repo *Repository, publication ports.SnapshotPublication) ports.SnapshotPublication {
72 t.Helper()70 t.Helper()
73 if publication.Generation == 1 {71 if publication.Generation == 1 {
M internal/app/daemon_integration_test.go +23 -3
@@ -579,7 +579,6 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) {
579579 t.Fatal("Serve callback returned after shutdown deadline while maintenance was blocked")
580580 case <-time.After(100 * time.Millisecond):
581581 }
582- assertLifecycleStagePending(t, callbackReturned, "Serve callback return while maintenance is blocked")
583582 assertLifecycleStagePending(t, catalogue.closed, "catalogue close while maintenance is blocked")
584583 assertLifecycleResultPending(t, wrapperReturned, "lifecycle wrapper return while maintenance is blocked")
585584 _, err := lifecycle.TryAcquire(runtimeDir)
@@ -963,8 +962,29 @@ func (t *lifecycleManualTimer) Reset(delay time.Duration) bool {
963962 }
964963 func (t *lifecycleManualTimer) Stop() bool { return !t.stopped.Swap(true) }
965964 func (t *lifecycleManualTimer) fire() {
966- if !t.stopped.Load() {
967- t.ch <- time.Now()
965+ if t.stopped.Load() {
966+ return
967+ }
968+ select {
969+ case t.ch <- time.Now():
970+ default:
971+ }
972+}
973+
974+func TestLifecycleManualTimerRepeatedFireIsNonBlocking(t *testing.T) {
975+ timer := &lifecycleManualTimer{t: t, ch: make(chan time.Time, 1)}
976+ timer.fire()
977+
978+ returned := make(chan struct{})
979+ go func() {
980+ timer.fire()
981+ close(returned)
982+ }()
983+
984+ select {
985+ case <-returned:
986+ case <-time.After(time.Second):
987+ t.Fatal("repeated manual timer fire blocked on a pending tick")
968988 }
969989 }
970990
@@ -579,7 +579,6 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) {
579 t.Fatal("Serve callback returned after shutdown deadline while maintenance was blocked")579 t.Fatal("Serve callback returned after shutdown deadline while maintenance was blocked")
580 case <-time.After(100 * time.Millisecond):580 case <-time.After(100 * time.Millisecond):
581 }581 }
582- assertLifecycleStagePending(t, callbackReturned, "Serve callback return while maintenance is blocked")
583 assertLifecycleStagePending(t, catalogue.closed, "catalogue close while maintenance is blocked")582 assertLifecycleStagePending(t, catalogue.closed, "catalogue close while maintenance is blocked")
584 assertLifecycleResultPending(t, wrapperReturned, "lifecycle wrapper return while maintenance is blocked")583 assertLifecycleResultPending(t, wrapperReturned, "lifecycle wrapper return while maintenance is blocked")
585 _, err := lifecycle.TryAcquire(runtimeDir)584 _, err := lifecycle.TryAcquire(runtimeDir)
@@ -963,8 +962,29 @@ func (t *lifecycleManualTimer) Reset(delay time.Duration) bool {
963 }962 }
964 func (t *lifecycleManualTimer) Stop() bool { return !t.stopped.Swap(true) }963 func (t *lifecycleManualTimer) Stop() bool { return !t.stopped.Swap(true) }
965 func (t *lifecycleManualTimer) fire() {964 func (t *lifecycleManualTimer) fire() {
966- if !t.stopped.Load() {965+ if t.stopped.Load() {
967- t.ch <- time.Now()966+ return
967+ }
968+ select {
969+ case t.ch <- time.Now():
970+ default:
971+ }
972+}
973+
974+func TestLifecycleManualTimerRepeatedFireIsNonBlocking(t *testing.T) {
975+ timer := &lifecycleManualTimer{t: t, ch: make(chan time.Time, 1)}
976+ timer.fire()
977+
978+ returned := make(chan struct{})
979+ go func() {
980+ timer.fire()
981+ close(returned)
982+ }()
983+
984+ select {
985+ case <-returned:
986+ case <-time.After(time.Second):
987+ t.Fatal("repeated manual timer fire blocked on a pending tick")
968 }988 }
969 }989 }
970 990
M internal/app/durable_crash_matrix_test.go +12 -12
@@ -40,12 +40,12 @@ func (h durableCrashHarness) open(t *testing.T) (*persist.Persister, *snapshot.R
4040 return persist.New(store), snapshot.NewRepository(h.snapshotDir), recoveryfs.New(h.stateDir)
4141 }
4242
43-func (h durableCrashHarness) recoverTwice(t *testing.T, assert func(*persist.Persister, *snapshot.Repository, *recoveryfs.Journal)) {
43+func (h durableCrashHarness) recoverTwice(ctx context.Context, t *testing.T, assert func(*persist.Persister, *snapshot.Repository, *recoveryfs.Journal)) {
4444 t.Helper()
4545 for attempt := range 2 {
4646 catalogue, repository, journal := h.open(t)
4747 coordinator := recovery.NewCoordinator(catalogue, repository, journal, nil)
48- require.NoError(t, coordinator.Recover(context.Background()), "recovery attempt %d", attempt+1)
48+ require.NoError(t, coordinator.Recover(ctx), "recovery attempt %d", attempt+1)
4949 assert(catalogue, repository, journal)
5050 require.NoError(t, catalogue.Close())
5151 }
@@ -131,7 +131,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) {
131131 require.True(t, boundary.triggered)
132132 require.NoError(t, catalogue.Close())
133133
134- h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {
134+ h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {
135135 record, exists, _ := catalogue.Record("work")
136136 require.True(t, exists)
137137 require.Equal(t, domain.IncarnationID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, record.IncarnationID)
@@ -159,7 +159,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) {
159159 require.NoError(t, catalogue.Close())
160160
161161 ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)}
162- h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
162+ h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
163163 stored, exists, _ := catalogue.Record(record.Name)
164164 require.True(t, exists)
165165 if boundaryName == "snapshot-publish" {
@@ -188,7 +188,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) {
188188 require.True(t, boundary.triggered)
189189 require.NoError(t, catalogue.Close())
190190
191- h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {
191+ h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {
192192 _, oldExists, _ := catalogue.Record("old")
193193 renamed, newExists, _ := catalogue.Record("new")
194194 require.False(t, oldExists)
@@ -227,10 +227,10 @@ func TestDurableTransactionCrashMatrix(t *testing.T) {
227227 require.True(t, boundary.triggered)
228228 require.NoError(t, catalogue.Close())
229229
230- h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
230+ h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
231231 _, exists, _ := catalogue.Record(record.Name)
232232 require.False(t, exists)
233- assertNoTombstones(t, repository)
233+ assertNoTombstones(ctx, t, repository)
234234 require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String()))
235235 require.Equal(t, sourceBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "snapshot")))
236236 require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "legacy.snap")))
@@ -274,7 +274,7 @@ func TestDurableDiscardCrashMatrix(t *testing.T) {
274274 require.True(t, boundary.triggered)
275275 require.NoError(t, catalogue.Close())
276276
277- h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) {
277+ h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) {
278278 got, exists, _ := catalogue.Record(old.Name)
279279 require.True(t, exists)
280280 require.Equal(t, newID, got.IncarnationID)
@@ -311,7 +311,7 @@ func TestDeletionRecoveryProtectsReusedNameFilesystemBytes(t *testing.T) {
311311 replacementBytes := readTree(t, replacementPath)
312312 require.NoError(t, catalogue.Close())
313313
314- h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
314+ h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
315315 got, exists, _ := catalogue.Record("work")
316316 require.True(t, exists)
317317 require.Equal(t, replacement, got)
@@ -319,7 +319,7 @@ func TestDeletionRecoveryProtectsReusedNameFilesystemBytes(t *testing.T) {
319319 require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "work.snap")))
320320 require.Equal(t, oldBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot")))
321321 require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", oldID.String()))
322- assertNoTombstones(t, repository)
322+ assertNoTombstones(ctx, t, repository)
323323 })
324324 }
325325
@@ -340,9 +340,9 @@ func durablePublication(t *testing.T, name string, id domain.IncarnationID, payl
340340 return ports.SnapshotPublication{IncarnationID: id, Name: name, Generation: 1, Manifest: manifest, Objects: []ports.SnapshotObject{tail, visible}}
341341 }
342342
343-func assertNoTombstones(t *testing.T, repository *snapshot.Repository) {
343+func assertNoTombstones(ctx context.Context, t *testing.T, repository *snapshot.Repository) {
344344 t.Helper()
345- page, err := repository.ListDeletionTombstones(context.Background(), ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 64 << 10})
345+ page, err := repository.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 64 << 10})
346346 require.NoError(t, err)
347347 require.True(t, page.Done)
348348 require.Empty(t, page.Tombstones)
@@ -40,12 +40,12 @@ func (h durableCrashHarness) open(t *testing.T) (*persist.Persister, *snapshot.R
40 return persist.New(store), snapshot.NewRepository(h.snapshotDir), recoveryfs.New(h.stateDir)40 return persist.New(store), snapshot.NewRepository(h.snapshotDir), recoveryfs.New(h.stateDir)
41 }41 }
42 42
43-func (h durableCrashHarness) recoverTwice(t *testing.T, assert func(*persist.Persister, *snapshot.Repository, *recoveryfs.Journal)) {43+func (h durableCrashHarness) recoverTwice(ctx context.Context, t *testing.T, assert func(*persist.Persister, *snapshot.Repository, *recoveryfs.Journal)) {
44 t.Helper()44 t.Helper()
45 for attempt := range 2 {45 for attempt := range 2 {
46 catalogue, repository, journal := h.open(t)46 catalogue, repository, journal := h.open(t)
47 coordinator := recovery.NewCoordinator(catalogue, repository, journal, nil)47 coordinator := recovery.NewCoordinator(catalogue, repository, journal, nil)
48- require.NoError(t, coordinator.Recover(context.Background()), "recovery attempt %d", attempt+1)48+ require.NoError(t, coordinator.Recover(ctx), "recovery attempt %d", attempt+1)
49 assert(catalogue, repository, journal)49 assert(catalogue, repository, journal)
50 require.NoError(t, catalogue.Close())50 require.NoError(t, catalogue.Close())
51 }51 }
@@ -131,7 +131,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) {
131 require.True(t, boundary.triggered)131 require.True(t, boundary.triggered)
132 require.NoError(t, catalogue.Close())132 require.NoError(t, catalogue.Close())
133 133
134- h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {134+ h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {
135 record, exists, _ := catalogue.Record("work")135 record, exists, _ := catalogue.Record("work")
136 require.True(t, exists)136 require.True(t, exists)
137 require.Equal(t, domain.IncarnationID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, record.IncarnationID)137 require.Equal(t, domain.IncarnationID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, record.IncarnationID)
@@ -159,7 +159,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) {
159 require.NoError(t, catalogue.Close())159 require.NoError(t, catalogue.Close())
160 160
161 ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)}161 ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)}
162- h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {162+ h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
163 stored, exists, _ := catalogue.Record(record.Name)163 stored, exists, _ := catalogue.Record(record.Name)
164 require.True(t, exists)164 require.True(t, exists)
165 if boundaryName == "snapshot-publish" {165 if boundaryName == "snapshot-publish" {
@@ -188,7 +188,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) {
188 require.True(t, boundary.triggered)188 require.True(t, boundary.triggered)
189 require.NoError(t, catalogue.Close())189 require.NoError(t, catalogue.Close())
190 190
191- h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {191+ h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {
192 _, oldExists, _ := catalogue.Record("old")192 _, oldExists, _ := catalogue.Record("old")
193 renamed, newExists, _ := catalogue.Record("new")193 renamed, newExists, _ := catalogue.Record("new")
194 require.False(t, oldExists)194 require.False(t, oldExists)
@@ -227,10 +227,10 @@ func TestDurableTransactionCrashMatrix(t *testing.T) {
227 require.True(t, boundary.triggered)227 require.True(t, boundary.triggered)
228 require.NoError(t, catalogue.Close())228 require.NoError(t, catalogue.Close())
229 229
230- h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {230+ h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
231 _, exists, _ := catalogue.Record(record.Name)231 _, exists, _ := catalogue.Record(record.Name)
232 require.False(t, exists)232 require.False(t, exists)
233- assertNoTombstones(t, repository)233+ assertNoTombstones(ctx, t, repository)
234 require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String()))234 require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String()))
235 require.Equal(t, sourceBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "snapshot")))235 require.Equal(t, sourceBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "snapshot")))
236 require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "legacy.snap")))236 require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "legacy.snap")))
@@ -274,7 +274,7 @@ func TestDurableDiscardCrashMatrix(t *testing.T) {
274 require.True(t, boundary.triggered)274 require.True(t, boundary.triggered)
275 require.NoError(t, catalogue.Close())275 require.NoError(t, catalogue.Close())
276 276
277- h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) {277+ h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) {
278 got, exists, _ := catalogue.Record(old.Name)278 got, exists, _ := catalogue.Record(old.Name)
279 require.True(t, exists)279 require.True(t, exists)
280 require.Equal(t, newID, got.IncarnationID)280 require.Equal(t, newID, got.IncarnationID)
@@ -311,7 +311,7 @@ func TestDeletionRecoveryProtectsReusedNameFilesystemBytes(t *testing.T) {
311 replacementBytes := readTree(t, replacementPath)311 replacementBytes := readTree(t, replacementPath)
312 require.NoError(t, catalogue.Close())312 require.NoError(t, catalogue.Close())
313 313
314- h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {314+ h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
315 got, exists, _ := catalogue.Record("work")315 got, exists, _ := catalogue.Record("work")
316 require.True(t, exists)316 require.True(t, exists)
317 require.Equal(t, replacement, got)317 require.Equal(t, replacement, got)
@@ -319,7 +319,7 @@ func TestDeletionRecoveryProtectsReusedNameFilesystemBytes(t *testing.T) {
319 require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "work.snap")))319 require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "work.snap")))
320 require.Equal(t, oldBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot")))320 require.Equal(t, oldBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot")))
321 require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", oldID.String()))321 require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", oldID.String()))
322- assertNoTombstones(t, repository)322+ assertNoTombstones(ctx, t, repository)
323 })323 })
324 }324 }
325 325
@@ -340,9 +340,9 @@ func durablePublication(t *testing.T, name string, id domain.IncarnationID, payl
340 return ports.SnapshotPublication{IncarnationID: id, Name: name, Generation: 1, Manifest: manifest, Objects: []ports.SnapshotObject{tail, visible}}340 return ports.SnapshotPublication{IncarnationID: id, Name: name, Generation: 1, Manifest: manifest, Objects: []ports.SnapshotObject{tail, visible}}
341 }341 }
342 342
343-func assertNoTombstones(t *testing.T, repository *snapshot.Repository) {343+func assertNoTombstones(ctx context.Context, t *testing.T, repository *snapshot.Repository) {
344 t.Helper()344 t.Helper()
345- page, err := repository.ListDeletionTombstones(context.Background(), ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 64 << 10})345+ page, err := repository.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 64 << 10})
346 require.NoError(t, err)346 require.NoError(t, err)
347 require.True(t, page.Done)347 require.True(t, page.Done)
348 require.Empty(t, page.Tombstones)348 require.Empty(t, page.Tombstones)
M internal/app/offline_kill_retry_test.go +28 -0
@@ -2,6 +2,7 @@ package app
22
33 import (
44 "context"
5+ "errors"
56 "os"
67 "path/filepath"
78 "testing"
@@ -13,6 +14,33 @@ import (
1314 "github.com/bnema/vev/internal/persist"
1415 )
1516
17+func TestRunOfflineNamedKillUsesOpenOrMigrateSeam(t *testing.T) {
18+ stateRoot := t.TempDir()
19+ t.Setenv("XDG_STATE_HOME", stateRoot)
20+ stateDir := filepath.Join(stateRoot, "vev")
21+
22+ p := newTestPersister(t, stateDir)
23+ now := time.Now().UnixNano()
24+ require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: domain.IncarnationID{1}, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh}))
25+ require.NoError(t, p.Close())
26+
27+ original := openOrMigrate
28+ t.Cleanup(func() { openOrMigrate = original })
29+ wantErr := errors.New("open seam called")
30+ called := false
31+ openOrMigrate = func(_ context.Context, deps persist.OpenDeps) (persist.OpenResult, error) {
32+ called = true
33+ require.Equal(t, stateDir, deps.StateDir)
34+ require.NotNil(t, deps.Random)
35+ require.NotNil(t, deps.SnapshotMigration)
36+ return persist.OpenResult{}, wantErr
37+ }
38+
39+ err := runOfflineNamedKill(context.Background(), "named")
40+ require.ErrorIs(t, err, wantErr)
41+ require.True(t, called)
42+}
43+
1644 func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) {
1745 tests := []struct {
1846 name string
@@ -2,6 +2,7 @@ package app
2 2
3 import (3 import (
4 "context"4 "context"
5+ "errors"
5 "os"6 "os"
6 "path/filepath"7 "path/filepath"
7 "testing"8 "testing"
@@ -13,6 +14,33 @@ import (
13 "github.com/bnema/vev/internal/persist"14 "github.com/bnema/vev/internal/persist"
14 )15 )
15 16
17+func TestRunOfflineNamedKillUsesOpenOrMigrateSeam(t *testing.T) {
18+ stateRoot := t.TempDir()
19+ t.Setenv("XDG_STATE_HOME", stateRoot)
20+ stateDir := filepath.Join(stateRoot, "vev")
21+
22+ p := newTestPersister(t, stateDir)
23+ now := time.Now().UnixNano()
24+ require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: domain.IncarnationID{1}, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh}))
25+ require.NoError(t, p.Close())
26+
27+ original := openOrMigrate
28+ t.Cleanup(func() { openOrMigrate = original })
29+ wantErr := errors.New("open seam called")
30+ called := false
31+ openOrMigrate = func(_ context.Context, deps persist.OpenDeps) (persist.OpenResult, error) {
32+ called = true
33+ require.Equal(t, stateDir, deps.StateDir)
34+ require.NotNil(t, deps.Random)
35+ require.NotNil(t, deps.SnapshotMigration)
36+ return persist.OpenResult{}, wantErr
37+ }
38+
39+ err := runOfflineNamedKill(context.Background(), "named")
40+ require.ErrorIs(t, err, wantErr)
41+ require.True(t, called)
42+}
43+
16 func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) {44 func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) {
17 tests := []struct {45 tests := []struct {
18 name string46 name string
M internal/app/run.go +10 -11
@@ -436,13 +436,18 @@ type lifecycleStartupDeps struct {
436436
437437 const lifecycleAcquireRetry = 25 * time.Millisecond
438438
439-func runWithLifecycleOwner(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart) error {
440- return runWithLifecycleOwnerDeps(ctx, runtimeDir, stateDir, start, lifecycleStartupDeps{
439+func lifecycleStartupDependencies(log *slog.Logger) lifecycleStartupDeps {
440+ return lifecycleStartupDeps{
441441 ensurePrivate: safedir.EnsurePrivate,
442442 acquire: func(ctx context.Context, runtimeDir string, retry time.Duration) (lifecycleOwnership, error) {
443443 return lifecycle.Acquire(ctx, runtimeDir, retry)
444444 },
445- })
445+ log: log,
446+ }
447+}
448+
449+func runWithLifecycleOwner(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart) error {
450+ return runWithLifecycleOwnerDeps(ctx, runtimeDir, stateDir, start, lifecycleStartupDependencies(nil))
446451 }
447452
448453 func runWithLifecycleOwnerDeps(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart, deps lifecycleStartupDeps) (retErr error) {
@@ -492,13 +497,7 @@ func runDaemon() (retErr error) {
492497 defer func() { retErr = errors.Join(retErr, logCloser.Close()) }()
493498 return runWithLifecycleOwnerDeps(ctx, ipc.SocketDir(), platform.StateDir(), func(ctx context.Context) error {
494499 return runDaemonOwnedWithLogger(ctx, log)
495- }, lifecycleStartupDeps{
496- ensurePrivate: safedir.EnsurePrivate,
497- acquire: func(ctx context.Context, runtimeDir string, retry time.Duration) (lifecycleOwnership, error) {
498- return lifecycle.Acquire(ctx, runtimeDir, retry)
499- },
500- log: log,
501- })
500+ }, lifecycleStartupDependencies(log))
502501 }
503502
504503 func logCatalogueRecovery(log *slog.Logger, records []domain.CatalogueRecord, recoveryMode string) {
@@ -1567,7 +1566,7 @@ func runOfflineNamedKill(ctx context.Context, name string) (retErr error) {
15671566 }
15681567
15691568 repository := snapshotadapter.NewRepository(snapshotDir())
1570- opened, err := persist.OpenOrMigrate(ctx, persist.OpenDeps{
1569+ opened, err := openOrMigrate(ctx, persist.OpenDeps{
15711570 StateDir: platform.StateDir(),
15721571 Random: rand.Reader,
15731572 SnapshotMigration: repository,
@@ -436,13 +436,18 @@ type lifecycleStartupDeps struct {
436 436
437 const lifecycleAcquireRetry = 25 * time.Millisecond437 const lifecycleAcquireRetry = 25 * time.Millisecond
438 438
439-func runWithLifecycleOwner(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart) error {439+func lifecycleStartupDependencies(log *slog.Logger) lifecycleStartupDeps {
440- return runWithLifecycleOwnerDeps(ctx, runtimeDir, stateDir, start, lifecycleStartupDeps{440+ return lifecycleStartupDeps{
441 ensurePrivate: safedir.EnsurePrivate,441 ensurePrivate: safedir.EnsurePrivate,
442 acquire: func(ctx context.Context, runtimeDir string, retry time.Duration) (lifecycleOwnership, error) {442 acquire: func(ctx context.Context, runtimeDir string, retry time.Duration) (lifecycleOwnership, error) {
443 return lifecycle.Acquire(ctx, runtimeDir, retry)443 return lifecycle.Acquire(ctx, runtimeDir, retry)
444 },444 },
445- })445+ log: log,
446+ }
447+}
448+
449+func runWithLifecycleOwner(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart) error {
450+ return runWithLifecycleOwnerDeps(ctx, runtimeDir, stateDir, start, lifecycleStartupDependencies(nil))
446 }451 }
447 452
448 func runWithLifecycleOwnerDeps(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart, deps lifecycleStartupDeps) (retErr error) {453 func runWithLifecycleOwnerDeps(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart, deps lifecycleStartupDeps) (retErr error) {
@@ -492,13 +497,7 @@ func runDaemon() (retErr error) {
492 defer func() { retErr = errors.Join(retErr, logCloser.Close()) }()497 defer func() { retErr = errors.Join(retErr, logCloser.Close()) }()
493 return runWithLifecycleOwnerDeps(ctx, ipc.SocketDir(), platform.StateDir(), func(ctx context.Context) error {498 return runWithLifecycleOwnerDeps(ctx, ipc.SocketDir(), platform.StateDir(), func(ctx context.Context) error {
494 return runDaemonOwnedWithLogger(ctx, log)499 return runDaemonOwnedWithLogger(ctx, log)
495- }, lifecycleStartupDeps{500+ }, lifecycleStartupDependencies(log))
496- ensurePrivate: safedir.EnsurePrivate,
497- acquire: func(ctx context.Context, runtimeDir string, retry time.Duration) (lifecycleOwnership, error) {
498- return lifecycle.Acquire(ctx, runtimeDir, retry)
499- },
500- log: log,
501- })
502 }501 }
503 502
504 func logCatalogueRecovery(log *slog.Logger, records []domain.CatalogueRecord, recoveryMode string) {503 func logCatalogueRecovery(log *slog.Logger, records []domain.CatalogueRecord, recoveryMode string) {
@@ -1567,7 +1566,7 @@ func runOfflineNamedKill(ctx context.Context, name string) (retErr error) {
1567 }1566 }
1568 1567
1569 repository := snapshotadapter.NewRepository(snapshotDir())1568 repository := snapshotadapter.NewRepository(snapshotDir())
1570- opened, err := persist.OpenOrMigrate(ctx, persist.OpenDeps{1569+ opened, err := openOrMigrate(ctx, persist.OpenDeps{
1571 StateDir: platform.StateDir(),1570 StateDir: platform.StateDir(),
1572 Random: rand.Reader,1571 Random: rand.Reader,
1573 SnapshotMigration: repository,1572 SnapshotMigration: repository,
M internal/app/spawn.go +3 -0
@@ -70,6 +70,9 @@ var defaultBackoff = backoffConfig{
7070 func retryAttempts[T any](ctx context.Context, cfg backoffConfig, attempt func() (T, bool, error)) (T, error) {
7171 deadline := time.Now().Add(cfg.total)
7272 backoff := cfg.initial
73+ if backoff <= 0 {
74+ backoff = time.Millisecond
75+ }
7376 for {
7477 if err := ctx.Err(); err != nil {
7578 var zero T
@@ -70,6 +70,9 @@ var defaultBackoff = backoffConfig{
70 func retryAttempts[T any](ctx context.Context, cfg backoffConfig, attempt func() (T, bool, error)) (T, error) {70 func retryAttempts[T any](ctx context.Context, cfg backoffConfig, attempt func() (T, bool, error)) (T, error) {
71 deadline := time.Now().Add(cfg.total)71 deadline := time.Now().Add(cfg.total)
72 backoff := cfg.initial72 backoff := cfg.initial
73+ if backoff <= 0 {
74+ backoff = time.Millisecond
75+ }
73 for {76 for {
74 if err := ctx.Err(); err != nil {77 if err := ctx.Err(); err != nil {
75 var zero T78 var zero T
M internal/app/spawn_test.go +15 -0
@@ -273,6 +273,21 @@ func TestRetryAttemptsChecksOperationErrorBeforeDeadline(t *testing.T) {
273273 }
274274 }
275275
276+func TestRetryAttemptsClampsNonPositiveInitialDelay(t *testing.T) {
277+ started := time.Now()
278+ calls := 0
279+ _, err := retryAttempts(context.Background(), backoffConfig{total: time.Second, max: time.Second}, func() (struct{}, bool, error) {
280+ calls++
281+ return struct{}{}, calls == 2, nil
282+ })
283+ if err != nil {
284+ t.Fatalf("retryAttempts: %v", err)
285+ }
286+ if elapsed := time.Since(started); elapsed < 500*time.Microsecond {
287+ t.Fatalf("retryAttempts retried after %s, want a positive initial delay", elapsed)
288+ }
289+}
290+
276291 func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) {
277292 want := portsmocks.NewMockTransport(t)
278293 var calls atomic.Int32
@@ -273,6 +273,21 @@ func TestRetryAttemptsChecksOperationErrorBeforeDeadline(t *testing.T) {
273 }273 }
274 }274 }
275 275
276+func TestRetryAttemptsClampsNonPositiveInitialDelay(t *testing.T) {
277+ started := time.Now()
278+ calls := 0
279+ _, err := retryAttempts(context.Background(), backoffConfig{total: time.Second, max: time.Second}, func() (struct{}, bool, error) {
280+ calls++
281+ return struct{}{}, calls == 2, nil
282+ })
283+ if err != nil {
284+ t.Fatalf("retryAttempts: %v", err)
285+ }
286+ if elapsed := time.Since(started); elapsed < 500*time.Microsecond {
287+ t.Fatalf("retryAttempts retried after %s, want a positive initial delay", elapsed)
288+ }
289+}
290+
276 func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) {291 func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) {
277 want := portsmocks.NewMockTransport(t)292 want := portsmocks.NewMockTransport(t)
278 var calls atomic.Int32293 var calls atomic.Int32
M internal/domain/recovery.go +2 -2
@@ -43,8 +43,6 @@ type CatalogueRecord struct {
4343 DegradedReason string
4444 }
4545
46-// CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue
47-// incarnation without carrying authority-owned recovery or checkpoint fields.
4846 // Equal reports canonical value equality between checkpoint references.
4947 func (r *CheckpointRef) Equal(other *CheckpointRef) bool {
5048 if r == nil || other == nil {
@@ -63,6 +61,8 @@ func (r CatalogueRecord) Equal(other CatalogueRecord) bool {
6361 r.Fallbacks[1].Equal(other.Fallbacks[1]) && r.DegradedReason == other.DegradedReason
6462 }
6563
64+// CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue
65+// incarnation without carrying authority-owned recovery or checkpoint fields.
6666 type CatalogueMetadataUpdate struct {
6767 Name string
6868 IncarnationID IncarnationID
@@ -43,8 +43,6 @@ type CatalogueRecord struct {
43 DegradedReason string43 DegradedReason string
44 }44 }
45 45
46-// CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue
47-// incarnation without carrying authority-owned recovery or checkpoint fields.
48 // Equal reports canonical value equality between checkpoint references.46 // Equal reports canonical value equality between checkpoint references.
49 func (r *CheckpointRef) Equal(other *CheckpointRef) bool {47 func (r *CheckpointRef) Equal(other *CheckpointRef) bool {
50 if r == nil || other == nil {48 if r == nil || other == nil {
@@ -63,6 +61,8 @@ func (r CatalogueRecord) Equal(other CatalogueRecord) bool {
63 r.Fallbacks[1].Equal(other.Fallbacks[1]) && r.DegradedReason == other.DegradedReason61 r.Fallbacks[1].Equal(other.Fallbacks[1]) && r.DegradedReason == other.DegradedReason
64 }62 }
65 63
64+// CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue
65+// incarnation without carrying authority-owned recovery or checkpoint fields.
66 type CatalogueMetadataUpdate struct {66 type CatalogueMetadataUpdate struct {
67 Name string67 Name string
68 IncarnationID IncarnationID68 IncarnationID IncarnationID
M internal/domain/recovery_test.go +14 -13
@@ -31,19 +31,15 @@ func TestIncarnationIDText(t *testing.T) {
3131 }
3232
3333 func TestCatalogueRecordValidate(t *testing.T) {
34- valid := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh}
35- require.NoError(t, valid.Validate())
36- valid.RecoveryState = RecoveryHealthy
37- valid.Committed = &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}
38- require.NoError(t, valid.Validate())
39- valid.Fallbacks[0] = &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{2}}
40- require.Error(t, valid.Validate(), "duplicate generations are malformed")
41-
4234 validRef := &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}
4335 tests := []struct {
44- name string
45- record CatalogueRecord
36+ name string
37+ record CatalogueRecord
38+ wantValid bool
4639 }{
40+ {name: "fresh valid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh}, wantValid: true},
41+ {name: "healthy with committed checkpoint valid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}}, wantValid: true},
42+ {name: "duplicate generation invalid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}, Fallbacks: [2]*CheckpointRef{{Generation: 7, ManifestDigest: [32]byte{2}}}}},
4743 {name: "invalid session name", record: CatalogueRecord{Name: "bad/name", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh}},
4844 {name: "zero incarnation", record: CatalogueRecord{Name: "work", RecoveryState: RecoveryFresh}},
4945 {name: "unknown recovery state", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryState(99)}},
@@ -59,7 +55,12 @@ func TestCatalogueRecordValidate(t *testing.T) {
5955 }
6056 for _, tt := range tests {
6157 t.Run(tt.name, func(t *testing.T) {
62- require.Error(t, tt.record.Validate())
58+ err := tt.record.Validate()
59+ if tt.wantValid {
60+ require.NoError(t, err)
61+ } else {
62+ require.Error(t, err)
63+ }
6364 })
6465 }
6566 }
@@ -69,8 +70,8 @@ func TestCatalogueRecordEqualUsesCheckpointValuesAndOrderedTabs(t *testing.T) {
6970 left := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, TabNames: []string{"shell", "logs"}, RecoveryState: RecoveryHealthy, Committed: &committed}
7071 right := left
7172 right.TabNames = append([]string(nil), left.TabNames...)
72- copy := committed
73- right.Committed = &copy
73+ committedCopy := committed
74+ right.Committed = &committedCopy
7475 require.True(t, left.Equal(right))
7576
7677 right.TabNames[0], right.TabNames[1] = right.TabNames[1], right.TabNames[0]
@@ -31,19 +31,15 @@ func TestIncarnationIDText(t *testing.T) {
31 }31 }
32 32
33 func TestCatalogueRecordValidate(t *testing.T) {33 func TestCatalogueRecordValidate(t *testing.T) {
34- valid := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh}
35- require.NoError(t, valid.Validate())
36- valid.RecoveryState = RecoveryHealthy
37- valid.Committed = &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}
38- require.NoError(t, valid.Validate())
39- valid.Fallbacks[0] = &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{2}}
40- require.Error(t, valid.Validate(), "duplicate generations are malformed")
41-
42 validRef := &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}34 validRef := &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}
43 tests := []struct {35 tests := []struct {
44- name string36+ name string
45- record CatalogueRecord37+ record CatalogueRecord
38+ wantValid bool
46 }{39 }{
40+ {name: "fresh valid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh}, wantValid: true},
41+ {name: "healthy with committed checkpoint valid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}}, wantValid: true},
42+ {name: "duplicate generation invalid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}, Fallbacks: [2]*CheckpointRef{{Generation: 7, ManifestDigest: [32]byte{2}}}}},
47 {name: "invalid session name", record: CatalogueRecord{Name: "bad/name", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh}},43 {name: "invalid session name", record: CatalogueRecord{Name: "bad/name", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh}},
48 {name: "zero incarnation", record: CatalogueRecord{Name: "work", RecoveryState: RecoveryFresh}},44 {name: "zero incarnation", record: CatalogueRecord{Name: "work", RecoveryState: RecoveryFresh}},
49 {name: "unknown recovery state", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryState(99)}},45 {name: "unknown recovery state", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryState(99)}},
@@ -59,7 +55,12 @@ func TestCatalogueRecordValidate(t *testing.T) {
59 }55 }
60 for _, tt := range tests {56 for _, tt := range tests {
61 t.Run(tt.name, func(t *testing.T) {57 t.Run(tt.name, func(t *testing.T) {
62- require.Error(t, tt.record.Validate())58+ err := tt.record.Validate()
59+ if tt.wantValid {
60+ require.NoError(t, err)
61+ } else {
62+ require.Error(t, err)
63+ }
63 })64 })
64 }65 }
65 }66 }
@@ -69,8 +70,8 @@ func TestCatalogueRecordEqualUsesCheckpointValuesAndOrderedTabs(t *testing.T) {
69 left := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, TabNames: []string{"shell", "logs"}, RecoveryState: RecoveryHealthy, Committed: &committed}70 left := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, TabNames: []string{"shell", "logs"}, RecoveryState: RecoveryHealthy, Committed: &committed}
70 right := left71 right := left
71 right.TabNames = append([]string(nil), left.TabNames...)72 right.TabNames = append([]string(nil), left.TabNames...)
72- copy := committed73+ committedCopy := committed
73- right.Committed = &copy74+ right.Committed = &committedCopy
74 require.True(t, left.Equal(right))75 require.True(t, left.Equal(right))
75 76
76 right.TabNames[0], right.TabNames[1] = right.TabNames[1], right.TabNames[0]77 right.TabNames[0], right.TabNames[1] = right.TabNames[1], right.TabNames[0]
M internal/persist/catalogue_test.go +19 -5
@@ -52,15 +52,29 @@ func testCatalogueRecordRoundTrip(t *testing.T) {
5252 func TestDecodeRecordValueWrapsMalformedAndValidationErrors(t *testing.T) {
5353 encoded, err := encodeRecordValue(validRecord("work", 1))
5454 require.NoError(t, err)
55- const recoveryStateOffset = 4 + 2 + 16 + 4 + 8 + 8 + 8 + 4
56- encoded[recoveryStateOffset] = 99
55+ encoded[recoveryStateFieldOffset(t, encoded)] = 99
5756
5857 _, err = decodeRecordValue("work", encoded)
5958 require.ErrorIs(t, err, errMalformedRecord)
60- joined, ok := err.(interface{ Unwrap() []error })
59+ require.ErrorContains(t, err, "invalid recovery state")
60+}
61+
62+func recoveryStateFieldOffset(t *testing.T, encoded []byte) int {
63+ t.Helper()
64+ r := valueReader{data: encoded}
65+ _, ok := r.take(len(catalogueMagic) + 2 + len(domain.IncarnationID{}))
66+ require.True(t, ok)
67+ _, ok = r.str()
6168 require.True(t, ok)
62- require.Len(t, joined.Unwrap(), 2)
63- require.ErrorContains(t, joined.Unwrap()[1], "invalid recovery state")
69+ _, ok = r.take(8 + 8 + 8)
70+ require.True(t, ok)
71+ count, ok := r.u32()
72+ require.True(t, ok)
73+ for range count {
74+ _, ok = r.str()
75+ require.True(t, ok)
76+ }
77+ return len(encoded) - r.remaining()
6478 }
6579
6680 func testCatalogueDuplicateIncarnations(t *testing.T) {
@@ -52,15 +52,29 @@ func testCatalogueRecordRoundTrip(t *testing.T) {
52 func TestDecodeRecordValueWrapsMalformedAndValidationErrors(t *testing.T) {52 func TestDecodeRecordValueWrapsMalformedAndValidationErrors(t *testing.T) {
53 encoded, err := encodeRecordValue(validRecord("work", 1))53 encoded, err := encodeRecordValue(validRecord("work", 1))
54 require.NoError(t, err)54 require.NoError(t, err)
55- const recoveryStateOffset = 4 + 2 + 16 + 4 + 8 + 8 + 8 + 455+ encoded[recoveryStateFieldOffset(t, encoded)] = 99
56- encoded[recoveryStateOffset] = 99
57 56
58 _, err = decodeRecordValue("work", encoded)57 _, err = decodeRecordValue("work", encoded)
59 require.ErrorIs(t, err, errMalformedRecord)58 require.ErrorIs(t, err, errMalformedRecord)
60- joined, ok := err.(interface{ Unwrap() []error })59+ require.ErrorContains(t, err, "invalid recovery state")
60+}
61+
62+func recoveryStateFieldOffset(t *testing.T, encoded []byte) int {
63+ t.Helper()
64+ r := valueReader{data: encoded}
65+ _, ok := r.take(len(catalogueMagic) + 2 + len(domain.IncarnationID{}))
66+ require.True(t, ok)
67+ _, ok = r.str()
61 require.True(t, ok)68 require.True(t, ok)
62- require.Len(t, joined.Unwrap(), 2)69+ _, ok = r.take(8 + 8 + 8)
63- require.ErrorContains(t, joined.Unwrap()[1], "invalid recovery state")70+ require.True(t, ok)
71+ count, ok := r.u32()
72+ require.True(t, ok)
73+ for range count {
74+ _, ok = r.str()
75+ require.True(t, ok)
76+ }
77+ return len(encoded) - r.remaining()
64 }78 }
65 79
66 func testCatalogueDuplicateIncarnations(t *testing.T) {80 func testCatalogueDuplicateIncarnations(t *testing.T) {
M internal/persist/migration.go +27 -9
@@ -419,6 +419,17 @@ func installMigratedCatalogue(stateDir string, records []domain.CatalogueRecord)
419419 if err := p.Close(); err != nil {
420420 return err
421421 }
422+ staged, opened, err := openCurrentCatalogue(stage, false)
423+ if err != nil {
424+ return err
425+ }
426+ if !catalogueRecordsEqual(records, opened) {
427+ _ = staged.Close()
428+ return errors.New("persist: staged catalogue validation mismatch")
429+ }
430+ if err := staged.Close(); err != nil {
431+ return err
432+ }
422433 source := StorePath(stage)
423434 target := StorePath(stateDir)
424435 if err := os.Rename(source, target); err != nil {
@@ -487,9 +498,13 @@ func decodeLegacyRecordV0(name string, value []byte) (legacyCatalogueRecordV0, e
487498 }
488499
489500 func writeMigrationRecord(path string, record MigrationRecord) error {
490- return atomicMigrationWrite(path, encodeMigrationRecord(record))
501+ encoded, err := encodeMigrationRecord(record)
502+ if err != nil {
503+ return err
504+ }
505+ return atomicMigrationWrite(path, encoded)
491506 }
492-func encodeMigrationRecord(record MigrationRecord) []byte {
507+func encodeMigrationRecord(record MigrationRecord) ([]byte, error) {
493508 out := append([]byte(migrationRecordMagic), byte(migrationRecordVersion>>8), byte(migrationRecordVersion), byte(record.Phase))
494509 names := make([]string, 0, len(record.Assignments))
495510 for name := range record.Assignments {
@@ -498,7 +513,11 @@ func encodeMigrationRecord(record MigrationRecord) []byte {
498513 sort.Strings(names)
499514 out = binary.BigEndian.AppendUint32(out, uint32(len(names)))
500515 for _, name := range names {
501- out = appendMigrationString(out, name)
516+ var err error
517+ out, err = appendCheckedString(out, name)
518+ if err != nil {
519+ return nil, err
520+ }
502521 id := record.Assignments[name]
503522 out = append(out, id[:]...)
504523 if ref := record.Validated[name]; ref != nil {
@@ -508,9 +527,12 @@ func encodeMigrationRecord(record MigrationRecord) []byte {
508527 } else {
509528 out = append(out, 0)
510529 }
511- out = appendMigrationString(out, record.Degraded[name])
530+ out, err = appendCheckedString(out, record.Degraded[name])
531+ if err != nil {
532+ return nil, err
533+ }
512534 }
513- return out
535+ return out, nil
514536 }
515537 func readMigrationRecord(path string) (MigrationRecord, error) {
516538 data, err := os.ReadFile(path)
@@ -573,10 +595,6 @@ func readMigrationRecord(path string) (MigrationRecord, error) {
573595 }
574596 return record, nil
575597 }
576-func appendMigrationString(out []byte, value string) []byte {
577- out = binary.BigEndian.AppendUint32(out, uint32(len(value)))
578- return append(out, value...)
579-}
580598 func atomicMigrationWrite(path string, data []byte) (retErr error) {
581599 dir := filepath.Dir(path)
582600 if err := safedir.EnsurePrivate(dir); err != nil {
@@ -419,6 +419,17 @@ func installMigratedCatalogue(stateDir string, records []domain.CatalogueRecord)
419 if err := p.Close(); err != nil {419 if err := p.Close(); err != nil {
420 return err420 return err
421 }421 }
422+ staged, opened, err := openCurrentCatalogue(stage, false)
423+ if err != nil {
424+ return err
425+ }
426+ if !catalogueRecordsEqual(records, opened) {
427+ _ = staged.Close()
428+ return errors.New("persist: staged catalogue validation mismatch")
429+ }
430+ if err := staged.Close(); err != nil {
431+ return err
432+ }
422 source := StorePath(stage)433 source := StorePath(stage)
423 target := StorePath(stateDir)434 target := StorePath(stateDir)
424 if err := os.Rename(source, target); err != nil {435 if err := os.Rename(source, target); err != nil {
@@ -487,9 +498,13 @@ func decodeLegacyRecordV0(name string, value []byte) (legacyCatalogueRecordV0, e
487 }498 }
488 499
489 func writeMigrationRecord(path string, record MigrationRecord) error {500 func writeMigrationRecord(path string, record MigrationRecord) error {
490- return atomicMigrationWrite(path, encodeMigrationRecord(record))501+ encoded, err := encodeMigrationRecord(record)
502+ if err != nil {
503+ return err
504+ }
505+ return atomicMigrationWrite(path, encoded)
491 }506 }
492-func encodeMigrationRecord(record MigrationRecord) []byte {507+func encodeMigrationRecord(record MigrationRecord) ([]byte, error) {
493 out := append([]byte(migrationRecordMagic), byte(migrationRecordVersion>>8), byte(migrationRecordVersion), byte(record.Phase))508 out := append([]byte(migrationRecordMagic), byte(migrationRecordVersion>>8), byte(migrationRecordVersion), byte(record.Phase))
494 names := make([]string, 0, len(record.Assignments))509 names := make([]string, 0, len(record.Assignments))
495 for name := range record.Assignments {510 for name := range record.Assignments {
@@ -498,7 +513,11 @@ func encodeMigrationRecord(record MigrationRecord) []byte {
498 sort.Strings(names)513 sort.Strings(names)
499 out = binary.BigEndian.AppendUint32(out, uint32(len(names)))514 out = binary.BigEndian.AppendUint32(out, uint32(len(names)))
500 for _, name := range names {515 for _, name := range names {
501- out = appendMigrationString(out, name)516+ var err error
517+ out, err = appendCheckedString(out, name)
518+ if err != nil {
519+ return nil, err
520+ }
502 id := record.Assignments[name]521 id := record.Assignments[name]
503 out = append(out, id[:]...)522 out = append(out, id[:]...)
504 if ref := record.Validated[name]; ref != nil {523 if ref := record.Validated[name]; ref != nil {
@@ -508,9 +527,12 @@ func encodeMigrationRecord(record MigrationRecord) []byte {
508 } else {527 } else {
509 out = append(out, 0)528 out = append(out, 0)
510 }529 }
511- out = appendMigrationString(out, record.Degraded[name])530+ out, err = appendCheckedString(out, record.Degraded[name])
531+ if err != nil {
532+ return nil, err
533+ }
512 }534 }
513- return out535+ return out, nil
514 }536 }
515 func readMigrationRecord(path string) (MigrationRecord, error) {537 func readMigrationRecord(path string) (MigrationRecord, error) {
516 data, err := os.ReadFile(path)538 data, err := os.ReadFile(path)
@@ -573,10 +595,6 @@ func readMigrationRecord(path string) (MigrationRecord, error) {
573 }595 }
574 return record, nil596 return record, nil
575 }597 }
576-func appendMigrationString(out []byte, value string) []byte {
577- out = binary.BigEndian.AppendUint32(out, uint32(len(value)))
578- return append(out, value...)
579-}
580 func atomicMigrationWrite(path string, data []byte) (retErr error) {598 func atomicMigrationWrite(path string, data []byte) (retErr error) {
581 dir := filepath.Dir(path)599 dir := filepath.Dir(path)
582 if err := safedir.EnsurePrivate(dir); err != nil {600 if err := safedir.EnsurePrivate(dir); err != nil {
M internal/persist/migration_test.go +22 -10
@@ -126,7 +126,7 @@ func TestMigrationExistingState(t *testing.T) {
126126 require.NoError(t, reopened.Catalogue.Close())
127127 }
128128
129-func TestMigrationOver4096(t *testing.T) {
129+func TestMigrationPreservesLargeGeneration(t *testing.T) {
130130 dir := t.TempDir()
131131 writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "large", Cwd: "/tmp"})
132132 migration := &migrationStub{heads: map[string]domain.CheckpointRef{"large": {Generation: 5001, ManifestDigest: [32]byte{1}}}, migrateErr: map[string]error{}}
@@ -248,16 +248,28 @@ func TestMigrationPreservesUncertain(t *testing.T) {
248248 require.LessOrEqual(t, len(result.Records[0].DegradedReason), migrationDegradedReasonMaxBytes)
249249 require.NotContains(t, result.Records[0].DegradedReason, adapterCause)
250250 require.Contains(t, logs.String(), adapterCause, "the full adapter cause must remain diagnosable")
251+}
251252
252- failClosedDir := t.TempDir()
253- writeLegacyCatalogue(t, failClosedDir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"})
254- _, err = OpenOrMigrate(context.Background(), OpenDeps{StateDir: failClosedDir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"work": ref}, migrateErr: map[string]error{"work": errors.New("repository unavailable")}}})
255- require.ErrorContains(t, err, "repository unavailable")
256- data, replayErr := kv.Replay(StorePath(failClosedDir))
257- require.NoError(t, replayErr)
258- _, legacyFormat, classifyErr := classifyCatalogue(data)
259- require.NoError(t, classifyErr)
260- require.True(t, legacyFormat, "migration-port failure must not install a new catalogue")
253+func TestMigrationFailsClosedOnSnapshotError(t *testing.T) {
254+ for _, tc := range []struct {
255+ name string
256+ err error
257+ }{
258+ {name: "repository unavailable", err: errors.New("repository unavailable")},
259+ } {
260+ t.Run(tc.name, func(t *testing.T) {
261+ dir := t.TempDir()
262+ writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"})
263+ ref := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}
264+ _, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"work": ref}, migrateErr: map[string]error{"work": tc.err}}})
265+ require.ErrorContains(t, err, tc.err.Error())
266+ data, replayErr := kv.Replay(StorePath(dir))
267+ require.NoError(t, replayErr)
268+ _, legacyFormat, classifyErr := classifyCatalogue(data)
269+ require.NoError(t, classifyErr)
270+ require.True(t, legacyFormat, "migration-port failure must not install a new catalogue")
271+ })
272+ }
261273 }
262274
263275 func TestOpenOrMigrateLegacyProbe(t *testing.T) {
@@ -126,7 +126,7 @@ func TestMigrationExistingState(t *testing.T) {
126 require.NoError(t, reopened.Catalogue.Close())126 require.NoError(t, reopened.Catalogue.Close())
127 }127 }
128 128
129-func TestMigrationOver4096(t *testing.T) {129+func TestMigrationPreservesLargeGeneration(t *testing.T) {
130 dir := t.TempDir()130 dir := t.TempDir()
131 writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "large", Cwd: "/tmp"})131 writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "large", Cwd: "/tmp"})
132 migration := &migrationStub{heads: map[string]domain.CheckpointRef{"large": {Generation: 5001, ManifestDigest: [32]byte{1}}}, migrateErr: map[string]error{}}132 migration := &migrationStub{heads: map[string]domain.CheckpointRef{"large": {Generation: 5001, ManifestDigest: [32]byte{1}}}, migrateErr: map[string]error{}}
@@ -248,16 +248,28 @@ func TestMigrationPreservesUncertain(t *testing.T) {
248 require.LessOrEqual(t, len(result.Records[0].DegradedReason), migrationDegradedReasonMaxBytes)248 require.LessOrEqual(t, len(result.Records[0].DegradedReason), migrationDegradedReasonMaxBytes)
249 require.NotContains(t, result.Records[0].DegradedReason, adapterCause)249 require.NotContains(t, result.Records[0].DegradedReason, adapterCause)
250 require.Contains(t, logs.String(), adapterCause, "the full adapter cause must remain diagnosable")250 require.Contains(t, logs.String(), adapterCause, "the full adapter cause must remain diagnosable")
251+}
251 252
252- failClosedDir := t.TempDir()253+func TestMigrationFailsClosedOnSnapshotError(t *testing.T) {
253- writeLegacyCatalogue(t, failClosedDir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"})254+ for _, tc := range []struct {
254- _, err = OpenOrMigrate(context.Background(), OpenDeps{StateDir: failClosedDir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"work": ref}, migrateErr: map[string]error{"work": errors.New("repository unavailable")}}})255+ name string
255- require.ErrorContains(t, err, "repository unavailable")256+ err error
256- data, replayErr := kv.Replay(StorePath(failClosedDir))257+ }{
257- require.NoError(t, replayErr)258+ {name: "repository unavailable", err: errors.New("repository unavailable")},
258- _, legacyFormat, classifyErr := classifyCatalogue(data)259+ } {
259- require.NoError(t, classifyErr)260+ t.Run(tc.name, func(t *testing.T) {
260- require.True(t, legacyFormat, "migration-port failure must not install a new catalogue")261+ dir := t.TempDir()
262+ writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"})
263+ ref := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}
264+ _, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"work": ref}, migrateErr: map[string]error{"work": tc.err}}})
265+ require.ErrorContains(t, err, tc.err.Error())
266+ data, replayErr := kv.Replay(StorePath(dir))
267+ require.NoError(t, replayErr)
268+ _, legacyFormat, classifyErr := classifyCatalogue(data)
269+ require.NoError(t, classifyErr)
270+ require.True(t, legacyFormat, "migration-port failure must not install a new catalogue")
271+ })
272+ }
261 }273 }
262 274
263 func TestOpenOrMigrateLegacyProbe(t *testing.T) {275 func TestOpenOrMigrateLegacyProbe(t *testing.T) {
M internal/persist/persist.go +69 -21
@@ -21,6 +21,10 @@ func StorePath(dir string) string { return filepath.Join(dir, filename) }
2121 type Persister struct {
2222 store ports.Store
2323 mu sync.Mutex
24+
25+ incarnationOwners map[domain.IncarnationID]string
26+ nameIncarnations map[string]domain.IncarnationID
27+ incarnationIndex bool
2428 }
2529
2630 // KVStore adapts the reusable WAL implementation to the persistence port.
@@ -146,19 +150,8 @@ func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) erro
146150 if len(records) == 0 {
147151 return errors.New("persist: empty catalogue batch")
148152 }
149- projected := make(map[string]domain.CatalogueRecord)
150- var decodeErr error
151- p.store.Range(func(key, value []byte) bool {
152- record, err := decodeRecordValue(string(key), value)
153- if err != nil {
154- decodeErr = err
155- return false
156- }
157- projected[string(key)] = record
158- return true
159- })
160- if decodeErr != nil {
161- return decodeErr
153+ if err := p.ensureIncarnationIndexLocked(); err != nil {
154+ return err
162155 }
163156 changes := make([]ports.StoreChange, 0, len(records))
164157 keys := make([]string, 0, len(records))
@@ -172,7 +165,6 @@ func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) erro
172165 return err
173166 }
174167 if record == nil {
175- delete(projected, name)
176168 changes = append(changes, ports.StoreChange{Key: []byte(name), Delete: true})
177169 continue
178170 }
@@ -183,20 +175,76 @@ func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) erro
183175 if err != nil {
184176 return err
185177 }
186- projected[name] = *record
187178 changes = append(changes, ports.StoreChange{Key: []byte(name), Value: value})
188179 }
189- all := make([]domain.CatalogueRecord, 0, len(projected))
190- for _, record := range projected {
191- all = append(all, record)
180+ incoming := make(map[domain.IncarnationID]string, len(records))
181+ for name, record := range records {
182+ if record == nil {
183+ continue
184+ }
185+ if previous, ok := incoming[record.IncarnationID]; ok && previous != name {
186+ return fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, name)
187+ }
188+ incoming[record.IncarnationID] = name
189+ if owner, ok := p.incarnationOwners[record.IncarnationID]; ok && owner != name {
190+ ownerNext, ownerTouched := records[owner]
191+ if !ownerTouched || (ownerNext != nil && ownerNext.IncarnationID == record.IncarnationID) {
192+ return fmt.Errorf("persist: duplicate incarnation for %q and %q", owner, name)
193+ }
194+ }
192195 }
193- if err := validateUniqueIncarnations(all); err != nil {
196+ if err := p.store.Batch(changes); err != nil {
197+ p.incarnationIndex = false
194198 return err
195199 }
196- if err := p.store.Batch(changes); err != nil {
200+ if err := p.store.Sync(); err != nil {
201+ p.incarnationIndex = false
197202 return err
198203 }
199- return p.store.Sync()
204+ for name := range records {
205+ if id, ok := p.nameIncarnations[name]; ok {
206+ delete(p.incarnationOwners, id)
207+ delete(p.nameIncarnations, name)
208+ }
209+ }
210+ for name, record := range records {
211+ if record != nil {
212+ p.incarnationOwners[record.IncarnationID] = name
213+ p.nameIncarnations[name] = record.IncarnationID
214+ }
215+ }
216+ return nil
217+}
218+
219+func (p *Persister) ensureIncarnationIndexLocked() error {
220+ if p.incarnationIndex {
221+ return nil
222+ }
223+ owners := make(map[domain.IncarnationID]string)
224+ names := make(map[string]domain.IncarnationID)
225+ var indexErr error
226+ p.store.Range(func(key, value []byte) bool {
227+ name := string(key)
228+ record, err := decodeRecordValue(name, value)
229+ if err != nil {
230+ indexErr = err
231+ return false
232+ }
233+ if previous, ok := owners[record.IncarnationID]; ok {
234+ indexErr = fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, name)
235+ return false
236+ }
237+ owners[record.IncarnationID] = name
238+ names[name] = record.IncarnationID
239+ return true
240+ })
241+ if indexErr != nil {
242+ return indexErr
243+ }
244+ p.incarnationOwners = owners
245+ p.nameIncarnations = names
246+ p.incarnationIndex = true
247+ return nil
200248 }
201249 func (p *Persister) Rename(oldName string, next domain.CatalogueRecord) error {
202250 if p == nil {
@@ -21,6 +21,10 @@ func StorePath(dir string) string { return filepath.Join(dir, filename) }
21 type Persister struct {21 type Persister struct {
22 store ports.Store22 store ports.Store
23 mu sync.Mutex23 mu sync.Mutex
24+
25+ incarnationOwners map[domain.IncarnationID]string
26+ nameIncarnations map[string]domain.IncarnationID
27+ incarnationIndex bool
24 }28 }
25 29
26 // KVStore adapts the reusable WAL implementation to the persistence port.30 // KVStore adapts the reusable WAL implementation to the persistence port.
@@ -146,19 +150,8 @@ func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) erro
146 if len(records) == 0 {150 if len(records) == 0 {
147 return errors.New("persist: empty catalogue batch")151 return errors.New("persist: empty catalogue batch")
148 }152 }
149- projected := make(map[string]domain.CatalogueRecord)153+ if err := p.ensureIncarnationIndexLocked(); err != nil {
150- var decodeErr error154+ return err
151- p.store.Range(func(key, value []byte) bool {
152- record, err := decodeRecordValue(string(key), value)
153- if err != nil {
154- decodeErr = err
155- return false
156- }
157- projected[string(key)] = record
158- return true
159- })
160- if decodeErr != nil {
161- return decodeErr
162 }155 }
163 changes := make([]ports.StoreChange, 0, len(records))156 changes := make([]ports.StoreChange, 0, len(records))
164 keys := make([]string, 0, len(records))157 keys := make([]string, 0, len(records))
@@ -172,7 +165,6 @@ func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) erro
172 return err165 return err
173 }166 }
174 if record == nil {167 if record == nil {
175- delete(projected, name)
176 changes = append(changes, ports.StoreChange{Key: []byte(name), Delete: true})168 changes = append(changes, ports.StoreChange{Key: []byte(name), Delete: true})
177 continue169 continue
178 }170 }
@@ -183,20 +175,76 @@ func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) erro
183 if err != nil {175 if err != nil {
184 return err176 return err
185 }177 }
186- projected[name] = *record
187 changes = append(changes, ports.StoreChange{Key: []byte(name), Value: value})178 changes = append(changes, ports.StoreChange{Key: []byte(name), Value: value})
188 }179 }
189- all := make([]domain.CatalogueRecord, 0, len(projected))180+ incoming := make(map[domain.IncarnationID]string, len(records))
190- for _, record := range projected {181+ for name, record := range records {
191- all = append(all, record)182+ if record == nil {
183+ continue
184+ }
185+ if previous, ok := incoming[record.IncarnationID]; ok && previous != name {
186+ return fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, name)
187+ }
188+ incoming[record.IncarnationID] = name
189+ if owner, ok := p.incarnationOwners[record.IncarnationID]; ok && owner != name {
190+ ownerNext, ownerTouched := records[owner]
191+ if !ownerTouched || (ownerNext != nil && ownerNext.IncarnationID == record.IncarnationID) {
192+ return fmt.Errorf("persist: duplicate incarnation for %q and %q", owner, name)
193+ }
194+ }
192 }195 }
193- if err := validateUniqueIncarnations(all); err != nil {196+ if err := p.store.Batch(changes); err != nil {
197+ p.incarnationIndex = false
194 return err198 return err
195 }199 }
196- if err := p.store.Batch(changes); err != nil {200+ if err := p.store.Sync(); err != nil {
201+ p.incarnationIndex = false
197 return err202 return err
198 }203 }
199- return p.store.Sync()204+ for name := range records {
205+ if id, ok := p.nameIncarnations[name]; ok {
206+ delete(p.incarnationOwners, id)
207+ delete(p.nameIncarnations, name)
208+ }
209+ }
210+ for name, record := range records {
211+ if record != nil {
212+ p.incarnationOwners[record.IncarnationID] = name
213+ p.nameIncarnations[name] = record.IncarnationID
214+ }
215+ }
216+ return nil
217+}
218+
219+func (p *Persister) ensureIncarnationIndexLocked() error {
220+ if p.incarnationIndex {
221+ return nil
222+ }
223+ owners := make(map[domain.IncarnationID]string)
224+ names := make(map[string]domain.IncarnationID)
225+ var indexErr error
226+ p.store.Range(func(key, value []byte) bool {
227+ name := string(key)
228+ record, err := decodeRecordValue(name, value)
229+ if err != nil {
230+ indexErr = err
231+ return false
232+ }
233+ if previous, ok := owners[record.IncarnationID]; ok {
234+ indexErr = fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, name)
235+ return false
236+ }
237+ owners[record.IncarnationID] = name
238+ names[name] = record.IncarnationID
239+ return true
240+ })
241+ if indexErr != nil {
242+ return indexErr
243+ }
244+ p.incarnationOwners = owners
245+ p.nameIncarnations = names
246+ p.incarnationIndex = true
247+ return nil
200 }248 }
201 func (p *Persister) Rename(oldName string, next domain.CatalogueRecord) error {249 func (p *Persister) Rename(oldName string, next domain.CatalogueRecord) error {
202 if p == nil {250 if p == nil {
M internal/persist/persist_test.go +1 -1
@@ -34,7 +34,7 @@ func TestCatalogueBatchSyncBehavior(t *testing.T) {
3434 return
3535 }
3636 }
37- }).Twice()
37+ }).Once()
3838 store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error {
3939 for _, change := range changes {
4040 if change.Delete {
@@ -34,7 +34,7 @@ func TestCatalogueBatchSyncBehavior(t *testing.T) {
34 return34 return
35 }35 }
36 }36 }
37- }).Twice()37+ }).Once()
38 store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error {38 store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error {
39 for _, change := range changes {39 for _, change := range changes {
40 if change.Delete {40 if change.Delete {
M internal/persist/record.go +2 -5
@@ -125,15 +125,12 @@ func decodeRecordValue(name string, value []byte) (domain.CatalogueRecord, error
125125 return record, nil
126126 }
127127
128-func appendString(buf []byte, value string) []byte {
129- buf = binary.BigEndian.AppendUint32(buf, uint32(len(value)))
130- return append(buf, value...)
131-}
132128 func appendCheckedString(buf []byte, value string) ([]byte, error) {
133129 if uint64(len(value)) > math.MaxUint32 {
134130 return nil, errors.New("persist: string too large")
135131 }
136- return appendString(buf, value), nil
132+ buf = binary.BigEndian.AppendUint32(buf, uint32(len(value)))
133+ return append(buf, value...), nil
137134 }
138135 func appendRef(buf []byte, ref *domain.CheckpointRef) []byte {
139136 if ref == nil {
@@ -125,15 +125,12 @@ func decodeRecordValue(name string, value []byte) (domain.CatalogueRecord, error
125 return record, nil125 return record, nil
126 }126 }
127 127
128-func appendString(buf []byte, value string) []byte {
129- buf = binary.BigEndian.AppendUint32(buf, uint32(len(value)))
130- return append(buf, value...)
131-}
132 func appendCheckedString(buf []byte, value string) ([]byte, error) {128 func appendCheckedString(buf []byte, value string) ([]byte, error) {
133 if uint64(len(value)) > math.MaxUint32 {129 if uint64(len(value)) > math.MaxUint32 {
134 return nil, errors.New("persist: string too large")130 return nil, errors.New("persist: string too large")
135 }131 }
136- return appendString(buf, value), nil132+ buf = binary.BigEndian.AppendUint32(buf, uint32(len(value)))
133+ return append(buf, value...), nil
137 }134 }
138 func appendRef(buf []byte, ref *domain.CheckpointRef) []byte {135 func appendRef(buf []byte, ref *domain.CheckpointRef) []byte {
139 if ref == nil {136 if ref == nil {
M internal/ports/wire_test.go +6 -0
@@ -765,6 +765,12 @@ func TestSessionInfoRecoveryState(t *testing.T) {
765765 assertTrailingGarbageFails(t, payload, UnmarshalSessions)
766766 })
767767 }
768+ for _, state := range []byte{0, 5} {
769+ payload := append([]byte(nil), tests[0].want...)
770+ payload[len(payload)-1] = state
771+ _, err := UnmarshalSessions(payload)
772+ require.Error(t, err, "state %d", state)
773+ }
768774 }
769775
770776 func TestSessionsGoldenAndRoundTrip(t *testing.T) {
@@ -765,6 +765,12 @@ func TestSessionInfoRecoveryState(t *testing.T) {
765 assertTrailingGarbageFails(t, payload, UnmarshalSessions)765 assertTrailingGarbageFails(t, payload, UnmarshalSessions)
766 })766 })
767 }767 }
768+ for _, state := range []byte{0, 5} {
769+ payload := append([]byte(nil), tests[0].want...)
770+ payload[len(payload)-1] = state
771+ _, err := UnmarshalSessions(payload)
772+ require.Error(t, err, "state %d", state)
773+ }
768 }774 }
769 775
770 func TestSessionsGoldenAndRoundTrip(t *testing.T) {776 func TestSessionsGoldenAndRoundTrip(t *testing.T) {
M internal/usecase/command/command_test.go +25 -7
@@ -6,13 +6,31 @@ import (
66 )
77
88 func TestParsePositiveUint64(t *testing.T) {
9- const maxUint64 = "18446744073709551615"
10- got, err := ParsePositiveUint64([]string{maxUint64})
11- if err != nil || got != ^uint64(0) {
12- t.Fatalf("ParsePositiveUint64(max) = %d, %v", got, err)
13- }
14- if _, err := ParsePositiveUint64([]string{"18446744073709551616"}); !errors.Is(err, ErrInvalidArguments) {
15- t.Fatalf("ParsePositiveUint64(overflow) error = %v", err)
9+ for _, tt := range []struct {
10+ name string
11+ args []string
12+ want uint64
13+ invalid bool
14+ }{
15+ {name: "maximum", args: []string{"18446744073709551615"}, want: ^uint64(0)},
16+ {name: "overflow", args: []string{"18446744073709551616"}, invalid: true},
17+ {name: "zero", args: []string{"0"}, invalid: true},
18+ {name: "leading zero", args: []string{"01"}, invalid: true},
19+ {name: "empty arguments", invalid: true},
20+ {name: "multiple arguments", args: []string{"1", "2"}, invalid: true},
21+ } {
22+ t.Run(tt.name, func(t *testing.T) {
23+ got, err := ParsePositiveUint64(tt.args)
24+ if tt.invalid {
25+ if !errors.Is(err, ErrInvalidArguments) {
26+ t.Fatalf("error = %v", err)
27+ }
28+ return
29+ }
30+ if err != nil || got != tt.want {
31+ t.Fatalf("ParsePositiveUint64() = %d, %v", got, err)
32+ }
33+ })
1634 }
1735 }
1836
@@ -6,13 +6,31 @@ import (
6 )6 )
7 7
8 func TestParsePositiveUint64(t *testing.T) {8 func TestParsePositiveUint64(t *testing.T) {
9- const maxUint64 = "18446744073709551615"9+ for _, tt := range []struct {
10- got, err := ParsePositiveUint64([]string{maxUint64})10+ name string
11- if err != nil || got != ^uint64(0) {11+ args []string
12- t.Fatalf("ParsePositiveUint64(max) = %d, %v", got, err)12+ want uint64
13- }13+ invalid bool
14- if _, err := ParsePositiveUint64([]string{"18446744073709551616"}); !errors.Is(err, ErrInvalidArguments) {14+ }{
15- t.Fatalf("ParsePositiveUint64(overflow) error = %v", err)15+ {name: "maximum", args: []string{"18446744073709551615"}, want: ^uint64(0)},
16+ {name: "overflow", args: []string{"18446744073709551616"}, invalid: true},
17+ {name: "zero", args: []string{"0"}, invalid: true},
18+ {name: "leading zero", args: []string{"01"}, invalid: true},
19+ {name: "empty arguments", invalid: true},
20+ {name: "multiple arguments", args: []string{"1", "2"}, invalid: true},
21+ } {
22+ t.Run(tt.name, func(t *testing.T) {
23+ got, err := ParsePositiveUint64(tt.args)
24+ if tt.invalid {
25+ if !errors.Is(err, ErrInvalidArguments) {
26+ t.Fatalf("error = %v", err)
27+ }
28+ return
29+ }
30+ if err != nil || got != tt.want {
31+ t.Fatalf("ParsePositiveUint64() = %d, %v", got, err)
32+ }
33+ })
16 }34 }
17 }35 }
18 36
M internal/usecase/daemon/catalogue.go +10 -9
@@ -25,15 +25,15 @@ func (d *Daemon) updateCatalogueMetadata(update domain.CatalogueMetadataUpdate)
2525 // session state locks during I/O. Only a successfully committed newer revision
2626 // subsumes an older caller. Failed revisions retain their rollback until later
2727 // failed mutations have unwound, keeping live metadata at the durable boundary.
28-func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update domain.CatalogueMetadataUpdate, rollback func() bool) error {
28+func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update domain.CatalogueMetadataUpdate, rollback func() bool) (bool, error) {
2929 if sess == nil {
30- return nil
30+ return false, nil
3131 }
3232 sess.metadataPersistMu.Lock()
3333 defer sess.metadataPersistMu.Unlock()
3434
3535 if sess.metadataDurableVersion >= version {
36- return nil
36+ return false, nil
3737 }
3838 if err := d.updateCatalogueMetadata(update); err != nil {
3939 if sess.metadataFailedRollbacks == nil {
@@ -48,8 +48,8 @@ func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update do
4848 if current > sess.metadataLiveVersion {
4949 sess.metadataLiveVersion = current
5050 }
51- sess.reconcileFailedMetadataLocked()
52- return err
51+ rollbackRejected := sess.reconcileFailedMetadataLocked()
52+ return rollbackRejected, err
5353 }
5454
5555 sess.metadataDurableVersion = version
@@ -58,24 +58,25 @@ func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update do
5858 delete(sess.metadataFailedRollbacks, failedVersion)
5959 }
6060 }
61- return nil
61+ return false, nil
6262 }
6363
6464 // reconcileFailedMetadataLocked unwinds only a contiguous failed suffix. An
6565 // older failure remains pending while a newer mutation is still writing.
6666 // Rollbacks run only after authority I/O has returned and acquire state locks in
6767 // the daemon-before-session order.
68-func (s *session) reconcileFailedMetadataLocked() {
68+func (s *session) reconcileFailedMetadataLocked() bool {
6969 for s.metadataLiveVersion > s.metadataDurableVersion {
7070 version := s.metadataLiveVersion
7171 rollback, failed := s.metadataFailedRollbacks[version]
7272 if !failed {
73- return
73+ return false
7474 }
7575 if rollback != nil && !rollback() {
76- return
76+ return true
7777 }
7878 delete(s.metadataFailedRollbacks, version)
7979 s.metadataLiveVersion--
8080 }
81+ return false
8182 }
@@ -25,15 +25,15 @@ func (d *Daemon) updateCatalogueMetadata(update domain.CatalogueMetadataUpdate)
25 // session state locks during I/O. Only a successfully committed newer revision25 // session state locks during I/O. Only a successfully committed newer revision
26 // subsumes an older caller. Failed revisions retain their rollback until later26 // subsumes an older caller. Failed revisions retain their rollback until later
27 // failed mutations have unwound, keeping live metadata at the durable boundary.27 // failed mutations have unwound, keeping live metadata at the durable boundary.
28-func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update domain.CatalogueMetadataUpdate, rollback func() bool) error {28+func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update domain.CatalogueMetadataUpdate, rollback func() bool) (bool, error) {
29 if sess == nil {29 if sess == nil {
30- return nil30+ return false, nil
31 }31 }
32 sess.metadataPersistMu.Lock()32 sess.metadataPersistMu.Lock()
33 defer sess.metadataPersistMu.Unlock()33 defer sess.metadataPersistMu.Unlock()
34 34
35 if sess.metadataDurableVersion >= version {35 if sess.metadataDurableVersion >= version {
36- return nil36+ return false, nil
37 }37 }
38 if err := d.updateCatalogueMetadata(update); err != nil {38 if err := d.updateCatalogueMetadata(update); err != nil {
39 if sess.metadataFailedRollbacks == nil {39 if sess.metadataFailedRollbacks == nil {
@@ -48,8 +48,8 @@ func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update do
48 if current > sess.metadataLiveVersion {48 if current > sess.metadataLiveVersion {
49 sess.metadataLiveVersion = current49 sess.metadataLiveVersion = current
50 }50 }
51- sess.reconcileFailedMetadataLocked()51+ rollbackRejected := sess.reconcileFailedMetadataLocked()
52- return err52+ return rollbackRejected, err
53 }53 }
54 54
55 sess.metadataDurableVersion = version55 sess.metadataDurableVersion = version
@@ -58,24 +58,25 @@ func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update do
58 delete(sess.metadataFailedRollbacks, failedVersion)58 delete(sess.metadataFailedRollbacks, failedVersion)
59 }59 }
60 }60 }
61- return nil61+ return false, nil
62 }62 }
63 63
64 // reconcileFailedMetadataLocked unwinds only a contiguous failed suffix. An64 // reconcileFailedMetadataLocked unwinds only a contiguous failed suffix. An
65 // older failure remains pending while a newer mutation is still writing.65 // older failure remains pending while a newer mutation is still writing.
66 // Rollbacks run only after authority I/O has returned and acquire state locks in66 // Rollbacks run only after authority I/O has returned and acquire state locks in
67 // the daemon-before-session order.67 // the daemon-before-session order.
68-func (s *session) reconcileFailedMetadataLocked() {68+func (s *session) reconcileFailedMetadataLocked() bool {
69 for s.metadataLiveVersion > s.metadataDurableVersion {69 for s.metadataLiveVersion > s.metadataDurableVersion {
70 version := s.metadataLiveVersion70 version := s.metadataLiveVersion
71 rollback, failed := s.metadataFailedRollbacks[version]71 rollback, failed := s.metadataFailedRollbacks[version]
72 if !failed {72 if !failed {
73- return73+ return false
74 }74 }
75 if rollback != nil && !rollback() {75 if rollback != nil && !rollback() {
76- return76+ return true
77 }77 }
78 delete(s.metadataFailedRollbacks, version)78 delete(s.metadataFailedRollbacks, version)
79 s.metadataLiveVersion--79 s.metadataLiveVersion--
80 }80 }
81+ return false
81 }82 }
A internal/usecase/daemon/coderabbit_regression_test.go +63 -0
@@ -0,0 +1,63 @@
1+package daemon
2+
3+import (
4+ "context"
5+ "sync"
6+ "testing"
7+
8+ "github.com/stretchr/testify/require"
9+
10+ "github.com/bnema/vev/internal/domain"
11+ "github.com/bnema/vev/internal/ports"
12+)
13+
14+func TestRestoreTransientLoadFailurePreservesHealthyAuthority(t *testing.T) {
15+ record := durableRecoveryRecord(0)
16+ repository := &durableRecoveryRepository{
17+ errors: map[string]error{record.Name: ports.ErrBudgetExhausted},
18+ loads: make(map[string]int),
19+ repairs: make(map[string]domain.CheckpointRef),
20+ }
21+ d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository)
22+
23+ d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))
24+
25+ persisted, ok, err := catalogue.Record(record.Name)
26+ require.NoError(t, err)
27+ require.True(t, ok)
28+ require.Equal(t, domain.RecoveryHealthy, persisted.RecoveryState)
29+ d.mu.Lock()
30+ entry := d.stopped[record.Name]
31+ d.mu.Unlock()
32+ require.Equal(t, runtimeRestoring, entry.state)
33+ require.NotEqual(t, runtimeDegraded, entry.state)
34+ select {
35+ case <-entry.restoreDone:
36+ default:
37+ t.Fatal("restore completion was not signaled")
38+ }
39+}
40+
41+func TestFinishRecordRestoreAlwaysClosesProvidedBarrier(t *testing.T) {
42+ d := newTestDaemon(t, nil, stubClock{})
43+ done := make(chan struct{})
44+
45+ d.finishRecordRestore(domain.CatalogueRecord{Name: "removed"}, nil, done)
46+
47+ select {
48+ case <-done:
49+ default:
50+ t.Fatal("restore completion was not signaled")
51+ }
52+}
53+
54+func TestSnapshotStopContextCancelIsIdempotent(t *testing.T) {
55+ deadline := &snapshotShutdownDeadline{done: make(chan struct{})}
56+ _, cancel := snapshotStopContext(deadline)
57+
58+ var wg sync.WaitGroup
59+ for range 8 {
60+ wg.Go(cancel)
61+ }
62+ wg.Wait()
63+}
@@ -0,0 +1,63 @@
1+package daemon
2+
3+import (
4+ "context"
5+ "sync"
6+ "testing"
7+
8+ "github.com/stretchr/testify/require"
9+
10+ "github.com/bnema/vev/internal/domain"
11+ "github.com/bnema/vev/internal/ports"
12+)
13+
14+func TestRestoreTransientLoadFailurePreservesHealthyAuthority(t *testing.T) {
15+ record := durableRecoveryRecord(0)
16+ repository := &durableRecoveryRepository{
17+ errors: map[string]error{record.Name: ports.ErrBudgetExhausted},
18+ loads: make(map[string]int),
19+ repairs: make(map[string]domain.CheckpointRef),
20+ }
21+ d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository)
22+
23+ d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))
24+
25+ persisted, ok, err := catalogue.Record(record.Name)
26+ require.NoError(t, err)
27+ require.True(t, ok)
28+ require.Equal(t, domain.RecoveryHealthy, persisted.RecoveryState)
29+ d.mu.Lock()
30+ entry := d.stopped[record.Name]
31+ d.mu.Unlock()
32+ require.Equal(t, runtimeRestoring, entry.state)
33+ require.NotEqual(t, runtimeDegraded, entry.state)
34+ select {
35+ case <-entry.restoreDone:
36+ default:
37+ t.Fatal("restore completion was not signaled")
38+ }
39+}
40+
41+func TestFinishRecordRestoreAlwaysClosesProvidedBarrier(t *testing.T) {
42+ d := newTestDaemon(t, nil, stubClock{})
43+ done := make(chan struct{})
44+
45+ d.finishRecordRestore(domain.CatalogueRecord{Name: "removed"}, nil, done)
46+
47+ select {
48+ case <-done:
49+ default:
50+ t.Fatal("restore completion was not signaled")
51+ }
52+}
53+
54+func TestSnapshotStopContextCancelIsIdempotent(t *testing.T) {
55+ deadline := &snapshotShutdownDeadline{done: make(chan struct{})}
56+ _, cancel := snapshotStopContext(deadline)
57+
58+ var wg sync.WaitGroup
59+ for range 8 {
60+ wg.Go(cancel)
61+ }
62+ wg.Wait()
63+}
M internal/usecase/daemon/control.go +4 -1
@@ -465,6 +465,9 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) {
465465 if ctx == nil {
466466 ctx = e.d.serveCtx
467467 }
468+ if ctx == nil {
469+ ctx = context.Background()
470+ }
468471 switch action {
469472 case "retry":
470473 if err := e.d.degradedRecovery.Retry(ctx, e.recoveryName); err != nil {
@@ -549,7 +552,7 @@ func (e controlExec) restoreExplicitRecovery(ctx context.Context) error {
549552 e.d.stopped[record.Name] = entry
550553 e.d.mu.Unlock()
551554 err = e.d.restoreRecord(ctx, record)
552- e.d.finishRecordRestore(record, err, done)
555+ e.d.finishExplicitRecordRestore(record, err, done)
553556 return err
554557 }
555558
@@ -465,6 +465,9 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) {
465 if ctx == nil {465 if ctx == nil {
466 ctx = e.d.serveCtx466 ctx = e.d.serveCtx
467 }467 }
468+ if ctx == nil {
469+ ctx = context.Background()
470+ }
468 switch action {471 switch action {
469 case "retry":472 case "retry":
470 if err := e.d.degradedRecovery.Retry(ctx, e.recoveryName); err != nil {473 if err := e.d.degradedRecovery.Retry(ctx, e.recoveryName); err != nil {
@@ -549,7 +552,7 @@ func (e controlExec) restoreExplicitRecovery(ctx context.Context) error {
549 e.d.stopped[record.Name] = entry552 e.d.stopped[record.Name] = entry
550 e.d.mu.Unlock()553 e.d.mu.Unlock()
551 err = e.d.restoreRecord(ctx, record)554 err = e.d.restoreRecord(ctx, record)
552- e.d.finishRecordRestore(record, err, done)555+ e.d.finishExplicitRecordRestore(record, err, done)
553 return err556 return err
554 }557 }
555 558
M internal/usecase/daemon/daemon.go +15 -5
@@ -416,18 +416,27 @@ func (d *Daemon) allocateLifecycleCreatedAtLocked() (int64, error) {
416416 }
417417
418418 func (d *Daemon) nowUnixNano() int64 {
419- if d == nil || d.clock == nil {
420- return time.Now().UnixNano()
421- }
422419 return d.clock.Now().UnixNano()
423420 }
424421
422+type systemClock struct{}
423+type systemTimer struct{ *time.Timer }
424+
425+func (systemClock) Now() time.Time { return time.Now() }
426+func (systemClock) NewTimer(delay time.Duration) ports.Timer {
427+ return systemTimer{Timer: time.NewTimer(delay)}
428+}
429+func (t systemTimer) C() <-chan time.Time { return t.Timer.C }
430+
425431 // New constructs a Daemon. ptys spawns PTY-backed children, clock drives the
426432 // render debounce, and log receives diagnostics (defaults to slog.Default).
427433 func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Option) *Daemon {
428434 if log == nil {
429435 log = slog.Default()
430436 }
437+ if clock == nil {
438+ clock = systemClock{}
439+ }
431440 shell := os.Getenv("SHELL")
432441 if shell == "" {
433442 shell = "/bin/sh"
@@ -613,7 +622,7 @@ func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error {
613622 closeErr = d.catalogue.Close()
614623 }
615624 if closeErr != nil {
616- d.log.Warn("closing session persister failed", "err", closeErr)
625+ d.log.Warn("closing session catalogue failed", "err", closeErr)
617626 }
618627 return nil
619628 }
@@ -745,8 +754,9 @@ func snapshotStopContext(deadline *snapshotShutdownDeadline) (context.Context, c
745754 case <-stop:
746755 }
747756 }()
757+ var stopOnce sync.Once
748758 return ctx, func() {
749- close(stop)
759+ stopOnce.Do(func() { close(stop) })
750760 cancel()
751761 }
752762 }
@@ -416,18 +416,27 @@ func (d *Daemon) allocateLifecycleCreatedAtLocked() (int64, error) {
416 }416 }
417 417
418 func (d *Daemon) nowUnixNano() int64 {418 func (d *Daemon) nowUnixNano() int64 {
419- if d == nil || d.clock == nil {
420- return time.Now().UnixNano()
421- }
422 return d.clock.Now().UnixNano()419 return d.clock.Now().UnixNano()
423 }420 }
424 421
422+type systemClock struct{}
423+type systemTimer struct{ *time.Timer }
424+
425+func (systemClock) Now() time.Time { return time.Now() }
426+func (systemClock) NewTimer(delay time.Duration) ports.Timer {
427+ return systemTimer{Timer: time.NewTimer(delay)}
428+}
429+func (t systemTimer) C() <-chan time.Time { return t.Timer.C }
430+
425 // New constructs a Daemon. ptys spawns PTY-backed children, clock drives the431 // New constructs a Daemon. ptys spawns PTY-backed children, clock drives the
426 // render debounce, and log receives diagnostics (defaults to slog.Default).432 // render debounce, and log receives diagnostics (defaults to slog.Default).
427 func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Option) *Daemon {433 func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Option) *Daemon {
428 if log == nil {434 if log == nil {
429 log = slog.Default()435 log = slog.Default()
430 }436 }
437+ if clock == nil {
438+ clock = systemClock{}
439+ }
431 shell := os.Getenv("SHELL")440 shell := os.Getenv("SHELL")
432 if shell == "" {441 if shell == "" {
433 shell = "/bin/sh"442 shell = "/bin/sh"
@@ -613,7 +622,7 @@ func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error {
613 closeErr = d.catalogue.Close()622 closeErr = d.catalogue.Close()
614 }623 }
615 if closeErr != nil {624 if closeErr != nil {
616- d.log.Warn("closing session persister failed", "err", closeErr)625+ d.log.Warn("closing session catalogue failed", "err", closeErr)
617 }626 }
618 return nil627 return nil
619 }628 }
@@ -745,8 +754,9 @@ func snapshotStopContext(deadline *snapshotShutdownDeadline) (context.Context, c
745 case <-stop:754 case <-stop:
746 }755 }
747 }()756 }()
757+ var stopOnce sync.Once
748 return ctx, func() {758 return ctx, func() {
749- close(stop)759+ stopOnce.Do(func() { close(stop) })
750 cancel()760 cancel()
751 }761 }
752 }762 }
M internal/usecase/daemon/daemon_test.go +6 -0
@@ -68,6 +68,12 @@ func newTestDaemon(t testing.TB, ptys ports.PTYFactory, clk ports.Clock) *Daemon
6868 return newTestDaemonWithCleanup(t, ptys, clk, true)
6969 }
7070
71+func createSessionForTest(d *Daemon, name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) {
72+ d.mu.Lock()
73+ defer d.mu.Unlock()
74+ return d.createSessionLocked(name, ephemeral, cwd, sz, term, env, restoredTabNames...)
75+}
76+
7177 func newTestDaemonWithCleanup(t testing.TB, ptys ports.PTYFactory, clk ports.Clock, registerCleanup bool) *Daemon {
7278 t.Helper()
7379 d := New(ptys, clk, slog.New(slog.NewTextHandler(io.Discard, nil)))
@@ -68,6 +68,12 @@ func newTestDaemon(t testing.TB, ptys ports.PTYFactory, clk ports.Clock) *Daemon
68 return newTestDaemonWithCleanup(t, ptys, clk, true)68 return newTestDaemonWithCleanup(t, ptys, clk, true)
69 }69 }
70 70
71+func createSessionForTest(d *Daemon, name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) {
72+ d.mu.Lock()
73+ defer d.mu.Unlock()
74+ return d.createSessionLocked(name, ephemeral, cwd, sz, term, env, restoredTabNames...)
75+}
76+
71 func newTestDaemonWithCleanup(t testing.TB, ptys ports.PTYFactory, clk ports.Clock, registerCleanup bool) *Daemon {77 func newTestDaemonWithCleanup(t testing.TB, ptys ports.PTYFactory, clk ports.Clock, registerCleanup bool) *Daemon {
72 t.Helper()78 t.Helper()
73 d := New(ptys, clk, slog.New(slog.NewTextHandler(io.Discard, nil)))79 d := New(ptys, clk, slog.New(slog.NewTextHandler(io.Discard, nil)))
M internal/usecase/daemon/durable_authority_test.go +13 -4
@@ -3,6 +3,9 @@ package daemon
33 import (
44 "context"
55 "crypto/rand"
6+ "testing"
7+
8+ "github.com/stretchr/testify/require"
69
710 "github.com/bnema/vev/internal/domain"
811 "github.com/bnema/vev/internal/persist"
@@ -24,16 +27,15 @@ func (testLifecycleRepository) DeleteDeletionTombstone(context.Context, domain.I
2427
2528 // WithStore is package-test composition glue. Production injects the same
2629 // recovery coordinator explicitly from internal/app.
27-func WithStore(store ports.Store) Option {
30+func WithStore(t testing.TB, store ports.Store) Option {
31+ t.Helper()
2832 return func(d *Daemon) {
2933 if store == nil {
3034 return
3135 }
3236 catalogue := persist.New(store)
3337 records, err := catalogue.Records()
34- if err != nil {
35- return
36- }
38+ require.NoError(t, err)
3739 d.catalogue = catalogue
3840 d.persistEnabled = true
3941 d.catalogueRecords = records
@@ -47,3 +49,10 @@ func WithStore(store ports.Store) Option {
4749 d.checkpointRecovery = authority
4850 }
4951 }
52+
53+func testPersister(t testing.TB, d *Daemon) *persist.Persister {
54+ t.Helper()
55+ persister, ok := d.catalogue.(*persist.Persister)
56+ require.True(t, ok, "catalogue type %T is not *persist.Persister", d.catalogue)
57+ return persister
58+}
@@ -3,6 +3,9 @@ package daemon
3 import (3 import (
4 "context"4 "context"
5 "crypto/rand"5 "crypto/rand"
6+ "testing"
7+
8+ "github.com/stretchr/testify/require"
6 9
7 "github.com/bnema/vev/internal/domain"10 "github.com/bnema/vev/internal/domain"
8 "github.com/bnema/vev/internal/persist"11 "github.com/bnema/vev/internal/persist"
@@ -24,16 +27,15 @@ func (testLifecycleRepository) DeleteDeletionTombstone(context.Context, domain.I
24 27
25 // WithStore is package-test composition glue. Production injects the same28 // WithStore is package-test composition glue. Production injects the same
26 // recovery coordinator explicitly from internal/app.29 // recovery coordinator explicitly from internal/app.
27-func WithStore(store ports.Store) Option {30+func WithStore(t testing.TB, store ports.Store) Option {
31+ t.Helper()
28 return func(d *Daemon) {32 return func(d *Daemon) {
29 if store == nil {33 if store == nil {
30 return34 return
31 }35 }
32 catalogue := persist.New(store)36 catalogue := persist.New(store)
33 records, err := catalogue.Records()37 records, err := catalogue.Records()
34- if err != nil {38+ require.NoError(t, err)
35- return
36- }
37 d.catalogue = catalogue39 d.catalogue = catalogue
38 d.persistEnabled = true40 d.persistEnabled = true
39 d.catalogueRecords = records41 d.catalogueRecords = records
@@ -47,3 +49,10 @@ func WithStore(store ports.Store) Option {
47 d.checkpointRecovery = authority49 d.checkpointRecovery = authority
48 }50 }
49 }51 }
52+
53+func testPersister(t testing.TB, d *Daemon) *persist.Persister {
54+ t.Helper()
55+ persister, ok := d.catalogue.(*persist.Persister)
56+ require.True(t, ok, "catalogue type %T is not *persist.Persister", d.catalogue)
57+ return persister
58+}
M internal/usecase/daemon/durable_recovery_test.go +149 -58
@@ -7,8 +7,10 @@ import (
77 "fmt"
88 "io"
99 "log/slog"
10+ "os"
1011 "sync"
1112 "sync/atomic"
13+ "syscall"
1214 "testing"
1315 "time"
1416
@@ -226,67 +228,73 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) {
226228 }
227229
228230 func TestCatalogueRestoreIndependent(t *testing.T) {
229- emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{})
230- emptyDone := make(chan struct{})
231- go func() {
232- emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue))
233- close(emptyDone)
234- }()
235- select {
236- case <-emptyDone:
237- case <-time.After(time.Second):
238- t.Fatal("empty catalogue restoration blocked")
239- }
240-
241- healthy := durableRecoveryRecord(0)
242- broken := durableRecoveryRecord(1)
243- repository := &durableRecoveryRepository{
244- generations: map[string]ports.SnapshotGeneration{healthy.Name: validGeneration(t, healthy)},
245- errors: map[string]error{broken.Name: ports.ErrBudgetExhausted},
246- loads: make(map[string]int),
247- repairs: make(map[string]domain.CheckpointRef),
248- }
249- d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository)
231+ t.Run("empty catalogue restoration", func(t *testing.T) {
232+ emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{})
233+ emptyDone := make(chan struct{})
234+ go func() {
235+ emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue))
236+ close(emptyDone)
237+ }()
238+ select {
239+ case <-emptyDone:
240+ case <-time.After(time.Second):
241+ t.Fatal("empty catalogue restoration blocked")
242+ }
243+ })
250244
251- d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))
245+ t.Run("healthy and broken restoration", func(t *testing.T) {
246+ healthy := durableRecoveryRecord(0)
247+ broken := durableRecoveryRecord(1)
248+ repository := &durableRecoveryRepository{
249+ generations: map[string]ports.SnapshotGeneration{healthy.Name: validGeneration(t, healthy)},
250+ errors: map[string]error{broken.Name: errors.New("corrupt checkpoint")},
251+ loads: make(map[string]int),
252+ repairs: make(map[string]domain.CheckpointRef),
253+ }
254+ d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository)
255+
256+ d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))
257+
258+ require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state)
259+ require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state)
260+ for _, name := range []string{healthy.Name, broken.Name} {
261+ select {
262+ case <-d.stopped[name].restoreDone:
263+ default:
264+ t.Fatalf("restore completion for %q was not closed", name)
265+ }
266+ }
267+ degraded, ok, _ := catalogue.Record(broken.Name)
268+ require.True(t, ok)
269+ require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState)
270+ require.Equal(t, 1, repository.loads[healthy.Name])
271+ require.Equal(t, 1, repository.loads[broken.Name])
272+ require.Empty(t, d.sessions, "a corrupt record must never open a replacement runtime")
273+ })
252274
253- require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state)
254- require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state)
255- for _, name := range []string{healthy.Name, broken.Name} {
256- select {
257- case <-d.stopped[name].restoreDone:
258- default:
259- t.Fatalf("restore completion for %q was not closed", name)
275+ t.Run("fallback promotion", func(t *testing.T) {
276+ fallback := durableRecoveryRecord(2)
277+ fallbackRef := domain.CheckpointRef{Generation: fallback.Committed.Generation - 1}
278+ fallbackSnapshotRecord := fallback
279+ fallbackSnapshotRecord.Committed = &fallbackRef
280+ fallbackGeneration := validGeneration(t, fallbackSnapshotRecord)
281+ fallbackRef.ManifestDigest = sha256.Sum256(fallbackGeneration.Manifest)
282+ fallback.Fallbacks[0] = &fallbackRef
283+ fallbackRepository := &durableRecoveryRepository{
284+ generations: map[string]ports.SnapshotGeneration{fallback.Name: fallbackGeneration},
285+ errors: make(map[string]error),
286+ loads: make(map[string]int),
287+ repairs: make(map[string]domain.CheckpointRef),
260288 }
261- }
262- degraded, ok, _ := catalogue.Record(broken.Name)
263- require.True(t, ok)
264- require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState)
265- require.Equal(t, 1, repository.loads[healthy.Name])
266- require.Equal(t, 1, repository.loads[broken.Name])
267- require.Empty(t, d.sessions, "a corrupt record must never open a replacement runtime")
268-
269- fallback := durableRecoveryRecord(2)
270- fallbackRef := domain.CheckpointRef{Generation: fallback.Committed.Generation - 1}
271- fallbackSnapshotRecord := fallback
272- fallbackSnapshotRecord.Committed = &fallbackRef
273- fallbackGeneration := validGeneration(t, fallbackSnapshotRecord)
274- fallbackRef.ManifestDigest = sha256.Sum256(fallbackGeneration.Manifest)
275- fallback.Fallbacks[0] = &fallbackRef
276- fallbackRepository := &durableRecoveryRepository{
277- generations: map[string]ports.SnapshotGeneration{fallback.Name: fallbackGeneration},
278- errors: make(map[string]error),
279- loads: make(map[string]int),
280- repairs: make(map[string]domain.CheckpointRef),
281- }
282- fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository)
283- fallbackDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, fallbackCatalogue))
284- promoted, ok, _ := fallbackCatalogue.Record(fallback.Name)
285- require.True(t, ok)
286- require.Equal(t, fallbackRef, *promoted.Committed)
287- require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state)
288- require.Equal(t, 3, fallbackRepository.loads[fallback.Name])
289- require.Equal(t, fallbackRef, fallbackRepository.repairs[fallback.IncarnationID.String()])
289+ fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository)
290+ fallbackDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, fallbackCatalogue))
291+ promoted, ok, _ := fallbackCatalogue.Record(fallback.Name)
292+ require.True(t, ok)
293+ require.Equal(t, fallbackRef, *promoted.Committed)
294+ require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state)
295+ require.Equal(t, 3, fallbackRepository.loads[fallback.Name])
296+ require.Equal(t, fallbackRef, fallbackRepository.repairs[fallback.IncarnationID.String()])
297+ })
290298 }
291299
292300 func TestCatalogueRestoreKeepsCommittedFallbackHealthyWhenHEADRepairFails(t *testing.T) {
@@ -433,6 +441,89 @@ func TestSessionRecoveryUsesIncomingCommandContext(t *testing.T) {
433441 require.True(t, stub.seen)
434442 }
435443
444+type temporaryRestoreError struct{}
445+
446+func (temporaryRestoreError) Error() string { return "temporary restore failure" }
447+func (temporaryRestoreError) Temporary() bool { return true }
448+
449+func TestRetryableRestoreLoadErrorClassification(t *testing.T) {
450+ tests := []struct {
451+ name string
452+ err error
453+ retryable bool
454+ }{
455+ {name: "budget exhausted", err: fmt.Errorf("wrapped: %w", ports.ErrBudgetExhausted), retryable: true},
456+ {name: "missing path", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.ENOENT})},
457+ {name: "permission denied", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EACCES})},
458+ {name: "invalid path operation", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EINVAL})},
459+ {name: "temporary path failure", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EAGAIN}), retryable: true},
460+ {name: "resource exhausted path", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.ENOSPC}), retryable: true},
461+ {name: "temporary interface", err: fmt.Errorf("wrapped: %w", temporaryRestoreError{}), retryable: true},
462+ {name: "corrupt data", err: errors.New("corrupt checkpoint")},
463+ {name: "context canceled", err: context.Canceled},
464+ {name: "context deadline exceeded", err: context.DeadlineExceeded},
465+ }
466+ for _, tt := range tests {
467+ t.Run(tt.name, func(t *testing.T) {
468+ require.Equal(t, tt.retryable, isRetryableRestoreLoadError(tt.err))
469+ })
470+ }
471+}
472+
473+func TestExplicitRecoveryRemainsRetryableAfterRepeatedTransientFailures(t *testing.T) {
474+ record := durableRecoveryRecord(0)
475+ repository := &durableRecoveryRepository{
476+ generations: map[string]ports.SnapshotGeneration{record.Name: validGeneration(t, record)},
477+ errors: map[string]error{record.Name: ports.ErrBudgetExhausted},
478+ loads: make(map[string]int),
479+ repairs: make(map[string]domain.CheckpointRef),
480+ }
481+ d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository)
482+ d.setStoppedRecovery(record, runtimeDegraded)
483+ exec := controlExec{d: d, recoveryName: record.Name}
484+
485+ var attemptBarriers []chan struct{}
486+ for range 2 {
487+ err := exec.restoreExplicitRecovery(t.Context())
488+ require.ErrorIs(t, err, errRetryableRestoreLoad)
489+
490+ d.mu.Lock()
491+ entry := d.stopped[record.Name]
492+ d.mu.Unlock()
493+ require.Equal(t, runtimeDegraded, entry.state)
494+ require.Equal(t, domain.RecoveryHealthy, entry.record.RecoveryState)
495+ select {
496+ case <-entry.restoreDone:
497+ default:
498+ t.Fatal("failed attempt completion barrier remained open")
499+ }
500+ for _, prior := range attemptBarriers {
501+ require.NotEqual(t, prior, entry.restoreDone, "a closed completion barrier must not be reused")
502+ }
503+ attemptBarriers = append(attemptBarriers, entry.restoreDone)
504+
505+ persisted, ok, catalogueErr := catalogue.Record(record.Name)
506+ require.NoError(t, catalogueErr)
507+ require.True(t, ok)
508+ require.Equal(t, record, persisted, "transient restore failure must preserve healthy durable authority")
509+ }
510+
511+ repository.mu.Lock()
512+ delete(repository.errors, record.Name)
513+ repository.mu.Unlock()
514+ require.NoError(t, exec.restoreExplicitRecovery(t.Context()))
515+ d.mu.Lock()
516+ require.Equal(t, runtimeHealthy, d.stopped[record.Name].state)
517+ d.mu.Unlock()
518+ persisted, ok, err := catalogue.Record(record.Name)
519+ require.NoError(t, err)
520+ require.True(t, ok)
521+ require.Equal(t, domain.RecoveryHealthy, persisted.RecoveryState)
522+ repository.mu.Lock()
523+ require.Equal(t, 3, repository.loads[record.Name])
524+ repository.mu.Unlock()
525+}
526+
436527 func TestExplicitRecoveryRejectsConcurrentRestore(t *testing.T) {
437528 record := durableRecoveryRecord(0)
438529 record.RecoveryState = domain.RecoveryDegraded
@@ -7,8 +7,10 @@ import (
7 "fmt"7 "fmt"
8 "io"8 "io"
9 "log/slog"9 "log/slog"
10+ "os"
10 "sync"11 "sync"
11 "sync/atomic"12 "sync/atomic"
13+ "syscall"
12 "testing"14 "testing"
13 "time"15 "time"
14 16
@@ -226,67 +228,73 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) {
226 }228 }
227 229
228 func TestCatalogueRestoreIndependent(t *testing.T) {230 func TestCatalogueRestoreIndependent(t *testing.T) {
229- emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{})231+ t.Run("empty catalogue restoration", func(t *testing.T) {
230- emptyDone := make(chan struct{})232+ emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{})
231- go func() {233+ emptyDone := make(chan struct{})
232- emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue))234+ go func() {
233- close(emptyDone)235+ emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue))
234- }()236+ close(emptyDone)
235- select {237+ }()
236- case <-emptyDone:238+ select {
237- case <-time.After(time.Second):239+ case <-emptyDone:
238- t.Fatal("empty catalogue restoration blocked")240+ case <-time.After(time.Second):
239- }241+ t.Fatal("empty catalogue restoration blocked")
240-242+ }
241- healthy := durableRecoveryRecord(0)243+ })
242- broken := durableRecoveryRecord(1)
243- repository := &durableRecoveryRepository{
244- generations: map[string]ports.SnapshotGeneration{healthy.Name: validGeneration(t, healthy)},
245- errors: map[string]error{broken.Name: ports.ErrBudgetExhausted},
246- loads: make(map[string]int),
247- repairs: make(map[string]domain.CheckpointRef),
248- }
249- d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository)
250 244
251- d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))245+ t.Run("healthy and broken restoration", func(t *testing.T) {
246+ healthy := durableRecoveryRecord(0)
247+ broken := durableRecoveryRecord(1)
248+ repository := &durableRecoveryRepository{
249+ generations: map[string]ports.SnapshotGeneration{healthy.Name: validGeneration(t, healthy)},
250+ errors: map[string]error{broken.Name: errors.New("corrupt checkpoint")},
251+ loads: make(map[string]int),
252+ repairs: make(map[string]domain.CheckpointRef),
253+ }
254+ d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository)
255+
256+ d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))
257+
258+ require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state)
259+ require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state)
260+ for _, name := range []string{healthy.Name, broken.Name} {
261+ select {
262+ case <-d.stopped[name].restoreDone:
263+ default:
264+ t.Fatalf("restore completion for %q was not closed", name)
265+ }
266+ }
267+ degraded, ok, _ := catalogue.Record(broken.Name)
268+ require.True(t, ok)
269+ require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState)
270+ require.Equal(t, 1, repository.loads[healthy.Name])
271+ require.Equal(t, 1, repository.loads[broken.Name])
272+ require.Empty(t, d.sessions, "a corrupt record must never open a replacement runtime")
273+ })
252 274
253- require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state)275+ t.Run("fallback promotion", func(t *testing.T) {
254- require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state)276+ fallback := durableRecoveryRecord(2)
255- for _, name := range []string{healthy.Name, broken.Name} {277+ fallbackRef := domain.CheckpointRef{Generation: fallback.Committed.Generation - 1}
256- select {278+ fallbackSnapshotRecord := fallback
257- case <-d.stopped[name].restoreDone:279+ fallbackSnapshotRecord.Committed = &fallbackRef
258- default:280+ fallbackGeneration := validGeneration(t, fallbackSnapshotRecord)
259- t.Fatalf("restore completion for %q was not closed", name)281+ fallbackRef.ManifestDigest = sha256.Sum256(fallbackGeneration.Manifest)
282+ fallback.Fallbacks[0] = &fallbackRef
283+ fallbackRepository := &durableRecoveryRepository{
284+ generations: map[string]ports.SnapshotGeneration{fallback.Name: fallbackGeneration},
285+ errors: make(map[string]error),
286+ loads: make(map[string]int),
287+ repairs: make(map[string]domain.CheckpointRef),
260 }288 }
261- }289+ fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository)
262- degraded, ok, _ := catalogue.Record(broken.Name)290+ fallbackDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, fallbackCatalogue))
263- require.True(t, ok)291+ promoted, ok, _ := fallbackCatalogue.Record(fallback.Name)
264- require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState)292+ require.True(t, ok)
265- require.Equal(t, 1, repository.loads[healthy.Name])293+ require.Equal(t, fallbackRef, *promoted.Committed)
266- require.Equal(t, 1, repository.loads[broken.Name])294+ require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state)
267- require.Empty(t, d.sessions, "a corrupt record must never open a replacement runtime")295+ require.Equal(t, 3, fallbackRepository.loads[fallback.Name])
268-296+ require.Equal(t, fallbackRef, fallbackRepository.repairs[fallback.IncarnationID.String()])
269- fallback := durableRecoveryRecord(2)297+ })
270- fallbackRef := domain.CheckpointRef{Generation: fallback.Committed.Generation - 1}
271- fallbackSnapshotRecord := fallback
272- fallbackSnapshotRecord.Committed = &fallbackRef
273- fallbackGeneration := validGeneration(t, fallbackSnapshotRecord)
274- fallbackRef.ManifestDigest = sha256.Sum256(fallbackGeneration.Manifest)
275- fallback.Fallbacks[0] = &fallbackRef
276- fallbackRepository := &durableRecoveryRepository{
277- generations: map[string]ports.SnapshotGeneration{fallback.Name: fallbackGeneration},
278- errors: make(map[string]error),
279- loads: make(map[string]int),
280- repairs: make(map[string]domain.CheckpointRef),
281- }
282- fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository)
283- fallbackDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, fallbackCatalogue))
284- promoted, ok, _ := fallbackCatalogue.Record(fallback.Name)
285- require.True(t, ok)
286- require.Equal(t, fallbackRef, *promoted.Committed)
287- require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state)
288- require.Equal(t, 3, fallbackRepository.loads[fallback.Name])
289- require.Equal(t, fallbackRef, fallbackRepository.repairs[fallback.IncarnationID.String()])
290 }298 }
291 299
292 func TestCatalogueRestoreKeepsCommittedFallbackHealthyWhenHEADRepairFails(t *testing.T) {300 func TestCatalogueRestoreKeepsCommittedFallbackHealthyWhenHEADRepairFails(t *testing.T) {
@@ -433,6 +441,89 @@ func TestSessionRecoveryUsesIncomingCommandContext(t *testing.T) {
433 require.True(t, stub.seen)441 require.True(t, stub.seen)
434 }442 }
435 443
444+type temporaryRestoreError struct{}
445+
446+func (temporaryRestoreError) Error() string { return "temporary restore failure" }
447+func (temporaryRestoreError) Temporary() bool { return true }
448+
449+func TestRetryableRestoreLoadErrorClassification(t *testing.T) {
450+ tests := []struct {
451+ name string
452+ err error
453+ retryable bool
454+ }{
455+ {name: "budget exhausted", err: fmt.Errorf("wrapped: %w", ports.ErrBudgetExhausted), retryable: true},
456+ {name: "missing path", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.ENOENT})},
457+ {name: "permission denied", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EACCES})},
458+ {name: "invalid path operation", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EINVAL})},
459+ {name: "temporary path failure", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EAGAIN}), retryable: true},
460+ {name: "resource exhausted path", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.ENOSPC}), retryable: true},
461+ {name: "temporary interface", err: fmt.Errorf("wrapped: %w", temporaryRestoreError{}), retryable: true},
462+ {name: "corrupt data", err: errors.New("corrupt checkpoint")},
463+ {name: "context canceled", err: context.Canceled},
464+ {name: "context deadline exceeded", err: context.DeadlineExceeded},
465+ }
466+ for _, tt := range tests {
467+ t.Run(tt.name, func(t *testing.T) {
468+ require.Equal(t, tt.retryable, isRetryableRestoreLoadError(tt.err))
469+ })
470+ }
471+}
472+
473+func TestExplicitRecoveryRemainsRetryableAfterRepeatedTransientFailures(t *testing.T) {
474+ record := durableRecoveryRecord(0)
475+ repository := &durableRecoveryRepository{
476+ generations: map[string]ports.SnapshotGeneration{record.Name: validGeneration(t, record)},
477+ errors: map[string]error{record.Name: ports.ErrBudgetExhausted},
478+ loads: make(map[string]int),
479+ repairs: make(map[string]domain.CheckpointRef),
480+ }
481+ d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository)
482+ d.setStoppedRecovery(record, runtimeDegraded)
483+ exec := controlExec{d: d, recoveryName: record.Name}
484+
485+ var attemptBarriers []chan struct{}
486+ for range 2 {
487+ err := exec.restoreExplicitRecovery(t.Context())
488+ require.ErrorIs(t, err, errRetryableRestoreLoad)
489+
490+ d.mu.Lock()
491+ entry := d.stopped[record.Name]
492+ d.mu.Unlock()
493+ require.Equal(t, runtimeDegraded, entry.state)
494+ require.Equal(t, domain.RecoveryHealthy, entry.record.RecoveryState)
495+ select {
496+ case <-entry.restoreDone:
497+ default:
498+ t.Fatal("failed attempt completion barrier remained open")
499+ }
500+ for _, prior := range attemptBarriers {
501+ require.NotEqual(t, prior, entry.restoreDone, "a closed completion barrier must not be reused")
502+ }
503+ attemptBarriers = append(attemptBarriers, entry.restoreDone)
504+
505+ persisted, ok, catalogueErr := catalogue.Record(record.Name)
506+ require.NoError(t, catalogueErr)
507+ require.True(t, ok)
508+ require.Equal(t, record, persisted, "transient restore failure must preserve healthy durable authority")
509+ }
510+
511+ repository.mu.Lock()
512+ delete(repository.errors, record.Name)
513+ repository.mu.Unlock()
514+ require.NoError(t, exec.restoreExplicitRecovery(t.Context()))
515+ d.mu.Lock()
516+ require.Equal(t, runtimeHealthy, d.stopped[record.Name].state)
517+ d.mu.Unlock()
518+ persisted, ok, err := catalogue.Record(record.Name)
519+ require.NoError(t, err)
520+ require.True(t, ok)
521+ require.Equal(t, domain.RecoveryHealthy, persisted.RecoveryState)
522+ repository.mu.Lock()
523+ require.Equal(t, 3, repository.loads[record.Name])
524+ repository.mu.Unlock()
525+}
526+
436 func TestExplicitRecoveryRejectsConcurrentRestore(t *testing.T) {527 func TestExplicitRecoveryRejectsConcurrentRestore(t *testing.T) {
437 record := durableRecoveryRecord(0)528 record := durableRecoveryRecord(0)
438 record.RecoveryState = domain.RecoveryDegraded529 record.RecoveryState = domain.RecoveryDegraded
M internal/usecase/daemon/maintenance.go +13 -14
@@ -119,7 +119,8 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
119119 // Reconcile before collection. Until a complete top-level scan reaches EOF,
120120 // an unseen forward orphan may still be repairable, so retention yields
121121 // without deleting any incarnation data.
122- next, decisions, err := deps.reconciler.Step(ctx, deps.cursor)
122+ previousCursor := deps.cursor
123+ next, decisions, err := deps.reconciler.Step(ctx, previousCursor)
123124 if err != nil {
124125 d.log.Warn("durable_reconciliation_tick", "action", "scan", "cursor", deps.cursor.DirectoryCookie, "done", false, "err", err)
125126 return false
@@ -151,7 +152,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
151152 }
152153 d.log.Info("durable_reconciliation_tick", "action", "scan", "cursor", next.DirectoryCookie, "done", next.DirectoryCookie == 0, "err", nil)
153154 if next.DirectoryCookie != 0 {
154- return true
155+ return next.DirectoryCookie != previousCursor.DirectoryCookie
155156 }
156157
157158 records, err := deps.catalogue.Records()
@@ -159,6 +160,15 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
159160 d.log.Warn("durable_maintenance_tick", "action", "catalogue-read", "done", false, "err", err)
160161 return false
161162 }
163+ d.mu.Lock()
164+ restoringIncarnations := make(map[domain.IncarnationID]struct{})
165+ for _, stopped := range d.stopped {
166+ if stopped.state == runtimeRestoring {
167+ restoringIncarnations[stopped.incarnation] = struct{}{}
168+ }
169+ }
170+ d.mu.Unlock()
171+
162172 incomplete := false
163173 for _, record := range records {
164174 if err := ctx.Err(); err != nil {
@@ -166,7 +176,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
166176 }
167177 _, unresolved := deps.unresolved[record.IncarnationID]
168178 _, uncertainRepair := deps.repairUncertain[record.IncarnationID]
169- restoring := d.incarnationRestoring(record.IncarnationID)
179+ _, restoring := restoringIncarnations[record.IncarnationID]
170180 plan := retentionPlan(record, unresolved || uncertainRepair, restoring)
171181 done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes})
172182 if err == nil && !done {
@@ -185,14 +195,3 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
185195 clear(deps.repairUncertain)
186196 return incomplete
187197 }
188-
189-func (d *Daemon) incarnationRestoring(id domain.IncarnationID) bool {
190- d.mu.Lock()
191- defer d.mu.Unlock()
192- for _, stopped := range d.stopped {
193- if stopped.incarnation == id && stopped.state == runtimeRestoring {
194- return true
195- }
196- }
197- return false
198-}
@@ -119,7 +119,8 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
119 // Reconcile before collection. Until a complete top-level scan reaches EOF,119 // Reconcile before collection. Until a complete top-level scan reaches EOF,
120 // an unseen forward orphan may still be repairable, so retention yields120 // an unseen forward orphan may still be repairable, so retention yields
121 // without deleting any incarnation data.121 // without deleting any incarnation data.
122- next, decisions, err := deps.reconciler.Step(ctx, deps.cursor)122+ previousCursor := deps.cursor
123+ next, decisions, err := deps.reconciler.Step(ctx, previousCursor)
123 if err != nil {124 if err != nil {
124 d.log.Warn("durable_reconciliation_tick", "action", "scan", "cursor", deps.cursor.DirectoryCookie, "done", false, "err", err)125 d.log.Warn("durable_reconciliation_tick", "action", "scan", "cursor", deps.cursor.DirectoryCookie, "done", false, "err", err)
125 return false126 return false
@@ -151,7 +152,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
151 }152 }
152 d.log.Info("durable_reconciliation_tick", "action", "scan", "cursor", next.DirectoryCookie, "done", next.DirectoryCookie == 0, "err", nil)153 d.log.Info("durable_reconciliation_tick", "action", "scan", "cursor", next.DirectoryCookie, "done", next.DirectoryCookie == 0, "err", nil)
153 if next.DirectoryCookie != 0 {154 if next.DirectoryCookie != 0 {
154- return true155+ return next.DirectoryCookie != previousCursor.DirectoryCookie
155 }156 }
156 157
157 records, err := deps.catalogue.Records()158 records, err := deps.catalogue.Records()
@@ -159,6 +160,15 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
159 d.log.Warn("durable_maintenance_tick", "action", "catalogue-read", "done", false, "err", err)160 d.log.Warn("durable_maintenance_tick", "action", "catalogue-read", "done", false, "err", err)
160 return false161 return false
161 }162 }
163+ d.mu.Lock()
164+ restoringIncarnations := make(map[domain.IncarnationID]struct{})
165+ for _, stopped := range d.stopped {
166+ if stopped.state == runtimeRestoring {
167+ restoringIncarnations[stopped.incarnation] = struct{}{}
168+ }
169+ }
170+ d.mu.Unlock()
171+
162 incomplete := false172 incomplete := false
163 for _, record := range records {173 for _, record := range records {
164 if err := ctx.Err(); err != nil {174 if err := ctx.Err(); err != nil {
@@ -166,7 +176,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
166 }176 }
167 _, unresolved := deps.unresolved[record.IncarnationID]177 _, unresolved := deps.unresolved[record.IncarnationID]
168 _, uncertainRepair := deps.repairUncertain[record.IncarnationID]178 _, uncertainRepair := deps.repairUncertain[record.IncarnationID]
169- restoring := d.incarnationRestoring(record.IncarnationID)179+ _, restoring := restoringIncarnations[record.IncarnationID]
170 plan := retentionPlan(record, unresolved || uncertainRepair, restoring)180 plan := retentionPlan(record, unresolved || uncertainRepair, restoring)
171 done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes})181 done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes})
172 if err == nil && !done {182 if err == nil && !done {
@@ -185,14 +195,3 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
185 clear(deps.repairUncertain)195 clear(deps.repairUncertain)
186 return incomplete196 return incomplete
187 }197 }
188-
189-func (d *Daemon) incarnationRestoring(id domain.IncarnationID) bool {
190- d.mu.Lock()
191- defer d.mu.Unlock()
192- for _, stopped := range d.stopped {
193- if stopped.incarnation == id && stopped.state == runtimeRestoring {
194- return true
195- }
196- }
197- return false
198-}
M internal/usecase/daemon/maintenance_test.go +18 -2
@@ -25,6 +25,22 @@ func (r *cursorMaintenanceReconciler) Step(_ context.Context, cursor ports.Recon
2525 return ports.ReconcileCursor{}, nil, nil
2626 }
2727
28+type stalledMaintenanceReconciler struct{}
29+
30+func (stalledMaintenanceReconciler) Step(_ context.Context, cursor ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error) {
31+ if cursor.DirectoryCookie == 0 {
32+ return ports.ReconcileCursor{DirectoryCookie: 7}, nil, nil
33+ }
34+ return cursor, nil, nil
35+}
36+
37+func TestMaintenanceDoesNotImmediatelyRetryStalledCursor(t *testing.T) {
38+ d := newTestDaemon(t, nil, stubClock{})
39+ d.maintenance = newMaintenanceDependencies(portsmocks.NewMockCatalogue(t), portsmocks.NewMockSnapshotRepository(t), stalledMaintenanceReconciler{}, nil)
40+ require.True(t, d.runDurableMaintenanceTick(context.Background()))
41+ require.False(t, d.runDurableMaintenanceTick(context.Background()))
42+}
43+
2844 func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) {
2945 record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}
3046 catalogue := portsmocks.NewMockCatalogue(t)
@@ -92,11 +108,11 @@ func TestMaintenanceRetentionUsesReconciliationOutcome(t *testing.T) {
92108 wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}},
93109 },
94110 {
95- name: "non-forward candidate kind does not pin",
111+ name: "non-forward candidate kind remains pinned",
96112 finding: ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
97113 Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}},
98114 }},
99- wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}},
115+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
100116 },
101117 {
102118 name: "unreadable candidate remains pinned",
@@ -25,6 +25,22 @@ func (r *cursorMaintenanceReconciler) Step(_ context.Context, cursor ports.Recon
25 return ports.ReconcileCursor{}, nil, nil25 return ports.ReconcileCursor{}, nil, nil
26 }26 }
27 27
28+type stalledMaintenanceReconciler struct{}
29+
30+func (stalledMaintenanceReconciler) Step(_ context.Context, cursor ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error) {
31+ if cursor.DirectoryCookie == 0 {
32+ return ports.ReconcileCursor{DirectoryCookie: 7}, nil, nil
33+ }
34+ return cursor, nil, nil
35+}
36+
37+func TestMaintenanceDoesNotImmediatelyRetryStalledCursor(t *testing.T) {
38+ d := newTestDaemon(t, nil, stubClock{})
39+ d.maintenance = newMaintenanceDependencies(portsmocks.NewMockCatalogue(t), portsmocks.NewMockSnapshotRepository(t), stalledMaintenanceReconciler{}, nil)
40+ require.True(t, d.runDurableMaintenanceTick(context.Background()))
41+ require.False(t, d.runDurableMaintenanceTick(context.Background()))
42+}
43+
28 func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) {44 func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) {
29 record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}45 record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}
30 catalogue := portsmocks.NewMockCatalogue(t)46 catalogue := portsmocks.NewMockCatalogue(t)
@@ -92,11 +108,11 @@ func TestMaintenanceRetentionUsesReconciliationOutcome(t *testing.T) {
92 wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}},108 wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}},
93 },109 },
94 {110 {
95- name: "non-forward candidate kind does not pin",111+ name: "non-forward candidate kind remains pinned",
96 finding: ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{112 finding: ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
97 Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}},113 Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}},
98 }},114 }},
99- wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}},115+ wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
100 },116 },
101 {117 {
102 name: "unreadable candidate remains pinned",118 name: "unreadable candidate remains pinned",
M internal/usecase/daemon/metadata_order_test.go +6 -2
@@ -205,8 +205,10 @@ func TestNewerMetadataFailureDoesNotSkipOlderSnapshot(t *testing.T) {
205205 return true
206206 }
207207
208- require.ErrorIs(t, d.persistSessionMetadata(sess, secondVersion, secondRecord.MetadataUpdate(), secondRollback), failure)
209- require.NoError(t, d.persistSessionMetadata(sess, firstVersion, firstRecord.MetadataUpdate(), firstRollback))
208+ _, err := d.persistSessionMetadata(sess, secondVersion, secondRecord.MetadataUpdate(), secondRollback)
209+ require.ErrorIs(t, err, failure)
210+ _, err = d.persistSessionMetadata(sess, firstVersion, firstRecord.MetadataUpdate(), firstRollback)
211+ require.NoError(t, err)
210212
211213 persisted, ok, err := catalogue.Record("work")
212214 require.NoError(t, err)
@@ -239,7 +241,9 @@ func TestLatestMetadataFailureRollsBackRename(t *testing.T) {
239241 require.NoError(t, err)
240242 require.True(t, ok)
241243 require.Empty(t, persisted.TabNames)
244+ sess.mu.Lock()
242245 require.Empty(t, sess.tabs[0].name)
246+ sess.mu.Unlock()
243247 }
244248
245249 func TestBlockedCwdWriteCannotOverwriteLaterTabMetadata(t *testing.T) {
@@ -205,8 +205,10 @@ func TestNewerMetadataFailureDoesNotSkipOlderSnapshot(t *testing.T) {
205 return true205 return true
206 }206 }
207 207
208- require.ErrorIs(t, d.persistSessionMetadata(sess, secondVersion, secondRecord.MetadataUpdate(), secondRollback), failure)208+ _, err := d.persistSessionMetadata(sess, secondVersion, secondRecord.MetadataUpdate(), secondRollback)
209- require.NoError(t, d.persistSessionMetadata(sess, firstVersion, firstRecord.MetadataUpdate(), firstRollback))209+ require.ErrorIs(t, err, failure)
210+ _, err = d.persistSessionMetadata(sess, firstVersion, firstRecord.MetadataUpdate(), firstRollback)
211+ require.NoError(t, err)
210 212
211 persisted, ok, err := catalogue.Record("work")213 persisted, ok, err := catalogue.Record("work")
212 require.NoError(t, err)214 require.NoError(t, err)
@@ -239,7 +241,9 @@ func TestLatestMetadataFailureRollsBackRename(t *testing.T) {
239 require.NoError(t, err)241 require.NoError(t, err)
240 require.True(t, ok)242 require.True(t, ok)
241 require.Empty(t, persisted.TabNames)243 require.Empty(t, persisted.TabNames)
244+ sess.mu.Lock()
242 require.Empty(t, sess.tabs[0].name)245 require.Empty(t, sess.tabs[0].name)
246+ sess.mu.Unlock()
243 }247 }
244 248
245 func TestBlockedCwdWriteCannotOverwriteLaterTabMetadata(t *testing.T) {249 func TestBlockedCwdWriteCannotOverwriteLaterTabMetadata(t *testing.T) {
M internal/usecase/daemon/picker_test.go +4 -2
@@ -293,7 +293,9 @@ func TestRestoreCancellationTransitionsBeforePickerCompletion(t *testing.T) {
293293 require.Error(t, <-result)
294294 <-restored
295295 require.Same(t, from, ac.currentSession())
296+ d.mu.Lock()
296297 entry := d.stopped[record.Name]
298+ d.mu.Unlock()
297299 require.Equal(t, runtimeDegraded, entry.state)
298300 require.Equal(t, "restore interrupted", entry.record.DegradedReason)
299301 select {
@@ -1133,7 +1135,7 @@ func TestPickerKillActiveSessionSnapshotDeleteRefusalReportsOnceAndKeepsPicker(t
11331135 WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause})(d)
11341136 target := d.sessions[domain.SessionID("recent")]
11351137 store, _ := newMockStore(t)
1136- WithStore(store)(d)
1138+ WithStore(t, store)(d)
11371139 target.mu.Lock()
11381140 if target.incarnation == (domain.IncarnationID{}) {
11391141 target.incarnation = domain.IncarnationID{1}
@@ -1175,7 +1177,7 @@ func TestPickerKillStoppedSessionPersistDeleteFailureSurfacesNoticeAndKeepsEntry
11751177 d, from, ac, sends := newManualSessionWithPTYs(t, p)
11761178 cause := errors.New("delete failed")
11771179 store, state := newMockStore(t)
1178- WithStore(store)(d)
1180+ WithStore(t, store)(d)
11791181 record := domain.CatalogueRecord{Name: "stopped", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/stopped", CreatedAt: 7, RecoveryState: domain.RecoveryFresh}
11801182 require.NoError(t, d.catalogue.Create(record))
11811183 state.mu.Lock()
@@ -293,7 +293,9 @@ func TestRestoreCancellationTransitionsBeforePickerCompletion(t *testing.T) {
293 require.Error(t, <-result)293 require.Error(t, <-result)
294 <-restored294 <-restored
295 require.Same(t, from, ac.currentSession())295 require.Same(t, from, ac.currentSession())
296+ d.mu.Lock()
296 entry := d.stopped[record.Name]297 entry := d.stopped[record.Name]
298+ d.mu.Unlock()
297 require.Equal(t, runtimeDegraded, entry.state)299 require.Equal(t, runtimeDegraded, entry.state)
298 require.Equal(t, "restore interrupted", entry.record.DegradedReason)300 require.Equal(t, "restore interrupted", entry.record.DegradedReason)
299 select {301 select {
@@ -1133,7 +1135,7 @@ func TestPickerKillActiveSessionSnapshotDeleteRefusalReportsOnceAndKeepsPicker(t
1133 WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause})(d)1135 WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause})(d)
1134 target := d.sessions[domain.SessionID("recent")]1136 target := d.sessions[domain.SessionID("recent")]
1135 store, _ := newMockStore(t)1137 store, _ := newMockStore(t)
1136- WithStore(store)(d)1138+ WithStore(t, store)(d)
1137 target.mu.Lock()1139 target.mu.Lock()
1138 if target.incarnation == (domain.IncarnationID{}) {1140 if target.incarnation == (domain.IncarnationID{}) {
1139 target.incarnation = domain.IncarnationID{1}1141 target.incarnation = domain.IncarnationID{1}
@@ -1175,7 +1177,7 @@ func TestPickerKillStoppedSessionPersistDeleteFailureSurfacesNoticeAndKeepsEntry
1175 d, from, ac, sends := newManualSessionWithPTYs(t, p)1177 d, from, ac, sends := newManualSessionWithPTYs(t, p)
1176 cause := errors.New("delete failed")1178 cause := errors.New("delete failed")
1177 store, state := newMockStore(t)1179 store, state := newMockStore(t)
1178- WithStore(store)(d)1180+ WithStore(t, store)(d)
1179 record := domain.CatalogueRecord{Name: "stopped", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/stopped", CreatedAt: 7, RecoveryState: domain.RecoveryFresh}1181 record := domain.CatalogueRecord{Name: "stopped", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/stopped", CreatedAt: 7, RecoveryState: domain.RecoveryFresh}
1180 require.NoError(t, d.catalogue.Create(record))1182 require.NoError(t, d.catalogue.Create(record))
1181 state.mu.Lock()1183 state.mu.Lock()
M internal/usecase/daemon/recovery_observability.go +1 -1
@@ -21,7 +21,7 @@ func (d *Daemon) logSessionDegraded(record domain.CatalogueRecord, reasonCode st
2121
2222 func (d *Daemon) logStartupRecoveryCounts(restoring int) {
2323 var records []domain.CatalogueRecord
24- if d.catalogue != nil {
24+ if d.persistEnabled && d.catalogue != nil {
2525 var err error
2626 records, err = d.catalogue.Records()
2727 if err != nil {
@@ -21,7 +21,7 @@ func (d *Daemon) logSessionDegraded(record domain.CatalogueRecord, reasonCode st
21 21
22 func (d *Daemon) logStartupRecoveryCounts(restoring int) {22 func (d *Daemon) logStartupRecoveryCounts(restoring int) {
23 var records []domain.CatalogueRecord23 var records []domain.CatalogueRecord
24- if d.catalogue != nil {24+ if d.persistEnabled && d.catalogue != nil {
25 var err error25 var err error
26 records, err = d.catalogue.Records()26 records, err = d.catalogue.Records()
27 if err != nil {27 if err != nil {
M internal/usecase/daemon/session.go +29 -12
@@ -203,20 +203,13 @@ func (d *Daemon) touchMRU(sess *session) {
203203 rollback := func() bool {
204204 return sess.mruAt.CompareAndSwap(seq, previous)
205205 }
206- if err := d.persistSessionMetadata(sess, version, record.MetadataUpdate(), rollback); err != nil {
206+ if _, err := d.persistSessionMetadata(sess, version, record.MetadataUpdate(), rollback); err != nil {
207207 d.log.Warn("touching persisted session recency failed", "err", err, "session", name)
208208 }
209209 }
210210 }
211211
212212 func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) {
213- // Historical focused tests call this helper directly. Acquire mu for those
214- // callers while preserving the production contract that an owning caller
215- // receives it still locked on return.
216- acquiredMu := d.mu.TryLock()
217- if acquiredMu {
218- defer d.mu.Unlock()
219- }
220213 env = copyEnvironment(env)
221214 if _, reserved := d.creating[name]; reserved {
222215 return nil, errSessionNameInUse
@@ -533,7 +526,22 @@ func (d *Daemon) createTab(sess *session, sz domain.Size) error {
533526 _ = pty.Close()
534527 return true
535528 }
536- if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {
529+ rollbackRejected, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback)
530+ if err != nil {
531+ if rollbackRejected {
532+ d.mu.Lock()
533+ if d.sessions[sess.id] == sess {
534+ sess.mu.Lock()
535+ sess.tabs = slices.DeleteFunc(sess.tabs, func(candidate *tab) bool { return candidate == tb })
536+ sess.active = min(sess.active, len(sess.tabs)-1)
537+ sess.mu.Unlock()
538+ }
539+ d.mu.Unlock()
540+ if tb.cancel != nil {
541+ tb.cancel()
542+ }
543+ tb.closeAllPanes()
544+ }
537545 return err
538546 }
539547 }
@@ -745,6 +753,7 @@ func (d *Daemon) renameSession(sess *session, name string) error {
745753 oldName := sess.name
746754 wasEphemeral := sess.ephemeral
747755 createdAt := sess.createdAt
756+ priorIncarnation := sess.incarnation
748757 if wasEphemeral {
749758 var err error
750759 createdAt, err = d.allocateLifecycleCreatedAtLocked()
@@ -772,6 +781,7 @@ func (d *Daemon) renameSession(sess *session, name string) error {
772781
773782 rollback := func(err error) error {
774783 sess.mu.Lock()
784+ sess.incarnation = priorIncarnation
775785 sess.renameInProgress = false
776786 if sess.renameDone != nil {
777787 close(sess.renameDone)
@@ -869,7 +879,7 @@ func (d *Daemon) renameTab(sess *session, tb *tab, name string) error {
869879 tb.name = oldName
870880 return true
871881 }
872- if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {
882+ if _, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {
873883 return err
874884 }
875885 return nil
@@ -969,8 +979,15 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error {
969979 sess.active = min(oldActive, len(sess.tabs)-1)
970980 return true
971981 }
972- if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {
982+ rollbackRejected, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback)
983+ if err != nil {
973984 d.log.Warn("persisting closed tab failed", "err", err, "session", name)
985+ if rollbackRejected {
986+ if tb.cancel != nil {
987+ tb.cancel()
988+ }
989+ tb.closeAllPanes()
990+ }
974991 return err
975992 }
976993 }
@@ -1510,7 +1527,7 @@ func (d *Daemon) refreshSessionCwd(sess *session) {
15101527 sess.cwd = oldCwd
15111528 return true
15121529 }
1513- if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {
1530+ if _, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {
15141531 d.log.Warn("touching persisted session cwd failed", "err", err, "session", name)
15151532 return
15161533 }
@@ -203,20 +203,13 @@ func (d *Daemon) touchMRU(sess *session) {
203 rollback := func() bool {203 rollback := func() bool {
204 return sess.mruAt.CompareAndSwap(seq, previous)204 return sess.mruAt.CompareAndSwap(seq, previous)
205 }205 }
206- if err := d.persistSessionMetadata(sess, version, record.MetadataUpdate(), rollback); err != nil {206+ if _, err := d.persistSessionMetadata(sess, version, record.MetadataUpdate(), rollback); err != nil {
207 d.log.Warn("touching persisted session recency failed", "err", err, "session", name)207 d.log.Warn("touching persisted session recency failed", "err", err, "session", name)
208 }208 }
209 }209 }
210 }210 }
211 211
212 func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) {212 func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) {
213- // Historical focused tests call this helper directly. Acquire mu for those
214- // callers while preserving the production contract that an owning caller
215- // receives it still locked on return.
216- acquiredMu := d.mu.TryLock()
217- if acquiredMu {
218- defer d.mu.Unlock()
219- }
220 env = copyEnvironment(env)213 env = copyEnvironment(env)
221 if _, reserved := d.creating[name]; reserved {214 if _, reserved := d.creating[name]; reserved {
222 return nil, errSessionNameInUse215 return nil, errSessionNameInUse
@@ -533,7 +526,22 @@ func (d *Daemon) createTab(sess *session, sz domain.Size) error {
533 _ = pty.Close()526 _ = pty.Close()
534 return true527 return true
535 }528 }
536- if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {529+ rollbackRejected, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback)
530+ if err != nil {
531+ if rollbackRejected {
532+ d.mu.Lock()
533+ if d.sessions[sess.id] == sess {
534+ sess.mu.Lock()
535+ sess.tabs = slices.DeleteFunc(sess.tabs, func(candidate *tab) bool { return candidate == tb })
536+ sess.active = min(sess.active, len(sess.tabs)-1)
537+ sess.mu.Unlock()
538+ }
539+ d.mu.Unlock()
540+ if tb.cancel != nil {
541+ tb.cancel()
542+ }
543+ tb.closeAllPanes()
544+ }
537 return err545 return err
538 }546 }
539 }547 }
@@ -745,6 +753,7 @@ func (d *Daemon) renameSession(sess *session, name string) error {
745 oldName := sess.name753 oldName := sess.name
746 wasEphemeral := sess.ephemeral754 wasEphemeral := sess.ephemeral
747 createdAt := sess.createdAt755 createdAt := sess.createdAt
756+ priorIncarnation := sess.incarnation
748 if wasEphemeral {757 if wasEphemeral {
749 var err error758 var err error
750 createdAt, err = d.allocateLifecycleCreatedAtLocked()759 createdAt, err = d.allocateLifecycleCreatedAtLocked()
@@ -772,6 +781,7 @@ func (d *Daemon) renameSession(sess *session, name string) error {
772 781
773 rollback := func(err error) error {782 rollback := func(err error) error {
774 sess.mu.Lock()783 sess.mu.Lock()
784+ sess.incarnation = priorIncarnation
775 sess.renameInProgress = false785 sess.renameInProgress = false
776 if sess.renameDone != nil {786 if sess.renameDone != nil {
777 close(sess.renameDone)787 close(sess.renameDone)
@@ -869,7 +879,7 @@ func (d *Daemon) renameTab(sess *session, tb *tab, name string) error {
869 tb.name = oldName879 tb.name = oldName
870 return true880 return true
871 }881 }
872- if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {882+ if _, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {
873 return err883 return err
874 }884 }
875 return nil885 return nil
@@ -969,8 +979,15 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error {
969 sess.active = min(oldActive, len(sess.tabs)-1)979 sess.active = min(oldActive, len(sess.tabs)-1)
970 return true980 return true
971 }981 }
972- if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {982+ rollbackRejected, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback)
983+ if err != nil {
973 d.log.Warn("persisting closed tab failed", "err", err, "session", name)984 d.log.Warn("persisting closed tab failed", "err", err, "session", name)
985+ if rollbackRejected {
986+ if tb.cancel != nil {
987+ tb.cancel()
988+ }
989+ tb.closeAllPanes()
990+ }
974 return err991 return err
975 }992 }
976 }993 }
@@ -1510,7 +1527,7 @@ func (d *Daemon) refreshSessionCwd(sess *session) {
1510 sess.cwd = oldCwd1527 sess.cwd = oldCwd
1511 return true1528 return true
1512 }1529 }
1513- if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {1530+ if _, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {
1514 d.log.Warn("touching persisted session cwd failed", "err", err, "session", name)1531 d.log.Warn("touching persisted session cwd failed", "err", err, "session", name)
1515 return1532 return
1516 }1533 }
M internal/usecase/daemon/session_purge_control_test.go +1 -1
@@ -15,7 +15,7 @@ func TestStoppedPurgeMetadataFailureRemainsFencedForRetry(t *testing.T) {
1515 repository := &retryablePurgeRepository{}
1616 WithSnapshotRepository(repository)(d)
1717 store, state := newMockStore(t)
18- WithStore(store)(d)
18+ WithStore(t, store)(d)
1919 record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}
2020 require.NoError(t, d.catalogue.Create(record))
2121 metadataErr := errors.New("metadata delete failed")
@@ -15,7 +15,7 @@ func TestStoppedPurgeMetadataFailureRemainsFencedForRetry(t *testing.T) {
15 repository := &retryablePurgeRepository{}15 repository := &retryablePurgeRepository{}
16 WithSnapshotRepository(repository)(d)16 WithSnapshotRepository(repository)(d)
17 store, state := newMockStore(t)17 store, state := newMockStore(t)
18- WithStore(store)(d)18+ WithStore(t, store)(d)
19 record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}19 record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}
20 require.NoError(t, d.catalogue.Create(record))20 require.NoError(t, d.catalogue.Create(record))
21 metadataErr := errors.New("metadata delete failed")21 metadataErr := errors.New("metadata delete failed")
M internal/usecase/daemon/session_purge_retry_test.go +10 -4
@@ -37,10 +37,11 @@ func (r *retryablePurgeRepository) WriteDeletionTombstone(ctx context.Context, t
3737 func (r *retryablePurgeRepository) QuarantineDeletionSources(ctx context.Context, tombstone domain.DeletionTombstone, _ bool) error {
3838 _, hasDeadline := ctx.Deadline()
3939 r.deadlineScoped = append(r.deadlineScoped, hasDeadline)
40- r.calls = append(r.calls, "incremental", "legacy")
40+ r.calls = append(r.calls, "incremental")
4141 if r.incrementalErr != nil {
4242 return r.incrementalErr
4343 }
44+ r.calls = append(r.calls, "legacy")
4445 return r.legacyErr
4546 }
4647 func (r *retryablePurgeRepository) DeleteDeletionTombstone(ctx context.Context, _ domain.IncarnationID) error {
@@ -74,7 +75,7 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) {
7475 repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy}
7576 WithSnapshotRepository(repository)(d)
7677 store, _ := newMockStore(t)
77- WithStore(store)(d)
78+ WithStore(t, store)(d)
7879 sess := newSnapshotTestSession(t, "work", false, "/work")
7980 record := sess.persistRecordLocked(1)
8081 record.RecoveryState = domain.RecoveryFresh
@@ -86,7 +87,11 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) {
8687
8788 require.Error(t, d.killSession(sess, ports.ReasonSessionKilled, true))
8889 require.True(t, repository.tombstoned["work"], "partial purge must fence startup restore/import")
89- require.Equal(t, []string{"tombstone", "incremental", "legacy"}, repository.calls)
90+ wantCalls := []string{"tombstone", "incremental"}
91+ if tt.incremental == nil {
92+ wantCalls = append(wantCalls, "legacy")
93+ }
94+ require.Equal(t, wantCalls, repository.calls)
9095 d.mu.Lock()
9196 stopped, ok := d.stopped["work"]
9297 d.mu.Unlock()
@@ -97,7 +102,8 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) {
97102 repository.legacyErr = nil
98103 require.NoError(t, d.retryStoppedPurge("work"))
99104 require.False(t, repository.tombstoned["work"])
100- require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls)
105+ wantCalls = append(wantCalls, "tombstone", "incremental", "legacy", "clear tombstone")
106+ require.Equal(t, wantCalls, repository.calls)
101107 require.NotEmpty(t, repository.deadlineScoped)
102108 for _, scoped := range repository.deadlineScoped {
103109 require.True(t, scoped, "every purge repository call must receive a deadline-scoped context")
@@ -37,10 +37,11 @@ func (r *retryablePurgeRepository) WriteDeletionTombstone(ctx context.Context, t
37 func (r *retryablePurgeRepository) QuarantineDeletionSources(ctx context.Context, tombstone domain.DeletionTombstone, _ bool) error {37 func (r *retryablePurgeRepository) QuarantineDeletionSources(ctx context.Context, tombstone domain.DeletionTombstone, _ bool) error {
38 _, hasDeadline := ctx.Deadline()38 _, hasDeadline := ctx.Deadline()
39 r.deadlineScoped = append(r.deadlineScoped, hasDeadline)39 r.deadlineScoped = append(r.deadlineScoped, hasDeadline)
40- r.calls = append(r.calls, "incremental", "legacy")40+ r.calls = append(r.calls, "incremental")
41 if r.incrementalErr != nil {41 if r.incrementalErr != nil {
42 return r.incrementalErr42 return r.incrementalErr
43 }43 }
44+ r.calls = append(r.calls, "legacy")
44 return r.legacyErr45 return r.legacyErr
45 }46 }
46 func (r *retryablePurgeRepository) DeleteDeletionTombstone(ctx context.Context, _ domain.IncarnationID) error {47 func (r *retryablePurgeRepository) DeleteDeletionTombstone(ctx context.Context, _ domain.IncarnationID) error {
@@ -74,7 +75,7 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) {
74 repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy}75 repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy}
75 WithSnapshotRepository(repository)(d)76 WithSnapshotRepository(repository)(d)
76 store, _ := newMockStore(t)77 store, _ := newMockStore(t)
77- WithStore(store)(d)78+ WithStore(t, store)(d)
78 sess := newSnapshotTestSession(t, "work", false, "/work")79 sess := newSnapshotTestSession(t, "work", false, "/work")
79 record := sess.persistRecordLocked(1)80 record := sess.persistRecordLocked(1)
80 record.RecoveryState = domain.RecoveryFresh81 record.RecoveryState = domain.RecoveryFresh
@@ -86,7 +87,11 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) {
86 87
87 require.Error(t, d.killSession(sess, ports.ReasonSessionKilled, true))88 require.Error(t, d.killSession(sess, ports.ReasonSessionKilled, true))
88 require.True(t, repository.tombstoned["work"], "partial purge must fence startup restore/import")89 require.True(t, repository.tombstoned["work"], "partial purge must fence startup restore/import")
89- require.Equal(t, []string{"tombstone", "incremental", "legacy"}, repository.calls)90+ wantCalls := []string{"tombstone", "incremental"}
91+ if tt.incremental == nil {
92+ wantCalls = append(wantCalls, "legacy")
93+ }
94+ require.Equal(t, wantCalls, repository.calls)
90 d.mu.Lock()95 d.mu.Lock()
91 stopped, ok := d.stopped["work"]96 stopped, ok := d.stopped["work"]
92 d.mu.Unlock()97 d.mu.Unlock()
@@ -97,7 +102,8 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) {
97 repository.legacyErr = nil102 repository.legacyErr = nil
98 require.NoError(t, d.retryStoppedPurge("work"))103 require.NoError(t, d.retryStoppedPurge("work"))
99 require.False(t, repository.tombstoned["work"])104 require.False(t, repository.tombstoned["work"])
100- require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls)105+ wantCalls = append(wantCalls, "tombstone", "incremental", "legacy", "clear tombstone")
106+ require.Equal(t, wantCalls, repository.calls)
101 require.NotEmpty(t, repository.deadlineScoped)107 require.NotEmpty(t, repository.deadlineScoped)
102 for _, scoped := range repository.deadlineScoped {108 for _, scoped := range repository.deadlineScoped {
103 require.True(t, scoped, "every purge repository call must receive a deadline-scoped context")109 require.True(t, scoped, "every purge repository call must receive a deadline-scoped context")
M internal/usecase/daemon/session_rename_purge_test.go +2 -3
@@ -7,7 +7,6 @@ import (
77 "github.com/stretchr/testify/require"
88
99 "github.com/bnema/vev/internal/domain"
10- "github.com/bnema/vev/internal/persist"
1110 portsmocks "github.com/bnema/vev/internal/ports/mocks"
1211 )
1312
@@ -25,10 +24,10 @@ func TestRenamePreservesIncarnationSnapshotSources(t *testing.T) {
2524 repository := &retryablePurgeRepository{legacyErr: tt.legacyErr}
2625 WithSnapshotRepository(repository)(d)
2726 store, state := newMockStore(t)
28- WithStore(store)(d)
27+ WithStore(t, store)(d)
2928 sess := newSnapshotTestSession(t, "old", false, "/work")
3029 d.sessions = map[domain.SessionID]*session{sess.id: sess}
31- require.NoError(t, d.catalogue.(*persist.Persister).Save(sess.persistRecordLocked(1)))
30+ require.NoError(t, testPersister(t, d).Save(sess.persistRecordLocked(1)))
3231
3332 require.NoError(t, d.renameSession(sess, "new"))
3433 require.Empty(t, repository.calls, "rename must not invoke source deletion, including a failing legacy source")
@@ -7,7 +7,6 @@ import (
7 "github.com/stretchr/testify/require"7 "github.com/stretchr/testify/require"
8 8
9 "github.com/bnema/vev/internal/domain"9 "github.com/bnema/vev/internal/domain"
10- "github.com/bnema/vev/internal/persist"
11 portsmocks "github.com/bnema/vev/internal/ports/mocks"10 portsmocks "github.com/bnema/vev/internal/ports/mocks"
12 )11 )
13 12
@@ -25,10 +24,10 @@ func TestRenamePreservesIncarnationSnapshotSources(t *testing.T) {
25 repository := &retryablePurgeRepository{legacyErr: tt.legacyErr}24 repository := &retryablePurgeRepository{legacyErr: tt.legacyErr}
26 WithSnapshotRepository(repository)(d)25 WithSnapshotRepository(repository)(d)
27 store, state := newMockStore(t)26 store, state := newMockStore(t)
28- WithStore(store)(d)27+ WithStore(t, store)(d)
29 sess := newSnapshotTestSession(t, "old", false, "/work")28 sess := newSnapshotTestSession(t, "old", false, "/work")
30 d.sessions = map[domain.SessionID]*session{sess.id: sess}29 d.sessions = map[domain.SessionID]*session{sess.id: sess}
31- require.NoError(t, d.catalogue.(*persist.Persister).Save(sess.persistRecordLocked(1)))30+ require.NoError(t, testPersister(t, d).Save(sess.persistRecordLocked(1)))
32 31
33 require.NoError(t, d.renameSession(sess, "new"))32 require.NoError(t, d.renameSession(sess, "new"))
34 require.Empty(t, repository.calls, "rename must not invoke source deletion, including a failing legacy source")33 require.Empty(t, repository.calls, "rename must not invoke source deletion, including a failing legacy source")
M internal/usecase/daemon/session_test.go +72 -60
@@ -702,18 +702,30 @@ func newMockStore(t *testing.T) (*portsmocks.MockStore, *mockStoreState) {
702702 return nil
703703 }).Maybe()
704704 store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error {
705+ if len(changes) == 0 {
706+ return errors.New("empty batch")
707+ }
705708 state.mu.Lock()
709+ seen := make(map[string]struct{}, len(changes))
710+ for _, change := range changes {
711+ key := string(change.Key)
712+ if _, duplicate := seen[key]; duplicate {
713+ state.mu.Unlock()
714+ return errors.New("duplicate batch key")
715+ }
716+ seen[key] = struct{}{}
717+ if change.Delete && state.deleteErr != nil {
718+ if err := state.deleteErr(key); err != nil {
719+ state.mu.Unlock()
720+ return err
721+ }
722+ }
723+ }
706724 deleted := false
707725 for _, change := range changes {
708726 key := string(change.Key)
709727 if change.Delete {
710728 state.dels = append(state.dels, key)
711- if state.deleteErr != nil {
712- if err := state.deleteErr(key); err != nil {
713- state.mu.Unlock()
714- return err
715- }
716- }
717729 delete(state.data, key)
718730 deleted = true
719731 continue
@@ -845,9 +857,9 @@ func TestNamedSessionCreationIsAllowedWithNilStore(t *testing.T) {
845857 pty, release := newBlockingPTY(t)
846858 defer release()
847859 d := newTestDaemon(t, newFactory(t, pty), stubClock{})
848- WithStore(nil)(d)
860+ WithStore(t, nil)(d)
849861
850- sess, err := d.createSessionLocked("work", false, "/tmp/work", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
862+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
851863 require.NoError(t, err)
852864 require.Equal(t, "work", sess.name)
853865 require.False(t, d.persistEnabled)
@@ -855,11 +867,11 @@ func TestNamedSessionCreationIsAllowedWithNilStore(t *testing.T) {
855867
856868 func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) {
857869 store, _ := newMockStore(t)
858- seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))
870+ seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store))
859871 incarnation := domain.IncarnationID{1}
860- require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh}))
872+ require.NoError(t, testPersister(t, seed).Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh}))
861873
862- d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))
874+ d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store))
863875 d.mu.Lock()
864876 stopped := d.stopped["work"]
865877 d.mu.Unlock()
@@ -876,7 +888,7 @@ func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) {
876888
877889 // TestDaemonNewWithoutPersistenceLogsNoLoadWarning covers the default,
878890 // persistence-free construction path (no WithStore/WithCatalogue option):
879-// d.catalogue.(*persist.Persister) is the persist.New(nil) sentinel, so LoadAll now returns
891+// testPersister(t, d) is the persist.New(nil) sentinel, so LoadAll now returns
880892 // errPersistenceUnavailable instead of an empty slice. That must not surface
881893 // as a "loading persisted sessions failed" warning, since nothing failed —
882894 // persistence simply was never configured.
@@ -890,7 +902,7 @@ func TestDaemonNewWithoutPersistenceLogsNoLoadWarning(t *testing.T) {
890902 }
891903
892904 func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) {
893- d := &Daemon{}
905+ d := &Daemon{clock: stubClock{}}
894906 sess := &session{}
895907 const goroutines = 16
896908 const iterations = 1000
@@ -937,9 +949,9 @@ func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) {
937949 func TestTouchMRUPersistsNamedButNotEphemeral(t *testing.T) {
938950 store, state := newMockStore(t)
939951 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
940- WithStore(store)(d)
952+ WithStore(t, store)(d)
941953 named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1, incarnation: domain.IncarnationID{1}}
942- require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh}))
954+ require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh}))
943955
944956 d.touchMRU(named)
945957 require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq)
@@ -957,11 +969,11 @@ func TestCreateSessionSeedsMRUFromStopped(t *testing.T) {
957969 defer release()
958970 store, state := newMockStore(t)
959971 d := newTestDaemon(t, newFactory(t, p), stubClock{})
960- WithStore(store)(d)
972+ WithStore(t, store)(d)
961973 d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 1, lastUsedSeq: 42}
962974 d.mruSeq.Store(42)
963975
964- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
976+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
965977 require.NoError(t, err)
966978 require.Equal(t, uint64(42), sess.mruAt.Load())
967979 require.Equal(t, uint64(42), state.record(t, "work").LastUsedSeq)
@@ -975,9 +987,9 @@ func TestCreateRenameKillPersistenceLifecycle(t *testing.T) {
975987 defer release2()
976988 store, state := newMockStore(t)
977989 d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{})
978- WithStore(store)(d)
990+ WithStore(t, store)(d)
979991
980- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
992+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
981993 require.NoError(t, err)
982994 require.True(t, state.has("work"))
983995 created := state.record(t, "work")
@@ -1006,13 +1018,13 @@ func TestRenameTabPersistsForNamedSession(t *testing.T) {
10061018 defer release()
10071019 store, _ := newMockStore(t)
10081020 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1009- WithStore(store)(d)
1021+ WithStore(t, store)(d)
10101022
1011- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
1023+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
10121024 require.NoError(t, err)
10131025 require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell"))
10141026
1015- records, err := d.catalogue.(*persist.Persister).LoadAll()
1027+ records, err := testPersister(t, d).LoadAll()
10161028 require.NoError(t, err)
10171029 require.Len(t, records, 1)
10181030 require.Equal(t, "work", records[0].Name)
@@ -1059,10 +1071,10 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) {
10591071 }
10601072 store, _ := newMockStore(t)
10611073 d := newTestDaemon(t, newFactorySeq(t, ptys...), stubClock{})
1062- WithStore(store)(d)
1074+ WithStore(t, store)(d)
10631075 sz := domain.Size{Cols: 80, Rows: 24}
10641076
1065- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
1077+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
10661078 require.NoError(t, err)
10671079 require.NoError(t, d.createTab(sess, sz))
10681080 require.NoError(t, d.createTab(sess, sz))
@@ -1071,7 +1083,7 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) {
10711083 }
10721084
10731085 require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false))
1074- records, err := d.catalogue.(*persist.Persister).LoadAll()
1086+ records, err := testPersister(t, d).LoadAll()
10751087 require.NoError(t, err)
10761088 require.Len(t, records, 1)
10771089 require.Equal(t, tt.want, records[0].TabNames)
@@ -1108,15 +1120,15 @@ func TestRenameTabDoesNotPersistForEphemeralSession(t *testing.T) {
11081120 defer release()
11091121 store, state := newMockStore(t)
11101122 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1111- WithStore(store)(d)
1123+ WithStore(t, store)(d)
11121124
1113- sess, err := d.createSessionLocked("0", true, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
1125+ sess, err := createSessionForTest(d, "0", true, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
11141126 require.NoError(t, err)
11151127 require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell"))
11161128 require.Equal(t, "shell", sess.tabs[0].name)
11171129
11181130 require.False(t, state.has("0"))
1119- records, err := d.catalogue.(*persist.Persister).LoadAll()
1131+ records, err := testPersister(t, d).LoadAll()
11201132 require.NoError(t, err)
11211133 require.Empty(t, records)
11221134 }
@@ -1129,8 +1141,8 @@ func TestAttachRestoresPersistedTabNames(t *testing.T) {
11291141 defer release2()
11301142 store, _ := newMockStore(t)
11311143 d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{})
1132- WithStore(store)(d)
1133- require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh}))
1144+ WithStore(t, store)(d)
1145+ require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh}))
11341146 d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}}
11351147 tr := portsmocks.NewMockTransport(t)
11361148 tr.EXPECT().Send(mock.Anything).Return(nil).Maybe()
@@ -1150,10 +1162,10 @@ func TestEphemeralRenamePromotesAndStoppedCollisionRejected(t *testing.T) {
11501162 defer release()
11511163 store, state := newMockStore(t)
11521164 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1153- WithStore(store)(d)
1165+ WithStore(t, store)(d)
11541166 d.stopped["taken"] = stoppedSession{name: "taken", cwd: "/tmp", createdAt: 1}
11551167
1156- sess, err := d.createSessionLocked("0", true, "/tmp/e", sz, terminalEnv{}, d.baseEnv)
1168+ sess, err := createSessionForTest(d, "0", true, "/tmp/e", sz, terminalEnv{}, d.baseEnv)
11571169 require.NoError(t, err)
11581170 require.False(t, state.has("0"))
11591171 require.EqualError(t, d.renameSession(sess, "taken"), "name already in use")
@@ -1167,9 +1179,9 @@ func TestEphemeralPromotionAssignsAndPersistsLifecycleIdentity(t *testing.T) {
11671179 store, state := newMockStore(t)
11681180 clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}}
11691181 d := newTestDaemon(t, newFactory(t, p), clock)
1170- WithStore(store)(d)
1182+ WithStore(t, store)(d)
11711183
1172- sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
1184+ sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
11731185 require.NoError(t, err)
11741186 require.Zero(t, sess.createdAt, "regression fixture: ephemerals have no lifecycle identity")
11751187
@@ -1194,9 +1206,9 @@ func TestEphemeralPromotionLifecyclePreventsStaleSameNamePaletteTarget(t *testin
11941206 clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}}
11951207 d.clock = clock
11961208 d.ptys = newFactorySeq(t, firstPTY, secondPTY)
1197- WithStore(store)(d)
1209+ WithStore(t, store)(d)
11981210
1199- first, err := d.createSessionLocked("0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv)
1211+ first, err := createSessionForTest(d, "0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv)
12001212 require.NoError(t, err)
12011213 require.NoError(t, d.renameSession(first, "named"))
12021214 staleCreatedAt := first.createdAt
@@ -1204,7 +1216,7 @@ func TestEphemeralPromotionLifecyclePreventsStaleSameNamePaletteTarget(t *testin
12041216 require.NoError(t, d.killSession(first, ports.ReasonSessionKilled, true))
12051217 d.sessWg.Wait()
12061218
1207- second, err := d.createSessionLocked("0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv)
1219+ second, err := createSessionForTest(d, "0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv)
12081220 require.NoError(t, err)
12091221 require.NoError(t, d.renameSession(second, "named"))
12101222 require.NotEqual(t, staleCreatedAt, second.createdAt)
@@ -1221,10 +1233,10 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T)
12211233 defer release()
12221234 store, state := newMockStore(t)
12231235 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1224- WithStore(store)(d)
1236+ WithStore(t, store)(d)
12251237 d.lastAllocatedCreatedAt = math.MaxInt64
12261238
1227- sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
1239+ sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
12281240 require.NoError(t, err)
12291241 require.EqualError(t, d.renameSession(sess, "named"), "daemon: lifecycle identities exhausted")
12301242 require.Equal(t, "0", sess.name)
@@ -1250,9 +1262,9 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T)
12501262 }).Once()
12511263 clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}}
12521264 d := newTestDaemon(t, newFactory(t, p), clock)
1253- WithStore(store)(d)
1265+ WithStore(t, store)(d)
12541266
1255- sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
1267+ sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
12561268 require.NoError(t, err)
12571269 require.EqualError(t, d.renameSession(sess, "named"), "disk full")
12581270 require.Equal(t, "0", sess.name)
@@ -1277,11 +1289,11 @@ func TestRefreshSessionCwdTouchesOnlyOnChange(t *testing.T) {
12771289 defer release()
12781290 store, state := newMockStore(t)
12791291 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1280- WithStore(store)(d)
1292+ WithStore(t, store)(d)
12811293 cwd := "/tmp/work"
12821294 WithCwdReader(func(int) (string, error) { return cwd, nil })(d)
12831295
1284- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
1296+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
12851297 require.NoError(t, err)
12861298 state.mu.Lock()
12871299 setsAfterCreate := state.sets
@@ -1305,7 +1317,7 @@ func TestAttachResumesStoppedSessionFromStoredCwd(t *testing.T) {
13051317 defer release()
13061318 store, _ := newMockStore(t)
13071319 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1308- WithStore(store)(d)
1320+ WithStore(t, store)(d)
13091321 cwd := t.TempDir()
13101322 d.stopped["work"] = stoppedSession{name: "work", cwd: cwd, createdAt: 1}
13111323 tr := portsmocks.NewMockTransport(t)
@@ -1329,7 +1341,7 @@ func TestAttachStoppedMissingCwdFallsBackToHome(t *testing.T) {
13291341 defer release()
13301342 store, _ := newMockStore(t)
13311343 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1332- WithStore(store)(d)
1344+ WithStore(t, store)(d)
13331345 d.stopped["work"] = stoppedSession{name: "work", cwd: "/definitely/missing/vev", createdAt: 1}
13341346 tr := portsmocks.NewMockTransport(t)
13351347 tr.EXPECT().Send(mock.Anything).Return(nil).Maybe()
@@ -1343,8 +1355,8 @@ func TestAttachStoppedMissingCwdFallsBackToHome(t *testing.T) {
13431355 func TestPickerStoppedTargetKillPurges(t *testing.T) {
13441356 store, state := newMockStore(t)
13451357 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
1346- WithStore(store)(d)
1347- require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh}))
1358+ WithStore(t, store)(d)
1359+ require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh}))
13481360 d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1}
13491361 require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true}))
13501362 require.False(t, state.has("old"))
@@ -1378,7 +1390,7 @@ func TestNewSessionAssignsStableIDsAndChildEnv(t *testing.T) {
13781390 d := newTestDaemon(t, f, stubClock{})
13791391 d.baseEnv = []string{"KEEP=1", "TERM=old", "COLORTERM=old", "TERM_PROGRAM=old", "VEV=old"}
13801392
1381- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{TrueColor: true}, d.baseEnv)
1393+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{TrueColor: true}, d.baseEnv)
13821394 require.NoError(t, err)
13831395 defer func() {
13841396 _ = d.killSession(sess, ports.ReasonServerShutdown, false)
@@ -1434,7 +1446,7 @@ func TestRenameSessionUnsafeNameRejected(t *testing.T) {
14341446 p, release := newBlockingPTY(t)
14351447 defer release()
14361448 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1437- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
1449+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
14381450 require.NoError(t, err)
14391451 require.ErrorIs(t, d.renameSession(sess, "my work"), domain.ErrInvalidSessionName)
14401452 }
@@ -1444,7 +1456,7 @@ func TestCreateTabPtyFailureIsUserError(t *testing.T) {
14441456 p, release := newBlockingPTY(t)
14451457 defer release()
14461458 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1447- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
1459+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
14481460 require.NoError(t, err)
14491461
14501462 cause := errors.New("fork/exec: no such file")
@@ -1471,12 +1483,12 @@ func TestNaturalExitStoppedButExplicitKillPurges(t *testing.T) {
14711483 defer release2()
14721484 store, state := newMockStore(t)
14731485 d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{})
1474- WithStore(store)(d)
1486+ WithStore(t, store)(d)
14751487 WithCwdReader(func(int) (string, error) { return "/tmp/latest", nil })(d)
14761488
1477- natural, err := d.createSessionLocked("natural", false, "/tmp/old", sz, terminalEnv{}, d.baseEnv)
1489+ natural, err := createSessionForTest(d, "natural", false, "/tmp/old", sz, terminalEnv{}, d.baseEnv)
14781490 require.NoError(t, err)
1479- other, err := d.createSessionLocked("other", false, "/tmp/other", sz, terminalEnv{}, d.baseEnv)
1491+ other, err := createSessionForTest(d, "other", false, "/tmp/other", sz, terminalEnv{}, d.baseEnv)
14801492 require.NoError(t, err)
14811493 _ = d.killSession(natural, ports.ReasonSessionKilled, false)
14821494 require.True(t, state.has("natural"))
@@ -1792,7 +1804,7 @@ func TestNamedSessionLifecycleTimestampsAreMonotonicAcrossClockRegression(t *tes
17921804
17931805 var got []int64
17941806 for _, name := range []string{"one", "two", "three"} {
1795- sess, err := d.createSessionLocked(name, false, "/tmp", sz, terminalEnv{}, d.baseEnv)
1807+ sess, err := createSessionForTest(d, name, false, "/tmp", sz, terminalEnv{}, d.baseEnv)
17961808 require.NoError(t, err)
17971809 got = append(got, sess.createdAt)
17981810 }
@@ -1805,7 +1817,7 @@ func TestNamedSessionLifecycleExhaustionDoesNotMutateSessionState(t *testing.T)
18051817 d.nextID = 17
18061818 d.stopped["retained"] = stoppedSession{name: "retained", cwd: "/tmp", createdAt: 9}
18071819
1808- sess, err := d.createSessionLocked("new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
1820+ sess, err := createSessionForTest(d, "new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
18091821
18101822 require.ErrorContains(t, err, "lifecycle identities exhausted")
18111823 require.Nil(t, sess)
@@ -1855,18 +1867,18 @@ func TestCatalogueRecordsConstructExpectedSessionRegistry(t *testing.T) {
18551867
18561868 func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) {
18571869 store, _ := newMockStore(t)
1858- seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))
1859- require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh}))
1870+ seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store))
1871+ require.NoError(t, testPersister(t, seed).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh}))
18601872
18611873 p, release := newBlockingPTY(t)
18621874 defer release()
18631875 clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}}
18641876 // Constructing with persistence must establish the lifecycle high-water mark.
1865- d := New(newFactory(t, p), clock, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))
1877+ d := New(newFactory(t, p), clock, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store))
18661878 d.serveCtx, d.serveCancel = context.WithCancel(context.Background())
18671879 t.Cleanup(d.serveCancel)
18681880
1869- sess, err := d.createSessionLocked("new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
1881+ sess, err := createSessionForTest(d, "new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
18701882 require.NoError(t, err)
18711883 require.Equal(t, int64(901), sess.createdAt)
18721884 }
@@ -1878,7 +1890,7 @@ func TestResumingStoppedSessionPreservesLifecycleIdentityInPersistence(t *testin
18781890 defer releaseTarget()
18791891 d, from, ac, _ := newManualSessionWithPTYs(t, fromPTY)
18801892 store, state := newMockStore(t)
1881- WithStore(store)(d)
1893+ WithStore(t, store)(d)
18821894 d.ptys = newFactory(t, targetPTY)
18831895 d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp", createdAt: 77}
18841896
@@ -702,18 +702,30 @@ func newMockStore(t *testing.T) (*portsmocks.MockStore, *mockStoreState) {
702 return nil702 return nil
703 }).Maybe()703 }).Maybe()
704 store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error {704 store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error {
705+ if len(changes) == 0 {
706+ return errors.New("empty batch")
707+ }
705 state.mu.Lock()708 state.mu.Lock()
709+ seen := make(map[string]struct{}, len(changes))
710+ for _, change := range changes {
711+ key := string(change.Key)
712+ if _, duplicate := seen[key]; duplicate {
713+ state.mu.Unlock()
714+ return errors.New("duplicate batch key")
715+ }
716+ seen[key] = struct{}{}
717+ if change.Delete && state.deleteErr != nil {
718+ if err := state.deleteErr(key); err != nil {
719+ state.mu.Unlock()
720+ return err
721+ }
722+ }
723+ }
706 deleted := false724 deleted := false
707 for _, change := range changes {725 for _, change := range changes {
708 key := string(change.Key)726 key := string(change.Key)
709 if change.Delete {727 if change.Delete {
710 state.dels = append(state.dels, key)728 state.dels = append(state.dels, key)
711- if state.deleteErr != nil {
712- if err := state.deleteErr(key); err != nil {
713- state.mu.Unlock()
714- return err
715- }
716- }
717 delete(state.data, key)729 delete(state.data, key)
718 deleted = true730 deleted = true
719 continue731 continue
@@ -845,9 +857,9 @@ func TestNamedSessionCreationIsAllowedWithNilStore(t *testing.T) {
845 pty, release := newBlockingPTY(t)857 pty, release := newBlockingPTY(t)
846 defer release()858 defer release()
847 d := newTestDaemon(t, newFactory(t, pty), stubClock{})859 d := newTestDaemon(t, newFactory(t, pty), stubClock{})
848- WithStore(nil)(d)860+ WithStore(t, nil)(d)
849 861
850- sess, err := d.createSessionLocked("work", false, "/tmp/work", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)862+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
851 require.NoError(t, err)863 require.NoError(t, err)
852 require.Equal(t, "work", sess.name)864 require.Equal(t, "work", sess.name)
853 require.False(t, d.persistEnabled)865 require.False(t, d.persistEnabled)
@@ -855,11 +867,11 @@ func TestNamedSessionCreationIsAllowedWithNilStore(t *testing.T) {
855 867
856 func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) {868 func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) {
857 store, _ := newMockStore(t)869 store, _ := newMockStore(t)
858- seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))870+ seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store))
859 incarnation := domain.IncarnationID{1}871 incarnation := domain.IncarnationID{1}
860- require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh}))872+ require.NoError(t, testPersister(t, seed).Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh}))
861 873
862- d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))874+ d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store))
863 d.mu.Lock()875 d.mu.Lock()
864 stopped := d.stopped["work"]876 stopped := d.stopped["work"]
865 d.mu.Unlock()877 d.mu.Unlock()
@@ -876,7 +888,7 @@ func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) {
876 888
877 // TestDaemonNewWithoutPersistenceLogsNoLoadWarning covers the default,889 // TestDaemonNewWithoutPersistenceLogsNoLoadWarning covers the default,
878 // persistence-free construction path (no WithStore/WithCatalogue option):890 // persistence-free construction path (no WithStore/WithCatalogue option):
879-// d.catalogue.(*persist.Persister) is the persist.New(nil) sentinel, so LoadAll now returns891+// testPersister(t, d) is the persist.New(nil) sentinel, so LoadAll now returns
880 // errPersistenceUnavailable instead of an empty slice. That must not surface892 // errPersistenceUnavailable instead of an empty slice. That must not surface
881 // as a "loading persisted sessions failed" warning, since nothing failed —893 // as a "loading persisted sessions failed" warning, since nothing failed —
882 // persistence simply was never configured.894 // persistence simply was never configured.
@@ -890,7 +902,7 @@ func TestDaemonNewWithoutPersistenceLogsNoLoadWarning(t *testing.T) {
890 }902 }
891 903
892 func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) {904 func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) {
893- d := &Daemon{}905+ d := &Daemon{clock: stubClock{}}
894 sess := &session{}906 sess := &session{}
895 const goroutines = 16907 const goroutines = 16
896 const iterations = 1000908 const iterations = 1000
@@ -937,9 +949,9 @@ func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) {
937 func TestTouchMRUPersistsNamedButNotEphemeral(t *testing.T) {949 func TestTouchMRUPersistsNamedButNotEphemeral(t *testing.T) {
938 store, state := newMockStore(t)950 store, state := newMockStore(t)
939 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})951 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
940- WithStore(store)(d)952+ WithStore(t, store)(d)
941 named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1, incarnation: domain.IncarnationID{1}}953 named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1, incarnation: domain.IncarnationID{1}}
942- require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh}))954+ require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh}))
943 955
944 d.touchMRU(named)956 d.touchMRU(named)
945 require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq)957 require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq)
@@ -957,11 +969,11 @@ func TestCreateSessionSeedsMRUFromStopped(t *testing.T) {
957 defer release()969 defer release()
958 store, state := newMockStore(t)970 store, state := newMockStore(t)
959 d := newTestDaemon(t, newFactory(t, p), stubClock{})971 d := newTestDaemon(t, newFactory(t, p), stubClock{})
960- WithStore(store)(d)972+ WithStore(t, store)(d)
961 d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 1, lastUsedSeq: 42}973 d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 1, lastUsedSeq: 42}
962 d.mruSeq.Store(42)974 d.mruSeq.Store(42)
963 975
964- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)976+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
965 require.NoError(t, err)977 require.NoError(t, err)
966 require.Equal(t, uint64(42), sess.mruAt.Load())978 require.Equal(t, uint64(42), sess.mruAt.Load())
967 require.Equal(t, uint64(42), state.record(t, "work").LastUsedSeq)979 require.Equal(t, uint64(42), state.record(t, "work").LastUsedSeq)
@@ -975,9 +987,9 @@ func TestCreateRenameKillPersistenceLifecycle(t *testing.T) {
975 defer release2()987 defer release2()
976 store, state := newMockStore(t)988 store, state := newMockStore(t)
977 d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{})989 d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{})
978- WithStore(store)(d)990+ WithStore(t, store)(d)
979 991
980- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)992+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
981 require.NoError(t, err)993 require.NoError(t, err)
982 require.True(t, state.has("work"))994 require.True(t, state.has("work"))
983 created := state.record(t, "work")995 created := state.record(t, "work")
@@ -1006,13 +1018,13 @@ func TestRenameTabPersistsForNamedSession(t *testing.T) {
1006 defer release()1018 defer release()
1007 store, _ := newMockStore(t)1019 store, _ := newMockStore(t)
1008 d := newTestDaemon(t, newFactory(t, p), stubClock{})1020 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1009- WithStore(store)(d)1021+ WithStore(t, store)(d)
1010 1022
1011- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)1023+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
1012 require.NoError(t, err)1024 require.NoError(t, err)
1013 require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell"))1025 require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell"))
1014 1026
1015- records, err := d.catalogue.(*persist.Persister).LoadAll()1027+ records, err := testPersister(t, d).LoadAll()
1016 require.NoError(t, err)1028 require.NoError(t, err)
1017 require.Len(t, records, 1)1029 require.Len(t, records, 1)
1018 require.Equal(t, "work", records[0].Name)1030 require.Equal(t, "work", records[0].Name)
@@ -1059,10 +1071,10 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) {
1059 }1071 }
1060 store, _ := newMockStore(t)1072 store, _ := newMockStore(t)
1061 d := newTestDaemon(t, newFactorySeq(t, ptys...), stubClock{})1073 d := newTestDaemon(t, newFactorySeq(t, ptys...), stubClock{})
1062- WithStore(store)(d)1074+ WithStore(t, store)(d)
1063 sz := domain.Size{Cols: 80, Rows: 24}1075 sz := domain.Size{Cols: 80, Rows: 24}
1064 1076
1065- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)1077+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
1066 require.NoError(t, err)1078 require.NoError(t, err)
1067 require.NoError(t, d.createTab(sess, sz))1079 require.NoError(t, d.createTab(sess, sz))
1068 require.NoError(t, d.createTab(sess, sz))1080 require.NoError(t, d.createTab(sess, sz))
@@ -1071,7 +1083,7 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) {
1071 }1083 }
1072 1084
1073 require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false))1085 require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false))
1074- records, err := d.catalogue.(*persist.Persister).LoadAll()1086+ records, err := testPersister(t, d).LoadAll()
1075 require.NoError(t, err)1087 require.NoError(t, err)
1076 require.Len(t, records, 1)1088 require.Len(t, records, 1)
1077 require.Equal(t, tt.want, records[0].TabNames)1089 require.Equal(t, tt.want, records[0].TabNames)
@@ -1108,15 +1120,15 @@ func TestRenameTabDoesNotPersistForEphemeralSession(t *testing.T) {
1108 defer release()1120 defer release()
1109 store, state := newMockStore(t)1121 store, state := newMockStore(t)
1110 d := newTestDaemon(t, newFactory(t, p), stubClock{})1122 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1111- WithStore(store)(d)1123+ WithStore(t, store)(d)
1112 1124
1113- sess, err := d.createSessionLocked("0", true, "/tmp/work", sz, terminalEnv{}, d.baseEnv)1125+ sess, err := createSessionForTest(d, "0", true, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
1114 require.NoError(t, err)1126 require.NoError(t, err)
1115 require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell"))1127 require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell"))
1116 require.Equal(t, "shell", sess.tabs[0].name)1128 require.Equal(t, "shell", sess.tabs[0].name)
1117 1129
1118 require.False(t, state.has("0"))1130 require.False(t, state.has("0"))
1119- records, err := d.catalogue.(*persist.Persister).LoadAll()1131+ records, err := testPersister(t, d).LoadAll()
1120 require.NoError(t, err)1132 require.NoError(t, err)
1121 require.Empty(t, records)1133 require.Empty(t, records)
1122 }1134 }
@@ -1129,8 +1141,8 @@ func TestAttachRestoresPersistedTabNames(t *testing.T) {
1129 defer release2()1141 defer release2()
1130 store, _ := newMockStore(t)1142 store, _ := newMockStore(t)
1131 d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{})1143 d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{})
1132- WithStore(store)(d)1144+ WithStore(t, store)(d)
1133- require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh}))1145+ require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh}))
1134 d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}}1146 d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}}
1135 tr := portsmocks.NewMockTransport(t)1147 tr := portsmocks.NewMockTransport(t)
1136 tr.EXPECT().Send(mock.Anything).Return(nil).Maybe()1148 tr.EXPECT().Send(mock.Anything).Return(nil).Maybe()
@@ -1150,10 +1162,10 @@ func TestEphemeralRenamePromotesAndStoppedCollisionRejected(t *testing.T) {
1150 defer release()1162 defer release()
1151 store, state := newMockStore(t)1163 store, state := newMockStore(t)
1152 d := newTestDaemon(t, newFactory(t, p), stubClock{})1164 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1153- WithStore(store)(d)1165+ WithStore(t, store)(d)
1154 d.stopped["taken"] = stoppedSession{name: "taken", cwd: "/tmp", createdAt: 1}1166 d.stopped["taken"] = stoppedSession{name: "taken", cwd: "/tmp", createdAt: 1}
1155 1167
1156- sess, err := d.createSessionLocked("0", true, "/tmp/e", sz, terminalEnv{}, d.baseEnv)1168+ sess, err := createSessionForTest(d, "0", true, "/tmp/e", sz, terminalEnv{}, d.baseEnv)
1157 require.NoError(t, err)1169 require.NoError(t, err)
1158 require.False(t, state.has("0"))1170 require.False(t, state.has("0"))
1159 require.EqualError(t, d.renameSession(sess, "taken"), "name already in use")1171 require.EqualError(t, d.renameSession(sess, "taken"), "name already in use")
@@ -1167,9 +1179,9 @@ func TestEphemeralPromotionAssignsAndPersistsLifecycleIdentity(t *testing.T) {
1167 store, state := newMockStore(t)1179 store, state := newMockStore(t)
1168 clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}}1180 clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}}
1169 d := newTestDaemon(t, newFactory(t, p), clock)1181 d := newTestDaemon(t, newFactory(t, p), clock)
1170- WithStore(store)(d)1182+ WithStore(t, store)(d)
1171 1183
1172- sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)1184+ sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
1173 require.NoError(t, err)1185 require.NoError(t, err)
1174 require.Zero(t, sess.createdAt, "regression fixture: ephemerals have no lifecycle identity")1186 require.Zero(t, sess.createdAt, "regression fixture: ephemerals have no lifecycle identity")
1175 1187
@@ -1194,9 +1206,9 @@ func TestEphemeralPromotionLifecyclePreventsStaleSameNamePaletteTarget(t *testin
1194 clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}}1206 clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}}
1195 d.clock = clock1207 d.clock = clock
1196 d.ptys = newFactorySeq(t, firstPTY, secondPTY)1208 d.ptys = newFactorySeq(t, firstPTY, secondPTY)
1197- WithStore(store)(d)1209+ WithStore(t, store)(d)
1198 1210
1199- first, err := d.createSessionLocked("0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv)1211+ first, err := createSessionForTest(d, "0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv)
1200 require.NoError(t, err)1212 require.NoError(t, err)
1201 require.NoError(t, d.renameSession(first, "named"))1213 require.NoError(t, d.renameSession(first, "named"))
1202 staleCreatedAt := first.createdAt1214 staleCreatedAt := first.createdAt
@@ -1204,7 +1216,7 @@ func TestEphemeralPromotionLifecyclePreventsStaleSameNamePaletteTarget(t *testin
1204 require.NoError(t, d.killSession(first, ports.ReasonSessionKilled, true))1216 require.NoError(t, d.killSession(first, ports.ReasonSessionKilled, true))
1205 d.sessWg.Wait()1217 d.sessWg.Wait()
1206 1218
1207- second, err := d.createSessionLocked("0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv)1219+ second, err := createSessionForTest(d, "0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv)
1208 require.NoError(t, err)1220 require.NoError(t, err)
1209 require.NoError(t, d.renameSession(second, "named"))1221 require.NoError(t, d.renameSession(second, "named"))
1210 require.NotEqual(t, staleCreatedAt, second.createdAt)1222 require.NotEqual(t, staleCreatedAt, second.createdAt)
@@ -1221,10 +1233,10 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T)
1221 defer release()1233 defer release()
1222 store, state := newMockStore(t)1234 store, state := newMockStore(t)
1223 d := newTestDaemon(t, newFactory(t, p), stubClock{})1235 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1224- WithStore(store)(d)1236+ WithStore(t, store)(d)
1225 d.lastAllocatedCreatedAt = math.MaxInt641237 d.lastAllocatedCreatedAt = math.MaxInt64
1226 1238
1227- sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)1239+ sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
1228 require.NoError(t, err)1240 require.NoError(t, err)
1229 require.EqualError(t, d.renameSession(sess, "named"), "daemon: lifecycle identities exhausted")1241 require.EqualError(t, d.renameSession(sess, "named"), "daemon: lifecycle identities exhausted")
1230 require.Equal(t, "0", sess.name)1242 require.Equal(t, "0", sess.name)
@@ -1250,9 +1262,9 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T)
1250 }).Once()1262 }).Once()
1251 clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}}1263 clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}}
1252 d := newTestDaemon(t, newFactory(t, p), clock)1264 d := newTestDaemon(t, newFactory(t, p), clock)
1253- WithStore(store)(d)1265+ WithStore(t, store)(d)
1254 1266
1255- sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)1267+ sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
1256 require.NoError(t, err)1268 require.NoError(t, err)
1257 require.EqualError(t, d.renameSession(sess, "named"), "disk full")1269 require.EqualError(t, d.renameSession(sess, "named"), "disk full")
1258 require.Equal(t, "0", sess.name)1270 require.Equal(t, "0", sess.name)
@@ -1277,11 +1289,11 @@ func TestRefreshSessionCwdTouchesOnlyOnChange(t *testing.T) {
1277 defer release()1289 defer release()
1278 store, state := newMockStore(t)1290 store, state := newMockStore(t)
1279 d := newTestDaemon(t, newFactory(t, p), stubClock{})1291 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1280- WithStore(store)(d)1292+ WithStore(t, store)(d)
1281 cwd := "/tmp/work"1293 cwd := "/tmp/work"
1282 WithCwdReader(func(int) (string, error) { return cwd, nil })(d)1294 WithCwdReader(func(int) (string, error) { return cwd, nil })(d)
1283 1295
1284- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)1296+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
1285 require.NoError(t, err)1297 require.NoError(t, err)
1286 state.mu.Lock()1298 state.mu.Lock()
1287 setsAfterCreate := state.sets1299 setsAfterCreate := state.sets
@@ -1305,7 +1317,7 @@ func TestAttachResumesStoppedSessionFromStoredCwd(t *testing.T) {
1305 defer release()1317 defer release()
1306 store, _ := newMockStore(t)1318 store, _ := newMockStore(t)
1307 d := newTestDaemon(t, newFactory(t, p), stubClock{})1319 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1308- WithStore(store)(d)1320+ WithStore(t, store)(d)
1309 cwd := t.TempDir()1321 cwd := t.TempDir()
1310 d.stopped["work"] = stoppedSession{name: "work", cwd: cwd, createdAt: 1}1322 d.stopped["work"] = stoppedSession{name: "work", cwd: cwd, createdAt: 1}
1311 tr := portsmocks.NewMockTransport(t)1323 tr := portsmocks.NewMockTransport(t)
@@ -1329,7 +1341,7 @@ func TestAttachStoppedMissingCwdFallsBackToHome(t *testing.T) {
1329 defer release()1341 defer release()
1330 store, _ := newMockStore(t)1342 store, _ := newMockStore(t)
1331 d := newTestDaemon(t, newFactory(t, p), stubClock{})1343 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1332- WithStore(store)(d)1344+ WithStore(t, store)(d)
1333 d.stopped["work"] = stoppedSession{name: "work", cwd: "/definitely/missing/vev", createdAt: 1}1345 d.stopped["work"] = stoppedSession{name: "work", cwd: "/definitely/missing/vev", createdAt: 1}
1334 tr := portsmocks.NewMockTransport(t)1346 tr := portsmocks.NewMockTransport(t)
1335 tr.EXPECT().Send(mock.Anything).Return(nil).Maybe()1347 tr.EXPECT().Send(mock.Anything).Return(nil).Maybe()
@@ -1343,8 +1355,8 @@ func TestAttachStoppedMissingCwdFallsBackToHome(t *testing.T) {
1343 func TestPickerStoppedTargetKillPurges(t *testing.T) {1355 func TestPickerStoppedTargetKillPurges(t *testing.T) {
1344 store, state := newMockStore(t)1356 store, state := newMockStore(t)
1345 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})1357 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
1346- WithStore(store)(d)1358+ WithStore(t, store)(d)
1347- require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh}))1359+ require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh}))
1348 d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1}1360 d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1}
1349 require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true}))1361 require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true}))
1350 require.False(t, state.has("old"))1362 require.False(t, state.has("old"))
@@ -1378,7 +1390,7 @@ func TestNewSessionAssignsStableIDsAndChildEnv(t *testing.T) {
1378 d := newTestDaemon(t, f, stubClock{})1390 d := newTestDaemon(t, f, stubClock{})
1379 d.baseEnv = []string{"KEEP=1", "TERM=old", "COLORTERM=old", "TERM_PROGRAM=old", "VEV=old"}1391 d.baseEnv = []string{"KEEP=1", "TERM=old", "COLORTERM=old", "TERM_PROGRAM=old", "VEV=old"}
1380 1392
1381- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{TrueColor: true}, d.baseEnv)1393+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{TrueColor: true}, d.baseEnv)
1382 require.NoError(t, err)1394 require.NoError(t, err)
1383 defer func() {1395 defer func() {
1384 _ = d.killSession(sess, ports.ReasonServerShutdown, false)1396 _ = d.killSession(sess, ports.ReasonServerShutdown, false)
@@ -1434,7 +1446,7 @@ func TestRenameSessionUnsafeNameRejected(t *testing.T) {
1434 p, release := newBlockingPTY(t)1446 p, release := newBlockingPTY(t)
1435 defer release()1447 defer release()
1436 d := newTestDaemon(t, newFactory(t, p), stubClock{})1448 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1437- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)1449+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
1438 require.NoError(t, err)1450 require.NoError(t, err)
1439 require.ErrorIs(t, d.renameSession(sess, "my work"), domain.ErrInvalidSessionName)1451 require.ErrorIs(t, d.renameSession(sess, "my work"), domain.ErrInvalidSessionName)
1440 }1452 }
@@ -1444,7 +1456,7 @@ func TestCreateTabPtyFailureIsUserError(t *testing.T) {
1444 p, release := newBlockingPTY(t)1456 p, release := newBlockingPTY(t)
1445 defer release()1457 defer release()
1446 d := newTestDaemon(t, newFactory(t, p), stubClock{})1458 d := newTestDaemon(t, newFactory(t, p), stubClock{})
1447- sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)1459+ sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv)
1448 require.NoError(t, err)1460 require.NoError(t, err)
1449 1461
1450 cause := errors.New("fork/exec: no such file")1462 cause := errors.New("fork/exec: no such file")
@@ -1471,12 +1483,12 @@ func TestNaturalExitStoppedButExplicitKillPurges(t *testing.T) {
1471 defer release2()1483 defer release2()
1472 store, state := newMockStore(t)1484 store, state := newMockStore(t)
1473 d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{})1485 d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{})
1474- WithStore(store)(d)1486+ WithStore(t, store)(d)
1475 WithCwdReader(func(int) (string, error) { return "/tmp/latest", nil })(d)1487 WithCwdReader(func(int) (string, error) { return "/tmp/latest", nil })(d)
1476 1488
1477- natural, err := d.createSessionLocked("natural", false, "/tmp/old", sz, terminalEnv{}, d.baseEnv)1489+ natural, err := createSessionForTest(d, "natural", false, "/tmp/old", sz, terminalEnv{}, d.baseEnv)
1478 require.NoError(t, err)1490 require.NoError(t, err)
1479- other, err := d.createSessionLocked("other", false, "/tmp/other", sz, terminalEnv{}, d.baseEnv)1491+ other, err := createSessionForTest(d, "other", false, "/tmp/other", sz, terminalEnv{}, d.baseEnv)
1480 require.NoError(t, err)1492 require.NoError(t, err)
1481 _ = d.killSession(natural, ports.ReasonSessionKilled, false)1493 _ = d.killSession(natural, ports.ReasonSessionKilled, false)
1482 require.True(t, state.has("natural"))1494 require.True(t, state.has("natural"))
@@ -1792,7 +1804,7 @@ func TestNamedSessionLifecycleTimestampsAreMonotonicAcrossClockRegression(t *tes
1792 1804
1793 var got []int641805 var got []int64
1794 for _, name := range []string{"one", "two", "three"} {1806 for _, name := range []string{"one", "two", "three"} {
1795- sess, err := d.createSessionLocked(name, false, "/tmp", sz, terminalEnv{}, d.baseEnv)1807+ sess, err := createSessionForTest(d, name, false, "/tmp", sz, terminalEnv{}, d.baseEnv)
1796 require.NoError(t, err)1808 require.NoError(t, err)
1797 got = append(got, sess.createdAt)1809 got = append(got, sess.createdAt)
1798 }1810 }
@@ -1805,7 +1817,7 @@ func TestNamedSessionLifecycleExhaustionDoesNotMutateSessionState(t *testing.T)
1805 d.nextID = 171817 d.nextID = 17
1806 d.stopped["retained"] = stoppedSession{name: "retained", cwd: "/tmp", createdAt: 9}1818 d.stopped["retained"] = stoppedSession{name: "retained", cwd: "/tmp", createdAt: 9}
1807 1819
1808- sess, err := d.createSessionLocked("new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)1820+ sess, err := createSessionForTest(d, "new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
1809 1821
1810 require.ErrorContains(t, err, "lifecycle identities exhausted")1822 require.ErrorContains(t, err, "lifecycle identities exhausted")
1811 require.Nil(t, sess)1823 require.Nil(t, sess)
@@ -1855,18 +1867,18 @@ func TestCatalogueRecordsConstructExpectedSessionRegistry(t *testing.T) {
1855 1867
1856 func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) {1868 func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) {
1857 store, _ := newMockStore(t)1869 store, _ := newMockStore(t)
1858- seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))1870+ seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store))
1859- require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh}))1871+ require.NoError(t, testPersister(t, seed).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh}))
1860 1872
1861 p, release := newBlockingPTY(t)1873 p, release := newBlockingPTY(t)
1862 defer release()1874 defer release()
1863 clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}}1875 clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}}
1864 // Constructing with persistence must establish the lifecycle high-water mark.1876 // Constructing with persistence must establish the lifecycle high-water mark.
1865- d := New(newFactory(t, p), clock, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))1877+ d := New(newFactory(t, p), clock, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store))
1866 d.serveCtx, d.serveCancel = context.WithCancel(context.Background())1878 d.serveCtx, d.serveCancel = context.WithCancel(context.Background())
1867 t.Cleanup(d.serveCancel)1879 t.Cleanup(d.serveCancel)
1868 1880
1869- sess, err := d.createSessionLocked("new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)1881+ sess, err := createSessionForTest(d, "new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv)
1870 require.NoError(t, err)1882 require.NoError(t, err)
1871 require.Equal(t, int64(901), sess.createdAt)1883 require.Equal(t, int64(901), sess.createdAt)
1872 }1884 }
@@ -1878,7 +1890,7 @@ func TestResumingStoppedSessionPreservesLifecycleIdentityInPersistence(t *testin
1878 defer releaseTarget()1890 defer releaseTarget()
1879 d, from, ac, _ := newManualSessionWithPTYs(t, fromPTY)1891 d, from, ac, _ := newManualSessionWithPTYs(t, fromPTY)
1880 store, state := newMockStore(t)1892 store, state := newMockStore(t)
1881- WithStore(store)(d)1893+ WithStore(t, store)(d)
1882 d.ptys = newFactory(t, targetPTY)1894 d.ptys = newFactory(t, targetPTY)
1883 d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp", createdAt: 77}1895 d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp", createdAt: 77}
1884 1896
M internal/usecase/daemon/snapshot_incremental.go +1 -2
@@ -1,7 +1,6 @@
11 package daemon
22
33 import (
4- "crypto/sha256"
54 "fmt"
65
76 "github.com/bnema/vev/internal/domain"
@@ -162,7 +161,7 @@ func (d *Daemon) incrementalPublication(capture *snapshotCapture) (ports.Snapsho
162161 if err != nil {
163162 return ports.SnapshotPublication{}, err
164163 }
165- capture.checkpoint = domain.CheckpointRef{Generation: capture.generation, ManifestDigest: sha256.Sum256(encoded)}
164+ capture.checkpoint = domain.CheckpointRef{Generation: capture.generation, ManifestDigest: snapcodec.ManifestDigest(encoded)}
166165 return ports.SnapshotPublication{IncarnationID: capture.incarnation, Name: capture.name, Generation: capture.generation, ParentCheckpoint: capture.parentCheckpoint, Manifest: encoded, Objects: objects}, nil
167166 }
168167
@@ -1,7 +1,6 @@
1 package daemon1 package daemon
2 2
3 import (3 import (
4- "crypto/sha256"
5 "fmt"4 "fmt"
6 5
7 "github.com/bnema/vev/internal/domain"6 "github.com/bnema/vev/internal/domain"
@@ -162,7 +161,7 @@ func (d *Daemon) incrementalPublication(capture *snapshotCapture) (ports.Snapsho
162 if err != nil {161 if err != nil {
163 return ports.SnapshotPublication{}, err162 return ports.SnapshotPublication{}, err
164 }163 }
165- capture.checkpoint = domain.CheckpointRef{Generation: capture.generation, ManifestDigest: sha256.Sum256(encoded)}164+ capture.checkpoint = domain.CheckpointRef{Generation: capture.generation, ManifestDigest: snapcodec.ManifestDigest(encoded)}
166 return ports.SnapshotPublication{IncarnationID: capture.incarnation, Name: capture.name, Generation: capture.generation, ParentCheckpoint: capture.parentCheckpoint, Manifest: encoded, Objects: objects}, nil165 return ports.SnapshotPublication{IncarnationID: capture.incarnation, Name: capture.name, Generation: capture.generation, ParentCheckpoint: capture.parentCheckpoint, Manifest: encoded, Objects: objects}, nil
167 }166 }
168 167
M internal/usecase/daemon/snapshot_incremental_test.go +1 -2
@@ -1,7 +1,6 @@
11 package daemon
22
33 import (
4- "crypto/sha256"
54 "testing"
65 "time"
76
@@ -131,7 +130,7 @@ func TestIncrementalPublicationReusesSealedChunkObject(t *testing.T) {
131130 if err != nil {
132131 t.Fatal(err)
133132 }
134- firstRef := domain.CheckpointRef{Generation: first.Generation, ManifestDigest: sha256.Sum256(first.Manifest)}
133+ firstRef := domain.CheckpointRef{Generation: first.Generation, ManifestDigest: snapcodec.ManifestDigest(first.Manifest)}
135134 capture.generation = 2
136135 capture.parentCheckpoint = &firstRef
137136 second, err := d.incrementalPublication(capture)
@@ -1,7 +1,6 @@
1 package daemon1 package daemon
2 2
3 import (3 import (
4- "crypto/sha256"
5 "testing"4 "testing"
6 "time"5 "time"
7 6
@@ -131,7 +130,7 @@ func TestIncrementalPublicationReusesSealedChunkObject(t *testing.T) {
131 if err != nil {130 if err != nil {
132 t.Fatal(err)131 t.Fatal(err)
133 }132 }
134- firstRef := domain.CheckpointRef{Generation: first.Generation, ManifestDigest: sha256.Sum256(first.Manifest)}133+ firstRef := domain.CheckpointRef{Generation: first.Generation, ManifestDigest: snapcodec.ManifestDigest(first.Manifest)}
135 capture.generation = 2134 capture.generation = 2
136 capture.parentCheckpoint = &firstRef135 capture.parentCheckpoint = &firstRef
137 second, err := d.incrementalPublication(capture)136 second, err := d.incrementalPublication(capture)
M internal/usecase/daemon/snapshot_restore.go +1 -1
@@ -59,8 +59,8 @@ func (d *Daemon) restoreSnapshots(ctx context.Context) {
5959 if d == nil {
6060 return
6161 }
62- defer d.closeRestoreDone()
6362 defer d.logStartupRecoveryCounts(0)
63+ defer d.closeRestoreDone()
6464 if ns := d.noticeStore; ns != nil {
6565 claimed, err := ns.Claim()
6666 if err != nil {
@@ -59,8 +59,8 @@ func (d *Daemon) restoreSnapshots(ctx context.Context) {
59 if d == nil {59 if d == nil {
60 return60 return
61 }61 }
62- defer d.closeRestoreDone()
63 defer d.logStartupRecoveryCounts(0)62 defer d.logStartupRecoveryCounts(0)
63+ defer d.closeRestoreDone()
64 if ns := d.noticeStore; ns != nil {64 if ns := d.noticeStore; ns != nil {
65 claimed, err := ns.Claim()65 claimed, err := ns.Claim()
66 if err != nil {66 if err != nil {
M internal/usecase/daemon/snapshot_restore_fixture_test.go +4 -4
@@ -20,18 +20,18 @@ func acceptancePublication(snapshot snapcodec.Session) (ports.SnapshotPublicatio
2020 for i, payload := range pane.SealedChunks {
2121 object, err := snapcodec.MarshalObject(snapcodec.HistoryChunk, payload)
2222 if err != nil {
23- return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q sealed chunk %d: marshal object: %w", tab.StableID, pane.ID, i, err)
23+ return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q sealed chunk %d: marshal object: %w", tab.StableID, pane.ID, i, err)
2424 }
2525 outPane.Sealed = append(outPane.Sealed, objectRef(snapcodec.HistoryChunk, object))
2626 objects = append(objects, object)
2727 }
2828 tail, err := snapcodec.MarshalObject(snapcodec.HistoryTail, pane.Tail)
2929 if err != nil {
30- return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q tail: marshal object: %w", tab.StableID, pane.ID, err)
30+ return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q tail: marshal object: %w", tab.StableID, pane.ID, err)
3131 }
3232 visible, err := snapcodec.MarshalObject(snapcodec.Visible, pane.Visible)
3333 if err != nil {
34- return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q visible: marshal object: %w", tab.StableID, pane.ID, err)
34+ return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q visible: marshal object: %w", tab.StableID, pane.ID, err)
3535 }
3636 outPane.Tail, outPane.Visible = objectRef(snapcodec.HistoryTail, tail), objectRef(snapcodec.Visible, visible)
3737 objects = append(objects, tail, visible)
@@ -41,7 +41,7 @@ func acceptancePublication(snapshot snapcodec.Session) (ports.SnapshotPublicatio
4141 }
4242 encoded, err := snapcodec.MarshalManifest(manifest)
4343 if err != nil {
44- return ports.SnapshotPublication{}, fmt.Errorf("snapshot: marshal legacy import manifest: %w", err)
44+ return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: marshal manifest: %w", err)
4545 }
4646 return ports.SnapshotPublication{IncarnationID: incarnation, Name: snapshot.Name, Generation: 1, Manifest: encoded, Objects: objects}, nil
4747 }
@@ -20,18 +20,18 @@ func acceptancePublication(snapshot snapcodec.Session) (ports.SnapshotPublicatio
20 for i, payload := range pane.SealedChunks {20 for i, payload := range pane.SealedChunks {
21 object, err := snapcodec.MarshalObject(snapcodec.HistoryChunk, payload)21 object, err := snapcodec.MarshalObject(snapcodec.HistoryChunk, payload)
22 if err != nil {22 if err != nil {
23- return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q sealed chunk %d: marshal object: %w", tab.StableID, pane.ID, i, err)23+ return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q sealed chunk %d: marshal object: %w", tab.StableID, pane.ID, i, err)
24 }24 }
25 outPane.Sealed = append(outPane.Sealed, objectRef(snapcodec.HistoryChunk, object))25 outPane.Sealed = append(outPane.Sealed, objectRef(snapcodec.HistoryChunk, object))
26 objects = append(objects, object)26 objects = append(objects, object)
27 }27 }
28 tail, err := snapcodec.MarshalObject(snapcodec.HistoryTail, pane.Tail)28 tail, err := snapcodec.MarshalObject(snapcodec.HistoryTail, pane.Tail)
29 if err != nil {29 if err != nil {
30- return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q tail: marshal object: %w", tab.StableID, pane.ID, err)30+ return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q tail: marshal object: %w", tab.StableID, pane.ID, err)
31 }31 }
32 visible, err := snapcodec.MarshalObject(snapcodec.Visible, pane.Visible)32 visible, err := snapcodec.MarshalObject(snapcodec.Visible, pane.Visible)
33 if err != nil {33 if err != nil {
34- return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q visible: marshal object: %w", tab.StableID, pane.ID, err)34+ return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q visible: marshal object: %w", tab.StableID, pane.ID, err)
35 }35 }
36 outPane.Tail, outPane.Visible = objectRef(snapcodec.HistoryTail, tail), objectRef(snapcodec.Visible, visible)36 outPane.Tail, outPane.Visible = objectRef(snapcodec.HistoryTail, tail), objectRef(snapcodec.Visible, visible)
37 objects = append(objects, tail, visible)37 objects = append(objects, tail, visible)
@@ -41,7 +41,7 @@ func acceptancePublication(snapshot snapcodec.Session) (ports.SnapshotPublicatio
41 }41 }
42 encoded, err := snapcodec.MarshalManifest(manifest)42 encoded, err := snapcodec.MarshalManifest(manifest)
43 if err != nil {43 if err != nil {
44- return ports.SnapshotPublication{}, fmt.Errorf("snapshot: marshal legacy import manifest: %w", err)44+ return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: marshal manifest: %w", err)
45 }45 }
46 return ports.SnapshotPublication{IncarnationID: incarnation, Name: snapshot.Name, Generation: 1, Manifest: encoded, Objects: objects}, nil46 return ports.SnapshotPublication{IncarnationID: incarnation, Name: snapshot.Name, Generation: 1, Manifest: encoded, Objects: objects}, nil
47 }47 }
M internal/usecase/daemon/snapshot_restore_incremental.go +50 -4
@@ -5,7 +5,9 @@ import (
55 "crypto/sha256"
66 "errors"
77 "fmt"
8+ "net"
89 "sync"
10+ "syscall"
911
1012 "github.com/bnema/vev/internal/domain"
1113 "github.com/bnema/vev/internal/ports"
@@ -14,6 +16,8 @@ import (
1416
1517 const catalogueRestoreConcurrency = 8
1618
19+var errRetryableRestoreLoad = errors.New("snapshot: retryable checkpoint load")
20+
1721 func initialRuntimeRecoveryState(record domain.CatalogueRecord) (runtimeRecoveryState, chan struct{}) {
1822 done := make(chan struct{})
1923 var state runtimeRecoveryState
@@ -131,7 +135,16 @@ func (d *Daemon) setStoppedRecovery(record domain.CatalogueRecord, state runtime
131135 }
132136
133137 func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) {
134- if restoreErr != nil {
138+ d.finishRecordRestoreAttempt(record, restoreErr, done, false)
139+}
140+
141+func (d *Daemon) finishExplicitRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) {
142+ d.finishRecordRestoreAttempt(record, restoreErr, done, true)
143+}
144+
145+func (d *Daemon) finishRecordRestoreAttempt(record domain.CatalogueRecord, restoreErr error, done chan struct{}, resetRetryable bool) {
146+ retryable := errors.Is(restoreErr, errRetryableRestoreLoad)
147+ if restoreErr != nil && !retryable {
135148 catalogueReadable := true
136149 if d.catalogue != nil {
137150 current, ok, readErr := d.catalogue.Record(record.Name)
@@ -157,7 +170,8 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e
157170 d.mu.Lock()
158171 entry, ok := d.stopped[record.Name]
159172 if ok {
160- if restoreErr != nil {
173+ switch {
174+ case restoreErr != nil && !retryable:
161175 entry.record = record
162176 entry.state = runtimeDegraded
163177 entry.cwd = record.Cwd
@@ -166,11 +180,14 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e
166180 entry.lastUsedSeq = record.LastUsedSeq
167181 entry.tabNames = append([]string(nil), record.TabNames...)
168182 d.stopped[record.Name] = entry
183+ case retryable && resetRetryable && entry.state == runtimeRestoring && entry.restoreDone == done:
184+ entry.state = runtimeDegraded
185+ d.stopped[record.Name] = entry
169186 }
170- closeRuntimeRestoreDoneLocked(done)
171187 }
188+ closeRuntimeRestoreDoneLocked(done)
172189 d.mu.Unlock()
173- if restoreErr != nil {
190+ if restoreErr != nil && !retryable {
174191 reasonCode := "checkpoint-invalid"
175192 if errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded) {
176193 reasonCode = "restore-interrupted"
@@ -210,6 +227,7 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor
210227 var selectedSnapshot snapcodec.Session
211228 selectedIndex := -1
212229 selectedFallback := false
230+ var retryableLoadErr error
213231 for i, candidate := range candidates {
214232 fallback := record.Committed == nil || i > 0
215233 generation, err := d.snapshotRepository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, candidate)
@@ -217,6 +235,9 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor
217235 if ctxErr := ctx.Err(); ctxErr != nil {
218236 return ctxErr
219237 }
238+ if isRetryableRestoreLoadError(err) {
239+ retryableLoadErr = err
240+ }
220241 d.logRejectedRestoreCandidate(record, candidate, fallback, "load-failed", err)
221242 continue
222243 }
@@ -241,6 +262,9 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor
241262 break
242263 }
243264 if selectedIndex < 0 {
265+ if retryableLoadErr != nil {
266+ return fmt.Errorf("%w: %v", errRetryableRestoreLoad, retryableLoadErr)
267+ }
244268 return errors.New("snapshot: no catalogue checkpoint validated")
245269 }
246270
@@ -276,6 +300,28 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor
276300 return nil
277301 }
278302
303+func isRetryableRestoreLoadError(err error) bool {
304+ if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
305+ return false
306+ }
307+ if errors.Is(err, ports.ErrBudgetExhausted) ||
308+ errors.Is(err, syscall.ENOSPC) ||
309+ errors.Is(err, syscall.ENOMEM) ||
310+ errors.Is(err, syscall.EMFILE) ||
311+ errors.Is(err, syscall.ENFILE) ||
312+ errors.Is(err, syscall.EDQUOT) {
313+ return true
314+ }
315+ if netErr, ok := errors.AsType[net.Error](err); ok && netErr.Timeout() {
316+ return true
317+ }
318+ temporary, ok := errors.AsType[interface {
319+ error
320+ Temporary() bool
321+ }](err)
322+ return ok && temporary.Temporary()
323+}
324+
279325 func (d *Daemon) logRejectedRestoreCandidate(record domain.CatalogueRecord, candidate domain.CheckpointRef, fallback bool, reason string, err error) {
280326 kind := "committed"
281327 if fallback {
@@ -5,7 +5,9 @@ import (
5 "crypto/sha256"5 "crypto/sha256"
6 "errors"6 "errors"
7 "fmt"7 "fmt"
8+ "net"
8 "sync"9 "sync"
10+ "syscall"
9 11
10 "github.com/bnema/vev/internal/domain"12 "github.com/bnema/vev/internal/domain"
11 "github.com/bnema/vev/internal/ports"13 "github.com/bnema/vev/internal/ports"
@@ -14,6 +16,8 @@ import (
14 16
15 const catalogueRestoreConcurrency = 817 const catalogueRestoreConcurrency = 8
16 18
19+var errRetryableRestoreLoad = errors.New("snapshot: retryable checkpoint load")
20+
17 func initialRuntimeRecoveryState(record domain.CatalogueRecord) (runtimeRecoveryState, chan struct{}) {21 func initialRuntimeRecoveryState(record domain.CatalogueRecord) (runtimeRecoveryState, chan struct{}) {
18 done := make(chan struct{})22 done := make(chan struct{})
19 var state runtimeRecoveryState23 var state runtimeRecoveryState
@@ -131,7 +135,16 @@ func (d *Daemon) setStoppedRecovery(record domain.CatalogueRecord, state runtime
131 }135 }
132 136
133 func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) {137 func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) {
134- if restoreErr != nil {138+ d.finishRecordRestoreAttempt(record, restoreErr, done, false)
139+}
140+
141+func (d *Daemon) finishExplicitRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) {
142+ d.finishRecordRestoreAttempt(record, restoreErr, done, true)
143+}
144+
145+func (d *Daemon) finishRecordRestoreAttempt(record domain.CatalogueRecord, restoreErr error, done chan struct{}, resetRetryable bool) {
146+ retryable := errors.Is(restoreErr, errRetryableRestoreLoad)
147+ if restoreErr != nil && !retryable {
135 catalogueReadable := true148 catalogueReadable := true
136 if d.catalogue != nil {149 if d.catalogue != nil {
137 current, ok, readErr := d.catalogue.Record(record.Name)150 current, ok, readErr := d.catalogue.Record(record.Name)
@@ -157,7 +170,8 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e
157 d.mu.Lock()170 d.mu.Lock()
158 entry, ok := d.stopped[record.Name]171 entry, ok := d.stopped[record.Name]
159 if ok {172 if ok {
160- if restoreErr != nil {173+ switch {
174+ case restoreErr != nil && !retryable:
161 entry.record = record175 entry.record = record
162 entry.state = runtimeDegraded176 entry.state = runtimeDegraded
163 entry.cwd = record.Cwd177 entry.cwd = record.Cwd
@@ -166,11 +180,14 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e
166 entry.lastUsedSeq = record.LastUsedSeq180 entry.lastUsedSeq = record.LastUsedSeq
167 entry.tabNames = append([]string(nil), record.TabNames...)181 entry.tabNames = append([]string(nil), record.TabNames...)
168 d.stopped[record.Name] = entry182 d.stopped[record.Name] = entry
183+ case retryable && resetRetryable && entry.state == runtimeRestoring && entry.restoreDone == done:
184+ entry.state = runtimeDegraded
185+ d.stopped[record.Name] = entry
169 }186 }
170- closeRuntimeRestoreDoneLocked(done)
171 }187 }
188+ closeRuntimeRestoreDoneLocked(done)
172 d.mu.Unlock()189 d.mu.Unlock()
173- if restoreErr != nil {190+ if restoreErr != nil && !retryable {
174 reasonCode := "checkpoint-invalid"191 reasonCode := "checkpoint-invalid"
175 if errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded) {192 if errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded) {
176 reasonCode = "restore-interrupted"193 reasonCode = "restore-interrupted"
@@ -210,6 +227,7 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor
210 var selectedSnapshot snapcodec.Session227 var selectedSnapshot snapcodec.Session
211 selectedIndex := -1228 selectedIndex := -1
212 selectedFallback := false229 selectedFallback := false
230+ var retryableLoadErr error
213 for i, candidate := range candidates {231 for i, candidate := range candidates {
214 fallback := record.Committed == nil || i > 0232 fallback := record.Committed == nil || i > 0
215 generation, err := d.snapshotRepository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, candidate)233 generation, err := d.snapshotRepository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, candidate)
@@ -217,6 +235,9 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor
217 if ctxErr := ctx.Err(); ctxErr != nil {235 if ctxErr := ctx.Err(); ctxErr != nil {
218 return ctxErr236 return ctxErr
219 }237 }
238+ if isRetryableRestoreLoadError(err) {
239+ retryableLoadErr = err
240+ }
220 d.logRejectedRestoreCandidate(record, candidate, fallback, "load-failed", err)241 d.logRejectedRestoreCandidate(record, candidate, fallback, "load-failed", err)
221 continue242 continue
222 }243 }
@@ -241,6 +262,9 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor
241 break262 break
242 }263 }
243 if selectedIndex < 0 {264 if selectedIndex < 0 {
265+ if retryableLoadErr != nil {
266+ return fmt.Errorf("%w: %v", errRetryableRestoreLoad, retryableLoadErr)
267+ }
244 return errors.New("snapshot: no catalogue checkpoint validated")268 return errors.New("snapshot: no catalogue checkpoint validated")
245 }269 }
246 270
@@ -276,6 +300,28 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor
276 return nil300 return nil
277 }301 }
278 302
303+func isRetryableRestoreLoadError(err error) bool {
304+ if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
305+ return false
306+ }
307+ if errors.Is(err, ports.ErrBudgetExhausted) ||
308+ errors.Is(err, syscall.ENOSPC) ||
309+ errors.Is(err, syscall.ENOMEM) ||
310+ errors.Is(err, syscall.EMFILE) ||
311+ errors.Is(err, syscall.ENFILE) ||
312+ errors.Is(err, syscall.EDQUOT) {
313+ return true
314+ }
315+ if netErr, ok := errors.AsType[net.Error](err); ok && netErr.Timeout() {
316+ return true
317+ }
318+ temporary, ok := errors.AsType[interface {
319+ error
320+ Temporary() bool
321+ }](err)
322+ return ok && temporary.Temporary()
323+}
324+
279 func (d *Daemon) logRejectedRestoreCandidate(record domain.CatalogueRecord, candidate domain.CheckpointRef, fallback bool, reason string, err error) {325 func (d *Daemon) logRejectedRestoreCandidate(record domain.CatalogueRecord, candidate domain.CheckpointRef, fallback bool, reason string, err error) {
280 kind := "committed"326 kind := "committed"
281 if fallback {327 if fallback {
M internal/usecase/daemon/snapshot_restore_test.go +5 -3
@@ -277,7 +277,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) {
277277 coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
278278 WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d)
279279 WithSnapshotRepository(repository)(d)
280- WithCheckpointCoordinator(coordinator)(d)
280+ WithRecoveryCoordinator(coordinator)(d)
281281 t.Cleanup(func() { release(); d.sessWg.Wait() })
282282
283283 d.restoreIncrementalSnapshots(context.Background())
@@ -290,7 +290,8 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) {
290290 updates := catalogue.MetadataUpdates()
291291 require.Len(t, updates, 1, "the authoritative Catalogue port must receive restored-session metadata updates")
292292 require.Equal(t, record.IncarnationID, updates[0].IncarnationID)
293- updated, ok, _ := catalogue.Record(record.Name)
293+ updated, ok, err := catalogue.Record(record.Name)
294+ require.NoError(t, err)
294295 require.True(t, ok)
295296 require.Equal(t, record.IncarnationID, updated.IncarnationID)
296297 require.Equal(t, record.RecoveryState, updated.RecoveryState)
@@ -309,7 +310,8 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) {
309310 require.Empty(t, failure)
310311 require.Len(t, repository.publishes, 1)
311312 require.Equal(t, record.Committed, repository.publishes[0].ParentCheckpoint)
312- published, ok, _ := catalogue.Record(record.Name)
313+ published, ok, err := catalogue.Record(record.Name)
314+ require.NoError(t, err)
313315 require.True(t, ok)
314316 require.Equal(t, uint64(10), published.Committed.Generation)
315317 require.Equal(t, committed, *published.Fallbacks[0])
@@ -277,7 +277,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) {
277 coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)277 coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
278 WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d)278 WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d)
279 WithSnapshotRepository(repository)(d)279 WithSnapshotRepository(repository)(d)
280- WithCheckpointCoordinator(coordinator)(d)280+ WithRecoveryCoordinator(coordinator)(d)
281 t.Cleanup(func() { release(); d.sessWg.Wait() })281 t.Cleanup(func() { release(); d.sessWg.Wait() })
282 282
283 d.restoreIncrementalSnapshots(context.Background())283 d.restoreIncrementalSnapshots(context.Background())
@@ -290,7 +290,8 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) {
290 updates := catalogue.MetadataUpdates()290 updates := catalogue.MetadataUpdates()
291 require.Len(t, updates, 1, "the authoritative Catalogue port must receive restored-session metadata updates")291 require.Len(t, updates, 1, "the authoritative Catalogue port must receive restored-session metadata updates")
292 require.Equal(t, record.IncarnationID, updates[0].IncarnationID)292 require.Equal(t, record.IncarnationID, updates[0].IncarnationID)
293- updated, ok, _ := catalogue.Record(record.Name)293+ updated, ok, err := catalogue.Record(record.Name)
294+ require.NoError(t, err)
294 require.True(t, ok)295 require.True(t, ok)
295 require.Equal(t, record.IncarnationID, updated.IncarnationID)296 require.Equal(t, record.IncarnationID, updated.IncarnationID)
296 require.Equal(t, record.RecoveryState, updated.RecoveryState)297 require.Equal(t, record.RecoveryState, updated.RecoveryState)
@@ -309,7 +310,8 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) {
309 require.Empty(t, failure)310 require.Empty(t, failure)
310 require.Len(t, repository.publishes, 1)311 require.Len(t, repository.publishes, 1)
311 require.Equal(t, record.Committed, repository.publishes[0].ParentCheckpoint)312 require.Equal(t, record.Committed, repository.publishes[0].ParentCheckpoint)
312- published, ok, _ := catalogue.Record(record.Name)313+ published, ok, err := catalogue.Record(record.Name)
314+ require.NoError(t, err)
313 require.True(t, ok)315 require.True(t, ok)
314 require.Equal(t, uint64(10), published.Committed.Generation)316 require.Equal(t, uint64(10), published.Committed.Generation)
315 require.Equal(t, committed, *published.Fallbacks[0])317 require.Equal(t, committed, *published.Fallbacks[0])
M internal/usecase/daemon/snapshot_scheduler_test.go +0 -1
@@ -819,7 +819,6 @@ func TestSnapshotSchedulerImmediateEligibilityAndStaleCapturesRemainDirty(t *tes
819819 now := time.Unix(100, 0)
820820 clock := newDeterministicSnapshotClock(now)
821821 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)
822- WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t))(d)
823822
824823 t.Run("no prior attempt is immediately eligible", func(t *testing.T) {
825824 published := make(chan ports.SnapshotPublication, 1)
@@ -819,7 +819,6 @@ func TestSnapshotSchedulerImmediateEligibilityAndStaleCapturesRemainDirty(t *tes
819 now := time.Unix(100, 0)819 now := time.Unix(100, 0)
820 clock := newDeterministicSnapshotClock(now)820 clock := newDeterministicSnapshotClock(now)
821 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)821 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)
822- WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t))(d)
823 822
824 t.Run("no prior attempt is immediately eligible", func(t *testing.T) {823 t.Run("no prior attempt is immediately eligible", func(t *testing.T) {
825 published := make(chan ports.SnapshotPublication, 1)824 published := make(chan ports.SnapshotPublication, 1)
M internal/usecase/daemon/status_test.go +2 -2
@@ -1066,9 +1066,9 @@ func TestBarStateForMRURestoredStoppedSessionsUsePersistedOrder(t *testing.T) {
10661066 cur := &session{id: "cur", name: "current", tabs: []*tab{{}}}
10671067 d.sessions[cur.id] = cur
10681068
1069- _, err := d.createSessionLocked("zeta", false, "/tmp/zeta", sz, terminalEnv{}, d.baseEnv)
1069+ _, err := createSessionForTest(d, "zeta", false, "/tmp/zeta", sz, terminalEnv{}, d.baseEnv)
10701070 require.NoError(t, err)
1071- _, err = d.createSessionLocked("alpha", false, "/tmp/alpha", sz, terminalEnv{}, d.baseEnv)
1071+ _, err = createSessionForTest(d, "alpha", false, "/tmp/alpha", sz, terminalEnv{}, d.baseEnv)
10721072 require.NoError(t, err)
10731073
10741074 state := d.barStateFor(cur, "")
@@ -1066,9 +1066,9 @@ func TestBarStateForMRURestoredStoppedSessionsUsePersistedOrder(t *testing.T) {
1066 cur := &session{id: "cur", name: "current", tabs: []*tab{{}}}1066 cur := &session{id: "cur", name: "current", tabs: []*tab{{}}}
1067 d.sessions[cur.id] = cur1067 d.sessions[cur.id] = cur
1068 1068
1069- _, err := d.createSessionLocked("zeta", false, "/tmp/zeta", sz, terminalEnv{}, d.baseEnv)1069+ _, err := createSessionForTest(d, "zeta", false, "/tmp/zeta", sz, terminalEnv{}, d.baseEnv)
1070 require.NoError(t, err)1070 require.NoError(t, err)
1071- _, err = d.createSessionLocked("alpha", false, "/tmp/alpha", sz, terminalEnv{}, d.baseEnv)1071+ _, err = createSessionForTest(d, "alpha", false, "/tmp/alpha", sz, terminalEnv{}, d.baseEnv)
1072 require.NoError(t, err)1072 require.NoError(t, err)
1073 1073
1074 state := d.barStateFor(cur, "")1074 state := d.barStateFor(cur, "")
M internal/usecase/recovery/checkpoint.go +4 -1
@@ -103,7 +103,10 @@ func (c *Coordinator) promoteFallbackLocked(ctx context.Context, name string, re
103103 candidates = append(candidates, record.Committed)
104104 candidates = append(candidates, record.Fallbacks[:]...)
105105 for _, candidate := range candidates {
106- if candidate == nil || *candidate == ref || candidate.Generation >= ref.Generation || len(alternatives) == 2 {
106+ if len(alternatives) == 2 {
107+ break
108+ }
109+ if candidate == nil || *candidate == ref || candidate.Generation >= ref.Generation {
107110 continue
108111 }
109112 if err := c.validateCheckpoint(ctx, record, *candidate); err == nil {
@@ -103,7 +103,10 @@ func (c *Coordinator) promoteFallbackLocked(ctx context.Context, name string, re
103 candidates = append(candidates, record.Committed)103 candidates = append(candidates, record.Committed)
104 candidates = append(candidates, record.Fallbacks[:]...)104 candidates = append(candidates, record.Fallbacks[:]...)
105 for _, candidate := range candidates {105 for _, candidate := range candidates {
106- if candidate == nil || *candidate == ref || candidate.Generation >= ref.Generation || len(alternatives) == 2 {106+ if len(alternatives) == 2 {
107+ break
108+ }
109+ if candidate == nil || *candidate == ref || candidate.Generation >= ref.Generation {
107 continue110 continue
108 }111 }
109 if err := c.validateCheckpoint(ctx, record, *candidate); err == nil {112 if err := c.validateCheckpoint(ctx, record, *candidate); err == nil {
M internal/usecase/recovery/coordinator.go +5 -1
@@ -123,7 +123,7 @@ type RecoveryConflict struct {
123123 Err error
124124 }
125125
126-// Conflicts returns the session-scoped items fenced by the last Recover calls.
126+// Conflicts returns the session-scoped items fenced by the most recent Recover call.
127127 func (c *Coordinator) Conflicts() []RecoveryConflict {
128128 if c == nil {
129129 return nil
@@ -264,6 +264,10 @@ func (c *Coordinator) Recover(ctx context.Context) error {
264264 if c == nil || c.catalogue == nil || c.repository == nil || c.journal == nil || c.locks == nil {
265265 return errors.New("recovery: incomplete coordinator dependencies")
266266 }
267+ c.conflictMu.Lock()
268+ c.conflicts = nil
269+ c.conflictMu.Unlock()
270+
267271 intents, err := c.journal.ListDiscards(ctx)
268272 if err != nil {
269273 return err
@@ -123,7 +123,7 @@ type RecoveryConflict struct {
123 Err error123 Err error
124 }124 }
125 125
126-// Conflicts returns the session-scoped items fenced by the last Recover calls.126+// Conflicts returns the session-scoped items fenced by the most recent Recover call.
127 func (c *Coordinator) Conflicts() []RecoveryConflict {127 func (c *Coordinator) Conflicts() []RecoveryConflict {
128 if c == nil {128 if c == nil {
129 return nil129 return nil
@@ -264,6 +264,10 @@ func (c *Coordinator) Recover(ctx context.Context) error {
264 if c == nil || c.catalogue == nil || c.repository == nil || c.journal == nil || c.locks == nil {264 if c == nil || c.catalogue == nil || c.repository == nil || c.journal == nil || c.locks == nil {
265 return errors.New("recovery: incomplete coordinator dependencies")265 return errors.New("recovery: incomplete coordinator dependencies")
266 }266 }
267+ c.conflictMu.Lock()
268+ c.conflicts = nil
269+ c.conflictMu.Unlock()
270+
267 intents, err := c.journal.ListDiscards(ctx)271 intents, err := c.journal.ListDiscards(ctx)
268 if err != nil {272 if err != nil {
269 return err273 return err
M internal/usecase/recovery/reconcile.go +6 -6
@@ -90,15 +90,15 @@ func classifyFinding(record domain.CatalogueRecord, finding ports.ReconcileFindi
9090 decision.ReasonCode = "incarnation-mismatch"
9191 return decision
9292 }
93- if finding.Status == ports.ReconcileValidated && record.Committed != nil && (finding.Kind != ports.ReconcileForwardOrphan || finding.Candidate.Ref.Generation <= record.Committed.Generation) {
93+ if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil {
9494 decision.Kind = ports.ReconcileKeepQuarantined
95- decision.ReasonCode = "not-forward"
96- decision.RetentionResolved = true
95+ decision.ReasonCode = "invalid-candidate"
9796 return decision
9897 }
99- if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil {
98+ if finding.Status == ports.ReconcileValidated && finding.Candidate.Ref.Generation <= record.Committed.Generation {
10099 decision.Kind = ports.ReconcileKeepQuarantined
101- decision.ReasonCode = "invalid-candidate"
100+ decision.ReasonCode = "not-forward"
101+ decision.RetentionResolved = true
102102 return decision
103103 }
104104 if finding.Status != ports.ReconcileValidated || !validForwardChain(*record.Committed, finding.Candidate, finding.AncestorChain) {
@@ -157,7 +157,7 @@ func (c *Coordinator) PublishReconciledCheckpoint(ctx context.Context, name stri
157157 if !ok {
158158 return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound
159159 }
160- if candidate.Name != name || candidate.IncarnationID != record.IncarnationID || record.Committed == nil || !validForwardChain(*record.Committed, candidate, ancestors) {
160+ if record.RecoveryState != domain.RecoveryHealthy || candidate.Name != name || candidate.IncarnationID != record.IncarnationID || record.Committed == nil || !validForwardChain(*record.Committed, candidate, ancestors) {
161161 return domain.CatalogueRecord{}, ErrCheckpointConflict
162162 }
163163 next := shiftedCheckpoint(record, candidate.Ref)
@@ -90,15 +90,15 @@ func classifyFinding(record domain.CatalogueRecord, finding ports.ReconcileFindi
90 decision.ReasonCode = "incarnation-mismatch"90 decision.ReasonCode = "incarnation-mismatch"
91 return decision91 return decision
92 }92 }
93- if finding.Status == ports.ReconcileValidated && record.Committed != nil && (finding.Kind != ports.ReconcileForwardOrphan || finding.Candidate.Ref.Generation <= record.Committed.Generation) {93+ if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil {
94 decision.Kind = ports.ReconcileKeepQuarantined94 decision.Kind = ports.ReconcileKeepQuarantined
95- decision.ReasonCode = "not-forward"95+ decision.ReasonCode = "invalid-candidate"
96- decision.RetentionResolved = true
97 return decision96 return decision
98 }97 }
99- if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil {98+ if finding.Status == ports.ReconcileValidated && finding.Candidate.Ref.Generation <= record.Committed.Generation {
100 decision.Kind = ports.ReconcileKeepQuarantined99 decision.Kind = ports.ReconcileKeepQuarantined
101- decision.ReasonCode = "invalid-candidate"100+ decision.ReasonCode = "not-forward"
101+ decision.RetentionResolved = true
102 return decision102 return decision
103 }103 }
104 if finding.Status != ports.ReconcileValidated || !validForwardChain(*record.Committed, finding.Candidate, finding.AncestorChain) {104 if finding.Status != ports.ReconcileValidated || !validForwardChain(*record.Committed, finding.Candidate, finding.AncestorChain) {
@@ -157,7 +157,7 @@ func (c *Coordinator) PublishReconciledCheckpoint(ctx context.Context, name stri
157 if !ok {157 if !ok {
158 return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound158 return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound
159 }159 }
160- if candidate.Name != name || candidate.IncarnationID != record.IncarnationID || record.Committed == nil || !validForwardChain(*record.Committed, candidate, ancestors) {160+ if record.RecoveryState != domain.RecoveryHealthy || candidate.Name != name || candidate.IncarnationID != record.IncarnationID || record.Committed == nil || !validForwardChain(*record.Committed, candidate, ancestors) {
161 return domain.CatalogueRecord{}, ErrCheckpointConflict161 return domain.CatalogueRecord{}, ErrCheckpointConflict
162 }162 }
163 next := shiftedCheckpoint(record, candidate.Ref)163 next := shiftedCheckpoint(record, candidate.Ref)
M internal/usecase/recovery/reconcile_test.go +19 -2
@@ -160,9 +160,9 @@ func TestForwardOrphanAdoption(t *testing.T) {
160160 {name: "generation reversal", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) {
161161 finding.Candidate.Ref.Generation = record.Committed.Generation
162162 }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "not-forward"},
163- {name: "not forward orphan", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) {
163+ {name: "non-orphan invalid candidate", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) {
164164 finding.Kind = ports.ReconcileInvalidCandidate
165- }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "not-forward"},
165+ }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "invalid-candidate"},
166166 } {
167167 t.Run(tc.name, func(t *testing.T) {
168168 record := healthyReconcileRecord()
@@ -191,6 +191,23 @@ func TestForwardOrphanAdoption(t *testing.T) {
191191 require.Empty(t, repository.repairs)
192192 })
193193
194+ for _, state := range []domain.RecoveryState{domain.RecoveryDegraded, domain.RecoveryDeleting} {
195+ t.Run("non-healthy record is not reconciled", func(t *testing.T) {
196+ record := healthyReconcileRecord()
197+ record.RecoveryState = state
198+ if state == domain.RecoveryDegraded {
199+ record.DegradedReason = "uncertain"
200+ } else {
201+ record.Committed = nil
202+ }
203+ candidate := forwardCandidate(healthyReconcileRecord(), 2, healthyReconcileRecord().Committed)
204+ coordinator := NewCoordinator(&checkpointCatalogue{record: record}, &reconcileRepository{}, nil, nil)
205+
206+ _, err := coordinator.PublishReconciledCheckpoint(context.Background(), record.Name, candidate, nil)
207+ require.ErrorIs(t, err, ErrCheckpointConflict)
208+ })
209+ }
210+
194211 t.Run("stale decision after concurrent commit", func(t *testing.T) {
195212 record := healthyReconcileRecord()
196213 candidate := forwardCandidate(record, 2, record.Committed)
@@ -160,9 +160,9 @@ func TestForwardOrphanAdoption(t *testing.T) {
160 {name: "generation reversal", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) {160 {name: "generation reversal", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) {
161 finding.Candidate.Ref.Generation = record.Committed.Generation161 finding.Candidate.Ref.Generation = record.Committed.Generation
162 }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "not-forward"},162 }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "not-forward"},
163- {name: "not forward orphan", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) {163+ {name: "non-orphan invalid candidate", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) {
164 finding.Kind = ports.ReconcileInvalidCandidate164 finding.Kind = ports.ReconcileInvalidCandidate
165- }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "not-forward"},165+ }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "invalid-candidate"},
166 } {166 } {
167 t.Run(tc.name, func(t *testing.T) {167 t.Run(tc.name, func(t *testing.T) {
168 record := healthyReconcileRecord()168 record := healthyReconcileRecord()
@@ -191,6 +191,23 @@ func TestForwardOrphanAdoption(t *testing.T) {
191 require.Empty(t, repository.repairs)191 require.Empty(t, repository.repairs)
192 })192 })
193 193
194+ for _, state := range []domain.RecoveryState{domain.RecoveryDegraded, domain.RecoveryDeleting} {
195+ t.Run("non-healthy record is not reconciled", func(t *testing.T) {
196+ record := healthyReconcileRecord()
197+ record.RecoveryState = state
198+ if state == domain.RecoveryDegraded {
199+ record.DegradedReason = "uncertain"
200+ } else {
201+ record.Committed = nil
202+ }
203+ candidate := forwardCandidate(healthyReconcileRecord(), 2, healthyReconcileRecord().Committed)
204+ coordinator := NewCoordinator(&checkpointCatalogue{record: record}, &reconcileRepository{}, nil, nil)
205+
206+ _, err := coordinator.PublishReconciledCheckpoint(context.Background(), record.Name, candidate, nil)
207+ require.ErrorIs(t, err, ErrCheckpointConflict)
208+ })
209+ }
210+
194 t.Run("stale decision after concurrent commit", func(t *testing.T) {211 t.Run("stale decision after concurrent commit", func(t *testing.T) {
195 record := healthyReconcileRecord()212 record := healthyReconcileRecord()
196 candidate := forwardCandidate(record, 2, record.Committed)213 candidate := forwardCandidate(record, 2, record.Committed)
M internal/usecase/recovery/transactions_test.go +8 -1
@@ -4,6 +4,7 @@ import (
44 "bytes"
55 "context"
66 "crypto/sha256"
7+ "encoding/hex"
78 "errors"
89 "sync"
910 "testing"
@@ -272,7 +273,9 @@ func TestDegradedExportIsReadOnly(t *testing.T) {
272273
273274 require.NoError(t, coordinator.Export(context.Background(), record.Name, &exported))
274275 require.NoError(t, coordinator.Export(context.Background(), record.Name, &repeated))
275- require.NotEmpty(t, exported.Bytes())
276+ golden, err := hex.DecodeString("5645565800010100000000000000000000000000000000000000000000030000000662726f6b656e000000086d616e69666573740000000201000000000000000000000000000000000000000000000000000000000000000000000566697273740200000000000000000000000000000000000000000000000000000000000000000000067365636f6e64")
277+ require.NoError(t, err)
278+ require.Equal(t, golden, exported.Bytes())
276279 require.Equal(t, exported.Bytes(), repeated.Bytes())
277280 require.Equal(t, record, catalogue.records[record.Name])
278281 require.Empty(t, catalogue.events)
@@ -314,6 +317,10 @@ func TestDiscardIncarnationConflict(t *testing.T) {
314317 require.Equal(t, old.Name, conflicts[0].Session)
315318 require.Equal(t, "discard-intent", conflicts[0].Kind)
316319 require.ErrorIs(t, conflicts[0].Err, ErrDiscardConflict)
320+
321+ delete(journal.intents, old.IncarnationID)
322+ require.NoError(t, coordinator.Recover(context.Background()))
323+ require.Empty(t, coordinator.Conflicts(), "a later recovery reports only its own conflicts")
317324 }
318325
319326 // A discard whose replacement is already durable, live, and checkpointed must
@@ -4,6 +4,7 @@ import (
4 "bytes"4 "bytes"
5 "context"5 "context"
6 "crypto/sha256"6 "crypto/sha256"
7+ "encoding/hex"
7 "errors"8 "errors"
8 "sync"9 "sync"
9 "testing"10 "testing"
@@ -272,7 +273,9 @@ func TestDegradedExportIsReadOnly(t *testing.T) {
272 273
273 require.NoError(t, coordinator.Export(context.Background(), record.Name, &exported))274 require.NoError(t, coordinator.Export(context.Background(), record.Name, &exported))
274 require.NoError(t, coordinator.Export(context.Background(), record.Name, &repeated))275 require.NoError(t, coordinator.Export(context.Background(), record.Name, &repeated))
275- require.NotEmpty(t, exported.Bytes())276+ golden, err := hex.DecodeString("5645565800010100000000000000000000000000000000000000000000030000000662726f6b656e000000086d616e69666573740000000201000000000000000000000000000000000000000000000000000000000000000000000566697273740200000000000000000000000000000000000000000000000000000000000000000000067365636f6e64")
277+ require.NoError(t, err)
278+ require.Equal(t, golden, exported.Bytes())
276 require.Equal(t, exported.Bytes(), repeated.Bytes())279 require.Equal(t, exported.Bytes(), repeated.Bytes())
277 require.Equal(t, record, catalogue.records[record.Name])280 require.Equal(t, record, catalogue.records[record.Name])
278 require.Empty(t, catalogue.events)281 require.Empty(t, catalogue.events)
@@ -314,6 +317,10 @@ func TestDiscardIncarnationConflict(t *testing.T) {
314 require.Equal(t, old.Name, conflicts[0].Session)317 require.Equal(t, old.Name, conflicts[0].Session)
315 require.Equal(t, "discard-intent", conflicts[0].Kind)318 require.Equal(t, "discard-intent", conflicts[0].Kind)
316 require.ErrorIs(t, conflicts[0].Err, ErrDiscardConflict)319 require.ErrorIs(t, conflicts[0].Err, ErrDiscardConflict)
320+
321+ delete(journal.intents, old.IncarnationID)
322+ require.NoError(t, coordinator.Recover(context.Background()))
323+ require.Empty(t, coordinator.Conflicts(), "a later recovery reports only its own conflicts")
317 }324 }
318 325
319 // A discard whose replacement is already durable, live, and checkpointed must326 // A discard whose replacement is already durable, live, and checkpointed must
M internal/usecase/snapshot/codec.go +9 -0
@@ -185,6 +185,15 @@ func writeProcess(w *payloadWriter, p *Process) error {
185185
186186 type payloadReader struct{ b []byte }
187187
188+func (r *payloadReader) getBytes(n int) ([]byte, error) {
189+ if n < 0 || len(r.b) < n {
190+ return nil, ErrShortPayload
191+ }
192+ b := r.b[:n]
193+ r.b = r.b[n:]
194+ return b, nil
195+}
196+
188197 func (r *payloadReader) getUint8() (uint8, error) {
189198 if len(r.b) < 1 {
190199 return 0, ErrShortPayload
@@ -185,6 +185,15 @@ func writeProcess(w *payloadWriter, p *Process) error {
185 185
186 type payloadReader struct{ b []byte }186 type payloadReader struct{ b []byte }
187 187
188+func (r *payloadReader) getBytes(n int) ([]byte, error) {
189+ if n < 0 || len(r.b) < n {
190+ return nil, ErrShortPayload
191+ }
192+ b := r.b[:n]
193+ r.b = r.b[n:]
194+ return b, nil
195+}
196+
188 func (r *payloadReader) getUint8() (uint8, error) {197 func (r *payloadReader) getUint8() (uint8, error) {
189 if len(r.b) < 1 {198 if len(r.b) < 1 {
190 return 0, ErrShortPayload199 return 0, ErrShortPayload
M internal/usecase/snapshot/codec_test.go +29 -0
@@ -14,6 +14,35 @@ import (
1414 "github.com/bnema/vev/pkg/vt"
1515 )
1616
17+func TestPayloadReaderGetBytes(t *testing.T) {
18+ for _, tc := range []struct {
19+ name string
20+ input []byte
21+ n int
22+ want []byte
23+ remaining []byte
24+ wantErr error
25+ }{
26+ {name: "reads and advances", input: []byte{1, 2, 3}, n: 2, want: []byte{1, 2}, remaining: []byte{3}},
27+ {name: "reads empty", input: []byte{1}, n: 0, want: []byte{}, remaining: []byte{1}},
28+ {name: "short payload does not advance", input: []byte{1}, n: 2, remaining: []byte{1}, wantErr: ErrShortPayload},
29+ } {
30+ t.Run(tc.name, func(t *testing.T) {
31+ r := payloadReader{b: append([]byte(nil), tc.input...)}
32+ got, err := r.getBytes(tc.n)
33+ if !errors.Is(err, tc.wantErr) {
34+ t.Fatalf("getBytes(%d) error = %v, want %v", tc.n, err, tc.wantErr)
35+ }
36+ if !bytes.Equal(got, tc.want) {
37+ t.Fatalf("getBytes(%d) = %v, want %v", tc.n, got, tc.want)
38+ }
39+ if !bytes.Equal(r.b, tc.remaining) {
40+ t.Fatalf("remaining bytes = %v, want %v", r.b, tc.remaining)
41+ }
42+ })
43+ }
44+}
45+
1746 func TestV3SnapshotRoundTripPreservesExactTerminalData(t *testing.T) {
1847 indexed := renderer.DefaultStyle()
1948 indexed.Bold, indexed.Inverse = true, true
@@ -14,6 +14,35 @@ import (
14 "github.com/bnema/vev/pkg/vt"14 "github.com/bnema/vev/pkg/vt"
15 )15 )
16 16
17+func TestPayloadReaderGetBytes(t *testing.T) {
18+ for _, tc := range []struct {
19+ name string
20+ input []byte
21+ n int
22+ want []byte
23+ remaining []byte
24+ wantErr error
25+ }{
26+ {name: "reads and advances", input: []byte{1, 2, 3}, n: 2, want: []byte{1, 2}, remaining: []byte{3}},
27+ {name: "reads empty", input: []byte{1}, n: 0, want: []byte{}, remaining: []byte{1}},
28+ {name: "short payload does not advance", input: []byte{1}, n: 2, remaining: []byte{1}, wantErr: ErrShortPayload},
29+ } {
30+ t.Run(tc.name, func(t *testing.T) {
31+ r := payloadReader{b: append([]byte(nil), tc.input...)}
32+ got, err := r.getBytes(tc.n)
33+ if !errors.Is(err, tc.wantErr) {
34+ t.Fatalf("getBytes(%d) error = %v, want %v", tc.n, err, tc.wantErr)
35+ }
36+ if !bytes.Equal(got, tc.want) {
37+ t.Fatalf("getBytes(%d) = %v, want %v", tc.n, got, tc.want)
38+ }
39+ if !bytes.Equal(r.b, tc.remaining) {
40+ t.Fatalf("remaining bytes = %v, want %v", r.b, tc.remaining)
41+ }
42+ })
43+ }
44+}
45+
17 func TestV3SnapshotRoundTripPreservesExactTerminalData(t *testing.T) {46 func TestV3SnapshotRoundTripPreservesExactTerminalData(t *testing.T) {
18 indexed := renderer.DefaultStyle()47 indexed := renderer.DefaultStyle()
19 indexed.Bold, indexed.Inverse = true, true48 indexed.Bold, indexed.Inverse = true, true
M internal/usecase/snapshot/manifest.go +8 -8
@@ -179,12 +179,12 @@ func decodeManifest(body []byte) (Manifest, error) {
179179 if err != nil {
180180 return Manifest{}, err
181181 }
182- if len(r.b) < len(domain.IncarnationID{}) {
183- return Manifest{}, ErrShortPayload
182+ incarnationBytes, err := r.getBytes(len(domain.IncarnationID{}))
183+ if err != nil {
184+ return Manifest{}, err
184185 }
185186 var incarnationID domain.IncarnationID
186- copy(incarnationID[:], r.b[:len(incarnationID)])
187- r.b = r.b[len(incarnationID):]
187+ copy(incarnationID[:], incarnationBytes)
188188 parentPresent, err := r.getUint8()
189189 if err != nil {
190190 return Manifest{}, err
@@ -198,11 +198,11 @@ func decodeManifest(body []byte) (Manifest, error) {
198198 if err != nil {
199199 return Manifest{}, err
200200 }
201- if len(r.b) < len(parent.ManifestDigest) {
202- return Manifest{}, ErrShortPayload
201+ digest, err := r.getBytes(len(parent.ManifestDigest))
202+ if err != nil {
203+ return Manifest{}, err
203204 }
204- copy(parent.ManifestDigest[:], r.b[:len(parent.ManifestDigest)])
205- r.b = r.b[len(parent.ManifestDigest):]
205+ copy(parent.ManifestDigest[:], digest)
206206 default:
207207 return Manifest{}, fmt.Errorf("%w: parent checkpoint presence", ErrInvalidData)
208208 }
@@ -179,12 +179,12 @@ func decodeManifest(body []byte) (Manifest, error) {
179 if err != nil {179 if err != nil {
180 return Manifest{}, err180 return Manifest{}, err
181 }181 }
182- if len(r.b) < len(domain.IncarnationID{}) {182+ incarnationBytes, err := r.getBytes(len(domain.IncarnationID{}))
183- return Manifest{}, ErrShortPayload183+ if err != nil {
184+ return Manifest{}, err
184 }185 }
185 var incarnationID domain.IncarnationID186 var incarnationID domain.IncarnationID
186- copy(incarnationID[:], r.b[:len(incarnationID)])187+ copy(incarnationID[:], incarnationBytes)
187- r.b = r.b[len(incarnationID):]
188 parentPresent, err := r.getUint8()188 parentPresent, err := r.getUint8()
189 if err != nil {189 if err != nil {
190 return Manifest{}, err190 return Manifest{}, err
@@ -198,11 +198,11 @@ func decodeManifest(body []byte) (Manifest, error) {
198 if err != nil {198 if err != nil {
199 return Manifest{}, err199 return Manifest{}, err
200 }200 }
201- if len(r.b) < len(parent.ManifestDigest) {201+ digest, err := r.getBytes(len(parent.ManifestDigest))
202- return Manifest{}, ErrShortPayload202+ if err != nil {
203+ return Manifest{}, err
203 }204 }
204- copy(parent.ManifestDigest[:], r.b[:len(parent.ManifestDigest)])205+ copy(parent.ManifestDigest[:], digest)
205- r.b = r.b[len(parent.ManifestDigest):]
206 default:206 default:
207 return Manifest{}, fmt.Errorf("%w: parent checkpoint presence", ErrInvalidData)207 return Manifest{}, fmt.Errorf("%w: parent checkpoint presence", ErrInvalidData)
208 }208 }
M internal/usecase/snapshot/manifest_preflight.go +7 -8
@@ -19,12 +19,12 @@ func preflightManifest(body []byte) error {
1919 if generation == 0 {
2020 return fmt.Errorf("%w: manifest generation", ErrInvalidData)
2121 }
22- if len(r.b) < len(domain.IncarnationID{}) {
23- return ErrShortPayload
22+ incarnationBytes, err := r.getBytes(len(domain.IncarnationID{}))
23+ if err != nil {
24+ return err
2425 }
2526 var incarnationID domain.IncarnationID
26- copy(incarnationID[:], r.b[:len(incarnationID)])
27- r.b = r.b[len(incarnationID):]
27+ copy(incarnationID[:], incarnationBytes)
2828 if incarnationID == (domain.IncarnationID{}) {
2929 return fmt.Errorf("%w: manifest incarnation", ErrInvalidData)
3030 }
@@ -39,11 +39,10 @@ func preflightManifest(body []byte) error {
3939 if err != nil {
4040 return err
4141 }
42- if len(r.b) < 32 {
43- return ErrShortPayload
42+ digest, err := r.getBytes(len(SnapshotDigest{}))
43+ if err != nil {
44+ return err
4445 }
45- digest := r.b[:32]
46- r.b = r.b[32:]
4746 if parentGeneration == 0 || parentGeneration >= generation || isZeroDigestBytes(digest) {
4847 return fmt.Errorf("%w: parent checkpoint", ErrInvalidData)
4948 }
@@ -19,12 +19,12 @@ func preflightManifest(body []byte) error {
19 if generation == 0 {19 if generation == 0 {
20 return fmt.Errorf("%w: manifest generation", ErrInvalidData)20 return fmt.Errorf("%w: manifest generation", ErrInvalidData)
21 }21 }
22- if len(r.b) < len(domain.IncarnationID{}) {22+ incarnationBytes, err := r.getBytes(len(domain.IncarnationID{}))
23- return ErrShortPayload23+ if err != nil {
24+ return err
24 }25 }
25 var incarnationID domain.IncarnationID26 var incarnationID domain.IncarnationID
26- copy(incarnationID[:], r.b[:len(incarnationID)])27+ copy(incarnationID[:], incarnationBytes)
27- r.b = r.b[len(incarnationID):]
28 if incarnationID == (domain.IncarnationID{}) {28 if incarnationID == (domain.IncarnationID{}) {
29 return fmt.Errorf("%w: manifest incarnation", ErrInvalidData)29 return fmt.Errorf("%w: manifest incarnation", ErrInvalidData)
30 }30 }
@@ -39,11 +39,10 @@ func preflightManifest(body []byte) error {
39 if err != nil {39 if err != nil {
40 return err40 return err
41 }41 }
42- if len(r.b) < 32 {42+ digest, err := r.getBytes(len(SnapshotDigest{}))
43- return ErrShortPayload43+ if err != nil {
44+ return err
44 }45 }
45- digest := r.b[:32]
46- r.b = r.b[32:]
47 if parentGeneration == 0 || parentGeneration >= generation || isZeroDigestBytes(digest) {46 if parentGeneration == 0 || parentGeneration >= generation || isZeroDigestBytes(digest) {
48 return fmt.Errorf("%w: parent checkpoint", ErrInvalidData)47 return fmt.Errorf("%w: parent checkpoint", ErrInvalidData)
49 }48 }
M internal/usecase/snapshot/manifest_preflight_test.go +12 -1
@@ -91,6 +91,9 @@ func manifestFieldOffsets(t *testing.T, encoded []byte) (nameOffset, nodeKindOff
9191 }
9292 _, err := r.getUint64()
9393 must(err)
94+ if len(r.b) < len(domain.IncarnationID{}) {
95+ t.Fatalf("manifest body has %d incarnation bytes, want at least %d", len(r.b), len(domain.IncarnationID{}))
96+ }
9497 r.b = r.b[len(domain.IncarnationID{}):]
9598 _, err = r.getUint8()
9699 must(err)
@@ -117,7 +120,15 @@ func manifestFieldOffsets(t *testing.T, encoded []byte) (nameOffset, nodeKindOff
117120 must(err)
118121 _, err = r.getString()
119122 must(err)
120- return nameOffset, len(encoded) - len(r.b)
123+ nodeKindOffset = len(encoded) - len(r.b)
124+ const wantName = "named"
125+ if nameOffset < 0 || nameOffset+len(wantName) > len(encoded) || string(encoded[nameOffset:nameOffset+len(wantName)]) != wantName {
126+ t.Fatalf("name offset %d does not point to %q", nameOffset, wantName)
127+ }
128+ if nodeKindOffset < 0 || nodeKindOffset >= len(encoded) || encoded[nodeKindOffset] != manifestNodeLeaf {
129+ t.Fatalf("node kind offset %d does not point to leaf kind", nodeKindOffset)
130+ }
131+ return nameOffset, nodeKindOffset
121132 }
122133
123134 func TestManifestObjectRefKinds(t *testing.T) {
@@ -91,6 +91,9 @@ func manifestFieldOffsets(t *testing.T, encoded []byte) (nameOffset, nodeKindOff
91 }91 }
92 _, err := r.getUint64()92 _, err := r.getUint64()
93 must(err)93 must(err)
94+ if len(r.b) < len(domain.IncarnationID{}) {
95+ t.Fatalf("manifest body has %d incarnation bytes, want at least %d", len(r.b), len(domain.IncarnationID{}))
96+ }
94 r.b = r.b[len(domain.IncarnationID{}):]97 r.b = r.b[len(domain.IncarnationID{}):]
95 _, err = r.getUint8()98 _, err = r.getUint8()
96 must(err)99 must(err)
@@ -117,7 +120,15 @@ func manifestFieldOffsets(t *testing.T, encoded []byte) (nameOffset, nodeKindOff
117 must(err)120 must(err)
118 _, err = r.getString()121 _, err = r.getString()
119 must(err)122 must(err)
120- return nameOffset, len(encoded) - len(r.b)123+ nodeKindOffset = len(encoded) - len(r.b)
124+ const wantName = "named"
125+ if nameOffset < 0 || nameOffset+len(wantName) > len(encoded) || string(encoded[nameOffset:nameOffset+len(wantName)]) != wantName {
126+ t.Fatalf("name offset %d does not point to %q", nameOffset, wantName)
127+ }
128+ if nodeKindOffset < 0 || nodeKindOffset >= len(encoded) || encoded[nodeKindOffset] != manifestNodeLeaf {
129+ t.Fatalf("node kind offset %d does not point to leaf kind", nodeKindOffset)
130+ }
131+ return nameOffset, nodeKindOffset
121 }132 }
122 133
123 func TestManifestObjectRefKinds(t *testing.T) {134 func TestManifestObjectRefKinds(t *testing.T) {
M pkg/kv/kv.go +1 -5
@@ -150,8 +150,7 @@ func openWithHooks(path string, hooks fsHooks) (*Store, error) {
150150 // retries from whichever complete file recovery selects.
151151 if result.Format != formatCurrent || s.shouldCompact() {
152152 if err := s.compactLocked(); err != nil {
153- _ = f.Close()
154- return cleanupLock(err)
153+ return nil, err
155154 }
156155 }
157156 return s, nil
@@ -720,9 +719,6 @@ func recoverCompaction(path string, hooks fsHooks) error {
720719 }
721720 return hooks.SyncDir(dir)
722721 }
723- if currentExists && currentErr != nil {
724- return fmt.Errorf("%w: no valid current", ErrCorruptWAL)
725- }
726722 return nil
727723 }
728724
@@ -150,8 +150,7 @@ func openWithHooks(path string, hooks fsHooks) (*Store, error) {
150 // retries from whichever complete file recovery selects.150 // retries from whichever complete file recovery selects.
151 if result.Format != formatCurrent || s.shouldCompact() {151 if result.Format != formatCurrent || s.shouldCompact() {
152 if err := s.compactLocked(); err != nil {152 if err := s.compactLocked(); err != nil {
153- _ = f.Close()153+ return nil, err
154- return cleanupLock(err)
155 }154 }
156 }155 }
157 return s, nil156 return s, nil
@@ -720,9 +719,6 @@ func recoverCompaction(path string, hooks fsHooks) error {
720 }719 }
721 return hooks.SyncDir(dir)720 return hooks.SyncDir(dir)
722 }721 }
723- if currentExists && currentErr != nil {
724- return fmt.Errorf("%w: no valid current", ErrCorruptWAL)
725- }
726 return nil722 return nil
727 }723 }
728 724
M pkg/kv/kv_test.go +19 -22
@@ -11,8 +11,17 @@ import (
1111 "path/filepath"
1212 "reflect"
1313 "testing"
14+
15+ "github.com/bnema/vev/pkg/safedir"
1416 )
1517
18+var compactionFaultPoints = []string{
19+ "before-write-next", "after-write-next", "before-sync-next", "after-sync-next",
20+ "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev",
21+ "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current",
22+ "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir",
23+}
24+
1625 func privatePath(t *testing.T) string {
1726 t.Helper()
1827 return filepath.Join(t.TempDir(), "vev", "store.kv")
@@ -56,7 +65,7 @@ func legacyFile(records ...[]byte) []byte {
5665
5766 func writeFile(t *testing.T, path string, data []byte) {
5867 t.Helper()
59- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
68+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
6069 t.Fatal(err)
6170 }
6271 if err := os.WriteFile(path, data, 0o600); err != nil {
@@ -173,7 +182,7 @@ func TestOpenTruncatesMidRecordTail(t *testing.T) {
173182 path := privatePath(t)
174183 first, _ := encodeRecord(opSet, []byte("a"), []byte("one"))
175184 second, _ := encodeRecord(opSet, []byte("b"), []byte("two"))
176- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
185+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
177186 t.Fatal(err)
178187 }
179188 if err := os.WriteFile(path, currentFile(first, second[:len(second)/2]), 0o600); err != nil {
@@ -206,7 +215,7 @@ func TestCRCcorruptionOfLastRecordFailsClosed(t *testing.T) {
206215 second, _ := encodeRecord(opSet, []byte("k"), []byte("new"))
207216 buf := currentFile(first, second)
208217 buf[fileHeaderLen+len(first)+recordHeaderLen+payloadPrefixLen+1] ^= 0xff
209- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
218+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
210219 t.Fatal(err)
211220 }
212221 if err := os.WriteFile(path, buf, 0o600); err != nil {
@@ -311,7 +320,7 @@ func TestReplayStrict(t *testing.T) {
311320 for _, tt := range tests {
312321 t.Run(tt.name, func(t *testing.T) {
313322 path := privatePath(t)
314- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
323+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
315324 t.Fatal(err)
316325 }
317326 if err := os.WriteFile(path, tt.wal, 0o600); err != nil {
@@ -361,7 +370,7 @@ func TestRecoverCompactionCleansStaleNextRegardlessOfPrev(t *testing.T) {
361370 for _, tt := range tests {
362371 t.Run(tt.name, func(t *testing.T) {
363372 path := privatePath(t)
364- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
373+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
365374 t.Fatal(err)
366375 }
367376 for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} {
@@ -486,7 +495,7 @@ func TestCompactionRecoveryMatrix(t *testing.T) {
486495 for _, tt := range tests {
487496 t.Run(tt.name, func(t *testing.T) {
488497 path := privatePath(t)
489- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
498+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
490499 t.Fatal(err)
491500 }
492501 for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} {
@@ -520,16 +529,10 @@ func TestCompactionRecoveryMatrix(t *testing.T) {
520529 })
521530 }
522531
523- faultPoints := []string{
524- "before-write-next", "after-write-next", "before-sync-next", "after-sync-next",
525- "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev",
526- "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current",
527- "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir",
528- }
529- for _, point := range faultPoints {
532+ for _, point := range compactionFaultPoints {
530533 t.Run("fault-"+point, func(t *testing.T) {
531534 path := privatePath(t)
532- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
535+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
533536 t.Fatal(err)
534537 }
535538 if err := os.WriteFile(path, oldFile, 0o600); err != nil {
@@ -632,7 +635,7 @@ func TestCompactionFilesystemOperationFailuresPoisonAndRecover(t *testing.T) {
632635 for _, tt := range tests {
633636 t.Run(tt.name, func(t *testing.T) {
634637 path := privatePath(t)
635- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
638+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
636639 t.Fatal(err)
637640 }
638641 if err := os.WriteFile(path, oldFile, 0o600); err != nil {
@@ -1083,13 +1086,7 @@ func TestLegacyUpgradeCrashSafety(t *testing.T) {
10831086 legacy := legacyFile(setA, setB)
10841087 want := map[string]string{"a": "one", "b": "two"}
10851088
1086- faultPoints := []string{
1087- "before-write-next", "after-write-next", "before-sync-next", "after-sync-next",
1088- "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev",
1089- "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current",
1090- "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir",
1091- }
1092- for _, point := range faultPoints {
1089+ for _, point := range compactionFaultPoints {
10931090 t.Run("fault-"+point, func(t *testing.T) {
10941091 path := privatePath(t)
10951092 writeFile(t, path, legacy)
@@ -11,8 +11,17 @@ import (
11 "path/filepath"11 "path/filepath"
12 "reflect"12 "reflect"
13 "testing"13 "testing"
14+
15+ "github.com/bnema/vev/pkg/safedir"
14 )16 )
15 17
18+var compactionFaultPoints = []string{
19+ "before-write-next", "after-write-next", "before-sync-next", "after-sync-next",
20+ "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev",
21+ "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current",
22+ "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir",
23+}
24+
16 func privatePath(t *testing.T) string {25 func privatePath(t *testing.T) string {
17 t.Helper()26 t.Helper()
18 return filepath.Join(t.TempDir(), "vev", "store.kv")27 return filepath.Join(t.TempDir(), "vev", "store.kv")
@@ -56,7 +65,7 @@ func legacyFile(records ...[]byte) []byte {
56 65
57 func writeFile(t *testing.T, path string, data []byte) {66 func writeFile(t *testing.T, path string, data []byte) {
58 t.Helper()67 t.Helper()
59- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {68+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
60 t.Fatal(err)69 t.Fatal(err)
61 }70 }
62 if err := os.WriteFile(path, data, 0o600); err != nil {71 if err := os.WriteFile(path, data, 0o600); err != nil {
@@ -173,7 +182,7 @@ func TestOpenTruncatesMidRecordTail(t *testing.T) {
173 path := privatePath(t)182 path := privatePath(t)
174 first, _ := encodeRecord(opSet, []byte("a"), []byte("one"))183 first, _ := encodeRecord(opSet, []byte("a"), []byte("one"))
175 second, _ := encodeRecord(opSet, []byte("b"), []byte("two"))184 second, _ := encodeRecord(opSet, []byte("b"), []byte("two"))
176- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {185+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
177 t.Fatal(err)186 t.Fatal(err)
178 }187 }
179 if err := os.WriteFile(path, currentFile(first, second[:len(second)/2]), 0o600); err != nil {188 if err := os.WriteFile(path, currentFile(first, second[:len(second)/2]), 0o600); err != nil {
@@ -206,7 +215,7 @@ func TestCRCcorruptionOfLastRecordFailsClosed(t *testing.T) {
206 second, _ := encodeRecord(opSet, []byte("k"), []byte("new"))215 second, _ := encodeRecord(opSet, []byte("k"), []byte("new"))
207 buf := currentFile(first, second)216 buf := currentFile(first, second)
208 buf[fileHeaderLen+len(first)+recordHeaderLen+payloadPrefixLen+1] ^= 0xff217 buf[fileHeaderLen+len(first)+recordHeaderLen+payloadPrefixLen+1] ^= 0xff
209- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {218+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
210 t.Fatal(err)219 t.Fatal(err)
211 }220 }
212 if err := os.WriteFile(path, buf, 0o600); err != nil {221 if err := os.WriteFile(path, buf, 0o600); err != nil {
@@ -311,7 +320,7 @@ func TestReplayStrict(t *testing.T) {
311 for _, tt := range tests {320 for _, tt := range tests {
312 t.Run(tt.name, func(t *testing.T) {321 t.Run(tt.name, func(t *testing.T) {
313 path := privatePath(t)322 path := privatePath(t)
314- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {323+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
315 t.Fatal(err)324 t.Fatal(err)
316 }325 }
317 if err := os.WriteFile(path, tt.wal, 0o600); err != nil {326 if err := os.WriteFile(path, tt.wal, 0o600); err != nil {
@@ -361,7 +370,7 @@ func TestRecoverCompactionCleansStaleNextRegardlessOfPrev(t *testing.T) {
361 for _, tt := range tests {370 for _, tt := range tests {
362 t.Run(tt.name, func(t *testing.T) {371 t.Run(tt.name, func(t *testing.T) {
363 path := privatePath(t)372 path := privatePath(t)
364- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {373+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
365 t.Fatal(err)374 t.Fatal(err)
366 }375 }
367 for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} {376 for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} {
@@ -486,7 +495,7 @@ func TestCompactionRecoveryMatrix(t *testing.T) {
486 for _, tt := range tests {495 for _, tt := range tests {
487 t.Run(tt.name, func(t *testing.T) {496 t.Run(tt.name, func(t *testing.T) {
488 path := privatePath(t)497 path := privatePath(t)
489- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {498+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
490 t.Fatal(err)499 t.Fatal(err)
491 }500 }
492 for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} {501 for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} {
@@ -520,16 +529,10 @@ func TestCompactionRecoveryMatrix(t *testing.T) {
520 })529 })
521 }530 }
522 531
523- faultPoints := []string{532+ for _, point := range compactionFaultPoints {
524- "before-write-next", "after-write-next", "before-sync-next", "after-sync-next",
525- "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev",
526- "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current",
527- "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir",
528- }
529- for _, point := range faultPoints {
530 t.Run("fault-"+point, func(t *testing.T) {533 t.Run("fault-"+point, func(t *testing.T) {
531 path := privatePath(t)534 path := privatePath(t)
532- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {535+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
533 t.Fatal(err)536 t.Fatal(err)
534 }537 }
535 if err := os.WriteFile(path, oldFile, 0o600); err != nil {538 if err := os.WriteFile(path, oldFile, 0o600); err != nil {
@@ -632,7 +635,7 @@ func TestCompactionFilesystemOperationFailuresPoisonAndRecover(t *testing.T) {
632 for _, tt := range tests {635 for _, tt := range tests {
633 t.Run(tt.name, func(t *testing.T) {636 t.Run(tt.name, func(t *testing.T) {
634 path := privatePath(t)637 path := privatePath(t)
635- if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {638+ if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
636 t.Fatal(err)639 t.Fatal(err)
637 }640 }
638 if err := os.WriteFile(path, oldFile, 0o600); err != nil {641 if err := os.WriteFile(path, oldFile, 0o600); err != nil {
@@ -1083,13 +1086,7 @@ func TestLegacyUpgradeCrashSafety(t *testing.T) {
1083 legacy := legacyFile(setA, setB)1086 legacy := legacyFile(setA, setB)
1084 want := map[string]string{"a": "one", "b": "two"}1087 want := map[string]string{"a": "one", "b": "two"}
1085 1088
1086- faultPoints := []string{1089+ for _, point := range compactionFaultPoints {
1087- "before-write-next", "after-write-next", "before-sync-next", "after-sync-next",
1088- "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev",
1089- "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current",
1090- "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir",
1091- }
1092- for _, point := range faultPoints {
1093 t.Run("fault-"+point, func(t *testing.T) {1090 t.Run("fault-"+point, func(t *testing.T) {
1094 path := privatePath(t)1091 path := privatePath(t)
1095 writeFile(t, path, legacy)1092 writeFile(t, path, legacy)

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 12

Tags 11

↑↓ moveenter openesc close