detached 979dbfb

fix(recovery): harden durable session invariants

commit
979dbfb6387143dee313544090da2238ce9508b9
author
brice · · 2026-07-26 07:25 UTC
parent
a6f1d55
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
80 files · +2738 -1260
M internal/adapters/noticefile/noticefile.go +2 -7
@@ -311,13 +311,8 @@ func appendNoticeFragment(line, fragment []byte) ([]byte, bool) {
311 }311 }
312 required := len(line) + len(fragment)312 required := len(line) + len(fragment)
313 if required > cap(line) {313 if required > cap(line) {
314- capacity := cap(line) * 2314+ capacity := max(cap(line)*2, noticeReadBufferSize)
315- if capacity < noticeReadBufferSize {315+ capacity = min(capacity, maxNoticeRecordSize)
316- capacity = noticeReadBufferSize
317- }
318- if capacity > maxNoticeRecordSize {
319- capacity = maxNoticeRecordSize
320- }
321 grown := make([]byte, len(line), capacity)316 grown := make([]byte, len(line), capacity)
322 copy(grown, line)317 copy(grown, line)
323 line = grown318 line = grown
M internal/adapters/recoveryfs/journal.go +26 -12
@@ -104,16 +104,33 @@ func (j *Journal) ListDiscards(ctx context.Context) ([]domain.DiscardIntent, err
104 if err != nil {104 if err != nil {
105 return nil, err105 return nil, err
106 }106 }
107- entries, readErr := dir.ReadDir(maxDiscardIntents + 1)107+ entries := make([]os.DirEntry, 0, maxDiscardIntents)
108- closeErr := dir.Close()108+ for {
109- if readErr != nil && !errors.Is(readErr, io.EOF) {109+ batch, readErr := dir.ReadDir(maxDiscardIntents + 1)
110- return nil, readErr110+ for _, entry := range batch {
111- }111+ if strings.HasPrefix(entry.Name(), ".intent-") && !entry.IsDir() {
112- if closeErr != nil {112+ continue
113- return nil, closeErr113+ }
114+ if entry.IsDir() || !strings.HasPrefix(entry.Name(), "discard-") {
115+ _ = dir.Close()
116+ return nil, errors.New("recovery journal: invalid transaction entry")
117+ }
118+ entries = append(entries, entry)
119+ if len(entries) > maxDiscardIntents {
120+ _ = dir.Close()
121+ return nil, errors.New("recovery journal: transaction entry limit exceeded")
122+ }
123+ }
124+ if errors.Is(readErr, io.EOF) {
125+ break
126+ }
127+ if readErr != nil {
128+ _ = dir.Close()
129+ return nil, readErr
130+ }
114 }131 }
115- if len(entries) > maxDiscardIntents {132+ if err := dir.Close(); err != nil {
116- return nil, errors.New("recovery journal: transaction entry limit exceeded")133+ return nil, err
117 }134 }
118 sort.Slice(entries, func(a, b int) bool { return entries[a].Name() < entries[b].Name() })135 sort.Slice(entries, func(a, b int) bool { return entries[a].Name() < entries[b].Name() })
119 out := make([]domain.DiscardIntent, 0, len(entries))136 out := make([]domain.DiscardIntent, 0, len(entries))
@@ -122,9 +139,6 @@ func (j *Journal) ListDiscards(ctx context.Context) ([]domain.DiscardIntent, err
122 if err := ctx.Err(); err != nil {139 if err := ctx.Err(); err != nil {
123 return nil, err140 return nil, err
124 }141 }
125- if entry.IsDir() || !strings.HasPrefix(entry.Name(), "discard-") {
126- return nil, errors.New("recovery journal: invalid transaction entry")
127- }
128 data, err := readFileBounded(filepath.Join(j.dir, entry.Name()), maxDiscardIntentSize)142 data, err := readFileBounded(filepath.Join(j.dir, entry.Name()), maxDiscardIntentSize)
129 if err != nil {143 if err != nil {
130 return nil, err144 return nil, err
M internal/adapters/recoveryfs/journal_test.go +13 -0
@@ -38,6 +38,7 @@ func TestDiscardCrashMatrix(t *testing.T) {
38 path := j.path(old)38 path := j.path(old)
39 encoded, err := os.ReadFile(path)39 encoded, err := os.ReadFile(path)
40 require.NoError(t, err)40 require.NoError(t, err)
41+ t.Cleanup(func() { require.NoError(t, os.WriteFile(path, encoded, 0o600)) })
41 require.NoError(t, os.WriteFile(path, append(encoded, 0), 0o600))42 require.NoError(t, os.WriteFile(path, append(encoded, 0), 0o600))
42 _, err = j.ListDiscards(ctx)43 _, err = j.ListDiscards(ctx)
43 require.Error(t, err)44 require.Error(t, err)
@@ -77,6 +78,18 @@ func TestRecoveryJournalRoundTripAndIdempotentDelete(t *testing.T) {
77 require.Empty(t, got)78 require.Empty(t, got)
78 }79 }
79 80
81+func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T) {
82+ j := New(t.TempDir())
83+ require.NoError(t, os.MkdirAll(j.dir, 0o700))
84+ for i := 0; i < maxDiscardIntents+1; i++ {
85+ require.NoError(t, os.WriteFile(filepath.Join(j.dir, ".intent-leftover-"+string(rune(0x1000+i))), nil, 0o600))
86+ }
87+
88+ got, err := j.ListDiscards(context.Background())
89+ require.NoError(t, err)
90+ require.Empty(t, got)
91+}
92+
80 func TestRecoveryJournalRejectsMalformedIntent(t *testing.T) {93 func TestRecoveryJournalRejectsMalformedIntent(t *testing.T) {
81 j := New(t.TempDir())94 j := New(t.TempDir())
82 require.Error(t, j.SaveDiscard(context.Background(), domain.DiscardIntent{}))95 require.Error(t, j.SaveDiscard(context.Background(), domain.DiscardIntent{}))
M internal/adapters/snapshot/atomic.go +4 -3
@@ -13,15 +13,16 @@ import (
13 )13 )
14 14
15 func (r *Repository) ensureSession(id domain.IncarnationID) error {15 func (r *Repository) ensureSession(id domain.IncarnationID) error {
16+ sessionPath := r.sessionPath(id)
16 for _, directory := range []struct {17 for _, directory := range []struct {
17 path string18 path string
18 phase string19 phase string
19 }{20 }{
20 {r.dir, "repository"},21 {r.dir, "repository"},
21 {filepath.Join(r.dir, repositorySessionsDir), "sessions"},22 {filepath.Join(r.dir, repositorySessionsDir), "sessions"},
22- {r.sessionPath(id), "session"},23+ {sessionPath, "session"},
23- {filepath.Join(r.sessionPath(id), repositoryObjectsDir), "objects"},24+ {filepath.Join(sessionPath, repositoryObjectsDir), "objects"},
24- {filepath.Join(r.sessionPath(id), repositoryGenerations), "generations"},25+ {filepath.Join(sessionPath, repositoryGenerations), "generations"},
25 } {26 } {
26 if err := r.ensurePrivateDirectoryPhase(directory.path, directory.phase); err != nil {27 if err := r.ensurePrivateDirectoryPhase(directory.path, directory.phase); err != nil {
27 return fmt.Errorf("create snapshot repository directory: %w", err)28 return fmt.Errorf("create snapshot repository directory: %w", err)
M internal/adapters/snapshot/legacy_load_test.go +0 -4
@@ -69,7 +69,6 @@ func (r *Repository) Load(ctx context.Context, name string) (ports.SnapshotGener
69 69
70 func (r *Repository) load(ctx context.Context, name, key string) (ports.SnapshotGeneration, error) {70 func (r *Repository) load(ctx context.Context, name, key string) (ports.SnapshotGeneration, error) {
71 preferred, preferredDigest, err := r.readLegacyHead(key)71 preferred, preferredDigest, err := r.readLegacyHead(key)
72- skipped := errors.Is(err, ErrInvalidHEAD)
73 if err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, ErrInvalidHEAD) {72 if err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, ErrInvalidHEAD) {
74 return ports.SnapshotGeneration{}, fmt.Errorf("read snapshot HEAD: %w", err)73 return ports.SnapshotGeneration{}, fmt.Errorf("read snapshot HEAD: %w", err)
75 }74 }
@@ -78,7 +77,6 @@ func (r *Repository) load(ctx context.Context, name, key string) (ports.Snapshot
78 if err == nil && sha256.Sum256(got.Manifest) == preferredDigest {77 if err == nil && sha256.Sum256(got.Manifest) == preferredDigest {
79 return got, nil78 return got, nil
80 }79 }
81- skipped = true
82 }80 }
83 81
84 budget := maxDirectoryTraversalEntries82 budget := maxDirectoryTraversalEntries
@@ -89,10 +87,8 @@ func (r *Repository) load(ctx context.Context, name, key string) (ports.Snapshot
89 for _, generation := range candidates {87 for _, generation := range candidates {
90 got, err := r.loadGeneration(ctx, name, key, generation)88 got, err := r.loadGeneration(ctx, name, key, generation)
91 if err != nil {89 if err != nil {
92- skipped = true
93 continue90 continue
94 }91 }
95- _ = skipped
96 return got, nil92 return got, nil
97 }93 }
98 return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name)94 return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name)
M internal/adapters/snapshot/load_bounds_test.go +10 -7
@@ -15,7 +15,7 @@ func TestRepositoryListStopsAtTraversalBudget(t *testing.T) {
15 if err := os.MkdirAll(sessions, 0o700); err != nil {15 if err := os.MkdirAll(sessions, 0o700); err != nil {
16 t.Fatal(err)16 t.Fatal(err)
17 }17 }
18- for i := 0; i < maxDirectoryTraversalEntries+1; i++ {18+ for i := range maxDirectoryTraversalEntries + 1 {
19 if err := os.Mkdir(filepath.Join(sessions, fmt.Sprintf("hostile-%05d", i)), 0o700); err != nil {19 if err := os.Mkdir(filepath.Join(sessions, fmt.Sprintf("hostile-%05d", i)), 0o700); err != nil {
20 t.Fatal(err)20 t.Fatal(err)
21 }21 }
@@ -34,7 +34,8 @@ func TestRepositoryLoadStopsAtGenerationTraversalBudget(t *testing.T) {
34 if err := os.MkdirAll(generations, 0o700); err != nil {34 if err := os.MkdirAll(generations, 0o700); err != nil {
35 t.Fatal(err)35 t.Fatal(err)
36 }36 }
37- for generation := uint64(1); generation <= uint64(maxDirectoryTraversalEntries+1); generation++ {37+ for i := range maxDirectoryTraversalEntries + 1 {
38+ generation := uint64(i + 1)
38 if err := os.WriteFile(filepath.Join(generations, generationFilename(generation)), nil, 0o600); err != nil {39 if err := os.WriteFile(filepath.Join(generations, generationFilename(generation)), nil, 0o600); err != nil {
39 t.Fatal(err)40 t.Fatal(err)
40 }41 }
@@ -48,8 +49,9 @@ func TestRepositoryLoadStopsAtGenerationTraversalBudget(t *testing.T) {
48 49
49 func TestRepositoryLoadFallsBackNewestToOldestWithoutGenerationEnumeration(t *testing.T) {50 func TestRepositoryLoadFallsBackNewestToOldestWithoutGenerationEnumeration(t *testing.T) {
50 repo := NewRepository(privateDir(t))51 repo := NewRepository(privateDir(t))
51- for generation := uint64(1); generation <= 3; generation++ {52+ for i := range 3 {
52- if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation)))); err != nil {53+ generation := uint64(i + 1)
54+ if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))); err != nil {
53 t.Fatal(err)55 t.Fatal(err)
54 }56 }
55 }57 }
@@ -70,12 +72,13 @@ func TestRepositoryLoadFallsBackNewestToOldestWithoutGenerationEnumeration(t *te
70 72
71 func TestRepositoryMaintainDoesNotCollectBeyondRetainedMetadataBudget(t *testing.T) {73 func TestRepositoryMaintainDoesNotCollectBeyondRetainedMetadataBudget(t *testing.T) {
72 repo := NewRepository(privateDir(t))74 repo := NewRepository(privateDir(t))
73- for generation := uint64(1); generation <= uint64(maxMaintenanceMarkedGenerations+1); generation++ {75+ for i := range maxMaintenanceMarkedGenerations + 1 {
74- if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation)))); err != nil {76+ generation := uint64(i + 1)
77+ if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))); err != nil {
75 t.Fatal(err)78 t.Fatal(err)
76 }79 }
77 }80 }
78- for pass := 0; pass < maxMaintenanceMarkedGenerations*3; pass++ {81+ for pass := range maxMaintenanceMarkedGenerations * 3 {
79 if err := repo.Maintain(context.Background()); err != nil {82 if err := repo.Maintain(context.Background()); err != nil {
80 t.Fatal(err)83 t.Fatal(err)
81 }84 }
M internal/adapters/snapshot/maintenance_delete.go +1 -22
@@ -644,28 +644,7 @@ func (r *Repository) SaveQuarantineDescriptor(ctx context.Context, descriptor do
644 644
645 func quarantineDescriptorsEqual(a, b domain.QuarantineDescriptor) bool {645 func quarantineDescriptorsEqual(a, b domain.QuarantineDescriptor) bool {
646 return a.OldIncarnation == b.OldIncarnation && a.ReplacementIncarnation == b.ReplacementIncarnation &&646 return a.OldIncarnation == b.OldIncarnation && a.ReplacementIncarnation == b.ReplacementIncarnation &&
647- a.SessionName == b.SessionName && a.Reason == b.Reason && catalogueRecordsEqual(a.OldRecord, b.OldRecord)647+ a.SessionName == b.SessionName && a.Reason == b.Reason && a.OldRecord.Equal(b.OldRecord)
648-}
649-
650-func catalogueRecordsEqual(a, b domain.CatalogueRecord) bool {
651- if a.Name != b.Name || a.IncarnationID != b.IncarnationID || a.Cwd != b.Cwd || a.CreatedAt != b.CreatedAt ||
652- a.UpdatedAt != b.UpdatedAt || a.LastUsedSeq != b.LastUsedSeq || a.RecoveryState != b.RecoveryState ||
653- a.DegradedReason != b.DegradedReason || len(a.TabNames) != len(b.TabNames) {
654- return false
655- }
656- for i := range a.TabNames {
657- if a.TabNames[i] != b.TabNames[i] {
658- return false
659- }
660- }
661- return checkpointRefsEqual(a.Committed, b.Committed) && checkpointRefsEqual(a.Fallbacks[0], b.Fallbacks[0]) && checkpointRefsEqual(a.Fallbacks[1], b.Fallbacks[1])
662-}
663-
664-func checkpointRefsEqual(a, b *domain.CheckpointRef) bool {
665- if a == nil || b == nil {
666- return a == nil && b == nil
667- }
668- return *a == *b
669 }648 }
670 649
671 func (r *Repository) QuarantineIncarnation(ctx context.Context, id domain.IncarnationID) error {650 func (r *Repository) QuarantineIncarnation(ctx context.Context, id domain.IncarnationID) error {
M internal/adapters/snapshot/maintenance_gc.go +88 -61
@@ -31,6 +31,37 @@ type manifestMaintenance struct {
31 complete bool31 complete bool
32 }32 }
33 33
34+type retentionEntryKind uint8
35+
36+const (
37+ retentionManifest retentionEntryKind = iota + 1
38+ retentionObject
39+)
40+
41+type retentionEntryPolicy struct {
42+ kind retentionEntryKind
43+ generations map[uint64]ports.SnapshotDigest
44+ objects map[ports.SnapshotDigest]struct{}
45+}
46+
47+func (p retentionEntryPolicy) delete(entry maintenanceDirEntry) bool {
48+ if entry.isDir {
49+ return false
50+ }
51+ switch p.kind {
52+ case retentionManifest:
53+ generation, canonical := parseGenerationFilename(entry.name)
54+ _, retained := p.generations[generation]
55+ return canonical && !retained
56+ case retentionObject:
57+ digest, canonical := parseObjectDigest(entry.name)
58+ _, retained := p.objects[digest]
59+ return canonical && !retained
60+ default:
61+ return false
62+ }
63+}
64+
34 type sessionMaintenance struct {65 type sessionMaintenance struct {
35 lock *sessionMutex66 lock *sessionMutex
36 token string67 token string
@@ -370,6 +401,44 @@ func keepSet(plan ports.RetentionPlan) map[uint64]ports.SnapshotDigest {
370 401
371 // MaintainSession incrementally validates and retains only catalogue-indexed402 // MaintainSession incrementally validates and retains only catalogue-indexed
372 // checkpoints. Every incarnation owns its budget and continuation state.403 // checkpoints. Every incarnation owns its budget and continuation state.
404+// processRetentionEntries charges stat-reported size for every entry, requeues
405+// the unadmitted suffix, and removes only canonical entries rejected by policy.
406+func (r *Repository) processRetentionEntries(ctx context.Context, dir, purpose string, entries []maintenanceDirEntry, budget *ports.MaintenanceBudget, consumedBefore bool, policy retentionEntryPolicy) (bool, error) {
407+ for i, entry := range entries {
408+ if err := ctx.Err(); err != nil {
409+ return false, err
410+ }
411+ path := filepath.Join(dir, entry.name)
412+ info, statErr := r.stat(path)
413+ if statErr != nil && !errors.Is(statErr, os.ErrNotExist) {
414+ return false, statErr
415+ }
416+ size := uint64(0)
417+ if statErr == nil && info.Size() > 0 {
418+ size = uint64(info.Size())
419+ }
420+ if budget.Entries == 0 || size > budget.Bytes {
421+ r.requeueMaintenanceEntries(dir, purpose, entries[i:])
422+ if !consumedBefore && i == 0 {
423+ return false, ErrMaintenanceBudgetTooSmall
424+ }
425+ return false, nil
426+ }
427+ budget.Entries--
428+ budget.Bytes -= size
429+ if !policy.delete(entry) {
430+ continue
431+ }
432+ if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
433+ return false, err
434+ }
435+ if err := r.syncDirectory(dir); err != nil {
436+ return false, err
437+ }
438+ }
439+ return true, nil
440+}
441+
373 func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPlan, budget ports.MaintenanceBudget) (done bool, err error) {442 func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPlan, budget ports.MaintenanceBudget) (done bool, err error) {
374 requested := budget443 requested := budget
375 if err := ctx.Err(); err != nil {444 if err := ctx.Err(); err != nil {
@@ -459,34 +528,14 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
459 if err != nil {528 if err != nil {
460 return false, err529 return false, err
461 }530 }
462- for i, entry := range entries {531+ dir := filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations)
463- path := r.manifestPath(plan.IncarnationID, 0)532+ consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes
464- path = filepath.Join(filepath.Dir(path), entry.name)533+ admitted, err := r.processRetentionEntries(ctx, dir, purpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionManifest, generations: keep})
465- info, statErr := r.stat(path)534+ if err != nil {
466- if statErr != nil && !errors.Is(statErr, os.ErrNotExist) {535+ return false, err
467- return false, statErr536+ }
468- }537+ if !admitted {
469- size := uint64(0)538+ return false, nil
470- if statErr == nil && info.Size() > 0 {
471- size = uint64(info.Size())
472- }
473- if budget.Entries == 0 || size > budget.Bytes {
474- r.requeueMaintenanceEntries(filepath.Dir(path), purpose, entries[i:])
475- return false, nil
476- }
477- budget.Entries--
478- budget.Bytes -= size
479- generation, canonical := parseGenerationFilename(entry.name)
480- if !entry.isDir && canonical {
481- if _, retained := keep[generation]; !retained {
482- if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
483- return false, err
484- }
485- if err := r.syncDirectory(filepath.Dir(path)); err != nil {
486- return false, err
487- }
488- }
489- }
490 }539 }
491 state.manifestsDone = done540 state.manifestsDone = done
492 if len(entries) == 0 && !done {541 if len(entries) == 0 && !done {
@@ -511,33 +560,13 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl
511 if err != nil {560 if err != nil {
512 return false, err561 return false, err
513 }562 }
514- for i, entry := range entries {563+ consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes
515- path := filepath.Join(dir, entry.name)564+ admitted, err := r.processRetentionEntries(ctx, dir, objectPurpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionObject, objects: state.references})
516- info, statErr := r.stat(path)565+ if err != nil {
517- if statErr != nil && !errors.Is(statErr, os.ErrNotExist) {566+ return false, err
518- return false, statErr567+ }
519- }568+ if !admitted {
520- size := uint64(0)569+ return false, nil
521- if statErr == nil && info.Size() > 0 {
522- size = uint64(info.Size())
523- }
524- if budget.Entries == 0 || size > budget.Bytes {
525- r.requeueMaintenanceEntries(dir, objectPurpose, entries[i:])
526- return false, nil
527- }
528- budget.Entries--
529- budget.Bytes -= size
530- digest, canonical := parseObjectDigest(entry.name)
531- if !entry.isDir && canonical {
532- if _, retained := state.references[digest]; !retained {
533- if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
534- return false, err
535- }
536- if err := r.syncDirectory(dir); err != nil {
537- return false, err
538- }
539- }
540- }
541 }570 }
542 if done {571 if done {
543 state.objectShard = ""572 state.objectShard = ""
@@ -587,15 +616,15 @@ func retentionToken(plan ports.RetentionPlan) string {
587 return b.String()616 return b.String()
588 }617 }
589 618
590-// validateMaintenanceCheckpoint admits every payload by stat-reported size
591-// before reading it. It validates directly instead of calling LoadCheckpoint,
592-// which would materialize a complete generation before maintenance could
593-// enforce its byte budget.
594 type validatedMaintenanceCheckpoint struct {619 type validatedMaintenanceCheckpoint struct {
595 refs map[ports.SnapshotDigest]codec.ObjectRef620 refs map[ports.SnapshotDigest]codec.ObjectRef
596 parent *domain.CheckpointRef621 parent *domain.CheckpointRef
597 }622 }
598 623
624+// validateMaintenanceCheckpoint admits every payload by stat-reported size
625+// before reading it. It validates directly instead of calling LoadCheckpoint,
626+// which would materialize a complete generation before maintenance could
627+// enforce its byte budget.
599 func (r *Repository) validateMaintenanceCheckpoint(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef, byteBudget uint64) (validatedMaintenanceCheckpoint, uint64, bool, error) {628 func (r *Repository) validateMaintenanceCheckpoint(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef, byteBudget uint64) (validatedMaintenanceCheckpoint, uint64, bool, error) {
600 manifestPath := r.manifestPath(id, ref.Generation)629 manifestPath := r.manifestPath(id, ref.Generation)
601 info, err := r.stat(manifestPath)630 info, err := r.stat(manifestPath)
@@ -692,8 +721,6 @@ func (r *Repository) Reconcile(ctx context.Context, records []domain.CatalogueRe
692 byIncarnation[record.IncarnationID] = record721 byIncarnation[record.IncarnationID] = record
693 }722 }
694 723
695- r.maintenanceMu.Lock()
696- defer r.maintenanceMu.Unlock()
697 root := filepath.Join(r.dir, repositorySessionsDir)724 root := filepath.Join(r.dir, repositorySessionsDir)
698 state := &maintenanceCursor{offset: offset}725 state := &maintenanceCursor{offset: offset}
699 findingCapacity := maintenanceBatch726 findingCapacity := maintenanceBatch
M internal/adapters/snapshot/maintenance_test.go +88 -16
@@ -1,6 +1,7 @@
1 package snapshot1 package snapshot
2 2
3 import (3 import (
4+ "cmp"
4 "context"5 "context"
5 "crypto/sha256"6 "crypto/sha256"
6 "encoding/binary"7 "encoding/binary"
@@ -11,8 +12,10 @@ import (
11 "path/filepath"12 "path/filepath"
12 "slices"13 "slices"
13 "strings"14 "strings"
15+ "sync"
14 "syscall"16 "syscall"
15 "testing"17 "testing"
18+ "time"
16 "unsafe"19 "unsafe"
17 20
18 "github.com/bnema/vev/internal/domain"21 "github.com/bnema/vev/internal/domain"
@@ -1043,21 +1046,17 @@ func TestRetentionSlots(t *testing.T) {
1043 }1046 }
1044 1047
1045 func TestUnresolvedDataPinned(t *testing.T) {1048 func TestUnresolvedDataPinned(t *testing.T) {
1046- for _, state := range []string{"degraded", "deleting", "migration", "transaction"} {1049+ repo := NewRepository(privateDir(t))
1047- t.Run(state, func(t *testing.T) {1050+ publications := publishMaintenanceGenerations(t, repo, "unresolved", 4)
1048- repo := NewRepository(privateDir(t))1051+ done, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: publications[0].IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1})
1049- publications := publishMaintenanceGenerations(t, repo, state, 4)1052+ if err != nil {
1050- done, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: publications[0].IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1})1053+ t.Fatal(err)
1051- if err != nil {1054+ }
1052- t.Fatal(err)1055+ if !done {
1053- }1056+ t.Fatal("pinned maintenance did not complete without traversing")
1054- if !done {1057+ }
1055- t.Fatal("pinned maintenance did not complete without traversing")1058+ if got := remainingMaintenanceGenerations(t, repo, publications[0].IncarnationID); !slices.Equal(got, []uint64{4, 3, 2, 1}) {
1056- }1059+ t.Fatalf("pinned generations = %v", got)
1057- if got := remainingMaintenanceGenerations(t, repo, publications[0].IncarnationID); !slices.Equal(got, []uint64{4, 3, 2, 1}) {
1058- t.Fatalf("pinned generations = %v", got)
1059- }
1060- })
1061 }1060 }
1062 }1061 }
1063 1062
@@ -1094,6 +1093,79 @@ func TestMaintenanceBudgetAdmitsPayloadsBeforeRead(t *testing.T) {
1094 }1093 }
1095 }1094 }
1096 1095
1096+func TestMaintainSessionDistinguishesInitialOversizeFromResumableExhaustion(t *testing.T) {
1097+ repo := NewRepository(privateDir(t))
1098+ publications := publishMaintenanceGenerations(t, repo, "budget-semantics", 2)
1099+ plan := ports.RetentionPlan{IncarnationID: publications[0].IncarnationID}
1100+ firstInfo, err := os.Stat(repo.manifestPath(plan.IncarnationID, 1))
1101+ if err != nil {
1102+ t.Fatal(err)
1103+ }
1104+ secondInfo, err := os.Stat(repo.manifestPath(plan.IncarnationID, 2))
1105+ if err != nil {
1106+ t.Fatal(err)
1107+ }
1108+ firstSize, secondSize := uint64(firstInfo.Size()), uint64(secondInfo.Size())
1109+
1110+ done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 2, Bytes: min(firstSize, secondSize) - 1})
1111+ if done || !errors.Is(err, ErrMaintenanceBudgetTooSmall) {
1112+ t.Fatalf("initial oversized item: done=%v err=%v", done, err)
1113+ }
1114+
1115+ done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 2, Bytes: firstSize + secondSize - 1})
1116+ if err != nil || done {
1117+ t.Fatalf("mid-pass exhaustion must be resumable: done=%v err=%v", done, err)
1118+ }
1119+ done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20})
1120+ if err != nil || !done {
1121+ t.Fatalf("resumed maintenance: done=%v err=%v", done, err)
1122+ }
1123+}
1124+
1125+func TestReconcileDoesNotHoldMaintenanceLockDuringPayloadRead(t *testing.T) {
1126+ repo := NewRepository(privateDir(t))
1127+ publications := publishMaintenanceGenerations(t, repo, "reconcile-lock", 2)
1128+ committed := checkpointRefForPublication(publications[0])
1129+ record := domain.CatalogueRecord{Name: "reconcile-lock", IncarnationID: publications[0].IncarnationID, RecoveryState: domain.RecoveryHealthy, Committed: &committed}
1130+
1131+ entered := make(chan struct{})
1132+ release := make(chan struct{})
1133+ var releaseOnce sync.Once
1134+ defer releaseOnce.Do(func() { close(release) })
1135+ repo.hooks.beforeMaintenancePayloadRead = func(string) {
1136+ select {
1137+ case <-entered:
1138+ default:
1139+ close(entered)
1140+ }
1141+ <-release
1142+ }
1143+ reconcileDone := make(chan error, 1)
1144+ go func() {
1145+ _, _, err := repo.Reconcile(context.Background(), []domain.CatalogueRecord{record}, ports.ReconcileCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20})
1146+ reconcileDone <- err
1147+ }()
1148+ <-entered
1149+
1150+ pinDone := make(chan error, 1)
1151+ go func() {
1152+ _, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1})
1153+ pinDone <- err
1154+ }()
1155+ select {
1156+ case err := <-pinDone:
1157+ if err != nil {
1158+ t.Fatal(err)
1159+ }
1160+ case <-time.After(time.Second):
1161+ t.Fatal("maintenance lock remained held during reconciliation payload I/O")
1162+ }
1163+ releaseOnce.Do(func() { close(release) })
1164+ if err := <-reconcileDone; err != nil {
1165+ t.Fatal(err)
1166+ }
1167+}
1168+
1097 func TestPerSessionBudgetIsolation(t *testing.T) {1169 func TestPerSessionBudgetIsolation(t *testing.T) {
1098 repo := NewRepository(privateDir(t))1170 repo := NewRepository(privateDir(t))
1099 large := publishMaintenanceGenerations(t, repo, "large", 4)1171 large := publishMaintenanceGenerations(t, repo, "large", 4)
@@ -1149,6 +1221,6 @@ func remainingMaintenanceGenerations(t *testing.T, repo *Repository, id domain.I
1149 generations = append(generations, generation)1221 generations = append(generations, generation)
1150 }1222 }
1151 }1223 }
1152- slices.SortFunc(generations, func(a, b uint64) int { return int(b) - int(a) })1224+ slices.SortFunc(generations, func(a, b uint64) int { return cmp.Compare(b, a) })
1153 return generations1225 return generations
1154 }1226 }
M internal/adapters/snapshot/migration_v0.go +6 -3
@@ -49,7 +49,7 @@ func decodeManifestV1(data []byte) (legacyManifestV1, error) {
49 objects := make([]legacyObjectV1, 0)49 objects := make([]legacyObjectV1, 0)
50 add := func(ref codec.ObjectRef) error {50 add := func(ref codec.ObjectRef) error {
51 if _, ok := seen[ref.Digest]; ok {51 if _, ok := seen[ref.Digest]; ok {
52- return errors.New("snapshot: duplicate legacy object digest")52+ return nil
53 }53 }
54 seen[ref.Digest] = struct{}{}54 seen[ref.Digest] = struct{}{}
55 objects = append(objects, legacyObjectV1{Kind: uint8(ref.Kind), Digest: ref.Digest, Size: uint64(ref.Size)})55 objects = append(objects, legacyObjectV1{Kind: uint8(ref.Kind), Digest: ref.Digest, Size: uint64(ref.Size)})
@@ -209,7 +209,7 @@ func uncertainLegacyError(operation string, cause error) error {
209 if cause == nil {209 if cause == nil {
210 return fmt.Errorf("%w: %s", ports.ErrLegacySnapshotUncertain, operation)210 return fmt.Errorf("%w: %s", ports.ErrLegacySnapshotUncertain, operation)
211 }211 }
212- return fmt.Errorf("%w: %s: %v", ports.ErrLegacySnapshotUncertain, operation, cause)212+ return fmt.Errorf("%w: %s: %w", ports.ErrLegacySnapshotUncertain, operation, cause)
213 }213 }
214 214
215 // MigrateV1Checkpoint is the migration-only admission path for an authoritative215 // MigrateV1Checkpoint is the migration-only admission path for an authoritative
@@ -244,7 +244,10 @@ func (r *Repository) MigrateV1Checkpoint(ctx context.Context, req ports.Snapshot
244 if manifest.Name != req.LegacyName || manifest.Generation != req.LegacyRef.Generation {244 if manifest.Name != req.LegacyName || manifest.Generation != req.LegacyRef.Generation {
245 return domain.CheckpointRef{}, uncertainLegacyError("manifest identity mismatch", nil)245 return domain.CheckpointRef{}, uncertainLegacyError("manifest identity mismatch", nil)
246 }246 }
247- key, _ := incarnationKey(req.IncarnationID)247+ key, err := incarnationKey(req.IncarnationID)
248+ if err != nil {
249+ return domain.CheckpointRef{}, err
250+ }
248 lock := r.lockSession(key)251 lock := r.lockSession(key)
249 defer r.unlockSession(lock)252 defer r.unlockSession(lock)
250 if err := r.ensureSession(req.IncarnationID); err != nil {253 if err := r.ensureSession(req.IncarnationID); err != nil {
M internal/adapters/snapshot/migration_v0_test.go +39 -3
@@ -4,6 +4,7 @@ import (
4 "context"4 "context"
5 "crypto/sha256"5 "crypto/sha256"
6 "encoding/binary"6 "encoding/binary"
7+ "errors"
7 "fmt"8 "fmt"
8 "hash/crc32"9 "hash/crc32"
9 "os"10 "os"
@@ -37,10 +38,45 @@ func TestLegacyManifestV1(t *testing.T) {
37 require.NoError(t, err)38 require.NoError(t, err)
38 require.Equal(t, "work", got.Name)39 require.Equal(t, "work", got.Name)
39 require.Equal(t, uint64(7), got.Generation)40 require.Equal(t, uint64(7), got.Generation)
40- for _, invalid := range [][]byte{encoded[:5], append(append([]byte(nil), encoded...), 0)} {41+ for size := range len(encoded) {
41- _, err := decodeManifestV1(invalid)42+ _, err := decodeManifestV1(encoded[:size])
42- require.Error(t, err)43+ require.Errorf(t, err, "prefix length %d", size)
43 }44 }
45+ _, err = decodeManifestV1(append(append([]byte(nil), encoded...), 0))
46+ require.Error(t, err)
47+}
48+
49+func TestLegacyManifestV1DeduplicatesObjectsPreservingFirst(t *testing.T) {
50+ chunk, err := codec.MarshalObject(codec.HistoryChunk, []byte("chunk"))
51+ require.NoError(t, err)
52+ tail, err := codec.MarshalObject(codec.HistoryTail, []byte("tail"))
53+ require.NoError(t, err)
54+ visible, err := codec.MarshalObject(codec.Visible, []byte("visible"))
55+ require.NoError(t, err)
56+ chunkRef := codec.ObjectRef{Kind: codec.HistoryChunk, Digest: chunk.Digest, Size: uint32(len(chunk.Data))}
57+ manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Sealed: []codec.ObjectRef{chunkRef, chunkRef}, Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}}
58+ modern, err := codec.MarshalManifest(manifest)
59+ require.NoError(t, err)
60+ body := append([]byte(nil), modern[legacyManifestHeaderSize:]...)
61+ body = append(body[:8], body[25:]...)
62+ legacy := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body))
63+ copy(legacy, "VEVM")
64+ binary.BigEndian.PutUint16(legacy[4:6], 1)
65+ binary.BigEndian.PutUint32(legacy[8:12], uint32(len(body)))
66+ binary.BigEndian.PutUint32(legacy[12:16], crc32.ChecksumIEEE(body))
67+ legacy = append(legacy, body...)
68+
69+ decoded, err := decodeManifestV1(legacy)
70+ require.NoError(t, err)
71+ require.Len(t, decoded.Objects, 3)
72+ require.Equal(t, chunk.Digest, decoded.Objects[0].Digest)
73+}
74+
75+func TestUncertainLegacyErrorWrapsSentinelAndCause(t *testing.T) {
76+ cause := errors.New("legacy cause")
77+ err := uncertainLegacyError("read", cause)
78+ require.ErrorIs(t, err, ports.ErrLegacySnapshotUncertain)
79+ require.ErrorIs(t, err, cause)
44 }80 }
45 81
46 func TestHasLegacyState(t *testing.T) {82 func TestHasLegacyState(t *testing.T) {
M internal/adapters/snapshot/publish_efficiency_test.go +7 -8
@@ -120,11 +120,11 @@ func TestRepositoryPublishVerifiesNecessaryExistingObjectOnce(t *testing.T) {
120 }120 }
121 second := repositoryPublicationAfter(t, repo, "named", 2, []byte("second"))121 second := repositoryPublicationAfter(t, repo, "named", 2, []byte("second"))
122 newTail := second.Objects[0]122 newTail := second.Objects[0]
123- key := legacyIncarnationID(second.Name).String()123+ path := repo.objectPath(second.IncarnationID, newTail.Digest)
124- if err := os.MkdirAll(filepath.Dir(repo.legacyObjectPath(key, newTail.Digest)), 0o700); err != nil {124+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
125 t.Fatal(err)125 t.Fatal(err)
126 }126 }
127- if err := os.WriteFile(repo.legacyObjectPath(key, newTail.Digest), newTail.Data, 0o600); err != nil {127+ if err := os.WriteFile(path, newTail.Data, 0o600); err != nil {
128 t.Fatal(err)128 t.Fatal(err)
129 }129 }
130 130
@@ -178,7 +178,6 @@ func largeIncrementalPublications(t testing.TB, name string, count int) (ports.S
178 178
179 func seedCompletePublication(t testing.TB, repo *Repository, publication ports.SnapshotPublication) {179 func seedCompletePublication(t testing.TB, repo *Repository, publication ports.SnapshotPublication) {
180 t.Helper()180 t.Helper()
181- key := legacyIncarnationID(publication.Name).String()
182 if err := repo.ensureSession(publication.IncarnationID); err != nil {181 if err := repo.ensureSession(publication.IncarnationID); err != nil {
183 t.Fatal(err)182 t.Fatal(err)
184 }183 }
@@ -187,17 +186,17 @@ func seedCompletePublication(t testing.TB, repo *Repository, publication ports.S
187 t.Fatal(err)186 t.Fatal(err)
188 }187 }
189 for _, object := range publication.Objects {188 for _, object := range publication.Objects {
190- if err := os.MkdirAll(filepath.Dir(repo.legacyObjectPath(key, object.Digest)), 0o700); err != nil {189+ if err := os.MkdirAll(filepath.Dir(repo.objectPath(publication.IncarnationID, object.Digest)), 0o700); err != nil {
191 t.Fatal(err)190 t.Fatal(err)
192 }191 }
193- if err := os.WriteFile(repo.legacyObjectPath(key, object.Digest), object.Data, 0o600); err != nil {192+ if err := os.WriteFile(repo.objectPath(publication.IncarnationID, object.Digest), object.Data, 0o600); err != nil {
194 t.Fatal(err)193 t.Fatal(err)
195 }194 }
196 }195 }
197- if err := os.WriteFile(repo.legacyManifestPath(key, publication.Generation), publication.Manifest, 0o600); err != nil {196+ if err := os.WriteFile(repo.manifestPath(publication.IncarnationID, publication.Generation), publication.Manifest, 0o600); err != nil {
198 t.Fatal(err)197 t.Fatal(err)
199 }198 }
200- if err := os.WriteFile(repo.legacyHeadPath(key), marshalHead(publication.Generation, sha256.Sum256(publication.Manifest)), 0o600); err != nil {199+ if err := os.WriteFile(repo.headPath(publication.IncarnationID), marshalHead(publication.Generation, sha256.Sum256(publication.Manifest)), 0o600); err != nil {
201 t.Fatal(err)200 t.Fatal(err)
202 }201 }
203 if refs := manifestRefs(manifest); refs == nil {202 if refs := manifestRefs(manifest); refs == nil {
M internal/adapters/snapshot/reconciliation_test.go +2 -2
@@ -2,13 +2,13 @@ package snapshot
2 2
3 import (3 import (
4 "context"4 "context"
5- "os"
6 "path/filepath"5 "path/filepath"
7 "testing"6 "testing"
8 7
9 "github.com/bnema/vev/internal/domain"8 "github.com/bnema/vev/internal/domain"
10 "github.com/bnema/vev/internal/ports"9 "github.com/bnema/vev/internal/ports"
11 codec "github.com/bnema/vev/internal/usecase/snapshot"10 codec "github.com/bnema/vev/internal/usecase/snapshot"
11+ "github.com/bnema/vev/pkg/safedir"
12 "github.com/stretchr/testify/require"12 "github.com/stretchr/testify/require"
13 )13 )
14 14
@@ -108,7 +108,7 @@ func TestReconciliationCursor(t *testing.T) {
108 sessions := filepath.Join(repo.dir, repositorySessionsDir)108 sessions := filepath.Join(repo.dir, repositorySessionsDir)
109 for i := byte(20); i < 26; i++ {109 for i := byte(20); i < 26; i++ {
110 id := domain.IncarnationID{i}110 id := domain.IncarnationID{i}
111- require.NoError(t, os.Mkdir(filepath.Join(sessions, id.String()), 0o700))111+ require.NoError(t, safedir.EnsurePrivate(filepath.Join(sessions, id.String())))
112 }112 }
113 113
114 cursor := ports.ReconcileCursor{}114 cursor := ports.ReconcileCursor{}
M internal/adapters/snapshot/recovery_crash_test.go +16 -7
@@ -13,6 +13,7 @@ import (
13 "github.com/bnema/vev/internal/persist"13 "github.com/bnema/vev/internal/persist"
14 "github.com/bnema/vev/internal/ports"14 "github.com/bnema/vev/internal/ports"
15 "github.com/bnema/vev/internal/usecase/recovery"15 "github.com/bnema/vev/internal/usecase/recovery"
16+ "github.com/bnema/vev/pkg/safedir"
16 "github.com/stretchr/testify/require"17 "github.com/stretchr/testify/require"
17 )18 )
18 19
@@ -57,7 +58,7 @@ func TestDeletionRestartsAtInternalQuarantineBoundaries(t *testing.T) {
57 record := domain.CatalogueRecord{Name: "work", IncarnationID: oldID, RecoveryState: domain.RecoveryFresh}58 record := domain.CatalogueRecord{Name: "work", IncarnationID: oldID, RecoveryState: domain.RecoveryFresh}
58 require.NoError(t, catalogue.Create(record))59 require.NoError(t, catalogue.Create(record))
59 sessionPath := repository.sessionPath(oldID)60 sessionPath := repository.sessionPath(oldID)
60- require.NoError(t, os.MkdirAll(sessionPath, 0o700))61+ require.NoError(t, safedir.EnsurePrivate(sessionPath))
61 require.NoError(t, os.WriteFile(filepath.Join(sessionPath, "durable"), []byte("old namespace"), 0o600))62 require.NoError(t, os.WriteFile(filepath.Join(sessionPath, "durable"), []byte("old namespace"), 0o600))
62 require.NoError(t, os.WriteFile(filepath.Join(h.snapshotDir, filenameForName(record.Name)), []byte("old legacy"), 0o600))63 require.NoError(t, os.WriteFile(filepath.Join(h.snapshotDir, filenameForName(record.Name)), []byte("old legacy"), 0o600))
63 tc.inject(repository)64 tc.inject(repository)
@@ -69,7 +70,8 @@ func TestDeletionRestartsAtInternalQuarantineBoundaries(t *testing.T) {
69 // Simulate a new process: none of the adapter or coordinator state survives.70 // Simulate a new process: none of the adapter or coordinator state survives.
70 catalogue, repository, journal = h.open(t)71 catalogue, repository, journal = h.open(t)
71 require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx))72 require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx))
72- _, exists := catalogue.Record(record.Name)73+ _, exists, err := catalogue.Record(record.Name)
74+ require.NoError(t, err)
73 require.False(t, exists)75 require.False(t, exists)
74 require.NoDirExists(t, repository.sessionPath(oldID))76 require.NoDirExists(t, repository.sessionPath(oldID))
75 require.Equal(t, []byte("old namespace"), mustReadCrashFile(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot", "durable")))77 require.Equal(t, []byte("old namespace"), mustReadCrashFile(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot", "durable")))
@@ -84,13 +86,14 @@ func TestDeletionRestartsAtInternalQuarantineBoundaries(t *testing.T) {
84 require.NoError(t, err)86 require.NoError(t, err)
85 require.Equal(t, newID, created.IncarnationID)87 require.Equal(t, newID, created.IncarnationID)
86 newPath := repository.sessionPath(newID)88 newPath := repository.sessionPath(newID)
87- require.NoError(t, os.MkdirAll(newPath, 0o700))89+ require.NoError(t, safedir.EnsurePrivate(newPath))
88 require.NoError(t, os.WriteFile(filepath.Join(newPath, "durable"), []byte("new namespace"), 0o600))90 require.NoError(t, os.WriteFile(filepath.Join(newPath, "durable"), []byte("new namespace"), 0o600))
89 require.NoError(t, catalogue.Close())91 require.NoError(t, catalogue.Close())
90 92
91 catalogue, repository, journal = h.open(t)93 catalogue, repository, journal = h.open(t)
92 require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx))94 require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx))
93- got, exists := catalogue.Record(record.Name)95+ got, exists, err := catalogue.Record(record.Name)
96+ require.NoError(t, err)
94 require.True(t, exists)97 require.True(t, exists)
95 require.Equal(t, newID, got.IncarnationID)98 require.Equal(t, newID, got.IncarnationID)
96 require.Equal(t, []byte("new namespace"), mustReadCrashFile(t, filepath.Join(repository.sessionPath(newID), "durable")))99 require.Equal(t, []byte("new namespace"), mustReadCrashFile(t, filepath.Join(repository.sessionPath(newID), "durable")))
@@ -114,20 +117,26 @@ func TestCreatePreCommitRestartLeavesNoAdoptableNamespace(t *testing.T) {
114 interruptedID := domain.IncarnationID{3}117 interruptedID := domain.IncarnationID{3}
115 _, err := recovery.NewCoordinator(rejectCreateCatalogue{Catalogue: catalogue}, repository, journal, bytes.NewReader(interruptedID[:])).Create(ctx, domain.CatalogueRecord{Name: "work"})118 _, err := recovery.NewCoordinator(rejectCreateCatalogue{Catalogue: catalogue}, repository, journal, bytes.NewReader(interruptedID[:])).Create(ctx, domain.CatalogueRecord{Name: "work"})
116 require.ErrorIs(t, err, errDeletionBoundaryCrash)119 require.ErrorIs(t, err, errDeletionBoundaryCrash)
117- require.Empty(t, catalogue.Records())120+ records, err := catalogue.Records()
121+ require.NoError(t, err)
122+ require.Empty(t, records)
118 require.NoDirExists(t, filepath.Join(h.snapshotDir, repositorySessionsDir))123 require.NoDirExists(t, filepath.Join(h.snapshotDir, repositorySessionsDir))
119 require.NoError(t, catalogue.Close())124 require.NoError(t, catalogue.Close())
120 125
121 catalogue, repository, journal = h.open(t)126 catalogue, repository, journal = h.open(t)
122 require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx))127 require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx))
123- require.Empty(t, catalogue.Records())128+ records, err = catalogue.Records()
129+ require.NoError(t, err)
130+ require.Empty(t, records)
124 require.NoDirExists(t, repository.sessionPath(interruptedID))131 require.NoDirExists(t, repository.sessionPath(interruptedID))
125 132
126 retryID := domain.IncarnationID{4}133 retryID := domain.IncarnationID{4}
127 created, err := recovery.NewCoordinator(catalogue, repository, journal, bytes.NewReader(retryID[:])).Create(ctx, domain.CatalogueRecord{Name: "work"})134 created, err := recovery.NewCoordinator(catalogue, repository, journal, bytes.NewReader(retryID[:])).Create(ctx, domain.CatalogueRecord{Name: "work"})
128 require.NoError(t, err)135 require.NoError(t, err)
129 require.Equal(t, retryID, created.IncarnationID)136 require.Equal(t, retryID, created.IncarnationID)
130- require.Len(t, catalogue.Records(), 1)137+ records, err = catalogue.Records()
138+ require.NoError(t, err)
139+ require.Len(t, records, 1)
131 require.NoDirExists(t, repository.sessionPath(interruptedID))140 require.NoDirExists(t, repository.sessionPath(interruptedID))
132 require.NoError(t, catalogue.Close())141 require.NoError(t, catalogue.Close())
133 }142 }
M internal/adapters/snapshot/recovery_observability_test.go +58 -20
@@ -14,25 +14,63 @@ import (
14 )14 )
15 15
16 func TestGCBudgetDiagnostic(t *testing.T) {16 func TestGCBudgetDiagnostic(t *testing.T) {
17- var buffer bytes.Buffer17+ for _, tc := range []struct {
18- log := slog.New(slog.NewJSONHandler(&buffer, nil))18+ name string
19- repository := NewRepository(t.TempDir(), log)19+ arrange func(*testing.T, *Repository) ports.RetentionPlan
20- id := domain.IncarnationID{1}20+ budget ports.MaintenanceBudget
21- done, err := repository.MaintainSession(context.Background(), ports.RetentionPlan{21+ wantDone bool
22- IncarnationID: id,22+ wantExhausted bool
23- Keep: []ports.CheckpointRef{{Generation: 3}, {Generation: 2}, {Generation: 1}},23+ wantConsumption bool
24- }, ports.MaintenanceBudget{})24+ }{
25- require.NoError(t, err)25+ {
26- require.False(t, done)26+ name: "exhausted",
27+ arrange: func(_ *testing.T, _ *Repository) ports.RetentionPlan {
28+ return ports.RetentionPlan{IncarnationID: domain.IncarnationID{1}, Keep: []ports.CheckpointRef{{Generation: 3}, {Generation: 2}, {Generation: 1}}}
29+ },
30+ wantExhausted: true,
31+ },
32+ {
33+ name: "sufficient",
34+ arrange: func(t *testing.T, repository *Repository) ports.RetentionPlan {
35+ publication := publishMaintenanceGenerations(t, repository, "diagnostic", 1)[0]
36+ return ports.RetentionPlan{IncarnationID: publication.IncarnationID}
37+ },
38+ budget: ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20},
39+ wantDone: true,
40+ wantConsumption: true,
41+ },
42+ } {
43+ t.Run(tc.name, func(t *testing.T) {
44+ var buffer bytes.Buffer
45+ log := slog.New(slog.NewJSONHandler(&buffer, nil))
46+ repository := NewRepository(privateDir(t), log)
47+ plan := tc.arrange(t, repository)
48+ buffer.Reset()
27 49
28- scanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes()))50+ done, err := repository.MaintainSession(context.Background(), plan, tc.budget)
29- require.True(t, scanner.Scan())51+ require.NoError(t, err)
30- var entry map[string]any52+ require.Equal(t, tc.wantDone, done)
31- require.NoError(t, json.Unmarshal(scanner.Bytes(), &entry))53+
32- require.NoError(t, scanner.Err())54+ scanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes()))
33- require.Equal(t, "snapshot_maintenance_progress", entry["msg"])55+ require.True(t, scanner.Scan())
34- require.Equal(t, id.String(), entry["incarnation"])56+ var entry map[string]any
35- require.EqualValues(t, 3, entry["retained"])57+ require.NoError(t, json.Unmarshal(scanner.Bytes(), &entry))
36- require.Contains(t, entry, "cursor")58+ require.NoError(t, scanner.Err())
37- require.Equal(t, true, entry["budget_exhausted"])59+ require.Equal(t, "snapshot_maintenance_progress", entry["msg"])
60+ require.Equal(t, plan.IncarnationID.String(), entry["incarnation"])
61+ require.Contains(t, entry, "cursor")
62+ require.Equal(t, tc.wantExhausted, entry["budget_exhausted"])
63+ consumedEntries, entriesOK := entry["consumed_entries"].(float64)
64+ consumedBytes, bytesOK := entry["consumed_bytes"].(float64)
65+ require.True(t, entriesOK)
66+ require.True(t, bytesOK)
67+ if tc.wantConsumption {
68+ require.Positive(t, consumedEntries)
69+ require.Positive(t, consumedBytes)
70+ } else {
71+ require.Zero(t, consumedEntries)
72+ require.Zero(t, consumedBytes)
73+ }
74+ })
75+ }
38 }76 }
M internal/app/daemon_integration_test.go +27 -18
@@ -453,15 +453,13 @@ func TestMultipleClientsOneLifecycleOwner(t *testing.T) {
453 errs := make(chan error, clients)453 errs := make(chan error, clients)
454 var wg sync.WaitGroup454 var wg sync.WaitGroup
455 for range clients {455 for range clients {
456- wg.Add(1)456+ wg.Go(func() {
457- go func() {
458- defer wg.Done()
459 _, err := ensureDaemonWithLifecycle(context.Background(), dir, dial, func() error {457 _, err := ensureDaemonWithLifecycle(context.Background(), dir, dial, func() error {
460 spawns.Add(1)458 spawns.Add(1)
461 return nil459 return nil
462 }, cfg)460 }, cfg)
463 errs <- err461 errs <- err
464- }()462+ })
465 }463 }
466 require.Never(t, func() bool { return spawns.Load() != 0 }, 50*time.Millisecond, time.Millisecond)464 require.Never(t, func() bool { return spawns.Load() != 0 }, 50*time.Millisecond, time.Millisecond)
467 close(ready)465 close(ready)
@@ -537,7 +535,7 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) {
537 closed: make(chan struct{}),535 closed: make(chan struct{}),
538 }536 }
539 repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots"))537 repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots"))
540- shutdownClock := newLifecycleShutdownClock()538+ shutdownClock := newLifecycleShutdownClock(t)
541 reconciler := &lifecycleBlockingReconciler{539 reconciler := &lifecycleBlockingReconciler{
542 entered: make(chan struct{}),540 entered: make(chan struct{}),
543 release: make(chan struct{}),541 release: make(chan struct{}),
@@ -564,7 +562,7 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) {
564 pty.NewFactory(),562 pty.NewFactory(),
565 shutdownClock,563 shutdownClock,
566 discardLog(),564 discardLog(),
567- daemon.WithSnapshotRepository(repository, nil),565+ daemon.WithSnapshotRepository(repository),
568 daemon.WithDurableMaintenance(catalogue, repository, reconciler, nil),566 daemon.WithDurableMaintenance(catalogue, repository, reconciler, nil),
569 daemon.WithCatalogue(catalogue, nil),567 daemon.WithCatalogue(catalogue, nil),
570 )568 )
@@ -679,7 +677,7 @@ func TestLifecycleOwnershipOutlivesRestorationWriter(t *testing.T) {
679 releaseRestore := func() { releaseOnce.Do(func() { close(blocking.release) }) }677 releaseRestore := func() { releaseOnce.Do(func() { close(blocking.release) }) }
680 t.Cleanup(releaseRestore)678 t.Cleanup(releaseRestore)
681 679
682- shutdownClock := newLifecycleShutdownClock()680+ shutdownClock := newLifecycleShutdownClock(t)
683 ctx, cancel := context.WithCancel(context.Background())681 ctx, cancel := context.WithCancel(context.Background())
684 t.Cleanup(cancel)682 t.Cleanup(cancel)
685 listenerReady := make(chan struct{})683 listenerReady := make(chan struct{})
@@ -707,7 +705,7 @@ func TestLifecycleOwnershipOutlivesRestorationWriter(t *testing.T) {
707 discardLog(),705 discardLog(),
708 daemon.WithShell("/bin/cat", nil),706 daemon.WithShell("/bin/cat", nil),
709 daemon.WithCatalogue(opened.Catalogue, opened.Records),707 daemon.WithCatalogue(opened.Catalogue, opened.Records),
710- daemon.WithSnapshotRepository(blocking, nil),708+ daemon.WithSnapshotRepository(blocking),
711 )709 )
712 return d.Serve(ctx, listener)710 return d.Serve(ctx, listener)
713 })711 })
@@ -764,7 +762,7 @@ func publishRestorableCheckpoint(t *testing.T, stateDir string, repository *snap
764 discardLog(),762 discardLog(),
765 daemon.WithShell("/bin/cat", nil),763 daemon.WithShell("/bin/cat", nil),
766 daemon.WithCatalogue(opened.Catalogue, opened.Records),764 daemon.WithCatalogue(opened.Catalogue, opened.Records),
767- daemon.WithSnapshotRepository(repository, nil),765+ daemon.WithSnapshotRepository(repository),
768 daemon.WithRecoveryCoordinator(coordinator),766 daemon.WithRecoveryCoordinator(coordinator),
769 )767 )
770 served := make(chan error, 1)768 served := make(chan error, 1)
@@ -773,7 +771,7 @@ func publishRestorableCheckpoint(t *testing.T, stateDir string, repository *snap
773 tr, _ := attach(t, runtimeDir, ports.IntentNew, name, domain.Size{Cols: 80, Rows: 24})771 tr, _ := attach(t, runtimeDir, ports.IntentNew, name, domain.Size{Cols: 80, Rows: 24})
774 require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgInput, Payload: ports.MarshalInput(ports.Input{Data: []byte("checkpoint me\n")})}))772 require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgInput, Payload: ports.MarshalInput(ports.Input{Data: []byte("checkpoint me\n")})}))
775 require.Eventually(t, func() bool {773 require.Eventually(t, func() bool {
776- record, ok := opened.Catalogue.Record(name)774+ record, ok, _ := opened.Catalogue.Record(name)
777 return ok && record.RecoveryState == domain.RecoveryHealthy && record.Committed != nil775 return ok && record.RecoveryState == domain.RecoveryHealthy && record.Committed != nil
778 }, 5*time.Second, 10*time.Millisecond, "session never committed a checkpoint")776 }, 5*time.Second, 10*time.Millisecond, "session never committed a checkpoint")
779 require.NoError(t, tr.Close())777 require.NoError(t, tr.Close())
@@ -826,7 +824,7 @@ func runBlockedSnapshotWriterShutdown(t *testing.T) blockedSnapshotWriterShutdow
826 t.Helper()824 t.Helper()
827 runtimeDir := filepath.Join(t.TempDir(), "runtime")825 runtimeDir := filepath.Join(t.TempDir(), "runtime")
828 stateDir := filepath.Join(t.TempDir(), "state")826 stateDir := filepath.Join(t.TempDir(), "state")
829- shutdownClock := newLifecycleShutdownClock()827+ shutdownClock := newLifecycleShutdownClock(t)
830 repository := newLifecycleBlockingRepository()828 repository := newLifecycleBlockingRepository()
831 ctx, cancel := context.WithCancel(context.Background())829 ctx, cancel := context.WithCancel(context.Background())
832 t.Cleanup(cancel)830 t.Cleanup(cancel)
@@ -849,7 +847,7 @@ func runBlockedSnapshotWriterShutdown(t *testing.T) blockedSnapshotWriterShutdow
849 shutdownClock,847 shutdownClock,
850 discardLog(),848 discardLog(),
851 daemon.WithShell("/bin/cat", nil),849 daemon.WithShell("/bin/cat", nil),
852- daemon.WithSnapshotRepository(repository, nil),850+ daemon.WithSnapshotRepository(repository),
853 )851 )
854 return d.Serve(ctx, observed)852 return d.Serve(ctx, observed)
855 })853 })
@@ -913,21 +911,28 @@ func (r *lifecycleBlockingRepository) Publish(context.Context, ports.SnapshotPub
913 }911 }
914 912
915 type lifecycleShutdownClock struct {913 type lifecycleShutdownClock struct {
914+ t *testing.T
916 base ports.Clock915 base ports.Clock
917 timers chan *lifecycleManualTimer916 timers chan *lifecycleManualTimer
918 }917 }
919 918
920-func newLifecycleShutdownClock() *lifecycleShutdownClock {919+func newLifecycleShutdownClock(t *testing.T) *lifecycleShutdownClock {
921- return &lifecycleShutdownClock{base: clock.New(), timers: make(chan *lifecycleManualTimer, 4)}920+ t.Helper()
921+ return &lifecycleShutdownClock{t: t, base: clock.New(), timers: make(chan *lifecycleManualTimer, 4)}
922 }922 }
923 923
924 func (c *lifecycleShutdownClock) Now() time.Time { return c.base.Now() }924 func (c *lifecycleShutdownClock) Now() time.Time { return c.base.Now() }
925 func (c *lifecycleShutdownClock) NewTimer(delay time.Duration) ports.Timer {925 func (c *lifecycleShutdownClock) NewTimer(delay time.Duration) ports.Timer {
926- if delay != time.Second {926+ if delay != daemon.SnapshotShutdownTimeout() {
927 return c.base.NewTimer(delay)927 return c.base.NewTimer(delay)
928 }928 }
929- timer := &lifecycleManualTimer{ch: make(chan time.Time, 1)}929+ timer := &lifecycleManualTimer{t: c.t, ch: make(chan time.Time, 1)}
930- c.timers <- timer930+ select {
931+ case c.timers <- timer:
932+ default:
933+ c.t.Errorf("unexpected additional snapshot shutdown timer with delay %s", delay)
934+ timer.fire()
935+ }
931 return timer936 return timer
932 }937 }
933 938
@@ -943,12 +948,16 @@ func (c *lifecycleShutdownClock) nextTimer(t *testing.T) *lifecycleManualTimer {
943 }948 }
944 949
945 type lifecycleManualTimer struct {950 type lifecycleManualTimer struct {
951+ t *testing.T
946 ch chan time.Time952 ch chan time.Time
947 stopped atomic.Bool953 stopped atomic.Bool
948 }954 }
949 955
950 func (t *lifecycleManualTimer) C() <-chan time.Time { return t.ch }956 func (t *lifecycleManualTimer) C() <-chan time.Time { return t.ch }
951-func (t *lifecycleManualTimer) Reset(time.Duration) bool {957+func (t *lifecycleManualTimer) Reset(delay time.Duration) bool {
958+ if delay != daemon.SnapshotShutdownTimeout() {
959+ t.t.Errorf("snapshot shutdown timer reset with delay %s, want %s", delay, daemon.SnapshotShutdownTimeout())
960+ }
952 t.stopped.Store(false)961 t.stopped.Store(false)
953 return false962 return false
954 }963 }
M internal/app/durable_crash_matrix_test.go +7 -7
@@ -132,7 +132,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) {
132 require.NoError(t, catalogue.Close())132 require.NoError(t, catalogue.Close())
133 133
134 h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {134 h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {
135- record, exists := catalogue.Record("work")135+ record, exists, _ := catalogue.Record("work")
136 require.True(t, exists)136 require.True(t, exists)
137 require.Equal(t, domain.IncarnationID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, record.IncarnationID)137 require.Equal(t, domain.IncarnationID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, record.IncarnationID)
138 require.Equal(t, domain.RecoveryFresh, record.RecoveryState)138 require.Equal(t, domain.RecoveryFresh, record.RecoveryState)
@@ -160,7 +160,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) {
160 160
161 ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)}161 ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)}
162 h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {162 h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
163- stored, exists := catalogue.Record(record.Name)163+ stored, exists, _ := catalogue.Record(record.Name)
164 require.True(t, exists)164 require.True(t, exists)
165 if boundaryName == "snapshot-publish" {165 if boundaryName == "snapshot-publish" {
166 require.Equal(t, domain.RecoveryFresh, stored.RecoveryState)166 require.Equal(t, domain.RecoveryFresh, stored.RecoveryState)
@@ -189,8 +189,8 @@ func TestDurableTransactionCrashMatrix(t *testing.T) {
189 require.NoError(t, catalogue.Close())189 require.NoError(t, catalogue.Close())
190 190
191 h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {191 h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {
192- _, oldExists := catalogue.Record("old")192+ _, oldExists, _ := catalogue.Record("old")
193- renamed, newExists := catalogue.Record("new")193+ renamed, newExists, _ := catalogue.Record("new")
194 require.False(t, oldExists)194 require.False(t, oldExists)
195 require.True(t, newExists)195 require.True(t, newExists)
196 require.Equal(t, record.IncarnationID, renamed.IncarnationID)196 require.Equal(t, record.IncarnationID, renamed.IncarnationID)
@@ -228,7 +228,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) {
228 require.NoError(t, catalogue.Close())228 require.NoError(t, catalogue.Close())
229 229
230 h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {230 h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
231- _, exists := catalogue.Record(record.Name)231+ _, exists, _ := catalogue.Record(record.Name)
232 require.False(t, exists)232 require.False(t, exists)
233 assertNoTombstones(t, repository)233 assertNoTombstones(t, repository)
234 require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String()))234 require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String()))
@@ -275,7 +275,7 @@ func TestDurableDiscardCrashMatrix(t *testing.T) {
275 require.NoError(t, catalogue.Close())275 require.NoError(t, catalogue.Close())
276 276
277 h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) {277 h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) {
278- got, exists := catalogue.Record(old.Name)278+ got, exists, _ := catalogue.Record(old.Name)
279 require.True(t, exists)279 require.True(t, exists)
280 require.Equal(t, newID, got.IncarnationID)280 require.Equal(t, newID, got.IncarnationID)
281 require.Equal(t, domain.RecoveryFresh, got.RecoveryState)281 require.Equal(t, domain.RecoveryFresh, got.RecoveryState)
@@ -312,7 +312,7 @@ func TestDeletionRecoveryProtectsReusedNameFilesystemBytes(t *testing.T) {
312 require.NoError(t, catalogue.Close())312 require.NoError(t, catalogue.Close())
313 313
314 h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {314 h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
315- got, exists := catalogue.Record("work")315+ got, exists, _ := catalogue.Record("work")
316 require.True(t, exists)316 require.True(t, exists)
317 require.Equal(t, replacement, got)317 require.Equal(t, replacement, got)
318 require.Equal(t, replacementBytes, readTree(t, replacementPath))318 require.Equal(t, replacementBytes, readTree(t, replacementPath))
M internal/app/offline_kill_retry_test.go +39 -46
@@ -14,50 +14,43 @@ import (
14 )14 )
15 15
16 func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) {16 func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) {
17- stateRoot, runtimeRoot := t.TempDir(), t.TempDir()17+ tests := []struct {
18- t.Setenv("XDG_STATE_HOME", stateRoot)18+ name string
19- t.Setenv("XDG_RUNTIME_DIR", runtimeRoot)19+ crashHook func(*testing.T, string)
20-20+ }{
21- id := domain.IncarnationID{1}21+ {name: "normal"},
22- p := newTestPersister(t, filepath.Join(stateRoot, "vev"))22+ {
23- now := time.Now().UnixNano()23+ name: ".prev crash hook",
24- require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: id, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh}))24+ crashHook: func(t *testing.T, stateDir string) {
25- require.NoError(t, p.Close())25+ t.Helper()
26- source := filepath.Join(stateRoot, "vev", "snapshots", "sessions", id.String())26+ storePath := persist.StorePath(stateDir)
27- require.NoError(t, os.MkdirAll(source, 0o700))27+ require.NoError(t, os.Rename(storePath, storePath+".prev"))
28- require.NoError(t, os.WriteFile(filepath.Join(source, "sentinel"), []byte("durable"), 0o600))28+ },
29-29+ },
30- require.NoError(t, runKill(context.Background(), "named", false, false))30+ }
31- require.NoFileExists(t, filepath.Join(source, "sentinel"))31+
32- require.FileExists(t, filepath.Join(stateRoot, "vev", "snapshots", "quarantine", id.String(), "snapshot", "sentinel"))32+ for _, tt := range tests {
33-}33+ t.Run(tt.name, func(t *testing.T) {
34-34+ stateRoot, runtimeRoot := t.TempDir(), t.TempDir()
35-// TestRunKillOfflineFindsSessionWhenOnlyPrevCandidateExists covers the same35+ t.Setenv("XDG_STATE_HOME", stateRoot)
36-// bug class as persist.LoadCatalogueReadOnly's fresh-install regression: a36+ t.Setenv("XDG_RUNTIME_DIR", runtimeRoot)
37-// crash mid-compaction can leave sessions.kv absent with only a .prev (or37+ stateDir := filepath.Join(stateRoot, "vev")
38-// .next) candidate on disk. `vev kill <name>` must still find and delete the38+
39-// session rather than reporting "no such session" from a bare os.Stat39+ id := domain.IncarnationID{1}
40-// precheck on the fixed sessions.kv path.40+ p := newTestPersister(t, stateDir)
41-func TestRunKillOfflineFindsSessionWhenOnlyPrevCandidateExists(t *testing.T) {41+ now := time.Now().UnixNano()
42- stateRoot, runtimeRoot := t.TempDir(), t.TempDir()42+ require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: id, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh}))
43- t.Setenv("XDG_STATE_HOME", stateRoot)43+ require.NoError(t, p.Close())
44- t.Setenv("XDG_RUNTIME_DIR", runtimeRoot)44+ source := filepath.Join(stateDir, "snapshots", "sessions", id.String())
45-45+ require.NoError(t, os.MkdirAll(source, 0o700))
46- id := domain.IncarnationID{1}46+ require.NoError(t, os.WriteFile(filepath.Join(source, "sentinel"), []byte("durable"), 0o600))
47- p := newTestPersister(t, filepath.Join(stateRoot, "vev"))47+ if tt.crashHook != nil {
48- now := time.Now().UnixNano()48+ tt.crashHook(t, stateDir)
49- require.NoError(t, p.Save(persist.Record{Name: "crashed", IncarnationID: id, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh}))49+ }
50- require.NoError(t, p.Close())50+
51- source := filepath.Join(stateRoot, "vev", "snapshots", "sessions", id.String())51+ require.NoError(t, runKill(context.Background(), "named", false, false))
52- require.NoError(t, os.MkdirAll(source, 0o700))52+ require.NoFileExists(t, filepath.Join(source, "sentinel"))
53- require.NoError(t, os.WriteFile(filepath.Join(source, "sentinel"), []byte("durable"), 0o600))53+ require.FileExists(t, filepath.Join(stateDir, "snapshots", "quarantine", id.String(), "snapshot", "sentinel"))
54-54+ })
55- // Simulate a crash mid-compaction: current renamed away to .prev, no55+ }
56- // sessions.kv on disk at all.
57- storePath := persist.StorePath(filepath.Join(stateRoot, "vev"))
58- require.NoError(t, os.Rename(storePath, storePath+".prev"))
59-
60- require.NoError(t, runKill(context.Background(), "crashed", false, false))
61- require.NoFileExists(t, filepath.Join(source, "sentinel"))
62- require.FileExists(t, filepath.Join(stateRoot, "vev", "snapshots", "quarantine", id.String(), "snapshot", "sentinel"))
63 }56 }
M internal/app/run.go +29 -8
@@ -583,18 +583,22 @@ type fencedRecoveryIdentity struct {
583 583
584 // fencedRecoveries resolves each conflict against the post-recovery catalogue584 // fencedRecoveries resolves each conflict against the post-recovery catalogue
585 // so a fenced session is reported with the incarnation it actually holds now.585 // so a fenced session is reported with the incarnation it actually holds now.
586-func fencedRecoveries(conflicts []recovery.RecoveryConflict, catalogue ports.Catalogue) []fencedRecoveryIdentity {586+func fencedRecoveries(conflicts []recovery.RecoveryConflict, catalogue ports.Catalogue) ([]fencedRecoveryIdentity, error) {
587 fenced := make([]fencedRecoveryIdentity, 0, len(conflicts))587 fenced := make([]fencedRecoveryIdentity, 0, len(conflicts))
588 for _, conflict := range conflicts {588 for _, conflict := range conflicts {
589 identity := fencedRecoveryIdentity{name: conflict.Session, kind: conflict.Kind, reasonCode: fencedRecoveryReasonCode(conflict.Err)}589 identity := fencedRecoveryIdentity{name: conflict.Session, kind: conflict.Kind, reasonCode: fencedRecoveryReasonCode(conflict.Err)}
590 if catalogue != nil {590 if catalogue != nil {
591- if record, ok := catalogue.Record(conflict.Session); ok {591+ record, ok, err := catalogue.Record(conflict.Session)
592+ if err != nil {
593+ return nil, err
594+ }
595+ if ok {
592 identity.id = record.IncarnationID596 identity.id = record.IncarnationID
593 }597 }
594 }598 }
595 fenced = append(fenced, identity)599 fenced = append(fenced, identity)
596 }600 }
597- return fenced601+ return fenced, nil
598 }602 }
599 603
600 func fencedRecoveryReasonCode(err error) string {604 func fencedRecoveryReasonCode(err error) string {
@@ -674,6 +678,13 @@ func runDaemonOwned(ctx context.Context) (retErr error) {
674 return runDaemonOwnedWithLogger(ctx, log)678 return runDaemonOwnedWithLogger(ctx, log)
675 }679 }
676 680
681+func recoverAuthoritativeCatalogue(ctx context.Context, coordinator *recovery.Coordinator, catalogue ports.Catalogue) ([]domain.CatalogueRecord, error) {
682+ if err := coordinator.Recover(ctx); err != nil {
683+ return nil, err
684+ }
685+ return catalogue.Records()
686+}
687+
677 func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr error) {688 func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr error) {
678 clk := clock.New()689 clk := clock.New()
679 observer, observerCloser, err := newPerformanceTrace(clk)690 observer, observerCloser, err := newPerformanceTrace(clk)
@@ -710,7 +721,7 @@ func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr err
710 daemonOpts = append(daemonOpts, daemon.WithBarScriptCommandRunner(shellcmd.New()))721 daemonOpts = append(daemonOpts, daemon.WithBarScriptCommandRunner(shellcmd.New()))
711 daemonOpts = append(daemonOpts, daemon.WithProcessInspector(platform.NewProcessInspector()), daemon.WithDirOrHome(platform.DirOrHome))722 daemonOpts = append(daemonOpts, daemon.WithProcessInspector(platform.NewProcessInspector()), daemon.WithDirOrHome(platform.DirOrHome))
712 snapshotRepository := snapshotadapter.NewRepository(snapshotDir(), log)723 snapshotRepository := snapshotadapter.NewRepository(snapshotDir(), log)
713- daemonOpts = append(daemonOpts, daemon.WithSnapshotRepository(snapshotRepository, snapshotRepository))724+ daemonOpts = append(daemonOpts, daemon.WithSnapshotRepository(snapshotRepository))
714 noticeStore := noticefile.New(platform.StateDir())725 noticeStore := noticefile.New(platform.StateDir())
715 daemonOpts = append(daemonOpts, daemon.WithNoticeStore(noticeStore))726 daemonOpts = append(daemonOpts, daemon.WithNoticeStore(noticeStore))
716 storePath := persist.StorePath(platform.StateDir())727 storePath := persist.StorePath(platform.StateDir())
@@ -736,13 +747,21 @@ func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr err
736 return errors.Join(fmt.Errorf("vev: inspect durable session transactions: %w", err), opened.Catalogue.Close())747 return errors.Join(fmt.Errorf("vev: inspect durable session transactions: %w", err), opened.Catalogue.Close())
737 }748 }
738 coordinator := recovery.NewCoordinator(opened.Catalogue, snapshotRepository, journal, rand.Reader)749 coordinator := recovery.NewCoordinator(opened.Catalogue, snapshotRepository, journal, rand.Reader)
739- if err := coordinator.Recover(ctx); err != nil {750+ // Recovery may roll catalogue changes forward, so reload the authoritative
740- return errors.Join(fmt.Errorf("vev: recover durable session transactions: %w", err), opened.Catalogue.Close())751+ // snapshot before constructing runtime reservations. A failed reload aborts
752+ // startup rather than treating durable sessions as absent.
753+ recoveredRecords, err := recoverAuthoritativeCatalogue(ctx, coordinator, opened.Catalogue)
754+ if err != nil {
755+ return errors.Join(fmt.Errorf("vev: recover and reload durable session transactions: %w", err), opened.Catalogue.Close())
741 }756 }
757+ opened.Records = recoveredRecords
742 // Recovery fences a session instead of failing startup when only that758 // Recovery fences a session instead of failing startup when only that
743 // session's durable state is self-inconsistent. The daemon then comes up759 // session's durable state is self-inconsistent. The daemon then comes up
744 // looking healthy, so every fenced session must be reported and persisted.760 // looking healthy, so every fenced session must be reported and persisted.
745- fenced := fencedRecoveries(coordinator.Conflicts(), opened.Catalogue)761+ fenced, err := fencedRecoveries(coordinator.Conflicts(), opened.Catalogue)
762+ if err != nil {
763+ return errors.Join(fmt.Errorf("vev: read fenced recovery identities: %w", err), opened.Catalogue.Close())
764+ }
746 logFencedRecoveries(log, fenced)765 logFencedRecoveries(log, fenced)
747 if err := persistTransactionRecoveryNotices(noticeStore, pendingRecoveries, clk.Now()); err != nil {766 if err := persistTransactionRecoveryNotices(noticeStore, pendingRecoveries, clk.Now()); err != nil {
748 log.Error("transaction_recovery_notice_failed", "err", err)767 log.Error("transaction_recovery_notice_failed", "err", err)
@@ -1558,7 +1577,9 @@ func runOfflineNamedKill(ctx context.Context, name string) (retErr error) {
1558 }1577 }
1559 defer func() { retErr = errors.Join(retErr, opened.Catalogue.Close()) }()1578 defer func() { retErr = errors.Join(retErr, opened.Catalogue.Close()) }()
1560 1579
1561- if _, ok := opened.Catalogue.Record(name); !ok {1580+ if _, ok, err := opened.Catalogue.Record(name); err != nil {
1581+ return fmt.Errorf("vev: reading stored session: %w", err)
1582+ } else if !ok {
1562 return fmt.Errorf("vev: no such session: %s", name)1583 return fmt.Errorf("vev: no such session: %s", name)
1563 }1584 }
1564 coordinator := recovery.NewCoordinator(opened.Catalogue, repository, recoveryfs.New(platform.StateDir()), rand.Reader)1585 coordinator := recovery.NewCoordinator(opened.Catalogue, repository, recoveryfs.New(platform.StateDir()), rand.Reader)
M internal/app/run_test.go +52 -19
@@ -19,6 +19,7 @@ import (
19 "time"19 "time"
20 20
21 "github.com/bnema/vev/internal/adapters/lifecycle"21 "github.com/bnema/vev/internal/adapters/lifecycle"
22+ "github.com/bnema/vev/internal/adapters/noticefile"
22 "github.com/bnema/vev/internal/adapters/recoveryfs"23 "github.com/bnema/vev/internal/adapters/recoveryfs"
23 "github.com/bnema/vev/internal/adapters/snapshot"24 "github.com/bnema/vev/internal/adapters/snapshot"
24 "github.com/bnema/vev/internal/domain"25 "github.com/bnema/vev/internal/domain"
@@ -110,9 +111,6 @@ func TestRecoveryObservability(t *testing.T) {
110 require.Equal(t, "clean", recovered["outcome"])111 require.Equal(t, "clean", recovered["outcome"])
111 require.NotContains(t, eventNames(entries), "interrupted_transaction_fenced")112 require.NotContains(t, eventNames(entries), "interrupted_transaction_fenced")
112 113
113- logBuffer.Reset()
114- log.Error("catalogue_validation_failed", "path", "/state/vev/sessions.kv", "reason_code", "corrupt")
115- require.NotContains(t, eventNames(decodeJSONLogs(t, logBuffer.Bytes())), "daemon_startup_complete")
116 require.NotContains(t, fmt.Sprint(requireEvent(t, entries, "interrupted_transaction_recovery_complete")), "terminal contents")114 require.NotContains(t, fmt.Sprint(requireEvent(t, entries, "interrupted_transaction_recovery_complete")), "terminal contents")
117 require.Equal(t, domain.NoticeSnapshotRestore, recoveryNotice.Code)115 require.Equal(t, domain.NoticeSnapshotRestore, recoveryNotice.Code)
118 require.Equal(t, domain.SessionID(domain.IncarnationID{9}.String()), recoveryNotice.SessionID)116 require.Equal(t, domain.SessionID(domain.IncarnationID{9}.String()), recoveryNotice.SessionID)
@@ -122,6 +120,27 @@ func TestRecoveryObservability(t *testing.T) {
122 // A session-scoped conflict no longer aborts startup, so the daemon comes up120 // A session-scoped conflict no longer aborts startup, so the daemon comes up
123 // looking healthy. Startup must therefore name the fenced session in the log,121 // looking healthy. Startup must therefore name the fenced session in the log,
124 // persist a notice for it, and never report the recovery as clean.122 // persist a notice for it, and never report the recovery as clean.
123+func TestStartupReloadsCatalogueAfterRolledForwardDeletion(t *testing.T) {
124+ ctx := t.Context()
125+ stateDir := filepath.Join(t.TempDir(), "vev")
126+ catalogue := newTestPersister(t, stateDir)
127+ t.Cleanup(func() { _ = catalogue.Close() })
128+ repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots"))
129+ record := domain.CatalogueRecord{
130+ Name: "work", IncarnationID: domain.IncarnationID{9}, Cwd: t.TempDir(),
131+ CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryDeleting,
132+ }
133+ require.NoError(t, catalogue.Save(record))
134+ coordinator := recovery.NewCoordinator(catalogue, repository, recoveryfs.New(stateDir), rand.Reader)
135+
136+ records, err := recoverAuthoritativeCatalogue(ctx, coordinator, catalogue)
137+ require.NoError(t, err)
138+ require.Empty(t, records, "daemon startup must not retain the pre-recovery reservation")
139+ _, exists, err := catalogue.Record(record.Name)
140+ require.NoError(t, err)
141+ require.False(t, exists, "rolled-forward deletion must remain absent in authoritative state")
142+}
143+
125 func TestFencedRecoveryObservability(t *testing.T) {144 func TestFencedRecoveryObservability(t *testing.T) {
126 ctx := context.Background()145 ctx := context.Background()
127 stateDir := filepath.Join(t.TempDir(), "vev")146 stateDir := filepath.Join(t.TempDir(), "vev")
@@ -139,7 +158,8 @@ func TestFencedRecoveryObservability(t *testing.T) {
139 158
140 coordinator := recovery.NewCoordinator(catalogue, repository, recoveryfs.New(stateDir), rand.Reader)159 coordinator := recovery.NewCoordinator(catalogue, repository, recoveryfs.New(stateDir), rand.Reader)
141 require.NoError(t, coordinator.Recover(ctx), "a session-scoped conflict must not abort startup")160 require.NoError(t, coordinator.Recover(ctx), "a session-scoped conflict must not abort startup")
142- fenced := fencedRecoveries(coordinator.Conflicts(), catalogue)161+ fenced, err := fencedRecoveries(coordinator.Conflicts(), catalogue)
162+ require.NoError(t, err)
143 require.Equal(t, []fencedRecoveryIdentity{{163 require.Equal(t, []fencedRecoveryIdentity{{
144 name: live.Name, id: live.IncarnationID, kind: "deletion-tombstone", reasonCode: "deletion-tombstone-conflict",164 name: live.Name, id: live.IncarnationID, kind: "deletion-tombstone", reasonCode: "deletion-tombstone-conflict",
145 }}, fenced)165 }}, fenced)
@@ -148,7 +168,9 @@ func TestFencedRecoveryObservability(t *testing.T) {
148 log := slog.New(slog.NewJSONHandler(&logBuffer, nil))168 log := slog.New(slog.NewJSONHandler(&logBuffer, nil))
149 logFencedRecoveries(log, fenced)169 logFencedRecoveries(log, fenced)
150 logTransactionRecovery(log, 1, len(fenced))170 logTransactionRecovery(log, 1, len(fenced))
151- logStartupRecoveryCounts(log, catalogue.Records(), 0)171+ records, err := catalogue.Records()
172+ require.NoError(t, err)
173+ logStartupRecoveryCounts(log, records, 0)
152 174
153 entries := decodeJSONLogs(t, logBuffer.Bytes())175 entries := decodeJSONLogs(t, logBuffer.Bytes())
154 event := requireEvent(t, entries, "interrupted_transaction_fenced")176 event := requireEvent(t, entries, "interrupted_transaction_fenced")
@@ -163,17 +185,22 @@ func TestFencedRecoveryObservability(t *testing.T) {
163 require.Equal(t, "fenced", recovered["outcome"])185 require.Equal(t, "fenced", recovered["outcome"])
164 require.Contains(t, eventNames(entries), "daemon_startup_complete")186 require.Contains(t, eventNames(entries), "daemon_startup_complete")
165 187
166- store := portsmocks.NewMockNoticeStore(t)188+ store := noticefile.New(stateDir)
167- var notice domain.Notification
168- store.EXPECT().Append(mock.Anything).Run(func(n domain.Notification) { notice = n }).Return(nil).Once()
169 require.NoError(t, persistFencedRecoveryNotices(store, fenced, time.Unix(1, 0)))189 require.NoError(t, persistFencedRecoveryNotices(store, fenced, time.Unix(1, 0)))
190+ reopened := noticefile.New(stateDir)
191+ pending, err := reopened.Claim()
192+ require.NoError(t, err)
193+ require.Len(t, pending, 1)
194+ notice := pending[0]
170 require.Equal(t, domain.NoticeSnapshotRestore, notice.Code)195 require.Equal(t, domain.NoticeSnapshotRestore, notice.Code)
171 require.Equal(t, domain.NoticeWarn, notice.Severity)196 require.Equal(t, domain.NoticeWarn, notice.Severity)
172 require.Equal(t, domain.SessionID(live.IncarnationID.String()), notice.SessionID)197 require.Equal(t, domain.SessionID(live.IncarnationID.String()), notice.SessionID)
173 require.Contains(t, notice.Details, "session="+live.Name)198 require.Contains(t, notice.Details, "session="+live.Name)
174 require.Contains(t, notice.Details, "reason_code=deletion-tombstone-conflict")199 require.Contains(t, notice.Details, "reason_code=deletion-tombstone-conflict")
200+ require.NoError(t, reopened.Ack())
175 201
176- degraded, ok := catalogue.Record(live.Name)202+ degraded, ok, err := catalogue.Record(live.Name)
203+ require.NoError(t, err)
177 require.True(t, ok)204 require.True(t, ok)
178 require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState)205 require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState)
179 }206 }
@@ -444,19 +471,25 @@ func TestLifecycleOwnershipPrecedesDaemonStartup(t *testing.T) {
444 stateDir := filepath.Join(stateRoot, "vev")471 stateDir := filepath.Join(stateRoot, "vev")
445 t.Setenv("XDG_RUNTIME_DIR", runtimeRoot)472 t.Setenv("XDG_RUNTIME_DIR", runtimeRoot)
446 t.Setenv("XDG_STATE_HOME", stateRoot)473 t.Setenv("XDG_STATE_HOME", stateRoot)
447- ctx, cancel := context.WithCancel(context.Background())474+ ctx, cancel := context.WithCancel(t.Context())
448- defer cancel()475+ result := make(chan error, 1)
449 go func() {476 go func() {
450- for {477+ result <- runWithLifecycleOwner(ctx, runtimeDir, stateDir, runDaemonOwned)
451- if _, err := os.Stat(filepath.Join(runtimeDir, "daemon.sock")); err == nil {
452- cancel()
453- return
454- }
455- time.Sleep(time.Millisecond)
456- }
457 }()478 }()
479+ t.Cleanup(func() {
480+ cancel()
481+ select {
482+ case err := <-result:
483+ require.NoError(t, err)
484+ case <-time.After(5 * time.Second):
485+ t.Error("daemon lifecycle did not stop after cancellation")
486+ }
487+ })
458 488
459- require.NoError(t, runWithLifecycleOwner(ctx, runtimeDir, stateDir, runDaemonOwned))489+ require.Eventually(t, func() bool {
490+ _, err := os.Stat(filepath.Join(runtimeDir, "daemon.sock"))
491+ return err == nil
492+ }, 5*time.Second, time.Millisecond, "daemon socket was not published")
460 require.FileExists(t, persist.StorePath(stateDir))493 require.FileExists(t, persist.StorePath(stateDir))
461 })494 })
462 495
M internal/app/spawn.go +45 -49
@@ -67,25 +67,29 @@ var defaultBackoff = backoffConfig{
67 total: 5 * time.Second,67 total: 5 * time.Second,
68 }68 }
69 69
70-func waitForLifecycleAvailability(ctx context.Context, runtimeDir string, cfg backoffConfig) (lifecycleOwnership, error) {70+func retryAttempts[T any](ctx context.Context, cfg backoffConfig, attempt func() (T, bool, error)) (T, error) {
71 deadline := time.Now().Add(cfg.total)71 deadline := time.Now().Add(cfg.total)
72 backoff := cfg.initial72 backoff := cfg.initial
73 for {73 for {
74 if err := ctx.Err(); err != nil {74 if err := ctx.Err(); err != nil {
75- return nil, err75+ var zero T
76+ return zero, err
76 }77 }
77- owner, err := daemonLifecycleProbe.TryAcquire(runtimeDir)78+ result, done, err := attempt()
78- if err == nil {79+ if err != nil {
79- return owner, nil80+ var zero T
81+ return zero, err
80 }82 }
81- if !errors.Is(err, lifecycle.ErrBusy) {83+ if done {
82- return nil, err84+ return result, nil
83 }85 }
84 if !time.Now().Before(deadline) {86 if !time.Now().Before(deadline) {
85- return nil, ErrDaemonUnreachable87+ var zero T
88+ return zero, ErrDaemonUnreachable
86 }89 }
87 if err := waitBackoff(ctx, backoff); err != nil {90 if err := waitBackoff(ctx, backoff); err != nil {
88- return nil, err91+ var zero T
92+ return zero, err
89 }93 }
90 if backoff *= 2; backoff > cfg.max {94 if backoff *= 2; backoff > cfg.max {
91 backoff = cfg.max95 backoff = cfg.max
@@ -93,33 +97,39 @@ func waitForLifecycleAvailability(ctx context.Context, runtimeDir string, cfg ba
93 }97 }
94 }98 }
95 99
96-func waitForDaemonOrLifecycle(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, lifecycleOwnership, error) {100+func waitForLifecycleAvailability(ctx context.Context, runtimeDir string, cfg backoffConfig) (lifecycleOwnership, error) {
97- deadline := time.Now().Add(cfg.total)101+ return retryAttempts(ctx, cfg, func() (lifecycleOwnership, bool, error) {
98- backoff := cfg.initial102+ owner, err := daemonLifecycleProbe.TryAcquire(runtimeDir)
99- for {103+ if err == nil {
100- if err := ctx.Err(); err != nil {104+ return owner, true, nil
101- return nil, nil, err105+ }
106+ if !errors.Is(err, lifecycle.ErrBusy) {
107+ return nil, false, err
102 }108 }
109+ return nil, false, nil
110+ })
111+}
112+
113+type daemonOrLifecycle struct {
114+ transport ports.Transport
115+ owner lifecycleOwnership
116+}
117+
118+func waitForDaemonOrLifecycle(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, lifecycleOwnership, error) {
119+ result, err := retryAttempts(ctx, cfg, func() (daemonOrLifecycle, bool, error) {
103 if transport, err := dial(ctx, dir); err == nil {120 if transport, err := dial(ctx, dir); err == nil {
104- return transport, nil, nil121+ return daemonOrLifecycle{transport: transport}, true, nil
105 }122 }
106 owner, err := daemonLifecycleProbe.TryAcquire(dir)123 owner, err := daemonLifecycleProbe.TryAcquire(dir)
107 if err == nil {124 if err == nil {
108- return nil, owner, nil125+ return daemonOrLifecycle{owner: owner}, true, nil
109 }126 }
110 if !errors.Is(err, lifecycle.ErrBusy) {127 if !errors.Is(err, lifecycle.ErrBusy) {
111- return nil, nil, err128+ return daemonOrLifecycle{}, false, err
112 }129 }
113- if !time.Now().Before(deadline) {130+ return daemonOrLifecycle{}, false, nil
114- return nil, nil, ErrDaemonUnreachable131+ })
115- }132+ return result.transport, result.owner, err
116- if err := waitBackoff(ctx, backoff); err != nil {
117- return nil, nil, err
118- }
119- if backoff *= 2; backoff > cfg.max {
120- backoff = cfg.max
121- }
122- }
123 }133 }
124 134
125 func waitBackoff(ctx context.Context, duration time.Duration) error {135 func waitBackoff(ctx context.Context, duration time.Duration) error {
@@ -224,28 +234,14 @@ func acquireSpawnLock(dir string) (release func(), acquired bool, err error) {
224 // retryDial dials repeatedly with exponential backoff until the daemon234 // retryDial dials repeatedly with exponential backoff until the daemon
225 // answers, the context is cancelled, or the total budget is exhausted.235 // answers, the context is cancelled, or the total budget is exhausted.
226 func retryDial(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, error) {236 func retryDial(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, error) {
227- deadline := time.Now().Add(cfg.total)237+ transport, err := retryAttempts(ctx, cfg, func() (ports.Transport, bool, error) {
228- backoff := cfg.initial238+ transport, err := dial(ctx, dir)
229- for {239+ return transport, err == nil, nil
230- if err := ctx.Err(); err != nil {240+ })
231- return nil, err241+ if errors.Is(err, ErrDaemonUnreachable) {
232- }242+ slog.Error("daemon did not become reachable before retry budget expired", "socket_dir", dir, "budget", cfg.total)
233- if t, err := dial(ctx, dir); err == nil {
234- return t, nil
235- }
236- if !time.Now().Before(deadline) {
237- slog.Error("daemon did not become reachable before retry budget expired", "socket_dir", dir, "budget", cfg.total)
238- return nil, ErrDaemonUnreachable
239- }
240- select {
241- case <-ctx.Done():
242- return nil, ctx.Err()
243- case <-time.After(backoff):
244- }
245- if backoff *= 2; backoff > cfg.max {
246- backoff = cfg.max
247- }
248 }243 }
244+ return transport, err
249 }245 }
250 246
251 // realDial is the production dialer.247 // realDial is the production dialer.
M internal/app/spawn_test.go +15 -0
@@ -258,6 +258,21 @@ func TestAcquireSpawnLockTakesOverStaleLock(t *testing.T) {
258 release()258 release()
259 }259 }
260 260
261+func TestRetryAttemptsChecksOperationErrorBeforeDeadline(t *testing.T) {
262+ attemptErr := errors.New("attempt failed")
263+ calls := 0
264+ _, err := retryAttempts(context.Background(), backoffConfig{}, func() (struct{}, bool, error) {
265+ calls++
266+ return struct{}{}, false, attemptErr
267+ })
268+ if !errors.Is(err, attemptErr) {
269+ t.Fatalf("retryAttempts error = %v, want %v", err, attemptErr)
270+ }
271+ if calls != 1 {
272+ t.Fatalf("attempt called %d times, want 1", calls)
273+ }
274+}
275+
261 func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) {276 func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) {
262 want := portsmocks.NewMockTransport(t)277 want := portsmocks.NewMockTransport(t)
263 var calls atomic.Int32278 var calls atomic.Int32
M internal/domain/recovery.go +19 -0
@@ -5,6 +5,7 @@ import (
5 "errors"5 "errors"
6 "fmt"6 "fmt"
7 "io"7 "io"
8+ "slices"
8 "strings"9 "strings"
9 )10 )
10 11
@@ -44,6 +45,24 @@ type CatalogueRecord struct {
44 45
45 // CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue46 // CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue
46 // incarnation without carrying authority-owned recovery or checkpoint fields.47 // incarnation without carrying authority-owned recovery or checkpoint fields.
48+// Equal reports canonical value equality between checkpoint references.
49+func (r *CheckpointRef) Equal(other *CheckpointRef) bool {
50+ if r == nil || other == nil {
51+ return r == nil && other == nil
52+ }
53+ return *r == *other
54+}
55+
56+// Equal reports canonical value equality, including ordered tab names and
57+// pointed-to checkpoint values rather than pointer identity.
58+func (r CatalogueRecord) Equal(other CatalogueRecord) bool {
59+ return r.Name == other.Name && r.IncarnationID == other.IncarnationID && r.Cwd == other.Cwd &&
60+ r.CreatedAt == other.CreatedAt && r.UpdatedAt == other.UpdatedAt && r.LastUsedSeq == other.LastUsedSeq &&
61+ slices.Equal(r.TabNames, other.TabNames) && r.RecoveryState == other.RecoveryState &&
62+ r.Committed.Equal(other.Committed) && r.Fallbacks[0].Equal(other.Fallbacks[0]) &&
63+ r.Fallbacks[1].Equal(other.Fallbacks[1]) && r.DegradedReason == other.DegradedReason
64+}
65+
47 type CatalogueMetadataUpdate struct {66 type CatalogueMetadataUpdate struct {
48 Name string67 Name string
49 IncarnationID IncarnationID68 IncarnationID IncarnationID
M internal/domain/recovery_test.go +22 -1
@@ -22,7 +22,12 @@ func TestIncarnationIDText(t *testing.T) {
22 var decoded IncarnationID22 var decoded IncarnationID
23 require.NoError(t, decoded.UnmarshalText(text))23 require.NoError(t, decoded.UnmarshalText(text))
24 require.Equal(t, id, decoded)24 require.Equal(t, id, decoded)
25- require.Error(t, decoded.UnmarshalText([]byte("AB")))25+ require.Error(t, decoded.UnmarshalText([]byte("AB0000000000000000000000000000CD")))
26+
27+ _, err = (IncarnationID{}).MarshalText()
28+ require.Error(t, err)
29+ require.Error(t, decoded.UnmarshalText([]byte("00000000000000000000000000000000")))
30+ require.Equal(t, id, decoded, "rejected zero text must not mutate the receiver")
26 }31 }
27 32
28 func TestCatalogueRecordValidate(t *testing.T) {33 func TestCatalogueRecordValidate(t *testing.T) {
@@ -50,6 +55,7 @@ func TestCatalogueRecordValidate(t *testing.T) {
50 {name: "zero checkpoint generation", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{ManifestDigest: [32]byte{1}}}},55 {name: "zero checkpoint generation", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{ManifestDigest: [32]byte{1}}}},
51 {name: "zero checkpoint digest", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7}}},56 {name: "zero checkpoint digest", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7}}},
52 {name: "second fallback without first", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: validRef, Fallbacks: [2]*CheckpointRef{nil, {Generation: 6, ManifestDigest: [32]byte{2}}}}},57 {name: "second fallback without first", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: validRef, Fallbacks: [2]*CheckpointRef{nil, {Generation: 6, ManifestDigest: [32]byte{2}}}}},
58+ {name: "fallback newer than committed", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 6, ManifestDigest: [32]byte{1}}, Fallbacks: [2]*CheckpointRef{{Generation: 7, ManifestDigest: [32]byte{2}}}}},
53 }59 }
54 for _, tt := range tests {60 for _, tt := range tests {
55 t.Run(tt.name, func(t *testing.T) {61 t.Run(tt.name, func(t *testing.T) {
@@ -58,6 +64,21 @@ func TestCatalogueRecordValidate(t *testing.T) {
58 }64 }
59 }65 }
60 66
67+func TestCatalogueRecordEqualUsesCheckpointValuesAndOrderedTabs(t *testing.T) {
68+ committed := CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}}
69+ left := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, TabNames: []string{"shell", "logs"}, RecoveryState: RecoveryHealthy, Committed: &committed}
70+ right := left
71+ right.TabNames = append([]string(nil), left.TabNames...)
72+ copy := committed
73+ right.Committed = &copy
74+ require.True(t, left.Equal(right))
75+
76+ right.TabNames[0], right.TabNames[1] = right.TabNames[1], right.TabNames[0]
77+ require.False(t, left.Equal(right))
78+ require.True(t, (*CheckpointRef)(nil).Equal(nil))
79+ require.False(t, left.Committed.Equal(nil))
80+}
81+
61 func TestDeletionTombstoneValidate(t *testing.T) {82 func TestDeletionTombstoneValidate(t *testing.T) {
62 tests := []struct {83 tests := []struct {
63 name string84 name string
M internal/persist/catalogue_test.go +31 -3
@@ -40,11 +40,29 @@ func testCatalogueRecordRoundTrip(t *testing.T) {
40 got, err := decodeRecordValue(record.Name, encoded)40 got, err := decodeRecordValue(record.Name, encoded)
41 require.NoError(t, err)41 require.NoError(t, err)
42 require.Equal(t, record, got)42 require.Equal(t, record, got)
43+ for prefix := range len(encoded) {
44+ _, err := decodeRecordValue(record.Name, encoded[:prefix])
45+ require.Error(t, err, "prefix length %d", prefix)
46+ }
43 require.Error(t, func() error { _, err := decodeRecordValue(record.Name, append(encoded, 0)); return err }())47 require.Error(t, func() error { _, err := decodeRecordValue(record.Name, append(encoded, 0)); return err }())
44 })48 })
45 }49 }
46 }50 }
47 51
52+func TestDecodeRecordValueWrapsMalformedAndValidationErrors(t *testing.T) {
53+ encoded, err := encodeRecordValue(validRecord("work", 1))
54+ require.NoError(t, err)
55+ const recoveryStateOffset = 4 + 2 + 16 + 4 + 8 + 8 + 8 + 4
56+ encoded[recoveryStateOffset] = 99
57+
58+ _, err = decodeRecordValue("work", encoded)
59+ require.ErrorIs(t, err, errMalformedRecord)
60+ joined, ok := err.(interface{ Unwrap() []error })
61+ require.True(t, ok)
62+ require.Len(t, joined.Unwrap(), 2)
63+ require.ErrorContains(t, joined.Unwrap()[1], "invalid recovery state")
64+}
65+
48 func testCatalogueDuplicateIncarnations(t *testing.T) {66 func testCatalogueDuplicateIncarnations(t *testing.T) {
49 id := domain.IncarnationID{1}67 id := domain.IncarnationID{1}
50 require.Error(t, validateUniqueIncarnations([]domain.CatalogueRecord{68 require.Error(t, validateUniqueIncarnations([]domain.CatalogueRecord{
@@ -110,7 +128,8 @@ func testCatalogueMetadataUpdatePreservesAuthority(t *testing.T) {
110 next := domain.CatalogueRecord{Name: "work", IncarnationID: original.IncarnationID, Cwd: "/new", UpdatedAt: 22, LastUsedSeq: 23, TabNames: []string{"editor", "logs"}}128 next := domain.CatalogueRecord{Name: "work", IncarnationID: original.IncarnationID, Cwd: "/new", UpdatedAt: 22, LastUsedSeq: 23, TabNames: []string{"editor", "logs"}}
111 update := next.MetadataUpdate()129 update := next.MetadataUpdate()
112 require.NoError(t, p.UpdateMetadata(update))130 require.NoError(t, p.UpdateMetadata(update))
113- got, ok := p.Record("work")131+ got, ok, err := p.Record("work")
132+ require.NoError(t, err)
114 require.True(t, ok)133 require.True(t, ok)
115 expected := original134 expected := original
116 expected.Cwd, expected.UpdatedAt, expected.LastUsedSeq, expected.TabNames = next.Cwd, next.UpdatedAt, next.LastUsedSeq, next.TabNames135 expected.Cwd, expected.UpdatedAt, expected.LastUsedSeq, expected.TabNames = next.Cwd, next.UpdatedAt, next.LastUsedSeq, next.TabNames
@@ -121,14 +140,16 @@ func testCatalogueMetadataUpdatePreservesAuthority(t *testing.T) {
121 Name: "work", IncarnationID: original.IncarnationID, LastUsedSeq: &lastUsedSeq,140 Name: "work", IncarnationID: original.IncarnationID, LastUsedSeq: &lastUsedSeq,
122 }))141 }))
123 expected.LastUsedSeq = lastUsedSeq142 expected.LastUsedSeq = lastUsedSeq
124- partiallyUpdated, ok := p.Record("work")143+ partiallyUpdated, ok, err := p.Record("work")
144+ require.NoError(t, err)
125 require.True(t, ok)145 require.True(t, ok)
126 require.Equal(t, expected, partiallyUpdated, "a partial metadata update must not clear other mutable or authority-owned fields")146 require.Equal(t, expected, partiallyUpdated, "a partial metadata update must not clear other mutable or authority-owned fields")
127 147
128 stale := update148 stale := update
129 stale.IncarnationID = domain.IncarnationID{2}149 stale.IncarnationID = domain.IncarnationID{2}
130 require.Error(t, p.UpdateMetadata(stale))150 require.Error(t, p.UpdateMetadata(stale))
131- unchanged, ok := p.Record("work")151+ unchanged, ok, err := p.Record("work")
152+ require.NoError(t, err)
132 require.True(t, ok)153 require.True(t, ok)
133 require.Equal(t, expected, unchanged)154 require.Equal(t, expected, unchanged)
134 }155 }
@@ -201,6 +222,13 @@ func testCatalogueLoadReadOnlyRecoversCrashStates(t *testing.T) {
201 writeCandidate(t, StorePath(dir)+".next", want)222 writeCandidate(t, StorePath(dir)+".next", want)
202 },223 },
203 },224 },
225+ {
226+ // Crash before a new installation publishes .next as current.
227+ name: "next-only-recovers-from-next",
228+ write: func(t *testing.T, dir string, want domain.CatalogueRecord) {
229+ writeCandidate(t, StorePath(dir)+".next", want)
230+ },
231+ },
204 {232 {
205 // Crash leaves only a predecessor behind; no current, no successor.233 // Crash leaves only a predecessor behind; no current, no successor.
206 name: "prev-only-recovers-from-prev",234 name: "prev-only-recovers-from-prev",
M internal/persist/migration.go +21 -5
@@ -6,6 +6,7 @@ import (
6 "errors"6 "errors"
7 "fmt"7 "fmt"
8 "io"8 "io"
9+ "log/slog"
9 "os"10 "os"
10 "path/filepath"11 "path/filepath"
11 "sort"12 "sort"
@@ -37,6 +38,8 @@ type OpenDeps struct {
37 StateDir string38 StateDir string
38 Random io.Reader39 Random io.Reader
39 SnapshotMigration ports.SnapshotMigration40 SnapshotMigration ports.SnapshotMigration
41+ // Logger receives full adapter causes; durable degraded reasons are sanitized.
42+ Logger *slog.Logger
40 // Fault is a test-only crash seam called after named durable boundaries.43 // Fault is a test-only crash seam called after named durable boundaries.
41 Fault func(string) error44 Fault func(string) error
42 }45 }
@@ -65,6 +68,9 @@ const (
65 catalogueFormatName = "durable-session-v1.format"68 catalogueFormatName = "durable-session-v1.format"
66 migrationRecordMagic = "VEVJ"69 migrationRecordMagic = "VEVJ"
67 migrationRecordVersion uint16 = 170 migrationRecordVersion uint16 = 1
71+
72+ migrationDegradedLegacySnapshotUncertain = "legacy_snapshot_uncertain"
73+ migrationDegradedReasonMaxBytes = 64
68 )74 )
69 75
70 func OpenOrMigrate(ctx context.Context, deps OpenDeps) (OpenResult, error) {76 func OpenOrMigrate(ctx context.Context, deps OpenDeps) (OpenResult, error) {
@@ -300,11 +306,12 @@ func resumeLegacyMigration(ctx context.Context, deps OpenDeps) (OpenResult, erro
300 if !errors.Is(err, ports.ErrLegacySnapshotUncertain) {306 if !errors.Is(err, ports.ErrLegacySnapshotUncertain) {
301 return OpenResult{}, fmt.Errorf("persist: migrate legacy snapshot %q: %w", legacy.Name, err)307 return OpenResult{}, fmt.Errorf("persist: migrate legacy snapshot %q: %w", legacy.Name, err)
302 }308 }
309+ migrationLogger(deps).Error("legacy snapshot migration uncertain", "session", legacy.Name, "err", err)
303 if ref.Generation != 0 && ref.ManifestDigest != ([32]byte{}) {310 if ref.Generation != 0 && ref.ManifestDigest != ([32]byte{}) {
304 legacyRef := ref311 legacyRef := ref
305 record.Validated[legacy.Name] = &legacyRef312 record.Validated[legacy.Name] = &legacyRef
306 }313 }
307- record.Degraded[legacy.Name] = err.Error()314+ record.Degraded[legacy.Name] = migrationDegradedLegacySnapshotUncertain
308 }315 }
309 if err := writeMigrationRecord(intent, record); err != nil {316 if err := writeMigrationRecord(intent, record); err != nil {
310 return OpenResult{}, err317 return OpenResult{}, err
@@ -374,6 +381,13 @@ func resumeLegacyMigration(ctx context.Context, deps OpenDeps) (OpenResult, erro
374 return OpenResult{Catalogue: catalogue, Records: opened, Migrated: true}, nil381 return OpenResult{Catalogue: catalogue, Records: opened, Migrated: true}, nil
375 }382 }
376 383
384+func migrationLogger(deps OpenDeps) *slog.Logger {
385+ if deps.Logger != nil {
386+ return deps.Logger
387+ }
388+ return slog.Default()
389+}
390+
377 func migrationBoundary(deps OpenDeps, name string) error {391 func migrationBoundary(deps OpenDeps, name string) error {
378 if deps.Fault != nil {392 if deps.Fault != nil {
379 return deps.Fault(name)393 return deps.Fault(name)
@@ -609,10 +623,12 @@ func catalogueRecordsEqual(a, b []domain.CatalogueRecord) bool {
609 if len(a) != len(b) {623 if len(a) != len(b) {
610 return false624 return false
611 }625 }
612- sort.Slice(a, func(i, j int) bool { return a[i].Name < a[j].Name })626+ left := append([]domain.CatalogueRecord(nil), a...)
613- sort.Slice(b, func(i, j int) bool { return b[i].Name < b[j].Name })627+ right := append([]domain.CatalogueRecord(nil), b...)
614- for i := range a {628+ sort.Slice(left, func(i, j int) bool { return left[i].Name < left[j].Name })
615- if a[i].Name != b[i].Name || a[i].IncarnationID != b[i].IncarnationID || a[i].RecoveryState != b[i].RecoveryState {629+ sort.Slice(right, func(i, j int) bool { return right[i].Name < right[j].Name })
630+ for i := range left {
631+ if !left[i].Equal(right[i]) {
616 return false632 return false
617 }633 }
618 }634 }
M internal/persist/migration_test.go +67 -15
@@ -1,24 +1,29 @@
1 package persist1 package persist
2 2
3 import (3 import (
4+ "bytes"
4 "context"5 "context"
5 "crypto/sha256"6 "crypto/sha256"
6 "encoding/binary"7 "encoding/binary"
7 "errors"8 "errors"
8 "io"9 "io"
10+ "log/slog"
9 "os"11 "os"
10 "path/filepath"12 "path/filepath"
11 "strconv"13 "strconv"
14+ "strings"
12 "testing"15 "testing"
13 16
14 "github.com/bnema/vev/internal/domain"17 "github.com/bnema/vev/internal/domain"
15 "github.com/bnema/vev/internal/ports"18 "github.com/bnema/vev/internal/ports"
16 "github.com/bnema/vev/pkg/kv"19 "github.com/bnema/vev/pkg/kv"
20+ "github.com/bnema/vev/pkg/safedir"
17 "github.com/stretchr/testify/require"21 "github.com/stretchr/testify/require"
18 )22 )
19 23
20 type migrationStub struct {24 type migrationStub struct {
21 legacy bool25 legacy bool
26+ probePaths []string
22 probeErr error27 probeErr error
23 heads map[string]domain.CheckpointRef28 heads map[string]domain.CheckpointRef
24 migrateErr map[string]error29 migrateErr map[string]error
@@ -26,7 +31,19 @@ type migrationStub struct {
26 sources map[string]string31 sources map[string]string
27 }32 }
28 33
29-func (s *migrationStub) HasLegacyState(context.Context) (bool, error) { return s.legacy, s.probeErr }34+func (s *migrationStub) HasLegacyState(context.Context) (bool, error) {
35+ if s.probeErr != nil || s.legacy {
36+ return s.legacy, s.probeErr
37+ }
38+ for _, path := range s.probePaths {
39+ if _, err := os.Stat(path); err == nil {
40+ return true, nil
41+ } else if !errors.Is(err, os.ErrNotExist) {
42+ return false, err
43+ }
44+ }
45+ return false, nil
46+}
30 func (s *migrationStub) ReadLegacyHEAD(_ context.Context, name string) (domain.CheckpointRef, error) {47 func (s *migrationStub) ReadLegacyHEAD(_ context.Context, name string) (domain.CheckpointRef, error) {
31 if source := s.sources[name]; source != "" {48 if source := s.sources[name]; source != "" {
32 if _, err := os.Stat(source); err != nil {49 if _, err := os.Stat(source); err != nil {
@@ -74,7 +91,7 @@ func encodeLegacyRecordV0(r legacyCatalogueRecordV0) []byte {
74 func writeLegacyCatalogue(t *testing.T, dir string, records ...legacyCatalogueRecordV0) string {91 func writeLegacyCatalogue(t *testing.T, dir string, records ...legacyCatalogueRecordV0) string {
75 t.Helper()92 t.Helper()
76 require.NoError(t, os.Chmod(dir, 0o700))93 require.NoError(t, os.Chmod(dir, 0o700))
77- require.NoError(t, os.MkdirAll(dir, 0o700))94+ require.NoError(t, safedir.EnsurePrivate(dir))
78 store, err := kv.Open(StorePath(dir))95 store, err := kv.Open(StorePath(dir))
79 require.NoError(t, err)96 require.NoError(t, err)
80 for _, record := range records {97 for _, record := range records {
@@ -198,7 +215,7 @@ func TestMigrationResumeMatrix(t *testing.T) {
198 }215 }
199 uncertain := recordByName(t, result.Records, "uncertain")216 uncertain := recordByName(t, result.Records, "uncertain")
200 require.Equal(t, domain.RecoveryDegraded, uncertain.RecoveryState)217 require.Equal(t, domain.RecoveryDegraded, uncertain.RecoveryState)
201- require.Contains(t, uncertain.DegradedReason, "corrupt legacy manifest")218+ require.Equal(t, migrationDegradedLegacySnapshotUncertain, uncertain.DegradedReason)
202 require.NoError(t, result.Catalogue.Close())219 require.NoError(t, result.Catalogue.Close())
203 220
204 reopened, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: io.LimitReader(&countingReader{}, 0), SnapshotMigration: restarted})221 reopened, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: io.LimitReader(&countingReader{}, 0), SnapshotMigration: restarted})
@@ -219,12 +236,18 @@ func TestMigrationPreservesUncertain(t *testing.T) {
219 dir := t.TempDir()236 dir := t.TempDir()
220 legacy := writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "broken", Cwd: "/tmp"})237 legacy := writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "broken", Cwd: "/tmp"})
221 ref := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}238 ref := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}
222- result, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"broken": ref}, migrateErr: map[string]error{"broken": errors.Join(ports.ErrLegacySnapshotUncertain, errors.New("invalid legacy manifest"))}}})239+ adapterCause := strings.Repeat("sensitive-adapter-detail", 64)
240+ var logs bytes.Buffer
241+ logger := slog.New(slog.NewTextHandler(&logs, nil))
242+ result, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"broken": ref}, migrateErr: map[string]error{"broken": errors.Join(ports.ErrLegacySnapshotUncertain, errors.New(adapterCause))}}, Logger: logger})
223 require.NoError(t, err)243 require.NoError(t, err)
224 require.NoError(t, result.Catalogue.Close())244 require.NoError(t, result.Catalogue.Close())
225 require.FileExists(t, legacy)245 require.FileExists(t, legacy)
226 require.Equal(t, domain.RecoveryDegraded, result.Records[0].RecoveryState)246 require.Equal(t, domain.RecoveryDegraded, result.Records[0].RecoveryState)
227- require.NotEmpty(t, result.Records[0].DegradedReason)247+ require.Equal(t, migrationDegradedLegacySnapshotUncertain, result.Records[0].DegradedReason)
248+ require.LessOrEqual(t, len(result.Records[0].DegradedReason), migrationDegradedReasonMaxBytes)
249+ require.NotContains(t, result.Records[0].DegradedReason, adapterCause)
250+ require.Contains(t, logs.String(), adapterCause, "the full adapter cause must remain diagnosable")
228 251
229 failClosedDir := t.TempDir()252 failClosedDir := t.TempDir()
230 writeLegacyCatalogue(t, failClosedDir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"})253 writeLegacyCatalogue(t, failClosedDir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"})
@@ -240,23 +263,36 @@ func TestMigrationPreservesUncertain(t *testing.T) {
240 func TestOpenOrMigrateLegacyProbe(t *testing.T) {263 func TestOpenOrMigrateLegacyProbe(t *testing.T) {
241 probeErr := errors.New("probe uncertain")264 probeErr := errors.New("probe uncertain")
242 for _, tc := range []struct {265 for _, tc := range []struct {
243- name string266+ name string
244- stub *migrationStub267+ stub *migrationStub
245- wantNew bool268+ arrange func(t *testing.T, dir string, stub *migrationStub)
246- wantErr error269+ wantNew bool
270+ wantLegacy bool
271+ wantErr error
247 }{272 }{
248- {"proven absent", &migrationStub{}, true, nil},273+ {name: "proven absent", stub: &migrationStub{}, wantNew: true},
249- {"legacy blob", &migrationStub{legacy: true}, false, nil},274+ {name: "legacy blob", stub: &migrationStub{}, wantLegacy: true, arrange: func(t *testing.T, dir string, stub *migrationStub) {
250- {"incremental namespace", &migrationStub{legacy: true}, false, nil},275+ path := filepath.Join(dir, "legacy.snapshot")
251- {"probe error", &migrationStub{probeErr: probeErr}, false, probeErr},276+ require.NoError(t, os.WriteFile(path, []byte("legacy"), 0o600))
277+ stub.probePaths = []string{path}
278+ }},
279+ {name: "incremental namespace", stub: &migrationStub{}, wantLegacy: true, arrange: func(t *testing.T, dir string, stub *migrationStub) {
280+ path := filepath.Join(dir, "snapshots")
281+ require.NoError(t, safedir.EnsurePrivate(path))
282+ stub.probePaths = []string{path}
283+ }},
284+ {name: "probe error", stub: &migrationStub{probeErr: probeErr}, wantErr: probeErr},
252 } {285 } {
253 t.Run(tc.name, func(t *testing.T) {286 t.Run(tc.name, func(t *testing.T) {
254 dir := t.TempDir()287 dir := t.TempDir()
255 require.NoError(t, os.Chmod(dir, 0o700))288 require.NoError(t, os.Chmod(dir, 0o700))
289+ if tc.arrange != nil {
290+ tc.arrange(t, dir, tc.stub)
291+ }
256 result, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: tc.stub})292 result, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: tc.stub})
257 if tc.wantErr != nil {293 if tc.wantErr != nil {
258 require.ErrorIs(t, err, tc.wantErr)294 require.ErrorIs(t, err, tc.wantErr)
259- } else if tc.stub.legacy {295+ } else if tc.wantLegacy {
260 require.Error(t, err)296 require.Error(t, err)
261 } else {297 } else {
262 require.NoError(t, err)298 require.NoError(t, err)
@@ -270,7 +306,7 @@ func TestOpenOrMigrateLegacyProbe(t *testing.T) {
270 require.NoError(t, reopened.Catalogue.Close())306 require.NoError(t, reopened.Catalogue.Close())
271 }307 }
272 }308 }
273- if tc.stub.legacy || tc.wantErr != nil {309+ if tc.wantLegacy || tc.wantErr != nil {
274 _, statErr := os.Stat(StorePath(dir))310 _, statErr := os.Stat(StorePath(dir))
275 require.ErrorIs(t, statErr, os.ErrNotExist)311 require.ErrorIs(t, statErr, os.ErrNotExist)
276 }312 }
@@ -278,6 +314,22 @@ func TestOpenOrMigrateLegacyProbe(t *testing.T) {
278 }314 }
279 }315 }
280 316
317+func TestCatalogueRecordsEqualIsCompleteAndNonMutating(t *testing.T) {
318+ committed := &domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}}
319+ left := []domain.CatalogueRecord{
320+ {Name: "z", IncarnationID: domain.IncarnationID{1}, Cwd: "/old", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3, TabNames: []string{"shell"}, RecoveryState: domain.RecoveryHealthy, Committed: committed},
321+ validRecord("a", 2),
322+ }
323+ right := []domain.CatalogueRecord{left[1], left[0]}
324+ leftOrder, rightOrder := left[0].Name, right[0].Name
325+ require.True(t, catalogueRecordsEqual(left, right))
326+ require.Equal(t, leftOrder, left[0].Name)
327+ require.Equal(t, rightOrder, right[0].Name)
328+
329+ right[1].Cwd = "/different"
330+ require.False(t, catalogueRecordsEqual(left, right), "installed catalogue validation must compare every record field")
331+}
332+
281 func TestAtomicMigrationWriteRemovesTempAfterRenameFailure(t *testing.T) {333 func TestAtomicMigrationWriteRemovesTempAfterRenameFailure(t *testing.T) {
282 dir := t.TempDir()334 dir := t.TempDir()
283 target := filepath.Join(dir, "target")335 target := filepath.Join(dir, "target")
M internal/persist/persist.go +13 -55
@@ -3,8 +3,6 @@ package persist
3 import (3 import (
4 "errors"4 "errors"
5 "fmt"5 "fmt"
6- "log/slog"
7- "os"
8 "path/filepath"6 "path/filepath"
9 "sort"7 "sort"
10 "sync"8 "sync"
@@ -23,25 +21,6 @@ func StorePath(dir string) string { return filepath.Join(dir, filename) }
23 type Persister struct {21 type Persister struct {
24 store ports.Store22 store ports.Store
25 mu sync.Mutex23 mu sync.Mutex
26- log *slog.Logger
27-}
28-
29-// Option configures a Persister at construction time.
30-type Option func(*Persister)
31-
32-// WithLogger directs diagnostics for failures that ports.Catalogue has no
33-// error return for (see Records) to log instead of the default logger.
34-func WithLogger(log *slog.Logger) Option {
35- return func(p *Persister) { p.log = log }
36-}
37-
38-// logger returns a nil-safe logger, defaulting to slog.Default() when no
39-// logger was configured.
40-func (p *Persister) logger() *slog.Logger {
41- if p != nil && p.log != nil {
42- return p.log
43- }
44- return slog.Default()
45 }24 }
46 25
47 // KVStore adapts the reusable WAL implementation to the persistence port.26 // KVStore adapts the reusable WAL implementation to the persistence port.
@@ -76,28 +55,15 @@ func Open(dir string) (*Persister, error) {
76 p, _, err := openCurrentCatalogue(dir, false)55 p, _, err := openCurrentCatalogue(dir, false)
77 return p, err56 return p, err
78 }57 }
79-func New(store ports.Store, opts ...Option) *Persister {58+func New(store ports.Store) *Persister { return &Persister{store: store} }
80- p := &Persister{store: store}
81- for _, opt := range opts {
82- opt(p)
83- }
84- return p
85-}
86 59
87 func openCurrentCatalogue(dir string, createProvenEmpty bool) (*Persister, []domain.CatalogueRecord, error) {60 func openCurrentCatalogue(dir string, createProvenEmpty bool) (*Persister, []domain.CatalogueRecord, error) {
88 path := StorePath(dir)61 path := StorePath(dir)
89- absent := true62+ present, err := catalogueCandidatesPresent(path)
90- for _, candidate := range []string{path, path + ".next", path + ".prev"} {63+ if err != nil {
91- _, err := os.Stat(candidate)64+ return nil, nil, err
92- if err == nil {
93- absent = false
94- continue
95- }
96- if !errors.Is(err, os.ErrNotExist) {
97- return nil, nil, err
98- }
99 }65 }
100- if absent && !createProvenEmpty {66+ if !present && !createProvenEmpty {
101 return nil, nil, fmt.Errorf("%w: no catalogue candidates", errPersistenceUnavailable)67 return nil, nil, fmt.Errorf("%w: no catalogue candidates", errPersistenceUnavailable)
102 }68 }
103 store, err := OpenStore(path)69 store, err := OpenStore(path)
@@ -148,30 +114,22 @@ func (p *Persister) Create(record domain.CatalogueRecord) error {
148 return p.applyLocked(map[string]*domain.CatalogueRecord{record.Name: &record})114 return p.applyLocked(map[string]*domain.CatalogueRecord{record.Name: &record})
149 }115 }
150 116
151-// Records satisfies ports.Catalogue, whose signature has no error return. A117+func (p *Persister) Records() ([]domain.CatalogueRecord, error) { return p.LoadCatalogue() }
152-// load failure (unavailable store, or a malformed record surfaced by118+func (p *Persister) Record(name string) (domain.CatalogueRecord, bool, error) {
153-// decodeAll) must not silently look like an empty catalogue, so it is logged
154-// here instead of being swallowed.
155-func (p *Persister) Records() []domain.CatalogueRecord {
156- records, err := p.LoadCatalogue()
157- if err != nil {
158- p.logger().Error("persist: loading catalogue records failed", "err", err)
159- return []domain.CatalogueRecord{}
160- }
161- return records
162-}
163-func (p *Persister) Record(name string) (domain.CatalogueRecord, bool) {
164 if p == nil || p.store == nil {119 if p == nil || p.store == nil {
165- return domain.CatalogueRecord{}, false120+ return domain.CatalogueRecord{}, false, errPersistenceUnavailable
166 }121 }
167 p.mu.Lock()122 p.mu.Lock()
168 defer p.mu.Unlock()123 defer p.mu.Unlock()
169 value, ok := p.store.Get([]byte(name))124 value, ok := p.store.Get([]byte(name))
170 if !ok {125 if !ok {
171- return domain.CatalogueRecord{}, false126+ return domain.CatalogueRecord{}, false, nil
172 }127 }
173 record, err := decodeRecordValue(name, value)128 record, err := decodeRecordValue(name, value)
174- return record, err == nil129+ if err != nil {
130+ return domain.CatalogueRecord{}, false, err
131+ }
132+ return record, true, nil
175 }133 }
176 func (p *Persister) Apply(records map[string]*domain.CatalogueRecord) error {134 func (p *Persister) Apply(records map[string]*domain.CatalogueRecord) error {
177 if p == nil {135 if p == nil {
M internal/persist/persist_test.go +22 -47
@@ -1,9 +1,7 @@
1 package persist1 package persist
2 2
3 import (3 import (
4- "bytes"
5 "errors"4 "errors"
6- "log/slog"
7 "path/filepath"5 "path/filepath"
8 "testing"6 "testing"
9 7
@@ -51,9 +49,17 @@ func TestCatalogueBatchSyncBehavior(t *testing.T) {
51 p := New(store)49 p := New(store)
52 one := validRecord("one", 1)50 one := validRecord("one", 1)
53 require.NoError(t, p.Replace("one", one))51 require.NoError(t, p.Replace("one", one))
52+ require.Contains(t, state, "one")
54 renamed := one53 renamed := one
55 renamed.Name = "renamed"54 renamed.Name = "renamed"
55+ renamed.Cwd = "/renamed"
56 require.NoError(t, p.Rename("one", renamed))56 require.NoError(t, p.Rename("one", renamed))
57+ require.NotContains(t, state, "one")
58+ encoded, ok := state["renamed"]
59+ require.True(t, ok)
60+ got, err := decodeRecordValue("renamed", encoded)
61+ require.NoError(t, err)
62+ require.Equal(t, renamed, got)
57 }63 }
58 64
59 func TestCatalogueBatchFailureDoesNotSync(t *testing.T) {65 func TestCatalogueBatchFailureDoesNotSync(t *testing.T) {
@@ -80,7 +86,9 @@ func TestCatalogueNilPersisterIsNoOp(t *testing.T) {
80 require.NoError(t, err)86 require.NoError(t, err)
81 require.Empty(t, records)87 require.Empty(t, records)
82 require.NoError(t, p.Close())88 require.NoError(t, p.Close())
83- require.Empty(t, p.Records())89+ got, err := p.Records()
90+ require.NoError(t, err)
91+ require.Empty(t, got)
84 }92 }
85 93
86 // TestCatalogueUnavailableStoreErrors covers a non-nil Persister with no94 // TestCatalogueUnavailableStoreErrors covers a non-nil Persister with no
@@ -102,51 +110,18 @@ func TestCatalogueUnavailableStoreErrors(t *testing.T) {
102 records, err := p.LoadAll()110 records, err := p.LoadAll()
103 require.ErrorIs(t, err, errPersistenceUnavailable)111 require.ErrorIs(t, err, errPersistenceUnavailable)
104 require.Empty(t, records)112 require.Empty(t, records)
105- require.Empty(t, p.Records())113+ got, err := p.Records()
114+ require.ErrorIs(t, err, errPersistenceUnavailable)
115+ require.Empty(t, got)
106 require.NoError(t, p.Close())116 require.NoError(t, p.Close())
107 }117 }
108 118
109-// TestCatalogueRecordsMakesLoadFailureObservable covers Records(), which119+func TestCatalogueRecordsPropagatesMalformedRecord(t *testing.T) {
110-// ports.Catalogue.Records() gives no error return for: a load failure must120+ store := portsmocks.NewMockStore(t)
111-// not silently look like an empty catalogue to every caller. Records()121+ store.EXPECT().Range(mock.Anything).Run(func(fn func([]byte, []byte) bool) {
112-// degrades to an empty slice but must emit the underlying error through its122+ fn([]byte("bad"), []byte{0, 1})
113-// (nil-safe, optional) logger rather than swallowing it outright.123+ })
114-func TestCatalogueRecordsMakesLoadFailureObservable(t *testing.T) {124+ records, err := New(store).Records()
115- tests := []struct {125+ require.Error(t, err)
116- name string126+ require.Empty(t, records)
117- build func(t *testing.T, log *slog.Logger) *Persister
118- }{
119- {
120- name: "unavailable-store",
121- build: func(t *testing.T, log *slog.Logger) *Persister {
122- return New(nil, WithLogger(log))
123- },
124- },
125- {
126- name: "malformed-record",
127- build: func(t *testing.T, log *slog.Logger) *Persister {
128- store := portsmocks.NewMockStore(t)
129- store.EXPECT().Range(mock.Anything).Run(func(fn func([]byte, []byte) bool) {
130- fn([]byte("bad"), []byte{0, 1})
131- })
132- return New(store, WithLogger(log))
133- },
134- },
135- }
136- for _, tt := range tests {
137- t.Run(tt.name, func(t *testing.T) {
138- var buf bytes.Buffer
139- p := tt.build(t, slog.New(slog.NewTextHandler(&buf, nil)))
140- require.Empty(t, p.Records())
141- require.Contains(t, buf.String(), "loading catalogue records failed")
142- })
143- }
144-}
145-
146-// TestCatalogueRecordsDefaultsToSlogDefaultLogger covers the nil-safe logger
147-// fallback: a Persister built without WithLogger must not panic when a load
148-// failure needs to be reported.
149-func TestCatalogueRecordsDefaultsToSlogDefaultLogger(t *testing.T) {
150- p := New(nil)
151- require.NotPanics(t, func() { require.Empty(t, p.Records()) })
152 }127 }
M internal/persist/record.go +6 -4
@@ -26,7 +26,11 @@ func encodeRecordValue(record domain.CatalogueRecord) ([]byte, error) {
26 buf = append(buf, catalogueMagic[:]...)26 buf = append(buf, catalogueMagic[:]...)
27 buf = binary.BigEndian.AppendUint16(buf, catalogueRecordVersion)27 buf = binary.BigEndian.AppendUint16(buf, catalogueRecordVersion)
28 buf = append(buf, record.IncarnationID[:]...)28 buf = append(buf, record.IncarnationID[:]...)
29- buf = appendString(buf, record.Cwd)29+ var err error
30+ buf, err = appendCheckedString(buf, record.Cwd)
31+ if err != nil {
32+ return nil, err
33+ }
30 buf = binary.BigEndian.AppendUint64(buf, uint64(record.CreatedAt))34 buf = binary.BigEndian.AppendUint64(buf, uint64(record.CreatedAt))
31 buf = binary.BigEndian.AppendUint64(buf, uint64(record.UpdatedAt))35 buf = binary.BigEndian.AppendUint64(buf, uint64(record.UpdatedAt))
32 buf = binary.BigEndian.AppendUint64(buf, record.LastUsedSeq)36 buf = binary.BigEndian.AppendUint64(buf, record.LastUsedSeq)
@@ -35,7 +39,6 @@ func encodeRecordValue(record domain.CatalogueRecord) ([]byte, error) {
35 }39 }
36 buf = binary.BigEndian.AppendUint32(buf, uint32(len(record.TabNames)))40 buf = binary.BigEndian.AppendUint32(buf, uint32(len(record.TabNames)))
37 for _, name := range record.TabNames {41 for _, name := range record.TabNames {
38- var err error
39 buf, err = appendCheckedString(buf, name)42 buf, err = appendCheckedString(buf, name)
40 if err != nil {43 if err != nil {
41 return nil, err44 return nil, err
@@ -45,7 +48,6 @@ func encodeRecordValue(record domain.CatalogueRecord) ([]byte, error) {
45 buf = appendRef(buf, record.Committed)48 buf = appendRef(buf, record.Committed)
46 buf = appendRef(buf, record.Fallbacks[0])49 buf = appendRef(buf, record.Fallbacks[0])
47 buf = appendRef(buf, record.Fallbacks[1])50 buf = appendRef(buf, record.Fallbacks[1])
48- var err error
49 buf, err = appendCheckedString(buf, record.DegradedReason)51 buf, err = appendCheckedString(buf, record.DegradedReason)
50 return buf, err52 return buf, err
51 }53 }
@@ -118,7 +120,7 @@ func decodeRecordValue(name string, value []byte) (domain.CatalogueRecord, error
118 return domain.CatalogueRecord{}, errMalformedRecord120 return domain.CatalogueRecord{}, errMalformedRecord
119 }121 }
120 if err := record.Validate(); err != nil {122 if err := record.Validate(); err != nil {
121- return domain.CatalogueRecord{}, fmt.Errorf("%w: %v", errMalformedRecord, err)123+ return domain.CatalogueRecord{}, errors.Join(errMalformedRecord, fmt.Errorf("persist: invalid catalogue record: %w", err))
122 }124 }
123 return record, nil125 return record, nil
124 }126 }
M internal/ports/mocks/recovery_mock.go +26 -11
@@ -1428,7 +1428,7 @@ func (_c *MockCatalogue_Delete_Call) RunAndReturn(run func(s string) error) *Moc
1428 }1428 }
1429 1429
1430 // Record provides a mock function for the type MockCatalogue1430 // Record provides a mock function for the type MockCatalogue
1431-func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool) {1431+func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool, error) {
1432 ret := _mock.Called(s)1432 ret := _mock.Called(s)
1433 1433
1434 if len(ret) == 0 {1434 if len(ret) == 0 {
@@ -1437,7 +1437,8 @@ func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool) {
1437 1437
1438 var r0 domain.CatalogueRecord1438 var r0 domain.CatalogueRecord
1439 var r1 bool1439 var r1 bool
1440- if returnFunc, ok := ret.Get(0).(func(string) (domain.CatalogueRecord, bool)); ok {1440+ var r2 error
1441+ if returnFunc, ok := ret.Get(0).(func(string) (domain.CatalogueRecord, bool, error)); ok {
1441 return returnFunc(s)1442 return returnFunc(s)
1442 }1443 }
1443 if returnFunc, ok := ret.Get(0).(func(string) domain.CatalogueRecord); ok {1444 if returnFunc, ok := ret.Get(0).(func(string) domain.CatalogueRecord); ok {
@@ -1450,7 +1451,12 @@ func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool) {
1450 } else {1451 } else {
1451 r1 = ret.Get(1).(bool)1452 r1 = ret.Get(1).(bool)
1452 }1453 }
1453- return r0, r11454+ if returnFunc, ok := ret.Get(2).(func(string) error); ok {
1455+ r2 = returnFunc(s)
1456+ } else {
1457+ r2 = ret.Error(2)
1458+ }
1459+ return r0, r1, r2
1454 }1460 }
1455 1461
1456 // MockCatalogue_Record_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Record'1462 // MockCatalogue_Record_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Record'
@@ -1477,18 +1483,18 @@ func (_c *MockCatalogue_Record_Call) Run(run func(s string)) *MockCatalogue_Reco
1477 return _c1483 return _c
1478 }1484 }
1479 1485
1480-func (_c *MockCatalogue_Record_Call) Return(catalogueRecord domain.CatalogueRecord, b bool) *MockCatalogue_Record_Call {1486+func (_c *MockCatalogue_Record_Call) Return(catalogueRecord domain.CatalogueRecord, b bool, err error) *MockCatalogue_Record_Call {
1481- _c.Call.Return(catalogueRecord, b)1487+ _c.Call.Return(catalogueRecord, b, err)
1482 return _c1488 return _c
1483 }1489 }
1484 1490
1485-func (_c *MockCatalogue_Record_Call) RunAndReturn(run func(s string) (domain.CatalogueRecord, bool)) *MockCatalogue_Record_Call {1491+func (_c *MockCatalogue_Record_Call) RunAndReturn(run func(s string) (domain.CatalogueRecord, bool, error)) *MockCatalogue_Record_Call {
1486 _c.Call.Return(run)1492 _c.Call.Return(run)
1487 return _c1493 return _c
1488 }1494 }
1489 1495
1490 // Records provides a mock function for the type MockCatalogue1496 // Records provides a mock function for the type MockCatalogue
1491-func (_mock *MockCatalogue) Records() []domain.CatalogueRecord {1497+func (_mock *MockCatalogue) Records() ([]domain.CatalogueRecord, error) {
1492 ret := _mock.Called()1498 ret := _mock.Called()
1493 1499
1494 if len(ret) == 0 {1500 if len(ret) == 0 {
@@ -1496,6 +1502,10 @@ func (_mock *MockCatalogue) Records() []domain.CatalogueRecord {
1496 }1502 }
1497 1503
1498 var r0 []domain.CatalogueRecord1504 var r0 []domain.CatalogueRecord
1505+ var r1 error
1506+ if returnFunc, ok := ret.Get(0).(func() ([]domain.CatalogueRecord, error)); ok {
1507+ return returnFunc()
1508+ }
1499 if returnFunc, ok := ret.Get(0).(func() []domain.CatalogueRecord); ok {1509 if returnFunc, ok := ret.Get(0).(func() []domain.CatalogueRecord); ok {
1500 r0 = returnFunc()1510 r0 = returnFunc()
1501 } else {1511 } else {
@@ -1503,7 +1513,12 @@ func (_mock *MockCatalogue) Records() []domain.CatalogueRecord {
1503 r0 = ret.Get(0).([]domain.CatalogueRecord)1513 r0 = ret.Get(0).([]domain.CatalogueRecord)
1504 }1514 }
1505 }1515 }
1506- return r01516+ if returnFunc, ok := ret.Get(1).(func() error); ok {
1517+ r1 = returnFunc()
1518+ } else {
1519+ r1 = ret.Error(1)
1520+ }
1521+ return r0, r1
1507 }1522 }
1508 1523
1509 // MockCatalogue_Records_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Records'1524 // MockCatalogue_Records_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Records'
@@ -1523,12 +1538,12 @@ func (_c *MockCatalogue_Records_Call) Run(run func()) *MockCatalogue_Records_Cal
1523 return _c1538 return _c
1524 }1539 }
1525 1540
1526-func (_c *MockCatalogue_Records_Call) Return(catalogueRecords []domain.CatalogueRecord) *MockCatalogue_Records_Call {1541+func (_c *MockCatalogue_Records_Call) Return(catalogueRecords []domain.CatalogueRecord, err error) *MockCatalogue_Records_Call {
1527- _c.Call.Return(catalogueRecords)1542+ _c.Call.Return(catalogueRecords, err)
1528 return _c1543 return _c
1529 }1544 }
1530 1545
1531-func (_c *MockCatalogue_Records_Call) RunAndReturn(run func() []domain.CatalogueRecord) *MockCatalogue_Records_Call {1546+func (_c *MockCatalogue_Records_Call) RunAndReturn(run func() ([]domain.CatalogueRecord, error)) *MockCatalogue_Records_Call {
1532 _c.Call.Return(run)1547 _c.Call.Return(run)
1533 return _c1548 return _c
1534 }1549 }
M internal/ports/recovery.go +2 -2
@@ -157,8 +157,8 @@ type DegradedRecoveryCoordinator interface {
157 }157 }
158 158
159 type Catalogue interface {159 type Catalogue interface {
160- Records() []domain.CatalogueRecord160+ Records() ([]domain.CatalogueRecord, error)
161- Record(string) (domain.CatalogueRecord, bool)161+ Record(string) (domain.CatalogueRecord, bool, error)
162 Create(domain.CatalogueRecord) error162 Create(domain.CatalogueRecord) error
163 UpdateMetadata(domain.CatalogueMetadataUpdate) error163 UpdateMetadata(domain.CatalogueMetadataUpdate) error
164 Replace(string, domain.CatalogueRecord) error164 Replace(string, domain.CatalogueRecord) error
M internal/ports/store.go +5 -0
@@ -21,6 +21,11 @@ type Store interface {
21 Get(key []byte) ([]byte, bool)21 Get(key []byte) ([]byte, bool)
22 Set(key, val []byte) error22 Set(key, val []byte) error
23 Delete(key []byte) error23 Delete(key []byte) error
24+ // Batch applies all changes atomically to in-memory state: an error leaves
25+ // every prior value unchanged. Implementations repair or discard any torn
26+ // WAL batch during replay so no partial batch becomes visible after restart.
27+ // Batch does not retain StoreChange key or value slices; the caller owns and
28+ // may mutate them after Batch returns.
24 Batch([]StoreChange) error29 Batch([]StoreChange) error
25 // Range iterates key/value pairs; fn returning false stops iteration early.30 // Range iterates key/value pairs; fn returning false stops iteration early.
26 // Each key and value is a stable copy owned by the caller, which may retain31 // Each key and value is a stable copy owned by the caller, which may retain
M internal/ports/wire_test.go +21 -6
@@ -744,12 +744,27 @@ func TestKillGoldenAndRoundTrip(t *testing.T) {
744 }744 }
745 745
746 func TestSessionInfoRecoveryState(t *testing.T) {746 func TestSessionInfoRecoveryState(t *testing.T) {
747- payload := MarshalSessions(Sessions{Sessions: []SessionInfo{{Name: "broken", State: SessionDegraded}}})747+ tests := []struct {
748- got, err := UnmarshalSessions(payload)748+ name string
749- require.NoError(t, err)749+ state SessionState
750- require.Equal(t, SessionDegraded, got.Sessions[0].State)750+ want []byte
751- assertAllPrefixesFail(t, payload, UnmarshalSessions)751+ }{
752- assertTrailingGarbageFails(t, payload, UnmarshalSessions)752+ {name: "running", state: SessionRunning, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1}},
753+ {name: "stopped", state: SessionStopped, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2}},
754+ {name: "restoring", state: SessionRestoring, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3}},
755+ {name: "degraded", state: SessionDegraded, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4}},
756+ }
757+ for _, tt := range tests {
758+ t.Run(tt.name, func(t *testing.T) {
759+ payload := MarshalSessions(Sessions{Sessions: []SessionInfo{{State: tt.state}}})
760+ require.Equal(t, tt.want, payload)
761+ got, err := UnmarshalSessions(payload)
762+ require.NoError(t, err)
763+ require.Equal(t, tt.state, got.Sessions[0].State)
764+ assertAllPrefixesFail(t, payload, UnmarshalSessions)
765+ assertTrailingGarbageFails(t, payload, UnmarshalSessions)
766+ })
767+ }
753 }768 }
754 769
755 func TestSessionsGoldenAndRoundTrip(t *testing.T) {770 func TestSessionsGoldenAndRoundTrip(t *testing.T) {
M internal/usecase/command/command.go +12 -3
@@ -8,9 +8,9 @@ import (
8 // ErrInvalidArguments reports arguments that do not have the exact command format.8 // ErrInvalidArguments reports arguments that do not have the exact command format.
9 var ErrInvalidArguments = errors.New("invalid command arguments")9 var ErrInvalidArguments = errors.New("invalid command arguments")
10 10
11-// ParsePositiveDecimal accepts exactly one base-10 positive decimal value.11+// ParsePositiveUint64 accepts exactly one base-10 positive decimal value.
12 // It deliberately rejects signs, whitespace, zero, and non-canonical forms.12 // It deliberately rejects signs, whitespace, zero, and non-canonical forms.
13-func ParsePositiveDecimal(args []string) (int, error) {13+func ParsePositiveUint64(args []string) (uint64, error) {
14 if len(args) != 1 || args[0] == "" || (len(args[0]) > 1 && args[0][0] == '0') {14 if len(args) != 1 || args[0] == "" || (len(args[0]) > 1 && args[0][0] == '0') {
15 return 0, ErrInvalidArguments15 return 0, ErrInvalidArguments
16 }16 }
@@ -19,13 +19,22 @@ func ParsePositiveDecimal(args []string) (int, error) {
19 return 0, ErrInvalidArguments19 return 0, ErrInvalidArguments
20 }20 }
21 }21 }
22- n, err := strconv.Atoi(args[0])22+ n, err := strconv.ParseUint(args[0], 10, 64)
23 if err != nil || n < 1 {23 if err != nil || n < 1 {
24 return 0, ErrInvalidArguments24 return 0, ErrInvalidArguments
25 }25 }
26 return n, nil26 return n, nil
27 }27 }
28 28
29+// ParsePositiveDecimal parses a positive decimal that must fit the platform int.
30+func ParsePositiveDecimal(args []string) (int, error) {
31+ n, err := ParsePositiveUint64(args)
32+ if err != nil || n > uint64(^uint(0)>>1) {
33+ return 0, ErrInvalidArguments
34+ }
35+ return int(n), nil
36+}
37+
29 // Context is the set of application actions available to commands.38 // Context is the set of application actions available to commands.
30 type Context interface {39 type Context interface {
31 CreateTab() error40 CreateTab() error
M internal/usecase/command/command_test.go +11 -0
@@ -5,6 +5,17 @@ import (
5 "testing"5 "testing"
6 )6 )
7 7
8+func TestParsePositiveUint64(t *testing.T) {
9+ const maxUint64 = "18446744073709551615"
10+ got, err := ParsePositiveUint64([]string{maxUint64})
11+ if err != nil || got != ^uint64(0) {
12+ t.Fatalf("ParsePositiveUint64(max) = %d, %v", got, err)
13+ }
14+ if _, err := ParsePositiveUint64([]string{"18446744073709551616"}); !errors.Is(err, ErrInvalidArguments) {
15+ t.Fatalf("ParsePositiveUint64(overflow) error = %v", err)
16+ }
17+}
18+
8 func TestParsePositiveDecimal(t *testing.T) {19 func TestParsePositiveDecimal(t *testing.T) {
9 for _, tt := range []struct {20 for _, tt := range []struct {
10 name string21 name string
M internal/usecase/command/registry.go +6 -3
@@ -1,6 +1,9 @@
1 package command1 package command
2 2
3-import "strings"3+import (
4+ "path/filepath"
5+ "strings"
6+)
4 7
5 // Registry returns all palette and control commands in stable display order.8 // Registry returns all palette and control commands in stable display order.
6 func Registry() []Command {9 func Registry() []Command {
@@ -101,11 +104,11 @@ func sessionRecoveryCommand() Command {
101 case len(args) == 1 && (args[0] == "retry" || args[0] == "discard"):104 case len(args) == 1 && (args[0] == "retry" || args[0] == "discard"):
102 action = args[0]105 action = args[0]
103 case len(args) == 2 && args[0] == "restore":106 case len(args) == 2 && args[0] == "restore":
104- if _, err := ParsePositiveDecimal(args[1:]); err != nil {107+ if _, err := ParsePositiveUint64(args[1:]); err != nil {
105 return ControlResult{}, ErrInvalidArguments108 return ControlResult{}, ErrInvalidArguments
106 }109 }
107 action, argument = args[0], args[1]110 action, argument = args[0], args[1]
108- case len(args) == 2 && args[0] == "export" && args[1] != "":111+ case len(args) == 2 && args[0] == "export" && filepath.IsAbs(args[1]):
109 action, argument = args[0], args[1]112 action, argument = args[0], args[1]
110 default:113 default:
111 return ControlResult{}, ErrInvalidArguments114 return ControlResult{}, ErrInvalidArguments
M internal/usecase/command/registry_test.go +22 -6
@@ -117,16 +117,29 @@ func TestSessionRecoveryCommand(t *testing.T) {
117 }{117 }{
118 {args: []string{"retry"}, wantCall: "session-recovery:retry:"},118 {args: []string{"retry"}, wantCall: "session-recovery:retry:"},
119 {args: []string{"restore", "7"}, wantCall: "session-recovery:restore:7"},119 {args: []string{"restore", "7"}, wantCall: "session-recovery:restore:7"},
120+ {args: []string{"restore", "18446744073709551615"}, wantCall: "session-recovery:restore:18446744073709551615"},
120 {args: []string{"export", "/tmp/export"}, wantCall: "session-recovery:export:/tmp/export"},121 {args: []string{"export", "/tmp/export"}, wantCall: "session-recovery:export:/tmp/export"},
121 {args: []string{"discard"}, wantCall: "session-recovery:discard:"},122 {args: []string{"discard"}, wantCall: "session-recovery:discard:"},
123+ {args: nil, wantErr: ErrInvalidArguments},
124+ {args: []string{"retry", "extra"}, wantErr: ErrInvalidArguments},
125+ {args: []string{"restore"}, wantErr: ErrInvalidArguments},
122 {args: []string{"restore", "0"}, wantErr: ErrInvalidArguments},126 {args: []string{"restore", "0"}, wantErr: ErrInvalidArguments},
127+ {args: []string{"restore", "01"}, wantErr: ErrInvalidArguments},
128+ {args: []string{"restore", "18446744073709551616"}, wantErr: ErrInvalidArguments},
129+ {args: []string{"export", "relative/path"}, wantErr: ErrInvalidArguments},
130+ {args: []string{"export"}, wantErr: ErrInvalidArguments},
131+ {args: []string{"discard", "extra"}, wantErr: ErrInvalidArguments},
123 {args: []string{"unknown"}, wantErr: ErrInvalidArguments},132 {args: []string{"unknown"}, wantErr: ErrInvalidArguments},
124 }133 }
125 for _, test := range tests {134 for _, test := range tests {
126- ctx := &controlSpy{}135+ ctx := &controlSpy{recoveryOutput: "recovery output"}
127- _, err := cmd.Control(ctx, test.args, ControlOptions{})136+ got, err := cmd.Control(ctx, test.args, ControlOptions{})
128- if !errors.Is(err, test.wantErr) || ctx.call != test.wantCall {137+ wantOutput := ""
129- t.Errorf("args %v: call/error = %q/%v, want %q/%v", test.args, ctx.call, err, test.wantCall, test.wantErr)138+ if test.wantCall != "" {
139+ wantOutput = ctx.recoveryOutput
140+ }
141+ if !errors.Is(err, test.wantErr) || ctx.call != test.wantCall || got.Output != wantOutput {
142+ t.Errorf("args %v: call/output/error = %q/%q/%v, want %q/%q/%v", test.args, ctx.call, got.Output, err, test.wantCall, wantOutput, test.wantErr)
130 }143 }
131 }144 }
132 }145 }
@@ -175,7 +188,10 @@ func TestControlHandlersValidateAndDelegate(t *testing.T) {
175 }188 }
176 }189 }
177 190
178-type controlSpy struct{ call string }191+type controlSpy struct {
192+ call string
193+ recoveryOutput string
194+}
179 195
180 func (s *controlSpy) record(call string) error { s.call = call; return nil }196 func (s *controlSpy) record(call string) error { s.call = call; return nil }
181 func (s *controlSpy) CreateTab() error { return s.record("new-tab") }197 func (s *controlSpy) CreateTab() error { return s.record("new-tab") }
@@ -205,7 +221,7 @@ func (s *controlSpy) Toast(level, message string) error {
205 return s.record("toast:" + level + ":" + message)221 return s.record("toast:" + level + ":" + message)
206 }222 }
207 func (s *controlSpy) SessionRecovery(action, argument string) (string, error) {223 func (s *controlSpy) SessionRecovery(action, argument string) (string, error) {
208- return "", s.record("session-recovery:" + action + ":" + argument)224+ return s.recoveryOutput, s.record("session-recovery:" + action + ":" + argument)
209 }225 }
210 func (s *controlSpy) ListSessions(json bool) (string, error) {226 func (s *controlSpy) ListSessions(json bool) (string, error) {
211 _ = s.record("list-sessions:" + strconv.FormatBool(json))227 _ = s.record("list-sessions:" + strconv.FormatBool(json))
M internal/usecase/daemon/catalogue.go +56 -11
@@ -7,30 +7,75 @@ import (
7 // Catalogue helpers keep persistence optional for ephemeral and focused unit7 // Catalogue helpers keep persistence optional for ephemeral and focused unit
8 // tests while ensuring every durable named-session mutation uses the authority8 // tests while ensuring every durable named-session mutation uses the authority
9 // port when one is configured.9 // port when one is configured.
10-func (d *Daemon) catalogueRecord(name string) (domain.CatalogueRecord, bool) {10+func (d *Daemon) catalogueRecord(name string) (domain.CatalogueRecord, bool, error) {
11 if d == nil || d.catalogue == nil {11 if d == nil || d.catalogue == nil {
12- return domain.CatalogueRecord{}, false12+ return domain.CatalogueRecord{}, false, nil
13 }13 }
14 return d.catalogue.Record(name)14 return d.catalogue.Record(name)
15 }15 }
16 16
17-func (d *Daemon) createCatalogueRecord(record domain.CatalogueRecord) error {17+func (d *Daemon) updateCatalogueMetadata(update domain.CatalogueMetadataUpdate) error {
18 if d == nil || d.catalogue == nil {18 if d == nil || d.catalogue == nil {
19 return nil19 return nil
20 }20 }
21- return d.catalogue.Create(record)21+ return d.catalogue.UpdateMetadata(update)
22 }22 }
23 23
24-func (d *Daemon) updateCatalogueMetadata(update domain.CatalogueMetadataUpdate) error {24+// persistSessionMetadata serializes authority writes without holding daemon or
25- if d == nil || d.catalogue == nil {25+// session state locks during I/O. Only a successfully committed newer revision
26+// subsumes an older caller. Failed revisions retain their rollback until later
27+// failed mutations have unwound, keeping live metadata at the durable boundary.
28+func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update domain.CatalogueMetadataUpdate, rollback func() bool) error {
29+ if sess == nil {
26 return nil30 return nil
27 }31 }
28- return d.catalogue.UpdateMetadata(update)32+ sess.metadataPersistMu.Lock()
29-}33+ defer sess.metadataPersistMu.Unlock()
30 34
31-func (d *Daemon) deleteCatalogueRecord(name string) error {35+ if sess.metadataDurableVersion >= version {
32- if d == nil || d.catalogue == nil {
33 return nil36 return nil
34 }37 }
35- return d.catalogue.Delete(name)38+ if err := d.updateCatalogueMetadata(update); err != nil {
39+ if sess.metadataFailedRollbacks == nil {
40+ sess.metadataFailedRollbacks = make(map[uint64]func() bool)
41+ }
42+ sess.metadataFailedRollbacks[version] = rollback
43+ // A later mutation may have been reserved while authority I/O blocked.
44+ // It must settle before an older failed mutation can be unwound.
45+ sess.mu.Lock()
46+ current := sess.metadataVersion
47+ sess.mu.Unlock()
48+ if current > sess.metadataLiveVersion {
49+ sess.metadataLiveVersion = current
50+ }
51+ sess.reconcileFailedMetadataLocked()
52+ return err
53+ }
54+
55+ sess.metadataDurableVersion = version
56+ for failedVersion := range sess.metadataFailedRollbacks {
57+ if failedVersion <= version {
58+ delete(sess.metadataFailedRollbacks, failedVersion)
59+ }
60+ }
61+ return nil
62+}
63+
64+// reconcileFailedMetadataLocked unwinds only a contiguous failed suffix. An
65+// older failure remains pending while a newer mutation is still writing.
66+// Rollbacks run only after authority I/O has returned and acquire state locks in
67+// the daemon-before-session order.
68+func (s *session) reconcileFailedMetadataLocked() {
69+ for s.metadataLiveVersion > s.metadataDurableVersion {
70+ version := s.metadataLiveVersion
71+ rollback, failed := s.metadataFailedRollbacks[version]
72+ if !failed {
73+ return
74+ }
75+ if rollback != nil && !rollback() {
76+ return
77+ }
78+ delete(s.metadataFailedRollbacks, version)
79+ s.metadataLiveVersion--
80+ }
36 }81 }
M internal/usecase/daemon/control.go +37 -22
@@ -6,6 +6,8 @@ import (
6 "errors"6 "errors"
7 "fmt"7 "fmt"
8 "os"8 "os"
9+ "path/filepath"
10+ "slices"
9 "sort"11 "sort"
10 "strconv"12 "strconv"
11 "strings"13 "strings"
@@ -30,14 +32,14 @@ func (d *Daemon) handleCommand(tr ports.Transport, f ports.Frame) {
30 _ = tr.Send(frameCommandResult(ports.CommandResult{Code: ports.ErrInternal, Text: "malformed command request"}))32 _ = tr.Send(frameCommandResult(ports.CommandResult{Code: ports.ErrInternal, Text: "malformed command request"}))
31 return33 return
32 }34 }
33- _ = tr.Send(frameCommandResult(d.dispatchCommand(request)))35+ _ = tr.Send(frameCommandResult(d.dispatchCommand(d.serveCtx, request)))
34 }36 }
35 37
36 func frameCommandResult(result ports.CommandResult) ports.Frame {38 func frameCommandResult(result ports.CommandResult) ports.Frame {
37 return ports.Frame{Type: ports.MsgCommandResult, Payload: ports.MarshalCommandResult(result)}39 return ports.Frame{Type: ports.MsgCommandResult, Payload: ports.MarshalCommandResult(result)}
38 }40 }
39 41
40-func (d *Daemon) dispatchCommand(request ports.CommandRequest) ports.CommandResult {42+func (d *Daemon) dispatchCommand(ctx context.Context, request ports.CommandRequest) ports.CommandResult {
41 cmd, ok := command.BySlug(request.Slug)43 cmd, ok := command.BySlug(request.Slug)
42 if !ok {44 if !ok {
43 return commandFailure(ports.ErrUnknownCommand, "unknown command: "+request.Slug)45 return commandFailure(ports.ErrUnknownCommand, "unknown command: "+request.Slug)
@@ -46,7 +48,7 @@ func (d *Daemon) dispatchCommand(request ports.CommandRequest) ports.CommandResu
46 return commandFailure(ports.ErrNotScriptable, request.Slug+" requires an attached client")48 return commandFailure(ports.ErrNotScriptable, request.Slug+" requires an attached client")
47 }49 }
48 if cmd.Target == command.TargetNone && !request.Self {50 if cmd.Target == command.TargetNone && !request.Self {
49- return d.runControl(cmd, controlExec{d: d, recoveryName: request.TargetSession}, request)51+ return d.runControl(cmd, controlExec{ctx: ctx, d: d, recoveryName: request.TargetSession}, request)
50 }52 }
51 53
52 sess, code, text := d.resolveTargetSession(request)54 sess, code, text := d.resolveTargetSession(request)
@@ -60,7 +62,7 @@ func (d *Daemon) dispatchCommand(request ports.CommandRequest) ports.CommandResu
60 if code != 0 {62 if code != 0 {
61 return commandFailure(code, text)63 return commandFailure(code, text)
62 }64 }
63- return d.runControl(cmd, controlExec{d: d, sess: sess, tab: tb, target: daemonActionTarget{session: sess, tab: tb, pane: pane}}, request)65+ return d.runControl(cmd, controlExec{ctx: ctx, d: d, sess: sess, tab: tb, target: daemonActionTarget{session: sess, tab: tb, pane: pane}}, request)
64 }66 }
65 67
66 func (d *Daemon) runControl(cmd command.Command, exec controlExec, request ports.CommandRequest) ports.CommandResult {68 func (d *Daemon) runControl(cmd command.Command, exec controlExec, request ports.CommandRequest) ports.CommandResult {
@@ -309,14 +311,7 @@ func (d *Daemon) hasDaemonActionPaneTarget(target daemonActionTarget) bool {
309 }311 }
310 target.session.mu.Lock()312 target.session.mu.Lock()
311 defer target.session.mu.Unlock()313 defer target.session.mu.Unlock()
312- foundTab := false314+ if !slices.Contains(target.session.tabs, target.tab) {
313- for _, tb := range target.session.tabs {
314- if tb == target.tab {
315- foundTab = true
316- break
317- }
318- }
319- if !foundTab {
320 return false315 return false
321 }316 }
322 target.tab.mu.Lock()317 target.tab.mu.Lock()
@@ -335,6 +330,7 @@ func (a daemonActions) switchRelative(sess *session, delta int) error {
335 // controlExec implements command.ControlContext against resolved daemon-owned330 // controlExec implements command.ControlContext against resolved daemon-owned
336 // targets. It deliberately contains no attached-client state.331 // targets. It deliberately contains no attached-client state.
337 type controlExec struct {332 type controlExec struct {
333+ ctx context.Context
338 d *Daemon334 d *Daemon
339 sess *session335 sess *session
340 tab *tab336 tab *tab
@@ -465,7 +461,10 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) {
465 if e.d == nil || e.d.degradedRecovery == nil || e.recoveryName == "" {461 if e.d == nil || e.d.degradedRecovery == nil || e.recoveryName == "" {
466 return "", command.ErrInvalidArguments462 return "", command.ErrInvalidArguments
467 }463 }
468- ctx := context.Background()464+ ctx := e.ctx
465+ if ctx == nil {
466+ ctx = e.d.serveCtx
467+ }
469 switch action {468 switch action {
470 case "retry":469 case "retry":
471 if err := e.d.degradedRecovery.Retry(ctx, e.recoveryName); err != nil {470 if err := e.d.degradedRecovery.Retry(ctx, e.recoveryName); err != nil {
@@ -480,7 +479,10 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) {
480 if err != nil || generation == 0 {479 if err != nil || generation == 0 {
481 return "", command.ErrInvalidArguments480 return "", command.ErrInvalidArguments
482 }481 }
483- record, ok := e.d.catalogue.Record(e.recoveryName)482+ record, ok, err := e.d.catalogue.Record(e.recoveryName)
483+ if err != nil {
484+ return "", err
485+ }
484 if !ok {486 if !ok {
485 return "", errors.New("session recovery record not found")487 return "", errors.New("session recovery record not found")
486 }488 }
@@ -494,6 +496,9 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) {
494 }496 }
495 return "", errors.New("requested recovery fallback is not available")497 return "", errors.New("requested recovery fallback is not available")
496 case "export":498 case "export":
499+ if !filepath.IsAbs(argument) {
500+ return "", command.ErrInvalidArguments
501+ }
497 file, err := os.OpenFile(argument, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)502 file, err := os.OpenFile(argument, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
498 if err != nil {503 if err != nil {
499 return "", err504 return "", err
@@ -520,20 +525,30 @@ func (e controlExec) restoreExplicitRecovery(ctx context.Context) error {
520 if e.d.catalogue == nil {525 if e.d.catalogue == nil {
521 return errors.New("durable session catalogue unavailable")526 return errors.New("durable session catalogue unavailable")
522 }527 }
523- record, ok := e.d.catalogue.Record(e.recoveryName)528+ record, ok, err := e.d.catalogue.Record(e.recoveryName)
529+ if err != nil {
530+ return err
531+ }
524 if !ok {532 if !ok {
525 return errors.New("session recovery record not found")533 return errors.New("session recovery record not found")
526 }534 }
527 done := make(chan struct{})535 done := make(chan struct{})
528 e.d.mu.Lock()536 e.d.mu.Lock()
529- if entry, exists := e.d.stopped[record.Name]; exists {537+ entry, exists := e.d.stopped[record.Name]
530- entry.record = record538+ if !exists || entry.incarnation != record.IncarnationID {
531- entry.state = runtimeRestoring539+ e.d.mu.Unlock()
532- entry.restoreDone = done540+ return errors.New("session recovery runtime record not found")
533- e.d.stopped[record.Name] = entry541+ }
534- }542+ if entry.purging || entry.state != runtimeDegraded {
543+ e.d.mu.Unlock()
544+ return errors.New("session recovery is not available in the current state")
545+ }
546+ entry.record = record
547+ entry.state = runtimeRestoring
548+ entry.restoreDone = done
549+ e.d.stopped[record.Name] = entry
535 e.d.mu.Unlock()550 e.d.mu.Unlock()
536- err := e.d.restoreRecord(ctx, record)551+ err = e.d.restoreRecord(ctx, record)
537 e.d.finishRecordRestore(record, err, done)552 e.d.finishRecordRestore(record, err, done)
538 return err553 return err
539 }554 }
M internal/usecase/daemon/daemon.go +72 -70
@@ -7,13 +7,13 @@ import (
7 "log/slog"7 "log/slog"
8 "math"8 "math"
9 "os"9 "os"
10+ "reflect"
10 "sort"11 "sort"
11 "sync"12 "sync"
12 "sync/atomic"13 "sync/atomic"
13 "time"14 "time"
14 15
15 "github.com/bnema/vev/internal/domain"16 "github.com/bnema/vev/internal/domain"
16- "github.com/bnema/vev/internal/persist"
17 "github.com/bnema/vev/internal/ports"17 "github.com/bnema/vev/internal/ports"
18 "github.com/bnema/vev/internal/usecase/keys"18 "github.com/bnema/vev/internal/usecase/keys"
19 )19 )
@@ -70,6 +70,8 @@ type Daemon struct {
70 mu sync.Mutex70 mu sync.Mutex
71 sessions map[domain.SessionID]*session71 sessions map[domain.SessionID]*session
72 stopped map[string]stoppedSession72 stopped map[string]stoppedSession
73+ // creating reserves names while durable creation I/O runs without mu.
74+ creating map[string]struct{}
73 nextID uint6475 nextID uint64
74 // lastAllocatedCreatedAt is the named-session lifecycle timestamp high-water76 // lastAllocatedCreatedAt is the named-session lifecycle timestamp high-water
75 // mark. It is guarded by mu and prevents a wall-clock regression from77 // mark. It is guarded by mu and prevents a wall-clock regression from
@@ -117,13 +119,11 @@ type Daemon struct {
117 shell string119 shell string
118 shellArgs []string120 shellArgs []string
119 shellOverride bool121 shellOverride bool
120- persist *persist.Persister
121 persistEnabled bool122 persistEnabled bool
122 catalogue ports.Catalogue123 catalogue ports.Catalogue
123 catalogueRecords []domain.CatalogueRecord124 catalogueRecords []domain.CatalogueRecord
124 catalogueRecordsProvided bool125 catalogueRecordsProvided bool
125- // snapshotRepository is the sole checkpoint contract. legacySnapshots is126+ // snapshotRepository is the sole checkpoint contract.
126- // read-only migration input and is never used for new writes.
127 snapshotRepository ports.SnapshotRepository127 snapshotRepository ports.SnapshotRepository
128 checkpointRecovery ports.CheckpointCoordinator128 checkpointRecovery ports.CheckpointCoordinator
129 lifecycleRecovery ports.SessionLifecycleCoordinator129 lifecycleRecovery ports.SessionLifecycleCoordinator
@@ -136,10 +136,12 @@ type Daemon struct {
136 // records, so it is a durable writer and is guarded by snapshotWorkerMu with136 // records, so it is a durable writer and is guarded by snapshotWorkerMu with
137 // the other two.137 // the other two.
138 restoreWorkerDone chan struct{}138 restoreWorkerDone chan struct{}
139- legacySnapshots ports.LegacySnapshotSource
140 snapsEnabled bool139 snapsEnabled bool
141 noticeStore ports.NoticeStore140 noticeStore ports.NoticeStore
142 snapshotJobs chan *snapshotCapture141 snapshotJobs chan *snapshotCapture
142+ // snapshotAdmitted contains every capture accepted by either worker queue,
143+ // including captures buffered in snapshotJobs. Guarded by snapshotWorkerMu.
144+ snapshotAdmitted map[*snapshotCapture]struct{}
143 // snapshotWake wakes the repository scheduler when a session becomes dirty145 // snapshotWake wakes the repository scheduler when a session becomes dirty
144 // or an attempt completes. It is never closed and producers only send146 // or an attempt completes. It is never closed and producers only send
145 // non-blockingly.147 // non-blockingly.
@@ -263,16 +265,6 @@ func WithShell(cmd string, args []string) Option {
263 }265 }
264 }266 }
265 267
266-// WithStore enables persisted named session metadata. A nil store keeps the
267-// daemon in no-op persistence mode.
268-func WithStore(store ports.Store) Option {
269- return func(d *Daemon) {
270- d.persist = persist.New(store)
271- d.catalogue = d.persist
272- d.persistEnabled = store != nil
273- }
274-}
275-
276 // WithCatalogue installs the singular catalogue and the strictly opened268 // WithCatalogue installs the singular catalogue and the strictly opened
277 // records that define the daemon's expected-session registry at startup.269 // records that define the daemon's expected-session registry at startup.
278 func WithCatalogue(catalogue ports.Catalogue, records []domain.CatalogueRecord) Option {270 func WithCatalogue(catalogue ports.Catalogue, records []domain.CatalogueRecord) Option {
@@ -303,26 +295,30 @@ func WithRecoveryCoordinator(coordinator ports.SessionLifecycleCoordinator) Opti
303 }295 }
304 }296 }
305 297
306-// WithSnapshotRepository enables content-addressed incremental snapshots. The298+// WithSnapshotRepository enables content-addressed incremental snapshots.
307-// legacy source is read-only migration input and is never used for new writes.299+func WithSnapshotRepository(repository ports.SnapshotRepository) Option {
308-func WithSnapshotRepository(repository any, legacy ports.LegacySnapshotSource) Option {
309 return func(d *Daemon) {300 return func(d *Daemon) {
310- if repository == nil {301+ if isNilSnapshotRepository(repository) {
311 return302 return
312 }303 }
313- typed, ok := repository.(ports.SnapshotRepository)304+ d.snapshotRepository = repository
314- if !ok {
315- // Pre-incarnation test repositories are intentionally not admitted to
316- // normal runtime paths. They remain accepted by this option only so
317- // package-level legacy tests can exercise their isolated helpers.
318- return
319- }
320- d.snapshotRepository = typed
321- d.legacySnapshots = legacy
322 d.snapsEnabled = true305 d.snapsEnabled = true
323 }306 }
324 }307 }
325 308
309+func isNilSnapshotRepository(repository ports.SnapshotRepository) bool {
310+ if repository == nil {
311+ return true
312+ }
313+ value := reflect.ValueOf(repository)
314+ switch value.Kind() {
315+ case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
316+ return value.IsNil()
317+ default:
318+ return false
319+ }
320+}
321+
326 // WithNoticeStore enables persisting undeliverable notices across daemon322 // WithNoticeStore enables persisting undeliverable notices across daemon
327 // restarts. A nil store keeps the daemon in no-op notice-persistence mode.323 // restarts. A nil store keeps the daemon in no-op notice-persistence mode.
328 func WithNoticeStore(store ports.NoticeStore) Option {324 func WithNoticeStore(store ports.NoticeStore) Option {
@@ -420,6 +416,9 @@ func (d *Daemon) allocateLifecycleCreatedAtLocked() (int64, error) {
420 }416 }
421 417
422 func (d *Daemon) nowUnixNano() int64 {418 func (d *Daemon) nowUnixNano() int64 {
419+ if d == nil || d.clock == nil {
420+ return time.Now().UnixNano()
421+ }
423 return d.clock.Now().UnixNano()422 return d.clock.Now().UnixNano()
424 }423 }
425 424
@@ -434,23 +433,24 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt
434 shell = "/bin/sh"433 shell = "/bin/sh"
435 }434 }
436 d := &Daemon{435 d := &Daemon{
437- sessions: make(map[domain.SessionID]*session),436+ sessions: make(map[domain.SessionID]*session),
438- stopped: make(map[string]stoppedSession),437+ stopped: make(map[string]stoppedSession),
439- parked: make(map[uint64]*parkedAttachment),438+ creating: make(map[string]struct{}),
440- ptys: ptys,439+ parked: make(map[uint64]*parkedAttachment),
441- clock: clock,440+ ptys: ptys,
442- log: log,441+ clock: clock,
443- baseEnv: os.Environ(),442+ log: log,
444- shell: shell,443+ baseEnv: os.Environ(),
445- persist: persist.New(nil),444+ shell: shell,
446- dirOrHome: dirOrHome,445+ dirOrHome: dirOrHome,
447- done: make(chan struct{}),446+ done: make(chan struct{}),
448- restoreDone: make(chan struct{}),447+ restoreDone: make(chan struct{}),
449- animWake: make(chan struct{}, 1),448+ animWake: make(chan struct{}, 1),
450- snapshotJobs: make(chan *snapshotCapture, snapshotQueueCapacity),449+ snapshotJobs: make(chan *snapshotCapture, snapshotQueueCapacity),
451- snapshotWake: make(chan struct{}, 1),450+ snapshotAdmitted: make(map[*snapshotCapture]struct{}),
452- notices: newNoticeCenter(),451+ snapshotWake: make(chan struct{}, 1),
453- resumeParkGrace: defaultResumeParkGrace,452+ notices: newNoticeCenter(),
453+ resumeParkGrace: defaultResumeParkGrace,
454 barScripts: &barScriptState{454 barScripts: &barScriptState{
455 cfg: barConfigFromDomain(domain.Defaults().Bar),455 cfg: barConfigFromDomain(domain.Defaults().Bar),
456 outputs: make(map[domain.SessionID]barScriptOutputs),456 outputs: make(map[domain.SessionID]barScriptOutputs),
@@ -470,9 +470,6 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt
470 for _, o := range opts {470 for _, o := range opts {
471 o(d)471 o(d)
472 }472 }
473- if d.persist == nil {
474- d.persist = persist.New(nil)
475- }
476 if d.dirOrHome == nil {473 if d.dirOrHome == nil {
477 d.dirOrHome = dirOrHome474 d.dirOrHome = dirOrHome
478 }475 }
@@ -488,20 +485,6 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt
488 d.restoreProcessAllowlist.Store(&allow)485 d.restoreProcessAllowlist.Store(&allow)
489 }486 }
490 records := d.catalogueRecords487 records := d.catalogueRecords
491- // Only attempt the load when persistence is actually configured
492- // (d.persistEnabled tracks the same store-is-non-nil condition
493- // persist.Persister uses to distinguish "no store" from "load failed").
494- // Without this gate, the default no-persistence Daemon (persist.New(nil))
495- // would log a false-alarm "loading persisted sessions failed" warning on
496- // every construction: nothing failed, persistence simply isn't enabled.
497- if !d.catalogueRecordsProvided && d.persistEnabled {
498- var err error
499- records, err = d.persist.LoadAll()
500- if err != nil {
501- d.log.Warn("loading persisted sessions failed", "err", err)
502- return d
503- }
504- }
505 var maxSeq uint64488 var maxSeq uint64
506 var maxCreatedAt int64489 var maxCreatedAt int64
507 hasCreatedAt := false490 hasCreatedAt := false
@@ -628,8 +611,6 @@ func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error {
628 var closeErr error611 var closeErr error
629 if d.catalogue != nil {612 if d.catalogue != nil {
630 closeErr = d.catalogue.Close()613 closeErr = d.catalogue.Close()
631- } else {
632- closeErr = d.persist.Close()
633 }614 }
634 if closeErr != nil {615 if closeErr != nil {
635 d.log.Warn("closing session persister failed", "err", closeErr)616 d.log.Warn("closing session persister failed", "err", closeErr)
@@ -662,11 +643,23 @@ func (d *Daemon) shutdownAllWithSnapshotDeadline(reason uint8, deadline *snapsho
662 for _, s := range snapshot {643 for _, s := range snapshot {
663 // Cancellation and PTY closure must not wait behind a teardown owner that644 // Cancellation and PTY closure must not wait behind a teardown owner that
664 // is blocked in snapshot publication or purge work.645 // is blocked in snapshot publication or purge work.
646+ s.mu.Lock()
647+ name := s.name
648+ ephemeral := s.ephemeral
649+ s.mu.Unlock()
665 s.stopInMemoryLifecycle()650 s.stopInMemoryLifecycle()
666 if err := d.killSessionWithSnapshotDeadline(s, reason, false, deadline); err != nil {651 if err := d.killSessionWithSnapshotDeadline(s, reason, false, deadline); err != nil {
667 checkpointIncomplete = true652 checkpointIncomplete = true
668 d.log.Error("closing session with unpersisted terminal state", "err", err)653 d.log.Error("closing session with unpersisted terminal state", "err", err)
669 }654 }
655+ if !ephemeral && deadline != nil {
656+ select {
657+ case <-deadline.Done():
658+ checkpointIncomplete = true
659+ d.persistShutdownSnapshotFailure(name, context.DeadlineExceeded)
660+ default:
661+ }
662+ }
670 }663 }
671 return checkpointIncomplete664 return checkpointIncomplete
672 }665 }
@@ -738,6 +731,12 @@ func snapshotStopContext(deadline *snapshotShutdownDeadline) (context.Context, c
738 if deadline == nil {731 if deadline == nil {
739 return ctx, cancel732 return ctx, cancel
740 }733 }
734+ select {
735+ case <-deadline.Done():
736+ cancel()
737+ return ctx, cancel
738+ default:
739+ }
741 stop := make(chan struct{})740 stop := make(chan struct{})
742 go func() {741 go func() {
743 select {742 select {
@@ -817,13 +816,16 @@ func (d *Daemon) handleList(tr ports.Transport) {
817 }816 }
818 state := ports.SessionStopped817 state := ports.SessionStopped
819 if stopped.purging {818 if stopped.purging {
819+ // Purge is the dominant externally visible state: restoration must
820+ // never make a deletion-reserved record appear attachable.
820 state = ports.SessionDegraded821 state = ports.SessionDegraded
821- }822+ } else {
822- switch stopped.state {823+ switch stopped.state {
823- case runtimeRestoring:824+ case runtimeRestoring:
824- state = ports.SessionRestoring825+ state = ports.SessionRestoring
825- case runtimeDegraded, runtimeDeleting:826+ case runtimeDegraded, runtimeDeleting:
826- state = ports.SessionDegraded827+ state = ports.SessionDegraded
828+ }
827 }829 }
828 infos = append(infos, ports.SessionInfo{Name: name, State: state})830 infos = append(infos, ports.SessionInfo{Name: name, State: state})
829 }831 }
A internal/usecase/daemon/durable_authority_test.go +49 -0
@@ -0,0 +1,49 @@
1+package daemon
2+
3+import (
4+ "context"
5+ "crypto/rand"
6+
7+ "github.com/bnema/vev/internal/domain"
8+ "github.com/bnema/vev/internal/persist"
9+ "github.com/bnema/vev/internal/ports"
10+ recoveryusecase "github.com/bnema/vev/internal/usecase/recovery"
11+)
12+
13+type testLifecycleRepository struct{ ports.SnapshotRepository }
14+
15+func (testLifecycleRepository) WriteDeletionTombstone(context.Context, domain.DeletionTombstone) error {
16+ return nil
17+}
18+func (testLifecycleRepository) QuarantineDeletionSources(context.Context, domain.DeletionTombstone, bool) error {
19+ return nil
20+}
21+func (testLifecycleRepository) DeleteDeletionTombstone(context.Context, domain.IncarnationID) error {
22+ return nil
23+}
24+
25+// WithStore is package-test composition glue. Production injects the same
26+// recovery coordinator explicitly from internal/app.
27+func WithStore(store ports.Store) Option {
28+ return func(d *Daemon) {
29+ if store == nil {
30+ return
31+ }
32+ catalogue := persist.New(store)
33+ records, err := catalogue.Records()
34+ if err != nil {
35+ return
36+ }
37+ d.catalogue = catalogue
38+ d.persistEnabled = true
39+ d.catalogueRecords = records
40+ d.catalogueRecordsProvided = true
41+ repository := d.snapshotRepository
42+ if repository == nil {
43+ repository = testLifecycleRepository{}
44+ }
45+ authority := recoveryusecase.NewCoordinator(catalogue, repository, nil, rand.Reader)
46+ d.lifecycleRecovery = authority
47+ d.checkpointRecovery = authority
48+ }
49+}
M internal/usecase/daemon/durable_recovery_test.go +126 -24
@@ -16,6 +16,8 @@ import (
16 16
17 "github.com/bnema/vev/internal/domain"17 "github.com/bnema/vev/internal/domain"
18 "github.com/bnema/vev/internal/ports"18 "github.com/bnema/vev/internal/ports"
19+ portsmocks "github.com/bnema/vev/internal/ports/mocks"
20+ "github.com/bnema/vev/internal/usecase/command"
19 recoveryusecase "github.com/bnema/vev/internal/usecase/recovery"21 recoveryusecase "github.com/bnema/vev/internal/usecase/recovery"
20 snapcodec "github.com/bnema/vev/internal/usecase/snapshot"22 snapcodec "github.com/bnema/vev/internal/usecase/snapshot"
21 )23 )
@@ -35,21 +37,28 @@ func newDurableRecoveryCatalogue(records []domain.CatalogueRecord) *durableRecov
35 return catalogue37 return catalogue
36 }38 }
37 39
38-func (c *durableRecoveryCatalogue) Records() []domain.CatalogueRecord {40+func (c *durableRecoveryCatalogue) Records() ([]domain.CatalogueRecord, error) {
39 c.mu.Lock()41 c.mu.Lock()
40 defer c.mu.Unlock()42 defer c.mu.Unlock()
41 records := make([]domain.CatalogueRecord, 0, len(c.records))43 records := make([]domain.CatalogueRecord, 0, len(c.records))
42 for _, record := range c.records {44 for _, record := range c.records {
43 records = append(records, record)45 records = append(records, record)
44 }46 }
45- return records47+ return records, nil
46 }48 }
47 49
48-func (c *durableRecoveryCatalogue) Record(name string) (domain.CatalogueRecord, bool) {50+func (c *durableRecoveryCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) {
49 c.mu.Lock()51 c.mu.Lock()
50 defer c.mu.Unlock()52 defer c.mu.Unlock()
51 record, ok := c.records[name]53 record, ok := c.records[name]
52- return record, ok54+ return record, ok, nil
55+}
56+
57+func mustDurableRecords(t *testing.T, c *durableRecoveryCatalogue) []domain.CatalogueRecord {
58+ t.Helper()
59+ records, err := c.Records()
60+ require.NoError(t, err)
61+ return records
53 }62 }
54 63
55 func (c *durableRecoveryCatalogue) Replace(name string, record domain.CatalogueRecord) error {64 func (c *durableRecoveryCatalogue) Replace(name string, record domain.CatalogueRecord) error {
@@ -111,6 +120,9 @@ type durableRecoveryRepository struct {
111 func (r *durableRecoveryRepository) LoadCheckpoint(_ context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) {120 func (r *durableRecoveryRepository) LoadCheckpoint(_ context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) {
112 r.mu.Lock()121 r.mu.Lock()
113 defer r.mu.Unlock()122 defer r.mu.Unlock()
123+ if r.loads == nil {
124+ r.loads = make(map[string]int)
125+ }
114 r.loads[name]++126 r.loads[name]++
115 if err := r.errors[name]; err != nil {127 if err := r.errors[name]; err != nil {
116 return ports.SnapshotGeneration{}, err128 return ports.SnapshotGeneration{}, err
@@ -129,6 +141,9 @@ func (r *durableRecoveryRepository) RepairHEAD(_ context.Context, id domain.Inca
129 if r.repairErr != nil {141 if r.repairErr != nil {
130 return r.repairErr142 return r.repairErr
131 }143 }
144+ if r.repairs == nil {
145+ r.repairs = make(map[string]domain.CheckpointRef)
146+ }
132 r.repairs[id.String()] = ref147 r.repairs[id.String()] = ref
133 return nil148 return nil
134 }149 }
@@ -169,7 +184,7 @@ func newDurableRecoveryDaemon(t testing.TB, records []domain.CatalogueRecord, re
169 t.Helper()184 t.Helper()
170 catalogue := newDurableRecoveryCatalogue(records)185 catalogue := newDurableRecoveryCatalogue(records)
171 coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)186 coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
172- d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, records), WithSnapshotRepository(repository, nil), WithCheckpointCoordinator(coordinator))187+ d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, records), WithSnapshotRepository(repository), WithCheckpointCoordinator(coordinator))
173 d.serveCtx, d.serveCancel = context.WithCancel(context.Background())188 d.serveCtx, d.serveCancel = context.WithCancel(context.Background())
174 t.Cleanup(d.serveCancel)189 t.Cleanup(d.serveCancel)
175 return d, catalogue190 return d, catalogue
@@ -188,7 +203,7 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) {
188 catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{prior})203 catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{prior})
189 repository := &durableRecoveryRepository{}204 repository := &durableRecoveryRepository{}
190 coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)205 coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
191- d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, []domain.CatalogueRecord{prior}), WithSnapshotRepository(repository, nil), WithCheckpointCoordinator(coordinator))206+ d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, []domain.CatalogueRecord{prior}), WithSnapshotRepository(repository), WithCheckpointCoordinator(coordinator))
192 startSnapshotEncodeWorker(t, d)207 startSnapshotEncodeWorker(t, d)
193 208
194 sess := newSnapshotTestSession(t, prior.Name, false, prior.Cwd)209 sess := newSnapshotTestSession(t, prior.Name, false, prior.Cwd)
@@ -204,7 +219,7 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) {
204 219
205 require.True(t, d.scheduleFinalSnapshot(sess))220 require.True(t, d.scheduleFinalSnapshot(sess))
206 awaitSnapshotIdle(t, sess)221 awaitSnapshotIdle(t, sess)
207- got, ok := catalogue.Record(prior.Name)222+ got, ok, _ := catalogue.Record(prior.Name)
208 require.True(t, ok)223 require.True(t, ok)
209 require.Equal(t, prior, got, "a timed-out final checkpoint must not change authoritative bytes")224 require.Equal(t, prior, got, "a timed-out final checkpoint must not change authoritative bytes")
210 require.Equal(t, priorRef, *sess.snapshotPublishedCheckpoint)225 require.Equal(t, priorRef, *sess.snapshotPublishedCheckpoint)
@@ -214,7 +229,7 @@ func TestCatalogueRestoreIndependent(t *testing.T) {
214 emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{})229 emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{})
215 emptyDone := make(chan struct{})230 emptyDone := make(chan struct{})
216 go func() {231 go func() {
217- emptyDaemon.restoreCatalogue(context.Background(), emptyCatalogue.Records())232+ emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue))
218 close(emptyDone)233 close(emptyDone)
219 }()234 }()
220 select {235 select {
@@ -233,7 +248,7 @@ func TestCatalogueRestoreIndependent(t *testing.T) {
233 }248 }
234 d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository)249 d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository)
235 250
236- d.restoreCatalogue(context.Background(), catalogue.Records())251+ d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))
237 252
238 require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state)253 require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state)
239 require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state)254 require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state)
@@ -244,7 +259,7 @@ func TestCatalogueRestoreIndependent(t *testing.T) {
244 t.Fatalf("restore completion for %q was not closed", name)259 t.Fatalf("restore completion for %q was not closed", name)
245 }260 }
246 }261 }
247- degraded, ok := catalogue.Record(broken.Name)262+ degraded, ok, _ := catalogue.Record(broken.Name)
248 require.True(t, ok)263 require.True(t, ok)
249 require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState)264 require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState)
250 require.Equal(t, 1, repository.loads[healthy.Name])265 require.Equal(t, 1, repository.loads[healthy.Name])
@@ -265,8 +280,8 @@ func TestCatalogueRestoreIndependent(t *testing.T) {
265 repairs: make(map[string]domain.CheckpointRef),280 repairs: make(map[string]domain.CheckpointRef),
266 }281 }
267 fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository)282 fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository)
268- fallbackDaemon.restoreCatalogue(context.Background(), fallbackCatalogue.Records())283+ fallbackDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, fallbackCatalogue))
269- promoted, ok := fallbackCatalogue.Record(fallback.Name)284+ promoted, ok, _ := fallbackCatalogue.Record(fallback.Name)
270 require.True(t, ok)285 require.True(t, ok)
271 require.Equal(t, fallbackRef, *promoted.Committed)286 require.Equal(t, fallbackRef, *promoted.Committed)
272 require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state)287 require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state)
@@ -291,9 +306,9 @@ func TestCatalogueRestoreKeepsCommittedFallbackHealthyWhenHEADRepairFails(t *tes
291 }306 }
292 307
293 d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository)308 d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository)
294- d.restoreCatalogue(context.Background(), catalogue.Records())309+ d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))
295 310
296- committed, ok := catalogue.Record(record.Name)311+ committed, ok, _ := catalogue.Record(record.Name)
297 require.True(t, ok)312 require.True(t, ok)
298 require.Equal(t, domain.RecoveryHealthy, committed.RecoveryState)313 require.Equal(t, domain.RecoveryHealthy, committed.RecoveryState)
299 require.Equal(t, fallbackRef, *committed.Committed)314 require.Equal(t, fallbackRef, *committed.Committed)
@@ -302,7 +317,7 @@ func TestCatalogueRestoreKeepsCommittedFallbackHealthyWhenHEADRepairFails(t *tes
302 317
303 repository.repairErr = nil318 repository.repairErr = nil
304 next, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{committed}, repository)319 next, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{committed}, repository)
305- next.restoreCatalogue(context.Background(), catalogue.Records())320+ next.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))
306 require.Equal(t, runtimeHealthy, next.stopped[record.Name].state)321 require.Equal(t, runtimeHealthy, next.stopped[record.Name].state)
307 require.Equal(t, 2, repository.repairCalls, "the next startup retries the pending HEAD repair")322 require.Equal(t, 2, repository.repairCalls, "the next startup retries the pending HEAD repair")
308 require.Equal(t, fallbackRef, repository.repairs[record.IncarnationID.String()])323 require.Equal(t, fallbackRef, repository.repairs[record.IncarnationID.String()])
@@ -318,7 +333,7 @@ func TestCatalogueRestoreAggregateOver4096(t *testing.T) {
318 }333 }
319 d, catalogue := newDurableRecoveryDaemon(t, records, repository)334 d, catalogue := newDurableRecoveryDaemon(t, records, repository)
320 335
321- d.restoreCatalogue(context.Background(), catalogue.Records())336+ d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))
322 337
323 require.Len(t, d.stopped, count)338 require.Len(t, d.stopped, count)
324 for _, record := range records {339 for _, record := range records {
@@ -334,7 +349,7 @@ func TestCatalogueRestoreSingleSessionOver4096(t *testing.T) {
334 349
335 // The repository may contain any number of stale entries; restoration has no350 // The repository may contain any number of stale entries; restoration has no
336 // discovery API and therefore performs exactly one direct catalogue lookup.351 // discovery API and therefore performs exactly one direct catalogue lookup.
337- d.restoreCatalogue(context.Background(), catalogue.Records())352+ d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))
338 353
339 require.Equal(t, runtimeHealthy, d.stopped[record.Name].state)354 require.Equal(t, runtimeHealthy, d.stopped[record.Name].state)
340 require.Equal(t, 1, repository.loads[record.Name])355 require.Equal(t, 1, repository.loads[record.Name])
@@ -347,10 +362,10 @@ func TestCatalogueRestoreIncarnationMismatch(t *testing.T) {
347 repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: generation}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)}362 repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: generation}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)}
348 d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository)363 d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository)
349 364
350- d.restoreCatalogue(context.Background(), catalogue.Records())365+ d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))
351 366
352 require.Equal(t, runtimeDegraded, d.stopped[record.Name].state)367 require.Equal(t, runtimeDegraded, d.stopped[record.Name].state)
353- got, _ := catalogue.Record(record.Name)368+ got, _, _ := catalogue.Record(record.Name)
354 require.Equal(t, domain.RecoveryDegraded, got.RecoveryState)369 require.Equal(t, domain.RecoveryDegraded, got.RecoveryState)
355 }370 }
356 371
@@ -391,6 +406,59 @@ func (s explicitRecoveryStub) Discard(context.Context, string, string) (domain.C
391 return s.record, nil406 return s.record, nil
392 }407 }
393 408
409+type contextRecoveryStub struct {
410+ explicitRecoveryStub
411+ wantKey any
412+ want any
413+ seen bool
414+ err error
415+}
416+
417+func (s *contextRecoveryStub) Retry(ctx context.Context, _ string) error {
418+ s.seen = ctx.Value(s.wantKey) == s.want
419+ return s.err
420+}
421+
422+func TestSessionRecoveryUsesIncomingCommandContext(t *testing.T) {
423+ type contextKey struct{}
424+ key := contextKey{}
425+ ctx := context.WithValue(t.Context(), key, "request")
426+ wantErr := errors.New("stop after context capture")
427+ stub := &contextRecoveryStub{wantKey: key, want: "request", err: wantErr}
428+ d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
429+ d.degradedRecovery = stub
430+
431+ _, err := (controlExec{ctx: ctx, d: d, recoveryName: "work"}).SessionRecovery("retry", "")
432+ require.ErrorIs(t, err, wantErr)
433+ require.True(t, stub.seen)
434+}
435+
436+func TestExplicitRecoveryRejectsConcurrentRestore(t *testing.T) {
437+ record := durableRecoveryRecord(0)
438+ record.RecoveryState = domain.RecoveryDegraded
439+ record.DegradedReason = "checkpoint validation failed"
440+ d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{})
441+ d.mu.Lock()
442+ entry := d.stopped[record.Name]
443+ entry.state = runtimeRestoring
444+ originalDone := entry.restoreDone
445+ d.stopped[record.Name] = entry
446+ d.mu.Unlock()
447+
448+ err := (controlExec{d: d, recoveryName: record.Name}).restoreExplicitRecovery(t.Context())
449+ require.ErrorContains(t, err, "current state")
450+ d.mu.Lock()
451+ require.Equal(t, originalDone, d.stopped[record.Name].restoreDone)
452+ d.mu.Unlock()
453+}
454+
455+func TestSessionRecoveryExportRequiresAbsolutePath(t *testing.T) {
456+ d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
457+ d.degradedRecovery = explicitRecoveryStub{}
458+ _, err := (controlExec{ctx: t.Context(), d: d, recoveryName: "work"}).SessionRecovery("export", "relative.snapshot")
459+ require.ErrorIs(t, err, command.ErrInvalidArguments)
460+}
461+
394 func TestSessionRecoveryCommand(t *testing.T) {462 func TestSessionRecoveryCommand(t *testing.T) {
395 degraded := durableRecoveryRecord(2)463 degraded := durableRecoveryRecord(2)
396 degraded.RecoveryState = domain.RecoveryDegraded464 degraded.RecoveryState = domain.RecoveryDegraded
@@ -428,6 +496,20 @@ func TestListShowsDegraded(t *testing.T) {
428 }, listed.Sessions)496 }, listed.Sessions)
429 }497 }
430 498
499+func TestListPurgingDominatesRestoringState(t *testing.T) {
500+ record := durableRecoveryRecord(0)
501+ d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{})
502+ d.mu.Lock()
503+ entry := d.stopped[record.Name]
504+ entry.purging = true
505+ entry.state = runtimeRestoring
506+ d.stopped[record.Name] = entry
507+ d.mu.Unlock()
508+
509+ listed := listSessions(t, d)
510+ require.Equal(t, []ports.SessionInfo{{Name: record.Name, State: ports.SessionDegraded}}, listed.Sessions)
511+}
512+
431 func TestAttachWaitsForRestore(t *testing.T) {513 func TestAttachWaitsForRestore(t *testing.T) {
432 record := durableRecoveryRecord(0)514 record := durableRecoveryRecord(0)
433 factory := &recoveryCountingPTYFactory{}515 factory := &recoveryCountingPTYFactory{}
@@ -449,7 +531,9 @@ func TestAttachWaitsForRestore(t *testing.T) {
449 require.Zero(t, factory.calls.Load())531 require.Zero(t, factory.calls.Load())
450 532
451 d.setStoppedRecovery(record, runtimeDegraded)533 d.setStoppedRecovery(record, runtimeDegraded)
452- closeRuntimeRestoreDone(d.recordRestoreDone(record.Name))534+ d.mu.Lock()
535+ closeRuntimeRestoreDoneLocked(d.stopped[record.Name].restoreDone)
536+ d.mu.Unlock()
453 var protocolError *protoErr537 var protocolError *protoErr
454 require.ErrorAs(t, <-result, &protocolError)538 require.ErrorAs(t, <-result, &protocolError)
455 require.Equal(t, ports.ErrSessionDegraded, protocolError.code)539 require.Equal(t, ports.ErrSessionDegraded, protocolError.code)
@@ -463,7 +547,7 @@ func TestRestoreCancellationTransitionsBeforeAttachCompletion(t *testing.T) {
463 restoreCtx, cancelRestore := context.WithCancel(context.Background())547 restoreCtx, cancelRestore := context.WithCancel(context.Background())
464 restored := make(chan struct{})548 restored := make(chan struct{})
465 go func() {549 go func() {
466- d.restoreCatalogue(restoreCtx, catalogue.Records())550+ d.restoreCatalogue(restoreCtx, mustDurableRecords(t, catalogue))
467 close(restored)551 close(restored)
468 }()552 }()
469 <-repository.started553 <-repository.started
@@ -488,14 +572,16 @@ func TestRestoreCancellationTransitionsBeforeAttachCompletion(t *testing.T) {
488 default:572 default:
489 t.Fatal("restore completion was not signaled")573 t.Fatal("restore completion was not signaled")
490 }574 }
491- committed, _ := catalogue.Record(record.Name)575+ committed, _, _ := catalogue.Record(record.Name)
492 require.Equal(t, entry.record, committed)576 require.Equal(t, entry.record, committed)
493 }577 }
494 578
495 func TestWaitForTargetRestoreRejectsClosedRestoringChannel(t *testing.T) {579 func TestWaitForTargetRestoreRejectsClosedRestoringChannel(t *testing.T) {
496 record := durableRecoveryRecord(0)580 record := durableRecoveryRecord(0)
497 d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{})581 d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{})
498- closeRuntimeRestoreDone(d.stopped[record.Name].restoreDone)582+ d.mu.Lock()
583+ closeRuntimeRestoreDoneLocked(d.stopped[record.Name].restoreDone)
584+ d.mu.Unlock()
499 585
500 result := make(chan error, 1)586 result := make(chan error, 1)
501 go func() { result <- d.waitForTargetRestore(record.Name) }()587 go func() { result <- d.waitForTargetRestore(record.Name) }()
@@ -552,6 +638,22 @@ func TestAttachRejectsDegraded(t *testing.T) {
552 require.Zero(t, factory.calls.Load())638 require.Zero(t, factory.calls.Load())
553 }639 }
554 640
641+func TestRestoreUnavailableReleasesPerRecordWaiters(t *testing.T) {
642+ record := durableRecoveryRecord(0)
643+ d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, nil)
644+ d.closeRestoreDone()
645+
646+ d.mu.Lock()
647+ entry := d.stopped[record.Name]
648+ d.mu.Unlock()
649+ require.Equal(t, runtimeDegraded, entry.state)
650+ select {
651+ case <-entry.restoreDone:
652+ default:
653+ t.Fatal("per-record restore waiter remained blocked when repository was unavailable")
654+ }
655+}
656+
555 func TestCatalogueRestoreDegradedVisible(t *testing.T) {657 func TestCatalogueRestoreDegradedVisible(t *testing.T) {
556 record := durableRecoveryRecord(0)658 record := durableRecoveryRecord(0)
557 record.RecoveryState = domain.RecoveryDegraded659 record.RecoveryState = domain.RecoveryDegraded
@@ -559,7 +661,7 @@ func TestCatalogueRestoreDegradedVisible(t *testing.T) {
559 repository := &durableRecoveryRepository{generations: make(map[string]ports.SnapshotGeneration), errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)}661 repository := &durableRecoveryRepository{generations: make(map[string]ports.SnapshotGeneration), errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)}
560 d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository)662 d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository)
561 663
562- d.restoreCatalogue(context.Background(), catalogue.Records())664+ d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue))
563 665
564 require.Equal(t, runtimeDegraded, d.stopped[record.Name].state)666 require.Equal(t, runtimeDegraded, d.stopped[record.Name].state)
565 require.Zero(t, repository.loads[record.Name])667 require.Zero(t, repository.loads[record.Name])
D internal/usecase/daemon/legacy_snapshot_test.go +0 -8
@@ -1,8 +0,0 @@
1-package daemon
2-
3-import "context"
4-
5-// importLegacySnapshots is retained only for tests covering the retired
6-// pre-migration bridge. Normal runtime legacy decoding lives exclusively in
7-// the snapshot migration adapter.
8-func (d *Daemon) importLegacySnapshots(context.Context) {}
M internal/usecase/daemon/maintenance.go +33 -8
@@ -18,6 +18,9 @@ type maintenanceReconciler interface {
18 Step(context.Context, ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error)18 Step(context.Context, ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error)
19 }19 }
20 20
21+// maintenanceDependencies is exclusively owned by runDurableMaintenance's
22+// single goroutine after startup. Shutdown joins that goroutine before these
23+// cursor and repair sets can be observed or reused.
21 type maintenanceDependencies struct {24 type maintenanceDependencies struct {
22 catalogue ports.Catalogue25 catalogue ports.Catalogue
23 repository ports.SnapshotRepository26 repository ports.SnapshotRepository
@@ -88,7 +91,13 @@ func (d *Daemon) runDurableMaintenance(ctx context.Context) {
88 case <-d.restoreDone:91 case <-d.restoreDone:
89 }92 }
90 for {93 for {
91- d.runDurableMaintenanceTick(ctx)94+ incomplete := d.runDurableMaintenanceTick(ctx)
95+ if ctx.Err() != nil {
96+ return
97+ }
98+ if incomplete {
99+ continue
100+ }
92 timer := d.clock.NewTimer(maintenanceInterval)101 timer := d.clock.NewTimer(maintenanceInterval)
93 select {102 select {
94 case <-ctx.Done():103 case <-ctx.Done():
@@ -99,10 +108,12 @@ func (d *Daemon) runDurableMaintenance(ctx context.Context) {
99 }108 }
100 }109 }
101 110
102-func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) {111+// runDurableMaintenanceTick reports whether bounded work remains and should be
112+// continued immediately rather than delayed until the periodic interval.
113+func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
103 deps := &d.maintenance114 deps := &d.maintenance
104 if deps.catalogue == nil || deps.repository == nil || deps.reconciler == nil {115 if deps.catalogue == nil || deps.repository == nil || deps.reconciler == nil {
105- return116+ return false
106 }117 }
107 118
108 // Reconcile before collection. Until a complete top-level scan reaches EOF,119 // Reconcile before collection. Until a complete top-level scan reaches EOF,
@@ -111,11 +122,15 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) {
111 next, decisions, err := deps.reconciler.Step(ctx, deps.cursor)122 next, decisions, err := deps.reconciler.Step(ctx, deps.cursor)
112 if err != nil {123 if err != nil {
113 d.log.Warn("durable_reconciliation_tick", "action", "scan", "cursor", deps.cursor.DirectoryCookie, "done", false, "err", err)124 d.log.Warn("durable_reconciliation_tick", "action", "scan", "cursor", deps.cursor.DirectoryCookie, "done", false, "err", err)
114- return125+ return false
115 }126 }
116 deps.cursor = next127 deps.cursor = next
117 for _, decision := range decisions {128 for _, decision := range decisions {
118- record, ok := deps.catalogue.Record(decision.Name)129+ record, ok, err := deps.catalogue.Record(decision.Name)
130+ if err != nil {
131+ d.log.Warn("durable_reconciliation_tick", "action", "catalogue-read", "done", false, "err", err)
132+ return false
133+ }
119 if ok && !decision.RetentionResolved {134 if ok && !decision.RetentionResolved {
120 deps.repairUncertain[record.IncarnationID] = struct{}{}135 deps.repairUncertain[record.IncarnationID] = struct{}{}
121 }136 }
@@ -136,18 +151,27 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) {
136 }151 }
137 d.log.Info("durable_reconciliation_tick", "action", "scan", "cursor", next.DirectoryCookie, "done", next.DirectoryCookie == 0, "err", nil)152 d.log.Info("durable_reconciliation_tick", "action", "scan", "cursor", next.DirectoryCookie, "done", next.DirectoryCookie == 0, "err", nil)
138 if next.DirectoryCookie != 0 {153 if next.DirectoryCookie != 0 {
139- return154+ return true
140 }155 }
141 156
142- for _, record := range deps.catalogue.Records() {157+ records, err := deps.catalogue.Records()
158+ if err != nil {
159+ d.log.Warn("durable_maintenance_tick", "action", "catalogue-read", "done", false, "err", err)
160+ return false
161+ }
162+ incomplete := false
163+ for _, record := range records {
143 if err := ctx.Err(); err != nil {164 if err := ctx.Err(); err != nil {
144- return165+ return false
145 }166 }
146 _, unresolved := deps.unresolved[record.IncarnationID]167 _, unresolved := deps.unresolved[record.IncarnationID]
147 _, uncertainRepair := deps.repairUncertain[record.IncarnationID]168 _, uncertainRepair := deps.repairUncertain[record.IncarnationID]
148 restoring := d.incarnationRestoring(record.IncarnationID)169 restoring := d.incarnationRestoring(record.IncarnationID)
149 plan := retentionPlan(record, unresolved || uncertainRepair, restoring)170 plan := retentionPlan(record, unresolved || uncertainRepair, restoring)
150 done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes})171 done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes})
172+ if err == nil && !done {
173+ incomplete = true
174+ }
151 d.log.Info("durable_maintenance_session",175 d.log.Info("durable_maintenance_session",
152 "incarnation", record.IncarnationID.String(),176 "incarnation", record.IncarnationID.String(),
153 "action", "retention",177 "action", "retention",
@@ -159,6 +183,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) {
159 )183 )
160 }184 }
161 clear(deps.repairUncertain)185 clear(deps.repairUncertain)
186+ return incomplete
162 }187 }
163 188
164 func (d *Daemon) incarnationRestoring(id domain.IncarnationID) bool {189 func (d *Daemon) incarnationRestoring(id domain.IncarnationID) bool {
M internal/usecase/daemon/maintenance_test.go +4 -4
@@ -28,7 +28,7 @@ func (r *cursorMaintenanceReconciler) Step(_ context.Context, cursor ports.Recon
28 func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) {28 func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) {
29 record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}29 record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}
30 catalogue := portsmocks.NewMockCatalogue(t)30 catalogue := portsmocks.NewMockCatalogue(t)
31- catalogue.EXPECT().Records().Return([]domain.CatalogueRecord{record}).Once()31+ catalogue.EXPECT().Records().Return([]domain.CatalogueRecord{record}, nil).Once()
32 repository := portsmocks.NewMockSnapshotRepository(t)32 repository := portsmocks.NewMockSnapshotRepository(t)
33 repository.EXPECT().MaintainSession(mock.Anything, mock.MatchedBy(func(plan ports.RetentionPlan) bool {33 repository.EXPECT().MaintainSession(mock.Anything, mock.MatchedBy(func(plan ports.RetentionPlan) bool {
34 return plan.IncarnationID == record.IncarnationID && !plan.PinAll && len(plan.Keep) == 034 return plan.IncarnationID == record.IncarnationID && !plan.PinAll && len(plan.Keep) == 0
@@ -37,9 +37,9 @@ func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) {
37 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})37 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
38 WithDurableMaintenance(catalogue, repository, reconciler, nil)(d)38 WithDurableMaintenance(catalogue, repository, reconciler, nil)(d)
39 39
40- d.runDurableMaintenanceTick(context.Background())40+ require.True(t, d.runDurableMaintenanceTick(context.Background()), "partial reconciliation must continue immediately")
41- d.runDurableMaintenanceTick(context.Background())41+ require.False(t, d.runDurableMaintenanceTick(context.Background()))
42- d.runDurableMaintenanceTick(context.Background())42+ require.True(t, d.runDurableMaintenanceTick(context.Background()), "the next partial pass must also continue immediately")
43 if len(reconciler.seen) != 3 || reconciler.seen[0].DirectoryCookie != 0 || reconciler.seen[1].DirectoryCookie != 7 || reconciler.seen[2].DirectoryCookie != 0 {43 if len(reconciler.seen) != 3 || reconciler.seen[0].DirectoryCookie != 0 || reconciler.seen[1].DirectoryCookie != 7 || reconciler.seen[2].DirectoryCookie != 0 {
44 t.Fatalf("reconciliation cursors = %v, want [0 7 0]", reconciler.seen)44 t.Fatalf("reconciliation cursors = %v, want [0 7 0]", reconciler.seen)
45 }45 }
A internal/usecase/daemon/metadata_order_test.go +285 -0
@@ -0,0 +1,285 @@
1+package daemon
2+
3+import (
4+ "errors"
5+ "sync"
6+ "testing"
7+ "time"
8+
9+ "github.com/stretchr/testify/require"
10+
11+ "github.com/bnema/vev/internal/domain"
12+ "github.com/bnema/vev/internal/ports"
13+)
14+
15+type blockingMetadataCatalogue struct {
16+ ports.Catalogue
17+
18+ mu sync.Mutex
19+ calls int
20+ firstStarted chan struct{}
21+ releaseFirst chan struct{}
22+ firstErr error
23+ secondErr error
24+}
25+
26+func (c *blockingMetadataCatalogue) UpdateMetadata(update domain.CatalogueMetadataUpdate) error {
27+ c.mu.Lock()
28+ c.calls++
29+ call := c.calls
30+ c.mu.Unlock()
31+ if call == 1 {
32+ close(c.firstStarted)
33+ <-c.releaseFirst
34+ if c.firstErr != nil {
35+ return c.firstErr
36+ }
37+ }
38+ if call == 2 && c.secondErr != nil {
39+ return c.secondErr
40+ }
41+ return c.Catalogue.UpdateMetadata(update)
42+}
43+
44+func newBlockingCatalogue(sess *session, firstErr error) (*durableRecoveryCatalogue, *blockingMetadataCatalogue) {
45+ record := sess.persistRecordLocked(sess.createdAt)
46+ catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record})
47+ return catalogue, &blockingMetadataCatalogue{
48+ Catalogue: catalogue,
49+ firstStarted: make(chan struct{}),
50+ releaseFirst: make(chan struct{}),
51+ firstErr: firstErr,
52+ }
53+}
54+
55+func TestMetadataWritesPreserveLaterMutationWhenFirstWriteIsBlocked(t *testing.T) {
56+ firstFailure := errors.New("first metadata write failed")
57+ for _, tt := range []struct {
58+ name string
59+ firstErr error
60+ }{
61+ {name: "delayed success"},
62+ {name: "delayed failure does not roll back newer state", firstErr: firstFailure},
63+ } {
64+ t.Run(tt.name, func(t *testing.T) {
65+ sess := newSnapshotTestSession(t, "work", false, "/work")
66+ catalogue, blocking := newBlockingCatalogue(sess, tt.firstErr)
67+ d := newTestDaemon(t, nil, stubClock{})
68+ d.catalogue = blocking
69+ d.sessions[sess.id] = sess
70+
71+ firstDone := make(chan error, 1)
72+ go func() { firstDone <- d.renameTab(sess, sess.tabs[0], "first") }()
73+ select {
74+ case <-blocking.firstStarted:
75+ case <-time.After(testWaitTimeout):
76+ t.Fatal("first metadata write did not block")
77+ }
78+
79+ secondDone := make(chan error, 1)
80+ go func() { secondDone <- d.renameTab(sess, sess.tabs[0], "second") }()
81+ require.Eventually(t, func() bool {
82+ sess.mu.Lock()
83+ defer sess.mu.Unlock()
84+ return sess.tabs[0].name == "second"
85+ }, testWaitTimeout, time.Millisecond, "later mutation did not proceed while storage was blocked")
86+
87+ close(blocking.releaseFirst)
88+ firstResult := <-firstDone
89+ if tt.firstErr == nil {
90+ require.NoError(t, firstResult)
91+ } else {
92+ require.ErrorIs(t, firstResult, tt.firstErr, "an in-memory revision cannot subsume a failed durable write")
93+ }
94+ require.NoError(t, <-secondDone)
95+
96+ persisted, ok, err := catalogue.Record("work")
97+ require.NoError(t, err)
98+ require.True(t, ok)
99+ require.Equal(t, []string{"second"}, persisted.TabNames)
100+ require.Equal(t, "second", sess.tabs[0].name)
101+ })
102+ }
103+}
104+
105+func TestTwoConsecutiveMetadataFailuresAreBothReported(t *testing.T) {
106+ firstFailure := errors.New("first metadata write failed")
107+ secondFailure := errors.New("second metadata write failed")
108+ sess := newSnapshotTestSession(t, "work", false, "/work")
109+ catalogue, blocking := newBlockingCatalogue(sess, firstFailure)
110+ blocking.secondErr = secondFailure
111+ close(blocking.releaseFirst)
112+ d := newTestDaemon(t, nil, stubClock{})
113+ d.catalogue = blocking
114+ d.sessions[sess.id] = sess
115+
116+ require.ErrorIs(t, d.renameTab(sess, sess.tabs[0], "first"), firstFailure)
117+ require.ErrorIs(t, d.renameTab(sess, sess.tabs[0], "second"), secondFailure)
118+
119+ persisted, ok, err := catalogue.Record("work")
120+ require.NoError(t, err)
121+ require.True(t, ok)
122+ require.Empty(t, persisted.TabNames)
123+ sess.mu.Lock()
124+ require.Empty(t, sess.tabs[0].name)
125+ sess.mu.Unlock()
126+}
127+
128+func TestConsecutiveMetadataFailuresRestoreDurableState(t *testing.T) {
129+ firstFailure := errors.New("first metadata write failed")
130+ secondFailure := errors.New("second metadata write failed")
131+ sess := newSnapshotTestSession(t, "work", false, "/work")
132+ catalogue, blocking := newBlockingCatalogue(sess, firstFailure)
133+ blocking.secondErr = secondFailure
134+ d := newTestDaemon(t, nil, stubClock{})
135+ d.catalogue = blocking
136+ d.sessions[sess.id] = sess
137+
138+ firstDone := make(chan error, 1)
139+ go func() { firstDone <- d.renameTab(sess, sess.tabs[0], "first") }()
140+ select {
141+ case <-blocking.firstStarted:
142+ case <-time.After(testWaitTimeout):
143+ t.Fatal("first metadata write did not block")
144+ }
145+ secondDone := make(chan error, 1)
146+ go func() { secondDone <- d.renameTab(sess, sess.tabs[0], "second") }()
147+ require.Eventually(t, func() bool {
148+ sess.mu.Lock()
149+ defer sess.mu.Unlock()
150+ return sess.tabs[0].name == "second"
151+ }, testWaitTimeout, time.Millisecond)
152+
153+ close(blocking.releaseFirst)
154+ require.ErrorIs(t, <-firstDone, firstFailure)
155+ require.ErrorIs(t, <-secondDone, secondFailure)
156+
157+ persisted, ok, err := catalogue.Record("work")
158+ require.NoError(t, err)
159+ require.True(t, ok)
160+ require.Empty(t, persisted.TabNames)
161+ sess.mu.Lock()
162+ require.Empty(t, sess.tabs[0].name)
163+ sess.mu.Unlock()
164+}
165+
166+func TestNewerMetadataFailureDoesNotSkipOlderSnapshot(t *testing.T) {
167+ failure := errors.New("newer metadata write failed")
168+ sess := newSnapshotTestSession(t, "work", false, "/work")
169+ catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{sess.persistRecordLocked(sess.createdAt)})
170+ blocking := &blockingMetadataCatalogue{
171+ Catalogue: catalogue,
172+ firstStarted: make(chan struct{}),
173+ releaseFirst: make(chan struct{}),
174+ firstErr: failure,
175+ }
176+ close(blocking.releaseFirst)
177+ d := newTestDaemon(t, nil, stubClock{})
178+ d.catalogue = blocking
179+ d.sessions[sess.id] = sess
180+ tb := sess.tabs[0]
181+
182+ sess.mu.Lock()
183+ tb.name = "first"
184+ firstRecord, firstVersion := sess.nextPersistRecordLocked(sess.createdAt + 1)
185+ tb.name = "second"
186+ secondRecord, secondVersion := sess.nextPersistRecordLocked(sess.createdAt + 2)
187+ sess.mu.Unlock()
188+
189+ secondRollback := func() bool {
190+ sess.mu.Lock()
191+ defer sess.mu.Unlock()
192+ if tb.name != "second" {
193+ return false
194+ }
195+ tb.name = "first"
196+ return true
197+ }
198+ firstRollback := func() bool {
199+ sess.mu.Lock()
200+ defer sess.mu.Unlock()
201+ if tb.name != "first" {
202+ return false
203+ }
204+ tb.name = ""
205+ return true
206+ }
207+
208+ require.ErrorIs(t, d.persistSessionMetadata(sess, secondVersion, secondRecord.MetadataUpdate(), secondRollback), failure)
209+ require.NoError(t, d.persistSessionMetadata(sess, firstVersion, firstRecord.MetadataUpdate(), firstRollback))
210+
211+ persisted, ok, err := catalogue.Record("work")
212+ require.NoError(t, err)
213+ require.True(t, ok)
214+ require.Equal(t, []string{"first"}, persisted.TabNames)
215+ sess.mu.Lock()
216+ require.Equal(t, "first", tb.name)
217+ sess.mu.Unlock()
218+}
219+
220+func TestLatestMetadataFailureRollsBackRename(t *testing.T) {
221+ failure := errors.New("metadata write failed")
222+ sess := newSnapshotTestSession(t, "work", false, "/work")
223+ catalogue, blocking := newBlockingCatalogue(sess, failure)
224+ d := newTestDaemon(t, nil, stubClock{})
225+ d.catalogue = blocking
226+ d.sessions[sess.id] = sess
227+
228+ done := make(chan error, 1)
229+ go func() { done <- d.renameTab(sess, sess.tabs[0], "failed") }()
230+ select {
231+ case <-blocking.firstStarted:
232+ case <-time.After(testWaitTimeout):
233+ t.Fatal("metadata write did not block")
234+ }
235+ close(blocking.releaseFirst)
236+ require.ErrorIs(t, <-done, failure)
237+
238+ persisted, ok, err := catalogue.Record("work")
239+ require.NoError(t, err)
240+ require.True(t, ok)
241+ require.Empty(t, persisted.TabNames)
242+ require.Empty(t, sess.tabs[0].name)
243+}
244+
245+func TestBlockedCwdWriteCannotOverwriteLaterTabMetadata(t *testing.T) {
246+ sess := newSnapshotTestSession(t, "work", false, "/work")
247+ catalogue, blocking := newBlockingCatalogue(sess, nil)
248+ d := newTestDaemon(t, nil, stubClock{})
249+ d.catalogue = blocking
250+ d.sessions[sess.id] = sess
251+ d.procCwd = func(int) (string, error) { return "/later-cwd", nil }
252+
253+ cwdDone := make(chan struct{})
254+ go func() {
255+ d.refreshSessionCwd(sess)
256+ close(cwdDone)
257+ }()
258+ select {
259+ case <-blocking.firstStarted:
260+ case <-time.After(testWaitTimeout):
261+ t.Fatal("cwd metadata write did not block")
262+ }
263+
264+ renameDone := make(chan error, 1)
265+ go func() { renameDone <- d.renameTab(sess, sess.tabs[0], "later-tab") }()
266+ require.Eventually(t, func() bool {
267+ sess.mu.Lock()
268+ defer sess.mu.Unlock()
269+ return sess.tabs[0].name == "later-tab"
270+ }, testWaitTimeout, time.Millisecond)
271+
272+ close(blocking.releaseFirst)
273+ select {
274+ case <-cwdDone:
275+ case <-time.After(testWaitTimeout):
276+ t.Fatal("cwd refresh deadlocked")
277+ }
278+ require.NoError(t, <-renameDone)
279+
280+ persisted, ok, err := catalogue.Record("work")
281+ require.NoError(t, err)
282+ require.True(t, ok)
283+ require.Equal(t, "/later-cwd", persisted.Cwd)
284+ require.Equal(t, []string{"later-tab"}, persisted.TabNames)
285+}
M internal/usecase/daemon/performance_benchmark_test.go +3 -3
@@ -638,7 +638,7 @@ func publishBenchmarkSnapshot(b testing.TB, fixture *performanceFixture, generat
638 638
639 func mutateBenchmarkVisible(fixture *performanceFixture, operation int) {639 func mutateBenchmarkVisible(fixture *performanceFixture, operation int) {
640 fixture.activePane.mu.Lock()640 fixture.activePane.mu.Lock()
641- fixture.activePane.screen.Write([]byte(fmt.Sprintf("\x1b[1;1Hvisible-%08d", operation)))641+ fixture.activePane.screen.Write(fmt.Appendf(nil, "\x1b[1;1Hvisible-%08d", operation))
642 fixture.activePane.mu.Unlock()642 fixture.activePane.mu.Unlock()
643 }643 }
644 644
@@ -653,7 +653,7 @@ func mutateBenchmarkTail(fixture *performanceFixture, operation int) {
653 }653 }
654 654
655 func mutateBenchmarkSealedChunk(fixture *performanceFixture, operation int) {655 func mutateBenchmarkSealedChunk(fixture *performanceFixture, operation int) {
656- for row := 0; row < 256; row++ {656+ for row := range 256 {
657 mutateBenchmarkTail(fixture, operation*256+row)657 mutateBenchmarkTail(fixture, operation*256+row)
658 }658 }
659 }659 }
@@ -1065,7 +1065,7 @@ func newPerformanceFixtureWithCleanup(t testing.TB, config performanceConfig, re
1065 liveWrites: [][]byte{[]byte("\x1b[1;1HA\x1b[2;2HA"), []byte("\x1b[1;1HB\x1b[2;2HB")},1065 liveWrites: [][]byte{[]byte("\x1b[1;1HA\x1b[2;2HA"), []byte("\x1b[1;1HB\x1b[2;2HB")},
1066 resizeSizes: [2]domain.Size{{Cols: 100, Rows: 30}, config.size},1066 resizeSizes: [2]domain.Size{{Cols: 100, Rows: 30}, config.size},
1067 }1067 }
1068- WithSnapshotRepository(fixture.snapshots, nil)(d)1068+ WithSnapshotRepository(fixture.snapshots)(d)
1069 d.startSnapshotEncodeWorker()1069 d.startSnapshotEncodeWorker()
1070 if registerCleanup {1070 if registerCleanup {
1071 t.Cleanup(d.stopSnapshotEncodeWorker)1071 t.Cleanup(d.stopSnapshotEncodeWorker)
M internal/usecase/daemon/picker_test.go +27 -34
@@ -25,26 +25,6 @@ import (
25 25
26 // --- test doubles -----------------------------------------------------------26 // --- test doubles -----------------------------------------------------------
27 27
28-// failingDeleteStore is a ports.Store whose Delete always fails, used to
29-// exercise the picker's kill-target error path without disturbing the
30-// generated mock's default Delete expectation.
31-type failingDeleteStore struct{ err error }
32-
33-func (failingDeleteStore) Get([]byte) ([]byte, bool) { return nil, false }
34-func (failingDeleteStore) Set([]byte, []byte) error { return nil }
35-func (s failingDeleteStore) Batch(changes []ports.StoreChange) error {
36- for _, change := range changes {
37- if change.Delete {
38- return s.err
39- }
40- }
41- return nil
42-}
43-func (s failingDeleteStore) Delete([]byte) error { return s.err }
44-func (failingDeleteStore) Range(func(k, v []byte) bool) {}
45-func (failingDeleteStore) Sync() error { return nil }
46-func (failingDeleteStore) Close() error { return nil }
47-
48 type refusingSnapshotDeleteRepository struct {28 type refusingSnapshotDeleteRepository struct {
49 noOpSnapshotRepository29 noOpSnapshotRepository
50 err error30 err error
@@ -277,8 +257,8 @@ func TestPickerWaitsForRestoringTargetBeforeSwitching(t *testing.T) {
277 d.mu.Lock()257 d.mu.Lock()
278 delete(d.stopped, record.Name)258 delete(d.stopped, record.Name)
279 d.sessions[target.id] = target259 d.sessions[target.id] = target
260+ closeRuntimeRestoreDoneLocked(done)
280 d.mu.Unlock()261 d.mu.Unlock()
281- closeRuntimeRestoreDone(done)
282 262
283 require.NoError(t, <-result)263 require.NoError(t, <-result)
284 require.Same(t, target, ac.currentSession())264 require.Same(t, target, ac.currentSession())
@@ -294,12 +274,12 @@ func TestRestoreCancellationTransitionsBeforePickerCompletion(t *testing.T) {
294 catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record})274 catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record})
295 coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)275 coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
296 WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d)276 WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d)
297- WithSnapshotRepository(repository, nil)(d)277+ WithSnapshotRepository(repository)(d)
298 WithCheckpointCoordinator(coordinator)(d)278 WithCheckpointCoordinator(coordinator)(d)
299 restoreCtx, cancelRestore := context.WithCancel(context.Background())279 restoreCtx, cancelRestore := context.WithCancel(context.Background())
300 restored := make(chan struct{})280 restored := make(chan struct{})
301 go func() {281 go func() {
302- d.restoreCatalogue(restoreCtx, catalogue.Records())282+ d.restoreCatalogue(restoreCtx, mustDurableRecords(t, catalogue))
303 close(restored)283 close(restored)
304 }()284 }()
305 <-repository.started285 <-repository.started
@@ -1150,8 +1130,18 @@ func TestPickerKillActiveSessionSnapshotDeleteRefusalReportsOnceAndKeepsPicker(t
1150 d, from, ac, sends, releases := newRecentNavigationTestSessions(t)1130 d, from, ac, sends, releases := newRecentNavigationTestSessions(t)
1151 defer releaseAll(releases)1131 defer releaseAll(releases)
1152 cause := errors.New("snapshot delete refused")1132 cause := errors.New("snapshot delete refused")
1153- WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause}, nil)(d)1133+ WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause})(d)
1154 target := d.sessions[domain.SessionID("recent")]1134 target := d.sessions[domain.SessionID("recent")]
1135+ store, _ := newMockStore(t)
1136+ WithStore(store)(d)
1137+ target.mu.Lock()
1138+ if target.incarnation == (domain.IncarnationID{}) {
1139+ target.incarnation = domain.IncarnationID{1}
1140+ }
1141+ record := target.persistRecordLocked(1)
1142+ target.mu.Unlock()
1143+ record.RecoveryState = domain.RecoveryFresh
1144+ require.NoError(t, d.catalogue.Create(record))
1155 1145
1156 d.enterPicker(from, ac)1146 d.enterPicker(from, ac)
1157 awaitFrame(t, sends, ports.MsgOutput)1147 awaitFrame(t, sends, ports.MsgOutput)
@@ -1184,8 +1174,14 @@ func TestPickerKillStoppedSessionPersistDeleteFailureSurfacesNoticeAndKeepsEntry
1184 defer release()1174 defer release()
1185 d, from, ac, sends := newManualSessionWithPTYs(t, p)1175 d, from, ac, sends := newManualSessionWithPTYs(t, p)
1186 cause := errors.New("delete failed")1176 cause := errors.New("delete failed")
1187- WithStore(failingDeleteStore{err: cause})(d)1177+ store, state := newMockStore(t)
1188- d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp/stopped", createdAt: 7}1178+ WithStore(store)(d)
1179+ record := domain.CatalogueRecord{Name: "stopped", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/stopped", CreatedAt: 7, RecoveryState: domain.RecoveryFresh}
1180+ require.NoError(t, d.catalogue.Create(record))
1181+ state.mu.Lock()
1182+ state.deleteErr = func(string) error { return cause }
1183+ state.mu.Unlock()
1184+ d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp/stopped", createdAt: 7, incarnation: record.IncarnationID, record: record}
1189 1185
1190 d.enterPicker(from, ac)1186 d.enterPicker(from, ac)
1191 awaitFrame(t, sends, ports.MsgOutput)1187 awaitFrame(t, sends, ports.MsgOutput)
@@ -1202,12 +1198,9 @@ func TestPickerKillStoppedSessionPersistDeleteFailureSurfacesNoticeAndKeepsEntry
1202 toasts := awaitToastCount(t, ac, 1)1198 toasts := awaitToastCount(t, ac, 1)
1203 require.Equal(t, domain.NoticePersistDelete, toasts[0].Code)1199 require.Equal(t, domain.NoticePersistDelete, toasts[0].Code)
1204 1200
1205- views, _ := d.pickerViews(from)1201+ d.mu.Lock()
1206- var stillListed bool1202+ stopped, retained := d.stopped["stopped"]
1207- for _, v := range views {1203+ d.mu.Unlock()
1208- if v.Stopped && v.Name == "stopped" {1204+ require.True(t, retained, "failed deletion must retain the reserved name")
1209- stillListed = true1205+ require.True(t, stopped.purging, "an uncertain deletion stays fenced from restore")
1210- }
1211- }
1212- require.True(t, stillListed, "entry must remain listed after refreshPicker when the delete failed")
1213 }1206 }
M internal/usecase/daemon/recovery_observability.go +6 -1
@@ -22,7 +22,12 @@ func (d *Daemon) logSessionDegraded(record domain.CatalogueRecord, reasonCode st
22 func (d *Daemon) logStartupRecoveryCounts(restoring int) {22 func (d *Daemon) logStartupRecoveryCounts(restoring int) {
23 var records []domain.CatalogueRecord23 var records []domain.CatalogueRecord
24 if d.catalogue != nil {24 if d.catalogue != nil {
25- records = d.catalogue.Records()25+ var err error
26+ records, err = d.catalogue.Records()
27+ if err != nil {
28+ d.log.Error("daemon_startup_catalogue_read_failed", "err", err)
29+ return
30+ }
26 } else {31 } else {
27 d.mu.Lock()32 d.mu.Lock()
28 records = make([]domain.CatalogueRecord, 0, len(d.stopped))33 records = make([]domain.CatalogueRecord, 0, len(d.stopped))
M internal/usecase/daemon/render_capture.go +37 -12
@@ -17,15 +17,21 @@ import (
17 // capture/compose/emit transaction holds that attachment's sendMu, so these17 // capture/compose/emit transaction holds that attachment's sendMu, so these
18 // buffers cannot be observed or reused before emission completes.18 // buffers cannot be observed or reused before emission completes.
19 type renderCaptureScratch struct {19 type renderCaptureScratch struct {
20- state capturedRenderState20+ state capturedRenderState
21- panes []capturedPaneRenderState21+ panes []capturedPaneRenderState
22- placements []layout.Placement22+ placements []layout.Placement
23- dividers []layout.Divider23+ dividers []layout.Divider
24- statusTabs []statusTab24+ layoutTab *tab
25- mru []recentSession25+ layoutGeneration uint64
26- ranked []rankedRecent26+ layoutArea domain.Rect
27- titleIDs []layout.PaneID27+ layoutFocus layout.PaneID
28- receipts []damageReceipt28+ layoutFingerprint string
29+ layoutValid bool
30+ statusTabs []statusTab
31+ mru []recentSession
32+ ranked []rankedRecent
33+ titleIDs []layout.PaneID
34+ receipts []damageReceipt
29 }35 }
30 36
31 type damageReceipt struct {37 type damageReceipt struct {
@@ -225,9 +231,28 @@ func capturePrimaryRenderState(
225 231
226 tb.mu.Lock()232 tb.mu.Lock()
227 defer tb.mu.Unlock()233 defer tb.mu.Unlock()
228- layoutSnap := solveTabLayoutLocked(tb)234+ area := domain.Rect{Width: tb.size.Cols, Height: tb.size.Rows}
229- scratch.placements = append(scratch.placements[:0], layoutSnap.placements...)235+ var focus layout.PaneID
230- scratch.dividers = append(scratch.dividers[:0], layoutSnap.dividers...)236+ if tb.tree != nil {
237+ focus = tb.tree.Focus
238+ }
239+ layoutSnap := tabLayoutSnapshot{
240+ area: area, focus: focus, placements: scratch.placements,
241+ dividers: scratch.dividers, fingerprint: scratch.layoutFingerprint, ok: scratch.layoutValid,
242+ }
243+ if scratch.layoutTab != tb || scratch.layoutGeneration != tb.layoutGeneration || scratch.layoutArea != area || scratch.layoutFocus != focus {
244+ layoutSnap = solveTabLayoutLocked(tb)
245+ scratch.placements = append(scratch.placements[:0], layoutSnap.placements...)
246+ scratch.dividers = append(scratch.dividers[:0], layoutSnap.dividers...)
247+ scratch.layoutTab = tb
248+ scratch.layoutGeneration = tb.layoutGeneration
249+ scratch.layoutArea = area
250+ scratch.layoutFocus = focus
251+ scratch.layoutFingerprint = layoutSnap.fingerprint
252+ scratch.layoutValid = layoutSnap.ok
253+ layoutSnap.placements = scratch.placements
254+ layoutSnap.dividers = scratch.dividers
255+ }
231 state := &scratch.state256 state := &scratch.state
232 *state = capturedRenderState{257 *state = capturedRenderState{
233 attachment: ac, lease: lease, reset: reset, bars: bars, theme: bars.theme,258 attachment: ac, lease: lease, reset: reset, bars: bars, theme: bars.theme,
M internal/usecase/daemon/session.go +263 -126
@@ -15,7 +15,6 @@ import (
15 "time"15 "time"
16 16
17 "github.com/bnema/vev/internal/domain"17 "github.com/bnema/vev/internal/domain"
18- "github.com/bnema/vev/internal/persist"
19 "github.com/bnema/vev/internal/ports"18 "github.com/bnema/vev/internal/ports"
20 "github.com/bnema/vev/internal/usecase/layout"19 "github.com/bnema/vev/internal/usecase/layout"
21 )20 )
@@ -47,22 +46,31 @@ type session struct {
47 // tab, or pane locks. The owner performs teardown without holding teardownMu;46 // tab, or pane locks. The owner performs teardown without holding teardownMu;
48 // duplicate daemon-shutdown callers can therefore bound their ownership wait47 // duplicate daemon-shutdown callers can therefore bound their ownership wait
49 // with Serve's shared snapshot deadline.48 // with Serve's shared snapshot deadline.
50- teardownMu sync.Mutex49+ teardownMu sync.Mutex
51- teardownActive bool50+ teardownActive bool
52- teardownDone chan struct{}51+ teardownDone chan struct{}
53- teardownWaiters uint52+ teardownWaiters uint
54- teardownChanged chan struct{}53+ teardownChanged chan struct{}
55- lifecycleStopOnce sync.Once54+ lifecycleStopOnce sync.Once
56- mu sync.Mutex // guards tabs, active, client, clipFiles, and clipboard queue state55+ mu sync.Mutex // guards tabs, active, client, clipFiles, and clipboard queue state
57- themeMu sync.Mutex56+ // metadataPersistMu serializes authority writes and post-I/O rollback. State
58- tabs []*tab57+ // mutation paths must release d.mu and mu before acquiring it. After durable
59- active int58+ // I/O completes it may acquire d.mu then mu to reconcile failed revisions.
60- client *attachedClient59+ metadataPersistMu sync.Mutex
61- clipboardQueue []clipboardForward60+ metadataVersion uint64 // guarded by mu; every durable metadata snapshot receives a revision
62- clipboardWorkerRunning bool61+ metadataDurableVersion uint64 // guarded by metadataPersistMu
63- renameInProgress bool62+ metadataLiveVersion uint64 // rollback cursor guarded by metadataPersistMu
64- cwd string63+ metadataFailedRollbacks map[uint64]func() bool // guarded by metadataPersistMu
65- terminal terminalEnv64+ themeMu sync.Mutex
65+ tabs []*tab
66+ active int
67+ client *attachedClient
68+ clipboardQueue []clipboardForward
69+ clipboardWorkerRunning bool
70+ renameInProgress bool
71+ renameDone chan struct{}
72+ cwd string
73+ terminal terminalEnv
66 // env is the authoritative immutable environment snapshot for future PTY children.74 // env is the authoritative immutable environment snapshot for future PTY children.
67 // It is guarded by mu and is always copied on ingress and egress.75 // It is guarded by mu and is always copied on ingress and egress.
68 env []string76 env []string
@@ -167,12 +175,14 @@ func (d *Daemon) touchMRU(sess *session) {
167 }175 }
168 seq := d.mruSeq.Add(1)176 seq := d.mruSeq.Add(1)
169 updated := false177 updated := false
178+ var previous uint64
170 for {179 for {
171 old := sess.mruAt.Load()180 old := sess.mruAt.Load()
172 if old >= seq {181 if old >= seq {
173 return182 return
174 }183 }
175 if sess.mruAt.CompareAndSwap(old, seq) {184 if sess.mruAt.CompareAndSwap(old, seq) {
185+ previous = old
176 updated = true186 updated = true
177 break187 break
178 }188 }
@@ -182,19 +192,49 @@ func (d *Daemon) touchMRU(sess *session) {
182 }192 }
183 sess.mu.Lock()193 sess.mu.Lock()
184 name := sess.name194 name := sess.name
185- incarnation := sess.incarnation
186 ephemeral := sess.ephemeral195 ephemeral := sess.ephemeral
196+ var record domain.CatalogueRecord
197+ var version uint64
198+ if !ephemeral {
199+ record, version = sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1)))
200+ }
187 sess.mu.Unlock()201 sess.mu.Unlock()
188 if !ephemeral {202 if !ephemeral {
189- if err := d.updateCatalogueMetadata(domain.CatalogueMetadataUpdate{Name: name, IncarnationID: incarnation, LastUsedSeq: &seq}); err != nil {203+ rollback := func() bool {
204+ return sess.mruAt.CompareAndSwap(seq, previous)
205+ }
206+ if err := d.persistSessionMetadata(sess, version, record.MetadataUpdate(), rollback); err != nil {
190 d.log.Warn("touching persisted session recency failed", "err", err, "session", name)207 d.log.Warn("touching persisted session recency failed", "err", err, "session", name)
191 }208 }
192 }209 }
193 }210 }
194 211
195 func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) {212 func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) {
213+ // Historical focused tests call this helper directly. Acquire mu for those
214+ // callers while preserving the production contract that an owning caller
215+ // receives it still locked on return.
216+ acquiredMu := d.mu.TryLock()
217+ if acquiredMu {
218+ defer d.mu.Unlock()
219+ }
196 env = copyEnvironment(env)220 env = copyEnvironment(env)
221+ if _, reserved := d.creating[name]; reserved {
222+ return nil, errSessionNameInUse
223+ }
224+ d.creating[name] = struct{}{}
225+ defer delete(d.creating, name)
197 stopped, resuming := d.stopped[name]226 stopped, resuming := d.stopped[name]
227+ var authoritative domain.CatalogueRecord
228+ var authoritativeExists bool
229+ if !ephemeral && resuming && d.persistEnabled {
230+ d.mu.Unlock()
231+ var err error
232+ authoritative, authoritativeExists, err = d.catalogueRecord(name)
233+ d.mu.Lock()
234+ if err != nil {
235+ return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't read session catalogue", err)
236+ }
237+ }
198 var createdAt int64238 var createdAt int64
199 var incarnation domain.IncarnationID239 var incarnation domain.IncarnationID
200 if !ephemeral {240 if !ephemeral {
@@ -203,9 +243,9 @@ func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz
203 // the persisted identity rather than receive a fresh timestamp.243 // the persisted identity rather than receive a fresh timestamp.
204 createdAt = stopped.createdAt244 createdAt = stopped.createdAt
205 incarnation = stopped.incarnation245 incarnation = stopped.incarnation
206- if authoritative, ok := d.catalogueRecord(name); ok {246+ if authoritativeExists {
207- incarnation = authoritative.IncarnationID
208 createdAt = authoritative.CreatedAt247 createdAt = authoritative.CreatedAt
248+ incarnation = authoritative.IncarnationID
209 }249 }
210 if createdAt > d.lastAllocatedCreatedAt {250 if createdAt > d.lastAllocatedCreatedAt {
211 d.lastAllocatedCreatedAt = createdAt251 d.lastAllocatedCreatedAt = createdAt
@@ -263,6 +303,9 @@ func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz
263 if !ephemeral && resuming {303 if !ephemeral && resuming {
264 lastUsedSeq = stopped.lastUsedSeq304 lastUsedSeq = stopped.lastUsedSeq
265 }305 }
306+ if lastUsedSeq == 0 {
307+ lastUsedSeq = d.mruSeq.Add(1)
308+ }
266 sess := &session{309 sess := &session{
267 id: id,310 id: id,
268 name: name,311 name: name,
@@ -286,34 +329,50 @@ func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz
286 sess.snapEligible.Store(!ephemeral && name != "")329 sess.snapEligible.Store(!ephemeral && name != "")
287 if !ephemeral {330 if !ephemeral {
288 if d.persistEnabled {331 if d.persistEnabled {
289- record := persist.Record{Name: name, IncarnationID: incarnation, Cwd: cwd, CreatedAt: createdAt, UpdatedAt: createdAt, LastUsedSeq: lastUsedSeq, TabNames: names, RecoveryState: domain.RecoveryFresh}332+ record := domain.CatalogueRecord{Name: name, IncarnationID: incarnation, Cwd: cwd, CreatedAt: createdAt, UpdatedAt: createdAt, LastUsedSeq: lastUsedSeq, TabNames: names, RecoveryState: domain.RecoveryFresh}
333+ if d.lifecycleRecovery == nil {
334+ closeTabs(tabs)
335+ cancel()
336+ return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", errors.New("durable session authority is not configured"))
337+ }
338+ // The name reservation remains visible while durable authority I/O runs,
339+ // but daemon/session mutexes are released so storage cannot block routing.
340+ d.mu.Unlock()
290 var err error341 var err error
291- if resuming {342+ if resuming && authoritativeExists {
292- if _, ok := d.catalogueRecord(name); ok {343+ err = d.updateCatalogueMetadata(record.MetadataUpdate())
293- err = d.updateCatalogueMetadata(record.MetadataUpdate())
294- } else if d.lifecycleRecovery != nil {
295- record, err = d.lifecycleRecovery.Create(context.Background(), record)
296- } else {
297- err = d.createCatalogueRecord(record)
298- }
299- } else if d.lifecycleRecovery != nil {
300- record, err = d.lifecycleRecovery.Create(context.Background(), record)
301 } else {344 } else {
302- err = d.createCatalogueRecord(record)345+ record, err = d.lifecycleRecovery.Create(d.serveCtx, record)
303 }346 }
347+ d.mu.Lock()
304 if err != nil {348 if err != nil {
305 closeTabs(tabs)349 closeTabs(tabs)
306 cancel()350 cancel()
307 return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", err)351 return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", err)
308 }352 }
353+ if d.closing {
354+ // Creation lost its race with shutdown. Roll back a newly committed
355+ // authority record without holding mu; resumed metadata remains owned
356+ // by the existing stopped lifecycle.
357+ if !resuming {
358+ d.mu.Unlock()
359+ rollbackCtx, rollbackCancel := context.WithTimeout(context.WithoutCancel(d.serveCtx), snapshotFinalFlushTimeout)
360+ rollbackErr := d.lifecycleRecovery.Delete(rollbackCtx, name)
361+ rollbackCancel()
362+ d.mu.Lock()
363+ if rollbackErr != nil {
364+ d.log.Warn("rolling back session creation during shutdown failed", "session", name, "err", rollbackErr)
365+ }
366+ }
367+ closeTabs(tabs)
368+ cancel()
369+ return nil, errors.New("daemon is shutting down")
370+ }
309 sess.incarnation = record.IncarnationID371 sess.incarnation = record.IncarnationID
310 }372 }
311 delete(d.stopped, name)373 delete(d.stopped, name)
312 }374 }
313 d.sessions[id] = sess375 d.sessions[id] = sess
314- if lastUsedSeq == 0 {
315- d.touchMRU(sess)
316- }
317 d.log.Info("session created", "session", name, "id", id, "ephemeral", ephemeral)376 d.log.Info("session created", "session", name, "id", id, "ephemeral", ephemeral)
318 for i, tb := range tabs {377 for i, tb := range tabs {
319 d.log.Info("tab created", "session", name, "tab", i)378 d.log.Info("tab created", "session", name, "tab", i)
@@ -445,25 +504,41 @@ func (d *Daemon) createTab(sess *session, sz domain.Size) error {
445 sess.tabs = append(sess.tabs, tb)504 sess.tabs = append(sess.tabs, tb)
446 sess.active = len(sess.tabs) - 1505 sess.active = len(sess.tabs) - 1
447 tabIndex := sess.active506 tabIndex := sess.active
448- record := sess.persistRecordLocked(time.Now().UnixNano())507+ record, metadataVersion := sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1)))
449 ephemeral := sess.ephemeral508 ephemeral := sess.ephemeral
509+ sess.mu.Unlock()
510+ d.mu.Unlock()
450 if !ephemeral {511 if !ephemeral {
451- if err := d.updateCatalogueMetadata(record.MetadataUpdate()); err != nil {512+ rollback := func() bool {
452- sess.tabs = sess.tabs[:len(sess.tabs)-1]513+ d.mu.Lock()
453- sess.active = oldActive514+ defer d.mu.Unlock()
515+ if d.sessions[sess.id] != sess {
516+ return true
517+ }
518+ sess.mu.Lock()
519+ if !slices.Contains(sess.tabs, tb) {
520+ sess.mu.Unlock()
521+ return true
522+ }
523+ if sess.tabs[len(sess.tabs)-1] != tb {
524+ sess.mu.Unlock()
525+ return false
526+ }
527+ sess.tabs = slices.DeleteFunc(sess.tabs, func(candidate *tab) bool { return candidate == tb })
528+ sess.active = min(oldActive, len(sess.tabs)-1)
529+ sess.mu.Unlock()
454 if tb.cancel != nil {530 if tb.cancel != nil {
455 tb.cancel()531 tb.cancel()
456 }532 }
457- sess.mu.Unlock()
458- d.mu.Unlock()
459 _ = pty.Close()533 _ = pty.Close()
534+ return true
535+ }
536+ if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {
460 return err537 return err
461 }538 }
462 }539 }
463- sess.mu.Unlock()
464 d.log.Info("tab created", "session", name, "tab", tabIndex)540 d.log.Info("tab created", "session", name, "tab", tabIndex)
465 d.startTabGoroutines(sess, tb)541 d.startTabGoroutines(sess, tb)
466- d.mu.Unlock()
467 d.activateTab(sess, tb)542 d.activateTab(sess, tb)
468 markSnapshotDirty(sess)543 markSnapshotDirty(sess)
469 return nil544 return nil
@@ -686,51 +761,43 @@ func (d *Daemon) renameSession(sess *session, name string) error {
686 }761 }
687 }762 }
688 lastUsedSeq := sess.mruAt.Load()763 lastUsedSeq := sess.mruAt.Load()
689- record := sess.persistRecordLocked(d.nowUnixNano())764+ record := sess.persistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1)))
690 record.Name = name765 record.Name = name
691 record.CreatedAt = createdAt766 record.CreatedAt = createdAt
692 record.LastUsedSeq = lastUsedSeq767 record.LastUsedSeq = lastUsedSeq
693 sess.renameInProgress = true768 sess.renameInProgress = true
769+ sess.renameDone = make(chan struct{})
694 sess.mu.Unlock()770 sess.mu.Unlock()
695 d.mu.Unlock()771 d.mu.Unlock()
696 772
697 rollback := func(err error) error {773 rollback := func(err error) error {
698 sess.mu.Lock()774 sess.mu.Lock()
699 sess.renameInProgress = false775 sess.renameInProgress = false
776+ if sess.renameDone != nil {
777+ close(sess.renameDone)
778+ sess.renameDone = nil
779+ }
700 sess.mu.Unlock()780 sess.mu.Unlock()
701 return err781 return err
702 }782 }
703 // Durable named renames are one atomic catalogue batch. The incarnation-keyed783 // Durable named renames are one atomic catalogue batch. The incarnation-keyed
704 // snapshot namespace is intentionally untouched.784 // snapshot namespace is intentionally untouched.
705 if (wasEphemeral || oldName != name) && d.persistEnabled {785 if (wasEphemeral || oldName != name) && d.persistEnabled {
706- if d.lifecycleRecovery != nil {786+ if d.lifecycleRecovery == nil {
787+ return rollback(errors.New("durable session authority is not configured"))
788+ }
789+ {
707 var committed domain.CatalogueRecord790 var committed domain.CatalogueRecord
708 var err error791 var err error
709 if wasEphemeral {792 if wasEphemeral {
710- committed, err = d.lifecycleRecovery.Create(context.Background(), record)793+ committed, err = d.lifecycleRecovery.Create(d.serveCtx, record)
711 } else {794 } else {
712- committed, err = d.lifecycleRecovery.Rename(context.Background(), oldName, name)795+ committed, err = d.lifecycleRecovery.Rename(d.serveCtx, oldName, name)
713 }796 }
714 if err != nil {797 if err != nil {
715 return rollback(err)798 return rollback(err)
716 }799 }
717 record = committed800 record = committed
718- } else if wasEphemeral {
719- if err := d.createCatalogueRecord(record); err != nil {
720- return rollback(err)
721- }
722- } else {
723- if current, ok := d.catalogueRecord(oldName); ok {
724- current.Name = name
725- current.Cwd = record.Cwd
726- current.UpdatedAt = record.UpdatedAt
727- current.LastUsedSeq = record.LastUsedSeq
728- current.TabNames = append([]string(nil), record.TabNames...)
729- record = current
730- }
731- if err := d.catalogue.Rename(oldName, record); err != nil {
732- return rollback(err)
733- }
734 }801 }
735 }802 }
736 803
@@ -745,6 +812,10 @@ func (d *Daemon) renameSession(sess *session, name string) error {
745 sess.incarnation = record.IncarnationID812 sess.incarnation = record.IncarnationID
746 sess.ephemeral = false813 sess.ephemeral = false
747 sess.renameInProgress = false814 sess.renameInProgress = false
815+ if sess.renameDone != nil {
816+ close(sess.renameDone)
817+ sess.renameDone = nil
818+ }
748 sess.snapEligible.Store(name != "")819 sess.snapEligible.Store(name != "")
749 sess.mu.Unlock()820 sess.mu.Unlock()
750 d.mu.Unlock()821 d.mu.Unlock()
@@ -774,22 +845,42 @@ func (d *Daemon) renameTab(sess *session, tb *tab, name string) error {
774 }845 }
775 oldName := tb.name846 oldName := tb.name
776 tb.name = name847 tb.name = name
777- record := sess.persistRecordLocked(time.Now().UnixNano())848+ record, metadataVersion := sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1)))
778 ephemeral := sess.ephemeral849 ephemeral := sess.ephemeral
779- if !ephemeral {
780- if err := d.updateCatalogueMetadata(record.MetadataUpdate()); err != nil {
781- tb.name = oldName
782- sess.mu.Unlock()
783- d.mu.Unlock()
784- return err
785- }
786- }
787 sess.mu.Unlock()850 sess.mu.Unlock()
788 d.mu.Unlock()851 d.mu.Unlock()
852+ if ephemeral {
853+ return nil
854+ }
855+ rollback := func() bool {
856+ d.mu.Lock()
857+ defer d.mu.Unlock()
858+ if d.sessions[sess.id] != sess {
859+ return true
860+ }
861+ sess.mu.Lock()
862+ defer sess.mu.Unlock()
863+ if !slices.Contains(sess.tabs, tb) {
864+ return true
865+ }
866+ if tb.name != name {
867+ return false
868+ }
869+ tb.name = oldName
870+ return true
871+ }
872+ if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {
873+ return err
874+ }
789 return nil875 return nil
790 }876 }
791 877
792-func (s *session) persistRecordLocked(updatedAt int64) persist.Record {878+func (s *session) nextPersistRecordLocked(updatedAt int64) (domain.CatalogueRecord, uint64) {
879+ s.metadataVersion++
880+ return s.persistRecordLocked(updatedAt), s.metadataVersion
881+}
882+
883+func (s *session) persistRecordLocked(updatedAt int64) domain.CatalogueRecord {
793 createdAt := s.createdAt884 createdAt := s.createdAt
794 tabNames := make([]string, len(s.tabs))885 tabNames := make([]string, len(s.tabs))
795 lastCustom := -1886 lastCustom := -1
@@ -804,7 +895,7 @@ func (s *session) persistRecordLocked(updatedAt int64) persist.Record {
804 } else {895 } else {
805 tabNames = tabNames[:lastCustom+1]896 tabNames = tabNames[:lastCustom+1]
806 }897 }
807- return persist.Record{Name: s.name, IncarnationID: s.incarnation, Cwd: s.cwd, CreatedAt: createdAt, UpdatedAt: updatedAt, LastUsedSeq: s.mruAt.Load(), TabNames: tabNames, RecoveryState: domain.RecoveryFresh}898+ return domain.CatalogueRecord{Name: s.name, IncarnationID: s.incarnation, Cwd: s.cwd, CreatedAt: createdAt, UpdatedAt: updatedAt, LastUsedSeq: s.mruAt.Load(), TabNames: tabNames, RecoveryState: domain.RecoveryFresh}
808 }899 }
809 900
810 func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error {901 func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error {
@@ -843,8 +934,10 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error {
843 return nil934 return nil
844 }935 }
845 ringing := tb.attention936 ringing := tb.attention
937+ oldActive := sess.active
846 wasActive := idx == sess.active938 wasActive := idx == sess.active
847 sess.tabs = append(sess.tabs[:idx], sess.tabs[idx+1:]...)939 sess.tabs = append(sess.tabs[:idx], sess.tabs[idx+1:]...)
940+ tabsAfterClose := len(sess.tabs)
848 if sess.active >= len(sess.tabs) {941 if sess.active >= len(sess.tabs) {
849 sess.active = len(sess.tabs) - 1942 sess.active = len(sess.tabs) - 1
850 } else if idx < sess.active {943 } else if idx < sess.active {
@@ -853,15 +946,34 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error {
853 destination := sess.tabs[sess.active]946 destination := sess.tabs[sess.active]
854 ac := sess.client947 ac := sess.client
855 name := sess.name948 name := sess.name
856- record := sess.persistRecordLocked(time.Now().UnixNano())949+ record, metadataVersion := sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1)))
857 ephemeral := sess.ephemeral950 ephemeral := sess.ephemeral
951+ sess.mu.Unlock()
952+ d.mu.Unlock()
858 if !ephemeral {953 if !ephemeral {
859- if err := d.updateCatalogueMetadata(record.MetadataUpdate()); err != nil {954+ rollback := func() bool {
955+ d.mu.Lock()
956+ defer d.mu.Unlock()
957+ if d.sessions[sess.id] != sess {
958+ return true
959+ }
960+ sess.mu.Lock()
961+ defer sess.mu.Unlock()
962+ if slices.Contains(sess.tabs, tb) {
963+ return true
964+ }
965+ if len(sess.tabs) != tabsAfterClose {
966+ return false
967+ }
968+ sess.tabs = slices.Insert(sess.tabs, min(idx, len(sess.tabs)), tb)
969+ sess.active = min(oldActive, len(sess.tabs)-1)
970+ return true
971+ }
972+ if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {
860 d.log.Warn("persisting closed tab failed", "err", err, "session", name)973 d.log.Warn("persisting closed tab failed", "err", err, "session", name)
974+ return err
861 }975 }
862 }976 }
863- sess.mu.Unlock()
864- d.mu.Unlock()
865 d.log.Info("tab closed", "session", name)977 d.log.Info("tab closed", "session", name)
866 markSnapshotDirty(sess)978 markSnapshotDirty(sess)
867 979
@@ -900,60 +1012,48 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error {
900 // beginSnapshotPurge commits a logical named-session kill before its live1012 // beginSnapshotPurge commits a logical named-session kill before its live
901 // identity or persisted metadata is removed. It never runs under daemon or1013 // identity or persisted metadata is removed. It never runs under daemon or
902 // session locks.1014 // session locks.
903-func (d *Daemon) beginSnapshotPurge(name string, incarnation domain.IncarnationID) error {1015+func (d *Daemon) beginSnapshotPurge(_ string, _ domain.IncarnationID) error {
904- if d.lifecycleRecovery != nil {1016+ if d.persistEnabled && d.lifecycleRecovery == nil {
905- // The lifecycle coordinator performs the complete ordered protocol after1017+ return errors.New("durable session authority is not configured")
906- // snapshot writers have joined.
907- return nil
908- }
909- if d.snapshotRepository == nil || name == "" {
910- return nil
911 }1018 }
912- return d.snapshotRepository.WriteDeletionTombstone(context.Background(), domain.DeletionTombstone{Name: name, IncarnationID: incarnation})1019+ // The lifecycle coordinator performs the complete ordered protocol after
1020+ // snapshot writers have joined.
1021+ return nil
913 }1022 }
914 1023
915 // finishSnapshotPurge deletes both independently durable snapshot sources,1024 // finishSnapshotPurge deletes both independently durable snapshot sources,
916 // then metadata, and only then clears the tombstone. Any failure deliberately1025 // then metadata, and only then clears the tombstone. Any failure deliberately
917 // leaves the marker in place so startup cannot restore or import the name and1026 // leaves the marker in place so startup cannot restore or import the name and
918 // a later live/offline kill can retry the idempotent source deletes.1027 // a later live/offline kill can retry the idempotent source deletes.
919-func (d *Daemon) finishSnapshotPurge(name string, incarnation domain.IncarnationID) error {1028+func (d *Daemon) finishSnapshotPurge(ctx context.Context, name string, _ domain.IncarnationID) error {
920- if d.lifecycleRecovery != nil {1029+ if !d.persistEnabled {
921- return d.lifecycleRecovery.Delete(context.Background(), name)1030+ return nil
922- }
923- if d.snapshotRepository == nil {
924- return d.deleteCatalogueRecord(name)
925 }1031 }
926- if err := d.snapshotRepository.QuarantineDeletionSources(context.Background(), domain.DeletionTombstone{Name: name, IncarnationID: incarnation}, false); err != nil {1032+ if d.lifecycleRecovery == nil {
927- return err1033+ return errors.New("durable session authority is not configured")
928 }1034 }
929- if err := d.deleteCatalogueRecord(name); err != nil {1035+ if ctx == nil {
930- return err1036+ ctx = d.serveCtx
931 }1037 }
932- return d.snapshotRepository.DeleteDeletionTombstone(context.Background(), incarnation)1038+ purgeCtx, cancel := context.WithTimeout(ctx, snapshotFinalFlushTimeout)
1039+ defer cancel()
1040+ return d.lifecycleRecovery.Delete(purgeCtx, name)
933 }1041 }
934 1042
935 // retryStoppedPurge completes a previously closed session's durable purge.1043 // retryStoppedPurge completes a previously closed session's durable purge.
936 // It keeps the stopped record hidden until all sources and metadata cross their1044 // It keeps the stopped record hidden until all sources and metadata cross their
937 // deletion boundaries.1045 // deletion boundaries.
938 func (d *Daemon) retryStoppedPurge(name string) error {1046 func (d *Daemon) retryStoppedPurge(name string) error {
1047+ return d.retryStoppedPurgeContext(d.serveCtx, name)
1048+}
1049+
1050+func (d *Daemon) retryStoppedPurgeContext(ctx context.Context, name string) error {
939 d.mu.Lock()1051 d.mu.Lock()
940 stopped, ok := d.stopped[name]1052 stopped, ok := d.stopped[name]
941 if !ok {1053 if !ok {
942 d.mu.Unlock()1054 d.mu.Unlock()
943 return nil1055 return nil
944 }1056 }
945- // Without a repository there is no restore/import source to fence, so the
946- // stopped record retains the metadata-only deletion behavior.
947- if d.snapshotRepository == nil {
948- d.mu.Unlock()
949- if err := d.deleteCatalogueRecord(name); err != nil {
950- return err
951- }
952- d.mu.Lock()
953- delete(d.stopped, name)
954- d.mu.Unlock()
955- return nil
956- }
957 stopped.purging = true1057 stopped.purging = true
958 d.stopped[name] = stopped1058 d.stopped[name] = stopped
959 d.mu.Unlock()1059 d.mu.Unlock()
@@ -961,7 +1061,7 @@ func (d *Daemon) retryStoppedPurge(name string) error {
961 if err := d.beginSnapshotPurge(name, stopped.incarnation); err != nil {1061 if err := d.beginSnapshotPurge(name, stopped.incarnation); err != nil {
962 return err1062 return err
963 }1063 }
964- if err := d.finishSnapshotPurge(name, stopped.incarnation); err != nil {1064+ if err := d.finishSnapshotPurge(ctx, name, stopped.incarnation); err != nil {
965 return err1065 return err
966 }1066 }
967 d.mu.Lock()1067 d.mu.Lock()
@@ -1089,6 +1189,23 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu
1089 }1189 }
1090 defer sess.finishTeardown()1190 defer sess.finishTeardown()
1091 1191
1192+ // A rename reserves durable identity while coordinator I/O runs unlocked.
1193+ // Teardown waits for that reservation before selecting the purge name.
1194+ sess.mu.Lock()
1195+ renameDone := sess.renameDone
1196+ sess.mu.Unlock()
1197+ if renameDone != nil {
1198+ if deadline == nil {
1199+ <-renameDone
1200+ } else {
1201+ select {
1202+ case <-renameDone:
1203+ case <-deadline.Done():
1204+ return context.DeadlineExceeded
1205+ }
1206+ }
1207+ }
1208+
1092 d.mu.Lock()1209 d.mu.Lock()
1093 current := d.sessions[sess.id]1210 current := d.sessions[sess.id]
1094 d.mu.Unlock()1211 d.mu.Unlock()
@@ -1106,13 +1223,14 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu
1106 sess.mu.Lock()1223 sess.mu.Lock()
1107 isEphemeral := sess.ephemeral1224 isEphemeral := sess.ephemeral
1108 name := sess.name1225 name := sess.name
1226+ incarnation := sess.incarnation
1109 sess.mu.Unlock()1227 sess.mu.Unlock()
1110 // Commit the tombstone before changing live identity or metadata. A1228 // Commit the tombstone before changing live identity or metadata. A
1111 // repository delete must never race an older publication that could recreate1229 // repository delete must never race an older publication that could recreate
1112 // the just-quarantined name. Do not hold daemon or session locks while1230 // the just-quarantined name. Do not hold daemon or session locks while
1113 // tombstoning or joining: a repository implementation may block on I/O.1231 // tombstoning or joining: a repository implementation may block on I/O.
1114 if purge && !isEphemeral {1232 if purge && !isEphemeral {
1115- if err := d.beginSnapshotPurge(name, sess.incarnation); err != nil {1233+ if err := d.beginSnapshotPurge(name, incarnation); err != nil {
1116 return err1234 return err
1117 }1235 }
1118 if d.snapshotRepository != nil {1236 if d.snapshotRepository != nil {
@@ -1188,7 +1306,7 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu
1188 ephemeral := sess.ephemeral1306 ephemeral := sess.ephemeral
1189 sess.mu.Unlock()1307 sess.mu.Unlock()
1190 if !ephemeral {1308 if !ephemeral {
1191- stopped := stoppedSession{name: stoppedName, cwd: stoppedCwd, createdAt: createdAt, incarnation: sess.incarnation, lastUsedSeq: sess.mruAt.Load(), tabNames: tabNames, purging: purge}1309+ stopped := stoppedSession{name: stoppedName, cwd: stoppedCwd, createdAt: createdAt, incarnation: incarnation, lastUsedSeq: sess.mruAt.Load(), tabNames: tabNames, purging: purge}
1192 d.stopped[stoppedName] = stopped1310 d.stopped[stoppedName] = stopped
1193 }1311 }
1194 empty := len(d.sessions) == 01312 empty := len(d.sessions) == 0
@@ -1248,7 +1366,7 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu
1248 // tombstone and hidden stopped record when any source or metadata operation1366 // tombstone and hidden stopped record when any source or metadata operation
1249 // fails; retryStoppedPurge (including a repeated live kill) resumes safely.1367 // fails; retryStoppedPurge (including a repeated live kill) resumes safely.
1250 if !ephemeral && purge {1368 if !ephemeral && purge {
1251- if err := d.finishSnapshotPurge(stoppedName, sess.incarnation); err != nil {1369+ if err := d.finishSnapshotPurge(d.serveCtx, stoppedName, incarnation); err != nil {
1252 purgeErr = errors.Join(purgeErr, err)1370 purgeErr = errors.Join(purgeErr, err)
1253 d.log.Warn("finishing session snapshot purge failed", "err", err, "session", stoppedName)1371 d.log.Warn("finishing session snapshot purge failed", "err", err, "session", stoppedName)
1254 } else {1372 } else {
@@ -1302,8 +1420,11 @@ func (d *Daemon) nameLiveOrStoppedLocked(name string) bool {
1302 if d.findByNameLocked(name) != nil {1420 if d.findByNameLocked(name) != nil {
1303 return true1421 return true
1304 }1422 }
1305- _, ok := d.stopped[name]1423+ if _, ok := d.stopped[name]; ok {
1306- return ok1424+ return true
1425+ }
1426+ _, reserved := d.creating[name]
1427+ return reserved
1307 }1428 }
1308 1429
1309 func (d *Daemon) cwdSampler(ctx context.Context) {1430 func (d *Daemon) cwdSampler(ctx context.Context) {
@@ -1363,19 +1484,38 @@ func (d *Daemon) refreshSessionCwd(sess *session) {
1363 d.mu.Unlock()1484 d.mu.Unlock()
1364 return1485 return
1365 }1486 }
1487+ oldCwd := sess.cwd
1366 sess.cwd = cwd1488 sess.cwd = cwd
1367 name := sess.name1489 name := sess.name
1368- incarnation := sess.incarnation
1369 ephemeral := sess.ephemeral1490 ephemeral := sess.ephemeral
1491+ var record domain.CatalogueRecord
1492+ var metadataVersion uint64
1493+ if !ephemeral {
1494+ record, metadataVersion = sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1)))
1495+ }
1370 sess.mu.Unlock()1496 sess.mu.Unlock()
1497+ d.mu.Unlock()
1371 if !ephemeral {1498 if !ephemeral {
1372- updatedAt := time.Now().UnixNano()1499+ rollback := func() bool {
1373- if err := d.updateCatalogueMetadata(domain.CatalogueMetadataUpdate{Name: name, IncarnationID: incarnation, Cwd: &cwd, UpdatedAt: &updatedAt}); err != nil {1500+ d.mu.Lock()
1501+ defer d.mu.Unlock()
1502+ if d.sessions[sess.id] != sess {
1503+ return true
1504+ }
1505+ sess.mu.Lock()
1506+ defer sess.mu.Unlock()
1507+ if sess.cwd != cwd {
1508+ return false
1509+ }
1510+ sess.cwd = oldCwd
1511+ return true
1512+ }
1513+ if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil {
1374 d.log.Warn("touching persisted session cwd failed", "err", err, "session", name)1514 d.log.Warn("touching persisted session cwd failed", "err", err, "session", name)
1515+ return
1375 }1516 }
1376 markSnapshotDirty(sess)1517 markSnapshotDirty(sess)
1377 }1518 }
1378- d.mu.Unlock()
1379 }1519 }
1380 1520
1381 // childEnv retains the daemon-environment helper for daemon-local legacy callers.1521 // childEnv retains the daemon-environment helper for daemon-local legacy callers.
@@ -1389,11 +1529,8 @@ func copyEnvironment(env []string) []string {
1389 }1529 }
1390 1530
1391 func environmentEntry(entry string) (name, value string, ok bool) {1531 func environmentEntry(entry string) (name, value string, ok bool) {
1392- i := strings.IndexByte(entry, '=')1532+ name, value, ok = strings.Cut(entry, "=")
1393- if i < 0 {1533+ return name, value, ok
1394- return "", "", false
1395- }
1396- return entry[:i], entry[i+1:], true
1397 }1534 }
1398 1535
1399 func (d *Daemon) ptyCommand(env []string) (string, []string) {1536 func (d *Daemon) ptyCommand(env []string) (string, []string) {
M internal/usecase/daemon/session_purge_control_test.go +5 -20
@@ -4,40 +4,25 @@ import (
4 "errors"4 "errors"
5 "testing"5 "testing"
6 6
7- "github.com/stretchr/testify/mock"
8 "github.com/stretchr/testify/require"7 "github.com/stretchr/testify/require"
9 8
10- "github.com/bnema/vev/internal/ports"9+ "github.com/bnema/vev/internal/domain"
11 portsmocks "github.com/bnema/vev/internal/ports/mocks"10 portsmocks "github.com/bnema/vev/internal/ports/mocks"
12 )11 )
13 12
14-func TestHandleKillStoppedUsesDurableBothSourcePurge(t *testing.T) {
15- d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
16- repository := &retryablePurgeRepository{legacyErr: errors.New("legacy delete failed")}
17- WithSnapshotRepository(repository, repository)(d)
18- d.stopped["work"] = stoppedSession{name: "work"}
19-
20- transport := portsmocks.NewMockTransport(t)
21- transport.EXPECT().Send(mock.Anything).Return(nil).Once()
22- transport.EXPECT().Close().Return(nil).Once()
23- d.handleKill(transport, ports.Frame{Type: ports.MsgKill, Payload: ports.MarshalKill(ports.Kill{Name: "work"})})
24-
25- require.Equal(t, []string{"tombstone", "incremental", "legacy"}, repository.calls)
26- require.True(t, repository.tombstoned["work"])
27- require.True(t, d.stopped["work"].purging)
28-}
29-
30 func TestStoppedPurgeMetadataFailureRemainsFencedForRetry(t *testing.T) {13 func TestStoppedPurgeMetadataFailureRemainsFencedForRetry(t *testing.T) {
31 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})14 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
32 repository := &retryablePurgeRepository{}15 repository := &retryablePurgeRepository{}
33- WithSnapshotRepository(repository, repository)(d)16+ WithSnapshotRepository(repository)(d)
34 store, state := newMockStore(t)17 store, state := newMockStore(t)
35 WithStore(store)(d)18 WithStore(store)(d)
19+ record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}
20+ require.NoError(t, d.catalogue.Create(record))
36 metadataErr := errors.New("metadata delete failed")21 metadataErr := errors.New("metadata delete failed")
37 state.mu.Lock()22 state.mu.Lock()
38 state.deleteErr = func(string) error { return metadataErr }23 state.deleteErr = func(string) error { return metadataErr }
39 state.mu.Unlock()24 state.mu.Unlock()
40- d.stopped["work"] = stoppedSession{name: "work"}25+ d.stopped["work"] = stoppedSession{name: "work", incarnation: record.IncarnationID, record: record}
41 26
42 require.ErrorIs(t, d.retryStoppedPurge("work"), metadataErr)27 require.ErrorIs(t, d.retryStoppedPurge("work"), metadataErr)
43 require.True(t, repository.tombstoned["work"])28 require.True(t, repository.tombstoned["work"])
M internal/usecase/daemon/session_purge_retry_test.go +20 -4
@@ -18,12 +18,15 @@ type retryablePurgeRepository struct {
18 legacyErr error18 legacyErr error
19 tombstoned map[string]bool19 tombstoned map[string]bool
20 calls []string20 calls []string
21+ deadlineScoped []bool
21 }22 }
22 23
23 func (r *retryablePurgeRepository) Publish(context.Context, ports.SnapshotPublication) error {24 func (r *retryablePurgeRepository) Publish(context.Context, ports.SnapshotPublication) error {
24 return nil25 return nil
25 }26 }
26-func (r *retryablePurgeRepository) WriteDeletionTombstone(_ context.Context, tombstone domain.DeletionTombstone) error {27+func (r *retryablePurgeRepository) WriteDeletionTombstone(ctx context.Context, tombstone domain.DeletionTombstone) error {
28+ _, hasDeadline := ctx.Deadline()
29+ r.deadlineScoped = append(r.deadlineScoped, hasDeadline)
27 if r.tombstoned == nil {30 if r.tombstoned == nil {
28 r.tombstoned = make(map[string]bool)31 r.tombstoned = make(map[string]bool)
29 }32 }
@@ -31,14 +34,18 @@ func (r *retryablePurgeRepository) WriteDeletionTombstone(_ context.Context, tom
31 r.tombstoned[tombstone.Name] = true34 r.tombstoned[tombstone.Name] = true
32 return nil35 return nil
33 }36 }
34-func (r *retryablePurgeRepository) QuarantineDeletionSources(_ context.Context, tombstone domain.DeletionTombstone, _ bool) error {37+func (r *retryablePurgeRepository) QuarantineDeletionSources(ctx context.Context, tombstone domain.DeletionTombstone, _ bool) error {
38+ _, hasDeadline := ctx.Deadline()
39+ r.deadlineScoped = append(r.deadlineScoped, hasDeadline)
35 r.calls = append(r.calls, "incremental", "legacy")40 r.calls = append(r.calls, "incremental", "legacy")
36 if r.incrementalErr != nil {41 if r.incrementalErr != nil {
37 return r.incrementalErr42 return r.incrementalErr
38 }43 }
39 return r.legacyErr44 return r.legacyErr
40 }45 }
41-func (r *retryablePurgeRepository) DeleteDeletionTombstone(_ context.Context, _ domain.IncarnationID) error {46+func (r *retryablePurgeRepository) DeleteDeletionTombstone(ctx context.Context, _ domain.IncarnationID) error {
47+ _, hasDeadline := ctx.Deadline()
48+ r.deadlineScoped = append(r.deadlineScoped, hasDeadline)
42 r.calls = append(r.calls, "clear tombstone")49 r.calls = append(r.calls, "clear tombstone")
43 for name := range r.tombstoned {50 for name := range r.tombstoned {
44 delete(r.tombstoned, name)51 delete(r.tombstoned, name)
@@ -65,8 +72,13 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) {
65 t.Run(tt.name, func(t *testing.T) {72 t.Run(tt.name, func(t *testing.T) {
66 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})73 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
67 repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy}74 repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy}
68- WithSnapshotRepository(repository, repository)(d)75+ WithSnapshotRepository(repository)(d)
76+ store, _ := newMockStore(t)
77+ WithStore(store)(d)
69 sess := newSnapshotTestSession(t, "work", false, "/work")78 sess := newSnapshotTestSession(t, "work", false, "/work")
79+ record := sess.persistRecordLocked(1)
80+ record.RecoveryState = domain.RecoveryFresh
81+ require.NoError(t, d.catalogue.Create(record))
70 pty, ok := sess.tabs[0].panes["pane-1"].pty.(*portsmocks.MockPTY)82 pty, ok := sess.tabs[0].panes["pane-1"].pty.(*portsmocks.MockPTY)
71 require.True(t, ok, "snapshot test pane must use MockPTY")83 require.True(t, ok, "snapshot test pane must use MockPTY")
72 pty.EXPECT().Close().Return(nil).Once()84 pty.EXPECT().Close().Return(nil).Once()
@@ -86,6 +98,10 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) {
86 require.NoError(t, d.retryStoppedPurge("work"))98 require.NoError(t, d.retryStoppedPurge("work"))
87 require.False(t, repository.tombstoned["work"])99 require.False(t, repository.tombstoned["work"])
88 require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls)100 require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls)
101+ require.NotEmpty(t, repository.deadlineScoped)
102+ for _, scoped := range repository.deadlineScoped {
103+ require.True(t, scoped, "every purge repository call must receive a deadline-scoped context")
104+ }
89 d.mu.Lock()105 d.mu.Lock()
90 _, ok = d.stopped["work"]106 _, ok = d.stopped["work"]
91 d.mu.Unlock()107 d.mu.Unlock()
M internal/usecase/daemon/session_rename_purge_test.go +26 -29
@@ -1,43 +1,40 @@
1 package daemon1 package daemon
2 2
3 import (3 import (
4+ "errors"
4 "testing"5 "testing"
5 6
6 "github.com/stretchr/testify/require"7 "github.com/stretchr/testify/require"
7 8
8 "github.com/bnema/vev/internal/domain"9 "github.com/bnema/vev/internal/domain"
10+ "github.com/bnema/vev/internal/persist"
9 portsmocks "github.com/bnema/vev/internal/ports/mocks"11 portsmocks "github.com/bnema/vev/internal/ports/mocks"
10 )12 )
11 13
12 func TestRenamePreservesIncarnationSnapshotSources(t *testing.T) {14 func TestRenamePreservesIncarnationSnapshotSources(t *testing.T) {
13- d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})15+ legacyDeleteErr := errors.New("legacy delete failed")
14- repository := &retryablePurgeRepository{}16+ for _, tt := range []struct {
15- WithSnapshotRepository(repository, repository)(d)17+ name string
16- store, state := newMockStore(t)18+ legacyErr error
17- WithStore(store)(d)19+ }{
18- sess := newSnapshotTestSession(t, "old", false, "/work")20+ {name: "snapshot sources available"},
19- d.sessions = map[domain.SessionID]*session{sess.id: sess}21+ {name: "unrelated legacy source delete would fail", legacyErr: legacyDeleteErr},
20- require.NoError(t, d.persist.Save(sess.persistRecordLocked(1)))22+ } {
23+ t.Run(tt.name, func(t *testing.T) {
24+ d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
25+ repository := &retryablePurgeRepository{legacyErr: tt.legacyErr}
26+ WithSnapshotRepository(repository)(d)
27+ store, state := newMockStore(t)
28+ WithStore(store)(d)
29+ sess := newSnapshotTestSession(t, "old", false, "/work")
30+ d.sessions = map[domain.SessionID]*session{sess.id: sess}
31+ require.NoError(t, d.catalogue.(*persist.Persister).Save(sess.persistRecordLocked(1)))
21 32
22- require.NoError(t, d.renameSession(sess, "new"))33+ require.NoError(t, d.renameSession(sess, "new"))
23- require.Empty(t, repository.calls, "rename must not touch incarnation-keyed snapshot storage")34+ require.Empty(t, repository.calls, "rename must not invoke source deletion, including a failing legacy source")
24- require.False(t, state.has("old"))35+ require.False(t, state.has("old"))
25- require.True(t, state.has("new"))36+ require.True(t, state.has("new"))
26- require.Equal(t, sess.incarnation, state.record(t, "new").IncarnationID)37+ require.Equal(t, sess.incarnation, state.record(t, "new").IncarnationID)
27-}38+ })
28-39+ }
29-func TestRenameIgnoresUnrelatedLegacySourceFailure(t *testing.T) {
30- d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
31- repository := &retryablePurgeRepository{}
32- WithSnapshotRepository(repository, repository)(d)
33- store, state := newMockStore(t)
34- WithStore(store)(d)
35- sess := newSnapshotTestSession(t, "old", false, "/work")
36- d.sessions = map[domain.SessionID]*session{sess.id: sess}
37- require.NoError(t, d.persist.Save(sess.persistRecordLocked(1)))
38-
39- require.NoError(t, d.renameSession(sess, "new"))
40- require.Empty(t, repository.calls)
41- require.False(t, state.has("old"))
42- require.True(t, state.has("new"))
43 }40 }
M internal/usecase/daemon/session_test.go +54 -28
@@ -806,11 +806,27 @@ func (s newStaticStore) Get(key []byte) ([]byte, bool) {
806 func (s newStaticStore) Set(_, _ []byte) error { return nil }806 func (s newStaticStore) Set(_, _ []byte) error { return nil }
807 func (s newStaticStore) Delete(_ []byte) error { return nil }807 func (s newStaticStore) Delete(_ []byte) error { return nil }
808 func (s newStaticStore) Batch(changes []ports.StoreChange) error {808 func (s newStaticStore) Batch(changes []ports.StoreChange) error {
809- for _, change := range changes {809+ if len(changes) == 0 {
810+ return errors.New("empty batch")
811+ }
812+ prepared := make([]ports.StoreChange, len(changes))
813+ seen := make(map[string]struct{}, len(changes))
814+ for i, change := range changes {
815+ key := string(change.Key)
816+ if _, exists := seen[key]; exists {
817+ return errors.New("duplicate batch key")
818+ }
819+ if change.Delete && len(change.Value) != 0 {
820+ return errors.New("delete batch change has value")
821+ }
822+ seen[key] = struct{}{}
823+ prepared[i] = ports.StoreChange{Key: append([]byte(nil), change.Key...), Value: append([]byte(nil), change.Value...), Delete: change.Delete}
824+ }
825+ for _, change := range prepared {
810 if change.Delete {826 if change.Delete {
811 delete(s, string(change.Key))827 delete(s, string(change.Key))
812 } else {828 } else {
813- s[string(change.Key)] = append([]byte(nil), change.Value...)829+ s[string(change.Key)] = change.Value
814 }830 }
815 }831 }
816 return nil832 return nil
@@ -841,19 +857,26 @@ func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) {
841 store, _ := newMockStore(t)857 store, _ := newMockStore(t)
842 seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))858 seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))
843 incarnation := domain.IncarnationID{1}859 incarnation := domain.IncarnationID{1}
844- require.NoError(t, seed.persist.Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh}))860+ require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh}))
845 861
846 d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))862 d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))
847 d.mu.Lock()863 d.mu.Lock()
848 stopped := d.stopped["work"]864 stopped := d.stopped["work"]
849 d.mu.Unlock()865 d.mu.Unlock()
850- require.Equal(t, stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, incarnation: incarnation, lastUsedSeq: 9}, stopped)866+ require.Equal(t, "work", stopped.name)
867+ require.Equal(t, "/tmp/work", stopped.cwd)
868+ require.Equal(t, int64(7), stopped.createdAt)
869+ require.Equal(t, incarnation, stopped.incarnation)
870+ require.Equal(t, uint64(9), stopped.lastUsedSeq)
871+ require.Equal(t, domain.RecoveryFresh, stopped.record.RecoveryState)
872+ require.Equal(t, runtimeFresh, stopped.state)
873+ require.NotNil(t, stopped.restoreDone)
851 require.Equal(t, uint64(9), d.mruSeq.Load())874 require.Equal(t, uint64(9), d.mruSeq.Load())
852 }875 }
853 876
854 // TestDaemonNewWithoutPersistenceLogsNoLoadWarning covers the default,877 // TestDaemonNewWithoutPersistenceLogsNoLoadWarning covers the default,
855 // persistence-free construction path (no WithStore/WithCatalogue option):878 // persistence-free construction path (no WithStore/WithCatalogue option):
856-// d.persist is the persist.New(nil) sentinel, so LoadAll now returns879+// d.catalogue.(*persist.Persister) is the persist.New(nil) sentinel, so LoadAll now returns
857 // errPersistenceUnavailable instead of an empty slice. That must not surface880 // errPersistenceUnavailable instead of an empty slice. That must not surface
858 // as a "loading persisted sessions failed" warning, since nothing failed —881 // as a "loading persisted sessions failed" warning, since nothing failed —
859 // persistence simply was never configured.882 // persistence simply was never configured.
@@ -866,20 +889,6 @@ func TestDaemonNewWithoutPersistenceLogsNoLoadWarning(t *testing.T) {
866 require.NotContains(t, logBuf.String(), "loading persisted sessions failed")889 require.NotContains(t, logBuf.String(), "loading persisted sessions failed")
867 }890 }
868 891
869-// TestDaemonNewWithConfiguredStoreStillLogsLoadFailure is the inverse of the
870-// above: when persistence IS configured (a real store via WithStore) and the
871-// load genuinely fails (an undecodable catalogue record here), the warning
872-// must still fire.
873-func TestDaemonNewWithConfiguredStoreStillLogsLoadFailure(t *testing.T) {
874- store, state := newMockStore(t)
875- state.data["bad"] = []byte{0, 1} // not a valid encoded catalogue record
876- var logBuf strings.Builder
877- d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(&logBuf, nil)), WithStore(store))
878- require.NotNil(t, d)
879- require.True(t, d.persistEnabled)
880- require.Contains(t, logBuf.String(), "loading persisted sessions failed")
881-}
882-
883 func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) {892 func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) {
884 d := &Daemon{}893 d := &Daemon{}
885 sess := &session{}894 sess := &session{}
@@ -930,7 +939,7 @@ func TestTouchMRUPersistsNamedButNotEphemeral(t *testing.T) {
930 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})939 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
931 WithStore(store)(d)940 WithStore(store)(d)
932 named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1, incarnation: domain.IncarnationID{1}}941 named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1, incarnation: domain.IncarnationID{1}}
933- require.NoError(t, d.persist.Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh}))942+ require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh}))
934 943
935 d.touchMRU(named)944 d.touchMRU(named)
936 require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq)945 require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq)
@@ -1003,7 +1012,7 @@ func TestRenameTabPersistsForNamedSession(t *testing.T) {
1003 require.NoError(t, err)1012 require.NoError(t, err)
1004 require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell"))1013 require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell"))
1005 1014
1006- records, err := d.persist.LoadAll()1015+ records, err := d.catalogue.(*persist.Persister).LoadAll()
1007 require.NoError(t, err)1016 require.NoError(t, err)
1008 require.Len(t, records, 1)1017 require.Len(t, records, 1)
1009 require.Equal(t, "work", records[0].Name)1018 require.Equal(t, "work", records[0].Name)
@@ -1013,6 +1022,23 @@ func TestRenameTabPersistsForNamedSession(t *testing.T) {
1013 require.Equal(t, []string{"shell"}, records[0].TabNames)1022 require.Equal(t, []string{"shell"}, records[0].TabNames)
1014 }1023 }
1015 1024
1025+func TestRenameTabDoesNotHoldDaemonOrSessionMutexDuringCatalogueIO(t *testing.T) {
1026+ d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
1027+ sess := newSnapshotTestSession(t, "work", false, "/work")
1028+ d.sessions[sess.id] = sess
1029+ catalogue := portsmocks.NewMockCatalogue(t)
1030+ WithCatalogue(catalogue, nil)(d)
1031+ catalogue.EXPECT().UpdateMetadata(mock.Anything).RunAndReturn(func(domain.CatalogueMetadataUpdate) error {
1032+ require.True(t, d.mu.TryLock(), "daemon mutex held during catalogue I/O")
1033+ d.mu.Unlock()
1034+ require.True(t, sess.mu.TryLock(), "session mutex held during catalogue I/O")
1035+ sess.mu.Unlock()
1036+ return nil
1037+ }).Once()
1038+
1039+ require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell"))
1040+}
1041+
1016 func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) {1042 func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) {
1017 tests := []struct {1043 tests := []struct {
1018 name string1044 name string
@@ -1045,7 +1071,7 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) {
1045 }1071 }
1046 1072
1047 require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false))1073 require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false))
1048- records, err := d.persist.LoadAll()1074+ records, err := d.catalogue.(*persist.Persister).LoadAll()
1049 require.NoError(t, err)1075 require.NoError(t, err)
1050 require.Len(t, records, 1)1076 require.Len(t, records, 1)
1051 require.Equal(t, tt.want, records[0].TabNames)1077 require.Equal(t, tt.want, records[0].TabNames)
@@ -1090,7 +1116,7 @@ func TestRenameTabDoesNotPersistForEphemeralSession(t *testing.T) {
1090 require.Equal(t, "shell", sess.tabs[0].name)1116 require.Equal(t, "shell", sess.tabs[0].name)
1091 1117
1092 require.False(t, state.has("0"))1118 require.False(t, state.has("0"))
1093- records, err := d.persist.LoadAll()1119+ records, err := d.catalogue.(*persist.Persister).LoadAll()
1094 require.NoError(t, err)1120 require.NoError(t, err)
1095 require.Empty(t, records)1121 require.Empty(t, records)
1096 }1122 }
@@ -1104,7 +1130,7 @@ func TestAttachRestoresPersistedTabNames(t *testing.T) {
1104 store, _ := newMockStore(t)1130 store, _ := newMockStore(t)
1105 d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{})1131 d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{})
1106 WithStore(store)(d)1132 WithStore(store)(d)
1107- require.NoError(t, d.persist.Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh}))1133+ require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh}))
1108 d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}}1134 d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}}
1109 tr := portsmocks.NewMockTransport(t)1135 tr := portsmocks.NewMockTransport(t)
1110 tr.EXPECT().Send(mock.Anything).Return(nil).Maybe()1136 tr.EXPECT().Send(mock.Anything).Return(nil).Maybe()
@@ -1213,8 +1239,8 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T)
1213 defer release()1239 defer release()
1214 store := portsmocks.NewMockStore(t)1240 store := portsmocks.NewMockStore(t)
1215 var attempted map[string][]byte1241 var attempted map[string][]byte
1216- store.EXPECT().Range(mock.Anything).Run(func(func([]byte, []byte) bool) {}).Twice()1242+ store.EXPECT().Range(mock.Anything).Run(func(func([]byte, []byte) bool) {}).Times(3)
1217- store.EXPECT().Get([]byte("named")).Return(nil, false).Twice()1243+ store.EXPECT().Get([]byte("named")).Return(nil, false).Times(4)
1218 store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error {1244 store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error {
1219 attempted = make(map[string][]byte, len(changes))1245 attempted = make(map[string][]byte, len(changes))
1220 for _, change := range changes {1246 for _, change := range changes {
@@ -1318,7 +1344,7 @@ func TestPickerStoppedTargetKillPurges(t *testing.T) {
1318 store, state := newMockStore(t)1344 store, state := newMockStore(t)
1319 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})1345 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
1320 WithStore(store)(d)1346 WithStore(store)(d)
1321- require.NoError(t, d.persist.Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh}))1347+ require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh}))
1322 d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1}1348 d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1}
1323 require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true}))1349 require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true}))
1324 require.False(t, state.has("old"))1350 require.False(t, state.has("old"))
@@ -1830,7 +1856,7 @@ func TestCatalogueRecordsConstructExpectedSessionRegistry(t *testing.T) {
1830 func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) {1856 func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) {
1831 store, _ := newMockStore(t)1857 store, _ := newMockStore(t)
1832 seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))1858 seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store))
1833- require.NoError(t, seed.persist.Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh}))1859+ require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh}))
1834 1860
1835 p, release := newBlockingPTY(t)1861 p, release := newBlockingPTY(t)
1836 defer release()1862 defer release()
M internal/usecase/daemon/snapshot.go +6 -0
@@ -47,6 +47,9 @@ type snapshotCapture struct {
47 publicationContext context.Context47 publicationContext context.Context
48 sealedRefs map[*vt.HistoryChunk]snapcodec.ObjectRef // set by the single encoder worker48 sealedRefs map[*vt.HistoryChunk]snapcodec.ObjectRef // set by the single encoder worker
49 coordinatorDiscarded bool // guarded by session.snapshotMu49 coordinatorDiscarded bool // guarded by session.snapshotMu
50+ // normalWorkerAdmitted is set before snapshotJobs admission and immutable
51+ // until completion; final-queue captures leave it false.
52+ normalWorkerAdmitted bool
50 finishOnce sync.Once53 finishOnce sync.Once
51 }54 }
52 55
@@ -98,3 +101,6 @@ const (
98 snapshotInterval = 2 * time.Minute101 snapshotInterval = 2 * time.Minute
99 snapshotFinalFlushTimeout = time.Second102 snapshotFinalFlushTimeout = time.Second
100 )103 )
104+
105+// SnapshotShutdownTimeout reports the shared final-checkpoint shutdown budget.
106+func SnapshotShutdownTimeout() time.Duration { return snapshotFinalFlushTimeout }
M internal/usecase/daemon/snapshot_coordinator.go +13 -0
@@ -108,6 +108,14 @@ func (d *Daemon) waitForSnapshotGenerationWithDeadline(sess *session, generation
108 return d.waitForSnapshotGeneration(sess, generation)108 return d.waitForSnapshotGeneration(sess, generation)
109 }109 }
110 for {110 for {
111+ // Once the shared deadline is observable, timeout wins even if publication
112+ // completion becomes ready concurrently. This preserves the timed-out
113+ // session identity and avoids nondeterministic success after the budget.
114+ select {
115+ case <-deadline.Done():
116+ return false
117+ default:
118+ }
111 sess.snapshotMu.Lock()119 sess.snapshotMu.Lock()
112 published := sess.snapshotPublishedMutationRevision >= generation120 published := sess.snapshotPublishedMutationRevision >= generation
113 changed := sess.snapshotChangeLocked()121 changed := sess.snapshotChangeLocked()
@@ -207,6 +215,11 @@ func (d *Daemon) finishSnapshotCapture(capture *snapshotCapture, succeeded bool)
207 capture.session.signalSnapshotChangedLocked()215 capture.session.signalSnapshotChangedLocked()
208 wake := capture.session.snapshotWake216 wake := capture.session.snapshotWake
209 capture.session.snapshotMu.Unlock()217 capture.session.snapshotMu.Unlock()
218+ if capture.normalWorkerAdmitted {
219+ d.snapshotWorkerMu.Lock()
220+ delete(d.snapshotAdmitted, capture)
221+ d.snapshotWorkerMu.Unlock()
222+ }
210 if wake != nil {223 if wake != nil {
211 select {224 select {
212 case wake <- struct{}{}:225 case wake <- struct{}{}:
M internal/usecase/daemon/snapshot_generation_test.go +2 -2
@@ -17,7 +17,7 @@ import (
17 func TestSnapshotMultipleMutationsPublishNextRepositoryGeneration(t *testing.T) {17 func TestSnapshotMultipleMutationsPublishNextRepositoryGeneration(t *testing.T) {
18 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})18 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
19 repository := portsmocks.NewMockSnapshotRepository(t)19 repository := portsmocks.NewMockSnapshotRepository(t)
20- WithSnapshotRepository(repository, nil)(d)20+ WithSnapshotRepository(repository)(d)
21 startSnapshotEncodeWorker(t, d)21 startSnapshotEncodeWorker(t, d)
22 22
23 publications := make(chan ports.SnapshotPublication, 1)23 publications := make(chan ports.SnapshotPublication, 1)
@@ -44,7 +44,7 @@ func TestSnapshotMultipleMutationsPublishNextRepositoryGeneration(t *testing.T)
44 func TestSnapshotRetryKeepsRepositoryGeneration(t *testing.T) {44 func TestSnapshotRetryKeepsRepositoryGeneration(t *testing.T) {
45 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})45 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
46 repository := portsmocks.NewMockSnapshotRepository(t)46 repository := portsmocks.NewMockSnapshotRepository(t)
47- WithSnapshotRepository(repository, nil)(d)47+ WithSnapshotRepository(repository)(d)
48 startSnapshotEncodeWorker(t, d)48 startSnapshotEncodeWorker(t, d)
49 49
50 publications := make(chan ports.SnapshotPublication, 2)50 publications := make(chan ports.SnapshotPublication, 2)
D internal/usecase/daemon/snapshot_import_test.go +0 -163
@@ -1,163 +0,0 @@
1-package daemon
2-
3-import (
4- "context"
5- "encoding/base64"
6- "errors"
7- "os"
8- "path/filepath"
9- "testing"
10-
11- "github.com/stretchr/testify/require"
12-
13- "github.com/bnema/vev/internal/ports"
14- snapcodec "github.com/bnema/vev/internal/usecase/snapshot"
15-)
16-
17-type snapshotAcceptanceLegacySource struct {
18- blobs []ports.LegacySnapshot
19- loadCalls int
20- deleteCalls []string
21- deleteErr error
22-}
23-
24-func (s *snapshotAcceptanceLegacySource) LoadLegacy(ctx context.Context) ([]ports.LegacySnapshot, error) {
25- if err := ctx.Err(); err != nil {
26- return nil, err
27- }
28- s.loadCalls++
29- out := make([]ports.LegacySnapshot, len(s.blobs))
30- for i, blob := range s.blobs {
31- out[i] = ports.LegacySnapshot{Name: blob.Name, Data: append([]byte(nil), blob.Data...)}
32- }
33- return out, nil
34-}
35-
36-func (s *snapshotAcceptanceLegacySource) DeleteVerifiedLegacy(ctx context.Context, blob ports.LegacySnapshot) error {
37- return s.DeleteLegacy(ctx, blob.Name)
38-}
39-
40-func (s *snapshotAcceptanceLegacySource) DeleteLegacy(ctx context.Context, name string) error {
41- if err := ctx.Err(); err != nil {
42- return err
43- }
44- s.deleteCalls = append(s.deleteCalls, name)
45- if s.deleteErr != nil {
46- return s.deleteErr
47- }
48- for i, blob := range s.blobs {
49- if blob.Name == name {
50- s.blobs = append(s.blobs[:i], s.blobs[i+1:]...)
51- break
52- }
53- }
54- return nil
55-}
56-
57-func legacyAcceptanceSnapshot(t *testing.T) snapcodec.Session {
58- t.Helper()
59- data, err := os.ReadFile(filepath.Join("testdata", "legacy-v3-empty.b64"))
60- require.NoError(t, err)
61- encoded, err := base64.StdEncoding.DecodeString(string(data))
62- require.NoError(t, err)
63- snapshot, err := snapcodec.Unmarshal(encoded)
64- require.NoError(t, err)
65- return snapshot
66-}
67-
68-func legacyAcceptanceBlob(t *testing.T, snapshot snapcodec.Session) ports.LegacySnapshot {
69- t.Helper()
70- fixture := legacyAcceptanceSnapshot(t)
71- require.Equal(t, fixture, snapshot, "legacy fixture must be the imported session")
72- data, err := os.ReadFile(filepath.Join("testdata", "legacy-v3-empty.b64"))
73- require.NoError(t, err)
74- encoded, err := base64.StdEncoding.DecodeString(string(data))
75- require.NoError(t, err)
76- return ports.LegacySnapshot{Name: snapshot.Name, Data: encoded}
77-}
78-
79-func TestLegacyPublicationMarshalObjectErrorsIdentifyPaneAndStage(t *testing.T) {
80- for _, tt := range []struct {
81- name string
82- mutate func(*snapcodec.Pane)
83- stage string
84- }{
85- {name: "sealed chunk", mutate: func(pane *snapcodec.Pane) { pane.SealedChunks = [][]byte{nil} }, stage: "sealed chunk 0"},
86- {name: "tail", mutate: func(pane *snapcodec.Pane) { pane.Tail = nil }, stage: "tail"},
87- {name: "visible", mutate: func(pane *snapcodec.Pane) { pane.Visible = nil }, stage: "visible"},
88- } {
89- t.Run(tt.name, func(t *testing.T) {
90- pane := snapcodec.Pane{ID: "pane-1", StableID: "pane-stable", Tail: []byte("tail"), Visible: []byte("visible")}
91- tt.mutate(&pane)
92- snapshot := snapcodec.Session{Name: "legacy", Tabs: []snapcodec.Tab{{StableID: "tab-1", Panes: []snapcodec.Pane{pane}}}}
93-
94- _, err := legacyPublication(snapshot)
95- require.Error(t, err)
96- require.ErrorContains(t, err, `tab "tab-1"`)
97- require.ErrorContains(t, err, `pane "pane-1"`)
98- require.ErrorContains(t, err, tt.stage)
99- require.ErrorContains(t, err, "marshal object")
100- })
101- }
102-}
103-
104-func TestImportLegacySnapshotsIsRetiredRuntimeNoop(t *testing.T) {
105- snapshot := legacyAcceptanceSnapshot(t)
106- repository := &snapshotAcceptanceRepository{generations: make(map[string]ports.SnapshotGeneration)}
107- legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}}
108- d := newTestDaemon(t, nil, stubClock{})
109- WithSnapshotRepository(repository, legacy)(d)
110-
111- d.importLegacySnapshots(context.Background())
112-
113- // Migration is adapter-owned. The daemon must never name-scan, publish, or
114- // delete a legacy source during normal recovery.
115- require.Empty(t, repository.publishes)
116- require.Empty(t, legacy.deleteCalls)
117- require.Len(t, legacy.blobs, 1)
118-}
119-
120-func TestImportLegacyExistingIncrementalSkipsLegacy(t *testing.T) {
121- snapshot := legacyAcceptanceSnapshot(t)
122- repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: make(map[string]ports.SnapshotGeneration)}
123- legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}}
124- d := newTestDaemon(t, nil, stubClock{})
125- WithSnapshotRepository(repository, legacy)(d)
126-
127- d.importLegacySnapshots(context.Background())
128-
129- require.Empty(t, repository.publishes)
130- require.Empty(t, legacy.deleteCalls)
131- require.Len(t, legacy.blobs, 1)
132-}
133-
134-func TestImportLegacyRetainsSourceUntilExactReloadVerification(t *testing.T) {
135- snapshot := legacyAcceptanceSnapshot(t)
136- for _, test := range []struct {
137- name string
138- publish error
139- load error
140- mutate func(ports.SnapshotGeneration, ports.SnapshotPublication) ports.SnapshotGeneration
141- }{
142- {"publish failure", errors.New("publish failed"), nil, nil},
143- {"reload failure", nil, errors.New("reload failed"), nil},
144- {"manifest metadata mismatch", nil, nil, func(g ports.SnapshotGeneration, _ ports.SnapshotPublication) ports.SnapshotGeneration {
145- g.Generation++
146- return g
147- }},
148- {"manifest bytes mismatch", nil, nil, func(g ports.SnapshotGeneration, _ ports.SnapshotPublication) ports.SnapshotGeneration {
149- g.Manifest[len(g.Manifest)-1] ^= 1
150- return g
151- }},
152- } {
153- t.Run(test.name, func(t *testing.T) {
154- repository := &snapshotAcceptanceRepository{generations: make(map[string]ports.SnapshotGeneration), publishErr: test.publish, loadErr: test.load, loadMutate: test.mutate}
155- legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}}
156- d := newTestDaemon(t, nil, stubClock{})
157- WithSnapshotRepository(repository, legacy)(d)
158- d.importLegacySnapshots(context.Background())
159- require.Empty(t, legacy.deleteCalls)
160- require.Len(t, legacy.blobs, 1)
161- })
162- }
163-}
M internal/usecase/daemon/snapshot_queue.go +6 -0
@@ -107,10 +107,16 @@ func (d *Daemon) enqueueSnapshotCapture(capture *snapshotCapture) bool {
107 if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil {107 if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil {
108 return false108 return false
109 }109 }
110+ capture.normalWorkerAdmitted = true
110 select {111 select {
111 case d.snapshotJobs <- capture:112 case d.snapshotJobs <- capture:
113+ if d.snapshotAdmitted == nil {
114+ d.snapshotAdmitted = make(map[*snapshotCapture]struct{})
115+ }
116+ d.snapshotAdmitted[capture] = struct{}{}
112 return true117 return true
113 default:118 default:
119+ capture.normalWorkerAdmitted = false
114 return false120 return false
115 }121 }
116 }122 }
M internal/usecase/daemon/snapshot_restore.go +23 -6
@@ -3,6 +3,7 @@ package daemon
3 import (3 import (
4 "context"4 "context"
5 "crypto/rand"5 "crypto/rand"
6+ "errors"
6 "fmt"7 "fmt"
7 "math"8 "math"
8 9
@@ -13,6 +14,20 @@ import (
13 )14 )
14 15
15 func (d *Daemon) closeRestoreDone() {16 func (d *Daemon) closeRestoreDone() {
17+ d.mu.Lock()
18+ for name, entry := range d.stopped {
19+ if entry.state != runtimeRestoring {
20+ continue
21+ }
22+ entry.state = runtimeDegraded
23+ entry.record.RecoveryState = domain.RecoveryDegraded
24+ if entry.record.DegradedReason == "" {
25+ entry.record.DegradedReason = "restore unavailable"
26+ }
27+ d.stopped[name] = entry
28+ closeRuntimeRestoreDoneLocked(entry.restoreDone)
29+ }
30+ d.mu.Unlock()
16 d.restoreOnce.Do(func() { close(d.restoreDone) })31 d.restoreOnce.Do(func() { close(d.restoreDone) })
17 }32 }
18 33
@@ -293,13 +308,15 @@ func (d *Daemon) persistAndRegisterRestoredSession(ctx context.Context, sess *se
293 }308 }
294 }309 }
295 }310 }
296- // An existing catalogue checkpoint already authorizes this runtime. Only a311+ // Restoration is catalogue-authorized. Missing or unreadable authority is a
297- // legacy snapshot without a catalogue record creates fresh metadata here.312+ // hard failure; migration creates catalogue records before daemon startup.
298 if d.persistEnabled {313 if d.persistEnabled {
299- if _, ok := d.catalogueRecord(sess.name); !ok {314+ _, ok, err := d.catalogueRecord(sess.name)
300- if err := d.createCatalogueRecord(sess.persistRecordLocked(sess.createdAt)); err != nil {315+ if err != nil {
301- return false, err316+ return false, err
302- }317+ }
318+ if !ok {
319+ return false, errors.New("snapshot: restored session is absent from catalogue")
303 }320 }
304 }321 }
305 d.mu.Lock()322 d.mu.Lock()
R internal/usecase/daemon/snapshot_legacy_import.go → internal/usecase/daemon/snapshot_restore_fixture_test.go +2 -3 renamed, 91% similar
@@ -8,9 +8,8 @@ import (
8 snapcodec "github.com/bnema/vev/internal/usecase/snapshot"8 snapcodec "github.com/bnema/vev/internal/usecase/snapshot"
9 )9 )
10 10
11-func legacyPublication(snapshot snapcodec.Session) (ports.SnapshotPublication, error) {11+func acceptancePublication(snapshot snapcodec.Session) (ports.SnapshotPublication, error) {
12- // Legacy payloads have no durable identity; this test-only conversion uses12+ // acceptancePublication converts a restore fixture to repository form.
13- // the fixture's stable synthetic identity.
14 incarnation := domain.IncarnationID{1}13 incarnation := domain.IncarnationID{1}
15 manifest := snapcodec.Manifest{Generation: 1, IncarnationID: incarnation, Name: snapshot.Name, CreatedAt: snapshot.CreatedAt, Active: snapshot.Active, Tabs: make([]snapcodec.ManifestTab, 0, len(snapshot.Tabs))}14 manifest := snapcodec.Manifest{Generation: 1, IncarnationID: incarnation, Name: snapshot.Name, CreatedAt: snapshot.CreatedAt, Active: snapshot.Active, Tabs: make([]snapcodec.ManifestTab, 0, len(snapshot.Tabs))}
16 objects := make([]ports.SnapshotObject, 0)15 objects := make([]ports.SnapshotObject, 0)
M internal/usecase/daemon/snapshot_restore_incremental.go +53 -10
@@ -43,7 +43,12 @@ func (d *Daemon) restoreIncrementalSnapshots(ctx context.Context) {
43 if d.catalogue == nil {43 if d.catalogue == nil {
44 return44 return
45 }45 }
46- d.restoreCatalogue(ctx, d.catalogue.Records())46+ records, err := d.catalogue.Records()
47+ if err != nil {
48+ d.log.Error("loading catalogue for snapshot restoration failed", "err", err)
49+ return
50+ }
51+ d.restoreCatalogue(ctx, records)
47 }52 }
48 53
49 // restoreCatalogue gives every expected record an independent bounded job.54 // restoreCatalogue gives every expected record an independent bounded job.
@@ -95,7 +100,9 @@ func (d *Daemon) recordRestoreDone(name string) chan struct{} {
95 return d.stopped[name].restoreDone100 return d.stopped[name].restoreDone
96 }101 }
97 102
98-func closeRuntimeRestoreDone(done chan struct{}) {103+// closeRuntimeRestoreDoneLocked closes a per-record restoration barrier.
104+// Caller must hold d.mu so state transition and waiter release are atomic.
105+func closeRuntimeRestoreDoneLocked(done chan struct{}) {
99 if done == nil {106 if done == nil {
100 return107 return
101 }108 }
@@ -125,8 +132,13 @@ func (d *Daemon) setStoppedRecovery(record domain.CatalogueRecord, state runtime
125 132
126 func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) {133 func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) {
127 if restoreErr != nil {134 if restoreErr != nil {
135+ catalogueReadable := true
128 if d.catalogue != nil {136 if d.catalogue != nil {
129- if current, ok := d.catalogue.Record(record.Name); ok && current.IncarnationID == record.IncarnationID {137+ current, ok, readErr := d.catalogue.Record(record.Name)
138+ if readErr != nil {
139+ catalogueReadable = false
140+ d.log.Error("reading catalogue before degradation failed", "session", record.Name, "err", readErr)
141+ } else if ok && current.IncarnationID == record.IncarnationID {
130 record = current142 record = current
131 }143 }
132 }144 }
@@ -135,7 +147,7 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e
135 if errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded) {147 if errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded) {
136 record.DegradedReason = "restore interrupted"148 record.DegradedReason = "restore interrupted"
137 }149 }
138- if d.catalogue != nil {150+ if d.catalogue != nil && catalogueReadable {
139 if err := d.catalogue.Replace(record.Name, record); err != nil {151 if err := d.catalogue.Replace(record.Name, record); err != nil {
140 d.log.Error("marking session degraded failed", "session", record.Name, "err", err)152 d.log.Error("marking session degraded failed", "session", record.Name, "err", err)
141 }153 }
@@ -155,7 +167,7 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e
155 entry.tabNames = append([]string(nil), record.TabNames...)167 entry.tabNames = append([]string(nil), record.TabNames...)
156 d.stopped[record.Name] = entry168 d.stopped[record.Name] = entry
157 }169 }
158- closeRuntimeRestoreDone(done)170+ closeRuntimeRestoreDoneLocked(done)
159 }171 }
160 d.mu.Unlock()172 d.mu.Unlock()
161 if restoreErr != nil {173 if restoreErr != nil {
@@ -197,26 +209,42 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor
197 var selectedGeneration ports.SnapshotGeneration209 var selectedGeneration ports.SnapshotGeneration
198 var selectedSnapshot snapcodec.Session210 var selectedSnapshot snapcodec.Session
199 selectedIndex := -1211 selectedIndex := -1
212+ selectedFallback := false
200 for i, candidate := range candidates {213 for i, candidate := range candidates {
214+ fallback := record.Committed == nil || i > 0
201 generation, err := d.snapshotRepository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, candidate)215 generation, err := d.snapshotRepository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, candidate)
202 if err != nil {216 if err != nil {
203 if ctxErr := ctx.Err(); ctxErr != nil {217 if ctxErr := ctx.Err(); ctxErr != nil {
204 return ctxErr218 return ctxErr
205 }219 }
220+ d.logRejectedRestoreCandidate(record, candidate, fallback, "load-failed", err)
206 continue221 continue
207 }222 }
208- snapshot, err := sessionFromGeneration(generation)223+ snapshot, decodeErr := sessionFromGeneration(generation)
209- if err != nil || generation.IncarnationID != record.IncarnationID || generation.Name != record.Name || generation.Generation != candidate.Generation {224+ if decodeErr != nil {
225+ d.logRejectedRestoreCandidate(record, candidate, fallback, "decode-failed", decodeErr)
210 continue226 continue
211 }227 }
212- selected, selectedGeneration, selectedSnapshot, selectedIndex = candidate, generation, snapshot, i228+ if generation.IncarnationID != record.IncarnationID {
229+ d.logRejectedRestoreCandidate(record, candidate, fallback, "incarnation-mismatch", nil)
230+ continue
231+ }
232+ if generation.Name != record.Name {
233+ d.logRejectedRestoreCandidate(record, candidate, fallback, "name-mismatch", nil)
234+ continue
235+ }
236+ if generation.Generation != candidate.Generation {
237+ d.logRejectedRestoreCandidate(record, candidate, fallback, "generation-mismatch", nil)
238+ continue
239+ }
240+ selected, selectedGeneration, selectedSnapshot, selectedIndex, selectedFallback = candidate, generation, snapshot, i, fallback
213 break241 break
214 }242 }
215 if selectedIndex < 0 {243 if selectedIndex < 0 {
216 return errors.New("snapshot: no catalogue checkpoint validated")244 return errors.New("snapshot: no catalogue checkpoint validated")
217 }245 }
218 246
219- if selectedIndex > 0 {247+ if selectedFallback {
220 if d.checkpointRecovery == nil {248 if d.checkpointRecovery == nil {
221 return errors.New("snapshot: checkpoint coordinator unavailable for fallback promotion")249 return errors.New("snapshot: checkpoint coordinator unavailable for fallback promotion")
222 }250 }
@@ -244,10 +272,25 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor
244 return err272 return err
245 }273 }
246 d.setStoppedRecovery(record, runtimeHealthy)274 d.setStoppedRecovery(record, runtimeHealthy)
247- d.logSessionRestoreComplete(record, selected.Generation, selectedIndex > 0)275+ d.logSessionRestoreComplete(record, selected.Generation, selectedFallback)
248 return nil276 return nil
249 }277 }
250 278
279+func (d *Daemon) logRejectedRestoreCandidate(record domain.CatalogueRecord, candidate domain.CheckpointRef, fallback bool, reason string, err error) {
280+ kind := "committed"
281+ if fallback {
282+ kind = "fallback"
283+ }
284+ d.log.Warn("snapshot_restore_candidate_rejected",
285+ "session", record.Name,
286+ "incarnation", record.IncarnationID.String(),
287+ "generation", candidate.Generation,
288+ "candidate", kind,
289+ "reason_code", reason,
290+ "err", err,
291+ )
292+}
293+
251 func checkpointCandidates(record domain.CatalogueRecord) []domain.CheckpointRef {294 func checkpointCandidates(record domain.CatalogueRecord) []domain.CheckpointRef {
252 candidates := make([]domain.CheckpointRef, 0, 3)295 candidates := make([]domain.CheckpointRef, 0, 3)
253 if record.Committed != nil {296 if record.Committed != nil {
M internal/usecase/daemon/snapshot_restore_test.go +6 -7
@@ -175,7 +175,7 @@ func restoreAcceptanceSession(t *testing.T, name string) snapcodec.Session {
175 175
176 func acceptanceGeneration(t *testing.T, snapshot snapcodec.Session, generation uint64) ports.SnapshotGeneration {176 func acceptanceGeneration(t *testing.T, snapshot snapcodec.Session, generation uint64) ports.SnapshotGeneration {
177 t.Helper()177 t.Helper()
178- publication, err := legacyPublication(snapshot)178+ publication, err := acceptancePublication(snapshot)
179 require.NoError(t, err)179 require.NoError(t, err)
180 manifest, err := snapcodec.UnmarshalManifest(publication.Manifest)180 manifest, err := snapcodec.UnmarshalManifest(publication.Manifest)
181 require.NoError(t, err)181 require.NoError(t, err)
@@ -218,13 +218,12 @@ func TestRestoreIncrementalGenerationAcceptance(t *testing.T) {
218 repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: map[string]ports.SnapshotGeneration{snapshot.Name: generation}}218 repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: map[string]ports.SnapshotGeneration{snapshot.Name: generation}}
219 pty, release := newBlockingPTY(t)219 pty, release := newBlockingPTY(t)
220 d := newTestDaemon(t, newFactory(t, pty), stubClock{})220 d := newTestDaemon(t, newFactory(t, pty), stubClock{})
221- store, _ := newMockStore(t)
222- WithStore(store)(d)
223 checkpoint := domain.CheckpointRef{Generation: generation.Generation, ManifestDigest: snapcodec.ManifestDigest(generation.Manifest)}221 checkpoint := domain.CheckpointRef{Generation: generation.Generation, ManifestDigest: snapcodec.ManifestDigest(generation.Manifest)}
224 record := domain.CatalogueRecord{Name: snapshot.Name, IncarnationID: generation.IncarnationID, Cwd: "/snapshot/cwd", CreatedAt: int64(snapshot.CreatedAt), RecoveryState: domain.RecoveryHealthy, Committed: &checkpoint}222 record := domain.CatalogueRecord{Name: snapshot.Name, IncarnationID: generation.IncarnationID, Cwd: "/snapshot/cwd", CreatedAt: int64(snapshot.CreatedAt), RecoveryState: domain.RecoveryHealthy, Committed: &checkpoint}
225 catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record})223 catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record})
226 WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d)224 WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d)
227- WithSnapshotRepository(repository, nil)(d)225+ WithSnapshotRepository(repository)(d)
226+ WithRecoveryCoordinator(recoveryusecase.NewCoordinator(catalogue, repository, nil, nil))(d)
228 t.Cleanup(func() { release(); d.sessWg.Wait() })227 t.Cleanup(func() { release(); d.sessWg.Wait() })
229 228
230 d.restoreIncrementalSnapshots(context.Background())229 d.restoreIncrementalSnapshots(context.Background())
@@ -277,7 +276,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) {
277 d := newTestDaemon(t, newFactory(t, pty), stubClock{})276 d := newTestDaemon(t, newFactory(t, pty), stubClock{})
278 coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)277 coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
279 WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d)278 WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d)
280- WithSnapshotRepository(repository, nil)(d)279+ WithSnapshotRepository(repository)(d)
281 WithCheckpointCoordinator(coordinator)(d)280 WithCheckpointCoordinator(coordinator)(d)
282 t.Cleanup(func() { release(); d.sessWg.Wait() })281 t.Cleanup(func() { release(); d.sessWg.Wait() })
283 282
@@ -291,7 +290,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) {
291 updates := catalogue.MetadataUpdates()290 updates := catalogue.MetadataUpdates()
292 require.Len(t, updates, 1, "the authoritative Catalogue port must receive restored-session metadata updates")291 require.Len(t, updates, 1, "the authoritative Catalogue port must receive restored-session metadata updates")
293 require.Equal(t, record.IncarnationID, updates[0].IncarnationID)292 require.Equal(t, record.IncarnationID, updates[0].IncarnationID)
294- updated, ok := catalogue.Record(record.Name)293+ updated, ok, _ := catalogue.Record(record.Name)
295 require.True(t, ok)294 require.True(t, ok)
296 require.Equal(t, record.IncarnationID, updated.IncarnationID)295 require.Equal(t, record.IncarnationID, updated.IncarnationID)
297 require.Equal(t, record.RecoveryState, updated.RecoveryState)296 require.Equal(t, record.RecoveryState, updated.RecoveryState)
@@ -310,7 +309,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) {
310 require.Empty(t, failure)309 require.Empty(t, failure)
311 require.Len(t, repository.publishes, 1)310 require.Len(t, repository.publishes, 1)
312 require.Equal(t, record.Committed, repository.publishes[0].ParentCheckpoint)311 require.Equal(t, record.Committed, repository.publishes[0].ParentCheckpoint)
313- published, ok := catalogue.Record(record.Name)312+ published, ok, _ := catalogue.Record(record.Name)
314 require.True(t, ok)313 require.True(t, ok)
315 require.Equal(t, uint64(10), published.Committed.Generation)314 require.Equal(t, uint64(10), published.Committed.Generation)
316 require.Equal(t, committed, *published.Fallbacks[0])315 require.Equal(t, committed, *published.Fallbacks[0])
M internal/usecase/daemon/snapshot_scheduler_test.go +27 -21
@@ -19,7 +19,7 @@ import (
19 func TestSnapshotCoordinatorQuarantineCancelsPublicationBeforeDelete(t *testing.T) {19 func TestSnapshotCoordinatorQuarantineCancelsPublicationBeforeDelete(t *testing.T) {
20 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})20 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
21 repository := portsmocks.NewMockSnapshotRepository(t)21 repository := portsmocks.NewMockSnapshotRepository(t)
22- WithSnapshotRepository(repository, nil)(d)22+ WithSnapshotRepository(repository)(d)
23 startSnapshotEncodeWorker(t, d)23 startSnapshotEncodeWorker(t, d)
24 24
25 started := make(chan struct{})25 started := make(chan struct{})
@@ -48,7 +48,7 @@ func TestSnapshotCoordinatorQuarantineCancelsPublicationBeforeDelete(t *testing.
48 func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) {48 func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) {
49 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})49 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
50 repository := portsmocks.NewMockSnapshotRepository(t)50 repository := portsmocks.NewMockSnapshotRepository(t)
51- WithSnapshotRepository(repository, nil)(d)51+ WithSnapshotRepository(repository)(d)
52 startSnapshotEncodeWorker(t, d)52 startSnapshotEncodeWorker(t, d)
53 53
54 started := make(chan struct{})54 started := make(chan struct{})
@@ -84,7 +84,7 @@ func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) {
84 func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) {84 func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) {
85 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})85 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
86 repository := portsmocks.NewMockSnapshotRepository(t)86 repository := portsmocks.NewMockSnapshotRepository(t)
87- WithSnapshotRepository(repository, nil)(d)87+ WithSnapshotRepository(repository)(d)
88 startSnapshotEncodeWorker(t, d)88 startSnapshotEncodeWorker(t, d)
89 89
90 started := make(chan struct{})90 started := make(chan struct{})
@@ -123,7 +123,7 @@ func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) {
123 func TestForcedSnapshotSchedulesSuccessorForRoutineCaptureQueuedBehindWorker(t *testing.T) {123 func TestForcedSnapshotSchedulesSuccessorForRoutineCaptureQueuedBehindWorker(t *testing.T) {
124 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})124 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
125 repository := portsmocks.NewMockSnapshotRepository(t)125 repository := portsmocks.NewMockSnapshotRepository(t)
126- WithSnapshotRepository(repository, nil)(d)126+ WithSnapshotRepository(repository)(d)
127 startSnapshotEncodeWorker(t, d)127 startSnapshotEncodeWorker(t, d)
128 128
129 blockerStarted := make(chan struct{})129 blockerStarted := make(chan struct{})
@@ -164,7 +164,7 @@ func TestForcedSnapshotSchedulesSuccessorForRoutineCaptureQueuedBehindWorker(t *
164 func TestMultipleForcedSnapshotsCoalesceToOneSuccessor(t *testing.T) {164 func TestMultipleForcedSnapshotsCoalesceToOneSuccessor(t *testing.T) {
165 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})165 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
166 repository := portsmocks.NewMockSnapshotRepository(t)166 repository := portsmocks.NewMockSnapshotRepository(t)
167- WithSnapshotRepository(repository, nil)(d)167+ WithSnapshotRepository(repository)(d)
168 startSnapshotEncodeWorker(t, d)168 startSnapshotEncodeWorker(t, d)
169 169
170 started := make(chan struct{})170 started := make(chan struct{})
@@ -207,7 +207,7 @@ func TestForcedSnapshotShutdownTimeoutRetainsRetryableStateAndNotice(t *testing.
207 clock := newSnapshotDeadlineClock()207 clock := newSnapshotDeadlineClock()
208 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)208 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)
209 repository := portsmocks.NewMockSnapshotRepository(t)209 repository := portsmocks.NewMockSnapshotRepository(t)
210- WithSnapshotRepository(repository, nil)(d)210+ WithSnapshotRepository(repository)(d)
211 notices := portsmocks.NewMockNoticeStore(t)211 notices := portsmocks.NewMockNoticeStore(t)
212 WithNoticeStore(notices)(d)212 WithNoticeStore(notices)(d)
213 startSnapshotEncodeWorker(t, d)213 startSnapshotEncodeWorker(t, d)
@@ -290,7 +290,7 @@ func TestServeShutdownDeadlineStillJoinsPaneExitWriter(t *testing.T) {
290 clock := newServeShutdownClock()290 clock := newServeShutdownClock()
291 d := newTestDaemon(t, newFactory(t, pty), clock)291 d := newTestDaemon(t, newFactory(t, pty), clock)
292 repository := portsmocks.NewMockSnapshotRepository(t)292 repository := portsmocks.NewMockSnapshotRepository(t)
293- WithSnapshotRepository(repository, nil)(d)293+ WithSnapshotRepository(repository)(d)
294 294
295 publicationStarted := make(chan struct{})295 publicationStarted := make(chan struct{})
296 releasePublication := make(chan struct{})296 releasePublication := make(chan struct{})
@@ -305,7 +305,7 @@ func TestServeShutdownDeadlineStillJoinsPaneExitWriter(t *testing.T) {
305 305
306 tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24}))306 tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24}))
307 listener := serveSnapshotListener(t, tr)307 listener := serveSnapshotListener(t, tr)
308- ctx, cancel := context.WithCancel(context.Background())308+ ctx, cancel := context.WithCancel(t.Context())
309 defer cancel()309 defer cancel()
310 served := make(chan error, 1)310 served := make(chan error, 1)
311 go func() { served <- d.Serve(ctx, listener) }()311 go func() { served <- d.Serve(ctx, listener) }()
@@ -382,7 +382,7 @@ func awaitTeardownWaiters(t *testing.T, sess *session, want uint) {
382 func TestConcurrentSessionTeardownPublishesOneFinalSnapshot(t *testing.T) {382 func TestConcurrentSessionTeardownPublishesOneFinalSnapshot(t *testing.T) {
383 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})383 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
384 repository := portsmocks.NewMockSnapshotRepository(t)384 repository := portsmocks.NewMockSnapshotRepository(t)
385- WithSnapshotRepository(repository, nil)(d)385+ WithSnapshotRepository(repository)(d)
386 startSnapshotEncodeWorker(t, d)386 startSnapshotEncodeWorker(t, d)
387 387
388 publicationStarted := make(chan struct{})388 publicationStarted := make(chan struct{})
@@ -456,7 +456,7 @@ func TestServeShutdownCheckpointsBeforeStoppingSnapshotWorker(t *testing.T) {
456 defer releasePTY()456 defer releasePTY()
457 d := newTestDaemon(t, newFactory(t, p), stubClock{})457 d := newTestDaemon(t, newFactory(t, p), stubClock{})
458 repository := portsmocks.NewMockSnapshotRepository(t)458 repository := portsmocks.NewMockSnapshotRepository(t)
459- WithSnapshotRepository(repository, nil)(d)459+ WithSnapshotRepository(repository)(d)
460 460
461 published := make(chan ports.SnapshotPublication, 2)461 published := make(chan ports.SnapshotPublication, 2)
462 allowTerminalPublication := make(chan struct{})462 allowTerminalPublication := make(chan struct{})
@@ -470,7 +470,7 @@ func TestServeShutdownCheckpointsBeforeStoppingSnapshotWorker(t *testing.T) {
470 470
471 tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24}))471 tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24}))
472 l := serveSnapshotListener(t, tr)472 l := serveSnapshotListener(t, tr)
473- ctx, cancel := context.WithCancel(context.Background())473+ ctx, cancel := context.WithCancel(t.Context())
474 defer cancel()474 defer cancel()
475 served := make(chan error, 1)475 served := make(chan error, 1)
476 go func() { served <- d.Serve(ctx, l) }()476 go func() { served <- d.Serve(ctx, l) }()
@@ -500,14 +500,15 @@ func TestServeShutdownDeadlineStillJoinsUncooperativeSnapshotRepository(t *testi
500 clock := newServeShutdownClock()500 clock := newServeShutdownClock()
501 d := newTestDaemon(t, newFactory(t, p), clock)501 d := newTestDaemon(t, newFactory(t, p), clock)
502 repository := portsmocks.NewMockSnapshotRepository(t)502 repository := portsmocks.NewMockSnapshotRepository(t)
503- WithSnapshotRepository(repository, nil)(d)503+ WithSnapshotRepository(repository)(d)
504 noticeStore := portsmocks.NewMockNoticeStore(t)504 noticeStore := portsmocks.NewMockNoticeStore(t)
505 WithNoticeStore(noticeStore)(d)505 WithNoticeStore(noticeStore)(d)
506 noticeStore.EXPECT().Claim().Return(nil, nil).Maybe()506 noticeStore.EXPECT().Claim().Return(nil, nil).Maybe()
507 noticeStore.EXPECT().Ack().Return(nil).Maybe()507 noticeStore.EXPECT().Ack().Return(nil).Maybe()
508+ noticePersisted := make(chan struct{})
508 noticeStore.EXPECT().Append(mock.MatchedBy(func(n domain.Notification) bool {509 noticeStore.EXPECT().Append(mock.MatchedBy(func(n domain.Notification) bool {
509 return n.Code == domain.NoticeSnapshotWrite && n.Severity == domain.NoticeError510 return n.Code == domain.NoticeSnapshotWrite && n.Severity == domain.NoticeError
510- })).Return(nil).Once()511+ })).Run(func(domain.Notification) { close(noticePersisted) }).Return(nil).Once()
511 512
512 firstPublicationCompleted := make(chan struct{})513 firstPublicationCompleted := make(chan struct{})
513 terminalPublicationStarted := make(chan struct{})514 terminalPublicationStarted := make(chan struct{})
@@ -526,7 +527,7 @@ func TestServeShutdownDeadlineStillJoinsUncooperativeSnapshotRepository(t *testi
526 527
527 tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24}))528 tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24}))
528 l := serveSnapshotListener(t, tr)529 l := serveSnapshotListener(t, tr)
529- ctx, cancel := context.WithCancel(context.Background())530+ ctx, cancel := context.WithCancel(t.Context())
530 defer cancel()531 defer cancel()
531 served := make(chan error, 1)532 served := make(chan error, 1)
532 go func() { served <- d.Serve(ctx, l) }()533 go func() { served <- d.Serve(ctx, l) }()
@@ -551,6 +552,11 @@ func TestServeShutdownDeadlineStillJoinsUncooperativeSnapshotRepository(t *testi
551 d.snapshotWorkerMu.Unlock()552 d.snapshotWorkerMu.Unlock()
552 require.NotNil(t, workerDone)553 require.NotNil(t, workerDone)
553 clock.nextFinalTimer(t).fire()554 clock.nextFinalTimer(t).fire()
555+ select {
556+ case <-noticePersisted:
557+ case <-time.After(testWaitTimeout):
558+ t.Fatal("shutdown timeout notice was not persisted before the writer join")
559+ }
554 560
555 select {561 select {
556 case err := <-served:562 case err := <-served:
@@ -632,7 +638,7 @@ func TestRoutineSnapshotEligibilityStartsAtCompletionAndForcedDoesNotMoveIt(t *t
632 clock := &snapshotSchedulerClock{now: time.Unix(100, 0)}638 clock := &snapshotSchedulerClock{now: time.Unix(100, 0)}
633 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)639 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)
634 repository := portsmocks.NewMockSnapshotRepository(t)640 repository := portsmocks.NewMockSnapshotRepository(t)
635- WithSnapshotRepository(repository, nil)(d)641+ WithSnapshotRepository(repository)(d)
636 startSnapshotEncodeWorker(t, d)642 startSnapshotEncodeWorker(t, d)
637 643
638 var publishes atomic.Int32644 var publishes atomic.Int32
@@ -780,7 +786,7 @@ func TestSnapshotCompletionSchedulesRoutineRetriesFromCompletion(t *testing.T) {
780 clock := newDeterministicSnapshotClock(now)786 clock := newDeterministicSnapshotClock(now)
781 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)787 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)
782 repository := portsmocks.NewMockSnapshotRepository(t)788 repository := portsmocks.NewMockSnapshotRepository(t)
783- WithSnapshotRepository(repository, nil)(d)789+ WithSnapshotRepository(repository)(d)
784 790
785 for _, tt := range []struct {791 for _, tt := range []struct {
786 name string792 name string
@@ -813,12 +819,12 @@ func TestSnapshotSchedulerImmediateEligibilityAndStaleCapturesRemainDirty(t *tes
813 now := time.Unix(100, 0)819 now := time.Unix(100, 0)
814 clock := newDeterministicSnapshotClock(now)820 clock := newDeterministicSnapshotClock(now)
815 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)821 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)
816- WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t), nil)(d)822+ WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t))(d)
817 823
818 t.Run("no prior attempt is immediately eligible", func(t *testing.T) {824 t.Run("no prior attempt is immediately eligible", func(t *testing.T) {
819 published := make(chan ports.SnapshotPublication, 1)825 published := make(chan ports.SnapshotPublication, 1)
820 repository := portsmocks.NewMockSnapshotRepository(t)826 repository := portsmocks.NewMockSnapshotRepository(t)
821- WithSnapshotRepository(repository, nil)(d)827+ WithSnapshotRepository(repository)(d)
822 startSnapshotEncodeWorker(t, d)828 startSnapshotEncodeWorker(t, d)
823 repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error {829 repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error {
824 published <- publication830 published <- publication
@@ -873,7 +879,7 @@ func TestSnapshotWorkerQueueIsBoundedAndForcedCapturesCoalesce(t *testing.T) {
873 clock := newDeterministicSnapshotClock(now)879 clock := newDeterministicSnapshotClock(now)
874 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)880 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)
875 repository := portsmocks.NewMockSnapshotRepository(t)881 repository := portsmocks.NewMockSnapshotRepository(t)
876- WithSnapshotRepository(repository, nil)(d)882+ WithSnapshotRepository(repository)(d)
877 startSnapshotEncodeWorker(t, d)883 startSnapshotEncodeWorker(t, d)
878 884
879 started := make(chan struct{})885 started := make(chan struct{})
@@ -931,7 +937,7 @@ func TestSnapshotRepositorySaverUsesEarliestDeadlineAndRecomputes(t *testing.T)
931 clock := newDeterministicSnapshotClock(now)937 clock := newDeterministicSnapshotClock(now)
932 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)938 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock)
933 repository := portsmocks.NewMockSnapshotRepository(t)939 repository := portsmocks.NewMockSnapshotRepository(t)
934- WithSnapshotRepository(repository, nil)(d)940+ WithSnapshotRepository(repository)(d)
935 startSnapshotEncodeWorker(t, d)941 startSnapshotEncodeWorker(t, d)
936 942
937 published := make(chan ports.SnapshotPublication, 1)943 published := make(chan ports.SnapshotPublication, 1)
@@ -946,7 +952,7 @@ func TestSnapshotRepositorySaverUsesEarliestDeadlineAndRecomputes(t *testing.T)
946 later.snapshotNextEligibleAt = now.Add(10 * time.Minute)952 later.snapshotNextEligibleAt = now.Add(10 * time.Minute)
947 d.sessions[later.id] = later953 d.sessions[later.id] = later
948 954
949- ctx, cancel := context.WithCancel(context.Background())955+ ctx, cancel := context.WithCancel(t.Context())
950 done := make(chan struct{})956 done := make(chan struct{})
951 go func() {957 go func() {
952 d.snapshotRepositorySaver(ctx)958 d.snapshotRepositorySaver(ctx)
M internal/usecase/daemon/snapshot_test.go +39 -1
@@ -17,10 +17,45 @@ import (
17 17
18 // TestSnapshotWorkerPublishesContentAddressedCapture verifies that all new18 // TestSnapshotWorkerPublishesContentAddressedCapture verifies that all new
19 // checkpoints use the repository publication contract.19 // checkpoints use the repository publication contract.
20+func TestWithSnapshotRepositoryRejectsTypedNil(t *testing.T) {
21+ var repository *snapshotAcceptanceRepository
22+ d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
23+
24+ WithSnapshotRepository(repository)(d)
25+
26+ require.Nil(t, d.snapshotRepository)
27+ require.False(t, d.snapsEnabled)
28+}
29+
30+func TestDurableWriterFailureNamesIncludesBufferedCapture(t *testing.T) {
31+ d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
32+ capture := &snapshotCapture{name: "work", session: newSnapshotTestSession(t, "work", false, "/work")}
33+ d.snapshotWorkerMu.Lock()
34+ d.snapshotAdmitted[capture] = struct{}{}
35+ d.snapshotJobs <- capture
36+ d.snapshotWorkerMu.Unlock()
37+
38+ require.Equal(t, []string{"work"}, d.durableWriterFailureNames())
39+}
40+
41+func TestCheckpointPublicationWithoutCommittedRefFailsCapture(t *testing.T) {
42+ d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
43+ WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t))(d)
44+ coordinator := portsmocks.NewMockCheckpointCoordinator(t)
45+ WithCheckpointCoordinator(coordinator)(d)
46+ startSnapshotEncodeWorker(t, d)
47+ coordinator.EXPECT().PublishCheckpoint(mock.Anything, "work", mock.Anything).Return(domain.CatalogueRecord{Name: "work"}, nil).Once()
48+
49+ sess := newSnapshotTestSession(t, "work", false, "/work")
50+ require.True(t, d.captureSession(sess))
51+ awaitSnapshotIdle(t, sess)
52+ require.True(t, sess.snapDirty.Load(), "nil committed publication must remain retryable")
53+}
54+
20 func TestSnapshotWorkerPublishesContentAddressedCapture(t *testing.T) {55 func TestSnapshotWorkerPublishesContentAddressedCapture(t *testing.T) {
21 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})56 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
22 repository := portsmocks.NewMockSnapshotRepository(t)57 repository := portsmocks.NewMockSnapshotRepository(t)
23- WithSnapshotRepository(repository, nil)(d)58+ WithSnapshotRepository(repository)(d)
24 startSnapshotEncodeWorker(t, d)59 startSnapshotEncodeWorker(t, d)
25 60
26 repository.EXPECT().Publish(mock.Anything, mock.MatchedBy(func(p ports.SnapshotPublication) bool {61 repository.EXPECT().Publish(mock.Anything, mock.MatchedBy(func(p ports.SnapshotPublication) bool {
@@ -57,6 +92,9 @@ func newSnapshotTestSession(t *testing.T, name string, ephemeral bool, cwd strin
57 // focused test sinks that only need to observe publication or deletion calls.92 // focused test sinks that only need to observe publication or deletion calls.
58 type noOpSnapshotRepository struct{}93 type noOpSnapshotRepository struct{}
59 94
95+var _ ports.SnapshotRepository = noOpSnapshotRepository{}
96+
97+func (noOpSnapshotRepository) Publish(context.Context, ports.SnapshotPublication) error { return nil }
60 func (noOpSnapshotRepository) LoadCheckpoint(context.Context, domain.IncarnationID, string, ports.CheckpointRef) (ports.SnapshotGeneration, error) {98 func (noOpSnapshotRepository) LoadCheckpoint(context.Context, domain.IncarnationID, string, ports.CheckpointRef) (ports.SnapshotGeneration, error) {
61 return ports.SnapshotGeneration{}, errors.New("unused")99 return ports.SnapshotGeneration{}, errors.New("unused")
62 }100 }
M internal/usecase/daemon/snapshot_worker.go +18 -3
@@ -2,6 +2,7 @@ package daemon
2 2
3 import (3 import (
4 "context"4 "context"
5+ "errors"
5 "sort"6 "sort"
6 7
7 "github.com/bnema/vev/internal/domain"8 "github.com/bnema/vev/internal/domain"
@@ -134,8 +135,12 @@ func (d *Daemon) publishSnapshotCapture(workerCtx context.Context, workerID uint
134 if d.checkpointRecovery != nil {135 if d.checkpointRecovery != nil {
135 var record domain.CatalogueRecord136 var record domain.CatalogueRecord
136 record, err = d.checkpointRecovery.PublishCheckpoint(publicationContext, capture.name, publication)137 record, err = d.checkpointRecovery.PublishCheckpoint(publicationContext, capture.name, publication)
137- if err == nil && record.Committed != nil {138+ if err == nil {
138- capture.checkpoint = *record.Committed139+ if record.Committed == nil {
140+ err = errors.New("snapshot: checkpoint publication returned no committed checkpoint")
141+ } else {
142+ capture.checkpoint = *record.Committed
143+ }
139 }144 }
140 } else {145 } else {
141 err = d.snapshotRepository.Publish(publicationContext, publication)146 err = d.snapshotRepository.Publish(publicationContext, publication)
@@ -217,8 +222,12 @@ func (d *Daemon) StopDurableWriters(ctx context.Context) []string {
217 case <-done:222 case <-done:
218 return nil223 return nil
219 case <-ctx.Done():224 case <-ctx.Done():
225+ // Snapshot identity before cancellation can let a cooperative worker clear
226+ // its in-flight capture. The caller persists timeout notices before the
227+ // unconditional ownership join.
228+ names := d.durableWriterFailureNames()
220 cancel()229 cancel()
221- return d.durableWriterFailureNames()230+ return names
222 }231 }
223 }232 }
224 233
@@ -267,6 +276,9 @@ func (d *Daemon) requestDurableWriterStop() (context.CancelFunc, <-chan struct{}
267 return cancel, d.snapshotWorkerDone276 return cancel, d.snapshotWorkerDone
268 }277 }
269 278
279+// durableWriterFailureNames includes admitted buffered captures as well as the
280+// active and final queues. snapshotAdmitted tracks normal captures from queue
281+// admission through completion, so worker dequeue cannot make one disappear.
270 func (d *Daemon) durableWriterFailureNames() []string {282 func (d *Daemon) durableWriterFailureNames() []string {
271 d.snapshotWorkerMu.Lock()283 d.snapshotWorkerMu.Lock()
272 defer d.snapshotWorkerMu.Unlock()284 defer d.snapshotWorkerMu.Unlock()
@@ -280,6 +292,9 @@ func (d *Daemon) durableWriterFailureNames() []string {
280 for _, capture := range d.snapshotFinalJobs {292 for _, capture := range d.snapshotFinalJobs {
281 add(capture)293 add(capture)
282 }294 }
295+ for capture := range d.snapshotAdmitted {
296+ add(capture)
297+ }
283 names := make([]string, 0, len(seen))298 names := make([]string, 0, len(seen))
284 for name := range seen {299 for name := range seen {
285 names = append(names, name)300 names = append(names, name)
M internal/usecase/recovery/checkpoint.go +12 -15
@@ -37,7 +37,10 @@ func (c *Coordinator) PublishCheckpoint(ctx context.Context, name string, public
37 unlock := c.locks.Lock([]string{name})37 unlock := c.locks.Lock([]string{name})
38 defer unlock()38 defer unlock()
39 39
40- record, ok := c.catalogue.Record(name)40+ record, ok, err := c.catalogue.Record(name)
41+ if err != nil {
42+ return domain.CatalogueRecord{}, err
43+ }
41 if !ok {44 if !ok {
42 return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound45 return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound
43 }46 }
@@ -74,7 +77,10 @@ func (c *Coordinator) PromoteFallback(ctx context.Context, name string, ref doma
74 }77 }
75 78
76 func (c *Coordinator) promoteFallbackLocked(ctx context.Context, name string, ref domain.CheckpointRef) (ports.FallbackPromotionOutcome, error) {79 func (c *Coordinator) promoteFallbackLocked(ctx context.Context, name string, ref domain.CheckpointRef) (ports.FallbackPromotionOutcome, error) {
77- record, ok := c.catalogue.Record(name)80+ record, ok, err := c.catalogue.Record(name)
81+ if err != nil {
82+ return ports.FallbackPromotionOutcome{}, err
83+ }
78 if !ok {84 if !ok {
79 return ports.FallbackPromotionOutcome{}, ErrCheckpointRecordNotFound85 return ports.FallbackPromotionOutcome{}, ErrCheckpointRecordNotFound
80 }86 }
@@ -129,24 +135,15 @@ func (c *Coordinator) validateCheckpoint(ctx context.Context, record domain.Cata
129 if err != nil {135 if err != nil {
130 return err136 return err
131 }137 }
132- if generation.IncarnationID != record.IncarnationID || generation.Name != record.Name || generation.Generation != ref.Generation ||138+ return validateLoadedGeneration(record, ref, generation)
133- sha256.Sum256(generation.Manifest) != ref.ManifestDigest {
134- return ErrCheckpointConflict
135- }
136- return nil
137 }139 }
138 140
139-func equalCheckpointRefs(a, b *domain.CheckpointRef) bool {141+func equalCheckpointRefs(a, b *domain.CheckpointRef) bool { return a.Equal(b) }
140- if a == nil || b == nil {
141- return a == nil && b == nil
142- }
143- return *a == *b
144-}
145 142
146 func copyCheckpointRef(ref *domain.CheckpointRef) *domain.CheckpointRef {143 func copyCheckpointRef(ref *domain.CheckpointRef) *domain.CheckpointRef {
147 if ref == nil {144 if ref == nil {
148 return nil145 return nil
149 }146 }
150- copy := *ref147+ cloned := *ref
151- return &copy148+ return &cloned
152 }149 }
M internal/usecase/recovery/checkpoint_test.go +28 -26
@@ -20,13 +20,15 @@ type checkpointCatalogue struct {
20 events *[]string20 events *[]string
21 }21 }
22 22
23-func (c *checkpointCatalogue) Records() []domain.CatalogueRecord {23+func (c *checkpointCatalogue) Records() ([]domain.CatalogueRecord, error) {
24- return []domain.CatalogueRecord{c.record}24+ c.mu.Lock()
25+ defer c.mu.Unlock()
26+ return []domain.CatalogueRecord{c.record}, nil
25 }27 }
26-func (c *checkpointCatalogue) Record(name string) (domain.CatalogueRecord, bool) {28+func (c *checkpointCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) {
27 c.mu.Lock()29 c.mu.Lock()
28 defer c.mu.Unlock()30 defer c.mu.Unlock()
29- return c.record, c.record.Name == name31+ return c.record, c.record.Name == name, nil
30 }32 }
31 func (c *checkpointCatalogue) Create(domain.CatalogueRecord) error { return nil }33 func (c *checkpointCatalogue) Create(domain.CatalogueRecord) error { return nil }
32 func (c *checkpointCatalogue) UpdateMetadata(domain.CatalogueMetadataUpdate) error {34 func (c *checkpointCatalogue) UpdateMetadata(domain.CatalogueMetadataUpdate) error {
@@ -127,33 +129,33 @@ func runCheckpointCommit(t *testing.T, count int) (domain.CatalogueRecord, *chec
127 catalogue.record, err = coordinator.PublishCheckpoint(context.Background(), "work", checkpointPublication(catalogue.record, generation))129 catalogue.record, err = coordinator.PublishCheckpoint(context.Background(), "work", checkpointPublication(catalogue.record, generation))
128 require.NoError(t, err)130 require.NoError(t, err)
129 }131 }
130- for i := 0; i < count; i++ {132+ for i := range count {
131 require.Equal(t, []string{"repository", "catalogue"}, events[i*2:i*2+2])133 require.Equal(t, []string{"repository", "catalogue"}, events[i*2:i*2+2])
132 }134 }
133 return catalogue.record, catalogue, repository135 return catalogue.record, catalogue, repository
134 }136 }
135 137
136-func TestCheckpointCommitFirst(t *testing.T) {138+func TestCheckpointCommits(t *testing.T) {
137- record, catalogue, _ := runCheckpointCommit(t, 1)139+ for _, tc := range []struct {
138- require.Equal(t, uint64(1), record.Committed.Generation)140+ name string
139- require.Empty(t, populatedGenerations(record.Fallbacks))141+ count int
140- require.Equal(t, 1, catalogue.replaces)142+ wantCommitted uint64
141-}143+ wantFallbacks []uint64
142-144+ }{
143-func TestCheckpointCommitSecond(t *testing.T) {145+ {name: "first", count: 1, wantCommitted: 1},
144- record, catalogue, _ := runCheckpointCommit(t, 2)146+ {name: "second", count: 2, wantCommitted: 2, wantFallbacks: []uint64{1}},
145- require.Equal(t, uint64(2), record.Committed.Generation)147+ {name: "third", count: 3, wantCommitted: 3, wantFallbacks: []uint64{2, 1}},
146- require.Equal(t, []uint64{1}, populatedGenerations(record.Fallbacks))148+ {name: "fourth", count: 4, wantCommitted: 4, wantFallbacks: []uint64{3, 2}},
147- require.Equal(t, 2, catalogue.replaces)149+ } {
148-}150+ t.Run(tc.name, func(t *testing.T) {
149-151+ record, catalogue, _ := runCheckpointCommit(t, tc.count)
150-func TestCheckpointCommitThird(t *testing.T) {152+ require.Equal(t, tc.wantCommitted, record.Committed.Generation)
151- record, catalogue, _ := runCheckpointCommit(t, 4)153+ require.Equal(t, tc.wantFallbacks, populatedGenerations(record.Fallbacks))
152- require.Equal(t, uint64(4), record.Committed.Generation)154+ require.Equal(t, domain.RecoveryHealthy, record.RecoveryState)
153- require.Equal(t, []uint64{3, 2}, populatedGenerations(record.Fallbacks))155+ require.Empty(t, record.DegradedReason)
154- require.Equal(t, domain.RecoveryHealthy, record.RecoveryState)156+ require.Equal(t, tc.count, catalogue.replaces)
155- require.Empty(t, record.DegradedReason)157+ })
156- require.Equal(t, 4, catalogue.replaces)158+ }
157 }159 }
158 160
159 func TestCheckpointCommitThirdPromotesOnlyValidatedDirectFallbacks(t *testing.T) {161 func TestCheckpointCommitThirdPromotesOnlyValidatedDirectFallbacks(t *testing.T) {
M internal/usecase/recovery/coordinator.go +32 -8
@@ -148,7 +148,9 @@ func (c *Coordinator) Create(ctx context.Context, record domain.CatalogueRecord)
148 }148 }
149 unlock := c.locks.Lock([]string{record.Name})149 unlock := c.locks.Lock([]string{record.Name})
150 defer unlock()150 defer unlock()
151- if _, exists := c.catalogue.Record(record.Name); exists {151+ if _, exists, err := c.catalogue.Record(record.Name); err != nil {
152+ return domain.CatalogueRecord{}, err
153+ } else if exists {
152 return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q already exists", record.Name)154 return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q already exists", record.Name)
153 }155 }
154 id, err := domain.NewIncarnationID(c.random)156 id, err := domain.NewIncarnationID(c.random)
@@ -180,11 +182,16 @@ func (c *Coordinator) Rename(ctx context.Context, oldName, newName string) (doma
180 }182 }
181 unlock := c.locks.Lock([]string{oldName, newName})183 unlock := c.locks.Lock([]string{oldName, newName})
182 defer unlock()184 defer unlock()
183- record, ok := c.catalogue.Record(oldName)185+ record, ok, err := c.catalogue.Record(oldName)
186+ if err != nil {
187+ return domain.CatalogueRecord{}, err
188+ }
184 if !ok {189 if !ok {
185 return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q not found", oldName)190 return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q not found", oldName)
186 }191 }
187- if existing, exists := c.catalogue.Record(newName); exists && (newName != oldName || existing.IncarnationID != record.IncarnationID) {192+ if existing, exists, err := c.catalogue.Record(newName); err != nil {
193+ return domain.CatalogueRecord{}, err
194+ } else if exists && (newName != oldName || existing.IncarnationID != record.IncarnationID) {
188 return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q already exists", newName)195 return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q already exists", newName)
189 }196 }
190 record.Name = newName197 record.Name = newName
@@ -207,7 +214,10 @@ func (c *Coordinator) Delete(ctx context.Context, name string) error {
207 }214 }
208 unlock := c.locks.Lock([]string{name})215 unlock := c.locks.Lock([]string{name})
209 defer unlock()216 defer unlock()
210- record, ok := c.catalogue.Record(name)217+ record, ok, err := c.catalogue.Record(name)
218+ if err != nil {
219+ return err
220+ }
211 if !ok {221 if !ok {
212 return nil222 return nil
213 }223 }
@@ -267,13 +277,21 @@ func (c *Coordinator) Recover(ctx context.Context) error {
267 }277 }
268 }278 }
269 }279 }
270- for _, candidate := range c.catalogue.Records() {280+ records, err := c.catalogue.Records()
281+ if err != nil {
282+ return err
283+ }
284+ for _, candidate := range records {
271 if candidate.RecoveryState != domain.RecoveryDeleting {285 if candidate.RecoveryState != domain.RecoveryDeleting {
272 continue286 continue
273 }287 }
274 unlock := c.locks.Lock([]string{candidate.Name})288 unlock := c.locks.Lock([]string{candidate.Name})
275 var deleteErr error289 var deleteErr error
276- current, ok := c.catalogue.Record(candidate.Name)290+ current, ok, readErr := c.catalogue.Record(candidate.Name)
291+ if readErr != nil {
292+ unlock()
293+ return readErr
294+ }
277 if ok && current.IncarnationID == candidate.IncarnationID && current.RecoveryState == domain.RecoveryDeleting {295 if ok && current.IncarnationID == candidate.IncarnationID && current.RecoveryState == domain.RecoveryDeleting {
278 deleteErr = c.deleteLocked(ctx, current)296 deleteErr = c.deleteLocked(ctx, current)
279 }297 }
@@ -323,7 +341,10 @@ func (c *Coordinator) fenceSession(name, kind, reason string, cause error) error
323 341
324 unlock := c.locks.Lock([]string{name})342 unlock := c.locks.Lock([]string{name})
325 defer unlock()343 defer unlock()
326- record, ok := c.catalogue.Record(name)344+ record, ok, err := c.catalogue.Record(name)
345+ if err != nil {
346+ return err
347+ }
327 if !ok || record.RecoveryState != domain.RecoveryHealthy {348 if !ok || record.RecoveryState != domain.RecoveryHealthy {
328 // Fresh records cannot represent degraded state (it requires a committed349 // Fresh records cannot represent degraded state (it requires a committed
329 // checkpoint), and deleting or already degraded records must keep the350 // checkpoint), and deleting or already degraded records must keep the
@@ -354,7 +375,10 @@ func (c *Coordinator) recoverDeletionTombstone(ctx context.Context, tombstone do
354 }375 }
355 unlock := c.locks.Lock([]string{tombstone.Name})376 unlock := c.locks.Lock([]string{tombstone.Name})
356 defer unlock()377 defer unlock()
357- record, exists := c.catalogue.Record(tombstone.Name)378+ record, exists, err := c.catalogue.Record(tombstone.Name)
379+ if err != nil {
380+ return err
381+ }
358 includeLegacyName := true382 includeLegacyName := true
359 if exists {383 if exists {
360 switch {384 switch {
M internal/usecase/recovery/discard.go +25 -15
@@ -39,7 +39,10 @@ func (c *Coordinator) Retry(ctx context.Context, name string) error {
39 }39 }
40 unlock := c.locks.Lock([]string{name})40 unlock := c.locks.Lock([]string{name})
41 defer unlock()41 defer unlock()
42- record, ok := c.catalogue.Record(name)42+ record, ok, err := c.catalogue.Record(name)
43+ if err != nil {
44+ return err
45+ }
43 if !ok {46 if !ok {
44 return ErrRecoveryRecordNotFound47 return ErrRecoveryRecordNotFound
45 }48 }
@@ -66,7 +69,10 @@ func (c *Coordinator) RestoreFallback(ctx context.Context, name string, ref doma
66 }69 }
67 unlock := c.locks.Lock([]string{name})70 unlock := c.locks.Lock([]string{name})
68 defer unlock()71 defer unlock()
69- record, ok := c.catalogue.Record(name)72+ record, ok, err := c.catalogue.Record(name)
73+ if err != nil {
74+ return err
75+ }
70 if !ok {76 if !ok {
71 return ErrRecoveryRecordNotFound77 return ErrRecoveryRecordNotFound
72 }78 }
@@ -88,7 +94,10 @@ func (c *Coordinator) Export(ctx context.Context, name string, w io.Writer) erro
88 }94 }
89 unlock := c.locks.Lock([]string{name})95 unlock := c.locks.Lock([]string{name})
90 defer unlock()96 defer unlock()
91- record, ok := c.catalogue.Record(name)97+ record, ok, err := c.catalogue.Record(name)
98+ if err != nil {
99+ return err
100+ }
92 if !ok {101 if !ok {
93 return ErrRecoveryRecordNotFound102 return ErrRecoveryRecordNotFound
94 }103 }
@@ -167,7 +176,10 @@ func (c *Coordinator) Discard(ctx context.Context, name, reason string) (domain.
167 }176 }
168 unlock := c.locks.Lock([]string{name})177 unlock := c.locks.Lock([]string{name})
169 defer unlock()178 defer unlock()
170- record, ok := c.catalogue.Record(name)179+ record, ok, err := c.catalogue.Record(name)
180+ if err != nil {
181+ return domain.CatalogueRecord{}, err
182+ }
171 if !ok {183 if !ok {
172 return domain.CatalogueRecord{}, ErrRecoveryRecordNotFound184 return domain.CatalogueRecord{}, ErrRecoveryRecordNotFound
173 }185 }
@@ -207,9 +219,15 @@ func (c *Coordinator) recoverDiscardLocked(ctx context.Context, intent domain.Di
207 if err := validateDiscardIntent(intent); err != nil {219 if err := validateDiscardIntent(intent); err != nil {
208 return err220 return err
209 }221 }
210- current, ok := c.catalogue.Record(intent.SessionName)222+ current, ok, err := c.catalogue.Record(intent.SessionName)
223+ if err != nil {
224+ return err
225+ }
211 if !ok {226 if !ok {
212- return fmt.Errorf("%w: missing session %q", ErrDiscardConflict, intent.SessionName)227+ // Catalogue deletion is terminal for this session identity. There is no
228+ // live record to replace or quarantine, so retaining the intent would
229+ // manufacture a permanent startup conflict.
230+ return c.journal.DeleteDiscard(ctx, intent.OldIncarnation)
213 }231 }
214 next := freshReplacement(intent)232 next := freshReplacement(intent)
215 if err := next.Validate(); err != nil {233 if err := next.Validate(); err != nil {
@@ -217,7 +235,7 @@ func (c *Coordinator) recoverDiscardLocked(ctx context.Context, intent domain.Di
217 }235 }
218 switch current.IncarnationID {236 switch current.IncarnationID {
219 case intent.OldIncarnation:237 case intent.OldIncarnation:
220- if !catalogueRecordsEqual(current, intent.OldRecord) {238+ if !current.Equal(intent.OldRecord) {
221 return fmt.Errorf("%w: old record changed", ErrDiscardConflict)239 return fmt.Errorf("%w: old record changed", ErrDiscardConflict)
222 }240 }
223 case intent.NewIncarnation:241 case intent.NewIncarnation:
@@ -275,11 +293,3 @@ func freshReplacement(intent domain.DiscardIntent) domain.CatalogueRecord {
275 next.DegradedReason = ""293 next.DegradedReason = ""
276 return next294 return next
277 }295 }
278-
279-func catalogueRecordsEqual(a, b domain.CatalogueRecord) bool {
280- return a.Name == b.Name && a.IncarnationID == b.IncarnationID && a.Cwd == b.Cwd &&
281- a.CreatedAt == b.CreatedAt && a.UpdatedAt == b.UpdatedAt && a.LastUsedSeq == b.LastUsedSeq &&
282- slices.Equal(a.TabNames, b.TabNames) && a.RecoveryState == b.RecoveryState &&
283- equalCheckpointRefs(a.Committed, b.Committed) && equalCheckpointRefs(a.Fallbacks[0], b.Fallbacks[0]) &&
284- equalCheckpointRefs(a.Fallbacks[1], b.Fallbacks[1]) && a.DegradedReason == b.DegradedReason
285-}
M internal/usecase/recovery/reconcile.go +26 -5
@@ -29,23 +29,41 @@ func (r *Reconciler) Step(ctx context.Context, cursor ports.ReconcileCursor) (po
29 if err := ctx.Err(); err != nil {29 if err := ctx.Err(); err != nil {
30 return cursor, nil, err30 return cursor, nil, err
31 }31 }
32- next, findings, err := r.repository.Reconcile(ctx, r.catalogue.Records(), cursor, r.budget)32+ records, err := r.catalogue.Records()
33+ if err != nil {
34+ return cursor, nil, err
35+ }
36+ next, findings, err := r.repository.Reconcile(ctx, records, cursor, r.budget)
33 if err != nil {37 if err != nil {
34 return cursor, nil, err38 return cursor, nil, err
35 }39 }
36 decisions := make([]ports.ReconcileDecision, 0, len(findings))40 decisions := make([]ports.ReconcileDecision, 0, len(findings))
41+ consumed := ports.MaintenanceBudget{}
37 for _, finding := range findings {42 for _, finding := range findings {
38- if finding.Consumed.Entries > r.budget.Entries || finding.Consumed.Bytes > r.budget.Bytes {43+ // Compare against the remainder before adding. This both enforces the
44+ // page-wide budget and avoids uint64 wraparound.
45+ if finding.Consumed.Entries > r.budget.Entries-consumed.Entries ||
46+ finding.Consumed.Bytes > r.budget.Bytes-consumed.Bytes {
39 return cursor, nil, errors.New("recovery: repository exceeded reconciliation budget")47 return cursor, nil, errors.New("recovery: repository exceeded reconciliation budget")
40 }48 }
41- record, ok := r.catalogue.Record(finding.Candidate.Name)49+ consumed.Entries += finding.Consumed.Entries
50+ consumed.Bytes += finding.Consumed.Bytes
51+ record, ok, err := r.catalogue.Record(finding.Candidate.Name)
52+ if err != nil {
53+ return cursor, nil, err
54+ }
42 if !ok {55 if !ok {
43 record = domain.CatalogueRecord{}56 record = domain.CatalogueRecord{}
44 }57 }
45 decision := classifyFinding(record, finding)58 decision := classifyFinding(record, finding)
46 if decision.Kind == ports.ReconcileAdopt {59 if decision.Kind == ports.ReconcileAdopt {
47 if _, err := r.coordinator.PublishReconciledCheckpoint(ctx, finding.Candidate.Name, finding.Candidate, finding.AncestorChain); err != nil {60 if _, err := r.coordinator.PublishReconciledCheckpoint(ctx, finding.Candidate.Name, finding.Candidate, finding.AncestorChain); err != nil {
48- return cursor, nil, err61+ if !errors.Is(err, ErrCheckpointConflict) && !errors.Is(err, ErrCheckpointRecordNotFound) {
62+ return cursor, nil, err
63+ }
64+ decision.Kind = ports.ReconcileDefer
65+ decision.ReasonCode = "catalogue-conflict"
66+ decision.RetentionResolved = false
49 }67 }
50 }68 }
51 decisions = append(decisions, decision)69 decisions = append(decisions, decision)
@@ -132,7 +150,10 @@ func (c *Coordinator) PublishReconciledCheckpoint(ctx context.Context, name stri
132 if err := ctx.Err(); err != nil {150 if err := ctx.Err(); err != nil {
133 return domain.CatalogueRecord{}, err151 return domain.CatalogueRecord{}, err
134 }152 }
135- record, ok := c.catalogue.Record(name)153+ record, ok, err := c.catalogue.Record(name)
154+ if err != nil {
155+ return domain.CatalogueRecord{}, err
156+ }
136 if !ok {157 if !ok {
137 return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound158 return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound
138 }159 }
M internal/usecase/recovery/reconcile_test.go +77 -0
@@ -24,6 +24,33 @@ type reconcilePage struct {
24 err error24 err error
25 }25 }
26 26
27+type reconcileConflictCatalogue struct {
28+ *transactionCatalogue
29+ stale domain.CatalogueRecord
30+ reads int
31+}
32+
33+func (c *reconcileConflictCatalogue) Records() ([]domain.CatalogueRecord, error) {
34+ records, err := c.transactionCatalogue.Records()
35+ if err != nil {
36+ return nil, err
37+ }
38+ for i := range records {
39+ if records[i].Name == c.stale.Name {
40+ records[i] = c.stale
41+ }
42+ }
43+ return records, nil
44+}
45+
46+func (c *reconcileConflictCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) {
47+ if name == c.stale.Name && c.reads == 0 {
48+ c.reads++
49+ return c.stale, true, nil
50+ }
51+ return c.transactionCatalogue.Record(name)
52+}
53+
27 func (r *reconcileRepository) Reconcile(_ context.Context, _ []domain.CatalogueRecord, cursor ports.ReconcileCursor, budget ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error) {54 func (r *reconcileRepository) Reconcile(_ context.Context, _ []domain.CatalogueRecord, cursor ports.ReconcileCursor, budget ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error) {
28 r.seenCursors = append(r.seenCursors, cursor)55 r.seenCursors = append(r.seenCursors, cursor)
29 r.seenBudgets = append(r.seenBudgets, budget)56 r.seenBudgets = append(r.seenBudgets, budget)
@@ -64,6 +91,56 @@ func TestReconciliationCursor(t *testing.T) {
64 require.Equal(t, []ports.MaintenanceBudget{{Entries: 2, Bytes: 32}, {Entries: 2, Bytes: 32}}, repository.seenBudgets)91 require.Equal(t, []ports.MaintenanceBudget{{Entries: 2, Bytes: 32}, {Entries: 2, Bytes: 32}}, repository.seenBudgets)
65 }92 }
66 93
94+func TestReconciliationRejectsCumulativeAndOverflowingBudgets(t *testing.T) {
95+ record := healthyReconcileRecord()
96+ finding := ports.ReconcileFinding{Status: ports.ReconcileBudgetExhausted, Candidate: ports.ReconcileCandidate{Name: record.Name}}
97+ for _, tc := range []struct {
98+ name string
99+ budget ports.MaintenanceBudget
100+ consumed []ports.MaintenanceBudget
101+ }{
102+ {name: "cumulative entries", budget: ports.MaintenanceBudget{Entries: 3, Bytes: 10}, consumed: []ports.MaintenanceBudget{{Entries: 2, Bytes: 1}, {Entries: 2, Bytes: 1}}},
103+ {name: "cumulative bytes", budget: ports.MaintenanceBudget{Entries: 10, Bytes: 3}, consumed: []ports.MaintenanceBudget{{Entries: 1, Bytes: 2}, {Entries: 1, Bytes: 2}}},
104+ {name: "overflow safe", budget: ports.MaintenanceBudget{Entries: ^uint64(0), Bytes: ^uint64(0)}, consumed: []ports.MaintenanceBudget{{Entries: ^uint64(0), Bytes: ^uint64(0)}, {Entries: 1, Bytes: 1}}},
105+ } {
106+ t.Run(tc.name, func(t *testing.T) {
107+ findings := make([]ports.ReconcileFinding, len(tc.consumed))
108+ for i, usage := range tc.consumed {
109+ findings[i] = finding
110+ findings[i].Consumed = usage
111+ }
112+ repository := &reconcileRepository{pages: map[uint64]reconcilePage{0: {findings: findings}}}
113+ catalogue := &checkpointCatalogue{record: record}
114+ reconciler := NewReconciler(NewCoordinator(catalogue, repository, nil, nil), catalogue, repository, tc.budget)
115+ _, _, err := reconciler.Step(context.Background(), ports.ReconcileCursor{})
116+ require.EqualError(t, err, "recovery: repository exceeded reconciliation budget")
117+ })
118+ }
119+}
120+
121+func TestReconciliationDefersConflictAndContinues(t *testing.T) {
122+ first := healthyReconcileRecord()
123+ first.Name = "first"
124+ staleRecord := first
125+ stale := forwardCandidate(staleRecord, 2, staleRecord.Committed)
126+ first = shiftedCheckpoint(first, domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{9}})
127+ second := healthyReconcileRecord()
128+ second.Name = "second"
129+ adoptable := forwardCandidate(second, 2, second.Committed)
130+ catalogue := &reconcileConflictCatalogue{transactionCatalogue: newTransactionCatalogue(first, second), stale: staleRecord}
131+ repository := &reconcileRepository{pages: map[uint64]reconcilePage{0: {findings: []ports.ReconcileFinding{
132+ {Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: stale, Consumed: ports.MaintenanceBudget{Entries: 1, Bytes: 1}},
133+ {Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: adoptable, Consumed: ports.MaintenanceBudget{Entries: 1, Bytes: 1}},
134+ }}}}
135+ reconciler := NewReconciler(NewCoordinator(catalogue, repository, nil, nil), catalogue, repository, ports.MaintenanceBudget{Entries: 2, Bytes: 2})
136+
137+ _, decisions, err := reconciler.Step(context.Background(), ports.ReconcileCursor{})
138+ require.NoError(t, err)
139+ require.Equal(t, []ports.ReconcileDecisionKind{ports.ReconcileDefer, ports.ReconcileAdopt}, []ports.ReconcileDecisionKind{decisions[0].Kind, decisions[1].Kind})
140+ require.Equal(t, "catalogue-conflict", decisions[0].ReasonCode)
141+ require.Equal(t, adoptable.Ref, *catalogue.records[second.Name].Committed)
142+}
143+
67 func TestForwardOrphanAdoption(t *testing.T) {144 func TestForwardOrphanAdoption(t *testing.T) {
68 for _, tc := range []struct {145 for _, tc := range []struct {
69 name string146 name string
M internal/usecase/recovery/transactions_test.go +21 -10
@@ -27,20 +27,20 @@ func newTransactionCatalogue(records ...domain.CatalogueRecord) *transactionCata
27 return catalogue27 return catalogue
28 }28 }
29 29
30-func (c *transactionCatalogue) Records() []domain.CatalogueRecord {30+func (c *transactionCatalogue) Records() ([]domain.CatalogueRecord, error) {
31 c.mu.Lock()31 c.mu.Lock()
32 defer c.mu.Unlock()32 defer c.mu.Unlock()
33 out := make([]domain.CatalogueRecord, 0, len(c.records))33 out := make([]domain.CatalogueRecord, 0, len(c.records))
34 for _, record := range c.records {34 for _, record := range c.records {
35 out = append(out, record)35 out = append(out, record)
36 }36 }
37- return out37+ return out, nil
38 }38 }
39-func (c *transactionCatalogue) Record(name string) (domain.CatalogueRecord, bool) {39+func (c *transactionCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) {
40 c.mu.Lock()40 c.mu.Lock()
41 defer c.mu.Unlock()41 defer c.mu.Unlock()
42 record, ok := c.records[name]42 record, ok := c.records[name]
43- return record, ok43+ return record, ok, nil
44 }44 }
45 func (c *transactionCatalogue) Create(record domain.CatalogueRecord) error {45 func (c *transactionCatalogue) Create(record domain.CatalogueRecord) error {
46 c.mu.Lock()46 c.mu.Lock()
@@ -233,16 +233,13 @@ func TestFallbackPromotion(t *testing.T) {
233 fallback := *record.Fallbacks[0]233 fallback := *record.Fallbacks[0]
234 catalogue := newTransactionCatalogue(record)234 catalogue := newTransactionCatalogue(record)
235 repository := newTransactionRepository()235 repository := newTransactionRepository()
236- repository.generations[fallback] = ports.SnapshotGeneration{236+ manifest := []byte("fallback")
237- IncarnationID: record.IncarnationID, Name: record.Name, Generation: fallback.Generation,237+ fallback.ManifestDigest = checkpointDigest(manifest)
238- Manifest: []byte("fallback"),
239- }
240- fallback.ManifestDigest = checkpointDigest(repository.generations[*record.Fallbacks[0]].Manifest)
241 record.Fallbacks[0] = &fallback238 record.Fallbacks[0] = &fallback
242 catalogue.records[record.Name] = record239 catalogue.records[record.Name] = record
243 repository.generations[fallback] = ports.SnapshotGeneration{240 repository.generations[fallback] = ports.SnapshotGeneration{
244 IncarnationID: record.IncarnationID, Name: record.Name, Generation: fallback.Generation,241 IncarnationID: record.IncarnationID, Name: record.Name, Generation: fallback.Generation,
245- Manifest: []byte("fallback"),242+ Manifest: manifest,
246 }243 }
247 coordinator := NewCoordinator(catalogue, repository, newTransactionJournal(), nil)244 coordinator := NewCoordinator(catalogue, repository, newTransactionJournal(), nil)
248 245
@@ -283,6 +280,20 @@ func TestDegradedExportIsReadOnly(t *testing.T) {
283 280
284 // A third incarnation still fails closed for that session: its intent and281 // A third incarnation still fails closed for that session: its intent and
285 // record are retained untouched, but startup is no longer aborted.282 // record are retained untouched, but startup is no longer aborted.
283+func TestDiscardRecoveryRemovesIntentWhenCatalogueRecordIsAbsent(t *testing.T) {
284+ old := degradedTransactionRecord()
285+ intent := domain.DiscardIntent{OldRecord: old, OldIncarnation: old.IncarnationID, NewIncarnation: domain.IncarnationID{2}, SessionName: old.Name, Reason: "discard"}
286+ catalogue := newTransactionCatalogue()
287+ journal := newTransactionJournal()
288+ journal.intents[old.IncarnationID] = intent
289+ coordinator := NewCoordinator(catalogue, newTransactionRepository(), journal, nil)
290+
291+ require.NoError(t, coordinator.Recover(context.Background()))
292+ require.NotContains(t, journal.intents, old.IncarnationID)
293+ require.Equal(t, []string{"delete-intent"}, journal.events)
294+ require.Empty(t, coordinator.Conflicts())
295+}
296+
286 func TestDiscardIncarnationConflict(t *testing.T) {297 func TestDiscardIncarnationConflict(t *testing.T) {
287 old := degradedTransactionRecord()298 old := degradedTransactionRecord()
288 intent := domain.DiscardIntent{OldRecord: old, OldIncarnation: old.IncarnationID, NewIncarnation: domain.IncarnationID{2}, SessionName: old.Name, Reason: "discard"}299 intent := domain.DiscardIntent{OldRecord: old, OldIncarnation: old.IncarnationID, NewIncarnation: domain.IncarnationID{2}, SessionName: old.Name, Reason: "discard"}
M internal/usecase/snapshot/manifest_preflight.go +4 -1
@@ -16,7 +16,10 @@ func preflightManifest(body []byte) error {
16 if err != nil {16 if err != nil {
17 return err17 return err
18 }18 }
19- if generation == 0 || len(r.b) < len(domain.IncarnationID{}) {19+ if generation == 0 {
20+ return fmt.Errorf("%w: manifest generation", ErrInvalidData)
21+ }
22+ if len(r.b) < len(domain.IncarnationID{}) {
20 return ErrShortPayload23 return ErrShortPayload
21 }24 }
22 var incarnationID domain.IncarnationID25 var incarnationID domain.IncarnationID
M internal/usecase/snapshot/manifest_preflight_test.go +55 -3
@@ -40,6 +40,7 @@ func TestManifestEnvelopePreflightParity(t *testing.T) {
40 if err != nil {40 if err != nil {
41 t.Fatal(err)41 t.Fatal(err)
42 }42 }
43+ nameOffset, nodeKindOffset := manifestFieldOffsets(t, encoded)
43 mutations := []struct {44 mutations := []struct {
44 name string45 name string
45 edit func([]byte) []byte46 edit func([]byte) []byte
@@ -51,9 +52,21 @@ func TestManifestEnvelopePreflightParity(t *testing.T) {
51 {"body length", func(b []byte) []byte { binary.BigEndian.PutUint32(b[8:12], uint32(len(b))); return b }},52 {"body length", func(b []byte) []byte { binary.BigEndian.PutUint32(b[8:12], uint32(len(b))); return b }},
52 {"crc", func(b []byte) []byte { b[12] ^= 1; return b }},53 {"crc", func(b []byte) []byte { b[12] ^= 1; return b }},
53 {"trailing", func(b []byte) []byte { return append(b, 0) }},54 {"trailing", func(b []byte) []byte { return append(b, 0) }},
54- {"body name", func(b []byte) []byte { b[26] ^= 1; binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])); return b }},55+ {"body name", func(b []byte) []byte {
55- {"body generation", func(b []byte) []byte { b[16] ^= 1; binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])); return b }},56+ b[nameOffset] = 0
56- {"body node kind", func(b []byte) []byte { b[65] = 99; binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])); return b }},57+ binary.BigEndian.PutUint32(b[12:16], checksum(b[16:]))
58+ return b
59+ }},
60+ {"body generation", func(b []byte) []byte {
61+ clear(b[manifestHeaderSize : manifestHeaderSize+8])
62+ binary.BigEndian.PutUint32(b[12:16], checksum(b[16:]))
63+ return b
64+ }},
65+ {"body node kind", func(b []byte) []byte {
66+ b[nodeKindOffset] = 99
67+ binary.BigEndian.PutUint32(b[12:16], checksum(b[16:]))
68+ return b
69+ }},
57 }70 }
58 for _, tc := range mutations {71 for _, tc := range mutations {
59 t.Run(tc.name, func(t *testing.T) {72 t.Run(tc.name, func(t *testing.T) {
@@ -68,6 +81,45 @@ func TestManifestEnvelopePreflightParity(t *testing.T) {
68 }81 }
69 }82 }
70 83
84+func manifestFieldOffsets(t *testing.T, encoded []byte) (nameOffset, nodeKindOffset int) {
85+ t.Helper()
86+ r := payloadReader{b: encoded[manifestHeaderSize:]}
87+ must := func(err error) {
88+ if err != nil {
89+ t.Fatal(err)
90+ }
91+ }
92+ _, err := r.getUint64()
93+ must(err)
94+ r.b = r.b[len(domain.IncarnationID{}):]
95+ _, err = r.getUint8()
96+ must(err)
97+ nameLengthOffset := len(encoded) - len(r.b)
98+ name, err := r.getString()
99+ must(err)
100+ nameOffset = nameLengthOffset + 2
101+ if len(name) == 0 {
102+ t.Fatal("test manifest has empty name")
103+ }
104+ _, err = r.getUint64()
105+ must(err)
106+ _, err = r.getUint16()
107+ must(err)
108+ _, err = r.getUint16()
109+ must(err)
110+ _, err = r.getString()
111+ must(err)
112+ _, err = r.getUint16()
113+ must(err)
114+ _, err = r.getUint16()
115+ must(err)
116+ _, err = r.getUint64()
117+ must(err)
118+ _, err = r.getString()
119+ must(err)
120+ return nameOffset, len(encoded) - len(r.b)
121+}
122+
71 func TestManifestObjectRefKinds(t *testing.T) {123 func TestManifestObjectRefKinds(t *testing.T) {
72 for _, tc := range []struct {124 for _, tc := range []struct {
73 name string125 name string
M pkg/kv/codec.go +12 -7
@@ -19,8 +19,9 @@ const (
19 // place by Open; nothing ever writes this framing again.19 // place by Open; nothing ever writes this framing again.
20 legacyHeaderLen = 820 legacyHeaderLen = 8
21 21
22- // Current framing. Every file starts with fileMagic plus a uint16 format22+ // Current framing. Every file starts with fileMagic, a uint16 format
23- // version, and every record carries a self-verifying header:23+ // version, and CRC32 over that six-byte identity. Every record carries a
24+ // self-verifying header:
24 //25 //
25 // [0:4] payload length, big endian26 // [0:4] payload length, big endian
26 // [4:8] CRC32(payload)27 // [4:8] CRC32(payload)
@@ -29,10 +30,13 @@ const (
29 // The header checksum is what makes the length trustworthy: a corrupt30 // The header checksum is what makes the length trustworthy: a corrupt
30 // length now fails closed instead of masquerading as a torn tail and31 // length now fails closed instead of masquerading as a torn tail and
31 // silently truncating every record that follows it.32 // silently truncating every record that follows it.
32- recordHeaderLen = 1233+ recordHeaderLen = 12
33- fileMagicLen = 434+ fileMagicLen = 4
34- fileHeaderLen = fileMagicLen + 235+ fileHeaderV1Len = fileMagicLen + 2
35- formatVersion = 136+ fileHeaderBodyLen = fileHeaderV1Len
37+ fileHeaderLen = fileHeaderBodyLen + 4
38+ formatVersionV1 = 1
39+ formatVersion = 2
36 40
37 // maxPayloadLen bounds any allocation driven by a length field. Catalogue41 // maxPayloadLen bounds any allocation driven by a length field. Catalogue
38 // records and batches are orders of magnitude smaller than this.42 // records and batches are orders of magnitude smaller than this.
@@ -145,7 +149,8 @@ func encodeBatch(changes []BatchChange) ([]byte, error) {
145 func fileHeader() []byte {149 func fileHeader() []byte {
146 buf := make([]byte, fileHeaderLen)150 buf := make([]byte, fileHeaderLen)
147 copy(buf, fileMagic[:])151 copy(buf, fileMagic[:])
148- binary.BigEndian.PutUint16(buf[fileMagicLen:], formatVersion)152+ binary.BigEndian.PutUint16(buf[fileMagicLen:fileHeaderBodyLen], formatVersion)
153+ binary.BigEndian.PutUint32(buf[fileHeaderBodyLen:], crc32.ChecksumIEEE(buf[:fileHeaderBodyLen]))
149 return buf154 return buf
150 }155 }
151 156
M pkg/kv/kv.go +100 -48
@@ -36,16 +36,30 @@ const (
36 // formatLegacy is a file written before the record header carried its own36 // formatLegacy is a file written before the record header carried its own
37 // checksum. It is read with the original semantics and upgraded at Open.37 // checksum. It is read with the original semantics and upgraded at Open.
38 formatLegacy walFormat = iota38 formatLegacy walFormat = iota
39+ // formatCurrentV1 is the released VEVK v1 framing with its six-byte file
40+ // header. Its record headers are identical to the current format.
41+ formatCurrentV1
39 formatCurrent42 formatCurrent
40 )43 )
41 44
42 func (f walFormat) headerLen() int64 {45 func (f walFormat) headerLen() int64 {
43- if f == formatCurrent {46+ if f != formatLegacy {
44 return recordHeaderLen47 return recordHeaderLen
45 }48 }
46 return legacyHeaderLen49 return legacyHeaderLen
47 }50 }
48 51
52+func (f walFormat) fileHeaderLen() int64 {
53+ switch f {
54+ case formatCurrentV1:
55+ return fileHeaderV1Len
56+ case formatCurrent:
57+ return fileHeaderLen
58+ default:
59+ return 0
60+ }
61+}
62+
49 type headerStatus uint863 type headerStatus uint8
50 64
51 const (65 const (
@@ -466,10 +480,7 @@ func replayFile(f *os.File) (ReplayResult, error) {
466 if err != nil {480 if err != nil {
467 return ReplayResult{}, err481 return ReplayResult{}, err
468 }482 }
469- result := ReplayResult{Data: make(map[string][]byte), EntrySize: make(map[string]int64), Format: format}483+ result := ReplayResult{Data: make(map[string][]byte), EntrySize: make(map[string]int64), Format: format, LastGood: format.fileHeaderLen()}
470- if format == formatCurrent {
471- result.LastGood = fileHeaderLen
472- }
473 for {484 for {
474 header, status, err := readRecordHeader(f, format, fileSize, result.LastGood)485 header, status, err := readRecordHeader(f, format, fileSize, result.LastGood)
475 if err != nil {486 if err != nil {
@@ -501,7 +512,7 @@ func replayFile(f *os.File) (ReplayResult, error) {
501 records = []record{rec}512 records = []record{rec}
502 }513 }
503 if err != nil {514 if err != nil {
504- return ReplayResult{}, fmt.Errorf("%w: invalid record at offset %d: %v", ErrCorruptWAL, result.LastGood, err)515+ return ReplayResult{}, fmt.Errorf("%w: invalid record at offset %d: %w", ErrCorruptWAL, result.LastGood, err)
505 }516 }
506 size := format.headerLen() + int64(len(payload))517 size := format.headerLen() + int64(len(payload))
507 applyRecords(result.Data, result.EntrySize, &result.Live, records, size)518 applyRecords(result.Data, result.EntrySize, &result.Live, records, size)
@@ -513,23 +524,60 @@ func replayFile(f *os.File) (ReplayResult, error) {
513 // readFileFormat consumes the file header when one is present and leaves the524 // readFileFormat consumes the file header when one is present and leaves the
514 // reader positioned at the first record either way.525 // reader positioned at the first record either way.
515 func readFileFormat(f io.ReadSeeker, fileSize int64) (walFormat, error) {526 func readFileFormat(f io.ReadSeeker, fileSize int64) (walFormat, error) {
516- if fileSize < fileHeaderLen {527+ if fileSize == 0 {
517 return formatLegacy, nil528 return formatLegacy, nil
518 }529 }
519- header := make([]byte, fileHeaderLen)530+ probeLen := min(fileSize, int64(fileHeaderLen))
520- if _, err := io.ReadFull(f, header); err != nil {531+ probe := make([]byte, probeLen)
532+ if _, err := io.ReadFull(f, probe); err != nil {
521 return formatLegacy, err533 return formatLegacy, err
522 }534 }
523- if !bytes.Equal(header[:fileMagicLen], fileMagic[:]) {535+ if fileSize < fileMagicLen {
536+ if bytes.Equal(probe, fileMagic[:len(probe)]) {
537+ return formatLegacy, fmt.Errorf("%w: incomplete VEVK header", ErrCorruptWAL)
538+ }
524 if _, err := f.Seek(0, io.SeekStart); err != nil {539 if _, err := f.Seek(0, io.SeekStart); err != nil {
525 return formatLegacy, err540 return formatLegacy, err
526 }541 }
527 return formatLegacy, nil542 return formatLegacy, nil
528 }543 }
529- if version := binary.BigEndian.Uint16(header[fileMagicLen:]); version != formatVersion {544+ if !bytes.Equal(probe[:fileMagicLen], fileMagic[:]) {
545+ // A complete v2 identity checksum authenticates the intended magic as
546+ // well as the version. Detect it before legacy replay so damage to the
547+ // first magic byte cannot masquerade as a small legacy length and be
548+ // truncated or compacted away.
549+ if len(probe) == fileHeaderLen &&
550+ binary.BigEndian.Uint16(probe[fileMagicLen:fileHeaderBodyLen]) == formatVersion &&
551+ binary.BigEndian.Uint32(probe[fileHeaderBodyLen:]) == binary.BigEndian.Uint32(fileHeader()[fileHeaderBodyLen:]) {
552+ return formatLegacy, fmt.Errorf("%w: damaged VEVK v2 magic", ErrCorruptWAL)
553+ }
554+ if _, err := f.Seek(0, io.SeekStart); err != nil {
555+ return formatLegacy, err
556+ }
557+ return formatLegacy, nil
558+ }
559+ if fileSize < fileHeaderV1Len {
560+ return formatLegacy, fmt.Errorf("%w: incomplete VEVK header", ErrCorruptWAL)
561+ }
562+ prefix := probe[:fileHeaderV1Len]
563+ if _, err := f.Seek(fileHeaderV1Len, io.SeekStart); err != nil {
564+ return formatLegacy, err
565+ }
566+ switch version := binary.BigEndian.Uint16(prefix[fileMagicLen:]); version {
567+ case formatVersionV1:
568+ return formatCurrentV1, nil
569+ case formatVersion:
570+ checksum := make([]byte, fileHeaderLen-fileHeaderV1Len)
571+ if _, err := io.ReadFull(f, checksum); err != nil {
572+ return formatLegacy, fmt.Errorf("%w: incomplete VEVK v2 header", ErrCorruptWAL)
573+ }
574+ if crc32.ChecksumIEEE(prefix) != binary.BigEndian.Uint32(checksum) {
575+ return formatLegacy, fmt.Errorf("%w: file header checksum mismatch", ErrCorruptWAL)
576+ }
577+ return formatCurrent, nil
578+ default:
530 return formatLegacy, fmt.Errorf("%w: unsupported format version %d", ErrCorruptWAL, version)579 return formatLegacy, fmt.Errorf("%w: unsupported format version %d", ErrCorruptWAL, version)
531 }580 }
532- return formatCurrent, nil
533 }581 }
534 582
535 type recordHeader struct {583 type recordHeader struct {
@@ -554,15 +602,18 @@ func readRecordHeader(f io.Reader, format walFormat, fileSize, offset int64) (re
554 payloadLen: binary.BigEndian.Uint32(buf[0:4]),602 payloadLen: binary.BigEndian.Uint32(buf[0:4]),
555 payloadCRC: binary.BigEndian.Uint32(buf[4:8]),603 payloadCRC: binary.BigEndian.Uint32(buf[4:8]),
556 }604 }
557- if format == formatCurrent {605+ if format != formatLegacy {
558 if crc32.ChecksumIEEE(buf[0:8]) != binary.BigEndian.Uint32(buf[8:recordHeaderLen]) {606 if crc32.ChecksumIEEE(buf[0:8]) != binary.BigEndian.Uint32(buf[8:recordHeaderLen]) {
559 return recordHeader{}, headerOK, fmt.Errorf("%w: record header CRC mismatch at offset %d", ErrCorruptWAL, offset)607 return recordHeader{}, headerOK, fmt.Errorf("%w: record header CRC mismatch at offset %d", ErrCorruptWAL, offset)
560 }608 }
561 // The length is now trustworthy, so an implausible one is corruption609 // The length is now trustworthy, so an implausible one is corruption
562 // rather than a short tail. Checked before any allocation.610 // rather than a short tail. Checked before any allocation.
563- if header.payloadLen > maxPayloadLen {611+ }
564- return recordHeader{}, headerOK, fmt.Errorf("%w: record length %d exceeds maximum at offset %d", ErrCorruptWAL, header.payloadLen, offset)612+ // Every framing is allocation-bounded. In legacy framing an oversized
565- }613+ // declaration remains corruption rather than driving an attacker-sized
614+ // allocation; ordinary short tails retain their released truncation semantics.
615+ if header.payloadLen > maxPayloadLen {
616+ return recordHeader{}, headerOK, fmt.Errorf("%w: record length %d exceeds maximum at offset %d", ErrCorruptWAL, header.payloadLen, offset)
566 }617 }
567 if int64(header.payloadLen) > fileSize-offset-format.headerLen() {618 if int64(header.payloadLen) > fileSize-offset-format.headerLen() {
568 return recordHeader{}, headerTorn, nil619 return recordHeader{}, headerTorn, nil
@@ -577,9 +628,9 @@ func applyRecords(data map[string][]byte, sizes map[string]int64, live *int64, r
577 setCount++628 setCount++
578 }629 }
579 }630 }
580- setSize := size631+ setSize, remainder := size, int64(0)
581 if setCount > 1 {632 if setCount > 1 {
582- setSize = size / setCount633+ setSize, remainder = size/setCount, size%setCount
583 }634 }
584 for _, rec := range records {635 for _, rec := range records {
585 key := string(rec.key)636 key := string(rec.key)
@@ -587,9 +638,14 @@ func applyRecords(data map[string][]byte, sizes map[string]int64, live *int64, r
587 *live -= old638 *live -= old
588 }639 }
589 if rec.op == opSet {640 if rec.op == opSet {
641+ entrySize := setSize
642+ if remainder > 0 {
643+ entrySize++
644+ remainder--
645+ }
590 data[key] = append([]byte(nil), rec.value...)646 data[key] = append([]byte(nil), rec.value...)
591- sizes[key] = setSize647+ sizes[key] = entrySize
592- *live += setSize648+ *live += entrySize
593 } else {649 } else {
594 delete(data, key)650 delete(data, key)
595 delete(sizes, key)651 delete(sizes, key)
@@ -598,7 +654,7 @@ func applyRecords(data map[string][]byte, sizes map[string]int64, live *int64, r
598 }654 }
599 655
600 func recoverCompaction(path string, hooks fsHooks) error {656 func recoverCompaction(path string, hooks fsHooks) error {
601- _, currentExists, currentErr := candidate(path)657+ currentResult, currentExists, currentErr := candidate(path)
602 nextResult, nextExists, nextErr := candidate(path + ".next")658 nextResult, nextExists, nextErr := candidate(path + ".next")
603 _, prevExists, prevErr := candidate(path + ".prev")659 _, prevExists, prevErr := candidate(path + ".prev")
604 if nextErr == nil && nextResult.TornTail {660 if nextErr == nil && nextResult.TornTail {
@@ -606,6 +662,12 @@ func recoverCompaction(path string, hooks fsHooks) error {
606 }662 }
607 dir := filepath.Dir(path)663 dir := filepath.Dir(path)
608 if currentExists && currentErr == nil {664 if currentExists && currentErr == nil {
665+ if currentResult.TornTail && prevExists {
666+ // A predecessor proves compaction was in flight. The torn current is
667+ // ambiguous, so preserve both candidates and fail closed rather than
668+ // truncating current and deleting the last complete predecessor.
669+ return fmt.Errorf("%w: torn current with predecessor", ErrCorruptWAL)
670+ }
609 if nextExists {671 if nextExists {
610 if err := hooks.Remove(path + ".next"); err != nil {672 if err := hooks.Remove(path + ".next"); err != nil {
611 return err673 return err
@@ -625,33 +687,23 @@ func recoverCompaction(path string, hooks fsHooks) error {
625 return nil687 return nil
626 }688 }
627 if currentExists && currentErr != nil {689 if currentExists && currentErr != nil {
628- if !prevExists || prevErr != nil {690+ // Corruption does not prove which bytes are disposable. Preserve every
629- return fmt.Errorf("%w: no valid current or predecessor", ErrCorruptWAL)691+ // candidate for diagnosis/recovery instead of deleting current or a valid
630- }692+ // predecessor and fail closed.
631- if err := hooks.Remove(path); err != nil {693+ return fmt.Errorf("%w: invalid current candidate", ErrCorruptWAL)
632- return err694+ }
633- }695+ if !currentExists && (nextExists || prevExists) {
634- if err := hooks.Rename(path+".prev", path); err != nil {696+ // A complete .next is the newest fully published compaction candidate and
635- return err697+ // wins even when .prev exists but is corrupt. Otherwise a complete .prev
636- }698+ // is the only safe rollback. Invalid candidates are never selected.
637- if err := hooks.SyncDir(dir); err != nil {699+ source := ""
638- return err
639- }
640- if nextExists {
641- if err := hooks.Remove(path + ".next"); err != nil {
642- return err
643- }
644- return hooks.SyncDir(dir)
645- }
646- return nil
647- }
648- if !currentExists && prevExists {
649- if prevErr != nil {
650- return fmt.Errorf("%w: invalid predecessor", ErrCorruptWAL)
651- }
652- source := path + ".prev"
653 if nextExists && nextErr == nil {700 if nextExists && nextErr == nil {
654 source = path + ".next"701 source = path + ".next"
702+ } else if prevExists && prevErr == nil {
703+ source = path + ".prev"
704+ }
705+ if source == "" {
706+ return fmt.Errorf("%w: no valid current, successor, or predecessor", ErrCorruptWAL)
655 }707 }
656 if err := hooks.Rename(source, path); err != nil {708 if err := hooks.Rename(source, path); err != nil {
657 return err709 return err
@@ -668,8 +720,8 @@ func recoverCompaction(path string, hooks fsHooks) error {
668 }720 }
669 return hooks.SyncDir(dir)721 return hooks.SyncDir(dir)
670 }722 }
671- if nextExists || (currentExists && currentErr != nil) || (prevExists && prevErr != nil) {723+ if currentExists && currentErr != nil {
672- return fmt.Errorf("%w: no valid current or predecessor", ErrCorruptWAL)724+ return fmt.Errorf("%w: no valid current", ErrCorruptWAL)
673 }725 }
674 return nil726 return nil
675 }727 }
M pkg/kv/kv_test.go +168 -19
@@ -4,7 +4,9 @@ import (
4 "bytes"4 "bytes"
5 "encoding/binary"5 "encoding/binary"
6 "errors"6 "errors"
7+ "fmt"
7 "hash/crc32"8 "hash/crc32"
9+ "maps"
8 "os"10 "os"
9 "path/filepath"11 "path/filepath"
10 "reflect"12 "reflect"
@@ -26,6 +28,17 @@ func currentFile(records ...[]byte) []byte {
26 return buf28 return buf
27 }29 }
28 30
31+// v1File is the previously released VEVK framing: magic and uint16 version,
32+// followed directly by the same self-checksummed records used today.
33+func v1File(records ...[]byte) []byte {
34+ buf := append([]byte(nil), fileMagic[:]...)
35+ buf = binary.BigEndian.AppendUint16(buf, formatVersionV1)
36+ for _, record := range records {
37+ buf = append(buf, record...)
38+ }
39+ return buf
40+}
41+
29 // legacyFile reproduces the framing written by the released version: no file42 // legacyFile reproduces the framing written by the released version: no file
30 // header, and a record header of length plus payload CRC only. Compatibility43 // header, and a record header of length plus payload CRC only. Compatibility
31 // with these files is a hard requirement, so the fixture stays in the tests.44 // with these files is a hard requirement, so the fixture stays in the tests.
@@ -378,6 +391,34 @@ func TestRecoverCompactionCleansStaleNextRegardlessOfPrev(t *testing.T) {
378 }391 }
379 }392 }
380 393
394+func TestReplayPreservesWrappedDecodeCause(t *testing.T) {
395+ path := privatePath(t)
396+ writeFile(t, path, currentFile(rawRecord([]byte{0xff, 0, 0})))
397+ f, err := os.Open(path)
398+ if err != nil {
399+ t.Fatal(err)
400+ }
401+ defer func() { _ = f.Close() }()
402+ _, err = replayFile(f)
403+ if !errors.Is(err, ErrCorruptWAL) || !errors.Is(err, errBadRecord) {
404+ t.Fatalf("replayFile error = %v, want wrapped ErrCorruptWAL and errBadRecord", err)
405+ }
406+}
407+
408+func TestBatchLiveSizeAccountsForRemainderExactly(t *testing.T) {
409+ data := map[string][]byte{}
410+ sizes := map[string]int64{}
411+ var live int64
412+ applyRecords(data, sizes, &live, []record{
413+ {op: opSet, key: []byte("a"), value: []byte("one")},
414+ {op: opSet, key: []byte("b"), value: []byte("two")},
415+ {op: opSet, key: []byte("c"), value: []byte("three")},
416+ }, 10)
417+ if live != 10 || sizes["a"]+sizes["b"]+sizes["c"] != 10 {
418+ t.Fatalf("live sizes = %v, live = %d, want exact total 10", sizes, live)
419+ }
420+}
421+
381 func TestDecodeBatchRejectsImpossibleCountBeforeAllocating(t *testing.T) {422 func TestDecodeBatchRejectsImpossibleCountBeforeAllocating(t *testing.T) {
382 payload := []byte{opBatch, 0xff, 0xff, 0xff, 0xff}423 payload := []byte{opBatch, 0xff, 0xff, 0xff, 0xff}
383 allocs := testing.AllocsPerRun(10, func() {424 allocs := testing.AllocsPerRun(10, func() {
@@ -429,14 +470,17 @@ func TestCompactionRecoveryMatrix(t *testing.T) {
429 name string470 name string
430 current, next, prev []byte471 current, next, prev []byte
431 want string472 want string
473+ wantErr bool
432 }{474 }{
433 {name: "current-alone", current: oldFile, want: "old"},475 {name: "current-alone", current: oldFile, want: "old"},
434 {name: "current-next", current: oldFile, next: newFile, want: "old"},476 {name: "current-next", current: oldFile, next: newFile, want: "old"},
435 {name: "prev-next", prev: oldFile, next: newFile, want: "new"},477 {name: "prev-next", prev: oldFile, next: newFile, want: "new"},
436- {name: "prev-invalid-next", prev: oldFile, next: currentFile(rawRecord([]byte{0xff, 0, 0})), want: "old"},478+ {name: "next-alone", next: newFile, want: "new"},
479+ {name: "invalid-prev-valid-next", prev: currentFile(rawRecord([]byte{0xff, 0, 0})), next: newFile, want: "new"},
480+ {name: "valid-prev-invalid-next", prev: oldFile, next: currentFile(rawRecord([]byte{0xff, 0, 0})), want: "old"},
437 {name: "prev-torn-next", prev: oldFile, next: newFile[:len(newFile)-1], want: "old"},481 {name: "prev-torn-next", prev: oldFile, next: newFile[:len(newFile)-1], want: "old"},
438 {name: "current-prev", current: newFile, prev: oldFile, want: "new"},482 {name: "current-prev", current: newFile, prev: oldFile, want: "new"},
439- {name: "invalid-current-prev", current: currentFile(rawRecord([]byte{0xff, 0, 0})), prev: oldFile, want: "old"},483+ {name: "invalid-current-prev", current: currentFile(rawRecord([]byte{0xff, 0, 0})), prev: oldFile, wantErr: true},
440 {name: "prev-alone", prev: oldFile, want: "old"},484 {name: "prev-alone", prev: oldFile, want: "old"},
441 }485 }
442 for _, tt := range tests {486 for _, tt := range tests {
@@ -452,7 +496,18 @@ func TestCompactionRecoveryMatrix(t *testing.T) {
452 }496 }
453 }497 }
454 }498 }
455- if err := recoverCompaction(path, defaultFSHooks()); err != nil {499+ before := candidateBytes(t, path)
500+ err := recoverCompaction(path, defaultFSHooks())
501+ if tt.wantErr {
502+ if !errors.Is(err, ErrCorruptWAL) {
503+ t.Fatalf("recoverCompaction error = %v, want ErrCorruptWAL", err)
504+ }
505+ if got := candidateBytes(t, path); !reflect.DeepEqual(got, before) {
506+ t.Fatalf("uncertain candidates changed: got %#v, want %#v", got, before)
507+ }
508+ return
509+ }
510+ if err != nil {
456 t.Fatal(err)511 t.Fatal(err)
457 }512 }
458 got, err := Replay(path)513 got, err := Replay(path)
@@ -701,25 +756,85 @@ func rawBatchPayload(changes []BatchChange) []byte {
701 return payload756 return payload
702 }757 }
703 758
704-// An oversized length must never drive an allocation. In a legacy file the759+// An oversized length must never drive an allocation. Legacy framing is
705-// length is unprotected, so it stays a torn tail; in the current format the760+// accepted only after complete strict replay, while the current header checksum
706-// header checksum proves the length was written that way, which is corruption.761+// proves an oversized length was written that way; both fail closed.
762+func TestCorruptedCurrentHeaderNeverFallsBackToLegacyOrDeletesPredecessor(t *testing.T) {
763+ record, _ := encodeRecord(opSet, []byte("version"), []byte("current"))
764+ previous, _ := encodeRecord(opSet, []byte("version"), []byte("previous"))
765+
766+ tests := []struct {
767+ name string
768+ current []byte
769+ standalone bool
770+ }{
771+ {name: "torn file header", current: append([]byte(nil), fileHeader()[:fileHeaderLen-1]...), standalone: true},
772+ {name: "torn current record", current: currentFile(record)[:fileHeaderLen+recordHeaderLen+1]},
773+ }
774+ for i := range fileMagicLen {
775+ corrupted := currentFile(record)
776+ corrupted[i] = 0
777+ tests = append(tests, struct {
778+ name string
779+ current []byte
780+ standalone bool
781+ }{name: fmt.Sprintf("corrupted magic byte %d", i), current: corrupted, standalone: true})
782+ }
783+ for length := 1; length < fileHeaderLen; length++ {
784+ tests = append(tests, struct {
785+ name string
786+ current []byte
787+ standalone bool
788+ }{name: fmt.Sprintf("torn header prefix %d", length), current: append([]byte(nil), fileHeader()[:length]...), standalone: true})
789+ }
790+
791+ for _, tt := range tests {
792+ t.Run(tt.name, func(t *testing.T) {
793+ for _, withCandidates := range []bool{false, true} {
794+ if !withCandidates && !tt.standalone {
795+ continue
796+ }
797+ name := "standalone"
798+ if withCandidates {
799+ name = "with candidates"
800+ }
801+ t.Run(name, func(t *testing.T) {
802+ path := privatePath(t)
803+ writeFile(t, path, tt.current)
804+ if withCandidates {
805+ if err := os.WriteFile(path+".prev", currentFile(previous), 0o600); err != nil {
806+ t.Fatal(err)
807+ }
808+ if err := os.WriteFile(path+".next", currentFile(record), 0o600); err != nil {
809+ t.Fatal(err)
810+ }
811+ }
812+ before := candidateBytes(t, path)
813+ if _, err := Open(path); !errors.Is(err, ErrCorruptWAL) {
814+ t.Fatalf("Open error = %v, want ErrCorruptWAL", err)
815+ }
816+ if got := candidateBytes(t, path); !reflect.DeepEqual(got, before) {
817+ t.Fatalf("candidates changed: got %#v, want %#v", got, before)
818+ }
819+ })
820+ }
821+ })
822+ }
823+}
824+
707 func TestReplayRejectsOversizedLengthWithoutAllocating(t *testing.T) {825 func TestReplayRejectsOversizedLengthWithoutAllocating(t *testing.T) {
708 first, _ := encodeRecord(opSet, []byte("a"), []byte("one"))826 first, _ := encodeRecord(opSet, []byte("a"), []byte("one"))
709 legacyHeader := make([]byte, legacyHeaderLen)827 legacyHeader := make([]byte, legacyHeaderLen)
710 binary.BigEndian.PutUint32(legacyHeader[0:4], 1<<31)828 binary.BigEndian.PutUint32(legacyHeader[0:4], 1<<31)
711 829
712- t.Run("legacy length is a torn tail", func(t *testing.T) {830+ t.Run("legacy incomplete record fails closed", func(t *testing.T) {
713 path := privatePath(t)831 path := privatePath(t)
714- writeFile(t, path, append(legacyFile(first), legacyHeader...))832+ wal := append(legacyFile(first), legacyHeader...)
715- s, err := Open(path)833+ writeFile(t, path, wal)
716- if err != nil {834+ if _, err := Open(path); !errors.Is(err, ErrCorruptWAL) {
717- t.Fatal(err)835+ t.Fatalf("Open error = %v, want ErrCorruptWAL", err)
718- }
719- defer closeStore(t, s)
720- if got, ok := s.Get([]byte("a")); !ok || string(got) != "one" {
721- t.Fatalf("good record missing before bad length: %q %v", got, ok)
722 }836 }
837+ requireFileBytes(t, path, wal)
723 })838 })
724 839
725 t.Run("current oversized length fails closed", func(t *testing.T) {840 t.Run("current oversized length fails closed", func(t *testing.T) {
@@ -851,13 +966,49 @@ func TestCurrentFormatReplayDecisionTable(t *testing.T) {
851 966
852 // A file written by the released version must keep working: it opens with the967 // A file written by the released version must keep working: it opens with the
853 // original semantics, is upgraded in place, and reopens identically.968 // original semantics, is upgraded in place, and reopens identically.
969+func TestVEVKV1FileOpensAndUpgradesInPlace(t *testing.T) {
970+ setA, _ := encodeRecord(opSet, []byte("a"), []byte("one"))
971+ setB, _ := encodeRecord(opSet, []byte("b"), bytes.Repeat([]byte("x"), 1<<20))
972+ for _, tc := range []struct {
973+ name string
974+ wal []byte
975+ }{
976+ {name: "ordinary catalogue payload", wal: v1File(setA)},
977+ {name: "prior large catalogue payload", wal: v1File(setB)},
978+ } {
979+ t.Run(tc.name, func(t *testing.T) {
980+ path := privatePath(t)
981+ writeFile(t, path, tc.wal)
982+ store, err := Open(path)
983+ if err != nil {
984+ t.Fatal(err)
985+ }
986+ closeStore(t, store)
987+ upgraded, err := os.ReadFile(path)
988+ if err != nil {
989+ t.Fatal(err)
990+ }
991+ if !bytes.Equal(upgraded[:fileHeaderLen], fileHeader()) {
992+ t.Fatalf("upgraded header = %x, want %x", upgraded[:fileHeaderLen], fileHeader())
993+ }
994+ })
995+ }
996+}
997+
998+func TestVEVKV1RejectsOversizedRecordBeforeAllocation(t *testing.T) {
999+ path := privatePath(t)
1000+ writeFile(t, path, v1File(oversizedRecordHeader(maxPayloadLen+1)))
1001+ if _, err := Open(path); !errors.Is(err, ErrCorruptWAL) {
1002+ t.Fatalf("Open error = %v, want ErrCorruptWAL", err)
1003+ }
1004+}
1005+
854 func TestLegacyFileOpensAndUpgradesInPlace(t *testing.T) {1006 func TestLegacyFileOpensAndUpgradesInPlace(t *testing.T) {
855 setA, _ := encodeRecord(opSet, []byte("a"), []byte("one"))1007 setA, _ := encodeRecord(opSet, []byte("a"), []byte("one"))
856 setB, _ := encodeRecord(opSet, []byte("b"), []byte("two"))1008 setB, _ := encodeRecord(opSet, []byte("b"), []byte("two"))
857 replaceA, _ := encodeRecord(opSet, []byte("a"), []byte("uno"))1009 replaceA, _ := encodeRecord(opSet, []byte("a"), []byte("uno"))
858 delB, _ := encodeRecord(opDel, []byte("b"), nil)1010 delB, _ := encodeRecord(opDel, []byte("b"), nil)
859 torn, _ := encodeRecord(opSet, []byte("c"), []byte("three"))1011 torn, _ := encodeRecord(opSet, []byte("c"), []byte("three"))
860-
861 tests := []struct {1012 tests := []struct {
862 name string1013 name string
863 wal []byte1014 wal []byte
@@ -896,9 +1047,7 @@ func TestLegacyFileOpensAndUpgradesInPlace(t *testing.T) {
896 }1047 }
897 defer closeStore(t, reopened)1048 defer closeStore(t, reopened)
898 want := map[string]string{"added": "after-upgrade"}1049 want := map[string]string{"added": "after-upgrade"}
899- for k, v := range tt.want {1050+ maps.Copy(want, tt.want)
900- want[k] = v
901- }
902 requireContents(t, reopened, want)1051 requireContents(t, reopened, want)
903 replayed, err := Replay(path)1052 replayed, err := Replay(path)
904 if err != nil {1053 if err != nil {

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 12

Tags 11

↑↓ moveenter openesc close