detached d5c98ee

refactor(recovery): replace intent journals with idempotent operations

commit
d5c98eeb0210b71ca8f224a0968d1b2fb2774fe0
author
brice · · 2026-07-26 11:18 UTC
parent
4102d7e
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
37 files · +262 -3828
Unified
UnifiedSplit
M internal/adapters/recoveryfs/atomic.go +6 -2
@@ -8,13 +8,17 @@ import (
88 "github.com/bnema/vev/pkg/safedir"
99 )
1010
11+type atomicWriter struct {
12+ hooks journalHooks
13+}
14+
1115 type journalHooks struct {
1216 syncFile func(string) error
1317 rename func(string, string) error
1418 syncDirectory func(string) error
1519 }
1620
17-func (j *Journal) atomicWrite(path string, data []byte) (retErr error) {
21+func (j *atomicWriter) atomicWrite(path string, data []byte) (retErr error) {
1822 dir := filepath.Dir(path)
1923 if err := safedir.EnsurePrivate(dir); err != nil {
2024 return err
@@ -55,7 +59,7 @@ func (j *Journal) atomicWrite(path string, data []byte) (retErr error) {
5559 return j.syncDir(dir)
5660 }
5761
58-func (j *Journal) syncDir(dir string) error {
62+func (j *atomicWriter) syncDir(dir string) error {
5963 if j.hooks.syncDirectory != nil {
6064 return j.hooks.syncDirectory(dir)
6165 }
D internal/adapters/recoveryfs/journal.go +0 -198
@@ -1,198 +0,0 @@
1-package recoveryfs
2-
3-import (
4- "bytes"
5- "context"
6- "encoding/json"
7- "errors"
8- "fmt"
9- "io"
10- "os"
11- "path/filepath"
12- "sort"
13- "strings"
14-
15- "github.com/bnema/vev/internal/domain"
16- "github.com/bnema/vev/pkg/safedir"
17-)
18-
19-const (
20- discardVersion = 1
21- maxDiscardIntents = 1024
22- maxDiscardIntentSize = 64 << 10
23-)
24-
25-type Journal struct {
26- dir string
27- hooks journalHooks
28-}
29-
30-type discardFile struct {
31- Version int `json:"version"`
32- Intent domain.DiscardIntent `json:"intent"`
33-}
34-
35-func New(stateDir string) *Journal { return &Journal{dir: filepath.Join(stateDir, "transactions")} }
36-
37-func validateDiscard(intent domain.DiscardIntent) error {
38- if err := intent.OldRecord.Validate(); err != nil {
39- return fmt.Errorf("recovery journal: old record: %w", err)
40- }
41- if intent.SessionName != intent.OldRecord.Name || intent.OldIncarnation != intent.OldRecord.IncarnationID {
42- return errors.New("recovery journal: inconsistent old identity")
43- }
44- if intent.NewIncarnation == (domain.IncarnationID{}) || intent.NewIncarnation == intent.OldIncarnation || intent.Reason == "" {
45- return errors.New("recovery journal: incomplete discard intent")
46- }
47- return nil
48-}
49-
50-func encodeDiscard(intent domain.DiscardIntent) ([]byte, error) {
51- if err := validateDiscard(intent); err != nil {
52- return nil, err
53- }
54- return json.Marshal(discardFile{Version: discardVersion, Intent: intent})
55-}
56-
57-func decodeDiscard(data []byte) (domain.DiscardIntent, error) {
58- decoder := json.NewDecoder(bytes.NewReader(data))
59- decoder.DisallowUnknownFields()
60- var file discardFile
61- if err := decoder.Decode(&file); err != nil {
62- return domain.DiscardIntent{}, err
63- }
64- if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
65- if err != nil {
66- return domain.DiscardIntent{}, err
67- }
68- return domain.DiscardIntent{}, errors.New("recovery journal: trailing data")
69- }
70- if file.Version != discardVersion {
71- return domain.DiscardIntent{}, errors.New("recovery journal: unsupported version")
72- }
73- return file.Intent, validateDiscard(file.Intent)
74-}
75-
76-func (j *Journal) path(id domain.IncarnationID) string {
77- return filepath.Join(j.dir, "discard-"+id.String())
78-}
79-
80-func (j *Journal) SaveDiscard(ctx context.Context, intent domain.DiscardIntent) error {
81- if err := ctx.Err(); err != nil {
82- return err
83- }
84- data, err := encodeDiscard(intent)
85- if err != nil {
86- return err
87- }
88- path := j.path(intent.OldIncarnation)
89- if existing, err := readFileBounded(path, maxDiscardIntentSize); err == nil {
90- if !bytes.Equal(existing, data) {
91- return errors.New("recovery journal: discard identity collision")
92- }
93- // Re-sync the parent on idempotent retry: the prior install may have
94- // succeeded while its durability acknowledgement failed.
95- return j.syncDir(filepath.Dir(path))
96- } else if !errors.Is(err, os.ErrNotExist) {
97- return err
98- }
99- return j.atomicWrite(path, data)
100-}
101-
102-func (j *Journal) ListDiscards(ctx context.Context) ([]domain.DiscardIntent, error) {
103- dir, err := os.Open(j.dir)
104- if errors.Is(err, os.ErrNotExist) {
105- return nil, nil
106- }
107- if err != nil {
108- return nil, err
109- }
110- entries := make([]os.DirEntry, 0, maxDiscardIntents)
111- for {
112- batch, readErr := dir.ReadDir(maxDiscardIntents + 1)
113- for _, entry := range batch {
114- if strings.HasPrefix(entry.Name(), ".intent-") && !entry.IsDir() {
115- continue
116- }
117- if entry.IsDir() || !strings.HasPrefix(entry.Name(), "discard-") {
118- _ = dir.Close()
119- return nil, errors.New("recovery journal: invalid transaction entry")
120- }
121- entries = append(entries, entry)
122- if len(entries) > maxDiscardIntents {
123- _ = dir.Close()
124- return nil, errors.New("recovery journal: transaction entry limit exceeded")
125- }
126- }
127- if errors.Is(readErr, io.EOF) {
128- break
129- }
130- if readErr != nil {
131- _ = dir.Close()
132- return nil, readErr
133- }
134- }
135- if err := dir.Close(); err != nil {
136- return nil, err
137- }
138- sort.Slice(entries, func(a, b int) bool { return entries[a].Name() < entries[b].Name() })
139- out := make([]domain.DiscardIntent, 0, len(entries))
140- seen := make(map[domain.IncarnationID]struct{}, len(entries))
141- for _, entry := range entries {
142- if err := ctx.Err(); err != nil {
143- return nil, err
144- }
145- data, err := readFileBounded(filepath.Join(j.dir, entry.Name()), maxDiscardIntentSize)
146- if err != nil {
147- return nil, err
148- }
149- intent, err := decodeDiscard(data)
150- if err != nil {
151- return nil, err
152- }
153- if entry.Name() != filepath.Base(j.path(intent.OldIncarnation)) {
154- return nil, errors.New("recovery journal: non-canonical intent filename")
155- }
156- if _, ok := seen[intent.OldIncarnation]; ok {
157- return nil, errors.New("recovery journal: duplicate discard identity")
158- }
159- seen[intent.OldIncarnation] = struct{}{}
160- out = append(out, intent)
161- }
162- return out, nil
163-}
164-
165-func (j *Journal) DeleteDiscard(ctx context.Context, id domain.IncarnationID) error {
166- if err := ctx.Err(); err != nil {
167- return err
168- }
169- if id == (domain.IncarnationID{}) {
170- return errors.New("recovery journal: zero discard identity")
171- }
172- if err := safedir.EnsurePrivate(j.dir); err != nil {
173- return err
174- }
175- if err := os.Remove(j.path(id)); err != nil && !errors.Is(err, os.ErrNotExist) {
176- return err
177- }
178- return j.syncDir(j.dir)
179-}
180-
181-func readFileBounded(path string, limit int64) ([]byte, error) {
182- file, err := os.Open(path)
183- if err != nil {
184- return nil, err
185- }
186- data, readErr := io.ReadAll(io.LimitReader(file, limit+1))
187- closeErr := file.Close()
188- if readErr != nil {
189- return nil, readErr
190- }
191- if closeErr != nil {
192- return nil, closeErr
193- }
194- if int64(len(data)) > limit {
195- return nil, errors.New("recovery journal: intent size limit exceeded")
196- }
197- return data, nil
198-}
D internal/adapters/recoveryfs/journal_test.go +0 -145
@@ -1,145 +0,0 @@
1-package recoveryfs
2-
3-import (
4- "context"
5- "errors"
6- "fmt"
7- "os"
8- "path/filepath"
9- "testing"
10-
11- "github.com/bnema/vev/internal/domain"
12- "github.com/bnema/vev/pkg/safedir"
13- "github.com/stretchr/testify/require"
14-)
15-
16-func TestDiscardCrashMatrix(t *testing.T) {
17- ctx := context.Background()
18- state := t.TempDir()
19- j := New(state)
20- old := domain.IncarnationID{1}
21- intent := domain.DiscardIntent{
22- OldRecord: domain.CatalogueRecord{
23- Name: "work", IncarnationID: old, RecoveryState: domain.RecoveryDegraded,
24- Committed: &domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}, DegradedReason: "invalid",
25- },
26- OldIncarnation: old, NewIncarnation: domain.IncarnationID{2}, SessionName: "work", Reason: "discard",
27- }
28- require.NoError(t, j.SaveDiscard(ctx, intent))
29-
30- t.Run("existing intent cannot be overwritten", func(t *testing.T) {
31- changed := intent
32- changed.Reason = "different"
33- require.Error(t, j.SaveDiscard(ctx, changed))
34- got, err := j.ListDiscards(ctx)
35- require.NoError(t, err)
36- require.Equal(t, []domain.DiscardIntent{intent}, got)
37- })
38-
39- t.Run("malformed intent remains visible", func(t *testing.T) {
40- path := j.path(old)
41- encoded, err := os.ReadFile(path)
42- require.NoError(t, err)
43- t.Cleanup(func() { require.NoError(t, os.WriteFile(path, encoded, 0o600)) })
44- require.NoError(t, os.WriteFile(path, append(encoded, 0), 0o600))
45- _, err = j.ListDiscards(ctx)
46- require.Error(t, err)
47- _, statErr := os.Stat(path)
48- require.NoError(t, statErr)
49- require.Error(t, j.SaveDiscard(ctx, intent))
50- })
51-
52- t.Run("delete sync failure rolls forward", func(t *testing.T) {
53- injected := errors.New("injected delete directory sync failure")
54- j.hooks.syncDirectory = func(string) error { return injected }
55- require.ErrorIs(t, j.DeleteDiscard(ctx, old), injected)
56- j.hooks = journalHooks{}
57- require.NoError(t, j.DeleteDiscard(ctx, old))
58- got, err := j.ListDiscards(ctx)
59- require.NoError(t, err)
60- require.Empty(t, got)
61- })
62-}
63-
64-func TestRecoveryJournalRoundTripAndIdempotentDelete(t *testing.T) {
65- j := New(t.TempDir())
66- old := domain.IncarnationID{1}
67- intent := domain.DiscardIntent{
68- OldRecord: domain.CatalogueRecord{Name: "work", IncarnationID: old, RecoveryState: domain.RecoveryDegraded, Committed: &domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}, DegradedReason: "invalid"},
69- OldIncarnation: old, NewIncarnation: domain.IncarnationID{2}, SessionName: "work", Reason: "discard",
70- }
71- require.NoError(t, j.SaveDiscard(context.Background(), intent))
72- got, err := j.ListDiscards(context.Background())
73- require.NoError(t, err)
74- require.Equal(t, []domain.DiscardIntent{intent}, got)
75- require.NoError(t, j.DeleteDiscard(context.Background(), old))
76- require.NoError(t, j.DeleteDiscard(context.Background(), old))
77- got, err = j.ListDiscards(context.Background())
78- require.NoError(t, err)
79- require.Empty(t, got)
80-}
81-
82-func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T) {
83- j := New(t.TempDir())
84- require.NoError(t, safedir.EnsurePrivate(j.dir))
85- for i := range maxDiscardIntents + 1 {
86- require.NoError(t, os.WriteFile(filepath.Join(j.dir, fmt.Sprintf(".intent-leftover-%04d", i)), nil, 0o600))
87- }
88-
89- got, err := j.ListDiscards(context.Background())
90- require.NoError(t, err)
91- require.Empty(t, got)
92-}
93-
94-func TestRecoveryJournalRejectsMalformedIntent(t *testing.T) {
95- j := New(t.TempDir())
96- require.Error(t, j.SaveDiscard(context.Background(), domain.DiscardIntent{}))
97-
98- _, err := decodeDiscard([]byte(`{"version":1,"intent":{}} !`))
99- require.ErrorContains(t, err, "invalid character")
100-}
101-
102-func TestRecoveryJournalBoundsEnumerationAndIntentReads(t *testing.T) {
103- state := t.TempDir()
104- j := New(state)
105- require.NoError(t, safedir.EnsurePrivate(j.dir))
106- for i := range maxDiscardIntents + 1 {
107- require.NoError(t, os.WriteFile(filepath.Join(j.dir, fmt.Sprintf("discard-extra-%04d", i)), nil, 0o600))
108- }
109- _, err := j.ListDiscards(context.Background())
110- require.ErrorContains(t, err, "entry limit exceeded")
111-
112- require.NoError(t, os.RemoveAll(j.dir))
113- require.NoError(t, safedir.EnsurePrivate(j.dir))
114- require.NoError(t, os.WriteFile(filepath.Join(j.dir, "discard-oversized"), make([]byte, maxDiscardIntentSize+1), 0o600))
115- _, err = j.ListDiscards(context.Background())
116- require.ErrorContains(t, err, "size limit exceeded")
117-}
118-
119-func TestRecoveryJournalFailsClosedAtDurabilityBoundaries(t *testing.T) {
120- for _, boundary := range []string{"file sync", "rename", "directory sync"} {
121- t.Run(boundary, func(t *testing.T) {
122- j := New(t.TempDir())
123- old := domain.IncarnationID{1}
124- intent := domain.DiscardIntent{
125- OldRecord: domain.CatalogueRecord{Name: "work", IncarnationID: old, RecoveryState: domain.RecoveryDegraded, Committed: &domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}, DegradedReason: "invalid"},
126- OldIncarnation: old, NewIncarnation: domain.IncarnationID{2}, SessionName: "work", Reason: "discard",
127- }
128- injected := errors.New("injected durability failure")
129- switch boundary {
130- case "file sync":
131- j.hooks.syncFile = func(string) error { return injected }
132- case "rename":
133- j.hooks.rename = func(string, string) error { return injected }
134- case "directory sync":
135- j.hooks.syncDirectory = func(string) error { return injected }
136- }
137- require.ErrorIs(t, j.SaveDiscard(context.Background(), intent), injected)
138- j.hooks = journalHooks{}
139- require.NoError(t, j.SaveDiscard(context.Background(), intent))
140- got, err := j.ListDiscards(context.Background())
141- require.NoError(t, err)
142- require.Equal(t, []domain.DiscardIntent{intent}, got)
143- })
144- }
145-}
M internal/adapters/snapshot/maintenance_delete.go +13 -135
@@ -19,10 +19,9 @@ import (
1919 )
2020
2121 const (
22- deletionTombstoneVersion = uint16(1)
23- deletionTombstoneMagic = "VEVD"
24- deletionTombstonesDir = "deletions"
25- quarantineDescriptorVersion = 1
22+ deletionTombstoneVersion = uint16(1)
23+ deletionTombstoneMagic = "VEVD"
24+ deletionTombstonesDir = "deletions"
2625 )
2726
2827 var ErrMaintenanceBudgetTooSmall = errors.New("snapshot: maintenance budget too small")
@@ -550,151 +549,30 @@ func (r *Repository) maintainQuarantine(ctx context.Context, budget *int) (chang
550549 return changed, false, nil
551550 }
552551
553-func (r *Repository) QuarantineDeletionSources(ctx context.Context, tombstone domain.DeletionTombstone, includeLegacyName bool) error {
554- if err := tombstone.Validate(); err != nil {
555- return err
556- }
552+func (r *Repository) DeleteIncarnation(ctx context.Context, id domain.IncarnationID) error {
557553 if err := ctx.Err(); err != nil {
558554 return err
559555 }
560- key, err := incarnationKey(tombstone.IncarnationID)
556+ key, err := incarnationKey(id)
561557 if err != nil {
562558 return err
563559 }
564560 lock := r.lockSession(key)
565561 defer r.unlockSession(lock)
566562 r.invalidateStorageEpoch(key)
567- quarantineDir := filepath.Join(r.dir, "quarantine", tombstone.IncarnationID.String())
568- if err := r.ensurePrivateDirectory(quarantineDir); err != nil {
569- return err
570- }
571- if err := r.quarantineSource(r.sessionPath(tombstone.IncarnationID), filepath.Join(quarantineDir, "snapshot")); err != nil {
572- return err
573- }
574- if hook := r.hooks.afterDeletionIncarnationQuarantine; hook != nil {
575- if err := hook(); err != nil {
576- return err
577- }
578- }
579- if includeLegacyName {
580- if err := r.quarantineSource(filepath.Join(r.dir, filenameForName(tombstone.Name)), filepath.Join(quarantineDir, "legacy.snap")); err != nil {
581- return err
582- }
583- if hook := r.hooks.afterDeletionLegacyQuarantine; hook != nil {
584- if err := hook(); err != nil {
585- return err
586- }
587- }
588- }
589- return nil
590-}
591-
592-func (r *Repository) quarantineSource(source, target string) error {
593- if _, err := r.stat(target); err == nil {
594- if _, sourceErr := r.stat(source); errors.Is(sourceErr, os.ErrNotExist) {
595- if err := r.syncDirectory(filepath.Dir(source)); err != nil {
596- return err
597- }
598- if filepath.Clean(filepath.Dir(source)) != filepath.Clean(filepath.Dir(target)) {
599- return r.syncDirectory(filepath.Dir(target))
600- }
601- return nil
602- }
603- return fmt.Errorf("snapshot: quarantine target already exists")
604- } else if !errors.Is(err, os.ErrNotExist) {
605- return err
606- }
607- if _, err := r.stat(source); errors.Is(err, os.ErrNotExist) {
608- return nil
609- } else if err != nil {
610- return err
611- }
612- if err := r.rename(source, target); err != nil {
613- return err
614- }
615- if err := r.syncDirectory(filepath.Dir(source)); err != nil {
616- return err
617- }
618- if filepath.Clean(filepath.Dir(source)) != filepath.Clean(filepath.Dir(target)) {
619- return r.syncDirectory(filepath.Dir(target))
620- }
621- return nil
622-}
623-
624-type quarantineDescriptorFile struct {
625- Version int `json:"version"`
626- Descriptor domain.QuarantineDescriptor `json:"descriptor"`
627-}
628-
629-func validateQuarantineDescriptor(descriptor domain.QuarantineDescriptor) error {
630- if err := descriptor.OldRecord.Validate(); err != nil {
631- return fmt.Errorf("snapshot: quarantine old record: %w", err)
632- }
633- if descriptor.SessionName != descriptor.OldRecord.Name || descriptor.OldIncarnation != descriptor.OldRecord.IncarnationID {
634- return errors.New("snapshot: inconsistent quarantine identity")
635- }
636- if descriptor.ReplacementIncarnation == (domain.IncarnationID{}) || descriptor.ReplacementIncarnation == descriptor.OldIncarnation || descriptor.Reason == "" {
637- return errors.New("snapshot: incomplete quarantine descriptor")
638- }
639- return nil
640-}
641-
642-func encodeQuarantineDescriptor(descriptor domain.QuarantineDescriptor) ([]byte, error) {
643- if err := validateQuarantineDescriptor(descriptor); err != nil {
644- return nil, err
645- }
646- return json.Marshal(quarantineDescriptorFile{Version: quarantineDescriptorVersion, Descriptor: descriptor})
647-}
648563
649-func decodeQuarantineDescriptor(encoded []byte) (domain.QuarantineDescriptor, error) {
650- decoder := json.NewDecoder(bytes.NewReader(encoded))
651- decoder.DisallowUnknownFields()
652- var file quarantineDescriptorFile
653- if err := decoder.Decode(&file); err != nil {
654- return domain.QuarantineDescriptor{}, err
655- }
656- if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
657- return domain.QuarantineDescriptor{}, errors.New("snapshot: trailing quarantine descriptor data")
564+ path := r.sessionPath(id)
565+ if err := os.RemoveAll(path); err != nil {
566+ return fmt.Errorf("remove snapshot incarnation %s: %w", id.String(), safeFilesystemError(err))
658567 }
659- if file.Version != quarantineDescriptorVersion {
660- return domain.QuarantineDescriptor{}, errors.New("snapshot: unsupported quarantine descriptor version")
661- }
662- return file.Descriptor, validateQuarantineDescriptor(file.Descriptor)
663-}
664-
665-func (r *Repository) SaveQuarantineDescriptor(ctx context.Context, descriptor domain.QuarantineDescriptor) error {
666568 if err := ctx.Err(); err != nil {
667569 return err
668570 }
669- encoded, err := encodeQuarantineDescriptor(descriptor)
670- if err != nil {
671- return err
672- }
673- dir := filepath.Join(r.dir, "quarantine", descriptor.OldIncarnation.String())
674- if err := r.ensurePrivateDirectory(dir); err != nil {
571+ parent := filepath.Dir(path)
572+ if _, err := os.Lstat(parent); errors.Is(err, os.ErrNotExist) {
573+ return nil
574+ } else if err != nil {
675575 return err
676576 }
677- return r.writeImmutable(filepath.Join(dir, "record"), encoded, func(existing []byte) error {
678- decoded, err := decodeQuarantineDescriptor(existing)
679- if err != nil {
680- return err
681- }
682- if !quarantineDescriptorsEqual(decoded, descriptor) || !bytes.Equal(existing, encoded) {
683- return errors.New("snapshot: quarantine descriptor identity mismatch")
684- }
685- return nil
686- })
687-}
688-
689-func quarantineDescriptorsEqual(a, b domain.QuarantineDescriptor) bool {
690- return a.OldIncarnation == b.OldIncarnation && a.ReplacementIncarnation == b.ReplacementIncarnation &&
691- a.SessionName == b.SessionName && a.Reason == b.Reason && a.OldRecord.Equal(b.OldRecord)
692-}
693-
694-func (r *Repository) QuarantineIncarnation(ctx context.Context, id domain.IncarnationID) error {
695- return r.QuarantineDeletionSources(ctx, domain.DeletionTombstone{Name: "quarantined", IncarnationID: id}, false)
696-}
697-
698-func (r *Repository) DeleteIncarnation(context.Context, domain.IncarnationID) error {
699- return errors.New("snapshot: direct incarnation deletion not implemented")
577+ return r.syncDirectory(parent)
700578 }
D internal/adapters/snapshot/maintenance_delete_test.go +0 -474
@@ -1,474 +0,0 @@
1-package snapshot
2-
3-import (
4- "context"
5- "encoding/base64"
6- "encoding/hex"
7- "errors"
8- "fmt"
9- "os"
10- "path/filepath"
11- "reflect"
12- "slices"
13- "testing"
14- "time"
15-
16- "github.com/bnema/vev/internal/domain"
17- "github.com/bnema/vev/internal/ports"
18-)
19-
20-func TestDiscardCrashMatrix(t *testing.T) {
21- ctx := context.Background()
22- old := domain.IncarnationID{1}
23- replacement := domain.IncarnationID{2}
24- record := domain.CatalogueRecord{
25- Name: "broken", IncarnationID: old, RecoveryState: domain.RecoveryDegraded,
26- Committed: &domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}, DegradedReason: "corrupt",
27- }
28- descriptor := domain.QuarantineDescriptor{
29- OldRecord: record, OldIncarnation: old, ReplacementIncarnation: replacement,
30- SessionName: record.Name, Reason: "explicit discard",
31- }
32-
33- t.Run("descriptor is immutable and idempotent", func(t *testing.T) {
34- repo := NewRepository(privateDir(t))
35- if err := repo.SaveQuarantineDescriptor(ctx, descriptor); err != nil {
36- t.Fatal(err)
37- }
38- if err := repo.SaveQuarantineDescriptor(ctx, descriptor); err != nil {
39- t.Fatalf("idempotent retry: %v", err)
40- }
41- path := filepath.Join(repo.dir, "quarantine", old.String(), "record")
42- encoded, err := os.ReadFile(path)
43- if err != nil {
44- t.Fatal(err)
45- }
46- got, err := decodeQuarantineDescriptor(encoded)
47- if err != nil || !reflect.DeepEqual(got, descriptor) {
48- t.Fatalf("descriptor round trip = %#v, %v", got, err)
49- }
50- changed := descriptor
51- changed.Reason = "different"
52- if err := repo.SaveQuarantineDescriptor(ctx, changed); err == nil {
53- t.Fatal("descriptor overwrite accepted")
54- }
55- if err := os.WriteFile(path, append(encoded, 0), 0o600); err != nil {
56- t.Fatal(err)
57- }
58- if err := repo.SaveQuarantineDescriptor(ctx, descriptor); err == nil {
59- t.Fatal("malformed existing descriptor accepted")
60- }
61- })
62-
63- for _, boundary := range []string{"descriptor-file-sync", "descriptor-install", "descriptor-directory-sync"} {
64- t.Run(boundary, func(t *testing.T) {
65- repo := NewRepository(privateDir(t))
66- dir := filepath.Join(repo.dir, "quarantine", old.String())
67- if err := repo.ensurePrivateDirectory(dir); err != nil {
68- t.Fatal(err)
69- }
70- injected := errors.New("injected descriptor durability failure")
71- switch boundary {
72- case "descriptor-file-sync":
73- repo.hooks.syncFile = func(string) error { return injected }
74- case "descriptor-install":
75- repo.hooks.installImmutable = func(string) error { return injected }
76- case "descriptor-directory-sync":
77- repo.hooks.syncDirectory = func(path string) error {
78- if path == dir {
79- return injected
80- }
81- return nil
82- }
83- }
84- if err := repo.SaveQuarantineDescriptor(ctx, descriptor); !errors.Is(err, injected) {
85- t.Fatalf("SaveQuarantineDescriptor() error = %v", err)
86- }
87- repo.hooks = repositoryHooks{}
88- if err := repo.SaveQuarantineDescriptor(ctx, descriptor); err != nil {
89- t.Fatalf("retry: %v", err)
90- }
91- })
92- }
93-
94- for _, boundary := range []string{"rename", "source-directory-sync", "destination-directory-sync"} {
95- t.Run(boundary, func(t *testing.T) {
96- repo := NewRepository(privateDir(t))
97- source := repo.sessionPath(old)
98- destinationDir := filepath.Join(repo.dir, "quarantine", old.String())
99- if err := repo.ensurePrivateDirectory(source); err != nil {
100- t.Fatal(err)
101- }
102- if err := repo.ensurePrivateDirectory(destinationDir); err != nil {
103- t.Fatal(err)
104- }
105- if err := os.WriteFile(filepath.Join(source, "payload"), []byte("uncertain"), 0o600); err != nil {
106- t.Fatal(err)
107- }
108- injected := errors.New("injected quarantine durability failure")
109- switch boundary {
110- case "rename":
111- repo.hooks.rename = func(string) error { return injected }
112- case "source-directory-sync":
113- repo.hooks.syncDirectory = func(path string) error {
114- if path == filepath.Dir(source) {
115- return injected
116- }
117- return nil
118- }
119- case "destination-directory-sync":
120- repo.hooks.syncDirectory = func(path string) error {
121- if path == destinationDir {
122- return injected
123- }
124- return nil
125- }
126- }
127- if err := repo.QuarantineIncarnation(ctx, old); !errors.Is(err, injected) {
128- t.Fatalf("QuarantineIncarnation() error = %v", err)
129- }
130- repo.hooks = repositoryHooks{}
131- if err := repo.QuarantineIncarnation(ctx, old); err != nil {
132- t.Fatalf("retry: %v", err)
133- }
134- got, err := os.ReadFile(filepath.Join(destinationDir, "snapshot", "payload"))
135- if err != nil || string(got) != "uncertain" {
136- t.Fatalf("quarantined payload = %q, %v", got, err)
137- }
138- })
139- }
140-
141- t.Run("quarantine never removes source data", func(t *testing.T) {
142- repo := NewRepository(privateDir(t))
143- source := repo.sessionPath(old)
144- if err := repo.ensurePrivateDirectory(source); err != nil {
145- t.Fatal(err)
146- }
147- payload := filepath.Join(source, "payload")
148- if err := os.WriteFile(payload, []byte("uncertain"), 0o600); err != nil {
149- t.Fatal(err)
150- }
151- if err := repo.SaveQuarantineDescriptor(ctx, descriptor); err != nil {
152- t.Fatal(err)
153- }
154- if err := repo.QuarantineIncarnation(ctx, old); err != nil {
155- t.Fatal(err)
156- }
157- target := filepath.Join(repo.dir, "quarantine", old.String(), "snapshot", "payload")
158- got, err := os.ReadFile(target)
159- if err != nil || string(got) != "uncertain" {
160- t.Fatalf("quarantined payload = %q, %v", got, err)
161- }
162- if err := repo.QuarantineIncarnation(ctx, old); err != nil {
163- t.Fatalf("idempotent retry: %v", err)
164- }
165- })
166-}
167-
168-func TestQuarantineDeletionSourcesSerializesConcurrentPublication(t *testing.T) {
169- repo := NewRepository(privateDir(t))
170- id := domain.IncarnationID{9}
171- tombstone := domain.DeletionTombstone{Name: "work", IncarnationID: id}
172- publication := incarnationPublication(t, id, "work", 1, nil)
173- if err := repo.Publish(context.Background(), publication); err != nil {
174- t.Fatal(err)
175- }
176-
177- quarantined := make(chan struct{})
178- release := make(chan struct{})
179- repo.hooks.afterDeletionIncarnationQuarantine = func() error {
180- close(quarantined)
181- <-release
182- return nil
183- }
184- quarantineDone := make(chan error, 1)
185- go func() { quarantineDone <- repo.QuarantineDeletionSources(context.Background(), tombstone, false) }()
186- <-quarantined
187-
188- publishAttempted := make(chan struct{})
189- repo.hooks.beforeSessionLock = func(string) { close(publishAttempted) }
190- publishDone := make(chan error, 1)
191- go func() { publishDone <- repo.Publish(context.Background(), publication) }()
192- select {
193- case <-publishAttempted:
194- case <-time.After(time.Second):
195- t.Fatal("Publish did not attempt the incarnation lock")
196- }
197- select {
198- case err := <-publishDone:
199- t.Fatalf("Publish completed while quarantine held the incarnation lock: %v", err)
200- case <-time.After(25 * time.Millisecond):
201- }
202- close(release)
203- if err := <-quarantineDone; err != nil {
204- t.Fatal(err)
205- }
206- if err := <-publishDone; err != nil {
207- t.Fatal(err)
208- }
209- if _, err := os.Stat(repo.headPath(id)); err != nil {
210- t.Fatalf("republished HEAD: %v", err)
211- }
212- if _, err := os.Stat(filepath.Join(repo.dir, "quarantine", id.String(), "snapshot", repositoryHead)); err != nil {
213- t.Fatalf("quarantined HEAD: %v", err)
214- }
215-}
216-
217-func TestDeletionTombstoneCodec(t *testing.T) {
218- want := domain.DeletionTombstone{Name: "work", IncarnationID: domain.IncarnationID{1}}
219- encoded, err := encodeDeletionTombstone(want)
220- if err != nil {
221- t.Fatal(err)
222- }
223- if got := hex.EncodeToString(encoded); got != "5645564400010004776f726b01000000000000000000000000000000" {
224- t.Fatalf("encoded tombstone = %s", got)
225- }
226- got, err := decodeDeletionTombstone(encoded)
227- if err != nil {
228- t.Fatal(err)
229- }
230- if got != want {
231- t.Fatalf("round trip = %#v", got)
232- }
233- for n := range len(encoded) {
234- if _, err := decodeDeletionTombstone(encoded[:n]); err == nil {
235- t.Fatalf("prefix %d accepted", n)
236- }
237- }
238- if _, err := decodeDeletionTombstone(append(encoded, 0)); err == nil {
239- t.Fatal("trailing byte accepted")
240- }
241- for _, test := range []struct {
242- name string
243- mutate func([]byte) []byte
244- }{
245- {name: "wrong magic", mutate: func(data []byte) []byte { data[0] ^= 0xff; return data }},
246- {name: "wrong version", mutate: func(data []byte) []byte { data[5]++; return data }},
247- {name: "invalid name", mutate: func(data []byte) []byte { copy(data[8:12], "../x"); return data }},
248- {name: "zero ID", mutate: func(data []byte) []byte { clear(data[len(data)-16:]); return data }},
249- } {
250- t.Run(test.name, func(t *testing.T) {
251- malformed := test.mutate(append([]byte(nil), encoded...))
252- if _, err := decodeDeletionTombstone(malformed); err == nil {
253- t.Fatal("malformed tombstone accepted")
254- }
255- })
256- }
257-
258- for _, boundary := range []string{"file-sync", "install", "directory-sync"} {
259- t.Run(boundary, func(t *testing.T) {
260- repo := NewRepository(privateDir(t))
261- dir := filepath.Join(repo.dir, deletionTombstonesDir)
262- if err := repo.ensurePrivateDirectory(dir); err != nil {
263- t.Fatal(err)
264- }
265- injected := errors.New("injected tombstone persistence failure")
266- switch boundary {
267- case "file-sync":
268- repo.hooks.syncFile = func(string) error { return injected }
269- case "install":
270- repo.hooks.installImmutable = func(string) error { return injected }
271- case "directory-sync":
272- repo.hooks.syncDirectory = func(path string) error {
273- if path == dir {
274- return injected
275- }
276- return nil
277- }
278- }
279- if err := repo.WriteDeletionTombstone(context.Background(), want); !errors.Is(err, injected) {
280- t.Fatalf("WriteDeletionTombstone() error = %v", err)
281- }
282- repo.hooks = repositoryHooks{}
283- if err := repo.WriteDeletionTombstone(context.Background(), want); err != nil {
284- t.Fatalf("retry: %v", err)
285- }
286- })
287- }
288-}
289-
290-func TestDeletionTombstoneListingResumesAcrossRepositoryRecreation(t *testing.T) {
291- dir := privateDir(t)
292- ctx := context.Background()
293- want := []domain.DeletionTombstone{
294- {Name: "first", IncarnationID: domain.IncarnationID{15: 1}},
295- {Name: "second", IncarnationID: domain.IncarnationID{15: 2}},
296- {Name: "third", IncarnationID: domain.IncarnationID{15: 3}},
297- }
298- repo := NewRepository(dir)
299- for _, index := range []int{2, 0, 1} {
300- if err := repo.WriteDeletionTombstone(ctx, want[index]); err != nil {
301- t.Fatal(err)
302- }
303- }
304-
305- var got []domain.DeletionTombstone
306- cursor := ports.DeletionTombstoneCursor{}
307- reads := 0
308- for calls := 0; ; calls++ {
309- if calls == 32 {
310- t.Fatal("listing did not complete")
311- }
312- // A new Repository has no in-memory scan state. Every continuation must
313- // therefore carry everything needed to resume the bounded directory scan.
314- repo = NewRepository(dir)
315- repo.hooks.beforeDeletionTombstoneRead = func(string) { reads++ }
316- page, err := repo.ListDeletionTombstones(ctx, cursor, ports.MaintenanceBudget{Entries: 1, Bytes: 1024})
317- if err != nil {
318- t.Fatal(err)
319- }
320- got = append(got, page.Tombstones...)
321- if page.Done {
322- break
323- }
324- if page.Next.After == "" || page.Next.After == cursor.After {
325- t.Fatalf("continuation did not advance: before=%q after=%q", cursor.After, page.Next.After)
326- }
327- cursor = page.Next
328- }
329- if !slices.Equal(got, want) {
330- t.Fatalf("tombstones = %#v, want %#v", got, want)
331- }
332- if reads != len(want) {
333- t.Fatalf("tombstone object reads = %d, want %d", reads, len(want))
334- }
335-}
336-
337-func TestDeletionTombstoneListingBatchesSortedCandidatesWithinScanBudget(t *testing.T) {
338- dir := privateDir(t)
339- repo := NewRepository(dir)
340- ctx := context.Background()
341- want := make([]domain.DeletionTombstone, 5)
342- for i := range want {
343- want[i] = domain.DeletionTombstone{Name: fmt.Sprintf("session-%d", i), IncarnationID: domain.IncarnationID{15: byte(i + 1)}}
344- }
345- for _, i := range []int{4, 1, 3, 0, 2} {
346- if err := repo.WriteDeletionTombstone(ctx, want[i]); err != nil {
347- t.Fatal(err)
348- }
349- }
350-
351- cursor := ports.DeletionTombstoneCursor{}
352- var got []domain.DeletionTombstone
353- reads := 0
354- for calls := range 64 {
355- repo = NewRepository(dir)
356- repo.hooks.beforeDeletionTombstoneRead = func(string) { reads++ }
357- entryBudget := uint64(2)
358- if calls > 0 {
359- entryBudget = 1 // A resumed caller may reduce its per-pass budget.
360- }
361- page, err := repo.ListDeletionTombstones(ctx, cursor, ports.MaintenanceBudget{Entries: entryBudget, Bytes: 4096})
362- if err != nil {
363- t.Fatal(err)
364- }
365- if uint64(len(page.Tombstones)) > entryBudget {
366- t.Fatalf("page %d returned %d tombstones over entry budget %d", calls, len(page.Tombstones), entryBudget)
367- }
368- got = append(got, page.Tombstones...)
369- if page.Done {
370- break
371- }
372- cursor = page.Next
373- }
374- if !slices.Equal(got, want) {
375- t.Fatalf("tombstones = %#v, want %#v", got, want)
376- }
377- if reads != len(want) {
378- t.Fatalf("payload reads = %d, want %d", reads, len(want))
379- }
380-}
381-
382-func TestMalformedDeletionTombstone(t *testing.T) {
383- dir := privateDir(t)
384- repo := NewRepository(dir)
385- tombstone := domain.DeletionTombstone{Name: "work", IncarnationID: domain.IncarnationID{7}}
386- if err := repo.WriteDeletionTombstone(context.Background(), tombstone); err != nil {
387- t.Fatal(err)
388- }
389- path := repo.deletionTombstonePath(tombstone.IncarnationID)
390- if err := os.WriteFile(path, []byte("malformed"), 0o600); err != nil {
391- t.Fatal(err)
392- }
393- if _, err := repo.ListDeletionTombstones(context.Background(), ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 1024}); err == nil {
394- t.Fatal("malformed tombstone was skipped")
395- }
396- if _, err := os.Stat(path); err != nil {
397- t.Fatalf("malformed tombstone was removed: %v", err)
398- }
399-}
400-
401-func TestDeletionTombstoneListingRejectsMalformedContinuation(t *testing.T) {
402- repo := NewRepository(privateDir(t))
403- ctx := context.Background()
404- after := domain.IncarnationID{15: 2}.String() + ".tombstone"
405-
406- tests := []struct {
407- name string
408- cursor ports.DeletionTombstoneCursor
409- }{
410- {name: "repository-memory token", cursor: ports.DeletionTombstoneCursor{After: "@1"}},
411- {name: "non-advancing empty state", cursor: rawDeletionListingCursor(t, `{}`)},
412- {name: "non-canonical key", cursor: rawDeletionListingCursor(t, `{"after":"ABC.tombstone"}`)},
413- {name: "candidate does not advance", cursor: rawDeletionListingCursor(t, `{"after":"`+after+`","candidate":"`+after+`","found":1}`)},
414- {name: "unknown field", cursor: rawDeletionListingCursor(t, `{"unknown":1}`)},
415- }
416- for _, test := range tests {
417- t.Run(test.name, func(t *testing.T) {
418- if _, err := repo.ListDeletionTombstones(ctx, test.cursor, ports.MaintenanceBudget{Entries: 1, Bytes: 1024}); err == nil {
419- t.Fatal("malformed continuation accepted")
420- }
421- })
422- }
423-}
424-
425-func rawDeletionListingCursor(t *testing.T, state string) ports.DeletionTombstoneCursor {
426- t.Helper()
427- return ports.DeletionTombstoneCursor{After: deletionListingCursorPrefix + base64.RawURLEncoding.EncodeToString([]byte(state))}
428-}
429-
430-func TestDeletionTombstoneListing(t *testing.T) {
431- repo := NewRepository(privateDir(t))
432- ctx := context.Background()
433- first := domain.DeletionTombstone{Name: "a", IncarnationID: domain.IncarnationID{1}}
434- second := domain.DeletionTombstone{Name: "b", IncarnationID: domain.IncarnationID{2}}
435- if err := repo.WriteDeletionTombstone(ctx, second); err != nil {
436- t.Fatal(err)
437- }
438- if err := repo.WriteDeletionTombstone(ctx, first); err != nil {
439- t.Fatal(err)
440- }
441- page, err := repo.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 3, Bytes: 1024})
442- if err != nil {
443- t.Fatal(err)
444- }
445- if !page.Done || !slices.Equal(page.Tombstones, []domain.DeletionTombstone{first, second}) || page.Next.After == "" {
446- t.Fatalf("batched page = %#v", page)
447- }
448- if _, err := repo.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 3, Bytes: 1}); err == nil {
449- t.Fatal("non-advancing budget accepted")
450- }
451-
452- reads := 0
453- repo.hooks.beforeDeletionTombstoneRead = func(string) { reads++ }
454- page, err = repo.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 1, Bytes: 1024})
455- if err != nil || reads != 0 || len(page.Tombstones) != 0 || page.Next.After == "" {
456- t.Fatalf("bounded scan read %d objects: page=%#v err=%v", reads, page, err)
457- }
458-
459- injected := errors.New("injected directory sync failure")
460- tombstoneDir := filepath.Join(repo.dir, deletionTombstonesDir)
461- repo.hooks.syncDirectory = func(path string) error {
462- if path == tombstoneDir {
463- return injected
464- }
465- return nil
466- }
467- if err := repo.DeleteDeletionTombstone(ctx, first.IncarnationID); !errors.Is(err, injected) {
468- t.Fatalf("DeleteDeletionTombstone() error = %v", err)
469- }
470- repo.hooks = repositoryHooks{}
471- if err := repo.DeleteDeletionTombstone(ctx, first.IncarnationID); err != nil {
472- t.Fatalf("DeleteDeletionTombstone() retry = %v", err)
473- }
474-}
M internal/adapters/snapshot/maintenance_gc.go +0 -188
@@ -755,194 +755,6 @@ func (r *Repository) validateMaintenanceCheckpoint(ctx context.Context, id domai
755755 return validatedMaintenanceCheckpoint{refs: refs, parent: manifest.ParentCheckpoint}, consumed, true, nil
756756 }
757757
758-// Reconcile scans top-level incarnation namespaces from an external seek
759-// cookie. Checkpoint-chain work receives a fresh budget for every directory.
760-func (r *Repository) Reconcile(ctx context.Context, records []domain.CatalogueRecord, cursor ports.ReconcileCursor, budget ports.MaintenanceBudget) (nextCursor ports.ReconcileCursor, findings []ports.ReconcileFinding, retErr error) {
761- requested := budget
762- defer func() {
763- consumed := ports.MaintenanceBudget{}
764- for _, finding := range findings {
765- consumed.Entries += finding.Consumed.Entries
766- consumed.Bytes += finding.Consumed.Bytes
767- r.log.Info("snapshot_reconciliation_progress",
768- "incarnation", finding.Candidate.IncarnationID.String(),
769- "action", finding.Kind,
770- "status", finding.Status,
771- "consumed_entries", finding.Consumed.Entries,
772- "consumed_bytes", finding.Consumed.Bytes,
773- "cursor", finding.Cursor.DirectoryCookie,
774- )
775- }
776- r.log.Info("snapshot_reconciliation_complete",
777- "action", "scan",
778- "cursor", nextCursor.DirectoryCookie,
779- "requested_entries", requested.Entries,
780- "requested_bytes", requested.Bytes,
781- "consumed_entries", consumed.Entries,
782- "consumed_bytes", consumed.Bytes,
783- "done", nextCursor.DirectoryCookie == 0 && retErr == nil,
784- "err", retErr,
785- )
786- }()
787- if err := ctx.Err(); err != nil {
788- return cursor, nil, err
789- }
790- if budget.Entries == 0 || budget.Bytes == 0 {
791- return cursor, nil, nil
792- }
793- offset := int64(cursor.DirectoryCookie)
794- if offset < 0 {
795- return cursor, nil, errors.New("snapshot: invalid reconciliation cursor")
796- }
797- byIncarnation := make(map[domain.IncarnationID]domain.CatalogueRecord, len(records))
798- for _, record := range records {
799- byIncarnation[record.IncarnationID] = record
800- }
801-
802- root := filepath.Join(r.dir, repositorySessionsDir)
803- state := &maintenanceCursor{offset: offset}
804- findingCapacity := maintenanceBatch
805- if budget.Entries < maintenanceBatch {
806- findingCapacity = int(budget.Entries)
807- }
808- findings = make([]ports.ReconcileFinding, 0, findingCapacity)
809- stepConsumed := ports.MaintenanceBudget{}
810- resumeCursor := cursor
811- for scanned := uint64(0); scanned < budget.Entries && stepConsumed.Entries < budget.Entries; scanned++ {
812- entries, done, err := r.readMaintenanceDirent(root, 1, state)
813- if errors.Is(err, os.ErrNotExist) {
814- return ports.ReconcileCursor{}, findings, nil
815- }
816- if err != nil {
817- return cursor, nil, err
818- }
819- if len(entries) == 0 {
820- if done {
821- return ports.ReconcileCursor{}, findings, nil
822- }
823- continue
824- }
825- entry := entries[0]
826- nextCursor = ports.ReconcileCursor{DirectoryCookie: uint64(entry.cookie)}
827- if !entry.isDir || isQuarantine(entry.name) {
828- resumeCursor = nextCursor
829- if done {
830- return ports.ReconcileCursor{}, findings, nil
831- }
832- continue
833- }
834- findingIndex := len(findings)
835- consumedBeforeCandidate := stepConsumed
836- var id domain.IncarnationID
837- if err := id.UnmarshalText([]byte(entry.name)); err != nil {
838- findings = append(findings, ports.ReconcileFinding{Kind: ports.ReconcileUnknownIncarnation, Status: ports.ReconcileQuarantined, Candidate: ports.ReconcileCandidate{Name: entry.name}, Cursor: nextCursor, Consumed: ports.MaintenanceBudget{Entries: 1}})
839- } else if record, ok := byIncarnation[id]; !ok {
840- findings = append(findings, ports.ReconcileFinding{Kind: ports.ReconcileUnknownIncarnation, Status: ports.ReconcileQuarantined, Candidate: ports.ReconcileCandidate{Name: entry.name, IncarnationID: id}, Cursor: nextCursor, Consumed: ports.MaintenanceBudget{Entries: 1}})
841- } else {
842- remaining := ports.MaintenanceBudget{Entries: budget.Entries - stepConsumed.Entries, Bytes: budget.Bytes - stepConsumed.Bytes}
843- finding := r.reconcileIncarnation(ctx, record, remaining)
844- finding.Cursor = nextCursor
845- findings = append(findings, finding)
846- }
847- if len(findings) > findingIndex {
848- stepConsumed.Entries += findings[findingIndex].Consumed.Entries
849- stepConsumed.Bytes += findings[findingIndex].Consumed.Bytes
850- if findings[findingIndex].Status == ports.ReconcileBudgetExhausted {
851- // If a fresh per-call budget cannot admit this candidate, retrying
852- // the same fixed budget can never succeed. Report the bounded
853- // exhaustion once and deliberately advance so later entries remain
854- // reachable. A candidate reached after earlier work is retried on a
855- // fresh call because it may fit that call's full budget.
856- if consumedBeforeCandidate.Entries == 0 && consumedBeforeCandidate.Bytes == 0 {
857- return nextCursor, findings, nil
858- }
859- return resumeCursor, findings, nil
860- }
861- }
862- resumeCursor = nextCursor
863- if done {
864- return ports.ReconcileCursor{}, findings, nil
865- }
866- }
867- return nextCursor, findings, nil
868-}
869-
870-func (r *Repository) reconcileIncarnation(ctx context.Context, record domain.CatalogueRecord, budget ports.MaintenanceBudget) ports.ReconcileFinding {
871- finding := ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileQuarantined, Candidate: ports.ReconcileCandidate{Name: record.Name, IncarnationID: record.IncarnationID}, Consumed: ports.MaintenanceBudget{Entries: 1}}
872- if record.Committed == nil {
873- return finding
874- }
875- headPath := r.headPath(record.IncarnationID)
876- info, err := r.stat(headPath)
877- if err != nil || info.Size() < 0 {
878- return finding
879- }
880- headBytes := uint64(info.Size())
881- if headBytes > budget.Bytes-finding.Consumed.Bytes {
882- finding.Status = ports.ReconcileBudgetExhausted
883- return finding
884- }
885- finding.Consumed.Bytes += headBytes
886- if hook := r.hooks.beforeMaintenancePayloadRead; hook != nil {
887- hook(headPath)
888- }
889- generation, digest, err := r.readHead(record.IncarnationID)
890- if err != nil {
891- return finding
892- }
893- candidateRef := domain.CheckpointRef{Generation: generation, ManifestDigest: digest}
894- finding.Candidate.Ref = candidateRef
895- if generation <= record.Committed.Generation {
896- finding.Status = ports.ReconcileValidated
897- return finding
898- }
899- finding.Kind = ports.ReconcileForwardOrphan
900-
901- candidate, exhausted, err := r.reconcileLoad(ctx, record, candidateRef, &finding.Consumed, budget)
902- if exhausted {
903- finding.Status = ports.ReconcileBudgetExhausted
904- return finding
905- }
906- if err != nil {
907- return finding
908- }
909- finding.Candidate.Parent = candidate.parent
910- parent := candidate.parent
911- for parent != nil && *parent != *record.Committed {
912- ancestor, exhausted, err := r.reconcileLoad(ctx, record, *parent, &finding.Consumed, budget)
913- if exhausted {
914- finding.Status = ports.ReconcileBudgetExhausted
915- return finding
916- }
917- if err != nil {
918- return finding
919- }
920- finding.AncestorChain = append(finding.AncestorChain, ports.ValidatedCheckpoint{Ref: *parent, Parent: ancestor.parent})
921- parent = ancestor.parent
922- }
923- if parent == nil {
924- return finding
925- }
926- finding.Status = ports.ReconcileValidated
927- return finding
928-}
929-
930-func (r *Repository) reconcileLoad(ctx context.Context, record domain.CatalogueRecord, ref domain.CheckpointRef, consumed *ports.MaintenanceBudget, budget ports.MaintenanceBudget) (validatedMaintenanceCheckpoint, bool, error) {
931- if consumed.Entries >= budget.Entries || consumed.Bytes >= budget.Bytes {
932- return validatedMaintenanceCheckpoint{}, true, nil
933- }
934- validated, bytes, admitted, err := r.validateMaintenanceCheckpoint(ctx, record.IncarnationID, ref, budget.Bytes-consumed.Bytes)
935- consumed.Bytes += bytes
936- if err != nil {
937- return validatedMaintenanceCheckpoint{}, false, err
938- }
939- if !admitted {
940- return validatedMaintenanceCheckpoint{}, true, nil
941- }
942- consumed.Entries++
943- return validated, false, nil
944-}
945-
946758 func retainedReferences(marked map[uint64]manifestMaintenance, conservative bool) map[ports.SnapshotDigest]struct{} {
947759 if conservative {
948760 references := make(map[ports.SnapshotDigest]struct{})
M internal/adapters/snapshot/maintenance_test.go +0 -41
@@ -1220,47 +1220,6 @@ func TestMaintainSessionDistinguishesInitialOversizeFromResumableExhaustion(t *t
12201220 }
12211221 }
12221222
1223-func TestReconcileDoesNotHoldMaintenanceLockDuringPayloadRead(t *testing.T) {
1224- repo := NewRepository(privateDir(t))
1225- publications := publishMaintenanceGenerations(t, repo, "reconcile-lock", 2)
1226- committed := checkpointRefForPublication(publications[0])
1227- record := domain.CatalogueRecord{Name: "reconcile-lock", IncarnationID: publications[0].IncarnationID, RecoveryState: domain.RecoveryHealthy, Committed: &committed}
1228-
1229- entered := make(chan struct{})
1230- release := make(chan struct{})
1231- var releaseOnce sync.Once
1232- defer releaseOnce.Do(func() { close(release) })
1233- var enteredOnce sync.Once
1234- repo.hooks.beforeMaintenancePayloadRead = func(string) {
1235- enteredOnce.Do(func() { close(entered) })
1236- <-release
1237- }
1238- reconcileDone := make(chan error, 1)
1239- go func() {
1240- _, _, err := repo.Reconcile(context.Background(), []domain.CatalogueRecord{record}, ports.ReconcileCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20})
1241- reconcileDone <- err
1242- }()
1243- <-entered
1244-
1245- pinDone := make(chan error, 1)
1246- go func() {
1247- _, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1})
1248- pinDone <- err
1249- }()
1250- select {
1251- case err := <-pinDone:
1252- if err != nil {
1253- t.Fatal(err)
1254- }
1255- case <-time.After(remainingTestTime(t) / 2):
1256- t.Fatal("maintenance lock remained held during reconciliation payload I/O")
1257- }
1258- releaseOnce.Do(func() { close(release) })
1259- if err := <-reconcileDone; err != nil {
1260- t.Fatal(err)
1261- }
1262-}
1263-
12641223 func TestMaintainSessionYieldsOnEmptyNonDoneObjectShardBatch(t *testing.T) {
12651224 repo := NewRepository(privateDir(t))
12661225 publication := publishMaintenanceGenerations(t, repo, "empty-shard-batch", 1)[0]
D internal/adapters/snapshot/reconciliation_test.go +0 -268
@@ -1,268 +0,0 @@
1-package snapshot
2-
3-import (
4- "context"
5- "io"
6- "path/filepath"
7- "syscall"
8- "testing"
9-
10- "github.com/bnema/vev/internal/domain"
11- "github.com/bnema/vev/internal/ports"
12- codec "github.com/bnema/vev/internal/usecase/snapshot"
13- "github.com/bnema/vev/pkg/safedir"
14- "github.com/stretchr/testify/require"
15-)
16-
17-type fakeReconcileDrainDirectory struct {
18- names []string
19- position int
20-}
21-
22-func (d *fakeReconcileDrainDirectory) Seek(offset int64, whence int) (int64, error) {
23- if whence == io.SeekCurrent {
24- return int64(d.position), nil
25- }
26- d.position = int(offset)
27- return offset, nil
28-}
29-
30-func (d *fakeReconcileDrainDirectory) ReadDirent(buffer []byte) (int, error) {
31- if d.position >= len(d.names) {
32- return 0, nil
33- }
34- batch := maintenanceDirentBatch(d.names[d.position:]...)
35- d.position = len(d.names)
36- return copy(buffer, batch), nil
37-}
38-
39-func (d *fakeReconcileDrainDirectory) Close() error { return nil }
40-
41-func configureDrainingReconcileScanner(repo *Repository, names []string) {
42- repo.hooks.openMaintenanceDirectory = func(string) (maintenanceDirectory, error) {
43- return &fakeReconcileDrainDirectory{names: names}, nil
44- }
45- repo.hooks.maintenanceDirentConfig = &maintenanceDirentConfig{
46- drainBuffer: true,
47- cookie: func(file maintenanceDirectory, _ *syscall.Dirent, remaining int) (int64, error) {
48- end, err := file.Seek(0, io.SeekCurrent)
49- return end - int64(remaining), err
50- },
51- }
52-}
53-
54-func TestReconciliationDrainedBatchHasNoSkippedOrDuplicateEntries(t *testing.T) {
55- repo := NewRepository(privateDir(t))
56- sessions := filepath.Join(repo.dir, repositorySessionsDir)
57- names := make([]string, 3)
58- for i := range names {
59- id := domain.IncarnationID{byte(i + 20)}
60- names[i] = id.String()
61- require.NoError(t, safedir.EnsurePrivate(filepath.Join(sessions, names[i])))
62- }
63- configureDrainingReconcileScanner(repo, names)
64-
65- cursor := ports.ReconcileCursor{}
66- var got []string
67- for calls := 0; ; calls++ {
68- next, findings, err := repo.Reconcile(context.Background(), nil, cursor, ports.MaintenanceBudget{Entries: 2, Bytes: 1})
69- require.NoError(t, err)
70- for _, finding := range findings {
71- got = append(got, finding.Candidate.Name)
72- }
73- if next.DirectoryCookie == 0 {
74- break
75- }
76- require.Greater(t, next.DirectoryCookie, cursor.DirectoryCookie)
77- cursor = next
78- require.Less(t, calls, 3)
79- }
80- require.Equal(t, names, got)
81-}
82-
83-func TestReconciliationReadsEachCandidatePayloadOnceWithinBudget(t *testing.T) {
84- repo := NewRepository(privateDir(t))
85- first := publishMaintenanceGenerations(t, repo, "first", 2)
86- second := publishMaintenanceGenerations(t, repo, "second", 2)
87-
88- records := make([]domain.CatalogueRecord, 0, 2)
89- expectedPaths := make(map[string]struct{})
90- for _, publications := range [][]ports.SnapshotPublication{first, second} {
91- committed := checkpointRefForPublication(publications[0])
92- records = append(records, domain.CatalogueRecord{
93- Name: publications[0].Name,
94- IncarnationID: publications[0].IncarnationID,
95- RecoveryState: domain.RecoveryHealthy,
96- Committed: &committed,
97- })
98- candidate := checkpointRefForPublication(publications[1])
99- expectedPaths[repo.headPath(publications[0].IncarnationID)] = struct{}{}
100- expectedPaths[repo.manifestPath(publications[0].IncarnationID, candidate.Generation)] = struct{}{}
101- manifest, err := codec.UnmarshalManifest(publications[1].Manifest)
102- require.NoError(t, err)
103- for digest := range manifestRefs(manifest) {
104- expectedPaths[repo.objectPath(publications[0].IncarnationID, digest)] = struct{}{}
105- }
106- }
107-
108- var budgetBytes uint64
109- for path := range expectedPaths {
110- info, err := repo.stat(path)
111- require.NoError(t, err)
112- budgetBytes += uint64(info.Size())
113- }
114- reads := make(map[string]int)
115- var actualBytes uint64
116- repo.hooks.beforePayloadRead = func(path string) {
117- info, err := repo.stat(path)
118- require.NoError(t, err)
119- reads[path]++
120- actualBytes += uint64(info.Size())
121- }
122-
123- next, findings, err := repo.Reconcile(context.Background(), records, ports.ReconcileCursor{}, ports.MaintenanceBudget{Entries: 16, Bytes: budgetBytes})
124- require.NoError(t, err)
125- require.Zero(t, next)
126- require.Len(t, findings, 2)
127- var consumedBytes uint64
128- for _, finding := range findings {
129- require.Equal(t, ports.ReconcileForwardOrphan, finding.Kind)
130- require.Equal(t, ports.ReconcileValidated, finding.Status)
131- consumedBytes += finding.Consumed.Bytes
132- }
133- require.Equal(t, consumedBytes, actualBytes, "every payload read must be charged")
134- require.LessOrEqual(t, actualBytes, budgetBytes)
135- require.Equal(t, expectedPaths, func() map[string]struct{} {
136- got := make(map[string]struct{}, len(reads))
137- for path := range reads {
138- got[path] = struct{}{}
139- }
140- return got
141- }())
142- for path, count := range reads {
143- require.Equalf(t, 1, count, "payload %q was reread", path)
144- }
145-
146- reads = make(map[string]int)
147- actualBytes = 0
148- limitedBudget := budgetBytes - 1
149- _, findings, err = repo.Reconcile(context.Background(), records, ports.ReconcileCursor{}, ports.MaintenanceBudget{Entries: 16, Bytes: limitedBudget})
150- require.NoError(t, err)
151- consumedBytes = 0
152- exhausted := false
153- for _, finding := range findings {
154- consumedBytes += finding.Consumed.Bytes
155- exhausted = exhausted || finding.Status == ports.ReconcileBudgetExhausted
156- }
157- require.True(t, exhausted, "reconciliation must yield rather than read past its budget")
158- require.Equal(t, consumedBytes, actualBytes)
159- require.LessOrEqual(t, actualBytes, limitedBudget)
160- for path, count := range reads {
161- require.Equalf(t, 1, count, "payload %q was reread in a budget-limited step", path)
162- }
163-}
164-
165-func TestReconciliationIntrinsicallyOversizedCandidateMakesFixedBudgetProgress(t *testing.T) {
166- tests := []struct {
167- name string
168- budget ports.MaintenanceBudget
169- }{
170- {name: "entry cost", budget: ports.MaintenanceBudget{Entries: 1, Bytes: 8 << 20}},
171- {name: "byte cost", budget: ports.MaintenanceBudget{Entries: 8, Bytes: 1}},
172- }
173- for _, tc := range tests {
174- t.Run(tc.name, func(t *testing.T) {
175- repo := NewRepository(privateDir(t))
176- publications := publishMaintenanceGenerations(t, repo, "work", 2)
177- committed := checkpointRefForPublication(publications[0])
178- record := domain.CatalogueRecord{Name: "work", IncarnationID: publications[0].IncarnationID, RecoveryState: domain.RecoveryHealthy, Committed: &committed}
179- unknown := domain.IncarnationID{99}
180- require.NoError(t, safedir.EnsurePrivate(filepath.Join(repo.dir, repositorySessionsDir, unknown.String())))
181- configureDrainingReconcileScanner(repo, []string{record.IncarnationID.String(), unknown.String()})
182-
183- next, findings, err := repo.Reconcile(context.Background(), []domain.CatalogueRecord{record}, ports.ReconcileCursor{}, tc.budget)
184- require.NoError(t, err)
185- require.Equal(t, uint64(1), next.DirectoryCookie)
186- require.Len(t, findings, 1)
187- require.Equal(t, ports.ReconcileBudgetExhausted, findings[0].Status)
188- require.LessOrEqual(t, findings[0].Consumed.Entries, tc.budget.Entries)
189- require.LessOrEqual(t, findings[0].Consumed.Bytes, tc.budget.Bytes)
190-
191- later, findings, err := repo.Reconcile(context.Background(), []domain.CatalogueRecord{record}, next, tc.budget)
192- require.NoError(t, err)
193- require.Len(t, findings, 1)
194- require.Equal(t, unknown, findings[0].Candidate.IncarnationID)
195- require.Equal(t, ports.ReconcileQuarantined, findings[0].Status)
196- if later.DirectoryCookie == 0 {
197- return
198- }
199- require.Greater(t, later.DirectoryCookie, next.DirectoryCookie)
200-
201- done, findings, err := repo.Reconcile(context.Background(), []domain.CatalogueRecord{record}, later, tc.budget)
202- require.NoError(t, err)
203- require.Zero(t, done)
204- require.Empty(t, findings, "oversized candidate must not be retried")
205- })
206- }
207-}
208-
209-func TestReconciliationCursor(t *testing.T) {
210- repo := NewRepository(privateDir(t))
211- publications := publishMaintenanceGenerations(t, repo, "work", 2)
212- committed := checkpointRefForPublication(publications[0])
213- record := domain.CatalogueRecord{
214- Name: "work",
215- IncarnationID: publications[0].IncarnationID,
216- RecoveryState: domain.RecoveryHealthy,
217- Committed: &committed,
218- }
219-
220- sessions := filepath.Join(repo.dir, repositorySessionsDir)
221- for i := byte(20); i < 26; i++ {
222- id := domain.IncarnationID{i}
223- require.NoError(t, safedir.EnsurePrivate(filepath.Join(sessions, id.String())))
224- }
225-
226- cursor := ports.ReconcileCursor{}
227- var findings []ports.ReconcileFinding
228- calls := 0
229- for {
230- next, page, err := repo.Reconcile(context.Background(), []domain.CatalogueRecord{record}, cursor, ports.MaintenanceBudget{Entries: 2, Bytes: 8 << 20})
231- require.NoError(t, err)
232- for _, finding := range page {
233- require.LessOrEqual(t, finding.Consumed.Entries, uint64(2))
234- require.LessOrEqual(t, finding.Consumed.Bytes, uint64(8<<20))
235- }
236- findings = append(findings, page...)
237- calls++
238- if next.DirectoryCookie == 0 {
239- break
240- }
241- require.NotEqual(t, cursor, next, "reconciliation cursor must advance")
242- cursor = next
243- require.Less(t, calls, 10)
244- }
245- require.Greater(t, calls, 1, "top-level traversal must resume across bounded calls")
246-
247- unknown := make(map[domain.IncarnationID]struct{})
248- adoptable := 0
249- for _, finding := range findings {
250- switch finding.Kind {
251- case ports.ReconcileUnknownIncarnation:
252- require.Equal(t, ports.ReconcileQuarantined, finding.Status)
253- _, duplicate := unknown[finding.Candidate.IncarnationID]
254- require.False(t, duplicate, "unknown incarnation was returned twice")
255- unknown[finding.Candidate.IncarnationID] = struct{}{}
256- case ports.ReconcileForwardOrphan:
257- if finding.Status == ports.ReconcileBudgetExhausted {
258- continue
259- }
260- require.Equal(t, ports.ReconcileValidated, finding.Status)
261- require.Equal(t, uint64(2), finding.Candidate.Ref.Generation)
262- require.Equal(t, committed, *finding.Candidate.Parent)
263- adoptable++
264- }
265- }
266- require.Len(t, unknown, 6)
267- require.Equal(t, 1, adoptable, "candidate must reach a terminal result exactly once")
268-}
D internal/adapters/snapshot/recovery_crash_test.go +0 -149
@@ -1,149 +0,0 @@
1-package snapshot
2-
3-import (
4- "bytes"
5- "context"
6- "errors"
7- "os"
8- "path/filepath"
9- "testing"
10-
11- "github.com/bnema/vev/internal/adapters/recoveryfs"
12- "github.com/bnema/vev/internal/domain"
13- "github.com/bnema/vev/internal/persist"
14- "github.com/bnema/vev/internal/ports"
15- "github.com/bnema/vev/internal/usecase/recovery"
16- "github.com/bnema/vev/pkg/safedir"
17- "github.com/stretchr/testify/require"
18-)
19-
20-var errDeletionBoundaryCrash = errors.New("simulated process interruption")
21-
22-type recoveryCrashHarness struct {
23- stateDir string
24- snapshotDir string
25-}
26-
27-func newRecoveryCrashHarness(t *testing.T) recoveryCrashHarness {
28- t.Helper()
29- stateDir := t.TempDir()
30- require.NoError(t, os.Chmod(stateDir, 0o700))
31- return recoveryCrashHarness{stateDir: stateDir, snapshotDir: filepath.Join(stateDir, "snapshots")}
32-}
33-
34-func (h recoveryCrashHarness) open(t *testing.T) (*persist.Persister, *Repository, *recoveryfs.Journal) {
35- t.Helper()
36- store, err := persist.OpenStore(persist.StorePath(h.stateDir))
37- require.NoError(t, err)
38- return persist.New(store), NewRepository(h.snapshotDir), recoveryfs.New(h.stateDir)
39-}
40-
41-func TestDeletionRestartsAtInternalQuarantineBoundaries(t *testing.T) {
42- ctx := context.Background()
43- for _, tc := range []struct {
44- name string
45- inject func(*Repository)
46- }{
47- {name: "after incarnation namespace", inject: func(repository *Repository) {
48- repository.hooks.afterDeletionIncarnationQuarantine = func() error { return errDeletionBoundaryCrash }
49- }},
50- {name: "after legacy name", inject: func(repository *Repository) {
51- repository.hooks.afterDeletionLegacyQuarantine = func() error { return errDeletionBoundaryCrash }
52- }},
53- } {
54- t.Run(tc.name, func(t *testing.T) {
55- h := newRecoveryCrashHarness(t)
56- catalogue, repository, journal := h.open(t)
57- oldID := domain.IncarnationID{1}
58- record := domain.CatalogueRecord{Name: "work", IncarnationID: oldID, RecoveryState: domain.RecoveryFresh}
59- require.NoError(t, catalogue.Create(record))
60- sessionPath := repository.sessionPath(oldID)
61- require.NoError(t, safedir.EnsurePrivate(sessionPath))
62- require.NoError(t, os.WriteFile(filepath.Join(sessionPath, "durable"), []byte("old namespace"), 0o600))
63- require.NoError(t, os.WriteFile(filepath.Join(h.snapshotDir, filenameForName(record.Name)), []byte("old legacy"), 0o600))
64- tc.inject(repository)
65-
66- coordinator := recovery.NewCoordinator(catalogue, repository, journal, nil)
67- require.ErrorIs(t, coordinator.Delete(ctx, record.Name), errDeletionBoundaryCrash)
68- require.NoError(t, catalogue.Close())
69-
70- // Simulate a new process: none of the adapter or coordinator state survives.
71- catalogue, repository, journal = h.open(t)
72- require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx))
73- _, exists, err := catalogue.Record(record.Name)
74- require.NoError(t, err)
75- require.False(t, exists)
76- require.NoDirExists(t, repository.sessionPath(oldID))
77- require.Equal(t, []byte("old namespace"), mustReadCrashFile(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot", "durable")))
78- require.Equal(t, []byte("old legacy"), mustReadCrashFile(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "legacy.snap")))
79- page, err := repository.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 16, Bytes: 4096})
80- require.NoError(t, err)
81- require.Empty(t, page.Tombstones)
82-
83- // Reusing the name after roll-forward must be safe on another restart.
84- newID := domain.IncarnationID{2}
85- created, err := recovery.NewCoordinator(catalogue, repository, journal, bytes.NewReader(newID[:])).Create(ctx, domain.CatalogueRecord{Name: record.Name})
86- require.NoError(t, err)
87- require.Equal(t, newID, created.IncarnationID)
88- newPath := repository.sessionPath(newID)
89- require.NoError(t, safedir.EnsurePrivate(newPath))
90- require.NoError(t, os.WriteFile(filepath.Join(newPath, "durable"), []byte("new namespace"), 0o600))
91- require.NoError(t, catalogue.Close())
92-
93- catalogue, repository, journal = h.open(t)
94- require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx))
95- got, exists, err := catalogue.Record(record.Name)
96- require.NoError(t, err)
97- require.True(t, exists)
98- require.Equal(t, newID, got.IncarnationID)
99- require.Equal(t, []byte("new namespace"), mustReadCrashFile(t, filepath.Join(repository.sessionPath(newID), "durable")))
100- require.NoError(t, catalogue.Close())
101- })
102- }
103-}
104-
105-type rejectCreateCatalogue struct {
106- ports.Catalogue
107-}
108-
109-func (c rejectCreateCatalogue) Create(domain.CatalogueRecord) error {
110- return errDeletionBoundaryCrash
111-}
112-
113-func TestCreatePreCommitRestartLeavesNoAdoptableNamespace(t *testing.T) {
114- ctx := context.Background()
115- h := newRecoveryCrashHarness(t)
116- catalogue, repository, journal := h.open(t)
117- interruptedID := domain.IncarnationID{3}
118- _, err := recovery.NewCoordinator(rejectCreateCatalogue{Catalogue: catalogue}, repository, journal, bytes.NewReader(interruptedID[:])).Create(ctx, domain.CatalogueRecord{Name: "work"})
119- require.ErrorIs(t, err, errDeletionBoundaryCrash)
120- records, err := catalogue.Records()
121- require.NoError(t, err)
122- require.Empty(t, records)
123- require.NoDirExists(t, filepath.Join(h.snapshotDir, repositorySessionsDir))
124- require.NoError(t, catalogue.Close())
125-
126- catalogue, repository, journal = h.open(t)
127- require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx))
128- records, err = catalogue.Records()
129- require.NoError(t, err)
130- require.Empty(t, records)
131- require.NoDirExists(t, repository.sessionPath(interruptedID))
132-
133- retryID := domain.IncarnationID{4}
134- created, err := recovery.NewCoordinator(catalogue, repository, journal, bytes.NewReader(retryID[:])).Create(ctx, domain.CatalogueRecord{Name: "work"})
135- require.NoError(t, err)
136- require.Equal(t, retryID, created.IncarnationID)
137- records, err = catalogue.Records()
138- require.NoError(t, err)
139- require.Len(t, records, 1)
140- require.NoDirExists(t, repository.sessionPath(interruptedID))
141- require.NoError(t, catalogue.Close())
142-}
143-
144-func mustReadCrashFile(t *testing.T, path string) []byte {
145- t.Helper()
146- data, err := os.ReadFile(path)
147- require.NoError(t, err)
148- return data
149-}
M internal/adapters/snapshot/repository.go +0 -4
@@ -78,10 +78,6 @@ type repositoryHooks struct {
7878 maintenanceDirentConfig *maintenanceDirentConfig
7979 beforeDirectoryRead func(string)
8080 beforeDeletionTombstoneRead func(string)
81- // afterDeletionIncarnationQuarantine and afterDeletionLegacyQuarantine expose
82- // the two completed deletion rename+sync boundaries to package tests.
83- afterDeletionIncarnationQuarantine func() error
84- afterDeletionLegacyQuarantine func() error
8581 // beforeMaintenanceWork observes each budgeted quarantine filesystem step.
8682 // It is test-only instrumentation for hostile traversal bounds.
8783 beforeMaintenanceWork func(string)
M internal/app/daemon_integration_test.go +16 -14
@@ -23,7 +23,6 @@ import (
2323 "github.com/bnema/vev/internal/adapters/ipc"
2424 "github.com/bnema/vev/internal/adapters/lifecycle"
2525 "github.com/bnema/vev/internal/adapters/pty"
26- "github.com/bnema/vev/internal/adapters/recoveryfs"
2726 "github.com/bnema/vev/internal/adapters/snapshot"
2827 "github.com/bnema/vev/internal/domain"
2928 "github.com/bnema/vev/internal/persist"
@@ -534,17 +533,19 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) {
534533 Catalogue: newTestPersister(t, stateDir),
535534 closed: make(chan struct{}),
536535 }
536+ require.NoError(t, catalogue.Create(domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}))
537537 repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots"))
538538 shutdownClock := newLifecycleShutdownClock(t)
539- reconciler := &lifecycleBlockingReconciler{
540- entered: make(chan struct{}),
541- release: make(chan struct{}),
542- returned: make(chan struct{}),
539+ maintenanceRepository := &lifecycleBlockingMaintenanceRepository{
540+ SnapshotRepository: repository,
541+ entered: make(chan struct{}),
542+ release: make(chan struct{}),
543+ returned: make(chan struct{}),
543544 }
544545 ctx, cancel := context.WithCancel(context.Background())
545546 t.Cleanup(cancel)
546547 var releaseOnce sync.Once
547- releaseMaintenance := func() { releaseOnce.Do(func() { close(reconciler.release) }) }
548+ releaseMaintenance := func() { releaseOnce.Do(func() { close(maintenanceRepository.release) }) }
548549 t.Cleanup(releaseMaintenance)
549550
550551 listenerReady := make(chan struct{})
@@ -563,7 +564,7 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) {
563564 shutdownClock,
564565 discardLog(),
565566 daemon.WithSnapshotRepository(repository),
566- daemon.WithDurableMaintenance(catalogue, repository, reconciler, nil),
567+ daemon.WithDurableMaintenance(catalogue, maintenanceRepository, nil),
567568 daemon.WithCatalogue(catalogue, nil),
568569 )
569570 return d.Serve(ctx, listener)
@@ -571,7 +572,7 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) {
571572 }()
572573
573574 awaitLifecycleStage(t, listenerReady, "daemon listener")
574- awaitLifecycleStage(t, reconciler.entered, "maintenance repository call")
575+ awaitLifecycleStage(t, maintenanceRepository.entered, "maintenance repository call")
575576 cancel()
576577 shutdownClock.nextTimer(t).fire()
577578 select {
@@ -585,7 +586,7 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) {
585586 require.ErrorIs(t, err, lifecycle.ErrBusy)
586587
587588 releaseMaintenance()
588- awaitLifecycleStage(t, reconciler.returned, "maintenance repository return")
589+ awaitLifecycleStage(t, maintenanceRepository.returned, "maintenance repository return")
589590 awaitLifecycleStage(t, catalogue.closed, "catalogue close after maintenance")
590591 awaitLifecycleStage(t, callbackReturned, "Serve callback return after catalogue close")
591592 require.NoError(t, awaitLifecycleResult(t, wrapperReturned, "lifecycle wrapper return"))
@@ -594,7 +595,8 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) {
594595 require.NoError(t, owner.Release())
595596 }
596597
597-type lifecycleBlockingReconciler struct {
598+type lifecycleBlockingMaintenanceRepository struct {
599+ ports.SnapshotRepository
598600 entered chan struct{}
599601 release chan struct{}
600602 returned chan struct{}
@@ -602,11 +604,11 @@ type lifecycleBlockingReconciler struct {
602604 returnOnce sync.Once
603605 }
604606
605-func (r *lifecycleBlockingReconciler) Step(context.Context, ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error) {
607+func (r *lifecycleBlockingMaintenanceRepository) MaintainSession(context.Context, ports.RetentionPlan, ports.MaintenanceBudget) (bool, error) {
606608 r.enterOnce.Do(func() { close(r.entered) })
607609 <-r.release // Intentionally ignore cancellation: lifecycle ownership must outlive this call.
608610 r.returnOnce.Do(func() { close(r.returned) })
609- return ports.ReconcileCursor{}, nil, nil
611+ return true, nil
610612 }
611613
612614 type lifecycleObservedCatalogue struct {
@@ -743,7 +745,7 @@ func publishRestorableCheckpoint(t *testing.T, stateDir string, repository *snap
743745
744746 opened, err := persist.OpenOrCreate(stateDir)
745747 require.NoError(t, err)
746- coordinator := recovery.NewCoordinator(opened.Catalogue, repository, recoveryfs.New(stateDir), rand.Reader)
748+ coordinator := recovery.NewCoordinator(opened.Catalogue, repository, rand.Reader)
747749 require.NoError(t, coordinator.Recover(ctx))
748750 listener, err := ipc.Listen(runtimeDir)
749751 require.NoError(t, err)
@@ -1054,7 +1056,7 @@ func TestLifecycleSocketCloseCatalogueRace(t *testing.T) {
10541056 if err != nil {
10551057 return err
10561058 }
1057- coordinator := recovery.NewCoordinator(opened.Catalogue, snapshotRepository, recoveryfs.New(stateDir), rand.Reader)
1059+ coordinator := recovery.NewCoordinator(opened.Catalogue, snapshotRepository, rand.Reader)
10581060 if err := coordinator.Recover(ctx); err != nil {
10591061 return errors.Join(err, opened.Catalogue.Close())
10601062 }
D internal/app/durable_crash_matrix_test.go +0 -389
@@ -1,389 +0,0 @@
1-package app
2-
3-import (
4- "bytes"
5- "context"
6- "crypto/sha256"
7- "errors"
8- "os"
9- "path/filepath"
10- "testing"
11-
12- "github.com/bnema/vev/internal/adapters/recoveryfs"
13- "github.com/bnema/vev/internal/adapters/snapshot"
14- "github.com/bnema/vev/internal/domain"
15- "github.com/bnema/vev/internal/persist"
16- "github.com/bnema/vev/internal/ports"
17- "github.com/bnema/vev/internal/usecase/recovery"
18- snapcodec "github.com/bnema/vev/internal/usecase/snapshot"
19- "github.com/stretchr/testify/require"
20-)
21-
22-var errInjectedCrash = errors.New("injected crash after durable boundary")
23-
24-type durableCrashHarness struct {
25- stateDir string
26- snapshotDir string
27-}
28-
29-func newDurableCrashHarness(t *testing.T) durableCrashHarness {
30- t.Helper()
31- stateDir := t.TempDir()
32- require.NoError(t, os.Chmod(stateDir, 0o700))
33- return durableCrashHarness{stateDir: stateDir, snapshotDir: filepath.Join(stateDir, "snapshots")}
34-}
35-
36-func (h durableCrashHarness) open(t *testing.T) (*persist.Persister, *snapshot.Repository, *recoveryfs.Journal) {
37- t.Helper()
38- store, err := persist.OpenStore(persist.StorePath(h.stateDir))
39- require.NoError(t, err)
40- return persist.New(store), snapshot.NewRepository(h.snapshotDir), recoveryfs.New(h.stateDir)
41-}
42-
43-func (h durableCrashHarness) recoverTwice(ctx context.Context, t *testing.T, assert func(*persist.Persister, *snapshot.Repository, *recoveryfs.Journal)) {
44- t.Helper()
45- for attempt := range 2 {
46- catalogue, repository, journal := h.open(t)
47- coordinator := recovery.NewCoordinator(catalogue, repository, journal, nil)
48- require.NoError(t, coordinator.Recover(ctx), "recovery attempt %d", attempt+1)
49- assert(catalogue, repository, journal)
50- require.NoError(t, catalogue.Close())
51- }
52-}
53-
54-type durableBoundary struct {
55- want string
56- triggered bool
57-}
58-
59-func (b *durableBoundary) after(name string, err error) error {
60- if err != nil || name != b.want {
61- return err
62- }
63- b.triggered = true
64- return errInjectedCrash
65-}
66-
67-type crashCatalogue struct {
68- ports.Catalogue
69- boundary *durableBoundary
70-}
71-
72-func (c crashCatalogue) Create(record domain.CatalogueRecord) error {
73- return c.boundary.after("catalogue-create", c.Catalogue.Create(record))
74-}
75-func (c crashCatalogue) Replace(name string, record domain.CatalogueRecord) error {
76- return c.boundary.after("catalogue-replace", c.Catalogue.Replace(name, record))
77-}
78-func (c crashCatalogue) Rename(oldName string, record domain.CatalogueRecord) error {
79- return c.boundary.after("catalogue-rename", c.Catalogue.Rename(oldName, record))
80-}
81-func (c crashCatalogue) Delete(name string) error {
82- return c.boundary.after("catalogue-delete", c.Catalogue.Delete(name))
83-}
84-
85-type crashRepository struct {
86- ports.SnapshotRepository
87- boundary *durableBoundary
88-}
89-
90-func (r crashRepository) Publish(ctx context.Context, publication ports.SnapshotPublication) error {
91- return r.boundary.after("snapshot-publish", r.SnapshotRepository.Publish(ctx, publication))
92-}
93-func (r crashRepository) WriteDeletionTombstone(ctx context.Context, tombstone domain.DeletionTombstone) error {
94- return r.boundary.after("deletion-tombstone", r.SnapshotRepository.WriteDeletionTombstone(ctx, tombstone))
95-}
96-func (r crashRepository) QuarantineDeletionSources(ctx context.Context, tombstone domain.DeletionTombstone, includeLegacy bool) error {
97- return r.boundary.after("deletion-quarantine", r.SnapshotRepository.QuarantineDeletionSources(ctx, tombstone, includeLegacy))
98-}
99-func (r crashRepository) DeleteDeletionTombstone(ctx context.Context, id domain.IncarnationID) error {
100- return r.boundary.after("deletion-tombstone-delete", r.SnapshotRepository.DeleteDeletionTombstone(ctx, id))
101-}
102-func (r crashRepository) SaveQuarantineDescriptor(ctx context.Context, descriptor domain.QuarantineDescriptor) error {
103- return r.boundary.after("discard-descriptor", r.SnapshotRepository.SaveQuarantineDescriptor(ctx, descriptor))
104-}
105-func (r crashRepository) QuarantineIncarnation(ctx context.Context, id domain.IncarnationID) error {
106- return r.boundary.after("discard-quarantine", r.SnapshotRepository.QuarantineIncarnation(ctx, id))
107-}
108-
109-type crashJournal struct {
110- ports.RecoveryJournal
111- boundary *durableBoundary
112-}
113-
114-func (j crashJournal) SaveDiscard(ctx context.Context, intent domain.DiscardIntent) error {
115- return j.boundary.after("discard-intent", j.RecoveryJournal.SaveDiscard(ctx, intent))
116-}
117-func (j crashJournal) DeleteDiscard(ctx context.Context, id domain.IncarnationID) error {
118- return j.boundary.after("discard-intent-delete", j.RecoveryJournal.DeleteDiscard(ctx, id))
119-}
120-
121-func TestDurableTransactionCrashMatrix(t *testing.T) {
122- ctx := context.Background()
123-
124- t.Run("create/catalogue-create", func(t *testing.T) {
125- h := newDurableCrashHarness(t)
126- catalogue, repository, journal := h.open(t)
127- boundary := &durableBoundary{want: "catalogue-create"}
128- coordinator := recovery.NewCoordinator(crashCatalogue{Catalogue: catalogue, boundary: boundary}, repository, journal, bytes.NewReader(bytes.Repeat([]byte{1}, 16)))
129- _, err := coordinator.Create(ctx, domain.CatalogueRecord{Name: "work", Cwd: "/tmp"})
130- require.ErrorIs(t, err, errInjectedCrash)
131- require.True(t, boundary.triggered)
132- require.NoError(t, catalogue.Close())
133-
134- h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {
135- record, exists, err := catalogue.Record("work")
136- require.NoError(t, err)
137- require.True(t, exists)
138- require.Equal(t, domain.IncarnationID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, record.IncarnationID)
139- require.Equal(t, domain.RecoveryFresh, record.RecoveryState)
140- })
141- })
142-
143- for _, boundaryName := range []string{"snapshot-publish", "catalogue-replace"} {
144- t.Run("checkpoint/"+boundaryName, func(t *testing.T) {
145- h := newDurableCrashHarness(t)
146- catalogue, repository, journal := h.open(t)
147- record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{2}, RecoveryState: domain.RecoveryFresh}
148- require.NoError(t, catalogue.Create(record))
149- publication := durablePublication(t, record.Name, record.IncarnationID, []byte("checkpoint"))
150- boundary := &durableBoundary{want: boundaryName}
151- coordinator := recovery.NewCoordinator(
152- crashCatalogue{Catalogue: catalogue, boundary: boundary},
153- crashRepository{SnapshotRepository: repository, boundary: boundary},
154- journal,
155- nil,
156- )
157- _, err := coordinator.PublishCheckpoint(ctx, record.Name, publication)
158- require.ErrorIs(t, err, errInjectedCrash)
159- require.True(t, boundary.triggered)
160- require.NoError(t, catalogue.Close())
161-
162- ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)}
163- h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
164- stored, exists, err := catalogue.Record(record.Name)
165- require.NoError(t, err)
166- require.True(t, exists)
167- if boundaryName == "snapshot-publish" {
168- require.Equal(t, domain.RecoveryFresh, stored.RecoveryState)
169- require.Nil(t, stored.Committed)
170- } else {
171- require.Equal(t, domain.RecoveryHealthy, stored.RecoveryState)
172- require.Equal(t, ref, *stored.Committed)
173- }
174- generation, err := repository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, ref)
175- require.NoError(t, err)
176- require.Equal(t, publication.Manifest, generation.Manifest)
177- })
178- })
179- }
180-
181- t.Run("rename/catalogue-rename", func(t *testing.T) {
182- h := newDurableCrashHarness(t)
183- catalogue, repository, journal := h.open(t)
184- record := domain.CatalogueRecord{Name: "old", IncarnationID: domain.IncarnationID{3}, RecoveryState: domain.RecoveryFresh}
185- require.NoError(t, catalogue.Create(record))
186- boundary := &durableBoundary{want: "catalogue-rename"}
187- coordinator := recovery.NewCoordinator(crashCatalogue{Catalogue: catalogue, boundary: boundary}, repository, journal, nil)
188- _, err := coordinator.Rename(ctx, "old", "new")
189- require.ErrorIs(t, err, errInjectedCrash)
190- require.True(t, boundary.triggered)
191- require.NoError(t, catalogue.Close())
192-
193- h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) {
194- _, oldExists, err := catalogue.Record("old")
195- require.NoError(t, err)
196- renamed, newExists, err := catalogue.Record("new")
197- require.NoError(t, err)
198- require.False(t, oldExists)
199- require.True(t, newExists)
200- require.Equal(t, record.IncarnationID, renamed.IncarnationID)
201- })
202- })
203-
204- deleteBoundaries := []string{"catalogue-replace", "deletion-tombstone", "deletion-quarantine", "catalogue-delete", "deletion-tombstone-delete", "transaction-complete"}
205- for _, boundaryName := range deleteBoundaries {
206- t.Run("delete/"+boundaryName, func(t *testing.T) {
207- h := newDurableCrashHarness(t)
208- catalogue, repository, journal := h.open(t)
209- record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{4}, RecoveryState: domain.RecoveryFresh}
210- require.NoError(t, catalogue.Create(record))
211- publication := durablePublication(t, record.Name, record.IncarnationID, []byte("delete-source"))
212- require.NoError(t, repository.Publish(ctx, publication))
213- legacy := []byte{0, 1, 2, 3, 0xff}
214- require.NoError(t, os.WriteFile(filepath.Join(h.snapshotDir, "work.snap"), legacy, 0o600))
215- sourceBytes := readTree(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String()))
216-
217- boundary := &durableBoundary{want: boundaryName}
218- coordinator := recovery.NewCoordinator(
219- crashCatalogue{Catalogue: catalogue, boundary: boundary},
220- crashRepository{SnapshotRepository: repository, boundary: boundary},
221- journal,
222- nil,
223- )
224- err := coordinator.Delete(ctx, record.Name)
225- if boundaryName == "transaction-complete" {
226- require.NoError(t, err)
227- boundary.triggered = true // interruption before the caller transfers lifecycle ownership
228- } else {
229- require.ErrorIs(t, err, errInjectedCrash)
230- }
231- require.True(t, boundary.triggered)
232- require.NoError(t, catalogue.Close())
233-
234- h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
235- _, exists, err := catalogue.Record(record.Name)
236- require.NoError(t, err)
237- require.False(t, exists)
238- assertNoTombstones(ctx, t, repository)
239- require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String()))
240- require.Equal(t, sourceBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "snapshot")))
241- require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "legacy.snap")))
242- })
243- })
244- }
245-}
246-
247-func TestDurableDiscardCrashMatrix(t *testing.T) {
248- ctx := context.Background()
249- oldID := domain.IncarnationID{5}
250- newID := domain.IncarnationID{9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9}
251- boundaries := []string{"discard-intent", "discard-descriptor", "discard-quarantine", "catalogue-replace", "discard-intent-delete", "runtime-exposure"}
252-
253- for _, boundaryName := range boundaries {
254- t.Run(boundaryName, func(t *testing.T) {
255- h := newDurableCrashHarness(t)
256- catalogue, repository, journal := h.open(t)
257- publication := durablePublication(t, "broken", oldID, []byte("discard-source"))
258- require.NoError(t, repository.Publish(ctx, publication))
259- ref := &domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)}
260- old := domain.CatalogueRecord{Name: "broken", IncarnationID: oldID, RecoveryState: domain.RecoveryDegraded, Committed: ref, DegradedReason: "checkpoint unreadable"}
261- require.NoError(t, catalogue.Create(old))
262- sourceBytes := readTree(t, filepath.Join(h.snapshotDir, "sessions", oldID.String()))
263-
264- boundary := &durableBoundary{want: boundaryName}
265- coordinator := recovery.NewCoordinator(
266- crashCatalogue{Catalogue: catalogue, boundary: boundary},
267- crashRepository{SnapshotRepository: repository, boundary: boundary},
268- crashJournal{RecoveryJournal: journal, boundary: boundary},
269- bytes.NewReader(newID[:]),
270- )
271- replacement, err := coordinator.Discard(ctx, old.Name, "explicit discard")
272- if boundaryName == "runtime-exposure" {
273- require.NoError(t, err)
274- require.Equal(t, newID, replacement.IncarnationID)
275- boundary.triggered = true // crash before the returned runtime is installed
276- } else {
277- require.ErrorIs(t, err, errInjectedCrash)
278- }
279- require.True(t, boundary.triggered)
280- require.NoError(t, catalogue.Close())
281-
282- h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) {
283- got, exists, err := catalogue.Record(old.Name)
284- require.NoError(t, err)
285- require.True(t, exists)
286- require.Equal(t, newID, got.IncarnationID)
287- require.Equal(t, domain.RecoveryFresh, got.RecoveryState)
288- require.Nil(t, got.Committed)
289- require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", oldID.String()))
290- require.Equal(t, sourceBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot")))
291- intents, err := journal.ListDiscards(ctx)
292- require.NoError(t, err)
293- require.Empty(t, intents)
294- require.FileExists(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "record"))
295- })
296- })
297- }
298-}
299-
300-func TestDeletionRecoveryProtectsReusedNameFilesystemBytes(t *testing.T) {
301- ctx := context.Background()
302- h := newDurableCrashHarness(t)
303- catalogue, repository, _ := h.open(t)
304- oldID := domain.IncarnationID{6}
305- replacementID := domain.IncarnationID{7}
306- oldPublication := durablePublication(t, "work", oldID, []byte("old-incarnation"))
307- replacementPublication := durablePublication(t, "work", replacementID, []byte("replacement-incarnation"))
308- require.NoError(t, repository.Publish(ctx, oldPublication))
309- require.NoError(t, repository.Publish(ctx, replacementPublication))
310- replacement := domain.CatalogueRecord{Name: "work", IncarnationID: replacementID, RecoveryState: domain.RecoveryFresh}
311- require.NoError(t, catalogue.Create(replacement))
312- require.NoError(t, repository.WriteDeletionTombstone(ctx, domain.DeletionTombstone{Name: "work", IncarnationID: oldID}))
313- legacy := []byte{0xde, 0xad, 0x00, 0xbe, 0xef}
314- require.NoError(t, os.WriteFile(filepath.Join(h.snapshotDir, "work.snap"), legacy, 0o600))
315- oldBytes := readTree(t, filepath.Join(h.snapshotDir, "sessions", oldID.String()))
316- replacementPath := filepath.Join(h.snapshotDir, "sessions", replacementID.String())
317- replacementBytes := readTree(t, replacementPath)
318- require.NoError(t, catalogue.Close())
319-
320- h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) {
321- got, exists, err := catalogue.Record("work")
322- require.NoError(t, err)
323- require.True(t, exists)
324- require.Equal(t, replacement, got)
325- require.Equal(t, replacementBytes, readTree(t, replacementPath))
326- require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "work.snap")))
327- require.Equal(t, oldBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot")))
328- require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", oldID.String()))
329- assertNoTombstones(ctx, t, repository)
330- })
331-}
332-
333-func durablePublication(t *testing.T, name string, id domain.IncarnationID, payload []byte) ports.SnapshotPublication {
334- t.Helper()
335- tail, err := snapcodec.MarshalObject(snapcodec.HistoryTail, payload)
336- require.NoError(t, err)
337- visible, err := snapcodec.MarshalObject(snapcodec.Visible, []byte("visible"))
338- require.NoError(t, err)
339- manifest, err := snapcodec.MarshalManifest(snapcodec.Manifest{
340- Generation: 1, IncarnationID: id, Name: name,
341- Tabs: []snapcodec.ManifestTab{{Cols: 1, Rows: 1, Panes: []snapcodec.ManifestPane{{
342- ID: "p", Tail: snapcodec.ObjectRef{Kind: snapcodec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))},
343- Visible: snapcodec.ObjectRef{Kind: snapcodec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))},
344- }}}},
345- })
346- require.NoError(t, err)
347- return ports.SnapshotPublication{IncarnationID: id, Name: name, Generation: 1, Manifest: manifest, Objects: []ports.SnapshotObject{tail, visible}}
348-}
349-
350-func assertNoTombstones(ctx context.Context, t *testing.T, repository *snapshot.Repository) {
351- t.Helper()
352- page, err := repository.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 64 << 10})
353- require.NoError(t, err)
354- require.True(t, page.Done)
355- require.Empty(t, page.Tombstones)
356-}
357-
358-func mustReadFile(t *testing.T, path string) []byte {
359- t.Helper()
360- data, err := os.ReadFile(path)
361- require.NoError(t, err)
362- return data
363-}
364-
365-func readTree(t *testing.T, root string) map[string][]byte {
366- t.Helper()
367- files := make(map[string][]byte)
368- err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
369- if err != nil {
370- return err
371- }
372- if entry.IsDir() {
373- return nil
374- }
375- relative, err := filepath.Rel(root, path)
376- if err != nil {
377- return err
378- }
379- data, err := os.ReadFile(path)
380- if err != nil {
381- return err
382- }
383- files[relative] = data
384- return nil
385- })
386- require.NoError(t, err)
387- require.NotEmpty(t, files)
388- return files
389-}
M internal/app/offline_kill_retry_test.go +2 -2
@@ -75,8 +75,8 @@ func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) {
7575 }
7676
7777 require.NoError(t, runKill(context.Background(), "named", false, false))
78- require.NoFileExists(t, filepath.Join(source, "sentinel"))
79- require.FileExists(t, filepath.Join(stateDir, "snapshots", "quarantine", id.String(), "snapshot", "sentinel"))
78+ require.NoDirExists(t, source)
79+ require.NoDirExists(t, filepath.Join(stateDir, "snapshots", "quarantine", id.String()))
8080 })
8181 }
8282 }
M internal/app/run.go +5 -19
@@ -36,7 +36,6 @@ import (
3636 "github.com/bnema/vev/internal/adapters/noticefile"
3737 "github.com/bnema/vev/internal/adapters/observability"
3838 "github.com/bnema/vev/internal/adapters/pty"
39- "github.com/bnema/vev/internal/adapters/recoveryfs"
4039 remoteadapter "github.com/bnema/vev/internal/adapters/remote"
4140 "github.com/bnema/vev/internal/adapters/shellcmd"
4241 snapshotadapter "github.com/bnema/vev/internal/adapters/snapshot"
@@ -521,20 +520,13 @@ type interruptedRecoveryIdentity struct {
521520 id domain.IncarnationID
522521 }
523522
524-func pendingInterruptedRecoveries(ctx context.Context, records []domain.CatalogueRecord, journal ports.RecoveryJournal, repository ports.SnapshotRepository) ([]interruptedRecoveryIdentity, error) {
523+func pendingInterruptedRecoveries(ctx context.Context, records []domain.CatalogueRecord, repository ports.SnapshotRepository) ([]interruptedRecoveryIdentity, error) {
525524 pending := make([]interruptedRecoveryIdentity, 0)
526525 for _, record := range records {
527526 if record.RecoveryState == domain.RecoveryDeleting {
528527 pending = append(pending, interruptedRecoveryIdentity{name: record.Name, id: record.IncarnationID})
529528 }
530529 }
531- intents, err := journal.ListDiscards(ctx)
532- if err != nil {
533- return nil, err
534- }
535- for _, intent := range intents {
536- pending = append(pending, interruptedRecoveryIdentity{name: intent.SessionName, id: intent.OldIncarnation})
537- }
538530 cursor := ports.DeletionTombstoneCursor{}
539531 for {
540532 page, err := repository.ListDeletionTombstones(ctx, cursor, ports.MaintenanceBudget{Entries: 64, Bytes: 64 << 10})
@@ -602,10 +594,6 @@ func fencedRecoveries(conflicts []recovery.RecoveryConflict, catalogue ports.Cat
602594
603595 func fencedRecoveryReasonCode(err error) string {
604596 switch {
605- case errors.Is(err, recovery.ErrDiscardConflict):
606- return "discard-intent-conflict"
607- case errors.Is(err, recovery.ErrDiscardIntentInvalid):
608- return "discard-intent-invalid"
609597 case errors.Is(err, recovery.ErrDeletionTombstoneConflict):
610598 return "deletion-tombstone-conflict"
611599 case errors.Is(err, recovery.ErrDeletionTombstoneInvalid):
@@ -738,12 +726,11 @@ func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr err
738726 recoveryMode = "new-install"
739727 }
740728 logCatalogueRecovery(log, opened.Records, recoveryMode)
741- journal := recoveryfs.New(platform.StateDir())
742- pendingRecoveries, err := pendingInterruptedRecoveries(ctx, opened.Records, journal, snapshotRepository)
729+ pendingRecoveries, err := pendingInterruptedRecoveries(ctx, opened.Records, snapshotRepository)
743730 if err != nil {
744731 return errors.Join(fmt.Errorf("vev: inspect durable session transactions: %w", err), opened.Catalogue.Close())
745732 }
746- coordinator := recovery.NewCoordinator(opened.Catalogue, snapshotRepository, journal, rand.Reader)
733+ coordinator := recovery.NewCoordinator(opened.Catalogue, snapshotRepository, rand.Reader)
747734 // Recovery may roll catalogue changes forward, so reload the authoritative
748735 // snapshot before constructing runtime reservations. A failed reload aborts
749736 // startup rather than treating durable sessions as absent.
@@ -774,8 +761,7 @@ func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr err
774761 for _, pending := range pendingRecoveries {
775762 unresolved = append(unresolved, pending.id)
776763 }
777- reconciler := recovery.NewReconciler(coordinator, opened.Catalogue, snapshotRepository, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20})
778- daemonOpts = append(daemonOpts, daemon.WithDurableMaintenance(opened.Catalogue, snapshotRepository, reconciler, unresolved))
764+ daemonOpts = append(daemonOpts, daemon.WithDurableMaintenance(opened.Catalogue, snapshotRepository, unresolved))
779765 log.Info("session persistence enabled", "path", storePath)
780766 daemonOpts = append(daemonOpts, daemon.WithCatalogue(opened.Catalogue, opened.Records))
781767
@@ -1577,7 +1563,7 @@ func runOfflineNamedKill(ctx context.Context, name string) (retErr error) {
15771563 } else if !ok {
15781564 return fmt.Errorf("vev: no such session: %s", name)
15791565 }
1580- coordinator := recovery.NewCoordinator(opened.Catalogue, repository, recoveryfs.New(platform.StateDir()), rand.Reader)
1566+ coordinator := recovery.NewCoordinator(opened.Catalogue, repository, rand.Reader)
15811567 if err := coordinator.Recover(ctx); err != nil {
15821568 return fmt.Errorf("vev: recovering stored session transactions: %w", err)
15831569 }
M internal/app/run_test.go +1 -2
@@ -21,7 +21,6 @@ import (
2121
2222 "github.com/bnema/vev/internal/adapters/lifecycle"
2323 "github.com/bnema/vev/internal/adapters/noticefile"
24- "github.com/bnema/vev/internal/adapters/recoveryfs"
2524 "github.com/bnema/vev/internal/adapters/snapshot"
2625 "github.com/bnema/vev/internal/domain"
2726 "github.com/bnema/vev/internal/persist"
@@ -132,7 +131,7 @@ func TestStartupReloadsCatalogueAfterRolledForwardDeletion(t *testing.T) {
132131 CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryDeleting,
133132 }
134133 require.NoError(t, catalogue.Save(record))
135- coordinator := recovery.NewCoordinator(catalogue, repository, recoveryfs.New(stateDir), rand.Reader)
134+ coordinator := recovery.NewCoordinator(catalogue, repository, rand.Reader)
136135
137136 records, err := recoverAuthoritativeCatalogue(ctx, coordinator, catalogue)
138137 require.NoError(t, err)
M internal/domain/recovery.go +0 -18
@@ -81,24 +81,6 @@ func (r CatalogueRecord) MetadataUpdate() CatalogueMetadataUpdate {
8181 }
8282 }
8383
84-// DiscardIntent records a requested replacement of uncertain persisted state.
85-type DiscardIntent struct {
86- OldRecord CatalogueRecord
87- OldIncarnation IncarnationID
88- NewIncarnation IncarnationID
89- SessionName string
90- Reason string
91-}
92-
93-// QuarantineDescriptor records persisted state retained during replacement.
94-type QuarantineDescriptor struct {
95- OldRecord CatalogueRecord
96- OldIncarnation IncarnationID
97- ReplacementIncarnation IncarnationID
98- SessionName string
99- Reason string
100-}
101-
10284 // DeletionTombstone identifies exactly one deleted session incarnation.
10385 type DeletionTombstone struct {
10486 Name string
M internal/ports/mocks/recovery_mock.go +13 -494
@@ -12,209 +12,6 @@ import (
1212 mock "github.com/stretchr/testify/mock"
1313 )
1414
15-// NewMockRecoveryJournal creates a new instance of MockRecoveryJournal. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
16-// The first argument is typically a *testing.T value.
17-func NewMockRecoveryJournal(t interface {
18- mock.TestingT
19- Cleanup(func())
20-}) *MockRecoveryJournal {
21- mock := &MockRecoveryJournal{}
22- mock.Mock.Test(t)
23-
24- t.Cleanup(func() { mock.AssertExpectations(t) })
25-
26- return mock
27-}
28-
29-// MockRecoveryJournal is an autogenerated mock type for the RecoveryJournal type
30-type MockRecoveryJournal struct {
31- mock.Mock
32-}
33-
34-type MockRecoveryJournal_Expecter struct {
35- mock *mock.Mock
36-}
37-
38-func (_m *MockRecoveryJournal) EXPECT() *MockRecoveryJournal_Expecter {
39- return &MockRecoveryJournal_Expecter{mock: &_m.Mock}
40-}
41-
42-// DeleteDiscard provides a mock function for the type MockRecoveryJournal
43-func (_mock *MockRecoveryJournal) DeleteDiscard(context1 context.Context, incarnationID domain.IncarnationID) error {
44- ret := _mock.Called(context1, incarnationID)
45-
46- if len(ret) == 0 {
47- panic("no return value specified for DeleteDiscard")
48- }
49-
50- var r0 error
51- if returnFunc, ok := ret.Get(0).(func(context.Context, domain.IncarnationID) error); ok {
52- r0 = returnFunc(context1, incarnationID)
53- } else {
54- r0 = ret.Error(0)
55- }
56- return r0
57-}
58-
59-// MockRecoveryJournal_DeleteDiscard_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteDiscard'
60-type MockRecoveryJournal_DeleteDiscard_Call struct {
61- *mock.Call
62-}
63-
64-// DeleteDiscard is a helper method to define mock.On call
65-// - context1 context.Context
66-// - incarnationID domain.IncarnationID
67-func (_e *MockRecoveryJournal_Expecter) DeleteDiscard(context1 any, incarnationID any) *MockRecoveryJournal_DeleteDiscard_Call {
68- return &MockRecoveryJournal_DeleteDiscard_Call{Call: _e.mock.On("DeleteDiscard", context1, incarnationID)}
69-}
70-
71-func (_c *MockRecoveryJournal_DeleteDiscard_Call) Run(run func(context1 context.Context, incarnationID domain.IncarnationID)) *MockRecoveryJournal_DeleteDiscard_Call {
72- _c.Call.Run(func(args mock.Arguments) {
73- var arg0 context.Context
74- if args[0] != nil {
75- arg0 = args[0].(context.Context)
76- }
77- var arg1 domain.IncarnationID
78- if args[1] != nil {
79- arg1 = args[1].(domain.IncarnationID)
80- }
81- run(
82- arg0,
83- arg1,
84- )
85- })
86- return _c
87-}
88-
89-func (_c *MockRecoveryJournal_DeleteDiscard_Call) Return(err error) *MockRecoveryJournal_DeleteDiscard_Call {
90- _c.Call.Return(err)
91- return _c
92-}
93-
94-func (_c *MockRecoveryJournal_DeleteDiscard_Call) RunAndReturn(run func(context1 context.Context, incarnationID domain.IncarnationID) error) *MockRecoveryJournal_DeleteDiscard_Call {
95- _c.Call.Return(run)
96- return _c
97-}
98-
99-// ListDiscards provides a mock function for the type MockRecoveryJournal
100-func (_mock *MockRecoveryJournal) ListDiscards(context1 context.Context) ([]domain.DiscardIntent, error) {
101- ret := _mock.Called(context1)
102-
103- if len(ret) == 0 {
104- panic("no return value specified for ListDiscards")
105- }
106-
107- var r0 []domain.DiscardIntent
108- var r1 error
109- if returnFunc, ok := ret.Get(0).(func(context.Context) ([]domain.DiscardIntent, error)); ok {
110- return returnFunc(context1)
111- }
112- if returnFunc, ok := ret.Get(0).(func(context.Context) []domain.DiscardIntent); ok {
113- r0 = returnFunc(context1)
114- } else {
115- if ret.Get(0) != nil {
116- r0 = ret.Get(0).([]domain.DiscardIntent)
117- }
118- }
119- if returnFunc, ok := ret.Get(1).(func(context.Context) error); ok {
120- r1 = returnFunc(context1)
121- } else {
122- r1 = ret.Error(1)
123- }
124- return r0, r1
125-}
126-
127-// MockRecoveryJournal_ListDiscards_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListDiscards'
128-type MockRecoveryJournal_ListDiscards_Call struct {
129- *mock.Call
130-}
131-
132-// ListDiscards is a helper method to define mock.On call
133-// - context1 context.Context
134-func (_e *MockRecoveryJournal_Expecter) ListDiscards(context1 any) *MockRecoveryJournal_ListDiscards_Call {
135- return &MockRecoveryJournal_ListDiscards_Call{Call: _e.mock.On("ListDiscards", context1)}
136-}
137-
138-func (_c *MockRecoveryJournal_ListDiscards_Call) Run(run func(context1 context.Context)) *MockRecoveryJournal_ListDiscards_Call {
139- _c.Call.Run(func(args mock.Arguments) {
140- var arg0 context.Context
141- if args[0] != nil {
142- arg0 = args[0].(context.Context)
143- }
144- run(
145- arg0,
146- )
147- })
148- return _c
149-}
150-
151-func (_c *MockRecoveryJournal_ListDiscards_Call) Return(discardIntents []domain.DiscardIntent, err error) *MockRecoveryJournal_ListDiscards_Call {
152- _c.Call.Return(discardIntents, err)
153- return _c
154-}
155-
156-func (_c *MockRecoveryJournal_ListDiscards_Call) RunAndReturn(run func(context1 context.Context) ([]domain.DiscardIntent, error)) *MockRecoveryJournal_ListDiscards_Call {
157- _c.Call.Return(run)
158- return _c
159-}
160-
161-// SaveDiscard provides a mock function for the type MockRecoveryJournal
162-func (_mock *MockRecoveryJournal) SaveDiscard(context1 context.Context, discardIntent domain.DiscardIntent) error {
163- ret := _mock.Called(context1, discardIntent)
164-
165- if len(ret) == 0 {
166- panic("no return value specified for SaveDiscard")
167- }
168-
169- var r0 error
170- if returnFunc, ok := ret.Get(0).(func(context.Context, domain.DiscardIntent) error); ok {
171- r0 = returnFunc(context1, discardIntent)
172- } else {
173- r0 = ret.Error(0)
174- }
175- return r0
176-}
177-
178-// MockRecoveryJournal_SaveDiscard_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SaveDiscard'
179-type MockRecoveryJournal_SaveDiscard_Call struct {
180- *mock.Call
181-}
182-
183-// SaveDiscard is a helper method to define mock.On call
184-// - context1 context.Context
185-// - discardIntent domain.DiscardIntent
186-func (_e *MockRecoveryJournal_Expecter) SaveDiscard(context1 any, discardIntent any) *MockRecoveryJournal_SaveDiscard_Call {
187- return &MockRecoveryJournal_SaveDiscard_Call{Call: _e.mock.On("SaveDiscard", context1, discardIntent)}
188-}
189-
190-func (_c *MockRecoveryJournal_SaveDiscard_Call) Run(run func(context1 context.Context, discardIntent domain.DiscardIntent)) *MockRecoveryJournal_SaveDiscard_Call {
191- _c.Call.Run(func(args mock.Arguments) {
192- var arg0 context.Context
193- if args[0] != nil {
194- arg0 = args[0].(context.Context)
195- }
196- var arg1 domain.DiscardIntent
197- if args[1] != nil {
198- arg1 = args[1].(domain.DiscardIntent)
199- }
200- run(
201- arg0,
202- arg1,
203- )
204- })
205- return _c
206-}
207-
208-func (_c *MockRecoveryJournal_SaveDiscard_Call) Return(err error) *MockRecoveryJournal_SaveDiscard_Call {
209- _c.Call.Return(err)
210- return _c
211-}
212-
213-func (_c *MockRecoveryJournal_SaveDiscard_Call) RunAndReturn(run func(context1 context.Context, discardIntent domain.DiscardIntent) error) *MockRecoveryJournal_SaveDiscard_Call {
214- _c.Call.Return(run)
215- return _c
216-}
217-
21815 // NewMockCheckpointCoordinator creates a new instance of MockCheckpointCoordinator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
21916 // The first argument is typically a *testing.T value.
22017 func NewMockCheckpointCoordinator(t interface {
@@ -636,29 +433,20 @@ func (_m *MockDegradedRecoveryCoordinator) EXPECT() *MockDegradedRecoveryCoordin
636433 }
637434
638435 // Discard provides a mock function for the type MockDegradedRecoveryCoordinator
639-func (_mock *MockDegradedRecoveryCoordinator) Discard(context1 context.Context, s string, s1 string) (domain.CatalogueRecord, error) {
640- ret := _mock.Called(context1, s, s1)
436+func (_mock *MockDegradedRecoveryCoordinator) Discard(context1 context.Context, s string) error {
437+ ret := _mock.Called(context1, s)
641438
642439 if len(ret) == 0 {
643440 panic("no return value specified for Discard")
644441 }
645442
646- var r0 domain.CatalogueRecord
647- var r1 error
648- if returnFunc, ok := ret.Get(0).(func(context.Context, string, string) (domain.CatalogueRecord, error)); ok {
649- return returnFunc(context1, s, s1)
650- }
651- if returnFunc, ok := ret.Get(0).(func(context.Context, string, string) domain.CatalogueRecord); ok {
652- r0 = returnFunc(context1, s, s1)
653- } else {
654- r0 = ret.Get(0).(domain.CatalogueRecord)
655- }
656- if returnFunc, ok := ret.Get(1).(func(context.Context, string, string) error); ok {
657- r1 = returnFunc(context1, s, s1)
443+ var r0 error
444+ if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok {
445+ r0 = returnFunc(context1, s)
658446 } else {
659- r1 = ret.Error(1)
447+ r0 = ret.Error(0)
660448 }
661- return r0, r1
449+ return r0
662450 }
663451
664452 // MockDegradedRecoveryCoordinator_Discard_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Discard'
@@ -669,12 +457,11 @@ type MockDegradedRecoveryCoordinator_Discard_Call struct {
669457 // Discard is a helper method to define mock.On call
670458 // - context1 context.Context
671459 // - s string
672-// - s1 string
673-func (_e *MockDegradedRecoveryCoordinator_Expecter) Discard(context1 any, s any, s1 any) *MockDegradedRecoveryCoordinator_Discard_Call {
674- return &MockDegradedRecoveryCoordinator_Discard_Call{Call: _e.mock.On("Discard", context1, s, s1)}
460+func (_e *MockDegradedRecoveryCoordinator_Expecter) Discard(context1 any, s any) *MockDegradedRecoveryCoordinator_Discard_Call {
461+ return &MockDegradedRecoveryCoordinator_Discard_Call{Call: _e.mock.On("Discard", context1, s)}
675462 }
676463
677-func (_c *MockDegradedRecoveryCoordinator_Discard_Call) Run(run func(context1 context.Context, s string, s1 string)) *MockDegradedRecoveryCoordinator_Discard_Call {
464+func (_c *MockDegradedRecoveryCoordinator_Discard_Call) Run(run func(context1 context.Context, s string)) *MockDegradedRecoveryCoordinator_Discard_Call {
678465 _c.Call.Run(func(args mock.Arguments) {
679466 var arg0 context.Context
680467 if args[0] != nil {
@@ -684,25 +471,20 @@ func (_c *MockDegradedRecoveryCoordinator_Discard_Call) Run(run func(context1 co
684471 if args[1] != nil {
685472 arg1 = args[1].(string)
686473 }
687- var arg2 string
688- if args[2] != nil {
689- arg2 = args[2].(string)
690- }
691474 run(
692475 arg0,
693476 arg1,
694- arg2,
695477 )
696478 })
697479 return _c
698480 }
699481
700-func (_c *MockDegradedRecoveryCoordinator_Discard_Call) Return(catalogueRecord domain.CatalogueRecord, err error) *MockDegradedRecoveryCoordinator_Discard_Call {
701- _c.Call.Return(catalogueRecord, err)
482+func (_c *MockDegradedRecoveryCoordinator_Discard_Call) Return(err error) *MockDegradedRecoveryCoordinator_Discard_Call {
483+ _c.Call.Return(err)
702484 return _c
703485 }
704486
705-func (_c *MockDegradedRecoveryCoordinator_Discard_Call) RunAndReturn(run func(context1 context.Context, s string, s1 string) (domain.CatalogueRecord, error)) *MockDegradedRecoveryCoordinator_Discard_Call {
487+func (_c *MockDegradedRecoveryCoordinator_Discard_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockDegradedRecoveryCoordinator_Discard_Call {
706488 _c.Call.Return(run)
707489 return _c
708490 }
@@ -1586,212 +1368,6 @@ func (_c *MockSnapshotRepository_Publish_Call) RunAndReturn(run func(context1 co
15861368 return _c
15871369 }
15881370
1589-// QuarantineDeletionSources provides a mock function for the type MockSnapshotRepository
1590-func (_mock *MockSnapshotRepository) QuarantineDeletionSources(context1 context.Context, deletionTombstone domain.DeletionTombstone, b bool) error {
1591- ret := _mock.Called(context1, deletionTombstone, b)
1592-
1593- if len(ret) == 0 {
1594- panic("no return value specified for QuarantineDeletionSources")
1595- }
1596-
1597- var r0 error
1598- if returnFunc, ok := ret.Get(0).(func(context.Context, domain.DeletionTombstone, bool) error); ok {
1599- r0 = returnFunc(context1, deletionTombstone, b)
1600- } else {
1601- r0 = ret.Error(0)
1602- }
1603- return r0
1604-}
1605-
1606-// MockSnapshotRepository_QuarantineDeletionSources_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'QuarantineDeletionSources'
1607-type MockSnapshotRepository_QuarantineDeletionSources_Call struct {
1608- *mock.Call
1609-}
1610-
1611-// QuarantineDeletionSources is a helper method to define mock.On call
1612-// - context1 context.Context
1613-// - deletionTombstone domain.DeletionTombstone
1614-// - b bool
1615-func (_e *MockSnapshotRepository_Expecter) QuarantineDeletionSources(context1 any, deletionTombstone any, b any) *MockSnapshotRepository_QuarantineDeletionSources_Call {
1616- return &MockSnapshotRepository_QuarantineDeletionSources_Call{Call: _e.mock.On("QuarantineDeletionSources", context1, deletionTombstone, b)}
1617-}
1618-
1619-func (_c *MockSnapshotRepository_QuarantineDeletionSources_Call) Run(run func(context1 context.Context, deletionTombstone domain.DeletionTombstone, b bool)) *MockSnapshotRepository_QuarantineDeletionSources_Call {
1620- _c.Call.Run(func(args mock.Arguments) {
1621- var arg0 context.Context
1622- if args[0] != nil {
1623- arg0 = args[0].(context.Context)
1624- }
1625- var arg1 domain.DeletionTombstone
1626- if args[1] != nil {
1627- arg1 = args[1].(domain.DeletionTombstone)
1628- }
1629- var arg2 bool
1630- if args[2] != nil {
1631- arg2 = args[2].(bool)
1632- }
1633- run(
1634- arg0,
1635- arg1,
1636- arg2,
1637- )
1638- })
1639- return _c
1640-}
1641-
1642-func (_c *MockSnapshotRepository_QuarantineDeletionSources_Call) Return(err error) *MockSnapshotRepository_QuarantineDeletionSources_Call {
1643- _c.Call.Return(err)
1644- return _c
1645-}
1646-
1647-func (_c *MockSnapshotRepository_QuarantineDeletionSources_Call) RunAndReturn(run func(context1 context.Context, deletionTombstone domain.DeletionTombstone, b bool) error) *MockSnapshotRepository_QuarantineDeletionSources_Call {
1648- _c.Call.Return(run)
1649- return _c
1650-}
1651-
1652-// QuarantineIncarnation provides a mock function for the type MockSnapshotRepository
1653-func (_mock *MockSnapshotRepository) QuarantineIncarnation(context1 context.Context, incarnationID domain.IncarnationID) error {
1654- ret := _mock.Called(context1, incarnationID)
1655-
1656- if len(ret) == 0 {
1657- panic("no return value specified for QuarantineIncarnation")
1658- }
1659-
1660- var r0 error
1661- if returnFunc, ok := ret.Get(0).(func(context.Context, domain.IncarnationID) error); ok {
1662- r0 = returnFunc(context1, incarnationID)
1663- } else {
1664- r0 = ret.Error(0)
1665- }
1666- return r0
1667-}
1668-
1669-// MockSnapshotRepository_QuarantineIncarnation_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'QuarantineIncarnation'
1670-type MockSnapshotRepository_QuarantineIncarnation_Call struct {
1671- *mock.Call
1672-}
1673-
1674-// QuarantineIncarnation is a helper method to define mock.On call
1675-// - context1 context.Context
1676-// - incarnationID domain.IncarnationID
1677-func (_e *MockSnapshotRepository_Expecter) QuarantineIncarnation(context1 any, incarnationID any) *MockSnapshotRepository_QuarantineIncarnation_Call {
1678- return &MockSnapshotRepository_QuarantineIncarnation_Call{Call: _e.mock.On("QuarantineIncarnation", context1, incarnationID)}
1679-}
1680-
1681-func (_c *MockSnapshotRepository_QuarantineIncarnation_Call) Run(run func(context1 context.Context, incarnationID domain.IncarnationID)) *MockSnapshotRepository_QuarantineIncarnation_Call {
1682- _c.Call.Run(func(args mock.Arguments) {
1683- var arg0 context.Context
1684- if args[0] != nil {
1685- arg0 = args[0].(context.Context)
1686- }
1687- var arg1 domain.IncarnationID
1688- if args[1] != nil {
1689- arg1 = args[1].(domain.IncarnationID)
1690- }
1691- run(
1692- arg0,
1693- arg1,
1694- )
1695- })
1696- return _c
1697-}
1698-
1699-func (_c *MockSnapshotRepository_QuarantineIncarnation_Call) Return(err error) *MockSnapshotRepository_QuarantineIncarnation_Call {
1700- _c.Call.Return(err)
1701- return _c
1702-}
1703-
1704-func (_c *MockSnapshotRepository_QuarantineIncarnation_Call) RunAndReturn(run func(context1 context.Context, incarnationID domain.IncarnationID) error) *MockSnapshotRepository_QuarantineIncarnation_Call {
1705- _c.Call.Return(run)
1706- return _c
1707-}
1708-
1709-// Reconcile provides a mock function for the type MockSnapshotRepository
1710-func (_mock *MockSnapshotRepository) Reconcile(context1 context.Context, catalogueRecords []domain.CatalogueRecord, reconcileCursor ports.ReconcileCursor, maintenanceBudget ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error) {
1711- ret := _mock.Called(context1, catalogueRecords, reconcileCursor, maintenanceBudget)
1712-
1713- if len(ret) == 0 {
1714- panic("no return value specified for Reconcile")
1715- }
1716-
1717- var r0 ports.ReconcileCursor
1718- var r1 []ports.ReconcileFinding
1719- var r2 error
1720- if returnFunc, ok := ret.Get(0).(func(context.Context, []domain.CatalogueRecord, ports.ReconcileCursor, ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error)); ok {
1721- return returnFunc(context1, catalogueRecords, reconcileCursor, maintenanceBudget)
1722- }
1723- if returnFunc, ok := ret.Get(0).(func(context.Context, []domain.CatalogueRecord, ports.ReconcileCursor, ports.MaintenanceBudget) ports.ReconcileCursor); ok {
1724- r0 = returnFunc(context1, catalogueRecords, reconcileCursor, maintenanceBudget)
1725- } else {
1726- r0 = ret.Get(0).(ports.ReconcileCursor)
1727- }
1728- if returnFunc, ok := ret.Get(1).(func(context.Context, []domain.CatalogueRecord, ports.ReconcileCursor, ports.MaintenanceBudget) []ports.ReconcileFinding); ok {
1729- r1 = returnFunc(context1, catalogueRecords, reconcileCursor, maintenanceBudget)
1730- } else {
1731- if ret.Get(1) != nil {
1732- r1 = ret.Get(1).([]ports.ReconcileFinding)
1733- }
1734- }
1735- if returnFunc, ok := ret.Get(2).(func(context.Context, []domain.CatalogueRecord, ports.ReconcileCursor, ports.MaintenanceBudget) error); ok {
1736- r2 = returnFunc(context1, catalogueRecords, reconcileCursor, maintenanceBudget)
1737- } else {
1738- r2 = ret.Error(2)
1739- }
1740- return r0, r1, r2
1741-}
1742-
1743-// MockSnapshotRepository_Reconcile_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Reconcile'
1744-type MockSnapshotRepository_Reconcile_Call struct {
1745- *mock.Call
1746-}
1747-
1748-// Reconcile is a helper method to define mock.On call
1749-// - context1 context.Context
1750-// - catalogueRecords []domain.CatalogueRecord
1751-// - reconcileCursor ports.ReconcileCursor
1752-// - maintenanceBudget ports.MaintenanceBudget
1753-func (_e *MockSnapshotRepository_Expecter) Reconcile(context1 any, catalogueRecords any, reconcileCursor any, maintenanceBudget any) *MockSnapshotRepository_Reconcile_Call {
1754- return &MockSnapshotRepository_Reconcile_Call{Call: _e.mock.On("Reconcile", context1, catalogueRecords, reconcileCursor, maintenanceBudget)}
1755-}
1756-
1757-func (_c *MockSnapshotRepository_Reconcile_Call) Run(run func(context1 context.Context, catalogueRecords []domain.CatalogueRecord, reconcileCursor ports.ReconcileCursor, maintenanceBudget ports.MaintenanceBudget)) *MockSnapshotRepository_Reconcile_Call {
1758- _c.Call.Run(func(args mock.Arguments) {
1759- var arg0 context.Context
1760- if args[0] != nil {
1761- arg0 = args[0].(context.Context)
1762- }
1763- var arg1 []domain.CatalogueRecord
1764- if args[1] != nil {
1765- arg1 = args[1].([]domain.CatalogueRecord)
1766- }
1767- var arg2 ports.ReconcileCursor
1768- if args[2] != nil {
1769- arg2 = args[2].(ports.ReconcileCursor)
1770- }
1771- var arg3 ports.MaintenanceBudget
1772- if args[3] != nil {
1773- arg3 = args[3].(ports.MaintenanceBudget)
1774- }
1775- run(
1776- arg0,
1777- arg1,
1778- arg2,
1779- arg3,
1780- )
1781- })
1782- return _c
1783-}
1784-
1785-func (_c *MockSnapshotRepository_Reconcile_Call) Return(reconcileCursor1 ports.ReconcileCursor, reconcileFindings []ports.ReconcileFinding, err error) *MockSnapshotRepository_Reconcile_Call {
1786- _c.Call.Return(reconcileCursor1, reconcileFindings, err)
1787- return _c
1788-}
1789-
1790-func (_c *MockSnapshotRepository_Reconcile_Call) RunAndReturn(run func(context1 context.Context, catalogueRecords []domain.CatalogueRecord, reconcileCursor ports.ReconcileCursor, maintenanceBudget ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error)) *MockSnapshotRepository_Reconcile_Call {
1791- _c.Call.Return(run)
1792- return _c
1793-}
1794-
17951371 // RepairHEAD provides a mock function for the type MockSnapshotRepository
17961372 func (_mock *MockSnapshotRepository) RepairHEAD(context1 context.Context, incarnationID domain.IncarnationID, v ports.CheckpointRef) error {
17971373 ret := _mock.Called(context1, incarnationID, v)
@@ -1855,63 +1431,6 @@ func (_c *MockSnapshotRepository_RepairHEAD_Call) RunAndReturn(run func(context1
18551431 return _c
18561432 }
18571433
1858-// SaveQuarantineDescriptor provides a mock function for the type MockSnapshotRepository
1859-func (_mock *MockSnapshotRepository) SaveQuarantineDescriptor(context1 context.Context, quarantineDescriptor domain.QuarantineDescriptor) error {
1860- ret := _mock.Called(context1, quarantineDescriptor)
1861-
1862- if len(ret) == 0 {
1863- panic("no return value specified for SaveQuarantineDescriptor")
1864- }
1865-
1866- var r0 error
1867- if returnFunc, ok := ret.Get(0).(func(context.Context, domain.QuarantineDescriptor) error); ok {
1868- r0 = returnFunc(context1, quarantineDescriptor)
1869- } else {
1870- r0 = ret.Error(0)
1871- }
1872- return r0
1873-}
1874-
1875-// MockSnapshotRepository_SaveQuarantineDescriptor_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SaveQuarantineDescriptor'
1876-type MockSnapshotRepository_SaveQuarantineDescriptor_Call struct {
1877- *mock.Call
1878-}
1879-
1880-// SaveQuarantineDescriptor is a helper method to define mock.On call
1881-// - context1 context.Context
1882-// - quarantineDescriptor domain.QuarantineDescriptor
1883-func (_e *MockSnapshotRepository_Expecter) SaveQuarantineDescriptor(context1 any, quarantineDescriptor any) *MockSnapshotRepository_SaveQuarantineDescriptor_Call {
1884- return &MockSnapshotRepository_SaveQuarantineDescriptor_Call{Call: _e.mock.On("SaveQuarantineDescriptor", context1, quarantineDescriptor)}
1885-}
1886-
1887-func (_c *MockSnapshotRepository_SaveQuarantineDescriptor_Call) Run(run func(context1 context.Context, quarantineDescriptor domain.QuarantineDescriptor)) *MockSnapshotRepository_SaveQuarantineDescriptor_Call {
1888- _c.Call.Run(func(args mock.Arguments) {
1889- var arg0 context.Context
1890- if args[0] != nil {
1891- arg0 = args[0].(context.Context)
1892- }
1893- var arg1 domain.QuarantineDescriptor
1894- if args[1] != nil {
1895- arg1 = args[1].(domain.QuarantineDescriptor)
1896- }
1897- run(
1898- arg0,
1899- arg1,
1900- )
1901- })
1902- return _c
1903-}
1904-
1905-func (_c *MockSnapshotRepository_SaveQuarantineDescriptor_Call) Return(err error) *MockSnapshotRepository_SaveQuarantineDescriptor_Call {
1906- _c.Call.Return(err)
1907- return _c
1908-}
1909-
1910-func (_c *MockSnapshotRepository_SaveQuarantineDescriptor_Call) RunAndReturn(run func(context1 context.Context, quarantineDescriptor domain.QuarantineDescriptor) error) *MockSnapshotRepository_SaveQuarantineDescriptor_Call {
1911- _c.Call.Return(run)
1912- return _c
1913-}
1914-
19151434 // WriteDeletionTombstone provides a mock function for the type MockSnapshotRepository
19161435 func (_mock *MockSnapshotRepository) WriteDeletionTombstone(context1 context.Context, deletionTombstone domain.DeletionTombstone) error {
19171436 ret := _mock.Called(context1, deletionTombstone)
M internal/ports/recovery.go +1 -68
@@ -38,8 +38,6 @@ type MaintenanceBudget struct {
3838 Bytes uint64
3939 }
4040
41-type ReconcileCursor struct{ DirectoryCookie uint64 }
42-
4341 type DeletionTombstoneCursor struct{ After string }
4442
4543 type DeletionTombstonePage struct {
@@ -48,69 +46,8 @@ type DeletionTombstonePage struct {
4846 Done bool
4947 }
5048
51-type ReconcileFindingKind uint8
52-
53-const (
54- ReconcileForwardOrphan ReconcileFindingKind = iota + 1
55- ReconcileUnknownIncarnation
56- ReconcileInvalidCandidate
57-)
58-
59-type ReconcileValidationStatus uint8
60-
61-const (
62- ReconcileValidated ReconcileValidationStatus = iota + 1
63- ReconcileQuarantined
64- ReconcileBudgetExhausted
65-)
66-
67-type ReconcileCandidate struct {
68- IncarnationID domain.IncarnationID
69- Name string
70- Ref domain.CheckpointRef
71- Parent *domain.CheckpointRef
72-}
73-
74-type ValidatedCheckpoint struct {
75- Ref domain.CheckpointRef
76- Parent *domain.CheckpointRef
77-}
78-
79-type ReconcileFinding struct {
80- Kind ReconcileFindingKind
81- Status ReconcileValidationStatus
82- Candidate ReconcileCandidate
83- AncestorChain []ValidatedCheckpoint
84- Cursor ReconcileCursor
85- Consumed MaintenanceBudget
86-}
87-
88-type ReconcileDecisionKind uint8
89-
90-const (
91- ReconcileAdopt ReconcileDecisionKind = iota + 1
92- ReconcileKeepQuarantined
93- ReconcileDefer
94-)
95-
96-type ReconcileDecision struct {
97- Kind ReconcileDecisionKind
98- Name string
99- Candidate *domain.CheckpointRef
100- ReasonCode string
101- // RetentionResolved permits collection after a conclusive adoption or
102- // not-forward decision. Its zero value conservatively pins known sessions.
103- RetentionResolved bool
104-}
105-
10649 var ErrBudgetExhausted = errors.New("snapshot maintenance budget exhausted")
10750
108-type RecoveryJournal interface {
109- SaveDiscard(context.Context, domain.DiscardIntent) error
110- ListDiscards(context.Context) ([]domain.DiscardIntent, error)
111- DeleteDiscard(context.Context, domain.IncarnationID) error
112-}
113-
11451 type CheckpointCoordinator interface {
11552 PublishCheckpoint(context.Context, string, SnapshotPublication) (domain.CatalogueRecord, error)
11653 }
@@ -127,7 +64,7 @@ type SessionLifecycleCoordinator interface {
12764
12865 // DegradedRecoveryCoordinator exposes only the explicit operator discard action.
12966 type DegradedRecoveryCoordinator interface {
130- Discard(context.Context, string, string) (domain.CatalogueRecord, error)
67+ Discard(context.Context, string) error
13168 }
13269
13370 type Catalogue interface {
@@ -147,11 +84,7 @@ type SnapshotRepository interface {
14784 RepairHEAD(context.Context, domain.IncarnationID, CheckpointRef) error
14885 WriteDeletionTombstone(context.Context, domain.DeletionTombstone) error
14986 ListDeletionTombstones(context.Context, DeletionTombstoneCursor, MaintenanceBudget) (DeletionTombstonePage, error)
150- QuarantineDeletionSources(context.Context, domain.DeletionTombstone, bool) error
15187 DeleteDeletionTombstone(context.Context, domain.IncarnationID) error
152- SaveQuarantineDescriptor(context.Context, domain.QuarantineDescriptor) error
153- QuarantineIncarnation(context.Context, domain.IncarnationID) error
15488 DeleteIncarnation(context.Context, domain.IncarnationID) error
15589 MaintainSession(context.Context, RetentionPlan, MaintenanceBudget) (bool, error)
156- Reconcile(context.Context, []domain.CatalogueRecord, ReconcileCursor, MaintenanceBudget) (ReconcileCursor, []ReconcileFinding, error)
15790 }
M internal/usecase/daemon/control.go +9 -3
@@ -465,11 +465,17 @@ func (e controlExec) SessionRecovery(action string) (string, error) {
465465 if ctx == nil {
466466 ctx = context.Background()
467467 }
468- record, err := e.d.degradedRecovery.Discard(ctx, e.recoveryName, "explicit operator discard")
469- if err == nil {
468+ if err := e.d.degradedRecovery.Discard(ctx, e.recoveryName); err != nil {
469+ return "", err
470+ }
471+ record, ok, err := e.d.catalogue.Record(e.recoveryName)
472+ if err != nil {
473+ return "", err
474+ }
475+ if ok {
470476 e.d.setStoppedRecovery(record, runtimeFresh)
471477 }
472- return "", err
478+ return "", nil
473479 }
474480
475481 func (e controlExec) ListSessions(asJSON bool) (string, error) {
M internal/usecase/daemon/daemon.go +5 -6
@@ -325,12 +325,11 @@ func WithNoticeStore(store ports.NoticeStore) Option {
325325 return func(d *Daemon) { d.noticeStore = store }
326326 }
327327
328-// WithDurableMaintenance schedules bounded retention and reconciliation after
329-// catalogue-driven restoration. Each call receives an independent budget;
330-// unresolved incarnation IDs remain pinned for this daemon lifetime.
331-func WithDurableMaintenance(catalogue ports.Catalogue, repository ports.SnapshotRepository, reconciler maintenanceReconciler, unresolved []domain.IncarnationID) Option {
328+// WithDurableMaintenance schedules bounded retention after catalogue-driven
329+// restoration. Unresolved incarnation IDs remain pinned for this daemon lifetime.
330+func WithDurableMaintenance(catalogue ports.Catalogue, repository ports.SnapshotRepository, unresolved []domain.IncarnationID) Option {
332331 return func(d *Daemon) {
333- d.maintenance = newMaintenanceDependencies(catalogue, repository, reconciler, unresolved)
332+ d.maintenance = newMaintenanceDependencies(catalogue, repository, unresolved)
334333 }
335334 }
336335
@@ -548,7 +547,7 @@ func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error {
548547 } else {
549548 d.closeRestoreDone()
550549 }
551- if d.maintenance.reconciler != nil {
550+ if d.maintenance.catalogue != nil && d.maintenance.repository != nil {
552551 d.startDurableMaintenance()
553552 }
554553
M internal/usecase/daemon/durable_authority_test.go +2 -2
@@ -18,7 +18,7 @@ type testLifecycleRepository struct{ ports.SnapshotRepository }
1818 func (testLifecycleRepository) WriteDeletionTombstone(context.Context, domain.DeletionTombstone) error {
1919 return nil
2020 }
21-func (testLifecycleRepository) QuarantineDeletionSources(context.Context, domain.DeletionTombstone, bool) error {
21+func (testLifecycleRepository) DeleteIncarnation(context.Context, domain.IncarnationID) error {
2222 return nil
2323 }
2424 func (testLifecycleRepository) DeleteDeletionTombstone(context.Context, domain.IncarnationID) error {
@@ -44,7 +44,7 @@ func WithStore(t testing.TB, store ports.Store) Option {
4444 if repository == nil {
4545 repository = testLifecycleRepository{}
4646 }
47- authority := recoveryusecase.NewCoordinator(catalogue, repository, nil, rand.Reader)
47+ authority := recoveryusecase.NewCoordinator(catalogue, repository, rand.Reader)
4848 d.lifecycleRecovery = authority
4949 d.checkpointRecovery = authority
5050 }
M internal/usecase/daemon/durable_recovery_test.go +10 -9
@@ -192,7 +192,7 @@ func validGeneration(t testing.TB, record domain.CatalogueRecord) ports.Snapshot
192192 func newDurableRecoveryDaemon(t testing.TB, records []domain.CatalogueRecord, repository ports.SnapshotRepository) (*Daemon, *durableRecoveryCatalogue) {
193193 t.Helper()
194194 catalogue := newDurableRecoveryCatalogue(records)
195- coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
195+ coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil)
196196 d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, records), WithSnapshotRepository(repository), WithCheckpointCoordinator(coordinator))
197197 d.serveCtx, d.serveCancel = context.WithCancel(context.Background())
198198 t.Cleanup(d.serveCancel)
@@ -211,7 +211,7 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) {
211211 }
212212 catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{prior})
213213 repository := &durableRecoveryRepository{}
214- coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
214+ coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil)
215215 d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, []domain.CatalogueRecord{prior}), WithSnapshotRepository(repository), WithCheckpointCoordinator(coordinator))
216216 startSnapshotEncodeWorker(t, d)
217217
@@ -352,11 +352,12 @@ func (f *recoveryCountingPTYFactory) Open(context.Context, string, []string, []s
352352 }
353353
354354 type explicitRecoveryStub struct {
355- record domain.CatalogueRecord
355+ record domain.CatalogueRecord
356+ catalogue ports.Catalogue
356357 }
357358
358-func (s explicitRecoveryStub) Discard(context.Context, string, string) (domain.CatalogueRecord, error) {
359- return s.record, nil
359+func (s explicitRecoveryStub) Discard(_ context.Context, name string) error {
360+ return s.catalogue.Replace(name, s.record)
360361 }
361362
362363 type contextRecoveryStub struct {
@@ -366,9 +367,9 @@ type contextRecoveryStub struct {
366367 err error
367368 }
368369
369-func (s *contextRecoveryStub) Discard(ctx context.Context, _, _ string) (domain.CatalogueRecord, error) {
370+func (s *contextRecoveryStub) Discard(ctx context.Context, _ string) error {
370371 s.seen = ctx.Value(s.wantKey) == s.want
371- return domain.CatalogueRecord{}, s.err
372+ return s.err
372373 }
373374
374375 func TestSessionRecoveryUsesIncomingCommandContext(t *testing.T) {
@@ -481,13 +482,13 @@ func TestSessionRecoveryCommand(t *testing.T) {
481482 degraded := durableRecoveryRecord(2)
482483 degraded.RecoveryState = domain.RecoveryDegraded
483484 degraded.DegradedReason = "checkpoint validation failed"
484- d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{degraded}, &durableRecoveryRepository{})
485+ d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{degraded}, &durableRecoveryRepository{})
485486 fresh := degraded
486487 fresh.IncarnationID = domain.IncarnationID{9}
487488 fresh.RecoveryState = domain.RecoveryFresh
488489 fresh.Committed = nil
489490 fresh.DegradedReason = ""
490- d.degradedRecovery = explicitRecoveryStub{record: fresh}
491+ d.degradedRecovery = explicitRecoveryStub{record: fresh, catalogue: catalogue}
491492
492493 _, err := (controlExec{d: d, recoveryName: degraded.Name}).SessionRecovery("discard")
493494 require.NoError(t, err)
M internal/usecase/daemon/maintenance.go +8 -57
@@ -14,27 +14,19 @@ const (
1414 maintenanceBytes = 8 << 20
1515 )
1616
17-type maintenanceReconciler interface {
18- Step(context.Context, ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error)
19-}
20-
2117 // maintenanceDependencies is exclusively owned by runDurableMaintenance's
2218 // single goroutine after startup. Shutdown joins that goroutine before these
2319 // cursor and repair sets can be observed or reused.
2420 type maintenanceDependencies struct {
25- catalogue ports.Catalogue
26- repository ports.SnapshotRepository
27- reconciler maintenanceReconciler
28- unresolved map[domain.IncarnationID]struct{}
29- repairUncertain map[domain.IncarnationID]struct{}
30- cursor ports.ReconcileCursor
21+ catalogue ports.Catalogue
22+ repository ports.SnapshotRepository
23+ unresolved map[domain.IncarnationID]struct{}
3124 }
3225
33-func newMaintenanceDependencies(catalogue ports.Catalogue, repository ports.SnapshotRepository, reconciler maintenanceReconciler, unresolved []domain.IncarnationID) maintenanceDependencies {
26+func newMaintenanceDependencies(catalogue ports.Catalogue, repository ports.SnapshotRepository, unresolved []domain.IncarnationID) maintenanceDependencies {
3427 deps := maintenanceDependencies{
35- catalogue: catalogue, repository: repository, reconciler: reconciler,
36- unresolved: make(map[domain.IncarnationID]struct{}, len(unresolved)),
37- repairUncertain: make(map[domain.IncarnationID]struct{}),
28+ catalogue: catalogue, repository: repository,
29+ unresolved: make(map[domain.IncarnationID]struct{}, len(unresolved)),
3830 }
3931 for _, id := range unresolved {
4032 if id != (domain.IncarnationID{}) {
@@ -107,49 +99,10 @@ func (d *Daemon) runDurableMaintenance(ctx context.Context) {
10799 // continued immediately rather than delayed until the periodic interval.
108100 func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
109101 deps := &d.maintenance
110- if deps.catalogue == nil || deps.repository == nil || deps.reconciler == nil {
102+ if deps.catalogue == nil || deps.repository == nil {
111103 return false
112104 }
113105
114- // Reconcile before collection. Until a complete top-level scan reaches EOF,
115- // an unseen forward orphan may still be repairable, so retention yields
116- // without deleting any incarnation data.
117- previousCursor := deps.cursor
118- next, decisions, err := deps.reconciler.Step(ctx, previousCursor)
119- if err != nil {
120- d.log.Warn("durable_reconciliation_tick", "action", "scan", "cursor", deps.cursor.DirectoryCookie, "done", false, "err", err)
121- return false
122- }
123- deps.cursor = next
124- for _, decision := range decisions {
125- record, ok, err := deps.catalogue.Record(decision.Name)
126- if err != nil {
127- d.log.Warn("durable_reconciliation_tick", "action", "catalogue-read", "done", false, "err", err)
128- return false
129- }
130- if ok && !decision.RetentionResolved {
131- deps.repairUncertain[record.IncarnationID] = struct{}{}
132- }
133- // The catalogue lookup can miss (e.g. an unknown-incarnation finding for a
134- // name the catalogue never held); logging record.IncarnationID.String() in
135- // that case would emit a misleading all-zero incarnation, so the field is
136- // omitted rather than logged as a fabricated identity.
137- args := []any{
138- "session", decision.Name,
139- "action", decision.Kind,
140- "reason_code", decision.ReasonCode,
141- "cursor", next.DirectoryCookie,
142- }
143- if ok {
144- args = append([]any{"incarnation", record.IncarnationID.String()}, args...)
145- }
146- d.log.Info("durable_reconciliation_decision", args...)
147- }
148- d.log.Info("durable_reconciliation_tick", "action", "scan", "cursor", next.DirectoryCookie, "done", next.DirectoryCookie == 0, "err", nil)
149- if next.DirectoryCookie != 0 {
150- return next.DirectoryCookie != previousCursor.DirectoryCookie
151- }
152-
153106 records, err := deps.catalogue.Records()
154107 if err != nil {
155108 d.log.Warn("durable_maintenance_tick", "action", "catalogue-read", "done", false, "err", err)
@@ -170,9 +123,8 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
170123 return false
171124 }
172125 _, unresolved := deps.unresolved[record.IncarnationID]
173- _, uncertainRepair := deps.repairUncertain[record.IncarnationID]
174126 _, restoring := restoringIncarnations[record.IncarnationID]
175- plan := retentionPlan(record, unresolved || uncertainRepair, restoring)
127+ plan := retentionPlan(record, unresolved, restoring)
176128 done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes})
177129 if err == nil && !done {
178130 incomplete = true
@@ -187,6 +139,5 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
187139 "err", err,
188140 )
189141 }
190- clear(deps.repairUncertain)
191142 return incomplete
192143 }
M internal/usecase/daemon/maintenance_test.go +6 -152
@@ -2,46 +2,16 @@ package daemon
22
33 import (
44 "context"
5- "errors"
65 "testing"
76
87 "github.com/bnema/vev/internal/domain"
98 "github.com/bnema/vev/internal/ports"
109 portsmocks "github.com/bnema/vev/internal/ports/mocks"
11- recoveryusecase "github.com/bnema/vev/internal/usecase/recovery"
1210 "github.com/stretchr/testify/mock"
1311 "github.com/stretchr/testify/require"
1412 )
1513
16-type cursorMaintenanceReconciler struct {
17- seen []ports.ReconcileCursor
18-}
19-
20-func (r *cursorMaintenanceReconciler) Step(_ context.Context, cursor ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error) {
21- r.seen = append(r.seen, cursor)
22- if cursor.DirectoryCookie == 0 {
23- return ports.ReconcileCursor{DirectoryCookie: 7}, nil, nil
24- }
25- return ports.ReconcileCursor{}, nil, nil
26-}
27-
28-type stalledMaintenanceReconciler struct{}
29-
30-func (stalledMaintenanceReconciler) Step(_ context.Context, cursor ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error) {
31- if cursor.DirectoryCookie == 0 {
32- return ports.ReconcileCursor{DirectoryCookie: 7}, nil, nil
33- }
34- return cursor, nil, nil
35-}
36-
37-func TestMaintenanceDoesNotImmediatelyRetryStalledCursor(t *testing.T) {
38- d := newTestDaemon(t, nil, stubClock{})
39- d.maintenance = newMaintenanceDependencies(portsmocks.NewMockCatalogue(t), portsmocks.NewMockSnapshotRepository(t), stalledMaintenanceReconciler{}, nil)
40- require.True(t, d.runDurableMaintenanceTick(context.Background()))
41- require.False(t, d.runDurableMaintenanceTick(context.Background()))
42-}
43-
44-func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) {
14+func TestProductionMaintenanceUsesCatalogue(t *testing.T) {
4515 record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}
4616 catalogue := portsmocks.NewMockCatalogue(t)
4717 catalogue.EXPECT().Records().Return([]domain.CatalogueRecord{record}, nil).Once()
@@ -49,124 +19,10 @@ func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) {
4919 repository.EXPECT().MaintainSession(mock.Anything, mock.MatchedBy(func(plan ports.RetentionPlan) bool {
5020 return plan.IncarnationID == record.IncarnationID && !plan.PinAll && len(plan.Keep) == 0
5121 }), ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes}).Return(true, nil).Once()
52- reconciler := &cursorMaintenanceReconciler{}
5322 d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
54- WithDurableMaintenance(catalogue, repository, reconciler, nil)(d)
23+ WithDurableMaintenance(catalogue, repository, nil)(d)
5524
56- require.True(t, d.runDurableMaintenanceTick(context.Background()), "partial reconciliation must continue immediately")
5725 require.False(t, d.runDurableMaintenanceTick(context.Background()))
58- require.True(t, d.runDurableMaintenanceTick(context.Background()), "the next partial pass must also continue immediately")
59- if len(reconciler.seen) != 3 || reconciler.seen[0].DirectoryCookie != 0 || reconciler.seen[1].DirectoryCookie != 7 || reconciler.seen[2].DirectoryCookie != 0 {
60- t.Fatalf("reconciliation cursors = %v, want [0 7 0]", reconciler.seen)
61- }
62-}
63-
64-type reconciliationMaintenanceRepository struct {
65- ports.SnapshotRepository
66- findings []ports.ReconcileFinding
67- err error
68- plans []ports.RetentionPlan
69-}
70-
71-func (r *reconciliationMaintenanceRepository) Reconcile(context.Context, []domain.CatalogueRecord, ports.ReconcileCursor, ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error) {
72- return ports.ReconcileCursor{}, r.findings, r.err
73-}
74-
75-func (r *reconciliationMaintenanceRepository) MaintainSession(_ context.Context, plan ports.RetentionPlan, _ ports.MaintenanceBudget) (bool, error) {
76- r.plans = append(r.plans, plan)
77- return true, nil
78-}
79-
80-func TestMaintenanceRetentionUsesReconciliationOutcome(t *testing.T) {
81- committed := domain.CheckpointRef{Generation: 3, ManifestDigest: [32]byte{3}}
82- fallback1 := domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}}
83- fallback2 := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}
84- record := domain.CatalogueRecord{
85- Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryHealthy,
86- Committed: &committed,
87- }
88- wrongParent := domain.CheckpointRef{Generation: 3, ManifestDigest: [32]byte{9}}
89-
90- for _, tc := range []struct {
91- name string
92- finding ports.ReconcileFinding
93- reconcileErr error
94- wantPlan *ports.RetentionPlan
95- }{
96- {
97- name: "current HEAD is conclusively not forward",
98- finding: ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
99- Name: record.Name, IncarnationID: record.IncarnationID, Ref: committed, Parent: &fallback1,
100- }},
101- wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed}},
102- },
103- {
104- name: "known predecessor is conclusively not forward",
105- finding: ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
106- Name: record.Name, IncarnationID: record.IncarnationID, Ref: fallback1, Parent: &fallback2,
107- }},
108- wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed}},
109- },
110- {
111- name: "non-forward candidate kind remains pinned",
112- finding: ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
113- Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}},
114- }},
115- wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
116- },
117- {
118- name: "unreadable candidate remains pinned",
119- finding: ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileQuarantined, Candidate: ports.ReconcileCandidate{
120- Name: record.Name, IncarnationID: record.IncarnationID,
121- }},
122- wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
123- },
124- {
125- name: "quarantined forward candidate remains pinned",
126- finding: ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileQuarantined, Candidate: ports.ReconcileCandidate{
127- Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}}, Parent: &wrongParent,
128- }},
129- wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
130- },
131- {
132- name: "incarnation mismatch remains pinned",
133- finding: ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{
134- Name: record.Name, IncarnationID: domain.IncarnationID{9}, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}}, Parent: &committed,
135- }},
136- wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
137- },
138- {
139- name: "incomplete decision remains pinned",
140- finding: ports.ReconcileFinding{Status: ports.ReconcileBudgetExhausted, Candidate: ports.ReconcileCandidate{
141- Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}},
142- }},
143- wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true},
144- },
145- {
146- name: "reconciliation error withholds maintenance",
147- reconcileErr: errors.New("scan failed"),
148- },
149- } {
150- t.Run(tc.name, func(t *testing.T) {
151- catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record})
152- repository := &reconciliationMaintenanceRepository{err: tc.reconcileErr}
153- if tc.finding.Candidate.Name != "" {
154- repository.findings = []ports.ReconcileFinding{tc.finding}
155- }
156- coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
157- reconciler := recoveryusecase.NewReconciler(coordinator, catalogue, repository, ports.MaintenanceBudget{Entries: 8, Bytes: 1024})
158- d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
159- WithDurableMaintenance(catalogue, repository, reconciler, nil)(d)
160-
161- d.runDurableMaintenanceTick(context.Background())
162-
163- if tc.wantPlan == nil {
164- require.Empty(t, repository.plans)
165- return
166- }
167- require.Equal(t, []ports.RetentionPlan{*tc.wantPlan}, repository.plans)
168- })
169- }
17026 }
17127
17228 func TestRetentionPlanPinsEveryUnresolvedRuntimeState(t *testing.T) {
@@ -184,18 +40,16 @@ func TestRetentionPlanPinsEveryUnresolvedRuntimeState(t *testing.T) {
18440 wantPin bool
18541 }{
18642 {name: "healthy", record: healthy},
187- {name: "pending journal tombstone or repair", record: healthy, unresolved: true, wantPin: true},
43+ {name: "pending deletion", record: healthy, unresolved: true, wantPin: true},
18844 {name: "restoring", record: healthy, restoring: true, wantPin: true},
18945 {name: "degraded", record: func() domain.CatalogueRecord { r := healthy; r.RecoveryState = domain.RecoveryDegraded; return r }(), wantPin: true},
19046 {name: "deleting", record: func() domain.CatalogueRecord { r := healthy; r.RecoveryState = domain.RecoveryDeleting; return r }(), wantPin: true},
19147 } {
19248 t.Run(tc.name, func(t *testing.T) {
19349 plan := retentionPlan(tc.record, tc.unresolved, tc.restoring)
194- if plan.PinAll != tc.wantPin {
195- t.Fatalf("PinAll = %v, want %v", plan.PinAll, tc.wantPin)
196- }
197- if !tc.wantPin && len(plan.Keep) != 1 {
198- t.Fatalf("healthy keep set = %v, want committed only", plan.Keep)
50+ require.Equal(t, tc.wantPin, plan.PinAll)
51+ if !tc.wantPin {
52+ require.Equal(t, []ports.CheckpointRef{committed}, plan.Keep)
19953 }
20054 })
20155 }
M internal/usecase/daemon/picker_test.go +2 -2
@@ -33,7 +33,7 @@ type refusingSnapshotDeleteRepository struct {
3333 func (refusingSnapshotDeleteRepository) Publish(context.Context, ports.SnapshotPublication) error {
3434 return nil
3535 }
36-func (s refusingSnapshotDeleteRepository) QuarantineDeletionSources(context.Context, domain.DeletionTombstone, bool) error {
36+func (s refusingSnapshotDeleteRepository) DeleteIncarnation(context.Context, domain.IncarnationID) error {
3737 return s.err
3838 }
3939
@@ -272,7 +272,7 @@ func TestRestoreCancellationTransitionsBeforePickerCompletion(t *testing.T) {
272272 record.Name = "restoring"
273273 repository := &cancellationRecoveryRepository{started: make(chan struct{})}
274274 catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record})
275- coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
275+ coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil)
276276 WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d)
277277 WithSnapshotRepository(repository)(d)
278278 WithCheckpointCoordinator(coordinator)(d)
M internal/usecase/daemon/session_purge_control_test.go +2 -2
@@ -26,11 +26,11 @@ func TestStoppedPurgeMetadataFailureRemainsFencedForRetry(t *testing.T) {
2626
2727 require.ErrorIs(t, d.retryStoppedPurge("work"), metadataErr)
2828 require.True(t, repository.tombstoned["work"])
29- require.Equal(t, []string{"tombstone", "incremental", "legacy"}, repository.calls)
29+ require.Equal(t, []string{"tombstone", "delete incarnation"}, repository.calls)
3030
3131 state.mu.Lock()
3232 state.deleteErr = nil
3333 state.mu.Unlock()
3434 require.NoError(t, d.retryStoppedPurge("work"))
35- require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls)
35+ require.Equal(t, []string{"tombstone", "delete incarnation", "tombstone", "delete incarnation", "clear tombstone"}, repository.calls)
3636 }
M internal/usecase/daemon/session_purge_retry_test.go +32 -62
@@ -14,8 +14,7 @@ import (
1414
1515 type retryablePurgeRepository struct {
1616 noOpSnapshotRepository
17- incrementalErr error
18- legacyErr error
17+ deleteErr error
1918 tombstoned map[string]bool
2019 calls []string
2120 deadlineScoped []bool
@@ -24,6 +23,7 @@ type retryablePurgeRepository struct {
2423 func (r *retryablePurgeRepository) Publish(context.Context, ports.SnapshotPublication) error {
2524 return nil
2625 }
26+
2727 func (r *retryablePurgeRepository) WriteDeletionTombstone(ctx context.Context, tombstone domain.DeletionTombstone) error {
2828 _, hasDeadline := ctx.Deadline()
2929 r.deadlineScoped = append(r.deadlineScoped, hasDeadline)
@@ -34,76 +34,46 @@ func (r *retryablePurgeRepository) WriteDeletionTombstone(ctx context.Context, t
3434 r.tombstoned[tombstone.Name] = true
3535 return nil
3636 }
37-func (r *retryablePurgeRepository) QuarantineDeletionSources(ctx context.Context, tombstone domain.DeletionTombstone, _ bool) error {
37+
38+func (r *retryablePurgeRepository) DeleteIncarnation(ctx context.Context, _ domain.IncarnationID) error {
3839 _, hasDeadline := ctx.Deadline()
3940 r.deadlineScoped = append(r.deadlineScoped, hasDeadline)
40- r.calls = append(r.calls, "incremental")
41- if r.incrementalErr != nil {
42- return r.incrementalErr
43- }
44- r.calls = append(r.calls, "legacy")
45- return r.legacyErr
41+ r.calls = append(r.calls, "delete incarnation")
42+ return r.deleteErr
4643 }
44+
4745 func (r *retryablePurgeRepository) DeleteDeletionTombstone(ctx context.Context, _ domain.IncarnationID) error {
4846 _, hasDeadline := ctx.Deadline()
4947 r.deadlineScoped = append(r.deadlineScoped, hasDeadline)
5048 r.calls = append(r.calls, "clear tombstone")
51- for name := range r.tombstoned {
52- delete(r.tombstoned, name)
53- }
49+ clear(r.tombstoned)
5450 return nil
5551 }
56-func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) {
57- for _, tt := range []struct {
58- name string
59- incremental error
60- legacy error
61- }{
62- {name: "incremental failure", incremental: errors.New("incremental failed")},
63- {name: "legacy failure", legacy: errors.New("legacy failed")},
64- } {
65- t.Run(tt.name, func(t *testing.T) {
66- d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
67- repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy}
68- WithSnapshotRepository(repository)(d)
69- store, _ := newMockStore(t)
70- WithStore(t, store)(d)
71- sess := newSnapshotTestSession(t, "work", false, "/work")
72- record := sess.persistRecordLocked(1)
73- record.RecoveryState = domain.RecoveryFresh
74- require.NoError(t, d.catalogue.Create(record))
75- pty, ok := sess.tabs[0].panes["pane-1"].pty.(*portsmocks.MockPTY)
76- require.True(t, ok, "snapshot test pane must use MockPTY")
77- pty.EXPECT().Close().Return(nil).Once()
78- d.sessions = map[domain.SessionID]*session{sess.id: sess}
7952
80- require.Error(t, d.killSession(sess, ports.ReasonSessionKilled, true))
81- require.True(t, repository.tombstoned["work"], "partial purge must fence startup restore/import")
82- wantCalls := []string{"tombstone", "incremental"}
83- if tt.incremental == nil {
84- wantCalls = append(wantCalls, "legacy")
85- }
86- require.Equal(t, wantCalls, repository.calls)
87- d.mu.Lock()
88- stopped, ok := d.stopped["work"]
89- d.mu.Unlock()
90- require.True(t, ok)
91- require.True(t, stopped.purging)
53+func TestLivePurgeRetainsTombstoneAcrossPartialDeletion(t *testing.T) {
54+ d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
55+ repository := &retryablePurgeRepository{deleteErr: errors.New("delete failed")}
56+ WithSnapshotRepository(repository)(d)
57+ store, _ := newMockStore(t)
58+ WithStore(t, store)(d)
59+ sess := newSnapshotTestSession(t, "work", false, "/work")
60+ record := sess.persistRecordLocked(1)
61+ record.RecoveryState = domain.RecoveryFresh
62+ require.NoError(t, d.catalogue.Create(record))
63+ pty, ok := sess.tabs[0].panes["pane-1"].pty.(*portsmocks.MockPTY)
64+ require.True(t, ok)
65+ pty.EXPECT().Close().Return(nil).Once()
66+ d.sessions = map[domain.SessionID]*session{sess.id: sess}
67+
68+ require.Error(t, d.killSession(sess, ports.ReasonSessionKilled, true))
69+ require.True(t, repository.tombstoned["work"])
70+ require.Equal(t, []string{"tombstone", "delete incarnation"}, repository.calls)
9271
93- repository.incrementalErr = nil
94- repository.legacyErr = nil
95- require.NoError(t, d.retryStoppedPurge("work"))
96- require.False(t, repository.tombstoned["work"])
97- wantCalls = append(wantCalls, "tombstone", "incremental", "legacy", "clear tombstone")
98- require.Equal(t, wantCalls, repository.calls)
99- require.NotEmpty(t, repository.deadlineScoped)
100- for _, scoped := range repository.deadlineScoped {
101- require.True(t, scoped, "every purge repository call must receive a deadline-scoped context")
102- }
103- d.mu.Lock()
104- _, ok = d.stopped["work"]
105- d.mu.Unlock()
106- require.False(t, ok, "metadata is removed only after both source deletes and tombstone clear")
107- })
72+ repository.deleteErr = nil
73+ require.NoError(t, d.retryStoppedPurge("work"))
74+ require.False(t, repository.tombstoned["work"])
75+ require.Equal(t, []string{"tombstone", "delete incarnation", "tombstone", "delete incarnation", "clear tombstone"}, repository.calls)
76+ for _, scoped := range repository.deadlineScoped {
77+ require.True(t, scoped)
10878 }
10979 }
M internal/usecase/daemon/session_rename_purge_test.go +16 -28
@@ -1,7 +1,6 @@
11 package daemon
22
33 import (
4- "errors"
54 "testing"
65
76 "github.com/stretchr/testify/require"
@@ -11,32 +10,21 @@ import (
1110 )
1211
1312 func TestRenamePreservesIncarnationSnapshotSources(t *testing.T) {
14- legacyDeleteErr := errors.New("legacy delete failed")
15- for _, tt := range []struct {
16- name string
17- legacyErr error
18- }{
19- {name: "snapshot sources available"},
20- {name: "unrelated legacy source delete would fail", legacyErr: legacyDeleteErr},
21- } {
22- t.Run(tt.name, func(t *testing.T) {
23- d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
24- repository := &retryablePurgeRepository{legacyErr: tt.legacyErr}
25- WithSnapshotRepository(repository)(d)
26- store, state := newMockStore(t)
27- WithStore(t, store)(d)
28- sess := newSnapshotTestSession(t, "old", false, "/work")
29- d.sessions = map[domain.SessionID]*session{sess.id: sess}
30- sess.mu.Lock()
31- record := sess.persistRecordLocked(1)
32- sess.mu.Unlock()
33- require.NoError(t, testPersister(t, d).Save(record))
13+ d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
14+ repository := &retryablePurgeRepository{}
15+ WithSnapshotRepository(repository)(d)
16+ store, state := newMockStore(t)
17+ WithStore(t, store)(d)
18+ sess := newSnapshotTestSession(t, "old", false, "/work")
19+ d.sessions = map[domain.SessionID]*session{sess.id: sess}
20+ sess.mu.Lock()
21+ record := sess.persistRecordLocked(1)
22+ sess.mu.Unlock()
23+ require.NoError(t, testPersister(t, d).Save(record))
3424
35- require.NoError(t, d.renameSession(sess, "new"))
36- require.Empty(t, repository.calls, "rename must not invoke source deletion, including a failing legacy source")
37- require.False(t, state.has("old"))
38- require.True(t, state.has("new"))
39- require.Equal(t, sess.incarnation, state.record(t, "new").IncarnationID)
40- })
41- }
25+ require.NoError(t, d.renameSession(sess, "new"))
26+ require.Empty(t, repository.calls, "rename must not delete incarnation-keyed snapshots")
27+ require.False(t, state.has("old"))
28+ require.True(t, state.has("new"))
29+ require.Equal(t, sess.incarnation, state.record(t, "new").IncarnationID)
4230 }
M internal/usecase/daemon/snapshot_restore_test.go +2 -2
@@ -223,7 +223,7 @@ func TestRestoreIncrementalGenerationAcceptance(t *testing.T) {
223223 catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record})
224224 WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d)
225225 WithSnapshotRepository(repository)(d)
226- WithRecoveryCoordinator(recoveryusecase.NewCoordinator(catalogue, repository, nil, nil))(d)
226+ WithRecoveryCoordinator(recoveryusecase.NewCoordinator(catalogue, repository, nil))(d)
227227 t.Cleanup(func() { release(); d.sessWg.Wait() })
228228
229229 d.restoreIncrementalSnapshots(context.Background())
@@ -272,7 +272,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) {
272272
273273 pty, release := newBlockingPTY(t)
274274 d := newTestDaemon(t, newFactory(t, pty), stubClock{})
275- coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil)
275+ coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil)
276276 WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d)
277277 WithSnapshotRepository(repository)(d)
278278 WithRecoveryCoordinator(coordinator)(d)
M internal/usecase/daemon/snapshot_test.go +0 -12
@@ -107,27 +107,15 @@ func (noOpSnapshotRepository) WriteDeletionTombstone(context.Context, domain.Del
107107 func (noOpSnapshotRepository) ListDeletionTombstones(context.Context, ports.DeletionTombstoneCursor, ports.MaintenanceBudget) (ports.DeletionTombstonePage, error) {
108108 return ports.DeletionTombstonePage{Done: true}, nil
109109 }
110-func (noOpSnapshotRepository) QuarantineDeletionSources(context.Context, domain.DeletionTombstone, bool) error {
111- return nil
112-}
113110 func (noOpSnapshotRepository) DeleteDeletionTombstone(context.Context, domain.IncarnationID) error {
114111 return nil
115112 }
116-func (noOpSnapshotRepository) SaveQuarantineDescriptor(context.Context, domain.QuarantineDescriptor) error {
117- return nil
118-}
119-func (noOpSnapshotRepository) QuarantineIncarnation(context.Context, domain.IncarnationID) error {
120- return nil
121-}
122113 func (noOpSnapshotRepository) DeleteIncarnation(context.Context, domain.IncarnationID) error {
123114 return nil
124115 }
125116 func (noOpSnapshotRepository) MaintainSession(context.Context, ports.RetentionPlan, ports.MaintenanceBudget) (bool, error) {
126117 return true, nil
127118 }
128-func (noOpSnapshotRepository) Reconcile(context.Context, []domain.CatalogueRecord, ports.ReconcileCursor, ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error) {
129- return ports.ReconcileCursor{}, nil, nil
130-}
131119
132120 func awaitSnapshotIdle(t testing.TB, sess *session) {
133121 t.Helper()
M internal/usecase/recovery/checkpoint_test.go +2 -2
@@ -112,7 +112,7 @@ func runCheckpointCommit(t *testing.T, count int) (domain.CatalogueRecord, *chec
112112 events := make([]string, 0, count*2)
113113 catalogue := &checkpointCatalogue{record: checkpointRecord(), events: &events}
114114 repository := &checkpointRepository{events: &events}
115- coordinator := NewCoordinator(catalogue, repository, nil, nil)
115+ coordinator := NewCoordinator(catalogue, repository, nil)
116116 for generation := uint64(1); generation <= uint64(count); generation++ {
117117 var err error
118118 catalogue.record, err = coordinator.PublishCheckpoint(context.Background(), "work", checkpointPublication(catalogue.record, generation))
@@ -153,7 +153,7 @@ func TestCheckpointCommitPublishOrphan(t *testing.T) {
153153 cause := errors.New("catalogue sync failed")
154154 catalogue := &checkpointCatalogue{record: prior, replaceErr: cause}
155155 repository := &checkpointRepository{}
156- coordinator := NewCoordinator(catalogue, repository, nil, nil)
156+ coordinator := NewCoordinator(catalogue, repository, nil)
157157 publication := checkpointPublication(prior, 8)
158158
159159 _, err := coordinator.PublishCheckpoint(context.Background(), prior.Name, publication)
M internal/usecase/recovery/coordinator.go +12 -34
@@ -98,15 +98,13 @@ var (
9898 // Degraded reasons are content-free by construction: they never carry paths,
9999 // terminal content, or operator input.
100100 const (
101- degradedReasonDiscardIntent = "discard intent conflict"
102- degradedReasonTombstone = "deletion tombstone conflict"
103- degradedReasonRecord = "catalogue record conflict"
101+ degradedReasonTombstone = "deletion tombstone conflict"
102+ degradedReasonRecord = "catalogue record conflict"
104103 )
105104
106105 type Coordinator struct {
107106 catalogue ports.Catalogue
108107 repository ports.SnapshotRepository
109- journal ports.RecoveryJournal
110108 locks *KeyLocks
111109 random io.Reader
112110
@@ -133,8 +131,8 @@ func (c *Coordinator) Conflicts() []RecoveryConflict {
133131 return append([]RecoveryConflict(nil), c.conflicts...)
134132 }
135133
136-func NewCoordinator(catalogue ports.Catalogue, repository ports.SnapshotRepository, journal ports.RecoveryJournal, random io.Reader) *Coordinator {
137- return &Coordinator{catalogue: catalogue, repository: repository, journal: journal, locks: NewKeyLocks(), random: random}
134+func NewCoordinator(catalogue ports.Catalogue, repository ports.SnapshotRepository, random io.Reader) *Coordinator {
135+ return &Coordinator{catalogue: catalogue, repository: repository, locks: NewKeyLocks(), random: random}
138136 }
139137
140138 // Create commits fresh catalogue metadata before the caller creates or exposes
@@ -244,7 +242,7 @@ func (c *Coordinator) deleteLocked(ctx context.Context, record domain.CatalogueR
244242 if err := c.repository.WriteDeletionTombstone(ctx, tombstone); err != nil {
245243 return err
246244 }
247- if err := c.repository.QuarantineDeletionSources(ctx, tombstone, true); err != nil {
245+ if err := c.repository.DeleteIncarnation(ctx, record.IncarnationID); err != nil {
248246 return err
249247 }
250248 if err := c.catalogue.Delete(record.Name); err != nil {
@@ -260,29 +258,16 @@ func (c *Coordinator) deleteLocked(ctx context.Context, record domain.CatalogueR
260258 // A per-item failure that only proves one session's durable state disagrees
261259 // with itself fences that session and lets startup continue: healthy sessions
262260 // must still restore next to a corrupt neighbour. Infrastructure failures
263-// (listing, journal, repository, or catalogue IO, and a non-advancing cursor)
264-// still abort startup, because they say nothing about any single session.
261+// (listing, repository, or catalogue IO, and a non-advancing cursor) still
262+// abort startup, because they say nothing about any single session.
265263 func (c *Coordinator) Recover(ctx context.Context) error {
266- if c == nil || c.catalogue == nil || c.repository == nil || c.journal == nil || c.locks == nil {
264+ if c == nil || c.catalogue == nil || c.repository == nil || c.locks == nil {
267265 return errors.New("recovery: incomplete coordinator dependencies")
268266 }
269267 c.conflictMu.Lock()
270268 c.conflicts = nil
271269 c.conflictMu.Unlock()
272270
273- intents, err := c.journal.ListDiscards(ctx)
274- if err != nil {
275- return err
276- }
277- for _, intent := range intents {
278- // The intent is deliberately retained for a fenced session: only a
279- // successful roll-forward may remove it.
280- if err := c.recoverDiscard(ctx, intent); err != nil {
281- if err := c.fenceSession(intent.SessionName, "discard-intent", degradedReasonDiscardIntent, err); err != nil {
282- return err
283- }
284- }
285- }
286271 records, err := c.catalogue.Records()
287272 if err != nil {
288273 return err
@@ -370,8 +355,7 @@ func (c *Coordinator) fenceSession(name, kind, reason string, cause error) error
370355 // durable state is self-inconsistent. Everything else - IO, decode, traversal,
371356 // context cancellation - is treated as infrastructure and aborts startup.
372357 func isSessionScopedConflict(err error) bool {
373- return errors.Is(err, ErrDiscardConflict) || errors.Is(err, ErrDiscardIntentInvalid) ||
374- errors.Is(err, ErrDeletionTombstoneConflict) || errors.Is(err, ErrDeletionTombstoneInvalid) ||
358+ return errors.Is(err, ErrDeletionTombstoneConflict) || errors.Is(err, ErrDeletionTombstoneInvalid) ||
375359 errors.Is(err, ErrSessionRecordInvalid)
376360 }
377361
@@ -385,16 +369,10 @@ func (c *Coordinator) recoverDeletionTombstone(ctx context.Context, tombstone do
385369 if err != nil {
386370 return err
387371 }
388- includeLegacyName := true
389- if exists {
390- switch {
391- case record.IncarnationID != tombstone.IncarnationID:
392- includeLegacyName = false
393- case record.RecoveryState != domain.RecoveryDeleting:
394- return fmt.Errorf("%w: non-deleting session %q", ErrDeletionTombstoneConflict, tombstone.Name)
395- }
372+ if exists && record.IncarnationID == tombstone.IncarnationID && record.RecoveryState != domain.RecoveryDeleting {
373+ return fmt.Errorf("%w: non-deleting session %q", ErrDeletionTombstoneConflict, tombstone.Name)
396374 }
397- if err := c.repository.QuarantineDeletionSources(ctx, tombstone, includeLegacyName); err != nil {
375+ if err := c.repository.DeleteIncarnation(ctx, tombstone.IncarnationID); err != nil {
398376 return err
399377 }
400378 if exists && record.IncarnationID == tombstone.IncarnationID {
M internal/usecase/recovery/coordinator_test.go +2 -35
@@ -5,59 +5,26 @@ import (
55 "errors"
66 "testing"
77
8- "github.com/bnema/vev/internal/domain"
98 "github.com/bnema/vev/internal/ports"
109 portsmocks "github.com/bnema/vev/internal/ports/mocks"
1110 "github.com/stretchr/testify/require"
1211 )
1312
14-type journalStub struct {
15- intents []domain.DiscardIntent
16- err error
17-}
18-
19-func (j journalStub) SaveDiscard(context.Context, domain.DiscardIntent) error { return nil }
20-func (j journalStub) ListDiscards(context.Context) ([]domain.DiscardIntent, error) {
21- return j.intents, j.err
22-}
23-func (j journalStub) DeleteDiscard(context.Context, domain.IncarnationID) error { return nil }
24-
2513 func TestDeleteChecksCancellationBeforeCatalogueAccess(t *testing.T) {
2614 ctx, cancel := context.WithCancel(t.Context())
2715 cancel()
2816 catalogue := portsmocks.NewMockCatalogue(t)
2917 repository := portsmocks.NewMockSnapshotRepository(t)
30- coordinator := NewCoordinator(catalogue, repository, journalStub{}, nil)
18+ coordinator := NewCoordinator(catalogue, repository, nil)
3119
3220 require.ErrorIs(t, coordinator.Delete(ctx, "work"), context.Canceled)
3321 }
3422
35-func TestCoordinatorRecoverRollsPendingDiscardForward(t *testing.T) {
36- ctx := context.Background()
37- old := domain.CatalogueRecord{
38- Name: "broken", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryDegraded,
39- Committed: &domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}, DegradedReason: "corrupt",
40- }
41- intent := domain.DiscardIntent{
42- OldRecord: old, OldIncarnation: old.IncarnationID, NewIncarnation: domain.IncarnationID{2},
43- SessionName: old.Name, Reason: "discard",
44- }
45- repo := portsmocks.NewMockSnapshotRepository(t)
46- repo.EXPECT().SaveQuarantineDescriptor(ctx, quarantineDescriptor(intent)).Return(nil)
47- repo.EXPECT().QuarantineIncarnation(ctx, old.IncarnationID).Return(nil)
48- repo.EXPECT().ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, recoveryListingBudget).
49- Return(ports.DeletionTombstonePage{Done: true}, nil)
50- catalogue := newTransactionCatalogue(old)
51- c := NewCoordinator(catalogue, repo, journalStub{intents: []domain.DiscardIntent{intent}}, nil)
52- require.NoError(t, c.Recover(ctx))
53- require.Equal(t, intent.NewIncarnation, catalogue.records[old.Name].IncarnationID)
54-}
55-
5623 func TestCoordinatorRecoverPagesTombstonesAndPropagatesErrors(t *testing.T) {
5724 cause := errors.New("malformed tombstone")
5825 repo := portsmocks.NewMockSnapshotRepository(t)
5926 repo.EXPECT().ListDeletionTombstones(context.Background(), ports.DeletionTombstoneCursor{}, recoveryListingBudget).
6027 Return(ports.DeletionTombstonePage{}, cause)
61- c := NewCoordinator(newTransactionCatalogue(), repo, journalStub{}, nil)
28+ c := NewCoordinator(newTransactionCatalogue(), repo, nil)
6229 require.ErrorIs(t, c.Recover(context.Background()), cause)
6330 }
M internal/usecase/recovery/discard.go +29 -109
@@ -11,131 +11,51 @@ import (
1111 var (
1212 ErrRecoveryRecordNotFound = errors.New("recovery: session not found")
1313 ErrSessionNotDegraded = errors.New("recovery: session is not degraded")
14- ErrDiscardConflict = errors.New("recovery: discard intent conflicts with catalogue")
15- ErrDiscardIntentInvalid = errors.New("recovery: discard intent is unusable")
1614 )
1715
18-// Discard commits to a new incarnation by saving the complete intent before
19-// any quarantine or catalogue mutation. There is no rollback after that point.
20-func (c *Coordinator) Discard(ctx context.Context, name, reason string) (domain.CatalogueRecord, error) {
21- if c == nil || c.catalogue == nil || c.repository == nil || c.journal == nil || c.locks == nil || c.random == nil {
22- return domain.CatalogueRecord{}, errors.New("recovery: incomplete discard dependencies")
16+// Discard throws away a broken session's persisted state and replaces it with a
17+// fresh incarnation. It is retry-idempotent, not crash-resumable: a crash
18+// leaves either the old record (rerun discard) or a new record plus an orphan
19+// directory that startup garbage collection removes.
20+func (c *Coordinator) Discard(ctx context.Context, name string) error {
21+ if c == nil || c.catalogue == nil || c.repository == nil || c.locks == nil || c.random == nil {
22+ return errors.New("recovery: incomplete discard dependencies")
23+ }
24+ if err := ctx.Err(); err != nil {
25+ return err
2326 }
2427 unlock := c.locks.Lock([]string{name})
2528 defer unlock()
29+
2630 record, ok, err := c.catalogue.Record(name)
2731 if err != nil {
28- return domain.CatalogueRecord{}, err
32+ return err
2933 }
3034 if !ok {
31- return domain.CatalogueRecord{}, ErrRecoveryRecordNotFound
32- }
33- if record.RecoveryState != domain.RecoveryDegraded {
34- return domain.CatalogueRecord{}, ErrSessionNotDegraded
35- }
36- if reason == "" {
37- return domain.CatalogueRecord{}, errors.New("recovery: discard reason is required")
38- }
39- newID, err := domain.NewIncarnationID(c.random)
40- if err != nil {
41- return domain.CatalogueRecord{}, fmt.Errorf("recovery: generate replacement incarnation: %w", err)
35+ return fmt.Errorf("discard %q: %w", name, ErrRecoveryRecordNotFound)
4236 }
43- intent := domain.DiscardIntent{
44- OldRecord: record, OldIncarnation: record.IncarnationID, NewIncarnation: newID,
45- SessionName: record.Name, Reason: reason,
37+ if record.RecoveryState == domain.RecoveryFresh {
38+ return nil
4639 }
47- if err := c.journal.SaveDiscard(ctx, intent); err != nil {
48- return domain.CatalogueRecord{}, err
49- }
50- if err := c.recoverDiscardLocked(ctx, intent); err != nil {
51- return domain.CatalogueRecord{}, err
40+ if record.RecoveryState != domain.RecoveryDegraded {
41+ return ErrSessionNotDegraded
5242 }
53- return freshReplacement(intent), nil
54-}
5543
56-func (c *Coordinator) recoverDiscard(ctx context.Context, intent domain.DiscardIntent) error {
57- if c == nil || c.catalogue == nil || c.repository == nil || c.journal == nil || c.locks == nil {
58- return errors.New("recovery: incomplete discard recovery dependencies")
59- }
60- unlock := c.locks.Lock([]string{intent.SessionName})
61- defer unlock()
62- return c.recoverDiscardLocked(ctx, intent)
63-}
64-
65-func (c *Coordinator) recoverDiscardLocked(ctx context.Context, intent domain.DiscardIntent) error {
66- if err := validateDiscardIntent(intent); err != nil {
67- return err
68- }
69- current, ok, err := c.catalogue.Record(intent.SessionName)
44+ old := record.IncarnationID
45+ next, err := domain.NewIncarnationID(c.random)
7046 if err != nil {
71- return err
72- }
73- if !ok {
74- // Catalogue deletion is terminal for this session identity. There is no
75- // live record to replace or quarantine, so retaining the intent would
76- // manufacture a permanent startup conflict.
77- return c.journal.DeleteDiscard(ctx, intent.OldIncarnation)
47+ return fmt.Errorf("recovery: generate replacement incarnation: %w", err)
7848 }
79- next := freshReplacement(intent)
80- if err := next.Validate(); err != nil {
81- return fmt.Errorf("%w: invalid discard replacement: %w", ErrDiscardIntentInvalid, err)
49+ fresh := record
50+ fresh.IncarnationID = next
51+ fresh.RecoveryState = domain.RecoveryFresh
52+ fresh.Committed = nil
53+ fresh.DegradedReason = ""
54+ if err := fresh.Validate(); err != nil {
55+ return fmt.Errorf("recovery: invalid discard replacement: %w", err)
8256 }
83- switch current.IncarnationID {
84- case intent.OldIncarnation:
85- if !current.Equal(intent.OldRecord) {
86- return fmt.Errorf("%w: old record changed", ErrDiscardConflict)
87- }
88- case intent.NewIncarnation:
89- // Step 4 is already durable. The replacement is live from this point on,
90- // so it may legitimately have advanced to a committed checkpoint before a
91- // later step failed. Only its identity may be required here: demanding
92- // byte equality with the pristine replacement would fail closed forever
93- // on a session that is in fact healthy, and destroy committed state if it
94- // were overwritten. Steps 2-3 and 5 below are idempotent.
95- if current.Name != intent.SessionName {
96- return fmt.Errorf("%w: replacement session identity changed", ErrDiscardConflict)
97- }
98- default:
99- return fmt.Errorf("%w: unexpected incarnation", ErrDiscardConflict)
100- }
101-
102- if err := c.repository.SaveQuarantineDescriptor(ctx, quarantineDescriptor(intent)); err != nil {
57+ if err := c.catalogue.Replace(name, fresh); err != nil {
10358 return err
10459 }
105- if err := c.repository.QuarantineIncarnation(ctx, intent.OldIncarnation); err != nil {
106- return err
107- }
108- if current.IncarnationID == intent.OldIncarnation {
109- if err := c.catalogue.Replace(intent.SessionName, next); err != nil {
110- return err
111- }
112- }
113- return c.journal.DeleteDiscard(ctx, intent.OldIncarnation)
114-}
115-
116-func validateDiscardIntent(intent domain.DiscardIntent) error {
117- if err := intent.OldRecord.Validate(); err != nil {
118- return fmt.Errorf("%w: invalid old record: %w", ErrDiscardIntentInvalid, err)
119- }
120- if intent.SessionName != intent.OldRecord.Name || intent.OldIncarnation != intent.OldRecord.IncarnationID ||
121- intent.NewIncarnation == (domain.IncarnationID{}) || intent.NewIncarnation == intent.OldIncarnation || intent.Reason == "" {
122- return fmt.Errorf("%w: inconsistent intent fields", ErrDiscardIntentInvalid)
123- }
124- return nil
125-}
126-
127-func quarantineDescriptor(intent domain.DiscardIntent) domain.QuarantineDescriptor {
128- return domain.QuarantineDescriptor{
129- OldRecord: intent.OldRecord, OldIncarnation: intent.OldIncarnation,
130- ReplacementIncarnation: intent.NewIncarnation, SessionName: intent.SessionName, Reason: intent.Reason,
131- }
132-}
133-
134-func freshReplacement(intent domain.DiscardIntent) domain.CatalogueRecord {
135- next := intent.OldRecord
136- next.IncarnationID = intent.NewIncarnation
137- next.RecoveryState = domain.RecoveryFresh
138- next.Committed = nil
139- next.DegradedReason = ""
140- return next
60+ return c.repository.DeleteIncarnation(ctx, old)
14161 }
D internal/usecase/recovery/reconcile.go +0 -173
@@ -1,173 +0,0 @@
1-package recovery
2-
3-import (
4- "context"
5- "errors"
6- "fmt"
7-
8- "github.com/bnema/vev/internal/domain"
9- "github.com/bnema/vev/internal/ports"
10-)
11-
12-// Reconciler runs bounded repository maintenance after catalogue-driven
13-// restoration. Its cursor is caller-owned and can be resumed by a later pass.
14-type Reconciler struct {
15- coordinator *Coordinator
16- catalogue ports.Catalogue
17- repository ports.SnapshotRepository
18- budget ports.MaintenanceBudget
19-}
20-
21-func NewReconciler(coordinator *Coordinator, catalogue ports.Catalogue, repository ports.SnapshotRepository, budget ports.MaintenanceBudget) *Reconciler {
22- return &Reconciler{coordinator: coordinator, catalogue: catalogue, repository: repository, budget: budget}
23-}
24-
25-func (r *Reconciler) Step(ctx context.Context, cursor ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error) {
26- if r == nil || r.coordinator == nil || r.catalogue == nil || r.repository == nil || r.budget.Entries == 0 || r.budget.Bytes == 0 {
27- return cursor, nil, errors.New("recovery: incomplete reconciliation dependencies")
28- }
29- if err := ctx.Err(); err != nil {
30- return cursor, nil, err
31- }
32- records, err := r.catalogue.Records()
33- if err != nil {
34- return cursor, nil, err
35- }
36- next, findings, err := r.repository.Reconcile(ctx, records, cursor, r.budget)
37- if err != nil {
38- return cursor, nil, err
39- }
40- decisions := make([]ports.ReconcileDecision, 0, len(findings))
41- consumed := ports.MaintenanceBudget{}
42- for _, finding := range findings {
43- // Compare against the remainder before adding. This both enforces the
44- // page-wide budget and avoids uint64 wraparound.
45- if finding.Consumed.Entries > r.budget.Entries-consumed.Entries ||
46- finding.Consumed.Bytes > r.budget.Bytes-consumed.Bytes {
47- return cursor, nil, errors.New("recovery: repository exceeded reconciliation budget")
48- }
49- consumed.Entries += finding.Consumed.Entries
50- consumed.Bytes += finding.Consumed.Bytes
51- record, ok, err := r.catalogue.Record(finding.Candidate.Name)
52- if err != nil {
53- return cursor, nil, err
54- }
55- if !ok {
56- record = domain.CatalogueRecord{}
57- }
58- decision := classifyFinding(record, finding)
59- if decision.Kind == ports.ReconcileAdopt {
60- if _, err := r.coordinator.PublishReconciledCheckpoint(ctx, finding.Candidate.Name, finding.Candidate, finding.AncestorChain); err != nil {
61- if !errors.Is(err, ErrCheckpointConflict) && !errors.Is(err, ErrCheckpointRecordNotFound) {
62- return cursor, nil, err
63- }
64- decision.Kind = ports.ReconcileDefer
65- decision.ReasonCode = "catalogue-conflict"
66- decision.RetentionResolved = false
67- }
68- }
69- decisions = append(decisions, decision)
70- }
71- return next, decisions, nil
72-}
73-
74-func classifyFinding(record domain.CatalogueRecord, finding ports.ReconcileFinding) ports.ReconcileDecision {
75- decision := ports.ReconcileDecision{Name: finding.Candidate.Name}
76- candidate := finding.Candidate.Ref
77- decision.Candidate = &candidate
78- if finding.Status == ports.ReconcileBudgetExhausted {
79- decision.Kind = ports.ReconcileDefer
80- decision.ReasonCode = "budget-exhausted"
81- return decision
82- }
83- if finding.Kind == ports.ReconcileUnknownIncarnation || record.IncarnationID == (domain.IncarnationID{}) {
84- decision.Kind = ports.ReconcileKeepQuarantined
85- decision.ReasonCode = "unknown-incarnation"
86- return decision
87- }
88- if finding.Candidate.IncarnationID != record.IncarnationID {
89- decision.Kind = ports.ReconcileKeepQuarantined
90- decision.ReasonCode = "incarnation-mismatch"
91- return decision
92- }
93- if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil {
94- decision.Kind = ports.ReconcileKeepQuarantined
95- decision.ReasonCode = "invalid-candidate"
96- return decision
97- }
98- if finding.Status == ports.ReconcileValidated && finding.Candidate.Ref.Generation <= record.Committed.Generation {
99- decision.Kind = ports.ReconcileKeepQuarantined
100- decision.ReasonCode = "not-forward"
101- decision.RetentionResolved = true
102- return decision
103- }
104- if finding.Status != ports.ReconcileValidated || !validForwardChain(*record.Committed, finding.Candidate, finding.AncestorChain) {
105- decision.Kind = ports.ReconcileKeepQuarantined
106- decision.ReasonCode = "invalid-chain"
107- return decision
108- }
109- decision.Kind = ports.ReconcileAdopt
110- decision.RetentionResolved = true
111- return decision
112-}
113-
114-func validForwardChain(committed domain.CheckpointRef, candidate ports.ReconcileCandidate, ancestors []ports.ValidatedCheckpoint) bool {
115- if committed.Generation == 0 || candidate.Ref.Generation <= committed.Generation || candidate.Parent == nil {
116- return false
117- }
118- if len(ancestors) == 0 {
119- return *candidate.Parent == committed
120- }
121- if *candidate.Parent != ancestors[0].Ref || candidate.Ref.Generation <= ancestors[0].Ref.Generation {
122- return false
123- }
124- for i, ancestor := range ancestors {
125- if ancestor.Parent == nil || ancestor.Ref.Generation <= committed.Generation {
126- return false
127- }
128- if i+1 < len(ancestors) {
129- if ancestor.Ref.Generation <= ancestors[i+1].Ref.Generation || *ancestor.Parent != ancestors[i+1].Ref {
130- return false
131- }
132- continue
133- }
134- if *ancestor.Parent != committed {
135- return false
136- }
137- }
138- return true
139-}
140-
141-// PublishReconciledCheckpoint rechecks the chain while holding the session
142-// transaction lock. The candidate is already the durable HEAD and its payloads
143-// were validated by the adapter, so publication performs no repository reread.
144-func (c *Coordinator) PublishReconciledCheckpoint(ctx context.Context, name string, candidate ports.ReconcileCandidate, ancestors []ports.ValidatedCheckpoint) (domain.CatalogueRecord, error) {
145- if c == nil || c.catalogue == nil || c.repository == nil || c.locks == nil {
146- return domain.CatalogueRecord{}, errors.New("recovery: incomplete reconciled checkpoint dependencies")
147- }
148- unlock := c.locks.Lock([]string{name})
149- defer unlock()
150- if err := ctx.Err(); err != nil {
151- return domain.CatalogueRecord{}, err
152- }
153- record, ok, err := c.catalogue.Record(name)
154- if err != nil {
155- return domain.CatalogueRecord{}, err
156- }
157- if !ok {
158- return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound
159- }
160- if record.RecoveryState != domain.RecoveryHealthy || candidate.Name != name || candidate.IncarnationID != record.IncarnationID || record.Committed == nil || !validForwardChain(*record.Committed, candidate, ancestors) {
161- return domain.CatalogueRecord{}, ErrCheckpointConflict
162- }
163- next := record
164- next.Committed = copyCheckpointRef(&candidate.Ref)
165- next.DegradedReason = ""
166- if err := next.Validate(); err != nil {
167- return domain.CatalogueRecord{}, fmt.Errorf("recovery: invalid reconciled checkpoint transition: %w", err)
168- }
169- if err := c.catalogue.Replace(name, next); err != nil {
170- return domain.CatalogueRecord{}, err
171- }
172- return next, nil
173-}
D internal/usecase/recovery/reconcile_test.go +0 -248
@@ -1,248 +0,0 @@
1-package recovery
2-
3-import (
4- "context"
5- "fmt"
6- "testing"
7-
8- "github.com/bnema/vev/internal/domain"
9- "github.com/bnema/vev/internal/ports"
10- "github.com/stretchr/testify/require"
11-)
12-
13-type reconcileRepository struct {
14- ports.SnapshotRepository
15- pages map[uint64]reconcilePage
16- seenCursors []ports.ReconcileCursor
17- seenBudgets []ports.MaintenanceBudget
18- repairs []domain.CheckpointRef
19- events *[]string
20-}
21-
22-type reconcilePage struct {
23- next ports.ReconcileCursor
24- findings []ports.ReconcileFinding
25- err error
26-}
27-
28-type reconcileConflictCatalogue struct {
29- *transactionCatalogue
30- stale domain.CatalogueRecord
31- reads int
32-}
33-
34-func (c *reconcileConflictCatalogue) Records() ([]domain.CatalogueRecord, error) {
35- records, err := c.transactionCatalogue.Records()
36- if err != nil {
37- return nil, err
38- }
39- for i := range records {
40- if records[i].Name == c.stale.Name {
41- records[i] = c.stale
42- }
43- }
44- return records, nil
45-}
46-
47-func (c *reconcileConflictCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) {
48- if name == c.stale.Name && c.reads == 0 {
49- c.reads++
50- return c.stale, true, nil
51- }
52- return c.transactionCatalogue.Record(name)
53-}
54-
55-func (r *reconcileRepository) Reconcile(_ context.Context, _ []domain.CatalogueRecord, cursor ports.ReconcileCursor, budget ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error) {
56- r.seenCursors = append(r.seenCursors, cursor)
57- r.seenBudgets = append(r.seenBudgets, budget)
58- page := r.pages[cursor.DirectoryCookie]
59- return page.next, page.findings, page.err
60-}
61-
62-func (r *reconcileRepository) RepairHEAD(_ context.Context, _ domain.IncarnationID, ref ports.CheckpointRef) error {
63- if r.events != nil {
64- *r.events = append(*r.events, "head")
65- }
66- r.repairs = append(r.repairs, ref)
67- return nil
68-}
69-
70-func TestReconciliationCursor(t *testing.T) {
71- record := healthyReconcileRecord()
72- catalogue := &checkpointCatalogue{record: record}
73- candidate := forwardCandidate(record, 2, record.Committed)
74- repository := &reconcileRepository{pages: map[uint64]reconcilePage{
75- 0: {next: ports.ReconcileCursor{DirectoryCookie: 12}, findings: []ports.ReconcileFinding{{Status: ports.ReconcileBudgetExhausted, Cursor: ports.ReconcileCursor{DirectoryCookie: 12}, Consumed: ports.MaintenanceBudget{Entries: 1, Bytes: 16}}}},
76- 12: {next: ports.ReconcileCursor{DirectoryCookie: 24}, findings: []ports.ReconcileFinding{{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: candidate, Cursor: ports.ReconcileCursor{DirectoryCookie: 24}, Consumed: ports.MaintenanceBudget{Entries: 2, Bytes: 32}}}},
77- }}
78- coordinator := NewCoordinator(catalogue, repository, nil, nil)
79- reconciler := &Reconciler{coordinator: coordinator, catalogue: catalogue, repository: repository, budget: ports.MaintenanceBudget{Entries: 2, Bytes: 32}}
80-
81- next, decisions, err := reconciler.Step(context.Background(), ports.ReconcileCursor{})
82- require.NoError(t, err)
83- require.Equal(t, uint64(12), next.DirectoryCookie)
84- require.Equal(t, ports.ReconcileDefer, decisions[0].Kind)
85- require.Equal(t, "budget-exhausted", decisions[0].ReasonCode)
86-
87- next, decisions, err = reconciler.Step(context.Background(), next)
88- require.NoError(t, err)
89- require.Equal(t, uint64(24), next.DirectoryCookie)
90- require.Equal(t, ports.ReconcileAdopt, decisions[0].Kind)
91- require.Equal(t, []ports.ReconcileCursor{{}, {DirectoryCookie: 12}}, repository.seenCursors)
92- require.Equal(t, []ports.MaintenanceBudget{{Entries: 2, Bytes: 32}, {Entries: 2, Bytes: 32}}, repository.seenBudgets)
93-}
94-
95-func TestReconciliationRejectsCumulativeAndOverflowingBudgets(t *testing.T) {
96- record := healthyReconcileRecord()
97- finding := ports.ReconcileFinding{Status: ports.ReconcileBudgetExhausted, Candidate: ports.ReconcileCandidate{Name: record.Name}}
98- for _, tc := range []struct {
99- name string
100- budget ports.MaintenanceBudget
101- consumed []ports.MaintenanceBudget
102- }{
103- {name: "cumulative entries", budget: ports.MaintenanceBudget{Entries: 3, Bytes: 10}, consumed: []ports.MaintenanceBudget{{Entries: 2, Bytes: 1}, {Entries: 2, Bytes: 1}}},
104- {name: "cumulative bytes", budget: ports.MaintenanceBudget{Entries: 10, Bytes: 3}, consumed: []ports.MaintenanceBudget{{Entries: 1, Bytes: 2}, {Entries: 1, Bytes: 2}}},
105- {name: "overflow safe", budget: ports.MaintenanceBudget{Entries: ^uint64(0), Bytes: ^uint64(0)}, consumed: []ports.MaintenanceBudget{{Entries: ^uint64(0), Bytes: ^uint64(0)}, {Entries: 1, Bytes: 1}}},
106- } {
107- t.Run(tc.name, func(t *testing.T) {
108- findings := make([]ports.ReconcileFinding, len(tc.consumed))
109- for i, usage := range tc.consumed {
110- findings[i] = finding
111- findings[i].Consumed = usage
112- }
113- repository := &reconcileRepository{pages: map[uint64]reconcilePage{0: {findings: findings}}}
114- catalogue := &checkpointCatalogue{record: record}
115- reconciler := NewReconciler(NewCoordinator(catalogue, repository, nil, nil), catalogue, repository, tc.budget)
116- _, _, err := reconciler.Step(context.Background(), ports.ReconcileCursor{})
117- require.EqualError(t, err, "recovery: repository exceeded reconciliation budget")
118- })
119- }
120-}
121-
122-func TestReconciliationDefersConflictAndContinues(t *testing.T) {
123- first := healthyReconcileRecord()
124- first.Name = "first"
125- staleRecord := first
126- stale := forwardCandidate(staleRecord, 2, staleRecord.Committed)
127- first.Committed = &domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{9}}
128- second := healthyReconcileRecord()
129- second.Name = "second"
130- adoptable := forwardCandidate(second, 2, second.Committed)
131- catalogue := &reconcileConflictCatalogue{transactionCatalogue: newTransactionCatalogue(first, second), stale: staleRecord}
132- repository := &reconcileRepository{pages: map[uint64]reconcilePage{0: {findings: []ports.ReconcileFinding{
133- {Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: stale, Consumed: ports.MaintenanceBudget{Entries: 1, Bytes: 1}},
134- {Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: adoptable, Consumed: ports.MaintenanceBudget{Entries: 1, Bytes: 1}},
135- }}}}
136- reconciler := NewReconciler(NewCoordinator(catalogue, repository, nil, nil), catalogue, repository, ports.MaintenanceBudget{Entries: 2, Bytes: 2})
137-
138- _, decisions, err := reconciler.Step(context.Background(), ports.ReconcileCursor{})
139- require.NoError(t, err)
140- require.Equal(t, []ports.ReconcileDecisionKind{ports.ReconcileDefer, ports.ReconcileAdopt}, []ports.ReconcileDecisionKind{decisions[0].Kind, decisions[1].Kind})
141- require.Equal(t, "catalogue-conflict", decisions[0].ReasonCode)
142- require.Equal(t, adoptable.Ref, *catalogue.records[second.Name].Committed)
143-}
144-
145-func TestForwardOrphanAdoption(t *testing.T) {
146- for _, tc := range []struct {
147- name string
148- mutate func(domain.CatalogueRecord, *ports.ReconcileFinding)
149- wantKind ports.ReconcileDecisionKind
150- wantCause string
151- }{
152- {name: "direct child", wantKind: ports.ReconcileAdopt},
153- {name: "two parent chain", mutate: addReconcileAncestor, wantKind: ports.ReconcileAdopt},
154- {name: "wrong incarnation", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) {
155- finding.Candidate.IncarnationID = domain.IncarnationID{9}
156- }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "incarnation-mismatch"},
157- {name: "wrong digest", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) {
158- finding.Candidate.Parent.ManifestDigest[0]++
159- }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "invalid-chain"},
160- {name: "missing link", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) { finding.Candidate.Parent = nil }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "invalid-chain"},
161- {name: "generation reversal", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) {
162- finding.Candidate.Ref.Generation = record.Committed.Generation
163- }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "not-forward"},
164- {name: "non-orphan invalid candidate", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) {
165- finding.Kind = ports.ReconcileInvalidCandidate
166- }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "invalid-candidate"},
167- } {
168- t.Run(tc.name, func(t *testing.T) {
169- record := healthyReconcileRecord()
170- finding := ports.ReconcileFinding{Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: forwardCandidate(record, 2, record.Committed)}
171- if tc.mutate != nil {
172- tc.mutate(record, &finding)
173- }
174- decision := classifyFinding(record, finding)
175- require.Equal(t, tc.wantKind, decision.Kind)
176- require.Equal(t, tc.wantCause, decision.ReasonCode)
177- })
178- }
179-
180- t.Run("validated head publishes without repository reread", func(t *testing.T) {
181- record := healthyReconcileRecord()
182- events := []string{}
183- catalogue := &checkpointCatalogue{record: record, events: &events}
184- repository := &reconcileRepository{events: &events}
185- coordinator := NewCoordinator(catalogue, repository, nil, nil)
186- candidate := forwardCandidate(record, 2, record.Committed)
187-
188- next, err := coordinator.PublishReconciledCheckpoint(context.Background(), record.Name, candidate, nil)
189- require.NoError(t, err)
190- require.Equal(t, candidate.Ref, *next.Committed)
191- require.Equal(t, []string{"catalogue"}, events)
192- require.Empty(t, repository.repairs)
193- })
194-
195- for _, state := range []domain.RecoveryState{domain.RecoveryDegraded, domain.RecoveryDeleting} {
196- t.Run(fmt.Sprintf("non-healthy record %d is not reconciled", state), func(t *testing.T) {
197- record := healthyReconcileRecord()
198- record.RecoveryState = state
199- if state == domain.RecoveryDegraded {
200- record.DegradedReason = "uncertain"
201- } else {
202- record.Committed = nil
203- }
204- candidate := forwardCandidate(healthyReconcileRecord(), 2, healthyReconcileRecord().Committed)
205- coordinator := NewCoordinator(&checkpointCatalogue{record: record}, &reconcileRepository{}, nil, nil)
206-
207- _, err := coordinator.PublishReconciledCheckpoint(context.Background(), record.Name, candidate, nil)
208- require.ErrorIs(t, err, ErrCheckpointConflict)
209- })
210- }
211-
212- t.Run("stale decision after concurrent commit", func(t *testing.T) {
213- record := healthyReconcileRecord()
214- candidate := forwardCandidate(record, 2, record.Committed)
215- concurrent := domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{7}}
216- record.Committed = &concurrent
217- catalogue := &checkpointCatalogue{record: record}
218- repository := &reconcileRepository{}
219- coordinator := NewCoordinator(catalogue, repository, nil, nil)
220-
221- _, err := coordinator.PublishReconciledCheckpoint(context.Background(), record.Name, candidate, nil)
222- require.ErrorIs(t, err, ErrCheckpointConflict)
223- require.Empty(t, repository.repairs)
224- })
225-
226- t.Run("unknown identity remains quarantined", func(t *testing.T) {
227- finding := ports.ReconcileFinding{Kind: ports.ReconcileUnknownIncarnation, Status: ports.ReconcileQuarantined, Candidate: ports.ReconcileCandidate{Name: "unknown", IncarnationID: domain.IncarnationID{9}}}
228- decision := classifyFinding(domain.CatalogueRecord{}, finding)
229- require.Equal(t, ports.ReconcileKeepQuarantined, decision.Kind)
230- require.Equal(t, "unknown-incarnation", decision.ReasonCode)
231- })
232-}
233-
234-func healthyReconcileRecord() domain.CatalogueRecord {
235- ref := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}
236- return domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryHealthy, Committed: &ref}
237-}
238-
239-func forwardCandidate(record domain.CatalogueRecord, generation uint64, parent *domain.CheckpointRef) ports.ReconcileCandidate {
240- parentCopy := *parent
241- return ports.ReconcileCandidate{IncarnationID: record.IncarnationID, Name: record.Name, Ref: domain.CheckpointRef{Generation: generation, ManifestDigest: [32]byte{byte(generation)}}, Parent: &parentCopy}
242-}
243-
244-func addReconcileAncestor(record domain.CatalogueRecord, finding *ports.ReconcileFinding) {
245- ancestor := ports.ValidatedCheckpoint{Ref: domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}}, Parent: record.Committed}
246- finding.Candidate = forwardCandidate(record, 3, &ancestor.Ref)
247- finding.AncestorChain = []ports.ValidatedCheckpoint{ancestor}
248-}
M internal/usecase/recovery/transactions_test.go +66 -280
@@ -1,6 +1,7 @@
11 package recovery
22
33 import (
4+ "bytes"
45 "context"
56 "errors"
67 "sync"
@@ -12,9 +13,10 @@ import (
1213 )
1314
1415 type transactionCatalogue struct {
15- mu sync.Mutex
16- records map[string]domain.CatalogueRecord
17- events []string
16+ mu sync.Mutex
17+ records map[string]domain.CatalogueRecord
18+ events []string
19+ replaceErrOnce error
1820 }
1921
2022 func newTransactionCatalogue(records ...domain.CatalogueRecord) *transactionCatalogue {
@@ -34,12 +36,14 @@ func (c *transactionCatalogue) Records() ([]domain.CatalogueRecord, error) {
3436 }
3537 return out, nil
3638 }
39+
3740 func (c *transactionCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) {
3841 c.mu.Lock()
3942 defer c.mu.Unlock()
4043 record, ok := c.records[name]
4144 return record, ok, nil
4245 }
46+
4347 func (c *transactionCatalogue) Create(record domain.CatalogueRecord) error {
4448 c.mu.Lock()
4549 defer c.mu.Unlock()
@@ -50,10 +54,17 @@ func (c *transactionCatalogue) Create(record domain.CatalogueRecord) error {
5054 c.events = append(c.events, "create")
5155 return nil
5256 }
57+
5358 func (c *transactionCatalogue) UpdateMetadata(domain.CatalogueMetadataUpdate) error { return nil }
59+
5460 func (c *transactionCatalogue) Replace(name string, record domain.CatalogueRecord) error {
5561 c.mu.Lock()
5662 defer c.mu.Unlock()
63+ if c.replaceErrOnce != nil {
64+ err := c.replaceErrOnce
65+ c.replaceErrOnce = nil
66+ return err
67+ }
5768 if _, exists := c.records[name]; !exists {
5869 return errors.New("record missing")
5970 }
@@ -61,6 +72,7 @@ func (c *transactionCatalogue) Replace(name string, record domain.CatalogueRecor
6172 c.events = append(c.events, "replace:"+name)
6273 return nil
6374 }
75+
6476 func (c *transactionCatalogue) Rename(oldName string, record domain.CatalogueRecord) error {
6577 c.mu.Lock()
6678 defer c.mu.Unlock()
@@ -75,6 +87,7 @@ func (c *transactionCatalogue) Rename(oldName string, record domain.CatalogueRec
7587 c.events = append(c.events, "rename:"+oldName+":"+record.Name)
7688 return nil
7789 }
90+
7891 func (c *transactionCatalogue) Delete(name string) error {
7992 c.mu.Lock()
8093 defer c.mu.Unlock()
@@ -82,126 +95,25 @@ func (c *transactionCatalogue) Delete(name string) error {
8295 c.events = append(c.events, "delete:"+name)
8396 return nil
8497 }
98+
8599 func (c *transactionCatalogue) Close() error { return nil }
86100
87101 type transactionRepository struct {
88102 ports.SnapshotRepository
89- mu sync.Mutex
90- tombstones map[domain.IncarnationID]domain.DeletionTombstone
91- generations map[domain.CheckpointRef]ports.SnapshotGeneration
92- loadErr error
93- quarantineErr error
94- events []string
95- quarantine []bool
96- descriptors []domain.QuarantineDescriptor
97- quarantinedIDs []domain.IncarnationID
98- repaired []domain.CheckpointRef
103+ mu sync.Mutex
104+ deleteErrOnce error
105+ deletedIDs []domain.IncarnationID
99106 }
100107
101-func newTransactionRepository() *transactionRepository {
102- return &transactionRepository{
103- tombstones: make(map[domain.IncarnationID]domain.DeletionTombstone),
104- generations: make(map[domain.CheckpointRef]ports.SnapshotGeneration),
105- }
106-}
107-func (r *transactionRepository) LoadCheckpoint(_ context.Context, _ domain.IncarnationID, _ string, ref domain.CheckpointRef) (ports.SnapshotGeneration, error) {
108- r.mu.Lock()
109- defer r.mu.Unlock()
110- r.events = append(r.events, "load")
111- if r.loadErr != nil {
112- return ports.SnapshotGeneration{}, r.loadErr
113- }
114- generation, ok := r.generations[ref]
115- if !ok {
116- return ports.SnapshotGeneration{}, errors.New("checkpoint missing")
117- }
118- return generation, nil
119-}
120-func (r *transactionRepository) RepairHEAD(_ context.Context, _ domain.IncarnationID, ref domain.CheckpointRef) error {
121- r.mu.Lock()
122- defer r.mu.Unlock()
123- r.repaired = append(r.repaired, ref)
124- r.events = append(r.events, "repair-head")
125- return nil
126-}
127-func (r *transactionRepository) SaveQuarantineDescriptor(_ context.Context, descriptor domain.QuarantineDescriptor) error {
128- r.mu.Lock()
129- defer r.mu.Unlock()
130- r.descriptors = append(r.descriptors, descriptor)
131- r.events = append(r.events, "descriptor")
132- return nil
133-}
134-func (r *transactionRepository) QuarantineIncarnation(_ context.Context, id domain.IncarnationID) error {
135- r.mu.Lock()
136- defer r.mu.Unlock()
137- r.quarantinedIDs = append(r.quarantinedIDs, id)
138- r.events = append(r.events, "quarantine-incarnation")
139- return nil
140-}
141-func (r *transactionRepository) WriteDeletionTombstone(_ context.Context, tombstone domain.DeletionTombstone) error {
142- r.mu.Lock()
143- defer r.mu.Unlock()
144- r.tombstones[tombstone.IncarnationID] = tombstone
145- r.events = append(r.events, "tombstone")
146- return nil
147-}
148-func (r *transactionRepository) ListDeletionTombstones(_ context.Context, cursor ports.DeletionTombstoneCursor, _ ports.MaintenanceBudget) (ports.DeletionTombstonePage, error) {
149- r.mu.Lock()
150- defer r.mu.Unlock()
151- if cursor.After != "" || len(r.tombstones) == 0 {
152- return ports.DeletionTombstonePage{Done: true}, nil
153- }
154- out := make([]domain.DeletionTombstone, 0, len(r.tombstones))
155- for _, tombstone := range r.tombstones {
156- out = append(out, tombstone)
157- }
158- return ports.DeletionTombstonePage{Tombstones: out, Done: true}, nil
159-}
160-func (r *transactionRepository) QuarantineDeletionSources(_ context.Context, _ domain.DeletionTombstone, includeLegacyName bool) error {
161- r.mu.Lock()
162- defer r.mu.Unlock()
163- r.events = append(r.events, "quarantine")
164- r.quarantine = append(r.quarantine, includeLegacyName)
165- return r.quarantineErr
166-}
167-func (r *transactionRepository) DeleteDeletionTombstone(_ context.Context, id domain.IncarnationID) error {
108+func (r *transactionRepository) DeleteIncarnation(_ context.Context, id domain.IncarnationID) error {
168109 r.mu.Lock()
169110 defer r.mu.Unlock()
170- delete(r.tombstones, id)
171- r.events = append(r.events, "remove-tombstone")
172- return nil
173-}
174-
175-type transactionJournal struct {
176- mu sync.Mutex
177- intents map[domain.IncarnationID]domain.DiscardIntent
178- events []string
179-}
180-
181-func newTransactionJournal() *transactionJournal {
182- return &transactionJournal{intents: make(map[domain.IncarnationID]domain.DiscardIntent)}
183-}
184-func (j *transactionJournal) SaveDiscard(_ context.Context, intent domain.DiscardIntent) error {
185- j.mu.Lock()
186- defer j.mu.Unlock()
187- j.intents[intent.OldIncarnation] = intent
188- j.events = append(j.events, "intent")
189- return nil
190-}
191-func (j *transactionJournal) ListDiscards(context.Context) ([]domain.DiscardIntent, error) {
192- j.mu.Lock()
193- defer j.mu.Unlock()
194- out := make([]domain.DiscardIntent, 0, len(j.intents))
195- for _, intent := range j.intents {
196- out = append(out, intent)
111+ if r.deleteErrOnce != nil {
112+ err := r.deleteErrOnce
113+ r.deleteErrOnce = nil
114+ return err
197115 }
198- return out, nil
199-}
200-func (j *transactionJournal) DeleteDiscard(_ context.Context, id domain.IncarnationID) error {
201- j.mu.Lock()
202- defer j.mu.Unlock()
203- delete(j.intents, id)
204- j.events = append(j.events, "delete-intent")
116+ r.deletedIDs = append(r.deletedIDs, id)
205117 return nil
206118 }
207119
@@ -213,171 +125,45 @@ func degradedTransactionRecord() domain.CatalogueRecord {
213125 }
214126 }
215127
216-func TestDiscardRecoveryRemovesIntentWhenCatalogueRecordIsAbsent(t *testing.T) {
217- old := degradedTransactionRecord()
218- intent := domain.DiscardIntent{OldRecord: old, OldIncarnation: old.IncarnationID, NewIncarnation: domain.IncarnationID{2}, SessionName: old.Name, Reason: "discard"}
219- catalogue := newTransactionCatalogue()
220- journal := newTransactionJournal()
221- journal.intents[old.IncarnationID] = intent
222- coordinator := NewCoordinator(catalogue, newTransactionRepository(), journal, nil)
223-
224- require.NoError(t, coordinator.Recover(context.Background()))
225- require.NotContains(t, journal.intents, old.IncarnationID)
226- require.Equal(t, []string{"delete-intent"}, journal.events)
227- require.Empty(t, coordinator.Conflicts())
228-}
229-
230-func TestDiscardIncarnationConflict(t *testing.T) {
231- old := degradedTransactionRecord()
232- intent := domain.DiscardIntent{OldRecord: old, OldIncarnation: old.IncarnationID, NewIncarnation: domain.IncarnationID{2}, SessionName: old.Name, Reason: "discard"}
233- conflict := old
234- conflict.IncarnationID = domain.IncarnationID{3}
235- catalogue := newTransactionCatalogue(conflict)
236- journal := newTransactionJournal()
237- journal.intents[old.IncarnationID] = intent
238- repository := newTransactionRepository()
239- coordinator := NewCoordinator(catalogue, repository, journal, nil)
240-
241- require.NoError(t, coordinator.Recover(context.Background()))
242- require.Equal(t, conflict, catalogue.records[old.Name])
243- require.Contains(t, journal.intents, old.IncarnationID)
244- require.Empty(t, repository.events)
245- conflicts := coordinator.Conflicts()
246- require.Len(t, conflicts, 1)
247- require.Equal(t, old.Name, conflicts[0].Session)
248- require.Equal(t, "discard-intent", conflicts[0].Kind)
249- require.ErrorIs(t, conflicts[0].Err, ErrDiscardConflict)
250-
251- delete(journal.intents, old.IncarnationID)
252- require.NoError(t, coordinator.Recover(context.Background()))
253- require.Empty(t, coordinator.Conflicts(), "a later recovery reports only its own conflicts")
254-}
255-
256-// A discard whose replacement is already durable, live, and checkpointed must
257-// roll forward. Requiring byte equality with the pristine replacement would
258-// leave the intent unresolvable and abort every future startup.
259-func TestDiscardRecoveryRollsForwardEvolvedReplacement(t *testing.T) {
260- old := degradedTransactionRecord()
261- intent := domain.DiscardIntent{OldRecord: old, OldIncarnation: old.IncarnationID, NewIncarnation: domain.IncarnationID{2}, SessionName: old.Name, Reason: "discard"}
262- evolved := freshReplacement(intent)
263- evolved.RecoveryState = domain.RecoveryHealthy
264- evolved.Committed = &domain.CheckpointRef{Generation: 7, ManifestDigest: [32]byte{7}}
265- evolved.UpdatedAt = 1234
266- evolved.LastUsedSeq = 9
267- evolved.TabNames = []string{"one"}
268- require.NoError(t, evolved.Validate())
269-
270- catalogue := newTransactionCatalogue(evolved)
271- journal := newTransactionJournal()
272- journal.intents[old.IncarnationID] = intent
273- repository := newTransactionRepository()
274- coordinator := NewCoordinator(catalogue, repository, journal, nil)
275-
276- require.NoError(t, coordinator.Recover(context.Background()))
277- require.Empty(t, coordinator.Conflicts())
278- require.Equal(t, evolved, catalogue.records[old.Name], "committed replacement state must survive roll-forward")
279- require.Empty(t, catalogue.events, "the live replacement record must not be rewritten")
280- require.Equal(t, []domain.QuarantineDescriptor{quarantineDescriptor(intent)}, repository.descriptors)
281- require.Equal(t, []domain.IncarnationID{intent.OldIncarnation}, repository.quarantinedIDs)
282- require.NotContains(t, journal.intents, old.IncarnationID)
283-}
284-
285-// Goal 3: one session's self-inconsistent durable state must not stop a healthy
286-// neighbour from being recovered, and must not silently resolve itself either.
287-func TestRecoverFencesOneSessionAndContinues(t *testing.T) {
288- ctx := context.Background()
289- broken := degradedTransactionRecord()
290- intent := domain.DiscardIntent{OldRecord: broken, OldIncarnation: broken.IncarnationID, NewIncarnation: domain.IncarnationID{2}, SessionName: broken.Name, Reason: "discard"}
291- conflict := broken
292- conflict.IncarnationID = domain.IncarnationID{3}
293- deleting := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{4}, RecoveryState: domain.RecoveryDeleting}
294-
295- catalogue := newTransactionCatalogue(conflict, deleting)
296- journal := newTransactionJournal()
297- journal.intents[broken.IncarnationID] = intent
298- repository := newTransactionRepository()
299- coordinator := NewCoordinator(catalogue, repository, journal, nil)
300-
301- require.NoError(t, coordinator.Recover(ctx))
302- require.NotContains(t, catalogue.records, deleting.Name, "healthy deletion must complete next to a fenced session")
303- require.Empty(t, repository.tombstones)
304- require.Equal(t, conflict, catalogue.records[broken.Name])
305- require.Contains(t, journal.intents, broken.IncarnationID)
306- require.Len(t, coordinator.Conflicts(), 1)
307-}
308-
309-// A tombstone that disagrees with a live healthy record degrades exactly that
310-// record, keeps the tombstone, and leaves every sibling tombstone processed.
311-func TestRecoverDegradesSessionOnTombstoneConflict(t *testing.T) {
312- ctx := context.Background()
313- live := domain.CatalogueRecord{
314- Name: "live", IncarnationID: domain.IncarnationID{5}, RecoveryState: domain.RecoveryHealthy,
315- Committed: &domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}},
128+func TestDiscardIsRetryIdempotent(t *testing.T) {
129+ t.Parallel()
130+ failure := errors.New("injected discard failure")
131+ tests := []struct {
132+ name string
133+ failBefore bool
134+ }{
135+ {name: "retry from old catalogue record", failBefore: true},
136+ {name: "retry from fresh catalogue record"},
137+ }
138+ for _, tt := range tests {
139+ t.Run(tt.name, func(t *testing.T) {
140+ t.Parallel()
141+ old := degradedTransactionRecord()
142+ catalogue := newTransactionCatalogue(old)
143+ repository := &transactionRepository{}
144+ if tt.failBefore {
145+ catalogue.replaceErrOnce = failure
146+ } else {
147+ repository.deleteErrOnce = failure
148+ }
149+ ids := append(bytes.Repeat([]byte{2}, 16), bytes.Repeat([]byte{3}, 16)...)
150+ coordinator := NewCoordinator(catalogue, repository, bytes.NewReader(ids))
151+
152+ err := coordinator.Discard(context.Background(), old.Name)
153+ require.ErrorIs(t, err, failure)
154+ require.NoError(t, coordinator.Discard(context.Background(), old.Name))
155+
156+ got, ok, err := catalogue.Record(old.Name)
157+ require.NoError(t, err)
158+ require.True(t, ok)
159+ require.Nil(t, got.Committed)
160+ require.Equal(t, domain.RecoveryFresh, got.RecoveryState)
161+ require.NotEqual(t, old.IncarnationID, got.IncarnationID)
162+ if tt.failBefore {
163+ require.Contains(t, repository.deletedIDs, old.IncarnationID)
164+ } else {
165+ require.Empty(t, repository.deletedIDs, "startup garbage collection owns the orphaned old directory")
166+ }
167+ })
316168 }
317- conflicting := domain.DeletionTombstone{Name: live.Name, IncarnationID: live.IncarnationID}
318- orphan := domain.DeletionTombstone{Name: "gone", IncarnationID: domain.IncarnationID{6}}
319-
320- catalogue := newTransactionCatalogue(live)
321- repository := newTransactionRepository()
322- repository.tombstones[conflicting.IncarnationID] = conflicting
323- repository.tombstones[orphan.IncarnationID] = orphan
324- coordinator := NewCoordinator(catalogue, repository, journalStub{}, nil)
325-
326- require.NoError(t, coordinator.Recover(ctx))
327- got := catalogue.records[live.Name]
328- require.Equal(t, domain.RecoveryDegraded, got.RecoveryState)
329- require.Equal(t, degradedReasonTombstone, got.DegradedReason)
330- require.Equal(t, live.Committed, got.Committed)
331- require.Contains(t, repository.tombstones, conflicting.IncarnationID, "a fenced tombstone is never removed")
332- require.NotContains(t, repository.tombstones, orphan.IncarnationID)
333- conflicts := coordinator.Conflicts()
334- require.Len(t, conflicts, 1)
335- require.Equal(t, "deletion-tombstone", conflicts[0].Kind)
336- require.ErrorIs(t, conflicts[0].Err, ErrDeletionTombstoneConflict)
337-}
338-
339-// Infrastructure failures still abort startup: they say nothing about any one
340-// session, so continuing would mutate durable state on a broken substrate.
341-func TestRecoverFailsOnInfrastructureErrors(t *testing.T) {
342- ctx := context.Background()
343- cause := errors.New("io failure")
344- tombstone := domain.DeletionTombstone{Name: "work", IncarnationID: domain.IncarnationID{4}}
345-
346- t.Run("journal listing", func(t *testing.T) {
347- coordinator := NewCoordinator(newTransactionCatalogue(), newTransactionRepository(), journalStub{err: cause}, nil)
348- require.ErrorIs(t, coordinator.Recover(ctx), cause)
349- })
350- t.Run("quarantine write", func(t *testing.T) {
351- repository := newTransactionRepository()
352- repository.tombstones[tombstone.IncarnationID] = tombstone
353- repository.quarantineErr = cause
354- coordinator := NewCoordinator(newTransactionCatalogue(), repository, journalStub{}, nil)
355- require.ErrorIs(t, coordinator.Recover(ctx), cause)
356- require.Empty(t, coordinator.Conflicts())
357- })
358-}
359-
360-func TestDeleteRecoveryAfterCatalogueRemoval(t *testing.T) {
361- ctx := context.Background()
362- old := domain.DeletionTombstone{Name: "work", IncarnationID: domain.IncarnationID{4}}
363- t.Run("absent catalogue record", func(t *testing.T) {
364- catalogue := newTransactionCatalogue()
365- repository := newTransactionRepository()
366- repository.tombstones[old.IncarnationID] = old
367- coordinator := NewCoordinator(catalogue, repository, journalStub{}, nil)
368- require.NoError(t, coordinator.Recover(ctx))
369- require.Empty(t, repository.tombstones)
370- require.Equal(t, []bool{true}, repository.quarantine)
371- })
372- t.Run("reused name preserves new incarnation", func(t *testing.T) {
373- replacement := domain.CatalogueRecord{Name: old.Name, IncarnationID: domain.IncarnationID{9}, RecoveryState: domain.RecoveryFresh}
374- catalogue := newTransactionCatalogue(replacement)
375- repository := newTransactionRepository()
376- repository.tombstones[old.IncarnationID] = old
377- coordinator := NewCoordinator(catalogue, repository, journalStub{}, nil)
378- require.NoError(t, coordinator.Recover(ctx))
379- require.Equal(t, replacement, catalogue.records[old.Name])
380- require.Equal(t, []bool{false}, repository.quarantine)
381- require.Empty(t, repository.tombstones)
382- })
383169 }

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 12

Tags 11

↑↓ moveenter openesc close