detached e4c6c0f

feat(persist): make session catalogue strict and atomic

commit
e4c6c0fe0941eda3df3eef0e0ddc9f7358ded13e
author
brice · · 2026-07-25 19:34 UTC
parent
dfc95be
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
9 files · +1265 -671
Unified
UnifiedSplit
A internal/persist/catalogue_test.go +103 -0
@@ -0,0 +1,103 @@
1+package persist
2+
3+import (
4+ "os"
5+ "path/filepath"
6+ "testing"
7+
8+ "github.com/stretchr/testify/require"
9+
10+ "github.com/bnema/vev/internal/domain"
11+ "github.com/bnema/vev/pkg/kv"
12+)
13+
14+func TestCatalogue(t *testing.T) {
15+ t.Run("record-round-trip", testCatalogueRecordRoundTrip)
16+ t.Run("duplicate-incarnations", testCatalogueDuplicateIncarnations)
17+ t.Run("open-fails-closed", testCatalogueOpenFailsClosed)
18+ t.Run("apply-rename-replace", testCatalogueApplyRenameReplace)
19+ t.Run("read-only-malformed", testCatalogueLoadReadOnlyRejectsMalformedValue)
20+}
21+
22+func testCatalogueRecordRoundTrip(t *testing.T) {
23+ ref1 := &domain.CheckpointRef{Generation: 9, ManifestDigest: [32]byte{1}}
24+ ref2 := &domain.CheckpointRef{Generation: 8, ManifestDigest: [32]byte{2}}
25+ ref3 := &domain.CheckpointRef{Generation: 7, ManifestDigest: [32]byte{3}}
26+ tests := []domain.CatalogueRecord{
27+ {Name: "fresh", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", RecoveryState: domain.RecoveryFresh},
28+ {Name: "healthy", IncarnationID: domain.IncarnationID{2}, RecoveryState: domain.RecoveryHealthy, Committed: ref1},
29+ {Name: "healthy-one-fallback", IncarnationID: domain.IncarnationID{3}, RecoveryState: domain.RecoveryHealthy, Committed: ref1, Fallbacks: [2]*domain.CheckpointRef{ref2}},
30+ {Name: "degraded", IncarnationID: domain.IncarnationID{4}, RecoveryState: domain.RecoveryDegraded, Committed: ref1, Fallbacks: [2]*domain.CheckpointRef{ref2, ref3}, DegradedReason: "invalid manifest"},
31+ {Name: "deleting", IncarnationID: domain.IncarnationID{5}, RecoveryState: domain.RecoveryDeleting},
32+ }
33+ for _, record := range tests {
34+ t.Run(record.Name, func(t *testing.T) {
35+ encoded, err := encodeRecordValue(record)
36+ require.NoError(t, err)
37+ got, err := decodeRecordValue(record.Name, encoded)
38+ require.NoError(t, err)
39+ require.Equal(t, record, got)
40+ require.Error(t, func() error { _, err := decodeRecordValue(record.Name, append(encoded, 0)); return err }())
41+ })
42+ }
43+}
44+
45+func testCatalogueDuplicateIncarnations(t *testing.T) {
46+ id := domain.IncarnationID{1}
47+ require.Error(t, validateUniqueIncarnations([]domain.CatalogueRecord{
48+ {Name: "one", IncarnationID: id, RecoveryState: domain.RecoveryFresh},
49+ {Name: "two", IncarnationID: id, RecoveryState: domain.RecoveryFresh},
50+ }))
51+}
52+
53+func testCatalogueOpenFailsClosed(t *testing.T) {
54+ dir := privateDir(t)
55+ _, _, err := openCurrentCatalogue(dir, false)
56+ require.Error(t, err)
57+
58+ require.NoError(t, os.MkdirAll(dir, 0o700))
59+ require.NoError(t, os.WriteFile(StorePath(dir), rawCorruptCatalogueWAL(), 0o600))
60+ _, _, err = openCurrentCatalogue(dir, false)
61+ require.ErrorIs(t, err, kv.ErrCorruptWAL)
62+
63+ require.NoError(t, os.Remove(StorePath(dir)))
64+ require.NoError(t, os.WriteFile(StorePath(dir)+".prev", rawCorruptCatalogueWAL(), 0o600))
65+ _, _, err = openCurrentCatalogue(dir, false)
66+ require.ErrorIs(t, err, kv.ErrCorruptWAL)
67+}
68+
69+func testCatalogueApplyRenameReplace(t *testing.T) {
70+ dir := privateDir(t)
71+ p, records, err := openCurrentCatalogue(dir, true)
72+ require.NoError(t, err)
73+ require.Empty(t, records)
74+ defer func() { require.NoError(t, p.Close()) }()
75+
76+ one := domain.CatalogueRecord{Name: "one", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}
77+ two := domain.CatalogueRecord{Name: "two", IncarnationID: domain.IncarnationID{2}, RecoveryState: domain.RecoveryFresh}
78+ require.NoError(t, p.Apply(map[string]*domain.CatalogueRecord{"one": &one, "two": &two}))
79+ renamed := one
80+ renamed.Name = "renamed"
81+ require.NoError(t, p.Rename("one", renamed))
82+ two.Cwd = "/next"
83+ require.NoError(t, p.Replace("two", two))
84+
85+ got, err := p.LoadCatalogue()
86+ require.NoError(t, err)
87+ require.ElementsMatch(t, []domain.CatalogueRecord{renamed, two}, got)
88+}
89+
90+func rawCorruptCatalogueWAL() []byte {
91+ // A complete WAL record with an impossible CRC.
92+ return []byte{0, 0, 0, 3, 0, 0, 0, 0, 0xff, 0, 0}
93+}
94+
95+func testCatalogueLoadReadOnlyRejectsMalformedValue(t *testing.T) {
96+ dir := privateDir(t)
97+ store, err := kv.Open(filepath.Join(dir, filename))
98+ require.NoError(t, err)
99+ require.NoError(t, store.Set([]byte("work"), []byte("malformed")))
100+ require.NoError(t, store.Close())
101+ _, err = LoadCatalogueReadOnly(dir)
102+ require.Error(t, err)
103+}
@@ -0,0 +1,103 @@
1+package persist
2+
3+import (
4+ "os"
5+ "path/filepath"
6+ "testing"
7+
8+ "github.com/stretchr/testify/require"
9+
10+ "github.com/bnema/vev/internal/domain"
11+ "github.com/bnema/vev/pkg/kv"
12+)
13+
14+func TestCatalogue(t *testing.T) {
15+ t.Run("record-round-trip", testCatalogueRecordRoundTrip)
16+ t.Run("duplicate-incarnations", testCatalogueDuplicateIncarnations)
17+ t.Run("open-fails-closed", testCatalogueOpenFailsClosed)
18+ t.Run("apply-rename-replace", testCatalogueApplyRenameReplace)
19+ t.Run("read-only-malformed", testCatalogueLoadReadOnlyRejectsMalformedValue)
20+}
21+
22+func testCatalogueRecordRoundTrip(t *testing.T) {
23+ ref1 := &domain.CheckpointRef{Generation: 9, ManifestDigest: [32]byte{1}}
24+ ref2 := &domain.CheckpointRef{Generation: 8, ManifestDigest: [32]byte{2}}
25+ ref3 := &domain.CheckpointRef{Generation: 7, ManifestDigest: [32]byte{3}}
26+ tests := []domain.CatalogueRecord{
27+ {Name: "fresh", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", RecoveryState: domain.RecoveryFresh},
28+ {Name: "healthy", IncarnationID: domain.IncarnationID{2}, RecoveryState: domain.RecoveryHealthy, Committed: ref1},
29+ {Name: "healthy-one-fallback", IncarnationID: domain.IncarnationID{3}, RecoveryState: domain.RecoveryHealthy, Committed: ref1, Fallbacks: [2]*domain.CheckpointRef{ref2}},
30+ {Name: "degraded", IncarnationID: domain.IncarnationID{4}, RecoveryState: domain.RecoveryDegraded, Committed: ref1, Fallbacks: [2]*domain.CheckpointRef{ref2, ref3}, DegradedReason: "invalid manifest"},
31+ {Name: "deleting", IncarnationID: domain.IncarnationID{5}, RecoveryState: domain.RecoveryDeleting},
32+ }
33+ for _, record := range tests {
34+ t.Run(record.Name, func(t *testing.T) {
35+ encoded, err := encodeRecordValue(record)
36+ require.NoError(t, err)
37+ got, err := decodeRecordValue(record.Name, encoded)
38+ require.NoError(t, err)
39+ require.Equal(t, record, got)
40+ require.Error(t, func() error { _, err := decodeRecordValue(record.Name, append(encoded, 0)); return err }())
41+ })
42+ }
43+}
44+
45+func testCatalogueDuplicateIncarnations(t *testing.T) {
46+ id := domain.IncarnationID{1}
47+ require.Error(t, validateUniqueIncarnations([]domain.CatalogueRecord{
48+ {Name: "one", IncarnationID: id, RecoveryState: domain.RecoveryFresh},
49+ {Name: "two", IncarnationID: id, RecoveryState: domain.RecoveryFresh},
50+ }))
51+}
52+
53+func testCatalogueOpenFailsClosed(t *testing.T) {
54+ dir := privateDir(t)
55+ _, _, err := openCurrentCatalogue(dir, false)
56+ require.Error(t, err)
57+
58+ require.NoError(t, os.MkdirAll(dir, 0o700))
59+ require.NoError(t, os.WriteFile(StorePath(dir), rawCorruptCatalogueWAL(), 0o600))
60+ _, _, err = openCurrentCatalogue(dir, false)
61+ require.ErrorIs(t, err, kv.ErrCorruptWAL)
62+
63+ require.NoError(t, os.Remove(StorePath(dir)))
64+ require.NoError(t, os.WriteFile(StorePath(dir)+".prev", rawCorruptCatalogueWAL(), 0o600))
65+ _, _, err = openCurrentCatalogue(dir, false)
66+ require.ErrorIs(t, err, kv.ErrCorruptWAL)
67+}
68+
69+func testCatalogueApplyRenameReplace(t *testing.T) {
70+ dir := privateDir(t)
71+ p, records, err := openCurrentCatalogue(dir, true)
72+ require.NoError(t, err)
73+ require.Empty(t, records)
74+ defer func() { require.NoError(t, p.Close()) }()
75+
76+ one := domain.CatalogueRecord{Name: "one", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}
77+ two := domain.CatalogueRecord{Name: "two", IncarnationID: domain.IncarnationID{2}, RecoveryState: domain.RecoveryFresh}
78+ require.NoError(t, p.Apply(map[string]*domain.CatalogueRecord{"one": &one, "two": &two}))
79+ renamed := one
80+ renamed.Name = "renamed"
81+ require.NoError(t, p.Rename("one", renamed))
82+ two.Cwd = "/next"
83+ require.NoError(t, p.Replace("two", two))
84+
85+ got, err := p.LoadCatalogue()
86+ require.NoError(t, err)
87+ require.ElementsMatch(t, []domain.CatalogueRecord{renamed, two}, got)
88+}
89+
90+func rawCorruptCatalogueWAL() []byte {
91+ // A complete WAL record with an impossible CRC.
92+ return []byte{0, 0, 0, 3, 0, 0, 0, 0, 0xff, 0, 0}
93+}
94+
95+func testCatalogueLoadReadOnlyRejectsMalformedValue(t *testing.T) {
96+ dir := privateDir(t)
97+ store, err := kv.Open(filepath.Join(dir, filename))
98+ require.NoError(t, err)
99+ require.NoError(t, store.Set([]byte("work"), []byte("malformed")))
100+ require.NoError(t, store.Close())
101+ _, err = LoadCatalogueReadOnly(dir)
102+ require.Error(t, err)
103+}
M internal/persist/persist.go +159 -90
@@ -1,174 +1,243 @@
11 package persist
22
33 import (
4- "log/slog"
4+ "errors"
5+ "fmt"
6+ "os"
57 "path/filepath"
68 "sort"
79 "sync"
810
11+ "github.com/bnema/vev/internal/domain"
912 "github.com/bnema/vev/internal/ports"
1013 "github.com/bnema/vev/pkg/kv"
1114 )
1215
1316 const filename = "sessions.kv"
1417
15-// StorePath returns the canonical path for the persisted session metadata store.
16-func StorePath(dir string) string {
17- return filepath.Join(dir, filename)
18-}
18+var errPersistenceUnavailable = errors.New("persist: catalogue unavailable")
19+
20+func StorePath(dir string) string { return filepath.Join(dir, filename) }
1921
20-// Persister stores named session metadata. A nil Store makes all mutating
21-// operations no-ops, so callers can keep persistence optional.
2222 type Persister struct {
2323 store ports.Store
2424 mu sync.Mutex
2525 }
2626
27-// Open opens the session persister under dir.
27+// Open opens an existing strict catalogue and never creates an unproven empty one.
2828 func Open(dir string) (*Persister, error) {
29- store, err := kv.Open(StorePath(dir))
29+ p, _, err := openCurrentCatalogue(dir, false)
30+ return p, err
31+}
32+func New(store ports.Store) *Persister { return &Persister{store: store} }
33+
34+func openCurrentCatalogue(dir string, createProvenEmpty bool) (*Persister, []domain.CatalogueRecord, error) {
35+ path := StorePath(dir)
36+ absent := true
37+ for _, candidate := range []string{path, path + ".next", path + ".prev"} {
38+ _, err := os.Stat(candidate)
39+ if err == nil {
40+ absent = false
41+ continue
42+ }
43+ if !errors.Is(err, os.ErrNotExist) {
44+ return nil, nil, err
45+ }
46+ }
47+ if absent && !createProvenEmpty {
48+ return nil, nil, fmt.Errorf("%w: no catalogue candidates", errPersistenceUnavailable)
49+ }
50+ store, err := kv.Open(path)
3051 if err != nil {
31- return nil, err
52+ return nil, nil, err
3253 }
33- return New(store), nil
34-}
35-
36-// New wraps store in a Persister. Passing nil creates a no-op persister.
37-func New(store ports.Store) *Persister {
38- return &Persister{store: store}
54+ p := New(store)
55+ records, err := p.LoadCatalogue()
56+ if err != nil {
57+ return nil, nil, errors.Join(err, p.Close())
58+ }
59+ return p, records, nil
3960 }
4061
41-// LoadReadOnly replays the session store under dir without mutating it.
42-func LoadReadOnly(dir string) ([]Record, error) {
43- data, err := kv.Replay(StorePath(dir))
62+func LoadReadOnly(dir string) ([]domain.CatalogueRecord, error) { return LoadCatalogueReadOnly(dir) }
63+func LoadCatalogueReadOnly(dir string) ([]domain.CatalogueRecord, error) {
64+ path := StorePath(dir)
65+ if _, err := os.Stat(path); err != nil {
66+ return nil, err
67+ }
68+ data, err := kv.Replay(path)
4469 if err != nil {
4570 return nil, err
4671 }
4772 return decodeAll(data)
4873 }
4974
50-// Save persists a structural session record and syncs it to disk.
51-func (p *Persister) Save(r Record) error {
75+func (p *Persister) Save(record domain.CatalogueRecord) error { return p.Replace(record.Name, record) }
76+func (p *Persister) Apply(records map[string]*domain.CatalogueRecord) error {
5277 if p == nil || p.store == nil {
53- return nil
78+ return errPersistenceUnavailable
5479 }
5580 p.mu.Lock()
5681 defer p.mu.Unlock()
57- value, err := encodeRecordValue(r)
58- if err != nil {
82+ return p.applyLocked(records)
83+}
84+func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) error {
85+ if len(records) == 0 {
86+ return errors.New("persist: empty catalogue batch")
87+ }
88+ projected := make(map[string]domain.CatalogueRecord)
89+ var decodeErr error
90+ p.store.Range(func(key, value []byte) bool {
91+ record, err := decodeRecordValue(string(key), value)
92+ if err != nil {
93+ decodeErr = err
94+ return false
95+ }
96+ projected[string(key)] = record
97+ return true
98+ })
99+ if decodeErr != nil {
100+ return decodeErr
101+ }
102+ changes := make([]ports.StoreChange, 0, len(records))
103+ keys := make([]string, 0, len(records))
104+ for name := range records {
105+ keys = append(keys, name)
106+ }
107+ sort.Strings(keys)
108+ for _, name := range keys {
109+ record := records[name]
110+ if err := domain.ValidateSessionName(name); err != nil {
111+ return err
112+ }
113+ if record == nil {
114+ delete(projected, name)
115+ changes = append(changes, ports.StoreChange{Key: []byte(name), Delete: true})
116+ continue
117+ }
118+ if record.Name != name {
119+ return errors.New("persist: catalogue key/name mismatch")
120+ }
121+ value, err := encodeRecordValue(*record)
122+ if err != nil {
123+ return err
124+ }
125+ projected[name] = *record
126+ changes = append(changes, ports.StoreChange{Key: []byte(name), Value: value})
127+ }
128+ all := make([]domain.CatalogueRecord, 0, len(projected))
129+ for _, record := range projected {
130+ all = append(all, record)
131+ }
132+ if err := validateUniqueIncarnations(all); err != nil {
59133 return err
60134 }
61- if err := p.store.Set([]byte(r.Name), value); err != nil {
135+ if err := p.store.Batch(changes); err != nil {
62136 return err
63137 }
64138 return p.store.Sync()
65139 }
140+func (p *Persister) Rename(oldName string, next domain.CatalogueRecord) error {
141+ if oldName == next.Name {
142+ return errors.New("persist: rename requires distinct names")
143+ }
144+ return p.Apply(map[string]*domain.CatalogueRecord{oldName: nil, next.Name: &next})
145+}
146+func (p *Persister) Replace(name string, next domain.CatalogueRecord) error {
147+ return p.Apply(map[string]*domain.CatalogueRecord{name: &next})
148+}
66149
67-// Touch updates hot-path session cwd metadata without forcing an fsync.
150+// Touch updates metadata while retaining the durable identity and recovery state.
68151 func (p *Persister) Touch(name, cwd string, at int64) error {
69152 if p == nil || p.store == nil {
70- return nil
71- }
72- if name == "" {
73- return errEmptyName
153+ return errPersistenceUnavailable
74154 }
75155 p.mu.Lock()
76156 defer p.mu.Unlock()
77-
78- createdAt := at
79- lastUsedSeq := uint64(0)
80- var tabNames []string
81- if v, ok := p.store.Get([]byte(name)); ok {
82- if r, err := decodeRecordValue(name, v); err == nil {
83- createdAt = r.CreatedAt
84- lastUsedSeq = r.LastUsedSeq
85- tabNames = r.TabNames
86- }
157+ value, ok := p.store.Get([]byte(name))
158+ if !ok {
159+ return errors.New("persist: session not found")
87160 }
88-
89- value, err := encodeRecordValue(Record{Name: name, Cwd: cwd, CreatedAt: createdAt, UpdatedAt: at, LastUsedSeq: lastUsedSeq, TabNames: tabNames})
161+ record, err := decodeRecordValue(name, value)
90162 if err != nil {
91163 return err
92164 }
93- return p.store.Set([]byte(name), value)
165+ record.Cwd, record.UpdatedAt = cwd, at
166+ encoded, err := encodeRecordValue(record)
167+ if err != nil {
168+ return err
169+ }
170+ return p.store.Set([]byte(name), encoded)
94171 }
95-
96-// TouchMRU updates hot-path session recency without forcing an fsync.
97-func (p *Persister) TouchMRU(name string, lastUsedSeq uint64) error {
172+func (p *Persister) TouchMRU(name string, sequence uint64) error {
98173 if p == nil || p.store == nil {
99- return nil
100- }
101- if name == "" {
102- return errEmptyName
174+ return errPersistenceUnavailable
103175 }
104176 p.mu.Lock()
105177 defer p.mu.Unlock()
106-
107- r := Record{Name: name, LastUsedSeq: lastUsedSeq}
108- if v, ok := p.store.Get([]byte(name)); ok {
109- if decoded, err := decodeRecordValue(name, v); err == nil {
110- r = decoded
111- r.LastUsedSeq = lastUsedSeq
112- }
178+ value, ok := p.store.Get([]byte(name))
179+ if !ok {
180+ return errors.New("persist: session not found")
113181 }
114- value, err := encodeRecordValue(r)
182+ record, err := decodeRecordValue(name, value)
115183 if err != nil {
116184 return err
117185 }
118- return p.store.Set([]byte(name), value)
119-}
120-
121-// Delete removes a persisted session record and syncs the structural change.
122-func (p *Persister) Delete(name string) error {
123- if p == nil || p.store == nil {
124- return nil
125- }
126- if name == "" {
127- return errEmptyName
128- }
129- p.mu.Lock()
130- defer p.mu.Unlock()
131- if err := p.store.Delete([]byte(name)); err != nil {
186+ record.LastUsedSeq = sequence
187+ encoded, err := encodeRecordValue(record)
188+ if err != nil {
132189 return err
133190 }
134- return p.store.Sync()
191+ return p.store.Set([]byte(name), encoded)
135192 }
136-
137-// LoadAll returns all persisted session records sorted by name.
138-func (p *Persister) LoadAll() ([]Record, error) {
193+func (p *Persister) Delete(name string) error {
194+ return p.Apply(map[string]*domain.CatalogueRecord{name: nil})
195+}
196+func (p *Persister) LoadAll() ([]domain.CatalogueRecord, error) { return p.LoadCatalogue() }
197+func (p *Persister) LoadCatalogue() ([]domain.CatalogueRecord, error) {
139198 if p == nil || p.store == nil {
140- return nil, nil
199+ return nil, errPersistenceUnavailable
141200 }
142201 p.mu.Lock()
143202 defer p.mu.Unlock()
144203 data := make(map[string][]byte)
145- p.store.Range(func(k, v []byte) bool {
146- data[string(k)] = append([]byte(nil), v...)
147- return true
148- })
204+ p.store.Range(func(key, value []byte) bool { data[string(key)] = append([]byte(nil), value...); return true })
149205 return decodeAll(data)
150206 }
151-
152-// Close closes the underlying store.
153207 func (p *Persister) Close() error {
154208 if p == nil || p.store == nil {
155- return nil
209+ return errPersistenceUnavailable
156210 }
157211 p.mu.Lock()
158212 defer p.mu.Unlock()
159213 return p.store.Close()
160214 }
161215
162-func decodeAll(data map[string][]byte) ([]Record, error) {
163- records := make([]Record, 0, len(data))
216+func decodeAll(data map[string][]byte) ([]domain.CatalogueRecord, error) {
217+ records := make([]domain.CatalogueRecord, 0, len(data))
164218 for name, value := range data {
165- r, err := decodeRecordValue(name, value)
219+ record, err := decodeRecordValue(name, value)
166220 if err != nil {
167- slog.Warn("skipping malformed persisted session", "session", name, "err", err)
168- continue
221+ return nil, fmt.Errorf("persist: decode catalogue key %q: %w", name, err)
169222 }
170- records = append(records, r)
223+ records = append(records, record)
171224 }
172225 sort.Slice(records, func(i, j int) bool { return records[i].Name < records[j].Name })
226+ if err := validateUniqueIncarnations(records); err != nil {
227+ return nil, err
228+ }
173229 return records, nil
174230 }
231+func validateUniqueIncarnations(records []domain.CatalogueRecord) error {
232+ seen := make(map[domain.IncarnationID]string, len(records))
233+ for _, record := range records {
234+ if err := record.Validate(); err != nil {
235+ return err
236+ }
237+ if previous, ok := seen[record.IncarnationID]; ok {
238+ return fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, record.Name)
239+ }
240+ seen[record.IncarnationID] = record.Name
241+ }
242+ return nil
243+}
@@ -1,174 +1,243 @@
1 package persist1 package persist
2 2
3 import (3 import (
4- "log/slog"4+ "errors"
5+ "fmt"
6+ "os"
5 "path/filepath"7 "path/filepath"
6 "sort"8 "sort"
7 "sync"9 "sync"
8 10
11+ "github.com/bnema/vev/internal/domain"
9 "github.com/bnema/vev/internal/ports"12 "github.com/bnema/vev/internal/ports"
10 "github.com/bnema/vev/pkg/kv"13 "github.com/bnema/vev/pkg/kv"
11 )14 )
12 15
13 const filename = "sessions.kv"16 const filename = "sessions.kv"
14 17
15-// StorePath returns the canonical path for the persisted session metadata store.18+var errPersistenceUnavailable = errors.New("persist: catalogue unavailable")
16-func StorePath(dir string) string {19+
17- return filepath.Join(dir, filename)20+func StorePath(dir string) string { return filepath.Join(dir, filename) }
18-}
19 21
20-// Persister stores named session metadata. A nil Store makes all mutating
21-// operations no-ops, so callers can keep persistence optional.
22 type Persister struct {22 type Persister struct {
23 store ports.Store23 store ports.Store
24 mu sync.Mutex24 mu sync.Mutex
25 }25 }
26 26
27-// Open opens the session persister under dir.27+// Open opens an existing strict catalogue and never creates an unproven empty one.
28 func Open(dir string) (*Persister, error) {28 func Open(dir string) (*Persister, error) {
29- store, err := kv.Open(StorePath(dir))29+ p, _, err := openCurrentCatalogue(dir, false)
30+ return p, err
31+}
32+func New(store ports.Store) *Persister { return &Persister{store: store} }
33+
34+func openCurrentCatalogue(dir string, createProvenEmpty bool) (*Persister, []domain.CatalogueRecord, error) {
35+ path := StorePath(dir)
36+ absent := true
37+ for _, candidate := range []string{path, path + ".next", path + ".prev"} {
38+ _, err := os.Stat(candidate)
39+ if err == nil {
40+ absent = false
41+ continue
42+ }
43+ if !errors.Is(err, os.ErrNotExist) {
44+ return nil, nil, err
45+ }
46+ }
47+ if absent && !createProvenEmpty {
48+ return nil, nil, fmt.Errorf("%w: no catalogue candidates", errPersistenceUnavailable)
49+ }
50+ store, err := kv.Open(path)
30 if err != nil {51 if err != nil {
31- return nil, err52+ return nil, nil, err
32 }53 }
33- return New(store), nil54+ p := New(store)
34-}55+ records, err := p.LoadCatalogue()
35-56+ if err != nil {
36-// New wraps store in a Persister. Passing nil creates a no-op persister.57+ return nil, nil, errors.Join(err, p.Close())
37-func New(store ports.Store) *Persister {58+ }
38- return &Persister{store: store}59+ return p, records, nil
39 }60 }
40 61
41-// LoadReadOnly replays the session store under dir without mutating it.62+func LoadReadOnly(dir string) ([]domain.CatalogueRecord, error) { return LoadCatalogueReadOnly(dir) }
42-func LoadReadOnly(dir string) ([]Record, error) {63+func LoadCatalogueReadOnly(dir string) ([]domain.CatalogueRecord, error) {
43- data, err := kv.Replay(StorePath(dir))64+ path := StorePath(dir)
65+ if _, err := os.Stat(path); err != nil {
66+ return nil, err
67+ }
68+ data, err := kv.Replay(path)
44 if err != nil {69 if err != nil {
45 return nil, err70 return nil, err
46 }71 }
47 return decodeAll(data)72 return decodeAll(data)
48 }73 }
49 74
50-// Save persists a structural session record and syncs it to disk.75+func (p *Persister) Save(record domain.CatalogueRecord) error { return p.Replace(record.Name, record) }
51-func (p *Persister) Save(r Record) error {76+func (p *Persister) Apply(records map[string]*domain.CatalogueRecord) error {
52 if p == nil || p.store == nil {77 if p == nil || p.store == nil {
53- return nil78+ return errPersistenceUnavailable
54 }79 }
55 p.mu.Lock()80 p.mu.Lock()
56 defer p.mu.Unlock()81 defer p.mu.Unlock()
57- value, err := encodeRecordValue(r)82+ return p.applyLocked(records)
58- if err != nil {83+}
84+func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) error {
85+ if len(records) == 0 {
86+ return errors.New("persist: empty catalogue batch")
87+ }
88+ projected := make(map[string]domain.CatalogueRecord)
89+ var decodeErr error
90+ p.store.Range(func(key, value []byte) bool {
91+ record, err := decodeRecordValue(string(key), value)
92+ if err != nil {
93+ decodeErr = err
94+ return false
95+ }
96+ projected[string(key)] = record
97+ return true
98+ })
99+ if decodeErr != nil {
100+ return decodeErr
101+ }
102+ changes := make([]ports.StoreChange, 0, len(records))
103+ keys := make([]string, 0, len(records))
104+ for name := range records {
105+ keys = append(keys, name)
106+ }
107+ sort.Strings(keys)
108+ for _, name := range keys {
109+ record := records[name]
110+ if err := domain.ValidateSessionName(name); err != nil {
111+ return err
112+ }
113+ if record == nil {
114+ delete(projected, name)
115+ changes = append(changes, ports.StoreChange{Key: []byte(name), Delete: true})
116+ continue
117+ }
118+ if record.Name != name {
119+ return errors.New("persist: catalogue key/name mismatch")
120+ }
121+ value, err := encodeRecordValue(*record)
122+ if err != nil {
123+ return err
124+ }
125+ projected[name] = *record
126+ changes = append(changes, ports.StoreChange{Key: []byte(name), Value: value})
127+ }
128+ all := make([]domain.CatalogueRecord, 0, len(projected))
129+ for _, record := range projected {
130+ all = append(all, record)
131+ }
132+ if err := validateUniqueIncarnations(all); err != nil {
59 return err133 return err
60 }134 }
61- if err := p.store.Set([]byte(r.Name), value); err != nil {135+ if err := p.store.Batch(changes); err != nil {
62 return err136 return err
63 }137 }
64 return p.store.Sync()138 return p.store.Sync()
65 }139 }
140+func (p *Persister) Rename(oldName string, next domain.CatalogueRecord) error {
141+ if oldName == next.Name {
142+ return errors.New("persist: rename requires distinct names")
143+ }
144+ return p.Apply(map[string]*domain.CatalogueRecord{oldName: nil, next.Name: &next})
145+}
146+func (p *Persister) Replace(name string, next domain.CatalogueRecord) error {
147+ return p.Apply(map[string]*domain.CatalogueRecord{name: &next})
148+}
66 149
67-// Touch updates hot-path session cwd metadata without forcing an fsync.150+// Touch updates metadata while retaining the durable identity and recovery state.
68 func (p *Persister) Touch(name, cwd string, at int64) error {151 func (p *Persister) Touch(name, cwd string, at int64) error {
69 if p == nil || p.store == nil {152 if p == nil || p.store == nil {
70- return nil153+ return errPersistenceUnavailable
71- }
72- if name == "" {
73- return errEmptyName
74 }154 }
75 p.mu.Lock()155 p.mu.Lock()
76 defer p.mu.Unlock()156 defer p.mu.Unlock()
77-157+ value, ok := p.store.Get([]byte(name))
78- createdAt := at158+ if !ok {
79- lastUsedSeq := uint64(0)159+ return errors.New("persist: session not found")
80- var tabNames []string
81- if v, ok := p.store.Get([]byte(name)); ok {
82- if r, err := decodeRecordValue(name, v); err == nil {
83- createdAt = r.CreatedAt
84- lastUsedSeq = r.LastUsedSeq
85- tabNames = r.TabNames
86- }
87 }160 }
88-161+ record, err := decodeRecordValue(name, value)
89- value, err := encodeRecordValue(Record{Name: name, Cwd: cwd, CreatedAt: createdAt, UpdatedAt: at, LastUsedSeq: lastUsedSeq, TabNames: tabNames})
90 if err != nil {162 if err != nil {
91 return err163 return err
92 }164 }
93- return p.store.Set([]byte(name), value)165+ record.Cwd, record.UpdatedAt = cwd, at
166+ encoded, err := encodeRecordValue(record)
167+ if err != nil {
168+ return err
169+ }
170+ return p.store.Set([]byte(name), encoded)
94 }171 }
95-172+func (p *Persister) TouchMRU(name string, sequence uint64) error {
96-// TouchMRU updates hot-path session recency without forcing an fsync.
97-func (p *Persister) TouchMRU(name string, lastUsedSeq uint64) error {
98 if p == nil || p.store == nil {173 if p == nil || p.store == nil {
99- return nil174+ return errPersistenceUnavailable
100- }
101- if name == "" {
102- return errEmptyName
103 }175 }
104 p.mu.Lock()176 p.mu.Lock()
105 defer p.mu.Unlock()177 defer p.mu.Unlock()
106-178+ value, ok := p.store.Get([]byte(name))
107- r := Record{Name: name, LastUsedSeq: lastUsedSeq}179+ if !ok {
108- if v, ok := p.store.Get([]byte(name)); ok {180+ return errors.New("persist: session not found")
109- if decoded, err := decodeRecordValue(name, v); err == nil {
110- r = decoded
111- r.LastUsedSeq = lastUsedSeq
112- }
113 }181 }
114- value, err := encodeRecordValue(r)182+ record, err := decodeRecordValue(name, value)
115 if err != nil {183 if err != nil {
116 return err184 return err
117 }185 }
118- return p.store.Set([]byte(name), value)186+ record.LastUsedSeq = sequence
119-}187+ encoded, err := encodeRecordValue(record)
120-188+ if err != nil {
121-// Delete removes a persisted session record and syncs the structural change.
122-func (p *Persister) Delete(name string) error {
123- if p == nil || p.store == nil {
124- return nil
125- }
126- if name == "" {
127- return errEmptyName
128- }
129- p.mu.Lock()
130- defer p.mu.Unlock()
131- if err := p.store.Delete([]byte(name)); err != nil {
132 return err189 return err
133 }190 }
134- return p.store.Sync()191+ return p.store.Set([]byte(name), encoded)
135 }192 }
136-193+func (p *Persister) Delete(name string) error {
137-// LoadAll returns all persisted session records sorted by name.194+ return p.Apply(map[string]*domain.CatalogueRecord{name: nil})
138-func (p *Persister) LoadAll() ([]Record, error) {195+}
196+func (p *Persister) LoadAll() ([]domain.CatalogueRecord, error) { return p.LoadCatalogue() }
197+func (p *Persister) LoadCatalogue() ([]domain.CatalogueRecord, error) {
139 if p == nil || p.store == nil {198 if p == nil || p.store == nil {
140- return nil, nil199+ return nil, errPersistenceUnavailable
141 }200 }
142 p.mu.Lock()201 p.mu.Lock()
143 defer p.mu.Unlock()202 defer p.mu.Unlock()
144 data := make(map[string][]byte)203 data := make(map[string][]byte)
145- p.store.Range(func(k, v []byte) bool {204+ p.store.Range(func(key, value []byte) bool { data[string(key)] = append([]byte(nil), value...); return true })
146- data[string(k)] = append([]byte(nil), v...)
147- return true
148- })
149 return decodeAll(data)205 return decodeAll(data)
150 }206 }
151-
152-// Close closes the underlying store.
153 func (p *Persister) Close() error {207 func (p *Persister) Close() error {
154 if p == nil || p.store == nil {208 if p == nil || p.store == nil {
155- return nil209+ return errPersistenceUnavailable
156 }210 }
157 p.mu.Lock()211 p.mu.Lock()
158 defer p.mu.Unlock()212 defer p.mu.Unlock()
159 return p.store.Close()213 return p.store.Close()
160 }214 }
161 215
162-func decodeAll(data map[string][]byte) ([]Record, error) {216+func decodeAll(data map[string][]byte) ([]domain.CatalogueRecord, error) {
163- records := make([]Record, 0, len(data))217+ records := make([]domain.CatalogueRecord, 0, len(data))
164 for name, value := range data {218 for name, value := range data {
165- r, err := decodeRecordValue(name, value)219+ record, err := decodeRecordValue(name, value)
166 if err != nil {220 if err != nil {
167- slog.Warn("skipping malformed persisted session", "session", name, "err", err)221+ return nil, fmt.Errorf("persist: decode catalogue key %q: %w", name, err)
168- continue
169 }222 }
170- records = append(records, r)223+ records = append(records, record)
171 }224 }
172 sort.Slice(records, func(i, j int) bool { return records[i].Name < records[j].Name })225 sort.Slice(records, func(i, j int) bool { return records[i].Name < records[j].Name })
226+ if err := validateUniqueIncarnations(records); err != nil {
227+ return nil, err
228+ }
173 return records, nil229 return records, nil
174 }230 }
231+func validateUniqueIncarnations(records []domain.CatalogueRecord) error {
232+ seen := make(map[domain.IncarnationID]string, len(records))
233+ for _, record := range records {
234+ if err := record.Validate(); err != nil {
235+ return err
236+ }
237+ if previous, ok := seen[record.IncarnationID]; ok {
238+ return fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, record.Name)
239+ }
240+ seen[record.IncarnationID] = record.Name
241+ }
242+ return nil
243+}
M internal/persist/persist_test.go +48 -277
@@ -3,300 +3,71 @@ package persist
33 import (
44 "errors"
55 "path/filepath"
6- "sync"
76 "testing"
8- "time"
97
108 "github.com/stretchr/testify/mock"
119 "github.com/stretchr/testify/require"
1210
11+ "github.com/bnema/vev/internal/domain"
12+ "github.com/bnema/vev/internal/ports"
1313 portsmocks "github.com/bnema/vev/internal/ports/mocks"
1414 )
1515
16-func privateDir(t *testing.T) string {
17- t.Helper()
18- return filepath.Join(t.TempDir(), "vev")
16+func privateDir(t *testing.T) string { t.Helper(); return filepath.Join(t.TempDir(), "vev") }
17+func validRecord(name string, id byte) domain.CatalogueRecord {
18+ return domain.CatalogueRecord{Name: name, IncarnationID: domain.IncarnationID{id}, RecoveryState: domain.RecoveryFresh}
1919 }
2020
21-func TestRecordCodecRoundTrip(t *testing.T) {
22- r := Record{Name: "work", Cwd: "/tmp/project", CreatedAt: 11, UpdatedAt: 22, LastUsedSeq: 33, TabNames: []string{"shell", "logs"}}
23- value, err := encodeRecordValue(r)
21+func TestCatalogueDecodeAllFailsOnMalformedRecord(t *testing.T) {
22+ good, err := encodeRecordValue(validRecord("good", 1))
2423 require.NoError(t, err)
25-
26- got, err := decodeRecordValue(r.Name, value)
27- require.NoError(t, err)
28- require.Equal(t, r, got)
29-}
30-
31-func TestRecordCodecOldRecordDefaultsLastUsed(t *testing.T) {
32- old := []byte{0, 0, 0, 4, '/', 't', 'm', 'p'}
33- old = append(old, 0, 0, 0, 0, 0, 0, 0, 11)
34- old = append(old, 0, 0, 0, 0, 0, 0, 0, 22)
35-
36- got, err := decodeRecordValue("work", old)
37- require.NoError(t, err)
38- require.Equal(t, Record{Name: "work", Cwd: "/tmp", CreatedAt: 11, UpdatedAt: 22}, got)
39-}
40-
41-func TestRecordCodecRejectsMalformedData(t *testing.T) {
42- _, err := encodeRecordValue(Record{Cwd: "/tmp"})
43- require.ErrorIs(t, err, errEmptyName)
44-
45- badValues := [][]byte{
46- nil,
47- {0, 0, 0},
48- {0, 0, 0, 5, 'a'},
49- {0, 0, 0, 1, 'a', 0},
50- {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
51- }
52- for _, value := range badValues {
53- _, err := decodeRecordValue("name", value)
54- require.ErrorIs(t, err, errMalformedRecord)
55- }
56-
57- _, err = decodeRecordValue("", make([]byte, 20))
58- require.ErrorIs(t, err, errEmptyName)
59-}
60-
61-func TestNilPersisterNoOps(t *testing.T) {
62- var p *Persister
63- require.NoError(t, p.Save(Record{Name: "a", Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1}))
64- require.NoError(t, p.Touch("a", "/tmp/next", 2))
65- require.NoError(t, p.Delete("a"))
66- records, err := p.LoadAll()
67- require.NoError(t, err)
68- require.Empty(t, records)
69- require.NoError(t, p.Close())
70-
71- p = New(nil)
72- require.NoError(t, p.Save(Record{Name: "a", Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1}))
73- require.NoError(t, p.Touch("a", "/tmp/next", 2))
74- require.NoError(t, p.Delete("a"))
75- records, err = p.LoadAll()
76- require.NoError(t, err)
77- require.Empty(t, records)
78- require.NoError(t, p.Close())
79-}
80-
81-func TestPersisterSaveTouchDeleteLoadAll(t *testing.T) {
82- dir := privateDir(t)
83- p, err := Open(dir)
84- require.NoError(t, err)
85-
86- require.NoError(t, p.Save(Record{Name: "b", Cwd: "/b", CreatedAt: 10, UpdatedAt: 10, LastUsedSeq: 40, TabNames: []string{"shell"}}))
87- require.NoError(t, p.Save(Record{Name: "a", Cwd: "/a", CreatedAt: 20, UpdatedAt: 20, LastUsedSeq: 50}))
88- require.NoError(t, p.Touch("b", "/b/next", 30))
89- require.NoError(t, p.TouchMRU("b", 60))
90-
91- records, err := p.LoadAll()
92- require.NoError(t, err)
93- require.Equal(t, []Record{
94- {Name: "a", Cwd: "/a", CreatedAt: 20, UpdatedAt: 20, LastUsedSeq: 50},
95- {Name: "b", Cwd: "/b/next", CreatedAt: 10, UpdatedAt: 30, LastUsedSeq: 60, TabNames: []string{"shell"}},
96- }, records)
97-
98- require.NoError(t, p.Delete("a"))
99- records, err = p.LoadAll()
100- require.NoError(t, err)
101- require.Equal(t, []Record{{Name: "b", Cwd: "/b/next", CreatedAt: 10, UpdatedAt: 30, LastUsedSeq: 60, TabNames: []string{"shell"}}}, records)
102- require.NoError(t, p.Close())
103-}
104-
105-func TestDecodeAllSkipsMalformedRecords(t *testing.T) {
106- good, err := encodeRecordValue(Record{Name: "good", Cwd: "/ok", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3})
107- require.NoError(t, err)
108-
109- records, err := decodeAll(map[string][]byte{
110- "bad": []byte{0, 0, 0, 5, 'x'},
111- "good": good,
112- })
113- require.NoError(t, err)
114- require.Equal(t, []Record{{Name: "good", Cwd: "/ok", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3}}, records)
115-}
116-
117-func TestLoadReadOnlyUsesReplay(t *testing.T) {
118- dir := privateDir(t)
119- p, err := Open(dir)
120- require.NoError(t, err)
121- require.NoError(t, p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3}))
122- require.NoError(t, p.Close())
123-
124- records, err := LoadReadOnly(dir)
125- require.NoError(t, err)
126- require.Equal(t, []Record{{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3}}, records)
127-
128- // Missing stores replay as empty rather than creating the WAL.
129- missing := filepath.Join(t.TempDir(), "missing")
130- records, err = LoadReadOnly(missing)
131- require.NoError(t, err)
132- require.Empty(t, records)
133-}
134-
135-func TestSyncBehavior(t *testing.T) {
136- store, state := newMockStore(t)
137- p := New(store)
138-
139- require.NoError(t, p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 1}))
140- require.Equal(t, 1, state.sets)
141- require.Equal(t, 1, state.syncs)
142-
143- require.NoError(t, p.Touch("work", "/work/next", 2))
144- require.Equal(t, 2, state.sets)
145- require.Equal(t, 1, state.syncs)
146-
147- require.NoError(t, p.TouchMRU("work", 3))
148- require.Equal(t, 3, state.sets)
149- require.Equal(t, 1, state.syncs)
150-
151- require.NoError(t, p.Delete("work"))
152- require.Equal(t, 1, state.deletes)
153- require.Equal(t, 2, state.syncs)
24+ _, err = decodeAll(map[string][]byte{"bad": {0, 1}, "good": good})
25+ require.Error(t, err)
15426 }
15527
156-func TestPersisterSerializesReadModifyWriteWithStructuralSave(t *testing.T) {
157- store := newBlockingGetStore()
28+func TestCatalogueBatchSyncBehavior(t *testing.T) {
29+ store := portsmocks.NewMockStore(t)
30+ state := map[string][]byte{}
31+ store.EXPECT().Range(mock.Anything).Run(func(fn func([]byte, []byte) bool) {
32+ for key, value := range state {
33+ if !fn([]byte(key), value) {
34+ return
35+ }
36+ }
37+ }).Twice()
38+ store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error {
39+ for _, change := range changes {
40+ if change.Delete {
41+ delete(state, string(change.Key))
42+ } else {
43+ state[string(change.Key)] = append([]byte(nil), change.Value...)
44+ }
45+ }
46+ return nil
47+ }).Twice()
48+ store.EXPECT().Sync().Return(nil).Twice()
15849 p := New(store)
159- require.NoError(t, p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, TabNames: []string{"old"}}))
160- store.blockNextGet()
161-
162- touchDone := make(chan error, 1)
163- go func() { touchDone <- p.TouchMRU("work", 2) }()
164- <-store.getStarted
165-
166- saveDone := make(chan error, 1)
167- go func() {
168- saveDone <- p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 3, LastUsedSeq: 2, TabNames: []string{"new"}})
169- }()
170- select {
171- case err := <-saveDone:
172- t.Fatalf("structural save interleaved with TouchMRU: %v", err)
173- case <-time.After(20 * time.Millisecond):
174- }
175- close(store.releaseGet)
176- require.NoError(t, <-touchDone)
177- require.NoError(t, <-saveDone)
178-
179- records, err := p.LoadAll()
180- require.NoError(t, err)
181- require.Equal(t, []string{"new"}, records[0].TabNames)
50+ one := validRecord("one", 1)
51+ require.NoError(t, p.Replace("one", one))
52+ renamed := one
53+ renamed.Name = "renamed"
54+ require.NoError(t, p.Rename("one", renamed))
18255 }
18356
184-func TestSavePropagatesSetErrorWithoutSync(t *testing.T) {
185- store, state := newMockStore(t)
186- state.errSet = errors.New("set failed")
57+func TestCatalogueBatchFailureDoesNotSync(t *testing.T) {
58+ store := portsmocks.NewMockStore(t)
59+ batchErr := errors.New("batch failed")
60+ store.EXPECT().Range(mock.Anything).Run(func(func([]byte, []byte) bool) {})
61+ store.EXPECT().Batch(mock.Anything).Return(batchErr)
18762 p := New(store)
188-
189- err := p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 1})
190- require.ErrorIs(t, err, state.errSet)
191- require.Zero(t, state.syncs)
192-}
193-
194-type mockStoreState struct {
195- data map[string][]byte
196- sets int
197- deletes int
198- syncs int
199- closed bool
200- errSet error
201-}
202-
203-type blockingGetStore struct {
204- mu sync.Mutex
205- data map[string][]byte
206- block bool
207- getStarted chan struct{}
208- releaseGet chan struct{}
209- getStartOnce sync.Once
210-}
211-
212-func newBlockingGetStore() *blockingGetStore {
213- return &blockingGetStore{data: make(map[string][]byte), getStarted: make(chan struct{}), releaseGet: make(chan struct{})}
214-}
215-
216-func (s *blockingGetStore) blockNextGet() {
217- s.mu.Lock()
218- s.block = true
219- s.mu.Unlock()
220-}
221-
222-func (s *blockingGetStore) Get(key []byte) ([]byte, bool) {
223- s.mu.Lock()
224- v, ok := s.data[string(key)]
225- v = append([]byte(nil), v...)
226- block := s.block
227- if block {
228- s.block = false
229- }
230- s.mu.Unlock()
231- if block {
232- s.getStartOnce.Do(func() { close(s.getStarted) })
233- <-s.releaseGet
234- }
235- return v, ok
63+ err := p.Replace("one", validRecord("one", 1))
64+ require.ErrorIs(t, err, batchErr)
23665 }
23766
238-func (s *blockingGetStore) Set(key, value []byte) error {
239- s.mu.Lock()
240- s.data[string(key)] = append([]byte(nil), value...)
241- s.mu.Unlock()
242- return nil
243-}
244-
245-func (s *blockingGetStore) Delete(key []byte) error {
246- s.mu.Lock()
247- delete(s.data, string(key))
248- s.mu.Unlock()
249- return nil
250-}
251-
252-func (s *blockingGetStore) Range(fn func(k, v []byte) bool) {
253- s.mu.Lock()
254- defer s.mu.Unlock()
255- for key, value := range s.data {
256- if !fn([]byte(key), append([]byte(nil), value...)) {
257- return
258- }
259- }
260-}
261-
262-func (*blockingGetStore) Sync() error { return nil }
263-func (*blockingGetStore) Close() error { return nil }
264-
265-func newMockStore(t *testing.T) (*portsmocks.MockStore, *mockStoreState) {
266- t.Helper()
267- state := &mockStoreState{data: make(map[string][]byte)}
268- store := portsmocks.NewMockStore(t)
269- store.EXPECT().Get(mock.Anything).RunAndReturn(func(key []byte) ([]byte, bool) {
270- v, ok := state.data[string(key)]
271- return append([]byte(nil), v...), ok
272- }).Maybe()
273- store.EXPECT().Set(mock.Anything, mock.Anything).RunAndReturn(func(key, val []byte) error {
274- state.sets++
275- if state.errSet != nil {
276- return state.errSet
277- }
278- state.data[string(key)] = append([]byte(nil), val...)
279- return nil
280- }).Maybe()
281- store.EXPECT().Delete(mock.Anything).RunAndReturn(func(key []byte) error {
282- state.deletes++
283- delete(state.data, string(key))
284- return nil
285- }).Maybe()
286- store.EXPECT().Range(mock.Anything).Run(func(fn func(k, v []byte) bool) {
287- for k, v := range state.data {
288- if !fn([]byte(k), append([]byte(nil), v...)) {
289- return
290- }
291- }
292- }).Maybe()
293- store.EXPECT().Sync().RunAndReturn(func() error {
294- state.syncs++
295- return nil
296- }).Maybe()
297- store.EXPECT().Close().RunAndReturn(func() error {
298- state.closed = true
299- return nil
300- }).Maybe()
301- return store, state
67+func TestCatalogueNilPersisterFailsClosed(t *testing.T) {
68+ var p *Persister
69+ require.ErrorIs(t, p.Save(validRecord("one", 1)), errPersistenceUnavailable)
70+ _, err := p.LoadAll()
71+ require.ErrorIs(t, err, errPersistenceUnavailable)
72+ require.ErrorIs(t, p.Close(), errPersistenceUnavailable)
30273 }
@@ -3,300 +3,71 @@ package persist
3 import (3 import (
4 "errors"4 "errors"
5 "path/filepath"5 "path/filepath"
6- "sync"
7 "testing"6 "testing"
8- "time"
9 7
10 "github.com/stretchr/testify/mock"8 "github.com/stretchr/testify/mock"
11 "github.com/stretchr/testify/require"9 "github.com/stretchr/testify/require"
12 10
11+ "github.com/bnema/vev/internal/domain"
12+ "github.com/bnema/vev/internal/ports"
13 portsmocks "github.com/bnema/vev/internal/ports/mocks"13 portsmocks "github.com/bnema/vev/internal/ports/mocks"
14 )14 )
15 15
16-func privateDir(t *testing.T) string {16+func privateDir(t *testing.T) string { t.Helper(); return filepath.Join(t.TempDir(), "vev") }
17- t.Helper()17+func validRecord(name string, id byte) domain.CatalogueRecord {
18- return filepath.Join(t.TempDir(), "vev")18+ return domain.CatalogueRecord{Name: name, IncarnationID: domain.IncarnationID{id}, RecoveryState: domain.RecoveryFresh}
19 }19 }
20 20
21-func TestRecordCodecRoundTrip(t *testing.T) {21+func TestCatalogueDecodeAllFailsOnMalformedRecord(t *testing.T) {
22- r := Record{Name: "work", Cwd: "/tmp/project", CreatedAt: 11, UpdatedAt: 22, LastUsedSeq: 33, TabNames: []string{"shell", "logs"}}22+ good, err := encodeRecordValue(validRecord("good", 1))
23- value, err := encodeRecordValue(r)
24 require.NoError(t, err)23 require.NoError(t, err)
25-24+ _, err = decodeAll(map[string][]byte{"bad": {0, 1}, "good": good})
26- got, err := decodeRecordValue(r.Name, value)25+ require.Error(t, err)
27- require.NoError(t, err)
28- require.Equal(t, r, got)
29-}
30-
31-func TestRecordCodecOldRecordDefaultsLastUsed(t *testing.T) {
32- old := []byte{0, 0, 0, 4, '/', 't', 'm', 'p'}
33- old = append(old, 0, 0, 0, 0, 0, 0, 0, 11)
34- old = append(old, 0, 0, 0, 0, 0, 0, 0, 22)
35-
36- got, err := decodeRecordValue("work", old)
37- require.NoError(t, err)
38- require.Equal(t, Record{Name: "work", Cwd: "/tmp", CreatedAt: 11, UpdatedAt: 22}, got)
39-}
40-
41-func TestRecordCodecRejectsMalformedData(t *testing.T) {
42- _, err := encodeRecordValue(Record{Cwd: "/tmp"})
43- require.ErrorIs(t, err, errEmptyName)
44-
45- badValues := [][]byte{
46- nil,
47- {0, 0, 0},
48- {0, 0, 0, 5, 'a'},
49- {0, 0, 0, 1, 'a', 0},
50- {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
51- }
52- for _, value := range badValues {
53- _, err := decodeRecordValue("name", value)
54- require.ErrorIs(t, err, errMalformedRecord)
55- }
56-
57- _, err = decodeRecordValue("", make([]byte, 20))
58- require.ErrorIs(t, err, errEmptyName)
59-}
60-
61-func TestNilPersisterNoOps(t *testing.T) {
62- var p *Persister
63- require.NoError(t, p.Save(Record{Name: "a", Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1}))
64- require.NoError(t, p.Touch("a", "/tmp/next", 2))
65- require.NoError(t, p.Delete("a"))
66- records, err := p.LoadAll()
67- require.NoError(t, err)
68- require.Empty(t, records)
69- require.NoError(t, p.Close())
70-
71- p = New(nil)
72- require.NoError(t, p.Save(Record{Name: "a", Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1}))
73- require.NoError(t, p.Touch("a", "/tmp/next", 2))
74- require.NoError(t, p.Delete("a"))
75- records, err = p.LoadAll()
76- require.NoError(t, err)
77- require.Empty(t, records)
78- require.NoError(t, p.Close())
79-}
80-
81-func TestPersisterSaveTouchDeleteLoadAll(t *testing.T) {
82- dir := privateDir(t)
83- p, err := Open(dir)
84- require.NoError(t, err)
85-
86- require.NoError(t, p.Save(Record{Name: "b", Cwd: "/b", CreatedAt: 10, UpdatedAt: 10, LastUsedSeq: 40, TabNames: []string{"shell"}}))
87- require.NoError(t, p.Save(Record{Name: "a", Cwd: "/a", CreatedAt: 20, UpdatedAt: 20, LastUsedSeq: 50}))
88- require.NoError(t, p.Touch("b", "/b/next", 30))
89- require.NoError(t, p.TouchMRU("b", 60))
90-
91- records, err := p.LoadAll()
92- require.NoError(t, err)
93- require.Equal(t, []Record{
94- {Name: "a", Cwd: "/a", CreatedAt: 20, UpdatedAt: 20, LastUsedSeq: 50},
95- {Name: "b", Cwd: "/b/next", CreatedAt: 10, UpdatedAt: 30, LastUsedSeq: 60, TabNames: []string{"shell"}},
96- }, records)
97-
98- require.NoError(t, p.Delete("a"))
99- records, err = p.LoadAll()
100- require.NoError(t, err)
101- require.Equal(t, []Record{{Name: "b", Cwd: "/b/next", CreatedAt: 10, UpdatedAt: 30, LastUsedSeq: 60, TabNames: []string{"shell"}}}, records)
102- require.NoError(t, p.Close())
103-}
104-
105-func TestDecodeAllSkipsMalformedRecords(t *testing.T) {
106- good, err := encodeRecordValue(Record{Name: "good", Cwd: "/ok", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3})
107- require.NoError(t, err)
108-
109- records, err := decodeAll(map[string][]byte{
110- "bad": []byte{0, 0, 0, 5, 'x'},
111- "good": good,
112- })
113- require.NoError(t, err)
114- require.Equal(t, []Record{{Name: "good", Cwd: "/ok", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3}}, records)
115-}
116-
117-func TestLoadReadOnlyUsesReplay(t *testing.T) {
118- dir := privateDir(t)
119- p, err := Open(dir)
120- require.NoError(t, err)
121- require.NoError(t, p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3}))
122- require.NoError(t, p.Close())
123-
124- records, err := LoadReadOnly(dir)
125- require.NoError(t, err)
126- require.Equal(t, []Record{{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3}}, records)
127-
128- // Missing stores replay as empty rather than creating the WAL.
129- missing := filepath.Join(t.TempDir(), "missing")
130- records, err = LoadReadOnly(missing)
131- require.NoError(t, err)
132- require.Empty(t, records)
133-}
134-
135-func TestSyncBehavior(t *testing.T) {
136- store, state := newMockStore(t)
137- p := New(store)
138-
139- require.NoError(t, p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 1}))
140- require.Equal(t, 1, state.sets)
141- require.Equal(t, 1, state.syncs)
142-
143- require.NoError(t, p.Touch("work", "/work/next", 2))
144- require.Equal(t, 2, state.sets)
145- require.Equal(t, 1, state.syncs)
146-
147- require.NoError(t, p.TouchMRU("work", 3))
148- require.Equal(t, 3, state.sets)
149- require.Equal(t, 1, state.syncs)
150-
151- require.NoError(t, p.Delete("work"))
152- require.Equal(t, 1, state.deletes)
153- require.Equal(t, 2, state.syncs)
154 }26 }
155 27
156-func TestPersisterSerializesReadModifyWriteWithStructuralSave(t *testing.T) {28+func TestCatalogueBatchSyncBehavior(t *testing.T) {
157- store := newBlockingGetStore()29+ store := portsmocks.NewMockStore(t)
30+ state := map[string][]byte{}
31+ store.EXPECT().Range(mock.Anything).Run(func(fn func([]byte, []byte) bool) {
32+ for key, value := range state {
33+ if !fn([]byte(key), value) {
34+ return
35+ }
36+ }
37+ }).Twice()
38+ store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error {
39+ for _, change := range changes {
40+ if change.Delete {
41+ delete(state, string(change.Key))
42+ } else {
43+ state[string(change.Key)] = append([]byte(nil), change.Value...)
44+ }
45+ }
46+ return nil
47+ }).Twice()
48+ store.EXPECT().Sync().Return(nil).Twice()
158 p := New(store)49 p := New(store)
159- require.NoError(t, p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, TabNames: []string{"old"}}))50+ one := validRecord("one", 1)
160- store.blockNextGet()51+ require.NoError(t, p.Replace("one", one))
161-52+ renamed := one
162- touchDone := make(chan error, 1)53+ renamed.Name = "renamed"
163- go func() { touchDone <- p.TouchMRU("work", 2) }()54+ require.NoError(t, p.Rename("one", renamed))
164- <-store.getStarted
165-
166- saveDone := make(chan error, 1)
167- go func() {
168- saveDone <- p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 3, LastUsedSeq: 2, TabNames: []string{"new"}})
169- }()
170- select {
171- case err := <-saveDone:
172- t.Fatalf("structural save interleaved with TouchMRU: %v", err)
173- case <-time.After(20 * time.Millisecond):
174- }
175- close(store.releaseGet)
176- require.NoError(t, <-touchDone)
177- require.NoError(t, <-saveDone)
178-
179- records, err := p.LoadAll()
180- require.NoError(t, err)
181- require.Equal(t, []string{"new"}, records[0].TabNames)
182 }55 }
183 56
184-func TestSavePropagatesSetErrorWithoutSync(t *testing.T) {57+func TestCatalogueBatchFailureDoesNotSync(t *testing.T) {
185- store, state := newMockStore(t)58+ store := portsmocks.NewMockStore(t)
186- state.errSet = errors.New("set failed")59+ batchErr := errors.New("batch failed")
60+ store.EXPECT().Range(mock.Anything).Run(func(func([]byte, []byte) bool) {})
61+ store.EXPECT().Batch(mock.Anything).Return(batchErr)
187 p := New(store)62 p := New(store)
188-63+ err := p.Replace("one", validRecord("one", 1))
189- err := p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 1})64+ require.ErrorIs(t, err, batchErr)
190- require.ErrorIs(t, err, state.errSet)
191- require.Zero(t, state.syncs)
192-}
193-
194-type mockStoreState struct {
195- data map[string][]byte
196- sets int
197- deletes int
198- syncs int
199- closed bool
200- errSet error
201-}
202-
203-type blockingGetStore struct {
204- mu sync.Mutex
205- data map[string][]byte
206- block bool
207- getStarted chan struct{}
208- releaseGet chan struct{}
209- getStartOnce sync.Once
210-}
211-
212-func newBlockingGetStore() *blockingGetStore {
213- return &blockingGetStore{data: make(map[string][]byte), getStarted: make(chan struct{}), releaseGet: make(chan struct{})}
214-}
215-
216-func (s *blockingGetStore) blockNextGet() {
217- s.mu.Lock()
218- s.block = true
219- s.mu.Unlock()
220-}
221-
222-func (s *blockingGetStore) Get(key []byte) ([]byte, bool) {
223- s.mu.Lock()
224- v, ok := s.data[string(key)]
225- v = append([]byte(nil), v...)
226- block := s.block
227- if block {
228- s.block = false
229- }
230- s.mu.Unlock()
231- if block {
232- s.getStartOnce.Do(func() { close(s.getStarted) })
233- <-s.releaseGet
234- }
235- return v, ok
236 }65 }
237 66
238-func (s *blockingGetStore) Set(key, value []byte) error {67+func TestCatalogueNilPersisterFailsClosed(t *testing.T) {
239- s.mu.Lock()68+ var p *Persister
240- s.data[string(key)] = append([]byte(nil), value...)69+ require.ErrorIs(t, p.Save(validRecord("one", 1)), errPersistenceUnavailable)
241- s.mu.Unlock()70+ _, err := p.LoadAll()
242- return nil71+ require.ErrorIs(t, err, errPersistenceUnavailable)
243-}72+ require.ErrorIs(t, p.Close(), errPersistenceUnavailable)
244-
245-func (s *blockingGetStore) Delete(key []byte) error {
246- s.mu.Lock()
247- delete(s.data, string(key))
248- s.mu.Unlock()
249- return nil
250-}
251-
252-func (s *blockingGetStore) Range(fn func(k, v []byte) bool) {
253- s.mu.Lock()
254- defer s.mu.Unlock()
255- for key, value := range s.data {
256- if !fn([]byte(key), append([]byte(nil), value...)) {
257- return
258- }
259- }
260-}
261-
262-func (*blockingGetStore) Sync() error { return nil }
263-func (*blockingGetStore) Close() error { return nil }
264-
265-func newMockStore(t *testing.T) (*portsmocks.MockStore, *mockStoreState) {
266- t.Helper()
267- state := &mockStoreState{data: make(map[string][]byte)}
268- store := portsmocks.NewMockStore(t)
269- store.EXPECT().Get(mock.Anything).RunAndReturn(func(key []byte) ([]byte, bool) {
270- v, ok := state.data[string(key)]
271- return append([]byte(nil), v...), ok
272- }).Maybe()
273- store.EXPECT().Set(mock.Anything, mock.Anything).RunAndReturn(func(key, val []byte) error {
274- state.sets++
275- if state.errSet != nil {
276- return state.errSet
277- }
278- state.data[string(key)] = append([]byte(nil), val...)
279- return nil
280- }).Maybe()
281- store.EXPECT().Delete(mock.Anything).RunAndReturn(func(key []byte) error {
282- state.deletes++
283- delete(state.data, string(key))
284- return nil
285- }).Maybe()
286- store.EXPECT().Range(mock.Anything).Run(func(fn func(k, v []byte) bool) {
287- for k, v := range state.data {
288- if !fn([]byte(k), append([]byte(nil), v...)) {
289- return
290- }
291- }
292- }).Maybe()
293- store.EXPECT().Sync().RunAndReturn(func() error {
294- state.syncs++
295- return nil
296- }).Maybe()
297- store.EXPECT().Close().RunAndReturn(func() error {
298- state.closed = true
299- return nil
300- }).Maybe()
301- return store, state
302 }73 }
M internal/persist/record.go +196 -105
@@ -5,122 +5,213 @@ import (
55 "errors"
66 "fmt"
77 "math"
8+
9+ "github.com/bnema/vev/internal/domain"
810 )
911
10-// Record is the persisted metadata for a named session.
11-type Record struct {
12- Name string
13- Cwd string
14- CreatedAt int64
15- UpdatedAt int64
16- LastUsedSeq uint64
17- TabNames []string
18-}
12+const catalogueRecordVersion uint16 = 1
1913
20-var (
21- errEmptyName = errors.New("persist: empty session name")
22- errMalformedRecord = errors.New("persist: malformed record")
23-)
14+var errMalformedRecord = errors.New("persist: malformed catalogue record")
15+
16+var catalogueMagic = [4]byte{'V', 'E', 'V', 'C'}
17+
18+// Record is the durable catalogue record used by daemon code.
19+type Record = domain.CatalogueRecord
2420
25-func encodeRecordValue(r Record) ([]byte, error) {
26- if r.Name == "" {
27- return nil, errEmptyName
21+func encodeRecordValue(record domain.CatalogueRecord) ([]byte, error) {
22+ if err := record.Validate(); err != nil {
23+ return nil, fmt.Errorf("persist: invalid catalogue record: %w", err)
24+ }
25+ buf := make([]byte, 0, 128)
26+ buf = append(buf, catalogueMagic[:]...)
27+ buf = binary.BigEndian.AppendUint16(buf, catalogueRecordVersion)
28+ buf = append(buf, record.IncarnationID[:]...)
29+ buf = appendString(buf, record.Cwd)
30+ buf = binary.BigEndian.AppendUint64(buf, uint64(record.CreatedAt))
31+ buf = binary.BigEndian.AppendUint64(buf, uint64(record.UpdatedAt))
32+ buf = binary.BigEndian.AppendUint64(buf, record.LastUsedSeq)
33+ if uint64(len(record.TabNames)) > math.MaxUint32 {
34+ return nil, errors.New("persist: too many tab names")
35+ }
36+ buf = binary.BigEndian.AppendUint32(buf, uint32(len(record.TabNames)))
37+ for _, name := range record.TabNames {
38+ var err error
39+ buf, err = appendCheckedString(buf, name)
40+ if err != nil {
41+ return nil, err
42+ }
2843 }
29- if uint64(len(r.Cwd)) > math.MaxUint32 {
30- return nil, fmt.Errorf("persist: cwd too large")
44+ buf = append(buf, byte(record.RecoveryState))
45+ buf = appendRef(buf, record.Committed)
46+ buf = appendRef(buf, record.Fallbacks[0])
47+ buf = appendRef(buf, record.Fallbacks[1])
48+ var err error
49+ buf, err = appendCheckedString(buf, record.DegradedReason)
50+ return buf, err
51+}
52+
53+func decodeRecordValue(name string, value []byte) (domain.CatalogueRecord, error) {
54+ if err := domain.ValidateSessionName(name); err != nil {
55+ return domain.CatalogueRecord{}, err
56+ }
57+ reader := valueReader{data: value}
58+ magic, ok := reader.take(4)
59+ if !ok || string(magic) != string(catalogueMagic[:]) {
60+ return domain.CatalogueRecord{}, errMalformedRecord
61+ }
62+ version, ok := reader.u16()
63+ if !ok || version != catalogueRecordVersion {
64+ return domain.CatalogueRecord{}, errMalformedRecord
65+ }
66+ id, ok := reader.take(16)
67+ if !ok {
68+ return domain.CatalogueRecord{}, errMalformedRecord
69+ }
70+ record := domain.CatalogueRecord{Name: name}
71+ copy(record.IncarnationID[:], id)
72+ var valid bool
73+ if record.Cwd, valid = reader.str(); !valid {
74+ return domain.CatalogueRecord{}, errMalformedRecord
75+ }
76+ created, ok := reader.u64()
77+ if !ok {
78+ return domain.CatalogueRecord{}, errMalformedRecord
79+ }
80+ record.CreatedAt = int64(created)
81+ updated, ok := reader.u64()
82+ if !ok {
83+ return domain.CatalogueRecord{}, errMalformedRecord
84+ }
85+ record.UpdatedAt = int64(updated)
86+ if record.LastUsedSeq, ok = reader.u64(); !ok {
87+ return domain.CatalogueRecord{}, errMalformedRecord
88+ }
89+ count, ok := reader.u32()
90+ if !ok || uint64(count) > uint64(reader.remaining())/4 {
91+ return domain.CatalogueRecord{}, errMalformedRecord
92+ }
93+ if count > 0 {
94+ record.TabNames = make([]string, 0, int(count))
95+ }
96+ for range count {
97+ tab, ok := reader.str()
98+ if !ok {
99+ return domain.CatalogueRecord{}, errMalformedRecord
100+ }
101+ record.TabNames = append(record.TabNames, tab)
31102 }
32- if uint64(len(r.TabNames)) > math.MaxUint32 {
33- return nil, fmt.Errorf("persist: too many tab names")
103+ state, ok := reader.byte()
104+ if !ok {
105+ return domain.CatalogueRecord{}, errMalformedRecord
34106 }
35- tabNamesSize := 4
36- for _, name := range r.TabNames {
37- if uint64(len(name)) > math.MaxUint32 {
38- return nil, fmt.Errorf("persist: tab name too large")
39- }
40- tabNamesSize += 4 + len(name)
107+ record.RecoveryState = domain.RecoveryState(state)
108+ if record.Committed, ok = reader.ref(); !ok {
109+ return domain.CatalogueRecord{}, errMalformedRecord
41110 }
111+ if record.Fallbacks[0], ok = reader.ref(); !ok {
112+ return domain.CatalogueRecord{}, errMalformedRecord
113+ }
114+ if record.Fallbacks[1], ok = reader.ref(); !ok {
115+ return domain.CatalogueRecord{}, errMalformedRecord
116+ }
117+ if record.DegradedReason, ok = reader.str(); !ok || reader.remaining() != 0 {
118+ return domain.CatalogueRecord{}, errMalformedRecord
119+ }
120+ if err := record.Validate(); err != nil {
121+ return domain.CatalogueRecord{}, fmt.Errorf("%w: %v", errMalformedRecord, err)
122+ }
123+ return record, nil
124+}
42125
43- buf := make([]byte, 4+len(r.Cwd)+8+8+8+tabNamesSize)
44- binary.BigEndian.PutUint32(buf[:4], uint32(len(r.Cwd)))
45- copy(buf[4:], r.Cwd)
46- off := 4 + len(r.Cwd)
47- binary.BigEndian.PutUint64(buf[off:off+8], uint64(r.CreatedAt))
48- binary.BigEndian.PutUint64(buf[off+8:off+16], uint64(r.UpdatedAt))
49- binary.BigEndian.PutUint64(buf[off+16:off+24], r.LastUsedSeq)
50- off += 24
51- binary.BigEndian.PutUint32(buf[off:off+4], uint32(len(r.TabNames)))
52- off += 4
53- for _, name := range r.TabNames {
54- binary.BigEndian.PutUint32(buf[off:off+4], uint32(len(name)))
55- off += 4
56- copy(buf[off:], name)
57- off += len(name)
58- }
59- return buf, nil
126+func appendString(buf []byte, value string) []byte {
127+ buf = binary.BigEndian.AppendUint32(buf, uint32(len(value)))
128+ return append(buf, value...)
129+}
130+func appendCheckedString(buf []byte, value string) ([]byte, error) {
131+ if uint64(len(value)) > math.MaxUint32 {
132+ return nil, errors.New("persist: string too large")
133+ }
134+ return appendString(buf, value), nil
135+}
136+func appendRef(buf []byte, ref *domain.CheckpointRef) []byte {
137+ if ref == nil {
138+ return append(buf, 0)
139+ }
140+ buf = append(buf, 1)
141+ buf = binary.BigEndian.AppendUint64(buf, ref.Generation)
142+ return append(buf, ref.ManifestDigest[:]...)
60143 }
61144
62-func decodeRecordValue(name string, value []byte) (Record, error) {
63- if name == "" {
64- return Record{}, errEmptyName
65- }
66- if len(value) < 4 {
67- return Record{}, errMalformedRecord
68- }
69- cwdLen32 := binary.BigEndian.Uint32(value[:4])
70- remaining := len(value) - 4
71- if uint64(cwdLen32) > uint64(remaining) {
72- return Record{}, errMalformedRecord
73- }
74- cwdLen := int(cwdLen32)
75- baseLen := 4 + cwdLen + 8 + 8
76- if len(value) < baseLen {
77- return Record{}, errMalformedRecord
78- }
79- off := 4 + cwdLen
80- r := Record{
81- Name: name,
82- Cwd: string(value[4:off]),
83- CreatedAt: int64(binary.BigEndian.Uint64(value[off : off+8])),
84- UpdatedAt: int64(binary.BigEndian.Uint64(value[off+8 : off+16])),
85- }
86- if len(value) == baseLen {
87- return r, nil
88- }
89- if len(value) < baseLen+8 {
90- return Record{}, errMalformedRecord
91- }
92- if len(value) == baseLen+8 {
93- r.LastUsedSeq = binary.BigEndian.Uint64(value[off+16 : off+24])
94- return r, nil
95- }
96- r.LastUsedSeq = binary.BigEndian.Uint64(value[off+16 : off+24])
97- off += 24
98- if len(value)-off < 4 {
99- return Record{}, errMalformedRecord
100- }
101- tabCount32 := binary.BigEndian.Uint32(value[off : off+4])
102- off += 4
103- if uint64(tabCount32) > uint64(len(value)-off)/4 {
104- return Record{}, errMalformedRecord
105- }
106- if tabCount32 > 0 {
107- r.TabNames = make([]string, 0, int(tabCount32))
108- }
109- for range tabCount32 {
110- if len(value)-off < 4 {
111- return Record{}, errMalformedRecord
112- }
113- nameLen32 := binary.BigEndian.Uint32(value[off : off+4])
114- off += 4
115- if uint64(nameLen32) > uint64(len(value)-off) {
116- return Record{}, errMalformedRecord
117- }
118- nameLen := int(nameLen32)
119- r.TabNames = append(r.TabNames, string(value[off:off+nameLen]))
120- off += nameLen
145+type valueReader struct {
146+ data []byte
147+ off int
148+}
149+
150+func (r *valueReader) remaining() int { return len(r.data) - r.off }
151+func (r *valueReader) take(n int) ([]byte, bool) {
152+ if n < 0 || n > r.remaining() {
153+ return nil, false
154+ }
155+ value := r.data[r.off : r.off+n]
156+ r.off += n
157+ return value, true
158+}
159+func (r *valueReader) byte() (byte, bool) {
160+ value, ok := r.take(1)
161+ if !ok {
162+ return 0, false
163+ }
164+ return value[0], true
165+}
166+func (r *valueReader) u16() (uint16, bool) {
167+ value, ok := r.take(2)
168+ if !ok {
169+ return 0, false
170+ }
171+ return binary.BigEndian.Uint16(value), true
172+}
173+func (r *valueReader) u32() (uint32, bool) {
174+ value, ok := r.take(4)
175+ if !ok {
176+ return 0, false
177+ }
178+ return binary.BigEndian.Uint32(value), true
179+}
180+func (r *valueReader) u64() (uint64, bool) {
181+ value, ok := r.take(8)
182+ if !ok {
183+ return 0, false
184+ }
185+ return binary.BigEndian.Uint64(value), true
186+}
187+func (r *valueReader) str() (string, bool) {
188+ size, ok := r.u32()
189+ if !ok || uint64(size) > uint64(r.remaining()) {
190+ return "", false
191+ }
192+ value, _ := r.take(int(size))
193+ return string(value), true
194+}
195+func (r *valueReader) ref() (*domain.CheckpointRef, bool) {
196+ present, ok := r.byte()
197+ if !ok {
198+ return nil, false
199+ }
200+ if present == 0 {
201+ return nil, true
202+ }
203+ if present != 1 {
204+ return nil, false
205+ }
206+ generation, ok := r.u64()
207+ if !ok {
208+ return nil, false
121209 }
122- if off != len(value) {
123- return Record{}, errMalformedRecord
210+ digest, ok := r.take(32)
211+ if !ok {
212+ return nil, false
124213 }
125- return r, nil
214+ ref := &domain.CheckpointRef{Generation: generation}
215+ copy(ref.ManifestDigest[:], digest)
216+ return ref, true
126217 }
@@ -5,122 +5,213 @@ import (
5 "errors"5 "errors"
6 "fmt"6 "fmt"
7 "math"7 "math"
8+
9+ "github.com/bnema/vev/internal/domain"
8 )10 )
9 11
10-// Record is the persisted metadata for a named session.12+const catalogueRecordVersion uint16 = 1
11-type Record struct {
12- Name string
13- Cwd string
14- CreatedAt int64
15- UpdatedAt int64
16- LastUsedSeq uint64
17- TabNames []string
18-}
19 13
20-var (14+var errMalformedRecord = errors.New("persist: malformed catalogue record")
21- errEmptyName = errors.New("persist: empty session name")15+
22- errMalformedRecord = errors.New("persist: malformed record")16+var catalogueMagic = [4]byte{'V', 'E', 'V', 'C'}
23-)17+
18+// Record is the durable catalogue record used by daemon code.
19+type Record = domain.CatalogueRecord
24 20
25-func encodeRecordValue(r Record) ([]byte, error) {21+func encodeRecordValue(record domain.CatalogueRecord) ([]byte, error) {
26- if r.Name == "" {22+ if err := record.Validate(); err != nil {
27- return nil, errEmptyName23+ return nil, fmt.Errorf("persist: invalid catalogue record: %w", err)
24+ }
25+ buf := make([]byte, 0, 128)
26+ buf = append(buf, catalogueMagic[:]...)
27+ buf = binary.BigEndian.AppendUint16(buf, catalogueRecordVersion)
28+ buf = append(buf, record.IncarnationID[:]...)
29+ buf = appendString(buf, record.Cwd)
30+ buf = binary.BigEndian.AppendUint64(buf, uint64(record.CreatedAt))
31+ buf = binary.BigEndian.AppendUint64(buf, uint64(record.UpdatedAt))
32+ buf = binary.BigEndian.AppendUint64(buf, record.LastUsedSeq)
33+ if uint64(len(record.TabNames)) > math.MaxUint32 {
34+ return nil, errors.New("persist: too many tab names")
35+ }
36+ buf = binary.BigEndian.AppendUint32(buf, uint32(len(record.TabNames)))
37+ for _, name := range record.TabNames {
38+ var err error
39+ buf, err = appendCheckedString(buf, name)
40+ if err != nil {
41+ return nil, err
42+ }
28 }43 }
29- if uint64(len(r.Cwd)) > math.MaxUint32 {44+ buf = append(buf, byte(record.RecoveryState))
30- return nil, fmt.Errorf("persist: cwd too large")45+ buf = appendRef(buf, record.Committed)
46+ buf = appendRef(buf, record.Fallbacks[0])
47+ buf = appendRef(buf, record.Fallbacks[1])
48+ var err error
49+ buf, err = appendCheckedString(buf, record.DegradedReason)
50+ return buf, err
51+}
52+
53+func decodeRecordValue(name string, value []byte) (domain.CatalogueRecord, error) {
54+ if err := domain.ValidateSessionName(name); err != nil {
55+ return domain.CatalogueRecord{}, err
56+ }
57+ reader := valueReader{data: value}
58+ magic, ok := reader.take(4)
59+ if !ok || string(magic) != string(catalogueMagic[:]) {
60+ return domain.CatalogueRecord{}, errMalformedRecord
61+ }
62+ version, ok := reader.u16()
63+ if !ok || version != catalogueRecordVersion {
64+ return domain.CatalogueRecord{}, errMalformedRecord
65+ }
66+ id, ok := reader.take(16)
67+ if !ok {
68+ return domain.CatalogueRecord{}, errMalformedRecord
69+ }
70+ record := domain.CatalogueRecord{Name: name}
71+ copy(record.IncarnationID[:], id)
72+ var valid bool
73+ if record.Cwd, valid = reader.str(); !valid {
74+ return domain.CatalogueRecord{}, errMalformedRecord
75+ }
76+ created, ok := reader.u64()
77+ if !ok {
78+ return domain.CatalogueRecord{}, errMalformedRecord
79+ }
80+ record.CreatedAt = int64(created)
81+ updated, ok := reader.u64()
82+ if !ok {
83+ return domain.CatalogueRecord{}, errMalformedRecord
84+ }
85+ record.UpdatedAt = int64(updated)
86+ if record.LastUsedSeq, ok = reader.u64(); !ok {
87+ return domain.CatalogueRecord{}, errMalformedRecord
88+ }
89+ count, ok := reader.u32()
90+ if !ok || uint64(count) > uint64(reader.remaining())/4 {
91+ return domain.CatalogueRecord{}, errMalformedRecord
92+ }
93+ if count > 0 {
94+ record.TabNames = make([]string, 0, int(count))
95+ }
96+ for range count {
97+ tab, ok := reader.str()
98+ if !ok {
99+ return domain.CatalogueRecord{}, errMalformedRecord
100+ }
101+ record.TabNames = append(record.TabNames, tab)
31 }102 }
32- if uint64(len(r.TabNames)) > math.MaxUint32 {103+ state, ok := reader.byte()
33- return nil, fmt.Errorf("persist: too many tab names")104+ if !ok {
105+ return domain.CatalogueRecord{}, errMalformedRecord
34 }106 }
35- tabNamesSize := 4107+ record.RecoveryState = domain.RecoveryState(state)
36- for _, name := range r.TabNames {108+ if record.Committed, ok = reader.ref(); !ok {
37- if uint64(len(name)) > math.MaxUint32 {109+ return domain.CatalogueRecord{}, errMalformedRecord
38- return nil, fmt.Errorf("persist: tab name too large")
39- }
40- tabNamesSize += 4 + len(name)
41 }110 }
111+ if record.Fallbacks[0], ok = reader.ref(); !ok {
112+ return domain.CatalogueRecord{}, errMalformedRecord
113+ }
114+ if record.Fallbacks[1], ok = reader.ref(); !ok {
115+ return domain.CatalogueRecord{}, errMalformedRecord
116+ }
117+ if record.DegradedReason, ok = reader.str(); !ok || reader.remaining() != 0 {
118+ return domain.CatalogueRecord{}, errMalformedRecord
119+ }
120+ if err := record.Validate(); err != nil {
121+ return domain.CatalogueRecord{}, fmt.Errorf("%w: %v", errMalformedRecord, err)
122+ }
123+ return record, nil
124+}
42 125
43- buf := make([]byte, 4+len(r.Cwd)+8+8+8+tabNamesSize)126+func appendString(buf []byte, value string) []byte {
44- binary.BigEndian.PutUint32(buf[:4], uint32(len(r.Cwd)))127+ buf = binary.BigEndian.AppendUint32(buf, uint32(len(value)))
45- copy(buf[4:], r.Cwd)128+ return append(buf, value...)
46- off := 4 + len(r.Cwd)129+}
47- binary.BigEndian.PutUint64(buf[off:off+8], uint64(r.CreatedAt))130+func appendCheckedString(buf []byte, value string) ([]byte, error) {
48- binary.BigEndian.PutUint64(buf[off+8:off+16], uint64(r.UpdatedAt))131+ if uint64(len(value)) > math.MaxUint32 {
49- binary.BigEndian.PutUint64(buf[off+16:off+24], r.LastUsedSeq)132+ return nil, errors.New("persist: string too large")
50- off += 24133+ }
51- binary.BigEndian.PutUint32(buf[off:off+4], uint32(len(r.TabNames)))134+ return appendString(buf, value), nil
52- off += 4135+}
53- for _, name := range r.TabNames {136+func appendRef(buf []byte, ref *domain.CheckpointRef) []byte {
54- binary.BigEndian.PutUint32(buf[off:off+4], uint32(len(name)))137+ if ref == nil {
55- off += 4138+ return append(buf, 0)
56- copy(buf[off:], name)139+ }
57- off += len(name)140+ buf = append(buf, 1)
58- }141+ buf = binary.BigEndian.AppendUint64(buf, ref.Generation)
59- return buf, nil142+ return append(buf, ref.ManifestDigest[:]...)
60 }143 }
61 144
62-func decodeRecordValue(name string, value []byte) (Record, error) {145+type valueReader struct {
63- if name == "" {146+ data []byte
64- return Record{}, errEmptyName147+ off int
65- }148+}
66- if len(value) < 4 {149+
67- return Record{}, errMalformedRecord150+func (r *valueReader) remaining() int { return len(r.data) - r.off }
68- }151+func (r *valueReader) take(n int) ([]byte, bool) {
69- cwdLen32 := binary.BigEndian.Uint32(value[:4])152+ if n < 0 || n > r.remaining() {
70- remaining := len(value) - 4153+ return nil, false
71- if uint64(cwdLen32) > uint64(remaining) {154+ }
72- return Record{}, errMalformedRecord155+ value := r.data[r.off : r.off+n]
73- }156+ r.off += n
74- cwdLen := int(cwdLen32)157+ return value, true
75- baseLen := 4 + cwdLen + 8 + 8158+}
76- if len(value) < baseLen {159+func (r *valueReader) byte() (byte, bool) {
77- return Record{}, errMalformedRecord160+ value, ok := r.take(1)
78- }161+ if !ok {
79- off := 4 + cwdLen162+ return 0, false
80- r := Record{163+ }
81- Name: name,164+ return value[0], true
82- Cwd: string(value[4:off]),165+}
83- CreatedAt: int64(binary.BigEndian.Uint64(value[off : off+8])),166+func (r *valueReader) u16() (uint16, bool) {
84- UpdatedAt: int64(binary.BigEndian.Uint64(value[off+8 : off+16])),167+ value, ok := r.take(2)
85- }168+ if !ok {
86- if len(value) == baseLen {169+ return 0, false
87- return r, nil170+ }
88- }171+ return binary.BigEndian.Uint16(value), true
89- if len(value) < baseLen+8 {172+}
90- return Record{}, errMalformedRecord173+func (r *valueReader) u32() (uint32, bool) {
91- }174+ value, ok := r.take(4)
92- if len(value) == baseLen+8 {175+ if !ok {
93- r.LastUsedSeq = binary.BigEndian.Uint64(value[off+16 : off+24])176+ return 0, false
94- return r, nil177+ }
95- }178+ return binary.BigEndian.Uint32(value), true
96- r.LastUsedSeq = binary.BigEndian.Uint64(value[off+16 : off+24])179+}
97- off += 24180+func (r *valueReader) u64() (uint64, bool) {
98- if len(value)-off < 4 {181+ value, ok := r.take(8)
99- return Record{}, errMalformedRecord182+ if !ok {
100- }183+ return 0, false
101- tabCount32 := binary.BigEndian.Uint32(value[off : off+4])184+ }
102- off += 4185+ return binary.BigEndian.Uint64(value), true
103- if uint64(tabCount32) > uint64(len(value)-off)/4 {186+}
104- return Record{}, errMalformedRecord187+func (r *valueReader) str() (string, bool) {
105- }188+ size, ok := r.u32()
106- if tabCount32 > 0 {189+ if !ok || uint64(size) > uint64(r.remaining()) {
107- r.TabNames = make([]string, 0, int(tabCount32))190+ return "", false
108- }191+ }
109- for range tabCount32 {192+ value, _ := r.take(int(size))
110- if len(value)-off < 4 {193+ return string(value), true
111- return Record{}, errMalformedRecord194+}
112- }195+func (r *valueReader) ref() (*domain.CheckpointRef, bool) {
113- nameLen32 := binary.BigEndian.Uint32(value[off : off+4])196+ present, ok := r.byte()
114- off += 4197+ if !ok {
115- if uint64(nameLen32) > uint64(len(value)-off) {198+ return nil, false
116- return Record{}, errMalformedRecord199+ }
117- }200+ if present == 0 {
118- nameLen := int(nameLen32)201+ return nil, true
119- r.TabNames = append(r.TabNames, string(value[off:off+nameLen]))202+ }
120- off += nameLen203+ if present != 1 {
204+ return nil, false
205+ }
206+ generation, ok := r.u64()
207+ if !ok {
208+ return nil, false
121 }209 }
122- if off != len(value) {210+ digest, ok := r.take(32)
123- return Record{}, errMalformedRecord211+ if !ok {
212+ return nil, false
124 }213 }
125- return r, nil214+ ref := &domain.CheckpointRef{Generation: generation}
215+ copy(ref.ManifestDigest[:], digest)
216+ return ref, true
126 }217 }
M internal/ports/mocks/store_mock.go +51 -0
@@ -38,6 +38,57 @@ func (_m *MockStore) EXPECT() *MockStore_Expecter {
3838 return &MockStore_Expecter{mock: &_m.Mock}
3939 }
4040
41+// Batch provides a mock function for the type MockStore
42+func (_mock *MockStore) Batch(vs []ports.StoreChange) error {
43+ ret := _mock.Called(vs)
44+
45+ if len(ret) == 0 {
46+ panic("no return value specified for Batch")
47+ }
48+
49+ var r0 error
50+ if returnFunc, ok := ret.Get(0).(func([]ports.StoreChange) error); ok {
51+ r0 = returnFunc(vs)
52+ } else {
53+ r0 = ret.Error(0)
54+ }
55+ return r0
56+}
57+
58+// MockStore_Batch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Batch'
59+type MockStore_Batch_Call struct {
60+ *mock.Call
61+}
62+
63+// Batch is a helper method to define mock.On call
64+// - vs []ports.StoreChange
65+func (_e *MockStore_Expecter) Batch(vs any) *MockStore_Batch_Call {
66+ return &MockStore_Batch_Call{Call: _e.mock.On("Batch", vs)}
67+}
68+
69+func (_c *MockStore_Batch_Call) Run(run func(vs []ports.StoreChange)) *MockStore_Batch_Call {
70+ _c.Call.Run(func(args mock.Arguments) {
71+ var arg0 []ports.StoreChange
72+ if args[0] != nil {
73+ arg0 = args[0].([]ports.StoreChange)
74+ }
75+ run(
76+ arg0,
77+ )
78+ })
79+ return _c
80+}
81+
82+func (_c *MockStore_Batch_Call) Return(err error) *MockStore_Batch_Call {
83+ _c.Call.Return(err)
84+ return _c
85+}
86+
87+func (_c *MockStore_Batch_Call) RunAndReturn(run func(vs []ports.StoreChange) error) *MockStore_Batch_Call {
88+ _c.Call.Return(run)
89+ return _c
90+}
91+
4192 // Close provides a mock function for the type MockStore
4293 func (_mock *MockStore) Close() error {
4394 ret := _mock.Called()
@@ -38,6 +38,57 @@ func (_m *MockStore) EXPECT() *MockStore_Expecter {
38 return &MockStore_Expecter{mock: &_m.Mock}38 return &MockStore_Expecter{mock: &_m.Mock}
39 }39 }
40 40
41+// Batch provides a mock function for the type MockStore
42+func (_mock *MockStore) Batch(vs []ports.StoreChange) error {
43+ ret := _mock.Called(vs)
44+
45+ if len(ret) == 0 {
46+ panic("no return value specified for Batch")
47+ }
48+
49+ var r0 error
50+ if returnFunc, ok := ret.Get(0).(func([]ports.StoreChange) error); ok {
51+ r0 = returnFunc(vs)
52+ } else {
53+ r0 = ret.Error(0)
54+ }
55+ return r0
56+}
57+
58+// MockStore_Batch_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Batch'
59+type MockStore_Batch_Call struct {
60+ *mock.Call
61+}
62+
63+// Batch is a helper method to define mock.On call
64+// - vs []ports.StoreChange
65+func (_e *MockStore_Expecter) Batch(vs any) *MockStore_Batch_Call {
66+ return &MockStore_Batch_Call{Call: _e.mock.On("Batch", vs)}
67+}
68+
69+func (_c *MockStore_Batch_Call) Run(run func(vs []ports.StoreChange)) *MockStore_Batch_Call {
70+ _c.Call.Run(func(args mock.Arguments) {
71+ var arg0 []ports.StoreChange
72+ if args[0] != nil {
73+ arg0 = args[0].([]ports.StoreChange)
74+ }
75+ run(
76+ arg0,
77+ )
78+ })
79+ return _c
80+}
81+
82+func (_c *MockStore_Batch_Call) Return(err error) *MockStore_Batch_Call {
83+ _c.Call.Return(err)
84+ return _c
85+}
86+
87+func (_c *MockStore_Batch_Call) RunAndReturn(run func(vs []ports.StoreChange) error) *MockStore_Batch_Call {
88+ _c.Call.Return(run)
89+ return _c
90+}
91+
41 // Close provides a mock function for the type MockStore92 // Close provides a mock function for the type MockStore
42 func (_mock *MockStore) Close() error {93 func (_mock *MockStore) Close() error {
43 ret := _mock.Called()94 ret := _mock.Called()
M internal/ports/store.go +6 -0
@@ -3,8 +3,13 @@ package ports
33 import (
44 "context"
55 "crypto/sha256"
6+
7+ "github.com/bnema/vev/pkg/kv"
68 )
79
10+// StoreChange is one key mutation in an atomic store batch.
11+type StoreChange = kv.BatchChange
12+
813 // Store is a small byte-key/value persistence port.
914 //
1015 // Implementations may buffer writes; Sync is the durability barrier.
@@ -14,6 +19,7 @@ type Store interface {
1419 Get(key []byte) ([]byte, bool)
1520 Set(key, val []byte) error
1621 Delete(key []byte) error
22+ Batch([]StoreChange) error
1723 // Range iterates key/value pairs; fn returning false stops iteration early.
1824 // Each key and value is a stable copy owned by the caller, which may retain
1925 // or mutate either slice after Range returns.
@@ -3,8 +3,13 @@ package ports
3 import (3 import (
4 "context"4 "context"
5 "crypto/sha256"5 "crypto/sha256"
6+
7+ "github.com/bnema/vev/pkg/kv"
6 )8 )
7 9
10+// StoreChange is one key mutation in an atomic store batch.
11+type StoreChange = kv.BatchChange
12+
8 // Store is a small byte-key/value persistence port.13 // Store is a small byte-key/value persistence port.
9 //14 //
10 // Implementations may buffer writes; Sync is the durability barrier.15 // Implementations may buffer writes; Sync is the durability barrier.
@@ -14,6 +19,7 @@ type Store interface {
14 Get(key []byte) ([]byte, bool)19 Get(key []byte) ([]byte, bool)
15 Set(key, val []byte) error20 Set(key, val []byte) error
16 Delete(key []byte) error21 Delete(key []byte) error
22+ Batch([]StoreChange) error
17 // Range iterates key/value pairs; fn returning false stops iteration early.23 // Range iterates key/value pairs; fn returning false stops iteration early.
18 // Each key and value is a stable copy owned by the caller, which may retain24 // Each key and value is a stable copy owned by the caller, which may retain
19 // or mutate either slice after Range returns.25 // or mutate either slice after Range returns.
M pkg/kv/codec.go +124 -13
@@ -8,8 +8,9 @@ import (
88 )
99
1010 const (
11- opSet byte = 0
12- opDel byte = 1
11+ opSet byte = iota
12+ opDel
13+ opBatch
1314
1415 headerLen = 8
1516 payloadPrefixLen = 3
@@ -18,7 +19,10 @@ const (
1819 maxKeyLen = 1<<16 - 1
1920 )
2021
21-var errBadRecord = errors.New("bad record")
22+var (
23+ errBadRecord = errors.New("bad record")
24+ ErrCorruptWAL = errors.New("kv: corrupt WAL")
25+)
2226
2327 type record struct {
2428 op byte
@@ -26,6 +30,13 @@ type record struct {
2630 value []byte
2731 }
2832
33+// BatchChange is one key mutation contained in a single atomic WAL record.
34+type BatchChange struct {
35+ Key []byte
36+ Value []byte
37+ Delete bool
38+}
39+
2940 func encodeRecord(op byte, key, value []byte) ([]byte, error) {
3041 if len(key) > maxKeyLen {
3142 return nil, errors.New("key too large")
@@ -33,7 +44,6 @@ func encodeRecord(op byte, key, value []byte) ([]byte, error) {
3344 if op != opSet && op != opDel {
3445 return nil, errBadRecord
3546 }
36-
3747 payloadLen := payloadPrefixLen + len(key)
3848 if op == opSet {
3949 payloadLen += len(value)
@@ -41,19 +51,72 @@ func encodeRecord(op byte, key, value []byte) ([]byte, error) {
4151 if uint64(payloadLen) > math.MaxUint32 {
4252 return nil, errors.New("payload too large")
4353 }
44-
45- buf := make([]byte, headerLen+payloadLen)
46- payload := buf[headerLen:]
54+ payload := make([]byte, payloadLen)
4755 payload[0] = op
4856 binary.BigEndian.PutUint16(payload[1:3], uint16(len(key)))
4957 copy(payload[3:], key)
5058 if op == opSet {
5159 copy(payload[3+len(key):], value)
5260 }
61+ return framePayload(payload), nil
62+}
63+
64+func encodeBatch(changes []BatchChange) ([]byte, error) {
65+ if len(changes) == 0 {
66+ return nil, errors.New("kv: empty batch")
67+ }
68+ if uint64(len(changes)) > math.MaxUint32 {
69+ return nil, errors.New("kv: too many batch changes")
70+ }
71+ seen := make(map[string]struct{}, len(changes))
72+ payloadLen := 1 + 4
73+ for _, change := range changes {
74+ if len(change.Key) > maxKeyLen {
75+ return nil, errors.New("key too large")
76+ }
77+ if _, exists := seen[string(change.Key)]; exists {
78+ return nil, errors.New("kv: duplicate batch key")
79+ }
80+ seen[string(change.Key)] = struct{}{}
81+ valueLen := len(change.Value)
82+ if change.Delete {
83+ if valueLen != 0 {
84+ return nil, errors.New("kv: delete batch change has value")
85+ }
86+ }
87+ payloadLen += 1 + 2 + len(change.Key) + 4 + valueLen
88+ if uint64(payloadLen) > math.MaxUint32 {
89+ return nil, errors.New("payload too large")
90+ }
91+ }
92+ payload := make([]byte, payloadLen)
93+ payload[0] = opBatch
94+ binary.BigEndian.PutUint32(payload[1:5], uint32(len(changes)))
95+ off := 5
96+ for _, change := range changes {
97+ payload[off] = opSet
98+ if change.Delete {
99+ payload[off] = opDel
100+ }
101+ off++
102+ binary.BigEndian.PutUint16(payload[off:off+2], uint16(len(change.Key)))
103+ off += 2
104+ copy(payload[off:], change.Key)
105+ off += len(change.Key)
106+ binary.BigEndian.PutUint32(payload[off:off+4], uint32(len(change.Value)))
107+ off += 4
108+ copy(payload[off:], change.Value)
109+ off += len(change.Value)
110+ }
111+ return framePayload(payload), nil
112+}
53113
54- binary.BigEndian.PutUint32(buf[0:4], uint32(payloadLen))
114+func framePayload(payload []byte) []byte {
115+ buf := make([]byte, headerLen+len(payload))
116+ binary.BigEndian.PutUint32(buf[:4], uint32(len(payload)))
55117 binary.BigEndian.PutUint32(buf[4:8], crc32.ChecksumIEEE(payload))
56- return buf, nil
118+ copy(buf[headerLen:], payload)
119+ return buf
57120 }
58121
59122 func decodePayload(payload []byte) (record, error) {
@@ -71,11 +134,59 @@ func decodePayload(payload []byte) (record, error) {
71134 if op == opDel && len(payload) != payloadPrefixLen+keyLen {
72135 return record{}, errBadRecord
73136 }
74-
75137 key := append([]byte(nil), payload[payloadPrefixLen:payloadPrefixLen+keyLen]...)
76- var val []byte
138+ var value []byte
77139 if op == opSet {
78- val = append([]byte(nil), payload[payloadPrefixLen+keyLen:]...)
140+ value = append([]byte(nil), payload[payloadPrefixLen+keyLen:]...)
141+ }
142+ return record{op: op, key: key, value: value}, nil
143+}
144+
145+func decodeBatch(payload []byte) ([]record, error) {
146+ if len(payload) < 5 || payload[0] != opBatch {
147+ return nil, errBadRecord
148+ }
149+ count := binary.BigEndian.Uint32(payload[1:5])
150+ if count == 0 {
151+ return nil, errBadRecord
152+ }
153+ off := 5
154+ records := make([]record, 0, int(count))
155+ seen := make(map[string]struct{}, int(count))
156+ for range count {
157+ if len(payload)-off < 7 {
158+ return nil, errBadRecord
159+ }
160+ op := payload[off]
161+ off++
162+ if op != opSet && op != opDel {
163+ return nil, errBadRecord
164+ }
165+ keyLen := int(binary.BigEndian.Uint16(payload[off : off+2]))
166+ off += 2
167+ if keyLen > len(payload)-off-4 {
168+ return nil, errBadRecord
169+ }
170+ key := append([]byte(nil), payload[off:off+keyLen]...)
171+ off += keyLen
172+ valueLen := uint64(binary.BigEndian.Uint32(payload[off : off+4]))
173+ off += 4
174+ if valueLen > uint64(len(payload)-off) {
175+ return nil, errBadRecord
176+ }
177+ if op == opDel && valueLen != 0 {
178+ return nil, errBadRecord
179+ }
180+ value := append([]byte(nil), payload[off:off+int(valueLen)]...)
181+ off += int(valueLen)
182+ if _, exists := seen[string(key)]; exists {
183+ return nil, errBadRecord
184+ }
185+ seen[string(key)] = struct{}{}
186+ records = append(records, record{op: op, key: key, value: value})
187+ }
188+ if off != len(payload) {
189+ return nil, errBadRecord
79190 }
80- return record{op: op, key: key, value: val}, nil
191+ return records, nil
81192 }
@@ -8,8 +8,9 @@ import (
8 )8 )
9 9
10 const (10 const (
11- opSet byte = 011+ opSet byte = iota
12- opDel byte = 112+ opDel
13+ opBatch
13 14
14 headerLen = 815 headerLen = 8
15 payloadPrefixLen = 316 payloadPrefixLen = 3
@@ -18,7 +19,10 @@ const (
18 maxKeyLen = 1<<16 - 119 maxKeyLen = 1<<16 - 1
19 )20 )
20 21
21-var errBadRecord = errors.New("bad record")22+var (
23+ errBadRecord = errors.New("bad record")
24+ ErrCorruptWAL = errors.New("kv: corrupt WAL")
25+)
22 26
23 type record struct {27 type record struct {
24 op byte28 op byte
@@ -26,6 +30,13 @@ type record struct {
26 value []byte30 value []byte
27 }31 }
28 32
33+// BatchChange is one key mutation contained in a single atomic WAL record.
34+type BatchChange struct {
35+ Key []byte
36+ Value []byte
37+ Delete bool
38+}
39+
29 func encodeRecord(op byte, key, value []byte) ([]byte, error) {40 func encodeRecord(op byte, key, value []byte) ([]byte, error) {
30 if len(key) > maxKeyLen {41 if len(key) > maxKeyLen {
31 return nil, errors.New("key too large")42 return nil, errors.New("key too large")
@@ -33,7 +44,6 @@ func encodeRecord(op byte, key, value []byte) ([]byte, error) {
33 if op != opSet && op != opDel {44 if op != opSet && op != opDel {
34 return nil, errBadRecord45 return nil, errBadRecord
35 }46 }
36-
37 payloadLen := payloadPrefixLen + len(key)47 payloadLen := payloadPrefixLen + len(key)
38 if op == opSet {48 if op == opSet {
39 payloadLen += len(value)49 payloadLen += len(value)
@@ -41,19 +51,72 @@ func encodeRecord(op byte, key, value []byte) ([]byte, error) {
41 if uint64(payloadLen) > math.MaxUint32 {51 if uint64(payloadLen) > math.MaxUint32 {
42 return nil, errors.New("payload too large")52 return nil, errors.New("payload too large")
43 }53 }
44-54+ payload := make([]byte, payloadLen)
45- buf := make([]byte, headerLen+payloadLen)
46- payload := buf[headerLen:]
47 payload[0] = op55 payload[0] = op
48 binary.BigEndian.PutUint16(payload[1:3], uint16(len(key)))56 binary.BigEndian.PutUint16(payload[1:3], uint16(len(key)))
49 copy(payload[3:], key)57 copy(payload[3:], key)
50 if op == opSet {58 if op == opSet {
51 copy(payload[3+len(key):], value)59 copy(payload[3+len(key):], value)
52 }60 }
61+ return framePayload(payload), nil
62+}
63+
64+func encodeBatch(changes []BatchChange) ([]byte, error) {
65+ if len(changes) == 0 {
66+ return nil, errors.New("kv: empty batch")
67+ }
68+ if uint64(len(changes)) > math.MaxUint32 {
69+ return nil, errors.New("kv: too many batch changes")
70+ }
71+ seen := make(map[string]struct{}, len(changes))
72+ payloadLen := 1 + 4
73+ for _, change := range changes {
74+ if len(change.Key) > maxKeyLen {
75+ return nil, errors.New("key too large")
76+ }
77+ if _, exists := seen[string(change.Key)]; exists {
78+ return nil, errors.New("kv: duplicate batch key")
79+ }
80+ seen[string(change.Key)] = struct{}{}
81+ valueLen := len(change.Value)
82+ if change.Delete {
83+ if valueLen != 0 {
84+ return nil, errors.New("kv: delete batch change has value")
85+ }
86+ }
87+ payloadLen += 1 + 2 + len(change.Key) + 4 + valueLen
88+ if uint64(payloadLen) > math.MaxUint32 {
89+ return nil, errors.New("payload too large")
90+ }
91+ }
92+ payload := make([]byte, payloadLen)
93+ payload[0] = opBatch
94+ binary.BigEndian.PutUint32(payload[1:5], uint32(len(changes)))
95+ off := 5
96+ for _, change := range changes {
97+ payload[off] = opSet
98+ if change.Delete {
99+ payload[off] = opDel
100+ }
101+ off++
102+ binary.BigEndian.PutUint16(payload[off:off+2], uint16(len(change.Key)))
103+ off += 2
104+ copy(payload[off:], change.Key)
105+ off += len(change.Key)
106+ binary.BigEndian.PutUint32(payload[off:off+4], uint32(len(change.Value)))
107+ off += 4
108+ copy(payload[off:], change.Value)
109+ off += len(change.Value)
110+ }
111+ return framePayload(payload), nil
112+}
53 113
54- binary.BigEndian.PutUint32(buf[0:4], uint32(payloadLen))114+func framePayload(payload []byte) []byte {
115+ buf := make([]byte, headerLen+len(payload))
116+ binary.BigEndian.PutUint32(buf[:4], uint32(len(payload)))
55 binary.BigEndian.PutUint32(buf[4:8], crc32.ChecksumIEEE(payload))117 binary.BigEndian.PutUint32(buf[4:8], crc32.ChecksumIEEE(payload))
56- return buf, nil118+ copy(buf[headerLen:], payload)
119+ return buf
57 }120 }
58 121
59 func decodePayload(payload []byte) (record, error) {122 func decodePayload(payload []byte) (record, error) {
@@ -71,11 +134,59 @@ func decodePayload(payload []byte) (record, error) {
71 if op == opDel && len(payload) != payloadPrefixLen+keyLen {134 if op == opDel && len(payload) != payloadPrefixLen+keyLen {
72 return record{}, errBadRecord135 return record{}, errBadRecord
73 }136 }
74-
75 key := append([]byte(nil), payload[payloadPrefixLen:payloadPrefixLen+keyLen]...)137 key := append([]byte(nil), payload[payloadPrefixLen:payloadPrefixLen+keyLen]...)
76- var val []byte138+ var value []byte
77 if op == opSet {139 if op == opSet {
78- val = append([]byte(nil), payload[payloadPrefixLen+keyLen:]...)140+ value = append([]byte(nil), payload[payloadPrefixLen+keyLen:]...)
141+ }
142+ return record{op: op, key: key, value: value}, nil
143+}
144+
145+func decodeBatch(payload []byte) ([]record, error) {
146+ if len(payload) < 5 || payload[0] != opBatch {
147+ return nil, errBadRecord
148+ }
149+ count := binary.BigEndian.Uint32(payload[1:5])
150+ if count == 0 {
151+ return nil, errBadRecord
152+ }
153+ off := 5
154+ records := make([]record, 0, int(count))
155+ seen := make(map[string]struct{}, int(count))
156+ for range count {
157+ if len(payload)-off < 7 {
158+ return nil, errBadRecord
159+ }
160+ op := payload[off]
161+ off++
162+ if op != opSet && op != opDel {
163+ return nil, errBadRecord
164+ }
165+ keyLen := int(binary.BigEndian.Uint16(payload[off : off+2]))
166+ off += 2
167+ if keyLen > len(payload)-off-4 {
168+ return nil, errBadRecord
169+ }
170+ key := append([]byte(nil), payload[off:off+keyLen]...)
171+ off += keyLen
172+ valueLen := uint64(binary.BigEndian.Uint32(payload[off : off+4]))
173+ off += 4
174+ if valueLen > uint64(len(payload)-off) {
175+ return nil, errBadRecord
176+ }
177+ if op == opDel && valueLen != 0 {
178+ return nil, errBadRecord
179+ }
180+ value := append([]byte(nil), payload[off:off+int(valueLen)]...)
181+ off += int(valueLen)
182+ if _, exists := seen[string(key)]; exists {
183+ return nil, errBadRecord
184+ }
185+ seen[string(key)] = struct{}{}
186+ records = append(records, record{op: op, key: key, value: value})
187+ }
188+ if off != len(payload) {
189+ return nil, errBadRecord
79 }190 }
80- return record{op: op, key: key, value: val}, nil191+ return records, nil
81 }192 }
M pkg/kv/kv.go +364 -172
@@ -15,6 +15,28 @@ import (
1515 "github.com/bnema/vev/pkg/safedir"
1616 )
1717
18+// ReplayResult describes a strictly validated WAL prefix.
19+type ReplayResult struct {
20+ Data map[string][]byte
21+ EntrySize map[string]int64
22+ Total int64
23+ Live int64
24+ LastGood int64
25+ TornTail bool
26+}
27+
28+type fsHooks struct {
29+ Open func(string, int, os.FileMode) (*os.File, error)
30+ Remove func(string) error
31+ Rename func(string, string) error
32+ SyncDir func(string) error
33+ Fault func(string) error
34+}
35+
36+func defaultFSHooks() fsHooks {
37+ return fsHooks{Open: os.OpenFile, Remove: os.Remove, Rename: os.Rename, SyncDir: syncDir}
38+}
39+
1840 // Store is an append-only WAL-backed key/value store.
1941 type Store struct {
2042 mu sync.Mutex
@@ -26,55 +48,54 @@ type Store struct {
2648 total int64
2749 live int64
2850 closed bool
51+ hooks fsHooks
2952 }
3053
31-// Open opens or creates a store at path, replays valid records, truncates any
32-// corrupt or torn tail, and compacts if the file contains enough obsolete data.
33-func Open(path string) (*Store, error) {
54+// Open opens or creates a store, strictly recovers compaction, and truncates only a torn tail.
55+func Open(path string) (*Store, error) { return openWithHooks(path, defaultFSHooks()) }
56+
57+func openWithHooks(path string, hooks fsHooks) (*Store, error) {
3458 if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
3559 return nil, err
3660 }
37-
3861 lockFile, err := acquireLock(path)
3962 if err != nil {
4063 return nil, err
4164 }
42-
43- f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600)
65+ cleanupLock := func(err error) (*Store, error) { _ = releaseLock(lockFile); return nil, err }
66+ if err := recoverCompaction(path, hooks); err != nil {
67+ return cleanupLock(err)
68+ }
69+ f, err := hooks.Open(path, os.O_RDWR|os.O_CREATE, 0o600)
4470 if err != nil {
45- _ = releaseLock(lockFile)
46- return nil, err
71+ return cleanupLock(err)
4772 }
48-
49- data, sizes, total, live, lastGood, err := replayFile(f)
73+ result, err := replayFile(f)
5074 if err != nil {
5175 _ = f.Close()
52- _ = releaseLock(lockFile)
53- return nil, err
76+ return cleanupLock(err)
5477 }
55- if err := f.Truncate(lastGood); err != nil {
56- _ = f.Close()
57- _ = releaseLock(lockFile)
58- return nil, err
78+ if result.TornTail {
79+ if err := f.Truncate(result.LastGood); err != nil {
80+ _ = f.Close()
81+ return cleanupLock(err)
82+ }
5983 }
6084 if _, err := f.Seek(0, io.SeekEnd); err != nil {
6185 _ = f.Close()
62- _ = releaseLock(lockFile)
63- return nil, err
86+ return cleanupLock(err)
6487 }
65-
66- s := &Store{path: path, file: f, lockFile: lockFile, data: data, entrySize: sizes, total: total, live: live}
88+ s := &Store{path: path, file: f, lockFile: lockFile, data: result.Data, entrySize: result.EntrySize, total: result.Total, live: result.Live, hooks: hooks}
6789 if s.shouldCompact() {
6890 if err := s.compactLocked(); err != nil {
6991 _ = f.Close()
70- _ = releaseLock(lockFile)
71- return nil, err
92+ return cleanupLock(err)
7293 }
7394 }
7495 return s, nil
7596 }
7697
77-// Replay reads path without mutating it and returns the state from valid records.
98+// Replay reads path without mutating it and returns only strictly valid state.
7899 func Replay(path string) (map[string][]byte, error) {
79100 f, err := os.Open(path)
80101 if errors.Is(err, os.ErrNotExist) {
@@ -84,37 +105,50 @@ func Replay(path string) (map[string][]byte, error) {
84105 return nil, err
85106 }
86107 defer func() { _ = f.Close() }()
87-
88- data, _, _, _, _, err := replayFile(f)
108+ result, err := replayFile(f)
89109 if err != nil {
90110 return nil, err
91111 }
92- return data, nil
112+ return result.Data, nil
93113 }
94114
95-// Get returns a copy of the value for key.
96115 func (s *Store) Get(key []byte) ([]byte, bool) {
97116 s.mu.Lock()
98117 defer s.mu.Unlock()
99-
100118 v, ok := s.data[string(key)]
101- if !ok {
102- return nil, false
103- }
104- return append([]byte(nil), v...), true
119+ return append([]byte(nil), v...), ok
105120 }
106-
107-// Set appends a SET record. It does not fsync.
108121 func (s *Store) Set(key, val []byte) error {
109- return s.appendRecord(opSet, key, val)
122+ return s.appendEncoded(func() ([]byte, []record, error) {
123+ buf, err := encodeRecord(opSet, key, val)
124+ return buf, []record{{op: opSet, key: append([]byte(nil), key...), value: append([]byte(nil), val...)}}, err
125+ })
110126 }
111-
112-// Delete appends a DEL tombstone. It does not fsync.
113127 func (s *Store) Delete(key []byte) error {
114- return s.appendRecord(opDel, key, nil)
128+ return s.appendEncoded(func() ([]byte, []record, error) {
129+ buf, err := encodeRecord(opDel, key, nil)
130+ return buf, []record{{op: opDel, key: append([]byte(nil), key...)}}, err
131+ })
132+}
133+
134+// Batch appends all changes as one WAL record and applies them only after a complete write.
135+func (s *Store) Batch(changes []BatchChange) error {
136+ return s.appendEncoded(func() ([]byte, []record, error) {
137+ buf, err := encodeBatch(changes)
138+ if err != nil {
139+ return nil, nil, err
140+ }
141+ recs := make([]record, len(changes))
142+ for i, change := range changes {
143+ recs[i] = record{op: opSet, key: append([]byte(nil), change.Key...), value: append([]byte(nil), change.Value...)}
144+ if change.Delete {
145+ recs[i].op = opDel
146+ }
147+ }
148+ return buf, recs, nil
149+ })
115150 }
116151
117-// Range calls fn for each key/value pair, using copies. Iteration stops when fn returns false.
118152 func (s *Store) Range(fn func(k, v []byte) bool) {
119153 s.mu.Lock()
120154 keys := make([]string, 0, len(s.data))
@@ -122,20 +156,17 @@ func (s *Store) Range(fn func(k, v []byte) bool) {
122156 keys = append(keys, k)
123157 }
124158 sort.Strings(keys)
125- pairs := make([]struct{ k, v []byte }, 0, len(keys))
159+ pairs := make([]record, 0, len(keys))
126160 for _, k := range keys {
127- pairs = append(pairs, struct{ k, v []byte }{[]byte(k), append([]byte(nil), s.data[k]...)})
161+ pairs = append(pairs, record{key: []byte(k), value: append([]byte(nil), s.data[k]...)})
128162 }
129163 s.mu.Unlock()
130-
131- for _, p := range pairs {
132- if !fn(p.k, p.v) {
164+ for _, pair := range pairs {
165+ if !fn(pair.key, pair.value) {
133166 return
134167 }
135168 }
136169 }
137-
138-// Sync fsyncs the WAL file.
139170 func (s *Store) Sync() error {
140171 s.mu.Lock()
141172 defer s.mu.Unlock()
@@ -144,57 +175,40 @@ func (s *Store) Sync() error {
144175 }
145176 return s.file.Sync()
146177 }
147-
148-// Close fsyncs and closes the store.
149178 func (s *Store) Close() error {
150179 s.mu.Lock()
151180 defer s.mu.Unlock()
152181 if s.closed {
153182 return nil
154183 }
155- errSync := s.file.Sync()
156- errClose := s.file.Close()
157- errUnlock := releaseLock(s.lockFile)
184+ err := errors.Join(s.file.Sync(), s.file.Close(), releaseLock(s.lockFile))
158185 s.closed = true
159- if errSync != nil {
160- return errSync
161- }
162- if errClose != nil {
163- return errClose
164- }
165- return errUnlock
186+ return err
166187 }
167188
168189 func acquireLock(path string) (*os.File, error) {
169- lf, err := os.OpenFile(path+".lock", os.O_RDWR|os.O_CREATE, 0o600)
190+ f, err := os.OpenFile(path+".lock", os.O_RDWR|os.O_CREATE, 0o600)
170191 if err != nil {
171192 return nil, err
172193 }
173- if err := syscall.Flock(int(lf.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
174- _ = lf.Close()
194+ if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
195+ _ = f.Close()
175196 return nil, err
176197 }
177- return lf, nil
198+ return f, nil
178199 }
179-
180200 func releaseLock(f *os.File) error {
181201 if f == nil {
182202 return nil
183203 }
184- errUnlock := syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
185- errClose := f.Close()
186- if errUnlock != nil {
187- return errUnlock
188- }
189- return errClose
204+ return errors.Join(syscall.Flock(int(f.Fd()), syscall.LOCK_UN), f.Close())
190205 }
191206
192-func (s *Store) appendRecord(op byte, key, val []byte) error {
193- buf, err := encodeRecord(op, key, val)
207+func (s *Store) appendEncoded(encode func() ([]byte, []record, error)) error {
208+ buf, records, err := encode()
194209 if err != nil {
195210 return err
196211 }
197-
198212 s.mu.Lock()
199213 defer s.mu.Unlock()
200214 if s.closed {
@@ -204,180 +218,358 @@ func (s *Store) appendRecord(op byte, key, val []byte) error {
204218 if err != nil {
205219 return err
206220 }
207- n, err := s.file.Write(buf)
208- if err != nil || n != len(buf) {
209- writeErr := err
221+ n, writeErr := s.file.Write(buf)
222+ if writeErr != nil || n != len(buf) {
210223 if writeErr == nil {
211224 writeErr = io.ErrShortWrite
212225 }
213- if tErr := s.file.Truncate(pos); tErr != nil {
226+ if err := errors.Join(s.file.Truncate(pos), seekTo(s.file, pos)); err != nil {
214227 s.closed = true
215- return fmt.Errorf("write failed (%w) and recovery truncate failed: %w", writeErr, tErr)
216- }
217- if _, sErr := s.file.Seek(pos, io.SeekStart); sErr != nil {
218- s.closed = true
219- return fmt.Errorf("write failed (%w) and recovery seek failed: %w", writeErr, sErr)
228+ return fmt.Errorf("write failed (%w) and WAL repair failed: %w", writeErr, err)
220229 }
221230 return writeErr
222231 }
223-
224- k := string(key)
225- if old, ok := s.entrySize[k]; ok {
226- s.live -= old
227- }
232+ applyRecords(s.data, s.entrySize, &s.live, records, int64(len(buf)))
228233 s.total += int64(len(buf))
229- if op == opSet {
230- s.data[k] = append([]byte(nil), val...)
231- s.entrySize[k] = int64(len(buf))
232- s.live += int64(len(buf))
233- } else {
234- delete(s.data, k)
235- delete(s.entrySize, k)
236- }
237234 if s.shouldCompact() {
238235 return s.compactLocked()
239236 }
240237 return nil
241238 }
242-
239+func seekTo(f *os.File, pos int64) error { _, err := f.Seek(pos, io.SeekStart); return err }
243240 func (s *Store) shouldCompact() bool {
244241 return s.total > compactThreshold && float64(s.total-s.live)/float64(s.total) > compactWasteRatio
245242 }
246243
247244 func (s *Store) compactLocked() error {
248- tmp := s.path + ".compact"
249- tf, err := os.OpenFile(tmp, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600)
245+ next, prev := s.path+".next", s.path+".prev"
246+ if err := injectFault(s.hooks, "before-write-next"); err != nil {
247+ return err
248+ }
249+ f, err := s.hooks.Open(next, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600)
250250 if err != nil {
251251 return err
252252 }
253-
254253 keys := make([]string, 0, len(s.data))
255254 for k := range s.data {
256255 keys = append(keys, k)
257256 }
258257 sort.Strings(keys)
259-
260- newSizes := make(map[string]int64, len(keys))
261- var newTotal, newLive int64
262- for _, k := range keys {
263- buf, err := encodeRecord(opSet, []byte(k), s.data[k])
258+ for _, key := range keys {
259+ buf, err := encodeRecord(opSet, []byte(key), s.data[key])
264260 if err != nil {
265- _ = tf.Close()
266- _ = os.Remove(tmp)
261+ _ = f.Close()
267262 return err
268263 }
269- if _, err := tf.Write(buf); err != nil {
270- _ = tf.Close()
271- _ = os.Remove(tmp)
264+ if n, err := f.Write(buf); err != nil || n != len(buf) {
265+ _ = f.Close()
266+ if err == nil {
267+ err = io.ErrShortWrite
268+ }
272269 return err
273270 }
274- sz := int64(len(buf))
275- newSizes[k] = sz
276- newTotal += sz
277- newLive += sz
278271 }
279- if err := tf.Sync(); err != nil {
280- _ = tf.Close()
281- _ = os.Remove(tmp)
272+ if err := injectFault(s.hooks, "after-write-next"); err != nil {
273+ _ = f.Close()
274+ return err
275+ }
276+ if err := injectFault(s.hooks, "before-sync-next"); err != nil {
277+ _ = f.Close()
278+ return err
279+ }
280+ if err := f.Sync(); err != nil {
281+ _ = f.Close()
282+ return err
283+ }
284+ if err := injectFault(s.hooks, "after-sync-next"); err != nil {
285+ _ = f.Close()
286+ return err
287+ }
288+ if err := f.Close(); err != nil {
289+ return err
290+ }
291+ if err := validatePath(next); err != nil {
292+ return err
293+ }
294+ dir := filepath.Dir(s.path)
295+ if err := s.hooks.SyncDir(dir); err != nil {
296+ return err
297+ }
298+ if err := injectFault(s.hooks, "after-sync-dir-next"); err != nil {
299+ return err
300+ }
301+ if err := removeIfExists(s.hooks, prev); err != nil {
302+ return err
303+ }
304+ if err := injectFault(s.hooks, "after-remove-stale-prev"); err != nil {
305+ return err
306+ }
307+ if err := s.hooks.Rename(s.path, prev); err != nil {
308+ return err
309+ }
310+ if err := injectFault(s.hooks, "after-rename-current-prev"); err != nil {
311+ return err
312+ }
313+ if err := s.hooks.SyncDir(dir); err != nil {
314+ return err
315+ }
316+ if err := injectFault(s.hooks, "after-sync-dir-prev"); err != nil {
317+ return err
318+ }
319+ if err := s.hooks.Rename(next, s.path); err != nil {
320+ return err
321+ }
322+ if err := injectFault(s.hooks, "after-rename-next-current"); err != nil {
282323 return err
283324 }
284- if err := tf.Close(); err != nil {
285- _ = os.Remove(tmp)
325+ if err := s.hooks.SyncDir(dir); err != nil {
286326 return err
287327 }
288- bak := s.path + ".bak"
289- _ = os.Remove(bak)
290- if err := os.Rename(s.path, bak); err != nil {
291- _ = os.Remove(tmp)
328+ if err := injectFault(s.hooks, "after-sync-dir-current"); err != nil {
292329 return err
293330 }
294- if err := os.Rename(tmp, s.path); err != nil {
295- _ = os.Rename(bak, s.path)
296- _ = os.Remove(tmp)
331+ if err := injectFault(s.hooks, "before-reopen-current"); err != nil {
297332 return err
298333 }
299- f, err := os.OpenFile(s.path, os.O_RDWR, 0o600)
334+ current, err := s.hooks.Open(s.path, os.O_RDWR, 0o600)
300335 if err != nil {
301- _ = os.Remove(s.path)
302- _ = os.Rename(bak, s.path)
303336 return err
304337 }
305- if _, err := f.Seek(0, io.SeekEnd); err != nil {
306- _ = f.Close()
307- _ = os.Remove(s.path)
308- _ = os.Rename(bak, s.path)
338+ if err := injectFault(s.hooks, "after-reopen-current"); err != nil {
339+ _ = current.Close()
340+ return err
341+ }
342+ result, err := replayFile(current)
343+ if err != nil || result.TornTail {
344+ _ = current.Close()
345+ if err == nil {
346+ err = ErrCorruptWAL
347+ }
348+ return err
349+ }
350+ if _, err := current.Seek(0, io.SeekEnd); err != nil {
351+ _ = current.Close()
309352 return err
310353 }
311354 old := s.file
312- s.file = f
313- s.entrySize = newSizes
314- s.total = newTotal
315- s.live = newLive
316- _ = old.Close()
317- _ = os.Remove(bak)
318- return nil
355+ s.file = current
356+ s.data, s.entrySize, s.total, s.live = result.Data, result.EntrySize, result.Total, result.Live
357+ if err := old.Close(); err != nil {
358+ return err
359+ }
360+ if err := removeIfExists(s.hooks, prev); err != nil {
361+ return err
362+ }
363+ if err := injectFault(s.hooks, "after-remove-prev"); err != nil {
364+ return err
365+ }
366+ if err := s.hooks.SyncDir(dir); err != nil {
367+ return err
368+ }
369+ return injectFault(s.hooks, "after-final-sync-dir")
319370 }
320371
321-func replayFile(f *os.File) (map[string][]byte, map[string]int64, int64, int64, int64, error) {
372+func replayFile(f *os.File) (ReplayResult, error) {
322373 info, err := f.Stat()
323374 if err != nil {
324- return nil, nil, 0, 0, 0, err
375+ return ReplayResult{}, err
325376 }
326- fileSize := info.Size()
327377 if _, err := f.Seek(0, io.SeekStart); err != nil {
328- return nil, nil, 0, 0, 0, err
378+ return ReplayResult{}, err
329379 }
330- data := make(map[string][]byte)
331- sizes := make(map[string]int64)
332- var total, live, off int64
380+ fileSize := info.Size()
381+ result := ReplayResult{Data: make(map[string][]byte), EntrySize: make(map[string]int64)}
333382 header := make([]byte, headerLen)
334383 for {
335384 n, err := io.ReadFull(f, header)
336385 if errors.Is(err, io.EOF) {
337- break
386+ return result, nil
338387 }
339388 if errors.Is(err, io.ErrUnexpectedEOF) {
340389 _ = n
341- break
390+ result.TornTail = true
391+ return result, nil
342392 }
343393 if err != nil {
344- return nil, nil, 0, 0, 0, err
394+ return ReplayResult{}, err
345395 }
346-
347- payloadLen := binary.BigEndian.Uint32(header[0:4])
348- if int64(payloadLen) > fileSize-off-headerLen {
349- break
396+ payloadLen := binary.BigEndian.Uint32(header[:4])
397+ if int64(payloadLen) > fileSize-result.LastGood-headerLen {
398+ result.TornTail = true
399+ return result, nil
350400 }
351- wantCRC := binary.BigEndian.Uint32(header[4:8])
352401 payload := make([]byte, int(payloadLen))
353- if _, err := io.ReadFull(f, payload); errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
354- break
402+ if _, err := io.ReadFull(f, payload); errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
403+ result.TornTail = true
404+ return result, nil
355405 } else if err != nil {
356- return nil, nil, 0, 0, 0, err
406+ return ReplayResult{}, err
407+ }
408+ if crc32.ChecksumIEEE(payload) != binary.BigEndian.Uint32(header[4:8]) {
409+ return ReplayResult{}, fmt.Errorf("%w: CRC mismatch at offset %d", ErrCorruptWAL, result.LastGood)
357410 }
358- if crc32.ChecksumIEEE(payload) != wantCRC {
359- break
411+ var records []record
412+ if len(payload) > 0 && payload[0] == opBatch {
413+ records, err = decodeBatch(payload)
414+ } else {
415+ var rec record
416+ rec, err = decodePayload(payload)
417+ records = []record{rec}
360418 }
361- rec, err := decodePayload(payload)
362419 if err != nil {
363- break
420+ return ReplayResult{}, fmt.Errorf("%w: invalid record at offset %d: %v", ErrCorruptWAL, result.LastGood, err)
364421 }
422+ size := int64(headerLen + len(payload))
423+ applyRecords(result.Data, result.EntrySize, &result.Live, records, size)
424+ result.Total += size
425+ result.LastGood += size
426+ }
427+}
365428
366- sz := int64(headerLen + len(payload))
367- k := string(rec.key)
368- if old, ok := sizes[k]; ok {
369- live -= old
429+func applyRecords(data map[string][]byte, sizes map[string]int64, live *int64, records []record, size int64) {
430+ setCount := int64(0)
431+ for _, rec := range records {
432+ if rec.op == opSet {
433+ setCount++
434+ }
435+ }
436+ setSize := size
437+ if setCount > 1 {
438+ setSize = size / setCount
439+ }
440+ for _, rec := range records {
441+ key := string(rec.key)
442+ if old, ok := sizes[key]; ok {
443+ *live -= old
370444 }
371- total += sz
372445 if rec.op == opSet {
373- data[k] = rec.value
374- sizes[k] = sz
375- live += sz
446+ data[key] = append([]byte(nil), rec.value...)
447+ sizes[key] = setSize
448+ *live += setSize
376449 } else {
377- delete(data, k)
378- delete(sizes, k)
450+ delete(data, key)
451+ delete(sizes, key)
452+ }
453+ }
454+}
455+
456+func recoverCompaction(path string, hooks fsHooks) error {
457+ _, currentExists, currentErr := candidate(path)
458+ nextResult, nextExists, nextErr := candidate(path + ".next")
459+ _, prevExists, prevErr := candidate(path + ".prev")
460+ if nextErr == nil && nextResult.TornTail {
461+ nextErr = ErrCorruptWAL
462+ }
463+ dir := filepath.Dir(path)
464+ if currentExists && currentErr == nil {
465+ if nextExists && !prevExists {
466+ if err := hooks.Remove(path + ".next"); err != nil {
467+ return err
468+ }
469+ if err := hooks.SyncDir(dir); err != nil {
470+ return err
471+ }
472+ }
473+ if prevExists {
474+ if err := hooks.Remove(path + ".prev"); err != nil {
475+ return err
476+ }
477+ if err := hooks.SyncDir(dir); err != nil {
478+ return err
479+ }
480+ }
481+ return nil
482+ }
483+ if currentExists && currentErr != nil {
484+ if !prevExists || prevErr != nil {
485+ return fmt.Errorf("%w: no valid current or predecessor", ErrCorruptWAL)
486+ }
487+ if err := hooks.Remove(path); err != nil {
488+ return err
489+ }
490+ if err := hooks.Rename(path+".prev", path); err != nil {
491+ return err
492+ }
493+ if err := hooks.SyncDir(dir); err != nil {
494+ return err
379495 }
380- off += sz
496+ if nextExists {
497+ if err := hooks.Remove(path + ".next"); err != nil {
498+ return err
499+ }
500+ return hooks.SyncDir(dir)
501+ }
502+ return nil
503+ }
504+ if !currentExists && prevExists {
505+ if prevErr != nil {
506+ return fmt.Errorf("%w: invalid predecessor", ErrCorruptWAL)
507+ }
508+ source := path + ".prev"
509+ if nextExists && nextErr == nil {
510+ source = path + ".next"
511+ }
512+ if err := hooks.Rename(source, path); err != nil {
513+ return err
514+ }
515+ if err := hooks.SyncDir(dir); err != nil {
516+ return err
517+ }
518+ for _, stale := range []string{path + ".prev", path + ".next"} {
519+ if stale != source {
520+ if err := removeIfExists(hooks, stale); err != nil {
521+ return err
522+ }
523+ }
524+ }
525+ return hooks.SyncDir(dir)
526+ }
527+ if nextExists || (currentExists && currentErr != nil) || (prevExists && prevErr != nil) {
528+ return fmt.Errorf("%w: no valid current or predecessor", ErrCorruptWAL)
529+ }
530+ return nil
531+}
532+
533+func candidate(path string) (ReplayResult, bool, error) {
534+ f, err := os.Open(path)
535+ if errors.Is(err, os.ErrNotExist) {
536+ return ReplayResult{}, false, nil
537+ }
538+ if err != nil {
539+ return ReplayResult{}, true, err
540+ }
541+ defer func() { _ = f.Close() }()
542+ result, err := replayFile(f)
543+ return result, true, err
544+}
545+func validatePath(path string) error {
546+ result, exists, err := candidate(path)
547+ if err != nil {
548+ return err
549+ }
550+ if !exists || result.TornTail {
551+ return ErrCorruptWAL
552+ }
553+ return nil
554+}
555+func removeIfExists(hooks fsHooks, path string) error {
556+ err := hooks.Remove(path)
557+ if errors.Is(err, os.ErrNotExist) {
558+ return nil
559+ }
560+ return err
561+}
562+func injectFault(hooks fsHooks, point string) error {
563+ if hooks.Fault == nil {
564+ return nil
565+ }
566+ return hooks.Fault(point)
567+}
568+
569+func syncDir(dir string) error {
570+ f, err := os.Open(dir)
571+ if err != nil {
572+ return err
381573 }
382- return data, sizes, total, live, off, nil
574+ return errors.Join(f.Sync(), f.Close())
383575 }
@@ -15,6 +15,28 @@ import (
15 "github.com/bnema/vev/pkg/safedir"15 "github.com/bnema/vev/pkg/safedir"
16 )16 )
17 17
18+// ReplayResult describes a strictly validated WAL prefix.
19+type ReplayResult struct {
20+ Data map[string][]byte
21+ EntrySize map[string]int64
22+ Total int64
23+ Live int64
24+ LastGood int64
25+ TornTail bool
26+}
27+
28+type fsHooks struct {
29+ Open func(string, int, os.FileMode) (*os.File, error)
30+ Remove func(string) error
31+ Rename func(string, string) error
32+ SyncDir func(string) error
33+ Fault func(string) error
34+}
35+
36+func defaultFSHooks() fsHooks {
37+ return fsHooks{Open: os.OpenFile, Remove: os.Remove, Rename: os.Rename, SyncDir: syncDir}
38+}
39+
18 // Store is an append-only WAL-backed key/value store.40 // Store is an append-only WAL-backed key/value store.
19 type Store struct {41 type Store struct {
20 mu sync.Mutex42 mu sync.Mutex
@@ -26,55 +48,54 @@ type Store struct {
26 total int6448 total int64
27 live int6449 live int64
28 closed bool50 closed bool
51+ hooks fsHooks
29 }52 }
30 53
31-// Open opens or creates a store at path, replays valid records, truncates any54+// Open opens or creates a store, strictly recovers compaction, and truncates only a torn tail.
32-// corrupt or torn tail, and compacts if the file contains enough obsolete data.55+func Open(path string) (*Store, error) { return openWithHooks(path, defaultFSHooks()) }
33-func Open(path string) (*Store, error) {56+
57+func openWithHooks(path string, hooks fsHooks) (*Store, error) {
34 if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {58 if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil {
35 return nil, err59 return nil, err
36 }60 }
37-
38 lockFile, err := acquireLock(path)61 lockFile, err := acquireLock(path)
39 if err != nil {62 if err != nil {
40 return nil, err63 return nil, err
41 }64 }
42-65+ cleanupLock := func(err error) (*Store, error) { _ = releaseLock(lockFile); return nil, err }
43- f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600)66+ if err := recoverCompaction(path, hooks); err != nil {
67+ return cleanupLock(err)
68+ }
69+ f, err := hooks.Open(path, os.O_RDWR|os.O_CREATE, 0o600)
44 if err != nil {70 if err != nil {
45- _ = releaseLock(lockFile)71+ return cleanupLock(err)
46- return nil, err
47 }72 }
48-73+ result, err := replayFile(f)
49- data, sizes, total, live, lastGood, err := replayFile(f)
50 if err != nil {74 if err != nil {
51 _ = f.Close()75 _ = f.Close()
52- _ = releaseLock(lockFile)76+ return cleanupLock(err)
53- return nil, err
54 }77 }
55- if err := f.Truncate(lastGood); err != nil {78+ if result.TornTail {
56- _ = f.Close()79+ if err := f.Truncate(result.LastGood); err != nil {
57- _ = releaseLock(lockFile)80+ _ = f.Close()
58- return nil, err81+ return cleanupLock(err)
82+ }
59 }83 }
60 if _, err := f.Seek(0, io.SeekEnd); err != nil {84 if _, err := f.Seek(0, io.SeekEnd); err != nil {
61 _ = f.Close()85 _ = f.Close()
62- _ = releaseLock(lockFile)86+ return cleanupLock(err)
63- return nil, err
64 }87 }
65-88+ s := &Store{path: path, file: f, lockFile: lockFile, data: result.Data, entrySize: result.EntrySize, total: result.Total, live: result.Live, hooks: hooks}
66- s := &Store{path: path, file: f, lockFile: lockFile, data: data, entrySize: sizes, total: total, live: live}
67 if s.shouldCompact() {89 if s.shouldCompact() {
68 if err := s.compactLocked(); err != nil {90 if err := s.compactLocked(); err != nil {
69 _ = f.Close()91 _ = f.Close()
70- _ = releaseLock(lockFile)92+ return cleanupLock(err)
71- return nil, err
72 }93 }
73 }94 }
74 return s, nil95 return s, nil
75 }96 }
76 97
77-// Replay reads path without mutating it and returns the state from valid records.98+// Replay reads path without mutating it and returns only strictly valid state.
78 func Replay(path string) (map[string][]byte, error) {99 func Replay(path string) (map[string][]byte, error) {
79 f, err := os.Open(path)100 f, err := os.Open(path)
80 if errors.Is(err, os.ErrNotExist) {101 if errors.Is(err, os.ErrNotExist) {
@@ -84,37 +105,50 @@ func Replay(path string) (map[string][]byte, error) {
84 return nil, err105 return nil, err
85 }106 }
86 defer func() { _ = f.Close() }()107 defer func() { _ = f.Close() }()
87-108+ result, err := replayFile(f)
88- data, _, _, _, _, err := replayFile(f)
89 if err != nil {109 if err != nil {
90 return nil, err110 return nil, err
91 }111 }
92- return data, nil112+ return result.Data, nil
93 }113 }
94 114
95-// Get returns a copy of the value for key.
96 func (s *Store) Get(key []byte) ([]byte, bool) {115 func (s *Store) Get(key []byte) ([]byte, bool) {
97 s.mu.Lock()116 s.mu.Lock()
98 defer s.mu.Unlock()117 defer s.mu.Unlock()
99-
100 v, ok := s.data[string(key)]118 v, ok := s.data[string(key)]
101- if !ok {119+ return append([]byte(nil), v...), ok
102- return nil, false
103- }
104- return append([]byte(nil), v...), true
105 }120 }
106-
107-// Set appends a SET record. It does not fsync.
108 func (s *Store) Set(key, val []byte) error {121 func (s *Store) Set(key, val []byte) error {
109- return s.appendRecord(opSet, key, val)122+ return s.appendEncoded(func() ([]byte, []record, error) {
123+ buf, err := encodeRecord(opSet, key, val)
124+ return buf, []record{{op: opSet, key: append([]byte(nil), key...), value: append([]byte(nil), val...)}}, err
125+ })
110 }126 }
111-
112-// Delete appends a DEL tombstone. It does not fsync.
113 func (s *Store) Delete(key []byte) error {127 func (s *Store) Delete(key []byte) error {
114- return s.appendRecord(opDel, key, nil)128+ return s.appendEncoded(func() ([]byte, []record, error) {
129+ buf, err := encodeRecord(opDel, key, nil)
130+ return buf, []record{{op: opDel, key: append([]byte(nil), key...)}}, err
131+ })
132+}
133+
134+// Batch appends all changes as one WAL record and applies them only after a complete write.
135+func (s *Store) Batch(changes []BatchChange) error {
136+ return s.appendEncoded(func() ([]byte, []record, error) {
137+ buf, err := encodeBatch(changes)
138+ if err != nil {
139+ return nil, nil, err
140+ }
141+ recs := make([]record, len(changes))
142+ for i, change := range changes {
143+ recs[i] = record{op: opSet, key: append([]byte(nil), change.Key...), value: append([]byte(nil), change.Value...)}
144+ if change.Delete {
145+ recs[i].op = opDel
146+ }
147+ }
148+ return buf, recs, nil
149+ })
115 }150 }
116 151
117-// Range calls fn for each key/value pair, using copies. Iteration stops when fn returns false.
118 func (s *Store) Range(fn func(k, v []byte) bool) {152 func (s *Store) Range(fn func(k, v []byte) bool) {
119 s.mu.Lock()153 s.mu.Lock()
120 keys := make([]string, 0, len(s.data))154 keys := make([]string, 0, len(s.data))
@@ -122,20 +156,17 @@ func (s *Store) Range(fn func(k, v []byte) bool) {
122 keys = append(keys, k)156 keys = append(keys, k)
123 }157 }
124 sort.Strings(keys)158 sort.Strings(keys)
125- pairs := make([]struct{ k, v []byte }, 0, len(keys))159+ pairs := make([]record, 0, len(keys))
126 for _, k := range keys {160 for _, k := range keys {
127- pairs = append(pairs, struct{ k, v []byte }{[]byte(k), append([]byte(nil), s.data[k]...)})161+ pairs = append(pairs, record{key: []byte(k), value: append([]byte(nil), s.data[k]...)})
128 }162 }
129 s.mu.Unlock()163 s.mu.Unlock()
130-164+ for _, pair := range pairs {
131- for _, p := range pairs {165+ if !fn(pair.key, pair.value) {
132- if !fn(p.k, p.v) {
133 return166 return
134 }167 }
135 }168 }
136 }169 }
137-
138-// Sync fsyncs the WAL file.
139 func (s *Store) Sync() error {170 func (s *Store) Sync() error {
140 s.mu.Lock()171 s.mu.Lock()
141 defer s.mu.Unlock()172 defer s.mu.Unlock()
@@ -144,57 +175,40 @@ func (s *Store) Sync() error {
144 }175 }
145 return s.file.Sync()176 return s.file.Sync()
146 }177 }
147-
148-// Close fsyncs and closes the store.
149 func (s *Store) Close() error {178 func (s *Store) Close() error {
150 s.mu.Lock()179 s.mu.Lock()
151 defer s.mu.Unlock()180 defer s.mu.Unlock()
152 if s.closed {181 if s.closed {
153 return nil182 return nil
154 }183 }
155- errSync := s.file.Sync()184+ err := errors.Join(s.file.Sync(), s.file.Close(), releaseLock(s.lockFile))
156- errClose := s.file.Close()
157- errUnlock := releaseLock(s.lockFile)
158 s.closed = true185 s.closed = true
159- if errSync != nil {186+ return err
160- return errSync
161- }
162- if errClose != nil {
163- return errClose
164- }
165- return errUnlock
166 }187 }
167 188
168 func acquireLock(path string) (*os.File, error) {189 func acquireLock(path string) (*os.File, error) {
169- lf, err := os.OpenFile(path+".lock", os.O_RDWR|os.O_CREATE, 0o600)190+ f, err := os.OpenFile(path+".lock", os.O_RDWR|os.O_CREATE, 0o600)
170 if err != nil {191 if err != nil {
171 return nil, err192 return nil, err
172 }193 }
173- if err := syscall.Flock(int(lf.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {194+ if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
174- _ = lf.Close()195+ _ = f.Close()
175 return nil, err196 return nil, err
176 }197 }
177- return lf, nil198+ return f, nil
178 }199 }
179-
180 func releaseLock(f *os.File) error {200 func releaseLock(f *os.File) error {
181 if f == nil {201 if f == nil {
182 return nil202 return nil
183 }203 }
184- errUnlock := syscall.Flock(int(f.Fd()), syscall.LOCK_UN)204+ return errors.Join(syscall.Flock(int(f.Fd()), syscall.LOCK_UN), f.Close())
185- errClose := f.Close()
186- if errUnlock != nil {
187- return errUnlock
188- }
189- return errClose
190 }205 }
191 206
192-func (s *Store) appendRecord(op byte, key, val []byte) error {207+func (s *Store) appendEncoded(encode func() ([]byte, []record, error)) error {
193- buf, err := encodeRecord(op, key, val)208+ buf, records, err := encode()
194 if err != nil {209 if err != nil {
195 return err210 return err
196 }211 }
197-
198 s.mu.Lock()212 s.mu.Lock()
199 defer s.mu.Unlock()213 defer s.mu.Unlock()
200 if s.closed {214 if s.closed {
@@ -204,180 +218,358 @@ func (s *Store) appendRecord(op byte, key, val []byte) error {
204 if err != nil {218 if err != nil {
205 return err219 return err
206 }220 }
207- n, err := s.file.Write(buf)221+ n, writeErr := s.file.Write(buf)
208- if err != nil || n != len(buf) {222+ if writeErr != nil || n != len(buf) {
209- writeErr := err
210 if writeErr == nil {223 if writeErr == nil {
211 writeErr = io.ErrShortWrite224 writeErr = io.ErrShortWrite
212 }225 }
213- if tErr := s.file.Truncate(pos); tErr != nil {226+ if err := errors.Join(s.file.Truncate(pos), seekTo(s.file, pos)); err != nil {
214 s.closed = true227 s.closed = true
215- return fmt.Errorf("write failed (%w) and recovery truncate failed: %w", writeErr, tErr)228+ return fmt.Errorf("write failed (%w) and WAL repair failed: %w", writeErr, err)
216- }
217- if _, sErr := s.file.Seek(pos, io.SeekStart); sErr != nil {
218- s.closed = true
219- return fmt.Errorf("write failed (%w) and recovery seek failed: %w", writeErr, sErr)
220 }229 }
221 return writeErr230 return writeErr
222 }231 }
223-232+ applyRecords(s.data, s.entrySize, &s.live, records, int64(len(buf)))
224- k := string(key)
225- if old, ok := s.entrySize[k]; ok {
226- s.live -= old
227- }
228 s.total += int64(len(buf))233 s.total += int64(len(buf))
229- if op == opSet {
230- s.data[k] = append([]byte(nil), val...)
231- s.entrySize[k] = int64(len(buf))
232- s.live += int64(len(buf))
233- } else {
234- delete(s.data, k)
235- delete(s.entrySize, k)
236- }
237 if s.shouldCompact() {234 if s.shouldCompact() {
238 return s.compactLocked()235 return s.compactLocked()
239 }236 }
240 return nil237 return nil
241 }238 }
242-239+func seekTo(f *os.File, pos int64) error { _, err := f.Seek(pos, io.SeekStart); return err }
243 func (s *Store) shouldCompact() bool {240 func (s *Store) shouldCompact() bool {
244 return s.total > compactThreshold && float64(s.total-s.live)/float64(s.total) > compactWasteRatio241 return s.total > compactThreshold && float64(s.total-s.live)/float64(s.total) > compactWasteRatio
245 }242 }
246 243
247 func (s *Store) compactLocked() error {244 func (s *Store) compactLocked() error {
248- tmp := s.path + ".compact"245+ next, prev := s.path+".next", s.path+".prev"
249- tf, err := os.OpenFile(tmp, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600)246+ if err := injectFault(s.hooks, "before-write-next"); err != nil {
247+ return err
248+ }
249+ f, err := s.hooks.Open(next, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600)
250 if err != nil {250 if err != nil {
251 return err251 return err
252 }252 }
253-
254 keys := make([]string, 0, len(s.data))253 keys := make([]string, 0, len(s.data))
255 for k := range s.data {254 for k := range s.data {
256 keys = append(keys, k)255 keys = append(keys, k)
257 }256 }
258 sort.Strings(keys)257 sort.Strings(keys)
259-258+ for _, key := range keys {
260- newSizes := make(map[string]int64, len(keys))259+ buf, err := encodeRecord(opSet, []byte(key), s.data[key])
261- var newTotal, newLive int64
262- for _, k := range keys {
263- buf, err := encodeRecord(opSet, []byte(k), s.data[k])
264 if err != nil {260 if err != nil {
265- _ = tf.Close()261+ _ = f.Close()
266- _ = os.Remove(tmp)
267 return err262 return err
268 }263 }
269- if _, err := tf.Write(buf); err != nil {264+ if n, err := f.Write(buf); err != nil || n != len(buf) {
270- _ = tf.Close()265+ _ = f.Close()
271- _ = os.Remove(tmp)266+ if err == nil {
267+ err = io.ErrShortWrite
268+ }
272 return err269 return err
273 }270 }
274- sz := int64(len(buf))
275- newSizes[k] = sz
276- newTotal += sz
277- newLive += sz
278 }271 }
279- if err := tf.Sync(); err != nil {272+ if err := injectFault(s.hooks, "after-write-next"); err != nil {
280- _ = tf.Close()273+ _ = f.Close()
281- _ = os.Remove(tmp)274+ return err
275+ }
276+ if err := injectFault(s.hooks, "before-sync-next"); err != nil {
277+ _ = f.Close()
278+ return err
279+ }
280+ if err := f.Sync(); err != nil {
281+ _ = f.Close()
282+ return err
283+ }
284+ if err := injectFault(s.hooks, "after-sync-next"); err != nil {
285+ _ = f.Close()
286+ return err
287+ }
288+ if err := f.Close(); err != nil {
289+ return err
290+ }
291+ if err := validatePath(next); err != nil {
292+ return err
293+ }
294+ dir := filepath.Dir(s.path)
295+ if err := s.hooks.SyncDir(dir); err != nil {
296+ return err
297+ }
298+ if err := injectFault(s.hooks, "after-sync-dir-next"); err != nil {
299+ return err
300+ }
301+ if err := removeIfExists(s.hooks, prev); err != nil {
302+ return err
303+ }
304+ if err := injectFault(s.hooks, "after-remove-stale-prev"); err != nil {
305+ return err
306+ }
307+ if err := s.hooks.Rename(s.path, prev); err != nil {
308+ return err
309+ }
310+ if err := injectFault(s.hooks, "after-rename-current-prev"); err != nil {
311+ return err
312+ }
313+ if err := s.hooks.SyncDir(dir); err != nil {
314+ return err
315+ }
316+ if err := injectFault(s.hooks, "after-sync-dir-prev"); err != nil {
317+ return err
318+ }
319+ if err := s.hooks.Rename(next, s.path); err != nil {
320+ return err
321+ }
322+ if err := injectFault(s.hooks, "after-rename-next-current"); err != nil {
282 return err323 return err
283 }324 }
284- if err := tf.Close(); err != nil {325+ if err := s.hooks.SyncDir(dir); err != nil {
285- _ = os.Remove(tmp)
286 return err326 return err
287 }327 }
288- bak := s.path + ".bak"328+ if err := injectFault(s.hooks, "after-sync-dir-current"); err != nil {
289- _ = os.Remove(bak)
290- if err := os.Rename(s.path, bak); err != nil {
291- _ = os.Remove(tmp)
292 return err329 return err
293 }330 }
294- if err := os.Rename(tmp, s.path); err != nil {331+ if err := injectFault(s.hooks, "before-reopen-current"); err != nil {
295- _ = os.Rename(bak, s.path)
296- _ = os.Remove(tmp)
297 return err332 return err
298 }333 }
299- f, err := os.OpenFile(s.path, os.O_RDWR, 0o600)334+ current, err := s.hooks.Open(s.path, os.O_RDWR, 0o600)
300 if err != nil {335 if err != nil {
301- _ = os.Remove(s.path)
302- _ = os.Rename(bak, s.path)
303 return err336 return err
304 }337 }
305- if _, err := f.Seek(0, io.SeekEnd); err != nil {338+ if err := injectFault(s.hooks, "after-reopen-current"); err != nil {
306- _ = f.Close()339+ _ = current.Close()
307- _ = os.Remove(s.path)340+ return err
308- _ = os.Rename(bak, s.path)341+ }
342+ result, err := replayFile(current)
343+ if err != nil || result.TornTail {
344+ _ = current.Close()
345+ if err == nil {
346+ err = ErrCorruptWAL
347+ }
348+ return err
349+ }
350+ if _, err := current.Seek(0, io.SeekEnd); err != nil {
351+ _ = current.Close()
309 return err352 return err
310 }353 }
311 old := s.file354 old := s.file
312- s.file = f355+ s.file = current
313- s.entrySize = newSizes356+ s.data, s.entrySize, s.total, s.live = result.Data, result.EntrySize, result.Total, result.Live
314- s.total = newTotal357+ if err := old.Close(); err != nil {
315- s.live = newLive358+ return err
316- _ = old.Close()359+ }
317- _ = os.Remove(bak)360+ if err := removeIfExists(s.hooks, prev); err != nil {
318- return nil361+ return err
362+ }
363+ if err := injectFault(s.hooks, "after-remove-prev"); err != nil {
364+ return err
365+ }
366+ if err := s.hooks.SyncDir(dir); err != nil {
367+ return err
368+ }
369+ return injectFault(s.hooks, "after-final-sync-dir")
319 }370 }
320 371
321-func replayFile(f *os.File) (map[string][]byte, map[string]int64, int64, int64, int64, error) {372+func replayFile(f *os.File) (ReplayResult, error) {
322 info, err := f.Stat()373 info, err := f.Stat()
323 if err != nil {374 if err != nil {
324- return nil, nil, 0, 0, 0, err375+ return ReplayResult{}, err
325 }376 }
326- fileSize := info.Size()
327 if _, err := f.Seek(0, io.SeekStart); err != nil {377 if _, err := f.Seek(0, io.SeekStart); err != nil {
328- return nil, nil, 0, 0, 0, err378+ return ReplayResult{}, err
329 }379 }
330- data := make(map[string][]byte)380+ fileSize := info.Size()
331- sizes := make(map[string]int64)381+ result := ReplayResult{Data: make(map[string][]byte), EntrySize: make(map[string]int64)}
332- var total, live, off int64
333 header := make([]byte, headerLen)382 header := make([]byte, headerLen)
334 for {383 for {
335 n, err := io.ReadFull(f, header)384 n, err := io.ReadFull(f, header)
336 if errors.Is(err, io.EOF) {385 if errors.Is(err, io.EOF) {
337- break386+ return result, nil
338 }387 }
339 if errors.Is(err, io.ErrUnexpectedEOF) {388 if errors.Is(err, io.ErrUnexpectedEOF) {
340 _ = n389 _ = n
341- break390+ result.TornTail = true
391+ return result, nil
342 }392 }
343 if err != nil {393 if err != nil {
344- return nil, nil, 0, 0, 0, err394+ return ReplayResult{}, err
345 }395 }
346-396+ payloadLen := binary.BigEndian.Uint32(header[:4])
347- payloadLen := binary.BigEndian.Uint32(header[0:4])397+ if int64(payloadLen) > fileSize-result.LastGood-headerLen {
348- if int64(payloadLen) > fileSize-off-headerLen {398+ result.TornTail = true
349- break399+ return result, nil
350 }400 }
351- wantCRC := binary.BigEndian.Uint32(header[4:8])
352 payload := make([]byte, int(payloadLen))401 payload := make([]byte, int(payloadLen))
353- if _, err := io.ReadFull(f, payload); errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {402+ if _, err := io.ReadFull(f, payload); errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
354- break403+ result.TornTail = true
404+ return result, nil
355 } else if err != nil {405 } else if err != nil {
356- return nil, nil, 0, 0, 0, err406+ return ReplayResult{}, err
407+ }
408+ if crc32.ChecksumIEEE(payload) != binary.BigEndian.Uint32(header[4:8]) {
409+ return ReplayResult{}, fmt.Errorf("%w: CRC mismatch at offset %d", ErrCorruptWAL, result.LastGood)
357 }410 }
358- if crc32.ChecksumIEEE(payload) != wantCRC {411+ var records []record
359- break412+ if len(payload) > 0 && payload[0] == opBatch {
413+ records, err = decodeBatch(payload)
414+ } else {
415+ var rec record
416+ rec, err = decodePayload(payload)
417+ records = []record{rec}
360 }418 }
361- rec, err := decodePayload(payload)
362 if err != nil {419 if err != nil {
363- break420+ return ReplayResult{}, fmt.Errorf("%w: invalid record at offset %d: %v", ErrCorruptWAL, result.LastGood, err)
364 }421 }
422+ size := int64(headerLen + len(payload))
423+ applyRecords(result.Data, result.EntrySize, &result.Live, records, size)
424+ result.Total += size
425+ result.LastGood += size
426+ }
427+}
365 428
366- sz := int64(headerLen + len(payload))429+func applyRecords(data map[string][]byte, sizes map[string]int64, live *int64, records []record, size int64) {
367- k := string(rec.key)430+ setCount := int64(0)
368- if old, ok := sizes[k]; ok {431+ for _, rec := range records {
369- live -= old432+ if rec.op == opSet {
433+ setCount++
434+ }
435+ }
436+ setSize := size
437+ if setCount > 1 {
438+ setSize = size / setCount
439+ }
440+ for _, rec := range records {
441+ key := string(rec.key)
442+ if old, ok := sizes[key]; ok {
443+ *live -= old
370 }444 }
371- total += sz
372 if rec.op == opSet {445 if rec.op == opSet {
373- data[k] = rec.value446+ data[key] = append([]byte(nil), rec.value...)
374- sizes[k] = sz447+ sizes[key] = setSize
375- live += sz448+ *live += setSize
376 } else {449 } else {
377- delete(data, k)450+ delete(data, key)
378- delete(sizes, k)451+ delete(sizes, key)
452+ }
453+ }
454+}
455+
456+func recoverCompaction(path string, hooks fsHooks) error {
457+ _, currentExists, currentErr := candidate(path)
458+ nextResult, nextExists, nextErr := candidate(path + ".next")
459+ _, prevExists, prevErr := candidate(path + ".prev")
460+ if nextErr == nil && nextResult.TornTail {
461+ nextErr = ErrCorruptWAL
462+ }
463+ dir := filepath.Dir(path)
464+ if currentExists && currentErr == nil {
465+ if nextExists && !prevExists {
466+ if err := hooks.Remove(path + ".next"); err != nil {
467+ return err
468+ }
469+ if err := hooks.SyncDir(dir); err != nil {
470+ return err
471+ }
472+ }
473+ if prevExists {
474+ if err := hooks.Remove(path + ".prev"); err != nil {
475+ return err
476+ }
477+ if err := hooks.SyncDir(dir); err != nil {
478+ return err
479+ }
480+ }
481+ return nil
482+ }
483+ if currentExists && currentErr != nil {
484+ if !prevExists || prevErr != nil {
485+ return fmt.Errorf("%w: no valid current or predecessor", ErrCorruptWAL)
486+ }
487+ if err := hooks.Remove(path); err != nil {
488+ return err
489+ }
490+ if err := hooks.Rename(path+".prev", path); err != nil {
491+ return err
492+ }
493+ if err := hooks.SyncDir(dir); err != nil {
494+ return err
379 }495 }
380- off += sz496+ if nextExists {
497+ if err := hooks.Remove(path + ".next"); err != nil {
498+ return err
499+ }
500+ return hooks.SyncDir(dir)
501+ }
502+ return nil
503+ }
504+ if !currentExists && prevExists {
505+ if prevErr != nil {
506+ return fmt.Errorf("%w: invalid predecessor", ErrCorruptWAL)
507+ }
508+ source := path + ".prev"
509+ if nextExists && nextErr == nil {
510+ source = path + ".next"
511+ }
512+ if err := hooks.Rename(source, path); err != nil {
513+ return err
514+ }
515+ if err := hooks.SyncDir(dir); err != nil {
516+ return err
517+ }
518+ for _, stale := range []string{path + ".prev", path + ".next"} {
519+ if stale != source {
520+ if err := removeIfExists(hooks, stale); err != nil {
521+ return err
522+ }
523+ }
524+ }
525+ return hooks.SyncDir(dir)
526+ }
527+ if nextExists || (currentExists && currentErr != nil) || (prevExists && prevErr != nil) {
528+ return fmt.Errorf("%w: no valid current or predecessor", ErrCorruptWAL)
529+ }
530+ return nil
531+}
532+
533+func candidate(path string) (ReplayResult, bool, error) {
534+ f, err := os.Open(path)
535+ if errors.Is(err, os.ErrNotExist) {
536+ return ReplayResult{}, false, nil
537+ }
538+ if err != nil {
539+ return ReplayResult{}, true, err
540+ }
541+ defer func() { _ = f.Close() }()
542+ result, err := replayFile(f)
543+ return result, true, err
544+}
545+func validatePath(path string) error {
546+ result, exists, err := candidate(path)
547+ if err != nil {
548+ return err
549+ }
550+ if !exists || result.TornTail {
551+ return ErrCorruptWAL
552+ }
553+ return nil
554+}
555+func removeIfExists(hooks fsHooks, path string) error {
556+ err := hooks.Remove(path)
557+ if errors.Is(err, os.ErrNotExist) {
558+ return nil
559+ }
560+ return err
561+}
562+func injectFault(hooks fsHooks, point string) error {
563+ if hooks.Fault == nil {
564+ return nil
565+ }
566+ return hooks.Fault(point)
567+}
568+
569+func syncDir(dir string) error {
570+ f, err := os.Open(dir)
571+ if err != nil {
572+ return err
381 }573 }
382- return data, sizes, total, live, off, nil574+ return errors.Join(f.Sync(), f.Close())
383 }575 }
M pkg/kv/kv_test.go +214 -14
@@ -3,6 +3,8 @@ package kv
33 import (
44 "bytes"
55 "encoding/binary"
6+ "errors"
7+ "hash/crc32"
68 "os"
79 "path/filepath"
810 "testing"
@@ -149,7 +151,7 @@ func TestOpenTruncatesMidRecordTail(t *testing.T) {
149151 }
150152 }
151153
152-func TestCRCcorruptionOfLastRecordDropsIt(t *testing.T) {
154+func TestCRCcorruptionOfLastRecordFailsClosed(t *testing.T) {
153155 path := privatePath(t)
154156 first, _ := encodeRecord(opSet, []byte("k"), []byte("old"))
155157 second, _ := encodeRecord(opSet, []byte("k"), []byte("new"))
@@ -162,21 +164,16 @@ func TestCRCcorruptionOfLastRecordDropsIt(t *testing.T) {
162164 t.Fatal(err)
163165 }
164166
165- s, err := Open(path)
166- if err != nil {
167- t.Fatal(err)
168- }
169- defer closeStore(t, s)
170- got, ok := s.Get([]byte("k"))
171- if !ok || string(got) != "old" {
172- t.Fatalf("corrupt last record not dropped: %q %v", got, ok)
167+ _, err := Open(path)
168+ if !errors.Is(err, ErrCorruptWAL) {
169+ t.Fatalf("Open error = %v, want ErrCorruptWAL", err)
173170 }
174- info, err := os.Stat(path)
175- if err != nil {
176- t.Fatal(err)
171+ info, statErr := os.Stat(path)
172+ if statErr != nil {
173+ t.Fatal(statErr)
177174 }
178- if info.Size() != int64(len(first)) {
179- t.Fatalf("corrupt tail not truncated: size=%d want=%d", info.Size(), len(first))
175+ if info.Size() != int64(len(buf)) {
176+ t.Fatalf("corrupt WAL was mutated: size=%d want=%d", info.Size(), len(buf))
180177 }
181178 }
182179
@@ -237,6 +234,209 @@ func TestTombstoneSurvivesCompaction(t *testing.T) {
237234 }
238235 }
239236
237+func TestReplayStrict(t *testing.T) {
238+ valid, err := encodeRecord(opSet, []byte("old"), []byte("value"))
239+ if err != nil {
240+ t.Fatal(err)
241+ }
242+ batch, err := encodeBatch([]BatchChange{{Key: []byte("old"), Delete: true}, {Key: []byte("new"), Value: []byte("value")}})
243+ if err != nil {
244+ t.Fatal(err)
245+ }
246+
247+ tests := []struct {
248+ name string
249+ wal []byte
250+ torn bool
251+ corrupt bool
252+ }{
253+ {name: "torn-final-header", wal: append(append([]byte{}, valid...), batch[:headerLen-1]...), torn: true},
254+ {name: "torn-final-payload", wal: append(append([]byte{}, valid...), batch[:len(batch)-1]...), torn: true},
255+ {name: "final-crc", wal: corruptRecord(append(append([]byte{}, valid...), batch...), len(valid)+4), corrupt: true},
256+ {name: "middle-crc", wal: corruptRecord(append(append([]byte{}, valid...), batch...), 4), corrupt: true},
257+ {name: "invalid-op", wal: rawRecord([]byte{0xff, 0, 0}), corrupt: true},
258+ {name: "duplicate-key-in-batch", wal: rawRecord(rawBatchPayload([]BatchChange{{Key: []byte("x")}, {Key: []byte("x"), Delete: true}})), corrupt: true},
259+ {name: "trailing-batch-garbage", wal: rawRecord(append(rawBatchPayload([]BatchChange{{Key: []byte("x")}}), 0)), corrupt: true},
260+ }
261+ for _, tt := range tests {
262+ t.Run(tt.name, func(t *testing.T) {
263+ path := privatePath(t)
264+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
265+ t.Fatal(err)
266+ }
267+ if err := os.WriteFile(path, tt.wal, 0o600); err != nil {
268+ t.Fatal(err)
269+ }
270+ f, err := os.Open(path)
271+ if err != nil {
272+ t.Fatal(err)
273+ }
274+ result, replayErr := replayFile(f)
275+ _ = f.Close()
276+ if tt.corrupt {
277+ if !errors.Is(replayErr, ErrCorruptWAL) {
278+ t.Fatalf("replay error = %v, want ErrCorruptWAL", replayErr)
279+ }
280+ return
281+ }
282+ if replayErr != nil {
283+ t.Fatal(replayErr)
284+ }
285+ if string(result.Data["old"]) != "value" || result.Data["new"] != nil {
286+ t.Fatalf("torn batch was partially applied: %#v", result.Data)
287+ }
288+ if !result.TornTail || result.LastGood != int64(len(valid)) {
289+ t.Fatalf("result = %+v", result)
290+ }
291+ })
292+ }
293+}
294+
295+func TestBatchAtomicity(t *testing.T) {
296+ path := privatePath(t)
297+ s, err := Open(path)
298+ if err != nil {
299+ t.Fatal(err)
300+ }
301+ defer closeStore(t, s)
302+ if err := s.Set([]byte("old"), []byte("before")); err != nil {
303+ t.Fatal(err)
304+ }
305+ if err := s.Batch(nil); err == nil {
306+ t.Fatal("empty batch succeeded")
307+ }
308+ if err := s.Batch([]BatchChange{{Key: []byte("duplicate")}, {Key: []byte("duplicate"), Delete: true}}); err == nil {
309+ t.Fatal("duplicate-key batch succeeded")
310+ }
311+ if err := s.Batch([]BatchChange{{Key: []byte("old"), Delete: true}, {Key: []byte("new"), Value: []byte("same-incarnation")}}); err != nil {
312+ t.Fatal(err)
313+ }
314+ if err := s.Sync(); err != nil {
315+ t.Fatal(err)
316+ }
317+ if _, ok := s.Get([]byte("old")); ok {
318+ t.Fatal("old key remains")
319+ }
320+ got, ok := s.Get([]byte("new"))
321+ if !ok || !bytes.Equal(got, []byte("same-incarnation")) {
322+ t.Fatalf("new = %q, %v", got, ok)
323+ }
324+}
325+
326+func TestCompactionRecoveryMatrix(t *testing.T) {
327+ oldRecord, _ := encodeRecord(opSet, []byte("version"), []byte("old"))
328+ newRecord, _ := encodeRecord(opSet, []byte("version"), []byte("new"))
329+ tests := []struct {
330+ name string
331+ current, next, prev []byte
332+ want string
333+ }{
334+ {name: "current-alone", current: oldRecord, want: "old"},
335+ {name: "current-next", current: oldRecord, next: newRecord, want: "old"},
336+ {name: "prev-next", prev: oldRecord, next: newRecord, want: "new"},
337+ {name: "prev-invalid-next", prev: oldRecord, next: rawRecord([]byte{0xff, 0, 0}), want: "old"},
338+ {name: "prev-torn-next", prev: oldRecord, next: newRecord[:len(newRecord)-1], want: "old"},
339+ {name: "current-prev", current: newRecord, prev: oldRecord, want: "new"},
340+ {name: "invalid-current-prev", current: rawRecord([]byte{0xff, 0, 0}), prev: oldRecord, want: "old"},
341+ {name: "prev-alone", prev: oldRecord, want: "old"},
342+ }
343+ for _, tt := range tests {
344+ t.Run(tt.name, func(t *testing.T) {
345+ path := privatePath(t)
346+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
347+ t.Fatal(err)
348+ }
349+ for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} {
350+ if data != nil {
351+ if err := os.WriteFile(path+suffix, data, 0o600); err != nil {
352+ t.Fatal(err)
353+ }
354+ }
355+ }
356+ if err := recoverCompaction(path, defaultFSHooks()); err != nil {
357+ t.Fatal(err)
358+ }
359+ got, err := Replay(path)
360+ if err != nil {
361+ t.Fatal(err)
362+ }
363+ if string(got["version"]) != tt.want {
364+ t.Fatalf("version = %q, want %q", got["version"], tt.want)
365+ }
366+ })
367+ }
368+
369+ faultPoints := []string{
370+ "before-write-next", "after-write-next", "before-sync-next", "after-sync-next",
371+ "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev",
372+ "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current",
373+ "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir",
374+ }
375+ for _, point := range faultPoints {
376+ t.Run("fault-"+point, func(t *testing.T) {
377+ path := privatePath(t)
378+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
379+ t.Fatal(err)
380+ }
381+ if err := os.WriteFile(path, oldRecord, 0o600); err != nil {
382+ t.Fatal(err)
383+ }
384+ s, err := Open(path)
385+ if err != nil {
386+ t.Fatal(err)
387+ }
388+ s.data = map[string][]byte{"version": []byte("new")}
389+ s.hooks.Fault = func(got string) error {
390+ if got == point {
391+ return errors.New("injected " + point)
392+ }
393+ return nil
394+ }
395+ if err := s.compactLocked(); err == nil {
396+ t.Fatalf("fault %q did not interrupt compaction", point)
397+ }
398+ if s.file != nil {
399+ _ = s.file.Close()
400+ }
401+ _ = releaseLock(s.lockFile)
402+ if err := recoverCompaction(path, defaultFSHooks()); err != nil {
403+ t.Fatalf("recovery: %v", err)
404+ }
405+ got, err := Replay(path)
406+ if err != nil {
407+ t.Fatal(err)
408+ }
409+ version := string(got["version"])
410+ if (version != "old" && version != "new") || len(got) != 1 {
411+ t.Fatalf("mixed or empty recovered map: %#v", got)
412+ }
413+ })
414+ }
415+}
416+
417+func corruptRecord(buf []byte, offset int) []byte { buf[offset] ^= 0xff; return buf }
418+func rawRecord(payload []byte) []byte {
419+ buf := make([]byte, headerLen+len(payload))
420+ binary.BigEndian.PutUint32(buf[:4], uint32(len(payload)))
421+ binary.BigEndian.PutUint32(buf[4:8], crc32.ChecksumIEEE(payload))
422+ copy(buf[8:], payload)
423+ return buf
424+}
425+func rawBatchPayload(changes []BatchChange) []byte {
426+ payload := []byte{opBatch, 0, 0, 0, byte(len(changes))}
427+ for _, change := range changes {
428+ op := opSet
429+ if change.Delete {
430+ op = opDel
431+ }
432+ payload = append(payload, op, 0, byte(len(change.Key)))
433+ payload = append(payload, change.Key...)
434+ payload = binary.BigEndian.AppendUint32(payload, uint32(len(change.Value)))
435+ payload = append(payload, change.Value...)
436+ }
437+ return payload
438+}
439+
240440 func TestReplayStopsAtBadPayloadLengthWithoutAllocatingHugeBuffer(t *testing.T) {
241441 path := privatePath(t)
242442 first, _ := encodeRecord(opSet, []byte("a"), []byte("one"))
@@ -3,6 +3,8 @@ package kv
3 import (3 import (
4 "bytes"4 "bytes"
5 "encoding/binary"5 "encoding/binary"
6+ "errors"
7+ "hash/crc32"
6 "os"8 "os"
7 "path/filepath"9 "path/filepath"
8 "testing"10 "testing"
@@ -149,7 +151,7 @@ func TestOpenTruncatesMidRecordTail(t *testing.T) {
149 }151 }
150 }152 }
151 153
152-func TestCRCcorruptionOfLastRecordDropsIt(t *testing.T) {154+func TestCRCcorruptionOfLastRecordFailsClosed(t *testing.T) {
153 path := privatePath(t)155 path := privatePath(t)
154 first, _ := encodeRecord(opSet, []byte("k"), []byte("old"))156 first, _ := encodeRecord(opSet, []byte("k"), []byte("old"))
155 second, _ := encodeRecord(opSet, []byte("k"), []byte("new"))157 second, _ := encodeRecord(opSet, []byte("k"), []byte("new"))
@@ -162,21 +164,16 @@ func TestCRCcorruptionOfLastRecordDropsIt(t *testing.T) {
162 t.Fatal(err)164 t.Fatal(err)
163 }165 }
164 166
165- s, err := Open(path)167+ _, err := Open(path)
166- if err != nil {168+ if !errors.Is(err, ErrCorruptWAL) {
167- t.Fatal(err)169+ t.Fatalf("Open error = %v, want ErrCorruptWAL", err)
168- }
169- defer closeStore(t, s)
170- got, ok := s.Get([]byte("k"))
171- if !ok || string(got) != "old" {
172- t.Fatalf("corrupt last record not dropped: %q %v", got, ok)
173 }170 }
174- info, err := os.Stat(path)171+ info, statErr := os.Stat(path)
175- if err != nil {172+ if statErr != nil {
176- t.Fatal(err)173+ t.Fatal(statErr)
177 }174 }
178- if info.Size() != int64(len(first)) {175+ if info.Size() != int64(len(buf)) {
179- t.Fatalf("corrupt tail not truncated: size=%d want=%d", info.Size(), len(first))176+ t.Fatalf("corrupt WAL was mutated: size=%d want=%d", info.Size(), len(buf))
180 }177 }
181 }178 }
182 179
@@ -237,6 +234,209 @@ func TestTombstoneSurvivesCompaction(t *testing.T) {
237 }234 }
238 }235 }
239 236
237+func TestReplayStrict(t *testing.T) {
238+ valid, err := encodeRecord(opSet, []byte("old"), []byte("value"))
239+ if err != nil {
240+ t.Fatal(err)
241+ }
242+ batch, err := encodeBatch([]BatchChange{{Key: []byte("old"), Delete: true}, {Key: []byte("new"), Value: []byte("value")}})
243+ if err != nil {
244+ t.Fatal(err)
245+ }
246+
247+ tests := []struct {
248+ name string
249+ wal []byte
250+ torn bool
251+ corrupt bool
252+ }{
253+ {name: "torn-final-header", wal: append(append([]byte{}, valid...), batch[:headerLen-1]...), torn: true},
254+ {name: "torn-final-payload", wal: append(append([]byte{}, valid...), batch[:len(batch)-1]...), torn: true},
255+ {name: "final-crc", wal: corruptRecord(append(append([]byte{}, valid...), batch...), len(valid)+4), corrupt: true},
256+ {name: "middle-crc", wal: corruptRecord(append(append([]byte{}, valid...), batch...), 4), corrupt: true},
257+ {name: "invalid-op", wal: rawRecord([]byte{0xff, 0, 0}), corrupt: true},
258+ {name: "duplicate-key-in-batch", wal: rawRecord(rawBatchPayload([]BatchChange{{Key: []byte("x")}, {Key: []byte("x"), Delete: true}})), corrupt: true},
259+ {name: "trailing-batch-garbage", wal: rawRecord(append(rawBatchPayload([]BatchChange{{Key: []byte("x")}}), 0)), corrupt: true},
260+ }
261+ for _, tt := range tests {
262+ t.Run(tt.name, func(t *testing.T) {
263+ path := privatePath(t)
264+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
265+ t.Fatal(err)
266+ }
267+ if err := os.WriteFile(path, tt.wal, 0o600); err != nil {
268+ t.Fatal(err)
269+ }
270+ f, err := os.Open(path)
271+ if err != nil {
272+ t.Fatal(err)
273+ }
274+ result, replayErr := replayFile(f)
275+ _ = f.Close()
276+ if tt.corrupt {
277+ if !errors.Is(replayErr, ErrCorruptWAL) {
278+ t.Fatalf("replay error = %v, want ErrCorruptWAL", replayErr)
279+ }
280+ return
281+ }
282+ if replayErr != nil {
283+ t.Fatal(replayErr)
284+ }
285+ if string(result.Data["old"]) != "value" || result.Data["new"] != nil {
286+ t.Fatalf("torn batch was partially applied: %#v", result.Data)
287+ }
288+ if !result.TornTail || result.LastGood != int64(len(valid)) {
289+ t.Fatalf("result = %+v", result)
290+ }
291+ })
292+ }
293+}
294+
295+func TestBatchAtomicity(t *testing.T) {
296+ path := privatePath(t)
297+ s, err := Open(path)
298+ if err != nil {
299+ t.Fatal(err)
300+ }
301+ defer closeStore(t, s)
302+ if err := s.Set([]byte("old"), []byte("before")); err != nil {
303+ t.Fatal(err)
304+ }
305+ if err := s.Batch(nil); err == nil {
306+ t.Fatal("empty batch succeeded")
307+ }
308+ if err := s.Batch([]BatchChange{{Key: []byte("duplicate")}, {Key: []byte("duplicate"), Delete: true}}); err == nil {
309+ t.Fatal("duplicate-key batch succeeded")
310+ }
311+ if err := s.Batch([]BatchChange{{Key: []byte("old"), Delete: true}, {Key: []byte("new"), Value: []byte("same-incarnation")}}); err != nil {
312+ t.Fatal(err)
313+ }
314+ if err := s.Sync(); err != nil {
315+ t.Fatal(err)
316+ }
317+ if _, ok := s.Get([]byte("old")); ok {
318+ t.Fatal("old key remains")
319+ }
320+ got, ok := s.Get([]byte("new"))
321+ if !ok || !bytes.Equal(got, []byte("same-incarnation")) {
322+ t.Fatalf("new = %q, %v", got, ok)
323+ }
324+}
325+
326+func TestCompactionRecoveryMatrix(t *testing.T) {
327+ oldRecord, _ := encodeRecord(opSet, []byte("version"), []byte("old"))
328+ newRecord, _ := encodeRecord(opSet, []byte("version"), []byte("new"))
329+ tests := []struct {
330+ name string
331+ current, next, prev []byte
332+ want string
333+ }{
334+ {name: "current-alone", current: oldRecord, want: "old"},
335+ {name: "current-next", current: oldRecord, next: newRecord, want: "old"},
336+ {name: "prev-next", prev: oldRecord, next: newRecord, want: "new"},
337+ {name: "prev-invalid-next", prev: oldRecord, next: rawRecord([]byte{0xff, 0, 0}), want: "old"},
338+ {name: "prev-torn-next", prev: oldRecord, next: newRecord[:len(newRecord)-1], want: "old"},
339+ {name: "current-prev", current: newRecord, prev: oldRecord, want: "new"},
340+ {name: "invalid-current-prev", current: rawRecord([]byte{0xff, 0, 0}), prev: oldRecord, want: "old"},
341+ {name: "prev-alone", prev: oldRecord, want: "old"},
342+ }
343+ for _, tt := range tests {
344+ t.Run(tt.name, func(t *testing.T) {
345+ path := privatePath(t)
346+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
347+ t.Fatal(err)
348+ }
349+ for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} {
350+ if data != nil {
351+ if err := os.WriteFile(path+suffix, data, 0o600); err != nil {
352+ t.Fatal(err)
353+ }
354+ }
355+ }
356+ if err := recoverCompaction(path, defaultFSHooks()); err != nil {
357+ t.Fatal(err)
358+ }
359+ got, err := Replay(path)
360+ if err != nil {
361+ t.Fatal(err)
362+ }
363+ if string(got["version"]) != tt.want {
364+ t.Fatalf("version = %q, want %q", got["version"], tt.want)
365+ }
366+ })
367+ }
368+
369+ faultPoints := []string{
370+ "before-write-next", "after-write-next", "before-sync-next", "after-sync-next",
371+ "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev",
372+ "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current",
373+ "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir",
374+ }
375+ for _, point := range faultPoints {
376+ t.Run("fault-"+point, func(t *testing.T) {
377+ path := privatePath(t)
378+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
379+ t.Fatal(err)
380+ }
381+ if err := os.WriteFile(path, oldRecord, 0o600); err != nil {
382+ t.Fatal(err)
383+ }
384+ s, err := Open(path)
385+ if err != nil {
386+ t.Fatal(err)
387+ }
388+ s.data = map[string][]byte{"version": []byte("new")}
389+ s.hooks.Fault = func(got string) error {
390+ if got == point {
391+ return errors.New("injected " + point)
392+ }
393+ return nil
394+ }
395+ if err := s.compactLocked(); err == nil {
396+ t.Fatalf("fault %q did not interrupt compaction", point)
397+ }
398+ if s.file != nil {
399+ _ = s.file.Close()
400+ }
401+ _ = releaseLock(s.lockFile)
402+ if err := recoverCompaction(path, defaultFSHooks()); err != nil {
403+ t.Fatalf("recovery: %v", err)
404+ }
405+ got, err := Replay(path)
406+ if err != nil {
407+ t.Fatal(err)
408+ }
409+ version := string(got["version"])
410+ if (version != "old" && version != "new") || len(got) != 1 {
411+ t.Fatalf("mixed or empty recovered map: %#v", got)
412+ }
413+ })
414+ }
415+}
416+
417+func corruptRecord(buf []byte, offset int) []byte { buf[offset] ^= 0xff; return buf }
418+func rawRecord(payload []byte) []byte {
419+ buf := make([]byte, headerLen+len(payload))
420+ binary.BigEndian.PutUint32(buf[:4], uint32(len(payload)))
421+ binary.BigEndian.PutUint32(buf[4:8], crc32.ChecksumIEEE(payload))
422+ copy(buf[8:], payload)
423+ return buf
424+}
425+func rawBatchPayload(changes []BatchChange) []byte {
426+ payload := []byte{opBatch, 0, 0, 0, byte(len(changes))}
427+ for _, change := range changes {
428+ op := opSet
429+ if change.Delete {
430+ op = opDel
431+ }
432+ payload = append(payload, op, 0, byte(len(change.Key)))
433+ payload = append(payload, change.Key...)
434+ payload = binary.BigEndian.AppendUint32(payload, uint32(len(change.Value)))
435+ payload = append(payload, change.Value...)
436+ }
437+ return payload
438+}
439+
240 func TestReplayStopsAtBadPayloadLengthWithoutAllocatingHugeBuffer(t *testing.T) {440 func TestReplayStopsAtBadPayloadLengthWithoutAllocatingHugeBuffer(t *testing.T) {
241 path := privatePath(t)441 path := privatePath(t)
242 first, _ := encodeRecord(opSet, []byte("a"), []byte("one"))442 first, _ := encodeRecord(opSet, []byte("a"), []byte("one"))

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 12

Tags 11

↑↓ moveenter openesc close