refactor(snapshot): replace tombstoned deletion with startup garbage collection

commit: 6bbcb8d1eb4d00573a0629ccb8ed2591dffc23ee
author: brice · 2mo ago · 2026-07-26 11:34 UTC
parent: d5c98ee
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

39 files · +376 -4777

D internal/adapters/snapshot/dirent_cookie_darwin.go +0 -25

@@ -1,25 +0,0 @@
1	-	//go:build darwin
2	-
3	-	package snapshot
4	-
5	-	import (
6	-	"io"
7	-	"syscall"
8	-	"unsafe"
9	-	)
10	-
11	-	// syscall.ReadDirent on Darwin implements Getdirentries by storing the number
12	-	// of returned entries in the descriptor seek offset. Dirent.Seekoff is not
13	-	// that cursor. A max-sized record buffer lets each call be drained before the
14	-	// descriptor-maintained count is saved.
15	-	const maintenanceDirentBufferSize = int(unsafe.Sizeof(syscall.Dirent{}))
16	-
17	-	func directoryCookie(file maintenanceDirectory, _ *syscall.Dirent, remaining int) (int64, error) {
18	-	end, err := file.Seek(0, io.SeekCurrent)
19	-	if err != nil {
20	-	return 0, err
21	-	}
22	-	return end - int64(remaining), nil
23	-	}
24	-
25	-	func drainMaintenanceDirentBatch() bool { return true }

D internal/adapters/snapshot/dirent_cookie_darwin_test.go +0 -31

@@ -1,31 +0,0 @@
1	-	//go:build darwin
2	-
3	-	package snapshot
4	-
5	-	import (
6	-	"syscall"
7	-	"testing"
8	-	)
9	-
10	-	func TestDirectoryCookieUsesDescriptorEntryCount(t *testing.T) {
11	-	file := fakeMaintenanceDirectory{seekOffset: 42}
12	-	dirent := syscall.Dirent{Seekoff: 99}
13	-	got, err := directoryCookie(file, &dirent, 0)
14	-	if err != nil {
15	-	t.Fatal(err)
16	-	}
17	-	if got != 42 {
18	-	t.Fatalf("directoryCookie() = %d, want descriptor entry count 42", got)
19	-	}
20	-	}
21	-
22	-	func TestDirectoryCookieSubtractsBufferedEntries(t *testing.T) {
23	-	file := fakeMaintenanceDirectory{seekOffset: 42}
24	-	got, err := directoryCookie(file, &syscall.Dirent{}, 3)
25	-	if err != nil {
26	-	t.Fatal(err)
27	-	}
28	-	if got != 39 {
29	-	t.Fatalf("directoryCookie() = %d, want per-entry cursor 39", got)
30	-	}
31	-	}

D internal/adapters/snapshot/dirent_cookie_linux.go +0 -14

@@ -1,14 +0,0 @@
1	-	//go:build linux
2	-
3	-	package snapshot
4	-
5	-	import "syscall"
6	-
7	-	const maintenanceDirentBufferSize = 8192
8	-
9	-	// directoryCookie is d_off, the resumable getdents(2) cookie for Linux.
10	-	func directoryCookie(_ maintenanceDirectory, dirent *syscall.Dirent, _ int) (int64, error) {
11	-	return dirent.Off, nil
12	-	}
13	-
14	-	func drainMaintenanceDirentBatch() bool { return false }

D internal/adapters/snapshot/dirent_cookie_linux_test.go +0 -19

@@ -1,19 +0,0 @@
1	-	//go:build linux
2	-
3	-	package snapshot
4	-
5	-	import (
6	-	"syscall"
7	-	"testing"
8	-	)
9	-
10	-	func TestDirectoryCookieUsesLinuxOffset(t *testing.T) {
11	-	dirent := syscall.Dirent{Off: 42}
12	-	got, err := directoryCookie(fakeMaintenanceDirectory{}, &dirent, 0)
13	-	if err != nil {
14	-	t.Fatal(err)
15	-	}
16	-	if got != 42 {
17	-	t.Fatalf("directoryCookie() = %d, want 42", got)
18	-	}
19	-	}

D internal/adapters/snapshot/load_bounds_test.go +0 -52

@@ -1,52 +0,0 @@
1	-	package snapshot
2	-
3	-	import (
4	-	"context"
5	-	"fmt"
6	-	"os"
7	-	"testing"
8	-	)
9	-
10	-	func TestRepositoryMaintainDoesNotCollectBeyondRetainedMetadataBudget(t *testing.T) {
11	-	repo := NewRepository(privateDir(t))
12	-	for i := range maxMaintenanceMarkedGenerations + 1 {
13	-	generation := uint64(i + 1)
14	-	if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))); err != nil {
15	-	t.Fatal(err)
16	-	}
17	-	}
18	-	for pass := range maxMaintenanceMarkedGenerations * 3 {
19	-	if err := repo.Maintain(context.Background()); err != nil {
20	-	t.Fatal(err)
21	-	}
22	-	if repo.maintenanceSessions[legacyIncarnationID("named").String()] == nil {
23	-	break
24	-	}
25	-	if pass == maxMaintenanceMarkedGenerations*3-1 {
26	-	t.Fatal("maintenance did not complete after bounded object-shard traversal")
27	-	}
28	-	}
29	-	if _, err := os.Lstat(repo.legacyManifestPath(legacyIncarnationID("named").String(), 1)); err != nil {
30	-	t.Fatalf("old manifest collected after mark budget overflow: %v", err)
31	-	}
32	-	if state := repo.maintenanceSessions[legacyIncarnationID("named").String()]; state != nil {
33	-	t.Fatalf("overflow maintenance state retained = %#v, want reset", state)
34	-	}
35	-	}
36	-
37	-	func TestMaintenanceMarkStateIsCapped(t *testing.T) {
38	-	references := &sessionMaintenance{marked: make(map[uint64]manifestMaintenance)}
39	-	if !references.canRetainManifest(maxMaintenanceReferences) {
40	-	t.Fatal("state rejected references at its documented ceiling")
41	-	}
42	-	if references.canRetainManifest(1) {
43	-	t.Fatal("state accepted references beyond its documented ceiling")
44	-	}
45	-	generations := &sessionMaintenance{marked: make(map[uint64]manifestMaintenance)}
46	-	for generation := uint64(1); generation <= maxMaintenanceMarkedGenerations; generation++ {
47	-	generations.marked[generation] = manifestMaintenance{}
48	-	}
49	-	if generations.canRetainManifest(0) {
50	-	t.Fatal("state accepted generations beyond its documented ceiling")
51	-	}
52	-	}

D internal/adapters/snapshot/maintenance.go +0 -129

@@ -1,129 +0,0 @@
1	-	package snapshot
2	-
3	-	import (
4	-	"context"
5	-	"errors"
6	-	"fmt"
7	-	"os"
8	-	"path/filepath"
9	-	)
10	-
11	-	const maintenanceBatch = 64
12	-
13	-	// These caps bound metadata retained across resumable GC passes. If a session
14	-	// exceeds either cap, marking remains conservative and no destructive sweep or
15	-	// generation collection occurs in that cycle.
16	-	const (
17	-	maxMaintenanceMarkedGenerations = 128
18	-	maxMaintenanceReferences = 8192
19	-	)
20	-
21	-	// Maintain reaps a bounded amount of stale state. Continuation handles are
22	-	// repository-owned, so a stable prefix cannot starve entries later in a large
23	-	// directory. Cancellation and errors discard those handles.
24	-	func (r *Repository) Maintain(ctx context.Context) (err error) {
25	-	r.maintenanceMu.Lock()
26	-	defer r.maintenanceMu.Unlock()
27	-	if err := ctx.Err(); err != nil {
28	-	r.resetMaintenance()
29	-	return err
30	-	}
31	-	defer func() {
32	-	if err != nil {
33	-	r.resetMaintenance()
34	-	}
35	-	}()
36	-	if r.maintenanceCursors == nil {
37	-	r.maintenanceCursors = make(map[string]*maintenanceCursor)
38	-	r.maintenanceSessions = make(map[string]*sessionMaintenance)
39	-	}
40	-
41	-	budget := maintenanceBatch
42	-	if _, err := r.removeTemps(ctx, r.dir, &budget, "root-temps"); err != nil \|\| budget == 0 {
43	-	return err
44	-	}
45	-	if r.maintenanceQuarantine != nil {
46	-	changed, done, err := r.maintainQuarantine(ctx, &budget)
47	-	if err != nil {
48	-	return err
49	-	}
50	-	if changed {
51	-	if err := r.syncDirectory(filepath.Join(r.dir, repositorySessionsDir)); err != nil {
52	-	return fmt.Errorf("sync snapshot maintenance directory: %w", safeFilesystemError(err))
53	-	}
54	-	}
55	-	if done {
56	-	r.maintenanceQuarantine = nil
57	-	}
58	-	return nil
59	-	}
60	-	// Resume an incomplete session before discovering another one. This keeps
61	-	// retained marks (and their session lock/epoch references) bounded even
62	-	// when a hostile repository contains arbitrarily many partial sessions.
63	-	for key := range r.maintenanceSessions {
64	-	lock := r.lockSession(key)
65	-	err = r.maintainSession(ctx, key, &budget)
66	-	r.unlockSession(lock)
67	-	return err
68	-	}
69	-	sessions := filepath.Join(r.dir, repositorySessionsDir)
70	-	entries, _, err := r.readMaintenanceDir(sessions, maintenanceBatch, "sessions")
71	-	if errors.Is(err, os.ErrNotExist) {
72	-	return nil
73	-	}
74	-	if err != nil {
75	-	return fmt.Errorf("read snapshot maintenance sessions: %w", safeFilesystemError(err))
76	-	}
77	-	for i, entry := range entries {
78	-	if budget == 0 {
79	-	r.requeueMaintenanceEntries(sessions, "sessions", entries[i:])
80	-	return nil
81	-	}
82	-	if err := ctx.Err(); err != nil {
83	-	return err
84	-	}
85	-	path := filepath.Join(sessions, entry.name)
86	-	if isQuarantine(entry.name) {
87	-	r.maintenanceQuarantine = &quarantineMaintenance{root: path, current: path}
88	-	changed, done, err := r.maintainQuarantine(ctx, &budget)
89	-	if err != nil && !errors.Is(err, os.ErrNotExist) {
90	-	return fmt.Errorf("remove snapshot quarantine %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err))
91	-	}
92	-	if changed {
93	-	if err := ctx.Err(); err != nil {
94	-	return err
95	-	}
96	-	if err := r.syncDirectory(sessions); err != nil {
97	-	return fmt.Errorf("sync snapshot maintenance directory for %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err))
98	-	}
99	-	}
100	-	if done {
101	-	r.maintenanceQuarantine = nil
102	-	}
103	-	if i+1 < len(entries) {
104	-	r.requeueMaintenanceEntries(sessions, "sessions", entries[i+1:])
105	-	}
106	-	return nil
107	-	}
108	-	if !entry.isDir \|\| !canonicalSessionKey(entry.name) {
109	-	continue
110	-	}
111	-	lock := r.lockSession(entry.name)
112	-	err = r.maintainSession(ctx, entry.name, &budget)
113	-	r.unlockSession(lock)
114	-	if err != nil {
115	-	return err
116	-	}
117	-	if r.maintenanceSessions[entry.name] != nil {
118	-	return nil
119	-	}
120	-	}
121	-	return nil
122	-	}
123	-
124	-	// maintenanceCursor is deliberately reopenable metadata, not an open file.
125	-	// On Linux, d_off is the seek cookie returned by getdents(2); seeking a newly
126	-	// opened descriptor to it resumes the directory stream. Go's File.ReadDir
127	-	// cannot be used here: its 8 KiB internal getdents buffer advances the kernel
128	-	// offset beyond a short ReadDir result (see os/dir_unix.go), so saving SeekCur
129	-	// after ReadDir would skip buffered entries.

D internal/adapters/snapshot/maintenance_cursor.go +0 -307

@@ -1,307 +0,0 @@
1	-	package snapshot
2	-
3	-	import (
4	-	"context"
5	-	"errors"
6	-	"fmt"
7	-	"io"
8	-	"os"
9	-	"path/filepath"
10	-	"strings"
11	-	"syscall"
12	-	"unsafe"
13	-	)
14	-
15	-	// maxMaintenanceCursors bounds all retained directory seek cookies, including
16	-	// cursors for attacker-controlled object shards. Maintenance processes only
17	-	// one shard at a time, so normal operation stays well below this ceiling.
18	-	const maxMaintenanceCursors = 8
19	-
20	-	var errMaintenanceCursorLimit = errors.New("snapshot maintenance cursor limit")
21	-
22	-	type maintenanceCursor struct {
23	-	offset int64
24	-	pending []maintenanceDirEntry
25	-	done bool
26	-	}
27	-
28	-	type maintenanceDirEntry struct {
29	-	name string
30	-	isDir bool
31	-	cookie int64
32	-	}
33	-
34	-	type maintenanceDirentConfig struct {
35	-	drainBuffer bool
36	-	cookie func(maintenanceDirectory, *syscall.Dirent, int) (int64, error)
37	-	}
38	-
39	-	func (r *Repository) resetMaintenance() {
40	-	for key := range r.maintenanceSessions {
41	-	r.clearSessionMaintenance(key)
42	-	}
43	-	r.maintenanceCursors = nil
44	-	r.maintenanceSessions = nil
45	-	r.maintenanceQuarantine = nil
46	-	}
47	-
48	-	func maintenanceCursorID(dir, purpose string) string { return purpose + "\x00" + dir }
49	-
50	-	// readMaintenanceDir advances one named directory cursor. Its continuation is
51	-	// only a Linux seek cookie and at most one call's worth of pending names. The
52	-	// descriptor used to read a call is always closed before this method returns.
53	-	// Callers must return entries they could not process with
54	-	// requeueMaintenanceEntries before yielding for budget exhaustion.
55	-	func (r *Repository) readMaintenanceDir(dir string, n int, purpose string) ([]maintenanceDirEntry, bool, error) {
56	-	id := maintenanceCursorID(dir, purpose)
57	-	cursor := r.maintenanceCursors[id]
58	-	if cursor == nil {
59	-	if len(r.maintenanceCursors) >= maxMaintenanceCursors {
60	-	return nil, false, errMaintenanceCursorLimit
61	-	}
62	-	cursor = &maintenanceCursor{}
63	-	r.maintenanceCursors[id] = cursor
64	-	}
65	-
66	-	entries, done, err := r.readMaintenanceDirent(dir, n, cursor)
67	-	if err != nil {
68	-	delete(r.maintenanceCursors, id)
69	-	return nil, false, err
70	-	}
71	-	if done {
72	-	delete(r.maintenanceCursors, id)
73	-	}
74	-	return entries, done, nil
75	-	}
76	-
77	-	// maintenanceDirectory is the small syscall surface needed to resume a
78	-	// bounded getdents cursor. It permits deterministic fault injection without
79	-	// retaining directory descriptors between maintenance passes.
80	-	type maintenanceDirectory interface {
81	-	Seek(int64, int) (int64, error)
82	-	ReadDirent([]byte) (int, error)
83	-	Close() error
84	-	}
85	-
86	-	type osMaintenanceDirectory struct{ file *os.File }
87	-
88	-	func (d osMaintenanceDirectory) Seek(offset int64, whence int) (int64, error) {
89	-	return syscall.Seek(int(d.file.Fd()), offset, whence)
90	-	}
91	-
92	-	func (d osMaintenanceDirectory) ReadDirent(buffer []byte) (int, error) {
93	-	return syscall.ReadDirent(int(d.file.Fd()), buffer)
94	-	}
95	-
96	-	func (d osMaintenanceDirectory) Close() error { return d.file.Close() }
97	-
98	-	func (r *Repository) openMaintenanceDirectory(dir string) (maintenanceDirectory, error) {
99	-	if open := r.hooks.openMaintenanceDirectory; open != nil {
100	-	return open(dir)
101	-	}
102	-	file, err := r.openDirectory(dir)
103	-	if err != nil {
104	-	return nil, err
105	-	}
106	-	return osMaintenanceDirectory{file: file}, nil
107	-	}
108	-
109	-	// maintenanceDirectoryError adds stable operation context and removes any
110	-	// filesystem path while retaining the underlying cause for errors.Is.
111	-	func maintenanceDirectoryError(operation string, err error) error {
112	-	return fmt.Errorf("%s: %w", operation, safeFilesystemError(err))
113	-	}
114	-
115	-	// readMaintenanceDirent saves a resumable directory cursor. Linux d_off lets
116	-	// a large getdents batch stop at limit. Darwin's syscall.ReadDirent instead
117	-	// stores an entry count in the descriptor offset, so every returned buffer
118	-	// must be drained before that descriptor-maintained count is saved.
119	-	func (r Repository) readMaintenanceDirent(dir string, limit int, cursor maintenanceCursor) (entries []maintenanceDirEntry, done bool, err error) {
120	-	if len(cursor.pending) != 0 {
121	-	count := min(limit, len(cursor.pending))
122	-	entries = cursor.pending[:count]
123	-	cursor.pending = cursor.pending[count:]
124	-	if len(entries) != 0 {
125	-	cursor.offset = entries[len(entries)-1].cookie
126	-	}
127	-	return entries, cursor.done && len(cursor.pending) == 0, nil
128	-	}
129	-	config := maintenanceDirentConfig{drainBuffer: drainMaintenanceDirentBatch(), cookie: directoryCookie}
130	-	if r.hooks.maintenanceDirentConfig != nil {
131	-	config = *r.hooks.maintenanceDirentConfig
132	-	}
133	-	return r.readMaintenanceDirentWithDrain(dir, limit, cursor, config.drainBuffer, config.cookie)
134	-	}
135	-
136	-	// readMaintenanceDirentWithDrain contains the shared traversal logic. The
137	-	// explicit parameters permit platform-independent verification of Darwin's
138	-	// buffer-draining semantics with deterministic fake directories.
139	-	func (r Repository) readMaintenanceDirentWithDrain(dir string, limit int, cursor maintenanceCursor, drainBuffer bool, cookie func(maintenanceDirectory, *syscall.Dirent, int) (int64, error)) (entries []maintenanceDirEntry, done bool, err error) {
140	-	file, err := r.openMaintenanceDirectory(dir)
141	-	if err != nil {
142	-	return nil, false, maintenanceDirectoryError("open maintenance directory", err)
143	-	}
144	-	defer func() {
145	-	if closeErr := file.Close(); closeErr != nil {
146	-	closeErr = maintenanceDirectoryError("close maintenance directory", closeErr)
147	-	if err != nil {
148	-	err = errors.Join(err, closeErr)
149	-	} else {
150	-	entries = nil
151	-	done = false
152	-	err = closeErr
153	-	}
154	-	}
155	-	}()
156	-	if cursor.offset != 0 {
157	-	if _, err := file.Seek(cursor.offset, io.SeekStart); err != nil {
158	-	return nil, false, maintenanceDirectoryError("seek maintenance directory", err)
159	-	}
160	-	}
161	-
162	-	entries = make([]maintenanceDirEntry, 0, limit)
163	-	buffer := make([]byte, maintenanceDirentBufferSize)
164	-	for {
165	-	count, err := file.ReadDirent(buffer)
166	-	if err != nil {
167	-	return nil, false, maintenanceDirectoryError("read maintenance directory", err)
168	-	}
169	-	if count < 0 \|\| count > len(buffer) {
170	-	return nil, false, syscall.EIO
171	-	}
172	-	if count == 0 {
173	-	return entries, true, nil
174	-	}
175	-	remainingRecords, err := countMaintenanceDirents(buffer[:count])
176	-	if err != nil {
177	-	return nil, false, err
178	-	}
179	-	for data := buffer[:count]; len(data) != 0 && (len(entries) < limit \|\| drainBuffer); {
180	-	record, nameBytes, reclen, decodeErr := decodeMaintenanceDirent(data)
181	-	if decodeErr != nil {
182	-	return nil, false, decodeErr
183	-	}
184	-	data = data[reclen:]
185	-	remainingRecords--
186	-	// Advance past every raw record, including dot and disappeared entries.
187	-	offset, cookieErr := cookie(file, record, remainingRecords)
188	-	if cookieErr != nil {
189	-	return nil, false, maintenanceDirectoryError("tell maintenance directory", cookieErr)
190	-	}
191	-	cursor.offset = offset
192	-	if end := strings.IndexByte(string(nameBytes), 0); end >= 0 {
193	-	nameBytes = nameBytes[:end]
194	-	}
195	-	name := string(nameBytes)
196	-	if name == "." \|\| name == ".." \|\| name == "" {
197	-	continue
198	-	}
199	-	stat, err := r.stat(filepath.Join(dir, name))
200	-	if errors.Is(err, os.ErrNotExist) {
201	-	continue
202	-	}
203	-	if err != nil {
204	-	return nil, false, maintenanceDirectoryError("stat maintenance directory entry", err)
205	-	}
206	-	entries = append(entries, maintenanceDirEntry{name: name, isDir: stat.IsDir(), cookie: offset})
207	-	}
208	-	if len(entries) >= limit {
209	-	if drainBuffer && len(entries) > limit {
210	-	// The descriptor is past the whole buffer. Retain the excess and
211	-	// expose only the last entry actually consumed as the logical cursor.
212	-	cursor.pending = append(cursor.pending, entries[limit:]...)
213	-	cursor.offset = entries[limit-1].cookie
214	-	}
215	-	return entries[:limit], false, nil
216	-	}
217	-	}
218	-	}
219	-
220	-	func countMaintenanceDirents(data []byte) (int, error) {
221	-	count := 0
222	-	for len(data) != 0 {
223	-	_, _, reclen, err := decodeMaintenanceDirent(data)
224	-	if err != nil {
225	-	return 0, err
226	-	}
227	-	data = data[reclen:]
228	-	count++
229	-	}
230	-	return count, nil
231	-	}
232	-
233	-	// decodeMaintenanceDirent copies a bounded raw record into aligned storage
234	-	// before reading its fields. syscall.ReadDirent writes to a byte buffer, whose
235	-	// start and record boundaries are not guaranteed to satisfy syscall.Dirent's
236	-	// alignment requirements on Darwin.
237	-	func decodeMaintenanceDirent(data []byte) (*syscall.Dirent, []byte, int, error) {
238	-	nameOffset := int(unsafe.Offsetof(syscall.Dirent{}.Name))
239	-	if len(data) < nameOffset {
240	-	return nil, nil, 0, syscall.EIO
241	-	}
242	-
243	-	var record syscall.Dirent
244	-	header := unsafe.Slice((*byte)(unsafe.Pointer(&record)), nameOffset)
245	-	copy(header, data[:nameOffset])
246	-	reclen := int(record.Reclen)
247	-	if reclen < nameOffset \|\| reclen > len(data) \|\| reclen > int(unsafe.Sizeof(record)) {
248	-	return nil, nil, 0, syscall.EIO
249	-	}
250	-
251	-	recordBytes := unsafe.Slice((*byte)(unsafe.Pointer(&record)), reclen)
252	-	copy(recordBytes, data[:reclen])
253	-	name := unsafe.Slice((*byte)(unsafe.Pointer(&record.Name[0])), reclen-nameOffset)
254	-	return &record, name, reclen, nil
255	-	}
256	-
257	-	func (r *Repository) requeueMaintenanceEntries(dir, purpose string, entries []maintenanceDirEntry) {
258	-	if len(entries) == 0 {
259	-	return
260	-	}
261	-	id := maintenanceCursorID(dir, purpose)
262	-	cursor := r.maintenanceCursors[id]
263	-	if cursor == nil {
264	-	cursor = &maintenanceCursor{done: true}
265	-	r.maintenanceCursors[id] = cursor
266	-	}
267	-	cursor.pending = append(entries, cursor.pending...)
268	-	}
269	-
270	-	func (r Repository) removeTemps(ctx context.Context, dir string, budget int, purpose string) (bool, error) {
271	-	if *budget == 0 {
272	-	return false, nil
273	-	}
274	-	entries, done, err := r.readMaintenanceDir(dir, maintenanceBatch, purpose)
275	-	if errors.Is(err, os.ErrNotExist) {
276	-	return true, nil
277	-	}
278	-	if err != nil {
279	-	return false, fmt.Errorf("read snapshot maintenance directory: %w", safeFilesystemError(err))
280	-	}
281	-	for i, entry := range entries {
282	-	if *budget == 0 {
283	-	r.requeueMaintenanceEntries(dir, purpose, entries[i:])
284	-	return false, nil
285	-	}
286	-	if err := ctx.Err(); err != nil {
287	-	return false, err
288	-	}
289	-	if entry.isDir \|\| !strings.HasPrefix(entry.name, ".tmp-") {
290	-	continue
291	-	}
292	-	path := filepath.Join(dir, entry.name)
293	-	if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
294	-	return false, fmt.Errorf("remove snapshot maintenance file %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err))
295	-	}
296	-	if err := ctx.Err(); err != nil {
297	-	return false, err
298	-	}
299	-	if err := r.syncDirectory(dir); err != nil {
300	-	return false, fmt.Errorf("sync snapshot maintenance directory for %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err))
301	-	}
302	-	*budget--
303	-	}
304	-	return done, nil
305	-	}
306	-
307	-	// maintenancePath avoids exposing the repository root while retaining a

M internal/adapters/snapshot/maintenance_delete.go +2 -539

@@ -1,554 +1,17 @@
1	1		package snapshot
2	2
3	3		import (
4		-	"bytes"
5	4		"context"
6		-	"encoding/base64"
7		-	"encoding/binary"
8		-	"encoding/json"
9	5		"errors"
10	6		"fmt"
11		-	"io"
12	7		"os"
13	8		"path/filepath"
14		-	"sort"
15		-	"strings"
16	9
17	10		"github.com/bnema/vev/internal/domain"
18		-	"github.com/bnema/vev/internal/ports"
19	11		)
20	12
21		-	const (
22		-	deletionTombstoneVersion = uint16(1)
23		-	deletionTombstoneMagic = "VEVD"
24		-	deletionTombstonesDir = "deletions"
25		-	)
26		-
27		-	var ErrMaintenanceBudgetTooSmall = errors.New("snapshot: maintenance budget too small")
28		-
29		-	func encodeDeletionTombstone(tombstone domain.DeletionTombstone) ([]byte, error) {
30		-	if err := tombstone.Validate(); err != nil {
31		-	return nil, err
32		-	}
33		-	if len(tombstone.Name) > 200 {
34		-	return nil, fmt.Errorf("snapshot: deletion tombstone name too long")
35		-	}
36		-	encoded := make([]byte, 4+2+2+len(tombstone.Name)+len(tombstone.IncarnationID))
37		-	copy(encoded, deletionTombstoneMagic)
38		-	binary.BigEndian.PutUint16(encoded[4:6], deletionTombstoneVersion)
39		-	binary.BigEndian.PutUint16(encoded[6:8], uint16(len(tombstone.Name)))
40		-	copy(encoded[8:], tombstone.Name)
41		-	copy(encoded[8+len(tombstone.Name):], tombstone.IncarnationID[:])
42		-	return encoded, nil
43		-	}
44		-
45		-	func decodeDeletionTombstone(encoded []byte) (domain.DeletionTombstone, error) {
46		-	if len(encoded) < 8 {
47		-	return domain.DeletionTombstone{}, fmt.Errorf("snapshot: truncated deletion tombstone")
48		-	}
49		-	if string(encoded[:4]) != deletionTombstoneMagic \|\| binary.BigEndian.Uint16(encoded[4:6]) != deletionTombstoneVersion {
50		-	return domain.DeletionTombstone{}, fmt.Errorf("snapshot: invalid deletion tombstone header")
51		-	}
52		-	nameLength := int(binary.BigEndian.Uint16(encoded[6:8]))
53		-	want := 8 + nameLength + len(domain.IncarnationID{})
54		-	if len(encoded) != want {
55		-	return domain.DeletionTombstone{}, fmt.Errorf("snapshot: malformed deletion tombstone length")
56		-	}
57		-	tombstone := domain.DeletionTombstone{Name: string(encoded[8 : 8+nameLength])}
58		-	copy(tombstone.IncarnationID[:], encoded[8+nameLength:])
59		-	if err := tombstone.Validate(); err != nil {
60		-	return domain.DeletionTombstone{}, err
61		-	}
62		-	return tombstone, nil
63		-	}
64		-
65		-	func (r *Repository) deletionTombstonePath(id domain.IncarnationID) string {
66		-	return filepath.Join(r.dir, deletionTombstonesDir, id.String()+".tombstone")
67		-	}
68		-
69		-	func (r *Repository) WriteDeletionTombstone(ctx context.Context, tombstone domain.DeletionTombstone) error {
70		-	if err := ctx.Err(); err != nil {
71		-	return err
72		-	}
73		-	encoded, err := encodeDeletionTombstone(tombstone)
74		-	if err != nil {
75		-	return err
76		-	}
77		-	dir := filepath.Join(r.dir, deletionTombstonesDir)
78		-	if err := r.ensurePrivateDirectory(dir); err != nil {
79		-	return err
80		-	}
81		-	path := r.deletionTombstonePath(tombstone.IncarnationID)
82		-	return r.writeImmutable(path, encoded, func(existing []byte) error {
83		-	decoded, err := decodeDeletionTombstone(existing)
84		-	if err != nil \|\| decoded != tombstone {
85		-	return fmt.Errorf("snapshot: deletion tombstone identity mismatch")
86		-	}
87		-	return nil
88		-	})
89		-	}
90		-
91		-	const (
92		-	deletionListingCursorPrefix = "dt1:"
93		-	maxDeletionListingCursor = 64 << 10
94		-	)
95		-
96		-	type deletionListingState struct {
97		-	After string `json:"after,omitempty"`
98		-	Offset int64 `json:"offset,omitempty"`
99		-	Pending []maintenanceDirEntry `json:"pending,omitempty"`
100		-	PendingAt int `json:"pending_at,omitempty"`
101		-	ScanDone bool `json:"scan_done,omitempty"`
102		-	Candidates []string `json:"candidates,omitempty"`
103		-	More bool `json:"more,omitempty"`
104		-	BatchLimit int `json:"batch_limit,omitempty"`
105		-	}
106		-
107		-	// MarshalJSON keeps maintenanceDirEntry private while making a continuation
108		-	// self-contained. Pending entries are bounded by one directory read buffer.
109		-	func (entry maintenanceDirEntry) MarshalJSON() ([]byte, error) {
110		-	return json.Marshal(struct {
111		-	Name string `json:"name"`
112		-	IsDir bool `json:"is_dir,omitempty"`
113		-	}{Name: entry.name, IsDir: entry.isDir})
114		-	}
115		-
116		-	func (entry *maintenanceDirEntry) UnmarshalJSON(encoded []byte) error {
117		-	var wire struct {
118		-	Name string `json:"name"`
119		-	IsDir bool `json:"is_dir,omitempty"`
120		-	}
121		-	decoder := json.NewDecoder(bytes.NewReader(encoded))
122		-	decoder.DisallowUnknownFields()
123		-	if err := decoder.Decode(&wire); err != nil {
124		-	return err
125		-	}
126		-	entry.name, entry.isDir = wire.Name, wire.IsDir
127		-	return nil
128		-	}
129		-
130		-	func canonicalDeletionTombstoneFilename(name string) bool {
131		-	if len(name) != 32+len(".tombstone") \|\| !strings.HasSuffix(name, ".tombstone") {
132		-	return false
133		-	}
134		-	var id domain.IncarnationID
135		-	key := strings.TrimSuffix(name, ".tombstone")
136		-	return id.UnmarshalText([]byte(key)) == nil && id.String()+".tombstone" == name
137		-	}
138		-
139		-	func decodeDeletionListingCursor(cursor ports.DeletionTombstoneCursor) (deletionListingState, error) {
140		-	if cursor.After == "" {
141		-	return deletionListingState{}, nil
142		-	}
143		-	if len(cursor.After) > maxDeletionListingCursor \|\| !strings.HasPrefix(cursor.After, deletionListingCursorPrefix) {
144		-	return deletionListingState{}, fmt.Errorf("snapshot: malformed deletion tombstone continuation")
145		-	}
146		-	encoded, err := base64.RawURLEncoding.DecodeString(strings.TrimPrefix(cursor.After, deletionListingCursorPrefix))
147		-	if err != nil {
148		-	return deletionListingState{}, fmt.Errorf("snapshot: malformed deletion tombstone continuation: %w", err)
149		-	}
150		-	var state deletionListingState
151		-	decoder := json.NewDecoder(bytes.NewReader(encoded))
152		-	decoder.DisallowUnknownFields()
153		-	if err := decoder.Decode(&state); err != nil {
154		-	return deletionListingState{}, fmt.Errorf("snapshot: malformed deletion tombstone continuation: %w", err)
155		-	}
156		-	if decoder.Decode(&struct{}{}) != io.EOF {
157		-	return deletionListingState{}, fmt.Errorf("snapshot: malformed deletion tombstone continuation")
158		-	}
159		-	if err := validateDeletionListingState(state); err != nil {
160		-	return deletionListingState{}, err
161		-	}
162		-	return state, nil
163		-	}
164		-
165		-	func validateDeletionListingState(state deletionListingState) error {
166		-	if state.After != "" && !canonicalDeletionTombstoneFilename(state.After) {
167		-	return fmt.Errorf("snapshot: non-canonical deletion tombstone continuation key")
168		-	}
169		-	if state.Offset < 0 \|\| state.PendingAt < 0 \|\| state.PendingAt > len(state.Pending) \|\| state.BatchLimit < 0 \|\| state.BatchLimit > maintenanceBatch \|\| len(state.Candidates) > state.BatchLimit {
170		-	return fmt.Errorf("snapshot: malformed deletion tombstone continuation")
171		-	}
172		-	if state.After == "" && state.Offset == 0 && len(state.Pending) == 0 && !state.ScanDone && len(state.Candidates) == 0 {
173		-	return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation")
174		-	}
175		-	for i, candidate := range state.Candidates {
176		-	if !canonicalDeletionTombstoneFilename(candidate) \|\| candidate <= state.After \|\| (i > 0 && candidate <= state.Candidates[i-1]) {
177		-	return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation")
178		-	}
179		-	}
180		-	if len(state.Pending) > maintenanceDirentBufferSize {
181		-	return fmt.Errorf("snapshot: malformed deletion tombstone continuation")
182		-	}
183		-	for _, entry := range state.Pending {
184		-	if entry.isDir \|\| !canonicalDeletionTombstoneFilename(entry.name) {
185		-	return fmt.Errorf("snapshot: non-canonical deletion tombstone continuation entry")
186		-	}
187		-	}
188		-	return nil
189		-	}
190		-
191		-	func encodeDeletionListingCursor(state deletionListingState) (ports.DeletionTombstoneCursor, error) {
192		-	if err := validateDeletionListingState(state); err != nil {
193		-	return ports.DeletionTombstoneCursor{}, err
194		-	}
195		-	encoded, err := json.Marshal(state)
196		-	if err != nil {
197		-	return ports.DeletionTombstoneCursor{}, err
198		-	}
199		-	return ports.DeletionTombstoneCursor{After: deletionListingCursorPrefix + base64.RawURLEncoding.EncodeToString(encoded)}, nil
200		-	}
201		-
202		-	func (r Repository) readDeletionListingEntries(dir string, limit int, state deletionListingState) ([]maintenanceDirEntry, bool, error) {
203		-	if state.PendingAt < len(state.Pending) {
204		-	end := min(state.PendingAt+limit, len(state.Pending))
205		-	entries := state.Pending[state.PendingAt:end]
206		-	state.PendingAt = end
207		-	done := state.ScanDone && end == len(state.Pending)
208		-	return entries, done, nil
209		-	}
210		-	cursor := &maintenanceCursor{offset: state.Offset}
211		-	entries, done, err := r.readMaintenanceDirent(dir, limit, cursor)
212		-	state.Offset = cursor.offset
213		-	state.Pending = cursor.pending
214		-	state.PendingAt = 0
215		-	state.ScanDone = done \|\| cursor.done
216		-	return entries, done && len(cursor.pending) == 0, err
217		-	}
218		-
219		-	func (r *Repository) ListDeletionTombstones(ctx context.Context, cursor ports.DeletionTombstoneCursor, budget ports.MaintenanceBudget) (ports.DeletionTombstonePage, error) {
220		-	if budget.Entries == 0 \|\| budget.Bytes == 0 {
221		-	return ports.DeletionTombstonePage{}, ErrMaintenanceBudgetTooSmall
222		-	}
223		-	if err := ctx.Err(); err != nil {
224		-	return ports.DeletionTombstonePage{}, err
225		-	}
226		-	state, err := decodeDeletionListingCursor(cursor)
227		-	if err != nil {
228		-	return ports.DeletionTombstonePage{}, err
229		-	}
230		-	dir := filepath.Join(r.dir, deletionTombstonesDir)
231		-	r.maintenanceMu.Lock()
232		-	defer r.maintenanceMu.Unlock()
233		-
234		-	batchLimit := int(min(budget.Entries, uint64(maintenanceBatch)))
235		-	if state.BatchLimit == 0 {
236		-	state.BatchLimit = batchLimit
237		-	} else if state.BatchLimit > batchLimit {
238		-	state.BatchLimit = batchLimit
239		-	if len(state.Candidates) > state.BatchLimit {
240		-	state.Candidates = state.Candidates[:state.BatchLimit]
241		-	state.More = true
242		-	}
243		-	}
244		-	remainingEntries := budget.Entries
245		-	done := state.ScanDone
246		-	for !done && remainingEntries > 0 {
247		-	limit := remainingEntries
248		-	if maxInt := uint64(^uint(0) >> 1); limit > maxInt {
249		-	limit = maxInt
250		-	}
251		-	entries, batchDone, err := r.readDeletionListingEntries(dir, int(limit), &state)
252		-	if errors.Is(err, os.ErrNotExist) {
253		-	return ports.DeletionTombstonePage{Done: true}, nil
254		-	}
255		-	if err != nil {
256		-	return ports.DeletionTombstonePage{}, err
257		-	}
258		-	remainingEntries -= uint64(len(entries))
259		-	for _, entry := range entries {
260		-	if err := ctx.Err(); err != nil {
261		-	return ports.DeletionTombstonePage{}, err
262		-	}
263		-	if entry.isDir \|\| !canonicalDeletionTombstoneFilename(entry.name) {
264		-	return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: invalid deletion tombstone entry")
265		-	}
266		-	if entry.name <= state.After {
267		-	continue
268		-	}
269		-	at := sort.SearchStrings(state.Candidates, entry.name)
270		-	if at < len(state.Candidates) && state.Candidates[at] == entry.name {
271		-	continue
272		-	}
273		-	state.Candidates = append(state.Candidates, "")
274		-	copy(state.Candidates[at+1:], state.Candidates[at:])
275		-	state.Candidates[at] = entry.name
276		-	if len(state.Candidates) > state.BatchLimit {
277		-	state.Candidates = state.Candidates[:state.BatchLimit]
278		-	state.More = true
279		-	}
280		-	}
281		-	done = batchDone
282		-	if len(entries) == 0 && !batchDone {
283		-	break
284		-	}
285		-	}
286		-	if !done {
287		-	next, err := encodeDeletionListingCursor(state)
288		-	if err != nil {
289		-	return ports.DeletionTombstonePage{}, err
290		-	}
291		-	if next.After == cursor.After {
292		-	return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: deletion tombstone continuation did not advance")
293		-	}
294		-	return ports.DeletionTombstonePage{Next: next}, nil
295		-	}
296		-	if len(state.Candidates) == 0 {
297		-	return ports.DeletionTombstonePage{Done: true}, nil
298		-	}
299		-
300		-	remainingBytes := budget.Bytes
301		-	tombstones := make([]domain.DeletionTombstone, 0, len(state.Candidates))
302		-	last := ""
303		-	for _, candidate := range state.Candidates {
304		-	path := filepath.Join(dir, candidate)
305		-	info, err := r.stat(path)
306		-	if err != nil {
307		-	return ports.DeletionTombstonePage{}, err
308		-	}
309		-	charge := uint64(len(candidate))
310		-	if info.Size() < 0 \|\| uint64(info.Size()) > remainingBytes \|\| charge > remainingBytes-uint64(info.Size()) {
311		-	if len(tombstones) == 0 {
312		-	return ports.DeletionTombstonePage{}, ErrMaintenanceBudgetTooSmall
313		-	}
314		-	state.More = true
315		-	break
316		-	}
317		-	remainingBytes -= charge + uint64(info.Size())
318		-	if hook := r.hooks.beforeDeletionTombstoneRead; hook != nil {
319		-	hook(path)
320		-	}
321		-	data, err := r.readBounded(path)
322		-	if err != nil {
323		-	return ports.DeletionTombstonePage{}, err
324		-	}
325		-	decoded, err := decodeDeletionTombstone(data)
326		-	if err != nil {
327		-	return ports.DeletionTombstonePage{}, err
328		-	}
329		-	if candidate != decoded.IncarnationID.String()+".tombstone" {
330		-	return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: non-canonical deletion tombstone filename")
331		-	}
332		-	tombstones = append(tombstones, decoded)
333		-	last = candidate
334		-	}
335		-	next, err := encodeDeletionListingCursor(deletionListingState{After: last})
336		-	if err != nil {
337		-	return ports.DeletionTombstonePage{}, err
338		-	}
339		-	return ports.DeletionTombstonePage{Tombstones: tombstones, Next: next, Done: !state.More && len(tombstones) == len(state.Candidates)}, nil
340		-	}
341		-
342		-	func (r *Repository) DeleteDeletionTombstone(ctx context.Context, id domain.IncarnationID) error {
343		-	if err := ctx.Err(); err != nil {
344		-	return err
345		-	}
346		-	if _, err := incarnationKey(id); err != nil {
347		-	return err
348		-	}
349		-	path := r.deletionTombstonePath(id)
350		-	dir := filepath.Dir(path)
351		-	if _, err := os.Lstat(dir); errors.Is(err, os.ErrNotExist) {
352		-	return nil
353		-	} else if err != nil {
354		-	return err
355		-	}
356		-	if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
357		-	return err
358		-	}
359		-	return r.syncDirectory(dir)
360		-	}
361		-
362		-	// Delete makes a session unavailable by durably moving it out of the canonical
363		-	// namespace. Maintain reaps the private quarantine later; Delete never restores
364		-	// a quarantined directory.
365		-	func (r *Repository) Delete(ctx context.Context, name string) error {
366		-	if err := ctx.Err(); err != nil {
367		-	return err
368		-	}
369		-	key, err := r.legacyRepositoryKey(name)
370		-	if err != nil {
371		-	return err
372		-	}
373		-	lock := r.lockSession(key)
374		-	defer r.unlockSession(lock)
375		-	if err := ctx.Err(); err != nil {
376		-	return err
377		-	}
378		-
379		-	canonical := r.legacySessionPath(key)
380		-	sessions := filepath.Dir(canonical)
381		-	// A prior rename may have succeeded while its parent sync failed. Complete
382		-	// that durability boundary before considering a canonical directory: this
383		-	// also leaves a newly recreated session untouched.
384		-	pending, err := r.pendingQuarantine(sessions, key)
385		-	if err != nil {
386		-	return fmt.Errorf("stat deleting snapshot session %q: %w", name, safeFilesystemError(err))
387		-	}
388		-	if pending {
389		-	if err := ctx.Err(); err != nil {
390		-	return err
391		-	}
392		-	if err := r.syncDirectory(sessions); err != nil {
393		-	return fmt.Errorf("sync deleted snapshot session directory %q: %w", name, safeFilesystemError(err))
394		-	}
395		-	return nil
396		-	}
397		-	if dir, err := r.openDirectory(canonical); errors.Is(err, os.ErrNotExist) {
398		-	return nil
399		-	} else if err != nil {
400		-	return fmt.Errorf("stat snapshot session %q: %w", name, safeFilesystemError(err))
401		-	} else if err := dir.Close(); err != nil {
402		-	return fmt.Errorf("close snapshot session %q: %w", name, safeFilesystemError(err))
403		-	}
404		-	quarantine := filepath.Join(sessions, deletingSessionName(key))
405		-	if err := ctx.Err(); err != nil {
406		-	return err
407		-	}
408		-	// A renamed session can be replaced by a later publication. Invalidate any
409		-	// mark made for the old namespace before that replacement becomes possible.
410		-	r.invalidateStorageEpoch(key)
411		-	if err := r.rename(canonical, quarantine); err != nil {
412		-	return fmt.Errorf("quarantine snapshot session %q: %w", name, safeFilesystemError(err))
413		-	}
414		-	if err := ctx.Err(); err != nil {
415		-	return err
416		-	}
417		-	if err := r.syncDirectory(sessions); err != nil {
418		-	return fmt.Errorf("sync deleted snapshot session directory %q: %w", name, safeFilesystemError(err))
419		-	}
420		-	return nil
421		-	}
422		-
423		-	// deletingSessionName is a deterministic pending-delete record. Its presence
424		-	// is checked directly rather than scanning an attacker-controlled sessions
425		-	// directory, and it prevents a retry from deleting a session republished after
426		-	// the initial rename reached disk but its directory sync failed.
427		-	func deletingSessionName(key string) string { return ".deleting-" + key }
428		-
429		-	func (r *Repository) pendingQuarantine(dir, key string) (bool, error) {
430		-	path := filepath.Join(dir, deletingSessionName(key))
431		-	if hook := r.hooks.beforePendingQuarantineCheck; hook != nil {
432		-	hook(path)
433		-	}
434		-	file, err := r.openDirectory(path)
435		-	if errors.Is(err, os.ErrNotExist) {
436		-	return false, nil
437		-	}
438		-	if err != nil {
439		-	return false, err
440		-	}
441		-	return true, file.Close()
442		-	}
443		-
444		-	// Maintain reaps a bounded amount of stale state. Continuation handles are
445		-	// repository-owned, so a stable prefix cannot starve entries later in a large
446		-	// directory. Cancellation and errors discard those handles: a restarted
447		-	// repository (and the next fresh cycle) always begins at a safe boundary.
448		-	func isQuarantine(name string) bool { return strings.HasPrefix(name, ".deleting-") }
449		-
450		-	// quarantineMaintenance retains one root-to-current path, never an unbounded
451		-	// stack of directory handles or names. Parent paths are recovered after a
452		-	// removal, while descent resumes from current on the next budgeted step.
453		-	type quarantineMaintenance struct {
454		-	root string
455		-	current string
456		-	}
457		-
458		-	// consumeQuarantineWork charges every filesystem operation that can be driven
459		-	// by quarantine contents. This makes deep and wide trees subject to exactly
460		-	// the same maintenanceBatch ceiling as removal work.
461		-	func (r Repository) consumeQuarantineWork(budget int, operation string) bool {
462		-	if *budget == 0 {
463		-	return false
464		-	}
465		-	*budget--
466		-	if hook := r.hooks.beforeMaintenanceWork; hook != nil {
467		-	hook(operation)
468		-	}
469		-	return true
470		-	}
471		-
472		-	// maintainQuarantine makes one bounded depth-first traversal step at a time.
473		-	// It deliberately reopens a directory to select its first child: every
474		-	// successful child removal changes that prefix, so no directory cursor is
475		-	// retained at each hostile nesting level.
476		-	func (r Repository) maintainQuarantine(ctx context.Context, budget int) (changed, done bool, err error) {
477		-	state := r.maintenanceQuarantine
478		-	if state == nil {
479		-	return false, true, nil
480		-	}
481		-	for *budget > 0 {
482		-	if err := ctx.Err(); err != nil {
483		-	return changed, false, err
484		-	}
485		-	if !r.consumeQuarantineWork(budget, "stat") {
486		-	return changed, false, nil
487		-	}
488		-	stat, err := r.stat(state.current)
489		-	if errors.Is(err, os.ErrNotExist) {
490		-	if state.current == state.root {
491		-	return changed, true, nil
492		-	}
493		-	state.current = filepath.Dir(state.current)
494		-	continue
495		-	}
496		-	if err != nil {
497		-	return changed, false, err
498		-	}
499		-	if !stat.IsDir() {
500		-	if !r.consumeQuarantineWork(budget, "remove") {
501		-	return changed, false, nil
502		-	}
503		-	if err := r.remove(state.current); err != nil && !errors.Is(err, os.ErrNotExist) {
504		-	return changed, false, err
505		-	}
506		-	changed = true
507		-	state.current = filepath.Dir(state.current)
508		-	continue
509		-	}
510		-
511		-	if !r.consumeQuarantineWork(budget, "open") {
512		-	return changed, false, nil
513		-	}
514		-	dir, err := r.openDirectory(state.current)
515		-	if err != nil {
516		-	return changed, false, err
517		-	}
518		-	if !r.consumeQuarantineWork(budget, "read") {
519		-	_ = dir.Close()
520		-	return changed, false, nil
521		-	}
522		-	entries, readErr := dir.ReadDir(1)
523		-	closeErr := dir.Close()
524		-	if readErr != nil && !errors.Is(readErr, io.EOF) {
525		-	return changed, false, readErr
526		-	}
527		-	if closeErr != nil {
528		-	return changed, false, closeErr
529		-	}
530		-	if len(entries) == 0 {
531		-	if !r.consumeQuarantineWork(budget, "remove") {
532		-	return changed, false, nil
533		-	}
534		-	if err := r.remove(state.current); err != nil && !errors.Is(err, os.ErrNotExist) {
535		-	return changed, false, err
536		-	}
537		-	changed = true
538		-	if state.current == state.root {
539		-	return changed, true, nil
540		-	}
541		-	state.current = filepath.Dir(state.current)
542		-	continue
543		-	}
544		-	if !r.consumeQuarantineWork(budget, "descend") {
545		-	return changed, false, nil
546		-	}
547		-	state.current = filepath.Join(state.current, entries[0].Name())
548		-	}
549		-	return changed, false, nil
550		-	}
551		-
	13	+	// DeleteIncarnation removes one incarnation-keyed snapshot namespace. It is
	14	+	// retry-idempotent: an incarnation already absent is a successful deletion.
552	15		func (r *Repository) DeleteIncarnation(ctx context.Context, id domain.IncarnationID) error {
553	16		if err := ctx.Err(); err != nil {
554	17		return err

D internal/adapters/snapshot/maintenance_error.go +0 -38

@@ -1,38 +0,0 @@
1	-	package snapshot
2	-
3	-	import (
4	-	"errors"
5	-	"os"
6	-	"path/filepath"
7	-	"strings"
8	-	)
9	-
10	-	func maintenancePath(root, path string) string {
11	-	rel, err := filepath.Rel(root, path)
12	-	if err != nil \|\| rel == "." \|\| strings.HasPrefix(rel, ".."+string(filepath.Separator)) \|\| rel == ".." {
13	-	return filepath.Base(path)
14	-	}
15	-	return rel
16	-	}
17	-
18	-	type safePathError struct {
19	-	op string
20	-	cause error
21	-	}
22	-
23	-	func (e safePathError) Error() string { return e.op + ": " + e.cause.Error() }
24	-	func (e safePathError) Unwrap() error { return e.cause }
25	-
26	-	// safeFilesystemError preserves error matching without including a filesystem
27	-	// path from terminal-facing error text.
28	-	func safeFilesystemError(err error) error {
29	-	var linkError *os.LinkError
30	-	if errors.As(err, &linkError) {
31	-	return safePathError{op: linkError.Op, cause: linkError.Err}
32	-	}
33	-	var pathError *os.PathError
34	-	if errors.As(err, &pathError) {
35	-	return safePathError{op: pathError.Op, cause: pathError.Err}
36	-	}
37	-	return err
38	-	}

M internal/adapters/snapshot/maintenance_gc.go +148 -728


@@ -7,817 +7,237 @@ import (
7	7		"fmt"
8	8		"os"
9	9		"path/filepath"
10		-	"sort"
11		-	"strings"
12		-	"sync/atomic"
13	10
14	11		"github.com/bnema/vev/internal/domain"
15	12		"github.com/bnema/vev/internal/ports"
16	13		codec "github.com/bnema/vev/internal/usecase/snapshot"
17	14		)
18	15
19		-	type retentionMaintenance struct {
20		-	invalidated atomic.Bool
21		-	token string
22		-	cursors map[string]*maintenanceCursor
23		-	references map[ports.SnapshotDigest]struct{}
24		-	validateQueue []ports.CheckpointRef
25		-	validateIndex int
26		-	validated bool
27		-	manifestsDone bool
28		-	objectShard string
29		-	objectRootDone bool
30		-	}
31		-
32		-	type manifestMaintenance struct {
33		-	refs map[ports.SnapshotDigest]codec.ObjectRef
34		-	complete bool
35		-	}
36		-
37		-	type retentionEntryKind uint8
38		-
39		-	const (
40		-	retentionManifest retentionEntryKind = iota + 1
41		-	retentionObject
42		-	)
43		-
44		-	type retentionEntryPolicy struct {
45		-	kind retentionEntryKind
46		-	generations map[uint64]ports.SnapshotDigest
47		-	objects map[ports.SnapshotDigest]struct{}
48		-	}
49		-
50		-	func (p retentionEntryPolicy) delete(entry maintenanceDirEntry) bool {
51		-	if entry.isDir {
52		-	return false
53		-	}
54		-	switch p.kind {
55		-	case retentionManifest:
56		-	generation, canonical := parseGenerationFilename(entry.name)
57		-	_, retained := p.generations[generation]
58		-	return canonical && !retained
59		-	case retentionObject:
60		-	digest, canonical := parseObjectDigest(entry.name)
61		-	_, retained := p.objects[digest]
62		-	return canonical && !retained
63		-	default:
64		-	return false
65		-	}
66		-	}
67		-
68		-	type sessionMaintenance struct {
69		-	lock *sessionMutex
70		-	token string
71		-	epoch uint64
72		-	marked map[uint64]manifestMaintenance
73		-	referenceCount int
74		-	uncertain bool
75		-	conservative bool
76		-	markDone bool
77		-	manifestQueue []uint64
78		-	objectTempShard string
79		-	objectTempsDone bool
80		-	sweepShard string
81		-	sweepRootDone bool
82		-	}
83		-
84		-	// canRetainManifest reserves bounded state before retaining a manifest's
85		-	// reference map. The caller must leave the map unretained when it returns
86		-	// false, and make the cycle conservative instead.
87		-	func (s *sessionMaintenance) canRetainManifest(referenceCount int) bool {
88		-	if referenceCount < 0 \|\| len(s.marked) >= maxMaintenanceMarkedGenerations {
89		-	return false
90		-	}
91		-	if referenceCount > maxMaintenanceReferences-s.referenceCount {
92		-	return false
93		-	}
94		-	s.referenceCount += referenceCount
95		-	return true
96		-	}
97		-
98		-	func (r Repository) maintainSession(ctx context.Context, key string, budget int) error {
	16	+	// CollectGarbage applies the complete snapshot retention policy. The keep map
	17	+	// must come from a catalogue that loaded and validated successfully: only then
	18	+	// is an incarnation absent from keep known to be an orphan.
	19	+	func (r *Repository) CollectGarbage(ctx context.Context, keep map[domain.IncarnationID]domain.CheckpointRef) error {
99	20		if err := ctx.Err(); err != nil {
100	21		return err
101	22		}
102		-	state, err := r.sessionMaintenanceState(key)
103		-	if err != nil {
104		-	return err
105		-	}
106		-	if _, err := r.removeTemps(ctx, filepath.Join(r.legacySessionPath(key), repositoryGenerations), budget, "generation-temps:"+key); err != nil \|\| *budget == 0 {
107		-	return err
108		-	}
109		-	if !state.objectTempsDone {
110		-	done, err := r.removeObjectTemps(ctx, key, state, budget)
111		-	if err != nil \|\| !done \|\| *budget == 0 {
112		-	return err
113		-	}
114		-	}
115		-	if !state.markDone {
116		-	if err := r.markSession(ctx, key, state); err != nil {
117		-	return err
118		-	}
119		-	if !state.markDone {
120		-	return nil
121		-	}
122		-	}
123		-	if state.epoch != r.storageEpoch(key) {
124		-	// A publication may have left an unpublished manifest after this mark.
125		-	// Discard all mark and sweep cursors; the next pass starts from it.
126		-	r.clearSessionMaintenance(key)
127		-	return nil
128		-	}
129		-	if state.uncertain {
130		-	// An unreadable, invalid, or unretained manifest may reference any
131		-	// existing blob. Do not collect manifests or sweep in this cycle; retry
132		-	// marking from a fresh directory pass.
133		-	r.clearSessionMaintenance(key)
134		-	return nil
135		-	}
136		-	if err := r.removeObsoleteManifests(ctx, key, state, budget); err != nil \|\| *budget == 0 {
137		-	return err
138		-	}
139		-	if err := r.sweepSession(ctx, key, state, budget); err != nil {
140		-	return err
141		-	}
142		-	return nil
143		-	}
144		-
145		-	func (r Repository) sessionMaintenanceState(key string) (sessionMaintenance, error) {
146		-	token, conservative, err := r.maintenanceToken(key)
147		-	if err != nil {
148		-	return nil, err
149		-	}
150		-	state := r.maintenanceSessions[key]
151		-	if state == nil \|\| state.token != token \|\| state.epoch != r.storageEpoch(key) {
152		-	r.clearSessionMaintenance(key)
153		-	state = &sessionMaintenance{
154		-	lock: r.retainSessionState(key),
155		-	token: token,
156		-	epoch: r.storageEpoch(key),
157		-	conservative: conservative,
158		-	marked: make(map[uint64]manifestMaintenance),
159		-	}
160		-	r.maintenanceSessions[key] = state
161		-	}
162		-	return state, nil
163		-	}
164		-
165		-	// maintenanceToken is the publication boundary. A missing or corrupt HEAD is
166		-	// itself stable maintenance state: we retain every classified reference until a
167		-	// valid publication changes that state, rather than restarting the mark pass.
168		-	func (r *Repository) maintenanceToken(key string) (string, bool, error) {
169		-	data, exists, err := r.readOptionalBounded(r.legacyHeadPath(key))
170		-	if err != nil {
171		-	return "", false, err
172		-	}
173		-	if !exists {
174		-	return "missing", true, nil
175		-	}
176		-	sum := sha256.Sum256(data)
177		-	if _, _, err := r.readLegacyHead(key); err != nil {
178		-	return fmt.Sprintf("invalid:%x", sum), true, nil
179		-	}
180		-	return fmt.Sprintf("valid:%x", sum), false, nil
181		-	}
182		-
183		-	func (r *Repository) clearSessionMaintenance(key string) {
184		-	if state := r.maintenanceSessions[key]; state != nil && state.lock != nil {
185		-	r.releaseSessionReference(state.lock)
186		-	}
187		-	delete(r.maintenanceSessions, key)
188		-	prefix := "\x00" + filepath.Clean(r.legacySessionPath(key))
189		-	for id := range r.maintenanceCursors {
190		-	if strings.HasSuffix(id, prefix) \|\| strings.Contains(id, ":"+key+":") \|\| strings.Contains(id, ":"+key+"\x00") {
191		-	delete(r.maintenanceCursors, id)
192		-	}
193		-	}
194		-	}
195		-
196		-	func (r Repository) markSession(ctx context.Context, key string, state sessionMaintenance) error {
197		-	dir := filepath.Join(r.legacySessionPath(key), repositoryGenerations)
198		-	entries, done, err := r.readMaintenanceDir(dir, maintenanceBatch, "mark-generations:"+key)
	23	+	sessions := filepath.Join(r.dir, repositorySessionsDir)
	24	+	entries, err := r.readGarbageCollectionDirectory(sessions)
199	25		if errors.Is(err, os.ErrNotExist) {
200		-	state.markDone = true
201	26		return nil
202	27		}
203	28		if err != nil {
204		-	return fmt.Errorf("read snapshot generations: %w", safeFilesystemError(err))
	29	+	return fmt.Errorf("read snapshot incarnations: %w", safeFilesystemError(err))
205	30		}
	31	+
	32	+	var collected error
206	33		for _, entry := range entries {
207	34		if err := ctx.Err(); err != nil {
208		-	return err
209		-	}
210		-	if entry.isDir {
211		-	continue
	35	+	return errors.Join(collected, err)
212	36		}
213		-	generation, ok := parseGenerationFilename(entry.name)
214		-	if !ok {
	37	+	if !entry.IsDir() {
215	38		continue
216	39		}
217		-	data, err := r.readBounded(r.legacyManifestPath(key, generation))
218		-	if err != nil {
219		-	state.uncertain = true
220		-	continue
221		-	}
222		-	manifest, err := codec.UnmarshalManifest(data)
223		-	if err != nil \|\| manifest.Generation != generation \|\| manifest.IncarnationID.String() != key {
224		-	state.uncertain = true
	40	+	var id domain.IncarnationID
	41	+	if err := id.UnmarshalText([]byte(entry.Name())); err != nil {
225	42		continue
226	43		}
227		-	refs, validRefs := maintenanceManifestRefs(manifest)
228		-	if !validRefs \|\| !withinGenerationBudget(len(data), refs) \|\| !state.canRetainManifest(len(refs)) {
229		-	state.uncertain = true
230		-	continue
231		-	}
232		-	// Valid but incomplete generations are marked too. This protects blobs
233		-	// written before a failed or in-flight publication reaches HEAD.
234		-	_, loadErr := r.loadGeneration(ctx, manifest.Name, key, generation)
235		-	state.marked[generation] = manifestMaintenance{refs: refs, complete: loadErr == nil}
236		-	}
237		-	if done {
238		-	state.markDone = true
239		-	if !state.conservative {
240		-	state.manifestQueue = retainedManifestQueue(state.marked)
241		-	}
242		-	}
243		-	return nil
244		-	}
245		-
246		-	// maintenanceManifestRefs applies the GC reference ceiling while building its
247		-	// map, rather than first materializing a hostile manifest's entire reference
248		-	// set. Conflicting references are invalid just as they are for publication.
249		-	func maintenanceManifestRefs(manifest codec.Manifest) (map[ports.SnapshotDigest]codec.ObjectRef, bool) {
250		-	refs := make(map[ports.SnapshotDigest]codec.ObjectRef)
251		-	add := func(ref codec.ObjectRef) bool {
252		-	if old, ok := refs[ref.Digest]; ok {
253		-	return old == ref
254		-	}
255		-	if len(refs) == maxMaintenanceReferences {
256		-	return false
257		-	}
258		-	refs[ref.Digest] = ref
259		-	return true
260		-	}
261		-	for _, tab := range manifest.Tabs {
262		-	for _, pane := range tab.Panes {
263		-	for _, ref := range pane.Sealed {
264		-	if !add(ref) {
265		-	return nil, false
266		-	}
267		-	}
268		-	if !add(pane.Tail) \|\| !add(pane.Visible) {
269		-	return nil, false
	44	+	committed, known := keep[id]
	45	+	if !known {
	46	+	path := r.sessionPath(id)
	47	+	if err := os.RemoveAll(path); err != nil {
	48	+	collected = errors.Join(collected, fmt.Errorf("remove orphan snapshot incarnation %s: %w", id.String(), safeFilesystemError(err)))
	49	+	continue
270	50		}
	51	+	r.log.Info("snapshot_garbage_collected", "incarnation", id.String(), "action", "remove-incarnation")
	52	+	continue
271	53		}
272		-	}
273		-	return refs, true
274		-	}
275		-
276		-	// completeGenerationsDescending returns only complete generations in canonical
277		-	// newest-first order for both manifest and object retention.
278		-	func completeGenerationsDescending(marked map[uint64]manifestMaintenance) []uint64 {
279		-	complete := make([]uint64, 0, len(marked))
280		-	for generation, item := range marked {
281		-	if item.complete {
282		-	complete = append(complete, generation)
	54	+	if err := r.pruneGenerations(ctx, id, committed); err != nil {
	55	+	collected = errors.Join(collected, err)
283	56		}
284	57		}
285		-	sort.Slice(complete, func(i, j int) bool { return complete[i] > complete[j] })
286		-	return complete
287		-	}
288		-
289		-	func retainedManifestQueue(marked map[uint64]manifestMaintenance) []uint64 {
290		-	complete := completeGenerationsDescending(marked)
291		-	if len(complete) <= 2 {
292		-	return nil
	58	+	if err := r.syncDirectory(sessions); err != nil {
	59	+	collected = errors.Join(collected, fmt.Errorf("sync snapshot sessions directory: %w", safeFilesystemError(err)))
293	60		}
294		-	return complete[2:]
	61	+	return collected
295	62		}
296	63
297		-	func (r Repository) removeObsoleteManifests(ctx context.Context, key string, state sessionMaintenance, budget *int) error {
298		-	for len(state.manifestQueue) != 0 && *budget > 0 {
299		-	if err := ctx.Err(); err != nil {
300		-	return err
301		-	}
302		-	generation := state.manifestQueue[0]
303		-	path := r.legacyManifestPath(key, generation)
304		-	if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
305		-	return fmt.Errorf("remove snapshot manifest %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err))
306		-	}
307		-	if err := ctx.Err(); err != nil {
308		-	return err
309		-	}
310		-	if err := r.syncDirectory(filepath.Dir(path)); err != nil {
311		-	return fmt.Errorf("sync snapshot maintenance directory for %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err))
312		-	}
313		-	state.manifestQueue = state.manifestQueue[1:]
314		-	*budget--
	64	+	// pruneGenerations keeps the committed generation and its immediate
	65	+	// predecessor. Everything else, including a forward orphan newer than the
	66	+	// catalogue commit, is removed before unreferenced objects are swept.
	67	+	func (r *Repository) pruneGenerations(ctx context.Context, id domain.IncarnationID, committed domain.CheckpointRef) error {
	68	+	key, err := incarnationKey(id)
	69	+	if err != nil {
	70	+	return err
315	71		}
316		-	return nil
317		-	}
	72	+	lock := r.lockSession(key)
	73	+	defer r.unlockSession(lock)
318	74
319		-	// removeObjectTemps performs at most one shard traversal or one root-entry
320		-	// discovery per Maintain call. Empty attacker-controlled shards consume no
321		-	// removal budget, so this fixed step ceiling prevents an unbounded scan of
322		-	// them while retaining only the root cursor and one active shard cursor.
323		-	func (r Repository) removeObjectTemps(ctx context.Context, key string, state sessionMaintenance, budget *int) (bool, error) {
324		-	root := filepath.Join(r.legacySessionPath(key), repositoryObjectsDir)
325		-	if *budget == 0 {
326		-	return false, nil
327		-	}
328		-	if state.objectTempShard != "" {
329		-	done, err := r.removeTemps(ctx, filepath.Join(root, state.objectTempShard), budget, "object-temps:"+key+":"+state.objectTempShard)
330		-	if err != nil \|\| !done {
331		-	return false, err
332		-	}
333		-	state.objectTempShard = ""
334		-	return state.objectTempsDone, nil
335		-	}
336		-	if state.objectTempsDone {
337		-	return true, nil
338		-	}
339		-	entries, done, err := r.readMaintenanceDir(root, 1, "object-temps-shards:"+key)
	75	+	generationsDir := filepath.Join(r.sessionPath(id), repositoryGenerations)
	76	+	entries, err := r.readGarbageCollectionDirectory(generationsDir)
340	77		if errors.Is(err, os.ErrNotExist) {
341		-	state.objectTempsDone = true
342		-	return true, nil
	78	+	return nil
343	79		}
344	80		if err != nil {
345		-	return false, fmt.Errorf("read snapshot object shards: %w", safeFilesystemError(err))
346		-	}
347		-	state.objectTempsDone = done
348		-	if len(entries) != 0 && entries[0].isDir {
349		-	state.objectTempShard = entries[0].name
	81	+	return fmt.Errorf("read snapshot generations for %s: %w", id.String(), safeFilesystemError(err))
350	82		}
351		-	return state.objectTempsDone && state.objectTempShard == "", nil
352		-	}
353	83
354		-	func (r Repository) sweepSession(ctx context.Context, key string, state sessionMaintenance, budget *int) error {
355		-	// Every bounded sweep batch is tied to the mark's storage epoch. Do not use
356		-	// stale references after any publication or replacement.
357		-	if state.epoch != r.storageEpoch(key) {
358		-	r.clearSessionMaintenance(key)
359		-	return nil
	84	+	kept := make(map[uint64]struct{}, 2)
	85	+	if committed.Generation > 0 {
	86	+	kept[committed.Generation] = struct{}{}
360	87		}
361		-	referenced := retainedReferences(state.marked, state.conservative)
362		-	root := filepath.Join(r.legacySessionPath(key), repositoryObjectsDir)
363		-	if state.sweepShard != "" && *budget > 0 {
364		-	shard := state.sweepShard
365		-	done, err := r.sweepShard(ctx, filepath.Join(root, shard), referenced, budget, key, shard)
366		-	if err != nil {
367		-	return err
368		-	}
369		-	if !done {
370		-	return nil
371		-	}
372		-	state.sweepShard = ""
373		-	// Release this shard's cursor before discovering another shard.
374		-	return nil
375		-	}
376		-	if !state.sweepRootDone {
377		-	entries, done, err := r.readMaintenanceDir(root, 1, "sweep-shards:"+key)
378		-	if errors.Is(err, os.ErrNotExist) {
379		-	state.sweepRootDone = true
380		-	} else if err != nil {
381		-	return fmt.Errorf("read snapshot objects: %w", safeFilesystemError(err))
382		-	} else {
383		-	state.sweepRootDone = done
384		-	if len(entries) != 0 && entries[0].isDir {
385		-	state.sweepShard = entries[0].name
386		-	}
387		-	}
	88	+	if committed.Generation > 1 {
	89	+	kept[committed.Generation-1] = struct{}{}
388	90		}
389		-	if state.sweepRootDone && state.sweepShard == "" {
390		-	r.clearSessionMaintenance(key)
391		-	}
392		-	return nil
393		-	}
394		-
395		-	// keepSet is the complete catalogue-owned retention index. Directory order and
396		-	// HEAD never grant retention authority.
397		-	func keepSet(plan ports.RetentionPlan) map[uint64]ports.SnapshotDigest {
398		-	keep := make(map[uint64]ports.SnapshotDigest, len(plan.Keep))
399		-	for _, ref := range plan.Keep {
400		-	keep[ref.Generation] = ref.ManifestDigest
	91	+	references, err := r.referencesForGenerations(id, entries, kept, committed)
	92	+	if err != nil {
	93	+	return fmt.Errorf("mark snapshot objects for %s: %w", id.String(), err)
401	94		}
402		-	return keep
403		-	}
404	95
405		-	// MaintainSession incrementally validates and retains only catalogue-indexed
406		-	// checkpoints. Every incarnation owns its budget and continuation state.
407		-	// processRetentionEntries charges stat-reported size for every entry, requeues
408		-	// the unadmitted suffix, and removes only canonical entries rejected by policy.
409		-	func (r Repository) processRetentionEntries(ctx context.Context, state retentionMaintenance, dir, purpose string, entries []maintenanceDirEntry, budget *ports.MaintenanceBudget, consumedBefore bool, policy retentionEntryPolicy) (admitted bool, err error) {
410		-	removed := false
411		-	defer func() {
412		-	if !removed {
413		-	return
414		-	}
415		-	if syncErr := r.syncDirectory(dir); syncErr != nil {
416		-	admitted = false
417		-	err = errors.Join(err, syncErr)
418		-	}
419		-	}()
420		-	for i, entry := range entries {
421		-	if state.invalidated.Load() {
422		-	return false, nil
423		-	}
	96	+	removed := 0
	97	+	for _, entry := range entries {
424	98		if err := ctx.Err(); err != nil {
425		-	return false, err
426		-	}
427		-	path := filepath.Join(dir, entry.name)
428		-	info, statErr := r.stat(path)
429		-	if statErr != nil && !errors.Is(statErr, os.ErrNotExist) {
430		-	return false, statErr
431		-	}
432		-	size := uint64(0)
433		-	if statErr == nil && info.Size() > 0 {
434		-	size = uint64(info.Size())
435		-	}
436		-	if budget.Entries == 0 \|\| size > budget.Bytes {
437		-	r.requeueRetentionEntries(state, dir, purpose, entries[i:])
438		-	if !consumedBefore && i == 0 {
439		-	return false, ErrMaintenanceBudgetTooSmall
440		-	}
441		-	return false, nil
	99	+	return err
442	100		}
443		-	budget.Entries--
444		-	budget.Bytes -= size
445		-	if !policy.delete(entry) {
	101	+	generation, canonical := parseGenerationFilename(entry.Name())
	102	+	if !canonical \|\| entry.IsDir() {
446	103		continue
447	104		}
448		-	if err := r.remove(path); err != nil {
449		-	if !errors.Is(err, os.ErrNotExist) {
450		-	return false, err
451		-	}
452		-	} else {
453		-	removed = true
454		-	}
455		-	}
456		-	return true, nil
457		-	}
458		-
459		-	func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPlan, budget ports.MaintenanceBudget) (done bool, err error) {
460		-	requested := budget
461		-	if err := ctx.Err(); err != nil {
462		-	return false, err
463		-	}
464		-	key, err := incarnationKey(plan.IncarnationID)
465		-	if err != nil {
466		-	return false, err
467		-	}
468		-	defer func() {
469		-	r.log.Info("snapshot_maintenance_progress",
470		-	"incarnation", plan.IncarnationID.String(),
471		-	"action", "retention",
472		-	"retained", len(plan.Keep),
473		-	"cursor", key,
474		-	"consumed_entries", requested.Entries-budget.Entries,
475		-	"consumed_bytes", requested.Bytes-budget.Bytes,
476		-	"done", done,
477		-	"budget_exhausted", !done && err == nil,
478		-	"err", err,
479		-	)
480		-	}()
481		-	if plan.PinAll {
482		-	r.maintenanceMu.Lock()
483		-	if state := r.retentionSessions[key]; state != nil {
484		-	state.invalidated.Store(true)
485		-	}
486		-	r.clearRetentionMaintenance(key)
487		-	r.maintenanceMu.Unlock()
488		-	return true, nil
489		-	}
490		-	if budget.Entries == 0 \|\| budget.Bytes == 0 {
491		-	return false, ErrMaintenanceBudgetTooSmall
492		-	}
493		-
494		-	// Establish per-incarnation state under maintenanceMu, then retain only the
495		-	// session mutex across payload I/O. Never reacquire maintenanceMu while the
496		-	// session mutex is held; PinAll invalidates active state without waiting.
497		-	r.maintenanceMu.Lock()
498		-	lock := r.lockSession(key)
499		-	if r.retentionSessions == nil {
500		-	r.retentionSessions = make(map[string]*retentionMaintenance)
501		-	}
502		-
503		-	token := retentionToken(plan)
504		-	state := r.retentionSessions[key]
505		-	if state == nil \|\| state.token != token {
506		-	r.clearRetentionMaintenance(key)
507		-	state = &retentionMaintenance{
508		-	token: token,
509		-	cursors: make(map[string]*maintenanceCursor),
510		-	references: make(map[ports.SnapshotDigest]struct{}),
511		-	validateQueue: append([]ports.CheckpointRef(nil), plan.Keep...),
512		-	}
513		-	r.retentionSessions[key] = state
514		-	}
515		-	r.maintenanceMu.Unlock()
516		-	defer func() {
517		-	r.unlockSession(lock)
518		-	if done {
519		-	r.maintenanceMu.Lock()
520		-	if r.retentionSessions[key] == state {
521		-	delete(r.retentionSessions, key)
522		-	}
523		-	r.maintenanceMu.Unlock()
	105	+	if _, retain := kept[generation]; retain {
	106	+	continue
524	107		}
525		-	}()
526		-	keep := keepSet(plan)
527		-	if !state.validated {
528		-	for state.validateIndex < len(state.validateQueue) {
529		-	ref := state.validateQueue[state.validateIndex]
530		-	if budget.Entries == 0 {
531		-	return false, nil
532		-	}
533		-	validated, consumedBytes, admitted, err := r.validateMaintenanceCheckpoint(ctx, plan.IncarnationID, ref, budget.Bytes)
534		-	if err != nil {
535		-	return false, fmt.Errorf("validate retained manifest %d: %w", ref.Generation, err)
536		-	}
537		-	if !admitted {
538		-	if requested.Entries == budget.Entries && requested.Bytes == budget.Bytes {
539		-	return false, ErrMaintenanceBudgetTooSmall
540		-	}
541		-	return false, nil
542		-	}
543		-	budget.Entries--
544		-	budget.Bytes -= consumedBytes
545		-	for digest := range validated.refs {
546		-	state.references[digest] = struct{}{}
547		-	}
548		-	state.validateIndex++
	108	+	if err := r.remove(filepath.Join(generationsDir, entry.Name())); err != nil && !errors.Is(err, os.ErrNotExist) {
	109	+	return fmt.Errorf("remove snapshot generation %d for %s: %w", generation, id.String(), safeFilesystemError(err))
549	110		}
550		-	state.validated = true
	111	+	removed++
551	112		}
552		-
553		-	purpose := "retention-manifests:" + key + ":" + token
554		-	for !state.manifestsDone && budget.Entries > 0 && budget.Bytes > 0 {
555		-	limit := int(min(budget.Entries, uint64(maintenanceBatch)))
556		-	entries, done, err := r.readRetentionDir(state, filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations), limit, purpose)
557		-	if errors.Is(err, os.ErrNotExist) {
558		-	state.manifestsDone = true
559		-	break
560		-	}
561		-	if err != nil {
562		-	return false, err
563		-	}
564		-	dir := filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations)
565		-	consumedBefore := requested.Entries != budget.Entries \|\| requested.Bytes != budget.Bytes
566		-	admitted, err := r.processRetentionEntries(ctx, state, dir, purpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionManifest, generations: keep})
567		-	if err != nil {
568		-	return false, err
569		-	}
570		-	if !admitted {
571		-	return false, nil
572		-	}
573		-	state.manifestsDone = done
574		-	if len(entries) == 0 && !done {
575		-	return false, nil
	113	+	if removed > 0 {
	114	+	if err := r.syncDirectory(generationsDir); err != nil {
	115	+	return fmt.Errorf("sync snapshot generations for %s: %w", id.String(), safeFilesystemError(err))
576	116		}
	117	+	r.log.Info("snapshot_garbage_collected", "incarnation", id.String(), "action", "remove-generations", "removed", removed)
577	118		}
578		-	if !state.manifestsDone {
579		-	return false, nil
580		-	}
	119	+	return r.sweepUnreferencedObjects(ctx, id, references)
	120	+	}
581	121
582		-	objectsRoot := filepath.Join(r.sessionPath(plan.IncarnationID), repositoryObjectsDir)
583		-	for budget.Entries > 0 && budget.Bytes > 0 {
584		-	if state.objectShard != "" {
585		-	shard := state.objectShard
586		-	dir := filepath.Join(objectsRoot, shard)
587		-	objectPurpose := "retention-objects:" + key + ":" + shard + ":" + token
588		-	entries, done, err := r.readRetentionDir(state, dir, int(min(budget.Entries, uint64(maintenanceBatch))), objectPurpose)
589		-	if errors.Is(err, os.ErrNotExist) {
590		-	state.objectShard = ""
591		-	continue
592		-	}
593		-	if err != nil {
594		-	return false, err
595		-	}
596		-	consumedBefore := requested.Entries != budget.Entries \|\| requested.Bytes != budget.Bytes
597		-	admitted, err := r.processRetentionEntries(ctx, state, dir, objectPurpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionObject, objects: state.references})
598		-	if err != nil {
599		-	return false, err
600		-	}
601		-	if !admitted {
602		-	return false, nil
603		-	}
604		-	if done {
605		-	state.objectShard = ""
606		-	} else if len(entries) == 0 {
607		-	return false, nil
608		-	}
	122	+	func (r *Repository) referencesForGenerations(id domain.IncarnationID, entries []os.DirEntry, keep map[uint64]struct{}, committed domain.CheckpointRef) (map[ports.SnapshotDigest]struct{}, error) {
	123	+	references := make(map[ports.SnapshotDigest]struct{})
	124	+	committedFound := committed.Generation == 0
	125	+	for _, entry := range entries {
	126	+	generation, canonical := parseGenerationFilename(entry.Name())
	127	+	if !canonical \|\| entry.IsDir() {
609	128		continue
610	129		}
611		-	if state.objectRootDone {
612		-	return true, nil
613		-	}
614		-	entries, done, err := r.readRetentionDir(state, objectsRoot, 1, "retention-shards:"+key+":"+token)
615		-	if errors.Is(err, os.ErrNotExist) {
616		-	return true, nil
	130	+	if _, retained := keep[generation]; !retained {
	131	+	continue
617	132		}
	133	+	data, err := r.readBounded(r.manifestPath(id, generation))
618	134		if err != nil {
619		-	return false, err
620		-	}
621		-	state.objectRootDone = done
622		-	if len(entries) == 0 {
623		-	if !done {
624		-	return false, nil
625		-	}
626		-	continue
	135	+	return nil, err
627	136		}
628		-	budget.Entries--
629		-	if entries[0].isDir {
630		-	state.objectShard = entries[0].name
	137	+	manifest, err := codec.UnmarshalManifest(data)
	138	+	if err != nil \|\| manifest.IncarnationID != id \|\| manifest.Generation != generation {
	139	+	return nil, fmt.Errorf("invalid retained manifest generation %d", generation)
631	140		}
632		-	}
633		-	return false, nil
634		-	}
635		-
636		-	func (r *Repository) clearRetentionMaintenance(key string) {
637		-	delete(r.retentionSessions, key)
638		-	}
639		-
640		-	func (r Repository) readRetentionDir(state retentionMaintenance, dir string, n int, purpose string) ([]maintenanceDirEntry, bool, error) {
641		-	id := maintenanceCursorID(dir, purpose)
642		-	cursor := state.cursors[id]
643		-	if cursor != nil && cursor.pending != nil {
644		-	count := min(n, len(cursor.pending))
645		-	entries := cursor.pending[:count]
646		-	cursor.pending = cursor.pending[count:]
647		-	if len(entries) != 0 {
648		-	cursor.offset = entries[len(entries)-1].cookie
	141	+	if generation == committed.Generation {
	142	+	committedFound = true
	143	+	if sha256.Sum256(data) != committed.ManifestDigest {
	144	+	return nil, fmt.Errorf("committed manifest digest mismatch")
	145	+	}
649	146		}
650		-	done := cursor.done && len(cursor.pending) == 0
651		-	if done {
652		-	delete(state.cursors, id)
653		-	} else if count > 0 && len(cursor.pending) == 0 {
654		-	cursor.pending = nil
	147	+	refs := manifestRefs(manifest)
	148	+	if refs == nil \|\| !withinGenerationBudget(len(data), refs) {
	149	+	return nil, fmt.Errorf("invalid retained manifest references generation %d", generation)
655	150		}
656		-	return entries, done, nil
657		-	}
658		-	if cursor == nil {
659		-	if len(state.cursors) >= maxMaintenanceCursors {
660		-	return nil, false, errMaintenanceCursorLimit
	151	+	for digest := range refs {
	152	+	references[digest] = struct{}{}
661	153		}
662		-	cursor = &maintenanceCursor{}
663		-	state.cursors[id] = cursor
664		-	}
665		-	entries, done, err := r.readMaintenanceDirent(dir, n, cursor)
666		-	if err != nil \|\| done {
667		-	delete(state.cursors, id)
668		-	}
669		-	return entries, done, err
670		-	}
671		-
672		-	func (r Repository) requeueRetentionEntries(state retentionMaintenance, dir, purpose string, entries []maintenanceDirEntry) {
673		-	if len(entries) == 0 {
674		-	return
675	154		}
676		-	id := maintenanceCursorID(dir, purpose)
677		-	cursor := state.cursors[id]
678		-	if cursor == nil {
679		-	cursor = &maintenanceCursor{done: true}
680		-	state.cursors[id] = cursor
	155	+	if !committedFound {
	156	+	return nil, fmt.Errorf("committed manifest generation %d is missing", committed.Generation)
681	157		}
682		-	pending := make([]maintenanceDirEntry, 0, len(entries)+len(cursor.pending))
683		-	pending = append(pending, entries...)
684		-	pending = append(pending, cursor.pending...)
685		-	cursor.pending = pending
	158	+	return references, nil
686	159		}
687	160
688		-	func retentionToken(plan ports.RetentionPlan) string {
689		-	var b strings.Builder
690		-	b.WriteString(plan.IncarnationID.String())
691		-	for _, ref := range plan.Keep {
692		-	fmt.Fprintf(&b, ":%d:%x", ref.Generation, ref.ManifestDigest)
693		-	}
694		-	return b.String()
	161	+	type safePathError struct {
	162	+	op string
	163	+	cause error
695	164		}
696	165
697		-	type validatedMaintenanceCheckpoint struct {
698		-	refs map[ports.SnapshotDigest]codec.ObjectRef
699		-	parent *domain.CheckpointRef
700		-	}
	166	+	func (e safePathError) Error() string { return e.op + ": " + e.cause.Error() }
	167	+	func (e safePathError) Unwrap() error { return e.cause }
701	168
702		-	// validateMaintenanceCheckpoint admits every payload by stat-reported size
703		-	// before reading it. It validates directly instead of calling LoadCheckpoint,
704		-	// which would materialize a complete generation before maintenance could
705		-	// enforce its byte budget.
706		-	func (r *Repository) validateMaintenanceCheckpoint(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef, byteBudget uint64) (validatedMaintenanceCheckpoint, uint64, bool, error) {
707		-	manifestPath := r.manifestPath(id, ref.Generation)
708		-	info, err := r.stat(manifestPath)
709		-	if err != nil {
710		-	return validatedMaintenanceCheckpoint{}, 0, false, err
711		-	}
712		-	if info.Size() < 0 \|\| uint64(info.Size()) > byteBudget {
713		-	return validatedMaintenanceCheckpoint{}, 0, false, nil
714		-	}
715		-	consumed := uint64(info.Size())
716		-	if hook := r.hooks.beforeMaintenancePayloadRead; hook != nil {
717		-	hook(manifestPath)
718		-	}
719		-	data, err := r.readBounded(manifestPath)
720		-	if err != nil {
721		-	return validatedMaintenanceCheckpoint{}, consumed, false, err
	169	+	// safeFilesystemError preserves error matching without exposing repository
	170	+	// paths in terminal-facing errors.
	171	+	func safeFilesystemError(err error) error {
	172	+	var linkError *os.LinkError
	173	+	if errors.As(err, &linkError) {
	174	+	return safePathError{op: linkError.Op, cause: linkError.Err}
722	175		}
723		-	manifest, err := codec.UnmarshalManifest(data)
724		-	if err != nil \|\| manifest.Generation != ref.Generation \|\| manifest.IncarnationID != id \|\| sha256.Sum256(data) != ref.ManifestDigest {
725		-	return validatedMaintenanceCheckpoint{}, consumed, false, errors.New("catalogue reference mismatch")
726		-	}
727		-	refs, valid := maintenanceManifestRefs(manifest)
728		-	if !valid \|\| !withinGenerationBudget(len(data), refs) {
729		-	return validatedMaintenanceCheckpoint{}, consumed, false, errors.New("invalid manifest references")
730		-	}
731		-	for digest, objectRef := range refs {
732		-	if err := ctx.Err(); err != nil {
733		-	return validatedMaintenanceCheckpoint{}, consumed, false, err
734		-	}
735		-	path := r.objectPath(id, digest)
736		-	info, err := r.stat(path)
737		-	if err != nil {
738		-	return validatedMaintenanceCheckpoint{}, consumed, false, err
739		-	}
740		-	if info.Size() < 0 \|\| uint64(info.Size()) > byteBudget-consumed {
741		-	return validatedMaintenanceCheckpoint{}, consumed, false, nil
742		-	}
743		-	consumed += uint64(info.Size())
744		-	if hook := r.hooks.beforeMaintenancePayloadRead; hook != nil {
745		-	hook(path)
746		-	}
747		-	object, err := r.readBounded(path)
748		-	if err != nil {
749		-	return validatedMaintenanceCheckpoint{}, consumed, false, err
750		-	}
751		-	if sha256.Sum256(object) != digest \|\| !validObject(object, objectRef) {
752		-	return validatedMaintenanceCheckpoint{}, consumed, false, errors.New("invalid retained object")
753		-	}
	176	+	var pathError *os.PathError
	177	+	if errors.As(err, &pathError) {
	178	+	return safePathError{op: pathError.Op, cause: pathError.Err}
754	179		}
755		-	return validatedMaintenanceCheckpoint{refs: refs, parent: manifest.ParentCheckpoint}, consumed, true, nil
	180	+	return err
756	181		}
757	182
758		-	func retainedReferences(marked map[uint64]manifestMaintenance, conservative bool) map[ports.SnapshotDigest]struct{} {
759		-	if conservative {
760		-	references := make(map[ports.SnapshotDigest]struct{})
761		-	for _, item := range marked {
762		-	for digest := range item.refs {
763		-	references[digest] = struct{}{}
764		-	}
765		-	}
766		-	return references
767		-	}
768		-	complete := completeGenerationsDescending(marked)
769		-	keep := make(map[uint64]bool, 2)
770		-	for _, generation := range complete {
771		-	if len(keep) == 2 {
772		-	break
773		-	}
774		-	keep[generation] = true
	183	+	func (r *Repository) readGarbageCollectionDirectory(path string) (entries []os.DirEntry, err error) {
	184	+	directory, err := r.openDirectory(path)
	185	+	if err != nil {
	186	+	return nil, err
775	187		}
776		-	references := make(map[ports.SnapshotDigest]struct{})
777		-	for generation, item := range marked {
778		-	if !item.complete \|\| keep[generation] {
779		-	for digest := range item.refs {
780		-	references[digest] = struct{}{}
781		-	}
	188	+	defer func() {
	189	+	if closeErr := directory.Close(); closeErr != nil {
	190	+	err = errors.Join(err, closeErr)
782	191		}
783		-	}
784		-	return references
	192	+	}()
	193	+	return directory.ReadDir(-1)
785	194		}
786	195
787		-	func (r Repository) sweepShard(ctx context.Context, dir string, referenced map[ports.SnapshotDigest]struct{}, budget int, key, shard string) (bool, error) {
788		-	entries, done, err := r.readMaintenanceDir(dir, maintenanceBatch, "sweep-objects:"+key+":"+shard)
	196	+	func (r *Repository) sweepUnreferencedObjects(ctx context.Context, id domain.IncarnationID, references map[ports.SnapshotDigest]struct{}) error {
	197	+	objectsRoot := filepath.Join(r.sessionPath(id), repositoryObjectsDir)
	198	+	shards, err := r.readGarbageCollectionDirectory(objectsRoot)
789	199		if errors.Is(err, os.ErrNotExist) {
790		-	return true, nil
	200	+	return nil
791	201		}
792	202		if err != nil {
793		-	return false, fmt.Errorf("read snapshot object shard: %w", safeFilesystemError(err))
	203	+	return fmt.Errorf("read snapshot objects for %s: %w", id.String(), safeFilesystemError(err))
794	204		}
795		-	for i, object := range entries {
796		-	if *budget == 0 {
797		-	r.requeueMaintenanceEntries(dir, "sweep-objects:"+key+":"+shard, entries[i:])
798		-	return false, nil
799		-	}
	205	+	removed := 0
	206	+	for _, shard := range shards {
800	207		if err := ctx.Err(); err != nil {
801		-	return false, err
802		-	}
803		-	digest, ok := parseObjectDigest(object.name)
804		-	if !ok \|\| object.isDir {
805		-	continue
	208	+	return err
806	209		}
807		-	if _, used := referenced[digest]; used {
	210	+	if !shard.IsDir() \|\| len(shard.Name()) != 2 {
808	211		continue
809	212		}
810		-	path := filepath.Join(dir, object.name)
811		-	if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
812		-	return false, fmt.Errorf("remove snapshot object %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err))
	213	+	dir := filepath.Join(objectsRoot, shard.Name())
	214	+	objects, err := r.readGarbageCollectionDirectory(dir)
	215	+	if err != nil {
	216	+	return fmt.Errorf("read snapshot object shard for %s: %w", id.String(), safeFilesystemError(err))
813	217		}
814		-	if err := ctx.Err(); err != nil {
815		-	return false, err
	218	+	shardRemoved := false
	219	+	for _, object := range objects {
	220	+	digest, canonical := parseObjectDigest(object.Name())
	221	+	if !canonical \|\| object.IsDir() {
	222	+	continue
	223	+	}
	224	+	if _, retained := references[digest]; retained {
	225	+	continue
	226	+	}
	227	+	if err := r.remove(filepath.Join(dir, object.Name())); err != nil && !errors.Is(err, os.ErrNotExist) {
	228	+	return fmt.Errorf("remove snapshot object for %s: %w", id.String(), safeFilesystemError(err))
	229	+	}
	230	+	removed++
	231	+	shardRemoved = true
816	232		}
817		-	if err := r.syncDirectory(dir); err != nil {
818		-	return false, fmt.Errorf("sync snapshot maintenance directory for %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err))
	233	+	if shardRemoved {
	234	+	if err := r.syncDirectory(dir); err != nil {
	235	+	return fmt.Errorf("sync snapshot object shard for %s: %w", id.String(), safeFilesystemError(err))
	236	+	}
819	237		}
820		-	*budget--
821	238		}
822		-	return done, nil
	239	+	if removed > 0 {
	240	+	r.log.Info("snapshot_garbage_collected", "incarnation", id.String(), "action", "remove-objects", "removed", removed)
	241	+	}
	242	+	return nil
823	243		}

A internal/adapters/snapshot/maintenance_gc_test.go +91 -0

@@ -0,0 +1,91 @@
	1	+	package snapshot
	2	+
	3	+	import (
	4	+	"context"
	5	+	"crypto/sha256"
	6	+	"os"
	7	+	"path/filepath"
	8	+	"slices"
	9	+	"testing"
	10	+
	11	+	"github.com/bnema/vev/internal/domain"
	12	+	"github.com/bnema/vev/internal/ports"
	13	+	"github.com/stretchr/testify/require"
	14	+	)
	15	+
	16	+	func TestCollectGarbage(t *testing.T) {
	17	+	t.Parallel()
	18	+	tests := []struct {
	19	+	name string
	20	+	committed uint64
	21	+	known bool
	22	+	wantGenerations []uint64
	23	+	}{
	24	+	{name: "incarnation absent from catalogue is removed whole"},
	25	+	{name: "committed and immediate predecessor survive", committed: 3, known: true, wantGenerations: []uint64{2, 3}},
	26	+	{name: "forward orphan newer than committed is removed", committed: 3, known: true, wantGenerations: []uint64{2, 3}},
	27	+	}
	28	+	for _, tt := range tests {
	29	+	t.Run(tt.name, func(t *testing.T) {
	30	+	t.Parallel()
	31	+	repository := NewRepository(privateDir(t))
	32	+	publications := publishGarbageCollectionGenerations(t, repository, "work", 4)
	33	+	id := publications[0].IncarnationID
	34	+	keep := make(map[domain.IncarnationID]domain.CheckpointRef)
	35	+	if tt.known {
	36	+	publication := publications[tt.committed-1]
	37	+	keep[id] = domain.CheckpointRef{Generation: tt.committed, ManifestDigest: sha256.Sum256(publication.Manifest)}
	38	+	}
	39	+
	40	+	require.NoError(t, repository.CollectGarbage(context.Background(), keep))
	41	+	if !tt.known {
	42	+	_, err := os.Stat(repository.sessionPath(id))
	43	+	require.ErrorIs(t, err, os.ErrNotExist)
	44	+	return
	45	+	}
	46	+	require.Equal(t, tt.wantGenerations, garbageCollectionGenerations(t, repository, id))
	47	+	retainedObjects := make(map[ports.SnapshotDigest]struct{})
	48	+	for _, publication := range publications[tt.committed-2 : tt.committed] {
	49	+	for _, object := range publication.Objects {
	50	+	retainedObjects[object.Digest] = struct{}{}
	51	+	}
	52	+	}
	53	+	for _, publication := range publications {
	54	+	for _, object := range publication.Objects {
	55	+	_, retained := retainedObjects[object.Digest]
	56	+	_, err := os.Stat(repository.objectPath(id, object.Digest))
	57	+	if retained {
	58	+	require.NoError(t, err)
	59	+	} else {
	60	+	require.ErrorIs(t, err, os.ErrNotExist)
	61	+	}
	62	+	}
	63	+	}
	64	+	})
	65	+	}
	66	+	}
	67	+
	68	+	func publishGarbageCollectionGenerations(t testing.T, repository Repository, name string, count uint64) []ports.SnapshotPublication {
	69	+	t.Helper()
	70	+	publications := make([]ports.SnapshotPublication, 0, count)
	71	+	for generation := uint64(1); generation <= count; generation++ {
	72	+	publication := repositoryPublicationAfter(t, repository, name, generation, []byte{byte(generation)})
	73	+	require.NoError(t, repository.Publish(context.Background(), publication))
	74	+	publications = append(publications, publication)
	75	+	}
	76	+	return publications
	77	+	}
	78	+
	79	+	func garbageCollectionGenerations(t testing.T, repository Repository, id domain.IncarnationID) []uint64 {
	80	+	t.Helper()
	81	+	entries, err := os.ReadDir(filepath.Join(repository.sessionPath(id), repositoryGenerations))
	82	+	require.NoError(t, err)
	83	+	generations := make([]uint64, 0, len(entries))
	84	+	for _, entry := range entries {
	85	+	if generation, ok := parseGenerationFilename(entry.Name()); ok {
	86	+	generations = append(generations, generation)
	87	+	}
	88	+	}
	89	+	slices.Sort(generations)
	90	+	return generations
	91	+	}

D internal/adapters/snapshot/maintenance_hardening_test.go +0 -141

@@ -1,141 +0,0 @@
1	-	package snapshot
2	-
3	-	import (
4	-	"context"
5	-	"errors"
6	-	"fmt"
7	-	"os"
8	-	"path/filepath"
9	-	"testing"
10	-	)
11	-
12	-	func TestRepositoryMaintainBoundsCursorsAcrossHostileObjectShards(t *testing.T) {
13	-	repo := NewRepository(privateDir(t))
14	-	key := legacyIncarnationID("named").String()
15	-	objects := filepath.Join(repo.legacySessionPath(key), repositoryObjectsDir)
16	-	const shards = maintenanceBatch * 2
17	-	for i := 0; i < shards; i++ {
18	-	shard := filepath.Join(objects, fmt.Sprintf("%03d", i))
19	-	if err := os.MkdirAll(shard, 0o700); err != nil {
20	-	t.Fatal(err)
21	-	}
22	-	if err := os.WriteFile(filepath.Join(shard, ".tmp-stale"), []byte("stale"), 0o600); err != nil {
23	-	t.Fatal(err)
24	-	}
25	-	// Force a resumable cursor in every attacker-controlled shard.
26	-	for j := 0; j < maintenanceBatch+1; j++ {
27	-	if err := os.WriteFile(filepath.Join(shard, fmt.Sprintf("live-%03d", j)), []byte("live"), 0o600); err != nil {
28	-	t.Fatal(err)
29	-	}
30	-	}
31	-	}
32	-
33	-	for pass := 0; pass < shards*4; pass++ {
34	-	if err := repo.Maintain(context.Background()); err != nil {
35	-	t.Fatal(err)
36	-	}
37	-	if got := len(repo.maintenanceCursors); got > maxMaintenanceCursors {
38	-	t.Fatalf("retained maintenance cursors = %d, want at most %d", got, maxMaintenanceCursors)
39	-	}
40	-	}
41	-	for i := 0; i < shards; i++ {
42	-	_, err := os.Lstat(filepath.Join(objects, fmt.Sprintf("%03d", i), ".tmp-stale"))
43	-	if !errors.Is(err, os.ErrNotExist) {
44	-	t.Fatalf("stale temporary object in shard %d = %v, want removed", i, err)
45	-	}
46	-	}
47	-	}
48	-
49	-	func TestRepositoryMaintainBoundsEmptyObjectTempShards(t *testing.T) {
50	-	repo := NewRepository(privateDir(t))
51	-	key := legacyIncarnationID("named").String()
52	-	objects := filepath.Join(repo.legacySessionPath(key), repositoryObjectsDir)
53	-	const shards = maintenanceBatch * 2
54	-	for i := 0; i < shards; i++ {
55	-	if err := os.MkdirAll(filepath.Join(objects, fmt.Sprintf("%03d", i)), 0o700); err != nil {
56	-	t.Fatal(err)
57	-	}
58	-	}
59	-
60	-	openedShards := make(map[string]struct{}, shards)
61	-	opensThisPass := 0
62	-	tempPass := false
63	-	repo.hooks.openMaintenanceDirectory = func(dir string) (maintenanceDirectory, error) {
64	-	if tempPass && filepath.Dir(dir) == objects {
65	-	opensThisPass++
66	-	openedShards[dir] = struct{}{}
67	-	}
68	-	file, err := os.Open(dir)
69	-	if err != nil {
70	-	return nil, err
71	-	}
72	-	return osMaintenanceDirectory{file: file}, nil
73	-	}
74	-
75	-	for pass := 0; pass < shards*3; pass++ {
76	-	opensThisPass = 0
77	-	tempPass = true
78	-	if err := repo.Maintain(context.Background()); err != nil {
79	-	t.Fatal(err)
80	-	}
81	-	tempPass = false
82	-	if opensThisPass > 1 {
83	-	t.Fatalf("empty object shards opened in maintenance pass %d = %d, want at most 1", pass, opensThisPass)
84	-	}
85	-	state := repo.maintenanceSessions[key]
86	-	if state != nil && state.objectTempsDone {
87	-	if got := len(openedShards); got != shards {
88	-	t.Fatalf("empty object shards opened = %d, want %d", got, shards)
89	-	}
90	-	return
91	-	}
92	-	}
93	-	t.Fatal("empty object temp shards were not eventually traversed")
94	-	}
95	-
96	-	func TestRepositoryMaintainBoundsAndResumesDeepWideQuarantine(t *testing.T) {
97	-	repo := NewRepository(privateDir(t))
98	-	quarantine := filepath.Join(repo.dir, repositorySessionsDir, ".deleting-hostile")
99	-	deep := quarantine
100	-	for i := 0; i < maintenanceBatch*2; i++ {
101	-	deep = filepath.Join(deep, "d")
102	-	}
103	-	leaf := filepath.Join(deep, "leaf")
104	-	if err := os.MkdirAll(filepath.Dir(leaf), 0o700); err != nil {
105	-	t.Fatal(err)
106	-	}
107	-	if err := os.WriteFile(leaf, []byte("stale"), 0o600); err != nil {
108	-	t.Fatal(err)
109	-	}
110	-	for i := 0; i < maintenanceBatch*2; i++ {
111	-	if err := os.WriteFile(filepath.Join(quarantine, fmt.Sprintf("wide-%03d", i)), []byte("stale"), 0o600); err != nil {
112	-	t.Fatal(err)
113	-	}
114	-	}
115	-
116	-	work := 0
117	-	repo.hooks.beforeMaintenanceWork = func(string) { work++ }
118	-	if err := repo.Maintain(context.Background()); err != nil {
119	-	t.Fatal(err)
120	-	}
121	-	if work == 0 \|\| work > maintenanceBatch {
122	-	t.Fatalf("quarantine work in one maintenance call = %d, want 1..%d", work, maintenanceBatch)
123	-	}
124	-	if _, err := os.Lstat(leaf); err != nil {
125	-	t.Fatalf("deep quarantine leaf removed despite bounded traversal: %v", err)
126	-	}
127	-
128	-	for pass := 0; pass < 1000; pass++ {
129	-	work = 0
130	-	if err := repo.Maintain(context.Background()); err != nil {
131	-	t.Fatal(err)
132	-	}
133	-	if work > maintenanceBatch {
134	-	t.Fatalf("quarantine work in pass %d = %d, want at most %d", pass, work, maintenanceBatch)
135	-	}
136	-	if _, err := os.Lstat(quarantine); errors.Is(err, os.ErrNotExist) {
137	-	return
138	-	}
139	-	}
140	-	t.Fatal("deep and wide quarantine was not eventually removed")
141	-	}

D internal/adapters/snapshot/maintenance_test.go +0 -1354

@@ -1,1354 +0,0 @@
1	-	package snapshot
2	-
3	-	import (
4	-	"cmp"
5	-	"context"
6	-	"crypto/sha256"
7	-	"encoding/binary"
8	-	"errors"
9	-	"fmt"
10	-	"io"
11	-	"os"
12	-	"path/filepath"
13	-	"slices"
14	-	"strings"
15	-	"sync"
16	-	"syscall"
17	-	"testing"
18	-	"time"
19	-	"unsafe"
20	-
21	-	"github.com/bnema/vev/internal/domain"
22	-	"github.com/bnema/vev/internal/ports"
23	-	"github.com/bnema/vev/pkg/safedir"
24	-	)
25	-
26	-	func TestReadMaintenanceDirentPreservesCursorAndCloseFailuresWithoutPaths(t *testing.T) {
27	-	privatePath := filepath.Join(t.TempDir(), "private-maintenance-directory")
28	-	closeCause := errors.New("injected close failure")
29	-	seekCause := errors.New("injected seek failure")
30	-	readCause := errors.New("injected read failure")
31	-
32	-	for _, tc := range []struct {
33	-	name string
34	-	offset int64
35	-	seekErr error
36	-	readErr error
37	-	wantCause []error
38	-	wantOps []string
39	-	}{
40	-	{
41	-	name: "close only",
42	-	wantCause: []error{closeCause},
43	-	wantOps: []string{"close maintenance directory"},
44	-	},
45	-	{
46	-	name: "seek and close",
47	-	offset: 1,
48	-	seekErr: &os.PathError{Op: "seek", Path: privatePath, Err: seekCause},
49	-	wantCause: []error{seekCause, closeCause},
50	-	wantOps: []string{"seek maintenance directory", "close maintenance directory"},
51	-	},
52	-	{
53	-	name: "read and close",
54	-	readErr: &os.PathError{Op: "read", Path: privatePath, Err: readCause},
55	-	wantCause: []error{readCause, closeCause},
56	-	wantOps: []string{"read maintenance directory", "close maintenance directory"},
57	-	},
58	-	} {
59	-	t.Run(tc.name, func(t *testing.T) {
60	-	repo := NewRepository(privateDir(t))
61	-	repo.hooks.openMaintenanceDirectory = func(string) (maintenanceDirectory, error) {
62	-	return fakeMaintenanceDirectory{
63	-	seekErr: tc.seekErr,
64	-	readErr: tc.readErr,
65	-	closeErr: &os.PathError{Op: "close", Path: privatePath, Err: closeCause},
66	-	}, nil
67	-	}
68	-
69	-	_, _, err := repo.readMaintenanceDirent(privatePath, 1, &maintenanceCursor{offset: tc.offset})
70	-	if err == nil {
71	-	t.Fatal("readMaintenanceDirent error = nil, want failure")
72	-	}
73	-	for _, cause := range tc.wantCause {
74	-	if !errors.Is(err, cause) {
75	-	t.Errorf("errors.Is(%v, %v) = false, want true", err, cause)
76	-	}
77	-	}
78	-	for _, operation := range tc.wantOps {
79	-	if !strings.Contains(err.Error(), operation) {
80	-	t.Errorf("error = %q, want stable operation context %q", err, operation)
81	-	}
82	-	}
83	-	if strings.Contains(err.Error(), privatePath) {
84	-	t.Errorf("error leaks private path: %q", err)
85	-	}
86	-	})
87	-	}
88	-	}
89	-
90	-	func TestReadMaintenanceDirentRejectsRecordShorterThanNameOffset(t *testing.T) {
91	-	repo := NewRepository(privateDir(t))
92	-	nameOffset := int(unsafe.Offsetof(syscall.Dirent{}.Name))
93	-	recordData := make([]byte, nameOffset)
94	-	reclenOffset := unsafe.Offsetof(syscall.Dirent{}.Reclen)
95	-	binary.NativeEndian.PutUint16(recordData[reclenOffset:], 1)
96	-
97	-	repo.hooks.openMaintenanceDirectory = func(string) (maintenanceDirectory, error) {
98	-	return fakeMaintenanceDirectory{data: recordData}, nil
99	-	}
100	-	_, _, err := repo.readMaintenanceDirent("unused", 1, &maintenanceCursor{})
101	-	if !errors.Is(err, syscall.EIO) {
102	-	t.Fatalf("readMaintenanceDirent error = %v, want EIO", err)
103	-	}
104	-	}
105	-
106	-	func TestDecodeMaintenanceDirentBoundsRecords(t *testing.T) {
107	-	nameOffset := int(unsafe.Offsetof(syscall.Dirent{}.Name))
108	-	reclenOffset := unsafe.Offsetof(syscall.Dirent{}.Reclen)
109	-	valid := make([]byte, nameOffset+len("entry")+1)
110	-	binary.NativeEndian.PutUint16(valid[reclenOffset:], uint16(len(valid)))
111	-	copy(valid[nameOffset:], "entry")
112	-
113	-	tests := []struct {
114	-	name string
115	-	data []byte
116	-	wantName string
117	-	wantErr bool
118	-	}{
119	-	{name: "truncated header", data: make([]byte, nameOffset-1), wantErr: true},
120	-	{name: "short record", data: append([]byte(nil), valid...), wantErr: true},
121	-	{name: "truncated record", data: valid[:nameOffset], wantErr: true},
122	-	{name: "record exceeds aligned storage", data: make([]byte, int(unsafe.Sizeof(syscall.Dirent{}))+1), wantErr: true},
123	-	{name: "valid", data: valid, wantName: "entry"},
124	-	}
125	-	binary.NativeEndian.PutUint16(tests[1].data[reclenOffset:], uint16(nameOffset-1))
126	-	binary.NativeEndian.PutUint16(tests[2].data[reclenOffset:], uint16(len(valid)))
127	-	binary.NativeEndian.PutUint16(tests[3].data[reclenOffset:], uint16(len(tests[3].data)))
128	-
129	-	for _, tc := range tests {
130	-	t.Run(tc.name, func(t *testing.T) {
131	-	record, name, reclen, err := decodeMaintenanceDirent(tc.data)
132	-	if tc.wantErr {
133	-	if !errors.Is(err, syscall.EIO) {
134	-	t.Fatalf("decodeMaintenanceDirent() error = %v, want EIO", err)
135	-	}
136	-	return
137	-	}
138	-	if err != nil {
139	-	t.Fatal(err)
140	-	}
141	-	if record == nil {
142	-	t.Fatal("decodeMaintenanceDirent() record = nil")
143	-	}
144	-	if reclen != len(valid) {
145	-	t.Fatalf("decodeMaintenanceDirent() reclen = %d, want %d", reclen, len(valid))
146	-	}
147	-	if got := string(name[:len(tc.wantName)]); got != tc.wantName {
148	-	t.Fatalf("decodeMaintenanceDirent() name = %q, want %q", got, tc.wantName)
149	-	}
150	-	})
151	-	}
152	-	}
153	-
154	-	type fakeMaintenanceDirectory struct {
155	-	data []byte
156	-	seekOffset int64
157	-	seekErr error
158	-	readErr error
159	-	closeErr error
160	-	}
161	-
162	-	func (d fakeMaintenanceDirectory) Seek(int64, int) (int64, error) { return d.seekOffset, d.seekErr }
163	-	func (d fakeMaintenanceDirectory) ReadDirent(buffer []byte) (int, error) {
164	-	if d.readErr != nil {
165	-	return 0, d.readErr
166	-	}
167	-	return copy(buffer, d.data), nil
168	-	}
169	-	func (d fakeMaintenanceDirectory) Close() error { return d.closeErr }
170	-
171	-	func TestReadMaintenanceDirentDrainsDarwinBatchesWithoutStarvation(t *testing.T) {
172	-	dir := privateDir(t)
173	-	if err := os.Mkdir(dir, 0o700); err != nil {
174	-	t.Fatal(err)
175	-	}
176	-	batches := [][]string{
177	-	{"one", "two"},
178	-	{"three", "four", "five", "six"},
179	-	}
180	-	for _, batch := range batches {
181	-	for _, name := range batch {
182	-	if err := os.WriteFile(filepath.Join(dir, name), nil, 0o600); err != nil {
183	-	t.Fatal(err)
184	-	}
185	-	}
186	-	}
187	-
188	-	file := &fakeMultiBatchMaintenanceDirectory{batches: [][]byte{
189	-	maintenanceDirentBatch(batches[0]...),
190	-	maintenanceDirentBatch(batches[1]...),
191	-	}}
192	-	repo := NewRepository(dir)
193	-	repo.hooks.openMaintenanceDirectory = func(string) (maintenanceDirectory, error) { return file, nil }
194	-	cursor := &maintenanceCursor{}
195	-
196	-	entries, done, err := repo.readMaintenanceDirentWithDrain(dir, 4, cursor, true, maintenanceTestDirectoryCookie)
197	-	if err != nil {
198	-	t.Fatal(err)
199	-	}
200	-	if done {
201	-	t.Fatal("readMaintenanceDirentWithDrain() done = true, want false before EOF")
202	-	}
203	-	if got, want := maintenanceEntryNames(entries), []string{"one", "two", "three", "four"}; !slices.Equal(got, want) {
204	-	t.Fatalf("first entries = %v, want %v", got, want)
205	-	}
206	-	if got, want := maintenanceEntryNames(cursor.pending), []string{"five", "six"}; !slices.Equal(got, want) {
207	-	t.Fatalf("pending entries = %v, want exactly final-buffer excess %v", got, want)
208	-	}
209	-	if got := file.reads; got != 2 {
210	-	t.Fatalf("ReadDirent calls = %d, want 2 to fill the requested batch", got)
211	-	}
212	-
213	-	id := maintenanceCursorID(dir, "test")
214	-	repo.maintenanceCursors = map[string]*maintenanceCursor{id: cursor}
215	-	entries, done, err = repo.readMaintenanceDir(dir, 4, "test")
216	-	if err != nil {
217	-	t.Fatal(err)
218	-	}
219	-	if done {
220	-	t.Fatal("pending batch done = true, want false before EOF")
221	-	}
222	-	if got, want := maintenanceEntryNames(entries), []string{"five", "six"}; !slices.Equal(got, want) {
223	-	t.Fatalf("pending entries = %v, want %v", got, want)
224	-	}
225	-	if got := file.reads; got != 2 {
226	-	t.Fatalf("ReadDirent calls while delivering pending entries = %d, want 2", got)
227	-	}
228	-
229	-	entries, done, err = repo.readMaintenanceDir(dir, 4, "test")
230	-	if err != nil {
231	-	t.Fatal(err)
232	-	}
233	-	if !done {
234	-	t.Fatal("EOF batch done = false, want true")
235	-	}
236	-	if len(entries) != 0 {
237	-	t.Fatalf("EOF entries = %v, want none", entries)
238	-	}
239	-	if _, ok := repo.maintenanceCursors[id]; ok {
240	-	t.Fatal("EOF cursor retained after pending entries were delivered")
241	-	}
242	-	}
243	-
244	-	func maintenanceEntryNames(entries []maintenanceDirEntry) []string {
245	-	names := make([]string, len(entries))
246	-	for i, entry := range entries {
247	-	names[i] = entry.name
248	-	}
249	-	return names
250	-	}
251	-
252	-	func maintenanceDirentBatch(names ...string) []byte {
253	-	nameOffset := int(unsafe.Offsetof(syscall.Dirent{}.Name))
254	-	reclenOffset := unsafe.Offsetof(syscall.Dirent{}.Reclen)
255	-	var data []byte
256	-	for _, name := range names {
257	-	record := make([]byte, nameOffset+len(name)+1)
258	-	binary.NativeEndian.PutUint16(record[reclenOffset:], uint16(len(record)))
259	-	copy(record[nameOffset:], name)
260	-	data = append(data, record...)
261	-	}
262	-	return data
263	-	}
264	-
265	-	type fakeMultiBatchMaintenanceDirectory struct {
266	-	batches [][]byte
267	-	index int
268	-	reads int
269	-	}
270	-
271	-	func (d *fakeMultiBatchMaintenanceDirectory) Seek(offset int64, whence int) (int64, error) {
272	-	if whence == io.SeekCurrent {
273	-	return int64(d.index), nil
274	-	}
275	-	d.index = int(offset)
276	-	return offset, nil
277	-	}
278	-
279	-	func (d *fakeMultiBatchMaintenanceDirectory) ReadDirent(buffer []byte) (int, error) {
280	-	d.reads++
281	-	if d.index == len(d.batches) {
282	-	return 0, nil
283	-	}
284	-	batch := d.batches[d.index]
285	-	d.index++
286	-	return copy(buffer, batch), nil
287	-	}
288	-
289	-	func (d *fakeMultiBatchMaintenanceDirectory) Close() error { return nil }
290	-
291	-	func maintenanceTestDirectoryCookie(file maintenanceDirectory, _ *syscall.Dirent, _ int) (int64, error) {
292	-	return file.Seek(0, io.SeekCurrent)
293	-	}
294	-
295	-	func TestRepositoryDeleteQuarantinesCanonicalSessionBeforeCleanup(t *testing.T) {
296	-	repo := NewRepository(privateDir(t))
297	-	if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state"))); err != nil {
298	-	t.Fatal(err)
299	-	}
300	-	if err := repo.Delete(context.Background(), "named"); err != nil {
301	-	t.Fatal(err)
302	-	}
303	-
304	-	canonical := repo.legacySessionPath(legacyIncarnationID("named").String())
305	-	if _, err := os.Lstat(canonical); !errors.Is(err, os.ErrNotExist) {
306	-	t.Fatalf("canonical session after Delete = %v, want not exist", err)
307	-	}
308	-	entries, err := os.ReadDir(filepath.Join(repo.dir, repositorySessionsDir))
309	-	if err != nil {
310	-	t.Fatal(err)
311	-	}
312	-	if len(entries) != 1 \|\| !isQuarantine(entries[0].Name()) \|\| !entries[0].IsDir() {
313	-	t.Fatalf("session entries after Delete = %#v, want one quarantine", entries)
314	-	}
315	-	if err := repo.Maintain(context.Background()); err != nil {
316	-	t.Fatal(err)
317	-	}
318	-	entries, err = os.ReadDir(filepath.Join(repo.dir, repositorySessionsDir))
319	-	if err != nil {
320	-	t.Fatal(err)
321	-	}
322	-	if len(entries) != 0 {
323	-	t.Fatalf("session entries after Maintain = %#v, want none", entries)
324	-	}
325	-	}
326	-
327	-	func TestRepositoryDeleteReturnsRootSyncFailureAfterQuarantine(t *testing.T) {
328	-	repo := NewRepository(privateDir(t))
329	-	if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state"))); err != nil {
330	-	t.Fatal(err)
331	-	}
332	-	injected := errors.New("sessions sync")
333	-	repo.hooks.syncDirectory = func(string) error { return injected }
334	-	if err := repo.Delete(context.Background(), "named"); !errors.Is(err, injected) {
335	-	t.Fatalf("Delete error = %v, want root sync failure", err)
336	-	}
337	-	if _, err := os.Lstat(repo.legacySessionPath(legacyIncarnationID("named").String())); !errors.Is(err, os.ErrNotExist) {
338	-	t.Fatalf("canonical session after failed sync = %v, want not exist", err)
339	-	}
340	-	}
341	-
342	-	func TestRepositoryDeleteWaitsForPublication(t *testing.T) {
343	-	repo := NewRepository(privateDir(t))
344	-	if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("first"))); err != nil {
345	-	t.Fatal(err)
346	-	}
347	-	entered := make(chan struct{})
348	-	release := make(chan struct{})
349	-	repo.hooks.beforeHeadWrite = func(string) error {
350	-	close(entered)
351	-	<-release
352	-	return nil
353	-	}
354	-	publishDone := make(chan error, 1)
355	-	go func() {
356	-	publishDone <- repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", 2, []byte("second")))
357	-	}()
358	-	<-entered
359	-	deleteReached := make(chan struct{})
360	-	deleteRelease := make(chan struct{})
361	-	repo.hooks.beforeSessionLock = waitAtSessionLock(deleteReached, deleteRelease)
362	-	deleteDone := make(chan error, 1)
363	-	go func() { deleteDone <- repo.Delete(context.Background(), "named") }()
364	-	<-deleteReached
365	-	// Delete is parked at the session-lock boundary while Publish owns it.
366	-	close(deleteRelease)
367	-	close(release)
368	-	if err := <-publishDone; err != nil {
369	-	t.Fatal(err)
370	-	}
371	-	if err := <-deleteDone; err != nil {
372	-	t.Fatal(err)
373	-	}
374	-	if _, err := os.Lstat(repo.legacySessionPath(legacyIncarnationID("named").String())); !errors.Is(err, os.ErrNotExist) {
375	-	t.Fatalf("canonical session after concurrent Delete = %v, want not exist", err)
376	-	}
377	-	}
378	-
379	-	func TestRepositoryMaintainRetainsNewestTwoCompleteGenerations(t *testing.T) {
380	-	repo := NewRepository(privateDir(t))
381	-	for generation := uint64(1); generation <= 3; generation++ {
382	-	if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, []byte{byte(generation)})); err != nil {
383	-	t.Fatal(err)
384	-	}
385	-	}
386	-	key := legacyIncarnationID("named").String()
387	-	for pass := range maintenanceBatch {
388	-	if err := repo.Maintain(context.Background()); err != nil {
389	-	t.Fatal(err)
390	-	}
391	-	_, err := os.Lstat(repo.legacyManifestPath(key, 1))
392	-	if errors.Is(err, os.ErrNotExist) {
393	-	break
394	-	}
395	-	if err != nil {
396	-	t.Fatalf("old manifest during Maintain = %v", err)
397	-	}
398	-	if pass == maintenanceBatch-1 {
399	-	t.Fatal("old manifest was not eventually collected")
400	-	}
401	-	}
402	-	for _, generation := range []uint64{2, 3} {
403	-	if _, err := os.Lstat(repo.legacyManifestPath(key, generation)); err != nil {
404	-	t.Fatalf("retained manifest %d: %v", generation, err)
405	-	}
406	-	}
407	-	if _, err := os.Lstat(repo.legacyManifestPath(key, 1)); !errors.Is(err, os.ErrNotExist) {
408	-	t.Fatalf("old manifest after Maintain = %v, want not exist", err)
409	-	}
410	-	for _, generation := range []uint64{2, 3} {
411	-	if _, err := loadGenerationCheckpoint(context.Background(), repo, legacyIncarnationID("named"), "named", generation); err != nil {
412	-	t.Fatalf("LoadCheckpoint after collecting generation %d: %v", generation, err)
413	-	}
414	-	}
415	-	}
416	-
417	-	func TestRepositoryMaintainPreservesIncompleteManifestReferences(t *testing.T) {
418	-	repo := NewRepository(privateDir(t))
419	-	for generation := uint64(1); generation <= 3; generation++ {
420	-	if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, []byte{byte(generation)})); err != nil {
421	-	t.Fatal(err)
422	-	}
423	-	}
424	-	key := legacyIncarnationID("named").String()
425	-	manifest := repositoryPublication(t, "named", 3, []byte{3})
426	-	if err := os.Remove(repo.legacyObjectPath(key, manifest.Objects[1].Digest)); err != nil {
427	-	t.Fatal(err)
428	-	}
429	-	if err := repo.Maintain(context.Background()); err != nil {
430	-	t.Fatal(err)
431	-	}
432	-	if _, err := os.Lstat(repo.legacyManifestPath(key, 3)); err != nil {
433	-	t.Fatalf("incomplete manifest removed: %v", err)
434	-	}
435	-	if _, err := os.Lstat(repo.legacyObjectPath(key, manifest.Objects[0].Digest)); err != nil {
436	-	t.Fatalf("object referenced by incomplete manifest removed: %v", err)
437	-	}
438	-	}
439	-
440	-	func TestRepositoryCancellationAfterSessionLockWaitPreventsMutations(t *testing.T) {
441	-	t.Run("publish", func(t *testing.T) {
442	-	repo := NewRepository(privateDir(t))
443	-	key := legacyIncarnationID("named").String()
444	-	lock := repo.lockSession(key)
445	-	ctx, cancel := context.WithCancel(context.Background())
446	-	reached := make(chan struct{})
447	-	release := make(chan struct{})
448	-	repo.hooks.beforeSessionLock = waitAtSessionLock(reached, release)
449	-	done := make(chan error, 1)
450	-	go func() { done <- repo.Publish(ctx, repositoryPublication(t, "named", 1, []byte("state"))) }()
451	-	<-reached
452	-	cancel()
453	-	close(release)
454	-	repo.unlockSession(lock)
455	-	if err := <-done; !errors.Is(err, context.Canceled) {
456	-	t.Fatalf("Publish error = %v, want canceled", err)
457	-	}
458	-	if _, err := os.Lstat(repo.dir); !errors.Is(err, os.ErrNotExist) {
459	-	t.Fatalf("Publish mutated canceled repository: %v", err)
460	-	}
461	-	})
462	-	t.Run("load and delete", func(t *testing.T) {
463	-	repo := NewRepository(privateDir(t))
464	-	publication := repositoryPublication(t, "named", 1, []byte("state"))
465	-	if err := repo.Publish(context.Background(), publication); err != nil {
466	-	t.Fatal(err)
467	-	}
468	-	key := publication.IncarnationID.String()
469	-	for _, operation := range []struct {
470	-	name string
471	-	run func(context.Context) error
472	-	}{
473	-	{"load", func(ctx context.Context) error { _, err := loadPublication(ctx, repo, publication); return err }},
474	-	{"delete", func(ctx context.Context) error { return repo.Delete(ctx, "named") }},
475	-	} {
476	-	t.Run(operation.name, func(t *testing.T) {
477	-	lock := repo.lockSession(key)
478	-	ctx, cancel := context.WithCancel(context.Background())
479	-	reached := make(chan struct{})
480	-	release := make(chan struct{})
481	-	repo.hooks.beforeSessionLock = waitAtSessionLock(reached, release)
482	-	done := make(chan error, 1)
483	-	go func() { done <- operation.run(ctx) }()
484	-	<-reached
485	-	cancel()
486	-	close(release)
487	-	repo.unlockSession(lock)
488	-	if err := <-done; !errors.Is(err, context.Canceled) {
489	-	t.Fatalf("%s error = %v, want canceled", operation.name, err)
490	-	}
491	-	repo.hooks.beforeSessionLock = nil
492	-	if _, err := loadPublication(context.Background(), repo, publication); err != nil {
493	-	t.Fatalf("%s mutated session: %v", operation.name, err)
494	-	}
495	-	})
496	-	}
497	-	})
498	-	t.Run("maintain", func(t *testing.T) {
499	-	repo := NewRepository(privateDir(t))
500	-	publication := repositoryPublication(t, "named", 1, []byte("state"))
501	-	if err := repo.Publish(context.Background(), publication); err != nil {
502	-	t.Fatal(err)
503	-	}
504	-	lock := repo.lockSession(publication.IncarnationID.String())
505	-	ctx, cancel := context.WithCancel(context.Background())
506	-	reached := make(chan struct{})
507	-	release := make(chan struct{})
508	-	repo.hooks.beforeSessionLock = waitAtSessionLock(reached, release)
509	-	done := make(chan error, 1)
510	-	go func() { done <- repo.Maintain(ctx) }()
511	-	<-reached
512	-	cancel()
513	-	close(release)
514	-	repo.unlockSession(lock)
515	-	if err := <-done; !errors.Is(err, context.Canceled) {
516	-	t.Fatalf("Maintain error = %v, want canceled", err)
517	-	}
518	-	repo.hooks.beforeSessionLock = nil
519	-	if _, err := loadPublication(context.Background(), repo, publication); err != nil {
520	-	t.Fatalf("Maintain mutated session: %v", err)
521	-	}
522	-	})
523	-	}
524	-
525	-	func waitAtSessionLock(reached chan<- struct{}, release <-chan struct{}) func(string) {
526	-	return func(string) {
527	-	close(reached)
528	-	<-release
529	-	}
530	-	}
531	-
532	-	func TestRepositoryDeleteRetriesPendingQuarantineSyncWithoutDeletingRecreatedSession(t *testing.T) {
533	-	repo := NewRepository(privateDir(t))
534	-	if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("old"))); err != nil {
535	-	t.Fatal(err)
536	-	}
537	-	injected := errors.New("sessions sync")
538	-	repo.hooks.syncDirectory = func(string) error { return injected }
539	-	if err := repo.Delete(context.Background(), "named"); !errors.Is(err, injected) {
540	-	t.Fatalf("Delete error = %v, want sync failure", err)
541	-	}
542	-	repo.hooks.syncDirectory = nil
543	-	recreated := repositoryPublication(t, "named", 1, []byte("new"))
544	-	if err := repo.Publish(context.Background(), recreated); err != nil {
545	-	t.Fatal(err)
546	-	}
547	-	if err := repo.Delete(context.Background(), "named"); err != nil {
548	-	t.Fatalf("retry Delete: %v", err)
549	-	}
550	-	if _, err := loadPublication(context.Background(), repo, recreated); err != nil {
551	-	t.Fatalf("recreated session was deleted: %v", err)
552	-	}
553	-	}
554	-
555	-	func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) {
556	-	repo := NewRepository(privateDir(t))
557	-	quarantine := filepath.Join(repo.dir, repositorySessionsDir, ".deleting-named-test")
558	-	for i := range maintenanceBatch + 1 {
559	-	path := filepath.Join(quarantine, "nested", string(rune('a'+i)))
560	-	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
561	-	t.Fatal(err)
562	-	}
563	-	if err := os.WriteFile(path, []byte("stale"), 0o600); err != nil {
564	-	t.Fatal(err)
565	-	}
566	-	}
567	-	if err := repo.Maintain(context.Background()); err != nil {
568	-	t.Fatal(err)
569	-	}
570	-	if _, err := os.Lstat(quarantine); errors.Is(err, os.ErrNotExist) {
571	-	t.Fatalf("quarantine after one bounded call = %v, want remaining tree", err)
572	-	}
573	-	for range 12 {
574	-	if err := repo.Maintain(context.Background()); err != nil {
575	-	t.Fatal(err)
576	-	}
577	-	if _, err := os.Lstat(quarantine); errors.Is(err, os.ErrNotExist) {
578	-	return
579	-	}
580	-	}
581	-	if _, err := os.Lstat(quarantine); !errors.Is(err, os.ErrNotExist) {
582	-	t.Fatalf("quarantine after resumed maintenance = %v, want removed", err)
583	-	}
584	-	}
585	-
586	-	func TestRepositoryMaintainClosesContinuationDirectories(t *testing.T) {
587	-	repo := NewRepository(privateDir(t))
588	-	sessions := filepath.Join(repo.dir, repositorySessionsDir)
589	-	for i := range maintenanceBatch + 1 {
590	-	generations := filepath.Join(sessions, fmt.Sprintf("named-%03d", i), repositoryGenerations)
591	-	if err := os.MkdirAll(generations, 0o700); err != nil {
592	-	t.Fatal(err)
593	-	}
594	-	for j := range maintenanceBatch + 1 {
595	-	if err := os.WriteFile(filepath.Join(generations, fmt.Sprintf("unclassified-%03d", j)), []byte("state"), 0o600); err != nil {
596	-	t.Fatal(err)
597	-	}
598	-	}
599	-	}
600	-	before := openDescriptorCount(t)
601	-	if err := repo.Maintain(context.Background()); err != nil {
602	-	t.Fatal(err)
603	-	}
604	-	if got := openDescriptorCount(t); got > before+4 {
605	-	t.Fatalf("open descriptors after bounded maintenance = %d, want at most %d", got, before+4)
606	-	}
607	-	}
608	-
609	-	func TestRepositoryMaintainBoundsQueuedShardNames(t *testing.T) {
610	-	repo := NewRepository(privateDir(t))
611	-	key := legacyIncarnationID("named").String()
612	-	root := filepath.Join(repo.legacySessionPath(key), repositoryObjectsDir)
613	-	for i := range maintenanceBatch * 2 {
614	-	if err := os.MkdirAll(filepath.Join(root, fmt.Sprintf("%02x", i)), 0o700); err != nil {
615	-	t.Fatal(err)
616	-	}
617	-	}
618	-	repo.maintenanceCursors = make(map[string]*maintenanceCursor)
619	-	state := &sessionMaintenance{marked: make(map[uint64]manifestMaintenance), markDone: true}
620	-	for range maintenanceBatch + 2 {
621	-	budget := maintenanceBatch
622	-	if err := repo.sweepSession(context.Background(), key, state, &budget); err != nil {
623	-	t.Fatal(err)
624	-	}
625	-	if state.sweepShard != "" && len(repo.maintenanceCursors) > maxMaintenanceCursors {
626	-	t.Fatalf("retained maintenance cursors = %d, want at most %d", len(repo.maintenanceCursors), maxMaintenanceCursors)
627	-	}
628	-	}
629	-	}
630	-
631	-	func TestRepositoryMaintainResumesCursorAfterDirectoryMutation(t *testing.T) {
632	-	repo := NewRepository(privateDir(t))
633	-	if err := os.Mkdir(repo.dir, 0o700); err != nil {
634	-	t.Fatal(err)
635	-	}
636	-	for i := range maintenanceBatch + 1 {
637	-	if err := os.WriteFile(filepath.Join(repo.dir, fmt.Sprintf("live-%03d", i)), []byte("live"), 0o600); err != nil {
638	-	t.Fatal(err)
639	-	}
640	-	}
641	-	if err := repo.Maintain(context.Background()); err != nil {
642	-	t.Fatal(err)
643	-	}
644	-	// Mutating entries ahead of the saved cookie must not reset traversal to
645	-	// the prefix or lose a stale entry appended after that cookie.
646	-	if err := os.Remove(filepath.Join(repo.dir, "live-000")); err != nil {
647	-	t.Fatal(err)
648	-	}
649	-	stale := filepath.Join(repo.dir, ".tmp-appended")
650	-	if err := os.WriteFile(stale, []byte("stale"), 0o600); err != nil {
651	-	t.Fatal(err)
652	-	}
653	-	for range 4 {
654	-	if err := repo.Maintain(context.Background()); err != nil {
655	-	t.Fatal(err)
656	-	}
657	-	if _, err := os.Lstat(stale); errors.Is(err, os.ErrNotExist) {
658	-	return
659	-	}
660	-	}
661	-	t.Fatal("stale entry appended after cursor mutation was not eventually removed")
662	-	}
663	-
664	-	func TestRepositoryMaintainDoesNotStarveLaterTemporaryEntries(t *testing.T) {
665	-	repo := NewRepository(privateDir(t))
666	-	if err := os.Mkdir(repo.dir, 0o700); err != nil {
667	-	t.Fatal(err)
668	-	}
669	-	// The live entries precede the stale entries in directory insertion order.
670	-	// A fresh ReadDir from the start on every call would never reach the latter.
671	-	for i := range maintenanceBatch + 1 {
672	-	if err := os.WriteFile(filepath.Join(repo.dir, "live-"+string(rune('a'+i))), []byte("live"), 0o600); err != nil {
673	-	t.Fatal(err)
674	-	}
675	-	}
676	-	for i := range maintenanceBatch + 1 {
677	-	if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-later-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil {
678	-	t.Fatal(err)
679	-	}
680	-	}
681	-	for range 4 {
682	-	if err := repo.Maintain(context.Background()); err != nil {
683	-	t.Fatal(err)
684	-	}
685	-	}
686	-	entries, err := os.ReadDir(repo.dir)
687	-	if err != nil {
688	-	t.Fatal(err)
689	-	}
690	-	for _, entry := range entries {
691	-	if strings.HasPrefix(entry.Name(), ".tmp-") {
692	-	t.Fatalf("stale entry %q was starved", entry.Name())
693	-	}
694	-	}
695	-	}
696	-
697	-	func TestRepositoryMaintainMarksLargeGenerationSetBeforeSweeping(t *testing.T) {
698	-	repo := NewRepository(privateDir(t))
699	-	var newest ports.SnapshotPublication
700	-	for generation := uint64(1); generation <= maintenanceBatch+2; generation++ {
701	-	publication := repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))
702	-	if err := repo.Publish(context.Background(), publication); err != nil {
703	-	t.Fatal(err)
704	-	}
705	-	newest = publication
706	-	}
707	-	key := legacyIncarnationID("named").String()
708	-	stale := sha256.Sum256([]byte("stale object"))
709	-	stalePath := repo.legacyObjectPath(key, stale)
710	-	if err := os.MkdirAll(filepath.Dir(stalePath), 0o700); err != nil {
711	-	t.Fatal(err)
712	-	}
713	-	if err := os.WriteFile(stalePath, []byte("stale object"), 0o600); err != nil {
714	-	t.Fatal(err)
715	-	}
716	-
717	-	// The first call has classified only one manifest batch. It must not sweep
718	-	// even an otherwise stale object, nor a blob owned by a later manifest.
719	-	if err := repo.Maintain(context.Background()); err != nil {
720	-	t.Fatal(err)
721	-	}
722	-	if _, err := os.Lstat(stalePath); err != nil {
723	-	t.Fatalf("stale object swept before complete mark pass: %v", err)
724	-	}
725	-	for _, object := range newest.Objects {
726	-	if _, err := os.Lstat(repo.legacyObjectPath(key, object.Digest)); err != nil {
727	-	t.Fatalf("object referenced by not-yet-classified manifest removed: %v", err)
728	-	}
729	-	}
730	-
731	-	for i := range 600 {
732	-	if err := repo.Maintain(context.Background()); err != nil {
733	-	t.Fatal(err)
734	-	}
735	-	if _, err := os.Lstat(stalePath); errors.Is(err, os.ErrNotExist) {
736	-	break
737	-	}
738	-	if i == 599 {
739	-	t.Fatal("stale object was not eventually swept")
740	-	}
741	-	}
742	-	for _, generation := range []uint64{maintenanceBatch + 1, maintenanceBatch + 2} {
743	-	if _, err := os.Lstat(repo.legacyManifestPath(key, generation)); err != nil {
744	-	t.Fatalf("newest complete manifest %d removed: %v", generation, err)
745	-	}
746	-	}
747	-	}
748	-
749	-	func TestRepositoryMaintainUsesFixedRemovalBatch(t *testing.T) {
750	-	repo := NewRepository(privateDir(t))
751	-	if err := os.Mkdir(repo.dir, 0o700); err != nil {
752	-	t.Fatal(err)
753	-	}
754	-	for i := range maintenanceBatch + 1 {
755	-	if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil {
756	-	t.Fatal(err)
757	-	}
758	-	}
759	-	if err := repo.Maintain(context.Background()); err != nil {
760	-	t.Fatal(err)
761	-	}
762	-	entries, err := os.ReadDir(repo.dir)
763	-	if err != nil {
764	-	t.Fatal(err)
765	-	}
766	-	if len(entries) != 1 {
767	-	t.Fatalf("remaining stale entries = %d, want 1 after one batch", len(entries))
768	-	}
769	-	}
770	-
771	-	func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) {
772	-	repo := NewRepository(privateDir(t))
773	-	if err := os.MkdirAll(filepath.Join(repo.dir, repositorySessionsDir), 0o700); err != nil {
774	-	t.Fatal(err)
775	-	}
776	-	if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-consume-budget"), []byte("stale"), 0o600); err != nil {
777	-	t.Fatal(err)
778	-	}
779	-
780	-	removed := make(map[string]int)
781	-	for i := range maintenanceBatch + 1 {
782	-	path := filepath.Join(repo.dir, repositorySessionsDir, fmt.Sprintf(".deleting-%03d", i))
783	-	if err := os.Mkdir(path, 0o700); err != nil {
784	-	t.Fatal(err)
785	-	}
786	-	}
787	-	repo.hooks.remove = func(path string) error {
788	-	if strings.HasPrefix(filepath.Base(path), ".deleting-") {
789	-	removed[path]++
790	-	}
791	-	return nil
792	-	}
793	-
794	-	sessions := filepath.Join(repo.dir, repositorySessionsDir)
795	-	f, err := os.Open(sessions)
796	-	if err != nil {
797	-	t.Fatal(err)
798	-	}
799	-	fetched, err := f.ReadDir(maintenanceBatch)
800	-	if err != nil {
801	-	t.Fatal(err)
802	-	}
803	-	unread, err := f.ReadDir(1)
804	-	if err != nil {
805	-	t.Fatal(err)
806	-	}
807	-	if err := f.Close(); err != nil {
808	-	t.Fatal(err)
809	-	}
810	-	queued := fetched[len(fetched)-1].Name()
811	-	for i := range maintenanceBatch + 4 {
812	-	if err := repo.Maintain(context.Background()); err != nil {
813	-	t.Fatal(err)
814	-	}
815	-	if _, err := os.Lstat(filepath.Join(sessions, queued)); errors.Is(err, os.ErrNotExist) {
816	-	break
817	-	} else if i == maintenanceBatch+3 {
818	-	t.Fatalf("fetched but unprocessed session %q = %v, want eventually removed", queued, err)
819	-	}
820	-	}
821	-	if _, err := os.Lstat(filepath.Join(sessions, unread[0].Name())); err != nil {
822	-	t.Fatalf("unread session %q = %v, want queued entry to run first", unread[0].Name(), err)
823	-	}
824	-
825	-	for range maintenanceBatch + 4 {
826	-	if err := repo.Maintain(context.Background()); err != nil {
827	-	t.Fatal(err)
828	-	}
829	-	entries, err := os.ReadDir(sessions)
830	-	if err != nil {
831	-	t.Fatal(err)
832	-	}
833	-	if len(entries) == 0 {
834	-	break
835	-	}
836	-	}
837	-	entries, err := os.ReadDir(sessions)
838	-	if err != nil {
839	-	t.Fatal(err)
840	-	}
841	-	if len(entries) != 0 {
842	-	t.Fatalf("queued sessions after resumed maintenance = %d, want none", len(entries))
843	-	}
844	-	if len(removed) != maintenanceBatch+1 {
845	-	t.Fatalf("removed sessions = %d, want %d", len(removed), maintenanceBatch+1)
846	-	}
847	-	for path, calls := range removed {
848	-	if calls != 1 {
849	-	t.Fatalf("remove calls for %q = %d, want one", path, calls)
850	-	}
851	-	}
852	-	}
853	-
854	-	func TestRepositoryMaintainClassifiesUnpublishedGenerationsBeforeSweep(t *testing.T) {
855	-	for _, head := range []struct {
856	-	name string
857	-	set func(t testing.T, repo Repository, key string)
858	-	}{
859	-	{
860	-	name: "missing",
861	-	set: func(t testing.T, repo Repository, key string) {
862	-	t.Helper()
863	-	if err := os.Remove(repo.legacyHeadPath(key)); err != nil {
864	-	t.Fatal(err)
865	-	}
866	-	},
867	-	},
868	-	{
869	-	name: "corrupt",
870	-	set: func(t testing.T, repo Repository, key string) {
871	-	t.Helper()
872	-	if err := os.WriteFile(repo.legacyHeadPath(key), []byte("corrupt"), 0o600); err != nil {
873	-	t.Fatal(err)
874	-	}
875	-	},
876	-	},
877	-	} {
878	-	t.Run(head.name, func(t *testing.T) {
879	-	repo := NewRepository(privateDir(t))
880	-	key := legacyIncarnationID("named").String()
881	-	publications := make([]ports.SnapshotPublication, 0, maintenanceBatch+2)
882	-	for generation := uint64(1); generation <= maintenanceBatch+2; generation++ {
883	-	publication := repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))
884	-	if err := repo.Publish(context.Background(), publication); err != nil {
885	-	t.Fatal(err)
886	-	}
887	-	publications = append(publications, publication)
888	-	}
889	-	head.set(t, repo, key)
890	-	stale := sha256.Sum256([]byte("unpublished stale object"))
891	-	stalePath := repo.legacyObjectPath(key, stale)
892	-	if err := os.MkdirAll(filepath.Dir(stalePath), 0o700); err != nil {
893	-	t.Fatal(err)
894	-	}
895	-	if err := os.WriteFile(stalePath, []byte("unpublished stale object"), 0o600); err != nil {
896	-	t.Fatal(err)
897	-	}
898	-
899	-	if err := repo.Maintain(context.Background()); err != nil {
900	-	t.Fatal(err)
901	-	}
902	-	if _, err := os.Lstat(stalePath); err != nil {
903	-	t.Fatalf("stale object swept before full manifest classification: %v", err)
904	-	}
905	-	for _, publication := range publications {
906	-	if _, err := os.Lstat(repo.legacyObjectPath(key, publication.Objects[0].Digest)); err != nil {
907	-	t.Fatalf("object for unpublished generation %d removed early: %v", publication.Generation, err)
908	-	}
909	-	}
910	-
911	-	for i := range 600 {
912	-	if err := repo.Maintain(context.Background()); err != nil {
913	-	t.Fatal(err)
914	-	}
915	-	if _, err := os.Lstat(stalePath); errors.Is(err, os.ErrNotExist) {
916	-	break
917	-	} else if i == 599 {
918	-	t.Fatal("stale object was not swept after complete classification")
919	-	}
920	-	}
921	-	for _, publication := range publications {
922	-	if _, err := os.Lstat(repo.legacyObjectPath(key, publication.Objects[0].Digest)); err != nil {
923	-	t.Fatalf("object for unpublished generation %d removed: %v", publication.Generation, err)
924	-	}
925	-	}
926	-	})
927	-	}
928	-	}
929	-
930	-	func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) {
931	-	repo := NewRepository(privateDir(t))
932	-	key := legacyIncarnationID("named").String()
933	-	first := publicationWithTailShard(t, "named", 1, "ff")
934	-	stalePaths := make([]string, 0, maintenanceBatch+1)
935	-	for i := range maintenanceBatch + 1 {
936	-	digest := digestInShard(t, "00", i)
937	-	path := repo.legacyObjectPath(key, digest)
938	-	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
939	-	t.Fatal(err)
940	-	}
941	-	if err := os.WriteFile(path, []byte("stale"), 0o600); err != nil {
942	-	t.Fatal(err)
943	-	}
944	-	stalePaths = append(stalePaths, path)
945	-	}
946	-	if err := repo.Publish(context.Background(), first); err != nil {
947	-	t.Fatal(err)
948	-	}
949	-
950	-	// This completes marking but exhausts the sweep budget in the stale shard,
951	-	// leaving the next sweep batch pending.
952	-	if err := repo.Maintain(context.Background()); err != nil {
953	-	t.Fatal(err)
954	-	}
955	-	staleRemaining := false
956	-	for _, path := range stalePaths {
957	-	if _, err := os.Lstat(path); err == nil {
958	-	staleRemaining = true
959	-	break
960	-	}
961	-	}
962	-	if !staleRemaining {
963	-	t.Fatal("sweep did not leave a pending stale object")
964	-	}
965	-
966	-	second := publicationWithCurrentParent(t, repo, publicationWithTailShard(t, "named", 2, "ff"))
967	-	injected := errors.New("before HEAD")
968	-	repo.hooks.beforeHeadWrite = func(string) error { return injected }
969	-	if err := repo.Publish(context.Background(), second); !errors.Is(err, injected) {
970	-	t.Fatalf("Publish error = %v, want injected failure", err)
971	-	}
972	-	repo.hooks.beforeHeadWrite = nil
973	-
974	-	for i := range 20 {
975	-	if err := repo.Maintain(context.Background()); err != nil {
976	-	t.Fatal(err)
977	-	}
978	-	for _, object := range second.Objects {
979	-	if _, err := os.Lstat(repo.legacyObjectPath(key, object.Digest)); err != nil {
980	-	t.Fatalf("object referenced by failed publication removed on pass %d: %v", i, err)
981	-	}
982	-	}
983	-	staleRemaining = false
984	-	for _, path := range stalePaths {
985	-	if _, err := os.Lstat(path); err == nil {
986	-	staleRemaining = true
987	-	break
988	-	}
989	-	}
990	-	}
991	-	if staleRemaining {
992	-	t.Fatal("stale object was not eventually swept")
993	-	}
994	-	}
995	-
996	-	func publicationWithTailShard(t *testing.T, name string, generation uint64, shard string) ports.SnapshotPublication {
997	-	t.Helper()
998	-	for i := range 4096 {
999	-	publication := repositoryPublication(t, name, generation, fmt.Appendf(nil, "state-%d-%d", generation, i))
1000	-	if fmt.Sprintf("%02x", publication.Objects[0].Digest[0]) == shard {
1001	-	return publication
1002	-	}
1003	-	}
1004	-	t.Fatalf("did not find publication tail in shard %q", shard)
1005	-	return ports.SnapshotPublication{}
1006	-	}
1007	-
1008	-	func digestInShard(t *testing.T, shard string, n int) ports.SnapshotDigest {
1009	-	t.Helper()
1010	-	for i := range 4096 {
1011	-	digest := sha256.Sum256(fmt.Appendf(nil, "stale-%d-%d", n, i))
1012	-	if fmt.Sprintf("%02x", digest[0]) == shard {
1013	-	return digest
1014	-	}
1015	-	}
1016	-	t.Fatalf("did not find stale digest in shard %q", shard)
1017	-	return ports.SnapshotDigest{}
1018	-	}
1019	-
1020	-	func TestRetentionSlots(t *testing.T) {
1021	-	for _, tc := range []struct {
1022	-	name string
1023	-	refs []uint64
1024	-	want []uint64
1025	-	}{
1026	-	{name: "first checkpoint", refs: []uint64{1}, want: []uint64{1}},
1027	-	{name: "second checkpoint", refs: []uint64{2, 1}, want: []uint64{2, 1}},
1028	-	{name: "committed and two predecessors", refs: []uint64{4, 3, 2}, want: []uint64{4, 3, 2}},
1029	-	{name: "fallback hole", refs: []uint64{4, 2}, want: []uint64{4, 2}},
1030	-	} {
1031	-	t.Run(tc.name, func(t *testing.T) {
1032	-	repo := NewRepository(privateDir(t))
1033	-	publications := publishMaintenanceGenerations(t, repo, "work", 4)
1034	-	plan := ports.RetentionPlan{IncarnationID: publications[0].IncarnationID}
1035	-	for _, generation := range tc.refs {
1036	-	plan.Keep = append(plan.Keep, checkpointRefForPublication(publications[generation-1]))
1037	-	}
1038	-	done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20})
1039	-	if err != nil {
1040	-	t.Fatal(err)
1041	-	}
1042	-	if !done {
1043	-	t.Fatal("MaintainSession() done = false with sufficient budget")
1044	-	}
1045	-	if got := remainingMaintenanceGenerations(t, repo, plan.IncarnationID); !slices.Equal(got, tc.want) {
1046	-	t.Fatalf("remaining generations = %v, want %v", got, tc.want)
1047	-	}
1048	-	})
1049	-	}
1050	-	}
1051	-
1052	-	func TestUnresolvedDataPinned(t *testing.T) {
1053	-	repo := NewRepository(privateDir(t))
1054	-	publications := publishMaintenanceGenerations(t, repo, "unresolved", 4)
1055	-	done, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: publications[0].IncarnationID, PinAll: true}, ports.MaintenanceBudget{})
1056	-	if err != nil {
1057	-	t.Fatal(err)
1058	-	}
1059	-	if !done {
1060	-	t.Fatal("pinned maintenance did not complete without traversing")
1061	-	}
1062	-	if got := remainingMaintenanceGenerations(t, repo, publications[0].IncarnationID); !slices.Equal(got, []uint64{4, 3, 2, 1}) {
1063	-	t.Fatalf("pinned generations = %v", got)
1064	-	}
1065	-	}
1066	-
1067	-	func TestPinAllDoesNotWaitForSameSessionRetention(t *testing.T) {
1068	-	repo := NewRepository(privateDir(t))
1069	-	publications := publishMaintenanceGenerations(t, repo, "pin-active", 2)
1070	-	publication := publications[0]
1071	-	plan := ports.RetentionPlan{IncarnationID: publication.IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(publication)}}
1072	-
1073	-	entered := make(chan struct{})
1074	-	release := make(chan struct{})
1075	-	var enteredOnce sync.Once
1076	-	repo.hooks.beforeMaintenancePayloadRead = func(string) {
1077	-	enteredOnce.Do(func() { close(entered) })
1078	-	<-release
1079	-	}
1080	-	activeDone := make(chan error, 1)
1081	-	go func() {
1082	-	_, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20})
1083	-	activeDone <- err
1084	-	}()
1085	-	<-entered
1086	-
1087	-	pinDone := make(chan error, 1)
1088	-	go func() {
1089	-	_, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: publication.IncarnationID, PinAll: true}, ports.MaintenanceBudget{})
1090	-	pinDone <- err
1091	-	}()
1092	-	select {
1093	-	case err := <-pinDone:
1094	-	if err != nil {
1095	-	t.Fatal(err)
1096	-	}
1097	-	case <-time.After(remainingTestTime(t) / 2):
1098	-	close(release)
1099	-	t.Fatal("PinAll waited for active retention on the same incarnation")
1100	-	}
1101	-	close(release)
1102	-	if err := <-activeDone; err != nil {
1103	-	t.Fatal(err)
1104	-	}
1105	-	if got := remainingMaintenanceGenerations(t, repo, publication.IncarnationID); !slices.Equal(got, []uint64{2, 1}) {
1106	-	t.Fatalf("PinAll allowed active retention to delete generations: %v", got)
1107	-	}
1108	-	}
1109	-
1110	-	func TestMaintainSessionRejectsZeroBudget(t *testing.T) {
1111	-	repo := NewRepository(privateDir(t))
1112	-	id := domain.IncarnationID{1}
1113	-	done, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: id}, ports.MaintenanceBudget{})
1114	-	if done \|\| !errors.Is(err, ErrMaintenanceBudgetTooSmall) {
1115	-	t.Fatalf("zero budget: done=%v err=%v", done, err)
1116	-	}
1117	-	}
1118	-
1119	-	func TestProcessRetentionEntriesSyncsRemovalBatchOnce(t *testing.T) {
1120	-	repo := NewRepository(privateDir(t))
1121	-	dir := filepath.Join(repo.dir, "batch")
1122	-	if err := safedir.EnsurePrivate(dir); err != nil {
1123	-	t.Fatal(err)
1124	-	}
1125	-	entries := []maintenanceDirEntry{{name: generationFilename(1), cookie: 1}, {name: generationFilename(2), cookie: 2}}
1126	-	for _, entry := range entries {
1127	-	if err := os.WriteFile(filepath.Join(dir, entry.name), []byte("x"), 0o600); err != nil {
1128	-	t.Fatal(err)
1129	-	}
1130	-	}
1131	-	syncs := 0
1132	-	repo.hooks.syncDirectory = func(got string) error {
1133	-	if got == dir {
1134	-	syncs++
1135	-	}
1136	-	return nil
1137	-	}
1138	-	state := &retentionMaintenance{cursors: make(map[string]*maintenanceCursor)}
1139	-	budget := ports.MaintenanceBudget{Entries: 2, Bytes: 2}
1140	-	admitted, err := repo.processRetentionEntries(context.Background(), state, dir, "batch", entries, &budget, false, retentionEntryPolicy{kind: retentionManifest})
1141	-	if err != nil \|\| !admitted {
1142	-	t.Fatalf("batch processing: admitted=%v err=%v", admitted, err)
1143	-	}
1144	-	if syncs != 1 {
1145	-	t.Fatalf("directory syncs = %d, want 1", syncs)
1146	-	}
1147	-	}
1148	-
1149	-	func TestRequeueRetentionEntriesDoesNotAliasInput(t *testing.T) {
1150	-	repo := NewRepository(privateDir(t))
1151	-	state := &retentionMaintenance{cursors: make(map[string]*maintenanceCursor)}
1152	-	entries := []maintenanceDirEntry{{name: "one", cookie: 1}}
1153	-	repo.requeueRetentionEntries(state, "/dir", "purpose", entries)
1154	-	entries[0].name = "mutated"
1155	-	pending := state.cursors[maintenanceCursorID("/dir", "purpose")].pending
1156	-	if pending[0].name != "one" {
1157	-	t.Fatalf("queued entry aliased caller input: %q", pending[0].name)
1158	-	}
1159	-	}
1160	-
1161	-	func TestMaintenanceBudgetAdmitsPayloadsBeforeRead(t *testing.T) {
1162	-	repo := NewRepository(privateDir(t))
1163	-	publications := publishMaintenanceGenerations(t, repo, "budget", 1)
1164	-	plan := ports.RetentionPlan{IncarnationID: publications[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(publications[0])}}
1165	-
1166	-	var reads []string
1167	-	repo.hooks.beforeMaintenancePayloadRead = func(path string) { reads = append(reads, path) }
1168	-	done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: 1})
1169	-	if !errors.Is(err, ErrMaintenanceBudgetTooSmall) {
1170	-	t.Fatalf("undersized manifest budget error = %v, want ErrMaintenanceBudgetTooSmall", err)
1171	-	}
1172	-	if done \|\| len(reads) != 0 {
1173	-	t.Fatalf("undersized manifest budget: done=%v payload reads=%v", done, reads)
1174	-	}
1175	-
1176	-	manifestPath := repo.manifestPath(plan.IncarnationID, 1)
1177	-	info, err := repo.stat(manifestPath)
1178	-	if err != nil {
1179	-	t.Fatal(err)
1180	-	}
1181	-	reads = nil
1182	-	done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: uint64(info.Size())})
1183	-	if !errors.Is(err, ErrMaintenanceBudgetTooSmall) {
1184	-	t.Fatalf("manifest-only budget error = %v, want ErrMaintenanceBudgetTooSmall", err)
1185	-	}
1186	-	if done {
1187	-	t.Fatal("manifest-only budget unexpectedly admitted the generation objects")
1188	-	}
1189	-	if len(reads) != 1 \|\| reads[0] != manifestPath {
1190	-	t.Fatalf("payload reads = %v, want only admitted manifest %q", reads, manifestPath)
1191	-	}
1192	-	}
1193	-
1194	-	func TestMaintainSessionDistinguishesInitialOversizeFromResumableExhaustion(t *testing.T) {
1195	-	repo := NewRepository(privateDir(t))
1196	-	publications := publishMaintenanceGenerations(t, repo, "budget-semantics", 2)
1197	-	plan := ports.RetentionPlan{IncarnationID: publications[0].IncarnationID}
1198	-	firstInfo, err := os.Stat(repo.manifestPath(plan.IncarnationID, 1))
1199	-	if err != nil {
1200	-	t.Fatal(err)
1201	-	}
1202	-	secondInfo, err := os.Stat(repo.manifestPath(plan.IncarnationID, 2))
1203	-	if err != nil {
1204	-	t.Fatal(err)
1205	-	}
1206	-	firstSize, secondSize := uint64(firstInfo.Size()), uint64(secondInfo.Size())
1207	-
1208	-	done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 2, Bytes: min(firstSize, secondSize) - 1})
1209	-	if done \|\| !errors.Is(err, ErrMaintenanceBudgetTooSmall) {
1210	-	t.Fatalf("initial oversized item: done=%v err=%v", done, err)
1211	-	}
1212	-
1213	-	done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 2, Bytes: firstSize + secondSize - 1})
1214	-	if err != nil \|\| done {
1215	-	t.Fatalf("mid-pass exhaustion must be resumable: done=%v err=%v", done, err)
1216	-	}
1217	-	done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20})
1218	-	if err != nil \|\| !done {
1219	-	t.Fatalf("resumed maintenance: done=%v err=%v", done, err)
1220	-	}
1221	-	}
1222	-
1223	-	func TestMaintainSessionYieldsOnEmptyNonDoneObjectShardBatch(t *testing.T) {
1224	-	repo := NewRepository(privateDir(t))
1225	-	publication := publishMaintenanceGenerations(t, repo, "empty-shard-batch", 1)[0]
1226	-	plan := ports.RetentionPlan{IncarnationID: publication.IncarnationID}
1227	-	key := publication.IncarnationID.String()
1228	-	token := retentionToken(plan)
1229	-	root := filepath.Join(repo.sessionPath(plan.IncarnationID), repositoryObjectsDir)
1230	-	purpose := "retention-shards:" + key + ":" + token
1231	-	state := &retentionMaintenance{
1232	-	token: token,
1233	-	cursors: map[string]*maintenanceCursor{maintenanceCursorID(root, purpose): {pending: []maintenanceDirEntry{}}},
1234	-	references: make(map[ports.SnapshotDigest]struct{}),
1235	-	validated: true,
1236	-	manifestsDone: true,
1237	-	}
1238	-	repo.retentionSessions[key] = state
1239	-
1240	-	done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: 1024})
1241	-	if err != nil \|\| done {
1242	-	t.Fatalf("empty non-done shard batch: done=%v err=%v", done, err)
1243	-	}
1244	-	if state.objectRootDone {
1245	-	t.Fatal("empty non-done shard batch advanced object-root completion")
1246	-	}
1247	-	}
1248	-
1249	-	func TestMaintainSessionDoesNotHoldMaintenanceLockDuringPayloadIO(t *testing.T) {
1250	-	repo := NewRepository(privateDir(t))
1251	-	active := publishMaintenanceGenerations(t, repo, "active-lock", 1)
1252	-	other := publishMaintenanceGenerations(t, repo, "other-lock", 1)
1253	-	plan := ports.RetentionPlan{IncarnationID: active[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(active[0])}}
1254	-
1255	-	entered := make(chan struct{})
1256	-	release := make(chan struct{})
1257	-	var once sync.Once
1258	-	repo.hooks.beforeMaintenancePayloadRead = func(string) {
1259	-	once.Do(func() { close(entered) })
1260	-	<-release
1261	-	}
1262	-	activeDone := make(chan error, 1)
1263	-	go func() {
1264	-	_, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20})
1265	-	activeDone <- err
1266	-	}()
1267	-	<-entered
1268	-
1269	-	pinDone := make(chan error, 1)
1270	-	go func() {
1271	-	_, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: other[0].IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1})
1272	-	pinDone <- err
1273	-	}()
1274	-	select {
1275	-	case err := <-pinDone:
1276	-	if err != nil {
1277	-	t.Fatal(err)
1278	-	}
1279	-	case <-time.After(remainingTestTime(t) / 2):
1280	-	close(release)
1281	-	t.Fatal("unrelated PinAll was blocked by retention payload I/O")
1282	-	}
1283	-	close(release)
1284	-	if err := <-activeDone; err != nil {
1285	-	t.Fatal(err)
1286	-	}
1287	-	}
1288	-
1289	-	func remainingTestTime(t *testing.T) time.Duration {
1290	-	t.Helper()
1291	-	if deadline, ok := t.Deadline(); ok {
1292	-	return time.Until(deadline)
1293	-	}
1294	-	return 30 * time.Second
1295	-	}
1296	-
1297	-	func TestPerSessionBudgetIsolation(t *testing.T) {
1298	-	repo := NewRepository(privateDir(t))
1299	-	large := publishMaintenanceGenerations(t, repo, "large", 4)
1300	-	small := publishMaintenanceGenerations(t, repo, "small", 2)
1301	-	largePlan := ports.RetentionPlan{IncarnationID: large[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(large[3])}}
1302	-	smallPlan := ports.RetentionPlan{IncarnationID: small[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(small[1])}}
1303	-
1304	-	done, err := repo.MaintainSession(context.Background(), largePlan, ports.MaintenanceBudget{Entries: 1, Bytes: 1})
1305	-	if !errors.Is(err, ErrMaintenanceBudgetTooSmall) {
1306	-	t.Fatalf("large session error = %v, want ErrMaintenanceBudgetTooSmall", err)
1307	-	}
1308	-	if done {
1309	-	t.Fatal("large session unexpectedly exhausted no budget")
1310	-	}
1311	-	done, err = repo.MaintainSession(context.Background(), smallPlan, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20})
1312	-	if err != nil {
1313	-	t.Fatal(err)
1314	-	}
1315	-	if !done {
1316	-	t.Fatal("small session did not progress after unrelated exhaustion")
1317	-	}
1318	-	if got := remainingMaintenanceGenerations(t, repo, small[0].IncarnationID); !slices.Equal(got, []uint64{2}) {
1319	-	t.Fatalf("small generations = %v, want [2]", got)
1320	-	}
1321	-	}
1322	-
1323	-	func publishMaintenanceGenerations(t testing.T, repo Repository, name string, count uint64) []ports.SnapshotPublication {
1324	-	t.Helper()
1325	-	publications := make([]ports.SnapshotPublication, 0, count)
1326	-	for generation := uint64(1); generation <= count; generation++ {
1327	-	publication := repositoryPublicationAfter(t, repo, name, generation, []byte{byte(generation)})
1328	-	if err := repo.Publish(context.Background(), publication); err != nil {
1329	-	t.Fatal(err)
1330	-	}
1331	-	publications = append(publications, publication)
1332	-	}
1333	-	return publications
1334	-	}
1335	-
1336	-	func checkpointRefForPublication(publication ports.SnapshotPublication) domain.CheckpointRef {
1337	-	return domain.CheckpointRef{Generation: publication.Generation, ManifestDigest: sha256.Sum256(publication.Manifest)}
1338	-	}
1339	-
1340	-	func remainingMaintenanceGenerations(t testing.T, repo Repository, id domain.IncarnationID) []uint64 {
1341	-	t.Helper()
1342	-	entries, err := os.ReadDir(filepath.Join(repo.sessionPath(id), repositoryGenerations))
1343	-	if err != nil {
1344	-	t.Fatal(err)
1345	-	}
1346	-	generations := make([]uint64, 0, len(entries))
1347	-	for _, entry := range entries {
1348	-	if generation, ok := parseGenerationFilename(entry.Name()); ok {
1349	-	generations = append(generations, generation)
1350	-	}
1351	-	}
1352	-	slices.SortFunc(generations, func(a, b uint64) int { return cmp.Compare(b, a) })
1353	-	return generations
1354	-	}

M internal/adapters/snapshot/path_security_test.go +9 -4


@@ -8,6 +8,9 @@ import (
8	8		"strings"
9	9		"syscall"
10	10		"testing"
	11	+
	12	+	"github.com/bnema/vev/internal/domain"
	13	+	codec "github.com/bnema/vev/internal/usecase/snapshot"
11	14		)
12	15
13	16		func TestRepositoryRejectsSymlinkedRoot(t *testing.T) {

@@ -41,8 +44,9 @@ func TestRepositoryRejectsSymlinkedRoot(t *testing.T) {
41	44		if _, err := loadPublication(context.Background(), repo, publication); err == nil {
42	45		t.Fatal("LoadCheckpoint succeeded through replaced configured-root symlink")
43	46		}
44		-	if err := repo.Maintain(context.Background()); err == nil {
45		-	t.Fatal("Maintain succeeded through replaced configured-root symlink")
	47	+	keep := map[domain.IncarnationID]domain.CheckpointRef{publication.IncarnationID: {Generation: publication.Generation, ManifestDigest: codec.ManifestDigest(publication.Manifest)}}
	48	+	if err := repo.CollectGarbage(context.Background(), keep); err == nil {
	49	+	t.Fatal("CollectGarbage succeeded through replaced configured-root symlink")
46	50		}
47	51		loaded, err := loadPublication(context.Background(), external, externalPublication)
48	52		if err != nil {

@@ -168,8 +172,9 @@ func TestRepositoryRejectsSymlinkedGenerationAndObjectShards(t *testing.T) {
168	172		if _, err := loadPublication(context.Background(), repo, publication); err == nil {
169	173		t.Fatal("LoadCheckpoint succeeded through symlinked repository component")
170	174		}
171		-	if err := repo.Maintain(context.Background()); err == nil {
172		-	t.Fatal("Maintain succeeded through symlinked repository component")
	175	+	keep := map[domain.IncarnationID]domain.CheckpointRef{publication.IncarnationID: {Generation: publication.Generation, ManifestDigest: codec.ManifestDigest(publication.Manifest)}}
	176	+	if err := repo.CollectGarbage(context.Background(), keep); err == nil {
	177	+	t.Fatal("CollectGarbage succeeded through symlinked repository component")
173	178		}
174	179		got, err := os.ReadFile(guard)
175	180		if err != nil \|\| string(got) != "guard" {

D internal/adapters/snapshot/recovery_observability_test.go +0 -81

@@ -1,81 +0,0 @@
1	-	package snapshot
2	-
3	-	import (
4	-	"bufio"
5	-	"bytes"
6	-	"context"
7	-	"encoding/json"
8	-	"log/slog"
9	-	"testing"
10	-
11	-	"github.com/bnema/vev/internal/domain"
12	-	"github.com/bnema/vev/internal/ports"
13	-	"github.com/stretchr/testify/require"
14	-	)
15	-
16	-	func TestGCBudgetDiagnostic(t *testing.T) {
17	-	for _, tc := range []struct {
18	-	name string
19	-	arrange func(testing.T, Repository) ports.RetentionPlan
20	-	budget ports.MaintenanceBudget
21	-	wantDone bool
22	-	wantErr error
23	-	wantExhausted bool
24	-	wantConsumption bool
25	-	}{
26	-	{
27	-	name: "exhausted",
28	-	arrange: func(_ testing.T, _ Repository) ports.RetentionPlan {
29	-	return ports.RetentionPlan{IncarnationID: domain.IncarnationID{1}, Keep: []ports.CheckpointRef{{Generation: 3}, {Generation: 2}, {Generation: 1}}}
30	-	},
31	-	wantErr: ErrMaintenanceBudgetTooSmall,
32	-	},
33	-	{
34	-	name: "sufficient",
35	-	arrange: func(t testing.T, repository Repository) ports.RetentionPlan {
36	-	publication := publishMaintenanceGenerations(t, repository, "diagnostic", 1)[0]
37	-	return ports.RetentionPlan{IncarnationID: publication.IncarnationID}
38	-	},
39	-	budget: ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20},
40	-	wantDone: true,
41	-	wantConsumption: true,
42	-	},
43	-	} {
44	-	t.Run(tc.name, func(t *testing.T) {
45	-	var buffer bytes.Buffer
46	-	log := slog.New(slog.NewJSONHandler(&buffer, nil))
47	-	repository := NewRepository(privateDir(t), log)
48	-	plan := tc.arrange(t, repository)
49	-	buffer.Reset()
50	-
51	-	done, err := repository.MaintainSession(context.Background(), plan, tc.budget)
52	-	if tc.wantErr == nil {
53	-	require.NoError(t, err)
54	-	} else {
55	-	require.ErrorIs(t, err, tc.wantErr)
56	-	}
57	-	require.Equal(t, tc.wantDone, done)
58	-
59	-	scanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes()))
60	-	require.True(t, scanner.Scan())
61	-	var entry map[string]any
62	-	require.NoError(t, json.Unmarshal(scanner.Bytes(), &entry))
63	-	require.NoError(t, scanner.Err())
64	-	require.Equal(t, "snapshot_maintenance_progress", entry["msg"])
65	-	require.Equal(t, plan.IncarnationID.String(), entry["incarnation"])
66	-	require.Contains(t, entry, "cursor")
67	-	require.Equal(t, tc.wantExhausted, entry["budget_exhausted"])
68	-	consumedEntries, entriesOK := entry["consumed_entries"].(float64)
69	-	consumedBytes, bytesOK := entry["consumed_bytes"].(float64)
70	-	require.True(t, entriesOK)
71	-	require.True(t, bytesOK)
72	-	if tc.wantConsumption {
73	-	require.Positive(t, consumedEntries)
74	-	require.Positive(t, consumedBytes)
75	-	} else {
76	-	require.Zero(t, consumedEntries)
77	-	require.Zero(t, consumedBytes)
78	-	}
79	-	})
80	-	}
81	-	}

M internal/adapters/snapshot/repository.go +21 -44


@@ -29,15 +29,6 @@ type Repository struct {
29	29		locks map[string]*sessionMutex
30	30		storageEpochs map[string]uint64
31	31		nextEpoch uint64
32		-
33		-	// maintenanceMu owns bounded continuation metadata: seek cookies and
34		-	// pending directory entries. Directory descriptors are opened and closed
35		-	// for each maintenance call.
36		-	maintenanceMu sync.Mutex
37		-	maintenanceCursors map[string]*maintenanceCursor
38		-	maintenanceSessions map[string]*sessionMaintenance
39		-	retentionSessions map[string]*retentionMaintenance
40		-	maintenanceQuarantine *quarantineMaintenance
41	32		}
42	33
43	34		// repositoryHooks makes each persistence boundary fault-injectable. Hooks run

@@ -50,38 +41,27 @@ type repositoryHooks struct {
50	41		// the steady-state cost of retained history observable without exposing a
51	42		// production metrics surface.
52	43		beforeObjectRead func(string)
53		-	// beforeMaintenancePayloadRead instruments payload reads after maintenance
54		-	// has admitted their stat-reported size against the current byte budget.
55		-	beforeMaintenancePayloadRead func(string)
56	44		// beforePayloadRead observes every bounded repository payload read. It is
57	45		// test-only accounting instrumentation and runs immediately before the read.
58		-	beforePayloadRead func(string)
59		-	beforeObjectHash func([]byte)
60		-	beforeObjectCopy func([]byte)
61		-	beforeHeadRead func(string) error
62		-	beforeHeadWrite func(string) error
63		-	beforeTombstoneCheck func(string)
64		-	beforePendingQuarantineCheck func(string)
65		-	createTemp func(string) error
66		-	writeTemp func(string) error
67		-	syncFile func(string) error
68		-	closeFile func(string) error
69		-	installImmutable func(string) error
70		-	rename func(string) error
71		-	syncDirectory func(string) error
72		-	remove func(string) error
	46	+	beforePayloadRead func(string)
	47	+	beforeObjectHash func([]byte)
	48	+	beforeObjectCopy func([]byte)
	49	+	beforeHeadRead func(string) error
	50	+	beforeHeadWrite func(string) error
	51	+	createTemp func(string) error
	52	+	writeTemp func(string) error
	53	+	syncFile func(string) error
	54	+	closeFile func(string) error
	55	+	installImmutable func(string) error
	56	+	rename func(string) error
	57	+	syncDirectory func(string) error
	58	+	remove func(string) error
73	59		// afterOpenRoot and closeRoot make descriptor race and close-error paths
74	60		// deterministic in package tests. closeRoot never replaces the real close.
75		-	afterOpenRoot func()
76		-	closeRoot func() error
77		-	openMaintenanceDirectory func(string) (maintenanceDirectory, error)
78		-	maintenanceDirentConfig *maintenanceDirentConfig
79		-	beforeDirectoryRead func(string)
80		-	beforeDeletionTombstoneRead func(string)
81		-	// beforeMaintenanceWork observes each budgeted quarantine filesystem step.
82		-	// It is test-only instrumentation for hostile traversal bounds.
83		-	beforeMaintenanceWork func(string)
84		-	beforeSessionLock func(string)
	61	+	afterOpenRoot func()
	62	+	closeRoot func() error
	63	+	beforeDirectoryRead func(string)
	64	+	beforeSessionLock func(string)
85	65		}
86	66
87	67		var _ ports.SnapshotRepository = (*Repository)(nil)

@@ -94,13 +74,10 @@ func NewRepository(dir string, logs ...slog.Logger) Repository {
94	74		log = logs[0]
95	75		}
96	76		return &Repository{
97		-	dir: dir,
98		-	log: log,
99		-	locks: make(map[string]*sessionMutex),
100		-	storageEpochs: make(map[string]uint64),
101		-	maintenanceCursors: make(map[string]*maintenanceCursor),
102		-	maintenanceSessions: make(map[string]*sessionMaintenance),
103		-	retentionSessions: make(map[string]*retentionMaintenance),
	77	+	dir: dir,
	78	+	log: log,
	79	+	locks: make(map[string]*sessionMutex),
	80	+	storageEpochs: make(map[string]uint64),
104	81		}
105	82		}
106	83

D internal/adapters/snapshot/repository_maintenance_test.go +0 -69

@@ -1,69 +0,0 @@
1	-	package snapshot
2	-
3	-	import (
4	-	"context"
5	-	"errors"
6	-	"os"
7	-	"path/filepath"
8	-	"strings"
9	-	"testing"
10	-	)
11	-
12	-	func TestSafeFilesystemErrorDoesNotExposePath(t *testing.T) {
13	-	cause := errors.New("permission denied")
14	-	err := safeFilesystemError(&os.PathError{Op: "remove", Path: "\x1b[31munsafe", Err: cause})
15	-	if !errors.Is(err, cause) \|\| strings.Contains(err.Error(), "\x1b[31munsafe") {
16	-	t.Fatalf("safe filesystem error = %q", err)
17	-	}
18	-	}
19	-
20	-	func TestSafeFilesystemErrorDoesNotExposeLinkErrorPath(t *testing.T) {
21	-	err := safeFilesystemError(&os.LinkError{Op: "rename", Old: "/unsafe/old", New: "/unsafe/new", Err: os.ErrExist})
22	-	if !errors.Is(err, os.ErrExist) \|\| strings.Contains(err.Error(), "/unsafe/") {
23	-	t.Fatalf("safe link error = %q", err)
24	-	}
25	-	}
26	-
27	-	func TestRepositoryMaintenanceFilesystemErrorsHaveSafeContext(t *testing.T) {
28	-	injected := errors.New("injected filesystem failure")
29	-	cases := []struct {
30	-	name string
31	-	run func(*Repository) error
32	-	want string
33	-	}{
34	-	{
35	-	name: "delete sync",
36	-	run: func(repo *Repository) error {
37	-	publication := repositoryPublication(t, "named", 1, []byte("state"))
38	-	if err := repo.Publish(context.Background(), publication); err != nil {
39	-	t.Fatal(err)
40	-	}
41	-	repo.hooks.syncDirectory = func(string) error { return injected }
42	-	return repo.Delete(context.Background(), "named")
43	-	},
44	-	want: `sync deleted snapshot session directory "named"`,
45	-	},
46	-	{
47	-	name: "maintain remove",
48	-	run: func(repo *Repository) error {
49	-	if err := os.Mkdir(repo.dir, 0o700); err != nil {
50	-	t.Fatal(err)
51	-	}
52	-	if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-stale"), []byte("stale"), 0o600); err != nil {
53	-	t.Fatal(err)
54	-	}
55	-	repo.hooks.remove = func(string) error { return injected }
56	-	return repo.Maintain(context.Background())
57	-	},
58	-	want: `remove snapshot maintenance file ".tmp-stale"`,
59	-	},
60	-	}
61	-	for _, tc := range cases {
62	-	t.Run(tc.name, func(t *testing.T) {
63	-	err := tc.run(NewRepository(privateDir(t)))
64	-	if !errors.Is(err, injected) \|\| !strings.Contains(err.Error(), tc.want) {
65	-	t.Fatalf("error = %v, want context %q and injected error", err, tc.want)
66	-	}
67	-	})
68	-	}
69	-	}

D internal/adapters/snapshot/repository_phase6_test.go +0 -107

@@ -1,107 +0,0 @@
1	-	package snapshot
2	-
3	-	import (
4	-	"context"
5	-	"errors"
6	-	"fmt"
7	-	"os"
8	-	"path/filepath"
9	-	"sync"
10	-	"sync/atomic"
11	-	"testing"
12	-
13	-	"github.com/bnema/vev/internal/ports"
14	-	)
15	-
16	-	func TestRepositoryPublishDeleteManySessionsRaceFree(t *testing.T) {
17	-	repo := NewRepository(privateDir(t))
18	-	const sessions = 128
19	-
20	-	publications := make([]ports.SnapshotPublication, sessions)
21	-	for i := range publications {
22	-	name := fmt.Sprintf("named-%03d", i)
23	-	publications[i] = repositoryPublication(t, name, 1, []byte(name))
24	-	if err := repo.Publish(context.Background(), publications[i]); err != nil {
25	-	t.Fatalf("initial Publish(%q): %v", name, err)
26	-	}
27	-	}
28	-
29	-	var wg sync.WaitGroup
30	-	for _, publication := range publications {
31	-	wg.Add(2)
32	-	go func() {
33	-	defer wg.Done()
34	-	if err := repo.Publish(context.Background(), publication); err != nil {
35	-	t.Errorf("Publish(%q): %v", publication.Name, err)
36	-	}
37	-	}()
38	-	go func() {
39	-	defer wg.Done()
40	-	if err := repo.Delete(context.Background(), publication.Name); err != nil {
41	-	t.Errorf("Delete(%q): %v", publication.Name, err)
42	-	}
43	-	}()
44	-	}
45	-	wg.Wait()
46	-
47	-	locks, epochs := repo.sessionStateCounts()
48	-	if locks != 0 \|\| epochs != 0 {
49	-	t.Fatalf("retained session state = %d locks, %d epochs, want none", locks, epochs)
50	-	}
51	-	}
52	-
53	-	func TestRepositoryBoundsAndEvictsPartialMaintenanceSessionState(t *testing.T) {
54	-	repo := NewRepository(privateDir(t))
55	-	for _, key := range []string{"named-000", "named-001"} {
56	-	generations := filepath.Join(repo.legacySessionPath(key), repositoryGenerations)
57	-	for i := 0; i < maintenanceBatch+1; i++ {
58	-	if err := os.MkdirAll(generations, 0o700); err != nil {
59	-	t.Fatal(err)
60	-	}
61	-	if err := os.WriteFile(filepath.Join(generations, fmt.Sprintf("unclassified-%03d", i)), []byte("state"), 0o600); err != nil {
62	-	t.Fatal(err)
63	-	}
64	-	}
65	-	}
66	-	if err := repo.Maintain(context.Background()); err != nil {
67	-	t.Fatal(err)
68	-	}
69	-	locks, epochs := repo.sessionStateCounts()
70	-	if locks != 1 \|\| epochs != 0 {
71	-	t.Fatalf("partial maintenance state = %d locks, %d epochs, want one lock and no epochs", locks, epochs)
72	-	}
73	-
74	-	ctx, cancel := context.WithCancel(context.Background())
75	-	cancel()
76	-	if err := repo.Maintain(ctx); !errors.Is(err, context.Canceled) {
77	-	t.Fatalf("canceled Maintain error = %v, want context canceled", err)
78	-	}
79	-	locks, epochs = repo.sessionStateCounts()
80	-	if locks != 0 \|\| epochs != 0 {
81	-	t.Fatalf("session state after maintenance reset = %d locks, %d epochs, want none", locks, epochs)
82	-	}
83	-	}
84	-
85	-	func TestRepositoryDeleteUsesDeterministicPendingQuarantine(t *testing.T) {
86	-	repo := NewRepository(privateDir(t))
87	-	if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state"))); err != nil {
88	-	t.Fatal(err)
89	-	}
90	-	sessions := filepath.Join(repo.dir, repositorySessionsDir)
91	-	for i := 0; i < maxDirectoryTraversalEntries*2; i++ {
92	-	if err := os.Mkdir(filepath.Join(sessions, fmt.Sprintf("unrelated-%05d", i)), 0o700); err != nil {
93	-	t.Fatal(err)
94	-	}
95	-	}
96	-	var checks atomic.Int32
97	-	repo.hooks.beforePendingQuarantineCheck = func(string) { checks.Add(1) }
98	-	if err := repo.Delete(context.Background(), "named"); err != nil {
99	-	t.Fatal(err)
100	-	}
101	-	if got := checks.Load(); got != 1 {
102	-	t.Fatalf("pending quarantine checks = %d, want one deterministic lookup", got)
103	-	}
104	-	if _, err := os.Lstat(filepath.Join(sessions, deletingSessionName(legacyIncarnationID("named").String()))); err != nil {
105	-	t.Fatalf("deterministic quarantine missing: %v", err)
106	-	}
107	-	}

M internal/app/daemon_integration_test.go +5 -12


@@ -564,7 +564,7 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) {
564	564		shutdownClock,
565	565		discardLog(),
566	566		daemon.WithSnapshotRepository(repository),
567		-	daemon.WithDurableMaintenance(catalogue, maintenanceRepository, nil),
	567	+	daemon.WithDurableMaintenance(catalogue, maintenanceRepository),
568	568		daemon.WithCatalogue(catalogue, nil),
569	569		)
570	570		return d.Serve(ctx, listener)

@@ -604,11 +604,11 @@ type lifecycleBlockingMaintenanceRepository struct {
604	604		returnOnce sync.Once
605	605		}
606	606
607		-	func (r *lifecycleBlockingMaintenanceRepository) MaintainSession(context.Context, ports.RetentionPlan, ports.MaintenanceBudget) (bool, error) {
	607	+	func (r *lifecycleBlockingMaintenanceRepository) CollectGarbage(context.Context, map[domain.IncarnationID]domain.CheckpointRef) error {
608	608		r.enterOnce.Do(func() { close(r.entered) })
609	609		<-r.release // Intentionally ignore cancellation: lifecycle ownership must outlive this call.
610	610		r.returnOnce.Do(func() { close(r.returned) })
611		-	return true, nil
	611	+	return nil
612	612		}
613	613
614	614		type lifecycleObservedCatalogue struct {

@@ -746,7 +746,6 @@ func publishRestorableCheckpoint(t testing.T, stateDir string, repository snap
746	746		opened, err := persist.OpenOrCreate(stateDir)
747	747		require.NoError(t, err)
748	748		coordinator := recovery.NewCoordinator(opened.Catalogue, repository, rand.Reader)
749		-	require.NoError(t, coordinator.Recover(ctx))
750	749		listener, err := ipc.Listen(runtimeDir)
751	750		require.NoError(t, err)
752	751		d := daemon.New(

@@ -997,8 +996,6 @@ func (l *lifecycleObservedListener) Close() error {
997	996		func TestLifecycleSocketCloseCatalogueRace(t *testing.T) {
998	997		runtimeDir := filepath.Join(t.TempDir(), "runtime")
999	998		stateDir := filepath.Join(t.TempDir(), "state")
1000		-	snapshotRepository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots"))
1001		-
1002	999		oldOwner, err := lifecycle.TryAcquire(runtimeDir)
1003	1000		require.NoError(t, err)
1004	1001		oldOpened, err := persist.OpenOrCreate(stateDir)

@@ -1056,11 +1053,7 @@ func TestLifecycleSocketCloseCatalogueRace(t *testing.T) {
1056	1053		if err != nil {
1057	1054		return err
1058	1055		}
1059		-	coordinator := recovery.NewCoordinator(opened.Catalogue, snapshotRepository, rand.Reader)
1060		-	if err := coordinator.Recover(ctx); err != nil {
1061		-	return errors.Join(err, opened.Catalogue.Close())
1062		-	}
1063		-	startupEvents <- "durable-open-recover"
	1056	+	startupEvents <- "durable-open"
1064	1057
1065	1058		close(newListen)
1066	1059		listener, err := ipc.Listen(runtimeDir)

@@ -1111,7 +1104,7 @@ func TestLifecycleSocketCloseCatalogueRace(t *testing.T) {
1111	1104		<-teardownEvents,
1112	1105		<-teardownEvents,
1113	1106		})
1114		-	require.Equal(t, []string{"owner-acquired", "durable-open-recover", "listen"}, []string{
	1107	+	require.Equal(t, []string{"owner-acquired", "durable-open", "listen"}, []string{
1115	1108		<-startupEvents,
1116	1109		<-startupEvents,
1117	1110		<-startupEvents,

M internal/app/offline_kill_retry_test.go +0 -1


@@ -76,7 +76,6 @@ func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) {
76	76
77	77		require.NoError(t, runKill(context.Background(), "named", false, false))
78	78		require.NoDirExists(t, source)
79		-	require.NoDirExists(t, filepath.Join(stateDir, "snapshots", "quarantine", id.String()))
80	79		})
81	80		}
82	81		}

M internal/app/run.go +1 -172


@@ -504,134 +504,6 @@ func logCatalogueRecovery(log *slog.Logger, records []domain.CatalogueRecord, re
504	504		log.Info("catalogue_compaction_recovery_complete", "recovery", recoveryMode)
505	505		}
506	506
507		-	// logTransactionRecovery reports the recovery outcome. A startup that fenced
508		-	// any session is never reported as clean: the daemon comes up, but one or more
509		-	// sessions were skipped and still need an operator decision.
510		-	func logTransactionRecovery(log *slog.Logger, examined, fenced int) {
511		-	outcome := "clean"
512		-	if fenced > 0 {
513		-	outcome = "fenced"
514		-	}
515		-	log.Info("interrupted_transaction_recovery_complete", "sessions_examined", examined, "sessions_fenced", fenced, "outcome", outcome)
516		-	}
517		-
518		-	type interruptedRecoveryIdentity struct {
519		-	name string
520		-	id domain.IncarnationID
521		-	}
522		-
523		-	func pendingInterruptedRecoveries(ctx context.Context, records []domain.CatalogueRecord, repository ports.SnapshotRepository) ([]interruptedRecoveryIdentity, error) {
524		-	pending := make([]interruptedRecoveryIdentity, 0)
525		-	for _, record := range records {
526		-	if record.RecoveryState == domain.RecoveryDeleting {
527		-	pending = append(pending, interruptedRecoveryIdentity{name: record.Name, id: record.IncarnationID})
528		-	}
529		-	}
530		-	cursor := ports.DeletionTombstoneCursor{}
531		-	for {
532		-	page, err := repository.ListDeletionTombstones(ctx, cursor, ports.MaintenanceBudget{Entries: 64, Bytes: 64 << 10})
533		-	if err != nil {
534		-	return nil, err
535		-	}
536		-	for _, tombstone := range page.Tombstones {
537		-	pending = append(pending, interruptedRecoveryIdentity{name: tombstone.Name, id: tombstone.IncarnationID})
538		-	}
539		-	if page.Done {
540		-	return pending, nil
541		-	}
542		-	if page.Next.After == cursor.After {
543		-	return nil, errors.New("vev: recovery tombstone listing did not advance")
544		-	}
545		-	cursor = page.Next
546		-	}
547		-	}
548		-
549		-	func persistTransactionRecoveryNotices(store ports.NoticeStore, recovered []interruptedRecoveryIdentity, now time.Time) error {
550		-	for _, item := range recovered {
551		-	if err := store.Append(domain.Notification{
552		-	Code: domain.NoticeSnapshotRestore,
553		-	Severity: domain.NoticeWarn,
554		-	Message: "recovered an interrupted durable session transaction",
555		-	Details: "session=" + item.name + " incarnation=" + item.id.String(),
556		-	Time: now,
557		-	SessionID: domain.SessionID(item.id.String()),
558		-	}); err != nil {
559		-	return fmt.Errorf("persist interrupted transaction recovery notice for session %q incarnation %s: %w", item.name, item.id.String(), err)
560		-	}
561		-	}
562		-	return nil
563		-	}
564		-
565		-	// fencedRecoveryIdentity is one session that recovery fenced instead of failing
566		-	// startup for. Only the session identity and a sentinel-derived reason code are
567		-	// retained: raw causes can embed paths or operator input.
568		-	type fencedRecoveryIdentity struct {
569		-	name string
570		-	id domain.IncarnationID
571		-	kind string
572		-	reasonCode string
573		-	}
574		-
575		-	// fencedRecoveries resolves each conflict against the post-recovery catalogue
576		-	// so a fenced session is reported with the incarnation it actually holds now.
577		-	func fencedRecoveries(conflicts []recovery.RecoveryConflict, catalogue ports.Catalogue) ([]fencedRecoveryIdentity, error) {
578		-	fenced := make([]fencedRecoveryIdentity, 0, len(conflicts))
579		-	for _, conflict := range conflicts {
580		-	identity := fencedRecoveryIdentity{name: conflict.Session, kind: conflict.Kind, reasonCode: fencedRecoveryReasonCode(conflict.Err)}
581		-	if catalogue != nil {
582		-	record, ok, err := catalogue.Record(conflict.Session)
583		-	if err != nil {
584		-	return nil, err
585		-	}
586		-	if ok {
587		-	identity.id = record.IncarnationID
588		-	}
589		-	}
590		-	fenced = append(fenced, identity)
591		-	}
592		-	return fenced, nil
593		-	}
594		-
595		-	func fencedRecoveryReasonCode(err error) string {
596		-	switch {
597		-	case errors.Is(err, recovery.ErrDeletionTombstoneConflict):
598		-	return "deletion-tombstone-conflict"
599		-	case errors.Is(err, recovery.ErrDeletionTombstoneInvalid):
600		-	return "deletion-tombstone-invalid"
601		-	case errors.Is(err, recovery.ErrSessionRecordInvalid):
602		-	return "catalogue-record-invalid"
603		-	default:
604		-	return "unclassified"
605		-	}
606		-	}
607		-
608		-	func logFencedRecoveries(log *slog.Logger, fenced []fencedRecoveryIdentity) {
609		-	for _, item := range fenced {
610		-	log.Warn("interrupted_transaction_fenced",
611		-	"session", item.name,
612		-	"incarnation", item.id.String(),
613		-	"kind", item.kind,
614		-	"reason_code", item.reasonCode,
615		-	)
616		-	}
617		-	}
618		-
619		-	func persistFencedRecoveryNotices(store ports.NoticeStore, fenced []fencedRecoveryIdentity, now time.Time) error {
620		-	for _, item := range fenced {
621		-	if err := store.Append(domain.Notification{
622		-	Code: domain.NoticeSnapshotRestore,
623		-	Severity: domain.NoticeWarn,
624		-	Message: "skipped a durable session transaction that conflicts with its session state",
625		-	Details: "session=" + item.name + " incarnation=" + item.id.String() + " kind=" + item.kind + " reason_code=" + item.reasonCode,
626		-	Time: now,
627		-	SessionID: domain.SessionID(item.id.String()),
628		-	}); err != nil {
629		-	return fmt.Errorf("persist fenced transaction notice for session %q incarnation %s: %w", item.name, item.id.String(), err)
630		-	}
631		-	}
632		-	return nil
633		-	}
634		-
635	507		func logStartupRecoveryCounts(log *slog.Logger, records []domain.CatalogueRecord, restoring int) {
636	508		healthy, fresh, degraded := 0, 0, 0
637	509		for _, record := range records {

@@ -665,13 +537,6 @@ func runDaemonOwned(ctx context.Context) (retErr error) {
665	537		return runDaemonOwnedWithLogger(ctx, log)
666	538		}
667	539
668		-	func recoverAuthoritativeCatalogue(ctx context.Context, coordinator *recovery.Coordinator, catalogue ports.Catalogue) ([]domain.CatalogueRecord, error) {
669		-	if err := coordinator.Recover(ctx); err != nil {
670		-	return nil, err
671		-	}
672		-	return catalogue.Records()
673		-	}
674		-
675	540		func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr error) {
676	541		clk := clock.New()
677	542		observer, observerCloser, err := newPerformanceTrace(clk)

@@ -726,42 +591,9 @@ func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr err
726	591		recoveryMode = "new-install"
727	592		}
728	593		logCatalogueRecovery(log, opened.Records, recoveryMode)
729		-	pendingRecoveries, err := pendingInterruptedRecoveries(ctx, opened.Records, snapshotRepository)
730		-	if err != nil {
731		-	return errors.Join(fmt.Errorf("vev: inspect durable session transactions: %w", err), opened.Catalogue.Close())
732		-	}
733	594		coordinator := recovery.NewCoordinator(opened.Catalogue, snapshotRepository, rand.Reader)
734		-	// Recovery may roll catalogue changes forward, so reload the authoritative
735		-	// snapshot before constructing runtime reservations. A failed reload aborts
736		-	// startup rather than treating durable sessions as absent.
737		-	recoveredRecords, err := recoverAuthoritativeCatalogue(ctx, coordinator, opened.Catalogue)
738		-	if err != nil {
739		-	return errors.Join(fmt.Errorf("vev: recover and reload durable session transactions: %w", err), opened.Catalogue.Close())
740		-	}
741		-	opened.Records = recoveredRecords
742		-	// Recovery fences a session instead of failing startup when only that
743		-	// session's durable state is self-inconsistent. The daemon then comes up
744		-	// looking healthy, so every fenced session must be reported and persisted.
745		-	fenced, err := fencedRecoveries(coordinator.Conflicts(), opened.Catalogue)
746		-	if err != nil {
747		-	return errors.Join(fmt.Errorf("vev: read fenced recovery identities: %w", err), opened.Catalogue.Close())
748		-	}
749		-	logFencedRecoveries(log, fenced)
750		-	if err := persistTransactionRecoveryNotices(noticeStore, pendingRecoveries, clk.Now()); err != nil {
751		-	log.Error("transaction_recovery_notice_failed", "err", err)
752		-	return errors.Join(fmt.Errorf("vev: persist durable transaction recovery notice: %w", err), opened.Catalogue.Close())
753		-	}
754		-	if err := persistFencedRecoveryNotices(noticeStore, fenced, clk.Now()); err != nil {
755		-	log.Error("transaction_recovery_notice_failed", "reason_code", "fenced-notice-failed")
756		-	return errors.Join(fmt.Errorf("vev: persist fenced transaction notice: %w", err), opened.Catalogue.Close())
757		-	}
758		-	logTransactionRecovery(log, len(pendingRecoveries), len(fenced))
759	595		daemonOpts = append(daemonOpts, daemon.WithRecoveryCoordinator(coordinator))
760		-	unresolved := make([]domain.IncarnationID, 0, len(pendingRecoveries))
761		-	for _, pending := range pendingRecoveries {
762		-	unresolved = append(unresolved, pending.id)
763		-	}
764		-	daemonOpts = append(daemonOpts, daemon.WithDurableMaintenance(opened.Catalogue, snapshotRepository, unresolved))
	596	+	daemonOpts = append(daemonOpts, daemon.WithDurableMaintenance(opened.Catalogue, snapshotRepository))
765	597		log.Info("session persistence enabled", "path", storePath)
766	598		daemonOpts = append(daemonOpts, daemon.WithCatalogue(opened.Catalogue, opened.Records))
767	599

@@ -1564,9 +1396,6 @@ func runOfflineNamedKill(ctx context.Context, name string) (retErr error) {
1564	1396		return fmt.Errorf("vev: no such session: %s", name)
1565	1397		}
1566	1398		coordinator := recovery.NewCoordinator(opened.Catalogue, repository, rand.Reader)
1567		-	if err := coordinator.Recover(ctx); err != nil {
1568		-	return fmt.Errorf("vev: recovering stored session transactions: %w", err)
1569		-	}
1570	1399		if err := coordinator.Delete(ctx, name); err != nil {
1571	1400		return fmt.Errorf("vev: deleting stored session: %w", err)
1572	1401		}

M internal/app/run_test.go +2 -186


@@ -4,10 +4,8 @@ import (
4	4		"bufio"
5	5		"bytes"
6	6		"context"
7		-	"crypto/rand"
8	7		"encoding/json"
9	8		"errors"
10		-	"fmt"
11	9		"io"
12	10		"log/slog"
13	11		"net"

@@ -15,13 +13,10 @@ import (
15	13		"os/exec"
16	14		"path/filepath"
17	15		"strings"
18		-	"sync"
19	16		"testing"
20	17		"time"
21	18
22	19		"github.com/bnema/vev/internal/adapters/lifecycle"
23		-	"github.com/bnema/vev/internal/adapters/noticefile"
24		-	"github.com/bnema/vev/internal/adapters/snapshot"
25	20		"github.com/bnema/vev/internal/domain"
26	21		"github.com/bnema/vev/internal/persist"
27	22		"github.com/bnema/vev/internal/ports"

@@ -29,7 +24,6 @@ import (
29	24		"github.com/bnema/vev/internal/usecase/client"
30	25		"github.com/bnema/vev/internal/usecase/confirm"
31	26		"github.com/bnema/vev/internal/usecase/daemon"
32		-	"github.com/bnema/vev/internal/usecase/recovery"
33	27		"github.com/bnema/vev/pkg/kv"
34	28		"github.com/stretchr/testify/mock"
35	29		"github.com/stretchr/testify/require"

@@ -86,19 +80,14 @@ func TestRecoveryObservability(t *testing.T) {
86	80		{RecoveryState: domain.RecoveryFresh},
87	81		{RecoveryState: domain.RecoveryDegraded},
88	82		}
89		-	noticeStore := portsmocks.NewMockNoticeStore(t)
90		-	var recoveryNotice domain.Notification
91		-	noticeStore.EXPECT().Append(mock.Anything).Run(func(n domain.Notification) { recoveryNotice = n }).Return(nil).Once()
92	83		require.NoError(t, runWithLifecycleOwnerDeps(context.Background(), "/runtime/vev", "/state/vev", func(context.Context) error {
93	84		logCatalogueRecovery(log, records, "current")
94		-	logTransactionRecovery(log, 2, 0)
95		-	require.NoError(t, persistTransactionRecoveryNotices(noticeStore, []interruptedRecoveryIdentity{{name: "work", id: domain.IncarnationID{9}}}, time.Unix(1, 0)))
96	85		logStartupRecoveryCounts(log, records, 0)
97	86		return nil
98	87		}, deps))
99	88
100	89		entries := decodeJSONLogs(t, logBuffer.Bytes())
101		-	for _, name := range []string{"lifecycle_owner_wait", "lifecycle_owner_acquired", "catalogue_validated", "catalogue_compaction_recovery_complete", "interrupted_transaction_recovery_complete", "daemon_startup_complete", "lifecycle_owner_released"} {
	90	+	for _, name := range []string{"lifecycle_owner_wait", "lifecycle_owner_acquired", "catalogue_validated", "catalogue_compaction_recovery_complete", "daemon_startup_complete", "lifecycle_owner_released"} {
102	91		require.Contains(t, eventNames(entries), name)
103	92		}
104	93		startup := requireEvent(t, entries, "daemon_startup_complete")

@@ -106,176 +95,12 @@ func TestRecoveryObservability(t *testing.T) {
106	95		require.EqualValues(t, 1, startup["fresh"])
107	96		require.EqualValues(t, 0, startup["restoring"])
108	97		require.EqualValues(t, 1, startup["degraded"])
109		-	recovered := requireEvent(t, entries, "interrupted_transaction_recovery_complete")
110		-	require.EqualValues(t, 0, recovered["sessions_fenced"])
111		-	require.Equal(t, "clean", recovered["outcome"])
112		-	require.NotContains(t, eventNames(entries), "interrupted_transaction_fenced")
113		-
114		-	require.NotContains(t, fmt.Sprint(requireEvent(t, entries, "interrupted_transaction_recovery_complete")), "terminal contents")
115		-	require.Equal(t, domain.NoticeSnapshotRestore, recoveryNotice.Code)
116		-	require.Equal(t, domain.SessionID(domain.IncarnationID{9}.String()), recoveryNotice.SessionID)
117		-	require.NotContains(t, fmt.Sprint(recoveryNotice), "terminal contents")
	98	+	require.NotContains(t, eventNames(entries), "interrupted_transaction_recovery_complete")
118	99		}
119	100
120	101		// A session-scoped conflict no longer aborts startup, so the daemon comes up
121	102		// looking healthy. Startup must therefore name the fenced session in the log,
122	103		// persist a notice for it, and never report the recovery as clean.
123		-	func TestStartupReloadsCatalogueAfterRolledForwardDeletion(t *testing.T) {
124		-	ctx := t.Context()
125		-	stateDir := filepath.Join(t.TempDir(), "vev")
126		-	catalogue := newTestPersister(t, stateDir)
127		-	t.Cleanup(func() { _ = catalogue.Close() })
128		-	repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots"))
129		-	record := domain.CatalogueRecord{
130		-	Name: "work", IncarnationID: domain.IncarnationID{9}, Cwd: t.TempDir(),
131		-	CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryDeleting,
132		-	}
133		-	require.NoError(t, catalogue.Save(record))
134		-	coordinator := recovery.NewCoordinator(catalogue, repository, rand.Reader)
135		-
136		-	records, err := recoverAuthoritativeCatalogue(ctx, coordinator, catalogue)
137		-	require.NoError(t, err)
138		-	require.Empty(t, records, "daemon startup must not retain the pre-recovery reservation")
139		-	_, exists, err := catalogue.Record(record.Name)
140		-	require.NoError(t, err)
141		-	require.False(t, exists, "rolled-forward deletion must remain absent in authoritative state")
142		-	}
143		-
144		-	func TestFencedRecoveryObservability(t *testing.T) {
145		-	// Keep the runtime path short enough for Darwin's smaller unix socket limit.
146		-	runtimeRoot, err := os.MkdirTemp("/tmp", "vev-r-")
147		-	require.NoError(t, err)
148		-	t.Cleanup(func() { require.NoError(t, os.RemoveAll(runtimeRoot)) })
149		-	stateRoot := t.TempDir()
150		-	t.Setenv("XDG_RUNTIME_DIR", runtimeRoot)
151		-	t.Setenv("XDG_STATE_HOME", stateRoot)
152		-	t.Setenv("XDG_CONFIG_HOME", t.TempDir())
153		-	t.Setenv("VEV_PPROF_ADDR", "")
154		-
155		-	stateDir := filepath.Join(stateRoot, "vev")
156		-	repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots"))
157		-	catalogue := newTestPersister(t, stateDir)
158		-	live := domain.CatalogueRecord{
159		-	Name: "work", IncarnationID: domain.IncarnationID{7}, Cwd: t.TempDir(),
160		-	CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryHealthy,
161		-	Committed: &domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}},
162		-	}
163		-	require.NoError(t, catalogue.Save(live))
164		-	require.NoError(t, catalogue.Close(), "startup must reopen the on-disk catalogue")
165		-	require.NoError(t, repository.WriteDeletionTombstone(t.Context(), domain.DeletionTombstone{
166		-	Name: live.Name, IncarnationID: live.IncarnationID,
167		-	}))
168		-
169		-	// Pause immediately before socket publication so the test can reopen and
170		-	// claim the notice before the daemon's restore worker consumes it.
171		-	originalListenDaemon := listenDaemon
172		-	listenEntered := make(chan struct{})
173		-	allowListen := make(chan struct{})
174		-	var allowListenOnce sync.Once
175		-	releaseListen := func() { allowListenOnce.Do(func() { close(allowListen) }) }
176		-	listenDaemon = func(dir string, observer ports.SerializedRuntimeObserver) (ports.Listener, error) {
177		-	close(listenEntered)
178		-	<-allowListen
179		-	return originalListenDaemon(dir, observer)
180		-	}
181		-	t.Cleanup(func() {
182		-	releaseListen()
183		-	listenDaemon = originalListenDaemon
184		-	})
185		-
186		-	var logBuffer bytes.Buffer
187		-	log := slog.New(slog.NewJSONHandler(&logBuffer, nil))
188		-	ctx, cancel := context.WithCancel(t.Context())
189		-	started := make(chan error, 1)
190		-	go func() { started <- runDaemonOwnedWithLogger(ctx, log) }()
191		-
192		-	var shutdownErr error
193		-	var shutdownOnce sync.Once
194		-	shutdown := func() error {
195		-	shutdownOnce.Do(func() {
196		-	releaseListen()
197		-	cancel()
198		-	select {
199		-	case shutdownErr = <-started:
200		-	case <-time.After(5 * time.Second):
201		-	shutdownErr = errors.New("daemon did not stop after cancellation")
202		-	}
203		-	})
204		-	return shutdownErr
205		-	}
206		-	t.Cleanup(func() { require.NoError(t, shutdown()) })
207		-
208		-	select {
209		-	case <-listenEntered:
210		-	case <-time.After(5 * time.Second):
211		-	t.Fatal("startup did not reach socket publication")
212		-	}
213		-	reopenedNotices := noticefile.New(stateDir)
214		-	pending, err := reopenedNotices.Claim()
215		-	require.NoError(t, err)
216		-	require.Len(t, pending, 2, "startup persists both the interrupted and fenced recovery notices")
217		-	var fencedNotice *domain.Notification
218		-	for i := range pending {
219		-	if strings.Contains(pending[i].Details, "reason_code=deletion-tombstone-conflict") {
220		-	fencedNotice = &pending[i]
221		-	}
222		-	}
223		-	require.NotNil(t, fencedNotice)
224		-	require.Equal(t, domain.NoticeSnapshotRestore, fencedNotice.Code)
225		-	require.Equal(t, domain.NoticeWarn, fencedNotice.Severity)
226		-	require.Equal(t, domain.SessionID(live.IncarnationID.String()), fencedNotice.SessionID)
227		-	require.Contains(t, fencedNotice.Details, "session="+live.Name)
228		-	require.NotContains(t, fmt.Sprint(fencedNotice), "conflicts with catalogue")
229		-	require.NoError(t, reopenedNotices.Ack(), "notice claim lock must be released before daemon startup resumes")
230		-
231		-	releaseListen()
232		-	socketPath := filepath.Join(runtimeRoot, "vev", "daemon.sock")
233		-	require.Eventually(t, func() bool {
234		-	_, statErr := os.Stat(socketPath)
235		-	return statErr == nil
236		-	}, 5*time.Second, time.Millisecond, "fenced recovery must not prevent socket publication")
237		-	require.NoError(t, shutdown(), "fenced recovery must allow clean daemon startup and shutdown")
238		-
239		-	entries := decodeJSONLogs(t, logBuffer.Bytes())
240		-	event := requireEvent(t, entries, "interrupted_transaction_fenced")
241		-	require.Equal(t, "WARN", event["level"])
242		-	require.Equal(t, live.Name, event["session"])
243		-	require.Equal(t, live.IncarnationID.String(), event["incarnation"])
244		-	require.Equal(t, "deletion-tombstone", event["kind"])
245		-	require.Equal(t, "deletion-tombstone-conflict", event["reason_code"])
246		-	recovered := requireEvent(t, entries, "interrupted_transaction_recovery_complete")
247		-	require.EqualValues(t, 1, recovered["sessions_fenced"])
248		-	require.Equal(t, "fenced", recovered["outcome"])
249		-	startup := requireEvent(t, entries, "daemon_startup_complete")
250		-	require.EqualValues(t, 1, startup["degraded"])
251		-	require.NotContains(t, logBuffer.String(), "conflicts with catalogue", "raw conflict text must not be logged")
252		-
253		-	reopenedCatalogue := newTestPersister(t, stateDir)
254		-	t.Cleanup(func() { require.NoError(t, reopenedCatalogue.Close()) })
255		-	degraded, ok, err := reopenedCatalogue.Record(live.Name)
256		-	require.NoError(t, err)
257		-	require.True(t, ok)
258		-	require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState)
259		-	}
260		-
261		-	func TestFencedRecoveryNoticeFailurePropagates(t *testing.T) {
262		-	store := portsmocks.NewMockNoticeStore(t)
263		-	store.EXPECT().Append(mock.Anything).Return(errors.New("disk full")).Once()
264		-	err := persistFencedRecoveryNotices(store, []fencedRecoveryIdentity{{
265		-	name: "work", id: domain.IncarnationID{1}, kind: "discard-intent", reasonCode: "discard-intent-conflict",
266		-	}}, time.Unix(1, 0))
267		-	require.ErrorContains(t, err, "session \"work\"")
268		-	require.ErrorContains(t, err, domain.IncarnationID{1}.String())
269		-	}
270		-
271		-	func TestTransactionRecoveryNoticeFailurePropagates(t *testing.T) {
272		-	store := portsmocks.NewMockNoticeStore(t)
273		-	store.EXPECT().Append(mock.Anything).Return(errors.New("disk full")).Once()
274		-	err := persistTransactionRecoveryNotices(store, []interruptedRecoveryIdentity{{name: "work", id: domain.IncarnationID{1}}}, time.Unix(1, 0))
275		-	require.ErrorContains(t, err, "session \"work\"")
276		-	require.ErrorContains(t, err, domain.IncarnationID{1}.String())
277		-	}
278		-
279	104		type fakeLifecycleOwnership struct {
280	105		release func() error
281	106		}

@@ -917,8 +742,6 @@ func TestRunKillDeletesStoppedSessionWithoutDaemon(t *testing.T) {
917	742		if err := p.Close(); err != nil {
918	743		t.Fatalf("Close error = %v", err)
919	744		}
920		-	snapshots := snapshot.NewRepository(filepath.Join(stateRoot, "vev", "snapshots"))
921		-
922	745		got := captureStdout(t, func() {
923	746		if err := runKill(context.Background(), "stored", false, false); err != nil {
924	747		t.Fatalf("runKill error = %v", err)

@@ -934,13 +757,6 @@ func TestRunKillDeletesStoppedSessionWithoutDaemon(t *testing.T) {
934	757		if len(records) != 0 {
935	758		t.Fatalf("records after kill = %#v, want none", records)
936	759		}
937		-	page, err := snapshots.ListDeletionTombstones(context.Background(), ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 1, Bytes: 1024})
938		-	if err != nil {
939		-	t.Fatalf("list deletion tombstones: %v", err)
940		-	}
941		-	if len(page.Tombstones) != 0 {
942		-	t.Fatalf("deletion tombstones after kill = %#v, want none", page.Tombstones)
943		-	}
944	760		}
945	761
946	762		func captureStdout(t *testing.T, fn func()) string {

M internal/domain/recovery.go +0 -17


@@ -81,23 +81,6 @@ func (r CatalogueRecord) MetadataUpdate() CatalogueMetadataUpdate {
81	81		}
82	82		}
83	83
84		-	// DeletionTombstone identifies exactly one deleted session incarnation.
85		-	type DeletionTombstone struct {
86		-	Name string
87		-	IncarnationID IncarnationID
88		-	}
89		-
90		-	// Validate rejects incomplete deletion identities.
91		-	func (t DeletionTombstone) Validate() error {
92		-	if err := ValidateSessionName(t.Name); err != nil {
93		-	return err
94		-	}
95		-	if t.IncarnationID == (IncarnationID{}) {
96		-	return errors.New("zero deletion incarnation ID")
97		-	}
98		-	return nil
99		-	}
100		-
101	84		// NewIncarnationID reads a new non-zero identity from r.
102	85		func NewIncarnationID(r io.Reader) (IncarnationID, error) {
103	86		var id IncarnationID

M internal/domain/recovery_test.go +0 -22


@@ -76,25 +76,3 @@ func TestCatalogueRecordEqualUsesCheckpointValuesAndOrderedTabs(t *testing.T) {
76	76		require.True(t, (*CheckpointRef)(nil).Equal(nil))
77	77		require.False(t, left.Committed.Equal(nil))
78	78		}
79		-
80		-	func TestDeletionTombstoneValidate(t *testing.T) {
81		-	tests := []struct {
82		-	name string
83		-	tombstone DeletionTombstone
84		-	wantErr bool
85		-	}{
86		-	{name: "valid", tombstone: DeletionTombstone{Name: "work", IncarnationID: IncarnationID{1}}},
87		-	{name: "invalid name", tombstone: DeletionTombstone{Name: "bad/name", IncarnationID: IncarnationID{1}}, wantErr: true},
88		-	{name: "zero incarnation", tombstone: DeletionTombstone{Name: "work"}, wantErr: true},
89		-	}
90		-	for _, tt := range tests {
91		-	t.Run(tt.name, func(t *testing.T) {
92		-	err := tt.tombstone.Validate()
93		-	if tt.wantErr {
94		-	require.Error(t, err)
95		-	return
96		-	}
97		-	require.NoError(t, err)
98		-	})
99		-	}
100		-	}

M internal/ports/mocks/recovery_mock.go +17 -218


@@ -975,44 +975,44 @@ func (_m MockSnapshotRepository) EXPECT() MockSnapshotRepository_Expecter {
975	975		return &MockSnapshotRepository_Expecter{mock: &_m.Mock}
976	976		}
977	977
978		-	// DeleteDeletionTombstone provides a mock function for the type MockSnapshotRepository
979		-	func (_mock *MockSnapshotRepository) DeleteDeletionTombstone(context1 context.Context, incarnationID domain.IncarnationID) error {
980		-	ret := _mock.Called(context1, incarnationID)
	978	+	// CollectGarbage provides a mock function for the type MockSnapshotRepository
	979	+	func (_mock *MockSnapshotRepository) CollectGarbage(context1 context.Context, incarnationIDToCheckpointRef map[domain.IncarnationID]domain.CheckpointRef) error {
	980	+	ret := _mock.Called(context1, incarnationIDToCheckpointRef)
981	981
982	982		if len(ret) == 0 {
983		-	panic("no return value specified for DeleteDeletionTombstone")
	983	+	panic("no return value specified for CollectGarbage")
984	984		}
985	985
986	986		var r0 error
987		-	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.IncarnationID) error); ok {
988		-	r0 = returnFunc(context1, incarnationID)
	987	+	if returnFunc, ok := ret.Get(0).(func(context.Context, map[domain.IncarnationID]domain.CheckpointRef) error); ok {
	988	+	r0 = returnFunc(context1, incarnationIDToCheckpointRef)
989	989		} else {
990	990		r0 = ret.Error(0)
991	991		}
992	992		return r0
993	993		}
994	994
995		-	// MockSnapshotRepository_DeleteDeletionTombstone_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteDeletionTombstone'
996		-	type MockSnapshotRepository_DeleteDeletionTombstone_Call struct {
	995	+	// MockSnapshotRepository_CollectGarbage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CollectGarbage'
	996	+	type MockSnapshotRepository_CollectGarbage_Call struct {
997	997		*mock.Call
998	998		}
999	999
1000		-	// DeleteDeletionTombstone is a helper method to define mock.On call
	1000	+	// CollectGarbage is a helper method to define mock.On call
1001	1001		// - context1 context.Context
1002		-	// - incarnationID domain.IncarnationID
1003		-	func (_e MockSnapshotRepository_Expecter) DeleteDeletionTombstone(context1 any, incarnationID any) MockSnapshotRepository_DeleteDeletionTombstone_Call {
1004		-	return &MockSnapshotRepository_DeleteDeletionTombstone_Call{Call: _e.mock.On("DeleteDeletionTombstone", context1, incarnationID)}
	1002	+	// - incarnationIDToCheckpointRef map[domain.IncarnationID]domain.CheckpointRef
	1003	+	func (_e MockSnapshotRepository_Expecter) CollectGarbage(context1 any, incarnationIDToCheckpointRef any) MockSnapshotRepository_CollectGarbage_Call {
	1004	+	return &MockSnapshotRepository_CollectGarbage_Call{Call: _e.mock.On("CollectGarbage", context1, incarnationIDToCheckpointRef)}
1005	1005		}
1006	1006
1007		-	func (_c MockSnapshotRepository_DeleteDeletionTombstone_Call) Run(run func(context1 context.Context, incarnationID domain.IncarnationID)) MockSnapshotRepository_DeleteDeletionTombstone_Call {
	1007	+	func (_c MockSnapshotRepository_CollectGarbage_Call) Run(run func(context1 context.Context, incarnationIDToCheckpointRef map[domain.IncarnationID]domain.CheckpointRef)) MockSnapshotRepository_CollectGarbage_Call {
1008	1008		_c.Call.Run(func(args mock.Arguments) {
1009	1009		var arg0 context.Context
1010	1010		if args[0] != nil {
1011	1011		arg0 = args[0].(context.Context)
1012	1012		}
1013		-	var arg1 domain.IncarnationID
	1013	+	var arg1 map[domain.IncarnationID]domain.CheckpointRef
1014	1014		if args[1] != nil {
1015		-	arg1 = args[1].(domain.IncarnationID)
	1015	+	arg1 = args[1].(map[domain.IncarnationID]domain.CheckpointRef)
1016	1016		}
1017	1017		run(
1018	1018		arg0,

@@ -1022,12 +1022,12 @@ func (_c *MockSnapshotRepository_DeleteDeletionTombstone_Call) Run(run func(cont
1022	1022		return _c
1023	1023		}
1024	1024
1025		-	func (_c MockSnapshotRepository_DeleteDeletionTombstone_Call) Return(err error) MockSnapshotRepository_DeleteDeletionTombstone_Call {
	1025	+	func (_c MockSnapshotRepository_CollectGarbage_Call) Return(err error) MockSnapshotRepository_CollectGarbage_Call {
1026	1026		_c.Call.Return(err)
1027	1027		return _c
1028	1028		}
1029	1029
1030		-	func (_c MockSnapshotRepository_DeleteDeletionTombstone_Call) RunAndReturn(run func(context1 context.Context, incarnationID domain.IncarnationID) error) MockSnapshotRepository_DeleteDeletionTombstone_Call {
	1030	+	func (_c MockSnapshotRepository_CollectGarbage_Call) RunAndReturn(run func(context1 context.Context, incarnationIDToCheckpointRef map[domain.IncarnationID]domain.CheckpointRef) error) MockSnapshotRepository_CollectGarbage_Call {
1031	1031		_c.Call.Return(run)
1032	1032		return _c
1033	1033		}

@@ -1089,78 +1089,6 @@ func (_c *MockSnapshotRepository_DeleteIncarnation_Call) RunAndReturn(run func(c
1089	1089		return _c
1090	1090		}
1091	1091
1092		-	// ListDeletionTombstones provides a mock function for the type MockSnapshotRepository
1093		-	func (_mock *MockSnapshotRepository) ListDeletionTombstones(context1 context.Context, deletionTombstoneCursor ports.DeletionTombstoneCursor, maintenanceBudget ports.MaintenanceBudget) (ports.DeletionTombstonePage, error) {
1094		-	ret := _mock.Called(context1, deletionTombstoneCursor, maintenanceBudget)
1095		-
1096		-	if len(ret) == 0 {
1097		-	panic("no return value specified for ListDeletionTombstones")
1098		-	}
1099		-
1100		-	var r0 ports.DeletionTombstonePage
1101		-	var r1 error
1102		-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.DeletionTombstoneCursor, ports.MaintenanceBudget) (ports.DeletionTombstonePage, error)); ok {
1103		-	return returnFunc(context1, deletionTombstoneCursor, maintenanceBudget)
1104		-	}
1105		-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.DeletionTombstoneCursor, ports.MaintenanceBudget) ports.DeletionTombstonePage); ok {
1106		-	r0 = returnFunc(context1, deletionTombstoneCursor, maintenanceBudget)
1107		-	} else {
1108		-	r0 = ret.Get(0).(ports.DeletionTombstonePage)
1109		-	}
1110		-	if returnFunc, ok := ret.Get(1).(func(context.Context, ports.DeletionTombstoneCursor, ports.MaintenanceBudget) error); ok {
1111		-	r1 = returnFunc(context1, deletionTombstoneCursor, maintenanceBudget)
1112		-	} else {
1113		-	r1 = ret.Error(1)
1114		-	}
1115		-	return r0, r1
1116		-	}
1117		-
1118		-	// MockSnapshotRepository_ListDeletionTombstones_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListDeletionTombstones'
1119		-	type MockSnapshotRepository_ListDeletionTombstones_Call struct {
1120		-	*mock.Call
1121		-	}
1122		-
1123		-	// ListDeletionTombstones is a helper method to define mock.On call
1124		-	// - context1 context.Context
1125		-	// - deletionTombstoneCursor ports.DeletionTombstoneCursor
1126		-	// - maintenanceBudget ports.MaintenanceBudget
1127		-	func (_e MockSnapshotRepository_Expecter) ListDeletionTombstones(context1 any, deletionTombstoneCursor any, maintenanceBudget any) MockSnapshotRepository_ListDeletionTombstones_Call {
1128		-	return &MockSnapshotRepository_ListDeletionTombstones_Call{Call: _e.mock.On("ListDeletionTombstones", context1, deletionTombstoneCursor, maintenanceBudget)}
1129		-	}
1130		-
1131		-	func (_c MockSnapshotRepository_ListDeletionTombstones_Call) Run(run func(context1 context.Context, deletionTombstoneCursor ports.DeletionTombstoneCursor, maintenanceBudget ports.MaintenanceBudget)) MockSnapshotRepository_ListDeletionTombstones_Call {
1132		-	_c.Call.Run(func(args mock.Arguments) {
1133		-	var arg0 context.Context
1134		-	if args[0] != nil {
1135		-	arg0 = args[0].(context.Context)
1136		-	}
1137		-	var arg1 ports.DeletionTombstoneCursor
1138		-	if args[1] != nil {
1139		-	arg1 = args[1].(ports.DeletionTombstoneCursor)
1140		-	}
1141		-	var arg2 ports.MaintenanceBudget
1142		-	if args[2] != nil {
1143		-	arg2 = args[2].(ports.MaintenanceBudget)
1144		-	}
1145		-	run(
1146		-	arg0,
1147		-	arg1,
1148		-	arg2,
1149		-	)
1150		-	})
1151		-	return _c
1152		-	}
1153		-
1154		-	func (_c MockSnapshotRepository_ListDeletionTombstones_Call) Return(deletionTombstonePage ports.DeletionTombstonePage, err error) MockSnapshotRepository_ListDeletionTombstones_Call {
1155		-	_c.Call.Return(deletionTombstonePage, err)
1156		-	return _c
1157		-	}
1158		-
1159		-	func (_c MockSnapshotRepository_ListDeletionTombstones_Call) RunAndReturn(run func(context1 context.Context, deletionTombstoneCursor ports.DeletionTombstoneCursor, maintenanceBudget ports.MaintenanceBudget) (ports.DeletionTombstonePage, error)) MockSnapshotRepository_ListDeletionTombstones_Call {
1160		-	_c.Call.Return(run)
1161		-	return _c
1162		-	}
1163		-
1164	1092		// LoadCheckpoint provides a mock function for the type MockSnapshotRepository
1165	1093		func (_mock *MockSnapshotRepository) LoadCheckpoint(context1 context.Context, incarnationID domain.IncarnationID, s string, v ports.CheckpointRef) (ports.SnapshotGeneration, error) {
1166	1094		ret := _mock.Called(context1, incarnationID, s, v)

@@ -1239,78 +1167,6 @@ func (_c *MockSnapshotRepository_LoadCheckpoint_Call) RunAndReturn(run func(cont
1239	1167		return _c
1240	1168		}
1241	1169
1242		-	// MaintainSession provides a mock function for the type MockSnapshotRepository
1243		-	func (_mock *MockSnapshotRepository) MaintainSession(context1 context.Context, retentionPlan ports.RetentionPlan, maintenanceBudget ports.MaintenanceBudget) (bool, error) {
1244		-	ret := _mock.Called(context1, retentionPlan, maintenanceBudget)
1245		-
1246		-	if len(ret) == 0 {
1247		-	panic("no return value specified for MaintainSession")
1248		-	}
1249		-
1250		-	var r0 bool
1251		-	var r1 error
1252		-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.RetentionPlan, ports.MaintenanceBudget) (bool, error)); ok {
1253		-	return returnFunc(context1, retentionPlan, maintenanceBudget)
1254		-	}
1255		-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.RetentionPlan, ports.MaintenanceBudget) bool); ok {
1256		-	r0 = returnFunc(context1, retentionPlan, maintenanceBudget)
1257		-	} else {
1258		-	r0 = ret.Get(0).(bool)
1259		-	}
1260		-	if returnFunc, ok := ret.Get(1).(func(context.Context, ports.RetentionPlan, ports.MaintenanceBudget) error); ok {
1261		-	r1 = returnFunc(context1, retentionPlan, maintenanceBudget)
1262		-	} else {
1263		-	r1 = ret.Error(1)
1264		-	}
1265		-	return r0, r1
1266		-	}
1267		-
1268		-	// MockSnapshotRepository_MaintainSession_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MaintainSession'
1269		-	type MockSnapshotRepository_MaintainSession_Call struct {
1270		-	*mock.Call
1271		-	}
1272		-
1273		-	// MaintainSession is a helper method to define mock.On call
1274		-	// - context1 context.Context
1275		-	// - retentionPlan ports.RetentionPlan
1276		-	// - maintenanceBudget ports.MaintenanceBudget
1277		-	func (_e MockSnapshotRepository_Expecter) MaintainSession(context1 any, retentionPlan any, maintenanceBudget any) MockSnapshotRepository_MaintainSession_Call {
1278		-	return &MockSnapshotRepository_MaintainSession_Call{Call: _e.mock.On("MaintainSession", context1, retentionPlan, maintenanceBudget)}
1279		-	}
1280		-
1281		-	func (_c MockSnapshotRepository_MaintainSession_Call) Run(run func(context1 context.Context, retentionPlan ports.RetentionPlan, maintenanceBudget ports.MaintenanceBudget)) MockSnapshotRepository_MaintainSession_Call {
1282		-	_c.Call.Run(func(args mock.Arguments) {
1283		-	var arg0 context.Context
1284		-	if args[0] != nil {
1285		-	arg0 = args[0].(context.Context)
1286		-	}
1287		-	var arg1 ports.RetentionPlan
1288		-	if args[1] != nil {
1289		-	arg1 = args[1].(ports.RetentionPlan)
1290		-	}
1291		-	var arg2 ports.MaintenanceBudget
1292		-	if args[2] != nil {
1293		-	arg2 = args[2].(ports.MaintenanceBudget)
1294		-	}
1295		-	run(
1296		-	arg0,
1297		-	arg1,
1298		-	arg2,
1299		-	)
1300		-	})
1301		-	return _c
1302		-	}
1303		-
1304		-	func (_c MockSnapshotRepository_MaintainSession_Call) Return(b bool, err error) MockSnapshotRepository_MaintainSession_Call {
1305		-	_c.Call.Return(b, err)
1306		-	return _c
1307		-	}
1308		-
1309		-	func (_c MockSnapshotRepository_MaintainSession_Call) RunAndReturn(run func(context1 context.Context, retentionPlan ports.RetentionPlan, maintenanceBudget ports.MaintenanceBudget) (bool, error)) MockSnapshotRepository_MaintainSession_Call {
1310		-	_c.Call.Return(run)
1311		-	return _c
1312		-	}
1313		-
1314	1170		// Publish provides a mock function for the type MockSnapshotRepository
1315	1171		func (_mock *MockSnapshotRepository) Publish(context1 context.Context, snapshotPublication ports.SnapshotPublication) error {
1316	1172		ret := _mock.Called(context1, snapshotPublication)

@@ -1430,60 +1286,3 @@ func (_c *MockSnapshotRepository_RepairHEAD_Call) RunAndReturn(run func(context1
1430	1286		_c.Call.Return(run)
1431	1287		return _c
1432	1288		}
1433		-
1434		-	// WriteDeletionTombstone provides a mock function for the type MockSnapshotRepository
1435		-	func (_mock *MockSnapshotRepository) WriteDeletionTombstone(context1 context.Context, deletionTombstone domain.DeletionTombstone) error {
1436		-	ret := _mock.Called(context1, deletionTombstone)
1437		-
1438		-	if len(ret) == 0 {
1439		-	panic("no return value specified for WriteDeletionTombstone")
1440		-	}
1441		-
1442		-	var r0 error
1443		-	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.DeletionTombstone) error); ok {
1444		-	r0 = returnFunc(context1, deletionTombstone)
1445		-	} else {
1446		-	r0 = ret.Error(0)
1447		-	}
1448		-	return r0
1449		-	}
1450		-
1451		-	// MockSnapshotRepository_WriteDeletionTombstone_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'WriteDeletionTombstone'
1452		-	type MockSnapshotRepository_WriteDeletionTombstone_Call struct {
1453		-	*mock.Call
1454		-	}
1455		-
1456		-	// WriteDeletionTombstone is a helper method to define mock.On call
1457		-	// - context1 context.Context
1458		-	// - deletionTombstone domain.DeletionTombstone
1459		-	func (_e MockSnapshotRepository_Expecter) WriteDeletionTombstone(context1 any, deletionTombstone any) MockSnapshotRepository_WriteDeletionTombstone_Call {
1460		-	return &MockSnapshotRepository_WriteDeletionTombstone_Call{Call: _e.mock.On("WriteDeletionTombstone", context1, deletionTombstone)}
1461		-	}
1462		-
1463		-	func (_c MockSnapshotRepository_WriteDeletionTombstone_Call) Run(run func(context1 context.Context, deletionTombstone domain.DeletionTombstone)) MockSnapshotRepository_WriteDeletionTombstone_Call {
1464		-	_c.Call.Run(func(args mock.Arguments) {
1465		-	var arg0 context.Context
1466		-	if args[0] != nil {
1467		-	arg0 = args[0].(context.Context)
1468		-	}
1469		-	var arg1 domain.DeletionTombstone
1470		-	if args[1] != nil {
1471		-	arg1 = args[1].(domain.DeletionTombstone)
1472		-	}
1473		-	run(
1474		-	arg0,
1475		-	arg1,
1476		-	)
1477		-	})
1478		-	return _c
1479		-	}
1480		-
1481		-	func (_c MockSnapshotRepository_WriteDeletionTombstone_Call) Return(err error) MockSnapshotRepository_WriteDeletionTombstone_Call {
1482		-	_c.Call.Return(err)
1483		-	return _c
1484		-	}
1485		-
1486		-	func (_c MockSnapshotRepository_WriteDeletionTombstone_Call) RunAndReturn(run func(context1 context.Context, deletionTombstone domain.DeletionTombstone) error) MockSnapshotRepository_WriteDeletionTombstone_Call {
1487		-	_c.Call.Return(run)
1488		-	return _c
1489		-	}

M internal/ports/recovery.go +1 -26


@@ -2,7 +2,6 @@ package ports
2	2
3	3		import (
4	4		"context"
5		-	"errors"
6	5
7	6		"github.com/bnema/vev/internal/domain"
8	7		)

@@ -27,27 +26,6 @@ type SnapshotGeneration struct {
27	26		Objects map[SnapshotDigest][]byte
28	27		}
29	28
30		-	type RetentionPlan struct {
31		-	IncarnationID domain.IncarnationID
32		-	Keep []CheckpointRef
33		-	PinAll bool
34		-	}
35		-
36		-	type MaintenanceBudget struct {
37		-	Entries uint64
38		-	Bytes uint64
39		-	}
40		-
41		-	type DeletionTombstoneCursor struct{ After string }
42		-
43		-	type DeletionTombstonePage struct {
44		-	Tombstones []domain.DeletionTombstone
45		-	Next DeletionTombstoneCursor
46		-	Done bool
47		-	}
48		-
49		-	var ErrBudgetExhausted = errors.New("snapshot maintenance budget exhausted")
50		-
51	29		type CheckpointCoordinator interface {
52	30		PublishCheckpoint(context.Context, string, SnapshotPublication) (domain.CatalogueRecord, error)
53	31		}

@@ -82,9 +60,6 @@ type SnapshotRepository interface {
82	60		Publish(context.Context, SnapshotPublication) error
83	61		LoadCheckpoint(context.Context, domain.IncarnationID, string, CheckpointRef) (SnapshotGeneration, error)
84	62		RepairHEAD(context.Context, domain.IncarnationID, CheckpointRef) error
85		-	WriteDeletionTombstone(context.Context, domain.DeletionTombstone) error
86		-	ListDeletionTombstones(context.Context, DeletionTombstoneCursor, MaintenanceBudget) (DeletionTombstonePage, error)
87		-	DeleteDeletionTombstone(context.Context, domain.IncarnationID) error
88	63		DeleteIncarnation(context.Context, domain.IncarnationID) error
89		-	MaintainSession(context.Context, RetentionPlan, MaintenanceBudget) (bool, error)
	64	+	CollectGarbage(context.Context, map[domain.IncarnationID]domain.CheckpointRef) error
90	65		}

M internal/usecase/daemon/coderabbit_regression_test.go +2 -1


@@ -3,6 +3,7 @@ package daemon
3	3		import (
4	4		"context"
5	5		"sync"
	6	+	"syscall"
6	7		"testing"
7	8
8	9		"github.com/stretchr/testify/require"

@@ -14,7 +15,7 @@ import (
14	15		func TestRestoreTransientLoadFailureBecomesDiscardable(t *testing.T) {
15	16		record := durableRecoveryRecord(0)
16	17		repository := &durableRecoveryRepository{
17		-	errors: map[string]error{record.Name: ports.ErrBudgetExhausted},
	18	+	errors: map[string]error{record.Name: syscall.ENOSPC},
18	19		loads: make(map[string]int),
19	20		repairs: make(map[string]domain.CheckpointRef),
20	21		}

M internal/usecase/daemon/daemon.go +6 -7


@@ -325,11 +325,11 @@ func WithNoticeStore(store ports.NoticeStore) Option {
325	325		return func(d *Daemon) { d.noticeStore = store }
326	326		}
327	327
328		-	// WithDurableMaintenance schedules bounded retention after catalogue-driven
329		-	// restoration. Unresolved incarnation IDs remain pinned for this daemon lifetime.
330		-	func WithDurableMaintenance(catalogue ports.Catalogue, repository ports.SnapshotRepository, unresolved []domain.IncarnationID) Option {
	328	+	// WithDurableMaintenance schedules one garbage-collection pass after
	329	+	// catalogue-driven restoration completes.
	330	+	func WithDurableMaintenance(catalogue ports.Catalogue, repository ports.SnapshotRepository) Option {
331	331		return func(d *Daemon) {
332		-	d.maintenance = newMaintenanceDependencies(catalogue, repository, unresolved)
	332	+	d.maintenance = newMaintenanceDependencies(catalogue, repository)
333	333		}
334	334		}
335	335

@@ -865,9 +865,8 @@ func (d *Daemon) handleKill(tr ports.Transport, f ports.Frame) {
865	865		if target == nil {
866	866		if _, ok := d.stopped[k.Name]; ok {
867	867		d.mu.Unlock()
868		-	// Stopped sessions follow the same tombstone-backed transaction as
869		-	// live and offline purges. In particular, a retained legacy source
870		-	// must never outlive metadata after an interrupted control kill.
	868	+	// Stopped sessions use the same catalogue-first, incarnation-second
	869	+	// deletion order as live and offline purges.
871	870		if err := d.retryStoppedPurge(k.Name); err != nil {
872	871		d.log.Warn("deleting stopped session failed", "err", err, "session", k.Name)
873	872		_ = tr.Send(frameError(ports.ErrInternal, "deleting stopped session failed"))

M internal/usecase/daemon/durable_authority_test.go +1 -4


@@ -15,13 +15,10 @@ import (
15	15
16	16		type testLifecycleRepository struct{ ports.SnapshotRepository }
17	17
18		-	func (testLifecycleRepository) WriteDeletionTombstone(context.Context, domain.DeletionTombstone) error {
19		-	return nil
20		-	}
21	18		func (testLifecycleRepository) DeleteIncarnation(context.Context, domain.IncarnationID) error {
22	19		return nil
23	20		}
24		-	func (testLifecycleRepository) DeleteDeletionTombstone(context.Context, domain.IncarnationID) error {
	21	+	func (testLifecycleRepository) CollectGarbage(context.Context, map[domain.IncarnationID]domain.CheckpointRef) error {
25	22		return nil
26	23		}
27	24

M internal/usecase/daemon/durable_recovery_test.go +0 -1


@@ -397,7 +397,6 @@ func TestRetryableRestoreLoadErrorClassification(t *testing.T) {
397	397		err error
398	398		retryable bool
399	399		}{
400		-	{name: "budget exhausted", err: fmt.Errorf("wrapped: %w", ports.ErrBudgetExhausted), retryable: true},
401	400		{name: "missing path", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.ENOENT})},
402	401		{name: "permission denied", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EACCES})},
403	402		{name: "invalid path operation", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EINVAL})},

M internal/usecase/daemon/maintenance.go +21 -92


@@ -2,50 +2,18 @@ package daemon
2	2
3	3		import (
4	4		"context"
5		-	"time"
6	5
7	6		"github.com/bnema/vev/internal/domain"
8	7		"github.com/bnema/vev/internal/ports"
9	8		)
10	9
11		-	const (
12		-	maintenanceInterval = time.Minute
13		-	maintenanceEntries = 64
14		-	maintenanceBytes = 8 << 20
15		-	)
16		-
17		-	// maintenanceDependencies is exclusively owned by runDurableMaintenance's
18		-	// single goroutine after startup. Shutdown joins that goroutine before these
19		-	// cursor and repair sets can be observed or reused.
20	10		type maintenanceDependencies struct {
21	11		catalogue ports.Catalogue
22	12		repository ports.SnapshotRepository
23		-	unresolved map[domain.IncarnationID]struct{}
24		-	}
25		-
26		-	func newMaintenanceDependencies(catalogue ports.Catalogue, repository ports.SnapshotRepository, unresolved []domain.IncarnationID) maintenanceDependencies {
27		-	deps := maintenanceDependencies{
28		-	catalogue: catalogue, repository: repository,
29		-	unresolved: make(map[domain.IncarnationID]struct{}, len(unresolved)),
30		-	}
31		-	for _, id := range unresolved {
32		-	if id != (domain.IncarnationID{}) {
33		-	deps.unresolved[id] = struct{}{}
34		-	}
35		-	}
36		-	return deps
37	13		}
38	14
39		-	func retentionPlan(record domain.CatalogueRecord, unresolved, restoring bool) ports.RetentionPlan {
40		-	plan := ports.RetentionPlan{IncarnationID: record.IncarnationID}
41		-	if unresolved \|\| restoring \|\| record.RecoveryState == domain.RecoveryDegraded \|\| record.RecoveryState == domain.RecoveryDeleting {
42		-	plan.PinAll = true
43		-	return plan
44		-	}
45		-	if record.Committed != nil {
46		-	plan.Keep = append(plan.Keep, *record.Committed)
47		-	}
48		-	return plan
	15	+	func newMaintenanceDependencies(catalogue ports.Catalogue, repository ports.SnapshotRepository) maintenanceDependencies {
	16	+	return maintenanceDependencies{catalogue: catalogue, repository: repository}
49	17		}
50	18
51	19		func (d *Daemon) startDurableMaintenance() {

@@ -63,81 +31,42 @@ func (d *Daemon) startDurableMaintenance() {
63	31		done := d.maintenanceWorkerDone
64	32		d.snapshotWorkerMu.Unlock()
65	33
66		-	// Register the completion channel before launch so shutdown can never begin
67		-	// waiting while this durable writer is still unregistered.
68	34		go func() {
69	35		defer close(done)
70		-	d.runDurableMaintenance(ctx)
	36	+	d.runStartupGarbageCollection(ctx)
71	37		}()
72	38		}
73	39
74		-	func (d *Daemon) runDurableMaintenance(ctx context.Context) {
	40	+	// runStartupGarbageCollection waits for catalogue-driven restoration, then
	41	+	// takes a fresh catalogue snapshot and applies GC exactly once. A catalogue
	42	+	// read failure never produces a keep set, so destructive collection cannot run
	43	+	// without known-good catalogue state.
	44	+	func (d *Daemon) runStartupGarbageCollection(ctx context.Context) {
75	45		select {
76	46		case <-ctx.Done():
77	47		return
78	48		case <-d.restoreDone:
79	49		}
80		-	for {
81		-	incomplete := d.runDurableMaintenanceTick(ctx)
82		-	if ctx.Err() != nil {
83		-	return
84		-	}
85		-	if incomplete {
86		-	continue
87		-	}
88		-	timer := d.clock.NewTimer(maintenanceInterval)
89		-	select {
90		-	case <-ctx.Done():
91		-	timer.Stop()
92		-	return
93		-	case <-timer.C():
94		-	}
95		-	}
96		-	}
97		-
98		-	// runDurableMaintenanceTick reports whether bounded work remains and should be
99		-	// continued immediately rather than delayed until the periodic interval.
100		-	func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool {
101	50		deps := &d.maintenance
102	51		if deps.catalogue == nil \|\| deps.repository == nil {
103		-	return false
	52	+	return
104	53		}
105		-
106	54		records, err := deps.catalogue.Records()
107	55		if err != nil {
108		-	d.log.Warn("durable_maintenance_tick", "action", "catalogue-read", "done", false, "err", err)
109		-	return false
110		-	}
111		-	d.mu.Lock()
112		-	restoringIncarnations := make(map[domain.IncarnationID]struct{})
113		-	for _, stopped := range d.stopped {
114		-	if stopped.state == runtimeRestoring {
115		-	restoringIncarnations[stopped.incarnation] = struct{}{}
116		-	}
	56	+	d.log.Warn("snapshot_garbage_collection_skipped", "reason", "catalogue-read-failed", "err", err)
	57	+	return
117	58		}
118		-	d.mu.Unlock()
119		-
120		-	incomplete := false
	59	+	keep := make(map[domain.IncarnationID]domain.CheckpointRef, len(records))
121	60		for _, record := range records {
122		-	if err := ctx.Err(); err != nil {
123		-	return false
	61	+	var committed domain.CheckpointRef
	62	+	if record.Committed != nil {
	63	+	committed = *record.Committed
124	64		}
125		-	_, unresolved := deps.unresolved[record.IncarnationID]
126		-	_, restoring := restoringIncarnations[record.IncarnationID]
127		-	plan := retentionPlan(record, unresolved, restoring)
128		-	done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes})
129		-	if err == nil && !done {
130		-	incomplete = true
131		-	}
132		-	d.log.Info("durable_maintenance_session",
133		-	"incarnation", record.IncarnationID.String(),
134		-	"action", "retention",
135		-	"pin_all", plan.PinAll,
136		-	"requested_entries", maintenanceEntries,
137		-	"requested_bytes", maintenanceBytes,
138		-	"done", done,
139		-	"err", err,
140		-	)
	65	+	keep[record.IncarnationID] = committed
	66	+	}
	67	+	if err := deps.repository.CollectGarbage(ctx, keep); err != nil {
	68	+	d.log.Warn("snapshot_garbage_collection_failed", "incarnations", len(keep), "err", err)
	69	+	return
141	70		}
142		-	return incomplete
	71	+	d.log.Info("snapshot_garbage_collection_complete", "incarnations", len(keep))
143	72		}

M internal/usecase/daemon/maintenance_test.go +24 -36


@@ -2,55 +2,43 @@ package daemon
2	2
3	3		import (
4	4		"context"
	5	+	"errors"
5	6		"testing"
6	7
7	8		"github.com/bnema/vev/internal/domain"
8		-	"github.com/bnema/vev/internal/ports"
9	9		portsmocks "github.com/bnema/vev/internal/ports/mocks"
10	10		"github.com/stretchr/testify/mock"
11	11		"github.com/stretchr/testify/require"
12	12		)
13	13
14		-	func TestProductionMaintenanceUsesCatalogue(t *testing.T) {
15		-	record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh}
	14	+	func TestStartupGarbageCollectionUsesValidatedCatalogue(t *testing.T) {
	15	+	committed := domain.CheckpointRef{Generation: 3, ManifestDigest: [32]byte{3}}
	16	+	records := []domain.CatalogueRecord{
	17	+	{Name: "work", IncarnationID: domain.IncarnationID{1}, Committed: &committed},
	18	+	{Name: "fresh", IncarnationID: domain.IncarnationID{2}},
	19	+	}
16	20		catalogue := portsmocks.NewMockCatalogue(t)
17		-	catalogue.EXPECT().Records().Return([]domain.CatalogueRecord{record}, nil).Once()
	21	+	catalogue.EXPECT().Records().Return(records, nil).Once()
18	22		repository := portsmocks.NewMockSnapshotRepository(t)
19		-	repository.EXPECT().MaintainSession(mock.Anything, mock.MatchedBy(func(plan ports.RetentionPlan) bool {
20		-	return plan.IncarnationID == record.IncarnationID && !plan.PinAll && len(plan.Keep) == 0
21		-	}), ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes}).Return(true, nil).Once()
	23	+	repository.EXPECT().CollectGarbage(mock.Anything, mock.MatchedBy(func(keep map[domain.IncarnationID]domain.CheckpointRef) bool {
	24	+	return len(keep) == 2 && keep[records[0].IncarnationID] == committed && keep[records[1].IncarnationID] == (domain.CheckpointRef{})
	25	+	})).Return(nil).Once()
22	26		d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
23		-	WithDurableMaintenance(catalogue, repository, nil)(d)
	27	+	WithDurableMaintenance(catalogue, repository)(d)
	28	+	d.closeRestoreDone()
24	29
25		-	require.False(t, d.runDurableMaintenanceTick(context.Background()))
	30	+	d.runStartupGarbageCollection(context.Background())
26	31		}
27	32
28		-	func TestRetentionPlanPinsEveryUnresolvedRuntimeState(t *testing.T) {
29		-	committed := domain.CheckpointRef{Generation: 3, ManifestDigest: [32]byte{3}}
30		-	healthy := domain.CatalogueRecord{
31		-	Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryHealthy,
32		-	Committed: &committed,
33		-	}
	33	+	func TestStartupGarbageCollectionSkipsUnknownCatalogueState(t *testing.T) {
	34	+	catalogue := portsmocks.NewMockCatalogue(t)
	35	+	catalogue.EXPECT().Records().Return(nil, errors.New("catalogue read failed")).Once()
	36	+	repository := portsmocks.NewMockSnapshotRepository(t)
	37	+	d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
	38	+	WithDurableMaintenance(catalogue, repository)(d)
	39	+	d.closeRestoreDone()
34	40
35		-	for _, tc := range []struct {
36		-	name string
37		-	record domain.CatalogueRecord
38		-	unresolved bool
39		-	restoring bool
40		-	wantPin bool
41		-	}{
42		-	{name: "healthy", record: healthy},
43		-	{name: "pending deletion", record: healthy, unresolved: true, wantPin: true},
44		-	{name: "restoring", record: healthy, restoring: true, wantPin: true},
45		-	{name: "degraded", record: func() domain.CatalogueRecord { r := healthy; r.RecoveryState = domain.RecoveryDegraded; return r }(), wantPin: true},
46		-	{name: "deleting", record: func() domain.CatalogueRecord { r := healthy; r.RecoveryState = domain.RecoveryDeleting; return r }(), wantPin: true},
47		-	} {
48		-	t.Run(tc.name, func(t *testing.T) {
49		-	plan := retentionPlan(tc.record, tc.unresolved, tc.restoring)
50		-	require.Equal(t, tc.wantPin, plan.PinAll)
51		-	if !tc.wantPin {
52		-	require.Equal(t, []ports.CheckpointRef{committed}, plan.Keep)
53		-	}
54		-	})
55		-	}
	41	+	d.runStartupGarbageCollection(context.Background())
	42	+	repository.AssertNotCalled(t, "CollectGarbage", mock.Anything, mock.Anything)
	43	+	require.NotNil(t, d.maintenance.catalogue)
56	44		}

M internal/usecase/daemon/session.go +9 -15


@@ -728,10 +728,8 @@ func (d Daemon) renameSession(sess session, name string) error {
728	728		return err
729	729		}
730	730
731		-	// Reserve the rename while state is inspected, then perform persistence and
732		-	// repository I/O without daemon/session locks. The old identity is first
733		-	// quarantined and durably purged; committing the new record before that
734		-	// transaction completes could leave a restorable old legacy source.
	731	+	// Reserve the rename while state is inspected, then persist the new name
	732	+	// without changing the incarnation-keyed snapshot namespace.
735	733		d.mu.Lock()
736	734		if taken := d.findByNameLocked(name); taken != nil && taken != sess {
737	735		d.mu.Unlock()

@@ -1036,10 +1034,9 @@ func (d *Daemon) beginSnapshotPurge(_ string, _ domain.IncarnationID) error {
1036	1034		return nil
1037	1035		}
1038	1036
1039		-	// finishSnapshotPurge deletes both independently durable snapshot sources,
1040		-	// then metadata, and only then clears the tombstone. Any failure deliberately
1041		-	// leaves the marker in place so startup cannot restore or import the name and
1042		-	// a later live/offline kill can retry the idempotent source deletes.
	1037	+	// finishSnapshotPurge removes catalogue metadata first, then deletes the
	1038	+	// incarnation directory. Startup garbage collection removes the directory if
	1039	+	// the second step is interrupted.
1043	1040		func (d *Daemon) finishSnapshotPurge(ctx context.Context, name string, _ domain.IncarnationID) error {
1044	1041		if !d.persistEnabled {
1045	1042		return nil

@@ -1240,10 +1237,8 @@ func (d Daemon) killSessionWithSnapshotDeadline(sess session, reason uint8, pu
1240	1237		name := sess.name
1241	1238		incarnation := sess.incarnation
1242	1239		sess.mu.Unlock()
1243		-	// Commit the tombstone before changing live identity or metadata. A
1244		-	// repository delete must never race an older publication that could recreate
1245		-	// the just-quarantined name. Do not hold daemon or session locks while
1246		-	// tombstoning or joining: a repository implementation may block on I/O.
	1240	+	// Join snapshot publication before changing live identity or deleting the
	1241	+	// incarnation directory, so an older publication cannot recreate it.
1247	1242		if purge && !isEphemeral {
1248	1243		if err := d.beginSnapshotPurge(name, incarnation); err != nil {
1249	1244		return err

@@ -1377,9 +1372,8 @@ func (d Daemon) killSessionWithSnapshotDeadline(sess session, reason uint8, pu
1377	1372		}
1378	1373		}
1379	1374		// The coordinator and all panes are now stopped, so no producer can publish
1380		-	// another generation after this destructive source deletion. Keep the
1381		-	// tombstone and hidden stopped record when any source or metadata operation
1382		-	// fails; retryStoppedPurge (including a repeated live kill) resumes safely.
	1375	+	// another generation after this destructive source deletion. Keep the hidden
	1376	+	// stopped record when deletion fails so a repeated live kill can retry.
1383	1377		if !ephemeral && purge {
1384	1378		if err := d.finishSnapshotPurge(d.serveCtx, stoppedName, incarnation); err != nil {
1385	1379		purgeErr = errors.Join(purgeErr, err)

M internal/usecase/daemon/session_purge_control_test.go +2 -3


@@ -25,12 +25,11 @@ func TestStoppedPurgeMetadataFailureRemainsFencedForRetry(t *testing.T) {
25	25		d.stopped["work"] = stoppedSession{name: "work", incarnation: record.IncarnationID, record: record}
26	26
27	27		require.ErrorIs(t, d.retryStoppedPurge("work"), metadataErr)
28		-	require.True(t, repository.tombstoned["work"])
29		-	require.Equal(t, []string{"tombstone", "delete incarnation"}, repository.calls)
	28	+	require.Empty(t, repository.calls, "snapshot deletion must wait for catalogue removal")
30	29
31	30		state.mu.Lock()
32	31		state.deleteErr = nil
33	32		state.mu.Unlock()
34	33		require.NoError(t, d.retryStoppedPurge("work"))
35		-	require.Equal(t, []string{"tombstone", "delete incarnation", "tombstone", "delete incarnation", "clear tombstone"}, repository.calls)
	34	+	require.Equal(t, []string{"delete incarnation"}, repository.calls)
36	35		}

M internal/usecase/daemon/session_purge_retry_test.go +6 -35


@@ -15,26 +15,10 @@ import (
15	15		type retryablePurgeRepository struct {
16	16		noOpSnapshotRepository
17	17		deleteErr error
18		-	tombstoned map[string]bool
19	18		calls []string
20	19		deadlineScoped []bool
21	20		}
22	21
23		-	func (r *retryablePurgeRepository) Publish(context.Context, ports.SnapshotPublication) error {
24		-	return nil
25		-	}
26		-
27		-	func (r *retryablePurgeRepository) WriteDeletionTombstone(ctx context.Context, tombstone domain.DeletionTombstone) error {
28		-	_, hasDeadline := ctx.Deadline()
29		-	r.deadlineScoped = append(r.deadlineScoped, hasDeadline)
30		-	if r.tombstoned == nil {
31		-	r.tombstoned = make(map[string]bool)
32		-	}
33		-	r.calls = append(r.calls, "tombstone")
34		-	r.tombstoned[tombstone.Name] = true
35		-	return nil
36		-	}
37		-
38	22		func (r *retryablePurgeRepository) DeleteIncarnation(ctx context.Context, _ domain.IncarnationID) error {
39	23		_, hasDeadline := ctx.Deadline()
40	24		r.deadlineScoped = append(r.deadlineScoped, hasDeadline)

@@ -42,15 +26,7 @@ func (r *retryablePurgeRepository) DeleteIncarnation(ctx context.Context, _ doma
42	26		return r.deleteErr
43	27		}
44	28
45		-	func (r *retryablePurgeRepository) DeleteDeletionTombstone(ctx context.Context, _ domain.IncarnationID) error {
46		-	_, hasDeadline := ctx.Deadline()
47		-	r.deadlineScoped = append(r.deadlineScoped, hasDeadline)
48		-	r.calls = append(r.calls, "clear tombstone")
49		-	clear(r.tombstoned)
50		-	return nil
51		-	}
52		-
53		-	func TestLivePurgeRetainsTombstoneAcrossPartialDeletion(t *testing.T) {
	29	+	func TestLivePurgeLeavesFailedDirectoryDeletionForStartupGarbageCollection(t *testing.T) {
54	30		d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{})
55	31		repository := &retryablePurgeRepository{deleteErr: errors.New("delete failed")}
56	32		WithSnapshotRepository(repository)(d)

@@ -66,14 +42,9 @@ func TestLivePurgeRetainsTombstoneAcrossPartialDeletion(t *testing.T) {
66	42		d.sessions = map[domain.SessionID]*session{sess.id: sess}
67	43
68	44		require.Error(t, d.killSession(sess, ports.ReasonSessionKilled, true))
69		-	require.True(t, repository.tombstoned["work"])
70		-	require.Equal(t, []string{"tombstone", "delete incarnation"}, repository.calls)
71		-
72		-	repository.deleteErr = nil
73		-	require.NoError(t, d.retryStoppedPurge("work"))
74		-	require.False(t, repository.tombstoned["work"])
75		-	require.Equal(t, []string{"tombstone", "delete incarnation", "tombstone", "delete incarnation", "clear tombstone"}, repository.calls)
76		-	for _, scoped := range repository.deadlineScoped {
77		-	require.True(t, scoped)
78		-	}
	45	+	require.Equal(t, []string{"delete incarnation"}, repository.calls)
	46	+	_, exists, err := d.catalogue.Record("work")
	47	+	require.NoError(t, err)
	48	+	require.False(t, exists, "catalogue removal commits before best-effort directory deletion")
	49	+	require.Equal(t, []bool{true}, repository.deadlineScoped)
79	50		}

M internal/usecase/daemon/snapshot_restore_incremental.go +1 -2


@@ -260,8 +260,7 @@ func isRetryableRestoreLoadError(err error) bool {
260	260		if err == nil \|\| errors.Is(err, context.Canceled) \|\| errors.Is(err, context.DeadlineExceeded) {
261	261		return false
262	262		}
263		-	if errors.Is(err, ports.ErrBudgetExhausted) \|\|
264		-	errors.Is(err, syscall.ENOSPC) \|\|
	263	+	if errors.Is(err, syscall.ENOSPC) \|\|
265	264		errors.Is(err, syscall.ENOMEM) \|\|
266	265		errors.Is(err, syscall.EMFILE) \|\|
267	266		errors.Is(err, syscall.ENFILE) \|\|

M internal/usecase/daemon/snapshot_test.go +2 -11


@@ -101,20 +101,11 @@ func (noOpSnapshotRepository) LoadCheckpoint(context.Context, domain.Incarnation
101	101		func (noOpSnapshotRepository) RepairHEAD(context.Context, domain.IncarnationID, ports.CheckpointRef) error {
102	102		return nil
103	103		}
104		-	func (noOpSnapshotRepository) WriteDeletionTombstone(context.Context, domain.DeletionTombstone) error {
105		-	return nil
106		-	}
107		-	func (noOpSnapshotRepository) ListDeletionTombstones(context.Context, ports.DeletionTombstoneCursor, ports.MaintenanceBudget) (ports.DeletionTombstonePage, error) {
108		-	return ports.DeletionTombstonePage{Done: true}, nil
109		-	}
110		-	func (noOpSnapshotRepository) DeleteDeletionTombstone(context.Context, domain.IncarnationID) error {
111		-	return nil
112		-	}
113	104		func (noOpSnapshotRepository) DeleteIncarnation(context.Context, domain.IncarnationID) error {
114	105		return nil
115	106		}
116		-	func (noOpSnapshotRepository) MaintainSession(context.Context, ports.RetentionPlan, ports.MaintenanceBudget) (bool, error) {
117		-	return true, nil
	107	+	func (noOpSnapshotRepository) CollectGarbage(context.Context, map[domain.IncarnationID]domain.CheckpointRef) error {
	108	+	return nil
118	109		}
119	110
120	111		func awaitSnapshotIdle(t testing.TB, sess *session) {

M internal/usecase/recovery/coordinator.go +5 -204

@@ -1,5 +1,5 @@
1		-	// Package recovery coordinates crash-resumable operations across the catalogue
2		-	// and snapshot repository without depending on filesystem adapters.
	1	+	// Package recovery coordinates idempotent operations across the catalogue and
	2	+	// snapshot repository without depending on filesystem adapters.
3	3		package recovery
4	4
5	5		import (

@@ -14,8 +14,6 @@ import (
14	14		"github.com/bnema/vev/internal/ports"
15	15		)
16	16
17		-	var recoveryListingBudget = ports.MaintenanceBudget{Entries: 64, Bytes: 64 << 10}
18		-
19	17		type KeyLocks struct {
20	18		mu sync.Mutex
21	19		refs map[string]*keyLockRef

@@ -83,52 +81,11 @@ func compactStrings(values []string) []string {
83	81		return out
84	82		}
85	83
86		-	var (
87		-	// ErrDeletionTombstoneInvalid marks a tombstone whose own identity cannot be
88		-	// used. ErrDeletionTombstoneConflict marks a tombstone that disagrees with a
89		-	// live catalogue record. Both are session-scoped: they fence one session
90		-	// instead of failing startup.
91		-	ErrDeletionTombstoneInvalid = errors.New("recovery: deletion tombstone is unusable")
92		-	ErrDeletionTombstoneConflict = errors.New("recovery: deletion tombstone conflicts with catalogue")
93		-	// ErrSessionRecordInvalid marks a catalogue record whose own contents cannot
94		-	// be advanced deterministically.
95		-	ErrSessionRecordInvalid = errors.New("recovery: catalogue record is unusable")
96		-	)
97		-
98		-	// Degraded reasons are content-free by construction: they never carry paths,
99		-	// terminal content, or operator input.
100		-	const (
101		-	degradedReasonTombstone = "deletion tombstone conflict"
102		-	degradedReasonRecord = "catalogue record conflict"
103		-	)
104		-
105	84		type Coordinator struct {
106	85		catalogue ports.Catalogue
107	86		repository ports.SnapshotRepository
108	87		locks *KeyLocks
109	88		random io.Reader
110		-
111		-	conflictMu sync.Mutex
112		-	conflicts []RecoveryConflict
113		-	}
114		-
115		-	// RecoveryConflict is one session-scoped item Recover fenced off. Recover keeps
116		-	// the durable source (intent or tombstone) so a later operator action can still
117		-	// resolve it; startup continues for every other session.
118		-	type RecoveryConflict struct {
119		-	Session string
120		-	Kind string
121		-	Err error
122		-	}
123		-
124		-	// Conflicts returns the session-scoped items fenced by the most recent Recover call.
125		-	func (c *Coordinator) Conflicts() []RecoveryConflict {
126		-	if c == nil {
127		-	return nil
128		-	}
129		-	c.conflictMu.Lock()
130		-	defer c.conflictMu.Unlock()
131		-	return append([]RecoveryConflict(nil), c.conflicts...)
132	89		}
133	90
134	91		func NewCoordinator(catalogue ports.Catalogue, repository ports.SnapshotRepository, random io.Reader) *Coordinator {

@@ -203,8 +160,8 @@ func (c *Coordinator) Rename(ctx context.Context, oldName, newName string) (doma
203	160		return record, nil
204	161		}
205	162
206		-	// Delete executes the durable deletion protocol. Every completed boundary is
207		-	// independently sufficient for Recover to roll the operation forward.
	163	+	// Delete removes the catalogue record before deleting its incarnation. A crash
	164	+	// between those steps leaves an orphan that startup garbage collection removes.
208	165		func (c *Coordinator) Delete(ctx context.Context, name string) error {
209	166		if c == nil \|\| c.catalogue == nil \|\| c.repository == nil \|\| c.locks == nil {
210	167		return errors.New("recovery: incomplete delete dependencies")

@@ -221,164 +178,8 @@ func (c *Coordinator) Delete(ctx context.Context, name string) error {
221	178		if !ok {
222	179		return nil
223	180		}
224		-	return c.deleteLocked(ctx, record)
225		-	}
226		-
227		-	func (c *Coordinator) deleteLocked(ctx context.Context, record domain.CatalogueRecord) error {
228		-	if err := ctx.Err(); err != nil {
229		-	return err
230		-	}
231		-	if record.RecoveryState != domain.RecoveryDeleting {
232		-	record.RecoveryState = domain.RecoveryDeleting
233		-	record.DegradedReason = ""
234		-	if err := record.Validate(); err != nil {
235		-	return fmt.Errorf("%w: invalid deleting record: %w", ErrSessionRecordInvalid, err)
236		-	}
237		-	if err := c.catalogue.Replace(record.Name, record); err != nil {
238		-	return err
239		-	}
240		-	}
241		-	tombstone := domain.DeletionTombstone{Name: record.Name, IncarnationID: record.IncarnationID}
242		-	if err := c.repository.WriteDeletionTombstone(ctx, tombstone); err != nil {
243		-	return err
244		-	}
245		-	if err := c.repository.DeleteIncarnation(ctx, record.IncarnationID); err != nil {
246		-	return err
247		-	}
248	181		if err := c.catalogue.Delete(record.Name); err != nil {
249	182		return err
250	183		}
251		-	return c.repository.DeleteDeletionTombstone(ctx, record.IncarnationID)
252		-	}
253		-
254		-	// Recover rolls deleting records and strict deletion tombstones forward. The
255		-	// two sources are enumerated independently because catalogue removal precedes
256		-	// tombstone removal in the commit protocol.
257		-	//
258		-	// A per-item failure that only proves one session's durable state disagrees
259		-	// with itself fences that session and lets startup continue: healthy sessions
260		-	// must still restore next to a corrupt neighbour. Infrastructure failures
261		-	// (listing, repository, or catalogue IO, and a non-advancing cursor) still
262		-	// abort startup, because they say nothing about any single session.
263		-	func (c *Coordinator) Recover(ctx context.Context) error {
264		-	if c == nil \|\| c.catalogue == nil \|\| c.repository == nil \|\| c.locks == nil {
265		-	return errors.New("recovery: incomplete coordinator dependencies")
266		-	}
267		-	c.conflictMu.Lock()
268		-	c.conflicts = nil
269		-	c.conflictMu.Unlock()
270		-
271		-	records, err := c.catalogue.Records()
272		-	if err != nil {
273		-	return err
274		-	}
275		-	for _, candidate := range records {
276		-	if candidate.RecoveryState != domain.RecoveryDeleting {
277		-	continue
278		-	}
279		-	unlock := c.locks.Lock([]string{candidate.Name})
280		-	var deleteErr error
281		-	current, ok, readErr := c.catalogue.Record(candidate.Name)
282		-	if readErr != nil {
283		-	unlock()
284		-	return readErr
285		-	}
286		-	if ok && current.IncarnationID == candidate.IncarnationID && current.RecoveryState == domain.RecoveryDeleting {
287		-	deleteErr = c.deleteLocked(ctx, current)
288		-	}
289		-	unlock()
290		-	if deleteErr != nil {
291		-	if err := c.fenceSession(candidate.Name, "catalogue-record", degradedReasonRecord, deleteErr); err != nil {
292		-	return err
293		-	}
294		-	}
295		-	}
296		-
297		-	cursor := ports.DeletionTombstoneCursor{}
298		-	for {
299		-	page, err := c.repository.ListDeletionTombstones(ctx, cursor, recoveryListingBudget)
300		-	if err != nil {
301		-	return err
302		-	}
303		-	for _, tombstone := range page.Tombstones {
304		-	// A fenced tombstone is never deleted here: the next startup or an
305		-	// explicit operator action must still be able to observe it.
306		-	if err := c.recoverDeletionTombstone(ctx, tombstone); err != nil {
307		-	if err := c.fenceSession(tombstone.Name, "deletion-tombstone", degradedReasonTombstone, err); err != nil {
308		-	return err
309		-	}
310		-	}
311		-	}
312		-	if page.Done {
313		-	return nil
314		-	}
315		-	if page.Next.After == cursor.After {
316		-	return errors.New("recovery: tombstone listing did not advance")
317		-	}
318		-	cursor = page.Next
319		-	}
320		-	}
321		-
322		-	// fenceSession returns cause unchanged when it is not session-scoped, so the
323		-	// caller aborts startup. For a session-scoped cause it records the conflict and
324		-	// durably marks the session degraded when the record can represent that state.
325		-	func (c *Coordinator) fenceSession(name, kind, reason string, cause error) error {
326		-	if !isSessionScopedConflict(cause) {
327		-	return cause
328		-	}
329		-	c.conflictMu.Lock()
330		-	c.conflicts = append(c.conflicts, RecoveryConflict{Session: name, Kind: kind, Err: cause})
331		-	c.conflictMu.Unlock()
332		-
333		-	unlock := c.locks.Lock([]string{name})
334		-	defer unlock()
335		-	record, ok, err := c.catalogue.Record(name)
336		-	if err != nil {
337		-	return err
338		-	}
339		-	if !ok \|\| record.RecoveryState != domain.RecoveryHealthy {
340		-	// Fresh records cannot represent degraded state (it requires a committed
341		-	// checkpoint), and deleting or already degraded records must keep the
342		-	// state their own protocol owns. Fencing is then the recorded skip.
343		-	return nil
344		-	}
345		-	record.RecoveryState = domain.RecoveryDegraded
346		-	record.DegradedReason = reason
347		-	if err := record.Validate(); err != nil {
348		-	return nil
349		-	}
350		-	// A catalogue write failure here is infrastructure, not one session's data.
351		-	return c.catalogue.Replace(name, record)
352		-	}
353		-
354		-	// isSessionScopedConflict reports whether err proves only that one session's
355		-	// durable state is self-inconsistent. Everything else - IO, decode, traversal,
356		-	// context cancellation - is treated as infrastructure and aborts startup.
357		-	func isSessionScopedConflict(err error) bool {
358		-	return errors.Is(err, ErrDeletionTombstoneConflict) \|\| errors.Is(err, ErrDeletionTombstoneInvalid) \|\|
359		-	errors.Is(err, ErrSessionRecordInvalid)
360		-	}
361		-
362		-	func (c *Coordinator) recoverDeletionTombstone(ctx context.Context, tombstone domain.DeletionTombstone) error {
363		-	if err := tombstone.Validate(); err != nil {
364		-	return fmt.Errorf("%w: %w", ErrDeletionTombstoneInvalid, err)
365		-	}
366		-	unlock := c.locks.Lock([]string{tombstone.Name})
367		-	defer unlock()
368		-	record, exists, err := c.catalogue.Record(tombstone.Name)
369		-	if err != nil {
370		-	return err
371		-	}
372		-	if exists && record.IncarnationID == tombstone.IncarnationID && record.RecoveryState != domain.RecoveryDeleting {
373		-	return fmt.Errorf("%w: non-deleting session %q", ErrDeletionTombstoneConflict, tombstone.Name)
374		-	}
375		-	if err := c.repository.DeleteIncarnation(ctx, tombstone.IncarnationID); err != nil {
376		-	return err
377		-	}
378		-	if exists && record.IncarnationID == tombstone.IncarnationID {
379		-	if err := c.catalogue.Delete(tombstone.Name); err != nil {
380		-	return err
381		-	}
382		-	}
383		-	return c.repository.DeleteDeletionTombstone(ctx, tombstone.IncarnationID)
	184	+	return c.repository.DeleteIncarnation(ctx, record.IncarnationID)
384	185		}

D internal/usecase/recovery/coordinator_test.go +0 -30

@@ -1,30 +0,0 @@
1	-	package recovery
2	-
3	-	import (
4	-	"context"
5	-	"errors"
6	-	"testing"
7	-
8	-	"github.com/bnema/vev/internal/ports"
9	-	portsmocks "github.com/bnema/vev/internal/ports/mocks"
10	-	"github.com/stretchr/testify/require"
11	-	)
12	-
13	-	func TestDeleteChecksCancellationBeforeCatalogueAccess(t *testing.T) {
14	-	ctx, cancel := context.WithCancel(t.Context())
15	-	cancel()
16	-	catalogue := portsmocks.NewMockCatalogue(t)
17	-	repository := portsmocks.NewMockSnapshotRepository(t)
18	-	coordinator := NewCoordinator(catalogue, repository, nil)
19	-
20	-	require.ErrorIs(t, coordinator.Delete(ctx, "work"), context.Canceled)
21	-	}
22	-
23	-	func TestCoordinatorRecoverPagesTombstonesAndPropagatesErrors(t *testing.T) {
24	-	cause := errors.New("malformed tombstone")
25	-	repo := portsmocks.NewMockSnapshotRepository(t)
26	-	repo.EXPECT().ListDeletionTombstones(context.Background(), ports.DeletionTombstoneCursor{}, recoveryListingBudget).
27	-	Return(ports.DeletionTombstonePage{}, cause)
28	-	c := NewCoordinator(newTransactionCatalogue(), repo, nil)
29	-	require.ErrorIs(t, c.Recover(context.Background()), cause)
30	-	}