fix(recovery): tighten maintenance and restore safety
- commit
- 1f12e310ab6edacfbd5708008b4a9003292c2ba3
- author
- brice · · 2026-07-26 08:09 UTC
- parent
- 979dbfb
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
71 files · +1400 -567
M docs/durable-session-recovery.md +2 -2
| @@ -10,10 +10,10 @@ vev opens named-session state only while holding `$XDG_RUNTIME_DIR/vev/lifecycle | |||
| 10 | 10 | ||
| 11 | 11 | - `vev cmd -s NAME session-recovery retry` | |
| 12 | 12 | - `vev cmd -s NAME session-recovery restore GENERATION` | |
| 13 | - | - `vev cmd -s NAME session-recovery export PATH` | |
| 13 | + | - `vev cmd -s NAME session-recovery export /ABSOLUTE/PATH` | |
| 14 | 14 | - `vev cmd -s NAME session-recovery discard` | |
| 15 | 15 | ||
| 16 | - | Retry validates persisted state without replacing it. Restore validates and promotes the selected catalogue fallback. Export writes recoverable data without mutation. Discard creates a new incarnation and retains the old record and snapshots under `quarantine/` until an explicit later purge. | |
| 16 | + | Retry validates persisted state without replacing it. Restore validates and promotes the selected catalogue fallback. Export writes recoverable data without mutation and requires an absolute destination path. Discard creates a new incarnation and retains the old record and snapshots under `quarantine/` until an explicit later purge. | |
| 17 | 17 | ||
| 18 | 18 | ## Migration and retention | |
| 19 | 19 | ||
| @@ -10,10 +10,10 @@ vev opens named-session state only while holding `$XDG_RUNTIME_DIR/vev/lifecycle | |||||
| 10 | 10 | ||||
| 11 | - `vev cmd -s NAME session-recovery retry` | 11 | - `vev cmd -s NAME session-recovery retry` | ||
| 12 | - `vev cmd -s NAME session-recovery restore GENERATION` | 12 | - `vev cmd -s NAME session-recovery restore GENERATION` | ||
| 13 | - | - `vev cmd -s NAME session-recovery export PATH` | 13 | + | - `vev cmd -s NAME session-recovery export /ABSOLUTE/PATH` |
| 14 | - `vev cmd -s NAME session-recovery discard` | 14 | - `vev cmd -s NAME session-recovery discard` | ||
| 15 | 15 | ||||
| 16 | - | Retry validates persisted state without replacing it. Restore validates and promotes the selected catalogue fallback. Export writes recoverable data without mutation. Discard creates a new incarnation and retains the old record and snapshots under `quarantine/` until an explicit later purge. | 16 | + | Retry validates persisted state without replacing it. Restore validates and promotes the selected catalogue fallback. Export writes recoverable data without mutation and requires an absolute destination path. Discard creates a new incarnation and retains the old record and snapshots under `quarantine/` until an explicit later purge. |
| 17 | 17 | ||||
| 18 | ## Migration and retention | 18 | ## Migration and retention | ||
| 19 | 19 | ||||
M internal/adapters/lifecycle/flock.go +2 -0
| @@ -1,3 +1,5 @@ | |||
| 1 | + | //go:build unix | |
| 2 | + | ||
| 1 | 3 | // Package lifecycle provides exclusive process ownership of vev's durable | |
| 2 | 4 | // state for the complete daemon lifetime. | |
| 3 | 5 | package lifecycle | |
| @@ -1,3 +1,5 @@ | |||||
| 1 | + | //go:build unix | |||
| 2 | + | ||||
| 1 | // Package lifecycle provides exclusive process ownership of vev's durable | 3 | // Package lifecycle provides exclusive process ownership of vev's durable | ||
| 2 | // state for the complete daemon lifetime. | 4 | // state for the complete daemon lifetime. | ||
| 3 | package lifecycle | 5 | package lifecycle | ||
M internal/adapters/lifecycle/flock_test.go +4 -0
| @@ -49,6 +49,10 @@ func testLifecycleLockProcessContentionAndReacquire(t *testing.T) { | |||
| 49 | 49 | require.NoError(t, err) | |
| 50 | 50 | cmd.Stderr = os.Stderr | |
| 51 | 51 | require.NoError(t, cmd.Start()) | |
| 52 | + | defer func() { | |
| 53 | + | _ = stdin.Close() | |
| 54 | + | _ = cmd.Wait() | |
| 55 | + | }() | |
| 52 | 56 | ||
| 53 | 57 | line, err := bufio.NewReader(stdout).ReadString('\n') | |
| 54 | 58 | require.NoError(t, err) | |
| @@ -49,6 +49,10 @@ func testLifecycleLockProcessContentionAndReacquire(t *testing.T) { | |||||
| 49 | require.NoError(t, err) | 49 | require.NoError(t, err) | ||
| 50 | cmd.Stderr = os.Stderr | 50 | cmd.Stderr = os.Stderr | ||
| 51 | require.NoError(t, cmd.Start()) | 51 | require.NoError(t, cmd.Start()) | ||
| 52 | + | defer func() { | |||
| 53 | + | _ = stdin.Close() | |||
| 54 | + | _ = cmd.Wait() | |||
| 55 | + | }() | |||
| 52 | 56 | ||||
| 53 | line, err := bufio.NewReader(stdout).ReadString('\n') | 57 | line, err := bufio.NewReader(stdout).ReadString('\n') | ||
| 54 | require.NoError(t, err) | 58 | require.NoError(t, err) | ||
M internal/adapters/recoveryfs/journal.go +3 -0
| @@ -62,6 +62,9 @@ func decodeDiscard(data []byte) (domain.DiscardIntent, error) { | |||
| 62 | 62 | return domain.DiscardIntent{}, err | |
| 63 | 63 | } | |
| 64 | 64 | if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { | |
| 65 | + | if err != nil { | |
| 66 | + | return domain.DiscardIntent{}, err | |
| 67 | + | } | |
| 65 | 68 | return domain.DiscardIntent{}, errors.New("recovery journal: trailing data") | |
| 66 | 69 | } | |
| 67 | 70 | if file.Version != discardVersion { | |
| @@ -62,6 +62,9 @@ func decodeDiscard(data []byte) (domain.DiscardIntent, error) { | |||||
| 62 | return domain.DiscardIntent{}, err | 62 | return domain.DiscardIntent{}, err | ||
| 63 | } | 63 | } | ||
| 64 | if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { | 64 | if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { | ||
| 65 | + | if err != nil { | |||
| 66 | + | return domain.DiscardIntent{}, err | |||
| 67 | + | } | |||
| 65 | return domain.DiscardIntent{}, errors.New("recovery journal: trailing data") | 68 | return domain.DiscardIntent{}, errors.New("recovery journal: trailing data") | ||
| 66 | } | 69 | } | ||
| 67 | if file.Version != discardVersion { | 70 | if file.Version != discardVersion { | ||
M internal/adapters/recoveryfs/journal_test.go +8 -5
| @@ -3,6 +3,7 @@ package recoveryfs | |||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | 5 | "errors" | |
| 6 | + | "fmt" | |
| 6 | 7 | "os" | |
| 7 | 8 | "path/filepath" | |
| 8 | 9 | "testing" | |
| @@ -45,7 +46,6 @@ func TestDiscardCrashMatrix(t *testing.T) { | |||
| 45 | 46 | _, statErr := os.Stat(path) | |
| 46 | 47 | require.NoError(t, statErr) | |
| 47 | 48 | require.Error(t, j.SaveDiscard(ctx, intent)) | |
| 48 | - | require.NoError(t, os.WriteFile(path, encoded, 0o600)) | |
| 49 | 49 | }) | |
| 50 | 50 | ||
| 51 | 51 | t.Run("delete sync failure rolls forward", func(t *testing.T) { | |
| @@ -81,8 +81,8 @@ func TestRecoveryJournalRoundTripAndIdempotentDelete(t *testing.T) { | |||
| 81 | 81 | func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T) { | |
| 82 | 82 | j := New(t.TempDir()) | |
| 83 | 83 | require.NoError(t, os.MkdirAll(j.dir, 0o700)) | |
| 84 | - | for i := 0; i < maxDiscardIntents+1; i++ { | |
| 85 | - | require.NoError(t, os.WriteFile(filepath.Join(j.dir, ".intent-leftover-"+string(rune(0x1000+i))), nil, 0o600)) | |
| 84 | + | for i := range maxDiscardIntents + 1 { | |
| 85 | + | require.NoError(t, os.WriteFile(filepath.Join(j.dir, fmt.Sprintf(".intent-leftover-%04d", i)), nil, 0o600)) | |
| 86 | 86 | } | |
| 87 | 87 | ||
| 88 | 88 | got, err := j.ListDiscards(context.Background()) | |
| @@ -93,14 +93,17 @@ func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T) | |||
| 93 | 93 | func TestRecoveryJournalRejectsMalformedIntent(t *testing.T) { | |
| 94 | 94 | j := New(t.TempDir()) | |
| 95 | 95 | require.Error(t, j.SaveDiscard(context.Background(), domain.DiscardIntent{})) | |
| 96 | + | ||
| 97 | + | _, err := decodeDiscard([]byte(`{"version":1,"intent":{}} !`)) | |
| 98 | + | require.ErrorContains(t, err, "invalid character") | |
| 96 | 99 | } | |
| 97 | 100 | ||
| 98 | 101 | func TestRecoveryJournalBoundsEnumerationAndIntentReads(t *testing.T) { | |
| 99 | 102 | state := t.TempDir() | |
| 100 | 103 | j := New(state) | |
| 101 | 104 | require.NoError(t, os.MkdirAll(j.dir, 0o700)) | |
| 102 | - | for i := 0; i <= maxDiscardIntents; i++ { | |
| 103 | - | require.NoError(t, os.WriteFile(filepath.Join(j.dir, "discard-extra-"+string(rune(0x1000+i))), nil, 0o600)) | |
| 105 | + | for i := range maxDiscardIntents + 1 { | |
| 106 | + | require.NoError(t, os.WriteFile(filepath.Join(j.dir, fmt.Sprintf("discard-extra-%04d", i)), nil, 0o600)) | |
| 104 | 107 | } | |
| 105 | 108 | _, err := j.ListDiscards(context.Background()) | |
| 106 | 109 | require.ErrorContains(t, err, "entry limit exceeded") | |
| @@ -3,6 +3,7 @@ package recoveryfs | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | "errors" | 5 | "errors" | ||
| 6 | + | "fmt" | |||
| 6 | "os" | 7 | "os" | ||
| 7 | "path/filepath" | 8 | "path/filepath" | ||
| 8 | "testing" | 9 | "testing" | ||
| @@ -45,7 +46,6 @@ func TestDiscardCrashMatrix(t *testing.T) { | |||||
| 45 | _, statErr := os.Stat(path) | 46 | _, statErr := os.Stat(path) | ||
| 46 | require.NoError(t, statErr) | 47 | require.NoError(t, statErr) | ||
| 47 | require.Error(t, j.SaveDiscard(ctx, intent)) | 48 | require.Error(t, j.SaveDiscard(ctx, intent)) | ||
| 48 | - | require.NoError(t, os.WriteFile(path, encoded, 0o600)) | |||
| 49 | }) | 49 | }) | ||
| 50 | 50 | ||||
| 51 | t.Run("delete sync failure rolls forward", func(t *testing.T) { | 51 | t.Run("delete sync failure rolls forward", func(t *testing.T) { | ||
| @@ -81,8 +81,8 @@ func TestRecoveryJournalRoundTripAndIdempotentDelete(t *testing.T) { | |||||
| 81 | func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T) { | 81 | func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T) { | ||
| 82 | j := New(t.TempDir()) | 82 | j := New(t.TempDir()) | ||
| 83 | require.NoError(t, os.MkdirAll(j.dir, 0o700)) | 83 | require.NoError(t, os.MkdirAll(j.dir, 0o700)) | ||
| 84 | - | for i := 0; i < maxDiscardIntents+1; i++ { | 84 | + | for i := range maxDiscardIntents + 1 { |
| 85 | - | require.NoError(t, os.WriteFile(filepath.Join(j.dir, ".intent-leftover-"+string(rune(0x1000+i))), nil, 0o600)) | 85 | + | require.NoError(t, os.WriteFile(filepath.Join(j.dir, fmt.Sprintf(".intent-leftover-%04d", i)), nil, 0o600)) |
| 86 | } | 86 | } | ||
| 87 | 87 | ||||
| 88 | got, err := j.ListDiscards(context.Background()) | 88 | got, err := j.ListDiscards(context.Background()) | ||
| @@ -93,14 +93,17 @@ func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T) | |||||
| 93 | func TestRecoveryJournalRejectsMalformedIntent(t *testing.T) { | 93 | func TestRecoveryJournalRejectsMalformedIntent(t *testing.T) { | ||
| 94 | j := New(t.TempDir()) | 94 | j := New(t.TempDir()) | ||
| 95 | require.Error(t, j.SaveDiscard(context.Background(), domain.DiscardIntent{})) | 95 | require.Error(t, j.SaveDiscard(context.Background(), domain.DiscardIntent{})) | ||
| 96 | + | ||||
| 97 | + | _, err := decodeDiscard([]byte(`{"version":1,"intent":{}} !`)) | |||
| 98 | + | require.ErrorContains(t, err, "invalid character") | |||
| 96 | } | 99 | } | ||
| 97 | 100 | ||||
| 98 | func TestRecoveryJournalBoundsEnumerationAndIntentReads(t *testing.T) { | 101 | func TestRecoveryJournalBoundsEnumerationAndIntentReads(t *testing.T) { | ||
| 99 | state := t.TempDir() | 102 | state := t.TempDir() | ||
| 100 | j := New(state) | 103 | j := New(state) | ||
| 101 | require.NoError(t, os.MkdirAll(j.dir, 0o700)) | 104 | require.NoError(t, os.MkdirAll(j.dir, 0o700)) | ||
| 102 | - | for i := 0; i <= maxDiscardIntents; i++ { | 105 | + | for i := range maxDiscardIntents + 1 { |
| 103 | - | require.NoError(t, os.WriteFile(filepath.Join(j.dir, "discard-extra-"+string(rune(0x1000+i))), nil, 0o600)) | 106 | + | require.NoError(t, os.WriteFile(filepath.Join(j.dir, fmt.Sprintf("discard-extra-%04d", i)), nil, 0o600)) |
| 104 | } | 107 | } | ||
| 105 | _, err := j.ListDiscards(context.Background()) | 108 | _, err := j.ListDiscards(context.Background()) | ||
| 106 | require.ErrorContains(t, err, "entry limit exceeded") | 109 | require.ErrorContains(t, err, "entry limit exceeded") | ||
M internal/adapters/snapshot/direct_load_test.go +9 -0
| @@ -76,9 +76,18 @@ func TestRepairHEAD(t *testing.T) { | |||
| 76 | 76 | t.Fatal(err) | |
| 77 | 77 | } | |
| 78 | 78 | valid := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(pub.Manifest)} | |
| 79 | + | manifestReads := 0 | |
| 80 | + | repo.hooks.beforePayloadRead = func(path string) { | |
| 81 | + | if path == repo.manifestPath(id, valid.Generation) { | |
| 82 | + | manifestReads++ | |
| 83 | + | } | |
| 84 | + | } | |
| 79 | 85 | if err := repo.RepairHEAD(context.Background(), id, valid); err != nil { | |
| 80 | 86 | t.Fatal(err) | |
| 81 | 87 | } | |
| 88 | + | if manifestReads != 1 { | |
| 89 | + | t.Fatalf("RepairHEAD manifest reads = %d, want 1", manifestReads) | |
| 90 | + | } | |
| 82 | 91 | before, err := os.ReadFile(repo.headPath(id)) | |
| 83 | 92 | if err != nil { | |
| 84 | 93 | t.Fatal(err) | |
| @@ -76,9 +76,18 @@ func TestRepairHEAD(t *testing.T) { | |||||
| 76 | t.Fatal(err) | 76 | t.Fatal(err) | ||
| 77 | } | 77 | } | ||
| 78 | valid := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(pub.Manifest)} | 78 | valid := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(pub.Manifest)} | ||
| 79 | + | manifestReads := 0 | |||
| 80 | + | repo.hooks.beforePayloadRead = func(path string) { | |||
| 81 | + | if path == repo.manifestPath(id, valid.Generation) { | |||
| 82 | + | manifestReads++ | |||
| 83 | + | } | |||
| 84 | + | } | |||
| 79 | if err := repo.RepairHEAD(context.Background(), id, valid); err != nil { | 85 | if err := repo.RepairHEAD(context.Background(), id, valid); err != nil { | ||
| 80 | t.Fatal(err) | 86 | t.Fatal(err) | ||
| 81 | } | 87 | } | ||
| 88 | + | if manifestReads != 1 { | |||
| 89 | + | t.Fatalf("RepairHEAD manifest reads = %d, want 1", manifestReads) | |||
| 90 | + | } | |||
| 82 | before, err := os.ReadFile(repo.headPath(id)) | 91 | before, err := os.ReadFile(repo.headPath(id)) | ||
| 83 | if err != nil { | 92 | if err != nil { | ||
| 84 | t.Fatal(err) | 93 | t.Fatal(err) | ||
M internal/adapters/snapshot/legacy_load_test.go +6 -2
| @@ -47,8 +47,7 @@ func (r *Repository) Load(ctx context.Context, name string) (ports.SnapshotGener | |||
| 47 | 47 | if err := ctx.Err(); err != nil { | |
| 48 | 48 | return ports.SnapshotGeneration{}, err | |
| 49 | 49 | } | |
| 50 | - | id := legacyIncarnationID(name) | |
| 51 | - | key, err := incarnationKey(id) | |
| 50 | + | key, err := r.legacyRepositoryKey(name) | |
| 52 | 51 | if err != nil { | |
| 53 | 52 | return ports.SnapshotGeneration{}, err | |
| 54 | 53 | } | |
| @@ -84,13 +83,18 @@ func (r *Repository) load(ctx context.Context, name, key string) (ports.Snapshot | |||
| 84 | 83 | if err != nil { | |
| 85 | 84 | return ports.SnapshotGeneration{}, err | |
| 86 | 85 | } | |
| 86 | + | var lastLoadErr error | |
| 87 | 87 | for _, generation := range candidates { | |
| 88 | 88 | got, err := r.loadGeneration(ctx, name, key, generation) | |
| 89 | 89 | if err != nil { | |
| 90 | + | lastLoadErr = err | |
| 90 | 91 | continue | |
| 91 | 92 | } | |
| 92 | 93 | return got, nil | |
| 93 | 94 | } | |
| 95 | + | if lastLoadErr != nil { | |
| 96 | + | return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q: %w", name, lastLoadErr) | |
| 97 | + | } | |
| 94 | 98 | return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name) | |
| 95 | 99 | } | |
| 96 | 100 | ||
| @@ -47,8 +47,7 @@ func (r *Repository) Load(ctx context.Context, name string) (ports.SnapshotGener | |||||
| 47 | if err := ctx.Err(); err != nil { | 47 | if err := ctx.Err(); err != nil { | ||
| 48 | return ports.SnapshotGeneration{}, err | 48 | return ports.SnapshotGeneration{}, err | ||
| 49 | } | 49 | } | ||
| 50 | - | id := legacyIncarnationID(name) | 50 | + | key, err := r.legacyRepositoryKey(name) |
| 51 | - | key, err := incarnationKey(id) | |||
| 52 | if err != nil { | 51 | if err != nil { | ||
| 53 | return ports.SnapshotGeneration{}, err | 52 | return ports.SnapshotGeneration{}, err | ||
| 54 | } | 53 | } | ||
| @@ -84,13 +83,18 @@ func (r *Repository) load(ctx context.Context, name, key string) (ports.Snapshot | |||||
| 84 | if err != nil { | 83 | if err != nil { | ||
| 85 | return ports.SnapshotGeneration{}, err | 84 | return ports.SnapshotGeneration{}, err | ||
| 86 | } | 85 | } | ||
| 86 | + | var lastLoadErr error | |||
| 87 | for _, generation := range candidates { | 87 | for _, generation := range candidates { | ||
| 88 | got, err := r.loadGeneration(ctx, name, key, generation) | 88 | got, err := r.loadGeneration(ctx, name, key, generation) | ||
| 89 | if err != nil { | 89 | if err != nil { | ||
| 90 | + | lastLoadErr = err | |||
| 90 | continue | 91 | continue | ||
| 91 | } | 92 | } | ||
| 92 | return got, nil | 93 | return got, nil | ||
| 93 | } | 94 | } | ||
| 95 | + | if lastLoadErr != nil { | |||
| 96 | + | return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q: %w", name, lastLoadErr) | |||
| 97 | + | } | |||
| 94 | return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name) | 98 | return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name) | ||
| 95 | } | 99 | } | ||
| 96 | 100 | ||||
M internal/adapters/snapshot/load.go +15 -21
| @@ -76,43 +76,44 @@ func (r *Repository) LoadCheckpoint(ctx context.Context, id domain.IncarnationID | |||
| 76 | 76 | } | |
| 77 | 77 | lock := r.lockSession(key) | |
| 78 | 78 | defer r.unlockSession(lock) | |
| 79 | - | return r.loadCheckpointLocked(ctx, id, name, ref) | |
| 79 | + | generation, _, err := r.loadCheckpointLocked(ctx, id, name, ref) | |
| 80 | + | return generation, err | |
| 80 | 81 | } | |
| 81 | 82 | ||
| 82 | - | func (r *Repository) loadCheckpointLocked(ctx context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) { | |
| 83 | + | func (r *Repository) loadCheckpointLocked(ctx context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, codec.Manifest, error) { | |
| 83 | 84 | if ref.Generation == 0 || ref.ManifestDigest == ([32]byte{}) { | |
| 84 | - | return ports.SnapshotGeneration{}, fmt.Errorf("invalid checkpoint reference") | |
| 85 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid checkpoint reference") | |
| 85 | 86 | } | |
| 86 | 87 | data, err := r.readBounded(r.manifestPath(id, ref.Generation)) | |
| 87 | 88 | if err != nil { | |
| 88 | - | return ports.SnapshotGeneration{}, err | |
| 89 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, err | |
| 89 | 90 | } | |
| 90 | 91 | if sha256.Sum256(data) != ref.ManifestDigest { | |
| 91 | - | return ports.SnapshotGeneration{}, fmt.Errorf("manifest digest mismatch") | |
| 92 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("manifest digest mismatch") | |
| 92 | 93 | } | |
| 93 | 94 | manifest, err := codec.UnmarshalManifest(data) | |
| 94 | 95 | if err != nil || manifest.IncarnationID != id || manifest.Generation != ref.Generation { | |
| 95 | - | return ports.SnapshotGeneration{}, fmt.Errorf("invalid manifest") | |
| 96 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid manifest") | |
| 96 | 97 | } | |
| 97 | 98 | refs := manifestRefs(manifest) | |
| 98 | 99 | if refs == nil || !withinGenerationBudget(len(data), refs) { | |
| 99 | - | return ports.SnapshotGeneration{}, fmt.Errorf("snapshot generation too large") | |
| 100 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("snapshot generation too large") | |
| 100 | 101 | } | |
| 101 | 102 | objects := make(map[ports.SnapshotDigest][]byte, len(refs)) | |
| 102 | 103 | for digest, objectRef := range refs { | |
| 103 | 104 | if err := ctx.Err(); err != nil { | |
| 104 | - | return ports.SnapshotGeneration{}, err | |
| 105 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, err | |
| 105 | 106 | } | |
| 106 | 107 | object, err := r.readBounded(r.objectPath(id, digest)) | |
| 107 | 108 | if err != nil { | |
| 108 | - | return ports.SnapshotGeneration{}, err | |
| 109 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, err | |
| 109 | 110 | } | |
| 110 | 111 | if sha256.Sum256(object) != digest || !validObject(object, objectRef) { | |
| 111 | - | return ports.SnapshotGeneration{}, fmt.Errorf("invalid object") | |
| 112 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid object") | |
| 112 | 113 | } | |
| 113 | 114 | objects[digest] = object | |
| 114 | 115 | } | |
| 115 | - | return ports.SnapshotGeneration{IncarnationID: id, Name: name, Generation: ref.Generation, ParentCheckpoint: manifest.ParentCheckpoint, Manifest: data, Objects: objects}, nil | |
| 116 | + | return ports.SnapshotGeneration{IncarnationID: id, Name: name, Generation: ref.Generation, ParentCheckpoint: manifest.ParentCheckpoint, Manifest: data, Objects: objects}, manifest, nil | |
| 116 | 117 | } | |
| 117 | 118 | ||
| 118 | 119 | func (r *Repository) RepairHEAD(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef) error { | |
| @@ -125,19 +126,12 @@ func (r *Repository) RepairHEAD(ctx context.Context, id domain.IncarnationID, re | |||
| 125 | 126 | } | |
| 126 | 127 | lock := r.lockSession(key) | |
| 127 | 128 | defer r.unlockSession(lock) | |
| 128 | - | data, err := r.readBounded(r.manifestPath(id, ref.Generation)) | |
| 129 | + | _, manifest, err := r.loadCheckpointLocked(ctx, id, "", ref) | |
| 129 | 130 | if err != nil { | |
| 130 | 131 | return fmt.Errorf("validate checkpoint before HEAD repair: %w", err) | |
| 131 | 132 | } | |
| 132 | - | if sha256.Sum256(data) != ref.ManifestDigest { | |
| 133 | - | return fmt.Errorf("validate checkpoint before HEAD repair: manifest digest mismatch") | |
| 134 | - | } | |
| 135 | - | manifest, err := codec.UnmarshalManifest(data) | |
| 136 | - | if err != nil || manifest.IncarnationID != id || manifest.Generation != ref.Generation { | |
| 137 | - | return fmt.Errorf("validate checkpoint before HEAD repair: invalid manifest") | |
| 138 | - | } | |
| 139 | - | if _, err := r.loadCheckpointLocked(ctx, id, manifest.Name, ref); err != nil { | |
| 140 | - | return err | |
| 133 | + | if manifest.Name == "" { | |
| 134 | + | return fmt.Errorf("validate checkpoint before HEAD repair: invalid manifest name") | |
| 141 | 135 | } | |
| 142 | 136 | return r.writeMutable(r.headPath(id), marshalHead(ref.Generation, ref.ManifestDigest)) | |
| 143 | 137 | } | |
| @@ -76,43 +76,44 @@ func (r *Repository) LoadCheckpoint(ctx context.Context, id domain.IncarnationID | |||||
| 76 | } | 76 | } | ||
| 77 | lock := r.lockSession(key) | 77 | lock := r.lockSession(key) | ||
| 78 | defer r.unlockSession(lock) | 78 | defer r.unlockSession(lock) | ||
| 79 | - | return r.loadCheckpointLocked(ctx, id, name, ref) | 79 | + | generation, _, err := r.loadCheckpointLocked(ctx, id, name, ref) |
| 80 | + | return generation, err | |||
| 80 | } | 81 | } | ||
| 81 | 82 | ||||
| 82 | - | func (r *Repository) loadCheckpointLocked(ctx context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) { | 83 | + | func (r *Repository) loadCheckpointLocked(ctx context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, codec.Manifest, error) { |
| 83 | if ref.Generation == 0 || ref.ManifestDigest == ([32]byte{}) { | 84 | if ref.Generation == 0 || ref.ManifestDigest == ([32]byte{}) { | ||
| 84 | - | return ports.SnapshotGeneration{}, fmt.Errorf("invalid checkpoint reference") | 85 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid checkpoint reference") |
| 85 | } | 86 | } | ||
| 86 | data, err := r.readBounded(r.manifestPath(id, ref.Generation)) | 87 | data, err := r.readBounded(r.manifestPath(id, ref.Generation)) | ||
| 87 | if err != nil { | 88 | if err != nil { | ||
| 88 | - | return ports.SnapshotGeneration{}, err | 89 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, err |
| 89 | } | 90 | } | ||
| 90 | if sha256.Sum256(data) != ref.ManifestDigest { | 91 | if sha256.Sum256(data) != ref.ManifestDigest { | ||
| 91 | - | return ports.SnapshotGeneration{}, fmt.Errorf("manifest digest mismatch") | 92 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("manifest digest mismatch") |
| 92 | } | 93 | } | ||
| 93 | manifest, err := codec.UnmarshalManifest(data) | 94 | manifest, err := codec.UnmarshalManifest(data) | ||
| 94 | if err != nil || manifest.IncarnationID != id || manifest.Generation != ref.Generation { | 95 | if err != nil || manifest.IncarnationID != id || manifest.Generation != ref.Generation { | ||
| 95 | - | return ports.SnapshotGeneration{}, fmt.Errorf("invalid manifest") | 96 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid manifest") |
| 96 | } | 97 | } | ||
| 97 | refs := manifestRefs(manifest) | 98 | refs := manifestRefs(manifest) | ||
| 98 | if refs == nil || !withinGenerationBudget(len(data), refs) { | 99 | if refs == nil || !withinGenerationBudget(len(data), refs) { | ||
| 99 | - | return ports.SnapshotGeneration{}, fmt.Errorf("snapshot generation too large") | 100 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("snapshot generation too large") |
| 100 | } | 101 | } | ||
| 101 | objects := make(map[ports.SnapshotDigest][]byte, len(refs)) | 102 | objects := make(map[ports.SnapshotDigest][]byte, len(refs)) | ||
| 102 | for digest, objectRef := range refs { | 103 | for digest, objectRef := range refs { | ||
| 103 | if err := ctx.Err(); err != nil { | 104 | if err := ctx.Err(); err != nil { | ||
| 104 | - | return ports.SnapshotGeneration{}, err | 105 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, err |
| 105 | } | 106 | } | ||
| 106 | object, err := r.readBounded(r.objectPath(id, digest)) | 107 | object, err := r.readBounded(r.objectPath(id, digest)) | ||
| 107 | if err != nil { | 108 | if err != nil { | ||
| 108 | - | return ports.SnapshotGeneration{}, err | 109 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, err |
| 109 | } | 110 | } | ||
| 110 | if sha256.Sum256(object) != digest || !validObject(object, objectRef) { | 111 | if sha256.Sum256(object) != digest || !validObject(object, objectRef) { | ||
| 111 | - | return ports.SnapshotGeneration{}, fmt.Errorf("invalid object") | 112 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid object") |
| 112 | } | 113 | } | ||
| 113 | objects[digest] = object | 114 | objects[digest] = object | ||
| 114 | } | 115 | } | ||
| 115 | - | return ports.SnapshotGeneration{IncarnationID: id, Name: name, Generation: ref.Generation, ParentCheckpoint: manifest.ParentCheckpoint, Manifest: data, Objects: objects}, nil | 116 | + | return ports.SnapshotGeneration{IncarnationID: id, Name: name, Generation: ref.Generation, ParentCheckpoint: manifest.ParentCheckpoint, Manifest: data, Objects: objects}, manifest, nil |
| 116 | } | 117 | } | ||
| 117 | 118 | ||||
| 118 | func (r *Repository) RepairHEAD(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef) error { | 119 | func (r *Repository) RepairHEAD(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef) error { | ||
| @@ -125,19 +126,12 @@ func (r *Repository) RepairHEAD(ctx context.Context, id domain.IncarnationID, re | |||||
| 125 | } | 126 | } | ||
| 126 | lock := r.lockSession(key) | 127 | lock := r.lockSession(key) | ||
| 127 | defer r.unlockSession(lock) | 128 | defer r.unlockSession(lock) | ||
| 128 | - | data, err := r.readBounded(r.manifestPath(id, ref.Generation)) | 129 | + | _, manifest, err := r.loadCheckpointLocked(ctx, id, "", ref) |
| 129 | if err != nil { | 130 | if err != nil { | ||
| 130 | return fmt.Errorf("validate checkpoint before HEAD repair: %w", err) | 131 | return fmt.Errorf("validate checkpoint before HEAD repair: %w", err) | ||
| 131 | } | 132 | } | ||
| 132 | - | if sha256.Sum256(data) != ref.ManifestDigest { | 133 | + | if manifest.Name == "" { |
| 133 | - | return fmt.Errorf("validate checkpoint before HEAD repair: manifest digest mismatch") | 134 | + | return fmt.Errorf("validate checkpoint before HEAD repair: invalid manifest name") |
| 134 | - | } | |||
| 135 | - | manifest, err := codec.UnmarshalManifest(data) | |||
| 136 | - | if err != nil || manifest.IncarnationID != id || manifest.Generation != ref.Generation { | |||
| 137 | - | return fmt.Errorf("validate checkpoint before HEAD repair: invalid manifest") | |||
| 138 | - | } | |||
| 139 | - | if _, err := r.loadCheckpointLocked(ctx, id, manifest.Name, ref); err != nil { | |||
| 140 | - | return err | |||
| 141 | } | 135 | } | ||
| 142 | return r.writeMutable(r.headPath(id), marshalHead(ref.Generation, ref.ManifestDigest)) | 136 | return r.writeMutable(r.headPath(id), marshalHead(ref.Generation, ref.ManifestDigest)) | ||
| 143 | } | 137 | } | ||
M internal/adapters/snapshot/load_major_test.go +17 -1
| @@ -3,6 +3,7 @@ package snapshot | |||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | 5 | "os" | |
| 6 | + | "path/filepath" | |
| 6 | 7 | "syscall" | |
| 7 | 8 | "testing" | |
| 8 | 9 | ||
| @@ -29,14 +30,27 @@ func TestRepositoryCurrentGenerationPropagatesOperationalHeadErrors(t *testing.T | |||
| 29 | 30 | require.ErrorIs(t, err, syscall.EIO) | |
| 30 | 31 | } | |
| 31 | 32 | ||
| 33 | + | func TestRepositoryLoadUsesEstablishedLegacySessionKey(t *testing.T) { | |
| 34 | + | repo := NewRepository(privateDir(t)) | |
| 35 | + | publication := repositoryPublication(t, "named", 1, []byte("legacy-state")) | |
| 36 | + | require.NoError(t, repo.Publish(context.Background(), publication)) | |
| 37 | + | require.NoError(t, os.Rename(repo.sessionPath(publication.IncarnationID), filepath.Join(repo.dir, repositorySessionsDir, sessionKey(publication.Name)))) | |
| 38 | + | ||
| 39 | + | got, err := repo.Load(context.Background(), publication.Name) | |
| 40 | + | require.NoError(t, err) | |
| 41 | + | require.Equal(t, publication.Objects[0].Data, got.Objects[publication.Objects[0].Digest]) | |
| 42 | + | } | |
| 43 | + | ||
| 32 | 44 | func TestRepositoryLoadFallsBackFromInvalidHead(t *testing.T) { | |
| 33 | 45 | repo := NewRepository(privateDir(t)) | |
| 34 | - | require.NoError(t, repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state")))) | |
| 46 | + | publication := repositoryPublication(t, "named", 1, []byte("state")) | |
| 47 | + | require.NoError(t, repo.Publish(context.Background(), publication)) | |
| 35 | 48 | require.NoError(t, os.WriteFile(repo.legacyHeadPath(legacyIncarnationID("named").String()), []byte("invalid"), 0o600)) | |
| 36 | 49 | ||
| 37 | 50 | got, err := repo.Load(context.Background(), "named") | |
| 38 | 51 | require.NoError(t, err) | |
| 39 | 52 | require.Equal(t, uint64(1), got.Generation) | |
| 53 | + | require.Equal(t, publication.Objects[0].Data, got.Objects[publication.Objects[0].Digest]) | |
| 40 | 54 | } | |
| 41 | 55 | ||
| 42 | 56 | func TestRepositoryLoadFallsBackAcrossMultipleCorruptCandidates(t *testing.T) { | |
| @@ -53,6 +67,8 @@ func TestRepositoryLoadFallsBackAcrossMultipleCorruptCandidates(t *testing.T) { | |||
| 53 | 67 | got, err := repo.Load(context.Background(), "named") | |
| 54 | 68 | require.NoError(t, err) | |
| 55 | 69 | require.Equal(t, uint64(2), got.Generation) | |
| 70 | + | fallback := repositoryPublication(t, "named", 2, []byte{2}) | |
| 71 | + | require.Equal(t, fallback.Objects[0].Data, got.Objects[fallback.Objects[0].Digest]) | |
| 56 | 72 | } | |
| 57 | 73 | ||
| 58 | 74 | func TestRepositoryLoadHoldsSessionLockAcrossTombstoneCheck(t *testing.T) { | |
| @@ -3,6 +3,7 @@ package snapshot | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | "os" | 5 | "os" | ||
| 6 | + | "path/filepath" | |||
| 6 | "syscall" | 7 | "syscall" | ||
| 7 | "testing" | 8 | "testing" | ||
| 8 | 9 | ||||
| @@ -29,14 +30,27 @@ func TestRepositoryCurrentGenerationPropagatesOperationalHeadErrors(t *testing.T | |||||
| 29 | require.ErrorIs(t, err, syscall.EIO) | 30 | require.ErrorIs(t, err, syscall.EIO) | ||
| 30 | } | 31 | } | ||
| 31 | 32 | ||||
| 33 | + | func TestRepositoryLoadUsesEstablishedLegacySessionKey(t *testing.T) { | |||
| 34 | + | repo := NewRepository(privateDir(t)) | |||
| 35 | + | publication := repositoryPublication(t, "named", 1, []byte("legacy-state")) | |||
| 36 | + | require.NoError(t, repo.Publish(context.Background(), publication)) | |||
| 37 | + | require.NoError(t, os.Rename(repo.sessionPath(publication.IncarnationID), filepath.Join(repo.dir, repositorySessionsDir, sessionKey(publication.Name)))) | |||
| 38 | + | ||||
| 39 | + | got, err := repo.Load(context.Background(), publication.Name) | |||
| 40 | + | require.NoError(t, err) | |||
| 41 | + | require.Equal(t, publication.Objects[0].Data, got.Objects[publication.Objects[0].Digest]) | |||
| 42 | + | } | |||
| 43 | + | ||||
| 32 | func TestRepositoryLoadFallsBackFromInvalidHead(t *testing.T) { | 44 | func TestRepositoryLoadFallsBackFromInvalidHead(t *testing.T) { | ||
| 33 | repo := NewRepository(privateDir(t)) | 45 | repo := NewRepository(privateDir(t)) | ||
| 34 | - | require.NoError(t, repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state")))) | 46 | + | publication := repositoryPublication(t, "named", 1, []byte("state")) |
| 47 | + | require.NoError(t, repo.Publish(context.Background(), publication)) | |||
| 35 | require.NoError(t, os.WriteFile(repo.legacyHeadPath(legacyIncarnationID("named").String()), []byte("invalid"), 0o600)) | 48 | require.NoError(t, os.WriteFile(repo.legacyHeadPath(legacyIncarnationID("named").String()), []byte("invalid"), 0o600)) | ||
| 36 | 49 | ||||
| 37 | got, err := repo.Load(context.Background(), "named") | 50 | got, err := repo.Load(context.Background(), "named") | ||
| 38 | require.NoError(t, err) | 51 | require.NoError(t, err) | ||
| 39 | require.Equal(t, uint64(1), got.Generation) | 52 | require.Equal(t, uint64(1), got.Generation) | ||
| 53 | + | require.Equal(t, publication.Objects[0].Data, got.Objects[publication.Objects[0].Digest]) | |||
| 40 | } | 54 | } | ||
| 41 | 55 | ||||
| 42 | func TestRepositoryLoadFallsBackAcrossMultipleCorruptCandidates(t *testing.T) { | 56 | func TestRepositoryLoadFallsBackAcrossMultipleCorruptCandidates(t *testing.T) { | ||
| @@ -53,6 +67,8 @@ func TestRepositoryLoadFallsBackAcrossMultipleCorruptCandidates(t *testing.T) { | |||||
| 53 | got, err := repo.Load(context.Background(), "named") | 67 | got, err := repo.Load(context.Background(), "named") | ||
| 54 | require.NoError(t, err) | 68 | require.NoError(t, err) | ||
| 55 | require.Equal(t, uint64(2), got.Generation) | 69 | require.Equal(t, uint64(2), got.Generation) | ||
| 70 | + | fallback := repositoryPublication(t, "named", 2, []byte{2}) | |||
| 71 | + | require.Equal(t, fallback.Objects[0].Data, got.Objects[fallback.Objects[0].Digest]) | |||
| 56 | } | 72 | } | ||
| 57 | 73 | ||||
| 58 | func TestRepositoryLoadHoldsSessionLockAcrossTombstoneCheck(t *testing.T) { | 74 | func TestRepositoryLoadHoldsSessionLockAcrossTombstoneCheck(t *testing.T) { | ||
M internal/adapters/snapshot/maintenance_delete.go +110 -66
| @@ -11,6 +11,7 @@ import ( | |||
| 11 | 11 | "io" | |
| 12 | 12 | "os" | |
| 13 | 13 | "path/filepath" | |
| 14 | + | "sort" | |
| 14 | 15 | "strings" | |
| 15 | 16 | ||
| 16 | 17 | "github.com/bnema/vev/internal/domain" | |
| @@ -94,13 +95,14 @@ const ( | |||
| 94 | 95 | ) | |
| 95 | 96 | ||
| 96 | 97 | type deletionListingState struct { | |
| 97 | - | After string `json:"after,omitempty"` | |
| 98 | - | Offset int64 `json:"offset,omitempty"` | |
| 99 | - | Pending []maintenanceDirEntry `json:"pending,omitempty"` | |
| 100 | - | PendingAt int `json:"pending_at,omitempty"` | |
| 101 | - | ScanDone bool `json:"scan_done,omitempty"` | |
| 102 | - | Candidate string `json:"candidate,omitempty"` | |
| 103 | - | Found uint8 `json:"found,omitempty"` | |
| 98 | + | After string `json:"after,omitempty"` | |
| 99 | + | Offset int64 `json:"offset,omitempty"` | |
| 100 | + | Pending []maintenanceDirEntry `json:"pending,omitempty"` | |
| 101 | + | PendingAt int `json:"pending_at,omitempty"` | |
| 102 | + | ScanDone bool `json:"scan_done,omitempty"` | |
| 103 | + | Candidates []string `json:"candidates,omitempty"` | |
| 104 | + | More bool `json:"more,omitempty"` | |
| 105 | + | BatchLimit int `json:"batch_limit,omitempty"` | |
| 104 | 106 | } | |
| 105 | 107 | ||
| 106 | 108 | // MarshalJSON keeps maintenanceDirEntry private while making a continuation | |
| @@ -165,14 +167,16 @@ func validateDeletionListingState(state deletionListingState) error { | |||
| 165 | 167 | if state.After != "" && !canonicalDeletionTombstoneFilename(state.After) { | |
| 166 | 168 | return fmt.Errorf("snapshot: non-canonical deletion tombstone continuation key") | |
| 167 | 169 | } | |
| 168 | - | if state.Offset < 0 || state.PendingAt < 0 || state.PendingAt > len(state.Pending) || state.Found > 2 { | |
| 170 | + | if state.Offset < 0 || state.PendingAt < 0 || state.PendingAt > len(state.Pending) || state.BatchLimit < 0 || state.BatchLimit > maintenanceBatch || len(state.Candidates) > state.BatchLimit { | |
| 169 | 171 | return fmt.Errorf("snapshot: malformed deletion tombstone continuation") | |
| 170 | 172 | } | |
| 171 | - | if state.After == "" && state.Offset == 0 && len(state.Pending) == 0 && !state.ScanDone && state.Candidate == "" { | |
| 173 | + | if state.After == "" && state.Offset == 0 && len(state.Pending) == 0 && !state.ScanDone && len(state.Candidates) == 0 { | |
| 172 | 174 | return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation") | |
| 173 | 175 | } | |
| 174 | - | if (state.Candidate == "") != (state.Found == 0) || (state.Candidate != "" && (!canonicalDeletionTombstoneFilename(state.Candidate) || state.Candidate <= state.After)) { | |
| 175 | - | return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation") | |
| 176 | + | for i, candidate := range state.Candidates { | |
| 177 | + | if !canonicalDeletionTombstoneFilename(candidate) || candidate <= state.After || (i > 0 && candidate <= state.Candidates[i-1]) { | |
| 178 | + | return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation") | |
| 179 | + | } | |
| 176 | 180 | } | |
| 177 | 181 | if len(state.Pending) > maintenanceDirentBufferSize { | |
| 178 | 182 | return fmt.Errorf("snapshot: malformed deletion tombstone continuation") | |
| @@ -228,31 +232,56 @@ func (r *Repository) ListDeletionTombstones(ctx context.Context, cursor ports.De | |||
| 228 | 232 | r.maintenanceMu.Lock() | |
| 229 | 233 | defer r.maintenanceMu.Unlock() | |
| 230 | 234 | ||
| 231 | - | entryLimit := budget.Entries | |
| 232 | - | if maxInt := uint64(^uint(0) >> 1); entryLimit > maxInt { | |
| 233 | - | entryLimit = maxInt | |
| 234 | - | } | |
| 235 | - | entries, done, err := r.readDeletionListingEntries(dir, int(entryLimit), &state) | |
| 236 | - | if errors.Is(err, os.ErrNotExist) { | |
| 237 | - | return ports.DeletionTombstonePage{Done: true}, nil | |
| 238 | - | } | |
| 239 | - | if err != nil { | |
| 240 | - | return ports.DeletionTombstonePage{}, err | |
| 235 | + | batchLimit := int(min(budget.Entries, uint64(maintenanceBatch))) | |
| 236 | + | if state.BatchLimit == 0 { | |
| 237 | + | state.BatchLimit = batchLimit | |
| 238 | + | } else if state.BatchLimit > batchLimit { | |
| 239 | + | state.BatchLimit = batchLimit | |
| 240 | + | if len(state.Candidates) > state.BatchLimit { | |
| 241 | + | state.Candidates = state.Candidates[:state.BatchLimit] | |
| 242 | + | state.More = true | |
| 243 | + | } | |
| 241 | 244 | } | |
| 242 | - | for _, entry := range entries { | |
| 243 | - | if err := ctx.Err(); err != nil { | |
| 244 | - | return ports.DeletionTombstonePage{}, err | |
| 245 | + | remainingEntries := budget.Entries | |
| 246 | + | done := state.ScanDone | |
| 247 | + | for !done && remainingEntries > 0 { | |
| 248 | + | limit := remainingEntries | |
| 249 | + | if maxInt := uint64(^uint(0) >> 1); limit > maxInt { | |
| 250 | + | limit = maxInt | |
| 245 | 251 | } | |
| 246 | - | if entry.isDir || !canonicalDeletionTombstoneFilename(entry.name) { | |
| 247 | - | return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: invalid deletion tombstone entry") | |
| 252 | + | entries, batchDone, err := r.readDeletionListingEntries(dir, int(limit), &state) | |
| 253 | + | if errors.Is(err, os.ErrNotExist) { | |
| 254 | + | return ports.DeletionTombstonePage{Done: true}, nil | |
| 248 | 255 | } | |
| 249 | - | if entry.name > state.After { | |
| 250 | - | if state.Found < 2 { | |
| 251 | - | state.Found++ | |
| 256 | + | if err != nil { | |
| 257 | + | return ports.DeletionTombstonePage{}, err | |
| 258 | + | } | |
| 259 | + | remainingEntries -= uint64(len(entries)) | |
| 260 | + | for _, entry := range entries { | |
| 261 | + | if err := ctx.Err(); err != nil { | |
| 262 | + | return ports.DeletionTombstonePage{}, err | |
| 263 | + | } | |
| 264 | + | if entry.isDir || !canonicalDeletionTombstoneFilename(entry.name) { | |
| 265 | + | return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: invalid deletion tombstone entry") | |
| 266 | + | } | |
| 267 | + | if entry.name <= state.After { | |
| 268 | + | continue | |
| 252 | 269 | } | |
| 253 | - | if state.Candidate == "" || entry.name < state.Candidate { | |
| 254 | - | state.Candidate = entry.name | |
| 270 | + | at := sort.SearchStrings(state.Candidates, entry.name) | |
| 271 | + | if at < len(state.Candidates) && state.Candidates[at] == entry.name { | |
| 272 | + | continue | |
| 255 | 273 | } | |
| 274 | + | state.Candidates = append(state.Candidates, "") | |
| 275 | + | copy(state.Candidates[at+1:], state.Candidates[at:]) | |
| 276 | + | state.Candidates[at] = entry.name | |
| 277 | + | if len(state.Candidates) > state.BatchLimit { | |
| 278 | + | state.Candidates = state.Candidates[:state.BatchLimit] | |
| 279 | + | state.More = true | |
| 280 | + | } | |
| 281 | + | } | |
| 282 | + | done = batchDone | |
| 283 | + | if len(entries) == 0 && !batchDone { | |
| 284 | + | break | |
| 256 | 285 | } | |
| 257 | 286 | } | |
| 258 | 287 | if !done { | |
| @@ -265,41 +294,50 @@ func (r *Repository) ListDeletionTombstones(ctx context.Context, cursor ports.De | |||
| 265 | 294 | } | |
| 266 | 295 | return ports.DeletionTombstonePage{Next: next}, nil | |
| 267 | 296 | } | |
| 268 | - | if state.Candidate == "" { | |
| 297 | + | if len(state.Candidates) == 0 { | |
| 269 | 298 | return ports.DeletionTombstonePage{Done: true}, nil | |
| 270 | 299 | } | |
| 271 | - | path := filepath.Join(dir, state.Candidate) | |
| 272 | - | info, err := r.stat(path) | |
| 273 | - | if err != nil { | |
| 274 | - | return ports.DeletionTombstonePage{}, err | |
| 275 | - | } | |
| 276 | - | charge := uint64(len(state.Candidate)) | |
| 277 | - | if info.Size() < 0 || uint64(info.Size()) > budget.Bytes || charge > budget.Bytes-uint64(info.Size()) { | |
| 278 | - | return ports.DeletionTombstonePage{}, ErrMaintenanceBudgetTooSmall | |
| 279 | - | } | |
| 280 | - | if hook := r.hooks.beforeDeletionTombstoneRead; hook != nil { | |
| 281 | - | hook(path) | |
| 282 | - | } | |
| 283 | - | data, err := r.readBounded(path) | |
| 284 | - | if err != nil { | |
| 285 | - | return ports.DeletionTombstonePage{}, err | |
| 286 | - | } | |
| 287 | - | decoded, err := decodeDeletionTombstone(data) | |
| 288 | - | if err != nil { | |
| 289 | - | return ports.DeletionTombstonePage{}, err | |
| 290 | - | } | |
| 291 | - | if state.Candidate != decoded.IncarnationID.String()+".tombstone" { | |
| 292 | - | return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: non-canonical deletion tombstone filename") | |
| 300 | + | ||
| 301 | + | remainingBytes := budget.Bytes | |
| 302 | + | tombstones := make([]domain.DeletionTombstone, 0, len(state.Candidates)) | |
| 303 | + | last := "" | |
| 304 | + | for _, candidate := range state.Candidates { | |
| 305 | + | path := filepath.Join(dir, candidate) | |
| 306 | + | info, err := r.stat(path) | |
| 307 | + | if err != nil { | |
| 308 | + | return ports.DeletionTombstonePage{}, err | |
| 309 | + | } | |
| 310 | + | charge := uint64(len(candidate)) | |
| 311 | + | if info.Size() < 0 || uint64(info.Size()) > remainingBytes || charge > remainingBytes-uint64(info.Size()) { | |
| 312 | + | if len(tombstones) == 0 { | |
| 313 | + | return ports.DeletionTombstonePage{}, ErrMaintenanceBudgetTooSmall | |
| 314 | + | } | |
| 315 | + | state.More = true | |
| 316 | + | break | |
| 317 | + | } | |
| 318 | + | remainingBytes -= charge + uint64(info.Size()) | |
| 319 | + | if hook := r.hooks.beforeDeletionTombstoneRead; hook != nil { | |
| 320 | + | hook(path) | |
| 321 | + | } | |
| 322 | + | data, err := r.readBounded(path) | |
| 323 | + | if err != nil { | |
| 324 | + | return ports.DeletionTombstonePage{}, err | |
| 325 | + | } | |
| 326 | + | decoded, err := decodeDeletionTombstone(data) | |
| 327 | + | if err != nil { | |
| 328 | + | return ports.DeletionTombstonePage{}, err | |
| 329 | + | } | |
| 330 | + | if candidate != decoded.IncarnationID.String()+".tombstone" { | |
| 331 | + | return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: non-canonical deletion tombstone filename") | |
| 332 | + | } | |
| 333 | + | tombstones = append(tombstones, decoded) | |
| 334 | + | last = candidate | |
| 293 | 335 | } | |
| 294 | - | next, err := encodeDeletionListingCursor(deletionListingState{After: state.Candidate}) | |
| 336 | + | next, err := encodeDeletionListingCursor(deletionListingState{After: last}) | |
| 295 | 337 | if err != nil { | |
| 296 | 338 | return ports.DeletionTombstonePage{}, err | |
| 297 | 339 | } | |
| 298 | - | return ports.DeletionTombstonePage{ | |
| 299 | - | Tombstones: []domain.DeletionTombstone{decoded}, | |
| 300 | - | Next: next, | |
| 301 | - | Done: state.Found == 1, | |
| 302 | - | }, nil | |
| 340 | + | return ports.DeletionTombstonePage{Tombstones: tombstones, Next: next, Done: !state.More && len(tombstones) == len(state.Candidates)}, nil | |
| 303 | 341 | } | |
| 304 | 342 | ||
| 305 | 343 | func (r *Repository) DeleteDeletionTombstone(ctx context.Context, id domain.IncarnationID) error { | |
| @@ -329,8 +367,7 @@ func (r *Repository) Delete(ctx context.Context, name string) error { | |||
| 329 | 367 | if err := ctx.Err(); err != nil { | |
| 330 | 368 | return err | |
| 331 | 369 | } | |
| 332 | - | id := legacyIncarnationID(name) | |
| 333 | - | key, err := incarnationKey(id) | |
| 370 | + | key, err := r.legacyRepositoryKey(name) | |
| 334 | 371 | if err != nil { | |
| 335 | 372 | return err | |
| 336 | 373 | } | |
| @@ -340,7 +377,7 @@ func (r *Repository) Delete(ctx context.Context, name string) error { | |||
| 340 | 377 | return err | |
| 341 | 378 | } | |
| 342 | 379 | ||
| 343 | - | canonical := r.sessionPath(id) | |
| 380 | + | canonical := r.legacySessionPath(key) | |
| 344 | 381 | sessions := filepath.Dir(canonical) | |
| 345 | 382 | // A prior rename may have succeeded while its parent sync failed. Complete | |
| 346 | 383 | // that durability boundary before considering a canonical directory: this | |
| @@ -520,6 +557,13 @@ func (r *Repository) QuarantineDeletionSources(ctx context.Context, tombstone do | |||
| 520 | 557 | if err := ctx.Err(); err != nil { | |
| 521 | 558 | return err | |
| 522 | 559 | } | |
| 560 | + | key, err := incarnationKey(tombstone.IncarnationID) | |
| 561 | + | if err != nil { | |
| 562 | + | return err | |
| 563 | + | } | |
| 564 | + | lock := r.lockSession(key) | |
| 565 | + | defer r.unlockSession(lock) | |
| 566 | + | r.invalidateStorageEpoch(key) | |
| 523 | 567 | quarantineDir := filepath.Join(r.dir, "quarantine", tombstone.IncarnationID.String()) | |
| 524 | 568 | if err := r.ensurePrivateDirectory(quarantineDir); err != nil { | |
| 525 | 569 | return err | |
| @@ -546,8 +590,8 @@ func (r *Repository) QuarantineDeletionSources(ctx context.Context, tombstone do | |||
| 546 | 590 | } | |
| 547 | 591 | ||
| 548 | 592 | func (r *Repository) quarantineSource(source, target string) error { | |
| 549 | - | if _, err := os.Lstat(target); err == nil { | |
| 550 | - | if _, sourceErr := os.Lstat(source); errors.Is(sourceErr, os.ErrNotExist) { | |
| 593 | + | if _, err := r.stat(target); err == nil { | |
| 594 | + | if _, sourceErr := r.stat(source); errors.Is(sourceErr, os.ErrNotExist) { | |
| 551 | 595 | if err := r.syncDirectory(filepath.Dir(source)); err != nil { | |
| 552 | 596 | return err | |
| 553 | 597 | } | |
| @@ -560,7 +604,7 @@ func (r *Repository) quarantineSource(source, target string) error { | |||
| 560 | 604 | } else if !errors.Is(err, os.ErrNotExist) { | |
| 561 | 605 | return err | |
| 562 | 606 | } | |
| 563 | - | if _, err := os.Lstat(source); errors.Is(err, os.ErrNotExist) { | |
| 607 | + | if _, err := r.stat(source); errors.Is(err, os.ErrNotExist) { | |
| 564 | 608 | return nil | |
| 565 | 609 | } else if err != nil { | |
| 566 | 610 | return err | |
| @@ -11,6 +11,7 @@ import ( | |||||
| 11 | "io" | 11 | "io" | ||
| 12 | "os" | 12 | "os" | ||
| 13 | "path/filepath" | 13 | "path/filepath" | ||
| 14 | + | "sort" | |||
| 14 | "strings" | 15 | "strings" | ||
| 15 | 16 | ||||
| 16 | "github.com/bnema/vev/internal/domain" | 17 | "github.com/bnema/vev/internal/domain" | ||
| @@ -94,13 +95,14 @@ const ( | |||||
| 94 | ) | 95 | ) | ||
| 95 | 96 | ||||
| 96 | type deletionListingState struct { | 97 | type deletionListingState struct { | ||
| 97 | - | After string `json:"after,omitempty"` | 98 | + | After string `json:"after,omitempty"` |
| 98 | - | Offset int64 `json:"offset,omitempty"` | 99 | + | Offset int64 `json:"offset,omitempty"` |
| 99 | - | Pending []maintenanceDirEntry `json:"pending,omitempty"` | 100 | + | Pending []maintenanceDirEntry `json:"pending,omitempty"` |
| 100 | - | PendingAt int `json:"pending_at,omitempty"` | 101 | + | PendingAt int `json:"pending_at,omitempty"` |
| 101 | - | ScanDone bool `json:"scan_done,omitempty"` | 102 | + | ScanDone bool `json:"scan_done,omitempty"` |
| 102 | - | Candidate string `json:"candidate,omitempty"` | 103 | + | Candidates []string `json:"candidates,omitempty"` |
| 103 | - | Found uint8 `json:"found,omitempty"` | 104 | + | More bool `json:"more,omitempty"` |
| 105 | + | BatchLimit int `json:"batch_limit,omitempty"` | |||
| 104 | } | 106 | } | ||
| 105 | 107 | ||||
| 106 | // MarshalJSON keeps maintenanceDirEntry private while making a continuation | 108 | // MarshalJSON keeps maintenanceDirEntry private while making a continuation | ||
| @@ -165,14 +167,16 @@ func validateDeletionListingState(state deletionListingState) error { | |||||
| 165 | if state.After != "" && !canonicalDeletionTombstoneFilename(state.After) { | 167 | if state.After != "" && !canonicalDeletionTombstoneFilename(state.After) { | ||
| 166 | return fmt.Errorf("snapshot: non-canonical deletion tombstone continuation key") | 168 | return fmt.Errorf("snapshot: non-canonical deletion tombstone continuation key") | ||
| 167 | } | 169 | } | ||
| 168 | - | if state.Offset < 0 || state.PendingAt < 0 || state.PendingAt > len(state.Pending) || state.Found > 2 { | 170 | + | if state.Offset < 0 || state.PendingAt < 0 || state.PendingAt > len(state.Pending) || state.BatchLimit < 0 || state.BatchLimit > maintenanceBatch || len(state.Candidates) > state.BatchLimit { |
| 169 | return fmt.Errorf("snapshot: malformed deletion tombstone continuation") | 171 | return fmt.Errorf("snapshot: malformed deletion tombstone continuation") | ||
| 170 | } | 172 | } | ||
| 171 | - | if state.After == "" && state.Offset == 0 && len(state.Pending) == 0 && !state.ScanDone && state.Candidate == "" { | 173 | + | if state.After == "" && state.Offset == 0 && len(state.Pending) == 0 && !state.ScanDone && len(state.Candidates) == 0 { |
| 172 | return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation") | 174 | return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation") | ||
| 173 | } | 175 | } | ||
| 174 | - | if (state.Candidate == "") != (state.Found == 0) || (state.Candidate != "" && (!canonicalDeletionTombstoneFilename(state.Candidate) || state.Candidate <= state.After)) { | 176 | + | for i, candidate := range state.Candidates { |
| 175 | - | return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation") | 177 | + | if !canonicalDeletionTombstoneFilename(candidate) || candidate <= state.After || (i > 0 && candidate <= state.Candidates[i-1]) { |
| 178 | + | return fmt.Errorf("snapshot: non-advancing deletion tombstone continuation") | |||
| 179 | + | } | |||
| 176 | } | 180 | } | ||
| 177 | if len(state.Pending) > maintenanceDirentBufferSize { | 181 | if len(state.Pending) > maintenanceDirentBufferSize { | ||
| 178 | return fmt.Errorf("snapshot: malformed deletion tombstone continuation") | 182 | return fmt.Errorf("snapshot: malformed deletion tombstone continuation") | ||
| @@ -228,31 +232,56 @@ func (r *Repository) ListDeletionTombstones(ctx context.Context, cursor ports.De | |||||
| 228 | r.maintenanceMu.Lock() | 232 | r.maintenanceMu.Lock() | ||
| 229 | defer r.maintenanceMu.Unlock() | 233 | defer r.maintenanceMu.Unlock() | ||
| 230 | 234 | ||||
| 231 | - | entryLimit := budget.Entries | 235 | + | batchLimit := int(min(budget.Entries, uint64(maintenanceBatch))) |
| 232 | - | if maxInt := uint64(^uint(0) >> 1); entryLimit > maxInt { | 236 | + | if state.BatchLimit == 0 { |
| 233 | - | entryLimit = maxInt | 237 | + | state.BatchLimit = batchLimit |
| 234 | - | } | 238 | + | } else if state.BatchLimit > batchLimit { |
| 235 | - | entries, done, err := r.readDeletionListingEntries(dir, int(entryLimit), &state) | 239 | + | state.BatchLimit = batchLimit |
| 236 | - | if errors.Is(err, os.ErrNotExist) { | 240 | + | if len(state.Candidates) > state.BatchLimit { |
| 237 | - | return ports.DeletionTombstonePage{Done: true}, nil | 241 | + | state.Candidates = state.Candidates[:state.BatchLimit] |
| 238 | - | } | 242 | + | state.More = true |
| 239 | - | if err != nil { | 243 | + | } |
| 240 | - | return ports.DeletionTombstonePage{}, err | |||
| 241 | } | 244 | } | ||
| 242 | - | for _, entry := range entries { | 245 | + | remainingEntries := budget.Entries |
| 243 | - | if err := ctx.Err(); err != nil { | 246 | + | done := state.ScanDone |
| 244 | - | return ports.DeletionTombstonePage{}, err | 247 | + | for !done && remainingEntries > 0 { |
| 248 | + | limit := remainingEntries | |||
| 249 | + | if maxInt := uint64(^uint(0) >> 1); limit > maxInt { | |||
| 250 | + | limit = maxInt | |||
| 245 | } | 251 | } | ||
| 246 | - | if entry.isDir || !canonicalDeletionTombstoneFilename(entry.name) { | 252 | + | entries, batchDone, err := r.readDeletionListingEntries(dir, int(limit), &state) |
| 247 | - | return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: invalid deletion tombstone entry") | 253 | + | if errors.Is(err, os.ErrNotExist) { |
| 254 | + | return ports.DeletionTombstonePage{Done: true}, nil | |||
| 248 | } | 255 | } | ||
| 249 | - | if entry.name > state.After { | 256 | + | if err != nil { |
| 250 | - | if state.Found < 2 { | 257 | + | return ports.DeletionTombstonePage{}, err |
| 251 | - | state.Found++ | 258 | + | } |
| 259 | + | remainingEntries -= uint64(len(entries)) | |||
| 260 | + | for _, entry := range entries { | |||
| 261 | + | if err := ctx.Err(); err != nil { | |||
| 262 | + | return ports.DeletionTombstonePage{}, err | |||
| 263 | + | } | |||
| 264 | + | if entry.isDir || !canonicalDeletionTombstoneFilename(entry.name) { | |||
| 265 | + | return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: invalid deletion tombstone entry") | |||
| 266 | + | } | |||
| 267 | + | if entry.name <= state.After { | |||
| 268 | + | continue | |||
| 252 | } | 269 | } | ||
| 253 | - | if state.Candidate == "" || entry.name < state.Candidate { | 270 | + | at := sort.SearchStrings(state.Candidates, entry.name) |
| 254 | - | state.Candidate = entry.name | 271 | + | if at < len(state.Candidates) && state.Candidates[at] == entry.name { |
| 272 | + | continue | |||
| 255 | } | 273 | } | ||
| 274 | + | state.Candidates = append(state.Candidates, "") | |||
| 275 | + | copy(state.Candidates[at+1:], state.Candidates[at:]) | |||
| 276 | + | state.Candidates[at] = entry.name | |||
| 277 | + | if len(state.Candidates) > state.BatchLimit { | |||
| 278 | + | state.Candidates = state.Candidates[:state.BatchLimit] | |||
| 279 | + | state.More = true | |||
| 280 | + | } | |||
| 281 | + | } | |||
| 282 | + | done = batchDone | |||
| 283 | + | if len(entries) == 0 && !batchDone { | |||
| 284 | + | break | |||
| 256 | } | 285 | } | ||
| 257 | } | 286 | } | ||
| 258 | if !done { | 287 | if !done { | ||
| @@ -265,41 +294,50 @@ func (r *Repository) ListDeletionTombstones(ctx context.Context, cursor ports.De | |||||
| 265 | } | 294 | } | ||
| 266 | return ports.DeletionTombstonePage{Next: next}, nil | 295 | return ports.DeletionTombstonePage{Next: next}, nil | ||
| 267 | } | 296 | } | ||
| 268 | - | if state.Candidate == "" { | 297 | + | if len(state.Candidates) == 0 { |
| 269 | return ports.DeletionTombstonePage{Done: true}, nil | 298 | return ports.DeletionTombstonePage{Done: true}, nil | ||
| 270 | } | 299 | } | ||
| 271 | - | path := filepath.Join(dir, state.Candidate) | 300 | + | |
| 272 | - | info, err := r.stat(path) | 301 | + | remainingBytes := budget.Bytes |
| 273 | - | if err != nil { | 302 | + | tombstones := make([]domain.DeletionTombstone, 0, len(state.Candidates)) |
| 274 | - | return ports.DeletionTombstonePage{}, err | 303 | + | last := "" |
| 275 | - | } | 304 | + | for _, candidate := range state.Candidates { |
| 276 | - | charge := uint64(len(state.Candidate)) | 305 | + | path := filepath.Join(dir, candidate) |
| 277 | - | if info.Size() < 0 || uint64(info.Size()) > budget.Bytes || charge > budget.Bytes-uint64(info.Size()) { | 306 | + | info, err := r.stat(path) |
| 278 | - | return ports.DeletionTombstonePage{}, ErrMaintenanceBudgetTooSmall | 307 | + | if err != nil { |
| 279 | - | } | 308 | + | return ports.DeletionTombstonePage{}, err |
| 280 | - | if hook := r.hooks.beforeDeletionTombstoneRead; hook != nil { | 309 | + | } |
| 281 | - | hook(path) | 310 | + | charge := uint64(len(candidate)) |
| 282 | - | } | 311 | + | if info.Size() < 0 || uint64(info.Size()) > remainingBytes || charge > remainingBytes-uint64(info.Size()) { |
| 283 | - | data, err := r.readBounded(path) | 312 | + | if len(tombstones) == 0 { |
| 284 | - | if err != nil { | 313 | + | return ports.DeletionTombstonePage{}, ErrMaintenanceBudgetTooSmall |
| 285 | - | return ports.DeletionTombstonePage{}, err | 314 | + | } |
| 286 | - | } | 315 | + | state.More = true |
| 287 | - | decoded, err := decodeDeletionTombstone(data) | 316 | + | break |
| 288 | - | if err != nil { | 317 | + | } |
| 289 | - | return ports.DeletionTombstonePage{}, err | 318 | + | remainingBytes -= charge + uint64(info.Size()) |
| 290 | - | } | 319 | + | if hook := r.hooks.beforeDeletionTombstoneRead; hook != nil { |
| 291 | - | if state.Candidate != decoded.IncarnationID.String()+".tombstone" { | 320 | + | hook(path) |
| 292 | - | return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: non-canonical deletion tombstone filename") | 321 | + | } |
| 322 | + | data, err := r.readBounded(path) | |||
| 323 | + | if err != nil { | |||
| 324 | + | return ports.DeletionTombstonePage{}, err | |||
| 325 | + | } | |||
| 326 | + | decoded, err := decodeDeletionTombstone(data) | |||
| 327 | + | if err != nil { | |||
| 328 | + | return ports.DeletionTombstonePage{}, err | |||
| 329 | + | } | |||
| 330 | + | if candidate != decoded.IncarnationID.String()+".tombstone" { | |||
| 331 | + | return ports.DeletionTombstonePage{}, fmt.Errorf("snapshot: non-canonical deletion tombstone filename") | |||
| 332 | + | } | |||
| 333 | + | tombstones = append(tombstones, decoded) | |||
| 334 | + | last = candidate | |||
| 293 | } | 335 | } | ||
| 294 | - | next, err := encodeDeletionListingCursor(deletionListingState{After: state.Candidate}) | 336 | + | next, err := encodeDeletionListingCursor(deletionListingState{After: last}) |
| 295 | if err != nil { | 337 | if err != nil { | ||
| 296 | return ports.DeletionTombstonePage{}, err | 338 | return ports.DeletionTombstonePage{}, err | ||
| 297 | } | 339 | } | ||
| 298 | - | return ports.DeletionTombstonePage{ | 340 | + | return ports.DeletionTombstonePage{Tombstones: tombstones, Next: next, Done: !state.More && len(tombstones) == len(state.Candidates)}, nil |
| 299 | - | Tombstones: []domain.DeletionTombstone{decoded}, | |||
| 300 | - | Next: next, | |||
| 301 | - | Done: state.Found == 1, | |||
| 302 | - | }, nil | |||
| 303 | } | 341 | } | ||
| 304 | 342 | ||||
| 305 | func (r *Repository) DeleteDeletionTombstone(ctx context.Context, id domain.IncarnationID) error { | 343 | func (r *Repository) DeleteDeletionTombstone(ctx context.Context, id domain.IncarnationID) error { | ||
| @@ -329,8 +367,7 @@ func (r *Repository) Delete(ctx context.Context, name string) error { | |||||
| 329 | if err := ctx.Err(); err != nil { | 367 | if err := ctx.Err(); err != nil { | ||
| 330 | return err | 368 | return err | ||
| 331 | } | 369 | } | ||
| 332 | - | id := legacyIncarnationID(name) | 370 | + | key, err := r.legacyRepositoryKey(name) |
| 333 | - | key, err := incarnationKey(id) | |||
| 334 | if err != nil { | 371 | if err != nil { | ||
| 335 | return err | 372 | return err | ||
| 336 | } | 373 | } | ||
| @@ -340,7 +377,7 @@ func (r *Repository) Delete(ctx context.Context, name string) error { | |||||
| 340 | return err | 377 | return err | ||
| 341 | } | 378 | } | ||
| 342 | 379 | ||||
| 343 | - | canonical := r.sessionPath(id) | 380 | + | canonical := r.legacySessionPath(key) |
| 344 | sessions := filepath.Dir(canonical) | 381 | sessions := filepath.Dir(canonical) | ||
| 345 | // A prior rename may have succeeded while its parent sync failed. Complete | 382 | // A prior rename may have succeeded while its parent sync failed. Complete | ||
| 346 | // that durability boundary before considering a canonical directory: this | 383 | // that durability boundary before considering a canonical directory: this | ||
| @@ -520,6 +557,13 @@ func (r *Repository) QuarantineDeletionSources(ctx context.Context, tombstone do | |||||
| 520 | if err := ctx.Err(); err != nil { | 557 | if err := ctx.Err(); err != nil { | ||
| 521 | return err | 558 | return err | ||
| 522 | } | 559 | } | ||
| 560 | + | key, err := incarnationKey(tombstone.IncarnationID) | |||
| 561 | + | if err != nil { | |||
| 562 | + | return err | |||
| 563 | + | } | |||
| 564 | + | lock := r.lockSession(key) | |||
| 565 | + | defer r.unlockSession(lock) | |||
| 566 | + | r.invalidateStorageEpoch(key) | |||
| 523 | quarantineDir := filepath.Join(r.dir, "quarantine", tombstone.IncarnationID.String()) | 567 | quarantineDir := filepath.Join(r.dir, "quarantine", tombstone.IncarnationID.String()) | ||
| 524 | if err := r.ensurePrivateDirectory(quarantineDir); err != nil { | 568 | if err := r.ensurePrivateDirectory(quarantineDir); err != nil { | ||
| 525 | return err | 569 | return err | ||
| @@ -546,8 +590,8 @@ func (r *Repository) QuarantineDeletionSources(ctx context.Context, tombstone do | |||||
| 546 | } | 590 | } | ||
| 547 | 591 | ||||
| 548 | func (r *Repository) quarantineSource(source, target string) error { | 592 | func (r *Repository) quarantineSource(source, target string) error { | ||
| 549 | - | if _, err := os.Lstat(target); err == nil { | 593 | + | if _, err := r.stat(target); err == nil { |
| 550 | - | if _, sourceErr := os.Lstat(source); errors.Is(sourceErr, os.ErrNotExist) { | 594 | + | if _, sourceErr := r.stat(source); errors.Is(sourceErr, os.ErrNotExist) { |
| 551 | if err := r.syncDirectory(filepath.Dir(source)); err != nil { | 595 | if err := r.syncDirectory(filepath.Dir(source)); err != nil { | ||
| 552 | return err | 596 | return err | ||
| 553 | } | 597 | } | ||
| @@ -560,7 +604,7 @@ func (r *Repository) quarantineSource(source, target string) error { | |||||
| 560 | } else if !errors.Is(err, os.ErrNotExist) { | 604 | } else if !errors.Is(err, os.ErrNotExist) { | ||
| 561 | return err | 605 | return err | ||
| 562 | } | 606 | } | ||
| 563 | - | if _, err := os.Lstat(source); errors.Is(err, os.ErrNotExist) { | 607 | + | if _, err := r.stat(source); errors.Is(err, os.ErrNotExist) { |
| 564 | return nil | 608 | return nil | ||
| 565 | } else if err != nil { | 609 | } else if err != nil { | ||
| 566 | return err | 610 | return err | ||
M internal/adapters/snapshot/maintenance_delete_test.go +92 -11
| @@ -5,6 +5,7 @@ import ( | |||
| 5 | 5 | "encoding/base64" | |
| 6 | 6 | "encoding/hex" | |
| 7 | 7 | "errors" | |
| 8 | + | "fmt" | |
| 8 | 9 | "os" | |
| 9 | 10 | "path/filepath" | |
| 10 | 11 | "reflect" | |
| @@ -94,7 +95,7 @@ func TestDiscardCrashMatrix(t *testing.T) { | |||
| 94 | 95 | repo := NewRepository(privateDir(t)) | |
| 95 | 96 | source := repo.sessionPath(old) | |
| 96 | 97 | destinationDir := filepath.Join(repo.dir, "quarantine", old.String()) | |
| 97 | - | if err := os.MkdirAll(source, 0o700); err != nil { | |
| 98 | + | if err := repo.ensurePrivateDirectory(source); err != nil { | |
| 98 | 99 | t.Fatal(err) | |
| 99 | 100 | } | |
| 100 | 101 | if err := repo.ensurePrivateDirectory(destinationDir); err != nil { | |
| @@ -139,7 +140,7 @@ func TestDiscardCrashMatrix(t *testing.T) { | |||
| 139 | 140 | t.Run("quarantine never removes source data", func(t *testing.T) { | |
| 140 | 141 | repo := NewRepository(privateDir(t)) | |
| 141 | 142 | source := repo.sessionPath(old) | |
| 142 | - | if err := os.MkdirAll(source, 0o700); err != nil { | |
| 143 | + | if err := repo.ensurePrivateDirectory(source); err != nil { | |
| 143 | 144 | t.Fatal(err) | |
| 144 | 145 | } | |
| 145 | 146 | payload := filepath.Join(source, "payload") | |
| @@ -163,6 +164,48 @@ func TestDiscardCrashMatrix(t *testing.T) { | |||
| 163 | 164 | }) | |
| 164 | 165 | } | |
| 165 | 166 | ||
| 167 | + | func TestQuarantineDeletionSourcesSerializesConcurrentPublication(t *testing.T) { | |
| 168 | + | repo := NewRepository(privateDir(t)) | |
| 169 | + | id := domain.IncarnationID{9} | |
| 170 | + | tombstone := domain.DeletionTombstone{Name: "work", IncarnationID: id} | |
| 171 | + | publication := incarnationPublication(t, id, "work", 1, nil) | |
| 172 | + | if err := repo.Publish(context.Background(), publication); err != nil { | |
| 173 | + | t.Fatal(err) | |
| 174 | + | } | |
| 175 | + | ||
| 176 | + | quarantined := make(chan struct{}) | |
| 177 | + | release := make(chan struct{}) | |
| 178 | + | repo.hooks.afterDeletionIncarnationQuarantine = func() error { | |
| 179 | + | close(quarantined) | |
| 180 | + | <-release | |
| 181 | + | return nil | |
| 182 | + | } | |
| 183 | + | quarantineDone := make(chan error, 1) | |
| 184 | + | go func() { quarantineDone <- repo.QuarantineDeletionSources(context.Background(), tombstone, false) }() | |
| 185 | + | <-quarantined | |
| 186 | + | ||
| 187 | + | publishDone := make(chan error, 1) | |
| 188 | + | go func() { publishDone <- repo.Publish(context.Background(), publication) }() | |
| 189 | + | select { | |
| 190 | + | case err := <-publishDone: | |
| 191 | + | t.Fatalf("Publish completed while quarantine held the incarnation lock: %v", err) | |
| 192 | + | default: | |
| 193 | + | } | |
| 194 | + | close(release) | |
| 195 | + | if err := <-quarantineDone; err != nil { | |
| 196 | + | t.Fatal(err) | |
| 197 | + | } | |
| 198 | + | if err := <-publishDone; err != nil { | |
| 199 | + | t.Fatal(err) | |
| 200 | + | } | |
| 201 | + | if _, err := os.Stat(repo.headPath(id)); err != nil { | |
| 202 | + | t.Fatalf("republished HEAD: %v", err) | |
| 203 | + | } | |
| 204 | + | if _, err := os.Stat(filepath.Join(repo.dir, "quarantine", id.String(), "snapshot", repositoryHead)); err != nil { | |
| 205 | + | t.Fatalf("quarantined HEAD: %v", err) | |
| 206 | + | } | |
| 207 | + | } | |
| 208 | + | ||
| 166 | 209 | func TestDeletionTombstoneCodec(t *testing.T) { | |
| 167 | 210 | want := domain.DeletionTombstone{Name: "work", IncarnationID: domain.IncarnationID{1}} | |
| 168 | 211 | encoded, err := encodeDeletionTombstone(want) | |
| @@ -283,6 +326,51 @@ func TestDeletionTombstoneListingResumesAcrossRepositoryRecreation(t *testing.T) | |||
| 283 | 326 | } | |
| 284 | 327 | } | |
| 285 | 328 | ||
| 329 | + | func TestDeletionTombstoneListingBatchesSortedCandidatesWithinScanBudget(t *testing.T) { | |
| 330 | + | dir := privateDir(t) | |
| 331 | + | repo := NewRepository(dir) | |
| 332 | + | ctx := context.Background() | |
| 333 | + | want := make([]domain.DeletionTombstone, 5) | |
| 334 | + | for i := range want { | |
| 335 | + | want[i] = domain.DeletionTombstone{Name: fmt.Sprintf("session-%d", i), IncarnationID: domain.IncarnationID{15: byte(i + 1)}} | |
| 336 | + | } | |
| 337 | + | for _, i := range []int{4, 1, 3, 0, 2} { | |
| 338 | + | if err := repo.WriteDeletionTombstone(ctx, want[i]); err != nil { | |
| 339 | + | t.Fatal(err) | |
| 340 | + | } | |
| 341 | + | } | |
| 342 | + | ||
| 343 | + | cursor := ports.DeletionTombstoneCursor{} | |
| 344 | + | var got []domain.DeletionTombstone | |
| 345 | + | reads := 0 | |
| 346 | + | for calls := range 64 { | |
| 347 | + | repo = NewRepository(dir) | |
| 348 | + | repo.hooks.beforeDeletionTombstoneRead = func(string) { reads++ } | |
| 349 | + | entryBudget := uint64(2) | |
| 350 | + | if calls > 0 { | |
| 351 | + | entryBudget = 1 // A resumed caller may reduce its per-pass budget. | |
| 352 | + | } | |
| 353 | + | page, err := repo.ListDeletionTombstones(ctx, cursor, ports.MaintenanceBudget{Entries: entryBudget, Bytes: 4096}) | |
| 354 | + | if err != nil { | |
| 355 | + | t.Fatal(err) | |
| 356 | + | } | |
| 357 | + | if uint64(len(page.Tombstones)) > entryBudget { | |
| 358 | + | t.Fatalf("page %d returned %d tombstones over entry budget %d", calls, len(page.Tombstones), entryBudget) | |
| 359 | + | } | |
| 360 | + | got = append(got, page.Tombstones...) | |
| 361 | + | if page.Done { | |
| 362 | + | break | |
| 363 | + | } | |
| 364 | + | cursor = page.Next | |
| 365 | + | } | |
| 366 | + | if !slices.Equal(got, want) { | |
| 367 | + | t.Fatalf("tombstones = %#v, want %#v", got, want) | |
| 368 | + | } | |
| 369 | + | if reads != len(want) { | |
| 370 | + | t.Fatalf("payload reads = %d, want %d", reads, len(want)) | |
| 371 | + | } | |
| 372 | + | } | |
| 373 | + | ||
| 286 | 374 | func TestMalformedDeletionTombstone(t *testing.T) { | |
| 287 | 375 | dir := privateDir(t) | |
| 288 | 376 | repo := NewRepository(dir) | |
| @@ -346,15 +434,8 @@ func TestDeletionTombstoneListing(t *testing.T) { | |||
| 346 | 434 | if err != nil { | |
| 347 | 435 | t.Fatal(err) | |
| 348 | 436 | } | |
| 349 | - | if page.Done || len(page.Tombstones) != 1 || page.Tombstones[0] != first || page.Next.After == "" { | |
| 350 | - | t.Fatalf("first page = %#v", page) | |
| 351 | - | } | |
| 352 | - | page, err = repo.ListDeletionTombstones(ctx, page.Next, ports.MaintenanceBudget{Entries: 3, Bytes: 1024}) | |
| 353 | - | if err != nil { | |
| 354 | - | t.Fatal(err) | |
| 355 | - | } | |
| 356 | - | if !page.Done || len(page.Tombstones) != 1 || page.Tombstones[0] != second || page.Next.After == "" { | |
| 357 | - | t.Fatalf("second page = %#v", page) | |
| 437 | + | if !page.Done || !slices.Equal(page.Tombstones, []domain.DeletionTombstone{first, second}) || page.Next.After == "" { | |
| 438 | + | t.Fatalf("batched page = %#v", page) | |
| 358 | 439 | } | |
| 359 | 440 | if _, err := repo.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 3, Bytes: 1}); err == nil { | |
| 360 | 441 | t.Fatal("non-advancing budget accepted") | |
| @@ -5,6 +5,7 @@ import ( | |||||
| 5 | "encoding/base64" | 5 | "encoding/base64" | ||
| 6 | "encoding/hex" | 6 | "encoding/hex" | ||
| 7 | "errors" | 7 | "errors" | ||
| 8 | + | "fmt" | |||
| 8 | "os" | 9 | "os" | ||
| 9 | "path/filepath" | 10 | "path/filepath" | ||
| 10 | "reflect" | 11 | "reflect" | ||
| @@ -94,7 +95,7 @@ func TestDiscardCrashMatrix(t *testing.T) { | |||||
| 94 | repo := NewRepository(privateDir(t)) | 95 | repo := NewRepository(privateDir(t)) | ||
| 95 | source := repo.sessionPath(old) | 96 | source := repo.sessionPath(old) | ||
| 96 | destinationDir := filepath.Join(repo.dir, "quarantine", old.String()) | 97 | destinationDir := filepath.Join(repo.dir, "quarantine", old.String()) | ||
| 97 | - | if err := os.MkdirAll(source, 0o700); err != nil { | 98 | + | if err := repo.ensurePrivateDirectory(source); err != nil { |
| 98 | t.Fatal(err) | 99 | t.Fatal(err) | ||
| 99 | } | 100 | } | ||
| 100 | if err := repo.ensurePrivateDirectory(destinationDir); err != nil { | 101 | if err := repo.ensurePrivateDirectory(destinationDir); err != nil { | ||
| @@ -139,7 +140,7 @@ func TestDiscardCrashMatrix(t *testing.T) { | |||||
| 139 | t.Run("quarantine never removes source data", func(t *testing.T) { | 140 | t.Run("quarantine never removes source data", func(t *testing.T) { | ||
| 140 | repo := NewRepository(privateDir(t)) | 141 | repo := NewRepository(privateDir(t)) | ||
| 141 | source := repo.sessionPath(old) | 142 | source := repo.sessionPath(old) | ||
| 142 | - | if err := os.MkdirAll(source, 0o700); err != nil { | 143 | + | if err := repo.ensurePrivateDirectory(source); err != nil { |
| 143 | t.Fatal(err) | 144 | t.Fatal(err) | ||
| 144 | } | 145 | } | ||
| 145 | payload := filepath.Join(source, "payload") | 146 | payload := filepath.Join(source, "payload") | ||
| @@ -163,6 +164,48 @@ func TestDiscardCrashMatrix(t *testing.T) { | |||||
| 163 | }) | 164 | }) | ||
| 164 | } | 165 | } | ||
| 165 | 166 | ||||
| 167 | + | func TestQuarantineDeletionSourcesSerializesConcurrentPublication(t *testing.T) { | |||
| 168 | + | repo := NewRepository(privateDir(t)) | |||
| 169 | + | id := domain.IncarnationID{9} | |||
| 170 | + | tombstone := domain.DeletionTombstone{Name: "work", IncarnationID: id} | |||
| 171 | + | publication := incarnationPublication(t, id, "work", 1, nil) | |||
| 172 | + | if err := repo.Publish(context.Background(), publication); err != nil { | |||
| 173 | + | t.Fatal(err) | |||
| 174 | + | } | |||
| 175 | + | ||||
| 176 | + | quarantined := make(chan struct{}) | |||
| 177 | + | release := make(chan struct{}) | |||
| 178 | + | repo.hooks.afterDeletionIncarnationQuarantine = func() error { | |||
| 179 | + | close(quarantined) | |||
| 180 | + | <-release | |||
| 181 | + | return nil | |||
| 182 | + | } | |||
| 183 | + | quarantineDone := make(chan error, 1) | |||
| 184 | + | go func() { quarantineDone <- repo.QuarantineDeletionSources(context.Background(), tombstone, false) }() | |||
| 185 | + | <-quarantined | |||
| 186 | + | ||||
| 187 | + | publishDone := make(chan error, 1) | |||
| 188 | + | go func() { publishDone <- repo.Publish(context.Background(), publication) }() | |||
| 189 | + | select { | |||
| 190 | + | case err := <-publishDone: | |||
| 191 | + | t.Fatalf("Publish completed while quarantine held the incarnation lock: %v", err) | |||
| 192 | + | default: | |||
| 193 | + | } | |||
| 194 | + | close(release) | |||
| 195 | + | if err := <-quarantineDone; err != nil { | |||
| 196 | + | t.Fatal(err) | |||
| 197 | + | } | |||
| 198 | + | if err := <-publishDone; err != nil { | |||
| 199 | + | t.Fatal(err) | |||
| 200 | + | } | |||
| 201 | + | if _, err := os.Stat(repo.headPath(id)); err != nil { | |||
| 202 | + | t.Fatalf("republished HEAD: %v", err) | |||
| 203 | + | } | |||
| 204 | + | if _, err := os.Stat(filepath.Join(repo.dir, "quarantine", id.String(), "snapshot", repositoryHead)); err != nil { | |||
| 205 | + | t.Fatalf("quarantined HEAD: %v", err) | |||
| 206 | + | } | |||
| 207 | + | } | |||
| 208 | + | ||||
| 166 | func TestDeletionTombstoneCodec(t *testing.T) { | 209 | func TestDeletionTombstoneCodec(t *testing.T) { | ||
| 167 | want := domain.DeletionTombstone{Name: "work", IncarnationID: domain.IncarnationID{1}} | 210 | want := domain.DeletionTombstone{Name: "work", IncarnationID: domain.IncarnationID{1}} | ||
| 168 | encoded, err := encodeDeletionTombstone(want) | 211 | encoded, err := encodeDeletionTombstone(want) | ||
| @@ -283,6 +326,51 @@ func TestDeletionTombstoneListingResumesAcrossRepositoryRecreation(t *testing.T) | |||||
| 283 | } | 326 | } | ||
| 284 | } | 327 | } | ||
| 285 | 328 | ||||
| 329 | + | func TestDeletionTombstoneListingBatchesSortedCandidatesWithinScanBudget(t *testing.T) { | |||
| 330 | + | dir := privateDir(t) | |||
| 331 | + | repo := NewRepository(dir) | |||
| 332 | + | ctx := context.Background() | |||
| 333 | + | want := make([]domain.DeletionTombstone, 5) | |||
| 334 | + | for i := range want { | |||
| 335 | + | want[i] = domain.DeletionTombstone{Name: fmt.Sprintf("session-%d", i), IncarnationID: domain.IncarnationID{15: byte(i + 1)}} | |||
| 336 | + | } | |||
| 337 | + | for _, i := range []int{4, 1, 3, 0, 2} { | |||
| 338 | + | if err := repo.WriteDeletionTombstone(ctx, want[i]); err != nil { | |||
| 339 | + | t.Fatal(err) | |||
| 340 | + | } | |||
| 341 | + | } | |||
| 342 | + | ||||
| 343 | + | cursor := ports.DeletionTombstoneCursor{} | |||
| 344 | + | var got []domain.DeletionTombstone | |||
| 345 | + | reads := 0 | |||
| 346 | + | for calls := range 64 { | |||
| 347 | + | repo = NewRepository(dir) | |||
| 348 | + | repo.hooks.beforeDeletionTombstoneRead = func(string) { reads++ } | |||
| 349 | + | entryBudget := uint64(2) | |||
| 350 | + | if calls > 0 { | |||
| 351 | + | entryBudget = 1 // A resumed caller may reduce its per-pass budget. | |||
| 352 | + | } | |||
| 353 | + | page, err := repo.ListDeletionTombstones(ctx, cursor, ports.MaintenanceBudget{Entries: entryBudget, Bytes: 4096}) | |||
| 354 | + | if err != nil { | |||
| 355 | + | t.Fatal(err) | |||
| 356 | + | } | |||
| 357 | + | if uint64(len(page.Tombstones)) > entryBudget { | |||
| 358 | + | t.Fatalf("page %d returned %d tombstones over entry budget %d", calls, len(page.Tombstones), entryBudget) | |||
| 359 | + | } | |||
| 360 | + | got = append(got, page.Tombstones...) | |||
| 361 | + | if page.Done { | |||
| 362 | + | break | |||
| 363 | + | } | |||
| 364 | + | cursor = page.Next | |||
| 365 | + | } | |||
| 366 | + | if !slices.Equal(got, want) { | |||
| 367 | + | t.Fatalf("tombstones = %#v, want %#v", got, want) | |||
| 368 | + | } | |||
| 369 | + | if reads != len(want) { | |||
| 370 | + | t.Fatalf("payload reads = %d, want %d", reads, len(want)) | |||
| 371 | + | } | |||
| 372 | + | } | |||
| 373 | + | ||||
| 286 | func TestMalformedDeletionTombstone(t *testing.T) { | 374 | func TestMalformedDeletionTombstone(t *testing.T) { | ||
| 287 | dir := privateDir(t) | 375 | dir := privateDir(t) | ||
| 288 | repo := NewRepository(dir) | 376 | repo := NewRepository(dir) | ||
| @@ -346,15 +434,8 @@ func TestDeletionTombstoneListing(t *testing.T) { | |||||
| 346 | if err != nil { | 434 | if err != nil { | ||
| 347 | t.Fatal(err) | 435 | t.Fatal(err) | ||
| 348 | } | 436 | } | ||
| 349 | - | if page.Done || len(page.Tombstones) != 1 || page.Tombstones[0] != first || page.Next.After == "" { | 437 | + | if !page.Done || !slices.Equal(page.Tombstones, []domain.DeletionTombstone{first, second}) || page.Next.After == "" { |
| 350 | - | t.Fatalf("first page = %#v", page) | 438 | + | t.Fatalf("batched page = %#v", page) |
| 351 | - | } | |||
| 352 | - | page, err = repo.ListDeletionTombstones(ctx, page.Next, ports.MaintenanceBudget{Entries: 3, Bytes: 1024}) | |||
| 353 | - | if err != nil { | |||
| 354 | - | t.Fatal(err) | |||
| 355 | - | } | |||
| 356 | - | if !page.Done || len(page.Tombstones) != 1 || page.Tombstones[0] != second || page.Next.After == "" { | |||
| 357 | - | t.Fatalf("second page = %#v", page) | |||
| 358 | } | 439 | } | ||
| 359 | if _, err := repo.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 3, Bytes: 1}); err == nil { | 440 | if _, err := repo.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 3, Bytes: 1}); err == nil { | ||
| 360 | t.Fatal("non-advancing budget accepted") | 441 | t.Fatal("non-advancing budget accepted") | ||
M internal/adapters/snapshot/maintenance_gc.go +79 -21
| @@ -17,6 +17,7 @@ import ( | |||
| 17 | 17 | ||
| 18 | 18 | type retentionMaintenance struct { | |
| 19 | 19 | token string | |
| 20 | + | cursors map[string]*maintenanceCursor | |
| 20 | 21 | references map[ports.SnapshotDigest]struct{} | |
| 21 | 22 | validateQueue []ports.CheckpointRef | |
| 22 | 23 | validateIndex int | |
| @@ -403,7 +404,7 @@ func keepSet(plan ports.RetentionPlan) map[uint64]ports.SnapshotDigest { | |||
| 403 | 404 | // checkpoints. Every incarnation owns its budget and continuation state. | |
| 404 | 405 | // processRetentionEntries charges stat-reported size for every entry, requeues | |
| 405 | 406 | // the unadmitted suffix, and removes only canonical entries rejected by policy. | |
| 406 | - | func (r *Repository) processRetentionEntries(ctx context.Context, dir, purpose string, entries []maintenanceDirEntry, budget *ports.MaintenanceBudget, consumedBefore bool, policy retentionEntryPolicy) (bool, error) { | |
| 407 | + | func (r *Repository) processRetentionEntries(ctx context.Context, state *retentionMaintenance, dir, purpose string, entries []maintenanceDirEntry, budget *ports.MaintenanceBudget, consumedBefore bool, policy retentionEntryPolicy) (bool, error) { | |
| 407 | 408 | for i, entry := range entries { | |
| 408 | 409 | if err := ctx.Err(); err != nil { | |
| 409 | 410 | return false, err | |
| @@ -418,7 +419,7 @@ func (r *Repository) processRetentionEntries(ctx context.Context, dir, purpose s | |||
| 418 | 419 | size = uint64(info.Size()) | |
| 419 | 420 | } | |
| 420 | 421 | if budget.Entries == 0 || size > budget.Bytes { | |
| 421 | - | r.requeueMaintenanceEntries(dir, purpose, entries[i:]) | |
| 422 | + | r.requeueRetentionEntries(state, dir, purpose, entries[i:]) | |
| 422 | 423 | if !consumedBefore && i == 0 { | |
| 423 | 424 | return false, ErrMaintenanceBudgetTooSmall | |
| 424 | 425 | } | |
| @@ -462,8 +463,13 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||
| 462 | 463 | ) | |
| 463 | 464 | }() | |
| 464 | 465 | if plan.PinAll { | |
| 466 | + | // Lock order is maintenanceMu -> session mutex. Pinning the same | |
| 467 | + | // incarnation waits for its active retention pass, while unrelated | |
| 468 | + | // incarnations never wait for that pass's payload I/O. | |
| 465 | 469 | r.maintenanceMu.Lock() | |
| 470 | + | lock := r.lockSession(key) | |
| 466 | 471 | r.clearRetentionMaintenance(key) | |
| 472 | + | r.unlockSession(lock) | |
| 467 | 473 | r.maintenanceMu.Unlock() | |
| 468 | 474 | return true, nil | |
| 469 | 475 | } | |
| @@ -471,13 +477,11 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||
| 471 | 477 | return false, nil | |
| 472 | 478 | } | |
| 473 | 479 | ||
| 480 | + | // Establish per-incarnation state under maintenanceMu, then retain only the | |
| 481 | + | // session mutex across payload I/O. Never reacquire maintenanceMu while the | |
| 482 | + | // session mutex is held; PinAll relies on maintenanceMu -> session ordering. | |
| 474 | 483 | r.maintenanceMu.Lock() | |
| 475 | - | defer r.maintenanceMu.Unlock() | |
| 476 | 484 | lock := r.lockSession(key) | |
| 477 | - | defer r.unlockSession(lock) | |
| 478 | - | if r.maintenanceCursors == nil { | |
| 479 | - | r.maintenanceCursors = make(map[string]*maintenanceCursor) | |
| 480 | - | } | |
| 481 | 485 | if r.retentionSessions == nil { | |
| 482 | 486 | r.retentionSessions = make(map[string]*retentionMaintenance) | |
| 483 | 487 | } | |
| @@ -488,11 +492,23 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||
| 488 | 492 | r.clearRetentionMaintenance(key) | |
| 489 | 493 | state = &retentionMaintenance{ | |
| 490 | 494 | token: token, | |
| 495 | + | cursors: make(map[string]*maintenanceCursor), | |
| 491 | 496 | references: make(map[ports.SnapshotDigest]struct{}), | |
| 492 | 497 | validateQueue: append([]ports.CheckpointRef(nil), plan.Keep...), | |
| 493 | 498 | } | |
| 494 | 499 | r.retentionSessions[key] = state | |
| 495 | 500 | } | |
| 501 | + | r.maintenanceMu.Unlock() | |
| 502 | + | defer func() { | |
| 503 | + | r.unlockSession(lock) | |
| 504 | + | if done { | |
| 505 | + | r.maintenanceMu.Lock() | |
| 506 | + | if r.retentionSessions[key] == state { | |
| 507 | + | delete(r.retentionSessions, key) | |
| 508 | + | } | |
| 509 | + | r.maintenanceMu.Unlock() | |
| 510 | + | } | |
| 511 | + | }() | |
| 496 | 512 | keep := keepSet(plan) | |
| 497 | 513 | if !state.validated { | |
| 498 | 514 | for state.validateIndex < len(state.validateQueue) { | |
| @@ -505,6 +521,9 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||
| 505 | 521 | return false, fmt.Errorf("validate retained manifest %d: %w", ref.Generation, err) | |
| 506 | 522 | } | |
| 507 | 523 | if !admitted { | |
| 524 | + | if requested.Entries == budget.Entries && requested.Bytes == budget.Bytes { | |
| 525 | + | return false, ErrMaintenanceBudgetTooSmall | |
| 526 | + | } | |
| 508 | 527 | return false, nil | |
| 509 | 528 | } | |
| 510 | 529 | budget.Entries-- | |
| @@ -520,7 +539,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||
| 520 | 539 | purpose := "retention-manifests:" + key + ":" + token | |
| 521 | 540 | for !state.manifestsDone && budget.Entries > 0 && budget.Bytes > 0 { | |
| 522 | 541 | limit := int(min(budget.Entries, uint64(maintenanceBatch))) | |
| 523 | - | entries, done, err := r.readMaintenanceDir(filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations), limit, purpose) | |
| 542 | + | entries, done, err := r.readRetentionDir(state, filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations), limit, purpose) | |
| 524 | 543 | if errors.Is(err, os.ErrNotExist) { | |
| 525 | 544 | state.manifestsDone = true | |
| 526 | 545 | break | |
| @@ -530,7 +549,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||
| 530 | 549 | } | |
| 531 | 550 | dir := filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations) | |
| 532 | 551 | consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes | |
| 533 | - | admitted, err := r.processRetentionEntries(ctx, dir, purpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionManifest, generations: keep}) | |
| 552 | + | admitted, err := r.processRetentionEntries(ctx, state, dir, purpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionManifest, generations: keep}) | |
| 534 | 553 | if err != nil { | |
| 535 | 554 | return false, err | |
| 536 | 555 | } | |
| @@ -552,7 +571,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||
| 552 | 571 | shard := state.objectShard | |
| 553 | 572 | dir := filepath.Join(objectsRoot, shard) | |
| 554 | 573 | objectPurpose := "retention-objects:" + key + ":" + shard + ":" + token | |
| 555 | - | entries, done, err := r.readMaintenanceDir(dir, int(min(budget.Entries, uint64(maintenanceBatch))), objectPurpose) | |
| 574 | + | entries, done, err := r.readRetentionDir(state, dir, int(min(budget.Entries, uint64(maintenanceBatch))), objectPurpose) | |
| 556 | 575 | if errors.Is(err, os.ErrNotExist) { | |
| 557 | 576 | state.objectShard = "" | |
| 558 | 577 | continue | |
| @@ -561,7 +580,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||
| 561 | 580 | return false, err | |
| 562 | 581 | } | |
| 563 | 582 | consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes | |
| 564 | - | admitted, err := r.processRetentionEntries(ctx, dir, objectPurpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionObject, objects: state.references}) | |
| 583 | + | admitted, err := r.processRetentionEntries(ctx, state, dir, objectPurpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionObject, objects: state.references}) | |
| 565 | 584 | if err != nil { | |
| 566 | 585 | return false, err | |
| 567 | 586 | } | |
| @@ -576,23 +595,25 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||
| 576 | 595 | continue | |
| 577 | 596 | } | |
| 578 | 597 | if state.objectRootDone { | |
| 579 | - | delete(r.retentionSessions, key) | |
| 580 | 598 | return true, nil | |
| 581 | 599 | } | |
| 582 | - | entries, done, err := r.readMaintenanceDir(objectsRoot, 1, "retention-shards:"+key+":"+token) | |
| 600 | + | entries, done, err := r.readRetentionDir(state, objectsRoot, 1, "retention-shards:"+key+":"+token) | |
| 583 | 601 | if errors.Is(err, os.ErrNotExist) { | |
| 584 | - | delete(r.retentionSessions, key) | |
| 585 | 602 | return true, nil | |
| 586 | 603 | } | |
| 587 | 604 | if err != nil { | |
| 588 | 605 | return false, err | |
| 589 | 606 | } | |
| 590 | 607 | state.objectRootDone = done | |
| 591 | - | if len(entries) != 0 { | |
| 592 | - | budget.Entries-- | |
| 593 | - | if entries[0].isDir { | |
| 594 | - | state.objectShard = entries[0].name | |
| 608 | + | if len(entries) == 0 { | |
| 609 | + | if !done { | |
| 610 | + | return false, nil | |
| 595 | 611 | } | |
| 612 | + | continue | |
| 613 | + | } | |
| 614 | + | budget.Entries-- | |
| 615 | + | if entries[0].isDir { | |
| 616 | + | state.objectShard = entries[0].name | |
| 596 | 617 | } | |
| 597 | 618 | } | |
| 598 | 619 | return false, nil | |
| @@ -600,11 +621,48 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||
| 600 | 621 | ||
| 601 | 622 | func (r *Repository) clearRetentionMaintenance(key string) { | |
| 602 | 623 | delete(r.retentionSessions, key) | |
| 603 | - | for id := range r.maintenanceCursors { | |
| 604 | - | if strings.Contains(id, "retention-") && strings.Contains(id, key) { | |
| 605 | - | delete(r.maintenanceCursors, id) | |
| 624 | + | } | |
| 625 | + | ||
| 626 | + | func (r *Repository) readRetentionDir(state *retentionMaintenance, dir string, n int, purpose string) ([]maintenanceDirEntry, bool, error) { | |
| 627 | + | id := maintenanceCursorID(dir, purpose) | |
| 628 | + | cursor := state.cursors[id] | |
| 629 | + | if cursor != nil && cursor.pending != nil { | |
| 630 | + | count := min(n, len(cursor.pending)) | |
| 631 | + | entries := cursor.pending[:count] | |
| 632 | + | cursor.pending = cursor.pending[count:] | |
| 633 | + | done := cursor.done && len(cursor.pending) == 0 | |
| 634 | + | if done { | |
| 635 | + | delete(state.cursors, id) | |
| 636 | + | } else if count > 0 && len(cursor.pending) == 0 { | |
| 637 | + | cursor.pending = nil | |
| 638 | + | } | |
| 639 | + | return entries, done, nil | |
| 640 | + | } | |
| 641 | + | if cursor == nil { | |
| 642 | + | if len(state.cursors) >= maxMaintenanceCursors { | |
| 643 | + | return nil, false, errMaintenanceCursorLimit | |
| 606 | 644 | } | |
| 645 | + | cursor = &maintenanceCursor{} | |
| 646 | + | state.cursors[id] = cursor | |
| 647 | + | } | |
| 648 | + | entries, done, err := r.readMaintenanceDirent(dir, n, cursor) | |
| 649 | + | if err != nil || done { | |
| 650 | + | delete(state.cursors, id) | |
| 651 | + | } | |
| 652 | + | return entries, done, err | |
| 653 | + | } | |
| 654 | + | ||
| 655 | + | func (r *Repository) requeueRetentionEntries(state *retentionMaintenance, dir, purpose string, entries []maintenanceDirEntry) { | |
| 656 | + | if len(entries) == 0 { | |
| 657 | + | return | |
| 658 | + | } | |
| 659 | + | id := maintenanceCursorID(dir, purpose) | |
| 660 | + | cursor := state.cursors[id] | |
| 661 | + | if cursor == nil { | |
| 662 | + | cursor = &maintenanceCursor{done: true} | |
| 663 | + | state.cursors[id] = cursor | |
| 607 | 664 | } | |
| 665 | + | cursor.pending = append(entries, cursor.pending...) | |
| 608 | 666 | } | |
| 609 | 667 | ||
| 610 | 668 | func retentionToken(plan ports.RetentionPlan) string { | |
| @@ -17,6 +17,7 @@ import ( | |||||
| 17 | 17 | ||||
| 18 | type retentionMaintenance struct { | 18 | type retentionMaintenance struct { | ||
| 19 | token string | 19 | token string | ||
| 20 | + | cursors map[string]*maintenanceCursor | |||
| 20 | references map[ports.SnapshotDigest]struct{} | 21 | references map[ports.SnapshotDigest]struct{} | ||
| 21 | validateQueue []ports.CheckpointRef | 22 | validateQueue []ports.CheckpointRef | ||
| 22 | validateIndex int | 23 | validateIndex int | ||
| @@ -403,7 +404,7 @@ func keepSet(plan ports.RetentionPlan) map[uint64]ports.SnapshotDigest { | |||||
| 403 | // checkpoints. Every incarnation owns its budget and continuation state. | 404 | // checkpoints. Every incarnation owns its budget and continuation state. | ||
| 404 | // processRetentionEntries charges stat-reported size for every entry, requeues | 405 | // processRetentionEntries charges stat-reported size for every entry, requeues | ||
| 405 | // the unadmitted suffix, and removes only canonical entries rejected by policy. | 406 | // the unadmitted suffix, and removes only canonical entries rejected by policy. | ||
| 406 | - | func (r *Repository) processRetentionEntries(ctx context.Context, dir, purpose string, entries []maintenanceDirEntry, budget *ports.MaintenanceBudget, consumedBefore bool, policy retentionEntryPolicy) (bool, error) { | 407 | + | func (r *Repository) processRetentionEntries(ctx context.Context, state *retentionMaintenance, dir, purpose string, entries []maintenanceDirEntry, budget *ports.MaintenanceBudget, consumedBefore bool, policy retentionEntryPolicy) (bool, error) { |
| 407 | for i, entry := range entries { | 408 | for i, entry := range entries { | ||
| 408 | if err := ctx.Err(); err != nil { | 409 | if err := ctx.Err(); err != nil { | ||
| 409 | return false, err | 410 | return false, err | ||
| @@ -418,7 +419,7 @@ func (r *Repository) processRetentionEntries(ctx context.Context, dir, purpose s | |||||
| 418 | size = uint64(info.Size()) | 419 | size = uint64(info.Size()) | ||
| 419 | } | 420 | } | ||
| 420 | if budget.Entries == 0 || size > budget.Bytes { | 421 | if budget.Entries == 0 || size > budget.Bytes { | ||
| 421 | - | r.requeueMaintenanceEntries(dir, purpose, entries[i:]) | 422 | + | r.requeueRetentionEntries(state, dir, purpose, entries[i:]) |
| 422 | if !consumedBefore && i == 0 { | 423 | if !consumedBefore && i == 0 { | ||
| 423 | return false, ErrMaintenanceBudgetTooSmall | 424 | return false, ErrMaintenanceBudgetTooSmall | ||
| 424 | } | 425 | } | ||
| @@ -462,8 +463,13 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||||
| 462 | ) | 463 | ) | ||
| 463 | }() | 464 | }() | ||
| 464 | if plan.PinAll { | 465 | if plan.PinAll { | ||
| 466 | + | // Lock order is maintenanceMu -> session mutex. Pinning the same | |||
| 467 | + | // incarnation waits for its active retention pass, while unrelated | |||
| 468 | + | // incarnations never wait for that pass's payload I/O. | |||
| 465 | r.maintenanceMu.Lock() | 469 | r.maintenanceMu.Lock() | ||
| 470 | + | lock := r.lockSession(key) | |||
| 466 | r.clearRetentionMaintenance(key) | 471 | r.clearRetentionMaintenance(key) | ||
| 472 | + | r.unlockSession(lock) | |||
| 467 | r.maintenanceMu.Unlock() | 473 | r.maintenanceMu.Unlock() | ||
| 468 | return true, nil | 474 | return true, nil | ||
| 469 | } | 475 | } | ||
| @@ -471,13 +477,11 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||||
| 471 | return false, nil | 477 | return false, nil | ||
| 472 | } | 478 | } | ||
| 473 | 479 | ||||
| 480 | + | // Establish per-incarnation state under maintenanceMu, then retain only the | |||
| 481 | + | // session mutex across payload I/O. Never reacquire maintenanceMu while the | |||
| 482 | + | // session mutex is held; PinAll relies on maintenanceMu -> session ordering. | |||
| 474 | r.maintenanceMu.Lock() | 483 | r.maintenanceMu.Lock() | ||
| 475 | - | defer r.maintenanceMu.Unlock() | |||
| 476 | lock := r.lockSession(key) | 484 | lock := r.lockSession(key) | ||
| 477 | - | defer r.unlockSession(lock) | |||
| 478 | - | if r.maintenanceCursors == nil { | |||
| 479 | - | r.maintenanceCursors = make(map[string]*maintenanceCursor) | |||
| 480 | - | } | |||
| 481 | if r.retentionSessions == nil { | 485 | if r.retentionSessions == nil { | ||
| 482 | r.retentionSessions = make(map[string]*retentionMaintenance) | 486 | r.retentionSessions = make(map[string]*retentionMaintenance) | ||
| 483 | } | 487 | } | ||
| @@ -488,11 +492,23 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||||
| 488 | r.clearRetentionMaintenance(key) | 492 | r.clearRetentionMaintenance(key) | ||
| 489 | state = &retentionMaintenance{ | 493 | state = &retentionMaintenance{ | ||
| 490 | token: token, | 494 | token: token, | ||
| 495 | + | cursors: make(map[string]*maintenanceCursor), | |||
| 491 | references: make(map[ports.SnapshotDigest]struct{}), | 496 | references: make(map[ports.SnapshotDigest]struct{}), | ||
| 492 | validateQueue: append([]ports.CheckpointRef(nil), plan.Keep...), | 497 | validateQueue: append([]ports.CheckpointRef(nil), plan.Keep...), | ||
| 493 | } | 498 | } | ||
| 494 | r.retentionSessions[key] = state | 499 | r.retentionSessions[key] = state | ||
| 495 | } | 500 | } | ||
| 501 | + | r.maintenanceMu.Unlock() | |||
| 502 | + | defer func() { | |||
| 503 | + | r.unlockSession(lock) | |||
| 504 | + | if done { | |||
| 505 | + | r.maintenanceMu.Lock() | |||
| 506 | + | if r.retentionSessions[key] == state { | |||
| 507 | + | delete(r.retentionSessions, key) | |||
| 508 | + | } | |||
| 509 | + | r.maintenanceMu.Unlock() | |||
| 510 | + | } | |||
| 511 | + | }() | |||
| 496 | keep := keepSet(plan) | 512 | keep := keepSet(plan) | ||
| 497 | if !state.validated { | 513 | if !state.validated { | ||
| 498 | for state.validateIndex < len(state.validateQueue) { | 514 | for state.validateIndex < len(state.validateQueue) { | ||
| @@ -505,6 +521,9 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||||
| 505 | return false, fmt.Errorf("validate retained manifest %d: %w", ref.Generation, err) | 521 | return false, fmt.Errorf("validate retained manifest %d: %w", ref.Generation, err) | ||
| 506 | } | 522 | } | ||
| 507 | if !admitted { | 523 | if !admitted { | ||
| 524 | + | if requested.Entries == budget.Entries && requested.Bytes == budget.Bytes { | |||
| 525 | + | return false, ErrMaintenanceBudgetTooSmall | |||
| 526 | + | } | |||
| 508 | return false, nil | 527 | return false, nil | ||
| 509 | } | 528 | } | ||
| 510 | budget.Entries-- | 529 | budget.Entries-- | ||
| @@ -520,7 +539,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||||
| 520 | purpose := "retention-manifests:" + key + ":" + token | 539 | purpose := "retention-manifests:" + key + ":" + token | ||
| 521 | for !state.manifestsDone && budget.Entries > 0 && budget.Bytes > 0 { | 540 | for !state.manifestsDone && budget.Entries > 0 && budget.Bytes > 0 { | ||
| 522 | limit := int(min(budget.Entries, uint64(maintenanceBatch))) | 541 | limit := int(min(budget.Entries, uint64(maintenanceBatch))) | ||
| 523 | - | entries, done, err := r.readMaintenanceDir(filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations), limit, purpose) | 542 | + | entries, done, err := r.readRetentionDir(state, filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations), limit, purpose) |
| 524 | if errors.Is(err, os.ErrNotExist) { | 543 | if errors.Is(err, os.ErrNotExist) { | ||
| 525 | state.manifestsDone = true | 544 | state.manifestsDone = true | ||
| 526 | break | 545 | break | ||
| @@ -530,7 +549,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||||
| 530 | } | 549 | } | ||
| 531 | dir := filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations) | 550 | dir := filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations) | ||
| 532 | consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes | 551 | consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes | ||
| 533 | - | admitted, err := r.processRetentionEntries(ctx, dir, purpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionManifest, generations: keep}) | 552 | + | admitted, err := r.processRetentionEntries(ctx, state, dir, purpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionManifest, generations: keep}) |
| 534 | if err != nil { | 553 | if err != nil { | ||
| 535 | return false, err | 554 | return false, err | ||
| 536 | } | 555 | } | ||
| @@ -552,7 +571,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||||
| 552 | shard := state.objectShard | 571 | shard := state.objectShard | ||
| 553 | dir := filepath.Join(objectsRoot, shard) | 572 | dir := filepath.Join(objectsRoot, shard) | ||
| 554 | objectPurpose := "retention-objects:" + key + ":" + shard + ":" + token | 573 | objectPurpose := "retention-objects:" + key + ":" + shard + ":" + token | ||
| 555 | - | entries, done, err := r.readMaintenanceDir(dir, int(min(budget.Entries, uint64(maintenanceBatch))), objectPurpose) | 574 | + | entries, done, err := r.readRetentionDir(state, dir, int(min(budget.Entries, uint64(maintenanceBatch))), objectPurpose) |
| 556 | if errors.Is(err, os.ErrNotExist) { | 575 | if errors.Is(err, os.ErrNotExist) { | ||
| 557 | state.objectShard = "" | 576 | state.objectShard = "" | ||
| 558 | continue | 577 | continue | ||
| @@ -561,7 +580,7 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||||
| 561 | return false, err | 580 | return false, err | ||
| 562 | } | 581 | } | ||
| 563 | consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes | 582 | consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes | ||
| 564 | - | admitted, err := r.processRetentionEntries(ctx, dir, objectPurpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionObject, objects: state.references}) | 583 | + | admitted, err := r.processRetentionEntries(ctx, state, dir, objectPurpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionObject, objects: state.references}) |
| 565 | if err != nil { | 584 | if err != nil { | ||
| 566 | return false, err | 585 | return false, err | ||
| 567 | } | 586 | } | ||
| @@ -576,23 +595,25 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||||
| 576 | continue | 595 | continue | ||
| 577 | } | 596 | } | ||
| 578 | if state.objectRootDone { | 597 | if state.objectRootDone { | ||
| 579 | - | delete(r.retentionSessions, key) | |||
| 580 | return true, nil | 598 | return true, nil | ||
| 581 | } | 599 | } | ||
| 582 | - | entries, done, err := r.readMaintenanceDir(objectsRoot, 1, "retention-shards:"+key+":"+token) | 600 | + | entries, done, err := r.readRetentionDir(state, objectsRoot, 1, "retention-shards:"+key+":"+token) |
| 583 | if errors.Is(err, os.ErrNotExist) { | 601 | if errors.Is(err, os.ErrNotExist) { | ||
| 584 | - | delete(r.retentionSessions, key) | |||
| 585 | return true, nil | 602 | return true, nil | ||
| 586 | } | 603 | } | ||
| 587 | if err != nil { | 604 | if err != nil { | ||
| 588 | return false, err | 605 | return false, err | ||
| 589 | } | 606 | } | ||
| 590 | state.objectRootDone = done | 607 | state.objectRootDone = done | ||
| 591 | - | if len(entries) != 0 { | 608 | + | if len(entries) == 0 { |
| 592 | - | budget.Entries-- | 609 | + | if !done { |
| 593 | - | if entries[0].isDir { | 610 | + | return false, nil |
| 594 | - | state.objectShard = entries[0].name | |||
| 595 | } | 611 | } | ||
| 612 | + | continue | |||
| 613 | + | } | |||
| 614 | + | budget.Entries-- | |||
| 615 | + | if entries[0].isDir { | |||
| 616 | + | state.objectShard = entries[0].name | |||
| 596 | } | 617 | } | ||
| 597 | } | 618 | } | ||
| 598 | return false, nil | 619 | return false, nil | ||
| @@ -600,11 +621,48 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||||
| 600 | 621 | ||||
| 601 | func (r *Repository) clearRetentionMaintenance(key string) { | 622 | func (r *Repository) clearRetentionMaintenance(key string) { | ||
| 602 | delete(r.retentionSessions, key) | 623 | delete(r.retentionSessions, key) | ||
| 603 | - | for id := range r.maintenanceCursors { | 624 | + | } |
| 604 | - | if strings.Contains(id, "retention-") && strings.Contains(id, key) { | 625 | + | |
| 605 | - | delete(r.maintenanceCursors, id) | 626 | + | func (r *Repository) readRetentionDir(state *retentionMaintenance, dir string, n int, purpose string) ([]maintenanceDirEntry, bool, error) { |
| 627 | + | id := maintenanceCursorID(dir, purpose) | |||
| 628 | + | cursor := state.cursors[id] | |||
| 629 | + | if cursor != nil && cursor.pending != nil { | |||
| 630 | + | count := min(n, len(cursor.pending)) | |||
| 631 | + | entries := cursor.pending[:count] | |||
| 632 | + | cursor.pending = cursor.pending[count:] | |||
| 633 | + | done := cursor.done && len(cursor.pending) == 0 | |||
| 634 | + | if done { | |||
| 635 | + | delete(state.cursors, id) | |||
| 636 | + | } else if count > 0 && len(cursor.pending) == 0 { | |||
| 637 | + | cursor.pending = nil | |||
| 638 | + | } | |||
| 639 | + | return entries, done, nil | |||
| 640 | + | } | |||
| 641 | + | if cursor == nil { | |||
| 642 | + | if len(state.cursors) >= maxMaintenanceCursors { | |||
| 643 | + | return nil, false, errMaintenanceCursorLimit | |||
| 606 | } | 644 | } | ||
| 645 | + | cursor = &maintenanceCursor{} | |||
| 646 | + | state.cursors[id] = cursor | |||
| 647 | + | } | |||
| 648 | + | entries, done, err := r.readMaintenanceDirent(dir, n, cursor) | |||
| 649 | + | if err != nil || done { | |||
| 650 | + | delete(state.cursors, id) | |||
| 651 | + | } | |||
| 652 | + | return entries, done, err | |||
| 653 | + | } | |||
| 654 | + | ||||
| 655 | + | func (r *Repository) requeueRetentionEntries(state *retentionMaintenance, dir, purpose string, entries []maintenanceDirEntry) { | |||
| 656 | + | if len(entries) == 0 { | |||
| 657 | + | return | |||
| 658 | + | } | |||
| 659 | + | id := maintenanceCursorID(dir, purpose) | |||
| 660 | + | cursor := state.cursors[id] | |||
| 661 | + | if cursor == nil { | |||
| 662 | + | cursor = &maintenanceCursor{done: true} | |||
| 663 | + | state.cursors[id] = cursor | |||
| 607 | } | 664 | } | ||
| 665 | + | cursor.pending = append(entries, cursor.pending...) | |||
| 608 | } | 666 | } | ||
| 609 | 667 | ||||
| 610 | func retentionToken(plan ports.RetentionPlan) string { | 668 | func retentionToken(plan ports.RetentionPlan) string { | ||
M internal/adapters/snapshot/maintenance_test.go +107 -33
| @@ -383,7 +383,7 @@ func TestRepositoryMaintainRetainsNewestTwoCompleteGenerations(t *testing.T) { | |||
| 383 | 383 | } | |
| 384 | 384 | } | |
| 385 | 385 | key := legacyIncarnationID("named").String() | |
| 386 | - | for pass := 0; pass < maintenanceBatch; pass++ { | |
| 386 | + | for pass := range maintenanceBatch { | |
| 387 | 387 | if err := repo.Maintain(context.Background()); err != nil { | |
| 388 | 388 | t.Fatal(err) | |
| 389 | 389 | } | |
| @@ -551,7 +551,7 @@ func TestRepositoryDeleteRetriesPendingQuarantineSyncWithoutDeletingRecreatedSes | |||
| 551 | 551 | func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) { | |
| 552 | 552 | repo := NewRepository(privateDir(t)) | |
| 553 | 553 | quarantine := filepath.Join(repo.dir, repositorySessionsDir, ".deleting-named-test") | |
| 554 | - | for i := 0; i < maintenanceBatch+1; i++ { | |
| 554 | + | for i := range maintenanceBatch + 1 { | |
| 555 | 555 | path := filepath.Join(quarantine, "nested", string(rune('a'+i))) | |
| 556 | 556 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 557 | 557 | t.Fatal(err) | |
| @@ -566,7 +566,7 @@ func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) { | |||
| 566 | 566 | if _, err := os.Lstat(quarantine); errors.Is(err, os.ErrNotExist) { | |
| 567 | 567 | t.Fatalf("quarantine after one bounded call = %v, want remaining tree", err) | |
| 568 | 568 | } | |
| 569 | - | for i := 0; i < 12; i++ { | |
| 569 | + | for range 12 { | |
| 570 | 570 | if err := repo.Maintain(context.Background()); err != nil { | |
| 571 | 571 | t.Fatal(err) | |
| 572 | 572 | } | |
| @@ -582,12 +582,12 @@ func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) { | |||
| 582 | 582 | func TestRepositoryMaintainClosesContinuationDirectories(t *testing.T) { | |
| 583 | 583 | repo := NewRepository(privateDir(t)) | |
| 584 | 584 | sessions := filepath.Join(repo.dir, repositorySessionsDir) | |
| 585 | - | for i := 0; i < maintenanceBatch+1; i++ { | |
| 585 | + | for i := range maintenanceBatch + 1 { | |
| 586 | 586 | generations := filepath.Join(sessions, fmt.Sprintf("named-%03d", i), repositoryGenerations) | |
| 587 | 587 | if err := os.MkdirAll(generations, 0o700); err != nil { | |
| 588 | 588 | t.Fatal(err) | |
| 589 | 589 | } | |
| 590 | - | for j := 0; j < maintenanceBatch+1; j++ { | |
| 590 | + | for j := range maintenanceBatch + 1 { | |
| 591 | 591 | if err := os.WriteFile(filepath.Join(generations, fmt.Sprintf("unclassified-%03d", j)), []byte("state"), 0o600); err != nil { | |
| 592 | 592 | t.Fatal(err) | |
| 593 | 593 | } | |
| @@ -606,14 +606,14 @@ func TestRepositoryMaintainBoundsQueuedShardNames(t *testing.T) { | |||
| 606 | 606 | repo := NewRepository(privateDir(t)) | |
| 607 | 607 | key := legacyIncarnationID("named").String() | |
| 608 | 608 | root := filepath.Join(repo.legacySessionPath(key), repositoryObjectsDir) | |
| 609 | - | for i := 0; i < maintenanceBatch*2; i++ { | |
| 609 | + | for i := range maintenanceBatch * 2 { | |
| 610 | 610 | if err := os.MkdirAll(filepath.Join(root, fmt.Sprintf("%02x", i)), 0o700); err != nil { | |
| 611 | 611 | t.Fatal(err) | |
| 612 | 612 | } | |
| 613 | 613 | } | |
| 614 | 614 | repo.maintenanceCursors = make(map[string]*maintenanceCursor) | |
| 615 | 615 | state := &sessionMaintenance{marked: make(map[uint64]manifestMaintenance), markDone: true} | |
| 616 | - | for i := 0; i < maintenanceBatch+2; i++ { | |
| 616 | + | for range maintenanceBatch + 2 { | |
| 617 | 617 | budget := maintenanceBatch | |
| 618 | 618 | if err := repo.sweepSession(context.Background(), key, state, &budget); err != nil { | |
| 619 | 619 | t.Fatal(err) | |
| @@ -629,7 +629,7 @@ func TestRepositoryMaintainResumesCursorAfterDirectoryMutation(t *testing.T) { | |||
| 629 | 629 | if err := os.Mkdir(repo.dir, 0o700); err != nil { | |
| 630 | 630 | t.Fatal(err) | |
| 631 | 631 | } | |
| 632 | - | for i := 0; i < maintenanceBatch+1; i++ { | |
| 632 | + | for i := range maintenanceBatch + 1 { | |
| 633 | 633 | if err := os.WriteFile(filepath.Join(repo.dir, fmt.Sprintf("live-%03d", i)), []byte("live"), 0o600); err != nil { | |
| 634 | 634 | t.Fatal(err) | |
| 635 | 635 | } | |
| @@ -646,7 +646,7 @@ func TestRepositoryMaintainResumesCursorAfterDirectoryMutation(t *testing.T) { | |||
| 646 | 646 | if err := os.WriteFile(stale, []byte("stale"), 0o600); err != nil { | |
| 647 | 647 | t.Fatal(err) | |
| 648 | 648 | } | |
| 649 | - | for i := 0; i < 4; i++ { | |
| 649 | + | for range 4 { | |
| 650 | 650 | if err := repo.Maintain(context.Background()); err != nil { | |
| 651 | 651 | t.Fatal(err) | |
| 652 | 652 | } | |
| @@ -664,17 +664,17 @@ func TestRepositoryMaintainDoesNotStarveLaterTemporaryEntries(t *testing.T) { | |||
| 664 | 664 | } | |
| 665 | 665 | // The live entries precede the stale entries in directory insertion order. | |
| 666 | 666 | // A fresh ReadDir from the start on every call would never reach the latter. | |
| 667 | - | for i := 0; i < maintenanceBatch+1; i++ { | |
| 667 | + | for i := range maintenanceBatch + 1 { | |
| 668 | 668 | if err := os.WriteFile(filepath.Join(repo.dir, "live-"+string(rune('a'+i))), []byte("live"), 0o600); err != nil { | |
| 669 | 669 | t.Fatal(err) | |
| 670 | 670 | } | |
| 671 | 671 | } | |
| 672 | - | for i := 0; i < maintenanceBatch+1; i++ { | |
| 672 | + | for i := range maintenanceBatch + 1 { | |
| 673 | 673 | if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-later-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil { | |
| 674 | 674 | t.Fatal(err) | |
| 675 | 675 | } | |
| 676 | 676 | } | |
| 677 | - | for i := 0; i < 4; i++ { | |
| 677 | + | for range 4 { | |
| 678 | 678 | if err := repo.Maintain(context.Background()); err != nil { | |
| 679 | 679 | t.Fatal(err) | |
| 680 | 680 | } | |
| @@ -694,7 +694,7 @@ func TestRepositoryMaintainMarksLargeGenerationSetBeforeSweeping(t *testing.T) { | |||
| 694 | 694 | repo := NewRepository(privateDir(t)) | |
| 695 | 695 | var newest ports.SnapshotPublication | |
| 696 | 696 | for generation := uint64(1); generation <= maintenanceBatch+2; generation++ { | |
| 697 | - | publication := repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation))) | |
| 697 | + | publication := repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation)) | |
| 698 | 698 | if err := repo.Publish(context.Background(), publication); err != nil { | |
| 699 | 699 | t.Fatal(err) | |
| 700 | 700 | } | |
| @@ -724,7 +724,7 @@ func TestRepositoryMaintainMarksLargeGenerationSetBeforeSweeping(t *testing.T) { | |||
| 724 | 724 | } | |
| 725 | 725 | } | |
| 726 | 726 | ||
| 727 | - | for i := 0; i < 600; i++ { | |
| 727 | + | for i := range 600 { | |
| 728 | 728 | if err := repo.Maintain(context.Background()); err != nil { | |
| 729 | 729 | t.Fatal(err) | |
| 730 | 730 | } | |
| @@ -747,7 +747,7 @@ func TestRepositoryMaintainUsesFixedRemovalBatch(t *testing.T) { | |||
| 747 | 747 | if err := os.Mkdir(repo.dir, 0o700); err != nil { | |
| 748 | 748 | t.Fatal(err) | |
| 749 | 749 | } | |
| 750 | - | for i := 0; i < maintenanceBatch+1; i++ { | |
| 750 | + | for i := range maintenanceBatch + 1 { | |
| 751 | 751 | if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil { | |
| 752 | 752 | t.Fatal(err) | |
| 753 | 753 | } | |
| @@ -774,7 +774,7 @@ func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) { | |||
| 774 | 774 | } | |
| 775 | 775 | ||
| 776 | 776 | removed := make(map[string]int) | |
| 777 | - | for i := 0; i < maintenanceBatch+1; i++ { | |
| 777 | + | for i := range maintenanceBatch + 1 { | |
| 778 | 778 | path := filepath.Join(repo.dir, repositorySessionsDir, fmt.Sprintf(".deleting-%03d", i)) | |
| 779 | 779 | if err := os.Mkdir(path, 0o700); err != nil { | |
| 780 | 780 | t.Fatal(err) | |
| @@ -804,7 +804,7 @@ func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) { | |||
| 804 | 804 | t.Fatal(err) | |
| 805 | 805 | } | |
| 806 | 806 | queued := fetched[len(fetched)-1].Name() | |
| 807 | - | for i := 0; i < maintenanceBatch+4; i++ { | |
| 807 | + | for i := range maintenanceBatch + 4 { | |
| 808 | 808 | if err := repo.Maintain(context.Background()); err != nil { | |
| 809 | 809 | t.Fatal(err) | |
| 810 | 810 | } | |
| @@ -818,7 +818,7 @@ func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) { | |||
| 818 | 818 | t.Fatalf("unread session %q = %v, want queued entry to run first", unread[0].Name(), err) | |
| 819 | 819 | } | |
| 820 | 820 | ||
| 821 | - | for i := 0; i < maintenanceBatch+4; i++ { | |
| 821 | + | for range maintenanceBatch + 4 { | |
| 822 | 822 | if err := repo.Maintain(context.Background()); err != nil { | |
| 823 | 823 | t.Fatal(err) | |
| 824 | 824 | } | |
| @@ -876,7 +876,7 @@ func TestRepositoryMaintainClassifiesUnpublishedGenerationsBeforeSweep(t *testin | |||
| 876 | 876 | key := legacyIncarnationID("named").String() | |
| 877 | 877 | publications := make([]ports.SnapshotPublication, 0, maintenanceBatch+2) | |
| 878 | 878 | for generation := uint64(1); generation <= maintenanceBatch+2; generation++ { | |
| 879 | - | publication := repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation))) | |
| 879 | + | publication := repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation)) | |
| 880 | 880 | if err := repo.Publish(context.Background(), publication); err != nil { | |
| 881 | 881 | t.Fatal(err) | |
| 882 | 882 | } | |
| @@ -904,7 +904,7 @@ func TestRepositoryMaintainClassifiesUnpublishedGenerationsBeforeSweep(t *testin | |||
| 904 | 904 | } | |
| 905 | 905 | } | |
| 906 | 906 | ||
| 907 | - | for i := 0; i < 600; i++ { | |
| 907 | + | for i := range 600 { | |
| 908 | 908 | if err := repo.Maintain(context.Background()); err != nil { | |
| 909 | 909 | t.Fatal(err) | |
| 910 | 910 | } | |
| @@ -928,7 +928,7 @@ func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) { | |||
| 928 | 928 | key := legacyIncarnationID("named").String() | |
| 929 | 929 | first := publicationWithTailShard(t, "named", 1, "ff") | |
| 930 | 930 | stalePaths := make([]string, 0, maintenanceBatch+1) | |
| 931 | - | for i := 0; i < maintenanceBatch+1; i++ { | |
| 931 | + | for i := range maintenanceBatch + 1 { | |
| 932 | 932 | digest := digestInShard(t, "00", i) | |
| 933 | 933 | path := repo.legacyObjectPath(key, digest) | |
| 934 | 934 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| @@ -967,7 +967,7 @@ func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) { | |||
| 967 | 967 | } | |
| 968 | 968 | repo.hooks.beforeHeadWrite = nil | |
| 969 | 969 | ||
| 970 | - | for i := 0; i < 20; i++ { | |
| 970 | + | for i := range 20 { | |
| 971 | 971 | if err := repo.Maintain(context.Background()); err != nil { | |
| 972 | 972 | t.Fatal(err) | |
| 973 | 973 | } | |
| @@ -991,8 +991,8 @@ func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) { | |||
| 991 | 991 | ||
| 992 | 992 | func publicationWithTailShard(t *testing.T, name string, generation uint64, shard string) ports.SnapshotPublication { | |
| 993 | 993 | t.Helper() | |
| 994 | - | for i := 0; i < 4096; i++ { | |
| 995 | - | publication := repositoryPublication(t, name, generation, []byte(fmt.Sprintf("state-%d-%d", generation, i))) | |
| 994 | + | for i := range 4096 { | |
| 995 | + | publication := repositoryPublication(t, name, generation, fmt.Appendf(nil, "state-%d-%d", generation, i)) | |
| 996 | 996 | if fmt.Sprintf("%02x", publication.Objects[0].Digest[0]) == shard { | |
| 997 | 997 | return publication | |
| 998 | 998 | } | |
| @@ -1003,8 +1003,8 @@ func publicationWithTailShard(t *testing.T, name string, generation uint64, shar | |||
| 1003 | 1003 | ||
| 1004 | 1004 | func digestInShard(t *testing.T, shard string, n int) ports.SnapshotDigest { | |
| 1005 | 1005 | t.Helper() | |
| 1006 | - | for i := 0; i < 4096; i++ { | |
| 1007 | - | digest := sha256.Sum256([]byte(fmt.Sprintf("stale-%d-%d", n, i))) | |
| 1006 | + | for i := range 4096 { | |
| 1007 | + | digest := sha256.Sum256(fmt.Appendf(nil, "stale-%d-%d", n, i)) | |
| 1008 | 1008 | if fmt.Sprintf("%02x", digest[0]) == shard { | |
| 1009 | 1009 | return digest | |
| 1010 | 1010 | } | |
| @@ -1068,8 +1068,8 @@ func TestMaintenanceBudgetAdmitsPayloadsBeforeRead(t *testing.T) { | |||
| 1068 | 1068 | var reads []string | |
| 1069 | 1069 | repo.hooks.beforeMaintenancePayloadRead = func(path string) { reads = append(reads, path) } | |
| 1070 | 1070 | done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | |
| 1071 | - | if err != nil { | |
| 1072 | - | t.Fatal(err) | |
| 1071 | + | if !errors.Is(err, ErrMaintenanceBudgetTooSmall) { | |
| 1072 | + | t.Fatalf("undersized manifest budget error = %v, want ErrMaintenanceBudgetTooSmall", err) | |
| 1073 | 1073 | } | |
| 1074 | 1074 | if done || len(reads) != 0 { | |
| 1075 | 1075 | t.Fatalf("undersized manifest budget: done=%v payload reads=%v", done, reads) | |
| @@ -1082,8 +1082,8 @@ func TestMaintenanceBudgetAdmitsPayloadsBeforeRead(t *testing.T) { | |||
| 1082 | 1082 | } | |
| 1083 | 1083 | reads = nil | |
| 1084 | 1084 | done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: uint64(info.Size())}) | |
| 1085 | - | if err != nil { | |
| 1086 | - | t.Fatal(err) | |
| 1085 | + | if !errors.Is(err, ErrMaintenanceBudgetTooSmall) { | |
| 1086 | + | t.Fatalf("manifest-only budget error = %v, want ErrMaintenanceBudgetTooSmall", err) | |
| 1087 | 1087 | } | |
| 1088 | 1088 | if done { | |
| 1089 | 1089 | t.Fatal("manifest-only budget unexpectedly admitted the generation objects") | |
| @@ -1157,7 +1157,7 @@ func TestReconcileDoesNotHoldMaintenanceLockDuringPayloadRead(t *testing.T) { | |||
| 1157 | 1157 | if err != nil { | |
| 1158 | 1158 | t.Fatal(err) | |
| 1159 | 1159 | } | |
| 1160 | - | case <-time.After(time.Second): | |
| 1160 | + | case <-time.After(remainingTestTime(t) / 2): | |
| 1161 | 1161 | t.Fatal("maintenance lock remained held during reconciliation payload I/O") | |
| 1162 | 1162 | } | |
| 1163 | 1163 | releaseOnce.Do(func() { close(release) }) | |
| @@ -1166,6 +1166,80 @@ func TestReconcileDoesNotHoldMaintenanceLockDuringPayloadRead(t *testing.T) { | |||
| 1166 | 1166 | } | |
| 1167 | 1167 | } | |
| 1168 | 1168 | ||
| 1169 | + | func TestMaintainSessionYieldsOnEmptyNonDoneObjectShardBatch(t *testing.T) { | |
| 1170 | + | repo := NewRepository(privateDir(t)) | |
| 1171 | + | publication := publishMaintenanceGenerations(t, repo, "empty-shard-batch", 1)[0] | |
| 1172 | + | plan := ports.RetentionPlan{IncarnationID: publication.IncarnationID} | |
| 1173 | + | key := publication.IncarnationID.String() | |
| 1174 | + | token := retentionToken(plan) | |
| 1175 | + | root := filepath.Join(repo.sessionPath(plan.IncarnationID), repositoryObjectsDir) | |
| 1176 | + | purpose := "retention-shards:" + key + ":" + token | |
| 1177 | + | state := &retentionMaintenance{ | |
| 1178 | + | token: token, | |
| 1179 | + | cursors: map[string]*maintenanceCursor{maintenanceCursorID(root, purpose): {pending: []maintenanceDirEntry{}}}, | |
| 1180 | + | references: make(map[ports.SnapshotDigest]struct{}), | |
| 1181 | + | validated: true, | |
| 1182 | + | manifestsDone: true, | |
| 1183 | + | } | |
| 1184 | + | repo.retentionSessions[key] = state | |
| 1185 | + | ||
| 1186 | + | done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: 1024}) | |
| 1187 | + | if err != nil || done { | |
| 1188 | + | t.Fatalf("empty non-done shard batch: done=%v err=%v", done, err) | |
| 1189 | + | } | |
| 1190 | + | if state.objectRootDone { | |
| 1191 | + | t.Fatal("empty non-done shard batch advanced object-root completion") | |
| 1192 | + | } | |
| 1193 | + | } | |
| 1194 | + | ||
| 1195 | + | func TestMaintainSessionDoesNotHoldMaintenanceLockDuringPayloadIO(t *testing.T) { | |
| 1196 | + | repo := NewRepository(privateDir(t)) | |
| 1197 | + | active := publishMaintenanceGenerations(t, repo, "active-lock", 1) | |
| 1198 | + | other := publishMaintenanceGenerations(t, repo, "other-lock", 1) | |
| 1199 | + | plan := ports.RetentionPlan{IncarnationID: active[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(active[0])}} | |
| 1200 | + | ||
| 1201 | + | entered := make(chan struct{}) | |
| 1202 | + | release := make(chan struct{}) | |
| 1203 | + | var once sync.Once | |
| 1204 | + | repo.hooks.beforeMaintenancePayloadRead = func(string) { | |
| 1205 | + | once.Do(func() { close(entered) }) | |
| 1206 | + | <-release | |
| 1207 | + | } | |
| 1208 | + | activeDone := make(chan error, 1) | |
| 1209 | + | go func() { | |
| 1210 | + | _, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20}) | |
| 1211 | + | activeDone <- err | |
| 1212 | + | }() | |
| 1213 | + | <-entered | |
| 1214 | + | ||
| 1215 | + | pinDone := make(chan error, 1) | |
| 1216 | + | go func() { | |
| 1217 | + | _, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: other[0].IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | |
| 1218 | + | pinDone <- err | |
| 1219 | + | }() | |
| 1220 | + | select { | |
| 1221 | + | case err := <-pinDone: | |
| 1222 | + | if err != nil { | |
| 1223 | + | t.Fatal(err) | |
| 1224 | + | } | |
| 1225 | + | case <-time.After(remainingTestTime(t) / 2): | |
| 1226 | + | close(release) | |
| 1227 | + | t.Fatal("unrelated PinAll was blocked by retention payload I/O") | |
| 1228 | + | } | |
| 1229 | + | close(release) | |
| 1230 | + | if err := <-activeDone; err != nil { | |
| 1231 | + | t.Fatal(err) | |
| 1232 | + | } | |
| 1233 | + | } | |
| 1234 | + | ||
| 1235 | + | func remainingTestTime(t *testing.T) time.Duration { | |
| 1236 | + | t.Helper() | |
| 1237 | + | if deadline, ok := t.Deadline(); ok { | |
| 1238 | + | return time.Until(deadline) | |
| 1239 | + | } | |
| 1240 | + | return 30 * time.Second | |
| 1241 | + | } | |
| 1242 | + | ||
| 1169 | 1243 | func TestPerSessionBudgetIsolation(t *testing.T) { | |
| 1170 | 1244 | repo := NewRepository(privateDir(t)) | |
| 1171 | 1245 | large := publishMaintenanceGenerations(t, repo, "large", 4) | |
| @@ -1174,8 +1248,8 @@ func TestPerSessionBudgetIsolation(t *testing.T) { | |||
| 1174 | 1248 | smallPlan := ports.RetentionPlan{IncarnationID: small[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(small[1])}} | |
| 1175 | 1249 | ||
| 1176 | 1250 | done, err := repo.MaintainSession(context.Background(), largePlan, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | |
| 1177 | - | if err != nil { | |
| 1178 | - | t.Fatal(err) | |
| 1251 | + | if !errors.Is(err, ErrMaintenanceBudgetTooSmall) { | |
| 1252 | + | t.Fatalf("large session error = %v, want ErrMaintenanceBudgetTooSmall", err) | |
| 1179 | 1253 | } | |
| 1180 | 1254 | if done { | |
| 1181 | 1255 | t.Fatal("large session unexpectedly exhausted no budget") | |
| @@ -383,7 +383,7 @@ func TestRepositoryMaintainRetainsNewestTwoCompleteGenerations(t *testing.T) { | |||||
| 383 | } | 383 | } | ||
| 384 | } | 384 | } | ||
| 385 | key := legacyIncarnationID("named").String() | 385 | key := legacyIncarnationID("named").String() | ||
| 386 | - | for pass := 0; pass < maintenanceBatch; pass++ { | 386 | + | for pass := range maintenanceBatch { |
| 387 | if err := repo.Maintain(context.Background()); err != nil { | 387 | if err := repo.Maintain(context.Background()); err != nil { | ||
| 388 | t.Fatal(err) | 388 | t.Fatal(err) | ||
| 389 | } | 389 | } | ||
| @@ -551,7 +551,7 @@ func TestRepositoryDeleteRetriesPendingQuarantineSyncWithoutDeletingRecreatedSes | |||||
| 551 | func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) { | 551 | func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) { | ||
| 552 | repo := NewRepository(privateDir(t)) | 552 | repo := NewRepository(privateDir(t)) | ||
| 553 | quarantine := filepath.Join(repo.dir, repositorySessionsDir, ".deleting-named-test") | 553 | quarantine := filepath.Join(repo.dir, repositorySessionsDir, ".deleting-named-test") | ||
| 554 | - | for i := 0; i < maintenanceBatch+1; i++ { | 554 | + | for i := range maintenanceBatch + 1 { |
| 555 | path := filepath.Join(quarantine, "nested", string(rune('a'+i))) | 555 | path := filepath.Join(quarantine, "nested", string(rune('a'+i))) | ||
| 556 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 556 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | ||
| 557 | t.Fatal(err) | 557 | t.Fatal(err) | ||
| @@ -566,7 +566,7 @@ func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) { | |||||
| 566 | if _, err := os.Lstat(quarantine); errors.Is(err, os.ErrNotExist) { | 566 | if _, err := os.Lstat(quarantine); errors.Is(err, os.ErrNotExist) { | ||
| 567 | t.Fatalf("quarantine after one bounded call = %v, want remaining tree", err) | 567 | t.Fatalf("quarantine after one bounded call = %v, want remaining tree", err) | ||
| 568 | } | 568 | } | ||
| 569 | - | for i := 0; i < 12; i++ { | 569 | + | for range 12 { |
| 570 | if err := repo.Maintain(context.Background()); err != nil { | 570 | if err := repo.Maintain(context.Background()); err != nil { | ||
| 571 | t.Fatal(err) | 571 | t.Fatal(err) | ||
| 572 | } | 572 | } | ||
| @@ -582,12 +582,12 @@ func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) { | |||||
| 582 | func TestRepositoryMaintainClosesContinuationDirectories(t *testing.T) { | 582 | func TestRepositoryMaintainClosesContinuationDirectories(t *testing.T) { | ||
| 583 | repo := NewRepository(privateDir(t)) | 583 | repo := NewRepository(privateDir(t)) | ||
| 584 | sessions := filepath.Join(repo.dir, repositorySessionsDir) | 584 | sessions := filepath.Join(repo.dir, repositorySessionsDir) | ||
| 585 | - | for i := 0; i < maintenanceBatch+1; i++ { | 585 | + | for i := range maintenanceBatch + 1 { |
| 586 | generations := filepath.Join(sessions, fmt.Sprintf("named-%03d", i), repositoryGenerations) | 586 | generations := filepath.Join(sessions, fmt.Sprintf("named-%03d", i), repositoryGenerations) | ||
| 587 | if err := os.MkdirAll(generations, 0o700); err != nil { | 587 | if err := os.MkdirAll(generations, 0o700); err != nil { | ||
| 588 | t.Fatal(err) | 588 | t.Fatal(err) | ||
| 589 | } | 589 | } | ||
| 590 | - | for j := 0; j < maintenanceBatch+1; j++ { | 590 | + | for j := range maintenanceBatch + 1 { |
| 591 | if err := os.WriteFile(filepath.Join(generations, fmt.Sprintf("unclassified-%03d", j)), []byte("state"), 0o600); err != nil { | 591 | if err := os.WriteFile(filepath.Join(generations, fmt.Sprintf("unclassified-%03d", j)), []byte("state"), 0o600); err != nil { | ||
| 592 | t.Fatal(err) | 592 | t.Fatal(err) | ||
| 593 | } | 593 | } | ||
| @@ -606,14 +606,14 @@ func TestRepositoryMaintainBoundsQueuedShardNames(t *testing.T) { | |||||
| 606 | repo := NewRepository(privateDir(t)) | 606 | repo := NewRepository(privateDir(t)) | ||
| 607 | key := legacyIncarnationID("named").String() | 607 | key := legacyIncarnationID("named").String() | ||
| 608 | root := filepath.Join(repo.legacySessionPath(key), repositoryObjectsDir) | 608 | root := filepath.Join(repo.legacySessionPath(key), repositoryObjectsDir) | ||
| 609 | - | for i := 0; i < maintenanceBatch*2; i++ { | 609 | + | for i := range maintenanceBatch * 2 { |
| 610 | if err := os.MkdirAll(filepath.Join(root, fmt.Sprintf("%02x", i)), 0o700); err != nil { | 610 | if err := os.MkdirAll(filepath.Join(root, fmt.Sprintf("%02x", i)), 0o700); err != nil { | ||
| 611 | t.Fatal(err) | 611 | t.Fatal(err) | ||
| 612 | } | 612 | } | ||
| 613 | } | 613 | } | ||
| 614 | repo.maintenanceCursors = make(map[string]*maintenanceCursor) | 614 | repo.maintenanceCursors = make(map[string]*maintenanceCursor) | ||
| 615 | state := &sessionMaintenance{marked: make(map[uint64]manifestMaintenance), markDone: true} | 615 | state := &sessionMaintenance{marked: make(map[uint64]manifestMaintenance), markDone: true} | ||
| 616 | - | for i := 0; i < maintenanceBatch+2; i++ { | 616 | + | for range maintenanceBatch + 2 { |
| 617 | budget := maintenanceBatch | 617 | budget := maintenanceBatch | ||
| 618 | if err := repo.sweepSession(context.Background(), key, state, &budget); err != nil { | 618 | if err := repo.sweepSession(context.Background(), key, state, &budget); err != nil { | ||
| 619 | t.Fatal(err) | 619 | t.Fatal(err) | ||
| @@ -629,7 +629,7 @@ func TestRepositoryMaintainResumesCursorAfterDirectoryMutation(t *testing.T) { | |||||
| 629 | if err := os.Mkdir(repo.dir, 0o700); err != nil { | 629 | if err := os.Mkdir(repo.dir, 0o700); err != nil { | ||
| 630 | t.Fatal(err) | 630 | t.Fatal(err) | ||
| 631 | } | 631 | } | ||
| 632 | - | for i := 0; i < maintenanceBatch+1; i++ { | 632 | + | for i := range maintenanceBatch + 1 { |
| 633 | if err := os.WriteFile(filepath.Join(repo.dir, fmt.Sprintf("live-%03d", i)), []byte("live"), 0o600); err != nil { | 633 | if err := os.WriteFile(filepath.Join(repo.dir, fmt.Sprintf("live-%03d", i)), []byte("live"), 0o600); err != nil { | ||
| 634 | t.Fatal(err) | 634 | t.Fatal(err) | ||
| 635 | } | 635 | } | ||
| @@ -646,7 +646,7 @@ func TestRepositoryMaintainResumesCursorAfterDirectoryMutation(t *testing.T) { | |||||
| 646 | if err := os.WriteFile(stale, []byte("stale"), 0o600); err != nil { | 646 | if err := os.WriteFile(stale, []byte("stale"), 0o600); err != nil { | ||
| 647 | t.Fatal(err) | 647 | t.Fatal(err) | ||
| 648 | } | 648 | } | ||
| 649 | - | for i := 0; i < 4; i++ { | 649 | + | for range 4 { |
| 650 | if err := repo.Maintain(context.Background()); err != nil { | 650 | if err := repo.Maintain(context.Background()); err != nil { | ||
| 651 | t.Fatal(err) | 651 | t.Fatal(err) | ||
| 652 | } | 652 | } | ||
| @@ -664,17 +664,17 @@ func TestRepositoryMaintainDoesNotStarveLaterTemporaryEntries(t *testing.T) { | |||||
| 664 | } | 664 | } | ||
| 665 | // The live entries precede the stale entries in directory insertion order. | 665 | // The live entries precede the stale entries in directory insertion order. | ||
| 666 | // A fresh ReadDir from the start on every call would never reach the latter. | 666 | // A fresh ReadDir from the start on every call would never reach the latter. | ||
| 667 | - | for i := 0; i < maintenanceBatch+1; i++ { | 667 | + | for i := range maintenanceBatch + 1 { |
| 668 | if err := os.WriteFile(filepath.Join(repo.dir, "live-"+string(rune('a'+i))), []byte("live"), 0o600); err != nil { | 668 | if err := os.WriteFile(filepath.Join(repo.dir, "live-"+string(rune('a'+i))), []byte("live"), 0o600); err != nil { | ||
| 669 | t.Fatal(err) | 669 | t.Fatal(err) | ||
| 670 | } | 670 | } | ||
| 671 | } | 671 | } | ||
| 672 | - | for i := 0; i < maintenanceBatch+1; i++ { | 672 | + | for i := range maintenanceBatch + 1 { |
| 673 | if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-later-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil { | 673 | if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-later-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil { | ||
| 674 | t.Fatal(err) | 674 | t.Fatal(err) | ||
| 675 | } | 675 | } | ||
| 676 | } | 676 | } | ||
| 677 | - | for i := 0; i < 4; i++ { | 677 | + | for range 4 { |
| 678 | if err := repo.Maintain(context.Background()); err != nil { | 678 | if err := repo.Maintain(context.Background()); err != nil { | ||
| 679 | t.Fatal(err) | 679 | t.Fatal(err) | ||
| 680 | } | 680 | } | ||
| @@ -694,7 +694,7 @@ func TestRepositoryMaintainMarksLargeGenerationSetBeforeSweeping(t *testing.T) { | |||||
| 694 | repo := NewRepository(privateDir(t)) | 694 | repo := NewRepository(privateDir(t)) | ||
| 695 | var newest ports.SnapshotPublication | 695 | var newest ports.SnapshotPublication | ||
| 696 | for generation := uint64(1); generation <= maintenanceBatch+2; generation++ { | 696 | for generation := uint64(1); generation <= maintenanceBatch+2; generation++ { | ||
| 697 | - | publication := repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation))) | 697 | + | publication := repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation)) |
| 698 | if err := repo.Publish(context.Background(), publication); err != nil { | 698 | if err := repo.Publish(context.Background(), publication); err != nil { | ||
| 699 | t.Fatal(err) | 699 | t.Fatal(err) | ||
| 700 | } | 700 | } | ||
| @@ -724,7 +724,7 @@ func TestRepositoryMaintainMarksLargeGenerationSetBeforeSweeping(t *testing.T) { | |||||
| 724 | } | 724 | } | ||
| 725 | } | 725 | } | ||
| 726 | 726 | ||||
| 727 | - | for i := 0; i < 600; i++ { | 727 | + | for i := range 600 { |
| 728 | if err := repo.Maintain(context.Background()); err != nil { | 728 | if err := repo.Maintain(context.Background()); err != nil { | ||
| 729 | t.Fatal(err) | 729 | t.Fatal(err) | ||
| 730 | } | 730 | } | ||
| @@ -747,7 +747,7 @@ func TestRepositoryMaintainUsesFixedRemovalBatch(t *testing.T) { | |||||
| 747 | if err := os.Mkdir(repo.dir, 0o700); err != nil { | 747 | if err := os.Mkdir(repo.dir, 0o700); err != nil { | ||
| 748 | t.Fatal(err) | 748 | t.Fatal(err) | ||
| 749 | } | 749 | } | ||
| 750 | - | for i := 0; i < maintenanceBatch+1; i++ { | 750 | + | for i := range maintenanceBatch + 1 { |
| 751 | if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil { | 751 | if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil { | ||
| 752 | t.Fatal(err) | 752 | t.Fatal(err) | ||
| 753 | } | 753 | } | ||
| @@ -774,7 +774,7 @@ func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) { | |||||
| 774 | } | 774 | } | ||
| 775 | 775 | ||||
| 776 | removed := make(map[string]int) | 776 | removed := make(map[string]int) | ||
| 777 | - | for i := 0; i < maintenanceBatch+1; i++ { | 777 | + | for i := range maintenanceBatch + 1 { |
| 778 | path := filepath.Join(repo.dir, repositorySessionsDir, fmt.Sprintf(".deleting-%03d", i)) | 778 | path := filepath.Join(repo.dir, repositorySessionsDir, fmt.Sprintf(".deleting-%03d", i)) | ||
| 779 | if err := os.Mkdir(path, 0o700); err != nil { | 779 | if err := os.Mkdir(path, 0o700); err != nil { | ||
| 780 | t.Fatal(err) | 780 | t.Fatal(err) | ||
| @@ -804,7 +804,7 @@ func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) { | |||||
| 804 | t.Fatal(err) | 804 | t.Fatal(err) | ||
| 805 | } | 805 | } | ||
| 806 | queued := fetched[len(fetched)-1].Name() | 806 | queued := fetched[len(fetched)-1].Name() | ||
| 807 | - | for i := 0; i < maintenanceBatch+4; i++ { | 807 | + | for i := range maintenanceBatch + 4 { |
| 808 | if err := repo.Maintain(context.Background()); err != nil { | 808 | if err := repo.Maintain(context.Background()); err != nil { | ||
| 809 | t.Fatal(err) | 809 | t.Fatal(err) | ||
| 810 | } | 810 | } | ||
| @@ -818,7 +818,7 @@ func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) { | |||||
| 818 | t.Fatalf("unread session %q = %v, want queued entry to run first", unread[0].Name(), err) | 818 | t.Fatalf("unread session %q = %v, want queued entry to run first", unread[0].Name(), err) | ||
| 819 | } | 819 | } | ||
| 820 | 820 | ||||
| 821 | - | for i := 0; i < maintenanceBatch+4; i++ { | 821 | + | for range maintenanceBatch + 4 { |
| 822 | if err := repo.Maintain(context.Background()); err != nil { | 822 | if err := repo.Maintain(context.Background()); err != nil { | ||
| 823 | t.Fatal(err) | 823 | t.Fatal(err) | ||
| 824 | } | 824 | } | ||
| @@ -876,7 +876,7 @@ func TestRepositoryMaintainClassifiesUnpublishedGenerationsBeforeSweep(t *testin | |||||
| 876 | key := legacyIncarnationID("named").String() | 876 | key := legacyIncarnationID("named").String() | ||
| 877 | publications := make([]ports.SnapshotPublication, 0, maintenanceBatch+2) | 877 | publications := make([]ports.SnapshotPublication, 0, maintenanceBatch+2) | ||
| 878 | for generation := uint64(1); generation <= maintenanceBatch+2; generation++ { | 878 | for generation := uint64(1); generation <= maintenanceBatch+2; generation++ { | ||
| 879 | - | publication := repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation))) | 879 | + | publication := repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation)) |
| 880 | if err := repo.Publish(context.Background(), publication); err != nil { | 880 | if err := repo.Publish(context.Background(), publication); err != nil { | ||
| 881 | t.Fatal(err) | 881 | t.Fatal(err) | ||
| 882 | } | 882 | } | ||
| @@ -904,7 +904,7 @@ func TestRepositoryMaintainClassifiesUnpublishedGenerationsBeforeSweep(t *testin | |||||
| 904 | } | 904 | } | ||
| 905 | } | 905 | } | ||
| 906 | 906 | ||||
| 907 | - | for i := 0; i < 600; i++ { | 907 | + | for i := range 600 { |
| 908 | if err := repo.Maintain(context.Background()); err != nil { | 908 | if err := repo.Maintain(context.Background()); err != nil { | ||
| 909 | t.Fatal(err) | 909 | t.Fatal(err) | ||
| 910 | } | 910 | } | ||
| @@ -928,7 +928,7 @@ func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) { | |||||
| 928 | key := legacyIncarnationID("named").String() | 928 | key := legacyIncarnationID("named").String() | ||
| 929 | first := publicationWithTailShard(t, "named", 1, "ff") | 929 | first := publicationWithTailShard(t, "named", 1, "ff") | ||
| 930 | stalePaths := make([]string, 0, maintenanceBatch+1) | 930 | stalePaths := make([]string, 0, maintenanceBatch+1) | ||
| 931 | - | for i := 0; i < maintenanceBatch+1; i++ { | 931 | + | for i := range maintenanceBatch + 1 { |
| 932 | digest := digestInShard(t, "00", i) | 932 | digest := digestInShard(t, "00", i) | ||
| 933 | path := repo.legacyObjectPath(key, digest) | 933 | path := repo.legacyObjectPath(key, digest) | ||
| 934 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 934 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | ||
| @@ -967,7 +967,7 @@ func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) { | |||||
| 967 | } | 967 | } | ||
| 968 | repo.hooks.beforeHeadWrite = nil | 968 | repo.hooks.beforeHeadWrite = nil | ||
| 969 | 969 | ||||
| 970 | - | for i := 0; i < 20; i++ { | 970 | + | for i := range 20 { |
| 971 | if err := repo.Maintain(context.Background()); err != nil { | 971 | if err := repo.Maintain(context.Background()); err != nil { | ||
| 972 | t.Fatal(err) | 972 | t.Fatal(err) | ||
| 973 | } | 973 | } | ||
| @@ -991,8 +991,8 @@ func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) { | |||||
| 991 | 991 | ||||
| 992 | func publicationWithTailShard(t *testing.T, name string, generation uint64, shard string) ports.SnapshotPublication { | 992 | func publicationWithTailShard(t *testing.T, name string, generation uint64, shard string) ports.SnapshotPublication { | ||
| 993 | t.Helper() | 993 | t.Helper() | ||
| 994 | - | for i := 0; i < 4096; i++ { | 994 | + | for i := range 4096 { |
| 995 | - | publication := repositoryPublication(t, name, generation, []byte(fmt.Sprintf("state-%d-%d", generation, i))) | 995 | + | publication := repositoryPublication(t, name, generation, fmt.Appendf(nil, "state-%d-%d", generation, i)) |
| 996 | if fmt.Sprintf("%02x", publication.Objects[0].Digest[0]) == shard { | 996 | if fmt.Sprintf("%02x", publication.Objects[0].Digest[0]) == shard { | ||
| 997 | return publication | 997 | return publication | ||
| 998 | } | 998 | } | ||
| @@ -1003,8 +1003,8 @@ func publicationWithTailShard(t *testing.T, name string, generation uint64, shar | |||||
| 1003 | 1003 | ||||
| 1004 | func digestInShard(t *testing.T, shard string, n int) ports.SnapshotDigest { | 1004 | func digestInShard(t *testing.T, shard string, n int) ports.SnapshotDigest { | ||
| 1005 | t.Helper() | 1005 | t.Helper() | ||
| 1006 | - | for i := 0; i < 4096; i++ { | 1006 | + | for i := range 4096 { |
| 1007 | - | digest := sha256.Sum256([]byte(fmt.Sprintf("stale-%d-%d", n, i))) | 1007 | + | digest := sha256.Sum256(fmt.Appendf(nil, "stale-%d-%d", n, i)) |
| 1008 | if fmt.Sprintf("%02x", digest[0]) == shard { | 1008 | if fmt.Sprintf("%02x", digest[0]) == shard { | ||
| 1009 | return digest | 1009 | return digest | ||
| 1010 | } | 1010 | } | ||
| @@ -1068,8 +1068,8 @@ func TestMaintenanceBudgetAdmitsPayloadsBeforeRead(t *testing.T) { | |||||
| 1068 | var reads []string | 1068 | var reads []string | ||
| 1069 | repo.hooks.beforeMaintenancePayloadRead = func(path string) { reads = append(reads, path) } | 1069 | repo.hooks.beforeMaintenancePayloadRead = func(path string) { reads = append(reads, path) } | ||
| 1070 | done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | 1070 | done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | ||
| 1071 | - | if err != nil { | 1071 | + | if !errors.Is(err, ErrMaintenanceBudgetTooSmall) { |
| 1072 | - | t.Fatal(err) | 1072 | + | t.Fatalf("undersized manifest budget error = %v, want ErrMaintenanceBudgetTooSmall", err) |
| 1073 | } | 1073 | } | ||
| 1074 | if done || len(reads) != 0 { | 1074 | if done || len(reads) != 0 { | ||
| 1075 | t.Fatalf("undersized manifest budget: done=%v payload reads=%v", done, reads) | 1075 | t.Fatalf("undersized manifest budget: done=%v payload reads=%v", done, reads) | ||
| @@ -1082,8 +1082,8 @@ func TestMaintenanceBudgetAdmitsPayloadsBeforeRead(t *testing.T) { | |||||
| 1082 | } | 1082 | } | ||
| 1083 | reads = nil | 1083 | reads = nil | ||
| 1084 | done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: uint64(info.Size())}) | 1084 | done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: uint64(info.Size())}) | ||
| 1085 | - | if err != nil { | 1085 | + | if !errors.Is(err, ErrMaintenanceBudgetTooSmall) { |
| 1086 | - | t.Fatal(err) | 1086 | + | t.Fatalf("manifest-only budget error = %v, want ErrMaintenanceBudgetTooSmall", err) |
| 1087 | } | 1087 | } | ||
| 1088 | if done { | 1088 | if done { | ||
| 1089 | t.Fatal("manifest-only budget unexpectedly admitted the generation objects") | 1089 | t.Fatal("manifest-only budget unexpectedly admitted the generation objects") | ||
| @@ -1157,7 +1157,7 @@ func TestReconcileDoesNotHoldMaintenanceLockDuringPayloadRead(t *testing.T) { | |||||
| 1157 | if err != nil { | 1157 | if err != nil { | ||
| 1158 | t.Fatal(err) | 1158 | t.Fatal(err) | ||
| 1159 | } | 1159 | } | ||
| 1160 | - | case <-time.After(time.Second): | 1160 | + | case <-time.After(remainingTestTime(t) / 2): |
| 1161 | t.Fatal("maintenance lock remained held during reconciliation payload I/O") | 1161 | t.Fatal("maintenance lock remained held during reconciliation payload I/O") | ||
| 1162 | } | 1162 | } | ||
| 1163 | releaseOnce.Do(func() { close(release) }) | 1163 | releaseOnce.Do(func() { close(release) }) | ||
| @@ -1166,6 +1166,80 @@ func TestReconcileDoesNotHoldMaintenanceLockDuringPayloadRead(t *testing.T) { | |||||
| 1166 | } | 1166 | } | ||
| 1167 | } | 1167 | } | ||
| 1168 | 1168 | ||||
| 1169 | + | func TestMaintainSessionYieldsOnEmptyNonDoneObjectShardBatch(t *testing.T) { | |||
| 1170 | + | repo := NewRepository(privateDir(t)) | |||
| 1171 | + | publication := publishMaintenanceGenerations(t, repo, "empty-shard-batch", 1)[0] | |||
| 1172 | + | plan := ports.RetentionPlan{IncarnationID: publication.IncarnationID} | |||
| 1173 | + | key := publication.IncarnationID.String() | |||
| 1174 | + | token := retentionToken(plan) | |||
| 1175 | + | root := filepath.Join(repo.sessionPath(plan.IncarnationID), repositoryObjectsDir) | |||
| 1176 | + | purpose := "retention-shards:" + key + ":" + token | |||
| 1177 | + | state := &retentionMaintenance{ | |||
| 1178 | + | token: token, | |||
| 1179 | + | cursors: map[string]*maintenanceCursor{maintenanceCursorID(root, purpose): {pending: []maintenanceDirEntry{}}}, | |||
| 1180 | + | references: make(map[ports.SnapshotDigest]struct{}), | |||
| 1181 | + | validated: true, | |||
| 1182 | + | manifestsDone: true, | |||
| 1183 | + | } | |||
| 1184 | + | repo.retentionSessions[key] = state | |||
| 1185 | + | ||||
| 1186 | + | done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 1, Bytes: 1024}) | |||
| 1187 | + | if err != nil || done { | |||
| 1188 | + | t.Fatalf("empty non-done shard batch: done=%v err=%v", done, err) | |||
| 1189 | + | } | |||
| 1190 | + | if state.objectRootDone { | |||
| 1191 | + | t.Fatal("empty non-done shard batch advanced object-root completion") | |||
| 1192 | + | } | |||
| 1193 | + | } | |||
| 1194 | + | ||||
| 1195 | + | func TestMaintainSessionDoesNotHoldMaintenanceLockDuringPayloadIO(t *testing.T) { | |||
| 1196 | + | repo := NewRepository(privateDir(t)) | |||
| 1197 | + | active := publishMaintenanceGenerations(t, repo, "active-lock", 1) | |||
| 1198 | + | other := publishMaintenanceGenerations(t, repo, "other-lock", 1) | |||
| 1199 | + | plan := ports.RetentionPlan{IncarnationID: active[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(active[0])}} | |||
| 1200 | + | ||||
| 1201 | + | entered := make(chan struct{}) | |||
| 1202 | + | release := make(chan struct{}) | |||
| 1203 | + | var once sync.Once | |||
| 1204 | + | repo.hooks.beforeMaintenancePayloadRead = func(string) { | |||
| 1205 | + | once.Do(func() { close(entered) }) | |||
| 1206 | + | <-release | |||
| 1207 | + | } | |||
| 1208 | + | activeDone := make(chan error, 1) | |||
| 1209 | + | go func() { | |||
| 1210 | + | _, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20}) | |||
| 1211 | + | activeDone <- err | |||
| 1212 | + | }() | |||
| 1213 | + | <-entered | |||
| 1214 | + | ||||
| 1215 | + | pinDone := make(chan error, 1) | |||
| 1216 | + | go func() { | |||
| 1217 | + | _, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: other[0].IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | |||
| 1218 | + | pinDone <- err | |||
| 1219 | + | }() | |||
| 1220 | + | select { | |||
| 1221 | + | case err := <-pinDone: | |||
| 1222 | + | if err != nil { | |||
| 1223 | + | t.Fatal(err) | |||
| 1224 | + | } | |||
| 1225 | + | case <-time.After(remainingTestTime(t) / 2): | |||
| 1226 | + | close(release) | |||
| 1227 | + | t.Fatal("unrelated PinAll was blocked by retention payload I/O") | |||
| 1228 | + | } | |||
| 1229 | + | close(release) | |||
| 1230 | + | if err := <-activeDone; err != nil { | |||
| 1231 | + | t.Fatal(err) | |||
| 1232 | + | } | |||
| 1233 | + | } | |||
| 1234 | + | ||||
| 1235 | + | func remainingTestTime(t *testing.T) time.Duration { | |||
| 1236 | + | t.Helper() | |||
| 1237 | + | if deadline, ok := t.Deadline(); ok { | |||
| 1238 | + | return time.Until(deadline) | |||
| 1239 | + | } | |||
| 1240 | + | return 30 * time.Second | |||
| 1241 | + | } | |||
| 1242 | + | ||||
| 1169 | func TestPerSessionBudgetIsolation(t *testing.T) { | 1243 | func TestPerSessionBudgetIsolation(t *testing.T) { | ||
| 1170 | repo := NewRepository(privateDir(t)) | 1244 | repo := NewRepository(privateDir(t)) | ||
| 1171 | large := publishMaintenanceGenerations(t, repo, "large", 4) | 1245 | large := publishMaintenanceGenerations(t, repo, "large", 4) | ||
| @@ -1174,8 +1248,8 @@ func TestPerSessionBudgetIsolation(t *testing.T) { | |||||
| 1174 | smallPlan := ports.RetentionPlan{IncarnationID: small[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(small[1])}} | 1248 | smallPlan := ports.RetentionPlan{IncarnationID: small[0].IncarnationID, Keep: []ports.CheckpointRef{checkpointRefForPublication(small[1])}} | ||
| 1175 | 1249 | ||||
| 1176 | done, err := repo.MaintainSession(context.Background(), largePlan, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | 1250 | done, err := repo.MaintainSession(context.Background(), largePlan, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | ||
| 1177 | - | if err != nil { | 1251 | + | if !errors.Is(err, ErrMaintenanceBudgetTooSmall) { |
| 1178 | - | t.Fatal(err) | 1252 | + | t.Fatalf("large session error = %v, want ErrMaintenanceBudgetTooSmall", err) |
| 1179 | } | 1253 | } | ||
| 1180 | if done { | 1254 | if done { | ||
| 1181 | t.Fatal("large session unexpectedly exhausted no budget") | 1255 | t.Fatal("large session unexpectedly exhausted no budget") | ||
M internal/adapters/snapshot/migration_v0.go +12 -18
| @@ -45,32 +45,29 @@ func decodeManifestV1(data []byte) (legacyManifestV1, error) { | |||
| 45 | 45 | if err != nil { | |
| 46 | 46 | return legacyManifestV1{}, err | |
| 47 | 47 | } | |
| 48 | + | return legacyObjectsOf(manifest), nil | |
| 49 | + | } | |
| 50 | + | ||
| 51 | + | func legacyObjectsOf(manifest codec.Manifest) legacyManifestV1 { | |
| 48 | 52 | seen := make(map[ports.SnapshotDigest]struct{}) | |
| 49 | 53 | objects := make([]legacyObjectV1, 0) | |
| 50 | - | add := func(ref codec.ObjectRef) error { | |
| 54 | + | add := func(ref codec.ObjectRef) { | |
| 51 | 55 | if _, ok := seen[ref.Digest]; ok { | |
| 52 | - | return nil | |
| 56 | + | return | |
| 53 | 57 | } | |
| 54 | 58 | seen[ref.Digest] = struct{}{} | |
| 55 | 59 | objects = append(objects, legacyObjectV1{Kind: uint8(ref.Kind), Digest: ref.Digest, Size: uint64(ref.Size)}) | |
| 56 | - | return nil | |
| 57 | 60 | } | |
| 58 | 61 | for _, tab := range manifest.Tabs { | |
| 59 | 62 | for _, pane := range tab.Panes { | |
| 60 | 63 | for _, ref := range pane.Sealed { | |
| 61 | - | if err := add(ref); err != nil { | |
| 62 | - | return legacyManifestV1{}, err | |
| 63 | - | } | |
| 64 | - | } | |
| 65 | - | if err := add(pane.Tail); err != nil { | |
| 66 | - | return legacyManifestV1{}, err | |
| 67 | - | } | |
| 68 | - | if err := add(pane.Visible); err != nil { | |
| 69 | - | return legacyManifestV1{}, err | |
| 64 | + | add(ref) | |
| 70 | 65 | } | |
| 66 | + | add(pane.Tail) | |
| 67 | + | add(pane.Visible) | |
| 71 | 68 | } | |
| 72 | 69 | } | |
| 73 | - | return legacyManifestV1{Name: manifest.Name, Generation: manifest.Generation, Objects: objects}, nil | |
| 70 | + | return legacyManifestV1{Name: manifest.Name, Generation: manifest.Generation, Objects: objects} | |
| 74 | 71 | } | |
| 75 | 72 | ||
| 76 | 73 | func upgradeManifestV1(data []byte, id domain.IncarnationID) (codec.Manifest, []byte, error) { | |
| @@ -237,10 +234,7 @@ func (r *Repository) MigrateV1Checkpoint(ctx context.Context, req ports.Snapshot | |||
| 237 | 234 | if err != nil { | |
| 238 | 235 | return domain.CheckpointRef{}, uncertainLegacyError("decode manifest", err) | |
| 239 | 236 | } | |
| 240 | - | decoded, err := decodeManifestV1(data) | |
| 241 | - | if err != nil { | |
| 242 | - | return domain.CheckpointRef{}, uncertainLegacyError("decode manifest objects", err) | |
| 243 | - | } | |
| 237 | + | legacy := legacyObjectsOf(manifest) | |
| 244 | 238 | if manifest.Name != req.LegacyName || manifest.Generation != req.LegacyRef.Generation { | |
| 245 | 239 | return domain.CheckpointRef{}, uncertainLegacyError("manifest identity mismatch", nil) | |
| 246 | 240 | } | |
| @@ -253,7 +247,7 @@ func (r *Repository) MigrateV1Checkpoint(ctx context.Context, req ports.Snapshot | |||
| 253 | 247 | if err := r.ensureSession(req.IncarnationID); err != nil { | |
| 254 | 248 | return domain.CheckpointRef{}, err | |
| 255 | 249 | } | |
| 256 | - | for _, ref := range decoded.Objects { | |
| 250 | + | for _, ref := range legacy.Objects { | |
| 257 | 251 | if err := ctx.Err(); err != nil { | |
| 258 | 252 | return domain.CheckpointRef{}, err | |
| 259 | 253 | } | |
| @@ -45,32 +45,29 @@ func decodeManifestV1(data []byte) (legacyManifestV1, error) { | |||||
| 45 | if err != nil { | 45 | if err != nil { | ||
| 46 | return legacyManifestV1{}, err | 46 | return legacyManifestV1{}, err | ||
| 47 | } | 47 | } | ||
| 48 | + | return legacyObjectsOf(manifest), nil | |||
| 49 | + | } | |||
| 50 | + | ||||
| 51 | + | func legacyObjectsOf(manifest codec.Manifest) legacyManifestV1 { | |||
| 48 | seen := make(map[ports.SnapshotDigest]struct{}) | 52 | seen := make(map[ports.SnapshotDigest]struct{}) | ||
| 49 | objects := make([]legacyObjectV1, 0) | 53 | objects := make([]legacyObjectV1, 0) | ||
| 50 | - | add := func(ref codec.ObjectRef) error { | 54 | + | add := func(ref codec.ObjectRef) { |
| 51 | if _, ok := seen[ref.Digest]; ok { | 55 | if _, ok := seen[ref.Digest]; ok { | ||
| 52 | - | return nil | 56 | + | return |
| 53 | } | 57 | } | ||
| 54 | seen[ref.Digest] = struct{}{} | 58 | seen[ref.Digest] = struct{}{} | ||
| 55 | objects = append(objects, legacyObjectV1{Kind: uint8(ref.Kind), Digest: ref.Digest, Size: uint64(ref.Size)}) | 59 | objects = append(objects, legacyObjectV1{Kind: uint8(ref.Kind), Digest: ref.Digest, Size: uint64(ref.Size)}) | ||
| 56 | - | return nil | |||
| 57 | } | 60 | } | ||
| 58 | for _, tab := range manifest.Tabs { | 61 | for _, tab := range manifest.Tabs { | ||
| 59 | for _, pane := range tab.Panes { | 62 | for _, pane := range tab.Panes { | ||
| 60 | for _, ref := range pane.Sealed { | 63 | for _, ref := range pane.Sealed { | ||
| 61 | - | if err := add(ref); err != nil { | 64 | + | add(ref) |
| 62 | - | return legacyManifestV1{}, err | |||
| 63 | - | } | |||
| 64 | - | } | |||
| 65 | - | if err := add(pane.Tail); err != nil { | |||
| 66 | - | return legacyManifestV1{}, err | |||
| 67 | - | } | |||
| 68 | - | if err := add(pane.Visible); err != nil { | |||
| 69 | - | return legacyManifestV1{}, err | |||
| 70 | } | 65 | } | ||
| 66 | + | add(pane.Tail) | |||
| 67 | + | add(pane.Visible) | |||
| 71 | } | 68 | } | ||
| 72 | } | 69 | } | ||
| 73 | - | return legacyManifestV1{Name: manifest.Name, Generation: manifest.Generation, Objects: objects}, nil | 70 | + | return legacyManifestV1{Name: manifest.Name, Generation: manifest.Generation, Objects: objects} |
| 74 | } | 71 | } | ||
| 75 | 72 | ||||
| 76 | func upgradeManifestV1(data []byte, id domain.IncarnationID) (codec.Manifest, []byte, error) { | 73 | func upgradeManifestV1(data []byte, id domain.IncarnationID) (codec.Manifest, []byte, error) { | ||
| @@ -237,10 +234,7 @@ func (r *Repository) MigrateV1Checkpoint(ctx context.Context, req ports.Snapshot | |||||
| 237 | if err != nil { | 234 | if err != nil { | ||
| 238 | return domain.CheckpointRef{}, uncertainLegacyError("decode manifest", err) | 235 | return domain.CheckpointRef{}, uncertainLegacyError("decode manifest", err) | ||
| 239 | } | 236 | } | ||
| 240 | - | decoded, err := decodeManifestV1(data) | 237 | + | legacy := legacyObjectsOf(manifest) |
| 241 | - | if err != nil { | |||
| 242 | - | return domain.CheckpointRef{}, uncertainLegacyError("decode manifest objects", err) | |||
| 243 | - | } | |||
| 244 | if manifest.Name != req.LegacyName || manifest.Generation != req.LegacyRef.Generation { | 238 | if manifest.Name != req.LegacyName || manifest.Generation != req.LegacyRef.Generation { | ||
| 245 | return domain.CheckpointRef{}, uncertainLegacyError("manifest identity mismatch", nil) | 239 | return domain.CheckpointRef{}, uncertainLegacyError("manifest identity mismatch", nil) | ||
| 246 | } | 240 | } | ||
| @@ -253,7 +247,7 @@ func (r *Repository) MigrateV1Checkpoint(ctx context.Context, req ports.Snapshot | |||||
| 253 | if err := r.ensureSession(req.IncarnationID); err != nil { | 247 | if err := r.ensureSession(req.IncarnationID); err != nil { | ||
| 254 | return domain.CheckpointRef{}, err | 248 | return domain.CheckpointRef{}, err | ||
| 255 | } | 249 | } | ||
| 256 | - | for _, ref := range decoded.Objects { | 250 | + | for _, ref := range legacy.Objects { |
| 257 | if err := ctx.Err(); err != nil { | 251 | if err := ctx.Err(); err != nil { | ||
| 258 | return domain.CheckpointRef{}, err | 252 | return domain.CheckpointRef{}, err | ||
| 259 | } | 253 | } | ||
M internal/adapters/snapshot/migration_v0_test.go +26 -27
| @@ -24,7 +24,18 @@ func legacyManifestBytes(name string, generation uint64) []byte { | |||
| 24 | 24 | body = binary.BigEndian.AppendUint64(body, 1) | |
| 25 | 25 | body = binary.BigEndian.AppendUint16(body, 0) | |
| 26 | 26 | body = binary.BigEndian.AppendUint16(body, 0) | |
| 27 | - | out := make([]byte, 16, 16+len(body)) | |
| 27 | + | return legacyEnvelope(body) | |
| 28 | + | } | |
| 29 | + | ||
| 30 | + | func legacyBodyFromModern(modern []byte) []byte { | |
| 31 | + | body := append([]byte(nil), modern[legacyManifestHeaderSize:]...) | |
| 32 | + | const generationSize = 8 | |
| 33 | + | modernOnly := generationSize + len(domain.IncarnationID{}) + 1 // incarnation and parent flag follow generation in v2. | |
| 34 | + | return append(body[:generationSize], body[modernOnly:]...) | |
| 35 | + | } | |
| 36 | + | ||
| 37 | + | func legacyEnvelope(body []byte) []byte { | |
| 38 | + | out := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body)) | |
| 28 | 39 | copy(out, "VEVM") | |
| 29 | 40 | binary.BigEndian.PutUint16(out[4:6], 1) | |
| 30 | 41 | binary.BigEndian.PutUint32(out[8:12], uint32(len(body))) | |
| @@ -57,14 +68,7 @@ func TestLegacyManifestV1DeduplicatesObjectsPreservingFirst(t *testing.T) { | |||
| 57 | 68 | manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Sealed: []codec.ObjectRef{chunkRef, chunkRef}, Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}} | |
| 58 | 69 | modern, err := codec.MarshalManifest(manifest) | |
| 59 | 70 | require.NoError(t, err) | |
| 60 | - | body := append([]byte(nil), modern[legacyManifestHeaderSize:]...) | |
| 61 | - | body = append(body[:8], body[25:]...) | |
| 62 | - | legacy := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body)) | |
| 63 | - | copy(legacy, "VEVM") | |
| 64 | - | binary.BigEndian.PutUint16(legacy[4:6], 1) | |
| 65 | - | binary.BigEndian.PutUint32(legacy[8:12], uint32(len(body))) | |
| 66 | - | binary.BigEndian.PutUint32(legacy[12:16], crc32.ChecksumIEEE(body)) | |
| 67 | - | legacy = append(legacy, body...) | |
| 71 | + | legacy := legacyEnvelope(legacyBodyFromModern(modern)) | |
| 68 | 72 | ||
| 69 | 73 | decoded, err := decodeManifestV1(legacy) | |
| 70 | 74 | require.NoError(t, err) | |
| @@ -98,8 +102,8 @@ func TestHasLegacyState(t *testing.T) { | |||
| 98 | 102 | }, false, true}, | |
| 99 | 103 | } { | |
| 100 | 104 | t.Run(tc.name, func(t *testing.T) { | |
| 101 | - | dir := t.TempDir() | |
| 102 | - | require.NoError(t, os.Chmod(dir, 0o700)) | |
| 105 | + | dir := privateDir(t) | |
| 106 | + | require.NoError(t, NewRepository(dir).ensurePrivateDirectory(dir)) | |
| 103 | 107 | tc.setup(dir) | |
| 104 | 108 | got, err := NewRepository(dir).HasLegacyState(context.Background()) | |
| 105 | 109 | if tc.budget { | |
| @@ -139,14 +143,7 @@ func TestMigrateV1CheckpointValidatesObjectEnvelopeAndKind(t *testing.T) { | |||
| 139 | 143 | manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: object.Digest, Size: uint32(len(object.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}} | |
| 140 | 144 | modern, err := codec.MarshalManifest(manifest) | |
| 141 | 145 | require.NoError(t, err) | |
| 142 | - | body := append([]byte(nil), modern[legacyManifestHeaderSize:]...) | |
| 143 | - | body = append(body[:8], body[25:]...) | |
| 144 | - | legacyManifest := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body)) | |
| 145 | - | copy(legacyManifest, "VEVM") | |
| 146 | - | binary.BigEndian.PutUint16(legacyManifest[4:6], 1) | |
| 147 | - | binary.BigEndian.PutUint32(legacyManifest[8:12], uint32(len(body))) | |
| 148 | - | binary.BigEndian.PutUint32(legacyManifest[12:16], crc32.ChecksumIEEE(body)) | |
| 149 | - | legacyManifest = append(legacyManifest, body...) | |
| 146 | + | legacyManifest := legacyEnvelope(legacyBodyFromModern(modern)) | |
| 150 | 147 | key := sessionKey("work") | |
| 151 | 148 | for _, stored := range []ports.SnapshotObject{object, visible} { | |
| 152 | 149 | require.NoError(t, os.MkdirAll(filepath.Dir(repo.legacyObjectPath(key, stored.Digest)), 0o700)) | |
| @@ -165,23 +162,25 @@ func TestMigrateV1CheckpointValidatesObjectEnvelopeAndKind(t *testing.T) { | |||
| 165 | 162 | } | |
| 166 | 163 | ||
| 167 | 164 | func TestMigrateV1CheckpointAcceptsLegacyNilParentBeyondGenerationOne(t *testing.T) { | |
| 168 | - | dir := t.TempDir() | |
| 169 | - | require.NoError(t, os.Chmod(dir, 0o700)) | |
| 165 | + | dir := privateDir(t) | |
| 170 | 166 | repo := NewRepository(dir) | |
| 167 | + | require.NoError(t, repo.ensurePrivateDirectory(dir)) | |
| 171 | 168 | name := "work" | |
| 172 | 169 | key := sessionKey(name) | |
| 173 | 170 | generations := filepath.Join(dir, "sessions", key, "generations") | |
| 174 | 171 | require.NoError(t, os.MkdirAll(generations, 0o700)) | |
| 175 | - | for i := 1; i <= 5000; i++ { | |
| 172 | + | // Place the authoritative generation just beyond the bounded legacy scan. | |
| 173 | + | highestGeneration := maxDirectoryTraversalEntries + 1 | |
| 174 | + | for i := 1; i < highestGeneration; i++ { | |
| 176 | 175 | require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(uint64(i))), []byte("stale"), 0o600)) | |
| 177 | 176 | } | |
| 178 | - | legacy := legacyManifestBytes(name, 5001) | |
| 177 | + | legacy := legacyManifestBytes(name, uint64(highestGeneration)) | |
| 179 | 178 | legacyDigest := sha256.Sum256(legacy) | |
| 180 | - | require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(5001)), legacy, 0o600)) | |
| 181 | - | require.NoError(t, os.WriteFile(filepath.Join(dir, "sessions", key, "HEAD"), marshalHead(5001, legacyDigest), 0o600)) | |
| 179 | + | require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(uint64(highestGeneration))), legacy, 0o600)) | |
| 180 | + | require.NoError(t, os.WriteFile(filepath.Join(dir, "sessions", key, "HEAD"), marshalHead(uint64(highestGeneration), legacyDigest), 0o600)) | |
| 182 | 181 | ref, err := repo.ReadLegacyHEAD(context.Background(), name) | |
| 183 | 182 | require.NoError(t, err) | |
| 184 | - | require.Equal(t, uint64(5001), ref.Generation) | |
| 183 | + | require.Equal(t, uint64(highestGeneration), ref.Generation) | |
| 185 | 184 | id := domain.IncarnationID{1} | |
| 186 | 185 | migrated, err := repo.MigrateV1Checkpoint(context.Background(), ports.SnapshotMigrationRequest{LegacyName: name, IncarnationID: id, LegacyRef: ref}) | |
| 187 | 186 | require.NoError(t, err) | |
| @@ -191,7 +190,7 @@ func TestMigrateV1CheckpointAcceptsLegacyNilParentBeyondGenerationOne(t *testing | |||
| 191 | 190 | require.NoError(t, err) | |
| 192 | 191 | require.Equal(t, id, manifest.IncarnationID) | |
| 193 | 192 | require.Nil(t, manifest.ParentCheckpoint) | |
| 194 | - | for i := 1; i <= 5000; i++ { | |
| 193 | + | for i := 1; i < highestGeneration; i++ { | |
| 195 | 194 | require.FileExists(t, filepath.Join(generations, generationFilename(uint64(i)))) | |
| 196 | 195 | } | |
| 197 | 196 | } | |
| @@ -24,7 +24,18 @@ func legacyManifestBytes(name string, generation uint64) []byte { | |||||
| 24 | body = binary.BigEndian.AppendUint64(body, 1) | 24 | body = binary.BigEndian.AppendUint64(body, 1) | ||
| 25 | body = binary.BigEndian.AppendUint16(body, 0) | 25 | body = binary.BigEndian.AppendUint16(body, 0) | ||
| 26 | body = binary.BigEndian.AppendUint16(body, 0) | 26 | body = binary.BigEndian.AppendUint16(body, 0) | ||
| 27 | - | out := make([]byte, 16, 16+len(body)) | 27 | + | return legacyEnvelope(body) |
| 28 | + | } | |||
| 29 | + | ||||
| 30 | + | func legacyBodyFromModern(modern []byte) []byte { | |||
| 31 | + | body := append([]byte(nil), modern[legacyManifestHeaderSize:]...) | |||
| 32 | + | const generationSize = 8 | |||
| 33 | + | modernOnly := generationSize + len(domain.IncarnationID{}) + 1 // incarnation and parent flag follow generation in v2. | |||
| 34 | + | return append(body[:generationSize], body[modernOnly:]...) | |||
| 35 | + | } | |||
| 36 | + | ||||
| 37 | + | func legacyEnvelope(body []byte) []byte { | |||
| 38 | + | out := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body)) | |||
| 28 | copy(out, "VEVM") | 39 | copy(out, "VEVM") | ||
| 29 | binary.BigEndian.PutUint16(out[4:6], 1) | 40 | binary.BigEndian.PutUint16(out[4:6], 1) | ||
| 30 | binary.BigEndian.PutUint32(out[8:12], uint32(len(body))) | 41 | binary.BigEndian.PutUint32(out[8:12], uint32(len(body))) | ||
| @@ -57,14 +68,7 @@ func TestLegacyManifestV1DeduplicatesObjectsPreservingFirst(t *testing.T) { | |||||
| 57 | manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Sealed: []codec.ObjectRef{chunkRef, chunkRef}, Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}} | 68 | manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Sealed: []codec.ObjectRef{chunkRef, chunkRef}, Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}} | ||
| 58 | modern, err := codec.MarshalManifest(manifest) | 69 | modern, err := codec.MarshalManifest(manifest) | ||
| 59 | require.NoError(t, err) | 70 | require.NoError(t, err) | ||
| 60 | - | body := append([]byte(nil), modern[legacyManifestHeaderSize:]...) | 71 | + | legacy := legacyEnvelope(legacyBodyFromModern(modern)) |
| 61 | - | body = append(body[:8], body[25:]...) | |||
| 62 | - | legacy := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body)) | |||
| 63 | - | copy(legacy, "VEVM") | |||
| 64 | - | binary.BigEndian.PutUint16(legacy[4:6], 1) | |||
| 65 | - | binary.BigEndian.PutUint32(legacy[8:12], uint32(len(body))) | |||
| 66 | - | binary.BigEndian.PutUint32(legacy[12:16], crc32.ChecksumIEEE(body)) | |||
| 67 | - | legacy = append(legacy, body...) | |||
| 68 | 72 | ||||
| 69 | decoded, err := decodeManifestV1(legacy) | 73 | decoded, err := decodeManifestV1(legacy) | ||
| 70 | require.NoError(t, err) | 74 | require.NoError(t, err) | ||
| @@ -98,8 +102,8 @@ func TestHasLegacyState(t *testing.T) { | |||||
| 98 | }, false, true}, | 102 | }, false, true}, | ||
| 99 | } { | 103 | } { | ||
| 100 | t.Run(tc.name, func(t *testing.T) { | 104 | t.Run(tc.name, func(t *testing.T) { | ||
| 101 | - | dir := t.TempDir() | 105 | + | dir := privateDir(t) |
| 102 | - | require.NoError(t, os.Chmod(dir, 0o700)) | 106 | + | require.NoError(t, NewRepository(dir).ensurePrivateDirectory(dir)) |
| 103 | tc.setup(dir) | 107 | tc.setup(dir) | ||
| 104 | got, err := NewRepository(dir).HasLegacyState(context.Background()) | 108 | got, err := NewRepository(dir).HasLegacyState(context.Background()) | ||
| 105 | if tc.budget { | 109 | if tc.budget { | ||
| @@ -139,14 +143,7 @@ func TestMigrateV1CheckpointValidatesObjectEnvelopeAndKind(t *testing.T) { | |||||
| 139 | manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: object.Digest, Size: uint32(len(object.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}} | 143 | manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: object.Digest, Size: uint32(len(object.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}} | ||
| 140 | modern, err := codec.MarshalManifest(manifest) | 144 | modern, err := codec.MarshalManifest(manifest) | ||
| 141 | require.NoError(t, err) | 145 | require.NoError(t, err) | ||
| 142 | - | body := append([]byte(nil), modern[legacyManifestHeaderSize:]...) | 146 | + | legacyManifest := legacyEnvelope(legacyBodyFromModern(modern)) |
| 143 | - | body = append(body[:8], body[25:]...) | |||
| 144 | - | legacyManifest := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body)) | |||
| 145 | - | copy(legacyManifest, "VEVM") | |||
| 146 | - | binary.BigEndian.PutUint16(legacyManifest[4:6], 1) | |||
| 147 | - | binary.BigEndian.PutUint32(legacyManifest[8:12], uint32(len(body))) | |||
| 148 | - | binary.BigEndian.PutUint32(legacyManifest[12:16], crc32.ChecksumIEEE(body)) | |||
| 149 | - | legacyManifest = append(legacyManifest, body...) | |||
| 150 | key := sessionKey("work") | 147 | key := sessionKey("work") | ||
| 151 | for _, stored := range []ports.SnapshotObject{object, visible} { | 148 | for _, stored := range []ports.SnapshotObject{object, visible} { | ||
| 152 | require.NoError(t, os.MkdirAll(filepath.Dir(repo.legacyObjectPath(key, stored.Digest)), 0o700)) | 149 | require.NoError(t, os.MkdirAll(filepath.Dir(repo.legacyObjectPath(key, stored.Digest)), 0o700)) | ||
| @@ -165,23 +162,25 @@ func TestMigrateV1CheckpointValidatesObjectEnvelopeAndKind(t *testing.T) { | |||||
| 165 | } | 162 | } | ||
| 166 | 163 | ||||
| 167 | func TestMigrateV1CheckpointAcceptsLegacyNilParentBeyondGenerationOne(t *testing.T) { | 164 | func TestMigrateV1CheckpointAcceptsLegacyNilParentBeyondGenerationOne(t *testing.T) { | ||
| 168 | - | dir := t.TempDir() | 165 | + | dir := privateDir(t) |
| 169 | - | require.NoError(t, os.Chmod(dir, 0o700)) | |||
| 170 | repo := NewRepository(dir) | 166 | repo := NewRepository(dir) | ||
| 167 | + | require.NoError(t, repo.ensurePrivateDirectory(dir)) | |||
| 171 | name := "work" | 168 | name := "work" | ||
| 172 | key := sessionKey(name) | 169 | key := sessionKey(name) | ||
| 173 | generations := filepath.Join(dir, "sessions", key, "generations") | 170 | generations := filepath.Join(dir, "sessions", key, "generations") | ||
| 174 | require.NoError(t, os.MkdirAll(generations, 0o700)) | 171 | require.NoError(t, os.MkdirAll(generations, 0o700)) | ||
| 175 | - | for i := 1; i <= 5000; i++ { | 172 | + | // Place the authoritative generation just beyond the bounded legacy scan. |
| 173 | + | highestGeneration := maxDirectoryTraversalEntries + 1 | |||
| 174 | + | for i := 1; i < highestGeneration; i++ { | |||
| 176 | require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(uint64(i))), []byte("stale"), 0o600)) | 175 | require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(uint64(i))), []byte("stale"), 0o600)) | ||
| 177 | } | 176 | } | ||
| 178 | - | legacy := legacyManifestBytes(name, 5001) | 177 | + | legacy := legacyManifestBytes(name, uint64(highestGeneration)) |
| 179 | legacyDigest := sha256.Sum256(legacy) | 178 | legacyDigest := sha256.Sum256(legacy) | ||
| 180 | - | require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(5001)), legacy, 0o600)) | 179 | + | require.NoError(t, os.WriteFile(filepath.Join(generations, generationFilename(uint64(highestGeneration))), legacy, 0o600)) |
| 181 | - | require.NoError(t, os.WriteFile(filepath.Join(dir, "sessions", key, "HEAD"), marshalHead(5001, legacyDigest), 0o600)) | 180 | + | require.NoError(t, os.WriteFile(filepath.Join(dir, "sessions", key, "HEAD"), marshalHead(uint64(highestGeneration), legacyDigest), 0o600)) |
| 182 | ref, err := repo.ReadLegacyHEAD(context.Background(), name) | 181 | ref, err := repo.ReadLegacyHEAD(context.Background(), name) | ||
| 183 | require.NoError(t, err) | 182 | require.NoError(t, err) | ||
| 184 | - | require.Equal(t, uint64(5001), ref.Generation) | 183 | + | require.Equal(t, uint64(highestGeneration), ref.Generation) |
| 185 | id := domain.IncarnationID{1} | 184 | id := domain.IncarnationID{1} | ||
| 186 | migrated, err := repo.MigrateV1Checkpoint(context.Background(), ports.SnapshotMigrationRequest{LegacyName: name, IncarnationID: id, LegacyRef: ref}) | 185 | migrated, err := repo.MigrateV1Checkpoint(context.Background(), ports.SnapshotMigrationRequest{LegacyName: name, IncarnationID: id, LegacyRef: ref}) | ||
| 187 | require.NoError(t, err) | 186 | require.NoError(t, err) | ||
| @@ -191,7 +190,7 @@ func TestMigrateV1CheckpointAcceptsLegacyNilParentBeyondGenerationOne(t *testing | |||||
| 191 | require.NoError(t, err) | 190 | require.NoError(t, err) | ||
| 192 | require.Equal(t, id, manifest.IncarnationID) | 191 | require.Equal(t, id, manifest.IncarnationID) | ||
| 193 | require.Nil(t, manifest.ParentCheckpoint) | 192 | require.Nil(t, manifest.ParentCheckpoint) | ||
| 194 | - | for i := 1; i <= 5000; i++ { | 193 | + | for i := 1; i < highestGeneration; i++ { |
| 195 | require.FileExists(t, filepath.Join(generations, generationFilename(uint64(i)))) | 194 | require.FileExists(t, filepath.Join(generations, generationFilename(uint64(i)))) | ||
| 196 | } | 195 | } | ||
| 197 | } | 196 | } | ||
M internal/adapters/snapshot/path.go +18 -2
| @@ -3,7 +3,9 @@ package snapshot | |||
| 3 | 3 | import ( | |
| 4 | 4 | "crypto/sha256" | |
| 5 | 5 | "encoding/hex" | |
| 6 | + | "errors" | |
| 6 | 7 | "fmt" | |
| 8 | + | "os" | |
| 7 | 9 | "path/filepath" | |
| 8 | 10 | "regexp" | |
| 9 | 11 | "strconv" | |
| @@ -27,8 +29,9 @@ func filenameForName(name string) string { | |||
| 27 | 29 | return "@" + hex.EncodeToString(sum[:])[:40] + ".snap" | |
| 28 | 30 | } | |
| 29 | 31 | ||
| 30 | - | // legacyIncarnationID is used only by retired name-keyed compatibility | |
| 31 | - | // operations. New callers must supply the durable catalogue incarnation ID. | |
| 32 | + | // legacyIncarnationID identifies snapshots produced by the short-lived | |
| 33 | + | // incarnation bridge. Established name-keyed repositories use sessionKey. | |
| 34 | + | // New callers must supply the durable catalogue incarnation ID. | |
| 32 | 35 | func legacyIncarnationID(name string) domain.IncarnationID { | |
| 33 | 36 | id := domain.IncarnationID{} | |
| 34 | 37 | digest := sha256.Sum256([]byte(name)) | |
| @@ -61,6 +64,19 @@ func (r *Repository) headPath(id domain.IncarnationID) string { | |||
| 61 | 64 | func (r *Repository) legacySessionPath(key string) string { | |
| 62 | 65 | return filepath.Join(r.dir, repositorySessionsDir, key) | |
| 63 | 66 | } | |
| 67 | + | ||
| 68 | + | // legacyRepositoryKey resolves the established name-keyed namespace first. | |
| 69 | + | // The digest fallback keeps the retired bridge readable without changing any | |
| 70 | + | // durable catalogue incarnation identity. | |
| 71 | + | func (r *Repository) legacyRepositoryKey(name string) (string, error) { | |
| 72 | + | key := sessionKey(name) | |
| 73 | + | if _, err := r.stat(r.legacySessionPath(key)); err == nil { | |
| 74 | + | return key, nil | |
| 75 | + | } else if !errors.Is(err, os.ErrNotExist) { | |
| 76 | + | return "", err | |
| 77 | + | } | |
| 78 | + | return legacyIncarnationID(name).String(), nil | |
| 79 | + | } | |
| 64 | 80 | func (r *Repository) legacyObjectPath(key string, digest ports.SnapshotDigest) string { | |
| 65 | 81 | hexDigest := hex.EncodeToString(digest[:]) | |
| 66 | 82 | return filepath.Join(r.legacySessionPath(key), repositoryObjectsDir, hexDigest[:2], hexDigest) | |
| @@ -3,7 +3,9 @@ package snapshot | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "crypto/sha256" | 4 | "crypto/sha256" | ||
| 5 | "encoding/hex" | 5 | "encoding/hex" | ||
| 6 | + | "errors" | |||
| 6 | "fmt" | 7 | "fmt" | ||
| 8 | + | "os" | |||
| 7 | "path/filepath" | 9 | "path/filepath" | ||
| 8 | "regexp" | 10 | "regexp" | ||
| 9 | "strconv" | 11 | "strconv" | ||
| @@ -27,8 +29,9 @@ func filenameForName(name string) string { | |||||
| 27 | return "@" + hex.EncodeToString(sum[:])[:40] + ".snap" | 29 | return "@" + hex.EncodeToString(sum[:])[:40] + ".snap" | ||
| 28 | } | 30 | } | ||
| 29 | 31 | ||||
| 30 | - | // legacyIncarnationID is used only by retired name-keyed compatibility | 32 | + | // legacyIncarnationID identifies snapshots produced by the short-lived |
| 31 | - | // operations. New callers must supply the durable catalogue incarnation ID. | 33 | + | // incarnation bridge. Established name-keyed repositories use sessionKey. |
| 34 | + | // New callers must supply the durable catalogue incarnation ID. | |||
| 32 | func legacyIncarnationID(name string) domain.IncarnationID { | 35 | func legacyIncarnationID(name string) domain.IncarnationID { | ||
| 33 | id := domain.IncarnationID{} | 36 | id := domain.IncarnationID{} | ||
| 34 | digest := sha256.Sum256([]byte(name)) | 37 | digest := sha256.Sum256([]byte(name)) | ||
| @@ -61,6 +64,19 @@ func (r *Repository) headPath(id domain.IncarnationID) string { | |||||
| 61 | func (r *Repository) legacySessionPath(key string) string { | 64 | func (r *Repository) legacySessionPath(key string) string { | ||
| 62 | return filepath.Join(r.dir, repositorySessionsDir, key) | 65 | return filepath.Join(r.dir, repositorySessionsDir, key) | ||
| 63 | } | 66 | } | ||
| 67 | + | ||||
| 68 | + | // legacyRepositoryKey resolves the established name-keyed namespace first. | |||
| 69 | + | // The digest fallback keeps the retired bridge readable without changing any | |||
| 70 | + | // durable catalogue incarnation identity. | |||
| 71 | + | func (r *Repository) legacyRepositoryKey(name string) (string, error) { | |||
| 72 | + | key := sessionKey(name) | |||
| 73 | + | if _, err := r.stat(r.legacySessionPath(key)); err == nil { | |||
| 74 | + | return key, nil | |||
| 75 | + | } else if !errors.Is(err, os.ErrNotExist) { | |||
| 76 | + | return "", err | |||
| 77 | + | } | |||
| 78 | + | return legacyIncarnationID(name).String(), nil | |||
| 79 | + | } | |||
| 64 | func (r *Repository) legacyObjectPath(key string, digest ports.SnapshotDigest) string { | 80 | func (r *Repository) legacyObjectPath(key string, digest ports.SnapshotDigest) string { | ||
| 65 | hexDigest := hex.EncodeToString(digest[:]) | 81 | hexDigest := hex.EncodeToString(digest[:]) | ||
| 66 | return filepath.Join(r.legacySessionPath(key), repositoryObjectsDir, hexDigest[:2], hexDigest) | 82 | return filepath.Join(r.legacySessionPath(key), repositoryObjectsDir, hexDigest[:2], hexDigest) | ||
M internal/adapters/snapshot/publish_efficiency_test.go +1 -1
| @@ -148,7 +148,7 @@ func largeIncrementalPublications(t testing.TB, name string, count int) (ports.S | |||
| 148 | 148 | sealed := make([]codec.ObjectRef, 0, count) | |
| 149 | 149 | objects := make([]ports.SnapshotObject, 0, count+2) | |
| 150 | 150 | for i := range count { | |
| 151 | - | object, err := codec.MarshalObject(codec.HistoryChunk, []byte(fmt.Sprintf("history-%d", i))) | |
| 151 | + | object, err := codec.MarshalObject(codec.HistoryChunk, fmt.Appendf(nil, "history-%d", i)) | |
| 152 | 152 | if err != nil { | |
| 153 | 153 | t.Fatal(err) | |
| 154 | 154 | } | |
| @@ -148,7 +148,7 @@ func largeIncrementalPublications(t testing.TB, name string, count int) (ports.S | |||||
| 148 | sealed := make([]codec.ObjectRef, 0, count) | 148 | sealed := make([]codec.ObjectRef, 0, count) | ||
| 149 | objects := make([]ports.SnapshotObject, 0, count+2) | 149 | objects := make([]ports.SnapshotObject, 0, count+2) | ||
| 150 | for i := range count { | 150 | for i := range count { | ||
| 151 | - | object, err := codec.MarshalObject(codec.HistoryChunk, []byte(fmt.Sprintf("history-%d", i))) | 151 | + | object, err := codec.MarshalObject(codec.HistoryChunk, fmt.Appendf(nil, "history-%d", i)) |
| 152 | if err != nil { | 152 | if err != nil { | ||
| 153 | t.Fatal(err) | 153 | t.Fatal(err) | ||
| 154 | } | 154 | } | ||
M internal/adapters/snapshot/repository_test.go +4 -6
| @@ -2,7 +2,6 @@ package snapshot | |||
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | - | "crypto/sha256" | |
| 6 | 5 | "testing" | |
| 7 | 6 | ||
| 8 | 7 | "github.com/bnema/vev/internal/domain" | |
| @@ -44,7 +43,7 @@ func TestRepositoryDoesNotRewriteVerifiedImmutableBlob(t *testing.T) { | |||
| 44 | 43 | t.Fatal(err) | |
| 45 | 44 | } | |
| 46 | 45 | manifest.Generation = 2 | |
| 47 | - | parent := &domain.CheckpointRef{Generation: 1, ManifestDigest: codecDigestForTest(pub.Manifest)} | |
| 46 | + | parent := &domain.CheckpointRef{Generation: 1, ManifestDigest: codec.ManifestDigest(pub.Manifest)} | |
| 48 | 47 | manifest.ParentCheckpoint = parent | |
| 49 | 48 | pub.ParentCheckpoint = parent | |
| 50 | 49 | pub.Manifest, err = codec.MarshalManifest(manifest) | |
| @@ -59,15 +58,14 @@ func TestRepositoryDoesNotRewriteVerifiedImmutableBlob(t *testing.T) { | |||
| 59 | 58 | } | |
| 60 | 59 | } | |
| 61 | 60 | ||
| 62 | - | func codecDigestForTest(data []byte) [32]byte { | |
| 63 | - | return sha256.Sum256(data) | |
| 64 | - | } | |
| 65 | - | ||
| 66 | 61 | func repositoryPublicationAfter(t *testing.T, repo *Repository, name string, generation uint64, payload []byte) ports.SnapshotPublication { | |
| 67 | 62 | t.Helper() | |
| 68 | 63 | return publicationWithCurrentParent(t, repo, repositoryPublication(t, name, generation, payload)) | |
| 69 | 64 | } | |
| 70 | 65 | ||
| 66 | + | // publicationWithCurrentParent requires repositories to contain HEAD for every | |
| 67 | + | // publication after generation one. Callers such as prepareHeadStage and | |
| 68 | + | // seedCompletePublication must seed or publish the parent generation first. | |
| 71 | 69 | func publicationWithCurrentParent(t *testing.T, repo *Repository, publication ports.SnapshotPublication) ports.SnapshotPublication { | |
| 72 | 70 | t.Helper() | |
| 73 | 71 | if publication.Generation == 1 { | |
| @@ -2,7 +2,6 @@ package snapshot | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | - | "crypto/sha256" | |||
| 6 | "testing" | 5 | "testing" | ||
| 7 | 6 | ||||
| 8 | "github.com/bnema/vev/internal/domain" | 7 | "github.com/bnema/vev/internal/domain" | ||
| @@ -44,7 +43,7 @@ func TestRepositoryDoesNotRewriteVerifiedImmutableBlob(t *testing.T) { | |||||
| 44 | t.Fatal(err) | 43 | t.Fatal(err) | ||
| 45 | } | 44 | } | ||
| 46 | manifest.Generation = 2 | 45 | manifest.Generation = 2 | ||
| 47 | - | parent := &domain.CheckpointRef{Generation: 1, ManifestDigest: codecDigestForTest(pub.Manifest)} | 46 | + | parent := &domain.CheckpointRef{Generation: 1, ManifestDigest: codec.ManifestDigest(pub.Manifest)} |
| 48 | manifest.ParentCheckpoint = parent | 47 | manifest.ParentCheckpoint = parent | ||
| 49 | pub.ParentCheckpoint = parent | 48 | pub.ParentCheckpoint = parent | ||
| 50 | pub.Manifest, err = codec.MarshalManifest(manifest) | 49 | pub.Manifest, err = codec.MarshalManifest(manifest) | ||
| @@ -59,15 +58,14 @@ func TestRepositoryDoesNotRewriteVerifiedImmutableBlob(t *testing.T) { | |||||
| 59 | } | 58 | } | ||
| 60 | } | 59 | } | ||
| 61 | 60 | ||||
| 62 | - | func codecDigestForTest(data []byte) [32]byte { | |||
| 63 | - | return sha256.Sum256(data) | |||
| 64 | - | } | |||
| 65 | - | ||||
| 66 | func repositoryPublicationAfter(t *testing.T, repo *Repository, name string, generation uint64, payload []byte) ports.SnapshotPublication { | 61 | func repositoryPublicationAfter(t *testing.T, repo *Repository, name string, generation uint64, payload []byte) ports.SnapshotPublication { | ||
| 67 | t.Helper() | 62 | t.Helper() | ||
| 68 | return publicationWithCurrentParent(t, repo, repositoryPublication(t, name, generation, payload)) | 63 | return publicationWithCurrentParent(t, repo, repositoryPublication(t, name, generation, payload)) | ||
| 69 | } | 64 | } | ||
| 70 | 65 | ||||
| 66 | + | // publicationWithCurrentParent requires repositories to contain HEAD for every | |||
| 67 | + | // publication after generation one. Callers such as prepareHeadStage and | |||
| 68 | + | // seedCompletePublication must seed or publish the parent generation first. | |||
| 71 | func publicationWithCurrentParent(t *testing.T, repo *Repository, publication ports.SnapshotPublication) ports.SnapshotPublication { | 69 | func publicationWithCurrentParent(t *testing.T, repo *Repository, publication ports.SnapshotPublication) ports.SnapshotPublication { | ||
| 72 | t.Helper() | 70 | t.Helper() | ||
| 73 | if publication.Generation == 1 { | 71 | if publication.Generation == 1 { | ||
M internal/app/daemon_integration_test.go +23 -3
| @@ -579,7 +579,6 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) { | |||
| 579 | 579 | t.Fatal("Serve callback returned after shutdown deadline while maintenance was blocked") | |
| 580 | 580 | case <-time.After(100 * time.Millisecond): | |
| 581 | 581 | } | |
| 582 | - | assertLifecycleStagePending(t, callbackReturned, "Serve callback return while maintenance is blocked") | |
| 583 | 582 | assertLifecycleStagePending(t, catalogue.closed, "catalogue close while maintenance is blocked") | |
| 584 | 583 | assertLifecycleResultPending(t, wrapperReturned, "lifecycle wrapper return while maintenance is blocked") | |
| 585 | 584 | _, err := lifecycle.TryAcquire(runtimeDir) | |
| @@ -963,8 +962,29 @@ func (t *lifecycleManualTimer) Reset(delay time.Duration) bool { | |||
| 963 | 962 | } | |
| 964 | 963 | func (t *lifecycleManualTimer) Stop() bool { return !t.stopped.Swap(true) } | |
| 965 | 964 | func (t *lifecycleManualTimer) fire() { | |
| 966 | - | if !t.stopped.Load() { | |
| 967 | - | t.ch <- time.Now() | |
| 965 | + | if t.stopped.Load() { | |
| 966 | + | return | |
| 967 | + | } | |
| 968 | + | select { | |
| 969 | + | case t.ch <- time.Now(): | |
| 970 | + | default: | |
| 971 | + | } | |
| 972 | + | } | |
| 973 | + | ||
| 974 | + | func TestLifecycleManualTimerRepeatedFireIsNonBlocking(t *testing.T) { | |
| 975 | + | timer := &lifecycleManualTimer{t: t, ch: make(chan time.Time, 1)} | |
| 976 | + | timer.fire() | |
| 977 | + | ||
| 978 | + | returned := make(chan struct{}) | |
| 979 | + | go func() { | |
| 980 | + | timer.fire() | |
| 981 | + | close(returned) | |
| 982 | + | }() | |
| 983 | + | ||
| 984 | + | select { | |
| 985 | + | case <-returned: | |
| 986 | + | case <-time.After(time.Second): | |
| 987 | + | t.Fatal("repeated manual timer fire blocked on a pending tick") | |
| 968 | 988 | } | |
| 969 | 989 | } | |
| 970 | 990 | ||
| @@ -579,7 +579,6 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) { | |||||
| 579 | t.Fatal("Serve callback returned after shutdown deadline while maintenance was blocked") | 579 | t.Fatal("Serve callback returned after shutdown deadline while maintenance was blocked") | ||
| 580 | case <-time.After(100 * time.Millisecond): | 580 | case <-time.After(100 * time.Millisecond): | ||
| 581 | } | 581 | } | ||
| 582 | - | assertLifecycleStagePending(t, callbackReturned, "Serve callback return while maintenance is blocked") | |||
| 583 | assertLifecycleStagePending(t, catalogue.closed, "catalogue close while maintenance is blocked") | 582 | assertLifecycleStagePending(t, catalogue.closed, "catalogue close while maintenance is blocked") | ||
| 584 | assertLifecycleResultPending(t, wrapperReturned, "lifecycle wrapper return while maintenance is blocked") | 583 | assertLifecycleResultPending(t, wrapperReturned, "lifecycle wrapper return while maintenance is blocked") | ||
| 585 | _, err := lifecycle.TryAcquire(runtimeDir) | 584 | _, err := lifecycle.TryAcquire(runtimeDir) | ||
| @@ -963,8 +962,29 @@ func (t *lifecycleManualTimer) Reset(delay time.Duration) bool { | |||||
| 963 | } | 962 | } | ||
| 964 | func (t *lifecycleManualTimer) Stop() bool { return !t.stopped.Swap(true) } | 963 | func (t *lifecycleManualTimer) Stop() bool { return !t.stopped.Swap(true) } | ||
| 965 | func (t *lifecycleManualTimer) fire() { | 964 | func (t *lifecycleManualTimer) fire() { | ||
| 966 | - | if !t.stopped.Load() { | 965 | + | if t.stopped.Load() { |
| 967 | - | t.ch <- time.Now() | 966 | + | return |
| 967 | + | } | |||
| 968 | + | select { | |||
| 969 | + | case t.ch <- time.Now(): | |||
| 970 | + | default: | |||
| 971 | + | } | |||
| 972 | + | } | |||
| 973 | + | ||||
| 974 | + | func TestLifecycleManualTimerRepeatedFireIsNonBlocking(t *testing.T) { | |||
| 975 | + | timer := &lifecycleManualTimer{t: t, ch: make(chan time.Time, 1)} | |||
| 976 | + | timer.fire() | |||
| 977 | + | ||||
| 978 | + | returned := make(chan struct{}) | |||
| 979 | + | go func() { | |||
| 980 | + | timer.fire() | |||
| 981 | + | close(returned) | |||
| 982 | + | }() | |||
| 983 | + | ||||
| 984 | + | select { | |||
| 985 | + | case <-returned: | |||
| 986 | + | case <-time.After(time.Second): | |||
| 987 | + | t.Fatal("repeated manual timer fire blocked on a pending tick") | |||
| 968 | } | 988 | } | ||
| 969 | } | 989 | } | ||
| 970 | 990 | ||||
M internal/app/durable_crash_matrix_test.go +12 -12
| @@ -40,12 +40,12 @@ func (h durableCrashHarness) open(t *testing.T) (*persist.Persister, *snapshot.R | |||
| 40 | 40 | return persist.New(store), snapshot.NewRepository(h.snapshotDir), recoveryfs.New(h.stateDir) | |
| 41 | 41 | } | |
| 42 | 42 | ||
| 43 | - | func (h durableCrashHarness) recoverTwice(t *testing.T, assert func(*persist.Persister, *snapshot.Repository, *recoveryfs.Journal)) { | |
| 43 | + | func (h durableCrashHarness) recoverTwice(ctx context.Context, t *testing.T, assert func(*persist.Persister, *snapshot.Repository, *recoveryfs.Journal)) { | |
| 44 | 44 | t.Helper() | |
| 45 | 45 | for attempt := range 2 { | |
| 46 | 46 | catalogue, repository, journal := h.open(t) | |
| 47 | 47 | coordinator := recovery.NewCoordinator(catalogue, repository, journal, nil) | |
| 48 | - | require.NoError(t, coordinator.Recover(context.Background()), "recovery attempt %d", attempt+1) | |
| 48 | + | require.NoError(t, coordinator.Recover(ctx), "recovery attempt %d", attempt+1) | |
| 49 | 49 | assert(catalogue, repository, journal) | |
| 50 | 50 | require.NoError(t, catalogue.Close()) | |
| 51 | 51 | } | |
| @@ -131,7 +131,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||
| 131 | 131 | require.True(t, boundary.triggered) | |
| 132 | 132 | require.NoError(t, catalogue.Close()) | |
| 133 | 133 | ||
| 134 | - | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 134 | + | h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 135 | 135 | record, exists, _ := catalogue.Record("work") | |
| 136 | 136 | require.True(t, exists) | |
| 137 | 137 | require.Equal(t, domain.IncarnationID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, record.IncarnationID) | |
| @@ -159,7 +159,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||
| 159 | 159 | require.NoError(t, catalogue.Close()) | |
| 160 | 160 | ||
| 161 | 161 | ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)} | |
| 162 | - | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 162 | + | h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 163 | 163 | stored, exists, _ := catalogue.Record(record.Name) | |
| 164 | 164 | require.True(t, exists) | |
| 165 | 165 | if boundaryName == "snapshot-publish" { | |
| @@ -188,7 +188,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||
| 188 | 188 | require.True(t, boundary.triggered) | |
| 189 | 189 | require.NoError(t, catalogue.Close()) | |
| 190 | 190 | ||
| 191 | - | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 191 | + | h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 192 | 192 | _, oldExists, _ := catalogue.Record("old") | |
| 193 | 193 | renamed, newExists, _ := catalogue.Record("new") | |
| 194 | 194 | require.False(t, oldExists) | |
| @@ -227,10 +227,10 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||
| 227 | 227 | require.True(t, boundary.triggered) | |
| 228 | 228 | require.NoError(t, catalogue.Close()) | |
| 229 | 229 | ||
| 230 | - | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 230 | + | h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 231 | 231 | _, exists, _ := catalogue.Record(record.Name) | |
| 232 | 232 | require.False(t, exists) | |
| 233 | - | assertNoTombstones(t, repository) | |
| 233 | + | assertNoTombstones(ctx, t, repository) | |
| 234 | 234 | require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String())) | |
| 235 | 235 | require.Equal(t, sourceBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "snapshot"))) | |
| 236 | 236 | require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "legacy.snap"))) | |
| @@ -274,7 +274,7 @@ func TestDurableDiscardCrashMatrix(t *testing.T) { | |||
| 274 | 274 | require.True(t, boundary.triggered) | |
| 275 | 275 | require.NoError(t, catalogue.Close()) | |
| 276 | 276 | ||
| 277 | - | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) { | |
| 277 | + | h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) { | |
| 278 | 278 | got, exists, _ := catalogue.Record(old.Name) | |
| 279 | 279 | require.True(t, exists) | |
| 280 | 280 | require.Equal(t, newID, got.IncarnationID) | |
| @@ -311,7 +311,7 @@ func TestDeletionRecoveryProtectsReusedNameFilesystemBytes(t *testing.T) { | |||
| 311 | 311 | replacementBytes := readTree(t, replacementPath) | |
| 312 | 312 | require.NoError(t, catalogue.Close()) | |
| 313 | 313 | ||
| 314 | - | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 314 | + | h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 315 | 315 | got, exists, _ := catalogue.Record("work") | |
| 316 | 316 | require.True(t, exists) | |
| 317 | 317 | require.Equal(t, replacement, got) | |
| @@ -319,7 +319,7 @@ func TestDeletionRecoveryProtectsReusedNameFilesystemBytes(t *testing.T) { | |||
| 319 | 319 | require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "work.snap"))) | |
| 320 | 320 | require.Equal(t, oldBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot"))) | |
| 321 | 321 | require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", oldID.String())) | |
| 322 | - | assertNoTombstones(t, repository) | |
| 322 | + | assertNoTombstones(ctx, t, repository) | |
| 323 | 323 | }) | |
| 324 | 324 | } | |
| 325 | 325 | ||
| @@ -340,9 +340,9 @@ func durablePublication(t *testing.T, name string, id domain.IncarnationID, payl | |||
| 340 | 340 | return ports.SnapshotPublication{IncarnationID: id, Name: name, Generation: 1, Manifest: manifest, Objects: []ports.SnapshotObject{tail, visible}} | |
| 341 | 341 | } | |
| 342 | 342 | ||
| 343 | - | func assertNoTombstones(t *testing.T, repository *snapshot.Repository) { | |
| 343 | + | func assertNoTombstones(ctx context.Context, t *testing.T, repository *snapshot.Repository) { | |
| 344 | 344 | t.Helper() | |
| 345 | - | page, err := repository.ListDeletionTombstones(context.Background(), ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 64 << 10}) | |
| 345 | + | page, err := repository.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 64 << 10}) | |
| 346 | 346 | require.NoError(t, err) | |
| 347 | 347 | require.True(t, page.Done) | |
| 348 | 348 | require.Empty(t, page.Tombstones) | |
| @@ -40,12 +40,12 @@ func (h durableCrashHarness) open(t *testing.T) (*persist.Persister, *snapshot.R | |||||
| 40 | return persist.New(store), snapshot.NewRepository(h.snapshotDir), recoveryfs.New(h.stateDir) | 40 | return persist.New(store), snapshot.NewRepository(h.snapshotDir), recoveryfs.New(h.stateDir) | ||
| 41 | } | 41 | } | ||
| 42 | 42 | ||||
| 43 | - | func (h durableCrashHarness) recoverTwice(t *testing.T, assert func(*persist.Persister, *snapshot.Repository, *recoveryfs.Journal)) { | 43 | + | func (h durableCrashHarness) recoverTwice(ctx context.Context, t *testing.T, assert func(*persist.Persister, *snapshot.Repository, *recoveryfs.Journal)) { |
| 44 | t.Helper() | 44 | t.Helper() | ||
| 45 | for attempt := range 2 { | 45 | for attempt := range 2 { | ||
| 46 | catalogue, repository, journal := h.open(t) | 46 | catalogue, repository, journal := h.open(t) | ||
| 47 | coordinator := recovery.NewCoordinator(catalogue, repository, journal, nil) | 47 | coordinator := recovery.NewCoordinator(catalogue, repository, journal, nil) | ||
| 48 | - | require.NoError(t, coordinator.Recover(context.Background()), "recovery attempt %d", attempt+1) | 48 | + | require.NoError(t, coordinator.Recover(ctx), "recovery attempt %d", attempt+1) |
| 49 | assert(catalogue, repository, journal) | 49 | assert(catalogue, repository, journal) | ||
| 50 | require.NoError(t, catalogue.Close()) | 50 | require.NoError(t, catalogue.Close()) | ||
| 51 | } | 51 | } | ||
| @@ -131,7 +131,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||||
| 131 | require.True(t, boundary.triggered) | 131 | require.True(t, boundary.triggered) | ||
| 132 | require.NoError(t, catalogue.Close()) | 132 | require.NoError(t, catalogue.Close()) | ||
| 133 | 133 | ||||
| 134 | - | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { | 134 | + | h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { |
| 135 | record, exists, _ := catalogue.Record("work") | 135 | record, exists, _ := catalogue.Record("work") | ||
| 136 | require.True(t, exists) | 136 | require.True(t, exists) | ||
| 137 | require.Equal(t, domain.IncarnationID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, record.IncarnationID) | 137 | require.Equal(t, domain.IncarnationID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, record.IncarnationID) | ||
| @@ -159,7 +159,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||||
| 159 | require.NoError(t, catalogue.Close()) | 159 | require.NoError(t, catalogue.Close()) | ||
| 160 | 160 | ||||
| 161 | ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)} | 161 | ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)} | ||
| 162 | - | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | 162 | + | h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { |
| 163 | stored, exists, _ := catalogue.Record(record.Name) | 163 | stored, exists, _ := catalogue.Record(record.Name) | ||
| 164 | require.True(t, exists) | 164 | require.True(t, exists) | ||
| 165 | if boundaryName == "snapshot-publish" { | 165 | if boundaryName == "snapshot-publish" { | ||
| @@ -188,7 +188,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||||
| 188 | require.True(t, boundary.triggered) | 188 | require.True(t, boundary.triggered) | ||
| 189 | require.NoError(t, catalogue.Close()) | 189 | require.NoError(t, catalogue.Close()) | ||
| 190 | 190 | ||||
| 191 | - | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { | 191 | + | h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { |
| 192 | _, oldExists, _ := catalogue.Record("old") | 192 | _, oldExists, _ := catalogue.Record("old") | ||
| 193 | renamed, newExists, _ := catalogue.Record("new") | 193 | renamed, newExists, _ := catalogue.Record("new") | ||
| 194 | require.False(t, oldExists) | 194 | require.False(t, oldExists) | ||
| @@ -227,10 +227,10 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||||
| 227 | require.True(t, boundary.triggered) | 227 | require.True(t, boundary.triggered) | ||
| 228 | require.NoError(t, catalogue.Close()) | 228 | require.NoError(t, catalogue.Close()) | ||
| 229 | 229 | ||||
| 230 | - | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | 230 | + | h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { |
| 231 | _, exists, _ := catalogue.Record(record.Name) | 231 | _, exists, _ := catalogue.Record(record.Name) | ||
| 232 | require.False(t, exists) | 232 | require.False(t, exists) | ||
| 233 | - | assertNoTombstones(t, repository) | 233 | + | assertNoTombstones(ctx, t, repository) |
| 234 | require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String())) | 234 | require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String())) | ||
| 235 | require.Equal(t, sourceBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "snapshot"))) | 235 | require.Equal(t, sourceBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "snapshot"))) | ||
| 236 | require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "legacy.snap"))) | 236 | require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "quarantine", record.IncarnationID.String(), "legacy.snap"))) | ||
| @@ -274,7 +274,7 @@ func TestDurableDiscardCrashMatrix(t *testing.T) { | |||||
| 274 | require.True(t, boundary.triggered) | 274 | require.True(t, boundary.triggered) | ||
| 275 | require.NoError(t, catalogue.Close()) | 275 | require.NoError(t, catalogue.Close()) | ||
| 276 | 276 | ||||
| 277 | - | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) { | 277 | + | h.recoverTwice(ctx, t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) { |
| 278 | got, exists, _ := catalogue.Record(old.Name) | 278 | got, exists, _ := catalogue.Record(old.Name) | ||
| 279 | require.True(t, exists) | 279 | require.True(t, exists) | ||
| 280 | require.Equal(t, newID, got.IncarnationID) | 280 | require.Equal(t, newID, got.IncarnationID) | ||
| @@ -311,7 +311,7 @@ func TestDeletionRecoveryProtectsReusedNameFilesystemBytes(t *testing.T) { | |||||
| 311 | replacementBytes := readTree(t, replacementPath) | 311 | replacementBytes := readTree(t, replacementPath) | ||
| 312 | require.NoError(t, catalogue.Close()) | 312 | require.NoError(t, catalogue.Close()) | ||
| 313 | 313 | ||||
| 314 | - | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | 314 | + | h.recoverTwice(ctx, t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { |
| 315 | got, exists, _ := catalogue.Record("work") | 315 | got, exists, _ := catalogue.Record("work") | ||
| 316 | require.True(t, exists) | 316 | require.True(t, exists) | ||
| 317 | require.Equal(t, replacement, got) | 317 | require.Equal(t, replacement, got) | ||
| @@ -319,7 +319,7 @@ func TestDeletionRecoveryProtectsReusedNameFilesystemBytes(t *testing.T) { | |||||
| 319 | require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "work.snap"))) | 319 | require.Equal(t, legacy, mustReadFile(t, filepath.Join(h.snapshotDir, "work.snap"))) | ||
| 320 | require.Equal(t, oldBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot"))) | 320 | require.Equal(t, oldBytes, readTree(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot"))) | ||
| 321 | require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", oldID.String())) | 321 | require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", oldID.String())) | ||
| 322 | - | assertNoTombstones(t, repository) | 322 | + | assertNoTombstones(ctx, t, repository) |
| 323 | }) | 323 | }) | ||
| 324 | } | 324 | } | ||
| 325 | 325 | ||||
| @@ -340,9 +340,9 @@ func durablePublication(t *testing.T, name string, id domain.IncarnationID, payl | |||||
| 340 | return ports.SnapshotPublication{IncarnationID: id, Name: name, Generation: 1, Manifest: manifest, Objects: []ports.SnapshotObject{tail, visible}} | 340 | return ports.SnapshotPublication{IncarnationID: id, Name: name, Generation: 1, Manifest: manifest, Objects: []ports.SnapshotObject{tail, visible}} | ||
| 341 | } | 341 | } | ||
| 342 | 342 | ||||
| 343 | - | func assertNoTombstones(t *testing.T, repository *snapshot.Repository) { | 343 | + | func assertNoTombstones(ctx context.Context, t *testing.T, repository *snapshot.Repository) { |
| 344 | t.Helper() | 344 | t.Helper() | ||
| 345 | - | page, err := repository.ListDeletionTombstones(context.Background(), ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 64 << 10}) | 345 | + | page, err := repository.ListDeletionTombstones(ctx, ports.DeletionTombstoneCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 64 << 10}) |
| 346 | require.NoError(t, err) | 346 | require.NoError(t, err) | ||
| 347 | require.True(t, page.Done) | 347 | require.True(t, page.Done) | ||
| 348 | require.Empty(t, page.Tombstones) | 348 | require.Empty(t, page.Tombstones) | ||
M internal/app/offline_kill_retry_test.go +28 -0
| @@ -2,6 +2,7 @@ package app | |||
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | + | "errors" | |
| 5 | 6 | "os" | |
| 6 | 7 | "path/filepath" | |
| 7 | 8 | "testing" | |
| @@ -13,6 +14,33 @@ import ( | |||
| 13 | 14 | "github.com/bnema/vev/internal/persist" | |
| 14 | 15 | ) | |
| 15 | 16 | ||
| 17 | + | func TestRunOfflineNamedKillUsesOpenOrMigrateSeam(t *testing.T) { | |
| 18 | + | stateRoot := t.TempDir() | |
| 19 | + | t.Setenv("XDG_STATE_HOME", stateRoot) | |
| 20 | + | stateDir := filepath.Join(stateRoot, "vev") | |
| 21 | + | ||
| 22 | + | p := newTestPersister(t, stateDir) | |
| 23 | + | now := time.Now().UnixNano() | |
| 24 | + | require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: domain.IncarnationID{1}, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh})) | |
| 25 | + | require.NoError(t, p.Close()) | |
| 26 | + | ||
| 27 | + | original := openOrMigrate | |
| 28 | + | t.Cleanup(func() { openOrMigrate = original }) | |
| 29 | + | wantErr := errors.New("open seam called") | |
| 30 | + | called := false | |
| 31 | + | openOrMigrate = func(_ context.Context, deps persist.OpenDeps) (persist.OpenResult, error) { | |
| 32 | + | called = true | |
| 33 | + | require.Equal(t, stateDir, deps.StateDir) | |
| 34 | + | require.NotNil(t, deps.Random) | |
| 35 | + | require.NotNil(t, deps.SnapshotMigration) | |
| 36 | + | return persist.OpenResult{}, wantErr | |
| 37 | + | } | |
| 38 | + | ||
| 39 | + | err := runOfflineNamedKill(context.Background(), "named") | |
| 40 | + | require.ErrorIs(t, err, wantErr) | |
| 41 | + | require.True(t, called) | |
| 42 | + | } | |
| 43 | + | ||
| 16 | 44 | func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) { | |
| 17 | 45 | tests := []struct { | |
| 18 | 46 | name string | |
| @@ -2,6 +2,7 @@ package app | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | + | "errors" | |||
| 5 | "os" | 6 | "os" | ||
| 6 | "path/filepath" | 7 | "path/filepath" | ||
| 7 | "testing" | 8 | "testing" | ||
| @@ -13,6 +14,33 @@ import ( | |||||
| 13 | "github.com/bnema/vev/internal/persist" | 14 | "github.com/bnema/vev/internal/persist" | ||
| 14 | ) | 15 | ) | ||
| 15 | 16 | ||||
| 17 | + | func TestRunOfflineNamedKillUsesOpenOrMigrateSeam(t *testing.T) { | |||
| 18 | + | stateRoot := t.TempDir() | |||
| 19 | + | t.Setenv("XDG_STATE_HOME", stateRoot) | |||
| 20 | + | stateDir := filepath.Join(stateRoot, "vev") | |||
| 21 | + | ||||
| 22 | + | p := newTestPersister(t, stateDir) | |||
| 23 | + | now := time.Now().UnixNano() | |||
| 24 | + | require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: domain.IncarnationID{1}, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh})) | |||
| 25 | + | require.NoError(t, p.Close()) | |||
| 26 | + | ||||
| 27 | + | original := openOrMigrate | |||
| 28 | + | t.Cleanup(func() { openOrMigrate = original }) | |||
| 29 | + | wantErr := errors.New("open seam called") | |||
| 30 | + | called := false | |||
| 31 | + | openOrMigrate = func(_ context.Context, deps persist.OpenDeps) (persist.OpenResult, error) { | |||
| 32 | + | called = true | |||
| 33 | + | require.Equal(t, stateDir, deps.StateDir) | |||
| 34 | + | require.NotNil(t, deps.Random) | |||
| 35 | + | require.NotNil(t, deps.SnapshotMigration) | |||
| 36 | + | return persist.OpenResult{}, wantErr | |||
| 37 | + | } | |||
| 38 | + | ||||
| 39 | + | err := runOfflineNamedKill(context.Background(), "named") | |||
| 40 | + | require.ErrorIs(t, err, wantErr) | |||
| 41 | + | require.True(t, called) | |||
| 42 | + | } | |||
| 43 | + | ||||
| 16 | func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) { | 44 | func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) { | ||
| 17 | tests := []struct { | 45 | tests := []struct { | ||
| 18 | name string | 46 | name string | ||
M internal/app/run.go +10 -11
| @@ -436,13 +436,18 @@ type lifecycleStartupDeps struct { | |||
| 436 | 436 | ||
| 437 | 437 | const lifecycleAcquireRetry = 25 * time.Millisecond | |
| 438 | 438 | ||
| 439 | - | func runWithLifecycleOwner(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart) error { | |
| 440 | - | return runWithLifecycleOwnerDeps(ctx, runtimeDir, stateDir, start, lifecycleStartupDeps{ | |
| 439 | + | func lifecycleStartupDependencies(log *slog.Logger) lifecycleStartupDeps { | |
| 440 | + | return lifecycleStartupDeps{ | |
| 441 | 441 | ensurePrivate: safedir.EnsurePrivate, | |
| 442 | 442 | acquire: func(ctx context.Context, runtimeDir string, retry time.Duration) (lifecycleOwnership, error) { | |
| 443 | 443 | return lifecycle.Acquire(ctx, runtimeDir, retry) | |
| 444 | 444 | }, | |
| 445 | - | }) | |
| 445 | + | log: log, | |
| 446 | + | } | |
| 447 | + | } | |
| 448 | + | ||
| 449 | + | func runWithLifecycleOwner(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart) error { | |
| 450 | + | return runWithLifecycleOwnerDeps(ctx, runtimeDir, stateDir, start, lifecycleStartupDependencies(nil)) | |
| 446 | 451 | } | |
| 447 | 452 | ||
| 448 | 453 | func runWithLifecycleOwnerDeps(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart, deps lifecycleStartupDeps) (retErr error) { | |
| @@ -492,13 +497,7 @@ func runDaemon() (retErr error) { | |||
| 492 | 497 | defer func() { retErr = errors.Join(retErr, logCloser.Close()) }() | |
| 493 | 498 | return runWithLifecycleOwnerDeps(ctx, ipc.SocketDir(), platform.StateDir(), func(ctx context.Context) error { | |
| 494 | 499 | return runDaemonOwnedWithLogger(ctx, log) | |
| 495 | - | }, lifecycleStartupDeps{ | |
| 496 | - | ensurePrivate: safedir.EnsurePrivate, | |
| 497 | - | acquire: func(ctx context.Context, runtimeDir string, retry time.Duration) (lifecycleOwnership, error) { | |
| 498 | - | return lifecycle.Acquire(ctx, runtimeDir, retry) | |
| 499 | - | }, | |
| 500 | - | log: log, | |
| 501 | - | }) | |
| 500 | + | }, lifecycleStartupDependencies(log)) | |
| 502 | 501 | } | |
| 503 | 502 | ||
| 504 | 503 | func logCatalogueRecovery(log *slog.Logger, records []domain.CatalogueRecord, recoveryMode string) { | |
| @@ -1567,7 +1566,7 @@ func runOfflineNamedKill(ctx context.Context, name string) (retErr error) { | |||
| 1567 | 1566 | } | |
| 1568 | 1567 | ||
| 1569 | 1568 | repository := snapshotadapter.NewRepository(snapshotDir()) | |
| 1570 | - | opened, err := persist.OpenOrMigrate(ctx, persist.OpenDeps{ | |
| 1569 | + | opened, err := openOrMigrate(ctx, persist.OpenDeps{ | |
| 1571 | 1570 | StateDir: platform.StateDir(), | |
| 1572 | 1571 | Random: rand.Reader, | |
| 1573 | 1572 | SnapshotMigration: repository, | |
| @@ -436,13 +436,18 @@ type lifecycleStartupDeps struct { | |||||
| 436 | 436 | ||||
| 437 | const lifecycleAcquireRetry = 25 * time.Millisecond | 437 | const lifecycleAcquireRetry = 25 * time.Millisecond | ||
| 438 | 438 | ||||
| 439 | - | func runWithLifecycleOwner(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart) error { | 439 | + | func lifecycleStartupDependencies(log *slog.Logger) lifecycleStartupDeps { |
| 440 | - | return runWithLifecycleOwnerDeps(ctx, runtimeDir, stateDir, start, lifecycleStartupDeps{ | 440 | + | return lifecycleStartupDeps{ |
| 441 | ensurePrivate: safedir.EnsurePrivate, | 441 | ensurePrivate: safedir.EnsurePrivate, | ||
| 442 | acquire: func(ctx context.Context, runtimeDir string, retry time.Duration) (lifecycleOwnership, error) { | 442 | acquire: func(ctx context.Context, runtimeDir string, retry time.Duration) (lifecycleOwnership, error) { | ||
| 443 | return lifecycle.Acquire(ctx, runtimeDir, retry) | 443 | return lifecycle.Acquire(ctx, runtimeDir, retry) | ||
| 444 | }, | 444 | }, | ||
| 445 | - | }) | 445 | + | log: log, |
| 446 | + | } | |||
| 447 | + | } | |||
| 448 | + | ||||
| 449 | + | func runWithLifecycleOwner(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart) error { | |||
| 450 | + | return runWithLifecycleOwnerDeps(ctx, runtimeDir, stateDir, start, lifecycleStartupDependencies(nil)) | |||
| 446 | } | 451 | } | ||
| 447 | 452 | ||||
| 448 | func runWithLifecycleOwnerDeps(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart, deps lifecycleStartupDeps) (retErr error) { | 453 | func runWithLifecycleOwnerDeps(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart, deps lifecycleStartupDeps) (retErr error) { | ||
| @@ -492,13 +497,7 @@ func runDaemon() (retErr error) { | |||||
| 492 | defer func() { retErr = errors.Join(retErr, logCloser.Close()) }() | 497 | defer func() { retErr = errors.Join(retErr, logCloser.Close()) }() | ||
| 493 | return runWithLifecycleOwnerDeps(ctx, ipc.SocketDir(), platform.StateDir(), func(ctx context.Context) error { | 498 | return runWithLifecycleOwnerDeps(ctx, ipc.SocketDir(), platform.StateDir(), func(ctx context.Context) error { | ||
| 494 | return runDaemonOwnedWithLogger(ctx, log) | 499 | return runDaemonOwnedWithLogger(ctx, log) | ||
| 495 | - | }, lifecycleStartupDeps{ | 500 | + | }, lifecycleStartupDependencies(log)) |
| 496 | - | ensurePrivate: safedir.EnsurePrivate, | |||
| 497 | - | acquire: func(ctx context.Context, runtimeDir string, retry time.Duration) (lifecycleOwnership, error) { | |||
| 498 | - | return lifecycle.Acquire(ctx, runtimeDir, retry) | |||
| 499 | - | }, | |||
| 500 | - | log: log, | |||
| 501 | - | }) | |||
| 502 | } | 501 | } | ||
| 503 | 502 | ||||
| 504 | func logCatalogueRecovery(log *slog.Logger, records []domain.CatalogueRecord, recoveryMode string) { | 503 | func logCatalogueRecovery(log *slog.Logger, records []domain.CatalogueRecord, recoveryMode string) { | ||
| @@ -1567,7 +1566,7 @@ func runOfflineNamedKill(ctx context.Context, name string) (retErr error) { | |||||
| 1567 | } | 1566 | } | ||
| 1568 | 1567 | ||||
| 1569 | repository := snapshotadapter.NewRepository(snapshotDir()) | 1568 | repository := snapshotadapter.NewRepository(snapshotDir()) | ||
| 1570 | - | opened, err := persist.OpenOrMigrate(ctx, persist.OpenDeps{ | 1569 | + | opened, err := openOrMigrate(ctx, persist.OpenDeps{ |
| 1571 | StateDir: platform.StateDir(), | 1570 | StateDir: platform.StateDir(), | ||
| 1572 | Random: rand.Reader, | 1571 | Random: rand.Reader, | ||
| 1573 | SnapshotMigration: repository, | 1572 | SnapshotMigration: repository, | ||
M internal/app/spawn.go +3 -0
| @@ -70,6 +70,9 @@ var defaultBackoff = backoffConfig{ | |||
| 70 | 70 | func retryAttempts[T any](ctx context.Context, cfg backoffConfig, attempt func() (T, bool, error)) (T, error) { | |
| 71 | 71 | deadline := time.Now().Add(cfg.total) | |
| 72 | 72 | backoff := cfg.initial | |
| 73 | + | if backoff <= 0 { | |
| 74 | + | backoff = time.Millisecond | |
| 75 | + | } | |
| 73 | 76 | for { | |
| 74 | 77 | if err := ctx.Err(); err != nil { | |
| 75 | 78 | var zero T | |
| @@ -70,6 +70,9 @@ var defaultBackoff = backoffConfig{ | |||||
| 70 | func retryAttempts[T any](ctx context.Context, cfg backoffConfig, attempt func() (T, bool, error)) (T, error) { | 70 | func retryAttempts[T any](ctx context.Context, cfg backoffConfig, attempt func() (T, bool, error)) (T, error) { | ||
| 71 | deadline := time.Now().Add(cfg.total) | 71 | deadline := time.Now().Add(cfg.total) | ||
| 72 | backoff := cfg.initial | 72 | backoff := cfg.initial | ||
| 73 | + | if backoff <= 0 { | |||
| 74 | + | backoff = time.Millisecond | |||
| 75 | + | } | |||
| 73 | for { | 76 | for { | ||
| 74 | if err := ctx.Err(); err != nil { | 77 | if err := ctx.Err(); err != nil { | ||
| 75 | var zero T | 78 | var zero T | ||
M internal/app/spawn_test.go +15 -0
| @@ -273,6 +273,21 @@ func TestRetryAttemptsChecksOperationErrorBeforeDeadline(t *testing.T) { | |||
| 273 | 273 | } | |
| 274 | 274 | } | |
| 275 | 275 | ||
| 276 | + | func TestRetryAttemptsClampsNonPositiveInitialDelay(t *testing.T) { | |
| 277 | + | started := time.Now() | |
| 278 | + | calls := 0 | |
| 279 | + | _, err := retryAttempts(context.Background(), backoffConfig{total: time.Second, max: time.Second}, func() (struct{}, bool, error) { | |
| 280 | + | calls++ | |
| 281 | + | return struct{}{}, calls == 2, nil | |
| 282 | + | }) | |
| 283 | + | if err != nil { | |
| 284 | + | t.Fatalf("retryAttempts: %v", err) | |
| 285 | + | } | |
| 286 | + | if elapsed := time.Since(started); elapsed < 500*time.Microsecond { | |
| 287 | + | t.Fatalf("retryAttempts retried after %s, want a positive initial delay", elapsed) | |
| 288 | + | } | |
| 289 | + | } | |
| 290 | + | ||
| 276 | 291 | func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) { | |
| 277 | 292 | want := portsmocks.NewMockTransport(t) | |
| 278 | 293 | var calls atomic.Int32 | |
| @@ -273,6 +273,21 @@ func TestRetryAttemptsChecksOperationErrorBeforeDeadline(t *testing.T) { | |||||
| 273 | } | 273 | } | ||
| 274 | } | 274 | } | ||
| 275 | 275 | ||||
| 276 | + | func TestRetryAttemptsClampsNonPositiveInitialDelay(t *testing.T) { | |||
| 277 | + | started := time.Now() | |||
| 278 | + | calls := 0 | |||
| 279 | + | _, err := retryAttempts(context.Background(), backoffConfig{total: time.Second, max: time.Second}, func() (struct{}, bool, error) { | |||
| 280 | + | calls++ | |||
| 281 | + | return struct{}{}, calls == 2, nil | |||
| 282 | + | }) | |||
| 283 | + | if err != nil { | |||
| 284 | + | t.Fatalf("retryAttempts: %v", err) | |||
| 285 | + | } | |||
| 286 | + | if elapsed := time.Since(started); elapsed < 500*time.Microsecond { | |||
| 287 | + | t.Fatalf("retryAttempts retried after %s, want a positive initial delay", elapsed) | |||
| 288 | + | } | |||
| 289 | + | } | |||
| 290 | + | ||||
| 276 | func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) { | 291 | func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) { | ||
| 277 | want := portsmocks.NewMockTransport(t) | 292 | want := portsmocks.NewMockTransport(t) | ||
| 278 | var calls atomic.Int32 | 293 | var calls atomic.Int32 | ||
M internal/domain/recovery.go +2 -2
| @@ -43,8 +43,6 @@ type CatalogueRecord struct { | |||
| 43 | 43 | DegradedReason string | |
| 44 | 44 | } | |
| 45 | 45 | ||
| 46 | - | // CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue | |
| 47 | - | // incarnation without carrying authority-owned recovery or checkpoint fields. | |
| 48 | 46 | // Equal reports canonical value equality between checkpoint references. | |
| 49 | 47 | func (r *CheckpointRef) Equal(other *CheckpointRef) bool { | |
| 50 | 48 | if r == nil || other == nil { | |
| @@ -63,6 +61,8 @@ func (r CatalogueRecord) Equal(other CatalogueRecord) bool { | |||
| 63 | 61 | r.Fallbacks[1].Equal(other.Fallbacks[1]) && r.DegradedReason == other.DegradedReason | |
| 64 | 62 | } | |
| 65 | 63 | ||
| 64 | + | // CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue | |
| 65 | + | // incarnation without carrying authority-owned recovery or checkpoint fields. | |
| 66 | 66 | type CatalogueMetadataUpdate struct { | |
| 67 | 67 | Name string | |
| 68 | 68 | IncarnationID IncarnationID | |
| @@ -43,8 +43,6 @@ type CatalogueRecord struct { | |||||
| 43 | DegradedReason string | 43 | DegradedReason string | ||
| 44 | } | 44 | } | ||
| 45 | 45 | ||||
| 46 | - | // CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue | |||
| 47 | - | // incarnation without carrying authority-owned recovery or checkpoint fields. | |||
| 48 | // Equal reports canonical value equality between checkpoint references. | 46 | // Equal reports canonical value equality between checkpoint references. | ||
| 49 | func (r *CheckpointRef) Equal(other *CheckpointRef) bool { | 47 | func (r *CheckpointRef) Equal(other *CheckpointRef) bool { | ||
| 50 | if r == nil || other == nil { | 48 | if r == nil || other == nil { | ||
| @@ -63,6 +61,8 @@ func (r CatalogueRecord) Equal(other CatalogueRecord) bool { | |||||
| 63 | r.Fallbacks[1].Equal(other.Fallbacks[1]) && r.DegradedReason == other.DegradedReason | 61 | r.Fallbacks[1].Equal(other.Fallbacks[1]) && r.DegradedReason == other.DegradedReason | ||
| 64 | } | 62 | } | ||
| 65 | 63 | ||||
| 64 | + | // CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue | |||
| 65 | + | // incarnation without carrying authority-owned recovery or checkpoint fields. | |||
| 66 | type CatalogueMetadataUpdate struct { | 66 | type CatalogueMetadataUpdate struct { | ||
| 67 | Name string | 67 | Name string | ||
| 68 | IncarnationID IncarnationID | 68 | IncarnationID IncarnationID | ||
M internal/domain/recovery_test.go +14 -13
| @@ -31,19 +31,15 @@ func TestIncarnationIDText(t *testing.T) { | |||
| 31 | 31 | } | |
| 32 | 32 | ||
| 33 | 33 | func TestCatalogueRecordValidate(t *testing.T) { | |
| 34 | - | valid := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh} | |
| 35 | - | require.NoError(t, valid.Validate()) | |
| 36 | - | valid.RecoveryState = RecoveryHealthy | |
| 37 | - | valid.Committed = &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}} | |
| 38 | - | require.NoError(t, valid.Validate()) | |
| 39 | - | valid.Fallbacks[0] = &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{2}} | |
| 40 | - | require.Error(t, valid.Validate(), "duplicate generations are malformed") | |
| 41 | - | ||
| 42 | 34 | validRef := &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}} | |
| 43 | 35 | tests := []struct { | |
| 44 | - | name string | |
| 45 | - | record CatalogueRecord | |
| 36 | + | name string | |
| 37 | + | record CatalogueRecord | |
| 38 | + | wantValid bool | |
| 46 | 39 | }{ | |
| 40 | + | {name: "fresh valid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh}, wantValid: true}, | |
| 41 | + | {name: "healthy with committed checkpoint valid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}}, wantValid: true}, | |
| 42 | + | {name: "duplicate generation invalid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}, Fallbacks: [2]*CheckpointRef{{Generation: 7, ManifestDigest: [32]byte{2}}}}}, | |
| 47 | 43 | {name: "invalid session name", record: CatalogueRecord{Name: "bad/name", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh}}, | |
| 48 | 44 | {name: "zero incarnation", record: CatalogueRecord{Name: "work", RecoveryState: RecoveryFresh}}, | |
| 49 | 45 | {name: "unknown recovery state", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryState(99)}}, | |
| @@ -59,7 +55,12 @@ func TestCatalogueRecordValidate(t *testing.T) { | |||
| 59 | 55 | } | |
| 60 | 56 | for _, tt := range tests { | |
| 61 | 57 | t.Run(tt.name, func(t *testing.T) { | |
| 62 | - | require.Error(t, tt.record.Validate()) | |
| 58 | + | err := tt.record.Validate() | |
| 59 | + | if tt.wantValid { | |
| 60 | + | require.NoError(t, err) | |
| 61 | + | } else { | |
| 62 | + | require.Error(t, err) | |
| 63 | + | } | |
| 63 | 64 | }) | |
| 64 | 65 | } | |
| 65 | 66 | } | |
| @@ -69,8 +70,8 @@ func TestCatalogueRecordEqualUsesCheckpointValuesAndOrderedTabs(t *testing.T) { | |||
| 69 | 70 | left := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, TabNames: []string{"shell", "logs"}, RecoveryState: RecoveryHealthy, Committed: &committed} | |
| 70 | 71 | right := left | |
| 71 | 72 | right.TabNames = append([]string(nil), left.TabNames...) | |
| 72 | - | copy := committed | |
| 73 | - | right.Committed = © | |
| 73 | + | committedCopy := committed | |
| 74 | + | right.Committed = &committedCopy | |
| 74 | 75 | require.True(t, left.Equal(right)) | |
| 75 | 76 | ||
| 76 | 77 | right.TabNames[0], right.TabNames[1] = right.TabNames[1], right.TabNames[0] | |
| @@ -31,19 +31,15 @@ func TestIncarnationIDText(t *testing.T) { | |||||
| 31 | } | 31 | } | ||
| 32 | 32 | ||||
| 33 | func TestCatalogueRecordValidate(t *testing.T) { | 33 | func TestCatalogueRecordValidate(t *testing.T) { | ||
| 34 | - | valid := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh} | |||
| 35 | - | require.NoError(t, valid.Validate()) | |||
| 36 | - | valid.RecoveryState = RecoveryHealthy | |||
| 37 | - | valid.Committed = &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}} | |||
| 38 | - | require.NoError(t, valid.Validate()) | |||
| 39 | - | valid.Fallbacks[0] = &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{2}} | |||
| 40 | - | require.Error(t, valid.Validate(), "duplicate generations are malformed") | |||
| 41 | - | ||||
| 42 | validRef := &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}} | 34 | validRef := &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}} | ||
| 43 | tests := []struct { | 35 | tests := []struct { | ||
| 44 | - | name string | 36 | + | name string |
| 45 | - | record CatalogueRecord | 37 | + | record CatalogueRecord |
| 38 | + | wantValid bool | |||
| 46 | }{ | 39 | }{ | ||
| 40 | + | {name: "fresh valid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh}, wantValid: true}, | |||
| 41 | + | {name: "healthy with committed checkpoint valid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}}, wantValid: true}, | |||
| 42 | + | {name: "duplicate generation invalid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}, Fallbacks: [2]*CheckpointRef{{Generation: 7, ManifestDigest: [32]byte{2}}}}}, | |||
| 47 | {name: "invalid session name", record: CatalogueRecord{Name: "bad/name", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh}}, | 43 | {name: "invalid session name", record: CatalogueRecord{Name: "bad/name", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryFresh}}, | ||
| 48 | {name: "zero incarnation", record: CatalogueRecord{Name: "work", RecoveryState: RecoveryFresh}}, | 44 | {name: "zero incarnation", record: CatalogueRecord{Name: "work", RecoveryState: RecoveryFresh}}, | ||
| 49 | {name: "unknown recovery state", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryState(99)}}, | 45 | {name: "unknown recovery state", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryState(99)}}, | ||
| @@ -59,7 +55,12 @@ func TestCatalogueRecordValidate(t *testing.T) { | |||||
| 59 | } | 55 | } | ||
| 60 | for _, tt := range tests { | 56 | for _, tt := range tests { | ||
| 61 | t.Run(tt.name, func(t *testing.T) { | 57 | t.Run(tt.name, func(t *testing.T) { | ||
| 62 | - | require.Error(t, tt.record.Validate()) | 58 | + | err := tt.record.Validate() |
| 59 | + | if tt.wantValid { | |||
| 60 | + | require.NoError(t, err) | |||
| 61 | + | } else { | |||
| 62 | + | require.Error(t, err) | |||
| 63 | + | } | |||
| 63 | }) | 64 | }) | ||
| 64 | } | 65 | } | ||
| 65 | } | 66 | } | ||
| @@ -69,8 +70,8 @@ func TestCatalogueRecordEqualUsesCheckpointValuesAndOrderedTabs(t *testing.T) { | |||||
| 69 | left := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, TabNames: []string{"shell", "logs"}, RecoveryState: RecoveryHealthy, Committed: &committed} | 70 | left := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, TabNames: []string{"shell", "logs"}, RecoveryState: RecoveryHealthy, Committed: &committed} | ||
| 70 | right := left | 71 | right := left | ||
| 71 | right.TabNames = append([]string(nil), left.TabNames...) | 72 | right.TabNames = append([]string(nil), left.TabNames...) | ||
| 72 | - | copy := committed | 73 | + | committedCopy := committed |
| 73 | - | right.Committed = © | 74 | + | right.Committed = &committedCopy |
| 74 | require.True(t, left.Equal(right)) | 75 | require.True(t, left.Equal(right)) | ||
| 75 | 76 | ||||
| 76 | right.TabNames[0], right.TabNames[1] = right.TabNames[1], right.TabNames[0] | 77 | right.TabNames[0], right.TabNames[1] = right.TabNames[1], right.TabNames[0] | ||
M internal/persist/catalogue_test.go +19 -5
| @@ -52,15 +52,29 @@ func testCatalogueRecordRoundTrip(t *testing.T) { | |||
| 52 | 52 | func TestDecodeRecordValueWrapsMalformedAndValidationErrors(t *testing.T) { | |
| 53 | 53 | encoded, err := encodeRecordValue(validRecord("work", 1)) | |
| 54 | 54 | require.NoError(t, err) | |
| 55 | - | const recoveryStateOffset = 4 + 2 + 16 + 4 + 8 + 8 + 8 + 4 | |
| 56 | - | encoded[recoveryStateOffset] = 99 | |
| 55 | + | encoded[recoveryStateFieldOffset(t, encoded)] = 99 | |
| 57 | 56 | ||
| 58 | 57 | _, err = decodeRecordValue("work", encoded) | |
| 59 | 58 | require.ErrorIs(t, err, errMalformedRecord) | |
| 60 | - | joined, ok := err.(interface{ Unwrap() []error }) | |
| 59 | + | require.ErrorContains(t, err, "invalid recovery state") | |
| 60 | + | } | |
| 61 | + | ||
| 62 | + | func recoveryStateFieldOffset(t *testing.T, encoded []byte) int { | |
| 63 | + | t.Helper() | |
| 64 | + | r := valueReader{data: encoded} | |
| 65 | + | _, ok := r.take(len(catalogueMagic) + 2 + len(domain.IncarnationID{})) | |
| 66 | + | require.True(t, ok) | |
| 67 | + | _, ok = r.str() | |
| 61 | 68 | require.True(t, ok) | |
| 62 | - | require.Len(t, joined.Unwrap(), 2) | |
| 63 | - | require.ErrorContains(t, joined.Unwrap()[1], "invalid recovery state") | |
| 69 | + | _, ok = r.take(8 + 8 + 8) | |
| 70 | + | require.True(t, ok) | |
| 71 | + | count, ok := r.u32() | |
| 72 | + | require.True(t, ok) | |
| 73 | + | for range count { | |
| 74 | + | _, ok = r.str() | |
| 75 | + | require.True(t, ok) | |
| 76 | + | } | |
| 77 | + | return len(encoded) - r.remaining() | |
| 64 | 78 | } | |
| 65 | 79 | ||
| 66 | 80 | func testCatalogueDuplicateIncarnations(t *testing.T) { | |
| @@ -52,15 +52,29 @@ func testCatalogueRecordRoundTrip(t *testing.T) { | |||||
| 52 | func TestDecodeRecordValueWrapsMalformedAndValidationErrors(t *testing.T) { | 52 | func TestDecodeRecordValueWrapsMalformedAndValidationErrors(t *testing.T) { | ||
| 53 | encoded, err := encodeRecordValue(validRecord("work", 1)) | 53 | encoded, err := encodeRecordValue(validRecord("work", 1)) | ||
| 54 | require.NoError(t, err) | 54 | require.NoError(t, err) | ||
| 55 | - | const recoveryStateOffset = 4 + 2 + 16 + 4 + 8 + 8 + 8 + 4 | 55 | + | encoded[recoveryStateFieldOffset(t, encoded)] = 99 |
| 56 | - | encoded[recoveryStateOffset] = 99 | |||
| 57 | 56 | ||||
| 58 | _, err = decodeRecordValue("work", encoded) | 57 | _, err = decodeRecordValue("work", encoded) | ||
| 59 | require.ErrorIs(t, err, errMalformedRecord) | 58 | require.ErrorIs(t, err, errMalformedRecord) | ||
| 60 | - | joined, ok := err.(interface{ Unwrap() []error }) | 59 | + | require.ErrorContains(t, err, "invalid recovery state") |
| 60 | + | } | |||
| 61 | + | ||||
| 62 | + | func recoveryStateFieldOffset(t *testing.T, encoded []byte) int { | |||
| 63 | + | t.Helper() | |||
| 64 | + | r := valueReader{data: encoded} | |||
| 65 | + | _, ok := r.take(len(catalogueMagic) + 2 + len(domain.IncarnationID{})) | |||
| 66 | + | require.True(t, ok) | |||
| 67 | + | _, ok = r.str() | |||
| 61 | require.True(t, ok) | 68 | require.True(t, ok) | ||
| 62 | - | require.Len(t, joined.Unwrap(), 2) | 69 | + | _, ok = r.take(8 + 8 + 8) |
| 63 | - | require.ErrorContains(t, joined.Unwrap()[1], "invalid recovery state") | 70 | + | require.True(t, ok) |
| 71 | + | count, ok := r.u32() | |||
| 72 | + | require.True(t, ok) | |||
| 73 | + | for range count { | |||
| 74 | + | _, ok = r.str() | |||
| 75 | + | require.True(t, ok) | |||
| 76 | + | } | |||
| 77 | + | return len(encoded) - r.remaining() | |||
| 64 | } | 78 | } | ||
| 65 | 79 | ||||
| 66 | func testCatalogueDuplicateIncarnations(t *testing.T) { | 80 | func testCatalogueDuplicateIncarnations(t *testing.T) { | ||
M internal/persist/migration.go +27 -9
| @@ -419,6 +419,17 @@ func installMigratedCatalogue(stateDir string, records []domain.CatalogueRecord) | |||
| 419 | 419 | if err := p.Close(); err != nil { | |
| 420 | 420 | return err | |
| 421 | 421 | } | |
| 422 | + | staged, opened, err := openCurrentCatalogue(stage, false) | |
| 423 | + | if err != nil { | |
| 424 | + | return err | |
| 425 | + | } | |
| 426 | + | if !catalogueRecordsEqual(records, opened) { | |
| 427 | + | _ = staged.Close() | |
| 428 | + | return errors.New("persist: staged catalogue validation mismatch") | |
| 429 | + | } | |
| 430 | + | if err := staged.Close(); err != nil { | |
| 431 | + | return err | |
| 432 | + | } | |
| 422 | 433 | source := StorePath(stage) | |
| 423 | 434 | target := StorePath(stateDir) | |
| 424 | 435 | if err := os.Rename(source, target); err != nil { | |
| @@ -487,9 +498,13 @@ func decodeLegacyRecordV0(name string, value []byte) (legacyCatalogueRecordV0, e | |||
| 487 | 498 | } | |
| 488 | 499 | ||
| 489 | 500 | func writeMigrationRecord(path string, record MigrationRecord) error { | |
| 490 | - | return atomicMigrationWrite(path, encodeMigrationRecord(record)) | |
| 501 | + | encoded, err := encodeMigrationRecord(record) | |
| 502 | + | if err != nil { | |
| 503 | + | return err | |
| 504 | + | } | |
| 505 | + | return atomicMigrationWrite(path, encoded) | |
| 491 | 506 | } | |
| 492 | - | func encodeMigrationRecord(record MigrationRecord) []byte { | |
| 507 | + | func encodeMigrationRecord(record MigrationRecord) ([]byte, error) { | |
| 493 | 508 | out := append([]byte(migrationRecordMagic), byte(migrationRecordVersion>>8), byte(migrationRecordVersion), byte(record.Phase)) | |
| 494 | 509 | names := make([]string, 0, len(record.Assignments)) | |
| 495 | 510 | for name := range record.Assignments { | |
| @@ -498,7 +513,11 @@ func encodeMigrationRecord(record MigrationRecord) []byte { | |||
| 498 | 513 | sort.Strings(names) | |
| 499 | 514 | out = binary.BigEndian.AppendUint32(out, uint32(len(names))) | |
| 500 | 515 | for _, name := range names { | |
| 501 | - | out = appendMigrationString(out, name) | |
| 516 | + | var err error | |
| 517 | + | out, err = appendCheckedString(out, name) | |
| 518 | + | if err != nil { | |
| 519 | + | return nil, err | |
| 520 | + | } | |
| 502 | 521 | id := record.Assignments[name] | |
| 503 | 522 | out = append(out, id[:]...) | |
| 504 | 523 | if ref := record.Validated[name]; ref != nil { | |
| @@ -508,9 +527,12 @@ func encodeMigrationRecord(record MigrationRecord) []byte { | |||
| 508 | 527 | } else { | |
| 509 | 528 | out = append(out, 0) | |
| 510 | 529 | } | |
| 511 | - | out = appendMigrationString(out, record.Degraded[name]) | |
| 530 | + | out, err = appendCheckedString(out, record.Degraded[name]) | |
| 531 | + | if err != nil { | |
| 532 | + | return nil, err | |
| 533 | + | } | |
| 512 | 534 | } | |
| 513 | - | return out | |
| 535 | + | return out, nil | |
| 514 | 536 | } | |
| 515 | 537 | func readMigrationRecord(path string) (MigrationRecord, error) { | |
| 516 | 538 | data, err := os.ReadFile(path) | |
| @@ -573,10 +595,6 @@ func readMigrationRecord(path string) (MigrationRecord, error) { | |||
| 573 | 595 | } | |
| 574 | 596 | return record, nil | |
| 575 | 597 | } | |
| 576 | - | func appendMigrationString(out []byte, value string) []byte { | |
| 577 | - | out = binary.BigEndian.AppendUint32(out, uint32(len(value))) | |
| 578 | - | return append(out, value...) | |
| 579 | - | } | |
| 580 | 598 | func atomicMigrationWrite(path string, data []byte) (retErr error) { | |
| 581 | 599 | dir := filepath.Dir(path) | |
| 582 | 600 | if err := safedir.EnsurePrivate(dir); err != nil { | |
| @@ -419,6 +419,17 @@ func installMigratedCatalogue(stateDir string, records []domain.CatalogueRecord) | |||||
| 419 | if err := p.Close(); err != nil { | 419 | if err := p.Close(); err != nil { | ||
| 420 | return err | 420 | return err | ||
| 421 | } | 421 | } | ||
| 422 | + | staged, opened, err := openCurrentCatalogue(stage, false) | |||
| 423 | + | if err != nil { | |||
| 424 | + | return err | |||
| 425 | + | } | |||
| 426 | + | if !catalogueRecordsEqual(records, opened) { | |||
| 427 | + | _ = staged.Close() | |||
| 428 | + | return errors.New("persist: staged catalogue validation mismatch") | |||
| 429 | + | } | |||
| 430 | + | if err := staged.Close(); err != nil { | |||
| 431 | + | return err | |||
| 432 | + | } | |||
| 422 | source := StorePath(stage) | 433 | source := StorePath(stage) | ||
| 423 | target := StorePath(stateDir) | 434 | target := StorePath(stateDir) | ||
| 424 | if err := os.Rename(source, target); err != nil { | 435 | if err := os.Rename(source, target); err != nil { | ||
| @@ -487,9 +498,13 @@ func decodeLegacyRecordV0(name string, value []byte) (legacyCatalogueRecordV0, e | |||||
| 487 | } | 498 | } | ||
| 488 | 499 | ||||
| 489 | func writeMigrationRecord(path string, record MigrationRecord) error { | 500 | func writeMigrationRecord(path string, record MigrationRecord) error { | ||
| 490 | - | return atomicMigrationWrite(path, encodeMigrationRecord(record)) | 501 | + | encoded, err := encodeMigrationRecord(record) |
| 502 | + | if err != nil { | |||
| 503 | + | return err | |||
| 504 | + | } | |||
| 505 | + | return atomicMigrationWrite(path, encoded) | |||
| 491 | } | 506 | } | ||
| 492 | - | func encodeMigrationRecord(record MigrationRecord) []byte { | 507 | + | func encodeMigrationRecord(record MigrationRecord) ([]byte, error) { |
| 493 | out := append([]byte(migrationRecordMagic), byte(migrationRecordVersion>>8), byte(migrationRecordVersion), byte(record.Phase)) | 508 | out := append([]byte(migrationRecordMagic), byte(migrationRecordVersion>>8), byte(migrationRecordVersion), byte(record.Phase)) | ||
| 494 | names := make([]string, 0, len(record.Assignments)) | 509 | names := make([]string, 0, len(record.Assignments)) | ||
| 495 | for name := range record.Assignments { | 510 | for name := range record.Assignments { | ||
| @@ -498,7 +513,11 @@ func encodeMigrationRecord(record MigrationRecord) []byte { | |||||
| 498 | sort.Strings(names) | 513 | sort.Strings(names) | ||
| 499 | out = binary.BigEndian.AppendUint32(out, uint32(len(names))) | 514 | out = binary.BigEndian.AppendUint32(out, uint32(len(names))) | ||
| 500 | for _, name := range names { | 515 | for _, name := range names { | ||
| 501 | - | out = appendMigrationString(out, name) | 516 | + | var err error |
| 517 | + | out, err = appendCheckedString(out, name) | |||
| 518 | + | if err != nil { | |||
| 519 | + | return nil, err | |||
| 520 | + | } | |||
| 502 | id := record.Assignments[name] | 521 | id := record.Assignments[name] | ||
| 503 | out = append(out, id[:]...) | 522 | out = append(out, id[:]...) | ||
| 504 | if ref := record.Validated[name]; ref != nil { | 523 | if ref := record.Validated[name]; ref != nil { | ||
| @@ -508,9 +527,12 @@ func encodeMigrationRecord(record MigrationRecord) []byte { | |||||
| 508 | } else { | 527 | } else { | ||
| 509 | out = append(out, 0) | 528 | out = append(out, 0) | ||
| 510 | } | 529 | } | ||
| 511 | - | out = appendMigrationString(out, record.Degraded[name]) | 530 | + | out, err = appendCheckedString(out, record.Degraded[name]) |
| 531 | + | if err != nil { | |||
| 532 | + | return nil, err | |||
| 533 | + | } | |||
| 512 | } | 534 | } | ||
| 513 | - | return out | 535 | + | return out, nil |
| 514 | } | 536 | } | ||
| 515 | func readMigrationRecord(path string) (MigrationRecord, error) { | 537 | func readMigrationRecord(path string) (MigrationRecord, error) { | ||
| 516 | data, err := os.ReadFile(path) | 538 | data, err := os.ReadFile(path) | ||
| @@ -573,10 +595,6 @@ func readMigrationRecord(path string) (MigrationRecord, error) { | |||||
| 573 | } | 595 | } | ||
| 574 | return record, nil | 596 | return record, nil | ||
| 575 | } | 597 | } | ||
| 576 | - | func appendMigrationString(out []byte, value string) []byte { | |||
| 577 | - | out = binary.BigEndian.AppendUint32(out, uint32(len(value))) | |||
| 578 | - | return append(out, value...) | |||
| 579 | - | } | |||
| 580 | func atomicMigrationWrite(path string, data []byte) (retErr error) { | 598 | func atomicMigrationWrite(path string, data []byte) (retErr error) { | ||
| 581 | dir := filepath.Dir(path) | 599 | dir := filepath.Dir(path) | ||
| 582 | if err := safedir.EnsurePrivate(dir); err != nil { | 600 | if err := safedir.EnsurePrivate(dir); err != nil { | ||
M internal/persist/migration_test.go +22 -10
| @@ -126,7 +126,7 @@ func TestMigrationExistingState(t *testing.T) { | |||
| 126 | 126 | require.NoError(t, reopened.Catalogue.Close()) | |
| 127 | 127 | } | |
| 128 | 128 | ||
| 129 | - | func TestMigrationOver4096(t *testing.T) { | |
| 129 | + | func TestMigrationPreservesLargeGeneration(t *testing.T) { | |
| 130 | 130 | dir := t.TempDir() | |
| 131 | 131 | writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "large", Cwd: "/tmp"}) | |
| 132 | 132 | migration := &migrationStub{heads: map[string]domain.CheckpointRef{"large": {Generation: 5001, ManifestDigest: [32]byte{1}}}, migrateErr: map[string]error{}} | |
| @@ -248,16 +248,28 @@ func TestMigrationPreservesUncertain(t *testing.T) { | |||
| 248 | 248 | require.LessOrEqual(t, len(result.Records[0].DegradedReason), migrationDegradedReasonMaxBytes) | |
| 249 | 249 | require.NotContains(t, result.Records[0].DegradedReason, adapterCause) | |
| 250 | 250 | require.Contains(t, logs.String(), adapterCause, "the full adapter cause must remain diagnosable") | |
| 251 | + | } | |
| 251 | 252 | ||
| 252 | - | failClosedDir := t.TempDir() | |
| 253 | - | writeLegacyCatalogue(t, failClosedDir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"}) | |
| 254 | - | _, err = OpenOrMigrate(context.Background(), OpenDeps{StateDir: failClosedDir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"work": ref}, migrateErr: map[string]error{"work": errors.New("repository unavailable")}}}) | |
| 255 | - | require.ErrorContains(t, err, "repository unavailable") | |
| 256 | - | data, replayErr := kv.Replay(StorePath(failClosedDir)) | |
| 257 | - | require.NoError(t, replayErr) | |
| 258 | - | _, legacyFormat, classifyErr := classifyCatalogue(data) | |
| 259 | - | require.NoError(t, classifyErr) | |
| 260 | - | require.True(t, legacyFormat, "migration-port failure must not install a new catalogue") | |
| 253 | + | func TestMigrationFailsClosedOnSnapshotError(t *testing.T) { | |
| 254 | + | for _, tc := range []struct { | |
| 255 | + | name string | |
| 256 | + | err error | |
| 257 | + | }{ | |
| 258 | + | {name: "repository unavailable", err: errors.New("repository unavailable")}, | |
| 259 | + | } { | |
| 260 | + | t.Run(tc.name, func(t *testing.T) { | |
| 261 | + | dir := t.TempDir() | |
| 262 | + | writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"}) | |
| 263 | + | ref := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}} | |
| 264 | + | _, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"work": ref}, migrateErr: map[string]error{"work": tc.err}}}) | |
| 265 | + | require.ErrorContains(t, err, tc.err.Error()) | |
| 266 | + | data, replayErr := kv.Replay(StorePath(dir)) | |
| 267 | + | require.NoError(t, replayErr) | |
| 268 | + | _, legacyFormat, classifyErr := classifyCatalogue(data) | |
| 269 | + | require.NoError(t, classifyErr) | |
| 270 | + | require.True(t, legacyFormat, "migration-port failure must not install a new catalogue") | |
| 271 | + | }) | |
| 272 | + | } | |
| 261 | 273 | } | |
| 262 | 274 | ||
| 263 | 275 | func TestOpenOrMigrateLegacyProbe(t *testing.T) { | |
| @@ -126,7 +126,7 @@ func TestMigrationExistingState(t *testing.T) { | |||||
| 126 | require.NoError(t, reopened.Catalogue.Close()) | 126 | require.NoError(t, reopened.Catalogue.Close()) | ||
| 127 | } | 127 | } | ||
| 128 | 128 | ||||
| 129 | - | func TestMigrationOver4096(t *testing.T) { | 129 | + | func TestMigrationPreservesLargeGeneration(t *testing.T) { |
| 130 | dir := t.TempDir() | 130 | dir := t.TempDir() | ||
| 131 | writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "large", Cwd: "/tmp"}) | 131 | writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "large", Cwd: "/tmp"}) | ||
| 132 | migration := &migrationStub{heads: map[string]domain.CheckpointRef{"large": {Generation: 5001, ManifestDigest: [32]byte{1}}}, migrateErr: map[string]error{}} | 132 | migration := &migrationStub{heads: map[string]domain.CheckpointRef{"large": {Generation: 5001, ManifestDigest: [32]byte{1}}}, migrateErr: map[string]error{}} | ||
| @@ -248,16 +248,28 @@ func TestMigrationPreservesUncertain(t *testing.T) { | |||||
| 248 | require.LessOrEqual(t, len(result.Records[0].DegradedReason), migrationDegradedReasonMaxBytes) | 248 | require.LessOrEqual(t, len(result.Records[0].DegradedReason), migrationDegradedReasonMaxBytes) | ||
| 249 | require.NotContains(t, result.Records[0].DegradedReason, adapterCause) | 249 | require.NotContains(t, result.Records[0].DegradedReason, adapterCause) | ||
| 250 | require.Contains(t, logs.String(), adapterCause, "the full adapter cause must remain diagnosable") | 250 | require.Contains(t, logs.String(), adapterCause, "the full adapter cause must remain diagnosable") | ||
| 251 | + | } | |||
| 251 | 252 | ||||
| 252 | - | failClosedDir := t.TempDir() | 253 | + | func TestMigrationFailsClosedOnSnapshotError(t *testing.T) { |
| 253 | - | writeLegacyCatalogue(t, failClosedDir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"}) | 254 | + | for _, tc := range []struct { |
| 254 | - | _, err = OpenOrMigrate(context.Background(), OpenDeps{StateDir: failClosedDir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"work": ref}, migrateErr: map[string]error{"work": errors.New("repository unavailable")}}}) | 255 | + | name string |
| 255 | - | require.ErrorContains(t, err, "repository unavailable") | 256 | + | err error |
| 256 | - | data, replayErr := kv.Replay(StorePath(failClosedDir)) | 257 | + | }{ |
| 257 | - | require.NoError(t, replayErr) | 258 | + | {name: "repository unavailable", err: errors.New("repository unavailable")}, |
| 258 | - | _, legacyFormat, classifyErr := classifyCatalogue(data) | 259 | + | } { |
| 259 | - | require.NoError(t, classifyErr) | 260 | + | t.Run(tc.name, func(t *testing.T) { |
| 260 | - | require.True(t, legacyFormat, "migration-port failure must not install a new catalogue") | 261 | + | dir := t.TempDir() |
| 262 | + | writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"}) | |||
| 263 | + | ref := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}} | |||
| 264 | + | _, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"work": ref}, migrateErr: map[string]error{"work": tc.err}}}) | |||
| 265 | + | require.ErrorContains(t, err, tc.err.Error()) | |||
| 266 | + | data, replayErr := kv.Replay(StorePath(dir)) | |||
| 267 | + | require.NoError(t, replayErr) | |||
| 268 | + | _, legacyFormat, classifyErr := classifyCatalogue(data) | |||
| 269 | + | require.NoError(t, classifyErr) | |||
| 270 | + | require.True(t, legacyFormat, "migration-port failure must not install a new catalogue") | |||
| 271 | + | }) | |||
| 272 | + | } | |||
| 261 | } | 273 | } | ||
| 262 | 274 | ||||
| 263 | func TestOpenOrMigrateLegacyProbe(t *testing.T) { | 275 | func TestOpenOrMigrateLegacyProbe(t *testing.T) { | ||
M internal/persist/persist.go +69 -21
| @@ -21,6 +21,10 @@ func StorePath(dir string) string { return filepath.Join(dir, filename) } | |||
| 21 | 21 | type Persister struct { | |
| 22 | 22 | store ports.Store | |
| 23 | 23 | mu sync.Mutex | |
| 24 | + | ||
| 25 | + | incarnationOwners map[domain.IncarnationID]string | |
| 26 | + | nameIncarnations map[string]domain.IncarnationID | |
| 27 | + | incarnationIndex bool | |
| 24 | 28 | } | |
| 25 | 29 | ||
| 26 | 30 | // KVStore adapts the reusable WAL implementation to the persistence port. | |
| @@ -146,19 +150,8 @@ func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) erro | |||
| 146 | 150 | if len(records) == 0 { | |
| 147 | 151 | return errors.New("persist: empty catalogue batch") | |
| 148 | 152 | } | |
| 149 | - | projected := make(map[string]domain.CatalogueRecord) | |
| 150 | - | var decodeErr error | |
| 151 | - | p.store.Range(func(key, value []byte) bool { | |
| 152 | - | record, err := decodeRecordValue(string(key), value) | |
| 153 | - | if err != nil { | |
| 154 | - | decodeErr = err | |
| 155 | - | return false | |
| 156 | - | } | |
| 157 | - | projected[string(key)] = record | |
| 158 | - | return true | |
| 159 | - | }) | |
| 160 | - | if decodeErr != nil { | |
| 161 | - | return decodeErr | |
| 153 | + | if err := p.ensureIncarnationIndexLocked(); err != nil { | |
| 154 | + | return err | |
| 162 | 155 | } | |
| 163 | 156 | changes := make([]ports.StoreChange, 0, len(records)) | |
| 164 | 157 | keys := make([]string, 0, len(records)) | |
| @@ -172,7 +165,6 @@ func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) erro | |||
| 172 | 165 | return err | |
| 173 | 166 | } | |
| 174 | 167 | if record == nil { | |
| 175 | - | delete(projected, name) | |
| 176 | 168 | changes = append(changes, ports.StoreChange{Key: []byte(name), Delete: true}) | |
| 177 | 169 | continue | |
| 178 | 170 | } | |
| @@ -183,20 +175,76 @@ func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) erro | |||
| 183 | 175 | if err != nil { | |
| 184 | 176 | return err | |
| 185 | 177 | } | |
| 186 | - | projected[name] = *record | |
| 187 | 178 | changes = append(changes, ports.StoreChange{Key: []byte(name), Value: value}) | |
| 188 | 179 | } | |
| 189 | - | all := make([]domain.CatalogueRecord, 0, len(projected)) | |
| 190 | - | for _, record := range projected { | |
| 191 | - | all = append(all, record) | |
| 180 | + | incoming := make(map[domain.IncarnationID]string, len(records)) | |
| 181 | + | for name, record := range records { | |
| 182 | + | if record == nil { | |
| 183 | + | continue | |
| 184 | + | } | |
| 185 | + | if previous, ok := incoming[record.IncarnationID]; ok && previous != name { | |
| 186 | + | return fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, name) | |
| 187 | + | } | |
| 188 | + | incoming[record.IncarnationID] = name | |
| 189 | + | if owner, ok := p.incarnationOwners[record.IncarnationID]; ok && owner != name { | |
| 190 | + | ownerNext, ownerTouched := records[owner] | |
| 191 | + | if !ownerTouched || (ownerNext != nil && ownerNext.IncarnationID == record.IncarnationID) { | |
| 192 | + | return fmt.Errorf("persist: duplicate incarnation for %q and %q", owner, name) | |
| 193 | + | } | |
| 194 | + | } | |
| 192 | 195 | } | |
| 193 | - | if err := validateUniqueIncarnations(all); err != nil { | |
| 196 | + | if err := p.store.Batch(changes); err != nil { | |
| 197 | + | p.incarnationIndex = false | |
| 194 | 198 | return err | |
| 195 | 199 | } | |
| 196 | - | if err := p.store.Batch(changes); err != nil { | |
| 200 | + | if err := p.store.Sync(); err != nil { | |
| 201 | + | p.incarnationIndex = false | |
| 197 | 202 | return err | |
| 198 | 203 | } | |
| 199 | - | return p.store.Sync() | |
| 204 | + | for name := range records { | |
| 205 | + | if id, ok := p.nameIncarnations[name]; ok { | |
| 206 | + | delete(p.incarnationOwners, id) | |
| 207 | + | delete(p.nameIncarnations, name) | |
| 208 | + | } | |
| 209 | + | } | |
| 210 | + | for name, record := range records { | |
| 211 | + | if record != nil { | |
| 212 | + | p.incarnationOwners[record.IncarnationID] = name | |
| 213 | + | p.nameIncarnations[name] = record.IncarnationID | |
| 214 | + | } | |
| 215 | + | } | |
| 216 | + | return nil | |
| 217 | + | } | |
| 218 | + | ||
| 219 | + | func (p *Persister) ensureIncarnationIndexLocked() error { | |
| 220 | + | if p.incarnationIndex { | |
| 221 | + | return nil | |
| 222 | + | } | |
| 223 | + | owners := make(map[domain.IncarnationID]string) | |
| 224 | + | names := make(map[string]domain.IncarnationID) | |
| 225 | + | var indexErr error | |
| 226 | + | p.store.Range(func(key, value []byte) bool { | |
| 227 | + | name := string(key) | |
| 228 | + | record, err := decodeRecordValue(name, value) | |
| 229 | + | if err != nil { | |
| 230 | + | indexErr = err | |
| 231 | + | return false | |
| 232 | + | } | |
| 233 | + | if previous, ok := owners[record.IncarnationID]; ok { | |
| 234 | + | indexErr = fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, name) | |
| 235 | + | return false | |
| 236 | + | } | |
| 237 | + | owners[record.IncarnationID] = name | |
| 238 | + | names[name] = record.IncarnationID | |
| 239 | + | return true | |
| 240 | + | }) | |
| 241 | + | if indexErr != nil { | |
| 242 | + | return indexErr | |
| 243 | + | } | |
| 244 | + | p.incarnationOwners = owners | |
| 245 | + | p.nameIncarnations = names | |
| 246 | + | p.incarnationIndex = true | |
| 247 | + | return nil | |
| 200 | 248 | } | |
| 201 | 249 | func (p *Persister) Rename(oldName string, next domain.CatalogueRecord) error { | |
| 202 | 250 | if p == nil { | |
| @@ -21,6 +21,10 @@ func StorePath(dir string) string { return filepath.Join(dir, filename) } | |||||
| 21 | type Persister struct { | 21 | type Persister struct { | ||
| 22 | store ports.Store | 22 | store ports.Store | ||
| 23 | mu sync.Mutex | 23 | mu sync.Mutex | ||
| 24 | + | ||||
| 25 | + | incarnationOwners map[domain.IncarnationID]string | |||
| 26 | + | nameIncarnations map[string]domain.IncarnationID | |||
| 27 | + | incarnationIndex bool | |||
| 24 | } | 28 | } | ||
| 25 | 29 | ||||
| 26 | // KVStore adapts the reusable WAL implementation to the persistence port. | 30 | // KVStore adapts the reusable WAL implementation to the persistence port. | ||
| @@ -146,19 +150,8 @@ func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) erro | |||||
| 146 | if len(records) == 0 { | 150 | if len(records) == 0 { | ||
| 147 | return errors.New("persist: empty catalogue batch") | 151 | return errors.New("persist: empty catalogue batch") | ||
| 148 | } | 152 | } | ||
| 149 | - | projected := make(map[string]domain.CatalogueRecord) | 153 | + | if err := p.ensureIncarnationIndexLocked(); err != nil { |
| 150 | - | var decodeErr error | 154 | + | return err |
| 151 | - | p.store.Range(func(key, value []byte) bool { | |||
| 152 | - | record, err := decodeRecordValue(string(key), value) | |||
| 153 | - | if err != nil { | |||
| 154 | - | decodeErr = err | |||
| 155 | - | return false | |||
| 156 | - | } | |||
| 157 | - | projected[string(key)] = record | |||
| 158 | - | return true | |||
| 159 | - | }) | |||
| 160 | - | if decodeErr != nil { | |||
| 161 | - | return decodeErr | |||
| 162 | } | 155 | } | ||
| 163 | changes := make([]ports.StoreChange, 0, len(records)) | 156 | changes := make([]ports.StoreChange, 0, len(records)) | ||
| 164 | keys := make([]string, 0, len(records)) | 157 | keys := make([]string, 0, len(records)) | ||
| @@ -172,7 +165,6 @@ func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) erro | |||||
| 172 | return err | 165 | return err | ||
| 173 | } | 166 | } | ||
| 174 | if record == nil { | 167 | if record == nil { | ||
| 175 | - | delete(projected, name) | |||
| 176 | changes = append(changes, ports.StoreChange{Key: []byte(name), Delete: true}) | 168 | changes = append(changes, ports.StoreChange{Key: []byte(name), Delete: true}) | ||
| 177 | continue | 169 | continue | ||
| 178 | } | 170 | } | ||
| @@ -183,20 +175,76 @@ func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord) erro | |||||
| 183 | if err != nil { | 175 | if err != nil { | ||
| 184 | return err | 176 | return err | ||
| 185 | } | 177 | } | ||
| 186 | - | projected[name] = *record | |||
| 187 | changes = append(changes, ports.StoreChange{Key: []byte(name), Value: value}) | 178 | changes = append(changes, ports.StoreChange{Key: []byte(name), Value: value}) | ||
| 188 | } | 179 | } | ||
| 189 | - | all := make([]domain.CatalogueRecord, 0, len(projected)) | 180 | + | incoming := make(map[domain.IncarnationID]string, len(records)) |
| 190 | - | for _, record := range projected { | 181 | + | for name, record := range records { |
| 191 | - | all = append(all, record) | 182 | + | if record == nil { |
| 183 | + | continue | |||
| 184 | + | } | |||
| 185 | + | if previous, ok := incoming[record.IncarnationID]; ok && previous != name { | |||
| 186 | + | return fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, name) | |||
| 187 | + | } | |||
| 188 | + | incoming[record.IncarnationID] = name | |||
| 189 | + | if owner, ok := p.incarnationOwners[record.IncarnationID]; ok && owner != name { | |||
| 190 | + | ownerNext, ownerTouched := records[owner] | |||
| 191 | + | if !ownerTouched || (ownerNext != nil && ownerNext.IncarnationID == record.IncarnationID) { | |||
| 192 | + | return fmt.Errorf("persist: duplicate incarnation for %q and %q", owner, name) | |||
| 193 | + | } | |||
| 194 | + | } | |||
| 192 | } | 195 | } | ||
| 193 | - | if err := validateUniqueIncarnations(all); err != nil { | 196 | + | if err := p.store.Batch(changes); err != nil { |
| 197 | + | p.incarnationIndex = false | |||
| 194 | return err | 198 | return err | ||
| 195 | } | 199 | } | ||
| 196 | - | if err := p.store.Batch(changes); err != nil { | 200 | + | if err := p.store.Sync(); err != nil { |
| 201 | + | p.incarnationIndex = false | |||
| 197 | return err | 202 | return err | ||
| 198 | } | 203 | } | ||
| 199 | - | return p.store.Sync() | 204 | + | for name := range records { |
| 205 | + | if id, ok := p.nameIncarnations[name]; ok { | |||
| 206 | + | delete(p.incarnationOwners, id) | |||
| 207 | + | delete(p.nameIncarnations, name) | |||
| 208 | + | } | |||
| 209 | + | } | |||
| 210 | + | for name, record := range records { | |||
| 211 | + | if record != nil { | |||
| 212 | + | p.incarnationOwners[record.IncarnationID] = name | |||
| 213 | + | p.nameIncarnations[name] = record.IncarnationID | |||
| 214 | + | } | |||
| 215 | + | } | |||
| 216 | + | return nil | |||
| 217 | + | } | |||
| 218 | + | ||||
| 219 | + | func (p *Persister) ensureIncarnationIndexLocked() error { | |||
| 220 | + | if p.incarnationIndex { | |||
| 221 | + | return nil | |||
| 222 | + | } | |||
| 223 | + | owners := make(map[domain.IncarnationID]string) | |||
| 224 | + | names := make(map[string]domain.IncarnationID) | |||
| 225 | + | var indexErr error | |||
| 226 | + | p.store.Range(func(key, value []byte) bool { | |||
| 227 | + | name := string(key) | |||
| 228 | + | record, err := decodeRecordValue(name, value) | |||
| 229 | + | if err != nil { | |||
| 230 | + | indexErr = err | |||
| 231 | + | return false | |||
| 232 | + | } | |||
| 233 | + | if previous, ok := owners[record.IncarnationID]; ok { | |||
| 234 | + | indexErr = fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, name) | |||
| 235 | + | return false | |||
| 236 | + | } | |||
| 237 | + | owners[record.IncarnationID] = name | |||
| 238 | + | names[name] = record.IncarnationID | |||
| 239 | + | return true | |||
| 240 | + | }) | |||
| 241 | + | if indexErr != nil { | |||
| 242 | + | return indexErr | |||
| 243 | + | } | |||
| 244 | + | p.incarnationOwners = owners | |||
| 245 | + | p.nameIncarnations = names | |||
| 246 | + | p.incarnationIndex = true | |||
| 247 | + | return nil | |||
| 200 | } | 248 | } | ||
| 201 | func (p *Persister) Rename(oldName string, next domain.CatalogueRecord) error { | 249 | func (p *Persister) Rename(oldName string, next domain.CatalogueRecord) error { | ||
| 202 | if p == nil { | 250 | if p == nil { | ||
M internal/persist/persist_test.go +1 -1
| @@ -34,7 +34,7 @@ func TestCatalogueBatchSyncBehavior(t *testing.T) { | |||
| 34 | 34 | return | |
| 35 | 35 | } | |
| 36 | 36 | } | |
| 37 | - | }).Twice() | |
| 37 | + | }).Once() | |
| 38 | 38 | store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error { | |
| 39 | 39 | for _, change := range changes { | |
| 40 | 40 | if change.Delete { | |
| @@ -34,7 +34,7 @@ func TestCatalogueBatchSyncBehavior(t *testing.T) { | |||||
| 34 | return | 34 | return | ||
| 35 | } | 35 | } | ||
| 36 | } | 36 | } | ||
| 37 | - | }).Twice() | 37 | + | }).Once() |
| 38 | store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error { | 38 | store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error { | ||
| 39 | for _, change := range changes { | 39 | for _, change := range changes { | ||
| 40 | if change.Delete { | 40 | if change.Delete { | ||
M internal/persist/record.go +2 -5
| @@ -125,15 +125,12 @@ func decodeRecordValue(name string, value []byte) (domain.CatalogueRecord, error | |||
| 125 | 125 | return record, nil | |
| 126 | 126 | } | |
| 127 | 127 | ||
| 128 | - | func appendString(buf []byte, value string) []byte { | |
| 129 | - | buf = binary.BigEndian.AppendUint32(buf, uint32(len(value))) | |
| 130 | - | return append(buf, value...) | |
| 131 | - | } | |
| 132 | 128 | func appendCheckedString(buf []byte, value string) ([]byte, error) { | |
| 133 | 129 | if uint64(len(value)) > math.MaxUint32 { | |
| 134 | 130 | return nil, errors.New("persist: string too large") | |
| 135 | 131 | } | |
| 136 | - | return appendString(buf, value), nil | |
| 132 | + | buf = binary.BigEndian.AppendUint32(buf, uint32(len(value))) | |
| 133 | + | return append(buf, value...), nil | |
| 137 | 134 | } | |
| 138 | 135 | func appendRef(buf []byte, ref *domain.CheckpointRef) []byte { | |
| 139 | 136 | if ref == nil { | |
| @@ -125,15 +125,12 @@ func decodeRecordValue(name string, value []byte) (domain.CatalogueRecord, error | |||||
| 125 | return record, nil | 125 | return record, nil | ||
| 126 | } | 126 | } | ||
| 127 | 127 | ||||
| 128 | - | func appendString(buf []byte, value string) []byte { | |||
| 129 | - | buf = binary.BigEndian.AppendUint32(buf, uint32(len(value))) | |||
| 130 | - | return append(buf, value...) | |||
| 131 | - | } | |||
| 132 | func appendCheckedString(buf []byte, value string) ([]byte, error) { | 128 | func appendCheckedString(buf []byte, value string) ([]byte, error) { | ||
| 133 | if uint64(len(value)) > math.MaxUint32 { | 129 | if uint64(len(value)) > math.MaxUint32 { | ||
| 134 | return nil, errors.New("persist: string too large") | 130 | return nil, errors.New("persist: string too large") | ||
| 135 | } | 131 | } | ||
| 136 | - | return appendString(buf, value), nil | 132 | + | buf = binary.BigEndian.AppendUint32(buf, uint32(len(value))) |
| 133 | + | return append(buf, value...), nil | |||
| 137 | } | 134 | } | ||
| 138 | func appendRef(buf []byte, ref *domain.CheckpointRef) []byte { | 135 | func appendRef(buf []byte, ref *domain.CheckpointRef) []byte { | ||
| 139 | if ref == nil { | 136 | if ref == nil { | ||
M internal/ports/wire_test.go +6 -0
| @@ -765,6 +765,12 @@ func TestSessionInfoRecoveryState(t *testing.T) { | |||
| 765 | 765 | assertTrailingGarbageFails(t, payload, UnmarshalSessions) | |
| 766 | 766 | }) | |
| 767 | 767 | } | |
| 768 | + | for _, state := range []byte{0, 5} { | |
| 769 | + | payload := append([]byte(nil), tests[0].want...) | |
| 770 | + | payload[len(payload)-1] = state | |
| 771 | + | _, err := UnmarshalSessions(payload) | |
| 772 | + | require.Error(t, err, "state %d", state) | |
| 773 | + | } | |
| 768 | 774 | } | |
| 769 | 775 | ||
| 770 | 776 | func TestSessionsGoldenAndRoundTrip(t *testing.T) { | |
| @@ -765,6 +765,12 @@ func TestSessionInfoRecoveryState(t *testing.T) { | |||||
| 765 | assertTrailingGarbageFails(t, payload, UnmarshalSessions) | 765 | assertTrailingGarbageFails(t, payload, UnmarshalSessions) | ||
| 766 | }) | 766 | }) | ||
| 767 | } | 767 | } | ||
| 768 | + | for _, state := range []byte{0, 5} { | |||
| 769 | + | payload := append([]byte(nil), tests[0].want...) | |||
| 770 | + | payload[len(payload)-1] = state | |||
| 771 | + | _, err := UnmarshalSessions(payload) | |||
| 772 | + | require.Error(t, err, "state %d", state) | |||
| 773 | + | } | |||
| 768 | } | 774 | } | ||
| 769 | 775 | ||||
| 770 | func TestSessionsGoldenAndRoundTrip(t *testing.T) { | 776 | func TestSessionsGoldenAndRoundTrip(t *testing.T) { | ||
M internal/usecase/command/command_test.go +25 -7
| @@ -6,13 +6,31 @@ import ( | |||
| 6 | 6 | ) | |
| 7 | 7 | ||
| 8 | 8 | func TestParsePositiveUint64(t *testing.T) { | |
| 9 | - | const maxUint64 = "18446744073709551615" | |
| 10 | - | got, err := ParsePositiveUint64([]string{maxUint64}) | |
| 11 | - | if err != nil || got != ^uint64(0) { | |
| 12 | - | t.Fatalf("ParsePositiveUint64(max) = %d, %v", got, err) | |
| 13 | - | } | |
| 14 | - | if _, err := ParsePositiveUint64([]string{"18446744073709551616"}); !errors.Is(err, ErrInvalidArguments) { | |
| 15 | - | t.Fatalf("ParsePositiveUint64(overflow) error = %v", err) | |
| 9 | + | for _, tt := range []struct { | |
| 10 | + | name string | |
| 11 | + | args []string | |
| 12 | + | want uint64 | |
| 13 | + | invalid bool | |
| 14 | + | }{ | |
| 15 | + | {name: "maximum", args: []string{"18446744073709551615"}, want: ^uint64(0)}, | |
| 16 | + | {name: "overflow", args: []string{"18446744073709551616"}, invalid: true}, | |
| 17 | + | {name: "zero", args: []string{"0"}, invalid: true}, | |
| 18 | + | {name: "leading zero", args: []string{"01"}, invalid: true}, | |
| 19 | + | {name: "empty arguments", invalid: true}, | |
| 20 | + | {name: "multiple arguments", args: []string{"1", "2"}, invalid: true}, | |
| 21 | + | } { | |
| 22 | + | t.Run(tt.name, func(t *testing.T) { | |
| 23 | + | got, err := ParsePositiveUint64(tt.args) | |
| 24 | + | if tt.invalid { | |
| 25 | + | if !errors.Is(err, ErrInvalidArguments) { | |
| 26 | + | t.Fatalf("error = %v", err) | |
| 27 | + | } | |
| 28 | + | return | |
| 29 | + | } | |
| 30 | + | if err != nil || got != tt.want { | |
| 31 | + | t.Fatalf("ParsePositiveUint64() = %d, %v", got, err) | |
| 32 | + | } | |
| 33 | + | }) | |
| 16 | 34 | } | |
| 17 | 35 | } | |
| 18 | 36 | ||
| @@ -6,13 +6,31 @@ import ( | |||||
| 6 | ) | 6 | ) | ||
| 7 | 7 | ||||
| 8 | func TestParsePositiveUint64(t *testing.T) { | 8 | func TestParsePositiveUint64(t *testing.T) { | ||
| 9 | - | const maxUint64 = "18446744073709551615" | 9 | + | for _, tt := range []struct { |
| 10 | - | got, err := ParsePositiveUint64([]string{maxUint64}) | 10 | + | name string |
| 11 | - | if err != nil || got != ^uint64(0) { | 11 | + | args []string |
| 12 | - | t.Fatalf("ParsePositiveUint64(max) = %d, %v", got, err) | 12 | + | want uint64 |
| 13 | - | } | 13 | + | invalid bool |
| 14 | - | if _, err := ParsePositiveUint64([]string{"18446744073709551616"}); !errors.Is(err, ErrInvalidArguments) { | 14 | + | }{ |
| 15 | - | t.Fatalf("ParsePositiveUint64(overflow) error = %v", err) | 15 | + | {name: "maximum", args: []string{"18446744073709551615"}, want: ^uint64(0)}, |
| 16 | + | {name: "overflow", args: []string{"18446744073709551616"}, invalid: true}, | |||
| 17 | + | {name: "zero", args: []string{"0"}, invalid: true}, | |||
| 18 | + | {name: "leading zero", args: []string{"01"}, invalid: true}, | |||
| 19 | + | {name: "empty arguments", invalid: true}, | |||
| 20 | + | {name: "multiple arguments", args: []string{"1", "2"}, invalid: true}, | |||
| 21 | + | } { | |||
| 22 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 23 | + | got, err := ParsePositiveUint64(tt.args) | |||
| 24 | + | if tt.invalid { | |||
| 25 | + | if !errors.Is(err, ErrInvalidArguments) { | |||
| 26 | + | t.Fatalf("error = %v", err) | |||
| 27 | + | } | |||
| 28 | + | return | |||
| 29 | + | } | |||
| 30 | + | if err != nil || got != tt.want { | |||
| 31 | + | t.Fatalf("ParsePositiveUint64() = %d, %v", got, err) | |||
| 32 | + | } | |||
| 33 | + | }) | |||
| 16 | } | 34 | } | ||
| 17 | } | 35 | } | ||
| 18 | 36 | ||||
M internal/usecase/daemon/catalogue.go +10 -9
| @@ -25,15 +25,15 @@ func (d *Daemon) updateCatalogueMetadata(update domain.CatalogueMetadataUpdate) | |||
| 25 | 25 | // session state locks during I/O. Only a successfully committed newer revision | |
| 26 | 26 | // subsumes an older caller. Failed revisions retain their rollback until later | |
| 27 | 27 | // failed mutations have unwound, keeping live metadata at the durable boundary. | |
| 28 | - | func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update domain.CatalogueMetadataUpdate, rollback func() bool) error { | |
| 28 | + | func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update domain.CatalogueMetadataUpdate, rollback func() bool) (bool, error) { | |
| 29 | 29 | if sess == nil { | |
| 30 | - | return nil | |
| 30 | + | return false, nil | |
| 31 | 31 | } | |
| 32 | 32 | sess.metadataPersistMu.Lock() | |
| 33 | 33 | defer sess.metadataPersistMu.Unlock() | |
| 34 | 34 | ||
| 35 | 35 | if sess.metadataDurableVersion >= version { | |
| 36 | - | return nil | |
| 36 | + | return false, nil | |
| 37 | 37 | } | |
| 38 | 38 | if err := d.updateCatalogueMetadata(update); err != nil { | |
| 39 | 39 | if sess.metadataFailedRollbacks == nil { | |
| @@ -48,8 +48,8 @@ func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update do | |||
| 48 | 48 | if current > sess.metadataLiveVersion { | |
| 49 | 49 | sess.metadataLiveVersion = current | |
| 50 | 50 | } | |
| 51 | - | sess.reconcileFailedMetadataLocked() | |
| 52 | - | return err | |
| 51 | + | rollbackRejected := sess.reconcileFailedMetadataLocked() | |
| 52 | + | return rollbackRejected, err | |
| 53 | 53 | } | |
| 54 | 54 | ||
| 55 | 55 | sess.metadataDurableVersion = version | |
| @@ -58,24 +58,25 @@ func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update do | |||
| 58 | 58 | delete(sess.metadataFailedRollbacks, failedVersion) | |
| 59 | 59 | } | |
| 60 | 60 | } | |
| 61 | - | return nil | |
| 61 | + | return false, nil | |
| 62 | 62 | } | |
| 63 | 63 | ||
| 64 | 64 | // reconcileFailedMetadataLocked unwinds only a contiguous failed suffix. An | |
| 65 | 65 | // older failure remains pending while a newer mutation is still writing. | |
| 66 | 66 | // Rollbacks run only after authority I/O has returned and acquire state locks in | |
| 67 | 67 | // the daemon-before-session order. | |
| 68 | - | func (s *session) reconcileFailedMetadataLocked() { | |
| 68 | + | func (s *session) reconcileFailedMetadataLocked() bool { | |
| 69 | 69 | for s.metadataLiveVersion > s.metadataDurableVersion { | |
| 70 | 70 | version := s.metadataLiveVersion | |
| 71 | 71 | rollback, failed := s.metadataFailedRollbacks[version] | |
| 72 | 72 | if !failed { | |
| 73 | - | return | |
| 73 | + | return false | |
| 74 | 74 | } | |
| 75 | 75 | if rollback != nil && !rollback() { | |
| 76 | - | return | |
| 76 | + | return true | |
| 77 | 77 | } | |
| 78 | 78 | delete(s.metadataFailedRollbacks, version) | |
| 79 | 79 | s.metadataLiveVersion-- | |
| 80 | 80 | } | |
| 81 | + | return false | |
| 81 | 82 | } | |
| @@ -25,15 +25,15 @@ func (d *Daemon) updateCatalogueMetadata(update domain.CatalogueMetadataUpdate) | |||||
| 25 | // session state locks during I/O. Only a successfully committed newer revision | 25 | // session state locks during I/O. Only a successfully committed newer revision | ||
| 26 | // subsumes an older caller. Failed revisions retain their rollback until later | 26 | // subsumes an older caller. Failed revisions retain their rollback until later | ||
| 27 | // failed mutations have unwound, keeping live metadata at the durable boundary. | 27 | // failed mutations have unwound, keeping live metadata at the durable boundary. | ||
| 28 | - | func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update domain.CatalogueMetadataUpdate, rollback func() bool) error { | 28 | + | func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update domain.CatalogueMetadataUpdate, rollback func() bool) (bool, error) { |
| 29 | if sess == nil { | 29 | if sess == nil { | ||
| 30 | - | return nil | 30 | + | return false, nil |
| 31 | } | 31 | } | ||
| 32 | sess.metadataPersistMu.Lock() | 32 | sess.metadataPersistMu.Lock() | ||
| 33 | defer sess.metadataPersistMu.Unlock() | 33 | defer sess.metadataPersistMu.Unlock() | ||
| 34 | 34 | ||||
| 35 | if sess.metadataDurableVersion >= version { | 35 | if sess.metadataDurableVersion >= version { | ||
| 36 | - | return nil | 36 | + | return false, nil |
| 37 | } | 37 | } | ||
| 38 | if err := d.updateCatalogueMetadata(update); err != nil { | 38 | if err := d.updateCatalogueMetadata(update); err != nil { | ||
| 39 | if sess.metadataFailedRollbacks == nil { | 39 | if sess.metadataFailedRollbacks == nil { | ||
| @@ -48,8 +48,8 @@ func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update do | |||||
| 48 | if current > sess.metadataLiveVersion { | 48 | if current > sess.metadataLiveVersion { | ||
| 49 | sess.metadataLiveVersion = current | 49 | sess.metadataLiveVersion = current | ||
| 50 | } | 50 | } | ||
| 51 | - | sess.reconcileFailedMetadataLocked() | 51 | + | rollbackRejected := sess.reconcileFailedMetadataLocked() |
| 52 | - | return err | 52 | + | return rollbackRejected, err |
| 53 | } | 53 | } | ||
| 54 | 54 | ||||
| 55 | sess.metadataDurableVersion = version | 55 | sess.metadataDurableVersion = version | ||
| @@ -58,24 +58,25 @@ func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update do | |||||
| 58 | delete(sess.metadataFailedRollbacks, failedVersion) | 58 | delete(sess.metadataFailedRollbacks, failedVersion) | ||
| 59 | } | 59 | } | ||
| 60 | } | 60 | } | ||
| 61 | - | return nil | 61 | + | return false, nil |
| 62 | } | 62 | } | ||
| 63 | 63 | ||||
| 64 | // reconcileFailedMetadataLocked unwinds only a contiguous failed suffix. An | 64 | // reconcileFailedMetadataLocked unwinds only a contiguous failed suffix. An | ||
| 65 | // older failure remains pending while a newer mutation is still writing. | 65 | // older failure remains pending while a newer mutation is still writing. | ||
| 66 | // Rollbacks run only after authority I/O has returned and acquire state locks in | 66 | // Rollbacks run only after authority I/O has returned and acquire state locks in | ||
| 67 | // the daemon-before-session order. | 67 | // the daemon-before-session order. | ||
| 68 | - | func (s *session) reconcileFailedMetadataLocked() { | 68 | + | func (s *session) reconcileFailedMetadataLocked() bool { |
| 69 | for s.metadataLiveVersion > s.metadataDurableVersion { | 69 | for s.metadataLiveVersion > s.metadataDurableVersion { | ||
| 70 | version := s.metadataLiveVersion | 70 | version := s.metadataLiveVersion | ||
| 71 | rollback, failed := s.metadataFailedRollbacks[version] | 71 | rollback, failed := s.metadataFailedRollbacks[version] | ||
| 72 | if !failed { | 72 | if !failed { | ||
| 73 | - | return | 73 | + | return false |
| 74 | } | 74 | } | ||
| 75 | if rollback != nil && !rollback() { | 75 | if rollback != nil && !rollback() { | ||
| 76 | - | return | 76 | + | return true |
| 77 | } | 77 | } | ||
| 78 | delete(s.metadataFailedRollbacks, version) | 78 | delete(s.metadataFailedRollbacks, version) | ||
| 79 | s.metadataLiveVersion-- | 79 | s.metadataLiveVersion-- | ||
| 80 | } | 80 | } | ||
| 81 | + | return false | |||
| 81 | } | 82 | } | ||
A internal/usecase/daemon/coderabbit_regression_test.go +63 -0
| @@ -0,0 +1,63 @@ | |||
| 1 | + | package daemon | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "sync" | |
| 6 | + | "testing" | |
| 7 | + | ||
| 8 | + | "github.com/stretchr/testify/require" | |
| 9 | + | ||
| 10 | + | "github.com/bnema/vev/internal/domain" | |
| 11 | + | "github.com/bnema/vev/internal/ports" | |
| 12 | + | ) | |
| 13 | + | ||
| 14 | + | func TestRestoreTransientLoadFailurePreservesHealthyAuthority(t *testing.T) { | |
| 15 | + | record := durableRecoveryRecord(0) | |
| 16 | + | repository := &durableRecoveryRepository{ | |
| 17 | + | errors: map[string]error{record.Name: ports.ErrBudgetExhausted}, | |
| 18 | + | loads: make(map[string]int), | |
| 19 | + | repairs: make(map[string]domain.CheckpointRef), | |
| 20 | + | } | |
| 21 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 22 | + | ||
| 23 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 24 | + | ||
| 25 | + | persisted, ok, err := catalogue.Record(record.Name) | |
| 26 | + | require.NoError(t, err) | |
| 27 | + | require.True(t, ok) | |
| 28 | + | require.Equal(t, domain.RecoveryHealthy, persisted.RecoveryState) | |
| 29 | + | d.mu.Lock() | |
| 30 | + | entry := d.stopped[record.Name] | |
| 31 | + | d.mu.Unlock() | |
| 32 | + | require.Equal(t, runtimeRestoring, entry.state) | |
| 33 | + | require.NotEqual(t, runtimeDegraded, entry.state) | |
| 34 | + | select { | |
| 35 | + | case <-entry.restoreDone: | |
| 36 | + | default: | |
| 37 | + | t.Fatal("restore completion was not signaled") | |
| 38 | + | } | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | func TestFinishRecordRestoreAlwaysClosesProvidedBarrier(t *testing.T) { | |
| 42 | + | d := newTestDaemon(t, nil, stubClock{}) | |
| 43 | + | done := make(chan struct{}) | |
| 44 | + | ||
| 45 | + | d.finishRecordRestore(domain.CatalogueRecord{Name: "removed"}, nil, done) | |
| 46 | + | ||
| 47 | + | select { | |
| 48 | + | case <-done: | |
| 49 | + | default: | |
| 50 | + | t.Fatal("restore completion was not signaled") | |
| 51 | + | } | |
| 52 | + | } | |
| 53 | + | ||
| 54 | + | func TestSnapshotStopContextCancelIsIdempotent(t *testing.T) { | |
| 55 | + | deadline := &snapshotShutdownDeadline{done: make(chan struct{})} | |
| 56 | + | _, cancel := snapshotStopContext(deadline) | |
| 57 | + | ||
| 58 | + | var wg sync.WaitGroup | |
| 59 | + | for range 8 { | |
| 60 | + | wg.Go(cancel) | |
| 61 | + | } | |
| 62 | + | wg.Wait() | |
| 63 | + | } | |
| @@ -0,0 +1,63 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "sync" | |||
| 6 | + | "testing" | |||
| 7 | + | ||||
| 8 | + | "github.com/stretchr/testify/require" | |||
| 9 | + | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | + | "github.com/bnema/vev/internal/ports" | |||
| 12 | + | ) | |||
| 13 | + | ||||
| 14 | + | func TestRestoreTransientLoadFailurePreservesHealthyAuthority(t *testing.T) { | |||
| 15 | + | record := durableRecoveryRecord(0) | |||
| 16 | + | repository := &durableRecoveryRepository{ | |||
| 17 | + | errors: map[string]error{record.Name: ports.ErrBudgetExhausted}, | |||
| 18 | + | loads: make(map[string]int), | |||
| 19 | + | repairs: make(map[string]domain.CheckpointRef), | |||
| 20 | + | } | |||
| 21 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |||
| 22 | + | ||||
| 23 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |||
| 24 | + | ||||
| 25 | + | persisted, ok, err := catalogue.Record(record.Name) | |||
| 26 | + | require.NoError(t, err) | |||
| 27 | + | require.True(t, ok) | |||
| 28 | + | require.Equal(t, domain.RecoveryHealthy, persisted.RecoveryState) | |||
| 29 | + | d.mu.Lock() | |||
| 30 | + | entry := d.stopped[record.Name] | |||
| 31 | + | d.mu.Unlock() | |||
| 32 | + | require.Equal(t, runtimeRestoring, entry.state) | |||
| 33 | + | require.NotEqual(t, runtimeDegraded, entry.state) | |||
| 34 | + | select { | |||
| 35 | + | case <-entry.restoreDone: | |||
| 36 | + | default: | |||
| 37 | + | t.Fatal("restore completion was not signaled") | |||
| 38 | + | } | |||
| 39 | + | } | |||
| 40 | + | ||||
| 41 | + | func TestFinishRecordRestoreAlwaysClosesProvidedBarrier(t *testing.T) { | |||
| 42 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 43 | + | done := make(chan struct{}) | |||
| 44 | + | ||||
| 45 | + | d.finishRecordRestore(domain.CatalogueRecord{Name: "removed"}, nil, done) | |||
| 46 | + | ||||
| 47 | + | select { | |||
| 48 | + | case <-done: | |||
| 49 | + | default: | |||
| 50 | + | t.Fatal("restore completion was not signaled") | |||
| 51 | + | } | |||
| 52 | + | } | |||
| 53 | + | ||||
| 54 | + | func TestSnapshotStopContextCancelIsIdempotent(t *testing.T) { | |||
| 55 | + | deadline := &snapshotShutdownDeadline{done: make(chan struct{})} | |||
| 56 | + | _, cancel := snapshotStopContext(deadline) | |||
| 57 | + | ||||
| 58 | + | var wg sync.WaitGroup | |||
| 59 | + | for range 8 { | |||
| 60 | + | wg.Go(cancel) | |||
| 61 | + | } | |||
| 62 | + | wg.Wait() | |||
| 63 | + | } | |||
M internal/usecase/daemon/control.go +4 -1
| @@ -465,6 +465,9 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) { | |||
| 465 | 465 | if ctx == nil { | |
| 466 | 466 | ctx = e.d.serveCtx | |
| 467 | 467 | } | |
| 468 | + | if ctx == nil { | |
| 469 | + | ctx = context.Background() | |
| 470 | + | } | |
| 468 | 471 | switch action { | |
| 469 | 472 | case "retry": | |
| 470 | 473 | if err := e.d.degradedRecovery.Retry(ctx, e.recoveryName); err != nil { | |
| @@ -549,7 +552,7 @@ func (e controlExec) restoreExplicitRecovery(ctx context.Context) error { | |||
| 549 | 552 | e.d.stopped[record.Name] = entry | |
| 550 | 553 | e.d.mu.Unlock() | |
| 551 | 554 | err = e.d.restoreRecord(ctx, record) | |
| 552 | - | e.d.finishRecordRestore(record, err, done) | |
| 555 | + | e.d.finishExplicitRecordRestore(record, err, done) | |
| 553 | 556 | return err | |
| 554 | 557 | } | |
| 555 | 558 | ||
| @@ -465,6 +465,9 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) { | |||||
| 465 | if ctx == nil { | 465 | if ctx == nil { | ||
| 466 | ctx = e.d.serveCtx | 466 | ctx = e.d.serveCtx | ||
| 467 | } | 467 | } | ||
| 468 | + | if ctx == nil { | |||
| 469 | + | ctx = context.Background() | |||
| 470 | + | } | |||
| 468 | switch action { | 471 | switch action { | ||
| 469 | case "retry": | 472 | case "retry": | ||
| 470 | if err := e.d.degradedRecovery.Retry(ctx, e.recoveryName); err != nil { | 473 | if err := e.d.degradedRecovery.Retry(ctx, e.recoveryName); err != nil { | ||
| @@ -549,7 +552,7 @@ func (e controlExec) restoreExplicitRecovery(ctx context.Context) error { | |||||
| 549 | e.d.stopped[record.Name] = entry | 552 | e.d.stopped[record.Name] = entry | ||
| 550 | e.d.mu.Unlock() | 553 | e.d.mu.Unlock() | ||
| 551 | err = e.d.restoreRecord(ctx, record) | 554 | err = e.d.restoreRecord(ctx, record) | ||
| 552 | - | e.d.finishRecordRestore(record, err, done) | 555 | + | e.d.finishExplicitRecordRestore(record, err, done) |
| 553 | return err | 556 | return err | ||
| 554 | } | 557 | } | ||
| 555 | 558 | ||||
M internal/usecase/daemon/daemon.go +15 -5
| @@ -416,18 +416,27 @@ func (d *Daemon) allocateLifecycleCreatedAtLocked() (int64, error) { | |||
| 416 | 416 | } | |
| 417 | 417 | ||
| 418 | 418 | func (d *Daemon) nowUnixNano() int64 { | |
| 419 | - | if d == nil || d.clock == nil { | |
| 420 | - | return time.Now().UnixNano() | |
| 421 | - | } | |
| 422 | 419 | return d.clock.Now().UnixNano() | |
| 423 | 420 | } | |
| 424 | 421 | ||
| 422 | + | type systemClock struct{} | |
| 423 | + | type systemTimer struct{ *time.Timer } | |
| 424 | + | ||
| 425 | + | func (systemClock) Now() time.Time { return time.Now() } | |
| 426 | + | func (systemClock) NewTimer(delay time.Duration) ports.Timer { | |
| 427 | + | return systemTimer{Timer: time.NewTimer(delay)} | |
| 428 | + | } | |
| 429 | + | func (t systemTimer) C() <-chan time.Time { return t.Timer.C } | |
| 430 | + | ||
| 425 | 431 | // New constructs a Daemon. ptys spawns PTY-backed children, clock drives the | |
| 426 | 432 | // render debounce, and log receives diagnostics (defaults to slog.Default). | |
| 427 | 433 | func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Option) *Daemon { | |
| 428 | 434 | if log == nil { | |
| 429 | 435 | log = slog.Default() | |
| 430 | 436 | } | |
| 437 | + | if clock == nil { | |
| 438 | + | clock = systemClock{} | |
| 439 | + | } | |
| 431 | 440 | shell := os.Getenv("SHELL") | |
| 432 | 441 | if shell == "" { | |
| 433 | 442 | shell = "/bin/sh" | |
| @@ -613,7 +622,7 @@ func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error { | |||
| 613 | 622 | closeErr = d.catalogue.Close() | |
| 614 | 623 | } | |
| 615 | 624 | if closeErr != nil { | |
| 616 | - | d.log.Warn("closing session persister failed", "err", closeErr) | |
| 625 | + | d.log.Warn("closing session catalogue failed", "err", closeErr) | |
| 617 | 626 | } | |
| 618 | 627 | return nil | |
| 619 | 628 | } | |
| @@ -745,8 +754,9 @@ func snapshotStopContext(deadline *snapshotShutdownDeadline) (context.Context, c | |||
| 745 | 754 | case <-stop: | |
| 746 | 755 | } | |
| 747 | 756 | }() | |
| 757 | + | var stopOnce sync.Once | |
| 748 | 758 | return ctx, func() { | |
| 749 | - | close(stop) | |
| 759 | + | stopOnce.Do(func() { close(stop) }) | |
| 750 | 760 | cancel() | |
| 751 | 761 | } | |
| 752 | 762 | } | |
| @@ -416,18 +416,27 @@ func (d *Daemon) allocateLifecycleCreatedAtLocked() (int64, error) { | |||||
| 416 | } | 416 | } | ||
| 417 | 417 | ||||
| 418 | func (d *Daemon) nowUnixNano() int64 { | 418 | func (d *Daemon) nowUnixNano() int64 { | ||
| 419 | - | if d == nil || d.clock == nil { | |||
| 420 | - | return time.Now().UnixNano() | |||
| 421 | - | } | |||
| 422 | return d.clock.Now().UnixNano() | 419 | return d.clock.Now().UnixNano() | ||
| 423 | } | 420 | } | ||
| 424 | 421 | ||||
| 422 | + | type systemClock struct{} | |||
| 423 | + | type systemTimer struct{ *time.Timer } | |||
| 424 | + | ||||
| 425 | + | func (systemClock) Now() time.Time { return time.Now() } | |||
| 426 | + | func (systemClock) NewTimer(delay time.Duration) ports.Timer { | |||
| 427 | + | return systemTimer{Timer: time.NewTimer(delay)} | |||
| 428 | + | } | |||
| 429 | + | func (t systemTimer) C() <-chan time.Time { return t.Timer.C } | |||
| 430 | + | ||||
| 425 | // New constructs a Daemon. ptys spawns PTY-backed children, clock drives the | 431 | // New constructs a Daemon. ptys spawns PTY-backed children, clock drives the | ||
| 426 | // render debounce, and log receives diagnostics (defaults to slog.Default). | 432 | // render debounce, and log receives diagnostics (defaults to slog.Default). | ||
| 427 | func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Option) *Daemon { | 433 | func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Option) *Daemon { | ||
| 428 | if log == nil { | 434 | if log == nil { | ||
| 429 | log = slog.Default() | 435 | log = slog.Default() | ||
| 430 | } | 436 | } | ||
| 437 | + | if clock == nil { | |||
| 438 | + | clock = systemClock{} | |||
| 439 | + | } | |||
| 431 | shell := os.Getenv("SHELL") | 440 | shell := os.Getenv("SHELL") | ||
| 432 | if shell == "" { | 441 | if shell == "" { | ||
| 433 | shell = "/bin/sh" | 442 | shell = "/bin/sh" | ||
| @@ -613,7 +622,7 @@ func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error { | |||||
| 613 | closeErr = d.catalogue.Close() | 622 | closeErr = d.catalogue.Close() | ||
| 614 | } | 623 | } | ||
| 615 | if closeErr != nil { | 624 | if closeErr != nil { | ||
| 616 | - | d.log.Warn("closing session persister failed", "err", closeErr) | 625 | + | d.log.Warn("closing session catalogue failed", "err", closeErr) |
| 617 | } | 626 | } | ||
| 618 | return nil | 627 | return nil | ||
| 619 | } | 628 | } | ||
| @@ -745,8 +754,9 @@ func snapshotStopContext(deadline *snapshotShutdownDeadline) (context.Context, c | |||||
| 745 | case <-stop: | 754 | case <-stop: | ||
| 746 | } | 755 | } | ||
| 747 | }() | 756 | }() | ||
| 757 | + | var stopOnce sync.Once | |||
| 748 | return ctx, func() { | 758 | return ctx, func() { | ||
| 749 | - | close(stop) | 759 | + | stopOnce.Do(func() { close(stop) }) |
| 750 | cancel() | 760 | cancel() | ||
| 751 | } | 761 | } | ||
| 752 | } | 762 | } | ||
M internal/usecase/daemon/daemon_test.go +6 -0
| @@ -68,6 +68,12 @@ func newTestDaemon(t testing.TB, ptys ports.PTYFactory, clk ports.Clock) *Daemon | |||
| 68 | 68 | return newTestDaemonWithCleanup(t, ptys, clk, true) | |
| 69 | 69 | } | |
| 70 | 70 | ||
| 71 | + | func createSessionForTest(d *Daemon, name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) { | |
| 72 | + | d.mu.Lock() | |
| 73 | + | defer d.mu.Unlock() | |
| 74 | + | return d.createSessionLocked(name, ephemeral, cwd, sz, term, env, restoredTabNames...) | |
| 75 | + | } | |
| 76 | + | ||
| 71 | 77 | func newTestDaemonWithCleanup(t testing.TB, ptys ports.PTYFactory, clk ports.Clock, registerCleanup bool) *Daemon { | |
| 72 | 78 | t.Helper() | |
| 73 | 79 | d := New(ptys, clk, slog.New(slog.NewTextHandler(io.Discard, nil))) | |
| @@ -68,6 +68,12 @@ func newTestDaemon(t testing.TB, ptys ports.PTYFactory, clk ports.Clock) *Daemon | |||||
| 68 | return newTestDaemonWithCleanup(t, ptys, clk, true) | 68 | return newTestDaemonWithCleanup(t, ptys, clk, true) | ||
| 69 | } | 69 | } | ||
| 70 | 70 | ||||
| 71 | + | func createSessionForTest(d *Daemon, name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) { | |||
| 72 | + | d.mu.Lock() | |||
| 73 | + | defer d.mu.Unlock() | |||
| 74 | + | return d.createSessionLocked(name, ephemeral, cwd, sz, term, env, restoredTabNames...) | |||
| 75 | + | } | |||
| 76 | + | ||||
| 71 | func newTestDaemonWithCleanup(t testing.TB, ptys ports.PTYFactory, clk ports.Clock, registerCleanup bool) *Daemon { | 77 | func newTestDaemonWithCleanup(t testing.TB, ptys ports.PTYFactory, clk ports.Clock, registerCleanup bool) *Daemon { | ||
| 72 | t.Helper() | 78 | t.Helper() | ||
| 73 | d := New(ptys, clk, slog.New(slog.NewTextHandler(io.Discard, nil))) | 79 | d := New(ptys, clk, slog.New(slog.NewTextHandler(io.Discard, nil))) | ||
M internal/usecase/daemon/durable_authority_test.go +13 -4
| @@ -3,6 +3,9 @@ package daemon | |||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | 5 | "crypto/rand" | |
| 6 | + | "testing" | |
| 7 | + | ||
| 8 | + | "github.com/stretchr/testify/require" | |
| 6 | 9 | ||
| 7 | 10 | "github.com/bnema/vev/internal/domain" | |
| 8 | 11 | "github.com/bnema/vev/internal/persist" | |
| @@ -24,16 +27,15 @@ func (testLifecycleRepository) DeleteDeletionTombstone(context.Context, domain.I | |||
| 24 | 27 | ||
| 25 | 28 | // WithStore is package-test composition glue. Production injects the same | |
| 26 | 29 | // recovery coordinator explicitly from internal/app. | |
| 27 | - | func WithStore(store ports.Store) Option { | |
| 30 | + | func WithStore(t testing.TB, store ports.Store) Option { | |
| 31 | + | t.Helper() | |
| 28 | 32 | return func(d *Daemon) { | |
| 29 | 33 | if store == nil { | |
| 30 | 34 | return | |
| 31 | 35 | } | |
| 32 | 36 | catalogue := persist.New(store) | |
| 33 | 37 | records, err := catalogue.Records() | |
| 34 | - | if err != nil { | |
| 35 | - | return | |
| 36 | - | } | |
| 38 | + | require.NoError(t, err) | |
| 37 | 39 | d.catalogue = catalogue | |
| 38 | 40 | d.persistEnabled = true | |
| 39 | 41 | d.catalogueRecords = records | |
| @@ -47,3 +49,10 @@ func WithStore(store ports.Store) Option { | |||
| 47 | 49 | d.checkpointRecovery = authority | |
| 48 | 50 | } | |
| 49 | 51 | } | |
| 52 | + | ||
| 53 | + | func testPersister(t testing.TB, d *Daemon) *persist.Persister { | |
| 54 | + | t.Helper() | |
| 55 | + | persister, ok := d.catalogue.(*persist.Persister) | |
| 56 | + | require.True(t, ok, "catalogue type %T is not *persist.Persister", d.catalogue) | |
| 57 | + | return persister | |
| 58 | + | } | |
| @@ -3,6 +3,9 @@ package daemon | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | "crypto/rand" | 5 | "crypto/rand" | ||
| 6 | + | "testing" | |||
| 7 | + | ||||
| 8 | + | "github.com/stretchr/testify/require" | |||
| 6 | 9 | ||||
| 7 | "github.com/bnema/vev/internal/domain" | 10 | "github.com/bnema/vev/internal/domain" | ||
| 8 | "github.com/bnema/vev/internal/persist" | 11 | "github.com/bnema/vev/internal/persist" | ||
| @@ -24,16 +27,15 @@ func (testLifecycleRepository) DeleteDeletionTombstone(context.Context, domain.I | |||||
| 24 | 27 | ||||
| 25 | // WithStore is package-test composition glue. Production injects the same | 28 | // WithStore is package-test composition glue. Production injects the same | ||
| 26 | // recovery coordinator explicitly from internal/app. | 29 | // recovery coordinator explicitly from internal/app. | ||
| 27 | - | func WithStore(store ports.Store) Option { | 30 | + | func WithStore(t testing.TB, store ports.Store) Option { |
| 31 | + | t.Helper() | |||
| 28 | return func(d *Daemon) { | 32 | return func(d *Daemon) { | ||
| 29 | if store == nil { | 33 | if store == nil { | ||
| 30 | return | 34 | return | ||
| 31 | } | 35 | } | ||
| 32 | catalogue := persist.New(store) | 36 | catalogue := persist.New(store) | ||
| 33 | records, err := catalogue.Records() | 37 | records, err := catalogue.Records() | ||
| 34 | - | if err != nil { | 38 | + | require.NoError(t, err) |
| 35 | - | return | |||
| 36 | - | } | |||
| 37 | d.catalogue = catalogue | 39 | d.catalogue = catalogue | ||
| 38 | d.persistEnabled = true | 40 | d.persistEnabled = true | ||
| 39 | d.catalogueRecords = records | 41 | d.catalogueRecords = records | ||
| @@ -47,3 +49,10 @@ func WithStore(store ports.Store) Option { | |||||
| 47 | d.checkpointRecovery = authority | 49 | d.checkpointRecovery = authority | ||
| 48 | } | 50 | } | ||
| 49 | } | 51 | } | ||
| 52 | + | ||||
| 53 | + | func testPersister(t testing.TB, d *Daemon) *persist.Persister { | |||
| 54 | + | t.Helper() | |||
| 55 | + | persister, ok := d.catalogue.(*persist.Persister) | |||
| 56 | + | require.True(t, ok, "catalogue type %T is not *persist.Persister", d.catalogue) | |||
| 57 | + | return persister | |||
| 58 | + | } | |||
M internal/usecase/daemon/durable_recovery_test.go +149 -58
| @@ -7,8 +7,10 @@ import ( | |||
| 7 | 7 | "fmt" | |
| 8 | 8 | "io" | |
| 9 | 9 | "log/slog" | |
| 10 | + | "os" | |
| 10 | 11 | "sync" | |
| 11 | 12 | "sync/atomic" | |
| 13 | + | "syscall" | |
| 12 | 14 | "testing" | |
| 13 | 15 | "time" | |
| 14 | 16 | ||
| @@ -226,67 +228,73 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) { | |||
| 226 | 228 | } | |
| 227 | 229 | ||
| 228 | 230 | func TestCatalogueRestoreIndependent(t *testing.T) { | |
| 229 | - | emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{}) | |
| 230 | - | emptyDone := make(chan struct{}) | |
| 231 | - | go func() { | |
| 232 | - | emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue)) | |
| 233 | - | close(emptyDone) | |
| 234 | - | }() | |
| 235 | - | select { | |
| 236 | - | case <-emptyDone: | |
| 237 | - | case <-time.After(time.Second): | |
| 238 | - | t.Fatal("empty catalogue restoration blocked") | |
| 239 | - | } | |
| 240 | - | ||
| 241 | - | healthy := durableRecoveryRecord(0) | |
| 242 | - | broken := durableRecoveryRecord(1) | |
| 243 | - | repository := &durableRecoveryRepository{ | |
| 244 | - | generations: map[string]ports.SnapshotGeneration{healthy.Name: validGeneration(t, healthy)}, | |
| 245 | - | errors: map[string]error{broken.Name: ports.ErrBudgetExhausted}, | |
| 246 | - | loads: make(map[string]int), | |
| 247 | - | repairs: make(map[string]domain.CheckpointRef), | |
| 248 | - | } | |
| 249 | - | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository) | |
| 231 | + | t.Run("empty catalogue restoration", func(t *testing.T) { | |
| 232 | + | emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{}) | |
| 233 | + | emptyDone := make(chan struct{}) | |
| 234 | + | go func() { | |
| 235 | + | emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue)) | |
| 236 | + | close(emptyDone) | |
| 237 | + | }() | |
| 238 | + | select { | |
| 239 | + | case <-emptyDone: | |
| 240 | + | case <-time.After(time.Second): | |
| 241 | + | t.Fatal("empty catalogue restoration blocked") | |
| 242 | + | } | |
| 243 | + | }) | |
| 250 | 244 | ||
| 251 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 245 | + | t.Run("healthy and broken restoration", func(t *testing.T) { | |
| 246 | + | healthy := durableRecoveryRecord(0) | |
| 247 | + | broken := durableRecoveryRecord(1) | |
| 248 | + | repository := &durableRecoveryRepository{ | |
| 249 | + | generations: map[string]ports.SnapshotGeneration{healthy.Name: validGeneration(t, healthy)}, | |
| 250 | + | errors: map[string]error{broken.Name: errors.New("corrupt checkpoint")}, | |
| 251 | + | loads: make(map[string]int), | |
| 252 | + | repairs: make(map[string]domain.CheckpointRef), | |
| 253 | + | } | |
| 254 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository) | |
| 255 | + | ||
| 256 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 257 | + | ||
| 258 | + | require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state) | |
| 259 | + | require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state) | |
| 260 | + | for _, name := range []string{healthy.Name, broken.Name} { | |
| 261 | + | select { | |
| 262 | + | case <-d.stopped[name].restoreDone: | |
| 263 | + | default: | |
| 264 | + | t.Fatalf("restore completion for %q was not closed", name) | |
| 265 | + | } | |
| 266 | + | } | |
| 267 | + | degraded, ok, _ := catalogue.Record(broken.Name) | |
| 268 | + | require.True(t, ok) | |
| 269 | + | require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState) | |
| 270 | + | require.Equal(t, 1, repository.loads[healthy.Name]) | |
| 271 | + | require.Equal(t, 1, repository.loads[broken.Name]) | |
| 272 | + | require.Empty(t, d.sessions, "a corrupt record must never open a replacement runtime") | |
| 273 | + | }) | |
| 252 | 274 | ||
| 253 | - | require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state) | |
| 254 | - | require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state) | |
| 255 | - | for _, name := range []string{healthy.Name, broken.Name} { | |
| 256 | - | select { | |
| 257 | - | case <-d.stopped[name].restoreDone: | |
| 258 | - | default: | |
| 259 | - | t.Fatalf("restore completion for %q was not closed", name) | |
| 275 | + | t.Run("fallback promotion", func(t *testing.T) { | |
| 276 | + | fallback := durableRecoveryRecord(2) | |
| 277 | + | fallbackRef := domain.CheckpointRef{Generation: fallback.Committed.Generation - 1} | |
| 278 | + | fallbackSnapshotRecord := fallback | |
| 279 | + | fallbackSnapshotRecord.Committed = &fallbackRef | |
| 280 | + | fallbackGeneration := validGeneration(t, fallbackSnapshotRecord) | |
| 281 | + | fallbackRef.ManifestDigest = sha256.Sum256(fallbackGeneration.Manifest) | |
| 282 | + | fallback.Fallbacks[0] = &fallbackRef | |
| 283 | + | fallbackRepository := &durableRecoveryRepository{ | |
| 284 | + | generations: map[string]ports.SnapshotGeneration{fallback.Name: fallbackGeneration}, | |
| 285 | + | errors: make(map[string]error), | |
| 286 | + | loads: make(map[string]int), | |
| 287 | + | repairs: make(map[string]domain.CheckpointRef), | |
| 260 | 288 | } | |
| 261 | - | } | |
| 262 | - | degraded, ok, _ := catalogue.Record(broken.Name) | |
| 263 | - | require.True(t, ok) | |
| 264 | - | require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState) | |
| 265 | - | require.Equal(t, 1, repository.loads[healthy.Name]) | |
| 266 | - | require.Equal(t, 1, repository.loads[broken.Name]) | |
| 267 | - | require.Empty(t, d.sessions, "a corrupt record must never open a replacement runtime") | |
| 268 | - | ||
| 269 | - | fallback := durableRecoveryRecord(2) | |
| 270 | - | fallbackRef := domain.CheckpointRef{Generation: fallback.Committed.Generation - 1} | |
| 271 | - | fallbackSnapshotRecord := fallback | |
| 272 | - | fallbackSnapshotRecord.Committed = &fallbackRef | |
| 273 | - | fallbackGeneration := validGeneration(t, fallbackSnapshotRecord) | |
| 274 | - | fallbackRef.ManifestDigest = sha256.Sum256(fallbackGeneration.Manifest) | |
| 275 | - | fallback.Fallbacks[0] = &fallbackRef | |
| 276 | - | fallbackRepository := &durableRecoveryRepository{ | |
| 277 | - | generations: map[string]ports.SnapshotGeneration{fallback.Name: fallbackGeneration}, | |
| 278 | - | errors: make(map[string]error), | |
| 279 | - | loads: make(map[string]int), | |
| 280 | - | repairs: make(map[string]domain.CheckpointRef), | |
| 281 | - | } | |
| 282 | - | fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository) | |
| 283 | - | fallbackDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, fallbackCatalogue)) | |
| 284 | - | promoted, ok, _ := fallbackCatalogue.Record(fallback.Name) | |
| 285 | - | require.True(t, ok) | |
| 286 | - | require.Equal(t, fallbackRef, *promoted.Committed) | |
| 287 | - | require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state) | |
| 288 | - | require.Equal(t, 3, fallbackRepository.loads[fallback.Name]) | |
| 289 | - | require.Equal(t, fallbackRef, fallbackRepository.repairs[fallback.IncarnationID.String()]) | |
| 289 | + | fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository) | |
| 290 | + | fallbackDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, fallbackCatalogue)) | |
| 291 | + | promoted, ok, _ := fallbackCatalogue.Record(fallback.Name) | |
| 292 | + | require.True(t, ok) | |
| 293 | + | require.Equal(t, fallbackRef, *promoted.Committed) | |
| 294 | + | require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state) | |
| 295 | + | require.Equal(t, 3, fallbackRepository.loads[fallback.Name]) | |
| 296 | + | require.Equal(t, fallbackRef, fallbackRepository.repairs[fallback.IncarnationID.String()]) | |
| 297 | + | }) | |
| 290 | 298 | } | |
| 291 | 299 | ||
| 292 | 300 | func TestCatalogueRestoreKeepsCommittedFallbackHealthyWhenHEADRepairFails(t *testing.T) { | |
| @@ -433,6 +441,89 @@ func TestSessionRecoveryUsesIncomingCommandContext(t *testing.T) { | |||
| 433 | 441 | require.True(t, stub.seen) | |
| 434 | 442 | } | |
| 435 | 443 | ||
| 444 | + | type temporaryRestoreError struct{} | |
| 445 | + | ||
| 446 | + | func (temporaryRestoreError) Error() string { return "temporary restore failure" } | |
| 447 | + | func (temporaryRestoreError) Temporary() bool { return true } | |
| 448 | + | ||
| 449 | + | func TestRetryableRestoreLoadErrorClassification(t *testing.T) { | |
| 450 | + | tests := []struct { | |
| 451 | + | name string | |
| 452 | + | err error | |
| 453 | + | retryable bool | |
| 454 | + | }{ | |
| 455 | + | {name: "budget exhausted", err: fmt.Errorf("wrapped: %w", ports.ErrBudgetExhausted), retryable: true}, | |
| 456 | + | {name: "missing path", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.ENOENT})}, | |
| 457 | + | {name: "permission denied", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EACCES})}, | |
| 458 | + | {name: "invalid path operation", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EINVAL})}, | |
| 459 | + | {name: "temporary path failure", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EAGAIN}), retryable: true}, | |
| 460 | + | {name: "resource exhausted path", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.ENOSPC}), retryable: true}, | |
| 461 | + | {name: "temporary interface", err: fmt.Errorf("wrapped: %w", temporaryRestoreError{}), retryable: true}, | |
| 462 | + | {name: "corrupt data", err: errors.New("corrupt checkpoint")}, | |
| 463 | + | {name: "context canceled", err: context.Canceled}, | |
| 464 | + | {name: "context deadline exceeded", err: context.DeadlineExceeded}, | |
| 465 | + | } | |
| 466 | + | for _, tt := range tests { | |
| 467 | + | t.Run(tt.name, func(t *testing.T) { | |
| 468 | + | require.Equal(t, tt.retryable, isRetryableRestoreLoadError(tt.err)) | |
| 469 | + | }) | |
| 470 | + | } | |
| 471 | + | } | |
| 472 | + | ||
| 473 | + | func TestExplicitRecoveryRemainsRetryableAfterRepeatedTransientFailures(t *testing.T) { | |
| 474 | + | record := durableRecoveryRecord(0) | |
| 475 | + | repository := &durableRecoveryRepository{ | |
| 476 | + | generations: map[string]ports.SnapshotGeneration{record.Name: validGeneration(t, record)}, | |
| 477 | + | errors: map[string]error{record.Name: ports.ErrBudgetExhausted}, | |
| 478 | + | loads: make(map[string]int), | |
| 479 | + | repairs: make(map[string]domain.CheckpointRef), | |
| 480 | + | } | |
| 481 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 482 | + | d.setStoppedRecovery(record, runtimeDegraded) | |
| 483 | + | exec := controlExec{d: d, recoveryName: record.Name} | |
| 484 | + | ||
| 485 | + | var attemptBarriers []chan struct{} | |
| 486 | + | for range 2 { | |
| 487 | + | err := exec.restoreExplicitRecovery(t.Context()) | |
| 488 | + | require.ErrorIs(t, err, errRetryableRestoreLoad) | |
| 489 | + | ||
| 490 | + | d.mu.Lock() | |
| 491 | + | entry := d.stopped[record.Name] | |
| 492 | + | d.mu.Unlock() | |
| 493 | + | require.Equal(t, runtimeDegraded, entry.state) | |
| 494 | + | require.Equal(t, domain.RecoveryHealthy, entry.record.RecoveryState) | |
| 495 | + | select { | |
| 496 | + | case <-entry.restoreDone: | |
| 497 | + | default: | |
| 498 | + | t.Fatal("failed attempt completion barrier remained open") | |
| 499 | + | } | |
| 500 | + | for _, prior := range attemptBarriers { | |
| 501 | + | require.NotEqual(t, prior, entry.restoreDone, "a closed completion barrier must not be reused") | |
| 502 | + | } | |
| 503 | + | attemptBarriers = append(attemptBarriers, entry.restoreDone) | |
| 504 | + | ||
| 505 | + | persisted, ok, catalogueErr := catalogue.Record(record.Name) | |
| 506 | + | require.NoError(t, catalogueErr) | |
| 507 | + | require.True(t, ok) | |
| 508 | + | require.Equal(t, record, persisted, "transient restore failure must preserve healthy durable authority") | |
| 509 | + | } | |
| 510 | + | ||
| 511 | + | repository.mu.Lock() | |
| 512 | + | delete(repository.errors, record.Name) | |
| 513 | + | repository.mu.Unlock() | |
| 514 | + | require.NoError(t, exec.restoreExplicitRecovery(t.Context())) | |
| 515 | + | d.mu.Lock() | |
| 516 | + | require.Equal(t, runtimeHealthy, d.stopped[record.Name].state) | |
| 517 | + | d.mu.Unlock() | |
| 518 | + | persisted, ok, err := catalogue.Record(record.Name) | |
| 519 | + | require.NoError(t, err) | |
| 520 | + | require.True(t, ok) | |
| 521 | + | require.Equal(t, domain.RecoveryHealthy, persisted.RecoveryState) | |
| 522 | + | repository.mu.Lock() | |
| 523 | + | require.Equal(t, 3, repository.loads[record.Name]) | |
| 524 | + | repository.mu.Unlock() | |
| 525 | + | } | |
| 526 | + | ||
| 436 | 527 | func TestExplicitRecoveryRejectsConcurrentRestore(t *testing.T) { | |
| 437 | 528 | record := durableRecoveryRecord(0) | |
| 438 | 529 | record.RecoveryState = domain.RecoveryDegraded | |
| @@ -7,8 +7,10 @@ import ( | |||||
| 7 | "fmt" | 7 | "fmt" | ||
| 8 | "io" | 8 | "io" | ||
| 9 | "log/slog" | 9 | "log/slog" | ||
| 10 | + | "os" | |||
| 10 | "sync" | 11 | "sync" | ||
| 11 | "sync/atomic" | 12 | "sync/atomic" | ||
| 13 | + | "syscall" | |||
| 12 | "testing" | 14 | "testing" | ||
| 13 | "time" | 15 | "time" | ||
| 14 | 16 | ||||
| @@ -226,67 +228,73 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) { | |||||
| 226 | } | 228 | } | ||
| 227 | 229 | ||||
| 228 | func TestCatalogueRestoreIndependent(t *testing.T) { | 230 | func TestCatalogueRestoreIndependent(t *testing.T) { | ||
| 229 | - | emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{}) | 231 | + | t.Run("empty catalogue restoration", func(t *testing.T) { |
| 230 | - | emptyDone := make(chan struct{}) | 232 | + | emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{}) |
| 231 | - | go func() { | 233 | + | emptyDone := make(chan struct{}) |
| 232 | - | emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue)) | 234 | + | go func() { |
| 233 | - | close(emptyDone) | 235 | + | emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue)) |
| 234 | - | }() | 236 | + | close(emptyDone) |
| 235 | - | select { | 237 | + | }() |
| 236 | - | case <-emptyDone: | 238 | + | select { |
| 237 | - | case <-time.After(time.Second): | 239 | + | case <-emptyDone: |
| 238 | - | t.Fatal("empty catalogue restoration blocked") | 240 | + | case <-time.After(time.Second): |
| 239 | - | } | 241 | + | t.Fatal("empty catalogue restoration blocked") |
| 240 | - | 242 | + | } | |
| 241 | - | healthy := durableRecoveryRecord(0) | 243 | + | }) |
| 242 | - | broken := durableRecoveryRecord(1) | |||
| 243 | - | repository := &durableRecoveryRepository{ | |||
| 244 | - | generations: map[string]ports.SnapshotGeneration{healthy.Name: validGeneration(t, healthy)}, | |||
| 245 | - | errors: map[string]error{broken.Name: ports.ErrBudgetExhausted}, | |||
| 246 | - | loads: make(map[string]int), | |||
| 247 | - | repairs: make(map[string]domain.CheckpointRef), | |||
| 248 | - | } | |||
| 249 | - | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository) | |||
| 250 | 244 | ||||
| 251 | - | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | 245 | + | t.Run("healthy and broken restoration", func(t *testing.T) { |
| 246 | + | healthy := durableRecoveryRecord(0) | |||
| 247 | + | broken := durableRecoveryRecord(1) | |||
| 248 | + | repository := &durableRecoveryRepository{ | |||
| 249 | + | generations: map[string]ports.SnapshotGeneration{healthy.Name: validGeneration(t, healthy)}, | |||
| 250 | + | errors: map[string]error{broken.Name: errors.New("corrupt checkpoint")}, | |||
| 251 | + | loads: make(map[string]int), | |||
| 252 | + | repairs: make(map[string]domain.CheckpointRef), | |||
| 253 | + | } | |||
| 254 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository) | |||
| 255 | + | ||||
| 256 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |||
| 257 | + | ||||
| 258 | + | require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state) | |||
| 259 | + | require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state) | |||
| 260 | + | for _, name := range []string{healthy.Name, broken.Name} { | |||
| 261 | + | select { | |||
| 262 | + | case <-d.stopped[name].restoreDone: | |||
| 263 | + | default: | |||
| 264 | + | t.Fatalf("restore completion for %q was not closed", name) | |||
| 265 | + | } | |||
| 266 | + | } | |||
| 267 | + | degraded, ok, _ := catalogue.Record(broken.Name) | |||
| 268 | + | require.True(t, ok) | |||
| 269 | + | require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState) | |||
| 270 | + | require.Equal(t, 1, repository.loads[healthy.Name]) | |||
| 271 | + | require.Equal(t, 1, repository.loads[broken.Name]) | |||
| 272 | + | require.Empty(t, d.sessions, "a corrupt record must never open a replacement runtime") | |||
| 273 | + | }) | |||
| 252 | 274 | ||||
| 253 | - | require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state) | 275 | + | t.Run("fallback promotion", func(t *testing.T) { |
| 254 | - | require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state) | 276 | + | fallback := durableRecoveryRecord(2) |
| 255 | - | for _, name := range []string{healthy.Name, broken.Name} { | 277 | + | fallbackRef := domain.CheckpointRef{Generation: fallback.Committed.Generation - 1} |
| 256 | - | select { | 278 | + | fallbackSnapshotRecord := fallback |
| 257 | - | case <-d.stopped[name].restoreDone: | 279 | + | fallbackSnapshotRecord.Committed = &fallbackRef |
| 258 | - | default: | 280 | + | fallbackGeneration := validGeneration(t, fallbackSnapshotRecord) |
| 259 | - | t.Fatalf("restore completion for %q was not closed", name) | 281 | + | fallbackRef.ManifestDigest = sha256.Sum256(fallbackGeneration.Manifest) |
| 282 | + | fallback.Fallbacks[0] = &fallbackRef | |||
| 283 | + | fallbackRepository := &durableRecoveryRepository{ | |||
| 284 | + | generations: map[string]ports.SnapshotGeneration{fallback.Name: fallbackGeneration}, | |||
| 285 | + | errors: make(map[string]error), | |||
| 286 | + | loads: make(map[string]int), | |||
| 287 | + | repairs: make(map[string]domain.CheckpointRef), | |||
| 260 | } | 288 | } | ||
| 261 | - | } | 289 | + | fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository) |
| 262 | - | degraded, ok, _ := catalogue.Record(broken.Name) | 290 | + | fallbackDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, fallbackCatalogue)) |
| 263 | - | require.True(t, ok) | 291 | + | promoted, ok, _ := fallbackCatalogue.Record(fallback.Name) |
| 264 | - | require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState) | 292 | + | require.True(t, ok) |
| 265 | - | require.Equal(t, 1, repository.loads[healthy.Name]) | 293 | + | require.Equal(t, fallbackRef, *promoted.Committed) |
| 266 | - | require.Equal(t, 1, repository.loads[broken.Name]) | 294 | + | require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state) |
| 267 | - | require.Empty(t, d.sessions, "a corrupt record must never open a replacement runtime") | 295 | + | require.Equal(t, 3, fallbackRepository.loads[fallback.Name]) |
| 268 | - | 296 | + | require.Equal(t, fallbackRef, fallbackRepository.repairs[fallback.IncarnationID.String()]) | |
| 269 | - | fallback := durableRecoveryRecord(2) | 297 | + | }) |
| 270 | - | fallbackRef := domain.CheckpointRef{Generation: fallback.Committed.Generation - 1} | |||
| 271 | - | fallbackSnapshotRecord := fallback | |||
| 272 | - | fallbackSnapshotRecord.Committed = &fallbackRef | |||
| 273 | - | fallbackGeneration := validGeneration(t, fallbackSnapshotRecord) | |||
| 274 | - | fallbackRef.ManifestDigest = sha256.Sum256(fallbackGeneration.Manifest) | |||
| 275 | - | fallback.Fallbacks[0] = &fallbackRef | |||
| 276 | - | fallbackRepository := &durableRecoveryRepository{ | |||
| 277 | - | generations: map[string]ports.SnapshotGeneration{fallback.Name: fallbackGeneration}, | |||
| 278 | - | errors: make(map[string]error), | |||
| 279 | - | loads: make(map[string]int), | |||
| 280 | - | repairs: make(map[string]domain.CheckpointRef), | |||
| 281 | - | } | |||
| 282 | - | fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository) | |||
| 283 | - | fallbackDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, fallbackCatalogue)) | |||
| 284 | - | promoted, ok, _ := fallbackCatalogue.Record(fallback.Name) | |||
| 285 | - | require.True(t, ok) | |||
| 286 | - | require.Equal(t, fallbackRef, *promoted.Committed) | |||
| 287 | - | require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state) | |||
| 288 | - | require.Equal(t, 3, fallbackRepository.loads[fallback.Name]) | |||
| 289 | - | require.Equal(t, fallbackRef, fallbackRepository.repairs[fallback.IncarnationID.String()]) | |||
| 290 | } | 298 | } | ||
| 291 | 299 | ||||
| 292 | func TestCatalogueRestoreKeepsCommittedFallbackHealthyWhenHEADRepairFails(t *testing.T) { | 300 | func TestCatalogueRestoreKeepsCommittedFallbackHealthyWhenHEADRepairFails(t *testing.T) { | ||
| @@ -433,6 +441,89 @@ func TestSessionRecoveryUsesIncomingCommandContext(t *testing.T) { | |||||
| 433 | require.True(t, stub.seen) | 441 | require.True(t, stub.seen) | ||
| 434 | } | 442 | } | ||
| 435 | 443 | ||||
| 444 | + | type temporaryRestoreError struct{} | |||
| 445 | + | ||||
| 446 | + | func (temporaryRestoreError) Error() string { return "temporary restore failure" } | |||
| 447 | + | func (temporaryRestoreError) Temporary() bool { return true } | |||
| 448 | + | ||||
| 449 | + | func TestRetryableRestoreLoadErrorClassification(t *testing.T) { | |||
| 450 | + | tests := []struct { | |||
| 451 | + | name string | |||
| 452 | + | err error | |||
| 453 | + | retryable bool | |||
| 454 | + | }{ | |||
| 455 | + | {name: "budget exhausted", err: fmt.Errorf("wrapped: %w", ports.ErrBudgetExhausted), retryable: true}, | |||
| 456 | + | {name: "missing path", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.ENOENT})}, | |||
| 457 | + | {name: "permission denied", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EACCES})}, | |||
| 458 | + | {name: "invalid path operation", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EINVAL})}, | |||
| 459 | + | {name: "temporary path failure", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EAGAIN}), retryable: true}, | |||
| 460 | + | {name: "resource exhausted path", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.ENOSPC}), retryable: true}, | |||
| 461 | + | {name: "temporary interface", err: fmt.Errorf("wrapped: %w", temporaryRestoreError{}), retryable: true}, | |||
| 462 | + | {name: "corrupt data", err: errors.New("corrupt checkpoint")}, | |||
| 463 | + | {name: "context canceled", err: context.Canceled}, | |||
| 464 | + | {name: "context deadline exceeded", err: context.DeadlineExceeded}, | |||
| 465 | + | } | |||
| 466 | + | for _, tt := range tests { | |||
| 467 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 468 | + | require.Equal(t, tt.retryable, isRetryableRestoreLoadError(tt.err)) | |||
| 469 | + | }) | |||
| 470 | + | } | |||
| 471 | + | } | |||
| 472 | + | ||||
| 473 | + | func TestExplicitRecoveryRemainsRetryableAfterRepeatedTransientFailures(t *testing.T) { | |||
| 474 | + | record := durableRecoveryRecord(0) | |||
| 475 | + | repository := &durableRecoveryRepository{ | |||
| 476 | + | generations: map[string]ports.SnapshotGeneration{record.Name: validGeneration(t, record)}, | |||
| 477 | + | errors: map[string]error{record.Name: ports.ErrBudgetExhausted}, | |||
| 478 | + | loads: make(map[string]int), | |||
| 479 | + | repairs: make(map[string]domain.CheckpointRef), | |||
| 480 | + | } | |||
| 481 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |||
| 482 | + | d.setStoppedRecovery(record, runtimeDegraded) | |||
| 483 | + | exec := controlExec{d: d, recoveryName: record.Name} | |||
| 484 | + | ||||
| 485 | + | var attemptBarriers []chan struct{} | |||
| 486 | + | for range 2 { | |||
| 487 | + | err := exec.restoreExplicitRecovery(t.Context()) | |||
| 488 | + | require.ErrorIs(t, err, errRetryableRestoreLoad) | |||
| 489 | + | ||||
| 490 | + | d.mu.Lock() | |||
| 491 | + | entry := d.stopped[record.Name] | |||
| 492 | + | d.mu.Unlock() | |||
| 493 | + | require.Equal(t, runtimeDegraded, entry.state) | |||
| 494 | + | require.Equal(t, domain.RecoveryHealthy, entry.record.RecoveryState) | |||
| 495 | + | select { | |||
| 496 | + | case <-entry.restoreDone: | |||
| 497 | + | default: | |||
| 498 | + | t.Fatal("failed attempt completion barrier remained open") | |||
| 499 | + | } | |||
| 500 | + | for _, prior := range attemptBarriers { | |||
| 501 | + | require.NotEqual(t, prior, entry.restoreDone, "a closed completion barrier must not be reused") | |||
| 502 | + | } | |||
| 503 | + | attemptBarriers = append(attemptBarriers, entry.restoreDone) | |||
| 504 | + | ||||
| 505 | + | persisted, ok, catalogueErr := catalogue.Record(record.Name) | |||
| 506 | + | require.NoError(t, catalogueErr) | |||
| 507 | + | require.True(t, ok) | |||
| 508 | + | require.Equal(t, record, persisted, "transient restore failure must preserve healthy durable authority") | |||
| 509 | + | } | |||
| 510 | + | ||||
| 511 | + | repository.mu.Lock() | |||
| 512 | + | delete(repository.errors, record.Name) | |||
| 513 | + | repository.mu.Unlock() | |||
| 514 | + | require.NoError(t, exec.restoreExplicitRecovery(t.Context())) | |||
| 515 | + | d.mu.Lock() | |||
| 516 | + | require.Equal(t, runtimeHealthy, d.stopped[record.Name].state) | |||
| 517 | + | d.mu.Unlock() | |||
| 518 | + | persisted, ok, err := catalogue.Record(record.Name) | |||
| 519 | + | require.NoError(t, err) | |||
| 520 | + | require.True(t, ok) | |||
| 521 | + | require.Equal(t, domain.RecoveryHealthy, persisted.RecoveryState) | |||
| 522 | + | repository.mu.Lock() | |||
| 523 | + | require.Equal(t, 3, repository.loads[record.Name]) | |||
| 524 | + | repository.mu.Unlock() | |||
| 525 | + | } | |||
| 526 | + | ||||
| 436 | func TestExplicitRecoveryRejectsConcurrentRestore(t *testing.T) { | 527 | func TestExplicitRecoveryRejectsConcurrentRestore(t *testing.T) { | ||
| 437 | record := durableRecoveryRecord(0) | 528 | record := durableRecoveryRecord(0) | ||
| 438 | record.RecoveryState = domain.RecoveryDegraded | 529 | record.RecoveryState = domain.RecoveryDegraded | ||
M internal/usecase/daemon/maintenance.go +13 -14
| @@ -119,7 +119,8 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool { | |||
| 119 | 119 | // Reconcile before collection. Until a complete top-level scan reaches EOF, | |
| 120 | 120 | // an unseen forward orphan may still be repairable, so retention yields | |
| 121 | 121 | // without deleting any incarnation data. | |
| 122 | - | next, decisions, err := deps.reconciler.Step(ctx, deps.cursor) | |
| 122 | + | previousCursor := deps.cursor | |
| 123 | + | next, decisions, err := deps.reconciler.Step(ctx, previousCursor) | |
| 123 | 124 | if err != nil { | |
| 124 | 125 | d.log.Warn("durable_reconciliation_tick", "action", "scan", "cursor", deps.cursor.DirectoryCookie, "done", false, "err", err) | |
| 125 | 126 | return false | |
| @@ -151,7 +152,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool { | |||
| 151 | 152 | } | |
| 152 | 153 | d.log.Info("durable_reconciliation_tick", "action", "scan", "cursor", next.DirectoryCookie, "done", next.DirectoryCookie == 0, "err", nil) | |
| 153 | 154 | if next.DirectoryCookie != 0 { | |
| 154 | - | return true | |
| 155 | + | return next.DirectoryCookie != previousCursor.DirectoryCookie | |
| 155 | 156 | } | |
| 156 | 157 | ||
| 157 | 158 | records, err := deps.catalogue.Records() | |
| @@ -159,6 +160,15 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool { | |||
| 159 | 160 | d.log.Warn("durable_maintenance_tick", "action", "catalogue-read", "done", false, "err", err) | |
| 160 | 161 | return false | |
| 161 | 162 | } | |
| 163 | + | d.mu.Lock() | |
| 164 | + | restoringIncarnations := make(map[domain.IncarnationID]struct{}) | |
| 165 | + | for _, stopped := range d.stopped { | |
| 166 | + | if stopped.state == runtimeRestoring { | |
| 167 | + | restoringIncarnations[stopped.incarnation] = struct{}{} | |
| 168 | + | } | |
| 169 | + | } | |
| 170 | + | d.mu.Unlock() | |
| 171 | + | ||
| 162 | 172 | incomplete := false | |
| 163 | 173 | for _, record := range records { | |
| 164 | 174 | if err := ctx.Err(); err != nil { | |
| @@ -166,7 +176,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool { | |||
| 166 | 176 | } | |
| 167 | 177 | _, unresolved := deps.unresolved[record.IncarnationID] | |
| 168 | 178 | _, uncertainRepair := deps.repairUncertain[record.IncarnationID] | |
| 169 | - | restoring := d.incarnationRestoring(record.IncarnationID) | |
| 179 | + | _, restoring := restoringIncarnations[record.IncarnationID] | |
| 170 | 180 | plan := retentionPlan(record, unresolved || uncertainRepair, restoring) | |
| 171 | 181 | done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes}) | |
| 172 | 182 | if err == nil && !done { | |
| @@ -185,14 +195,3 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool { | |||
| 185 | 195 | clear(deps.repairUncertain) | |
| 186 | 196 | return incomplete | |
| 187 | 197 | } | |
| 188 | - | ||
| 189 | - | func (d *Daemon) incarnationRestoring(id domain.IncarnationID) bool { | |
| 190 | - | d.mu.Lock() | |
| 191 | - | defer d.mu.Unlock() | |
| 192 | - | for _, stopped := range d.stopped { | |
| 193 | - | if stopped.incarnation == id && stopped.state == runtimeRestoring { | |
| 194 | - | return true | |
| 195 | - | } | |
| 196 | - | } | |
| 197 | - | return false | |
| 198 | - | } | |
| @@ -119,7 +119,8 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool { | |||||
| 119 | // Reconcile before collection. Until a complete top-level scan reaches EOF, | 119 | // Reconcile before collection. Until a complete top-level scan reaches EOF, | ||
| 120 | // an unseen forward orphan may still be repairable, so retention yields | 120 | // an unseen forward orphan may still be repairable, so retention yields | ||
| 121 | // without deleting any incarnation data. | 121 | // without deleting any incarnation data. | ||
| 122 | - | next, decisions, err := deps.reconciler.Step(ctx, deps.cursor) | 122 | + | previousCursor := deps.cursor |
| 123 | + | next, decisions, err := deps.reconciler.Step(ctx, previousCursor) | |||
| 123 | if err != nil { | 124 | if err != nil { | ||
| 124 | d.log.Warn("durable_reconciliation_tick", "action", "scan", "cursor", deps.cursor.DirectoryCookie, "done", false, "err", err) | 125 | d.log.Warn("durable_reconciliation_tick", "action", "scan", "cursor", deps.cursor.DirectoryCookie, "done", false, "err", err) | ||
| 125 | return false | 126 | return false | ||
| @@ -151,7 +152,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool { | |||||
| 151 | } | 152 | } | ||
| 152 | d.log.Info("durable_reconciliation_tick", "action", "scan", "cursor", next.DirectoryCookie, "done", next.DirectoryCookie == 0, "err", nil) | 153 | d.log.Info("durable_reconciliation_tick", "action", "scan", "cursor", next.DirectoryCookie, "done", next.DirectoryCookie == 0, "err", nil) | ||
| 153 | if next.DirectoryCookie != 0 { | 154 | if next.DirectoryCookie != 0 { | ||
| 154 | - | return true | 155 | + | return next.DirectoryCookie != previousCursor.DirectoryCookie |
| 155 | } | 156 | } | ||
| 156 | 157 | ||||
| 157 | records, err := deps.catalogue.Records() | 158 | records, err := deps.catalogue.Records() | ||
| @@ -159,6 +160,15 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool { | |||||
| 159 | d.log.Warn("durable_maintenance_tick", "action", "catalogue-read", "done", false, "err", err) | 160 | d.log.Warn("durable_maintenance_tick", "action", "catalogue-read", "done", false, "err", err) | ||
| 160 | return false | 161 | return false | ||
| 161 | } | 162 | } | ||
| 163 | + | d.mu.Lock() | |||
| 164 | + | restoringIncarnations := make(map[domain.IncarnationID]struct{}) | |||
| 165 | + | for _, stopped := range d.stopped { | |||
| 166 | + | if stopped.state == runtimeRestoring { | |||
| 167 | + | restoringIncarnations[stopped.incarnation] = struct{}{} | |||
| 168 | + | } | |||
| 169 | + | } | |||
| 170 | + | d.mu.Unlock() | |||
| 171 | + | ||||
| 162 | incomplete := false | 172 | incomplete := false | ||
| 163 | for _, record := range records { | 173 | for _, record := range records { | ||
| 164 | if err := ctx.Err(); err != nil { | 174 | if err := ctx.Err(); err != nil { | ||
| @@ -166,7 +176,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool { | |||||
| 166 | } | 176 | } | ||
| 167 | _, unresolved := deps.unresolved[record.IncarnationID] | 177 | _, unresolved := deps.unresolved[record.IncarnationID] | ||
| 168 | _, uncertainRepair := deps.repairUncertain[record.IncarnationID] | 178 | _, uncertainRepair := deps.repairUncertain[record.IncarnationID] | ||
| 169 | - | restoring := d.incarnationRestoring(record.IncarnationID) | 179 | + | _, restoring := restoringIncarnations[record.IncarnationID] |
| 170 | plan := retentionPlan(record, unresolved || uncertainRepair, restoring) | 180 | plan := retentionPlan(record, unresolved || uncertainRepair, restoring) | ||
| 171 | done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes}) | 181 | done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes}) | ||
| 172 | if err == nil && !done { | 182 | if err == nil && !done { | ||
| @@ -185,14 +195,3 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool { | |||||
| 185 | clear(deps.repairUncertain) | 195 | clear(deps.repairUncertain) | ||
| 186 | return incomplete | 196 | return incomplete | ||
| 187 | } | 197 | } | ||
| 188 | - | ||||
| 189 | - | func (d *Daemon) incarnationRestoring(id domain.IncarnationID) bool { | |||
| 190 | - | d.mu.Lock() | |||
| 191 | - | defer d.mu.Unlock() | |||
| 192 | - | for _, stopped := range d.stopped { | |||
| 193 | - | if stopped.incarnation == id && stopped.state == runtimeRestoring { | |||
| 194 | - | return true | |||
| 195 | - | } | |||
| 196 | - | } | |||
| 197 | - | return false | |||
| 198 | - | } | |||
M internal/usecase/daemon/maintenance_test.go +18 -2
| @@ -25,6 +25,22 @@ func (r *cursorMaintenanceReconciler) Step(_ context.Context, cursor ports.Recon | |||
| 25 | 25 | return ports.ReconcileCursor{}, nil, nil | |
| 26 | 26 | } | |
| 27 | 27 | ||
| 28 | + | type stalledMaintenanceReconciler struct{} | |
| 29 | + | ||
| 30 | + | func (stalledMaintenanceReconciler) Step(_ context.Context, cursor ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error) { | |
| 31 | + | if cursor.DirectoryCookie == 0 { | |
| 32 | + | return ports.ReconcileCursor{DirectoryCookie: 7}, nil, nil | |
| 33 | + | } | |
| 34 | + | return cursor, nil, nil | |
| 35 | + | } | |
| 36 | + | ||
| 37 | + | func TestMaintenanceDoesNotImmediatelyRetryStalledCursor(t *testing.T) { | |
| 38 | + | d := newTestDaemon(t, nil, stubClock{}) | |
| 39 | + | d.maintenance = newMaintenanceDependencies(portsmocks.NewMockCatalogue(t), portsmocks.NewMockSnapshotRepository(t), stalledMaintenanceReconciler{}, nil) | |
| 40 | + | require.True(t, d.runDurableMaintenanceTick(context.Background())) | |
| 41 | + | require.False(t, d.runDurableMaintenanceTick(context.Background())) | |
| 42 | + | } | |
| 43 | + | ||
| 28 | 44 | func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) { | |
| 29 | 45 | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh} | |
| 30 | 46 | catalogue := portsmocks.NewMockCatalogue(t) | |
| @@ -92,11 +108,11 @@ func TestMaintenanceRetentionUsesReconciliationOutcome(t *testing.T) { | |||
| 92 | 108 | wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}}, | |
| 93 | 109 | }, | |
| 94 | 110 | { | |
| 95 | - | name: "non-forward candidate kind does not pin", | |
| 111 | + | name: "non-forward candidate kind remains pinned", | |
| 96 | 112 | finding: ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{ | |
| 97 | 113 | Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}}, | |
| 98 | 114 | }}, | |
| 99 | - | wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}}, | |
| 115 | + | wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true}, | |
| 100 | 116 | }, | |
| 101 | 117 | { | |
| 102 | 118 | name: "unreadable candidate remains pinned", | |
| @@ -25,6 +25,22 @@ func (r *cursorMaintenanceReconciler) Step(_ context.Context, cursor ports.Recon | |||||
| 25 | return ports.ReconcileCursor{}, nil, nil | 25 | return ports.ReconcileCursor{}, nil, nil | ||
| 26 | } | 26 | } | ||
| 27 | 27 | ||||
| 28 | + | type stalledMaintenanceReconciler struct{} | |||
| 29 | + | ||||
| 30 | + | func (stalledMaintenanceReconciler) Step(_ context.Context, cursor ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error) { | |||
| 31 | + | if cursor.DirectoryCookie == 0 { | |||
| 32 | + | return ports.ReconcileCursor{DirectoryCookie: 7}, nil, nil | |||
| 33 | + | } | |||
| 34 | + | return cursor, nil, nil | |||
| 35 | + | } | |||
| 36 | + | ||||
| 37 | + | func TestMaintenanceDoesNotImmediatelyRetryStalledCursor(t *testing.T) { | |||
| 38 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 39 | + | d.maintenance = newMaintenanceDependencies(portsmocks.NewMockCatalogue(t), portsmocks.NewMockSnapshotRepository(t), stalledMaintenanceReconciler{}, nil) | |||
| 40 | + | require.True(t, d.runDurableMaintenanceTick(context.Background())) | |||
| 41 | + | require.False(t, d.runDurableMaintenanceTick(context.Background())) | |||
| 42 | + | } | |||
| 43 | + | ||||
| 28 | func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) { | 44 | func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) { | ||
| 29 | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh} | 45 | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh} | ||
| 30 | catalogue := portsmocks.NewMockCatalogue(t) | 46 | catalogue := portsmocks.NewMockCatalogue(t) | ||
| @@ -92,11 +108,11 @@ func TestMaintenanceRetentionUsesReconciliationOutcome(t *testing.T) { | |||||
| 92 | wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}}, | 108 | wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}}, | ||
| 93 | }, | 109 | }, | ||
| 94 | { | 110 | { | ||
| 95 | - | name: "non-forward candidate kind does not pin", | 111 | + | name: "non-forward candidate kind remains pinned", |
| 96 | finding: ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{ | 112 | finding: ports.ReconcileFinding{Kind: ports.ReconcileInvalidCandidate, Status: ports.ReconcileValidated, Candidate: ports.ReconcileCandidate{ | ||
| 97 | Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}}, | 113 | Name: record.Name, IncarnationID: record.IncarnationID, Ref: domain.CheckpointRef{Generation: 4, ManifestDigest: [32]byte{4}}, | ||
| 98 | }}, | 114 | }}, | ||
| 99 | - | wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, Keep: []ports.CheckpointRef{committed, fallback1, fallback2}}, | 115 | + | wantPlan: &ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true}, |
| 100 | }, | 116 | }, | ||
| 101 | { | 117 | { | ||
| 102 | name: "unreadable candidate remains pinned", | 118 | name: "unreadable candidate remains pinned", | ||
M internal/usecase/daemon/metadata_order_test.go +6 -2
| @@ -205,8 +205,10 @@ func TestNewerMetadataFailureDoesNotSkipOlderSnapshot(t *testing.T) { | |||
| 205 | 205 | return true | |
| 206 | 206 | } | |
| 207 | 207 | ||
| 208 | - | require.ErrorIs(t, d.persistSessionMetadata(sess, secondVersion, secondRecord.MetadataUpdate(), secondRollback), failure) | |
| 209 | - | require.NoError(t, d.persistSessionMetadata(sess, firstVersion, firstRecord.MetadataUpdate(), firstRollback)) | |
| 208 | + | _, err := d.persistSessionMetadata(sess, secondVersion, secondRecord.MetadataUpdate(), secondRollback) | |
| 209 | + | require.ErrorIs(t, err, failure) | |
| 210 | + | _, err = d.persistSessionMetadata(sess, firstVersion, firstRecord.MetadataUpdate(), firstRollback) | |
| 211 | + | require.NoError(t, err) | |
| 210 | 212 | ||
| 211 | 213 | persisted, ok, err := catalogue.Record("work") | |
| 212 | 214 | require.NoError(t, err) | |
| @@ -239,7 +241,9 @@ func TestLatestMetadataFailureRollsBackRename(t *testing.T) { | |||
| 239 | 241 | require.NoError(t, err) | |
| 240 | 242 | require.True(t, ok) | |
| 241 | 243 | require.Empty(t, persisted.TabNames) | |
| 244 | + | sess.mu.Lock() | |
| 242 | 245 | require.Empty(t, sess.tabs[0].name) | |
| 246 | + | sess.mu.Unlock() | |
| 243 | 247 | } | |
| 244 | 248 | ||
| 245 | 249 | func TestBlockedCwdWriteCannotOverwriteLaterTabMetadata(t *testing.T) { | |
| @@ -205,8 +205,10 @@ func TestNewerMetadataFailureDoesNotSkipOlderSnapshot(t *testing.T) { | |||||
| 205 | return true | 205 | return true | ||
| 206 | } | 206 | } | ||
| 207 | 207 | ||||
| 208 | - | require.ErrorIs(t, d.persistSessionMetadata(sess, secondVersion, secondRecord.MetadataUpdate(), secondRollback), failure) | 208 | + | _, err := d.persistSessionMetadata(sess, secondVersion, secondRecord.MetadataUpdate(), secondRollback) |
| 209 | - | require.NoError(t, d.persistSessionMetadata(sess, firstVersion, firstRecord.MetadataUpdate(), firstRollback)) | 209 | + | require.ErrorIs(t, err, failure) |
| 210 | + | _, err = d.persistSessionMetadata(sess, firstVersion, firstRecord.MetadataUpdate(), firstRollback) | |||
| 211 | + | require.NoError(t, err) | |||
| 210 | 212 | ||||
| 211 | persisted, ok, err := catalogue.Record("work") | 213 | persisted, ok, err := catalogue.Record("work") | ||
| 212 | require.NoError(t, err) | 214 | require.NoError(t, err) | ||
| @@ -239,7 +241,9 @@ func TestLatestMetadataFailureRollsBackRename(t *testing.T) { | |||||
| 239 | require.NoError(t, err) | 241 | require.NoError(t, err) | ||
| 240 | require.True(t, ok) | 242 | require.True(t, ok) | ||
| 241 | require.Empty(t, persisted.TabNames) | 243 | require.Empty(t, persisted.TabNames) | ||
| 244 | + | sess.mu.Lock() | |||
| 242 | require.Empty(t, sess.tabs[0].name) | 245 | require.Empty(t, sess.tabs[0].name) | ||
| 246 | + | sess.mu.Unlock() | |||
| 243 | } | 247 | } | ||
| 244 | 248 | ||||
| 245 | func TestBlockedCwdWriteCannotOverwriteLaterTabMetadata(t *testing.T) { | 249 | func TestBlockedCwdWriteCannotOverwriteLaterTabMetadata(t *testing.T) { | ||
M internal/usecase/daemon/picker_test.go +4 -2
| @@ -293,7 +293,9 @@ func TestRestoreCancellationTransitionsBeforePickerCompletion(t *testing.T) { | |||
| 293 | 293 | require.Error(t, <-result) | |
| 294 | 294 | <-restored | |
| 295 | 295 | require.Same(t, from, ac.currentSession()) | |
| 296 | + | d.mu.Lock() | |
| 296 | 297 | entry := d.stopped[record.Name] | |
| 298 | + | d.mu.Unlock() | |
| 297 | 299 | require.Equal(t, runtimeDegraded, entry.state) | |
| 298 | 300 | require.Equal(t, "restore interrupted", entry.record.DegradedReason) | |
| 299 | 301 | select { | |
| @@ -1133,7 +1135,7 @@ func TestPickerKillActiveSessionSnapshotDeleteRefusalReportsOnceAndKeepsPicker(t | |||
| 1133 | 1135 | WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause})(d) | |
| 1134 | 1136 | target := d.sessions[domain.SessionID("recent")] | |
| 1135 | 1137 | store, _ := newMockStore(t) | |
| 1136 | - | WithStore(store)(d) | |
| 1138 | + | WithStore(t, store)(d) | |
| 1137 | 1139 | target.mu.Lock() | |
| 1138 | 1140 | if target.incarnation == (domain.IncarnationID{}) { | |
| 1139 | 1141 | target.incarnation = domain.IncarnationID{1} | |
| @@ -1175,7 +1177,7 @@ func TestPickerKillStoppedSessionPersistDeleteFailureSurfacesNoticeAndKeepsEntry | |||
| 1175 | 1177 | d, from, ac, sends := newManualSessionWithPTYs(t, p) | |
| 1176 | 1178 | cause := errors.New("delete failed") | |
| 1177 | 1179 | store, state := newMockStore(t) | |
| 1178 | - | WithStore(store)(d) | |
| 1180 | + | WithStore(t, store)(d) | |
| 1179 | 1181 | record := domain.CatalogueRecord{Name: "stopped", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/stopped", CreatedAt: 7, RecoveryState: domain.RecoveryFresh} | |
| 1180 | 1182 | require.NoError(t, d.catalogue.Create(record)) | |
| 1181 | 1183 | state.mu.Lock() | |
| @@ -293,7 +293,9 @@ func TestRestoreCancellationTransitionsBeforePickerCompletion(t *testing.T) { | |||||
| 293 | require.Error(t, <-result) | 293 | require.Error(t, <-result) | ||
| 294 | <-restored | 294 | <-restored | ||
| 295 | require.Same(t, from, ac.currentSession()) | 295 | require.Same(t, from, ac.currentSession()) | ||
| 296 | + | d.mu.Lock() | |||
| 296 | entry := d.stopped[record.Name] | 297 | entry := d.stopped[record.Name] | ||
| 298 | + | d.mu.Unlock() | |||
| 297 | require.Equal(t, runtimeDegraded, entry.state) | 299 | require.Equal(t, runtimeDegraded, entry.state) | ||
| 298 | require.Equal(t, "restore interrupted", entry.record.DegradedReason) | 300 | require.Equal(t, "restore interrupted", entry.record.DegradedReason) | ||
| 299 | select { | 301 | select { | ||
| @@ -1133,7 +1135,7 @@ func TestPickerKillActiveSessionSnapshotDeleteRefusalReportsOnceAndKeepsPicker(t | |||||
| 1133 | WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause})(d) | 1135 | WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause})(d) | ||
| 1134 | target := d.sessions[domain.SessionID("recent")] | 1136 | target := d.sessions[domain.SessionID("recent")] | ||
| 1135 | store, _ := newMockStore(t) | 1137 | store, _ := newMockStore(t) | ||
| 1136 | - | WithStore(store)(d) | 1138 | + | WithStore(t, store)(d) |
| 1137 | target.mu.Lock() | 1139 | target.mu.Lock() | ||
| 1138 | if target.incarnation == (domain.IncarnationID{}) { | 1140 | if target.incarnation == (domain.IncarnationID{}) { | ||
| 1139 | target.incarnation = domain.IncarnationID{1} | 1141 | target.incarnation = domain.IncarnationID{1} | ||
| @@ -1175,7 +1177,7 @@ func TestPickerKillStoppedSessionPersistDeleteFailureSurfacesNoticeAndKeepsEntry | |||||
| 1175 | d, from, ac, sends := newManualSessionWithPTYs(t, p) | 1177 | d, from, ac, sends := newManualSessionWithPTYs(t, p) | ||
| 1176 | cause := errors.New("delete failed") | 1178 | cause := errors.New("delete failed") | ||
| 1177 | store, state := newMockStore(t) | 1179 | store, state := newMockStore(t) | ||
| 1178 | - | WithStore(store)(d) | 1180 | + | WithStore(t, store)(d) |
| 1179 | record := domain.CatalogueRecord{Name: "stopped", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/stopped", CreatedAt: 7, RecoveryState: domain.RecoveryFresh} | 1181 | record := domain.CatalogueRecord{Name: "stopped", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/stopped", CreatedAt: 7, RecoveryState: domain.RecoveryFresh} | ||
| 1180 | require.NoError(t, d.catalogue.Create(record)) | 1182 | require.NoError(t, d.catalogue.Create(record)) | ||
| 1181 | state.mu.Lock() | 1183 | state.mu.Lock() | ||
M internal/usecase/daemon/recovery_observability.go +1 -1
| @@ -21,7 +21,7 @@ func (d *Daemon) logSessionDegraded(record domain.CatalogueRecord, reasonCode st | |||
| 21 | 21 | ||
| 22 | 22 | func (d *Daemon) logStartupRecoveryCounts(restoring int) { | |
| 23 | 23 | var records []domain.CatalogueRecord | |
| 24 | - | if d.catalogue != nil { | |
| 24 | + | if d.persistEnabled && d.catalogue != nil { | |
| 25 | 25 | var err error | |
| 26 | 26 | records, err = d.catalogue.Records() | |
| 27 | 27 | if err != nil { | |
| @@ -21,7 +21,7 @@ func (d *Daemon) logSessionDegraded(record domain.CatalogueRecord, reasonCode st | |||||
| 21 | 21 | ||||
| 22 | func (d *Daemon) logStartupRecoveryCounts(restoring int) { | 22 | func (d *Daemon) logStartupRecoveryCounts(restoring int) { | ||
| 23 | var records []domain.CatalogueRecord | 23 | var records []domain.CatalogueRecord | ||
| 24 | - | if d.catalogue != nil { | 24 | + | if d.persistEnabled && d.catalogue != nil { |
| 25 | var err error | 25 | var err error | ||
| 26 | records, err = d.catalogue.Records() | 26 | records, err = d.catalogue.Records() | ||
| 27 | if err != nil { | 27 | if err != nil { | ||
M internal/usecase/daemon/session.go +29 -12
| @@ -203,20 +203,13 @@ func (d *Daemon) touchMRU(sess *session) { | |||
| 203 | 203 | rollback := func() bool { | |
| 204 | 204 | return sess.mruAt.CompareAndSwap(seq, previous) | |
| 205 | 205 | } | |
| 206 | - | if err := d.persistSessionMetadata(sess, version, record.MetadataUpdate(), rollback); err != nil { | |
| 206 | + | if _, err := d.persistSessionMetadata(sess, version, record.MetadataUpdate(), rollback); err != nil { | |
| 207 | 207 | d.log.Warn("touching persisted session recency failed", "err", err, "session", name) | |
| 208 | 208 | } | |
| 209 | 209 | } | |
| 210 | 210 | } | |
| 211 | 211 | ||
| 212 | 212 | func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) { | |
| 213 | - | // Historical focused tests call this helper directly. Acquire mu for those | |
| 214 | - | // callers while preserving the production contract that an owning caller | |
| 215 | - | // receives it still locked on return. | |
| 216 | - | acquiredMu := d.mu.TryLock() | |
| 217 | - | if acquiredMu { | |
| 218 | - | defer d.mu.Unlock() | |
| 219 | - | } | |
| 220 | 213 | env = copyEnvironment(env) | |
| 221 | 214 | if _, reserved := d.creating[name]; reserved { | |
| 222 | 215 | return nil, errSessionNameInUse | |
| @@ -533,7 +526,22 @@ func (d *Daemon) createTab(sess *session, sz domain.Size) error { | |||
| 533 | 526 | _ = pty.Close() | |
| 534 | 527 | return true | |
| 535 | 528 | } | |
| 536 | - | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |
| 529 | + | rollbackRejected, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback) | |
| 530 | + | if err != nil { | |
| 531 | + | if rollbackRejected { | |
| 532 | + | d.mu.Lock() | |
| 533 | + | if d.sessions[sess.id] == sess { | |
| 534 | + | sess.mu.Lock() | |
| 535 | + | sess.tabs = slices.DeleteFunc(sess.tabs, func(candidate *tab) bool { return candidate == tb }) | |
| 536 | + | sess.active = min(sess.active, len(sess.tabs)-1) | |
| 537 | + | sess.mu.Unlock() | |
| 538 | + | } | |
| 539 | + | d.mu.Unlock() | |
| 540 | + | if tb.cancel != nil { | |
| 541 | + | tb.cancel() | |
| 542 | + | } | |
| 543 | + | tb.closeAllPanes() | |
| 544 | + | } | |
| 537 | 545 | return err | |
| 538 | 546 | } | |
| 539 | 547 | } | |
| @@ -745,6 +753,7 @@ func (d *Daemon) renameSession(sess *session, name string) error { | |||
| 745 | 753 | oldName := sess.name | |
| 746 | 754 | wasEphemeral := sess.ephemeral | |
| 747 | 755 | createdAt := sess.createdAt | |
| 756 | + | priorIncarnation := sess.incarnation | |
| 748 | 757 | if wasEphemeral { | |
| 749 | 758 | var err error | |
| 750 | 759 | createdAt, err = d.allocateLifecycleCreatedAtLocked() | |
| @@ -772,6 +781,7 @@ func (d *Daemon) renameSession(sess *session, name string) error { | |||
| 772 | 781 | ||
| 773 | 782 | rollback := func(err error) error { | |
| 774 | 783 | sess.mu.Lock() | |
| 784 | + | sess.incarnation = priorIncarnation | |
| 775 | 785 | sess.renameInProgress = false | |
| 776 | 786 | if sess.renameDone != nil { | |
| 777 | 787 | close(sess.renameDone) | |
| @@ -869,7 +879,7 @@ func (d *Daemon) renameTab(sess *session, tb *tab, name string) error { | |||
| 869 | 879 | tb.name = oldName | |
| 870 | 880 | return true | |
| 871 | 881 | } | |
| 872 | - | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |
| 882 | + | if _, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |
| 873 | 883 | return err | |
| 874 | 884 | } | |
| 875 | 885 | return nil | |
| @@ -969,8 +979,15 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | |||
| 969 | 979 | sess.active = min(oldActive, len(sess.tabs)-1) | |
| 970 | 980 | return true | |
| 971 | 981 | } | |
| 972 | - | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |
| 982 | + | rollbackRejected, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback) | |
| 983 | + | if err != nil { | |
| 973 | 984 | d.log.Warn("persisting closed tab failed", "err", err, "session", name) | |
| 985 | + | if rollbackRejected { | |
| 986 | + | if tb.cancel != nil { | |
| 987 | + | tb.cancel() | |
| 988 | + | } | |
| 989 | + | tb.closeAllPanes() | |
| 990 | + | } | |
| 974 | 991 | return err | |
| 975 | 992 | } | |
| 976 | 993 | } | |
| @@ -1510,7 +1527,7 @@ func (d *Daemon) refreshSessionCwd(sess *session) { | |||
| 1510 | 1527 | sess.cwd = oldCwd | |
| 1511 | 1528 | return true | |
| 1512 | 1529 | } | |
| 1513 | - | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |
| 1530 | + | if _, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |
| 1514 | 1531 | d.log.Warn("touching persisted session cwd failed", "err", err, "session", name) | |
| 1515 | 1532 | return | |
| 1516 | 1533 | } | |
| @@ -203,20 +203,13 @@ func (d *Daemon) touchMRU(sess *session) { | |||||
| 203 | rollback := func() bool { | 203 | rollback := func() bool { | ||
| 204 | return sess.mruAt.CompareAndSwap(seq, previous) | 204 | return sess.mruAt.CompareAndSwap(seq, previous) | ||
| 205 | } | 205 | } | ||
| 206 | - | if err := d.persistSessionMetadata(sess, version, record.MetadataUpdate(), rollback); err != nil { | 206 | + | if _, err := d.persistSessionMetadata(sess, version, record.MetadataUpdate(), rollback); err != nil { |
| 207 | d.log.Warn("touching persisted session recency failed", "err", err, "session", name) | 207 | d.log.Warn("touching persisted session recency failed", "err", err, "session", name) | ||
| 208 | } | 208 | } | ||
| 209 | } | 209 | } | ||
| 210 | } | 210 | } | ||
| 211 | 211 | ||||
| 212 | func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) { | 212 | func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) { | ||
| 213 | - | // Historical focused tests call this helper directly. Acquire mu for those | |||
| 214 | - | // callers while preserving the production contract that an owning caller | |||
| 215 | - | // receives it still locked on return. | |||
| 216 | - | acquiredMu := d.mu.TryLock() | |||
| 217 | - | if acquiredMu { | |||
| 218 | - | defer d.mu.Unlock() | |||
| 219 | - | } | |||
| 220 | env = copyEnvironment(env) | 213 | env = copyEnvironment(env) | ||
| 221 | if _, reserved := d.creating[name]; reserved { | 214 | if _, reserved := d.creating[name]; reserved { | ||
| 222 | return nil, errSessionNameInUse | 215 | return nil, errSessionNameInUse | ||
| @@ -533,7 +526,22 @@ func (d *Daemon) createTab(sess *session, sz domain.Size) error { | |||||
| 533 | _ = pty.Close() | 526 | _ = pty.Close() | ||
| 534 | return true | 527 | return true | ||
| 535 | } | 528 | } | ||
| 536 | - | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | 529 | + | rollbackRejected, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback) |
| 530 | + | if err != nil { | |||
| 531 | + | if rollbackRejected { | |||
| 532 | + | d.mu.Lock() | |||
| 533 | + | if d.sessions[sess.id] == sess { | |||
| 534 | + | sess.mu.Lock() | |||
| 535 | + | sess.tabs = slices.DeleteFunc(sess.tabs, func(candidate *tab) bool { return candidate == tb }) | |||
| 536 | + | sess.active = min(sess.active, len(sess.tabs)-1) | |||
| 537 | + | sess.mu.Unlock() | |||
| 538 | + | } | |||
| 539 | + | d.mu.Unlock() | |||
| 540 | + | if tb.cancel != nil { | |||
| 541 | + | tb.cancel() | |||
| 542 | + | } | |||
| 543 | + | tb.closeAllPanes() | |||
| 544 | + | } | |||
| 537 | return err | 545 | return err | ||
| 538 | } | 546 | } | ||
| 539 | } | 547 | } | ||
| @@ -745,6 +753,7 @@ func (d *Daemon) renameSession(sess *session, name string) error { | |||||
| 745 | oldName := sess.name | 753 | oldName := sess.name | ||
| 746 | wasEphemeral := sess.ephemeral | 754 | wasEphemeral := sess.ephemeral | ||
| 747 | createdAt := sess.createdAt | 755 | createdAt := sess.createdAt | ||
| 756 | + | priorIncarnation := sess.incarnation | |||
| 748 | if wasEphemeral { | 757 | if wasEphemeral { | ||
| 749 | var err error | 758 | var err error | ||
| 750 | createdAt, err = d.allocateLifecycleCreatedAtLocked() | 759 | createdAt, err = d.allocateLifecycleCreatedAtLocked() | ||
| @@ -772,6 +781,7 @@ func (d *Daemon) renameSession(sess *session, name string) error { | |||||
| 772 | 781 | ||||
| 773 | rollback := func(err error) error { | 782 | rollback := func(err error) error { | ||
| 774 | sess.mu.Lock() | 783 | sess.mu.Lock() | ||
| 784 | + | sess.incarnation = priorIncarnation | |||
| 775 | sess.renameInProgress = false | 785 | sess.renameInProgress = false | ||
| 776 | if sess.renameDone != nil { | 786 | if sess.renameDone != nil { | ||
| 777 | close(sess.renameDone) | 787 | close(sess.renameDone) | ||
| @@ -869,7 +879,7 @@ func (d *Daemon) renameTab(sess *session, tb *tab, name string) error { | |||||
| 869 | tb.name = oldName | 879 | tb.name = oldName | ||
| 870 | return true | 880 | return true | ||
| 871 | } | 881 | } | ||
| 872 | - | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | 882 | + | if _, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { |
| 873 | return err | 883 | return err | ||
| 874 | } | 884 | } | ||
| 875 | return nil | 885 | return nil | ||
| @@ -969,8 +979,15 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | |||||
| 969 | sess.active = min(oldActive, len(sess.tabs)-1) | 979 | sess.active = min(oldActive, len(sess.tabs)-1) | ||
| 970 | return true | 980 | return true | ||
| 971 | } | 981 | } | ||
| 972 | - | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | 982 | + | rollbackRejected, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback) |
| 983 | + | if err != nil { | |||
| 973 | d.log.Warn("persisting closed tab failed", "err", err, "session", name) | 984 | d.log.Warn("persisting closed tab failed", "err", err, "session", name) | ||
| 985 | + | if rollbackRejected { | |||
| 986 | + | if tb.cancel != nil { | |||
| 987 | + | tb.cancel() | |||
| 988 | + | } | |||
| 989 | + | tb.closeAllPanes() | |||
| 990 | + | } | |||
| 974 | return err | 991 | return err | ||
| 975 | } | 992 | } | ||
| 976 | } | 993 | } | ||
| @@ -1510,7 +1527,7 @@ func (d *Daemon) refreshSessionCwd(sess *session) { | |||||
| 1510 | sess.cwd = oldCwd | 1527 | sess.cwd = oldCwd | ||
| 1511 | return true | 1528 | return true | ||
| 1512 | } | 1529 | } | ||
| 1513 | - | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | 1530 | + | if _, err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { |
| 1514 | d.log.Warn("touching persisted session cwd failed", "err", err, "session", name) | 1531 | d.log.Warn("touching persisted session cwd failed", "err", err, "session", name) | ||
| 1515 | return | 1532 | return | ||
| 1516 | } | 1533 | } | ||
M internal/usecase/daemon/session_purge_control_test.go +1 -1
| @@ -15,7 +15,7 @@ func TestStoppedPurgeMetadataFailureRemainsFencedForRetry(t *testing.T) { | |||
| 15 | 15 | repository := &retryablePurgeRepository{} | |
| 16 | 16 | WithSnapshotRepository(repository)(d) | |
| 17 | 17 | store, state := newMockStore(t) | |
| 18 | - | WithStore(store)(d) | |
| 18 | + | WithStore(t, store)(d) | |
| 19 | 19 | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh} | |
| 20 | 20 | require.NoError(t, d.catalogue.Create(record)) | |
| 21 | 21 | metadataErr := errors.New("metadata delete failed") | |
| @@ -15,7 +15,7 @@ func TestStoppedPurgeMetadataFailureRemainsFencedForRetry(t *testing.T) { | |||||
| 15 | repository := &retryablePurgeRepository{} | 15 | repository := &retryablePurgeRepository{} | ||
| 16 | WithSnapshotRepository(repository)(d) | 16 | WithSnapshotRepository(repository)(d) | ||
| 17 | store, state := newMockStore(t) | 17 | store, state := newMockStore(t) | ||
| 18 | - | WithStore(store)(d) | 18 | + | WithStore(t, store)(d) |
| 19 | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh} | 19 | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh} | ||
| 20 | require.NoError(t, d.catalogue.Create(record)) | 20 | require.NoError(t, d.catalogue.Create(record)) | ||
| 21 | metadataErr := errors.New("metadata delete failed") | 21 | metadataErr := errors.New("metadata delete failed") | ||
M internal/usecase/daemon/session_purge_retry_test.go +10 -4
| @@ -37,10 +37,11 @@ func (r *retryablePurgeRepository) WriteDeletionTombstone(ctx context.Context, t | |||
| 37 | 37 | func (r *retryablePurgeRepository) QuarantineDeletionSources(ctx context.Context, tombstone domain.DeletionTombstone, _ bool) error { | |
| 38 | 38 | _, hasDeadline := ctx.Deadline() | |
| 39 | 39 | r.deadlineScoped = append(r.deadlineScoped, hasDeadline) | |
| 40 | - | r.calls = append(r.calls, "incremental", "legacy") | |
| 40 | + | r.calls = append(r.calls, "incremental") | |
| 41 | 41 | if r.incrementalErr != nil { | |
| 42 | 42 | return r.incrementalErr | |
| 43 | 43 | } | |
| 44 | + | r.calls = append(r.calls, "legacy") | |
| 44 | 45 | return r.legacyErr | |
| 45 | 46 | } | |
| 46 | 47 | func (r *retryablePurgeRepository) DeleteDeletionTombstone(ctx context.Context, _ domain.IncarnationID) error { | |
| @@ -74,7 +75,7 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) { | |||
| 74 | 75 | repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy} | |
| 75 | 76 | WithSnapshotRepository(repository)(d) | |
| 76 | 77 | store, _ := newMockStore(t) | |
| 77 | - | WithStore(store)(d) | |
| 78 | + | WithStore(t, store)(d) | |
| 78 | 79 | sess := newSnapshotTestSession(t, "work", false, "/work") | |
| 79 | 80 | record := sess.persistRecordLocked(1) | |
| 80 | 81 | record.RecoveryState = domain.RecoveryFresh | |
| @@ -86,7 +87,11 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) { | |||
| 86 | 87 | ||
| 87 | 88 | require.Error(t, d.killSession(sess, ports.ReasonSessionKilled, true)) | |
| 88 | 89 | require.True(t, repository.tombstoned["work"], "partial purge must fence startup restore/import") | |
| 89 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy"}, repository.calls) | |
| 90 | + | wantCalls := []string{"tombstone", "incremental"} | |
| 91 | + | if tt.incremental == nil { | |
| 92 | + | wantCalls = append(wantCalls, "legacy") | |
| 93 | + | } | |
| 94 | + | require.Equal(t, wantCalls, repository.calls) | |
| 90 | 95 | d.mu.Lock() | |
| 91 | 96 | stopped, ok := d.stopped["work"] | |
| 92 | 97 | d.mu.Unlock() | |
| @@ -97,7 +102,8 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) { | |||
| 97 | 102 | repository.legacyErr = nil | |
| 98 | 103 | require.NoError(t, d.retryStoppedPurge("work")) | |
| 99 | 104 | require.False(t, repository.tombstoned["work"]) | |
| 100 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls) | |
| 105 | + | wantCalls = append(wantCalls, "tombstone", "incremental", "legacy", "clear tombstone") | |
| 106 | + | require.Equal(t, wantCalls, repository.calls) | |
| 101 | 107 | require.NotEmpty(t, repository.deadlineScoped) | |
| 102 | 108 | for _, scoped := range repository.deadlineScoped { | |
| 103 | 109 | require.True(t, scoped, "every purge repository call must receive a deadline-scoped context") | |
| @@ -37,10 +37,11 @@ func (r *retryablePurgeRepository) WriteDeletionTombstone(ctx context.Context, t | |||||
| 37 | func (r *retryablePurgeRepository) QuarantineDeletionSources(ctx context.Context, tombstone domain.DeletionTombstone, _ bool) error { | 37 | func (r *retryablePurgeRepository) QuarantineDeletionSources(ctx context.Context, tombstone domain.DeletionTombstone, _ bool) error { | ||
| 38 | _, hasDeadline := ctx.Deadline() | 38 | _, hasDeadline := ctx.Deadline() | ||
| 39 | r.deadlineScoped = append(r.deadlineScoped, hasDeadline) | 39 | r.deadlineScoped = append(r.deadlineScoped, hasDeadline) | ||
| 40 | - | r.calls = append(r.calls, "incremental", "legacy") | 40 | + | r.calls = append(r.calls, "incremental") |
| 41 | if r.incrementalErr != nil { | 41 | if r.incrementalErr != nil { | ||
| 42 | return r.incrementalErr | 42 | return r.incrementalErr | ||
| 43 | } | 43 | } | ||
| 44 | + | r.calls = append(r.calls, "legacy") | |||
| 44 | return r.legacyErr | 45 | return r.legacyErr | ||
| 45 | } | 46 | } | ||
| 46 | func (r *retryablePurgeRepository) DeleteDeletionTombstone(ctx context.Context, _ domain.IncarnationID) error { | 47 | func (r *retryablePurgeRepository) DeleteDeletionTombstone(ctx context.Context, _ domain.IncarnationID) error { | ||
| @@ -74,7 +75,7 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) { | |||||
| 74 | repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy} | 75 | repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy} | ||
| 75 | WithSnapshotRepository(repository)(d) | 76 | WithSnapshotRepository(repository)(d) | ||
| 76 | store, _ := newMockStore(t) | 77 | store, _ := newMockStore(t) | ||
| 77 | - | WithStore(store)(d) | 78 | + | WithStore(t, store)(d) |
| 78 | sess := newSnapshotTestSession(t, "work", false, "/work") | 79 | sess := newSnapshotTestSession(t, "work", false, "/work") | ||
| 79 | record := sess.persistRecordLocked(1) | 80 | record := sess.persistRecordLocked(1) | ||
| 80 | record.RecoveryState = domain.RecoveryFresh | 81 | record.RecoveryState = domain.RecoveryFresh | ||
| @@ -86,7 +87,11 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) { | |||||
| 86 | 87 | ||||
| 87 | require.Error(t, d.killSession(sess, ports.ReasonSessionKilled, true)) | 88 | require.Error(t, d.killSession(sess, ports.ReasonSessionKilled, true)) | ||
| 88 | require.True(t, repository.tombstoned["work"], "partial purge must fence startup restore/import") | 89 | require.True(t, repository.tombstoned["work"], "partial purge must fence startup restore/import") | ||
| 89 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy"}, repository.calls) | 90 | + | wantCalls := []string{"tombstone", "incremental"} |
| 91 | + | if tt.incremental == nil { | |||
| 92 | + | wantCalls = append(wantCalls, "legacy") | |||
| 93 | + | } | |||
| 94 | + | require.Equal(t, wantCalls, repository.calls) | |||
| 90 | d.mu.Lock() | 95 | d.mu.Lock() | ||
| 91 | stopped, ok := d.stopped["work"] | 96 | stopped, ok := d.stopped["work"] | ||
| 92 | d.mu.Unlock() | 97 | d.mu.Unlock() | ||
| @@ -97,7 +102,8 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) { | |||||
| 97 | repository.legacyErr = nil | 102 | repository.legacyErr = nil | ||
| 98 | require.NoError(t, d.retryStoppedPurge("work")) | 103 | require.NoError(t, d.retryStoppedPurge("work")) | ||
| 99 | require.False(t, repository.tombstoned["work"]) | 104 | require.False(t, repository.tombstoned["work"]) | ||
| 100 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls) | 105 | + | wantCalls = append(wantCalls, "tombstone", "incremental", "legacy", "clear tombstone") |
| 106 | + | require.Equal(t, wantCalls, repository.calls) | |||
| 101 | require.NotEmpty(t, repository.deadlineScoped) | 107 | require.NotEmpty(t, repository.deadlineScoped) | ||
| 102 | for _, scoped := range repository.deadlineScoped { | 108 | for _, scoped := range repository.deadlineScoped { | ||
| 103 | require.True(t, scoped, "every purge repository call must receive a deadline-scoped context") | 109 | require.True(t, scoped, "every purge repository call must receive a deadline-scoped context") | ||
M internal/usecase/daemon/session_rename_purge_test.go +2 -3
| @@ -7,7 +7,6 @@ import ( | |||
| 7 | 7 | "github.com/stretchr/testify/require" | |
| 8 | 8 | ||
| 9 | 9 | "github.com/bnema/vev/internal/domain" | |
| 10 | - | "github.com/bnema/vev/internal/persist" | |
| 11 | 10 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | |
| 12 | 11 | ) | |
| 13 | 12 | ||
| @@ -25,10 +24,10 @@ func TestRenamePreservesIncarnationSnapshotSources(t *testing.T) { | |||
| 25 | 24 | repository := &retryablePurgeRepository{legacyErr: tt.legacyErr} | |
| 26 | 25 | WithSnapshotRepository(repository)(d) | |
| 27 | 26 | store, state := newMockStore(t) | |
| 28 | - | WithStore(store)(d) | |
| 27 | + | WithStore(t, store)(d) | |
| 29 | 28 | sess := newSnapshotTestSession(t, "old", false, "/work") | |
| 30 | 29 | d.sessions = map[domain.SessionID]*session{sess.id: sess} | |
| 31 | - | require.NoError(t, d.catalogue.(*persist.Persister).Save(sess.persistRecordLocked(1))) | |
| 30 | + | require.NoError(t, testPersister(t, d).Save(sess.persistRecordLocked(1))) | |
| 32 | 31 | ||
| 33 | 32 | require.NoError(t, d.renameSession(sess, "new")) | |
| 34 | 33 | require.Empty(t, repository.calls, "rename must not invoke source deletion, including a failing legacy source") | |
| @@ -7,7 +7,6 @@ import ( | |||||
| 7 | "github.com/stretchr/testify/require" | 7 | "github.com/stretchr/testify/require" | ||
| 8 | 8 | ||||
| 9 | "github.com/bnema/vev/internal/domain" | 9 | "github.com/bnema/vev/internal/domain" | ||
| 10 | - | "github.com/bnema/vev/internal/persist" | |||
| 11 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | 10 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | ||
| 12 | ) | 11 | ) | ||
| 13 | 12 | ||||
| @@ -25,10 +24,10 @@ func TestRenamePreservesIncarnationSnapshotSources(t *testing.T) { | |||||
| 25 | repository := &retryablePurgeRepository{legacyErr: tt.legacyErr} | 24 | repository := &retryablePurgeRepository{legacyErr: tt.legacyErr} | ||
| 26 | WithSnapshotRepository(repository)(d) | 25 | WithSnapshotRepository(repository)(d) | ||
| 27 | store, state := newMockStore(t) | 26 | store, state := newMockStore(t) | ||
| 28 | - | WithStore(store)(d) | 27 | + | WithStore(t, store)(d) |
| 29 | sess := newSnapshotTestSession(t, "old", false, "/work") | 28 | sess := newSnapshotTestSession(t, "old", false, "/work") | ||
| 30 | d.sessions = map[domain.SessionID]*session{sess.id: sess} | 29 | d.sessions = map[domain.SessionID]*session{sess.id: sess} | ||
| 31 | - | require.NoError(t, d.catalogue.(*persist.Persister).Save(sess.persistRecordLocked(1))) | 30 | + | require.NoError(t, testPersister(t, d).Save(sess.persistRecordLocked(1))) |
| 32 | 31 | ||||
| 33 | require.NoError(t, d.renameSession(sess, "new")) | 32 | require.NoError(t, d.renameSession(sess, "new")) | ||
| 34 | require.Empty(t, repository.calls, "rename must not invoke source deletion, including a failing legacy source") | 33 | require.Empty(t, repository.calls, "rename must not invoke source deletion, including a failing legacy source") | ||
M internal/usecase/daemon/session_test.go +72 -60
| @@ -702,18 +702,30 @@ func newMockStore(t *testing.T) (*portsmocks.MockStore, *mockStoreState) { | |||
| 702 | 702 | return nil | |
| 703 | 703 | }).Maybe() | |
| 704 | 704 | store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error { | |
| 705 | + | if len(changes) == 0 { | |
| 706 | + | return errors.New("empty batch") | |
| 707 | + | } | |
| 705 | 708 | state.mu.Lock() | |
| 709 | + | seen := make(map[string]struct{}, len(changes)) | |
| 710 | + | for _, change := range changes { | |
| 711 | + | key := string(change.Key) | |
| 712 | + | if _, duplicate := seen[key]; duplicate { | |
| 713 | + | state.mu.Unlock() | |
| 714 | + | return errors.New("duplicate batch key") | |
| 715 | + | } | |
| 716 | + | seen[key] = struct{}{} | |
| 717 | + | if change.Delete && state.deleteErr != nil { | |
| 718 | + | if err := state.deleteErr(key); err != nil { | |
| 719 | + | state.mu.Unlock() | |
| 720 | + | return err | |
| 721 | + | } | |
| 722 | + | } | |
| 723 | + | } | |
| 706 | 724 | deleted := false | |
| 707 | 725 | for _, change := range changes { | |
| 708 | 726 | key := string(change.Key) | |
| 709 | 727 | if change.Delete { | |
| 710 | 728 | state.dels = append(state.dels, key) | |
| 711 | - | if state.deleteErr != nil { | |
| 712 | - | if err := state.deleteErr(key); err != nil { | |
| 713 | - | state.mu.Unlock() | |
| 714 | - | return err | |
| 715 | - | } | |
| 716 | - | } | |
| 717 | 729 | delete(state.data, key) | |
| 718 | 730 | deleted = true | |
| 719 | 731 | continue | |
| @@ -845,9 +857,9 @@ func TestNamedSessionCreationIsAllowedWithNilStore(t *testing.T) { | |||
| 845 | 857 | pty, release := newBlockingPTY(t) | |
| 846 | 858 | defer release() | |
| 847 | 859 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | |
| 848 | - | WithStore(nil)(d) | |
| 860 | + | WithStore(t, nil)(d) | |
| 849 | 861 | ||
| 850 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 862 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 851 | 863 | require.NoError(t, err) | |
| 852 | 864 | require.Equal(t, "work", sess.name) | |
| 853 | 865 | require.False(t, d.persistEnabled) | |
| @@ -855,11 +867,11 @@ func TestNamedSessionCreationIsAllowedWithNilStore(t *testing.T) { | |||
| 855 | 867 | ||
| 856 | 868 | func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) { | |
| 857 | 869 | store, _ := newMockStore(t) | |
| 858 | - | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | |
| 870 | + | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store)) | |
| 859 | 871 | incarnation := domain.IncarnationID{1} | |
| 860 | - | require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh})) | |
| 872 | + | require.NoError(t, testPersister(t, seed).Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh})) | |
| 861 | 873 | ||
| 862 | - | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | |
| 874 | + | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store)) | |
| 863 | 875 | d.mu.Lock() | |
| 864 | 876 | stopped := d.stopped["work"] | |
| 865 | 877 | d.mu.Unlock() | |
| @@ -876,7 +888,7 @@ func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) { | |||
| 876 | 888 | ||
| 877 | 889 | // TestDaemonNewWithoutPersistenceLogsNoLoadWarning covers the default, | |
| 878 | 890 | // persistence-free construction path (no WithStore/WithCatalogue option): | |
| 879 | - | // d.catalogue.(*persist.Persister) is the persist.New(nil) sentinel, so LoadAll now returns | |
| 891 | + | // testPersister(t, d) is the persist.New(nil) sentinel, so LoadAll now returns | |
| 880 | 892 | // errPersistenceUnavailable instead of an empty slice. That must not surface | |
| 881 | 893 | // as a "loading persisted sessions failed" warning, since nothing failed — | |
| 882 | 894 | // persistence simply was never configured. | |
| @@ -890,7 +902,7 @@ func TestDaemonNewWithoutPersistenceLogsNoLoadWarning(t *testing.T) { | |||
| 890 | 902 | } | |
| 891 | 903 | ||
| 892 | 904 | func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) { | |
| 893 | - | d := &Daemon{} | |
| 905 | + | d := &Daemon{clock: stubClock{}} | |
| 894 | 906 | sess := &session{} | |
| 895 | 907 | const goroutines = 16 | |
| 896 | 908 | const iterations = 1000 | |
| @@ -937,9 +949,9 @@ func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) { | |||
| 937 | 949 | func TestTouchMRUPersistsNamedButNotEphemeral(t *testing.T) { | |
| 938 | 950 | store, state := newMockStore(t) | |
| 939 | 951 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 940 | - | WithStore(store)(d) | |
| 952 | + | WithStore(t, store)(d) | |
| 941 | 953 | named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1, incarnation: domain.IncarnationID{1}} | |
| 942 | - | require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) | |
| 954 | + | require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) | |
| 943 | 955 | ||
| 944 | 956 | d.touchMRU(named) | |
| 945 | 957 | require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq) | |
| @@ -957,11 +969,11 @@ func TestCreateSessionSeedsMRUFromStopped(t *testing.T) { | |||
| 957 | 969 | defer release() | |
| 958 | 970 | store, state := newMockStore(t) | |
| 959 | 971 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |
| 960 | - | WithStore(store)(d) | |
| 972 | + | WithStore(t, store)(d) | |
| 961 | 973 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 1, lastUsedSeq: 42} | |
| 962 | 974 | d.mruSeq.Store(42) | |
| 963 | 975 | ||
| 964 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 976 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 965 | 977 | require.NoError(t, err) | |
| 966 | 978 | require.Equal(t, uint64(42), sess.mruAt.Load()) | |
| 967 | 979 | require.Equal(t, uint64(42), state.record(t, "work").LastUsedSeq) | |
| @@ -975,9 +987,9 @@ func TestCreateRenameKillPersistenceLifecycle(t *testing.T) { | |||
| 975 | 987 | defer release2() | |
| 976 | 988 | store, state := newMockStore(t) | |
| 977 | 989 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | |
| 978 | - | WithStore(store)(d) | |
| 990 | + | WithStore(t, store)(d) | |
| 979 | 991 | ||
| 980 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 992 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 981 | 993 | require.NoError(t, err) | |
| 982 | 994 | require.True(t, state.has("work")) | |
| 983 | 995 | created := state.record(t, "work") | |
| @@ -1006,13 +1018,13 @@ func TestRenameTabPersistsForNamedSession(t *testing.T) { | |||
| 1006 | 1018 | defer release() | |
| 1007 | 1019 | store, _ := newMockStore(t) | |
| 1008 | 1020 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |
| 1009 | - | WithStore(store)(d) | |
| 1021 | + | WithStore(t, store)(d) | |
| 1010 | 1022 | ||
| 1011 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 1023 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 1012 | 1024 | require.NoError(t, err) | |
| 1013 | 1025 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | |
| 1014 | 1026 | ||
| 1015 | - | records, err := d.catalogue.(*persist.Persister).LoadAll() | |
| 1027 | + | records, err := testPersister(t, d).LoadAll() | |
| 1016 | 1028 | require.NoError(t, err) | |
| 1017 | 1029 | require.Len(t, records, 1) | |
| 1018 | 1030 | require.Equal(t, "work", records[0].Name) | |
| @@ -1059,10 +1071,10 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | |||
| 1059 | 1071 | } | |
| 1060 | 1072 | store, _ := newMockStore(t) | |
| 1061 | 1073 | d := newTestDaemon(t, newFactorySeq(t, ptys...), stubClock{}) | |
| 1062 | - | WithStore(store)(d) | |
| 1074 | + | WithStore(t, store)(d) | |
| 1063 | 1075 | sz := domain.Size{Cols: 80, Rows: 24} | |
| 1064 | 1076 | ||
| 1065 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 1077 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 1066 | 1078 | require.NoError(t, err) | |
| 1067 | 1079 | require.NoError(t, d.createTab(sess, sz)) | |
| 1068 | 1080 | require.NoError(t, d.createTab(sess, sz)) | |
| @@ -1071,7 +1083,7 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | |||
| 1071 | 1083 | } | |
| 1072 | 1084 | ||
| 1073 | 1085 | require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false)) | |
| 1074 | - | records, err := d.catalogue.(*persist.Persister).LoadAll() | |
| 1086 | + | records, err := testPersister(t, d).LoadAll() | |
| 1075 | 1087 | require.NoError(t, err) | |
| 1076 | 1088 | require.Len(t, records, 1) | |
| 1077 | 1089 | require.Equal(t, tt.want, records[0].TabNames) | |
| @@ -1108,15 +1120,15 @@ func TestRenameTabDoesNotPersistForEphemeralSession(t *testing.T) { | |||
| 1108 | 1120 | defer release() | |
| 1109 | 1121 | store, state := newMockStore(t) | |
| 1110 | 1122 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |
| 1111 | - | WithStore(store)(d) | |
| 1123 | + | WithStore(t, store)(d) | |
| 1112 | 1124 | ||
| 1113 | - | sess, err := d.createSessionLocked("0", true, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 1125 | + | sess, err := createSessionForTest(d, "0", true, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 1114 | 1126 | require.NoError(t, err) | |
| 1115 | 1127 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | |
| 1116 | 1128 | require.Equal(t, "shell", sess.tabs[0].name) | |
| 1117 | 1129 | ||
| 1118 | 1130 | require.False(t, state.has("0")) | |
| 1119 | - | records, err := d.catalogue.(*persist.Persister).LoadAll() | |
| 1131 | + | records, err := testPersister(t, d).LoadAll() | |
| 1120 | 1132 | require.NoError(t, err) | |
| 1121 | 1133 | require.Empty(t, records) | |
| 1122 | 1134 | } | |
| @@ -1129,8 +1141,8 @@ func TestAttachRestoresPersistedTabNames(t *testing.T) { | |||
| 1129 | 1141 | defer release2() | |
| 1130 | 1142 | store, _ := newMockStore(t) | |
| 1131 | 1143 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | |
| 1132 | - | WithStore(store)(d) | |
| 1133 | - | require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh})) | |
| 1144 | + | WithStore(t, store)(d) | |
| 1145 | + | require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh})) | |
| 1134 | 1146 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}} | |
| 1135 | 1147 | tr := portsmocks.NewMockTransport(t) | |
| 1136 | 1148 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | |
| @@ -1150,10 +1162,10 @@ func TestEphemeralRenamePromotesAndStoppedCollisionRejected(t *testing.T) { | |||
| 1150 | 1162 | defer release() | |
| 1151 | 1163 | store, state := newMockStore(t) | |
| 1152 | 1164 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |
| 1153 | - | WithStore(store)(d) | |
| 1165 | + | WithStore(t, store)(d) | |
| 1154 | 1166 | d.stopped["taken"] = stoppedSession{name: "taken", cwd: "/tmp", createdAt: 1} | |
| 1155 | 1167 | ||
| 1156 | - | sess, err := d.createSessionLocked("0", true, "/tmp/e", sz, terminalEnv{}, d.baseEnv) | |
| 1168 | + | sess, err := createSessionForTest(d, "0", true, "/tmp/e", sz, terminalEnv{}, d.baseEnv) | |
| 1157 | 1169 | require.NoError(t, err) | |
| 1158 | 1170 | require.False(t, state.has("0")) | |
| 1159 | 1171 | require.EqualError(t, d.renameSession(sess, "taken"), "name already in use") | |
| @@ -1167,9 +1179,9 @@ func TestEphemeralPromotionAssignsAndPersistsLifecycleIdentity(t *testing.T) { | |||
| 1167 | 1179 | store, state := newMockStore(t) | |
| 1168 | 1180 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}} | |
| 1169 | 1181 | d := newTestDaemon(t, newFactory(t, p), clock) | |
| 1170 | - | WithStore(store)(d) | |
| 1182 | + | WithStore(t, store)(d) | |
| 1171 | 1183 | ||
| 1172 | - | sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 1184 | + | sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 1173 | 1185 | require.NoError(t, err) | |
| 1174 | 1186 | require.Zero(t, sess.createdAt, "regression fixture: ephemerals have no lifecycle identity") | |
| 1175 | 1187 | ||
| @@ -1194,9 +1206,9 @@ func TestEphemeralPromotionLifecyclePreventsStaleSameNamePaletteTarget(t *testin | |||
| 1194 | 1206 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}} | |
| 1195 | 1207 | d.clock = clock | |
| 1196 | 1208 | d.ptys = newFactorySeq(t, firstPTY, secondPTY) | |
| 1197 | - | WithStore(store)(d) | |
| 1209 | + | WithStore(t, store)(d) | |
| 1198 | 1210 | ||
| 1199 | - | first, err := d.createSessionLocked("0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv) | |
| 1211 | + | first, err := createSessionForTest(d, "0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv) | |
| 1200 | 1212 | require.NoError(t, err) | |
| 1201 | 1213 | require.NoError(t, d.renameSession(first, "named")) | |
| 1202 | 1214 | staleCreatedAt := first.createdAt | |
| @@ -1204,7 +1216,7 @@ func TestEphemeralPromotionLifecyclePreventsStaleSameNamePaletteTarget(t *testin | |||
| 1204 | 1216 | require.NoError(t, d.killSession(first, ports.ReasonSessionKilled, true)) | |
| 1205 | 1217 | d.sessWg.Wait() | |
| 1206 | 1218 | ||
| 1207 | - | second, err := d.createSessionLocked("0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv) | |
| 1219 | + | second, err := createSessionForTest(d, "0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv) | |
| 1208 | 1220 | require.NoError(t, err) | |
| 1209 | 1221 | require.NoError(t, d.renameSession(second, "named")) | |
| 1210 | 1222 | require.NotEqual(t, staleCreatedAt, second.createdAt) | |
| @@ -1221,10 +1233,10 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T) | |||
| 1221 | 1233 | defer release() | |
| 1222 | 1234 | store, state := newMockStore(t) | |
| 1223 | 1235 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |
| 1224 | - | WithStore(store)(d) | |
| 1236 | + | WithStore(t, store)(d) | |
| 1225 | 1237 | d.lastAllocatedCreatedAt = math.MaxInt64 | |
| 1226 | 1238 | ||
| 1227 | - | sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 1239 | + | sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 1228 | 1240 | require.NoError(t, err) | |
| 1229 | 1241 | require.EqualError(t, d.renameSession(sess, "named"), "daemon: lifecycle identities exhausted") | |
| 1230 | 1242 | require.Equal(t, "0", sess.name) | |
| @@ -1250,9 +1262,9 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T) | |||
| 1250 | 1262 | }).Once() | |
| 1251 | 1263 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}} | |
| 1252 | 1264 | d := newTestDaemon(t, newFactory(t, p), clock) | |
| 1253 | - | WithStore(store)(d) | |
| 1265 | + | WithStore(t, store)(d) | |
| 1254 | 1266 | ||
| 1255 | - | sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 1267 | + | sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 1256 | 1268 | require.NoError(t, err) | |
| 1257 | 1269 | require.EqualError(t, d.renameSession(sess, "named"), "disk full") | |
| 1258 | 1270 | require.Equal(t, "0", sess.name) | |
| @@ -1277,11 +1289,11 @@ func TestRefreshSessionCwdTouchesOnlyOnChange(t *testing.T) { | |||
| 1277 | 1289 | defer release() | |
| 1278 | 1290 | store, state := newMockStore(t) | |
| 1279 | 1291 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |
| 1280 | - | WithStore(store)(d) | |
| 1292 | + | WithStore(t, store)(d) | |
| 1281 | 1293 | cwd := "/tmp/work" | |
| 1282 | 1294 | WithCwdReader(func(int) (string, error) { return cwd, nil })(d) | |
| 1283 | 1295 | ||
| 1284 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 1296 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 1285 | 1297 | require.NoError(t, err) | |
| 1286 | 1298 | state.mu.Lock() | |
| 1287 | 1299 | setsAfterCreate := state.sets | |
| @@ -1305,7 +1317,7 @@ func TestAttachResumesStoppedSessionFromStoredCwd(t *testing.T) { | |||
| 1305 | 1317 | defer release() | |
| 1306 | 1318 | store, _ := newMockStore(t) | |
| 1307 | 1319 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |
| 1308 | - | WithStore(store)(d) | |
| 1320 | + | WithStore(t, store)(d) | |
| 1309 | 1321 | cwd := t.TempDir() | |
| 1310 | 1322 | d.stopped["work"] = stoppedSession{name: "work", cwd: cwd, createdAt: 1} | |
| 1311 | 1323 | tr := portsmocks.NewMockTransport(t) | |
| @@ -1329,7 +1341,7 @@ func TestAttachStoppedMissingCwdFallsBackToHome(t *testing.T) { | |||
| 1329 | 1341 | defer release() | |
| 1330 | 1342 | store, _ := newMockStore(t) | |
| 1331 | 1343 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |
| 1332 | - | WithStore(store)(d) | |
| 1344 | + | WithStore(t, store)(d) | |
| 1333 | 1345 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/definitely/missing/vev", createdAt: 1} | |
| 1334 | 1346 | tr := portsmocks.NewMockTransport(t) | |
| 1335 | 1347 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | |
| @@ -1343,8 +1355,8 @@ func TestAttachStoppedMissingCwdFallsBackToHome(t *testing.T) { | |||
| 1343 | 1355 | func TestPickerStoppedTargetKillPurges(t *testing.T) { | |
| 1344 | 1356 | store, state := newMockStore(t) | |
| 1345 | 1357 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 1346 | - | WithStore(store)(d) | |
| 1347 | - | require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) | |
| 1358 | + | WithStore(t, store)(d) | |
| 1359 | + | require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) | |
| 1348 | 1360 | d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1} | |
| 1349 | 1361 | require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true})) | |
| 1350 | 1362 | require.False(t, state.has("old")) | |
| @@ -1378,7 +1390,7 @@ func TestNewSessionAssignsStableIDsAndChildEnv(t *testing.T) { | |||
| 1378 | 1390 | d := newTestDaemon(t, f, stubClock{}) | |
| 1379 | 1391 | d.baseEnv = []string{"KEEP=1", "TERM=old", "COLORTERM=old", "TERM_PROGRAM=old", "VEV=old"} | |
| 1380 | 1392 | ||
| 1381 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{TrueColor: true}, d.baseEnv) | |
| 1393 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{TrueColor: true}, d.baseEnv) | |
| 1382 | 1394 | require.NoError(t, err) | |
| 1383 | 1395 | defer func() { | |
| 1384 | 1396 | _ = d.killSession(sess, ports.ReasonServerShutdown, false) | |
| @@ -1434,7 +1446,7 @@ func TestRenameSessionUnsafeNameRejected(t *testing.T) { | |||
| 1434 | 1446 | p, release := newBlockingPTY(t) | |
| 1435 | 1447 | defer release() | |
| 1436 | 1448 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |
| 1437 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 1449 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 1438 | 1450 | require.NoError(t, err) | |
| 1439 | 1451 | require.ErrorIs(t, d.renameSession(sess, "my work"), domain.ErrInvalidSessionName) | |
| 1440 | 1452 | } | |
| @@ -1444,7 +1456,7 @@ func TestCreateTabPtyFailureIsUserError(t *testing.T) { | |||
| 1444 | 1456 | p, release := newBlockingPTY(t) | |
| 1445 | 1457 | defer release() | |
| 1446 | 1458 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |
| 1447 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 1459 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | |
| 1448 | 1460 | require.NoError(t, err) | |
| 1449 | 1461 | ||
| 1450 | 1462 | cause := errors.New("fork/exec: no such file") | |
| @@ -1471,12 +1483,12 @@ func TestNaturalExitStoppedButExplicitKillPurges(t *testing.T) { | |||
| 1471 | 1483 | defer release2() | |
| 1472 | 1484 | store, state := newMockStore(t) | |
| 1473 | 1485 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | |
| 1474 | - | WithStore(store)(d) | |
| 1486 | + | WithStore(t, store)(d) | |
| 1475 | 1487 | WithCwdReader(func(int) (string, error) { return "/tmp/latest", nil })(d) | |
| 1476 | 1488 | ||
| 1477 | - | natural, err := d.createSessionLocked("natural", false, "/tmp/old", sz, terminalEnv{}, d.baseEnv) | |
| 1489 | + | natural, err := createSessionForTest(d, "natural", false, "/tmp/old", sz, terminalEnv{}, d.baseEnv) | |
| 1478 | 1490 | require.NoError(t, err) | |
| 1479 | - | other, err := d.createSessionLocked("other", false, "/tmp/other", sz, terminalEnv{}, d.baseEnv) | |
| 1491 | + | other, err := createSessionForTest(d, "other", false, "/tmp/other", sz, terminalEnv{}, d.baseEnv) | |
| 1480 | 1492 | require.NoError(t, err) | |
| 1481 | 1493 | _ = d.killSession(natural, ports.ReasonSessionKilled, false) | |
| 1482 | 1494 | require.True(t, state.has("natural")) | |
| @@ -1792,7 +1804,7 @@ func TestNamedSessionLifecycleTimestampsAreMonotonicAcrossClockRegression(t *tes | |||
| 1792 | 1804 | ||
| 1793 | 1805 | var got []int64 | |
| 1794 | 1806 | for _, name := range []string{"one", "two", "three"} { | |
| 1795 | - | sess, err := d.createSessionLocked(name, false, "/tmp", sz, terminalEnv{}, d.baseEnv) | |
| 1807 | + | sess, err := createSessionForTest(d, name, false, "/tmp", sz, terminalEnv{}, d.baseEnv) | |
| 1796 | 1808 | require.NoError(t, err) | |
| 1797 | 1809 | got = append(got, sess.createdAt) | |
| 1798 | 1810 | } | |
| @@ -1805,7 +1817,7 @@ func TestNamedSessionLifecycleExhaustionDoesNotMutateSessionState(t *testing.T) | |||
| 1805 | 1817 | d.nextID = 17 | |
| 1806 | 1818 | d.stopped["retained"] = stoppedSession{name: "retained", cwd: "/tmp", createdAt: 9} | |
| 1807 | 1819 | ||
| 1808 | - | sess, err := d.createSessionLocked("new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 1820 | + | sess, err := createSessionForTest(d, "new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 1809 | 1821 | ||
| 1810 | 1822 | require.ErrorContains(t, err, "lifecycle identities exhausted") | |
| 1811 | 1823 | require.Nil(t, sess) | |
| @@ -1855,18 +1867,18 @@ func TestCatalogueRecordsConstructExpectedSessionRegistry(t *testing.T) { | |||
| 1855 | 1867 | ||
| 1856 | 1868 | func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) { | |
| 1857 | 1869 | store, _ := newMockStore(t) | |
| 1858 | - | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | |
| 1859 | - | require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh})) | |
| 1870 | + | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store)) | |
| 1871 | + | require.NoError(t, testPersister(t, seed).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh})) | |
| 1860 | 1872 | ||
| 1861 | 1873 | p, release := newBlockingPTY(t) | |
| 1862 | 1874 | defer release() | |
| 1863 | 1875 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}} | |
| 1864 | 1876 | // Constructing with persistence must establish the lifecycle high-water mark. | |
| 1865 | - | d := New(newFactory(t, p), clock, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | |
| 1877 | + | d := New(newFactory(t, p), clock, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store)) | |
| 1866 | 1878 | d.serveCtx, d.serveCancel = context.WithCancel(context.Background()) | |
| 1867 | 1879 | t.Cleanup(d.serveCancel) | |
| 1868 | 1880 | ||
| 1869 | - | sess, err := d.createSessionLocked("new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 1881 | + | sess, err := createSessionForTest(d, "new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |
| 1870 | 1882 | require.NoError(t, err) | |
| 1871 | 1883 | require.Equal(t, int64(901), sess.createdAt) | |
| 1872 | 1884 | } | |
| @@ -1878,7 +1890,7 @@ func TestResumingStoppedSessionPreservesLifecycleIdentityInPersistence(t *testin | |||
| 1878 | 1890 | defer releaseTarget() | |
| 1879 | 1891 | d, from, ac, _ := newManualSessionWithPTYs(t, fromPTY) | |
| 1880 | 1892 | store, state := newMockStore(t) | |
| 1881 | - | WithStore(store)(d) | |
| 1893 | + | WithStore(t, store)(d) | |
| 1882 | 1894 | d.ptys = newFactory(t, targetPTY) | |
| 1883 | 1895 | d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp", createdAt: 77} | |
| 1884 | 1896 | ||
| @@ -702,18 +702,30 @@ func newMockStore(t *testing.T) (*portsmocks.MockStore, *mockStoreState) { | |||||
| 702 | return nil | 702 | return nil | ||
| 703 | }).Maybe() | 703 | }).Maybe() | ||
| 704 | store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error { | 704 | store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error { | ||
| 705 | + | if len(changes) == 0 { | |||
| 706 | + | return errors.New("empty batch") | |||
| 707 | + | } | |||
| 705 | state.mu.Lock() | 708 | state.mu.Lock() | ||
| 709 | + | seen := make(map[string]struct{}, len(changes)) | |||
| 710 | + | for _, change := range changes { | |||
| 711 | + | key := string(change.Key) | |||
| 712 | + | if _, duplicate := seen[key]; duplicate { | |||
| 713 | + | state.mu.Unlock() | |||
| 714 | + | return errors.New("duplicate batch key") | |||
| 715 | + | } | |||
| 716 | + | seen[key] = struct{}{} | |||
| 717 | + | if change.Delete && state.deleteErr != nil { | |||
| 718 | + | if err := state.deleteErr(key); err != nil { | |||
| 719 | + | state.mu.Unlock() | |||
| 720 | + | return err | |||
| 721 | + | } | |||
| 722 | + | } | |||
| 723 | + | } | |||
| 706 | deleted := false | 724 | deleted := false | ||
| 707 | for _, change := range changes { | 725 | for _, change := range changes { | ||
| 708 | key := string(change.Key) | 726 | key := string(change.Key) | ||
| 709 | if change.Delete { | 727 | if change.Delete { | ||
| 710 | state.dels = append(state.dels, key) | 728 | state.dels = append(state.dels, key) | ||
| 711 | - | if state.deleteErr != nil { | |||
| 712 | - | if err := state.deleteErr(key); err != nil { | |||
| 713 | - | state.mu.Unlock() | |||
| 714 | - | return err | |||
| 715 | - | } | |||
| 716 | - | } | |||
| 717 | delete(state.data, key) | 729 | delete(state.data, key) | ||
| 718 | deleted = true | 730 | deleted = true | ||
| 719 | continue | 731 | continue | ||
| @@ -845,9 +857,9 @@ func TestNamedSessionCreationIsAllowedWithNilStore(t *testing.T) { | |||||
| 845 | pty, release := newBlockingPTY(t) | 857 | pty, release := newBlockingPTY(t) | ||
| 846 | defer release() | 858 | defer release() | ||
| 847 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | 859 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | ||
| 848 | - | WithStore(nil)(d) | 860 | + | WithStore(t, nil)(d) |
| 849 | 861 | ||||
| 850 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | 862 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) |
| 851 | require.NoError(t, err) | 863 | require.NoError(t, err) | ||
| 852 | require.Equal(t, "work", sess.name) | 864 | require.Equal(t, "work", sess.name) | ||
| 853 | require.False(t, d.persistEnabled) | 865 | require.False(t, d.persistEnabled) | ||
| @@ -855,11 +867,11 @@ func TestNamedSessionCreationIsAllowedWithNilStore(t *testing.T) { | |||||
| 855 | 867 | ||||
| 856 | func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) { | 868 | func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) { | ||
| 857 | store, _ := newMockStore(t) | 869 | store, _ := newMockStore(t) | ||
| 858 | - | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | 870 | + | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store)) |
| 859 | incarnation := domain.IncarnationID{1} | 871 | incarnation := domain.IncarnationID{1} | ||
| 860 | - | require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh})) | 872 | + | require.NoError(t, testPersister(t, seed).Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh})) |
| 861 | 873 | ||||
| 862 | - | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | 874 | + | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store)) |
| 863 | d.mu.Lock() | 875 | d.mu.Lock() | ||
| 864 | stopped := d.stopped["work"] | 876 | stopped := d.stopped["work"] | ||
| 865 | d.mu.Unlock() | 877 | d.mu.Unlock() | ||
| @@ -876,7 +888,7 @@ func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) { | |||||
| 876 | 888 | ||||
| 877 | // TestDaemonNewWithoutPersistenceLogsNoLoadWarning covers the default, | 889 | // TestDaemonNewWithoutPersistenceLogsNoLoadWarning covers the default, | ||
| 878 | // persistence-free construction path (no WithStore/WithCatalogue option): | 890 | // persistence-free construction path (no WithStore/WithCatalogue option): | ||
| 879 | - | // d.catalogue.(*persist.Persister) is the persist.New(nil) sentinel, so LoadAll now returns | 891 | + | // testPersister(t, d) is the persist.New(nil) sentinel, so LoadAll now returns |
| 880 | // errPersistenceUnavailable instead of an empty slice. That must not surface | 892 | // errPersistenceUnavailable instead of an empty slice. That must not surface | ||
| 881 | // as a "loading persisted sessions failed" warning, since nothing failed — | 893 | // as a "loading persisted sessions failed" warning, since nothing failed — | ||
| 882 | // persistence simply was never configured. | 894 | // persistence simply was never configured. | ||
| @@ -890,7 +902,7 @@ func TestDaemonNewWithoutPersistenceLogsNoLoadWarning(t *testing.T) { | |||||
| 890 | } | 902 | } | ||
| 891 | 903 | ||||
| 892 | func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) { | 904 | func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) { | ||
| 893 | - | d := &Daemon{} | 905 | + | d := &Daemon{clock: stubClock{}} |
| 894 | sess := &session{} | 906 | sess := &session{} | ||
| 895 | const goroutines = 16 | 907 | const goroutines = 16 | ||
| 896 | const iterations = 1000 | 908 | const iterations = 1000 | ||
| @@ -937,9 +949,9 @@ func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) { | |||||
| 937 | func TestTouchMRUPersistsNamedButNotEphemeral(t *testing.T) { | 949 | func TestTouchMRUPersistsNamedButNotEphemeral(t *testing.T) { | ||
| 938 | store, state := newMockStore(t) | 950 | store, state := newMockStore(t) | ||
| 939 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 951 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 940 | - | WithStore(store)(d) | 952 | + | WithStore(t, store)(d) |
| 941 | named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1, incarnation: domain.IncarnationID{1}} | 953 | named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1, incarnation: domain.IncarnationID{1}} | ||
| 942 | - | require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) | 954 | + | require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) |
| 943 | 955 | ||||
| 944 | d.touchMRU(named) | 956 | d.touchMRU(named) | ||
| 945 | require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq) | 957 | require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq) | ||
| @@ -957,11 +969,11 @@ func TestCreateSessionSeedsMRUFromStopped(t *testing.T) { | |||||
| 957 | defer release() | 969 | defer release() | ||
| 958 | store, state := newMockStore(t) | 970 | store, state := newMockStore(t) | ||
| 959 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 971 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 960 | - | WithStore(store)(d) | 972 | + | WithStore(t, store)(d) |
| 961 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 1, lastUsedSeq: 42} | 973 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 1, lastUsedSeq: 42} | ||
| 962 | d.mruSeq.Store(42) | 974 | d.mruSeq.Store(42) | ||
| 963 | 975 | ||||
| 964 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 976 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 965 | require.NoError(t, err) | 977 | require.NoError(t, err) | ||
| 966 | require.Equal(t, uint64(42), sess.mruAt.Load()) | 978 | require.Equal(t, uint64(42), sess.mruAt.Load()) | ||
| 967 | require.Equal(t, uint64(42), state.record(t, "work").LastUsedSeq) | 979 | require.Equal(t, uint64(42), state.record(t, "work").LastUsedSeq) | ||
| @@ -975,9 +987,9 @@ func TestCreateRenameKillPersistenceLifecycle(t *testing.T) { | |||||
| 975 | defer release2() | 987 | defer release2() | ||
| 976 | store, state := newMockStore(t) | 988 | store, state := newMockStore(t) | ||
| 977 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | 989 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | ||
| 978 | - | WithStore(store)(d) | 990 | + | WithStore(t, store)(d) |
| 979 | 991 | ||||
| 980 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 992 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 981 | require.NoError(t, err) | 993 | require.NoError(t, err) | ||
| 982 | require.True(t, state.has("work")) | 994 | require.True(t, state.has("work")) | ||
| 983 | created := state.record(t, "work") | 995 | created := state.record(t, "work") | ||
| @@ -1006,13 +1018,13 @@ func TestRenameTabPersistsForNamedSession(t *testing.T) { | |||||
| 1006 | defer release() | 1018 | defer release() | ||
| 1007 | store, _ := newMockStore(t) | 1019 | store, _ := newMockStore(t) | ||
| 1008 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1020 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1009 | - | WithStore(store)(d) | 1021 | + | WithStore(t, store)(d) |
| 1010 | 1022 | ||||
| 1011 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 1023 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 1012 | require.NoError(t, err) | 1024 | require.NoError(t, err) | ||
| 1013 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | 1025 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | ||
| 1014 | 1026 | ||||
| 1015 | - | records, err := d.catalogue.(*persist.Persister).LoadAll() | 1027 | + | records, err := testPersister(t, d).LoadAll() |
| 1016 | require.NoError(t, err) | 1028 | require.NoError(t, err) | ||
| 1017 | require.Len(t, records, 1) | 1029 | require.Len(t, records, 1) | ||
| 1018 | require.Equal(t, "work", records[0].Name) | 1030 | require.Equal(t, "work", records[0].Name) | ||
| @@ -1059,10 +1071,10 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | |||||
| 1059 | } | 1071 | } | ||
| 1060 | store, _ := newMockStore(t) | 1072 | store, _ := newMockStore(t) | ||
| 1061 | d := newTestDaemon(t, newFactorySeq(t, ptys...), stubClock{}) | 1073 | d := newTestDaemon(t, newFactorySeq(t, ptys...), stubClock{}) | ||
| 1062 | - | WithStore(store)(d) | 1074 | + | WithStore(t, store)(d) |
| 1063 | sz := domain.Size{Cols: 80, Rows: 24} | 1075 | sz := domain.Size{Cols: 80, Rows: 24} | ||
| 1064 | 1076 | ||||
| 1065 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 1077 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 1066 | require.NoError(t, err) | 1078 | require.NoError(t, err) | ||
| 1067 | require.NoError(t, d.createTab(sess, sz)) | 1079 | require.NoError(t, d.createTab(sess, sz)) | ||
| 1068 | require.NoError(t, d.createTab(sess, sz)) | 1080 | require.NoError(t, d.createTab(sess, sz)) | ||
| @@ -1071,7 +1083,7 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | |||||
| 1071 | } | 1083 | } | ||
| 1072 | 1084 | ||||
| 1073 | require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false)) | 1085 | require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false)) | ||
| 1074 | - | records, err := d.catalogue.(*persist.Persister).LoadAll() | 1086 | + | records, err := testPersister(t, d).LoadAll() |
| 1075 | require.NoError(t, err) | 1087 | require.NoError(t, err) | ||
| 1076 | require.Len(t, records, 1) | 1088 | require.Len(t, records, 1) | ||
| 1077 | require.Equal(t, tt.want, records[0].TabNames) | 1089 | require.Equal(t, tt.want, records[0].TabNames) | ||
| @@ -1108,15 +1120,15 @@ func TestRenameTabDoesNotPersistForEphemeralSession(t *testing.T) { | |||||
| 1108 | defer release() | 1120 | defer release() | ||
| 1109 | store, state := newMockStore(t) | 1121 | store, state := newMockStore(t) | ||
| 1110 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1122 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1111 | - | WithStore(store)(d) | 1123 | + | WithStore(t, store)(d) |
| 1112 | 1124 | ||||
| 1113 | - | sess, err := d.createSessionLocked("0", true, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 1125 | + | sess, err := createSessionForTest(d, "0", true, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 1114 | require.NoError(t, err) | 1126 | require.NoError(t, err) | ||
| 1115 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | 1127 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | ||
| 1116 | require.Equal(t, "shell", sess.tabs[0].name) | 1128 | require.Equal(t, "shell", sess.tabs[0].name) | ||
| 1117 | 1129 | ||||
| 1118 | require.False(t, state.has("0")) | 1130 | require.False(t, state.has("0")) | ||
| 1119 | - | records, err := d.catalogue.(*persist.Persister).LoadAll() | 1131 | + | records, err := testPersister(t, d).LoadAll() |
| 1120 | require.NoError(t, err) | 1132 | require.NoError(t, err) | ||
| 1121 | require.Empty(t, records) | 1133 | require.Empty(t, records) | ||
| 1122 | } | 1134 | } | ||
| @@ -1129,8 +1141,8 @@ func TestAttachRestoresPersistedTabNames(t *testing.T) { | |||||
| 1129 | defer release2() | 1141 | defer release2() | ||
| 1130 | store, _ := newMockStore(t) | 1142 | store, _ := newMockStore(t) | ||
| 1131 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | 1143 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | ||
| 1132 | - | WithStore(store)(d) | 1144 | + | WithStore(t, store)(d) |
| 1133 | - | require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh})) | 1145 | + | require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh})) |
| 1134 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}} | 1146 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}} | ||
| 1135 | tr := portsmocks.NewMockTransport(t) | 1147 | tr := portsmocks.NewMockTransport(t) | ||
| 1136 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | 1148 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | ||
| @@ -1150,10 +1162,10 @@ func TestEphemeralRenamePromotesAndStoppedCollisionRejected(t *testing.T) { | |||||
| 1150 | defer release() | 1162 | defer release() | ||
| 1151 | store, state := newMockStore(t) | 1163 | store, state := newMockStore(t) | ||
| 1152 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1164 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1153 | - | WithStore(store)(d) | 1165 | + | WithStore(t, store)(d) |
| 1154 | d.stopped["taken"] = stoppedSession{name: "taken", cwd: "/tmp", createdAt: 1} | 1166 | d.stopped["taken"] = stoppedSession{name: "taken", cwd: "/tmp", createdAt: 1} | ||
| 1155 | 1167 | ||||
| 1156 | - | sess, err := d.createSessionLocked("0", true, "/tmp/e", sz, terminalEnv{}, d.baseEnv) | 1168 | + | sess, err := createSessionForTest(d, "0", true, "/tmp/e", sz, terminalEnv{}, d.baseEnv) |
| 1157 | require.NoError(t, err) | 1169 | require.NoError(t, err) | ||
| 1158 | require.False(t, state.has("0")) | 1170 | require.False(t, state.has("0")) | ||
| 1159 | require.EqualError(t, d.renameSession(sess, "taken"), "name already in use") | 1171 | require.EqualError(t, d.renameSession(sess, "taken"), "name already in use") | ||
| @@ -1167,9 +1179,9 @@ func TestEphemeralPromotionAssignsAndPersistsLifecycleIdentity(t *testing.T) { | |||||
| 1167 | store, state := newMockStore(t) | 1179 | store, state := newMockStore(t) | ||
| 1168 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}} | 1180 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}} | ||
| 1169 | d := newTestDaemon(t, newFactory(t, p), clock) | 1181 | d := newTestDaemon(t, newFactory(t, p), clock) | ||
| 1170 | - | WithStore(store)(d) | 1182 | + | WithStore(t, store)(d) |
| 1171 | 1183 | ||||
| 1172 | - | sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | 1184 | + | sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) |
| 1173 | require.NoError(t, err) | 1185 | require.NoError(t, err) | ||
| 1174 | require.Zero(t, sess.createdAt, "regression fixture: ephemerals have no lifecycle identity") | 1186 | require.Zero(t, sess.createdAt, "regression fixture: ephemerals have no lifecycle identity") | ||
| 1175 | 1187 | ||||
| @@ -1194,9 +1206,9 @@ func TestEphemeralPromotionLifecyclePreventsStaleSameNamePaletteTarget(t *testin | |||||
| 1194 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}} | 1206 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}} | ||
| 1195 | d.clock = clock | 1207 | d.clock = clock | ||
| 1196 | d.ptys = newFactorySeq(t, firstPTY, secondPTY) | 1208 | d.ptys = newFactorySeq(t, firstPTY, secondPTY) | ||
| 1197 | - | WithStore(store)(d) | 1209 | + | WithStore(t, store)(d) |
| 1198 | 1210 | ||||
| 1199 | - | first, err := d.createSessionLocked("0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv) | 1211 | + | first, err := createSessionForTest(d, "0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv) |
| 1200 | require.NoError(t, err) | 1212 | require.NoError(t, err) | ||
| 1201 | require.NoError(t, d.renameSession(first, "named")) | 1213 | require.NoError(t, d.renameSession(first, "named")) | ||
| 1202 | staleCreatedAt := first.createdAt | 1214 | staleCreatedAt := first.createdAt | ||
| @@ -1204,7 +1216,7 @@ func TestEphemeralPromotionLifecyclePreventsStaleSameNamePaletteTarget(t *testin | |||||
| 1204 | require.NoError(t, d.killSession(first, ports.ReasonSessionKilled, true)) | 1216 | require.NoError(t, d.killSession(first, ports.ReasonSessionKilled, true)) | ||
| 1205 | d.sessWg.Wait() | 1217 | d.sessWg.Wait() | ||
| 1206 | 1218 | ||||
| 1207 | - | second, err := d.createSessionLocked("0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv) | 1219 | + | second, err := createSessionForTest(d, "0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv) |
| 1208 | require.NoError(t, err) | 1220 | require.NoError(t, err) | ||
| 1209 | require.NoError(t, d.renameSession(second, "named")) | 1221 | require.NoError(t, d.renameSession(second, "named")) | ||
| 1210 | require.NotEqual(t, staleCreatedAt, second.createdAt) | 1222 | require.NotEqual(t, staleCreatedAt, second.createdAt) | ||
| @@ -1221,10 +1233,10 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T) | |||||
| 1221 | defer release() | 1233 | defer release() | ||
| 1222 | store, state := newMockStore(t) | 1234 | store, state := newMockStore(t) | ||
| 1223 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1235 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1224 | - | WithStore(store)(d) | 1236 | + | WithStore(t, store)(d) |
| 1225 | d.lastAllocatedCreatedAt = math.MaxInt64 | 1237 | d.lastAllocatedCreatedAt = math.MaxInt64 | ||
| 1226 | 1238 | ||||
| 1227 | - | sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | 1239 | + | sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) |
| 1228 | require.NoError(t, err) | 1240 | require.NoError(t, err) | ||
| 1229 | require.EqualError(t, d.renameSession(sess, "named"), "daemon: lifecycle identities exhausted") | 1241 | require.EqualError(t, d.renameSession(sess, "named"), "daemon: lifecycle identities exhausted") | ||
| 1230 | require.Equal(t, "0", sess.name) | 1242 | require.Equal(t, "0", sess.name) | ||
| @@ -1250,9 +1262,9 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T) | |||||
| 1250 | }).Once() | 1262 | }).Once() | ||
| 1251 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}} | 1263 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}} | ||
| 1252 | d := newTestDaemon(t, newFactory(t, p), clock) | 1264 | d := newTestDaemon(t, newFactory(t, p), clock) | ||
| 1253 | - | WithStore(store)(d) | 1265 | + | WithStore(t, store)(d) |
| 1254 | 1266 | ||||
| 1255 | - | sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | 1267 | + | sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) |
| 1256 | require.NoError(t, err) | 1268 | require.NoError(t, err) | ||
| 1257 | require.EqualError(t, d.renameSession(sess, "named"), "disk full") | 1269 | require.EqualError(t, d.renameSession(sess, "named"), "disk full") | ||
| 1258 | require.Equal(t, "0", sess.name) | 1270 | require.Equal(t, "0", sess.name) | ||
| @@ -1277,11 +1289,11 @@ func TestRefreshSessionCwdTouchesOnlyOnChange(t *testing.T) { | |||||
| 1277 | defer release() | 1289 | defer release() | ||
| 1278 | store, state := newMockStore(t) | 1290 | store, state := newMockStore(t) | ||
| 1279 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1291 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1280 | - | WithStore(store)(d) | 1292 | + | WithStore(t, store)(d) |
| 1281 | cwd := "/tmp/work" | 1293 | cwd := "/tmp/work" | ||
| 1282 | WithCwdReader(func(int) (string, error) { return cwd, nil })(d) | 1294 | WithCwdReader(func(int) (string, error) { return cwd, nil })(d) | ||
| 1283 | 1295 | ||||
| 1284 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 1296 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 1285 | require.NoError(t, err) | 1297 | require.NoError(t, err) | ||
| 1286 | state.mu.Lock() | 1298 | state.mu.Lock() | ||
| 1287 | setsAfterCreate := state.sets | 1299 | setsAfterCreate := state.sets | ||
| @@ -1305,7 +1317,7 @@ func TestAttachResumesStoppedSessionFromStoredCwd(t *testing.T) { | |||||
| 1305 | defer release() | 1317 | defer release() | ||
| 1306 | store, _ := newMockStore(t) | 1318 | store, _ := newMockStore(t) | ||
| 1307 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1319 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1308 | - | WithStore(store)(d) | 1320 | + | WithStore(t, store)(d) |
| 1309 | cwd := t.TempDir() | 1321 | cwd := t.TempDir() | ||
| 1310 | d.stopped["work"] = stoppedSession{name: "work", cwd: cwd, createdAt: 1} | 1322 | d.stopped["work"] = stoppedSession{name: "work", cwd: cwd, createdAt: 1} | ||
| 1311 | tr := portsmocks.NewMockTransport(t) | 1323 | tr := portsmocks.NewMockTransport(t) | ||
| @@ -1329,7 +1341,7 @@ func TestAttachStoppedMissingCwdFallsBackToHome(t *testing.T) { | |||||
| 1329 | defer release() | 1341 | defer release() | ||
| 1330 | store, _ := newMockStore(t) | 1342 | store, _ := newMockStore(t) | ||
| 1331 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1343 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1332 | - | WithStore(store)(d) | 1344 | + | WithStore(t, store)(d) |
| 1333 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/definitely/missing/vev", createdAt: 1} | 1345 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/definitely/missing/vev", createdAt: 1} | ||
| 1334 | tr := portsmocks.NewMockTransport(t) | 1346 | tr := portsmocks.NewMockTransport(t) | ||
| 1335 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | 1347 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | ||
| @@ -1343,8 +1355,8 @@ func TestAttachStoppedMissingCwdFallsBackToHome(t *testing.T) { | |||||
| 1343 | func TestPickerStoppedTargetKillPurges(t *testing.T) { | 1355 | func TestPickerStoppedTargetKillPurges(t *testing.T) { | ||
| 1344 | store, state := newMockStore(t) | 1356 | store, state := newMockStore(t) | ||
| 1345 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 1357 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 1346 | - | WithStore(store)(d) | 1358 | + | WithStore(t, store)(d) |
| 1347 | - | require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) | 1359 | + | require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) |
| 1348 | d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1} | 1360 | d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1} | ||
| 1349 | require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true})) | 1361 | require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true})) | ||
| 1350 | require.False(t, state.has("old")) | 1362 | require.False(t, state.has("old")) | ||
| @@ -1378,7 +1390,7 @@ func TestNewSessionAssignsStableIDsAndChildEnv(t *testing.T) { | |||||
| 1378 | d := newTestDaemon(t, f, stubClock{}) | 1390 | d := newTestDaemon(t, f, stubClock{}) | ||
| 1379 | d.baseEnv = []string{"KEEP=1", "TERM=old", "COLORTERM=old", "TERM_PROGRAM=old", "VEV=old"} | 1391 | d.baseEnv = []string{"KEEP=1", "TERM=old", "COLORTERM=old", "TERM_PROGRAM=old", "VEV=old"} | ||
| 1380 | 1392 | ||||
| 1381 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{TrueColor: true}, d.baseEnv) | 1393 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{TrueColor: true}, d.baseEnv) |
| 1382 | require.NoError(t, err) | 1394 | require.NoError(t, err) | ||
| 1383 | defer func() { | 1395 | defer func() { | ||
| 1384 | _ = d.killSession(sess, ports.ReasonServerShutdown, false) | 1396 | _ = d.killSession(sess, ports.ReasonServerShutdown, false) | ||
| @@ -1434,7 +1446,7 @@ func TestRenameSessionUnsafeNameRejected(t *testing.T) { | |||||
| 1434 | p, release := newBlockingPTY(t) | 1446 | p, release := newBlockingPTY(t) | ||
| 1435 | defer release() | 1447 | defer release() | ||
| 1436 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1448 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1437 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 1449 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 1438 | require.NoError(t, err) | 1450 | require.NoError(t, err) | ||
| 1439 | require.ErrorIs(t, d.renameSession(sess, "my work"), domain.ErrInvalidSessionName) | 1451 | require.ErrorIs(t, d.renameSession(sess, "my work"), domain.ErrInvalidSessionName) | ||
| 1440 | } | 1452 | } | ||
| @@ -1444,7 +1456,7 @@ func TestCreateTabPtyFailureIsUserError(t *testing.T) { | |||||
| 1444 | p, release := newBlockingPTY(t) | 1456 | p, release := newBlockingPTY(t) | ||
| 1445 | defer release() | 1457 | defer release() | ||
| 1446 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1458 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1447 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 1459 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 1448 | require.NoError(t, err) | 1460 | require.NoError(t, err) | ||
| 1449 | 1461 | ||||
| 1450 | cause := errors.New("fork/exec: no such file") | 1462 | cause := errors.New("fork/exec: no such file") | ||
| @@ -1471,12 +1483,12 @@ func TestNaturalExitStoppedButExplicitKillPurges(t *testing.T) { | |||||
| 1471 | defer release2() | 1483 | defer release2() | ||
| 1472 | store, state := newMockStore(t) | 1484 | store, state := newMockStore(t) | ||
| 1473 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | 1485 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | ||
| 1474 | - | WithStore(store)(d) | 1486 | + | WithStore(t, store)(d) |
| 1475 | WithCwdReader(func(int) (string, error) { return "/tmp/latest", nil })(d) | 1487 | WithCwdReader(func(int) (string, error) { return "/tmp/latest", nil })(d) | ||
| 1476 | 1488 | ||||
| 1477 | - | natural, err := d.createSessionLocked("natural", false, "/tmp/old", sz, terminalEnv{}, d.baseEnv) | 1489 | + | natural, err := createSessionForTest(d, "natural", false, "/tmp/old", sz, terminalEnv{}, d.baseEnv) |
| 1478 | require.NoError(t, err) | 1490 | require.NoError(t, err) | ||
| 1479 | - | other, err := d.createSessionLocked("other", false, "/tmp/other", sz, terminalEnv{}, d.baseEnv) | 1491 | + | other, err := createSessionForTest(d, "other", false, "/tmp/other", sz, terminalEnv{}, d.baseEnv) |
| 1480 | require.NoError(t, err) | 1492 | require.NoError(t, err) | ||
| 1481 | _ = d.killSession(natural, ports.ReasonSessionKilled, false) | 1493 | _ = d.killSession(natural, ports.ReasonSessionKilled, false) | ||
| 1482 | require.True(t, state.has("natural")) | 1494 | require.True(t, state.has("natural")) | ||
| @@ -1792,7 +1804,7 @@ func TestNamedSessionLifecycleTimestampsAreMonotonicAcrossClockRegression(t *tes | |||||
| 1792 | 1804 | ||||
| 1793 | var got []int64 | 1805 | var got []int64 | ||
| 1794 | for _, name := range []string{"one", "two", "three"} { | 1806 | for _, name := range []string{"one", "two", "three"} { | ||
| 1795 | - | sess, err := d.createSessionLocked(name, false, "/tmp", sz, terminalEnv{}, d.baseEnv) | 1807 | + | sess, err := createSessionForTest(d, name, false, "/tmp", sz, terminalEnv{}, d.baseEnv) |
| 1796 | require.NoError(t, err) | 1808 | require.NoError(t, err) | ||
| 1797 | got = append(got, sess.createdAt) | 1809 | got = append(got, sess.createdAt) | ||
| 1798 | } | 1810 | } | ||
| @@ -1805,7 +1817,7 @@ func TestNamedSessionLifecycleExhaustionDoesNotMutateSessionState(t *testing.T) | |||||
| 1805 | d.nextID = 17 | 1817 | d.nextID = 17 | ||
| 1806 | d.stopped["retained"] = stoppedSession{name: "retained", cwd: "/tmp", createdAt: 9} | 1818 | d.stopped["retained"] = stoppedSession{name: "retained", cwd: "/tmp", createdAt: 9} | ||
| 1807 | 1819 | ||||
| 1808 | - | sess, err := d.createSessionLocked("new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | 1820 | + | sess, err := createSessionForTest(d, "new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) |
| 1809 | 1821 | ||||
| 1810 | require.ErrorContains(t, err, "lifecycle identities exhausted") | 1822 | require.ErrorContains(t, err, "lifecycle identities exhausted") | ||
| 1811 | require.Nil(t, sess) | 1823 | require.Nil(t, sess) | ||
| @@ -1855,18 +1867,18 @@ func TestCatalogueRecordsConstructExpectedSessionRegistry(t *testing.T) { | |||||
| 1855 | 1867 | ||||
| 1856 | func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) { | 1868 | func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) { | ||
| 1857 | store, _ := newMockStore(t) | 1869 | store, _ := newMockStore(t) | ||
| 1858 | - | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | 1870 | + | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store)) |
| 1859 | - | require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh})) | 1871 | + | require.NoError(t, testPersister(t, seed).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh})) |
| 1860 | 1872 | ||||
| 1861 | p, release := newBlockingPTY(t) | 1873 | p, release := newBlockingPTY(t) | ||
| 1862 | defer release() | 1874 | defer release() | ||
| 1863 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}} | 1875 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}} | ||
| 1864 | // Constructing with persistence must establish the lifecycle high-water mark. | 1876 | // Constructing with persistence must establish the lifecycle high-water mark. | ||
| 1865 | - | d := New(newFactory(t, p), clock, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | 1877 | + | d := New(newFactory(t, p), clock, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store)) |
| 1866 | d.serveCtx, d.serveCancel = context.WithCancel(context.Background()) | 1878 | d.serveCtx, d.serveCancel = context.WithCancel(context.Background()) | ||
| 1867 | t.Cleanup(d.serveCancel) | 1879 | t.Cleanup(d.serveCancel) | ||
| 1868 | 1880 | ||||
| 1869 | - | sess, err := d.createSessionLocked("new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | 1881 | + | sess, err := createSessionForTest(d, "new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) |
| 1870 | require.NoError(t, err) | 1882 | require.NoError(t, err) | ||
| 1871 | require.Equal(t, int64(901), sess.createdAt) | 1883 | require.Equal(t, int64(901), sess.createdAt) | ||
| 1872 | } | 1884 | } | ||
| @@ -1878,7 +1890,7 @@ func TestResumingStoppedSessionPreservesLifecycleIdentityInPersistence(t *testin | |||||
| 1878 | defer releaseTarget() | 1890 | defer releaseTarget() | ||
| 1879 | d, from, ac, _ := newManualSessionWithPTYs(t, fromPTY) | 1891 | d, from, ac, _ := newManualSessionWithPTYs(t, fromPTY) | ||
| 1880 | store, state := newMockStore(t) | 1892 | store, state := newMockStore(t) | ||
| 1881 | - | WithStore(store)(d) | 1893 | + | WithStore(t, store)(d) |
| 1882 | d.ptys = newFactory(t, targetPTY) | 1894 | d.ptys = newFactory(t, targetPTY) | ||
| 1883 | d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp", createdAt: 77} | 1895 | d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp", createdAt: 77} | ||
| 1884 | 1896 | ||||
M internal/usecase/daemon/snapshot_incremental.go +1 -2
| @@ -1,7 +1,6 @@ | |||
| 1 | 1 | package daemon | |
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | - | "crypto/sha256" | |
| 5 | 4 | "fmt" | |
| 6 | 5 | ||
| 7 | 6 | "github.com/bnema/vev/internal/domain" | |
| @@ -162,7 +161,7 @@ func (d *Daemon) incrementalPublication(capture *snapshotCapture) (ports.Snapsho | |||
| 162 | 161 | if err != nil { | |
| 163 | 162 | return ports.SnapshotPublication{}, err | |
| 164 | 163 | } | |
| 165 | - | capture.checkpoint = domain.CheckpointRef{Generation: capture.generation, ManifestDigest: sha256.Sum256(encoded)} | |
| 164 | + | capture.checkpoint = domain.CheckpointRef{Generation: capture.generation, ManifestDigest: snapcodec.ManifestDigest(encoded)} | |
| 166 | 165 | return ports.SnapshotPublication{IncarnationID: capture.incarnation, Name: capture.name, Generation: capture.generation, ParentCheckpoint: capture.parentCheckpoint, Manifest: encoded, Objects: objects}, nil | |
| 167 | 166 | } | |
| 168 | 167 | ||
| @@ -1,7 +1,6 @@ | |||||
| 1 | package daemon | 1 | package daemon | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | - | "crypto/sha256" | |||
| 5 | "fmt" | 4 | "fmt" | ||
| 6 | 5 | ||||
| 7 | "github.com/bnema/vev/internal/domain" | 6 | "github.com/bnema/vev/internal/domain" | ||
| @@ -162,7 +161,7 @@ func (d *Daemon) incrementalPublication(capture *snapshotCapture) (ports.Snapsho | |||||
| 162 | if err != nil { | 161 | if err != nil { | ||
| 163 | return ports.SnapshotPublication{}, err | 162 | return ports.SnapshotPublication{}, err | ||
| 164 | } | 163 | } | ||
| 165 | - | capture.checkpoint = domain.CheckpointRef{Generation: capture.generation, ManifestDigest: sha256.Sum256(encoded)} | 164 | + | capture.checkpoint = domain.CheckpointRef{Generation: capture.generation, ManifestDigest: snapcodec.ManifestDigest(encoded)} |
| 166 | return ports.SnapshotPublication{IncarnationID: capture.incarnation, Name: capture.name, Generation: capture.generation, ParentCheckpoint: capture.parentCheckpoint, Manifest: encoded, Objects: objects}, nil | 165 | return ports.SnapshotPublication{IncarnationID: capture.incarnation, Name: capture.name, Generation: capture.generation, ParentCheckpoint: capture.parentCheckpoint, Manifest: encoded, Objects: objects}, nil | ||
| 167 | } | 166 | } | ||
| 168 | 167 | ||||
M internal/usecase/daemon/snapshot_incremental_test.go +1 -2
| @@ -1,7 +1,6 @@ | |||
| 1 | 1 | package daemon | |
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | - | "crypto/sha256" | |
| 5 | 4 | "testing" | |
| 6 | 5 | "time" | |
| 7 | 6 | ||
| @@ -131,7 +130,7 @@ func TestIncrementalPublicationReusesSealedChunkObject(t *testing.T) { | |||
| 131 | 130 | if err != nil { | |
| 132 | 131 | t.Fatal(err) | |
| 133 | 132 | } | |
| 134 | - | firstRef := domain.CheckpointRef{Generation: first.Generation, ManifestDigest: sha256.Sum256(first.Manifest)} | |
| 133 | + | firstRef := domain.CheckpointRef{Generation: first.Generation, ManifestDigest: snapcodec.ManifestDigest(first.Manifest)} | |
| 135 | 134 | capture.generation = 2 | |
| 136 | 135 | capture.parentCheckpoint = &firstRef | |
| 137 | 136 | second, err := d.incrementalPublication(capture) | |
| @@ -1,7 +1,6 @@ | |||||
| 1 | package daemon | 1 | package daemon | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | - | "crypto/sha256" | |||
| 5 | "testing" | 4 | "testing" | ||
| 6 | "time" | 5 | "time" | ||
| 7 | 6 | ||||
| @@ -131,7 +130,7 @@ func TestIncrementalPublicationReusesSealedChunkObject(t *testing.T) { | |||||
| 131 | if err != nil { | 130 | if err != nil { | ||
| 132 | t.Fatal(err) | 131 | t.Fatal(err) | ||
| 133 | } | 132 | } | ||
| 134 | - | firstRef := domain.CheckpointRef{Generation: first.Generation, ManifestDigest: sha256.Sum256(first.Manifest)} | 133 | + | firstRef := domain.CheckpointRef{Generation: first.Generation, ManifestDigest: snapcodec.ManifestDigest(first.Manifest)} |
| 135 | capture.generation = 2 | 134 | capture.generation = 2 | ||
| 136 | capture.parentCheckpoint = &firstRef | 135 | capture.parentCheckpoint = &firstRef | ||
| 137 | second, err := d.incrementalPublication(capture) | 136 | second, err := d.incrementalPublication(capture) | ||
M internal/usecase/daemon/snapshot_restore.go +1 -1
| @@ -59,8 +59,8 @@ func (d *Daemon) restoreSnapshots(ctx context.Context) { | |||
| 59 | 59 | if d == nil { | |
| 60 | 60 | return | |
| 61 | 61 | } | |
| 62 | - | defer d.closeRestoreDone() | |
| 63 | 62 | defer d.logStartupRecoveryCounts(0) | |
| 63 | + | defer d.closeRestoreDone() | |
| 64 | 64 | if ns := d.noticeStore; ns != nil { | |
| 65 | 65 | claimed, err := ns.Claim() | |
| 66 | 66 | if err != nil { | |
| @@ -59,8 +59,8 @@ func (d *Daemon) restoreSnapshots(ctx context.Context) { | |||||
| 59 | if d == nil { | 59 | if d == nil { | ||
| 60 | return | 60 | return | ||
| 61 | } | 61 | } | ||
| 62 | - | defer d.closeRestoreDone() | |||
| 63 | defer d.logStartupRecoveryCounts(0) | 62 | defer d.logStartupRecoveryCounts(0) | ||
| 63 | + | defer d.closeRestoreDone() | |||
| 64 | if ns := d.noticeStore; ns != nil { | 64 | if ns := d.noticeStore; ns != nil { | ||
| 65 | claimed, err := ns.Claim() | 65 | claimed, err := ns.Claim() | ||
| 66 | if err != nil { | 66 | if err != nil { | ||
M internal/usecase/daemon/snapshot_restore_fixture_test.go +4 -4
| @@ -20,18 +20,18 @@ func acceptancePublication(snapshot snapcodec.Session) (ports.SnapshotPublicatio | |||
| 20 | 20 | for i, payload := range pane.SealedChunks { | |
| 21 | 21 | object, err := snapcodec.MarshalObject(snapcodec.HistoryChunk, payload) | |
| 22 | 22 | if err != nil { | |
| 23 | - | return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q sealed chunk %d: marshal object: %w", tab.StableID, pane.ID, i, err) | |
| 23 | + | return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q sealed chunk %d: marshal object: %w", tab.StableID, pane.ID, i, err) | |
| 24 | 24 | } | |
| 25 | 25 | outPane.Sealed = append(outPane.Sealed, objectRef(snapcodec.HistoryChunk, object)) | |
| 26 | 26 | objects = append(objects, object) | |
| 27 | 27 | } | |
| 28 | 28 | tail, err := snapcodec.MarshalObject(snapcodec.HistoryTail, pane.Tail) | |
| 29 | 29 | if err != nil { | |
| 30 | - | return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q tail: marshal object: %w", tab.StableID, pane.ID, err) | |
| 30 | + | return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q tail: marshal object: %w", tab.StableID, pane.ID, err) | |
| 31 | 31 | } | |
| 32 | 32 | visible, err := snapcodec.MarshalObject(snapcodec.Visible, pane.Visible) | |
| 33 | 33 | if err != nil { | |
| 34 | - | return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q visible: marshal object: %w", tab.StableID, pane.ID, err) | |
| 34 | + | return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q visible: marshal object: %w", tab.StableID, pane.ID, err) | |
| 35 | 35 | } | |
| 36 | 36 | outPane.Tail, outPane.Visible = objectRef(snapcodec.HistoryTail, tail), objectRef(snapcodec.Visible, visible) | |
| 37 | 37 | objects = append(objects, tail, visible) | |
| @@ -41,7 +41,7 @@ func acceptancePublication(snapshot snapcodec.Session) (ports.SnapshotPublicatio | |||
| 41 | 41 | } | |
| 42 | 42 | encoded, err := snapcodec.MarshalManifest(manifest) | |
| 43 | 43 | if err != nil { | |
| 44 | - | return ports.SnapshotPublication{}, fmt.Errorf("snapshot: marshal legacy import manifest: %w", err) | |
| 44 | + | return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: marshal manifest: %w", err) | |
| 45 | 45 | } | |
| 46 | 46 | return ports.SnapshotPublication{IncarnationID: incarnation, Name: snapshot.Name, Generation: 1, Manifest: encoded, Objects: objects}, nil | |
| 47 | 47 | } | |
| @@ -20,18 +20,18 @@ func acceptancePublication(snapshot snapcodec.Session) (ports.SnapshotPublicatio | |||||
| 20 | for i, payload := range pane.SealedChunks { | 20 | for i, payload := range pane.SealedChunks { | ||
| 21 | object, err := snapcodec.MarshalObject(snapcodec.HistoryChunk, payload) | 21 | object, err := snapcodec.MarshalObject(snapcodec.HistoryChunk, payload) | ||
| 22 | if err != nil { | 22 | if err != nil { | ||
| 23 | - | return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q sealed chunk %d: marshal object: %w", tab.StableID, pane.ID, i, err) | 23 | + | return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q sealed chunk %d: marshal object: %w", tab.StableID, pane.ID, i, err) |
| 24 | } | 24 | } | ||
| 25 | outPane.Sealed = append(outPane.Sealed, objectRef(snapcodec.HistoryChunk, object)) | 25 | outPane.Sealed = append(outPane.Sealed, objectRef(snapcodec.HistoryChunk, object)) | ||
| 26 | objects = append(objects, object) | 26 | objects = append(objects, object) | ||
| 27 | } | 27 | } | ||
| 28 | tail, err := snapcodec.MarshalObject(snapcodec.HistoryTail, pane.Tail) | 28 | tail, err := snapcodec.MarshalObject(snapcodec.HistoryTail, pane.Tail) | ||
| 29 | if err != nil { | 29 | if err != nil { | ||
| 30 | - | return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q tail: marshal object: %w", tab.StableID, pane.ID, err) | 30 | + | return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q tail: marshal object: %w", tab.StableID, pane.ID, err) |
| 31 | } | 31 | } | ||
| 32 | visible, err := snapcodec.MarshalObject(snapcodec.Visible, pane.Visible) | 32 | visible, err := snapcodec.MarshalObject(snapcodec.Visible, pane.Visible) | ||
| 33 | if err != nil { | 33 | if err != nil { | ||
| 34 | - | return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q visible: marshal object: %w", tab.StableID, pane.ID, err) | 34 | + | return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q visible: marshal object: %w", tab.StableID, pane.ID, err) |
| 35 | } | 35 | } | ||
| 36 | outPane.Tail, outPane.Visible = objectRef(snapcodec.HistoryTail, tail), objectRef(snapcodec.Visible, visible) | 36 | outPane.Tail, outPane.Visible = objectRef(snapcodec.HistoryTail, tail), objectRef(snapcodec.Visible, visible) | ||
| 37 | objects = append(objects, tail, visible) | 37 | objects = append(objects, tail, visible) | ||
| @@ -41,7 +41,7 @@ func acceptancePublication(snapshot snapcodec.Session) (ports.SnapshotPublicatio | |||||
| 41 | } | 41 | } | ||
| 42 | encoded, err := snapcodec.MarshalManifest(manifest) | 42 | encoded, err := snapcodec.MarshalManifest(manifest) | ||
| 43 | if err != nil { | 43 | if err != nil { | ||
| 44 | - | return ports.SnapshotPublication{}, fmt.Errorf("snapshot: marshal legacy import manifest: %w", err) | 44 | + | return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: marshal manifest: %w", err) |
| 45 | } | 45 | } | ||
| 46 | return ports.SnapshotPublication{IncarnationID: incarnation, Name: snapshot.Name, Generation: 1, Manifest: encoded, Objects: objects}, nil | 46 | return ports.SnapshotPublication{IncarnationID: incarnation, Name: snapshot.Name, Generation: 1, Manifest: encoded, Objects: objects}, nil | ||
| 47 | } | 47 | } | ||
M internal/usecase/daemon/snapshot_restore_incremental.go +50 -4
| @@ -5,7 +5,9 @@ import ( | |||
| 5 | 5 | "crypto/sha256" | |
| 6 | 6 | "errors" | |
| 7 | 7 | "fmt" | |
| 8 | + | "net" | |
| 8 | 9 | "sync" | |
| 10 | + | "syscall" | |
| 9 | 11 | ||
| 10 | 12 | "github.com/bnema/vev/internal/domain" | |
| 11 | 13 | "github.com/bnema/vev/internal/ports" | |
| @@ -14,6 +16,8 @@ import ( | |||
| 14 | 16 | ||
| 15 | 17 | const catalogueRestoreConcurrency = 8 | |
| 16 | 18 | ||
| 19 | + | var errRetryableRestoreLoad = errors.New("snapshot: retryable checkpoint load") | |
| 20 | + | ||
| 17 | 21 | func initialRuntimeRecoveryState(record domain.CatalogueRecord) (runtimeRecoveryState, chan struct{}) { | |
| 18 | 22 | done := make(chan struct{}) | |
| 19 | 23 | var state runtimeRecoveryState | |
| @@ -131,7 +135,16 @@ func (d *Daemon) setStoppedRecovery(record domain.CatalogueRecord, state runtime | |||
| 131 | 135 | } | |
| 132 | 136 | ||
| 133 | 137 | func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) { | |
| 134 | - | if restoreErr != nil { | |
| 138 | + | d.finishRecordRestoreAttempt(record, restoreErr, done, false) | |
| 139 | + | } | |
| 140 | + | ||
| 141 | + | func (d *Daemon) finishExplicitRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) { | |
| 142 | + | d.finishRecordRestoreAttempt(record, restoreErr, done, true) | |
| 143 | + | } | |
| 144 | + | ||
| 145 | + | func (d *Daemon) finishRecordRestoreAttempt(record domain.CatalogueRecord, restoreErr error, done chan struct{}, resetRetryable bool) { | |
| 146 | + | retryable := errors.Is(restoreErr, errRetryableRestoreLoad) | |
| 147 | + | if restoreErr != nil && !retryable { | |
| 135 | 148 | catalogueReadable := true | |
| 136 | 149 | if d.catalogue != nil { | |
| 137 | 150 | current, ok, readErr := d.catalogue.Record(record.Name) | |
| @@ -157,7 +170,8 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e | |||
| 157 | 170 | d.mu.Lock() | |
| 158 | 171 | entry, ok := d.stopped[record.Name] | |
| 159 | 172 | if ok { | |
| 160 | - | if restoreErr != nil { | |
| 173 | + | switch { | |
| 174 | + | case restoreErr != nil && !retryable: | |
| 161 | 175 | entry.record = record | |
| 162 | 176 | entry.state = runtimeDegraded | |
| 163 | 177 | entry.cwd = record.Cwd | |
| @@ -166,11 +180,14 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e | |||
| 166 | 180 | entry.lastUsedSeq = record.LastUsedSeq | |
| 167 | 181 | entry.tabNames = append([]string(nil), record.TabNames...) | |
| 168 | 182 | d.stopped[record.Name] = entry | |
| 183 | + | case retryable && resetRetryable && entry.state == runtimeRestoring && entry.restoreDone == done: | |
| 184 | + | entry.state = runtimeDegraded | |
| 185 | + | d.stopped[record.Name] = entry | |
| 169 | 186 | } | |
| 170 | - | closeRuntimeRestoreDoneLocked(done) | |
| 171 | 187 | } | |
| 188 | + | closeRuntimeRestoreDoneLocked(done) | |
| 172 | 189 | d.mu.Unlock() | |
| 173 | - | if restoreErr != nil { | |
| 190 | + | if restoreErr != nil && !retryable { | |
| 174 | 191 | reasonCode := "checkpoint-invalid" | |
| 175 | 192 | if errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded) { | |
| 176 | 193 | reasonCode = "restore-interrupted" | |
| @@ -210,6 +227,7 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||
| 210 | 227 | var selectedSnapshot snapcodec.Session | |
| 211 | 228 | selectedIndex := -1 | |
| 212 | 229 | selectedFallback := false | |
| 230 | + | var retryableLoadErr error | |
| 213 | 231 | for i, candidate := range candidates { | |
| 214 | 232 | fallback := record.Committed == nil || i > 0 | |
| 215 | 233 | generation, err := d.snapshotRepository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, candidate) | |
| @@ -217,6 +235,9 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||
| 217 | 235 | if ctxErr := ctx.Err(); ctxErr != nil { | |
| 218 | 236 | return ctxErr | |
| 219 | 237 | } | |
| 238 | + | if isRetryableRestoreLoadError(err) { | |
| 239 | + | retryableLoadErr = err | |
| 240 | + | } | |
| 220 | 241 | d.logRejectedRestoreCandidate(record, candidate, fallback, "load-failed", err) | |
| 221 | 242 | continue | |
| 222 | 243 | } | |
| @@ -241,6 +262,9 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||
| 241 | 262 | break | |
| 242 | 263 | } | |
| 243 | 264 | if selectedIndex < 0 { | |
| 265 | + | if retryableLoadErr != nil { | |
| 266 | + | return fmt.Errorf("%w: %v", errRetryableRestoreLoad, retryableLoadErr) | |
| 267 | + | } | |
| 244 | 268 | return errors.New("snapshot: no catalogue checkpoint validated") | |
| 245 | 269 | } | |
| 246 | 270 | ||
| @@ -276,6 +300,28 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||
| 276 | 300 | return nil | |
| 277 | 301 | } | |
| 278 | 302 | ||
| 303 | + | func isRetryableRestoreLoadError(err error) bool { | |
| 304 | + | if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { | |
| 305 | + | return false | |
| 306 | + | } | |
| 307 | + | if errors.Is(err, ports.ErrBudgetExhausted) || | |
| 308 | + | errors.Is(err, syscall.ENOSPC) || | |
| 309 | + | errors.Is(err, syscall.ENOMEM) || | |
| 310 | + | errors.Is(err, syscall.EMFILE) || | |
| 311 | + | errors.Is(err, syscall.ENFILE) || | |
| 312 | + | errors.Is(err, syscall.EDQUOT) { | |
| 313 | + | return true | |
| 314 | + | } | |
| 315 | + | if netErr, ok := errors.AsType[net.Error](err); ok && netErr.Timeout() { | |
| 316 | + | return true | |
| 317 | + | } | |
| 318 | + | temporary, ok := errors.AsType[interface { | |
| 319 | + | error | |
| 320 | + | Temporary() bool | |
| 321 | + | }](err) | |
| 322 | + | return ok && temporary.Temporary() | |
| 323 | + | } | |
| 324 | + | ||
| 279 | 325 | func (d *Daemon) logRejectedRestoreCandidate(record domain.CatalogueRecord, candidate domain.CheckpointRef, fallback bool, reason string, err error) { | |
| 280 | 326 | kind := "committed" | |
| 281 | 327 | if fallback { | |
| @@ -5,7 +5,9 @@ import ( | |||||
| 5 | "crypto/sha256" | 5 | "crypto/sha256" | ||
| 6 | "errors" | 6 | "errors" | ||
| 7 | "fmt" | 7 | "fmt" | ||
| 8 | + | "net" | |||
| 8 | "sync" | 9 | "sync" | ||
| 10 | + | "syscall" | |||
| 9 | 11 | ||||
| 10 | "github.com/bnema/vev/internal/domain" | 12 | "github.com/bnema/vev/internal/domain" | ||
| 11 | "github.com/bnema/vev/internal/ports" | 13 | "github.com/bnema/vev/internal/ports" | ||
| @@ -14,6 +16,8 @@ import ( | |||||
| 14 | 16 | ||||
| 15 | const catalogueRestoreConcurrency = 8 | 17 | const catalogueRestoreConcurrency = 8 | ||
| 16 | 18 | ||||
| 19 | + | var errRetryableRestoreLoad = errors.New("snapshot: retryable checkpoint load") | |||
| 20 | + | ||||
| 17 | func initialRuntimeRecoveryState(record domain.CatalogueRecord) (runtimeRecoveryState, chan struct{}) { | 21 | func initialRuntimeRecoveryState(record domain.CatalogueRecord) (runtimeRecoveryState, chan struct{}) { | ||
| 18 | done := make(chan struct{}) | 22 | done := make(chan struct{}) | ||
| 19 | var state runtimeRecoveryState | 23 | var state runtimeRecoveryState | ||
| @@ -131,7 +135,16 @@ func (d *Daemon) setStoppedRecovery(record domain.CatalogueRecord, state runtime | |||||
| 131 | } | 135 | } | ||
| 132 | 136 | ||||
| 133 | func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) { | 137 | func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) { | ||
| 134 | - | if restoreErr != nil { | 138 | + | d.finishRecordRestoreAttempt(record, restoreErr, done, false) |
| 139 | + | } | |||
| 140 | + | ||||
| 141 | + | func (d *Daemon) finishExplicitRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) { | |||
| 142 | + | d.finishRecordRestoreAttempt(record, restoreErr, done, true) | |||
| 143 | + | } | |||
| 144 | + | ||||
| 145 | + | func (d *Daemon) finishRecordRestoreAttempt(record domain.CatalogueRecord, restoreErr error, done chan struct{}, resetRetryable bool) { | |||
| 146 | + | retryable := errors.Is(restoreErr, errRetryableRestoreLoad) | |||
| 147 | + | if restoreErr != nil && !retryable { | |||
| 135 | catalogueReadable := true | 148 | catalogueReadable := true | ||
| 136 | if d.catalogue != nil { | 149 | if d.catalogue != nil { | ||
| 137 | current, ok, readErr := d.catalogue.Record(record.Name) | 150 | current, ok, readErr := d.catalogue.Record(record.Name) | ||
| @@ -157,7 +170,8 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e | |||||
| 157 | d.mu.Lock() | 170 | d.mu.Lock() | ||
| 158 | entry, ok := d.stopped[record.Name] | 171 | entry, ok := d.stopped[record.Name] | ||
| 159 | if ok { | 172 | if ok { | ||
| 160 | - | if restoreErr != nil { | 173 | + | switch { |
| 174 | + | case restoreErr != nil && !retryable: | |||
| 161 | entry.record = record | 175 | entry.record = record | ||
| 162 | entry.state = runtimeDegraded | 176 | entry.state = runtimeDegraded | ||
| 163 | entry.cwd = record.Cwd | 177 | entry.cwd = record.Cwd | ||
| @@ -166,11 +180,14 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e | |||||
| 166 | entry.lastUsedSeq = record.LastUsedSeq | 180 | entry.lastUsedSeq = record.LastUsedSeq | ||
| 167 | entry.tabNames = append([]string(nil), record.TabNames...) | 181 | entry.tabNames = append([]string(nil), record.TabNames...) | ||
| 168 | d.stopped[record.Name] = entry | 182 | d.stopped[record.Name] = entry | ||
| 183 | + | case retryable && resetRetryable && entry.state == runtimeRestoring && entry.restoreDone == done: | |||
| 184 | + | entry.state = runtimeDegraded | |||
| 185 | + | d.stopped[record.Name] = entry | |||
| 169 | } | 186 | } | ||
| 170 | - | closeRuntimeRestoreDoneLocked(done) | |||
| 171 | } | 187 | } | ||
| 188 | + | closeRuntimeRestoreDoneLocked(done) | |||
| 172 | d.mu.Unlock() | 189 | d.mu.Unlock() | ||
| 173 | - | if restoreErr != nil { | 190 | + | if restoreErr != nil && !retryable { |
| 174 | reasonCode := "checkpoint-invalid" | 191 | reasonCode := "checkpoint-invalid" | ||
| 175 | if errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded) { | 192 | if errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded) { | ||
| 176 | reasonCode = "restore-interrupted" | 193 | reasonCode = "restore-interrupted" | ||
| @@ -210,6 +227,7 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||||
| 210 | var selectedSnapshot snapcodec.Session | 227 | var selectedSnapshot snapcodec.Session | ||
| 211 | selectedIndex := -1 | 228 | selectedIndex := -1 | ||
| 212 | selectedFallback := false | 229 | selectedFallback := false | ||
| 230 | + | var retryableLoadErr error | |||
| 213 | for i, candidate := range candidates { | 231 | for i, candidate := range candidates { | ||
| 214 | fallback := record.Committed == nil || i > 0 | 232 | fallback := record.Committed == nil || i > 0 | ||
| 215 | generation, err := d.snapshotRepository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, candidate) | 233 | generation, err := d.snapshotRepository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, candidate) | ||
| @@ -217,6 +235,9 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||||
| 217 | if ctxErr := ctx.Err(); ctxErr != nil { | 235 | if ctxErr := ctx.Err(); ctxErr != nil { | ||
| 218 | return ctxErr | 236 | return ctxErr | ||
| 219 | } | 237 | } | ||
| 238 | + | if isRetryableRestoreLoadError(err) { | |||
| 239 | + | retryableLoadErr = err | |||
| 240 | + | } | |||
| 220 | d.logRejectedRestoreCandidate(record, candidate, fallback, "load-failed", err) | 241 | d.logRejectedRestoreCandidate(record, candidate, fallback, "load-failed", err) | ||
| 221 | continue | 242 | continue | ||
| 222 | } | 243 | } | ||
| @@ -241,6 +262,9 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||||
| 241 | break | 262 | break | ||
| 242 | } | 263 | } | ||
| 243 | if selectedIndex < 0 { | 264 | if selectedIndex < 0 { | ||
| 265 | + | if retryableLoadErr != nil { | |||
| 266 | + | return fmt.Errorf("%w: %v", errRetryableRestoreLoad, retryableLoadErr) | |||
| 267 | + | } | |||
| 244 | return errors.New("snapshot: no catalogue checkpoint validated") | 268 | return errors.New("snapshot: no catalogue checkpoint validated") | ||
| 245 | } | 269 | } | ||
| 246 | 270 | ||||
| @@ -276,6 +300,28 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||||
| 276 | return nil | 300 | return nil | ||
| 277 | } | 301 | } | ||
| 278 | 302 | ||||
| 303 | + | func isRetryableRestoreLoadError(err error) bool { | |||
| 304 | + | if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { | |||
| 305 | + | return false | |||
| 306 | + | } | |||
| 307 | + | if errors.Is(err, ports.ErrBudgetExhausted) || | |||
| 308 | + | errors.Is(err, syscall.ENOSPC) || | |||
| 309 | + | errors.Is(err, syscall.ENOMEM) || | |||
| 310 | + | errors.Is(err, syscall.EMFILE) || | |||
| 311 | + | errors.Is(err, syscall.ENFILE) || | |||
| 312 | + | errors.Is(err, syscall.EDQUOT) { | |||
| 313 | + | return true | |||
| 314 | + | } | |||
| 315 | + | if netErr, ok := errors.AsType[net.Error](err); ok && netErr.Timeout() { | |||
| 316 | + | return true | |||
| 317 | + | } | |||
| 318 | + | temporary, ok := errors.AsType[interface { | |||
| 319 | + | error | |||
| 320 | + | Temporary() bool | |||
| 321 | + | }](err) | |||
| 322 | + | return ok && temporary.Temporary() | |||
| 323 | + | } | |||
| 324 | + | ||||
| 279 | func (d *Daemon) logRejectedRestoreCandidate(record domain.CatalogueRecord, candidate domain.CheckpointRef, fallback bool, reason string, err error) { | 325 | func (d *Daemon) logRejectedRestoreCandidate(record domain.CatalogueRecord, candidate domain.CheckpointRef, fallback bool, reason string, err error) { | ||
| 280 | kind := "committed" | 326 | kind := "committed" | ||
| 281 | if fallback { | 327 | if fallback { | ||
M internal/usecase/daemon/snapshot_restore_test.go +5 -3
| @@ -277,7 +277,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { | |||
| 277 | 277 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | |
| 278 | 278 | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | |
| 279 | 279 | WithSnapshotRepository(repository)(d) | |
| 280 | - | WithCheckpointCoordinator(coordinator)(d) | |
| 280 | + | WithRecoveryCoordinator(coordinator)(d) | |
| 281 | 281 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | |
| 282 | 282 | ||
| 283 | 283 | d.restoreIncrementalSnapshots(context.Background()) | |
| @@ -290,7 +290,8 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { | |||
| 290 | 290 | updates := catalogue.MetadataUpdates() | |
| 291 | 291 | require.Len(t, updates, 1, "the authoritative Catalogue port must receive restored-session metadata updates") | |
| 292 | 292 | require.Equal(t, record.IncarnationID, updates[0].IncarnationID) | |
| 293 | - | updated, ok, _ := catalogue.Record(record.Name) | |
| 293 | + | updated, ok, err := catalogue.Record(record.Name) | |
| 294 | + | require.NoError(t, err) | |
| 294 | 295 | require.True(t, ok) | |
| 295 | 296 | require.Equal(t, record.IncarnationID, updated.IncarnationID) | |
| 296 | 297 | require.Equal(t, record.RecoveryState, updated.RecoveryState) | |
| @@ -309,7 +310,8 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { | |||
| 309 | 310 | require.Empty(t, failure) | |
| 310 | 311 | require.Len(t, repository.publishes, 1) | |
| 311 | 312 | require.Equal(t, record.Committed, repository.publishes[0].ParentCheckpoint) | |
| 312 | - | published, ok, _ := catalogue.Record(record.Name) | |
| 313 | + | published, ok, err := catalogue.Record(record.Name) | |
| 314 | + | require.NoError(t, err) | |
| 313 | 315 | require.True(t, ok) | |
| 314 | 316 | require.Equal(t, uint64(10), published.Committed.Generation) | |
| 315 | 317 | require.Equal(t, committed, *published.Fallbacks[0]) | |
| @@ -277,7 +277,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { | |||||
| 277 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | 277 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | ||
| 278 | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | 278 | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | ||
| 279 | WithSnapshotRepository(repository)(d) | 279 | WithSnapshotRepository(repository)(d) | ||
| 280 | - | WithCheckpointCoordinator(coordinator)(d) | 280 | + | WithRecoveryCoordinator(coordinator)(d) |
| 281 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | 281 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | ||
| 282 | 282 | ||||
| 283 | d.restoreIncrementalSnapshots(context.Background()) | 283 | d.restoreIncrementalSnapshots(context.Background()) | ||
| @@ -290,7 +290,8 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { | |||||
| 290 | updates := catalogue.MetadataUpdates() | 290 | updates := catalogue.MetadataUpdates() | ||
| 291 | require.Len(t, updates, 1, "the authoritative Catalogue port must receive restored-session metadata updates") | 291 | require.Len(t, updates, 1, "the authoritative Catalogue port must receive restored-session metadata updates") | ||
| 292 | require.Equal(t, record.IncarnationID, updates[0].IncarnationID) | 292 | require.Equal(t, record.IncarnationID, updates[0].IncarnationID) | ||
| 293 | - | updated, ok, _ := catalogue.Record(record.Name) | 293 | + | updated, ok, err := catalogue.Record(record.Name) |
| 294 | + | require.NoError(t, err) | |||
| 294 | require.True(t, ok) | 295 | require.True(t, ok) | ||
| 295 | require.Equal(t, record.IncarnationID, updated.IncarnationID) | 296 | require.Equal(t, record.IncarnationID, updated.IncarnationID) | ||
| 296 | require.Equal(t, record.RecoveryState, updated.RecoveryState) | 297 | require.Equal(t, record.RecoveryState, updated.RecoveryState) | ||
| @@ -309,7 +310,8 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { | |||||
| 309 | require.Empty(t, failure) | 310 | require.Empty(t, failure) | ||
| 310 | require.Len(t, repository.publishes, 1) | 311 | require.Len(t, repository.publishes, 1) | ||
| 311 | require.Equal(t, record.Committed, repository.publishes[0].ParentCheckpoint) | 312 | require.Equal(t, record.Committed, repository.publishes[0].ParentCheckpoint) | ||
| 312 | - | published, ok, _ := catalogue.Record(record.Name) | 313 | + | published, ok, err := catalogue.Record(record.Name) |
| 314 | + | require.NoError(t, err) | |||
| 313 | require.True(t, ok) | 315 | require.True(t, ok) | ||
| 314 | require.Equal(t, uint64(10), published.Committed.Generation) | 316 | require.Equal(t, uint64(10), published.Committed.Generation) | ||
| 315 | require.Equal(t, committed, *published.Fallbacks[0]) | 317 | require.Equal(t, committed, *published.Fallbacks[0]) | ||
M internal/usecase/daemon/snapshot_scheduler_test.go +0 -1
| @@ -819,7 +819,6 @@ func TestSnapshotSchedulerImmediateEligibilityAndStaleCapturesRemainDirty(t *tes | |||
| 819 | 819 | now := time.Unix(100, 0) | |
| 820 | 820 | clock := newDeterministicSnapshotClock(now) | |
| 821 | 821 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | |
| 822 | - | WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t))(d) | |
| 823 | 822 | ||
| 824 | 823 | t.Run("no prior attempt is immediately eligible", func(t *testing.T) { | |
| 825 | 824 | published := make(chan ports.SnapshotPublication, 1) | |
| @@ -819,7 +819,6 @@ func TestSnapshotSchedulerImmediateEligibilityAndStaleCapturesRemainDirty(t *tes | |||||
| 819 | now := time.Unix(100, 0) | 819 | now := time.Unix(100, 0) | ||
| 820 | clock := newDeterministicSnapshotClock(now) | 820 | clock := newDeterministicSnapshotClock(now) | ||
| 821 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 821 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 822 | - | WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t))(d) | |||
| 823 | 822 | ||||
| 824 | t.Run("no prior attempt is immediately eligible", func(t *testing.T) { | 823 | t.Run("no prior attempt is immediately eligible", func(t *testing.T) { | ||
| 825 | published := make(chan ports.SnapshotPublication, 1) | 824 | published := make(chan ports.SnapshotPublication, 1) | ||
M internal/usecase/daemon/status_test.go +2 -2
| @@ -1066,9 +1066,9 @@ func TestBarStateForMRURestoredStoppedSessionsUsePersistedOrder(t *testing.T) { | |||
| 1066 | 1066 | cur := &session{id: "cur", name: "current", tabs: []*tab{{}}} | |
| 1067 | 1067 | d.sessions[cur.id] = cur | |
| 1068 | 1068 | ||
| 1069 | - | _, err := d.createSessionLocked("zeta", false, "/tmp/zeta", sz, terminalEnv{}, d.baseEnv) | |
| 1069 | + | _, err := createSessionForTest(d, "zeta", false, "/tmp/zeta", sz, terminalEnv{}, d.baseEnv) | |
| 1070 | 1070 | require.NoError(t, err) | |
| 1071 | - | _, err = d.createSessionLocked("alpha", false, "/tmp/alpha", sz, terminalEnv{}, d.baseEnv) | |
| 1071 | + | _, err = createSessionForTest(d, "alpha", false, "/tmp/alpha", sz, terminalEnv{}, d.baseEnv) | |
| 1072 | 1072 | require.NoError(t, err) | |
| 1073 | 1073 | ||
| 1074 | 1074 | state := d.barStateFor(cur, "") | |
| @@ -1066,9 +1066,9 @@ func TestBarStateForMRURestoredStoppedSessionsUsePersistedOrder(t *testing.T) { | |||||
| 1066 | cur := &session{id: "cur", name: "current", tabs: []*tab{{}}} | 1066 | cur := &session{id: "cur", name: "current", tabs: []*tab{{}}} | ||
| 1067 | d.sessions[cur.id] = cur | 1067 | d.sessions[cur.id] = cur | ||
| 1068 | 1068 | ||||
| 1069 | - | _, err := d.createSessionLocked("zeta", false, "/tmp/zeta", sz, terminalEnv{}, d.baseEnv) | 1069 | + | _, err := createSessionForTest(d, "zeta", false, "/tmp/zeta", sz, terminalEnv{}, d.baseEnv) |
| 1070 | require.NoError(t, err) | 1070 | require.NoError(t, err) | ||
| 1071 | - | _, err = d.createSessionLocked("alpha", false, "/tmp/alpha", sz, terminalEnv{}, d.baseEnv) | 1071 | + | _, err = createSessionForTest(d, "alpha", false, "/tmp/alpha", sz, terminalEnv{}, d.baseEnv) |
| 1072 | require.NoError(t, err) | 1072 | require.NoError(t, err) | ||
| 1073 | 1073 | ||||
| 1074 | state := d.barStateFor(cur, "") | 1074 | state := d.barStateFor(cur, "") | ||
M internal/usecase/recovery/checkpoint.go +4 -1
| @@ -103,7 +103,10 @@ func (c *Coordinator) promoteFallbackLocked(ctx context.Context, name string, re | |||
| 103 | 103 | candidates = append(candidates, record.Committed) | |
| 104 | 104 | candidates = append(candidates, record.Fallbacks[:]...) | |
| 105 | 105 | for _, candidate := range candidates { | |
| 106 | - | if candidate == nil || *candidate == ref || candidate.Generation >= ref.Generation || len(alternatives) == 2 { | |
| 106 | + | if len(alternatives) == 2 { | |
| 107 | + | break | |
| 108 | + | } | |
| 109 | + | if candidate == nil || *candidate == ref || candidate.Generation >= ref.Generation { | |
| 107 | 110 | continue | |
| 108 | 111 | } | |
| 109 | 112 | if err := c.validateCheckpoint(ctx, record, *candidate); err == nil { | |
| @@ -103,7 +103,10 @@ func (c *Coordinator) promoteFallbackLocked(ctx context.Context, name string, re | |||||
| 103 | candidates = append(candidates, record.Committed) | 103 | candidates = append(candidates, record.Committed) | ||
| 104 | candidates = append(candidates, record.Fallbacks[:]...) | 104 | candidates = append(candidates, record.Fallbacks[:]...) | ||
| 105 | for _, candidate := range candidates { | 105 | for _, candidate := range candidates { | ||
| 106 | - | if candidate == nil || *candidate == ref || candidate.Generation >= ref.Generation || len(alternatives) == 2 { | 106 | + | if len(alternatives) == 2 { |
| 107 | + | break | |||
| 108 | + | } | |||
| 109 | + | if candidate == nil || *candidate == ref || candidate.Generation >= ref.Generation { | |||
| 107 | continue | 110 | continue | ||
| 108 | } | 111 | } | ||
| 109 | if err := c.validateCheckpoint(ctx, record, *candidate); err == nil { | 112 | if err := c.validateCheckpoint(ctx, record, *candidate); err == nil { | ||
M internal/usecase/recovery/coordinator.go +5 -1
| @@ -123,7 +123,7 @@ type RecoveryConflict struct { | |||
| 123 | 123 | Err error | |
| 124 | 124 | } | |
| 125 | 125 | ||
| 126 | - | // Conflicts returns the session-scoped items fenced by the last Recover calls. | |
| 126 | + | // Conflicts returns the session-scoped items fenced by the most recent Recover call. | |
| 127 | 127 | func (c *Coordinator) Conflicts() []RecoveryConflict { | |
| 128 | 128 | if c == nil { | |
| 129 | 129 | return nil | |
| @@ -264,6 +264,10 @@ func (c *Coordinator) Recover(ctx context.Context) error { | |||
| 264 | 264 | if c == nil || c.catalogue == nil || c.repository == nil || c.journal == nil || c.locks == nil { | |
| 265 | 265 | return errors.New("recovery: incomplete coordinator dependencies") | |
| 266 | 266 | } | |
| 267 | + | c.conflictMu.Lock() | |
| 268 | + | c.conflicts = nil | |
| 269 | + | c.conflictMu.Unlock() | |
| 270 | + | ||
| 267 | 271 | intents, err := c.journal.ListDiscards(ctx) | |
| 268 | 272 | if err != nil { | |
| 269 | 273 | return err | |
| @@ -123,7 +123,7 @@ type RecoveryConflict struct { | |||||
| 123 | Err error | 123 | Err error | ||
| 124 | } | 124 | } | ||
| 125 | 125 | ||||
| 126 | - | // Conflicts returns the session-scoped items fenced by the last Recover calls. | 126 | + | // Conflicts returns the session-scoped items fenced by the most recent Recover call. |
| 127 | func (c *Coordinator) Conflicts() []RecoveryConflict { | 127 | func (c *Coordinator) Conflicts() []RecoveryConflict { | ||
| 128 | if c == nil { | 128 | if c == nil { | ||
| 129 | return nil | 129 | return nil | ||
| @@ -264,6 +264,10 @@ func (c *Coordinator) Recover(ctx context.Context) error { | |||||
| 264 | if c == nil || c.catalogue == nil || c.repository == nil || c.journal == nil || c.locks == nil { | 264 | if c == nil || c.catalogue == nil || c.repository == nil || c.journal == nil || c.locks == nil { | ||
| 265 | return errors.New("recovery: incomplete coordinator dependencies") | 265 | return errors.New("recovery: incomplete coordinator dependencies") | ||
| 266 | } | 266 | } | ||
| 267 | + | c.conflictMu.Lock() | |||
| 268 | + | c.conflicts = nil | |||
| 269 | + | c.conflictMu.Unlock() | |||
| 270 | + | ||||
| 267 | intents, err := c.journal.ListDiscards(ctx) | 271 | intents, err := c.journal.ListDiscards(ctx) | ||
| 268 | if err != nil { | 272 | if err != nil { | ||
| 269 | return err | 273 | return err | ||
M internal/usecase/recovery/reconcile.go +6 -6
| @@ -90,15 +90,15 @@ func classifyFinding(record domain.CatalogueRecord, finding ports.ReconcileFindi | |||
| 90 | 90 | decision.ReasonCode = "incarnation-mismatch" | |
| 91 | 91 | return decision | |
| 92 | 92 | } | |
| 93 | - | if finding.Status == ports.ReconcileValidated && record.Committed != nil && (finding.Kind != ports.ReconcileForwardOrphan || finding.Candidate.Ref.Generation <= record.Committed.Generation) { | |
| 93 | + | if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil { | |
| 94 | 94 | decision.Kind = ports.ReconcileKeepQuarantined | |
| 95 | - | decision.ReasonCode = "not-forward" | |
| 96 | - | decision.RetentionResolved = true | |
| 95 | + | decision.ReasonCode = "invalid-candidate" | |
| 97 | 96 | return decision | |
| 98 | 97 | } | |
| 99 | - | if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil { | |
| 98 | + | if finding.Status == ports.ReconcileValidated && finding.Candidate.Ref.Generation <= record.Committed.Generation { | |
| 100 | 99 | decision.Kind = ports.ReconcileKeepQuarantined | |
| 101 | - | decision.ReasonCode = "invalid-candidate" | |
| 100 | + | decision.ReasonCode = "not-forward" | |
| 101 | + | decision.RetentionResolved = true | |
| 102 | 102 | return decision | |
| 103 | 103 | } | |
| 104 | 104 | if finding.Status != ports.ReconcileValidated || !validForwardChain(*record.Committed, finding.Candidate, finding.AncestorChain) { | |
| @@ -157,7 +157,7 @@ func (c *Coordinator) PublishReconciledCheckpoint(ctx context.Context, name stri | |||
| 157 | 157 | if !ok { | |
| 158 | 158 | return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound | |
| 159 | 159 | } | |
| 160 | - | if candidate.Name != name || candidate.IncarnationID != record.IncarnationID || record.Committed == nil || !validForwardChain(*record.Committed, candidate, ancestors) { | |
| 160 | + | if record.RecoveryState != domain.RecoveryHealthy || candidate.Name != name || candidate.IncarnationID != record.IncarnationID || record.Committed == nil || !validForwardChain(*record.Committed, candidate, ancestors) { | |
| 161 | 161 | return domain.CatalogueRecord{}, ErrCheckpointConflict | |
| 162 | 162 | } | |
| 163 | 163 | next := shiftedCheckpoint(record, candidate.Ref) | |
| @@ -90,15 +90,15 @@ func classifyFinding(record domain.CatalogueRecord, finding ports.ReconcileFindi | |||||
| 90 | decision.ReasonCode = "incarnation-mismatch" | 90 | decision.ReasonCode = "incarnation-mismatch" | ||
| 91 | return decision | 91 | return decision | ||
| 92 | } | 92 | } | ||
| 93 | - | if finding.Status == ports.ReconcileValidated && record.Committed != nil && (finding.Kind != ports.ReconcileForwardOrphan || finding.Candidate.Ref.Generation <= record.Committed.Generation) { | 93 | + | if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil { |
| 94 | decision.Kind = ports.ReconcileKeepQuarantined | 94 | decision.Kind = ports.ReconcileKeepQuarantined | ||
| 95 | - | decision.ReasonCode = "not-forward" | 95 | + | decision.ReasonCode = "invalid-candidate" |
| 96 | - | decision.RetentionResolved = true | |||
| 97 | return decision | 96 | return decision | ||
| 98 | } | 97 | } | ||
| 99 | - | if finding.Kind != ports.ReconcileForwardOrphan || record.Committed == nil { | 98 | + | if finding.Status == ports.ReconcileValidated && finding.Candidate.Ref.Generation <= record.Committed.Generation { |
| 100 | decision.Kind = ports.ReconcileKeepQuarantined | 99 | decision.Kind = ports.ReconcileKeepQuarantined | ||
| 101 | - | decision.ReasonCode = "invalid-candidate" | 100 | + | decision.ReasonCode = "not-forward" |
| 101 | + | decision.RetentionResolved = true | |||
| 102 | return decision | 102 | return decision | ||
| 103 | } | 103 | } | ||
| 104 | if finding.Status != ports.ReconcileValidated || !validForwardChain(*record.Committed, finding.Candidate, finding.AncestorChain) { | 104 | if finding.Status != ports.ReconcileValidated || !validForwardChain(*record.Committed, finding.Candidate, finding.AncestorChain) { | ||
| @@ -157,7 +157,7 @@ func (c *Coordinator) PublishReconciledCheckpoint(ctx context.Context, name stri | |||||
| 157 | if !ok { | 157 | if !ok { | ||
| 158 | return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound | 158 | return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound | ||
| 159 | } | 159 | } | ||
| 160 | - | if candidate.Name != name || candidate.IncarnationID != record.IncarnationID || record.Committed == nil || !validForwardChain(*record.Committed, candidate, ancestors) { | 160 | + | if record.RecoveryState != domain.RecoveryHealthy || candidate.Name != name || candidate.IncarnationID != record.IncarnationID || record.Committed == nil || !validForwardChain(*record.Committed, candidate, ancestors) { |
| 161 | return domain.CatalogueRecord{}, ErrCheckpointConflict | 161 | return domain.CatalogueRecord{}, ErrCheckpointConflict | ||
| 162 | } | 162 | } | ||
| 163 | next := shiftedCheckpoint(record, candidate.Ref) | 163 | next := shiftedCheckpoint(record, candidate.Ref) | ||
M internal/usecase/recovery/reconcile_test.go +19 -2
| @@ -160,9 +160,9 @@ func TestForwardOrphanAdoption(t *testing.T) { | |||
| 160 | 160 | {name: "generation reversal", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) { | |
| 161 | 161 | finding.Candidate.Ref.Generation = record.Committed.Generation | |
| 162 | 162 | }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "not-forward"}, | |
| 163 | - | {name: "not forward orphan", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) { | |
| 163 | + | {name: "non-orphan invalid candidate", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) { | |
| 164 | 164 | finding.Kind = ports.ReconcileInvalidCandidate | |
| 165 | - | }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "not-forward"}, | |
| 165 | + | }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "invalid-candidate"}, | |
| 166 | 166 | } { | |
| 167 | 167 | t.Run(tc.name, func(t *testing.T) { | |
| 168 | 168 | record := healthyReconcileRecord() | |
| @@ -191,6 +191,23 @@ func TestForwardOrphanAdoption(t *testing.T) { | |||
| 191 | 191 | require.Empty(t, repository.repairs) | |
| 192 | 192 | }) | |
| 193 | 193 | ||
| 194 | + | for _, state := range []domain.RecoveryState{domain.RecoveryDegraded, domain.RecoveryDeleting} { | |
| 195 | + | t.Run("non-healthy record is not reconciled", func(t *testing.T) { | |
| 196 | + | record := healthyReconcileRecord() | |
| 197 | + | record.RecoveryState = state | |
| 198 | + | if state == domain.RecoveryDegraded { | |
| 199 | + | record.DegradedReason = "uncertain" | |
| 200 | + | } else { | |
| 201 | + | record.Committed = nil | |
| 202 | + | } | |
| 203 | + | candidate := forwardCandidate(healthyReconcileRecord(), 2, healthyReconcileRecord().Committed) | |
| 204 | + | coordinator := NewCoordinator(&checkpointCatalogue{record: record}, &reconcileRepository{}, nil, nil) | |
| 205 | + | ||
| 206 | + | _, err := coordinator.PublishReconciledCheckpoint(context.Background(), record.Name, candidate, nil) | |
| 207 | + | require.ErrorIs(t, err, ErrCheckpointConflict) | |
| 208 | + | }) | |
| 209 | + | } | |
| 210 | + | ||
| 194 | 211 | t.Run("stale decision after concurrent commit", func(t *testing.T) { | |
| 195 | 212 | record := healthyReconcileRecord() | |
| 196 | 213 | candidate := forwardCandidate(record, 2, record.Committed) | |
| @@ -160,9 +160,9 @@ func TestForwardOrphanAdoption(t *testing.T) { | |||||
| 160 | {name: "generation reversal", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) { | 160 | {name: "generation reversal", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) { | ||
| 161 | finding.Candidate.Ref.Generation = record.Committed.Generation | 161 | finding.Candidate.Ref.Generation = record.Committed.Generation | ||
| 162 | }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "not-forward"}, | 162 | }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "not-forward"}, | ||
| 163 | - | {name: "not forward orphan", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) { | 163 | + | {name: "non-orphan invalid candidate", mutate: func(record domain.CatalogueRecord, finding *ports.ReconcileFinding) { |
| 164 | finding.Kind = ports.ReconcileInvalidCandidate | 164 | finding.Kind = ports.ReconcileInvalidCandidate | ||
| 165 | - | }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "not-forward"}, | 165 | + | }, wantKind: ports.ReconcileKeepQuarantined, wantCause: "invalid-candidate"}, |
| 166 | } { | 166 | } { | ||
| 167 | t.Run(tc.name, func(t *testing.T) { | 167 | t.Run(tc.name, func(t *testing.T) { | ||
| 168 | record := healthyReconcileRecord() | 168 | record := healthyReconcileRecord() | ||
| @@ -191,6 +191,23 @@ func TestForwardOrphanAdoption(t *testing.T) { | |||||
| 191 | require.Empty(t, repository.repairs) | 191 | require.Empty(t, repository.repairs) | ||
| 192 | }) | 192 | }) | ||
| 193 | 193 | ||||
| 194 | + | for _, state := range []domain.RecoveryState{domain.RecoveryDegraded, domain.RecoveryDeleting} { | |||
| 195 | + | t.Run("non-healthy record is not reconciled", func(t *testing.T) { | |||
| 196 | + | record := healthyReconcileRecord() | |||
| 197 | + | record.RecoveryState = state | |||
| 198 | + | if state == domain.RecoveryDegraded { | |||
| 199 | + | record.DegradedReason = "uncertain" | |||
| 200 | + | } else { | |||
| 201 | + | record.Committed = nil | |||
| 202 | + | } | |||
| 203 | + | candidate := forwardCandidate(healthyReconcileRecord(), 2, healthyReconcileRecord().Committed) | |||
| 204 | + | coordinator := NewCoordinator(&checkpointCatalogue{record: record}, &reconcileRepository{}, nil, nil) | |||
| 205 | + | ||||
| 206 | + | _, err := coordinator.PublishReconciledCheckpoint(context.Background(), record.Name, candidate, nil) | |||
| 207 | + | require.ErrorIs(t, err, ErrCheckpointConflict) | |||
| 208 | + | }) | |||
| 209 | + | } | |||
| 210 | + | ||||
| 194 | t.Run("stale decision after concurrent commit", func(t *testing.T) { | 211 | t.Run("stale decision after concurrent commit", func(t *testing.T) { | ||
| 195 | record := healthyReconcileRecord() | 212 | record := healthyReconcileRecord() | ||
| 196 | candidate := forwardCandidate(record, 2, record.Committed) | 213 | candidate := forwardCandidate(record, 2, record.Committed) | ||
M internal/usecase/recovery/transactions_test.go +8 -1
| @@ -4,6 +4,7 @@ import ( | |||
| 4 | 4 | "bytes" | |
| 5 | 5 | "context" | |
| 6 | 6 | "crypto/sha256" | |
| 7 | + | "encoding/hex" | |
| 7 | 8 | "errors" | |
| 8 | 9 | "sync" | |
| 9 | 10 | "testing" | |
| @@ -272,7 +273,9 @@ func TestDegradedExportIsReadOnly(t *testing.T) { | |||
| 272 | 273 | ||
| 273 | 274 | require.NoError(t, coordinator.Export(context.Background(), record.Name, &exported)) | |
| 274 | 275 | require.NoError(t, coordinator.Export(context.Background(), record.Name, &repeated)) | |
| 275 | - | require.NotEmpty(t, exported.Bytes()) | |
| 276 | + | golden, err := hex.DecodeString("5645565800010100000000000000000000000000000000000000000000030000000662726f6b656e000000086d616e69666573740000000201000000000000000000000000000000000000000000000000000000000000000000000566697273740200000000000000000000000000000000000000000000000000000000000000000000067365636f6e64") | |
| 277 | + | require.NoError(t, err) | |
| 278 | + | require.Equal(t, golden, exported.Bytes()) | |
| 276 | 279 | require.Equal(t, exported.Bytes(), repeated.Bytes()) | |
| 277 | 280 | require.Equal(t, record, catalogue.records[record.Name]) | |
| 278 | 281 | require.Empty(t, catalogue.events) | |
| @@ -314,6 +317,10 @@ func TestDiscardIncarnationConflict(t *testing.T) { | |||
| 314 | 317 | require.Equal(t, old.Name, conflicts[0].Session) | |
| 315 | 318 | require.Equal(t, "discard-intent", conflicts[0].Kind) | |
| 316 | 319 | require.ErrorIs(t, conflicts[0].Err, ErrDiscardConflict) | |
| 320 | + | ||
| 321 | + | delete(journal.intents, old.IncarnationID) | |
| 322 | + | require.NoError(t, coordinator.Recover(context.Background())) | |
| 323 | + | require.Empty(t, coordinator.Conflicts(), "a later recovery reports only its own conflicts") | |
| 317 | 324 | } | |
| 318 | 325 | ||
| 319 | 326 | // A discard whose replacement is already durable, live, and checkpointed must | |
| @@ -4,6 +4,7 @@ import ( | |||||
| 4 | "bytes" | 4 | "bytes" | ||
| 5 | "context" | 5 | "context" | ||
| 6 | "crypto/sha256" | 6 | "crypto/sha256" | ||
| 7 | + | "encoding/hex" | |||
| 7 | "errors" | 8 | "errors" | ||
| 8 | "sync" | 9 | "sync" | ||
| 9 | "testing" | 10 | "testing" | ||
| @@ -272,7 +273,9 @@ func TestDegradedExportIsReadOnly(t *testing.T) { | |||||
| 272 | 273 | ||||
| 273 | require.NoError(t, coordinator.Export(context.Background(), record.Name, &exported)) | 274 | require.NoError(t, coordinator.Export(context.Background(), record.Name, &exported)) | ||
| 274 | require.NoError(t, coordinator.Export(context.Background(), record.Name, &repeated)) | 275 | require.NoError(t, coordinator.Export(context.Background(), record.Name, &repeated)) | ||
| 275 | - | require.NotEmpty(t, exported.Bytes()) | 276 | + | golden, err := hex.DecodeString("5645565800010100000000000000000000000000000000000000000000030000000662726f6b656e000000086d616e69666573740000000201000000000000000000000000000000000000000000000000000000000000000000000566697273740200000000000000000000000000000000000000000000000000000000000000000000067365636f6e64") |
| 277 | + | require.NoError(t, err) | |||
| 278 | + | require.Equal(t, golden, exported.Bytes()) | |||
| 276 | require.Equal(t, exported.Bytes(), repeated.Bytes()) | 279 | require.Equal(t, exported.Bytes(), repeated.Bytes()) | ||
| 277 | require.Equal(t, record, catalogue.records[record.Name]) | 280 | require.Equal(t, record, catalogue.records[record.Name]) | ||
| 278 | require.Empty(t, catalogue.events) | 281 | require.Empty(t, catalogue.events) | ||
| @@ -314,6 +317,10 @@ func TestDiscardIncarnationConflict(t *testing.T) { | |||||
| 314 | require.Equal(t, old.Name, conflicts[0].Session) | 317 | require.Equal(t, old.Name, conflicts[0].Session) | ||
| 315 | require.Equal(t, "discard-intent", conflicts[0].Kind) | 318 | require.Equal(t, "discard-intent", conflicts[0].Kind) | ||
| 316 | require.ErrorIs(t, conflicts[0].Err, ErrDiscardConflict) | 319 | require.ErrorIs(t, conflicts[0].Err, ErrDiscardConflict) | ||
| 320 | + | ||||
| 321 | + | delete(journal.intents, old.IncarnationID) | |||
| 322 | + | require.NoError(t, coordinator.Recover(context.Background())) | |||
| 323 | + | require.Empty(t, coordinator.Conflicts(), "a later recovery reports only its own conflicts") | |||
| 317 | } | 324 | } | ||
| 318 | 325 | ||||
| 319 | // A discard whose replacement is already durable, live, and checkpointed must | 326 | // A discard whose replacement is already durable, live, and checkpointed must | ||
M internal/usecase/snapshot/codec.go +9 -0
| @@ -185,6 +185,15 @@ func writeProcess(w *payloadWriter, p *Process) error { | |||
| 185 | 185 | ||
| 186 | 186 | type payloadReader struct{ b []byte } | |
| 187 | 187 | ||
| 188 | + | func (r *payloadReader) getBytes(n int) ([]byte, error) { | |
| 189 | + | if n < 0 || len(r.b) < n { | |
| 190 | + | return nil, ErrShortPayload | |
| 191 | + | } | |
| 192 | + | b := r.b[:n] | |
| 193 | + | r.b = r.b[n:] | |
| 194 | + | return b, nil | |
| 195 | + | } | |
| 196 | + | ||
| 188 | 197 | func (r *payloadReader) getUint8() (uint8, error) { | |
| 189 | 198 | if len(r.b) < 1 { | |
| 190 | 199 | return 0, ErrShortPayload | |
| @@ -185,6 +185,15 @@ func writeProcess(w *payloadWriter, p *Process) error { | |||||
| 185 | 185 | ||||
| 186 | type payloadReader struct{ b []byte } | 186 | type payloadReader struct{ b []byte } | ||
| 187 | 187 | ||||
| 188 | + | func (r *payloadReader) getBytes(n int) ([]byte, error) { | |||
| 189 | + | if n < 0 || len(r.b) < n { | |||
| 190 | + | return nil, ErrShortPayload | |||
| 191 | + | } | |||
| 192 | + | b := r.b[:n] | |||
| 193 | + | r.b = r.b[n:] | |||
| 194 | + | return b, nil | |||
| 195 | + | } | |||
| 196 | + | ||||
| 188 | func (r *payloadReader) getUint8() (uint8, error) { | 197 | func (r *payloadReader) getUint8() (uint8, error) { | ||
| 189 | if len(r.b) < 1 { | 198 | if len(r.b) < 1 { | ||
| 190 | return 0, ErrShortPayload | 199 | return 0, ErrShortPayload | ||
M internal/usecase/snapshot/codec_test.go +29 -0
| @@ -14,6 +14,35 @@ import ( | |||
| 14 | 14 | "github.com/bnema/vev/pkg/vt" | |
| 15 | 15 | ) | |
| 16 | 16 | ||
| 17 | + | func TestPayloadReaderGetBytes(t *testing.T) { | |
| 18 | + | for _, tc := range []struct { | |
| 19 | + | name string | |
| 20 | + | input []byte | |
| 21 | + | n int | |
| 22 | + | want []byte | |
| 23 | + | remaining []byte | |
| 24 | + | wantErr error | |
| 25 | + | }{ | |
| 26 | + | {name: "reads and advances", input: []byte{1, 2, 3}, n: 2, want: []byte{1, 2}, remaining: []byte{3}}, | |
| 27 | + | {name: "reads empty", input: []byte{1}, n: 0, want: []byte{}, remaining: []byte{1}}, | |
| 28 | + | {name: "short payload does not advance", input: []byte{1}, n: 2, remaining: []byte{1}, wantErr: ErrShortPayload}, | |
| 29 | + | } { | |
| 30 | + | t.Run(tc.name, func(t *testing.T) { | |
| 31 | + | r := payloadReader{b: append([]byte(nil), tc.input...)} | |
| 32 | + | got, err := r.getBytes(tc.n) | |
| 33 | + | if !errors.Is(err, tc.wantErr) { | |
| 34 | + | t.Fatalf("getBytes(%d) error = %v, want %v", tc.n, err, tc.wantErr) | |
| 35 | + | } | |
| 36 | + | if !bytes.Equal(got, tc.want) { | |
| 37 | + | t.Fatalf("getBytes(%d) = %v, want %v", tc.n, got, tc.want) | |
| 38 | + | } | |
| 39 | + | if !bytes.Equal(r.b, tc.remaining) { | |
| 40 | + | t.Fatalf("remaining bytes = %v, want %v", r.b, tc.remaining) | |
| 41 | + | } | |
| 42 | + | }) | |
| 43 | + | } | |
| 44 | + | } | |
| 45 | + | ||
| 17 | 46 | func TestV3SnapshotRoundTripPreservesExactTerminalData(t *testing.T) { | |
| 18 | 47 | indexed := renderer.DefaultStyle() | |
| 19 | 48 | indexed.Bold, indexed.Inverse = true, true | |
| @@ -14,6 +14,35 @@ import ( | |||||
| 14 | "github.com/bnema/vev/pkg/vt" | 14 | "github.com/bnema/vev/pkg/vt" | ||
| 15 | ) | 15 | ) | ||
| 16 | 16 | ||||
| 17 | + | func TestPayloadReaderGetBytes(t *testing.T) { | |||
| 18 | + | for _, tc := range []struct { | |||
| 19 | + | name string | |||
| 20 | + | input []byte | |||
| 21 | + | n int | |||
| 22 | + | want []byte | |||
| 23 | + | remaining []byte | |||
| 24 | + | wantErr error | |||
| 25 | + | }{ | |||
| 26 | + | {name: "reads and advances", input: []byte{1, 2, 3}, n: 2, want: []byte{1, 2}, remaining: []byte{3}}, | |||
| 27 | + | {name: "reads empty", input: []byte{1}, n: 0, want: []byte{}, remaining: []byte{1}}, | |||
| 28 | + | {name: "short payload does not advance", input: []byte{1}, n: 2, remaining: []byte{1}, wantErr: ErrShortPayload}, | |||
| 29 | + | } { | |||
| 30 | + | t.Run(tc.name, func(t *testing.T) { | |||
| 31 | + | r := payloadReader{b: append([]byte(nil), tc.input...)} | |||
| 32 | + | got, err := r.getBytes(tc.n) | |||
| 33 | + | if !errors.Is(err, tc.wantErr) { | |||
| 34 | + | t.Fatalf("getBytes(%d) error = %v, want %v", tc.n, err, tc.wantErr) | |||
| 35 | + | } | |||
| 36 | + | if !bytes.Equal(got, tc.want) { | |||
| 37 | + | t.Fatalf("getBytes(%d) = %v, want %v", tc.n, got, tc.want) | |||
| 38 | + | } | |||
| 39 | + | if !bytes.Equal(r.b, tc.remaining) { | |||
| 40 | + | t.Fatalf("remaining bytes = %v, want %v", r.b, tc.remaining) | |||
| 41 | + | } | |||
| 42 | + | }) | |||
| 43 | + | } | |||
| 44 | + | } | |||
| 45 | + | ||||
| 17 | func TestV3SnapshotRoundTripPreservesExactTerminalData(t *testing.T) { | 46 | func TestV3SnapshotRoundTripPreservesExactTerminalData(t *testing.T) { | ||
| 18 | indexed := renderer.DefaultStyle() | 47 | indexed := renderer.DefaultStyle() | ||
| 19 | indexed.Bold, indexed.Inverse = true, true | 48 | indexed.Bold, indexed.Inverse = true, true | ||
M internal/usecase/snapshot/manifest.go +8 -8
| @@ -179,12 +179,12 @@ func decodeManifest(body []byte) (Manifest, error) { | |||
| 179 | 179 | if err != nil { | |
| 180 | 180 | return Manifest{}, err | |
| 181 | 181 | } | |
| 182 | - | if len(r.b) < len(domain.IncarnationID{}) { | |
| 183 | - | return Manifest{}, ErrShortPayload | |
| 182 | + | incarnationBytes, err := r.getBytes(len(domain.IncarnationID{})) | |
| 183 | + | if err != nil { | |
| 184 | + | return Manifest{}, err | |
| 184 | 185 | } | |
| 185 | 186 | var incarnationID domain.IncarnationID | |
| 186 | - | copy(incarnationID[:], r.b[:len(incarnationID)]) | |
| 187 | - | r.b = r.b[len(incarnationID):] | |
| 187 | + | copy(incarnationID[:], incarnationBytes) | |
| 188 | 188 | parentPresent, err := r.getUint8() | |
| 189 | 189 | if err != nil { | |
| 190 | 190 | return Manifest{}, err | |
| @@ -198,11 +198,11 @@ func decodeManifest(body []byte) (Manifest, error) { | |||
| 198 | 198 | if err != nil { | |
| 199 | 199 | return Manifest{}, err | |
| 200 | 200 | } | |
| 201 | - | if len(r.b) < len(parent.ManifestDigest) { | |
| 202 | - | return Manifest{}, ErrShortPayload | |
| 201 | + | digest, err := r.getBytes(len(parent.ManifestDigest)) | |
| 202 | + | if err != nil { | |
| 203 | + | return Manifest{}, err | |
| 203 | 204 | } | |
| 204 | - | copy(parent.ManifestDigest[:], r.b[:len(parent.ManifestDigest)]) | |
| 205 | - | r.b = r.b[len(parent.ManifestDigest):] | |
| 205 | + | copy(parent.ManifestDigest[:], digest) | |
| 206 | 206 | default: | |
| 207 | 207 | return Manifest{}, fmt.Errorf("%w: parent checkpoint presence", ErrInvalidData) | |
| 208 | 208 | } | |
| @@ -179,12 +179,12 @@ func decodeManifest(body []byte) (Manifest, error) { | |||||
| 179 | if err != nil { | 179 | if err != nil { | ||
| 180 | return Manifest{}, err | 180 | return Manifest{}, err | ||
| 181 | } | 181 | } | ||
| 182 | - | if len(r.b) < len(domain.IncarnationID{}) { | 182 | + | incarnationBytes, err := r.getBytes(len(domain.IncarnationID{})) |
| 183 | - | return Manifest{}, ErrShortPayload | 183 | + | if err != nil { |
| 184 | + | return Manifest{}, err | |||
| 184 | } | 185 | } | ||
| 185 | var incarnationID domain.IncarnationID | 186 | var incarnationID domain.IncarnationID | ||
| 186 | - | copy(incarnationID[:], r.b[:len(incarnationID)]) | 187 | + | copy(incarnationID[:], incarnationBytes) |
| 187 | - | r.b = r.b[len(incarnationID):] | |||
| 188 | parentPresent, err := r.getUint8() | 188 | parentPresent, err := r.getUint8() | ||
| 189 | if err != nil { | 189 | if err != nil { | ||
| 190 | return Manifest{}, err | 190 | return Manifest{}, err | ||
| @@ -198,11 +198,11 @@ func decodeManifest(body []byte) (Manifest, error) { | |||||
| 198 | if err != nil { | 198 | if err != nil { | ||
| 199 | return Manifest{}, err | 199 | return Manifest{}, err | ||
| 200 | } | 200 | } | ||
| 201 | - | if len(r.b) < len(parent.ManifestDigest) { | 201 | + | digest, err := r.getBytes(len(parent.ManifestDigest)) |
| 202 | - | return Manifest{}, ErrShortPayload | 202 | + | if err != nil { |
| 203 | + | return Manifest{}, err | |||
| 203 | } | 204 | } | ||
| 204 | - | copy(parent.ManifestDigest[:], r.b[:len(parent.ManifestDigest)]) | 205 | + | copy(parent.ManifestDigest[:], digest) |
| 205 | - | r.b = r.b[len(parent.ManifestDigest):] | |||
| 206 | default: | 206 | default: | ||
| 207 | return Manifest{}, fmt.Errorf("%w: parent checkpoint presence", ErrInvalidData) | 207 | return Manifest{}, fmt.Errorf("%w: parent checkpoint presence", ErrInvalidData) | ||
| 208 | } | 208 | } | ||
M internal/usecase/snapshot/manifest_preflight.go +7 -8
| @@ -19,12 +19,12 @@ func preflightManifest(body []byte) error { | |||
| 19 | 19 | if generation == 0 { | |
| 20 | 20 | return fmt.Errorf("%w: manifest generation", ErrInvalidData) | |
| 21 | 21 | } | |
| 22 | - | if len(r.b) < len(domain.IncarnationID{}) { | |
| 23 | - | return ErrShortPayload | |
| 22 | + | incarnationBytes, err := r.getBytes(len(domain.IncarnationID{})) | |
| 23 | + | if err != nil { | |
| 24 | + | return err | |
| 24 | 25 | } | |
| 25 | 26 | var incarnationID domain.IncarnationID | |
| 26 | - | copy(incarnationID[:], r.b[:len(incarnationID)]) | |
| 27 | - | r.b = r.b[len(incarnationID):] | |
| 27 | + | copy(incarnationID[:], incarnationBytes) | |
| 28 | 28 | if incarnationID == (domain.IncarnationID{}) { | |
| 29 | 29 | return fmt.Errorf("%w: manifest incarnation", ErrInvalidData) | |
| 30 | 30 | } | |
| @@ -39,11 +39,10 @@ func preflightManifest(body []byte) error { | |||
| 39 | 39 | if err != nil { | |
| 40 | 40 | return err | |
| 41 | 41 | } | |
| 42 | - | if len(r.b) < 32 { | |
| 43 | - | return ErrShortPayload | |
| 42 | + | digest, err := r.getBytes(len(SnapshotDigest{})) | |
| 43 | + | if err != nil { | |
| 44 | + | return err | |
| 44 | 45 | } | |
| 45 | - | digest := r.b[:32] | |
| 46 | - | r.b = r.b[32:] | |
| 47 | 46 | if parentGeneration == 0 || parentGeneration >= generation || isZeroDigestBytes(digest) { | |
| 48 | 47 | return fmt.Errorf("%w: parent checkpoint", ErrInvalidData) | |
| 49 | 48 | } | |
| @@ -19,12 +19,12 @@ func preflightManifest(body []byte) error { | |||||
| 19 | if generation == 0 { | 19 | if generation == 0 { | ||
| 20 | return fmt.Errorf("%w: manifest generation", ErrInvalidData) | 20 | return fmt.Errorf("%w: manifest generation", ErrInvalidData) | ||
| 21 | } | 21 | } | ||
| 22 | - | if len(r.b) < len(domain.IncarnationID{}) { | 22 | + | incarnationBytes, err := r.getBytes(len(domain.IncarnationID{})) |
| 23 | - | return ErrShortPayload | 23 | + | if err != nil { |
| 24 | + | return err | |||
| 24 | } | 25 | } | ||
| 25 | var incarnationID domain.IncarnationID | 26 | var incarnationID domain.IncarnationID | ||
| 26 | - | copy(incarnationID[:], r.b[:len(incarnationID)]) | 27 | + | copy(incarnationID[:], incarnationBytes) |
| 27 | - | r.b = r.b[len(incarnationID):] | |||
| 28 | if incarnationID == (domain.IncarnationID{}) { | 28 | if incarnationID == (domain.IncarnationID{}) { | ||
| 29 | return fmt.Errorf("%w: manifest incarnation", ErrInvalidData) | 29 | return fmt.Errorf("%w: manifest incarnation", ErrInvalidData) | ||
| 30 | } | 30 | } | ||
| @@ -39,11 +39,10 @@ func preflightManifest(body []byte) error { | |||||
| 39 | if err != nil { | 39 | if err != nil { | ||
| 40 | return err | 40 | return err | ||
| 41 | } | 41 | } | ||
| 42 | - | if len(r.b) < 32 { | 42 | + | digest, err := r.getBytes(len(SnapshotDigest{})) |
| 43 | - | return ErrShortPayload | 43 | + | if err != nil { |
| 44 | + | return err | |||
| 44 | } | 45 | } | ||
| 45 | - | digest := r.b[:32] | |||
| 46 | - | r.b = r.b[32:] | |||
| 47 | if parentGeneration == 0 || parentGeneration >= generation || isZeroDigestBytes(digest) { | 46 | if parentGeneration == 0 || parentGeneration >= generation || isZeroDigestBytes(digest) { | ||
| 48 | return fmt.Errorf("%w: parent checkpoint", ErrInvalidData) | 47 | return fmt.Errorf("%w: parent checkpoint", ErrInvalidData) | ||
| 49 | } | 48 | } | ||
M internal/usecase/snapshot/manifest_preflight_test.go +12 -1
| @@ -91,6 +91,9 @@ func manifestFieldOffsets(t *testing.T, encoded []byte) (nameOffset, nodeKindOff | |||
| 91 | 91 | } | |
| 92 | 92 | _, err := r.getUint64() | |
| 93 | 93 | must(err) | |
| 94 | + | if len(r.b) < len(domain.IncarnationID{}) { | |
| 95 | + | t.Fatalf("manifest body has %d incarnation bytes, want at least %d", len(r.b), len(domain.IncarnationID{})) | |
| 96 | + | } | |
| 94 | 97 | r.b = r.b[len(domain.IncarnationID{}):] | |
| 95 | 98 | _, err = r.getUint8() | |
| 96 | 99 | must(err) | |
| @@ -117,7 +120,15 @@ func manifestFieldOffsets(t *testing.T, encoded []byte) (nameOffset, nodeKindOff | |||
| 117 | 120 | must(err) | |
| 118 | 121 | _, err = r.getString() | |
| 119 | 122 | must(err) | |
| 120 | - | return nameOffset, len(encoded) - len(r.b) | |
| 123 | + | nodeKindOffset = len(encoded) - len(r.b) | |
| 124 | + | const wantName = "named" | |
| 125 | + | if nameOffset < 0 || nameOffset+len(wantName) > len(encoded) || string(encoded[nameOffset:nameOffset+len(wantName)]) != wantName { | |
| 126 | + | t.Fatalf("name offset %d does not point to %q", nameOffset, wantName) | |
| 127 | + | } | |
| 128 | + | if nodeKindOffset < 0 || nodeKindOffset >= len(encoded) || encoded[nodeKindOffset] != manifestNodeLeaf { | |
| 129 | + | t.Fatalf("node kind offset %d does not point to leaf kind", nodeKindOffset) | |
| 130 | + | } | |
| 131 | + | return nameOffset, nodeKindOffset | |
| 121 | 132 | } | |
| 122 | 133 | ||
| 123 | 134 | func TestManifestObjectRefKinds(t *testing.T) { | |
| @@ -91,6 +91,9 @@ func manifestFieldOffsets(t *testing.T, encoded []byte) (nameOffset, nodeKindOff | |||||
| 91 | } | 91 | } | ||
| 92 | _, err := r.getUint64() | 92 | _, err := r.getUint64() | ||
| 93 | must(err) | 93 | must(err) | ||
| 94 | + | if len(r.b) < len(domain.IncarnationID{}) { | |||
| 95 | + | t.Fatalf("manifest body has %d incarnation bytes, want at least %d", len(r.b), len(domain.IncarnationID{})) | |||
| 96 | + | } | |||
| 94 | r.b = r.b[len(domain.IncarnationID{}):] | 97 | r.b = r.b[len(domain.IncarnationID{}):] | ||
| 95 | _, err = r.getUint8() | 98 | _, err = r.getUint8() | ||
| 96 | must(err) | 99 | must(err) | ||
| @@ -117,7 +120,15 @@ func manifestFieldOffsets(t *testing.T, encoded []byte) (nameOffset, nodeKindOff | |||||
| 117 | must(err) | 120 | must(err) | ||
| 118 | _, err = r.getString() | 121 | _, err = r.getString() | ||
| 119 | must(err) | 122 | must(err) | ||
| 120 | - | return nameOffset, len(encoded) - len(r.b) | 123 | + | nodeKindOffset = len(encoded) - len(r.b) |
| 124 | + | const wantName = "named" | |||
| 125 | + | if nameOffset < 0 || nameOffset+len(wantName) > len(encoded) || string(encoded[nameOffset:nameOffset+len(wantName)]) != wantName { | |||
| 126 | + | t.Fatalf("name offset %d does not point to %q", nameOffset, wantName) | |||
| 127 | + | } | |||
| 128 | + | if nodeKindOffset < 0 || nodeKindOffset >= len(encoded) || encoded[nodeKindOffset] != manifestNodeLeaf { | |||
| 129 | + | t.Fatalf("node kind offset %d does not point to leaf kind", nodeKindOffset) | |||
| 130 | + | } | |||
| 131 | + | return nameOffset, nodeKindOffset | |||
| 121 | } | 132 | } | ||
| 122 | 133 | ||||
| 123 | func TestManifestObjectRefKinds(t *testing.T) { | 134 | func TestManifestObjectRefKinds(t *testing.T) { | ||
M pkg/kv/kv.go +1 -5
| @@ -150,8 +150,7 @@ func openWithHooks(path string, hooks fsHooks) (*Store, error) { | |||
| 150 | 150 | // retries from whichever complete file recovery selects. | |
| 151 | 151 | if result.Format != formatCurrent || s.shouldCompact() { | |
| 152 | 152 | if err := s.compactLocked(); err != nil { | |
| 153 | - | _ = f.Close() | |
| 154 | - | return cleanupLock(err) | |
| 153 | + | return nil, err | |
| 155 | 154 | } | |
| 156 | 155 | } | |
| 157 | 156 | return s, nil | |
| @@ -720,9 +719,6 @@ func recoverCompaction(path string, hooks fsHooks) error { | |||
| 720 | 719 | } | |
| 721 | 720 | return hooks.SyncDir(dir) | |
| 722 | 721 | } | |
| 723 | - | if currentExists && currentErr != nil { | |
| 724 | - | return fmt.Errorf("%w: no valid current", ErrCorruptWAL) | |
| 725 | - | } | |
| 726 | 722 | return nil | |
| 727 | 723 | } | |
| 728 | 724 | ||
| @@ -150,8 +150,7 @@ func openWithHooks(path string, hooks fsHooks) (*Store, error) { | |||||
| 150 | // retries from whichever complete file recovery selects. | 150 | // retries from whichever complete file recovery selects. | ||
| 151 | if result.Format != formatCurrent || s.shouldCompact() { | 151 | if result.Format != formatCurrent || s.shouldCompact() { | ||
| 152 | if err := s.compactLocked(); err != nil { | 152 | if err := s.compactLocked(); err != nil { | ||
| 153 | - | _ = f.Close() | 153 | + | return nil, err |
| 154 | - | return cleanupLock(err) | |||
| 155 | } | 154 | } | ||
| 156 | } | 155 | } | ||
| 157 | return s, nil | 156 | return s, nil | ||
| @@ -720,9 +719,6 @@ func recoverCompaction(path string, hooks fsHooks) error { | |||||
| 720 | } | 719 | } | ||
| 721 | return hooks.SyncDir(dir) | 720 | return hooks.SyncDir(dir) | ||
| 722 | } | 721 | } | ||
| 723 | - | if currentExists && currentErr != nil { | |||
| 724 | - | return fmt.Errorf("%w: no valid current", ErrCorruptWAL) | |||
| 725 | - | } | |||
| 726 | return nil | 722 | return nil | ||
| 727 | } | 723 | } | ||
| 728 | 724 | ||||
M pkg/kv/kv_test.go +19 -22
| @@ -11,8 +11,17 @@ import ( | |||
| 11 | 11 | "path/filepath" | |
| 12 | 12 | "reflect" | |
| 13 | 13 | "testing" | |
| 14 | + | ||
| 15 | + | "github.com/bnema/vev/pkg/safedir" | |
| 14 | 16 | ) | |
| 15 | 17 | ||
| 18 | + | var compactionFaultPoints = []string{ | |
| 19 | + | "before-write-next", "after-write-next", "before-sync-next", "after-sync-next", | |
| 20 | + | "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev", | |
| 21 | + | "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current", | |
| 22 | + | "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir", | |
| 23 | + | } | |
| 24 | + | ||
| 16 | 25 | func privatePath(t *testing.T) string { | |
| 17 | 26 | t.Helper() | |
| 18 | 27 | return filepath.Join(t.TempDir(), "vev", "store.kv") | |
| @@ -56,7 +65,7 @@ func legacyFile(records ...[]byte) []byte { | |||
| 56 | 65 | ||
| 57 | 66 | func writeFile(t *testing.T, path string, data []byte) { | |
| 58 | 67 | t.Helper() | |
| 59 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 68 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { | |
| 60 | 69 | t.Fatal(err) | |
| 61 | 70 | } | |
| 62 | 71 | if err := os.WriteFile(path, data, 0o600); err != nil { | |
| @@ -173,7 +182,7 @@ func TestOpenTruncatesMidRecordTail(t *testing.T) { | |||
| 173 | 182 | path := privatePath(t) | |
| 174 | 183 | first, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | |
| 175 | 184 | second, _ := encodeRecord(opSet, []byte("b"), []byte("two")) | |
| 176 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 185 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { | |
| 177 | 186 | t.Fatal(err) | |
| 178 | 187 | } | |
| 179 | 188 | if err := os.WriteFile(path, currentFile(first, second[:len(second)/2]), 0o600); err != nil { | |
| @@ -206,7 +215,7 @@ func TestCRCcorruptionOfLastRecordFailsClosed(t *testing.T) { | |||
| 206 | 215 | second, _ := encodeRecord(opSet, []byte("k"), []byte("new")) | |
| 207 | 216 | buf := currentFile(first, second) | |
| 208 | 217 | buf[fileHeaderLen+len(first)+recordHeaderLen+payloadPrefixLen+1] ^= 0xff | |
| 209 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 218 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { | |
| 210 | 219 | t.Fatal(err) | |
| 211 | 220 | } | |
| 212 | 221 | if err := os.WriteFile(path, buf, 0o600); err != nil { | |
| @@ -311,7 +320,7 @@ func TestReplayStrict(t *testing.T) { | |||
| 311 | 320 | for _, tt := range tests { | |
| 312 | 321 | t.Run(tt.name, func(t *testing.T) { | |
| 313 | 322 | path := privatePath(t) | |
| 314 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 323 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { | |
| 315 | 324 | t.Fatal(err) | |
| 316 | 325 | } | |
| 317 | 326 | if err := os.WriteFile(path, tt.wal, 0o600); err != nil { | |
| @@ -361,7 +370,7 @@ func TestRecoverCompactionCleansStaleNextRegardlessOfPrev(t *testing.T) { | |||
| 361 | 370 | for _, tt := range tests { | |
| 362 | 371 | t.Run(tt.name, func(t *testing.T) { | |
| 363 | 372 | path := privatePath(t) | |
| 364 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 373 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { | |
| 365 | 374 | t.Fatal(err) | |
| 366 | 375 | } | |
| 367 | 376 | for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} { | |
| @@ -486,7 +495,7 @@ func TestCompactionRecoveryMatrix(t *testing.T) { | |||
| 486 | 495 | for _, tt := range tests { | |
| 487 | 496 | t.Run(tt.name, func(t *testing.T) { | |
| 488 | 497 | path := privatePath(t) | |
| 489 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 498 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { | |
| 490 | 499 | t.Fatal(err) | |
| 491 | 500 | } | |
| 492 | 501 | for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} { | |
| @@ -520,16 +529,10 @@ func TestCompactionRecoveryMatrix(t *testing.T) { | |||
| 520 | 529 | }) | |
| 521 | 530 | } | |
| 522 | 531 | ||
| 523 | - | faultPoints := []string{ | |
| 524 | - | "before-write-next", "after-write-next", "before-sync-next", "after-sync-next", | |
| 525 | - | "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev", | |
| 526 | - | "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current", | |
| 527 | - | "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir", | |
| 528 | - | } | |
| 529 | - | for _, point := range faultPoints { | |
| 532 | + | for _, point := range compactionFaultPoints { | |
| 530 | 533 | t.Run("fault-"+point, func(t *testing.T) { | |
| 531 | 534 | path := privatePath(t) | |
| 532 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 535 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { | |
| 533 | 536 | t.Fatal(err) | |
| 534 | 537 | } | |
| 535 | 538 | if err := os.WriteFile(path, oldFile, 0o600); err != nil { | |
| @@ -632,7 +635,7 @@ func TestCompactionFilesystemOperationFailuresPoisonAndRecover(t *testing.T) { | |||
| 632 | 635 | for _, tt := range tests { | |
| 633 | 636 | t.Run(tt.name, func(t *testing.T) { | |
| 634 | 637 | path := privatePath(t) | |
| 635 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 638 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { | |
| 636 | 639 | t.Fatal(err) | |
| 637 | 640 | } | |
| 638 | 641 | if err := os.WriteFile(path, oldFile, 0o600); err != nil { | |
| @@ -1083,13 +1086,7 @@ func TestLegacyUpgradeCrashSafety(t *testing.T) { | |||
| 1083 | 1086 | legacy := legacyFile(setA, setB) | |
| 1084 | 1087 | want := map[string]string{"a": "one", "b": "two"} | |
| 1085 | 1088 | ||
| 1086 | - | faultPoints := []string{ | |
| 1087 | - | "before-write-next", "after-write-next", "before-sync-next", "after-sync-next", | |
| 1088 | - | "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev", | |
| 1089 | - | "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current", | |
| 1090 | - | "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir", | |
| 1091 | - | } | |
| 1092 | - | for _, point := range faultPoints { | |
| 1089 | + | for _, point := range compactionFaultPoints { | |
| 1093 | 1090 | t.Run("fault-"+point, func(t *testing.T) { | |
| 1094 | 1091 | path := privatePath(t) | |
| 1095 | 1092 | writeFile(t, path, legacy) | |
| @@ -11,8 +11,17 @@ import ( | |||||
| 11 | "path/filepath" | 11 | "path/filepath" | ||
| 12 | "reflect" | 12 | "reflect" | ||
| 13 | "testing" | 13 | "testing" | ||
| 14 | + | ||||
| 15 | + | "github.com/bnema/vev/pkg/safedir" | |||
| 14 | ) | 16 | ) | ||
| 15 | 17 | ||||
| 18 | + | var compactionFaultPoints = []string{ | |||
| 19 | + | "before-write-next", "after-write-next", "before-sync-next", "after-sync-next", | |||
| 20 | + | "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev", | |||
| 21 | + | "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current", | |||
| 22 | + | "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir", | |||
| 23 | + | } | |||
| 24 | + | ||||
| 16 | func privatePath(t *testing.T) string { | 25 | func privatePath(t *testing.T) string { | ||
| 17 | t.Helper() | 26 | t.Helper() | ||
| 18 | return filepath.Join(t.TempDir(), "vev", "store.kv") | 27 | return filepath.Join(t.TempDir(), "vev", "store.kv") | ||
| @@ -56,7 +65,7 @@ func legacyFile(records ...[]byte) []byte { | |||||
| 56 | 65 | ||||
| 57 | func writeFile(t *testing.T, path string, data []byte) { | 66 | func writeFile(t *testing.T, path string, data []byte) { | ||
| 58 | t.Helper() | 67 | t.Helper() | ||
| 59 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 68 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { |
| 60 | t.Fatal(err) | 69 | t.Fatal(err) | ||
| 61 | } | 70 | } | ||
| 62 | if err := os.WriteFile(path, data, 0o600); err != nil { | 71 | if err := os.WriteFile(path, data, 0o600); err != nil { | ||
| @@ -173,7 +182,7 @@ func TestOpenTruncatesMidRecordTail(t *testing.T) { | |||||
| 173 | path := privatePath(t) | 182 | path := privatePath(t) | ||
| 174 | first, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | 183 | first, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | ||
| 175 | second, _ := encodeRecord(opSet, []byte("b"), []byte("two")) | 184 | second, _ := encodeRecord(opSet, []byte("b"), []byte("two")) | ||
| 176 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 185 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { |
| 177 | t.Fatal(err) | 186 | t.Fatal(err) | ||
| 178 | } | 187 | } | ||
| 179 | if err := os.WriteFile(path, currentFile(first, second[:len(second)/2]), 0o600); err != nil { | 188 | if err := os.WriteFile(path, currentFile(first, second[:len(second)/2]), 0o600); err != nil { | ||
| @@ -206,7 +215,7 @@ func TestCRCcorruptionOfLastRecordFailsClosed(t *testing.T) { | |||||
| 206 | second, _ := encodeRecord(opSet, []byte("k"), []byte("new")) | 215 | second, _ := encodeRecord(opSet, []byte("k"), []byte("new")) | ||
| 207 | buf := currentFile(first, second) | 216 | buf := currentFile(first, second) | ||
| 208 | buf[fileHeaderLen+len(first)+recordHeaderLen+payloadPrefixLen+1] ^= 0xff | 217 | buf[fileHeaderLen+len(first)+recordHeaderLen+payloadPrefixLen+1] ^= 0xff | ||
| 209 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 218 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { |
| 210 | t.Fatal(err) | 219 | t.Fatal(err) | ||
| 211 | } | 220 | } | ||
| 212 | if err := os.WriteFile(path, buf, 0o600); err != nil { | 221 | if err := os.WriteFile(path, buf, 0o600); err != nil { | ||
| @@ -311,7 +320,7 @@ func TestReplayStrict(t *testing.T) { | |||||
| 311 | for _, tt := range tests { | 320 | for _, tt := range tests { | ||
| 312 | t.Run(tt.name, func(t *testing.T) { | 321 | t.Run(tt.name, func(t *testing.T) { | ||
| 313 | path := privatePath(t) | 322 | path := privatePath(t) | ||
| 314 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 323 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { |
| 315 | t.Fatal(err) | 324 | t.Fatal(err) | ||
| 316 | } | 325 | } | ||
| 317 | if err := os.WriteFile(path, tt.wal, 0o600); err != nil { | 326 | if err := os.WriteFile(path, tt.wal, 0o600); err != nil { | ||
| @@ -361,7 +370,7 @@ func TestRecoverCompactionCleansStaleNextRegardlessOfPrev(t *testing.T) { | |||||
| 361 | for _, tt := range tests { | 370 | for _, tt := range tests { | ||
| 362 | t.Run(tt.name, func(t *testing.T) { | 371 | t.Run(tt.name, func(t *testing.T) { | ||
| 363 | path := privatePath(t) | 372 | path := privatePath(t) | ||
| 364 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 373 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { |
| 365 | t.Fatal(err) | 374 | t.Fatal(err) | ||
| 366 | } | 375 | } | ||
| 367 | for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} { | 376 | for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} { | ||
| @@ -486,7 +495,7 @@ func TestCompactionRecoveryMatrix(t *testing.T) { | |||||
| 486 | for _, tt := range tests { | 495 | for _, tt := range tests { | ||
| 487 | t.Run(tt.name, func(t *testing.T) { | 496 | t.Run(tt.name, func(t *testing.T) { | ||
| 488 | path := privatePath(t) | 497 | path := privatePath(t) | ||
| 489 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 498 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { |
| 490 | t.Fatal(err) | 499 | t.Fatal(err) | ||
| 491 | } | 500 | } | ||
| 492 | for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} { | 501 | for suffix, data := range map[string][]byte{"": tt.current, ".next": tt.next, ".prev": tt.prev} { | ||
| @@ -520,16 +529,10 @@ func TestCompactionRecoveryMatrix(t *testing.T) { | |||||
| 520 | }) | 529 | }) | ||
| 521 | } | 530 | } | ||
| 522 | 531 | ||||
| 523 | - | faultPoints := []string{ | 532 | + | for _, point := range compactionFaultPoints { |
| 524 | - | "before-write-next", "after-write-next", "before-sync-next", "after-sync-next", | |||
| 525 | - | "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev", | |||
| 526 | - | "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current", | |||
| 527 | - | "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir", | |||
| 528 | - | } | |||
| 529 | - | for _, point := range faultPoints { | |||
| 530 | t.Run("fault-"+point, func(t *testing.T) { | 533 | t.Run("fault-"+point, func(t *testing.T) { | ||
| 531 | path := privatePath(t) | 534 | path := privatePath(t) | ||
| 532 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 535 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { |
| 533 | t.Fatal(err) | 536 | t.Fatal(err) | ||
| 534 | } | 537 | } | ||
| 535 | if err := os.WriteFile(path, oldFile, 0o600); err != nil { | 538 | if err := os.WriteFile(path, oldFile, 0o600); err != nil { | ||
| @@ -632,7 +635,7 @@ func TestCompactionFilesystemOperationFailuresPoisonAndRecover(t *testing.T) { | |||||
| 632 | for _, tt := range tests { | 635 | for _, tt := range tests { | ||
| 633 | t.Run(tt.name, func(t *testing.T) { | 636 | t.Run(tt.name, func(t *testing.T) { | ||
| 634 | path := privatePath(t) | 637 | path := privatePath(t) | ||
| 635 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 638 | + | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { |
| 636 | t.Fatal(err) | 639 | t.Fatal(err) | ||
| 637 | } | 640 | } | ||
| 638 | if err := os.WriteFile(path, oldFile, 0o600); err != nil { | 641 | if err := os.WriteFile(path, oldFile, 0o600); err != nil { | ||
| @@ -1083,13 +1086,7 @@ func TestLegacyUpgradeCrashSafety(t *testing.T) { | |||||
| 1083 | legacy := legacyFile(setA, setB) | 1086 | legacy := legacyFile(setA, setB) | ||
| 1084 | want := map[string]string{"a": "one", "b": "two"} | 1087 | want := map[string]string{"a": "one", "b": "two"} | ||
| 1085 | 1088 | ||||
| 1086 | - | faultPoints := []string{ | 1089 | + | for _, point := range compactionFaultPoints { |
| 1087 | - | "before-write-next", "after-write-next", "before-sync-next", "after-sync-next", | |||
| 1088 | - | "after-sync-dir-next", "after-remove-stale-prev", "after-rename-current-prev", | |||
| 1089 | - | "after-sync-dir-prev", "after-rename-next-current", "after-sync-dir-current", | |||
| 1090 | - | "before-reopen-current", "after-reopen-current", "after-remove-prev", "after-final-sync-dir", | |||
| 1091 | - | } | |||
| 1092 | - | for _, point := range faultPoints { | |||
| 1093 | t.Run("fault-"+point, func(t *testing.T) { | 1090 | t.Run("fault-"+point, func(t *testing.T) { | ||
| 1094 | path := privatePath(t) | 1091 | path := privatePath(t) | ||
| 1095 | writeFile(t, path, legacy) | 1092 | writeFile(t, path, legacy) | ||