fix(recovery): harden durable session invariants
- commit
- 979dbfb6387143dee313544090da2238ce9508b9
- author
- brice · · 2026-07-26 07:25 UTC
- parent
- a6f1d55
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
80 files · +2738 -1260
M internal/adapters/noticefile/noticefile.go +2 -7
| @@ -311,13 +311,8 @@ func appendNoticeFragment(line, fragment []byte) ([]byte, bool) { | |||||
| 311 | } | 311 | } | ||
| 312 | required := len(line) + len(fragment) | 312 | required := len(line) + len(fragment) | ||
| 313 | if required > cap(line) { | 313 | if required > cap(line) { | ||
| 314 | - | capacity := cap(line) * 2 | 314 | + | capacity := max(cap(line)*2, noticeReadBufferSize) |
| 315 | - | if capacity < noticeReadBufferSize { | 315 | + | capacity = min(capacity, maxNoticeRecordSize) |
| 316 | - | capacity = noticeReadBufferSize | |||
| 317 | - | } | |||
| 318 | - | if capacity > maxNoticeRecordSize { | |||
| 319 | - | capacity = maxNoticeRecordSize | |||
| 320 | - | } | |||
| 321 | grown := make([]byte, len(line), capacity) | 316 | grown := make([]byte, len(line), capacity) | ||
| 322 | copy(grown, line) | 317 | copy(grown, line) | ||
| 323 | line = grown | 318 | line = grown | ||
M internal/adapters/recoveryfs/journal.go +26 -12
| @@ -104,16 +104,33 @@ func (j *Journal) ListDiscards(ctx context.Context) ([]domain.DiscardIntent, err | |||||
| 104 | if err != nil { | 104 | if err != nil { | ||
| 105 | return nil, err | 105 | return nil, err | ||
| 106 | } | 106 | } | ||
| 107 | - | entries, readErr := dir.ReadDir(maxDiscardIntents + 1) | 107 | + | entries := make([]os.DirEntry, 0, maxDiscardIntents) |
| 108 | - | closeErr := dir.Close() | 108 | + | for { |
| 109 | - | if readErr != nil && !errors.Is(readErr, io.EOF) { | 109 | + | batch, readErr := dir.ReadDir(maxDiscardIntents + 1) |
| 110 | - | return nil, readErr | 110 | + | for _, entry := range batch { |
| 111 | - | } | 111 | + | if strings.HasPrefix(entry.Name(), ".intent-") && !entry.IsDir() { |
| 112 | - | if closeErr != nil { | 112 | + | continue |
| 113 | - | return nil, closeErr | 113 | + | } |
| 114 | + | if entry.IsDir() || !strings.HasPrefix(entry.Name(), "discard-") { | |||
| 115 | + | _ = dir.Close() | |||
| 116 | + | return nil, errors.New("recovery journal: invalid transaction entry") | |||
| 117 | + | } | |||
| 118 | + | entries = append(entries, entry) | |||
| 119 | + | if len(entries) > maxDiscardIntents { | |||
| 120 | + | _ = dir.Close() | |||
| 121 | + | return nil, errors.New("recovery journal: transaction entry limit exceeded") | |||
| 122 | + | } | |||
| 123 | + | } | |||
| 124 | + | if errors.Is(readErr, io.EOF) { | |||
| 125 | + | break | |||
| 126 | + | } | |||
| 127 | + | if readErr != nil { | |||
| 128 | + | _ = dir.Close() | |||
| 129 | + | return nil, readErr | |||
| 130 | + | } | |||
| 114 | } | 131 | } | ||
| 115 | - | if len(entries) > maxDiscardIntents { | 132 | + | if err := dir.Close(); err != nil { |
| 116 | - | return nil, errors.New("recovery journal: transaction entry limit exceeded") | 133 | + | return nil, err |
| 117 | } | 134 | } | ||
| 118 | sort.Slice(entries, func(a, b int) bool { return entries[a].Name() < entries[b].Name() }) | 135 | sort.Slice(entries, func(a, b int) bool { return entries[a].Name() < entries[b].Name() }) | ||
| 119 | out := make([]domain.DiscardIntent, 0, len(entries)) | 136 | out := make([]domain.DiscardIntent, 0, len(entries)) | ||
| @@ -122,9 +139,6 @@ func (j *Journal) ListDiscards(ctx context.Context) ([]domain.DiscardIntent, err | |||||
| 122 | if err := ctx.Err(); err != nil { | 139 | if err := ctx.Err(); err != nil { | ||
| 123 | return nil, err | 140 | return nil, err | ||
| 124 | } | 141 | } | ||
| 125 | - | if entry.IsDir() || !strings.HasPrefix(entry.Name(), "discard-") { | |||
| 126 | - | return nil, errors.New("recovery journal: invalid transaction entry") | |||
| 127 | - | } | |||
| 128 | data, err := readFileBounded(filepath.Join(j.dir, entry.Name()), maxDiscardIntentSize) | 142 | data, err := readFileBounded(filepath.Join(j.dir, entry.Name()), maxDiscardIntentSize) | ||
| 129 | if err != nil { | 143 | if err != nil { | ||
| 130 | return nil, err | 144 | return nil, err | ||
M internal/adapters/recoveryfs/journal_test.go +13 -0
| @@ -38,6 +38,7 @@ func TestDiscardCrashMatrix(t *testing.T) { | |||||
| 38 | path := j.path(old) | 38 | path := j.path(old) | ||
| 39 | encoded, err := os.ReadFile(path) | 39 | encoded, err := os.ReadFile(path) | ||
| 40 | require.NoError(t, err) | 40 | require.NoError(t, err) | ||
| 41 | + | t.Cleanup(func() { require.NoError(t, os.WriteFile(path, encoded, 0o600)) }) | |||
| 41 | require.NoError(t, os.WriteFile(path, append(encoded, 0), 0o600)) | 42 | require.NoError(t, os.WriteFile(path, append(encoded, 0), 0o600)) | ||
| 42 | _, err = j.ListDiscards(ctx) | 43 | _, err = j.ListDiscards(ctx) | ||
| 43 | require.Error(t, err) | 44 | require.Error(t, err) | ||
| @@ -77,6 +78,18 @@ func TestRecoveryJournalRoundTripAndIdempotentDelete(t *testing.T) { | |||||
| 77 | require.Empty(t, got) | 78 | require.Empty(t, got) | ||
| 78 | } | 79 | } | ||
| 79 | 80 | ||||
| 81 | + | func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T) { | |||
| 82 | + | j := New(t.TempDir()) | |||
| 83 | + | require.NoError(t, os.MkdirAll(j.dir, 0o700)) | |||
| 84 | + | for i := 0; i < maxDiscardIntents+1; i++ { | |||
| 85 | + | require.NoError(t, os.WriteFile(filepath.Join(j.dir, ".intent-leftover-"+string(rune(0x1000+i))), nil, 0o600)) | |||
| 86 | + | } | |||
| 87 | + | ||||
| 88 | + | got, err := j.ListDiscards(context.Background()) | |||
| 89 | + | require.NoError(t, err) | |||
| 90 | + | require.Empty(t, got) | |||
| 91 | + | } | |||
| 92 | + | ||||
| 80 | func TestRecoveryJournalRejectsMalformedIntent(t *testing.T) { | 93 | func TestRecoveryJournalRejectsMalformedIntent(t *testing.T) { | ||
| 81 | j := New(t.TempDir()) | 94 | j := New(t.TempDir()) | ||
| 82 | require.Error(t, j.SaveDiscard(context.Background(), domain.DiscardIntent{})) | 95 | require.Error(t, j.SaveDiscard(context.Background(), domain.DiscardIntent{})) | ||
M internal/adapters/snapshot/atomic.go +4 -3
| @@ -13,15 +13,16 @@ import ( | |||||
| 13 | ) | 13 | ) | ||
| 14 | 14 | ||||
| 15 | func (r *Repository) ensureSession(id domain.IncarnationID) error { | 15 | func (r *Repository) ensureSession(id domain.IncarnationID) error { | ||
| 16 | + | sessionPath := r.sessionPath(id) | |||
| 16 | for _, directory := range []struct { | 17 | for _, directory := range []struct { | ||
| 17 | path string | 18 | path string | ||
| 18 | phase string | 19 | phase string | ||
| 19 | }{ | 20 | }{ | ||
| 20 | {r.dir, "repository"}, | 21 | {r.dir, "repository"}, | ||
| 21 | {filepath.Join(r.dir, repositorySessionsDir), "sessions"}, | 22 | {filepath.Join(r.dir, repositorySessionsDir), "sessions"}, | ||
| 22 | - | {r.sessionPath(id), "session"}, | 23 | + | {sessionPath, "session"}, |
| 23 | - | {filepath.Join(r.sessionPath(id), repositoryObjectsDir), "objects"}, | 24 | + | {filepath.Join(sessionPath, repositoryObjectsDir), "objects"}, |
| 24 | - | {filepath.Join(r.sessionPath(id), repositoryGenerations), "generations"}, | 25 | + | {filepath.Join(sessionPath, repositoryGenerations), "generations"}, |
| 25 | } { | 26 | } { | ||
| 26 | if err := r.ensurePrivateDirectoryPhase(directory.path, directory.phase); err != nil { | 27 | if err := r.ensurePrivateDirectoryPhase(directory.path, directory.phase); err != nil { | ||
| 27 | return fmt.Errorf("create snapshot repository directory: %w", err) | 28 | return fmt.Errorf("create snapshot repository directory: %w", err) | ||
M internal/adapters/snapshot/legacy_load_test.go +0 -4
| @@ -69,7 +69,6 @@ func (r *Repository) Load(ctx context.Context, name string) (ports.SnapshotGener | |||||
| 69 | 69 | ||||
| 70 | func (r *Repository) load(ctx context.Context, name, key string) (ports.SnapshotGeneration, error) { | 70 | func (r *Repository) load(ctx context.Context, name, key string) (ports.SnapshotGeneration, error) { | ||
| 71 | preferred, preferredDigest, err := r.readLegacyHead(key) | 71 | preferred, preferredDigest, err := r.readLegacyHead(key) | ||
| 72 | - | skipped := errors.Is(err, ErrInvalidHEAD) | |||
| 73 | if err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, ErrInvalidHEAD) { | 72 | if err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, ErrInvalidHEAD) { | ||
| 74 | return ports.SnapshotGeneration{}, fmt.Errorf("read snapshot HEAD: %w", err) | 73 | return ports.SnapshotGeneration{}, fmt.Errorf("read snapshot HEAD: %w", err) | ||
| 75 | } | 74 | } | ||
| @@ -78,7 +77,6 @@ func (r *Repository) load(ctx context.Context, name, key string) (ports.Snapshot | |||||
| 78 | if err == nil && sha256.Sum256(got.Manifest) == preferredDigest { | 77 | if err == nil && sha256.Sum256(got.Manifest) == preferredDigest { | ||
| 79 | return got, nil | 78 | return got, nil | ||
| 80 | } | 79 | } | ||
| 81 | - | skipped = true | |||
| 82 | } | 80 | } | ||
| 83 | 81 | ||||
| 84 | budget := maxDirectoryTraversalEntries | 82 | budget := maxDirectoryTraversalEntries | ||
| @@ -89,10 +87,8 @@ func (r *Repository) load(ctx context.Context, name, key string) (ports.Snapshot | |||||
| 89 | for _, generation := range candidates { | 87 | for _, generation := range candidates { | ||
| 90 | got, err := r.loadGeneration(ctx, name, key, generation) | 88 | got, err := r.loadGeneration(ctx, name, key, generation) | ||
| 91 | if err != nil { | 89 | if err != nil { | ||
| 92 | - | skipped = true | |||
| 93 | continue | 90 | continue | ||
| 94 | } | 91 | } | ||
| 95 | - | _ = skipped | |||
| 96 | return got, nil | 92 | return got, nil | ||
| 97 | } | 93 | } | ||
| 98 | return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name) | 94 | return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name) | ||
M internal/adapters/snapshot/load_bounds_test.go +10 -7
| @@ -15,7 +15,7 @@ func TestRepositoryListStopsAtTraversalBudget(t *testing.T) { | |||||
| 15 | if err := os.MkdirAll(sessions, 0o700); err != nil { | 15 | if err := os.MkdirAll(sessions, 0o700); err != nil { | ||
| 16 | t.Fatal(err) | 16 | t.Fatal(err) | ||
| 17 | } | 17 | } | ||
| 18 | - | for i := 0; i < maxDirectoryTraversalEntries+1; i++ { | 18 | + | for i := range maxDirectoryTraversalEntries + 1 { |
| 19 | if err := os.Mkdir(filepath.Join(sessions, fmt.Sprintf("hostile-%05d", i)), 0o700); err != nil { | 19 | if err := os.Mkdir(filepath.Join(sessions, fmt.Sprintf("hostile-%05d", i)), 0o700); err != nil { | ||
| 20 | t.Fatal(err) | 20 | t.Fatal(err) | ||
| 21 | } | 21 | } | ||
| @@ -34,7 +34,8 @@ func TestRepositoryLoadStopsAtGenerationTraversalBudget(t *testing.T) { | |||||
| 34 | if err := os.MkdirAll(generations, 0o700); err != nil { | 34 | if err := os.MkdirAll(generations, 0o700); err != nil { | ||
| 35 | t.Fatal(err) | 35 | t.Fatal(err) | ||
| 36 | } | 36 | } | ||
| 37 | - | for generation := uint64(1); generation <= uint64(maxDirectoryTraversalEntries+1); generation++ { | 37 | + | for i := range maxDirectoryTraversalEntries + 1 { |
| 38 | + | generation := uint64(i + 1) | |||
| 38 | if err := os.WriteFile(filepath.Join(generations, generationFilename(generation)), nil, 0o600); err != nil { | 39 | if err := os.WriteFile(filepath.Join(generations, generationFilename(generation)), nil, 0o600); err != nil { | ||
| 39 | t.Fatal(err) | 40 | t.Fatal(err) | ||
| 40 | } | 41 | } | ||
| @@ -48,8 +49,9 @@ func TestRepositoryLoadStopsAtGenerationTraversalBudget(t *testing.T) { | |||||
| 48 | 49 | ||||
| 49 | func TestRepositoryLoadFallsBackNewestToOldestWithoutGenerationEnumeration(t *testing.T) { | 50 | func TestRepositoryLoadFallsBackNewestToOldestWithoutGenerationEnumeration(t *testing.T) { | ||
| 50 | repo := NewRepository(privateDir(t)) | 51 | repo := NewRepository(privateDir(t)) | ||
| 51 | - | for generation := uint64(1); generation <= 3; generation++ { | 52 | + | for i := range 3 { |
| 52 | - | if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation)))); err != nil { | 53 | + | generation := uint64(i + 1) |
| 54 | + | if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))); err != nil { | |||
| 53 | t.Fatal(err) | 55 | t.Fatal(err) | ||
| 54 | } | 56 | } | ||
| 55 | } | 57 | } | ||
| @@ -70,12 +72,13 @@ func TestRepositoryLoadFallsBackNewestToOldestWithoutGenerationEnumeration(t *te | |||||
| 70 | 72 | ||||
| 71 | func TestRepositoryMaintainDoesNotCollectBeyondRetainedMetadataBudget(t *testing.T) { | 73 | func TestRepositoryMaintainDoesNotCollectBeyondRetainedMetadataBudget(t *testing.T) { | ||
| 72 | repo := NewRepository(privateDir(t)) | 74 | repo := NewRepository(privateDir(t)) | ||
| 73 | - | for generation := uint64(1); generation <= uint64(maxMaintenanceMarkedGenerations+1); generation++ { | 75 | + | for i := range maxMaintenanceMarkedGenerations + 1 { |
| 74 | - | if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation)))); err != nil { | 76 | + | generation := uint64(i + 1) |
| 77 | + | if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))); err != nil { | |||
| 75 | t.Fatal(err) | 78 | t.Fatal(err) | ||
| 76 | } | 79 | } | ||
| 77 | } | 80 | } | ||
| 78 | - | for pass := 0; pass < maxMaintenanceMarkedGenerations*3; pass++ { | 81 | + | for pass := range maxMaintenanceMarkedGenerations * 3 { |
| 79 | if err := repo.Maintain(context.Background()); err != nil { | 82 | if err := repo.Maintain(context.Background()); err != nil { | ||
| 80 | t.Fatal(err) | 83 | t.Fatal(err) | ||
| 81 | } | 84 | } | ||
M internal/adapters/snapshot/maintenance_delete.go +1 -22
| @@ -644,28 +644,7 @@ func (r *Repository) SaveQuarantineDescriptor(ctx context.Context, descriptor do | |||||
| 644 | 644 | ||||
| 645 | func quarantineDescriptorsEqual(a, b domain.QuarantineDescriptor) bool { | 645 | func quarantineDescriptorsEqual(a, b domain.QuarantineDescriptor) bool { | ||
| 646 | return a.OldIncarnation == b.OldIncarnation && a.ReplacementIncarnation == b.ReplacementIncarnation && | 646 | return a.OldIncarnation == b.OldIncarnation && a.ReplacementIncarnation == b.ReplacementIncarnation && | ||
| 647 | - | a.SessionName == b.SessionName && a.Reason == b.Reason && catalogueRecordsEqual(a.OldRecord, b.OldRecord) | 647 | + | a.SessionName == b.SessionName && a.Reason == b.Reason && a.OldRecord.Equal(b.OldRecord) |
| 648 | - | } | |||
| 649 | - | ||||
| 650 | - | func catalogueRecordsEqual(a, b domain.CatalogueRecord) bool { | |||
| 651 | - | if a.Name != b.Name || a.IncarnationID != b.IncarnationID || a.Cwd != b.Cwd || a.CreatedAt != b.CreatedAt || | |||
| 652 | - | a.UpdatedAt != b.UpdatedAt || a.LastUsedSeq != b.LastUsedSeq || a.RecoveryState != b.RecoveryState || | |||
| 653 | - | a.DegradedReason != b.DegradedReason || len(a.TabNames) != len(b.TabNames) { | |||
| 654 | - | return false | |||
| 655 | - | } | |||
| 656 | - | for i := range a.TabNames { | |||
| 657 | - | if a.TabNames[i] != b.TabNames[i] { | |||
| 658 | - | return false | |||
| 659 | - | } | |||
| 660 | - | } | |||
| 661 | - | return checkpointRefsEqual(a.Committed, b.Committed) && checkpointRefsEqual(a.Fallbacks[0], b.Fallbacks[0]) && checkpointRefsEqual(a.Fallbacks[1], b.Fallbacks[1]) | |||
| 662 | - | } | |||
| 663 | - | ||||
| 664 | - | func checkpointRefsEqual(a, b *domain.CheckpointRef) bool { | |||
| 665 | - | if a == nil || b == nil { | |||
| 666 | - | return a == nil && b == nil | |||
| 667 | - | } | |||
| 668 | - | return *a == *b | |||
| 669 | } | 648 | } | ||
| 670 | 649 | ||||
| 671 | func (r *Repository) QuarantineIncarnation(ctx context.Context, id domain.IncarnationID) error { | 650 | func (r *Repository) QuarantineIncarnation(ctx context.Context, id domain.IncarnationID) error { | ||
M internal/adapters/snapshot/maintenance_gc.go +88 -61
| @@ -31,6 +31,37 @@ type manifestMaintenance struct { | |||||
| 31 | complete bool | 31 | complete bool | ||
| 32 | } | 32 | } | ||
| 33 | 33 | ||||
| 34 | + | type retentionEntryKind uint8 | |||
| 35 | + | ||||
| 36 | + | const ( | |||
| 37 | + | retentionManifest retentionEntryKind = iota + 1 | |||
| 38 | + | retentionObject | |||
| 39 | + | ) | |||
| 40 | + | ||||
| 41 | + | type retentionEntryPolicy struct { | |||
| 42 | + | kind retentionEntryKind | |||
| 43 | + | generations map[uint64]ports.SnapshotDigest | |||
| 44 | + | objects map[ports.SnapshotDigest]struct{} | |||
| 45 | + | } | |||
| 46 | + | ||||
| 47 | + | func (p retentionEntryPolicy) delete(entry maintenanceDirEntry) bool { | |||
| 48 | + | if entry.isDir { | |||
| 49 | + | return false | |||
| 50 | + | } | |||
| 51 | + | switch p.kind { | |||
| 52 | + | case retentionManifest: | |||
| 53 | + | generation, canonical := parseGenerationFilename(entry.name) | |||
| 54 | + | _, retained := p.generations[generation] | |||
| 55 | + | return canonical && !retained | |||
| 56 | + | case retentionObject: | |||
| 57 | + | digest, canonical := parseObjectDigest(entry.name) | |||
| 58 | + | _, retained := p.objects[digest] | |||
| 59 | + | return canonical && !retained | |||
| 60 | + | default: | |||
| 61 | + | return false | |||
| 62 | + | } | |||
| 63 | + | } | |||
| 64 | + | ||||
| 34 | type sessionMaintenance struct { | 65 | type sessionMaintenance struct { | ||
| 35 | lock *sessionMutex | 66 | lock *sessionMutex | ||
| 36 | token string | 67 | token string | ||
| @@ -370,6 +401,44 @@ func keepSet(plan ports.RetentionPlan) map[uint64]ports.SnapshotDigest { | |||||
| 370 | 401 | ||||
| 371 | // MaintainSession incrementally validates and retains only catalogue-indexed | 402 | // MaintainSession incrementally validates and retains only catalogue-indexed | ||
| 372 | // checkpoints. Every incarnation owns its budget and continuation state. | 403 | // checkpoints. Every incarnation owns its budget and continuation state. | ||
| 404 | + | // processRetentionEntries charges stat-reported size for every entry, requeues | |||
| 405 | + | // the unadmitted suffix, and removes only canonical entries rejected by policy. | |||
| 406 | + | func (r *Repository) processRetentionEntries(ctx context.Context, dir, purpose string, entries []maintenanceDirEntry, budget *ports.MaintenanceBudget, consumedBefore bool, policy retentionEntryPolicy) (bool, error) { | |||
| 407 | + | for i, entry := range entries { | |||
| 408 | + | if err := ctx.Err(); err != nil { | |||
| 409 | + | return false, err | |||
| 410 | + | } | |||
| 411 | + | path := filepath.Join(dir, entry.name) | |||
| 412 | + | info, statErr := r.stat(path) | |||
| 413 | + | if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { | |||
| 414 | + | return false, statErr | |||
| 415 | + | } | |||
| 416 | + | size := uint64(0) | |||
| 417 | + | if statErr == nil && info.Size() > 0 { | |||
| 418 | + | size = uint64(info.Size()) | |||
| 419 | + | } | |||
| 420 | + | if budget.Entries == 0 || size > budget.Bytes { | |||
| 421 | + | r.requeueMaintenanceEntries(dir, purpose, entries[i:]) | |||
| 422 | + | if !consumedBefore && i == 0 { | |||
| 423 | + | return false, ErrMaintenanceBudgetTooSmall | |||
| 424 | + | } | |||
| 425 | + | return false, nil | |||
| 426 | + | } | |||
| 427 | + | budget.Entries-- | |||
| 428 | + | budget.Bytes -= size | |||
| 429 | + | if !policy.delete(entry) { | |||
| 430 | + | continue | |||
| 431 | + | } | |||
| 432 | + | if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { | |||
| 433 | + | return false, err | |||
| 434 | + | } | |||
| 435 | + | if err := r.syncDirectory(dir); err != nil { | |||
| 436 | + | return false, err | |||
| 437 | + | } | |||
| 438 | + | } | |||
| 439 | + | return true, nil | |||
| 440 | + | } | |||
| 441 | + | ||||
| 373 | func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPlan, budget ports.MaintenanceBudget) (done bool, err error) { | 442 | func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPlan, budget ports.MaintenanceBudget) (done bool, err error) { | ||
| 374 | requested := budget | 443 | requested := budget | ||
| 375 | if err := ctx.Err(); err != nil { | 444 | if err := ctx.Err(); err != nil { | ||
| @@ -459,34 +528,14 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||||
| 459 | if err != nil { | 528 | if err != nil { | ||
| 460 | return false, err | 529 | return false, err | ||
| 461 | } | 530 | } | ||
| 462 | - | for i, entry := range entries { | 531 | + | dir := filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations) |
| 463 | - | path := r.manifestPath(plan.IncarnationID, 0) | 532 | + | consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes |
| 464 | - | path = filepath.Join(filepath.Dir(path), entry.name) | 533 | + | admitted, err := r.processRetentionEntries(ctx, dir, purpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionManifest, generations: keep}) |
| 465 | - | info, statErr := r.stat(path) | 534 | + | if err != nil { |
| 466 | - | if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { | 535 | + | return false, err |
| 467 | - | return false, statErr | 536 | + | } |
| 468 | - | } | 537 | + | if !admitted { |
| 469 | - | size := uint64(0) | 538 | + | return false, nil |
| 470 | - | if statErr == nil && info.Size() > 0 { | |||
| 471 | - | size = uint64(info.Size()) | |||
| 472 | - | } | |||
| 473 | - | if budget.Entries == 0 || size > budget.Bytes { | |||
| 474 | - | r.requeueMaintenanceEntries(filepath.Dir(path), purpose, entries[i:]) | |||
| 475 | - | return false, nil | |||
| 476 | - | } | |||
| 477 | - | budget.Entries-- | |||
| 478 | - | budget.Bytes -= size | |||
| 479 | - | generation, canonical := parseGenerationFilename(entry.name) | |||
| 480 | - | if !entry.isDir && canonical { | |||
| 481 | - | if _, retained := keep[generation]; !retained { | |||
| 482 | - | if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { | |||
| 483 | - | return false, err | |||
| 484 | - | } | |||
| 485 | - | if err := r.syncDirectory(filepath.Dir(path)); err != nil { | |||
| 486 | - | return false, err | |||
| 487 | - | } | |||
| 488 | - | } | |||
| 489 | - | } | |||
| 490 | } | 539 | } | ||
| 491 | state.manifestsDone = done | 540 | state.manifestsDone = done | ||
| 492 | if len(entries) == 0 && !done { | 541 | if len(entries) == 0 && !done { | ||
| @@ -511,33 +560,13 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||||
| 511 | if err != nil { | 560 | if err != nil { | ||
| 512 | return false, err | 561 | return false, err | ||
| 513 | } | 562 | } | ||
| 514 | - | for i, entry := range entries { | 563 | + | consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes |
| 515 | - | path := filepath.Join(dir, entry.name) | 564 | + | admitted, err := r.processRetentionEntries(ctx, dir, objectPurpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionObject, objects: state.references}) |
| 516 | - | info, statErr := r.stat(path) | 565 | + | if err != nil { |
| 517 | - | if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { | 566 | + | return false, err |
| 518 | - | return false, statErr | 567 | + | } |
| 519 | - | } | 568 | + | if !admitted { |
| 520 | - | size := uint64(0) | 569 | + | return false, nil |
| 521 | - | if statErr == nil && info.Size() > 0 { | |||
| 522 | - | size = uint64(info.Size()) | |||
| 523 | - | } | |||
| 524 | - | if budget.Entries == 0 || size > budget.Bytes { | |||
| 525 | - | r.requeueMaintenanceEntries(dir, objectPurpose, entries[i:]) | |||
| 526 | - | return false, nil | |||
| 527 | - | } | |||
| 528 | - | budget.Entries-- | |||
| 529 | - | budget.Bytes -= size | |||
| 530 | - | digest, canonical := parseObjectDigest(entry.name) | |||
| 531 | - | if !entry.isDir && canonical { | |||
| 532 | - | if _, retained := state.references[digest]; !retained { | |||
| 533 | - | if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { | |||
| 534 | - | return false, err | |||
| 535 | - | } | |||
| 536 | - | if err := r.syncDirectory(dir); err != nil { | |||
| 537 | - | return false, err | |||
| 538 | - | } | |||
| 539 | - | } | |||
| 540 | - | } | |||
| 541 | } | 570 | } | ||
| 542 | if done { | 571 | if done { | ||
| 543 | state.objectShard = "" | 572 | state.objectShard = "" | ||
| @@ -587,15 +616,15 @@ func retentionToken(plan ports.RetentionPlan) string { | |||||
| 587 | return b.String() | 616 | return b.String() | ||
| 588 | } | 617 | } | ||
| 589 | 618 | ||||
| 590 | - | // validateMaintenanceCheckpoint admits every payload by stat-reported size | |||
| 591 | - | // before reading it. It validates directly instead of calling LoadCheckpoint, | |||
| 592 | - | // which would materialize a complete generation before maintenance could | |||
| 593 | - | // enforce its byte budget. | |||
| 594 | type validatedMaintenanceCheckpoint struct { | 619 | type validatedMaintenanceCheckpoint struct { | ||
| 595 | refs map[ports.SnapshotDigest]codec.ObjectRef | 620 | refs map[ports.SnapshotDigest]codec.ObjectRef | ||
| 596 | parent *domain.CheckpointRef | 621 | parent *domain.CheckpointRef | ||
| 597 | } | 622 | } | ||
| 598 | 623 | ||||
| 624 | + | // validateMaintenanceCheckpoint admits every payload by stat-reported size | |||
| 625 | + | // before reading it. It validates directly instead of calling LoadCheckpoint, | |||
| 626 | + | // which would materialize a complete generation before maintenance could | |||
| 627 | + | // enforce its byte budget. | |||
| 599 | func (r *Repository) validateMaintenanceCheckpoint(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef, byteBudget uint64) (validatedMaintenanceCheckpoint, uint64, bool, error) { | 628 | func (r *Repository) validateMaintenanceCheckpoint(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef, byteBudget uint64) (validatedMaintenanceCheckpoint, uint64, bool, error) { | ||
| 600 | manifestPath := r.manifestPath(id, ref.Generation) | 629 | manifestPath := r.manifestPath(id, ref.Generation) | ||
| 601 | info, err := r.stat(manifestPath) | 630 | info, err := r.stat(manifestPath) | ||
| @@ -692,8 +721,6 @@ func (r *Repository) Reconcile(ctx context.Context, records []domain.CatalogueRe | |||||
| 692 | byIncarnation[record.IncarnationID] = record | 721 | byIncarnation[record.IncarnationID] = record | ||
| 693 | } | 722 | } | ||
| 694 | 723 | ||||
| 695 | - | r.maintenanceMu.Lock() | |||
| 696 | - | defer r.maintenanceMu.Unlock() | |||
| 697 | root := filepath.Join(r.dir, repositorySessionsDir) | 724 | root := filepath.Join(r.dir, repositorySessionsDir) | ||
| 698 | state := &maintenanceCursor{offset: offset} | 725 | state := &maintenanceCursor{offset: offset} | ||
| 699 | findingCapacity := maintenanceBatch | 726 | findingCapacity := maintenanceBatch | ||
M internal/adapters/snapshot/maintenance_test.go +88 -16
| @@ -1,6 +1,7 @@ | |||||
| 1 | package snapshot | 1 | package snapshot | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | + | "cmp" | |||
| 4 | "context" | 5 | "context" | ||
| 5 | "crypto/sha256" | 6 | "crypto/sha256" | ||
| 6 | "encoding/binary" | 7 | "encoding/binary" | ||
| @@ -11,8 +12,10 @@ import ( | |||||
| 11 | "path/filepath" | 12 | "path/filepath" | ||
| 12 | "slices" | 13 | "slices" | ||
| 13 | "strings" | 14 | "strings" | ||
| 15 | + | "sync" | |||
| 14 | "syscall" | 16 | "syscall" | ||
| 15 | "testing" | 17 | "testing" | ||
| 18 | + | "time" | |||
| 16 | "unsafe" | 19 | "unsafe" | ||
| 17 | 20 | ||||
| 18 | "github.com/bnema/vev/internal/domain" | 21 | "github.com/bnema/vev/internal/domain" | ||
| @@ -1043,21 +1046,17 @@ func TestRetentionSlots(t *testing.T) { | |||||
| 1043 | } | 1046 | } | ||
| 1044 | 1047 | ||||
| 1045 | func TestUnresolvedDataPinned(t *testing.T) { | 1048 | func TestUnresolvedDataPinned(t *testing.T) { | ||
| 1046 | - | for _, state := range []string{"degraded", "deleting", "migration", "transaction"} { | 1049 | + | repo := NewRepository(privateDir(t)) |
| 1047 | - | t.Run(state, func(t *testing.T) { | 1050 | + | publications := publishMaintenanceGenerations(t, repo, "unresolved", 4) |
| 1048 | - | repo := NewRepository(privateDir(t)) | 1051 | + | done, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: publications[0].IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) |
| 1049 | - | publications := publishMaintenanceGenerations(t, repo, state, 4) | 1052 | + | if err != nil { |
| 1050 | - | done, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: publications[0].IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | 1053 | + | t.Fatal(err) |
| 1051 | - | if err != nil { | 1054 | + | } |
| 1052 | - | t.Fatal(err) | 1055 | + | if !done { |
| 1053 | - | } | 1056 | + | t.Fatal("pinned maintenance did not complete without traversing") |
| 1054 | - | if !done { | 1057 | + | } |
| 1055 | - | t.Fatal("pinned maintenance did not complete without traversing") | 1058 | + | if got := remainingMaintenanceGenerations(t, repo, publications[0].IncarnationID); !slices.Equal(got, []uint64{4, 3, 2, 1}) { |
| 1056 | - | } | 1059 | + | t.Fatalf("pinned generations = %v", got) |
| 1057 | - | if got := remainingMaintenanceGenerations(t, repo, publications[0].IncarnationID); !slices.Equal(got, []uint64{4, 3, 2, 1}) { | |||
| 1058 | - | t.Fatalf("pinned generations = %v", got) | |||
| 1059 | - | } | |||
| 1060 | - | }) | |||
| 1061 | } | 1060 | } | ||
| 1062 | } | 1061 | } | ||
| 1063 | 1062 | ||||
| @@ -1094,6 +1093,79 @@ func TestMaintenanceBudgetAdmitsPayloadsBeforeRead(t *testing.T) { | |||||
| 1094 | } | 1093 | } | ||
| 1095 | } | 1094 | } | ||
| 1096 | 1095 | ||||
| 1096 | + | func TestMaintainSessionDistinguishesInitialOversizeFromResumableExhaustion(t *testing.T) { | |||
| 1097 | + | repo := NewRepository(privateDir(t)) | |||
| 1098 | + | publications := publishMaintenanceGenerations(t, repo, "budget-semantics", 2) | |||
| 1099 | + | plan := ports.RetentionPlan{IncarnationID: publications[0].IncarnationID} | |||
| 1100 | + | firstInfo, err := os.Stat(repo.manifestPath(plan.IncarnationID, 1)) | |||
| 1101 | + | if err != nil { | |||
| 1102 | + | t.Fatal(err) | |||
| 1103 | + | } | |||
| 1104 | + | secondInfo, err := os.Stat(repo.manifestPath(plan.IncarnationID, 2)) | |||
| 1105 | + | if err != nil { | |||
| 1106 | + | t.Fatal(err) | |||
| 1107 | + | } | |||
| 1108 | + | firstSize, secondSize := uint64(firstInfo.Size()), uint64(secondInfo.Size()) | |||
| 1109 | + | ||||
| 1110 | + | done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 2, Bytes: min(firstSize, secondSize) - 1}) | |||
| 1111 | + | if done || !errors.Is(err, ErrMaintenanceBudgetTooSmall) { | |||
| 1112 | + | t.Fatalf("initial oversized item: done=%v err=%v", done, err) | |||
| 1113 | + | } | |||
| 1114 | + | ||||
| 1115 | + | done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 2, Bytes: firstSize + secondSize - 1}) | |||
| 1116 | + | if err != nil || done { | |||
| 1117 | + | t.Fatalf("mid-pass exhaustion must be resumable: done=%v err=%v", done, err) | |||
| 1118 | + | } | |||
| 1119 | + | done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20}) | |||
| 1120 | + | if err != nil || !done { | |||
| 1121 | + | t.Fatalf("resumed maintenance: done=%v err=%v", done, err) | |||
| 1122 | + | } | |||
| 1123 | + | } | |||
| 1124 | + | ||||
| 1125 | + | func TestReconcileDoesNotHoldMaintenanceLockDuringPayloadRead(t *testing.T) { | |||
| 1126 | + | repo := NewRepository(privateDir(t)) | |||
| 1127 | + | publications := publishMaintenanceGenerations(t, repo, "reconcile-lock", 2) | |||
| 1128 | + | committed := checkpointRefForPublication(publications[0]) | |||
| 1129 | + | record := domain.CatalogueRecord{Name: "reconcile-lock", IncarnationID: publications[0].IncarnationID, RecoveryState: domain.RecoveryHealthy, Committed: &committed} | |||
| 1130 | + | ||||
| 1131 | + | entered := make(chan struct{}) | |||
| 1132 | + | release := make(chan struct{}) | |||
| 1133 | + | var releaseOnce sync.Once | |||
| 1134 | + | defer releaseOnce.Do(func() { close(release) }) | |||
| 1135 | + | repo.hooks.beforeMaintenancePayloadRead = func(string) { | |||
| 1136 | + | select { | |||
| 1137 | + | case <-entered: | |||
| 1138 | + | default: | |||
| 1139 | + | close(entered) | |||
| 1140 | + | } | |||
| 1141 | + | <-release | |||
| 1142 | + | } | |||
| 1143 | + | reconcileDone := make(chan error, 1) | |||
| 1144 | + | go func() { | |||
| 1145 | + | _, _, err := repo.Reconcile(context.Background(), []domain.CatalogueRecord{record}, ports.ReconcileCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20}) | |||
| 1146 | + | reconcileDone <- err | |||
| 1147 | + | }() | |||
| 1148 | + | <-entered | |||
| 1149 | + | ||||
| 1150 | + | pinDone := make(chan error, 1) | |||
| 1151 | + | go func() { | |||
| 1152 | + | _, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | |||
| 1153 | + | pinDone <- err | |||
| 1154 | + | }() | |||
| 1155 | + | select { | |||
| 1156 | + | case err := <-pinDone: | |||
| 1157 | + | if err != nil { | |||
| 1158 | + | t.Fatal(err) | |||
| 1159 | + | } | |||
| 1160 | + | case <-time.After(time.Second): | |||
| 1161 | + | t.Fatal("maintenance lock remained held during reconciliation payload I/O") | |||
| 1162 | + | } | |||
| 1163 | + | releaseOnce.Do(func() { close(release) }) | |||
| 1164 | + | if err := <-reconcileDone; err != nil { | |||
| 1165 | + | t.Fatal(err) | |||
| 1166 | + | } | |||
| 1167 | + | } | |||
| 1168 | + | ||||
| 1097 | func TestPerSessionBudgetIsolation(t *testing.T) { | 1169 | func TestPerSessionBudgetIsolation(t *testing.T) { | ||
| 1098 | repo := NewRepository(privateDir(t)) | 1170 | repo := NewRepository(privateDir(t)) | ||
| 1099 | large := publishMaintenanceGenerations(t, repo, "large", 4) | 1171 | large := publishMaintenanceGenerations(t, repo, "large", 4) | ||
| @@ -1149,6 +1221,6 @@ func remainingMaintenanceGenerations(t *testing.T, repo *Repository, id domain.I | |||||
| 1149 | generations = append(generations, generation) | 1221 | generations = append(generations, generation) | ||
| 1150 | } | 1222 | } | ||
| 1151 | } | 1223 | } | ||
| 1152 | - | slices.SortFunc(generations, func(a, b uint64) int { return int(b) - int(a) }) | 1224 | + | slices.SortFunc(generations, func(a, b uint64) int { return cmp.Compare(b, a) }) |
| 1153 | return generations | 1225 | return generations | ||
| 1154 | } | 1226 | } | ||
M internal/adapters/snapshot/migration_v0.go +6 -3
| @@ -49,7 +49,7 @@ func decodeManifestV1(data []byte) (legacyManifestV1, error) { | |||||
| 49 | objects := make([]legacyObjectV1, 0) | 49 | objects := make([]legacyObjectV1, 0) | ||
| 50 | add := func(ref codec.ObjectRef) error { | 50 | add := func(ref codec.ObjectRef) error { | ||
| 51 | if _, ok := seen[ref.Digest]; ok { | 51 | if _, ok := seen[ref.Digest]; ok { | ||
| 52 | - | return errors.New("snapshot: duplicate legacy object digest") | 52 | + | return nil |
| 53 | } | 53 | } | ||
| 54 | seen[ref.Digest] = struct{}{} | 54 | seen[ref.Digest] = struct{}{} | ||
| 55 | objects = append(objects, legacyObjectV1{Kind: uint8(ref.Kind), Digest: ref.Digest, Size: uint64(ref.Size)}) | 55 | objects = append(objects, legacyObjectV1{Kind: uint8(ref.Kind), Digest: ref.Digest, Size: uint64(ref.Size)}) | ||
| @@ -209,7 +209,7 @@ func uncertainLegacyError(operation string, cause error) error { | |||||
| 209 | if cause == nil { | 209 | if cause == nil { | ||
| 210 | return fmt.Errorf("%w: %s", ports.ErrLegacySnapshotUncertain, operation) | 210 | return fmt.Errorf("%w: %s", ports.ErrLegacySnapshotUncertain, operation) | ||
| 211 | } | 211 | } | ||
| 212 | - | return fmt.Errorf("%w: %s: %v", ports.ErrLegacySnapshotUncertain, operation, cause) | 212 | + | return fmt.Errorf("%w: %s: %w", ports.ErrLegacySnapshotUncertain, operation, cause) |
| 213 | } | 213 | } | ||
| 214 | 214 | ||||
| 215 | // MigrateV1Checkpoint is the migration-only admission path for an authoritative | 215 | // MigrateV1Checkpoint is the migration-only admission path for an authoritative | ||
| @@ -244,7 +244,10 @@ func (r *Repository) MigrateV1Checkpoint(ctx context.Context, req ports.Snapshot | |||||
| 244 | if manifest.Name != req.LegacyName || manifest.Generation != req.LegacyRef.Generation { | 244 | if manifest.Name != req.LegacyName || manifest.Generation != req.LegacyRef.Generation { | ||
| 245 | return domain.CheckpointRef{}, uncertainLegacyError("manifest identity mismatch", nil) | 245 | return domain.CheckpointRef{}, uncertainLegacyError("manifest identity mismatch", nil) | ||
| 246 | } | 246 | } | ||
| 247 | - | key, _ := incarnationKey(req.IncarnationID) | 247 | + | key, err := incarnationKey(req.IncarnationID) |
| 248 | + | if err != nil { | |||
| 249 | + | return domain.CheckpointRef{}, err | |||
| 250 | + | } | |||
| 248 | lock := r.lockSession(key) | 251 | lock := r.lockSession(key) | ||
| 249 | defer r.unlockSession(lock) | 252 | defer r.unlockSession(lock) | ||
| 250 | if err := r.ensureSession(req.IncarnationID); err != nil { | 253 | if err := r.ensureSession(req.IncarnationID); err != nil { | ||
M internal/adapters/snapshot/migration_v0_test.go +39 -3
| @@ -4,6 +4,7 @@ import ( | |||||
| 4 | "context" | 4 | "context" | ||
| 5 | "crypto/sha256" | 5 | "crypto/sha256" | ||
| 6 | "encoding/binary" | 6 | "encoding/binary" | ||
| 7 | + | "errors" | |||
| 7 | "fmt" | 8 | "fmt" | ||
| 8 | "hash/crc32" | 9 | "hash/crc32" | ||
| 9 | "os" | 10 | "os" | ||
| @@ -37,10 +38,45 @@ func TestLegacyManifestV1(t *testing.T) { | |||||
| 37 | require.NoError(t, err) | 38 | require.NoError(t, err) | ||
| 38 | require.Equal(t, "work", got.Name) | 39 | require.Equal(t, "work", got.Name) | ||
| 39 | require.Equal(t, uint64(7), got.Generation) | 40 | require.Equal(t, uint64(7), got.Generation) | ||
| 40 | - | for _, invalid := range [][]byte{encoded[:5], append(append([]byte(nil), encoded...), 0)} { | 41 | + | for size := range len(encoded) { |
| 41 | - | _, err := decodeManifestV1(invalid) | 42 | + | _, err := decodeManifestV1(encoded[:size]) |
| 42 | - | require.Error(t, err) | 43 | + | require.Errorf(t, err, "prefix length %d", size) |
| 43 | } | 44 | } | ||
| 45 | + | _, err = decodeManifestV1(append(append([]byte(nil), encoded...), 0)) | |||
| 46 | + | require.Error(t, err) | |||
| 47 | + | } | |||
| 48 | + | ||||
| 49 | + | func TestLegacyManifestV1DeduplicatesObjectsPreservingFirst(t *testing.T) { | |||
| 50 | + | chunk, err := codec.MarshalObject(codec.HistoryChunk, []byte("chunk")) | |||
| 51 | + | require.NoError(t, err) | |||
| 52 | + | tail, err := codec.MarshalObject(codec.HistoryTail, []byte("tail")) | |||
| 53 | + | require.NoError(t, err) | |||
| 54 | + | visible, err := codec.MarshalObject(codec.Visible, []byte("visible")) | |||
| 55 | + | require.NoError(t, err) | |||
| 56 | + | chunkRef := codec.ObjectRef{Kind: codec.HistoryChunk, Digest: chunk.Digest, Size: uint32(len(chunk.Data))} | |||
| 57 | + | manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Sealed: []codec.ObjectRef{chunkRef, chunkRef}, Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}} | |||
| 58 | + | modern, err := codec.MarshalManifest(manifest) | |||
| 59 | + | require.NoError(t, err) | |||
| 60 | + | body := append([]byte(nil), modern[legacyManifestHeaderSize:]...) | |||
| 61 | + | body = append(body[:8], body[25:]...) | |||
| 62 | + | legacy := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body)) | |||
| 63 | + | copy(legacy, "VEVM") | |||
| 64 | + | binary.BigEndian.PutUint16(legacy[4:6], 1) | |||
| 65 | + | binary.BigEndian.PutUint32(legacy[8:12], uint32(len(body))) | |||
| 66 | + | binary.BigEndian.PutUint32(legacy[12:16], crc32.ChecksumIEEE(body)) | |||
| 67 | + | legacy = append(legacy, body...) | |||
| 68 | + | ||||
| 69 | + | decoded, err := decodeManifestV1(legacy) | |||
| 70 | + | require.NoError(t, err) | |||
| 71 | + | require.Len(t, decoded.Objects, 3) | |||
| 72 | + | require.Equal(t, chunk.Digest, decoded.Objects[0].Digest) | |||
| 73 | + | } | |||
| 74 | + | ||||
| 75 | + | func TestUncertainLegacyErrorWrapsSentinelAndCause(t *testing.T) { | |||
| 76 | + | cause := errors.New("legacy cause") | |||
| 77 | + | err := uncertainLegacyError("read", cause) | |||
| 78 | + | require.ErrorIs(t, err, ports.ErrLegacySnapshotUncertain) | |||
| 79 | + | require.ErrorIs(t, err, cause) | |||
| 44 | } | 80 | } | ||
| 45 | 81 | ||||
| 46 | func TestHasLegacyState(t *testing.T) { | 82 | func TestHasLegacyState(t *testing.T) { | ||
M internal/adapters/snapshot/publish_efficiency_test.go +7 -8
| @@ -120,11 +120,11 @@ func TestRepositoryPublishVerifiesNecessaryExistingObjectOnce(t *testing.T) { | |||||
| 120 | } | 120 | } | ||
| 121 | second := repositoryPublicationAfter(t, repo, "named", 2, []byte("second")) | 121 | second := repositoryPublicationAfter(t, repo, "named", 2, []byte("second")) | ||
| 122 | newTail := second.Objects[0] | 122 | newTail := second.Objects[0] | ||
| 123 | - | key := legacyIncarnationID(second.Name).String() | 123 | + | path := repo.objectPath(second.IncarnationID, newTail.Digest) |
| 124 | - | if err := os.MkdirAll(filepath.Dir(repo.legacyObjectPath(key, newTail.Digest)), 0o700); err != nil { | 124 | + | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { |
| 125 | t.Fatal(err) | 125 | t.Fatal(err) | ||
| 126 | } | 126 | } | ||
| 127 | - | if err := os.WriteFile(repo.legacyObjectPath(key, newTail.Digest), newTail.Data, 0o600); err != nil { | 127 | + | if err := os.WriteFile(path, newTail.Data, 0o600); err != nil { |
| 128 | t.Fatal(err) | 128 | t.Fatal(err) | ||
| 129 | } | 129 | } | ||
| 130 | 130 | ||||
| @@ -178,7 +178,6 @@ func largeIncrementalPublications(t testing.TB, name string, count int) (ports.S | |||||
| 178 | 178 | ||||
| 179 | func seedCompletePublication(t testing.TB, repo *Repository, publication ports.SnapshotPublication) { | 179 | func seedCompletePublication(t testing.TB, repo *Repository, publication ports.SnapshotPublication) { | ||
| 180 | t.Helper() | 180 | t.Helper() | ||
| 181 | - | key := legacyIncarnationID(publication.Name).String() | |||
| 182 | if err := repo.ensureSession(publication.IncarnationID); err != nil { | 181 | if err := repo.ensureSession(publication.IncarnationID); err != nil { | ||
| 183 | t.Fatal(err) | 182 | t.Fatal(err) | ||
| 184 | } | 183 | } | ||
| @@ -187,17 +186,17 @@ func seedCompletePublication(t testing.TB, repo *Repository, publication ports.S | |||||
| 187 | t.Fatal(err) | 186 | t.Fatal(err) | ||
| 188 | } | 187 | } | ||
| 189 | for _, object := range publication.Objects { | 188 | for _, object := range publication.Objects { | ||
| 190 | - | if err := os.MkdirAll(filepath.Dir(repo.legacyObjectPath(key, object.Digest)), 0o700); err != nil { | 189 | + | if err := os.MkdirAll(filepath.Dir(repo.objectPath(publication.IncarnationID, object.Digest)), 0o700); err != nil { |
| 191 | t.Fatal(err) | 190 | t.Fatal(err) | ||
| 192 | } | 191 | } | ||
| 193 | - | if err := os.WriteFile(repo.legacyObjectPath(key, object.Digest), object.Data, 0o600); err != nil { | 192 | + | if err := os.WriteFile(repo.objectPath(publication.IncarnationID, object.Digest), object.Data, 0o600); err != nil { |
| 194 | t.Fatal(err) | 193 | t.Fatal(err) | ||
| 195 | } | 194 | } | ||
| 196 | } | 195 | } | ||
| 197 | - | if err := os.WriteFile(repo.legacyManifestPath(key, publication.Generation), publication.Manifest, 0o600); err != nil { | 196 | + | if err := os.WriteFile(repo.manifestPath(publication.IncarnationID, publication.Generation), publication.Manifest, 0o600); err != nil { |
| 198 | t.Fatal(err) | 197 | t.Fatal(err) | ||
| 199 | } | 198 | } | ||
| 200 | - | if err := os.WriteFile(repo.legacyHeadPath(key), marshalHead(publication.Generation, sha256.Sum256(publication.Manifest)), 0o600); err != nil { | 199 | + | if err := os.WriteFile(repo.headPath(publication.IncarnationID), marshalHead(publication.Generation, sha256.Sum256(publication.Manifest)), 0o600); err != nil { |
| 201 | t.Fatal(err) | 200 | t.Fatal(err) | ||
| 202 | } | 201 | } | ||
| 203 | if refs := manifestRefs(manifest); refs == nil { | 202 | if refs := manifestRefs(manifest); refs == nil { | ||
M internal/adapters/snapshot/reconciliation_test.go +2 -2
| @@ -2,13 +2,13 @@ package snapshot | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | - | "os" | |||
| 6 | "path/filepath" | 5 | "path/filepath" | ||
| 7 | "testing" | 6 | "testing" | ||
| 8 | 7 | ||||
| 9 | "github.com/bnema/vev/internal/domain" | 8 | "github.com/bnema/vev/internal/domain" | ||
| 10 | "github.com/bnema/vev/internal/ports" | 9 | "github.com/bnema/vev/internal/ports" | ||
| 11 | codec "github.com/bnema/vev/internal/usecase/snapshot" | 10 | codec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 11 | + | "github.com/bnema/vev/pkg/safedir" | |||
| 12 | "github.com/stretchr/testify/require" | 12 | "github.com/stretchr/testify/require" | ||
| 13 | ) | 13 | ) | ||
| 14 | 14 | ||||
| @@ -108,7 +108,7 @@ func TestReconciliationCursor(t *testing.T) { | |||||
| 108 | sessions := filepath.Join(repo.dir, repositorySessionsDir) | 108 | sessions := filepath.Join(repo.dir, repositorySessionsDir) | ||
| 109 | for i := byte(20); i < 26; i++ { | 109 | for i := byte(20); i < 26; i++ { | ||
| 110 | id := domain.IncarnationID{i} | 110 | id := domain.IncarnationID{i} | ||
| 111 | - | require.NoError(t, os.Mkdir(filepath.Join(sessions, id.String()), 0o700)) | 111 | + | require.NoError(t, safedir.EnsurePrivate(filepath.Join(sessions, id.String()))) |
| 112 | } | 112 | } | ||
| 113 | 113 | ||||
| 114 | cursor := ports.ReconcileCursor{} | 114 | cursor := ports.ReconcileCursor{} | ||
M internal/adapters/snapshot/recovery_crash_test.go +16 -7
| @@ -13,6 +13,7 @@ import ( | |||||
| 13 | "github.com/bnema/vev/internal/persist" | 13 | "github.com/bnema/vev/internal/persist" | ||
| 14 | "github.com/bnema/vev/internal/ports" | 14 | "github.com/bnema/vev/internal/ports" | ||
| 15 | "github.com/bnema/vev/internal/usecase/recovery" | 15 | "github.com/bnema/vev/internal/usecase/recovery" | ||
| 16 | + | "github.com/bnema/vev/pkg/safedir" | |||
| 16 | "github.com/stretchr/testify/require" | 17 | "github.com/stretchr/testify/require" | ||
| 17 | ) | 18 | ) | ||
| 18 | 19 | ||||
| @@ -57,7 +58,7 @@ func TestDeletionRestartsAtInternalQuarantineBoundaries(t *testing.T) { | |||||
| 57 | record := domain.CatalogueRecord{Name: "work", IncarnationID: oldID, RecoveryState: domain.RecoveryFresh} | 58 | record := domain.CatalogueRecord{Name: "work", IncarnationID: oldID, RecoveryState: domain.RecoveryFresh} | ||
| 58 | require.NoError(t, catalogue.Create(record)) | 59 | require.NoError(t, catalogue.Create(record)) | ||
| 59 | sessionPath := repository.sessionPath(oldID) | 60 | sessionPath := repository.sessionPath(oldID) | ||
| 60 | - | require.NoError(t, os.MkdirAll(sessionPath, 0o700)) | 61 | + | require.NoError(t, safedir.EnsurePrivate(sessionPath)) |
| 61 | require.NoError(t, os.WriteFile(filepath.Join(sessionPath, "durable"), []byte("old namespace"), 0o600)) | 62 | require.NoError(t, os.WriteFile(filepath.Join(sessionPath, "durable"), []byte("old namespace"), 0o600)) | ||
| 62 | require.NoError(t, os.WriteFile(filepath.Join(h.snapshotDir, filenameForName(record.Name)), []byte("old legacy"), 0o600)) | 63 | require.NoError(t, os.WriteFile(filepath.Join(h.snapshotDir, filenameForName(record.Name)), []byte("old legacy"), 0o600)) | ||
| 63 | tc.inject(repository) | 64 | tc.inject(repository) | ||
| @@ -69,7 +70,8 @@ func TestDeletionRestartsAtInternalQuarantineBoundaries(t *testing.T) { | |||||
| 69 | // Simulate a new process: none of the adapter or coordinator state survives. | 70 | // Simulate a new process: none of the adapter or coordinator state survives. | ||
| 70 | catalogue, repository, journal = h.open(t) | 71 | catalogue, repository, journal = h.open(t) | ||
| 71 | require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx)) | 72 | require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx)) | ||
| 72 | - | _, exists := catalogue.Record(record.Name) | 73 | + | _, exists, err := catalogue.Record(record.Name) |
| 74 | + | require.NoError(t, err) | |||
| 73 | require.False(t, exists) | 75 | require.False(t, exists) | ||
| 74 | require.NoDirExists(t, repository.sessionPath(oldID)) | 76 | require.NoDirExists(t, repository.sessionPath(oldID)) | ||
| 75 | require.Equal(t, []byte("old namespace"), mustReadCrashFile(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot", "durable"))) | 77 | require.Equal(t, []byte("old namespace"), mustReadCrashFile(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot", "durable"))) | ||
| @@ -84,13 +86,14 @@ func TestDeletionRestartsAtInternalQuarantineBoundaries(t *testing.T) { | |||||
| 84 | require.NoError(t, err) | 86 | require.NoError(t, err) | ||
| 85 | require.Equal(t, newID, created.IncarnationID) | 87 | require.Equal(t, newID, created.IncarnationID) | ||
| 86 | newPath := repository.sessionPath(newID) | 88 | newPath := repository.sessionPath(newID) | ||
| 87 | - | require.NoError(t, os.MkdirAll(newPath, 0o700)) | 89 | + | require.NoError(t, safedir.EnsurePrivate(newPath)) |
| 88 | require.NoError(t, os.WriteFile(filepath.Join(newPath, "durable"), []byte("new namespace"), 0o600)) | 90 | require.NoError(t, os.WriteFile(filepath.Join(newPath, "durable"), []byte("new namespace"), 0o600)) | ||
| 89 | require.NoError(t, catalogue.Close()) | 91 | require.NoError(t, catalogue.Close()) | ||
| 90 | 92 | ||||
| 91 | catalogue, repository, journal = h.open(t) | 93 | catalogue, repository, journal = h.open(t) | ||
| 92 | require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx)) | 94 | require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx)) | ||
| 93 | - | got, exists := catalogue.Record(record.Name) | 95 | + | got, exists, err := catalogue.Record(record.Name) |
| 96 | + | require.NoError(t, err) | |||
| 94 | require.True(t, exists) | 97 | require.True(t, exists) | ||
| 95 | require.Equal(t, newID, got.IncarnationID) | 98 | require.Equal(t, newID, got.IncarnationID) | ||
| 96 | require.Equal(t, []byte("new namespace"), mustReadCrashFile(t, filepath.Join(repository.sessionPath(newID), "durable"))) | 99 | require.Equal(t, []byte("new namespace"), mustReadCrashFile(t, filepath.Join(repository.sessionPath(newID), "durable"))) | ||
| @@ -114,20 +117,26 @@ func TestCreatePreCommitRestartLeavesNoAdoptableNamespace(t *testing.T) { | |||||
| 114 | interruptedID := domain.IncarnationID{3} | 117 | interruptedID := domain.IncarnationID{3} | ||
| 115 | _, err := recovery.NewCoordinator(rejectCreateCatalogue{Catalogue: catalogue}, repository, journal, bytes.NewReader(interruptedID[:])).Create(ctx, domain.CatalogueRecord{Name: "work"}) | 118 | _, err := recovery.NewCoordinator(rejectCreateCatalogue{Catalogue: catalogue}, repository, journal, bytes.NewReader(interruptedID[:])).Create(ctx, domain.CatalogueRecord{Name: "work"}) | ||
| 116 | require.ErrorIs(t, err, errDeletionBoundaryCrash) | 119 | require.ErrorIs(t, err, errDeletionBoundaryCrash) | ||
| 117 | - | require.Empty(t, catalogue.Records()) | 120 | + | records, err := catalogue.Records() |
| 121 | + | require.NoError(t, err) | |||
| 122 | + | require.Empty(t, records) | |||
| 118 | require.NoDirExists(t, filepath.Join(h.snapshotDir, repositorySessionsDir)) | 123 | require.NoDirExists(t, filepath.Join(h.snapshotDir, repositorySessionsDir)) | ||
| 119 | require.NoError(t, catalogue.Close()) | 124 | require.NoError(t, catalogue.Close()) | ||
| 120 | 125 | ||||
| 121 | catalogue, repository, journal = h.open(t) | 126 | catalogue, repository, journal = h.open(t) | ||
| 122 | require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx)) | 127 | require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx)) | ||
| 123 | - | require.Empty(t, catalogue.Records()) | 128 | + | records, err = catalogue.Records() |
| 129 | + | require.NoError(t, err) | |||
| 130 | + | require.Empty(t, records) | |||
| 124 | require.NoDirExists(t, repository.sessionPath(interruptedID)) | 131 | require.NoDirExists(t, repository.sessionPath(interruptedID)) | ||
| 125 | 132 | ||||
| 126 | retryID := domain.IncarnationID{4} | 133 | retryID := domain.IncarnationID{4} | ||
| 127 | created, err := recovery.NewCoordinator(catalogue, repository, journal, bytes.NewReader(retryID[:])).Create(ctx, domain.CatalogueRecord{Name: "work"}) | 134 | created, err := recovery.NewCoordinator(catalogue, repository, journal, bytes.NewReader(retryID[:])).Create(ctx, domain.CatalogueRecord{Name: "work"}) | ||
| 128 | require.NoError(t, err) | 135 | require.NoError(t, err) | ||
| 129 | require.Equal(t, retryID, created.IncarnationID) | 136 | require.Equal(t, retryID, created.IncarnationID) | ||
| 130 | - | require.Len(t, catalogue.Records(), 1) | 137 | + | records, err = catalogue.Records() |
| 138 | + | require.NoError(t, err) | |||
| 139 | + | require.Len(t, records, 1) | |||
| 131 | require.NoDirExists(t, repository.sessionPath(interruptedID)) | 140 | require.NoDirExists(t, repository.sessionPath(interruptedID)) | ||
| 132 | require.NoError(t, catalogue.Close()) | 141 | require.NoError(t, catalogue.Close()) | ||
| 133 | } | 142 | } | ||
M internal/adapters/snapshot/recovery_observability_test.go +58 -20
| @@ -14,25 +14,63 @@ import ( | |||||
| 14 | ) | 14 | ) | ||
| 15 | 15 | ||||
| 16 | func TestGCBudgetDiagnostic(t *testing.T) { | 16 | func TestGCBudgetDiagnostic(t *testing.T) { | ||
| 17 | - | var buffer bytes.Buffer | 17 | + | for _, tc := range []struct { |
| 18 | - | log := slog.New(slog.NewJSONHandler(&buffer, nil)) | 18 | + | name string |
| 19 | - | repository := NewRepository(t.TempDir(), log) | 19 | + | arrange func(*testing.T, *Repository) ports.RetentionPlan |
| 20 | - | id := domain.IncarnationID{1} | 20 | + | budget ports.MaintenanceBudget |
| 21 | - | done, err := repository.MaintainSession(context.Background(), ports.RetentionPlan{ | 21 | + | wantDone bool |
| 22 | - | IncarnationID: id, | 22 | + | wantExhausted bool |
| 23 | - | Keep: []ports.CheckpointRef{{Generation: 3}, {Generation: 2}, {Generation: 1}}, | 23 | + | wantConsumption bool |
| 24 | - | }, ports.MaintenanceBudget{}) | 24 | + | }{ |
| 25 | - | require.NoError(t, err) | 25 | + | { |
| 26 | - | require.False(t, done) | 26 | + | name: "exhausted", |
| 27 | + | arrange: func(_ *testing.T, _ *Repository) ports.RetentionPlan { | |||
| 28 | + | return ports.RetentionPlan{IncarnationID: domain.IncarnationID{1}, Keep: []ports.CheckpointRef{{Generation: 3}, {Generation: 2}, {Generation: 1}}} | |||
| 29 | + | }, | |||
| 30 | + | wantExhausted: true, | |||
| 31 | + | }, | |||
| 32 | + | { | |||
| 33 | + | name: "sufficient", | |||
| 34 | + | arrange: func(t *testing.T, repository *Repository) ports.RetentionPlan { | |||
| 35 | + | publication := publishMaintenanceGenerations(t, repository, "diagnostic", 1)[0] | |||
| 36 | + | return ports.RetentionPlan{IncarnationID: publication.IncarnationID} | |||
| 37 | + | }, | |||
| 38 | + | budget: ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20}, | |||
| 39 | + | wantDone: true, | |||
| 40 | + | wantConsumption: true, | |||
| 41 | + | }, | |||
| 42 | + | } { | |||
| 43 | + | t.Run(tc.name, func(t *testing.T) { | |||
| 44 | + | var buffer bytes.Buffer | |||
| 45 | + | log := slog.New(slog.NewJSONHandler(&buffer, nil)) | |||
| 46 | + | repository := NewRepository(privateDir(t), log) | |||
| 47 | + | plan := tc.arrange(t, repository) | |||
| 48 | + | buffer.Reset() | |||
| 27 | 49 | ||||
| 28 | - | scanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes())) | 50 | + | done, err := repository.MaintainSession(context.Background(), plan, tc.budget) |
| 29 | - | require.True(t, scanner.Scan()) | 51 | + | require.NoError(t, err) |
| 30 | - | var entry map[string]any | 52 | + | require.Equal(t, tc.wantDone, done) |
| 31 | - | require.NoError(t, json.Unmarshal(scanner.Bytes(), &entry)) | 53 | + | |
| 32 | - | require.NoError(t, scanner.Err()) | 54 | + | scanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes())) |
| 33 | - | require.Equal(t, "snapshot_maintenance_progress", entry["msg"]) | 55 | + | require.True(t, scanner.Scan()) |
| 34 | - | require.Equal(t, id.String(), entry["incarnation"]) | 56 | + | var entry map[string]any |
| 35 | - | require.EqualValues(t, 3, entry["retained"]) | 57 | + | require.NoError(t, json.Unmarshal(scanner.Bytes(), &entry)) |
| 36 | - | require.Contains(t, entry, "cursor") | 58 | + | require.NoError(t, scanner.Err()) |
| 37 | - | require.Equal(t, true, entry["budget_exhausted"]) | 59 | + | require.Equal(t, "snapshot_maintenance_progress", entry["msg"]) |
| 60 | + | require.Equal(t, plan.IncarnationID.String(), entry["incarnation"]) | |||
| 61 | + | require.Contains(t, entry, "cursor") | |||
| 62 | + | require.Equal(t, tc.wantExhausted, entry["budget_exhausted"]) | |||
| 63 | + | consumedEntries, entriesOK := entry["consumed_entries"].(float64) | |||
| 64 | + | consumedBytes, bytesOK := entry["consumed_bytes"].(float64) | |||
| 65 | + | require.True(t, entriesOK) | |||
| 66 | + | require.True(t, bytesOK) | |||
| 67 | + | if tc.wantConsumption { | |||
| 68 | + | require.Positive(t, consumedEntries) | |||
| 69 | + | require.Positive(t, consumedBytes) | |||
| 70 | + | } else { | |||
| 71 | + | require.Zero(t, consumedEntries) | |||
| 72 | + | require.Zero(t, consumedBytes) | |||
| 73 | + | } | |||
| 74 | + | }) | |||
| 75 | + | } | |||
| 38 | } | 76 | } | ||
M internal/app/daemon_integration_test.go +27 -18
| @@ -453,15 +453,13 @@ func TestMultipleClientsOneLifecycleOwner(t *testing.T) { | |||||
| 453 | errs := make(chan error, clients) | 453 | errs := make(chan error, clients) | ||
| 454 | var wg sync.WaitGroup | 454 | var wg sync.WaitGroup | ||
| 455 | for range clients { | 455 | for range clients { | ||
| 456 | - | wg.Add(1) | 456 | + | wg.Go(func() { |
| 457 | - | go func() { | |||
| 458 | - | defer wg.Done() | |||
| 459 | _, err := ensureDaemonWithLifecycle(context.Background(), dir, dial, func() error { | 457 | _, err := ensureDaemonWithLifecycle(context.Background(), dir, dial, func() error { | ||
| 460 | spawns.Add(1) | 458 | spawns.Add(1) | ||
| 461 | return nil | 459 | return nil | ||
| 462 | }, cfg) | 460 | }, cfg) | ||
| 463 | errs <- err | 461 | errs <- err | ||
| 464 | - | }() | 462 | + | }) |
| 465 | } | 463 | } | ||
| 466 | require.Never(t, func() bool { return spawns.Load() != 0 }, 50*time.Millisecond, time.Millisecond) | 464 | require.Never(t, func() bool { return spawns.Load() != 0 }, 50*time.Millisecond, time.Millisecond) | ||
| 467 | close(ready) | 465 | close(ready) | ||
| @@ -537,7 +535,7 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) { | |||||
| 537 | closed: make(chan struct{}), | 535 | closed: make(chan struct{}), | ||
| 538 | } | 536 | } | ||
| 539 | repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots")) | 537 | repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots")) | ||
| 540 | - | shutdownClock := newLifecycleShutdownClock() | 538 | + | shutdownClock := newLifecycleShutdownClock(t) |
| 541 | reconciler := &lifecycleBlockingReconciler{ | 539 | reconciler := &lifecycleBlockingReconciler{ | ||
| 542 | entered: make(chan struct{}), | 540 | entered: make(chan struct{}), | ||
| 543 | release: make(chan struct{}), | 541 | release: make(chan struct{}), | ||
| @@ -564,7 +562,7 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) { | |||||
| 564 | pty.NewFactory(), | 562 | pty.NewFactory(), | ||
| 565 | shutdownClock, | 563 | shutdownClock, | ||
| 566 | discardLog(), | 564 | discardLog(), | ||
| 567 | - | daemon.WithSnapshotRepository(repository, nil), | 565 | + | daemon.WithSnapshotRepository(repository), |
| 568 | daemon.WithDurableMaintenance(catalogue, repository, reconciler, nil), | 566 | daemon.WithDurableMaintenance(catalogue, repository, reconciler, nil), | ||
| 569 | daemon.WithCatalogue(catalogue, nil), | 567 | daemon.WithCatalogue(catalogue, nil), | ||
| 570 | ) | 568 | ) | ||
| @@ -679,7 +677,7 @@ func TestLifecycleOwnershipOutlivesRestorationWriter(t *testing.T) { | |||||
| 679 | releaseRestore := func() { releaseOnce.Do(func() { close(blocking.release) }) } | 677 | releaseRestore := func() { releaseOnce.Do(func() { close(blocking.release) }) } | ||
| 680 | t.Cleanup(releaseRestore) | 678 | t.Cleanup(releaseRestore) | ||
| 681 | 679 | ||||
| 682 | - | shutdownClock := newLifecycleShutdownClock() | 680 | + | shutdownClock := newLifecycleShutdownClock(t) |
| 683 | ctx, cancel := context.WithCancel(context.Background()) | 681 | ctx, cancel := context.WithCancel(context.Background()) | ||
| 684 | t.Cleanup(cancel) | 682 | t.Cleanup(cancel) | ||
| 685 | listenerReady := make(chan struct{}) | 683 | listenerReady := make(chan struct{}) | ||
| @@ -707,7 +705,7 @@ func TestLifecycleOwnershipOutlivesRestorationWriter(t *testing.T) { | |||||
| 707 | discardLog(), | 705 | discardLog(), | ||
| 708 | daemon.WithShell("/bin/cat", nil), | 706 | daemon.WithShell("/bin/cat", nil), | ||
| 709 | daemon.WithCatalogue(opened.Catalogue, opened.Records), | 707 | daemon.WithCatalogue(opened.Catalogue, opened.Records), | ||
| 710 | - | daemon.WithSnapshotRepository(blocking, nil), | 708 | + | daemon.WithSnapshotRepository(blocking), |
| 711 | ) | 709 | ) | ||
| 712 | return d.Serve(ctx, listener) | 710 | return d.Serve(ctx, listener) | ||
| 713 | }) | 711 | }) | ||
| @@ -764,7 +762,7 @@ func publishRestorableCheckpoint(t *testing.T, stateDir string, repository *snap | |||||
| 764 | discardLog(), | 762 | discardLog(), | ||
| 765 | daemon.WithShell("/bin/cat", nil), | 763 | daemon.WithShell("/bin/cat", nil), | ||
| 766 | daemon.WithCatalogue(opened.Catalogue, opened.Records), | 764 | daemon.WithCatalogue(opened.Catalogue, opened.Records), | ||
| 767 | - | daemon.WithSnapshotRepository(repository, nil), | 765 | + | daemon.WithSnapshotRepository(repository), |
| 768 | daemon.WithRecoveryCoordinator(coordinator), | 766 | daemon.WithRecoveryCoordinator(coordinator), | ||
| 769 | ) | 767 | ) | ||
| 770 | served := make(chan error, 1) | 768 | served := make(chan error, 1) | ||
| @@ -773,7 +771,7 @@ func publishRestorableCheckpoint(t *testing.T, stateDir string, repository *snap | |||||
| 773 | tr, _ := attach(t, runtimeDir, ports.IntentNew, name, domain.Size{Cols: 80, Rows: 24}) | 771 | tr, _ := attach(t, runtimeDir, ports.IntentNew, name, domain.Size{Cols: 80, Rows: 24}) | ||
| 774 | require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgInput, Payload: ports.MarshalInput(ports.Input{Data: []byte("checkpoint me\n")})})) | 772 | require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgInput, Payload: ports.MarshalInput(ports.Input{Data: []byte("checkpoint me\n")})})) | ||
| 775 | require.Eventually(t, func() bool { | 773 | require.Eventually(t, func() bool { | ||
| 776 | - | record, ok := opened.Catalogue.Record(name) | 774 | + | record, ok, _ := opened.Catalogue.Record(name) |
| 777 | return ok && record.RecoveryState == domain.RecoveryHealthy && record.Committed != nil | 775 | return ok && record.RecoveryState == domain.RecoveryHealthy && record.Committed != nil | ||
| 778 | }, 5*time.Second, 10*time.Millisecond, "session never committed a checkpoint") | 776 | }, 5*time.Second, 10*time.Millisecond, "session never committed a checkpoint") | ||
| 779 | require.NoError(t, tr.Close()) | 777 | require.NoError(t, tr.Close()) | ||
| @@ -826,7 +824,7 @@ func runBlockedSnapshotWriterShutdown(t *testing.T) blockedSnapshotWriterShutdow | |||||
| 826 | t.Helper() | 824 | t.Helper() | ||
| 827 | runtimeDir := filepath.Join(t.TempDir(), "runtime") | 825 | runtimeDir := filepath.Join(t.TempDir(), "runtime") | ||
| 828 | stateDir := filepath.Join(t.TempDir(), "state") | 826 | stateDir := filepath.Join(t.TempDir(), "state") | ||
| 829 | - | shutdownClock := newLifecycleShutdownClock() | 827 | + | shutdownClock := newLifecycleShutdownClock(t) |
| 830 | repository := newLifecycleBlockingRepository() | 828 | repository := newLifecycleBlockingRepository() | ||
| 831 | ctx, cancel := context.WithCancel(context.Background()) | 829 | ctx, cancel := context.WithCancel(context.Background()) | ||
| 832 | t.Cleanup(cancel) | 830 | t.Cleanup(cancel) | ||
| @@ -849,7 +847,7 @@ func runBlockedSnapshotWriterShutdown(t *testing.T) blockedSnapshotWriterShutdow | |||||
| 849 | shutdownClock, | 847 | shutdownClock, | ||
| 850 | discardLog(), | 848 | discardLog(), | ||
| 851 | daemon.WithShell("/bin/cat", nil), | 849 | daemon.WithShell("/bin/cat", nil), | ||
| 852 | - | daemon.WithSnapshotRepository(repository, nil), | 850 | + | daemon.WithSnapshotRepository(repository), |
| 853 | ) | 851 | ) | ||
| 854 | return d.Serve(ctx, observed) | 852 | return d.Serve(ctx, observed) | ||
| 855 | }) | 853 | }) | ||
| @@ -913,21 +911,28 @@ func (r *lifecycleBlockingRepository) Publish(context.Context, ports.SnapshotPub | |||||
| 913 | } | 911 | } | ||
| 914 | 912 | ||||
| 915 | type lifecycleShutdownClock struct { | 913 | type lifecycleShutdownClock struct { | ||
| 914 | + | t *testing.T | |||
| 916 | base ports.Clock | 915 | base ports.Clock | ||
| 917 | timers chan *lifecycleManualTimer | 916 | timers chan *lifecycleManualTimer | ||
| 918 | } | 917 | } | ||
| 919 | 918 | ||||
| 920 | - | func newLifecycleShutdownClock() *lifecycleShutdownClock { | 919 | + | func newLifecycleShutdownClock(t *testing.T) *lifecycleShutdownClock { |
| 921 | - | return &lifecycleShutdownClock{base: clock.New(), timers: make(chan *lifecycleManualTimer, 4)} | 920 | + | t.Helper() |
| 921 | + | return &lifecycleShutdownClock{t: t, base: clock.New(), timers: make(chan *lifecycleManualTimer, 4)} | |||
| 922 | } | 922 | } | ||
| 923 | 923 | ||||
| 924 | func (c *lifecycleShutdownClock) Now() time.Time { return c.base.Now() } | 924 | func (c *lifecycleShutdownClock) Now() time.Time { return c.base.Now() } | ||
| 925 | func (c *lifecycleShutdownClock) NewTimer(delay time.Duration) ports.Timer { | 925 | func (c *lifecycleShutdownClock) NewTimer(delay time.Duration) ports.Timer { | ||
| 926 | - | if delay != time.Second { | 926 | + | if delay != daemon.SnapshotShutdownTimeout() { |
| 927 | return c.base.NewTimer(delay) | 927 | return c.base.NewTimer(delay) | ||
| 928 | } | 928 | } | ||
| 929 | - | timer := &lifecycleManualTimer{ch: make(chan time.Time, 1)} | 929 | + | timer := &lifecycleManualTimer{t: c.t, ch: make(chan time.Time, 1)} |
| 930 | - | c.timers <- timer | 930 | + | select { |
| 931 | + | case c.timers <- timer: | |||
| 932 | + | default: | |||
| 933 | + | c.t.Errorf("unexpected additional snapshot shutdown timer with delay %s", delay) | |||
| 934 | + | timer.fire() | |||
| 935 | + | } | |||
| 931 | return timer | 936 | return timer | ||
| 932 | } | 937 | } | ||
| 933 | 938 | ||||
| @@ -943,12 +948,16 @@ func (c *lifecycleShutdownClock) nextTimer(t *testing.T) *lifecycleManualTimer { | |||||
| 943 | } | 948 | } | ||
| 944 | 949 | ||||
| 945 | type lifecycleManualTimer struct { | 950 | type lifecycleManualTimer struct { | ||
| 951 | + | t *testing.T | |||
| 946 | ch chan time.Time | 952 | ch chan time.Time | ||
| 947 | stopped atomic.Bool | 953 | stopped atomic.Bool | ||
| 948 | } | 954 | } | ||
| 949 | 955 | ||||
| 950 | func (t *lifecycleManualTimer) C() <-chan time.Time { return t.ch } | 956 | func (t *lifecycleManualTimer) C() <-chan time.Time { return t.ch } | ||
| 951 | - | func (t *lifecycleManualTimer) Reset(time.Duration) bool { | 957 | + | func (t *lifecycleManualTimer) Reset(delay time.Duration) bool { |
| 958 | + | if delay != daemon.SnapshotShutdownTimeout() { | |||
| 959 | + | t.t.Errorf("snapshot shutdown timer reset with delay %s, want %s", delay, daemon.SnapshotShutdownTimeout()) | |||
| 960 | + | } | |||
| 952 | t.stopped.Store(false) | 961 | t.stopped.Store(false) | ||
| 953 | return false | 962 | return false | ||
| 954 | } | 963 | } | ||
M internal/app/durable_crash_matrix_test.go +7 -7
| @@ -132,7 +132,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||||
| 132 | require.NoError(t, catalogue.Close()) | 132 | require.NoError(t, catalogue.Close()) | ||
| 133 | 133 | ||||
| 134 | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { | 134 | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { | ||
| 135 | - | record, exists := catalogue.Record("work") | 135 | + | record, exists, _ := catalogue.Record("work") |
| 136 | require.True(t, exists) | 136 | require.True(t, exists) | ||
| 137 | require.Equal(t, domain.IncarnationID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, record.IncarnationID) | 137 | require.Equal(t, domain.IncarnationID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, record.IncarnationID) | ||
| 138 | require.Equal(t, domain.RecoveryFresh, record.RecoveryState) | 138 | require.Equal(t, domain.RecoveryFresh, record.RecoveryState) | ||
| @@ -160,7 +160,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||||
| 160 | 160 | ||||
| 161 | ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)} | 161 | ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)} | ||
| 162 | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | 162 | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | ||
| 163 | - | stored, exists := catalogue.Record(record.Name) | 163 | + | stored, exists, _ := catalogue.Record(record.Name) |
| 164 | require.True(t, exists) | 164 | require.True(t, exists) | ||
| 165 | if boundaryName == "snapshot-publish" { | 165 | if boundaryName == "snapshot-publish" { | ||
| 166 | require.Equal(t, domain.RecoveryFresh, stored.RecoveryState) | 166 | require.Equal(t, domain.RecoveryFresh, stored.RecoveryState) | ||
| @@ -189,8 +189,8 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||||
| 189 | require.NoError(t, catalogue.Close()) | 189 | require.NoError(t, catalogue.Close()) | ||
| 190 | 190 | ||||
| 191 | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { | 191 | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { | ||
| 192 | - | _, oldExists := catalogue.Record("old") | 192 | + | _, oldExists, _ := catalogue.Record("old") |
| 193 | - | renamed, newExists := catalogue.Record("new") | 193 | + | renamed, newExists, _ := catalogue.Record("new") |
| 194 | require.False(t, oldExists) | 194 | require.False(t, oldExists) | ||
| 195 | require.True(t, newExists) | 195 | require.True(t, newExists) | ||
| 196 | require.Equal(t, record.IncarnationID, renamed.IncarnationID) | 196 | require.Equal(t, record.IncarnationID, renamed.IncarnationID) | ||
| @@ -228,7 +228,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||||
| 228 | require.NoError(t, catalogue.Close()) | 228 | require.NoError(t, catalogue.Close()) | ||
| 229 | 229 | ||||
| 230 | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | 230 | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | ||
| 231 | - | _, exists := catalogue.Record(record.Name) | 231 | + | _, exists, _ := catalogue.Record(record.Name) |
| 232 | require.False(t, exists) | 232 | require.False(t, exists) | ||
| 233 | assertNoTombstones(t, repository) | 233 | assertNoTombstones(t, repository) | ||
| 234 | require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String())) | 234 | require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String())) | ||
| @@ -275,7 +275,7 @@ func TestDurableDiscardCrashMatrix(t *testing.T) { | |||||
| 275 | require.NoError(t, catalogue.Close()) | 275 | require.NoError(t, catalogue.Close()) | ||
| 276 | 276 | ||||
| 277 | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) { | 277 | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) { | ||
| 278 | - | got, exists := catalogue.Record(old.Name) | 278 | + | got, exists, _ := catalogue.Record(old.Name) |
| 279 | require.True(t, exists) | 279 | require.True(t, exists) | ||
| 280 | require.Equal(t, newID, got.IncarnationID) | 280 | require.Equal(t, newID, got.IncarnationID) | ||
| 281 | require.Equal(t, domain.RecoveryFresh, got.RecoveryState) | 281 | require.Equal(t, domain.RecoveryFresh, got.RecoveryState) | ||
| @@ -312,7 +312,7 @@ func TestDeletionRecoveryProtectsReusedNameFilesystemBytes(t *testing.T) { | |||||
| 312 | require.NoError(t, catalogue.Close()) | 312 | require.NoError(t, catalogue.Close()) | ||
| 313 | 313 | ||||
| 314 | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | 314 | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | ||
| 315 | - | got, exists := catalogue.Record("work") | 315 | + | got, exists, _ := catalogue.Record("work") |
| 316 | require.True(t, exists) | 316 | require.True(t, exists) | ||
| 317 | require.Equal(t, replacement, got) | 317 | require.Equal(t, replacement, got) | ||
| 318 | require.Equal(t, replacementBytes, readTree(t, replacementPath)) | 318 | require.Equal(t, replacementBytes, readTree(t, replacementPath)) | ||
M internal/app/offline_kill_retry_test.go +39 -46
| @@ -14,50 +14,43 @@ import ( | |||||
| 14 | ) | 14 | ) | ||
| 15 | 15 | ||||
| 16 | func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) { | 16 | func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) { | ||
| 17 | - | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | 17 | + | tests := []struct { |
| 18 | - | t.Setenv("XDG_STATE_HOME", stateRoot) | 18 | + | name string |
| 19 | - | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | 19 | + | crashHook func(*testing.T, string) |
| 20 | - | 20 | + | }{ | |
| 21 | - | id := domain.IncarnationID{1} | 21 | + | {name: "normal"}, |
| 22 | - | p := newTestPersister(t, filepath.Join(stateRoot, "vev")) | 22 | + | { |
| 23 | - | now := time.Now().UnixNano() | 23 | + | name: ".prev crash hook", |
| 24 | - | require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: id, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh})) | 24 | + | crashHook: func(t *testing.T, stateDir string) { |
| 25 | - | require.NoError(t, p.Close()) | 25 | + | t.Helper() |
| 26 | - | source := filepath.Join(stateRoot, "vev", "snapshots", "sessions", id.String()) | 26 | + | storePath := persist.StorePath(stateDir) |
| 27 | - | require.NoError(t, os.MkdirAll(source, 0o700)) | 27 | + | require.NoError(t, os.Rename(storePath, storePath+".prev")) |
| 28 | - | require.NoError(t, os.WriteFile(filepath.Join(source, "sentinel"), []byte("durable"), 0o600)) | 28 | + | }, |
| 29 | - | 29 | + | }, | |
| 30 | - | require.NoError(t, runKill(context.Background(), "named", false, false)) | 30 | + | } |
| 31 | - | require.NoFileExists(t, filepath.Join(source, "sentinel")) | 31 | + | |
| 32 | - | require.FileExists(t, filepath.Join(stateRoot, "vev", "snapshots", "quarantine", id.String(), "snapshot", "sentinel")) | 32 | + | for _, tt := range tests { |
| 33 | - | } | 33 | + | t.Run(tt.name, func(t *testing.T) { |
| 34 | - | 34 | + | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | |
| 35 | - | // TestRunKillOfflineFindsSessionWhenOnlyPrevCandidateExists covers the same | 35 | + | t.Setenv("XDG_STATE_HOME", stateRoot) |
| 36 | - | // bug class as persist.LoadCatalogueReadOnly's fresh-install regression: a | 36 | + | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) |
| 37 | - | // crash mid-compaction can leave sessions.kv absent with only a .prev (or | 37 | + | stateDir := filepath.Join(stateRoot, "vev") |
| 38 | - | // .next) candidate on disk. `vev kill <name>` must still find and delete the | 38 | + | |
| 39 | - | // session rather than reporting "no such session" from a bare os.Stat | 39 | + | id := domain.IncarnationID{1} |
| 40 | - | // precheck on the fixed sessions.kv path. | 40 | + | p := newTestPersister(t, stateDir) |
| 41 | - | func TestRunKillOfflineFindsSessionWhenOnlyPrevCandidateExists(t *testing.T) { | 41 | + | now := time.Now().UnixNano() |
| 42 | - | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | 42 | + | require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: id, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh})) |
| 43 | - | t.Setenv("XDG_STATE_HOME", stateRoot) | 43 | + | require.NoError(t, p.Close()) |
| 44 | - | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | 44 | + | source := filepath.Join(stateDir, "snapshots", "sessions", id.String()) |
| 45 | - | 45 | + | require.NoError(t, os.MkdirAll(source, 0o700)) | |
| 46 | - | id := domain.IncarnationID{1} | 46 | + | require.NoError(t, os.WriteFile(filepath.Join(source, "sentinel"), []byte("durable"), 0o600)) |
| 47 | - | p := newTestPersister(t, filepath.Join(stateRoot, "vev")) | 47 | + | if tt.crashHook != nil { |
| 48 | - | now := time.Now().UnixNano() | 48 | + | tt.crashHook(t, stateDir) |
| 49 | - | require.NoError(t, p.Save(persist.Record{Name: "crashed", IncarnationID: id, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh})) | 49 | + | } |
| 50 | - | require.NoError(t, p.Close()) | 50 | + | |
| 51 | - | source := filepath.Join(stateRoot, "vev", "snapshots", "sessions", id.String()) | 51 | + | require.NoError(t, runKill(context.Background(), "named", false, false)) |
| 52 | - | require.NoError(t, os.MkdirAll(source, 0o700)) | 52 | + | require.NoFileExists(t, filepath.Join(source, "sentinel")) |
| 53 | - | require.NoError(t, os.WriteFile(filepath.Join(source, "sentinel"), []byte("durable"), 0o600)) | 53 | + | require.FileExists(t, filepath.Join(stateDir, "snapshots", "quarantine", id.String(), "snapshot", "sentinel")) |
| 54 | - | 54 | + | }) | |
| 55 | - | // Simulate a crash mid-compaction: current renamed away to .prev, no | 55 | + | } |
| 56 | - | // sessions.kv on disk at all. | |||
| 57 | - | storePath := persist.StorePath(filepath.Join(stateRoot, "vev")) | |||
| 58 | - | require.NoError(t, os.Rename(storePath, storePath+".prev")) | |||
| 59 | - | ||||
| 60 | - | require.NoError(t, runKill(context.Background(), "crashed", false, false)) | |||
| 61 | - | require.NoFileExists(t, filepath.Join(source, "sentinel")) | |||
| 62 | - | require.FileExists(t, filepath.Join(stateRoot, "vev", "snapshots", "quarantine", id.String(), "snapshot", "sentinel")) | |||
| 63 | } | 56 | } | ||
M internal/app/run.go +29 -8
| @@ -583,18 +583,22 @@ type fencedRecoveryIdentity struct { | |||||
| 583 | 583 | ||||
| 584 | // fencedRecoveries resolves each conflict against the post-recovery catalogue | 584 | // fencedRecoveries resolves each conflict against the post-recovery catalogue | ||
| 585 | // so a fenced session is reported with the incarnation it actually holds now. | 585 | // so a fenced session is reported with the incarnation it actually holds now. | ||
| 586 | - | func fencedRecoveries(conflicts []recovery.RecoveryConflict, catalogue ports.Catalogue) []fencedRecoveryIdentity { | 586 | + | func fencedRecoveries(conflicts []recovery.RecoveryConflict, catalogue ports.Catalogue) ([]fencedRecoveryIdentity, error) { |
| 587 | fenced := make([]fencedRecoveryIdentity, 0, len(conflicts)) | 587 | fenced := make([]fencedRecoveryIdentity, 0, len(conflicts)) | ||
| 588 | for _, conflict := range conflicts { | 588 | for _, conflict := range conflicts { | ||
| 589 | identity := fencedRecoveryIdentity{name: conflict.Session, kind: conflict.Kind, reasonCode: fencedRecoveryReasonCode(conflict.Err)} | 589 | identity := fencedRecoveryIdentity{name: conflict.Session, kind: conflict.Kind, reasonCode: fencedRecoveryReasonCode(conflict.Err)} | ||
| 590 | if catalogue != nil { | 590 | if catalogue != nil { | ||
| 591 | - | if record, ok := catalogue.Record(conflict.Session); ok { | 591 | + | record, ok, err := catalogue.Record(conflict.Session) |
| 592 | + | if err != nil { | |||
| 593 | + | return nil, err | |||
| 594 | + | } | |||
| 595 | + | if ok { | |||
| 592 | identity.id = record.IncarnationID | 596 | identity.id = record.IncarnationID | ||
| 593 | } | 597 | } | ||
| 594 | } | 598 | } | ||
| 595 | fenced = append(fenced, identity) | 599 | fenced = append(fenced, identity) | ||
| 596 | } | 600 | } | ||
| 597 | - | return fenced | 601 | + | return fenced, nil |
| 598 | } | 602 | } | ||
| 599 | 603 | ||||
| 600 | func fencedRecoveryReasonCode(err error) string { | 604 | func fencedRecoveryReasonCode(err error) string { | ||
| @@ -674,6 +678,13 @@ func runDaemonOwned(ctx context.Context) (retErr error) { | |||||
| 674 | return runDaemonOwnedWithLogger(ctx, log) | 678 | return runDaemonOwnedWithLogger(ctx, log) | ||
| 675 | } | 679 | } | ||
| 676 | 680 | ||||
| 681 | + | func recoverAuthoritativeCatalogue(ctx context.Context, coordinator *recovery.Coordinator, catalogue ports.Catalogue) ([]domain.CatalogueRecord, error) { | |||
| 682 | + | if err := coordinator.Recover(ctx); err != nil { | |||
| 683 | + | return nil, err | |||
| 684 | + | } | |||
| 685 | + | return catalogue.Records() | |||
| 686 | + | } | |||
| 687 | + | ||||
| 677 | func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr error) { | 688 | func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr error) { | ||
| 678 | clk := clock.New() | 689 | clk := clock.New() | ||
| 679 | observer, observerCloser, err := newPerformanceTrace(clk) | 690 | observer, observerCloser, err := newPerformanceTrace(clk) | ||
| @@ -710,7 +721,7 @@ func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr err | |||||
| 710 | daemonOpts = append(daemonOpts, daemon.WithBarScriptCommandRunner(shellcmd.New())) | 721 | daemonOpts = append(daemonOpts, daemon.WithBarScriptCommandRunner(shellcmd.New())) | ||
| 711 | daemonOpts = append(daemonOpts, daemon.WithProcessInspector(platform.NewProcessInspector()), daemon.WithDirOrHome(platform.DirOrHome)) | 722 | daemonOpts = append(daemonOpts, daemon.WithProcessInspector(platform.NewProcessInspector()), daemon.WithDirOrHome(platform.DirOrHome)) | ||
| 712 | snapshotRepository := snapshotadapter.NewRepository(snapshotDir(), log) | 723 | snapshotRepository := snapshotadapter.NewRepository(snapshotDir(), log) | ||
| 713 | - | daemonOpts = append(daemonOpts, daemon.WithSnapshotRepository(snapshotRepository, snapshotRepository)) | 724 | + | daemonOpts = append(daemonOpts, daemon.WithSnapshotRepository(snapshotRepository)) |
| 714 | noticeStore := noticefile.New(platform.StateDir()) | 725 | noticeStore := noticefile.New(platform.StateDir()) | ||
| 715 | daemonOpts = append(daemonOpts, daemon.WithNoticeStore(noticeStore)) | 726 | daemonOpts = append(daemonOpts, daemon.WithNoticeStore(noticeStore)) | ||
| 716 | storePath := persist.StorePath(platform.StateDir()) | 727 | storePath := persist.StorePath(platform.StateDir()) | ||
| @@ -736,13 +747,21 @@ func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr err | |||||
| 736 | return errors.Join(fmt.Errorf("vev: inspect durable session transactions: %w", err), opened.Catalogue.Close()) | 747 | return errors.Join(fmt.Errorf("vev: inspect durable session transactions: %w", err), opened.Catalogue.Close()) | ||
| 737 | } | 748 | } | ||
| 738 | coordinator := recovery.NewCoordinator(opened.Catalogue, snapshotRepository, journal, rand.Reader) | 749 | coordinator := recovery.NewCoordinator(opened.Catalogue, snapshotRepository, journal, rand.Reader) | ||
| 739 | - | if err := coordinator.Recover(ctx); err != nil { | 750 | + | // Recovery may roll catalogue changes forward, so reload the authoritative |
| 740 | - | return errors.Join(fmt.Errorf("vev: recover durable session transactions: %w", err), opened.Catalogue.Close()) | 751 | + | // snapshot before constructing runtime reservations. A failed reload aborts |
| 752 | + | // startup rather than treating durable sessions as absent. | |||
| 753 | + | recoveredRecords, err := recoverAuthoritativeCatalogue(ctx, coordinator, opened.Catalogue) | |||
| 754 | + | if err != nil { | |||
| 755 | + | return errors.Join(fmt.Errorf("vev: recover and reload durable session transactions: %w", err), opened.Catalogue.Close()) | |||
| 741 | } | 756 | } | ||
| 757 | + | opened.Records = recoveredRecords | |||
| 742 | // Recovery fences a session instead of failing startup when only that | 758 | // Recovery fences a session instead of failing startup when only that | ||
| 743 | // session's durable state is self-inconsistent. The daemon then comes up | 759 | // session's durable state is self-inconsistent. The daemon then comes up | ||
| 744 | // looking healthy, so every fenced session must be reported and persisted. | 760 | // looking healthy, so every fenced session must be reported and persisted. | ||
| 745 | - | fenced := fencedRecoveries(coordinator.Conflicts(), opened.Catalogue) | 761 | + | fenced, err := fencedRecoveries(coordinator.Conflicts(), opened.Catalogue) |
| 762 | + | if err != nil { | |||
| 763 | + | return errors.Join(fmt.Errorf("vev: read fenced recovery identities: %w", err), opened.Catalogue.Close()) | |||
| 764 | + | } | |||
| 746 | logFencedRecoveries(log, fenced) | 765 | logFencedRecoveries(log, fenced) | ||
| 747 | if err := persistTransactionRecoveryNotices(noticeStore, pendingRecoveries, clk.Now()); err != nil { | 766 | if err := persistTransactionRecoveryNotices(noticeStore, pendingRecoveries, clk.Now()); err != nil { | ||
| 748 | log.Error("transaction_recovery_notice_failed", "err", err) | 767 | log.Error("transaction_recovery_notice_failed", "err", err) | ||
| @@ -1558,7 +1577,9 @@ func runOfflineNamedKill(ctx context.Context, name string) (retErr error) { | |||||
| 1558 | } | 1577 | } | ||
| 1559 | defer func() { retErr = errors.Join(retErr, opened.Catalogue.Close()) }() | 1578 | defer func() { retErr = errors.Join(retErr, opened.Catalogue.Close()) }() | ||
| 1560 | 1579 | ||||
| 1561 | - | if _, ok := opened.Catalogue.Record(name); !ok { | 1580 | + | if _, ok, err := opened.Catalogue.Record(name); err != nil { |
| 1581 | + | return fmt.Errorf("vev: reading stored session: %w", err) | |||
| 1582 | + | } else if !ok { | |||
| 1562 | return fmt.Errorf("vev: no such session: %s", name) | 1583 | return fmt.Errorf("vev: no such session: %s", name) | ||
| 1563 | } | 1584 | } | ||
| 1564 | coordinator := recovery.NewCoordinator(opened.Catalogue, repository, recoveryfs.New(platform.StateDir()), rand.Reader) | 1585 | coordinator := recovery.NewCoordinator(opened.Catalogue, repository, recoveryfs.New(platform.StateDir()), rand.Reader) | ||
M internal/app/run_test.go +52 -19
| @@ -19,6 +19,7 @@ import ( | |||||
| 19 | "time" | 19 | "time" | ||
| 20 | 20 | ||||
| 21 | "github.com/bnema/vev/internal/adapters/lifecycle" | 21 | "github.com/bnema/vev/internal/adapters/lifecycle" | ||
| 22 | + | "github.com/bnema/vev/internal/adapters/noticefile" | |||
| 22 | "github.com/bnema/vev/internal/adapters/recoveryfs" | 23 | "github.com/bnema/vev/internal/adapters/recoveryfs" | ||
| 23 | "github.com/bnema/vev/internal/adapters/snapshot" | 24 | "github.com/bnema/vev/internal/adapters/snapshot" | ||
| 24 | "github.com/bnema/vev/internal/domain" | 25 | "github.com/bnema/vev/internal/domain" | ||
| @@ -110,9 +111,6 @@ func TestRecoveryObservability(t *testing.T) { | |||||
| 110 | require.Equal(t, "clean", recovered["outcome"]) | 111 | require.Equal(t, "clean", recovered["outcome"]) | ||
| 111 | require.NotContains(t, eventNames(entries), "interrupted_transaction_fenced") | 112 | require.NotContains(t, eventNames(entries), "interrupted_transaction_fenced") | ||
| 112 | 113 | ||||
| 113 | - | logBuffer.Reset() | |||
| 114 | - | log.Error("catalogue_validation_failed", "path", "/state/vev/sessions.kv", "reason_code", "corrupt") | |||
| 115 | - | require.NotContains(t, eventNames(decodeJSONLogs(t, logBuffer.Bytes())), "daemon_startup_complete") | |||
| 116 | require.NotContains(t, fmt.Sprint(requireEvent(t, entries, "interrupted_transaction_recovery_complete")), "terminal contents") | 114 | require.NotContains(t, fmt.Sprint(requireEvent(t, entries, "interrupted_transaction_recovery_complete")), "terminal contents") | ||
| 117 | require.Equal(t, domain.NoticeSnapshotRestore, recoveryNotice.Code) | 115 | require.Equal(t, domain.NoticeSnapshotRestore, recoveryNotice.Code) | ||
| 118 | require.Equal(t, domain.SessionID(domain.IncarnationID{9}.String()), recoveryNotice.SessionID) | 116 | require.Equal(t, domain.SessionID(domain.IncarnationID{9}.String()), recoveryNotice.SessionID) | ||
| @@ -122,6 +120,27 @@ func TestRecoveryObservability(t *testing.T) { | |||||
| 122 | // A session-scoped conflict no longer aborts startup, so the daemon comes up | 120 | // A session-scoped conflict no longer aborts startup, so the daemon comes up | ||
| 123 | // looking healthy. Startup must therefore name the fenced session in the log, | 121 | // looking healthy. Startup must therefore name the fenced session in the log, | ||
| 124 | // persist a notice for it, and never report the recovery as clean. | 122 | // persist a notice for it, and never report the recovery as clean. | ||
| 123 | + | func TestStartupReloadsCatalogueAfterRolledForwardDeletion(t *testing.T) { | |||
| 124 | + | ctx := t.Context() | |||
| 125 | + | stateDir := filepath.Join(t.TempDir(), "vev") | |||
| 126 | + | catalogue := newTestPersister(t, stateDir) | |||
| 127 | + | t.Cleanup(func() { _ = catalogue.Close() }) | |||
| 128 | + | repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots")) | |||
| 129 | + | record := domain.CatalogueRecord{ | |||
| 130 | + | Name: "work", IncarnationID: domain.IncarnationID{9}, Cwd: t.TempDir(), | |||
| 131 | + | CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryDeleting, | |||
| 132 | + | } | |||
| 133 | + | require.NoError(t, catalogue.Save(record)) | |||
| 134 | + | coordinator := recovery.NewCoordinator(catalogue, repository, recoveryfs.New(stateDir), rand.Reader) | |||
| 135 | + | ||||
| 136 | + | records, err := recoverAuthoritativeCatalogue(ctx, coordinator, catalogue) | |||
| 137 | + | require.NoError(t, err) | |||
| 138 | + | require.Empty(t, records, "daemon startup must not retain the pre-recovery reservation") | |||
| 139 | + | _, exists, err := catalogue.Record(record.Name) | |||
| 140 | + | require.NoError(t, err) | |||
| 141 | + | require.False(t, exists, "rolled-forward deletion must remain absent in authoritative state") | |||
| 142 | + | } | |||
| 143 | + | ||||
| 125 | func TestFencedRecoveryObservability(t *testing.T) { | 144 | func TestFencedRecoveryObservability(t *testing.T) { | ||
| 126 | ctx := context.Background() | 145 | ctx := context.Background() | ||
| 127 | stateDir := filepath.Join(t.TempDir(), "vev") | 146 | stateDir := filepath.Join(t.TempDir(), "vev") | ||
| @@ -139,7 +158,8 @@ func TestFencedRecoveryObservability(t *testing.T) { | |||||
| 139 | 158 | ||||
| 140 | coordinator := recovery.NewCoordinator(catalogue, repository, recoveryfs.New(stateDir), rand.Reader) | 159 | coordinator := recovery.NewCoordinator(catalogue, repository, recoveryfs.New(stateDir), rand.Reader) | ||
| 141 | require.NoError(t, coordinator.Recover(ctx), "a session-scoped conflict must not abort startup") | 160 | require.NoError(t, coordinator.Recover(ctx), "a session-scoped conflict must not abort startup") | ||
| 142 | - | fenced := fencedRecoveries(coordinator.Conflicts(), catalogue) | 161 | + | fenced, err := fencedRecoveries(coordinator.Conflicts(), catalogue) |
| 162 | + | require.NoError(t, err) | |||
| 143 | require.Equal(t, []fencedRecoveryIdentity{{ | 163 | require.Equal(t, []fencedRecoveryIdentity{{ | ||
| 144 | name: live.Name, id: live.IncarnationID, kind: "deletion-tombstone", reasonCode: "deletion-tombstone-conflict", | 164 | name: live.Name, id: live.IncarnationID, kind: "deletion-tombstone", reasonCode: "deletion-tombstone-conflict", | ||
| 145 | }}, fenced) | 165 | }}, fenced) | ||
| @@ -148,7 +168,9 @@ func TestFencedRecoveryObservability(t *testing.T) { | |||||
| 148 | log := slog.New(slog.NewJSONHandler(&logBuffer, nil)) | 168 | log := slog.New(slog.NewJSONHandler(&logBuffer, nil)) | ||
| 149 | logFencedRecoveries(log, fenced) | 169 | logFencedRecoveries(log, fenced) | ||
| 150 | logTransactionRecovery(log, 1, len(fenced)) | 170 | logTransactionRecovery(log, 1, len(fenced)) | ||
| 151 | - | logStartupRecoveryCounts(log, catalogue.Records(), 0) | 171 | + | records, err := catalogue.Records() |
| 172 | + | require.NoError(t, err) | |||
| 173 | + | logStartupRecoveryCounts(log, records, 0) | |||
| 152 | 174 | ||||
| 153 | entries := decodeJSONLogs(t, logBuffer.Bytes()) | 175 | entries := decodeJSONLogs(t, logBuffer.Bytes()) | ||
| 154 | event := requireEvent(t, entries, "interrupted_transaction_fenced") | 176 | event := requireEvent(t, entries, "interrupted_transaction_fenced") | ||
| @@ -163,17 +185,22 @@ func TestFencedRecoveryObservability(t *testing.T) { | |||||
| 163 | require.Equal(t, "fenced", recovered["outcome"]) | 185 | require.Equal(t, "fenced", recovered["outcome"]) | ||
| 164 | require.Contains(t, eventNames(entries), "daemon_startup_complete") | 186 | require.Contains(t, eventNames(entries), "daemon_startup_complete") | ||
| 165 | 187 | ||||
| 166 | - | store := portsmocks.NewMockNoticeStore(t) | 188 | + | store := noticefile.New(stateDir) |
| 167 | - | var notice domain.Notification | |||
| 168 | - | store.EXPECT().Append(mock.Anything).Run(func(n domain.Notification) { notice = n }).Return(nil).Once() | |||
| 169 | require.NoError(t, persistFencedRecoveryNotices(store, fenced, time.Unix(1, 0))) | 189 | require.NoError(t, persistFencedRecoveryNotices(store, fenced, time.Unix(1, 0))) | ||
| 190 | + | reopened := noticefile.New(stateDir) | |||
| 191 | + | pending, err := reopened.Claim() | |||
| 192 | + | require.NoError(t, err) | |||
| 193 | + | require.Len(t, pending, 1) | |||
| 194 | + | notice := pending[0] | |||
| 170 | require.Equal(t, domain.NoticeSnapshotRestore, notice.Code) | 195 | require.Equal(t, domain.NoticeSnapshotRestore, notice.Code) | ||
| 171 | require.Equal(t, domain.NoticeWarn, notice.Severity) | 196 | require.Equal(t, domain.NoticeWarn, notice.Severity) | ||
| 172 | require.Equal(t, domain.SessionID(live.IncarnationID.String()), notice.SessionID) | 197 | require.Equal(t, domain.SessionID(live.IncarnationID.String()), notice.SessionID) | ||
| 173 | require.Contains(t, notice.Details, "session="+live.Name) | 198 | require.Contains(t, notice.Details, "session="+live.Name) | ||
| 174 | require.Contains(t, notice.Details, "reason_code=deletion-tombstone-conflict") | 199 | require.Contains(t, notice.Details, "reason_code=deletion-tombstone-conflict") | ||
| 200 | + | require.NoError(t, reopened.Ack()) | |||
| 175 | 201 | ||||
| 176 | - | degraded, ok := catalogue.Record(live.Name) | 202 | + | degraded, ok, err := catalogue.Record(live.Name) |
| 203 | + | require.NoError(t, err) | |||
| 177 | require.True(t, ok) | 204 | require.True(t, ok) | ||
| 178 | require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState) | 205 | require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState) | ||
| 179 | } | 206 | } | ||
| @@ -444,19 +471,25 @@ func TestLifecycleOwnershipPrecedesDaemonStartup(t *testing.T) { | |||||
| 444 | stateDir := filepath.Join(stateRoot, "vev") | 471 | stateDir := filepath.Join(stateRoot, "vev") | ||
| 445 | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | 472 | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | ||
| 446 | t.Setenv("XDG_STATE_HOME", stateRoot) | 473 | t.Setenv("XDG_STATE_HOME", stateRoot) | ||
| 447 | - | ctx, cancel := context.WithCancel(context.Background()) | 474 | + | ctx, cancel := context.WithCancel(t.Context()) |
| 448 | - | defer cancel() | 475 | + | result := make(chan error, 1) |
| 449 | go func() { | 476 | go func() { | ||
| 450 | - | for { | 477 | + | result <- runWithLifecycleOwner(ctx, runtimeDir, stateDir, runDaemonOwned) |
| 451 | - | if _, err := os.Stat(filepath.Join(runtimeDir, "daemon.sock")); err == nil { | |||
| 452 | - | cancel() | |||
| 453 | - | return | |||
| 454 | - | } | |||
| 455 | - | time.Sleep(time.Millisecond) | |||
| 456 | - | } | |||
| 457 | }() | 478 | }() | ||
| 479 | + | t.Cleanup(func() { | |||
| 480 | + | cancel() | |||
| 481 | + | select { | |||
| 482 | + | case err := <-result: | |||
| 483 | + | require.NoError(t, err) | |||
| 484 | + | case <-time.After(5 * time.Second): | |||
| 485 | + | t.Error("daemon lifecycle did not stop after cancellation") | |||
| 486 | + | } | |||
| 487 | + | }) | |||
| 458 | 488 | ||||
| 459 | - | require.NoError(t, runWithLifecycleOwner(ctx, runtimeDir, stateDir, runDaemonOwned)) | 489 | + | require.Eventually(t, func() bool { |
| 490 | + | _, err := os.Stat(filepath.Join(runtimeDir, "daemon.sock")) | |||
| 491 | + | return err == nil | |||
| 492 | + | }, 5*time.Second, time.Millisecond, "daemon socket was not published") | |||
| 460 | require.FileExists(t, persist.StorePath(stateDir)) | 493 | require.FileExists(t, persist.StorePath(stateDir)) | ||
| 461 | }) | 494 | }) | ||
| 462 | 495 | ||||
M internal/app/spawn.go +45 -49
| @@ -67,25 +67,29 @@ var defaultBackoff = backoffConfig{ | |||||
| 67 | total: 5 * time.Second, | 67 | total: 5 * time.Second, | ||
| 68 | } | 68 | } | ||
| 69 | 69 | ||||
| 70 | - | func waitForLifecycleAvailability(ctx context.Context, runtimeDir string, cfg backoffConfig) (lifecycleOwnership, error) { | 70 | + | func retryAttempts[T any](ctx context.Context, cfg backoffConfig, attempt func() (T, bool, error)) (T, error) { |
| 71 | deadline := time.Now().Add(cfg.total) | 71 | deadline := time.Now().Add(cfg.total) | ||
| 72 | backoff := cfg.initial | 72 | backoff := cfg.initial | ||
| 73 | for { | 73 | for { | ||
| 74 | if err := ctx.Err(); err != nil { | 74 | if err := ctx.Err(); err != nil { | ||
| 75 | - | return nil, err | 75 | + | var zero T |
| 76 | + | return zero, err | |||
| 76 | } | 77 | } | ||
| 77 | - | owner, err := daemonLifecycleProbe.TryAcquire(runtimeDir) | 78 | + | result, done, err := attempt() |
| 78 | - | if err == nil { | 79 | + | if err != nil { |
| 79 | - | return owner, nil | 80 | + | var zero T |
| 81 | + | return zero, err | |||
| 80 | } | 82 | } | ||
| 81 | - | if !errors.Is(err, lifecycle.ErrBusy) { | 83 | + | if done { |
| 82 | - | return nil, err | 84 | + | return result, nil |
| 83 | } | 85 | } | ||
| 84 | if !time.Now().Before(deadline) { | 86 | if !time.Now().Before(deadline) { | ||
| 85 | - | return nil, ErrDaemonUnreachable | 87 | + | var zero T |
| 88 | + | return zero, ErrDaemonUnreachable | |||
| 86 | } | 89 | } | ||
| 87 | if err := waitBackoff(ctx, backoff); err != nil { | 90 | if err := waitBackoff(ctx, backoff); err != nil { | ||
| 88 | - | return nil, err | 91 | + | var zero T |
| 92 | + | return zero, err | |||
| 89 | } | 93 | } | ||
| 90 | if backoff *= 2; backoff > cfg.max { | 94 | if backoff *= 2; backoff > cfg.max { | ||
| 91 | backoff = cfg.max | 95 | backoff = cfg.max | ||
| @@ -93,33 +97,39 @@ func waitForLifecycleAvailability(ctx context.Context, runtimeDir string, cfg ba | |||||
| 93 | } | 97 | } | ||
| 94 | } | 98 | } | ||
| 95 | 99 | ||||
| 96 | - | func waitForDaemonOrLifecycle(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, lifecycleOwnership, error) { | 100 | + | func waitForLifecycleAvailability(ctx context.Context, runtimeDir string, cfg backoffConfig) (lifecycleOwnership, error) { |
| 97 | - | deadline := time.Now().Add(cfg.total) | 101 | + | return retryAttempts(ctx, cfg, func() (lifecycleOwnership, bool, error) { |
| 98 | - | backoff := cfg.initial | 102 | + | owner, err := daemonLifecycleProbe.TryAcquire(runtimeDir) |
| 99 | - | for { | 103 | + | if err == nil { |
| 100 | - | if err := ctx.Err(); err != nil { | 104 | + | return owner, true, nil |
| 101 | - | return nil, nil, err | 105 | + | } |
| 106 | + | if !errors.Is(err, lifecycle.ErrBusy) { | |||
| 107 | + | return nil, false, err | |||
| 102 | } | 108 | } | ||
| 109 | + | return nil, false, nil | |||
| 110 | + | }) | |||
| 111 | + | } | |||
| 112 | + | ||||
| 113 | + | type daemonOrLifecycle struct { | |||
| 114 | + | transport ports.Transport | |||
| 115 | + | owner lifecycleOwnership | |||
| 116 | + | } | |||
| 117 | + | ||||
| 118 | + | func waitForDaemonOrLifecycle(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, lifecycleOwnership, error) { | |||
| 119 | + | result, err := retryAttempts(ctx, cfg, func() (daemonOrLifecycle, bool, error) { | |||
| 103 | if transport, err := dial(ctx, dir); err == nil { | 120 | if transport, err := dial(ctx, dir); err == nil { | ||
| 104 | - | return transport, nil, nil | 121 | + | return daemonOrLifecycle{transport: transport}, true, nil |
| 105 | } | 122 | } | ||
| 106 | owner, err := daemonLifecycleProbe.TryAcquire(dir) | 123 | owner, err := daemonLifecycleProbe.TryAcquire(dir) | ||
| 107 | if err == nil { | 124 | if err == nil { | ||
| 108 | - | return nil, owner, nil | 125 | + | return daemonOrLifecycle{owner: owner}, true, nil |
| 109 | } | 126 | } | ||
| 110 | if !errors.Is(err, lifecycle.ErrBusy) { | 127 | if !errors.Is(err, lifecycle.ErrBusy) { | ||
| 111 | - | return nil, nil, err | 128 | + | return daemonOrLifecycle{}, false, err |
| 112 | } | 129 | } | ||
| 113 | - | if !time.Now().Before(deadline) { | 130 | + | return daemonOrLifecycle{}, false, nil |
| 114 | - | return nil, nil, ErrDaemonUnreachable | 131 | + | }) |
| 115 | - | } | 132 | + | return result.transport, result.owner, err |
| 116 | - | if err := waitBackoff(ctx, backoff); err != nil { | |||
| 117 | - | return nil, nil, err | |||
| 118 | - | } | |||
| 119 | - | if backoff *= 2; backoff > cfg.max { | |||
| 120 | - | backoff = cfg.max | |||
| 121 | - | } | |||
| 122 | - | } | |||
| 123 | } | 133 | } | ||
| 124 | 134 | ||||
| 125 | func waitBackoff(ctx context.Context, duration time.Duration) error { | 135 | func waitBackoff(ctx context.Context, duration time.Duration) error { | ||
| @@ -224,28 +234,14 @@ func acquireSpawnLock(dir string) (release func(), acquired bool, err error) { | |||||
| 224 | // retryDial dials repeatedly with exponential backoff until the daemon | 234 | // retryDial dials repeatedly with exponential backoff until the daemon | ||
| 225 | // answers, the context is cancelled, or the total budget is exhausted. | 235 | // answers, the context is cancelled, or the total budget is exhausted. | ||
| 226 | func retryDial(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, error) { | 236 | func retryDial(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, error) { | ||
| 227 | - | deadline := time.Now().Add(cfg.total) | 237 | + | transport, err := retryAttempts(ctx, cfg, func() (ports.Transport, bool, error) { |
| 228 | - | backoff := cfg.initial | 238 | + | transport, err := dial(ctx, dir) |
| 229 | - | for { | 239 | + | return transport, err == nil, nil |
| 230 | - | if err := ctx.Err(); err != nil { | 240 | + | }) |
| 231 | - | return nil, err | 241 | + | if errors.Is(err, ErrDaemonUnreachable) { |
| 232 | - | } | 242 | + | slog.Error("daemon did not become reachable before retry budget expired", "socket_dir", dir, "budget", cfg.total) |
| 233 | - | if t, err := dial(ctx, dir); err == nil { | |||
| 234 | - | return t, nil | |||
| 235 | - | } | |||
| 236 | - | if !time.Now().Before(deadline) { | |||
| 237 | - | slog.Error("daemon did not become reachable before retry budget expired", "socket_dir", dir, "budget", cfg.total) | |||
| 238 | - | return nil, ErrDaemonUnreachable | |||
| 239 | - | } | |||
| 240 | - | select { | |||
| 241 | - | case <-ctx.Done(): | |||
| 242 | - | return nil, ctx.Err() | |||
| 243 | - | case <-time.After(backoff): | |||
| 244 | - | } | |||
| 245 | - | if backoff *= 2; backoff > cfg.max { | |||
| 246 | - | backoff = cfg.max | |||
| 247 | - | } | |||
| 248 | } | 243 | } | ||
| 244 | + | return transport, err | |||
| 249 | } | 245 | } | ||
| 250 | 246 | ||||
| 251 | // realDial is the production dialer. | 247 | // realDial is the production dialer. | ||
M internal/app/spawn_test.go +15 -0
| @@ -258,6 +258,21 @@ func TestAcquireSpawnLockTakesOverStaleLock(t *testing.T) { | |||||
| 258 | release() | 258 | release() | ||
| 259 | } | 259 | } | ||
| 260 | 260 | ||||
| 261 | + | func TestRetryAttemptsChecksOperationErrorBeforeDeadline(t *testing.T) { | |||
| 262 | + | attemptErr := errors.New("attempt failed") | |||
| 263 | + | calls := 0 | |||
| 264 | + | _, err := retryAttempts(context.Background(), backoffConfig{}, func() (struct{}, bool, error) { | |||
| 265 | + | calls++ | |||
| 266 | + | return struct{}{}, false, attemptErr | |||
| 267 | + | }) | |||
| 268 | + | if !errors.Is(err, attemptErr) { | |||
| 269 | + | t.Fatalf("retryAttempts error = %v, want %v", err, attemptErr) | |||
| 270 | + | } | |||
| 271 | + | if calls != 1 { | |||
| 272 | + | t.Fatalf("attempt called %d times, want 1", calls) | |||
| 273 | + | } | |||
| 274 | + | } | |||
| 275 | + | ||||
| 261 | func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) { | 276 | func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) { | ||
| 262 | want := portsmocks.NewMockTransport(t) | 277 | want := portsmocks.NewMockTransport(t) | ||
| 263 | var calls atomic.Int32 | 278 | var calls atomic.Int32 | ||
M internal/domain/recovery.go +19 -0
| @@ -5,6 +5,7 @@ import ( | |||||
| 5 | "errors" | 5 | "errors" | ||
| 6 | "fmt" | 6 | "fmt" | ||
| 7 | "io" | 7 | "io" | ||
| 8 | + | "slices" | |||
| 8 | "strings" | 9 | "strings" | ||
| 9 | ) | 10 | ) | ||
| 10 | 11 | ||||
| @@ -44,6 +45,24 @@ type CatalogueRecord struct { | |||||
| 44 | 45 | ||||
| 45 | // CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue | 46 | // CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue | ||
| 46 | // incarnation without carrying authority-owned recovery or checkpoint fields. | 47 | // incarnation without carrying authority-owned recovery or checkpoint fields. | ||
| 48 | + | // Equal reports canonical value equality between checkpoint references. | |||
| 49 | + | func (r *CheckpointRef) Equal(other *CheckpointRef) bool { | |||
| 50 | + | if r == nil || other == nil { | |||
| 51 | + | return r == nil && other == nil | |||
| 52 | + | } | |||
| 53 | + | return *r == *other | |||
| 54 | + | } | |||
| 55 | + | ||||
| 56 | + | // Equal reports canonical value equality, including ordered tab names and | |||
| 57 | + | // pointed-to checkpoint values rather than pointer identity. | |||
| 58 | + | func (r CatalogueRecord) Equal(other CatalogueRecord) bool { | |||
| 59 | + | return r.Name == other.Name && r.IncarnationID == other.IncarnationID && r.Cwd == other.Cwd && | |||
| 60 | + | r.CreatedAt == other.CreatedAt && r.UpdatedAt == other.UpdatedAt && r.LastUsedSeq == other.LastUsedSeq && | |||
| 61 | + | slices.Equal(r.TabNames, other.TabNames) && r.RecoveryState == other.RecoveryState && | |||
| 62 | + | r.Committed.Equal(other.Committed) && r.Fallbacks[0].Equal(other.Fallbacks[0]) && | |||
| 63 | + | r.Fallbacks[1].Equal(other.Fallbacks[1]) && r.DegradedReason == other.DegradedReason | |||
| 64 | + | } | |||
| 65 | + | ||||
| 47 | type CatalogueMetadataUpdate struct { | 66 | type CatalogueMetadataUpdate struct { | ||
| 48 | Name string | 67 | Name string | ||
| 49 | IncarnationID IncarnationID | 68 | IncarnationID IncarnationID | ||
M internal/domain/recovery_test.go +22 -1
| @@ -22,7 +22,12 @@ func TestIncarnationIDText(t *testing.T) { | |||||
| 22 | var decoded IncarnationID | 22 | var decoded IncarnationID | ||
| 23 | require.NoError(t, decoded.UnmarshalText(text)) | 23 | require.NoError(t, decoded.UnmarshalText(text)) | ||
| 24 | require.Equal(t, id, decoded) | 24 | require.Equal(t, id, decoded) | ||
| 25 | - | require.Error(t, decoded.UnmarshalText([]byte("AB"))) | 25 | + | require.Error(t, decoded.UnmarshalText([]byte("AB0000000000000000000000000000CD"))) |
| 26 | + | ||||
| 27 | + | _, err = (IncarnationID{}).MarshalText() | |||
| 28 | + | require.Error(t, err) | |||
| 29 | + | require.Error(t, decoded.UnmarshalText([]byte("00000000000000000000000000000000"))) | |||
| 30 | + | require.Equal(t, id, decoded, "rejected zero text must not mutate the receiver") | |||
| 26 | } | 31 | } | ||
| 27 | 32 | ||||
| 28 | func TestCatalogueRecordValidate(t *testing.T) { | 33 | func TestCatalogueRecordValidate(t *testing.T) { | ||
| @@ -50,6 +55,7 @@ func TestCatalogueRecordValidate(t *testing.T) { | |||||
| 50 | {name: "zero checkpoint generation", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{ManifestDigest: [32]byte{1}}}}, | 55 | {name: "zero checkpoint generation", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{ManifestDigest: [32]byte{1}}}}, | ||
| 51 | {name: "zero checkpoint digest", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7}}}, | 56 | {name: "zero checkpoint digest", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7}}}, | ||
| 52 | {name: "second fallback without first", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: validRef, Fallbacks: [2]*CheckpointRef{nil, {Generation: 6, ManifestDigest: [32]byte{2}}}}}, | 57 | {name: "second fallback without first", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: validRef, Fallbacks: [2]*CheckpointRef{nil, {Generation: 6, ManifestDigest: [32]byte{2}}}}}, | ||
| 58 | + | {name: "fallback newer than committed", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 6, ManifestDigest: [32]byte{1}}, Fallbacks: [2]*CheckpointRef{{Generation: 7, ManifestDigest: [32]byte{2}}}}}, | |||
| 53 | } | 59 | } | ||
| 54 | for _, tt := range tests { | 60 | for _, tt := range tests { | ||
| 55 | t.Run(tt.name, func(t *testing.T) { | 61 | t.Run(tt.name, func(t *testing.T) { | ||
| @@ -58,6 +64,21 @@ func TestCatalogueRecordValidate(t *testing.T) { | |||||
| 58 | } | 64 | } | ||
| 59 | } | 65 | } | ||
| 60 | 66 | ||||
| 67 | + | func TestCatalogueRecordEqualUsesCheckpointValuesAndOrderedTabs(t *testing.T) { | |||
| 68 | + | committed := CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}} | |||
| 69 | + | left := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, TabNames: []string{"shell", "logs"}, RecoveryState: RecoveryHealthy, Committed: &committed} | |||
| 70 | + | right := left | |||
| 71 | + | right.TabNames = append([]string(nil), left.TabNames...) | |||
| 72 | + | copy := committed | |||
| 73 | + | right.Committed = © | |||
| 74 | + | require.True(t, left.Equal(right)) | |||
| 75 | + | ||||
| 76 | + | right.TabNames[0], right.TabNames[1] = right.TabNames[1], right.TabNames[0] | |||
| 77 | + | require.False(t, left.Equal(right)) | |||
| 78 | + | require.True(t, (*CheckpointRef)(nil).Equal(nil)) | |||
| 79 | + | require.False(t, left.Committed.Equal(nil)) | |||
| 80 | + | } | |||
| 81 | + | ||||
| 61 | func TestDeletionTombstoneValidate(t *testing.T) { | 82 | func TestDeletionTombstoneValidate(t *testing.T) { | ||
| 62 | tests := []struct { | 83 | tests := []struct { | ||
| 63 | name string | 84 | name string | ||
M internal/persist/catalogue_test.go +31 -3
| @@ -40,11 +40,29 @@ func testCatalogueRecordRoundTrip(t *testing.T) { | |||||
| 40 | got, err := decodeRecordValue(record.Name, encoded) | 40 | got, err := decodeRecordValue(record.Name, encoded) | ||
| 41 | require.NoError(t, err) | 41 | require.NoError(t, err) | ||
| 42 | require.Equal(t, record, got) | 42 | require.Equal(t, record, got) | ||
| 43 | + | for prefix := range len(encoded) { | |||
| 44 | + | _, err := decodeRecordValue(record.Name, encoded[:prefix]) | |||
| 45 | + | require.Error(t, err, "prefix length %d", prefix) | |||
| 46 | + | } | |||
| 43 | require.Error(t, func() error { _, err := decodeRecordValue(record.Name, append(encoded, 0)); return err }()) | 47 | require.Error(t, func() error { _, err := decodeRecordValue(record.Name, append(encoded, 0)); return err }()) | ||
| 44 | }) | 48 | }) | ||
| 45 | } | 49 | } | ||
| 46 | } | 50 | } | ||
| 47 | 51 | ||||
| 52 | + | func TestDecodeRecordValueWrapsMalformedAndValidationErrors(t *testing.T) { | |||
| 53 | + | encoded, err := encodeRecordValue(validRecord("work", 1)) | |||
| 54 | + | require.NoError(t, err) | |||
| 55 | + | const recoveryStateOffset = 4 + 2 + 16 + 4 + 8 + 8 + 8 + 4 | |||
| 56 | + | encoded[recoveryStateOffset] = 99 | |||
| 57 | + | ||||
| 58 | + | _, err = decodeRecordValue("work", encoded) | |||
| 59 | + | require.ErrorIs(t, err, errMalformedRecord) | |||
| 60 | + | joined, ok := err.(interface{ Unwrap() []error }) | |||
| 61 | + | require.True(t, ok) | |||
| 62 | + | require.Len(t, joined.Unwrap(), 2) | |||
| 63 | + | require.ErrorContains(t, joined.Unwrap()[1], "invalid recovery state") | |||
| 64 | + | } | |||
| 65 | + | ||||
| 48 | func testCatalogueDuplicateIncarnations(t *testing.T) { | 66 | func testCatalogueDuplicateIncarnations(t *testing.T) { | ||
| 49 | id := domain.IncarnationID{1} | 67 | id := domain.IncarnationID{1} | ||
| 50 | require.Error(t, validateUniqueIncarnations([]domain.CatalogueRecord{ | 68 | require.Error(t, validateUniqueIncarnations([]domain.CatalogueRecord{ | ||
| @@ -110,7 +128,8 @@ func testCatalogueMetadataUpdatePreservesAuthority(t *testing.T) { | |||||
| 110 | next := domain.CatalogueRecord{Name: "work", IncarnationID: original.IncarnationID, Cwd: "/new", UpdatedAt: 22, LastUsedSeq: 23, TabNames: []string{"editor", "logs"}} | 128 | next := domain.CatalogueRecord{Name: "work", IncarnationID: original.IncarnationID, Cwd: "/new", UpdatedAt: 22, LastUsedSeq: 23, TabNames: []string{"editor", "logs"}} | ||
| 111 | update := next.MetadataUpdate() | 129 | update := next.MetadataUpdate() | ||
| 112 | require.NoError(t, p.UpdateMetadata(update)) | 130 | require.NoError(t, p.UpdateMetadata(update)) | ||
| 113 | - | got, ok := p.Record("work") | 131 | + | got, ok, err := p.Record("work") |
| 132 | + | require.NoError(t, err) | |||
| 114 | require.True(t, ok) | 133 | require.True(t, ok) | ||
| 115 | expected := original | 134 | expected := original | ||
| 116 | expected.Cwd, expected.UpdatedAt, expected.LastUsedSeq, expected.TabNames = next.Cwd, next.UpdatedAt, next.LastUsedSeq, next.TabNames | 135 | expected.Cwd, expected.UpdatedAt, expected.LastUsedSeq, expected.TabNames = next.Cwd, next.UpdatedAt, next.LastUsedSeq, next.TabNames | ||
| @@ -121,14 +140,16 @@ func testCatalogueMetadataUpdatePreservesAuthority(t *testing.T) { | |||||
| 121 | Name: "work", IncarnationID: original.IncarnationID, LastUsedSeq: &lastUsedSeq, | 140 | Name: "work", IncarnationID: original.IncarnationID, LastUsedSeq: &lastUsedSeq, | ||
| 122 | })) | 141 | })) | ||
| 123 | expected.LastUsedSeq = lastUsedSeq | 142 | expected.LastUsedSeq = lastUsedSeq | ||
| 124 | - | partiallyUpdated, ok := p.Record("work") | 143 | + | partiallyUpdated, ok, err := p.Record("work") |
| 144 | + | require.NoError(t, err) | |||
| 125 | require.True(t, ok) | 145 | require.True(t, ok) | ||
| 126 | require.Equal(t, expected, partiallyUpdated, "a partial metadata update must not clear other mutable or authority-owned fields") | 146 | require.Equal(t, expected, partiallyUpdated, "a partial metadata update must not clear other mutable or authority-owned fields") | ||
| 127 | 147 | ||||
| 128 | stale := update | 148 | stale := update | ||
| 129 | stale.IncarnationID = domain.IncarnationID{2} | 149 | stale.IncarnationID = domain.IncarnationID{2} | ||
| 130 | require.Error(t, p.UpdateMetadata(stale)) | 150 | require.Error(t, p.UpdateMetadata(stale)) | ||
| 131 | - | unchanged, ok := p.Record("work") | 151 | + | unchanged, ok, err := p.Record("work") |
| 152 | + | require.NoError(t, err) | |||
| 132 | require.True(t, ok) | 153 | require.True(t, ok) | ||
| 133 | require.Equal(t, expected, unchanged) | 154 | require.Equal(t, expected, unchanged) | ||
| 134 | } | 155 | } | ||
| @@ -201,6 +222,13 @@ func testCatalogueLoadReadOnlyRecoversCrashStates(t *testing.T) { | |||||
| 201 | writeCandidate(t, StorePath(dir)+".next", want) | 222 | writeCandidate(t, StorePath(dir)+".next", want) | ||
| 202 | }, | 223 | }, | ||
| 203 | }, | 224 | }, | ||
| 225 | + | { | |||
| 226 | + | // Crash before a new installation publishes .next as current. | |||
| 227 | + | name: "next-only-recovers-from-next", | |||
| 228 | + | write: func(t *testing.T, dir string, want domain.CatalogueRecord) { | |||
| 229 | + | writeCandidate(t, StorePath(dir)+".next", want) | |||
| 230 | + | }, | |||
| 231 | + | }, | |||
| 204 | { | 232 | { | ||
| 205 | // Crash leaves only a predecessor behind; no current, no successor. | 233 | // Crash leaves only a predecessor behind; no current, no successor. | ||
| 206 | name: "prev-only-recovers-from-prev", | 234 | name: "prev-only-recovers-from-prev", | ||
M internal/persist/migration.go +21 -5
| @@ -6,6 +6,7 @@ import ( | |||||
| 6 | "errors" | 6 | "errors" | ||
| 7 | "fmt" | 7 | "fmt" | ||
| 8 | "io" | 8 | "io" | ||
| 9 | + | "log/slog" | |||
| 9 | "os" | 10 | "os" | ||
| 10 | "path/filepath" | 11 | "path/filepath" | ||
| 11 | "sort" | 12 | "sort" | ||
| @@ -37,6 +38,8 @@ type OpenDeps struct { | |||||
| 37 | StateDir string | 38 | StateDir string | ||
| 38 | Random io.Reader | 39 | Random io.Reader | ||
| 39 | SnapshotMigration ports.SnapshotMigration | 40 | SnapshotMigration ports.SnapshotMigration | ||
| 41 | + | // Logger receives full adapter causes; durable degraded reasons are sanitized. | |||
| 42 | + | Logger *slog.Logger | |||
| 40 | // Fault is a test-only crash seam called after named durable boundaries. | 43 | // Fault is a test-only crash seam called after named durable boundaries. | ||
| 41 | Fault func(string) error | 44 | Fault func(string) error | ||
| 42 | } | 45 | } | ||
| @@ -65,6 +68,9 @@ const ( | |||||
| 65 | catalogueFormatName = "durable-session-v1.format" | 68 | catalogueFormatName = "durable-session-v1.format" | ||
| 66 | migrationRecordMagic = "VEVJ" | 69 | migrationRecordMagic = "VEVJ" | ||
| 67 | migrationRecordVersion uint16 = 1 | 70 | migrationRecordVersion uint16 = 1 | ||
| 71 | + | ||||
| 72 | + | migrationDegradedLegacySnapshotUncertain = "legacy_snapshot_uncertain" | |||
| 73 | + | migrationDegradedReasonMaxBytes = 64 | |||
| 68 | ) | 74 | ) | ||
| 69 | 75 | ||||
| 70 | func OpenOrMigrate(ctx context.Context, deps OpenDeps) (OpenResult, error) { | 76 | func OpenOrMigrate(ctx context.Context, deps OpenDeps) (OpenResult, error) { | ||
| @@ -300,11 +306,12 @@ func resumeLegacyMigration(ctx context.Context, deps OpenDeps) (OpenResult, erro | |||||
| 300 | if !errors.Is(err, ports.ErrLegacySnapshotUncertain) { | 306 | if !errors.Is(err, ports.ErrLegacySnapshotUncertain) { | ||
| 301 | return OpenResult{}, fmt.Errorf("persist: migrate legacy snapshot %q: %w", legacy.Name, err) | 307 | return OpenResult{}, fmt.Errorf("persist: migrate legacy snapshot %q: %w", legacy.Name, err) | ||
| 302 | } | 308 | } | ||
| 309 | + | migrationLogger(deps).Error("legacy snapshot migration uncertain", "session", legacy.Name, "err", err) | |||
| 303 | if ref.Generation != 0 && ref.ManifestDigest != ([32]byte{}) { | 310 | if ref.Generation != 0 && ref.ManifestDigest != ([32]byte{}) { | ||
| 304 | legacyRef := ref | 311 | legacyRef := ref | ||
| 305 | record.Validated[legacy.Name] = &legacyRef | 312 | record.Validated[legacy.Name] = &legacyRef | ||
| 306 | } | 313 | } | ||
| 307 | - | record.Degraded[legacy.Name] = err.Error() | 314 | + | record.Degraded[legacy.Name] = migrationDegradedLegacySnapshotUncertain |
| 308 | } | 315 | } | ||
| 309 | if err := writeMigrationRecord(intent, record); err != nil { | 316 | if err := writeMigrationRecord(intent, record); err != nil { | ||
| 310 | return OpenResult{}, err | 317 | return OpenResult{}, err | ||
| @@ -374,6 +381,13 @@ func resumeLegacyMigration(ctx context.Context, deps OpenDeps) (OpenResult, erro | |||||
| 374 | return OpenResult{Catalogue: catalogue, Records: opened, Migrated: true}, nil | 381 | return OpenResult{Catalogue: catalogue, Records: opened, Migrated: true}, nil | ||
| 375 | } | 382 | } | ||
| 376 | 383 | ||||
| 384 | + | func migrationLogger(deps OpenDeps) *slog.Logger { | |||
| 385 | + | if deps.Logger != nil { | |||
| 386 | + | return deps.Logger | |||
| 387 | + | } | |||
| 388 | + | return slog.Default() | |||
| 389 | + | } | |||
| 390 | + | ||||
| 377 | func migrationBoundary(deps OpenDeps, name string) error { | 391 | func migrationBoundary(deps OpenDeps, name string) error { | ||
| 378 | if deps.Fault != nil { | 392 | if deps.Fault != nil { | ||
| 379 | return deps.Fault(name) | 393 | return deps.Fault(name) | ||
| @@ -609,10 +623,12 @@ func catalogueRecordsEqual(a, b []domain.CatalogueRecord) bool { | |||||
| 609 | if len(a) != len(b) { | 623 | if len(a) != len(b) { | ||
| 610 | return false | 624 | return false | ||
| 611 | } | 625 | } | ||
| 612 | - | sort.Slice(a, func(i, j int) bool { return a[i].Name < a[j].Name }) | 626 | + | left := append([]domain.CatalogueRecord(nil), a...) |
| 613 | - | sort.Slice(b, func(i, j int) bool { return b[i].Name < b[j].Name }) | 627 | + | right := append([]domain.CatalogueRecord(nil), b...) |
| 614 | - | for i := range a { | 628 | + | sort.Slice(left, func(i, j int) bool { return left[i].Name < left[j].Name }) |
| 615 | - | if a[i].Name != b[i].Name || a[i].IncarnationID != b[i].IncarnationID || a[i].RecoveryState != b[i].RecoveryState { | 629 | + | sort.Slice(right, func(i, j int) bool { return right[i].Name < right[j].Name }) |
| 630 | + | for i := range left { | |||
| 631 | + | if !left[i].Equal(right[i]) { | |||
| 616 | return false | 632 | return false | ||
| 617 | } | 633 | } | ||
| 618 | } | 634 | } | ||
M internal/persist/migration_test.go +67 -15
| @@ -1,24 +1,29 @@ | |||||
| 1 | package persist | 1 | package persist | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | + | "bytes" | |||
| 4 | "context" | 5 | "context" | ||
| 5 | "crypto/sha256" | 6 | "crypto/sha256" | ||
| 6 | "encoding/binary" | 7 | "encoding/binary" | ||
| 7 | "errors" | 8 | "errors" | ||
| 8 | "io" | 9 | "io" | ||
| 10 | + | "log/slog" | |||
| 9 | "os" | 11 | "os" | ||
| 10 | "path/filepath" | 12 | "path/filepath" | ||
| 11 | "strconv" | 13 | "strconv" | ||
| 14 | + | "strings" | |||
| 12 | "testing" | 15 | "testing" | ||
| 13 | 16 | ||||
| 14 | "github.com/bnema/vev/internal/domain" | 17 | "github.com/bnema/vev/internal/domain" | ||
| 15 | "github.com/bnema/vev/internal/ports" | 18 | "github.com/bnema/vev/internal/ports" | ||
| 16 | "github.com/bnema/vev/pkg/kv" | 19 | "github.com/bnema/vev/pkg/kv" | ||
| 20 | + | "github.com/bnema/vev/pkg/safedir" | |||
| 17 | "github.com/stretchr/testify/require" | 21 | "github.com/stretchr/testify/require" | ||
| 18 | ) | 22 | ) | ||
| 19 | 23 | ||||
| 20 | type migrationStub struct { | 24 | type migrationStub struct { | ||
| 21 | legacy bool | 25 | legacy bool | ||
| 26 | + | probePaths []string | |||
| 22 | probeErr error | 27 | probeErr error | ||
| 23 | heads map[string]domain.CheckpointRef | 28 | heads map[string]domain.CheckpointRef | ||
| 24 | migrateErr map[string]error | 29 | migrateErr map[string]error | ||
| @@ -26,7 +31,19 @@ type migrationStub struct { | |||||
| 26 | sources map[string]string | 31 | sources map[string]string | ||
| 27 | } | 32 | } | ||
| 28 | 33 | ||||
| 29 | - | func (s *migrationStub) HasLegacyState(context.Context) (bool, error) { return s.legacy, s.probeErr } | 34 | + | func (s *migrationStub) HasLegacyState(context.Context) (bool, error) { |
| 35 | + | if s.probeErr != nil || s.legacy { | |||
| 36 | + | return s.legacy, s.probeErr | |||
| 37 | + | } | |||
| 38 | + | for _, path := range s.probePaths { | |||
| 39 | + | if _, err := os.Stat(path); err == nil { | |||
| 40 | + | return true, nil | |||
| 41 | + | } else if !errors.Is(err, os.ErrNotExist) { | |||
| 42 | + | return false, err | |||
| 43 | + | } | |||
| 44 | + | } | |||
| 45 | + | return false, nil | |||
| 46 | + | } | |||
| 30 | func (s *migrationStub) ReadLegacyHEAD(_ context.Context, name string) (domain.CheckpointRef, error) { | 47 | func (s *migrationStub) ReadLegacyHEAD(_ context.Context, name string) (domain.CheckpointRef, error) { | ||
| 31 | if source := s.sources[name]; source != "" { | 48 | if source := s.sources[name]; source != "" { | ||
| 32 | if _, err := os.Stat(source); err != nil { | 49 | if _, err := os.Stat(source); err != nil { | ||
| @@ -74,7 +91,7 @@ func encodeLegacyRecordV0(r legacyCatalogueRecordV0) []byte { | |||||
| 74 | func writeLegacyCatalogue(t *testing.T, dir string, records ...legacyCatalogueRecordV0) string { | 91 | func writeLegacyCatalogue(t *testing.T, dir string, records ...legacyCatalogueRecordV0) string { | ||
| 75 | t.Helper() | 92 | t.Helper() | ||
| 76 | require.NoError(t, os.Chmod(dir, 0o700)) | 93 | require.NoError(t, os.Chmod(dir, 0o700)) | ||
| 77 | - | require.NoError(t, os.MkdirAll(dir, 0o700)) | 94 | + | require.NoError(t, safedir.EnsurePrivate(dir)) |
| 78 | store, err := kv.Open(StorePath(dir)) | 95 | store, err := kv.Open(StorePath(dir)) | ||
| 79 | require.NoError(t, err) | 96 | require.NoError(t, err) | ||
| 80 | for _, record := range records { | 97 | for _, record := range records { | ||
| @@ -198,7 +215,7 @@ func TestMigrationResumeMatrix(t *testing.T) { | |||||
| 198 | } | 215 | } | ||
| 199 | uncertain := recordByName(t, result.Records, "uncertain") | 216 | uncertain := recordByName(t, result.Records, "uncertain") | ||
| 200 | require.Equal(t, domain.RecoveryDegraded, uncertain.RecoveryState) | 217 | require.Equal(t, domain.RecoveryDegraded, uncertain.RecoveryState) | ||
| 201 | - | require.Contains(t, uncertain.DegradedReason, "corrupt legacy manifest") | 218 | + | require.Equal(t, migrationDegradedLegacySnapshotUncertain, uncertain.DegradedReason) |
| 202 | require.NoError(t, result.Catalogue.Close()) | 219 | require.NoError(t, result.Catalogue.Close()) | ||
| 203 | 220 | ||||
| 204 | reopened, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: io.LimitReader(&countingReader{}, 0), SnapshotMigration: restarted}) | 221 | reopened, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: io.LimitReader(&countingReader{}, 0), SnapshotMigration: restarted}) | ||
| @@ -219,12 +236,18 @@ func TestMigrationPreservesUncertain(t *testing.T) { | |||||
| 219 | dir := t.TempDir() | 236 | dir := t.TempDir() | ||
| 220 | legacy := writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "broken", Cwd: "/tmp"}) | 237 | legacy := writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "broken", Cwd: "/tmp"}) | ||
| 221 | ref := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}} | 238 | ref := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}} | ||
| 222 | - | result, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"broken": ref}, migrateErr: map[string]error{"broken": errors.Join(ports.ErrLegacySnapshotUncertain, errors.New("invalid legacy manifest"))}}}) | 239 | + | adapterCause := strings.Repeat("sensitive-adapter-detail", 64) |
| 240 | + | var logs bytes.Buffer | |||
| 241 | + | logger := slog.New(slog.NewTextHandler(&logs, nil)) | |||
| 242 | + | result, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"broken": ref}, migrateErr: map[string]error{"broken": errors.Join(ports.ErrLegacySnapshotUncertain, errors.New(adapterCause))}}, Logger: logger}) | |||
| 223 | require.NoError(t, err) | 243 | require.NoError(t, err) | ||
| 224 | require.NoError(t, result.Catalogue.Close()) | 244 | require.NoError(t, result.Catalogue.Close()) | ||
| 225 | require.FileExists(t, legacy) | 245 | require.FileExists(t, legacy) | ||
| 226 | require.Equal(t, domain.RecoveryDegraded, result.Records[0].RecoveryState) | 246 | require.Equal(t, domain.RecoveryDegraded, result.Records[0].RecoveryState) | ||
| 227 | - | require.NotEmpty(t, result.Records[0].DegradedReason) | 247 | + | require.Equal(t, migrationDegradedLegacySnapshotUncertain, result.Records[0].DegradedReason) |
| 248 | + | require.LessOrEqual(t, len(result.Records[0].DegradedReason), migrationDegradedReasonMaxBytes) | |||
| 249 | + | require.NotContains(t, result.Records[0].DegradedReason, adapterCause) | |||
| 250 | + | require.Contains(t, logs.String(), adapterCause, "the full adapter cause must remain diagnosable") | |||
| 228 | 251 | ||||
| 229 | failClosedDir := t.TempDir() | 252 | failClosedDir := t.TempDir() | ||
| 230 | writeLegacyCatalogue(t, failClosedDir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"}) | 253 | writeLegacyCatalogue(t, failClosedDir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"}) | ||
| @@ -240,23 +263,36 @@ func TestMigrationPreservesUncertain(t *testing.T) { | |||||
| 240 | func TestOpenOrMigrateLegacyProbe(t *testing.T) { | 263 | func TestOpenOrMigrateLegacyProbe(t *testing.T) { | ||
| 241 | probeErr := errors.New("probe uncertain") | 264 | probeErr := errors.New("probe uncertain") | ||
| 242 | for _, tc := range []struct { | 265 | for _, tc := range []struct { | ||
| 243 | - | name string | 266 | + | name string |
| 244 | - | stub *migrationStub | 267 | + | stub *migrationStub |
| 245 | - | wantNew bool | 268 | + | arrange func(t *testing.T, dir string, stub *migrationStub) |
| 246 | - | wantErr error | 269 | + | wantNew bool |
| 270 | + | wantLegacy bool | |||
| 271 | + | wantErr error | |||
| 247 | }{ | 272 | }{ | ||
| 248 | - | {"proven absent", &migrationStub{}, true, nil}, | 273 | + | {name: "proven absent", stub: &migrationStub{}, wantNew: true}, |
| 249 | - | {"legacy blob", &migrationStub{legacy: true}, false, nil}, | 274 | + | {name: "legacy blob", stub: &migrationStub{}, wantLegacy: true, arrange: func(t *testing.T, dir string, stub *migrationStub) { |
| 250 | - | {"incremental namespace", &migrationStub{legacy: true}, false, nil}, | 275 | + | path := filepath.Join(dir, "legacy.snapshot") |
| 251 | - | {"probe error", &migrationStub{probeErr: probeErr}, false, probeErr}, | 276 | + | require.NoError(t, os.WriteFile(path, []byte("legacy"), 0o600)) |
| 277 | + | stub.probePaths = []string{path} | |||
| 278 | + | }}, | |||
| 279 | + | {name: "incremental namespace", stub: &migrationStub{}, wantLegacy: true, arrange: func(t *testing.T, dir string, stub *migrationStub) { | |||
| 280 | + | path := filepath.Join(dir, "snapshots") | |||
| 281 | + | require.NoError(t, safedir.EnsurePrivate(path)) | |||
| 282 | + | stub.probePaths = []string{path} | |||
| 283 | + | }}, | |||
| 284 | + | {name: "probe error", stub: &migrationStub{probeErr: probeErr}, wantErr: probeErr}, | |||
| 252 | } { | 285 | } { | ||
| 253 | t.Run(tc.name, func(t *testing.T) { | 286 | t.Run(tc.name, func(t *testing.T) { | ||
| 254 | dir := t.TempDir() | 287 | dir := t.TempDir() | ||
| 255 | require.NoError(t, os.Chmod(dir, 0o700)) | 288 | require.NoError(t, os.Chmod(dir, 0o700)) | ||
| 289 | + | if tc.arrange != nil { | |||
| 290 | + | tc.arrange(t, dir, tc.stub) | |||
| 291 | + | } | |||
| 256 | result, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: tc.stub}) | 292 | result, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: tc.stub}) | ||
| 257 | if tc.wantErr != nil { | 293 | if tc.wantErr != nil { | ||
| 258 | require.ErrorIs(t, err, tc.wantErr) | 294 | require.ErrorIs(t, err, tc.wantErr) | ||
| 259 | - | } else if tc.stub.legacy { | 295 | + | } else if tc.wantLegacy { |
| 260 | require.Error(t, err) | 296 | require.Error(t, err) | ||
| 261 | } else { | 297 | } else { | ||
| 262 | require.NoError(t, err) | 298 | require.NoError(t, err) | ||
| @@ -270,7 +306,7 @@ func TestOpenOrMigrateLegacyProbe(t *testing.T) { | |||||
| 270 | require.NoError(t, reopened.Catalogue.Close()) | 306 | require.NoError(t, reopened.Catalogue.Close()) | ||
| 271 | } | 307 | } | ||
| 272 | } | 308 | } | ||
| 273 | - | if tc.stub.legacy || tc.wantErr != nil { | 309 | + | if tc.wantLegacy || tc.wantErr != nil { |
| 274 | _, statErr := os.Stat(StorePath(dir)) | 310 | _, statErr := os.Stat(StorePath(dir)) | ||
| 275 | require.ErrorIs(t, statErr, os.ErrNotExist) | 311 | require.ErrorIs(t, statErr, os.ErrNotExist) | ||
| 276 | } | 312 | } | ||
| @@ -278,6 +314,22 @@ func TestOpenOrMigrateLegacyProbe(t *testing.T) { | |||||
| 278 | } | 314 | } | ||
| 279 | } | 315 | } | ||
| 280 | 316 | ||||
| 317 | + | func TestCatalogueRecordsEqualIsCompleteAndNonMutating(t *testing.T) { | |||
| 318 | + | committed := &domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}} | |||
| 319 | + | left := []domain.CatalogueRecord{ | |||
| 320 | + | {Name: "z", IncarnationID: domain.IncarnationID{1}, Cwd: "/old", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3, TabNames: []string{"shell"}, RecoveryState: domain.RecoveryHealthy, Committed: committed}, | |||
| 321 | + | validRecord("a", 2), | |||
| 322 | + | } | |||
| 323 | + | right := []domain.CatalogueRecord{left[1], left[0]} | |||
| 324 | + | leftOrder, rightOrder := left[0].Name, right[0].Name | |||
| 325 | + | require.True(t, catalogueRecordsEqual(left, right)) | |||
| 326 | + | require.Equal(t, leftOrder, left[0].Name) | |||
| 327 | + | require.Equal(t, rightOrder, right[0].Name) | |||
| 328 | + | ||||
| 329 | + | right[1].Cwd = "/different" | |||
| 330 | + | require.False(t, catalogueRecordsEqual(left, right), "installed catalogue validation must compare every record field") | |||
| 331 | + | } | |||
| 332 | + | ||||
| 281 | func TestAtomicMigrationWriteRemovesTempAfterRenameFailure(t *testing.T) { | 333 | func TestAtomicMigrationWriteRemovesTempAfterRenameFailure(t *testing.T) { | ||
| 282 | dir := t.TempDir() | 334 | dir := t.TempDir() | ||
| 283 | target := filepath.Join(dir, "target") | 335 | target := filepath.Join(dir, "target") | ||
M internal/persist/persist.go +13 -55
| @@ -3,8 +3,6 @@ package persist | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "errors" | 4 | "errors" | ||
| 5 | "fmt" | 5 | "fmt" | ||
| 6 | - | "log/slog" | |||
| 7 | - | "os" | |||
| 8 | "path/filepath" | 6 | "path/filepath" | ||
| 9 | "sort" | 7 | "sort" | ||
| 10 | "sync" | 8 | "sync" | ||
| @@ -23,25 +21,6 @@ func StorePath(dir string) string { return filepath.Join(dir, filename) } | |||||
| 23 | type Persister struct { | 21 | type Persister struct { | ||
| 24 | store ports.Store | 22 | store ports.Store | ||
| 25 | mu sync.Mutex | 23 | mu sync.Mutex | ||
| 26 | - | log *slog.Logger | |||
| 27 | - | } | |||
| 28 | - | ||||
| 29 | - | // Option configures a Persister at construction time. | |||
| 30 | - | type Option func(*Persister) | |||
| 31 | - | ||||
| 32 | - | // WithLogger directs diagnostics for failures that ports.Catalogue has no | |||
| 33 | - | // error return for (see Records) to log instead of the default logger. | |||
| 34 | - | func WithLogger(log *slog.Logger) Option { | |||
| 35 | - | return func(p *Persister) { p.log = log } | |||
| 36 | - | } | |||
| 37 | - | ||||
| 38 | - | // logger returns a nil-safe logger, defaulting to slog.Default() when no | |||
| 39 | - | // logger was configured. | |||
| 40 | - | func (p *Persister) logger() *slog.Logger { | |||
| 41 | - | if p != nil && p.log != nil { | |||
| 42 | - | return p.log | |||
| 43 | - | } | |||
| 44 | - | return slog.Default() | |||
| 45 | } | 24 | } | ||
| 46 | 25 | ||||
| 47 | // KVStore adapts the reusable WAL implementation to the persistence port. | 26 | // KVStore adapts the reusable WAL implementation to the persistence port. | ||
| @@ -76,28 +55,15 @@ func Open(dir string) (*Persister, error) { | |||||
| 76 | p, _, err := openCurrentCatalogue(dir, false) | 55 | p, _, err := openCurrentCatalogue(dir, false) | ||
| 77 | return p, err | 56 | return p, err | ||
| 78 | } | 57 | } | ||
| 79 | - | func New(store ports.Store, opts ...Option) *Persister { | 58 | + | func New(store ports.Store) *Persister { return &Persister{store: store} } |
| 80 | - | p := &Persister{store: store} | |||
| 81 | - | for _, opt := range opts { | |||
| 82 | - | opt(p) | |||
| 83 | - | } | |||
| 84 | - | return p | |||
| 85 | - | } | |||
| 86 | 59 | ||||
| 87 | func openCurrentCatalogue(dir string, createProvenEmpty bool) (*Persister, []domain.CatalogueRecord, error) { | 60 | func openCurrentCatalogue(dir string, createProvenEmpty bool) (*Persister, []domain.CatalogueRecord, error) { | ||
| 88 | path := StorePath(dir) | 61 | path := StorePath(dir) | ||
| 89 | - | absent := true | 62 | + | present, err := catalogueCandidatesPresent(path) |
| 90 | - | for _, candidate := range []string{path, path + ".next", path + ".prev"} { | 63 | + | if err != nil { |
| 91 | - | _, err := os.Stat(candidate) | 64 | + | return nil, nil, err |
| 92 | - | if err == nil { | |||
| 93 | - | absent = false | |||
| 94 | - | continue | |||
| 95 | - | } | |||
| 96 | - | if !errors.Is(err, os.ErrNotExist) { | |||
| 97 | - | return nil, nil, err | |||
| 98 | - | } | |||
| 99 | } | 65 | } | ||
| 100 | - | if absent && !createProvenEmpty { | 66 | + | if !present && !createProvenEmpty { |
| 101 | return nil, nil, fmt.Errorf("%w: no catalogue candidates", errPersistenceUnavailable) | 67 | return nil, nil, fmt.Errorf("%w: no catalogue candidates", errPersistenceUnavailable) | ||
| 102 | } | 68 | } | ||
| 103 | store, err := OpenStore(path) | 69 | store, err := OpenStore(path) | ||
| @@ -148,30 +114,22 @@ func (p *Persister) Create(record domain.CatalogueRecord) error { | |||||
| 148 | return p.applyLocked(map[string]*domain.CatalogueRecord{record.Name: &record}) | 114 | return p.applyLocked(map[string]*domain.CatalogueRecord{record.Name: &record}) | ||
| 149 | } | 115 | } | ||
| 150 | 116 | ||||
| 151 | - | // Records satisfies ports.Catalogue, whose signature has no error return. A | 117 | + | func (p *Persister) Records() ([]domain.CatalogueRecord, error) { return p.LoadCatalogue() } |
| 152 | - | // load failure (unavailable store, or a malformed record surfaced by | 118 | + | func (p *Persister) Record(name string) (domain.CatalogueRecord, bool, error) { |
| 153 | - | // decodeAll) must not silently look like an empty catalogue, so it is logged | |||
| 154 | - | // here instead of being swallowed. | |||
| 155 | - | func (p *Persister) Records() []domain.CatalogueRecord { | |||
| 156 | - | records, err := p.LoadCatalogue() | |||
| 157 | - | if err != nil { | |||
| 158 | - | p.logger().Error("persist: loading catalogue records failed", "err", err) | |||
| 159 | - | return []domain.CatalogueRecord{} | |||
| 160 | - | } | |||
| 161 | - | return records | |||
| 162 | - | } | |||
| 163 | - | func (p *Persister) Record(name string) (domain.CatalogueRecord, bool) { | |||
| 164 | if p == nil || p.store == nil { | 119 | if p == nil || p.store == nil { | ||
| 165 | - | return domain.CatalogueRecord{}, false | 120 | + | return domain.CatalogueRecord{}, false, errPersistenceUnavailable |
| 166 | } | 121 | } | ||
| 167 | p.mu.Lock() | 122 | p.mu.Lock() | ||
| 168 | defer p.mu.Unlock() | 123 | defer p.mu.Unlock() | ||
| 169 | value, ok := p.store.Get([]byte(name)) | 124 | value, ok := p.store.Get([]byte(name)) | ||
| 170 | if !ok { | 125 | if !ok { | ||
| 171 | - | return domain.CatalogueRecord{}, false | 126 | + | return domain.CatalogueRecord{}, false, nil |
| 172 | } | 127 | } | ||
| 173 | record, err := decodeRecordValue(name, value) | 128 | record, err := decodeRecordValue(name, value) | ||
| 174 | - | return record, err == nil | 129 | + | if err != nil { |
| 130 | + | return domain.CatalogueRecord{}, false, err | |||
| 131 | + | } | |||
| 132 | + | return record, true, nil | |||
| 175 | } | 133 | } | ||
| 176 | func (p *Persister) Apply(records map[string]*domain.CatalogueRecord) error { | 134 | func (p *Persister) Apply(records map[string]*domain.CatalogueRecord) error { | ||
| 177 | if p == nil { | 135 | if p == nil { | ||
M internal/persist/persist_test.go +22 -47
| @@ -1,9 +1,7 @@ | |||||
| 1 | package persist | 1 | package persist | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | - | "bytes" | |||
| 5 | "errors" | 4 | "errors" | ||
| 6 | - | "log/slog" | |||
| 7 | "path/filepath" | 5 | "path/filepath" | ||
| 8 | "testing" | 6 | "testing" | ||
| 9 | 7 | ||||
| @@ -51,9 +49,17 @@ func TestCatalogueBatchSyncBehavior(t *testing.T) { | |||||
| 51 | p := New(store) | 49 | p := New(store) | ||
| 52 | one := validRecord("one", 1) | 50 | one := validRecord("one", 1) | ||
| 53 | require.NoError(t, p.Replace("one", one)) | 51 | require.NoError(t, p.Replace("one", one)) | ||
| 52 | + | require.Contains(t, state, "one") | |||
| 54 | renamed := one | 53 | renamed := one | ||
| 55 | renamed.Name = "renamed" | 54 | renamed.Name = "renamed" | ||
| 55 | + | renamed.Cwd = "/renamed" | |||
| 56 | require.NoError(t, p.Rename("one", renamed)) | 56 | require.NoError(t, p.Rename("one", renamed)) | ||
| 57 | + | require.NotContains(t, state, "one") | |||
| 58 | + | encoded, ok := state["renamed"] | |||
| 59 | + | require.True(t, ok) | |||
| 60 | + | got, err := decodeRecordValue("renamed", encoded) | |||
| 61 | + | require.NoError(t, err) | |||
| 62 | + | require.Equal(t, renamed, got) | |||
| 57 | } | 63 | } | ||
| 58 | 64 | ||||
| 59 | func TestCatalogueBatchFailureDoesNotSync(t *testing.T) { | 65 | func TestCatalogueBatchFailureDoesNotSync(t *testing.T) { | ||
| @@ -80,7 +86,9 @@ func TestCatalogueNilPersisterIsNoOp(t *testing.T) { | |||||
| 80 | require.NoError(t, err) | 86 | require.NoError(t, err) | ||
| 81 | require.Empty(t, records) | 87 | require.Empty(t, records) | ||
| 82 | require.NoError(t, p.Close()) | 88 | require.NoError(t, p.Close()) | ||
| 83 | - | require.Empty(t, p.Records()) | 89 | + | got, err := p.Records() |
| 90 | + | require.NoError(t, err) | |||
| 91 | + | require.Empty(t, got) | |||
| 84 | } | 92 | } | ||
| 85 | 93 | ||||
| 86 | // TestCatalogueUnavailableStoreErrors covers a non-nil Persister with no | 94 | // TestCatalogueUnavailableStoreErrors covers a non-nil Persister with no | ||
| @@ -102,51 +110,18 @@ func TestCatalogueUnavailableStoreErrors(t *testing.T) { | |||||
| 102 | records, err := p.LoadAll() | 110 | records, err := p.LoadAll() | ||
| 103 | require.ErrorIs(t, err, errPersistenceUnavailable) | 111 | require.ErrorIs(t, err, errPersistenceUnavailable) | ||
| 104 | require.Empty(t, records) | 112 | require.Empty(t, records) | ||
| 105 | - | require.Empty(t, p.Records()) | 113 | + | got, err := p.Records() |
| 114 | + | require.ErrorIs(t, err, errPersistenceUnavailable) | |||
| 115 | + | require.Empty(t, got) | |||
| 106 | require.NoError(t, p.Close()) | 116 | require.NoError(t, p.Close()) | ||
| 107 | } | 117 | } | ||
| 108 | 118 | ||||
| 109 | - | // TestCatalogueRecordsMakesLoadFailureObservable covers Records(), which | 119 | + | func TestCatalogueRecordsPropagatesMalformedRecord(t *testing.T) { |
| 110 | - | // ports.Catalogue.Records() gives no error return for: a load failure must | 120 | + | store := portsmocks.NewMockStore(t) |
| 111 | - | // not silently look like an empty catalogue to every caller. Records() | 121 | + | store.EXPECT().Range(mock.Anything).Run(func(fn func([]byte, []byte) bool) { |
| 112 | - | // degrades to an empty slice but must emit the underlying error through its | 122 | + | fn([]byte("bad"), []byte{0, 1}) |
| 113 | - | // (nil-safe, optional) logger rather than swallowing it outright. | 123 | + | }) |
| 114 | - | func TestCatalogueRecordsMakesLoadFailureObservable(t *testing.T) { | 124 | + | records, err := New(store).Records() |
| 115 | - | tests := []struct { | 125 | + | require.Error(t, err) |
| 116 | - | name string | 126 | + | require.Empty(t, records) |
| 117 | - | build func(t *testing.T, log *slog.Logger) *Persister | |||
| 118 | - | }{ | |||
| 119 | - | { | |||
| 120 | - | name: "unavailable-store", | |||
| 121 | - | build: func(t *testing.T, log *slog.Logger) *Persister { | |||
| 122 | - | return New(nil, WithLogger(log)) | |||
| 123 | - | }, | |||
| 124 | - | }, | |||
| 125 | - | { | |||
| 126 | - | name: "malformed-record", | |||
| 127 | - | build: func(t *testing.T, log *slog.Logger) *Persister { | |||
| 128 | - | store := portsmocks.NewMockStore(t) | |||
| 129 | - | store.EXPECT().Range(mock.Anything).Run(func(fn func([]byte, []byte) bool) { | |||
| 130 | - | fn([]byte("bad"), []byte{0, 1}) | |||
| 131 | - | }) | |||
| 132 | - | return New(store, WithLogger(log)) | |||
| 133 | - | }, | |||
| 134 | - | }, | |||
| 135 | - | } | |||
| 136 | - | for _, tt := range tests { | |||
| 137 | - | t.Run(tt.name, func(t *testing.T) { | |||
| 138 | - | var buf bytes.Buffer | |||
| 139 | - | p := tt.build(t, slog.New(slog.NewTextHandler(&buf, nil))) | |||
| 140 | - | require.Empty(t, p.Records()) | |||
| 141 | - | require.Contains(t, buf.String(), "loading catalogue records failed") | |||
| 142 | - | }) | |||
| 143 | - | } | |||
| 144 | - | } | |||
| 145 | - | ||||
| 146 | - | // TestCatalogueRecordsDefaultsToSlogDefaultLogger covers the nil-safe logger | |||
| 147 | - | // fallback: a Persister built without WithLogger must not panic when a load | |||
| 148 | - | // failure needs to be reported. | |||
| 149 | - | func TestCatalogueRecordsDefaultsToSlogDefaultLogger(t *testing.T) { | |||
| 150 | - | p := New(nil) | |||
| 151 | - | require.NotPanics(t, func() { require.Empty(t, p.Records()) }) | |||
| 152 | } | 127 | } | ||
M internal/persist/record.go +6 -4
| @@ -26,7 +26,11 @@ func encodeRecordValue(record domain.CatalogueRecord) ([]byte, error) { | |||||
| 26 | buf = append(buf, catalogueMagic[:]...) | 26 | buf = append(buf, catalogueMagic[:]...) | ||
| 27 | buf = binary.BigEndian.AppendUint16(buf, catalogueRecordVersion) | 27 | buf = binary.BigEndian.AppendUint16(buf, catalogueRecordVersion) | ||
| 28 | buf = append(buf, record.IncarnationID[:]...) | 28 | buf = append(buf, record.IncarnationID[:]...) | ||
| 29 | - | buf = appendString(buf, record.Cwd) | 29 | + | var err error |
| 30 | + | buf, err = appendCheckedString(buf, record.Cwd) | |||
| 31 | + | if err != nil { | |||
| 32 | + | return nil, err | |||
| 33 | + | } | |||
| 30 | buf = binary.BigEndian.AppendUint64(buf, uint64(record.CreatedAt)) | 34 | buf = binary.BigEndian.AppendUint64(buf, uint64(record.CreatedAt)) | ||
| 31 | buf = binary.BigEndian.AppendUint64(buf, uint64(record.UpdatedAt)) | 35 | buf = binary.BigEndian.AppendUint64(buf, uint64(record.UpdatedAt)) | ||
| 32 | buf = binary.BigEndian.AppendUint64(buf, record.LastUsedSeq) | 36 | buf = binary.BigEndian.AppendUint64(buf, record.LastUsedSeq) | ||
| @@ -35,7 +39,6 @@ func encodeRecordValue(record domain.CatalogueRecord) ([]byte, error) { | |||||
| 35 | } | 39 | } | ||
| 36 | buf = binary.BigEndian.AppendUint32(buf, uint32(len(record.TabNames))) | 40 | buf = binary.BigEndian.AppendUint32(buf, uint32(len(record.TabNames))) | ||
| 37 | for _, name := range record.TabNames { | 41 | for _, name := range record.TabNames { | ||
| 38 | - | var err error | |||
| 39 | buf, err = appendCheckedString(buf, name) | 42 | buf, err = appendCheckedString(buf, name) | ||
| 40 | if err != nil { | 43 | if err != nil { | ||
| 41 | return nil, err | 44 | return nil, err | ||
| @@ -45,7 +48,6 @@ func encodeRecordValue(record domain.CatalogueRecord) ([]byte, error) { | |||||
| 45 | buf = appendRef(buf, record.Committed) | 48 | buf = appendRef(buf, record.Committed) | ||
| 46 | buf = appendRef(buf, record.Fallbacks[0]) | 49 | buf = appendRef(buf, record.Fallbacks[0]) | ||
| 47 | buf = appendRef(buf, record.Fallbacks[1]) | 50 | buf = appendRef(buf, record.Fallbacks[1]) | ||
| 48 | - | var err error | |||
| 49 | buf, err = appendCheckedString(buf, record.DegradedReason) | 51 | buf, err = appendCheckedString(buf, record.DegradedReason) | ||
| 50 | return buf, err | 52 | return buf, err | ||
| 51 | } | 53 | } | ||
| @@ -118,7 +120,7 @@ func decodeRecordValue(name string, value []byte) (domain.CatalogueRecord, error | |||||
| 118 | return domain.CatalogueRecord{}, errMalformedRecord | 120 | return domain.CatalogueRecord{}, errMalformedRecord | ||
| 119 | } | 121 | } | ||
| 120 | if err := record.Validate(); err != nil { | 122 | if err := record.Validate(); err != nil { | ||
| 121 | - | return domain.CatalogueRecord{}, fmt.Errorf("%w: %v", errMalformedRecord, err) | 123 | + | return domain.CatalogueRecord{}, errors.Join(errMalformedRecord, fmt.Errorf("persist: invalid catalogue record: %w", err)) |
| 122 | } | 124 | } | ||
| 123 | return record, nil | 125 | return record, nil | ||
| 124 | } | 126 | } | ||
M internal/ports/mocks/recovery_mock.go +26 -11
| @@ -1428,7 +1428,7 @@ func (_c *MockCatalogue_Delete_Call) RunAndReturn(run func(s string) error) *Moc | |||||
| 1428 | } | 1428 | } | ||
| 1429 | 1429 | ||||
| 1430 | // Record provides a mock function for the type MockCatalogue | 1430 | // Record provides a mock function for the type MockCatalogue | ||
| 1431 | - | func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool) { | 1431 | + | func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool, error) { |
| 1432 | ret := _mock.Called(s) | 1432 | ret := _mock.Called(s) | ||
| 1433 | 1433 | ||||
| 1434 | if len(ret) == 0 { | 1434 | if len(ret) == 0 { | ||
| @@ -1437,7 +1437,8 @@ func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool) { | |||||
| 1437 | 1437 | ||||
| 1438 | var r0 domain.CatalogueRecord | 1438 | var r0 domain.CatalogueRecord | ||
| 1439 | var r1 bool | 1439 | var r1 bool | ||
| 1440 | - | if returnFunc, ok := ret.Get(0).(func(string) (domain.CatalogueRecord, bool)); ok { | 1440 | + | var r2 error |
| 1441 | + | if returnFunc, ok := ret.Get(0).(func(string) (domain.CatalogueRecord, bool, error)); ok { | |||
| 1441 | return returnFunc(s) | 1442 | return returnFunc(s) | ||
| 1442 | } | 1443 | } | ||
| 1443 | if returnFunc, ok := ret.Get(0).(func(string) domain.CatalogueRecord); ok { | 1444 | if returnFunc, ok := ret.Get(0).(func(string) domain.CatalogueRecord); ok { | ||
| @@ -1450,7 +1451,12 @@ func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool) { | |||||
| 1450 | } else { | 1451 | } else { | ||
| 1451 | r1 = ret.Get(1).(bool) | 1452 | r1 = ret.Get(1).(bool) | ||
| 1452 | } | 1453 | } | ||
| 1453 | - | return r0, r1 | 1454 | + | if returnFunc, ok := ret.Get(2).(func(string) error); ok { |
| 1455 | + | r2 = returnFunc(s) | |||
| 1456 | + | } else { | |||
| 1457 | + | r2 = ret.Error(2) | |||
| 1458 | + | } | |||
| 1459 | + | return r0, r1, r2 | |||
| 1454 | } | 1460 | } | ||
| 1455 | 1461 | ||||
| 1456 | // MockCatalogue_Record_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Record' | 1462 | // MockCatalogue_Record_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Record' | ||
| @@ -1477,18 +1483,18 @@ func (_c *MockCatalogue_Record_Call) Run(run func(s string)) *MockCatalogue_Reco | |||||
| 1477 | return _c | 1483 | return _c | ||
| 1478 | } | 1484 | } | ||
| 1479 | 1485 | ||||
| 1480 | - | func (_c *MockCatalogue_Record_Call) Return(catalogueRecord domain.CatalogueRecord, b bool) *MockCatalogue_Record_Call { | 1486 | + | func (_c *MockCatalogue_Record_Call) Return(catalogueRecord domain.CatalogueRecord, b bool, err error) *MockCatalogue_Record_Call { |
| 1481 | - | _c.Call.Return(catalogueRecord, b) | 1487 | + | _c.Call.Return(catalogueRecord, b, err) |
| 1482 | return _c | 1488 | return _c | ||
| 1483 | } | 1489 | } | ||
| 1484 | 1490 | ||||
| 1485 | - | func (_c *MockCatalogue_Record_Call) RunAndReturn(run func(s string) (domain.CatalogueRecord, bool)) *MockCatalogue_Record_Call { | 1491 | + | func (_c *MockCatalogue_Record_Call) RunAndReturn(run func(s string) (domain.CatalogueRecord, bool, error)) *MockCatalogue_Record_Call { |
| 1486 | _c.Call.Return(run) | 1492 | _c.Call.Return(run) | ||
| 1487 | return _c | 1493 | return _c | ||
| 1488 | } | 1494 | } | ||
| 1489 | 1495 | ||||
| 1490 | // Records provides a mock function for the type MockCatalogue | 1496 | // Records provides a mock function for the type MockCatalogue | ||
| 1491 | - | func (_mock *MockCatalogue) Records() []domain.CatalogueRecord { | 1497 | + | func (_mock *MockCatalogue) Records() ([]domain.CatalogueRecord, error) { |
| 1492 | ret := _mock.Called() | 1498 | ret := _mock.Called() | ||
| 1493 | 1499 | ||||
| 1494 | if len(ret) == 0 { | 1500 | if len(ret) == 0 { | ||
| @@ -1496,6 +1502,10 @@ func (_mock *MockCatalogue) Records() []domain.CatalogueRecord { | |||||
| 1496 | } | 1502 | } | ||
| 1497 | 1503 | ||||
| 1498 | var r0 []domain.CatalogueRecord | 1504 | var r0 []domain.CatalogueRecord | ||
| 1505 | + | var r1 error | |||
| 1506 | + | if returnFunc, ok := ret.Get(0).(func() ([]domain.CatalogueRecord, error)); ok { | |||
| 1507 | + | return returnFunc() | |||
| 1508 | + | } | |||
| 1499 | if returnFunc, ok := ret.Get(0).(func() []domain.CatalogueRecord); ok { | 1509 | if returnFunc, ok := ret.Get(0).(func() []domain.CatalogueRecord); ok { | ||
| 1500 | r0 = returnFunc() | 1510 | r0 = returnFunc() | ||
| 1501 | } else { | 1511 | } else { | ||
| @@ -1503,7 +1513,12 @@ func (_mock *MockCatalogue) Records() []domain.CatalogueRecord { | |||||
| 1503 | r0 = ret.Get(0).([]domain.CatalogueRecord) | 1513 | r0 = ret.Get(0).([]domain.CatalogueRecord) | ||
| 1504 | } | 1514 | } | ||
| 1505 | } | 1515 | } | ||
| 1506 | - | return r0 | 1516 | + | if returnFunc, ok := ret.Get(1).(func() error); ok { |
| 1517 | + | r1 = returnFunc() | |||
| 1518 | + | } else { | |||
| 1519 | + | r1 = ret.Error(1) | |||
| 1520 | + | } | |||
| 1521 | + | return r0, r1 | |||
| 1507 | } | 1522 | } | ||
| 1508 | 1523 | ||||
| 1509 | // MockCatalogue_Records_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Records' | 1524 | // MockCatalogue_Records_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Records' | ||
| @@ -1523,12 +1538,12 @@ func (_c *MockCatalogue_Records_Call) Run(run func()) *MockCatalogue_Records_Cal | |||||
| 1523 | return _c | 1538 | return _c | ||
| 1524 | } | 1539 | } | ||
| 1525 | 1540 | ||||
| 1526 | - | func (_c *MockCatalogue_Records_Call) Return(catalogueRecords []domain.CatalogueRecord) *MockCatalogue_Records_Call { | 1541 | + | func (_c *MockCatalogue_Records_Call) Return(catalogueRecords []domain.CatalogueRecord, err error) *MockCatalogue_Records_Call { |
| 1527 | - | _c.Call.Return(catalogueRecords) | 1542 | + | _c.Call.Return(catalogueRecords, err) |
| 1528 | return _c | 1543 | return _c | ||
| 1529 | } | 1544 | } | ||
| 1530 | 1545 | ||||
| 1531 | - | func (_c *MockCatalogue_Records_Call) RunAndReturn(run func() []domain.CatalogueRecord) *MockCatalogue_Records_Call { | 1546 | + | func (_c *MockCatalogue_Records_Call) RunAndReturn(run func() ([]domain.CatalogueRecord, error)) *MockCatalogue_Records_Call { |
| 1532 | _c.Call.Return(run) | 1547 | _c.Call.Return(run) | ||
| 1533 | return _c | 1548 | return _c | ||
| 1534 | } | 1549 | } | ||
M internal/ports/recovery.go +2 -2
| @@ -157,8 +157,8 @@ type DegradedRecoveryCoordinator interface { | |||||
| 157 | } | 157 | } | ||
| 158 | 158 | ||||
| 159 | type Catalogue interface { | 159 | type Catalogue interface { | ||
| 160 | - | Records() []domain.CatalogueRecord | 160 | + | Records() ([]domain.CatalogueRecord, error) |
| 161 | - | Record(string) (domain.CatalogueRecord, bool) | 161 | + | Record(string) (domain.CatalogueRecord, bool, error) |
| 162 | Create(domain.CatalogueRecord) error | 162 | Create(domain.CatalogueRecord) error | ||
| 163 | UpdateMetadata(domain.CatalogueMetadataUpdate) error | 163 | UpdateMetadata(domain.CatalogueMetadataUpdate) error | ||
| 164 | Replace(string, domain.CatalogueRecord) error | 164 | Replace(string, domain.CatalogueRecord) error | ||
M internal/ports/store.go +5 -0
| @@ -21,6 +21,11 @@ type Store interface { | |||||
| 21 | Get(key []byte) ([]byte, bool) | 21 | Get(key []byte) ([]byte, bool) | ||
| 22 | Set(key, val []byte) error | 22 | Set(key, val []byte) error | ||
| 23 | Delete(key []byte) error | 23 | Delete(key []byte) error | ||
| 24 | + | // Batch applies all changes atomically to in-memory state: an error leaves | |||
| 25 | + | // every prior value unchanged. Implementations repair or discard any torn | |||
| 26 | + | // WAL batch during replay so no partial batch becomes visible after restart. | |||
| 27 | + | // Batch does not retain StoreChange key or value slices; the caller owns and | |||
| 28 | + | // may mutate them after Batch returns. | |||
| 24 | Batch([]StoreChange) error | 29 | Batch([]StoreChange) error | ||
| 25 | // Range iterates key/value pairs; fn returning false stops iteration early. | 30 | // Range iterates key/value pairs; fn returning false stops iteration early. | ||
| 26 | // Each key and value is a stable copy owned by the caller, which may retain | 31 | // Each key and value is a stable copy owned by the caller, which may retain | ||
M internal/ports/wire_test.go +21 -6
| @@ -744,12 +744,27 @@ func TestKillGoldenAndRoundTrip(t *testing.T) { | |||||
| 744 | } | 744 | } | ||
| 745 | 745 | ||||
| 746 | func TestSessionInfoRecoveryState(t *testing.T) { | 746 | func TestSessionInfoRecoveryState(t *testing.T) { | ||
| 747 | - | payload := MarshalSessions(Sessions{Sessions: []SessionInfo{{Name: "broken", State: SessionDegraded}}}) | 747 | + | tests := []struct { |
| 748 | - | got, err := UnmarshalSessions(payload) | 748 | + | name string |
| 749 | - | require.NoError(t, err) | 749 | + | state SessionState |
| 750 | - | require.Equal(t, SessionDegraded, got.Sessions[0].State) | 750 | + | want []byte |
| 751 | - | assertAllPrefixesFail(t, payload, UnmarshalSessions) | 751 | + | }{ |
| 752 | - | assertTrailingGarbageFails(t, payload, UnmarshalSessions) | 752 | + | {name: "running", state: SessionRunning, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1}}, |
| 753 | + | {name: "stopped", state: SessionStopped, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2}}, | |||
| 754 | + | {name: "restoring", state: SessionRestoring, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3}}, | |||
| 755 | + | {name: "degraded", state: SessionDegraded, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4}}, | |||
| 756 | + | } | |||
| 757 | + | for _, tt := range tests { | |||
| 758 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 759 | + | payload := MarshalSessions(Sessions{Sessions: []SessionInfo{{State: tt.state}}}) | |||
| 760 | + | require.Equal(t, tt.want, payload) | |||
| 761 | + | got, err := UnmarshalSessions(payload) | |||
| 762 | + | require.NoError(t, err) | |||
| 763 | + | require.Equal(t, tt.state, got.Sessions[0].State) | |||
| 764 | + | assertAllPrefixesFail(t, payload, UnmarshalSessions) | |||
| 765 | + | assertTrailingGarbageFails(t, payload, UnmarshalSessions) | |||
| 766 | + | }) | |||
| 767 | + | } | |||
| 753 | } | 768 | } | ||
| 754 | 769 | ||||
| 755 | func TestSessionsGoldenAndRoundTrip(t *testing.T) { | 770 | func TestSessionsGoldenAndRoundTrip(t *testing.T) { | ||
M internal/usecase/command/command.go +12 -3
| @@ -8,9 +8,9 @@ import ( | |||||
| 8 | // ErrInvalidArguments reports arguments that do not have the exact command format. | 8 | // ErrInvalidArguments reports arguments that do not have the exact command format. | ||
| 9 | var ErrInvalidArguments = errors.New("invalid command arguments") | 9 | var ErrInvalidArguments = errors.New("invalid command arguments") | ||
| 10 | 10 | ||||
| 11 | - | // ParsePositiveDecimal accepts exactly one base-10 positive decimal value. | 11 | + | // ParsePositiveUint64 accepts exactly one base-10 positive decimal value. |
| 12 | // It deliberately rejects signs, whitespace, zero, and non-canonical forms. | 12 | // It deliberately rejects signs, whitespace, zero, and non-canonical forms. | ||
| 13 | - | func ParsePositiveDecimal(args []string) (int, error) { | 13 | + | func ParsePositiveUint64(args []string) (uint64, error) { |
| 14 | if len(args) != 1 || args[0] == "" || (len(args[0]) > 1 && args[0][0] == '0') { | 14 | if len(args) != 1 || args[0] == "" || (len(args[0]) > 1 && args[0][0] == '0') { | ||
| 15 | return 0, ErrInvalidArguments | 15 | return 0, ErrInvalidArguments | ||
| 16 | } | 16 | } | ||
| @@ -19,13 +19,22 @@ func ParsePositiveDecimal(args []string) (int, error) { | |||||
| 19 | return 0, ErrInvalidArguments | 19 | return 0, ErrInvalidArguments | ||
| 20 | } | 20 | } | ||
| 21 | } | 21 | } | ||
| 22 | - | n, err := strconv.Atoi(args[0]) | 22 | + | n, err := strconv.ParseUint(args[0], 10, 64) |
| 23 | if err != nil || n < 1 { | 23 | if err != nil || n < 1 { | ||
| 24 | return 0, ErrInvalidArguments | 24 | return 0, ErrInvalidArguments | ||
| 25 | } | 25 | } | ||
| 26 | return n, nil | 26 | return n, nil | ||
| 27 | } | 27 | } | ||
| 28 | 28 | ||||
| 29 | + | // ParsePositiveDecimal parses a positive decimal that must fit the platform int. | |||
| 30 | + | func ParsePositiveDecimal(args []string) (int, error) { | |||
| 31 | + | n, err := ParsePositiveUint64(args) | |||
| 32 | + | if err != nil || n > uint64(^uint(0)>>1) { | |||
| 33 | + | return 0, ErrInvalidArguments | |||
| 34 | + | } | |||
| 35 | + | return int(n), nil | |||
| 36 | + | } | |||
| 37 | + | ||||
| 29 | // Context is the set of application actions available to commands. | 38 | // Context is the set of application actions available to commands. | ||
| 30 | type Context interface { | 39 | type Context interface { | ||
| 31 | CreateTab() error | 40 | CreateTab() error | ||
M internal/usecase/command/command_test.go +11 -0
| @@ -5,6 +5,17 @@ import ( | |||||
| 5 | "testing" | 5 | "testing" | ||
| 6 | ) | 6 | ) | ||
| 7 | 7 | ||||
| 8 | + | func TestParsePositiveUint64(t *testing.T) { | |||
| 9 | + | const maxUint64 = "18446744073709551615" | |||
| 10 | + | got, err := ParsePositiveUint64([]string{maxUint64}) | |||
| 11 | + | if err != nil || got != ^uint64(0) { | |||
| 12 | + | t.Fatalf("ParsePositiveUint64(max) = %d, %v", got, err) | |||
| 13 | + | } | |||
| 14 | + | if _, err := ParsePositiveUint64([]string{"18446744073709551616"}); !errors.Is(err, ErrInvalidArguments) { | |||
| 15 | + | t.Fatalf("ParsePositiveUint64(overflow) error = %v", err) | |||
| 16 | + | } | |||
| 17 | + | } | |||
| 18 | + | ||||
| 8 | func TestParsePositiveDecimal(t *testing.T) { | 19 | func TestParsePositiveDecimal(t *testing.T) { | ||
| 9 | for _, tt := range []struct { | 20 | for _, tt := range []struct { | ||
| 10 | name string | 21 | name string | ||
M internal/usecase/command/registry.go +6 -3
| @@ -1,6 +1,9 @@ | |||||
| 1 | package command | 1 | package command | ||
| 2 | 2 | ||||
| 3 | - | import "strings" | 3 | + | import ( |
| 4 | + | "path/filepath" | |||
| 5 | + | "strings" | |||
| 6 | + | ) | |||
| 4 | 7 | ||||
| 5 | // Registry returns all palette and control commands in stable display order. | 8 | // Registry returns all palette and control commands in stable display order. | ||
| 6 | func Registry() []Command { | 9 | func Registry() []Command { | ||
| @@ -101,11 +104,11 @@ func sessionRecoveryCommand() Command { | |||||
| 101 | case len(args) == 1 && (args[0] == "retry" || args[0] == "discard"): | 104 | case len(args) == 1 && (args[0] == "retry" || args[0] == "discard"): | ||
| 102 | action = args[0] | 105 | action = args[0] | ||
| 103 | case len(args) == 2 && args[0] == "restore": | 106 | case len(args) == 2 && args[0] == "restore": | ||
| 104 | - | if _, err := ParsePositiveDecimal(args[1:]); err != nil { | 107 | + | if _, err := ParsePositiveUint64(args[1:]); err != nil { |
| 105 | return ControlResult{}, ErrInvalidArguments | 108 | return ControlResult{}, ErrInvalidArguments | ||
| 106 | } | 109 | } | ||
| 107 | action, argument = args[0], args[1] | 110 | action, argument = args[0], args[1] | ||
| 108 | - | case len(args) == 2 && args[0] == "export" && args[1] != "": | 111 | + | case len(args) == 2 && args[0] == "export" && filepath.IsAbs(args[1]): |
| 109 | action, argument = args[0], args[1] | 112 | action, argument = args[0], args[1] | ||
| 110 | default: | 113 | default: | ||
| 111 | return ControlResult{}, ErrInvalidArguments | 114 | return ControlResult{}, ErrInvalidArguments | ||
M internal/usecase/command/registry_test.go +22 -6
| @@ -117,16 +117,29 @@ func TestSessionRecoveryCommand(t *testing.T) { | |||||
| 117 | }{ | 117 | }{ | ||
| 118 | {args: []string{"retry"}, wantCall: "session-recovery:retry:"}, | 118 | {args: []string{"retry"}, wantCall: "session-recovery:retry:"}, | ||
| 119 | {args: []string{"restore", "7"}, wantCall: "session-recovery:restore:7"}, | 119 | {args: []string{"restore", "7"}, wantCall: "session-recovery:restore:7"}, | ||
| 120 | + | {args: []string{"restore", "18446744073709551615"}, wantCall: "session-recovery:restore:18446744073709551615"}, | |||
| 120 | {args: []string{"export", "/tmp/export"}, wantCall: "session-recovery:export:/tmp/export"}, | 121 | {args: []string{"export", "/tmp/export"}, wantCall: "session-recovery:export:/tmp/export"}, | ||
| 121 | {args: []string{"discard"}, wantCall: "session-recovery:discard:"}, | 122 | {args: []string{"discard"}, wantCall: "session-recovery:discard:"}, | ||
| 123 | + | {args: nil, wantErr: ErrInvalidArguments}, | |||
| 124 | + | {args: []string{"retry", "extra"}, wantErr: ErrInvalidArguments}, | |||
| 125 | + | {args: []string{"restore"}, wantErr: ErrInvalidArguments}, | |||
| 122 | {args: []string{"restore", "0"}, wantErr: ErrInvalidArguments}, | 126 | {args: []string{"restore", "0"}, wantErr: ErrInvalidArguments}, | ||
| 127 | + | {args: []string{"restore", "01"}, wantErr: ErrInvalidArguments}, | |||
| 128 | + | {args: []string{"restore", "18446744073709551616"}, wantErr: ErrInvalidArguments}, | |||
| 129 | + | {args: []string{"export", "relative/path"}, wantErr: ErrInvalidArguments}, | |||
| 130 | + | {args: []string{"export"}, wantErr: ErrInvalidArguments}, | |||
| 131 | + | {args: []string{"discard", "extra"}, wantErr: ErrInvalidArguments}, | |||
| 123 | {args: []string{"unknown"}, wantErr: ErrInvalidArguments}, | 132 | {args: []string{"unknown"}, wantErr: ErrInvalidArguments}, | ||
| 124 | } | 133 | } | ||
| 125 | for _, test := range tests { | 134 | for _, test := range tests { | ||
| 126 | - | ctx := &controlSpy{} | 135 | + | ctx := &controlSpy{recoveryOutput: "recovery output"} |
| 127 | - | _, err := cmd.Control(ctx, test.args, ControlOptions{}) | 136 | + | got, err := cmd.Control(ctx, test.args, ControlOptions{}) |
| 128 | - | if !errors.Is(err, test.wantErr) || ctx.call != test.wantCall { | 137 | + | wantOutput := "" |
| 129 | - | t.Errorf("args %v: call/error = %q/%v, want %q/%v", test.args, ctx.call, err, test.wantCall, test.wantErr) | 138 | + | if test.wantCall != "" { |
| 139 | + | wantOutput = ctx.recoveryOutput | |||
| 140 | + | } | |||
| 141 | + | if !errors.Is(err, test.wantErr) || ctx.call != test.wantCall || got.Output != wantOutput { | |||
| 142 | + | t.Errorf("args %v: call/output/error = %q/%q/%v, want %q/%q/%v", test.args, ctx.call, got.Output, err, test.wantCall, wantOutput, test.wantErr) | |||
| 130 | } | 143 | } | ||
| 131 | } | 144 | } | ||
| 132 | } | 145 | } | ||
| @@ -175,7 +188,10 @@ func TestControlHandlersValidateAndDelegate(t *testing.T) { | |||||
| 175 | } | 188 | } | ||
| 176 | } | 189 | } | ||
| 177 | 190 | ||||
| 178 | - | type controlSpy struct{ call string } | 191 | + | type controlSpy struct { |
| 192 | + | call string | |||
| 193 | + | recoveryOutput string | |||
| 194 | + | } | |||
| 179 | 195 | ||||
| 180 | func (s *controlSpy) record(call string) error { s.call = call; return nil } | 196 | func (s *controlSpy) record(call string) error { s.call = call; return nil } | ||
| 181 | func (s *controlSpy) CreateTab() error { return s.record("new-tab") } | 197 | func (s *controlSpy) CreateTab() error { return s.record("new-tab") } | ||
| @@ -205,7 +221,7 @@ func (s *controlSpy) Toast(level, message string) error { | |||||
| 205 | return s.record("toast:" + level + ":" + message) | 221 | return s.record("toast:" + level + ":" + message) | ||
| 206 | } | 222 | } | ||
| 207 | func (s *controlSpy) SessionRecovery(action, argument string) (string, error) { | 223 | func (s *controlSpy) SessionRecovery(action, argument string) (string, error) { | ||
| 208 | - | return "", s.record("session-recovery:" + action + ":" + argument) | 224 | + | return s.recoveryOutput, s.record("session-recovery:" + action + ":" + argument) |
| 209 | } | 225 | } | ||
| 210 | func (s *controlSpy) ListSessions(json bool) (string, error) { | 226 | func (s *controlSpy) ListSessions(json bool) (string, error) { | ||
| 211 | _ = s.record("list-sessions:" + strconv.FormatBool(json)) | 227 | _ = s.record("list-sessions:" + strconv.FormatBool(json)) | ||
M internal/usecase/daemon/catalogue.go +56 -11
| @@ -7,30 +7,75 @@ import ( | |||||
| 7 | // Catalogue helpers keep persistence optional for ephemeral and focused unit | 7 | // Catalogue helpers keep persistence optional for ephemeral and focused unit | ||
| 8 | // tests while ensuring every durable named-session mutation uses the authority | 8 | // tests while ensuring every durable named-session mutation uses the authority | ||
| 9 | // port when one is configured. | 9 | // port when one is configured. | ||
| 10 | - | func (d *Daemon) catalogueRecord(name string) (domain.CatalogueRecord, bool) { | 10 | + | func (d *Daemon) catalogueRecord(name string) (domain.CatalogueRecord, bool, error) { |
| 11 | if d == nil || d.catalogue == nil { | 11 | if d == nil || d.catalogue == nil { | ||
| 12 | - | return domain.CatalogueRecord{}, false | 12 | + | return domain.CatalogueRecord{}, false, nil |
| 13 | } | 13 | } | ||
| 14 | return d.catalogue.Record(name) | 14 | return d.catalogue.Record(name) | ||
| 15 | } | 15 | } | ||
| 16 | 16 | ||||
| 17 | - | func (d *Daemon) createCatalogueRecord(record domain.CatalogueRecord) error { | 17 | + | func (d *Daemon) updateCatalogueMetadata(update domain.CatalogueMetadataUpdate) error { |
| 18 | if d == nil || d.catalogue == nil { | 18 | if d == nil || d.catalogue == nil { | ||
| 19 | return nil | 19 | return nil | ||
| 20 | } | 20 | } | ||
| 21 | - | return d.catalogue.Create(record) | 21 | + | return d.catalogue.UpdateMetadata(update) |
| 22 | } | 22 | } | ||
| 23 | 23 | ||||
| 24 | - | func (d *Daemon) updateCatalogueMetadata(update domain.CatalogueMetadataUpdate) error { | 24 | + | // persistSessionMetadata serializes authority writes without holding daemon or |
| 25 | - | if d == nil || d.catalogue == nil { | 25 | + | // session state locks during I/O. Only a successfully committed newer revision |
| 26 | + | // subsumes an older caller. Failed revisions retain their rollback until later | |||
| 27 | + | // failed mutations have unwound, keeping live metadata at the durable boundary. | |||
| 28 | + | func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update domain.CatalogueMetadataUpdate, rollback func() bool) error { | |||
| 29 | + | if sess == nil { | |||
| 26 | return nil | 30 | return nil | ||
| 27 | } | 31 | } | ||
| 28 | - | return d.catalogue.UpdateMetadata(update) | 32 | + | sess.metadataPersistMu.Lock() |
| 29 | - | } | 33 | + | defer sess.metadataPersistMu.Unlock() |
| 30 | 34 | ||||
| 31 | - | func (d *Daemon) deleteCatalogueRecord(name string) error { | 35 | + | if sess.metadataDurableVersion >= version { |
| 32 | - | if d == nil || d.catalogue == nil { | |||
| 33 | return nil | 36 | return nil | ||
| 34 | } | 37 | } | ||
| 35 | - | return d.catalogue.Delete(name) | 38 | + | if err := d.updateCatalogueMetadata(update); err != nil { |
| 39 | + | if sess.metadataFailedRollbacks == nil { | |||
| 40 | + | sess.metadataFailedRollbacks = make(map[uint64]func() bool) | |||
| 41 | + | } | |||
| 42 | + | sess.metadataFailedRollbacks[version] = rollback | |||
| 43 | + | // A later mutation may have been reserved while authority I/O blocked. | |||
| 44 | + | // It must settle before an older failed mutation can be unwound. | |||
| 45 | + | sess.mu.Lock() | |||
| 46 | + | current := sess.metadataVersion | |||
| 47 | + | sess.mu.Unlock() | |||
| 48 | + | if current > sess.metadataLiveVersion { | |||
| 49 | + | sess.metadataLiveVersion = current | |||
| 50 | + | } | |||
| 51 | + | sess.reconcileFailedMetadataLocked() | |||
| 52 | + | return err | |||
| 53 | + | } | |||
| 54 | + | ||||
| 55 | + | sess.metadataDurableVersion = version | |||
| 56 | + | for failedVersion := range sess.metadataFailedRollbacks { | |||
| 57 | + | if failedVersion <= version { | |||
| 58 | + | delete(sess.metadataFailedRollbacks, failedVersion) | |||
| 59 | + | } | |||
| 60 | + | } | |||
| 61 | + | return nil | |||
| 62 | + | } | |||
| 63 | + | ||||
| 64 | + | // reconcileFailedMetadataLocked unwinds only a contiguous failed suffix. An | |||
| 65 | + | // older failure remains pending while a newer mutation is still writing. | |||
| 66 | + | // Rollbacks run only after authority I/O has returned and acquire state locks in | |||
| 67 | + | // the daemon-before-session order. | |||
| 68 | + | func (s *session) reconcileFailedMetadataLocked() { | |||
| 69 | + | for s.metadataLiveVersion > s.metadataDurableVersion { | |||
| 70 | + | version := s.metadataLiveVersion | |||
| 71 | + | rollback, failed := s.metadataFailedRollbacks[version] | |||
| 72 | + | if !failed { | |||
| 73 | + | return | |||
| 74 | + | } | |||
| 75 | + | if rollback != nil && !rollback() { | |||
| 76 | + | return | |||
| 77 | + | } | |||
| 78 | + | delete(s.metadataFailedRollbacks, version) | |||
| 79 | + | s.metadataLiveVersion-- | |||
| 80 | + | } | |||
| 36 | } | 81 | } | ||
M internal/usecase/daemon/control.go +37 -22
| @@ -6,6 +6,8 @@ import ( | |||||
| 6 | "errors" | 6 | "errors" | ||
| 7 | "fmt" | 7 | "fmt" | ||
| 8 | "os" | 8 | "os" | ||
| 9 | + | "path/filepath" | |||
| 10 | + | "slices" | |||
| 9 | "sort" | 11 | "sort" | ||
| 10 | "strconv" | 12 | "strconv" | ||
| 11 | "strings" | 13 | "strings" | ||
| @@ -30,14 +32,14 @@ func (d *Daemon) handleCommand(tr ports.Transport, f ports.Frame) { | |||||
| 30 | _ = tr.Send(frameCommandResult(ports.CommandResult{Code: ports.ErrInternal, Text: "malformed command request"})) | 32 | _ = tr.Send(frameCommandResult(ports.CommandResult{Code: ports.ErrInternal, Text: "malformed command request"})) | ||
| 31 | return | 33 | return | ||
| 32 | } | 34 | } | ||
| 33 | - | _ = tr.Send(frameCommandResult(d.dispatchCommand(request))) | 35 | + | _ = tr.Send(frameCommandResult(d.dispatchCommand(d.serveCtx, request))) |
| 34 | } | 36 | } | ||
| 35 | 37 | ||||
| 36 | func frameCommandResult(result ports.CommandResult) ports.Frame { | 38 | func frameCommandResult(result ports.CommandResult) ports.Frame { | ||
| 37 | return ports.Frame{Type: ports.MsgCommandResult, Payload: ports.MarshalCommandResult(result)} | 39 | return ports.Frame{Type: ports.MsgCommandResult, Payload: ports.MarshalCommandResult(result)} | ||
| 38 | } | 40 | } | ||
| 39 | 41 | ||||
| 40 | - | func (d *Daemon) dispatchCommand(request ports.CommandRequest) ports.CommandResult { | 42 | + | func (d *Daemon) dispatchCommand(ctx context.Context, request ports.CommandRequest) ports.CommandResult { |
| 41 | cmd, ok := command.BySlug(request.Slug) | 43 | cmd, ok := command.BySlug(request.Slug) | ||
| 42 | if !ok { | 44 | if !ok { | ||
| 43 | return commandFailure(ports.ErrUnknownCommand, "unknown command: "+request.Slug) | 45 | return commandFailure(ports.ErrUnknownCommand, "unknown command: "+request.Slug) | ||
| @@ -46,7 +48,7 @@ func (d *Daemon) dispatchCommand(request ports.CommandRequest) ports.CommandResu | |||||
| 46 | return commandFailure(ports.ErrNotScriptable, request.Slug+" requires an attached client") | 48 | return commandFailure(ports.ErrNotScriptable, request.Slug+" requires an attached client") | ||
| 47 | } | 49 | } | ||
| 48 | if cmd.Target == command.TargetNone && !request.Self { | 50 | if cmd.Target == command.TargetNone && !request.Self { | ||
| 49 | - | return d.runControl(cmd, controlExec{d: d, recoveryName: request.TargetSession}, request) | 51 | + | return d.runControl(cmd, controlExec{ctx: ctx, d: d, recoveryName: request.TargetSession}, request) |
| 50 | } | 52 | } | ||
| 51 | 53 | ||||
| 52 | sess, code, text := d.resolveTargetSession(request) | 54 | sess, code, text := d.resolveTargetSession(request) | ||
| @@ -60,7 +62,7 @@ func (d *Daemon) dispatchCommand(request ports.CommandRequest) ports.CommandResu | |||||
| 60 | if code != 0 { | 62 | if code != 0 { | ||
| 61 | return commandFailure(code, text) | 63 | return commandFailure(code, text) | ||
| 62 | } | 64 | } | ||
| 63 | - | return d.runControl(cmd, controlExec{d: d, sess: sess, tab: tb, target: daemonActionTarget{session: sess, tab: tb, pane: pane}}, request) | 65 | + | return d.runControl(cmd, controlExec{ctx: ctx, d: d, sess: sess, tab: tb, target: daemonActionTarget{session: sess, tab: tb, pane: pane}}, request) |
| 64 | } | 66 | } | ||
| 65 | 67 | ||||
| 66 | func (d *Daemon) runControl(cmd command.Command, exec controlExec, request ports.CommandRequest) ports.CommandResult { | 68 | func (d *Daemon) runControl(cmd command.Command, exec controlExec, request ports.CommandRequest) ports.CommandResult { | ||
| @@ -309,14 +311,7 @@ func (d *Daemon) hasDaemonActionPaneTarget(target daemonActionTarget) bool { | |||||
| 309 | } | 311 | } | ||
| 310 | target.session.mu.Lock() | 312 | target.session.mu.Lock() | ||
| 311 | defer target.session.mu.Unlock() | 313 | defer target.session.mu.Unlock() | ||
| 312 | - | foundTab := false | 314 | + | if !slices.Contains(target.session.tabs, target.tab) { |
| 313 | - | for _, tb := range target.session.tabs { | |||
| 314 | - | if tb == target.tab { | |||
| 315 | - | foundTab = true | |||
| 316 | - | break | |||
| 317 | - | } | |||
| 318 | - | } | |||
| 319 | - | if !foundTab { | |||
| 320 | return false | 315 | return false | ||
| 321 | } | 316 | } | ||
| 322 | target.tab.mu.Lock() | 317 | target.tab.mu.Lock() | ||
| @@ -335,6 +330,7 @@ func (a daemonActions) switchRelative(sess *session, delta int) error { | |||||
| 335 | // controlExec implements command.ControlContext against resolved daemon-owned | 330 | // controlExec implements command.ControlContext against resolved daemon-owned | ||
| 336 | // targets. It deliberately contains no attached-client state. | 331 | // targets. It deliberately contains no attached-client state. | ||
| 337 | type controlExec struct { | 332 | type controlExec struct { | ||
| 333 | + | ctx context.Context | |||
| 338 | d *Daemon | 334 | d *Daemon | ||
| 339 | sess *session | 335 | sess *session | ||
| 340 | tab *tab | 336 | tab *tab | ||
| @@ -465,7 +461,10 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) { | |||||
| 465 | if e.d == nil || e.d.degradedRecovery == nil || e.recoveryName == "" { | 461 | if e.d == nil || e.d.degradedRecovery == nil || e.recoveryName == "" { | ||
| 466 | return "", command.ErrInvalidArguments | 462 | return "", command.ErrInvalidArguments | ||
| 467 | } | 463 | } | ||
| 468 | - | ctx := context.Background() | 464 | + | ctx := e.ctx |
| 465 | + | if ctx == nil { | |||
| 466 | + | ctx = e.d.serveCtx | |||
| 467 | + | } | |||
| 469 | switch action { | 468 | switch action { | ||
| 470 | case "retry": | 469 | case "retry": | ||
| 471 | if err := e.d.degradedRecovery.Retry(ctx, e.recoveryName); err != nil { | 470 | if err := e.d.degradedRecovery.Retry(ctx, e.recoveryName); err != nil { | ||
| @@ -480,7 +479,10 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) { | |||||
| 480 | if err != nil || generation == 0 { | 479 | if err != nil || generation == 0 { | ||
| 481 | return "", command.ErrInvalidArguments | 480 | return "", command.ErrInvalidArguments | ||
| 482 | } | 481 | } | ||
| 483 | - | record, ok := e.d.catalogue.Record(e.recoveryName) | 482 | + | record, ok, err := e.d.catalogue.Record(e.recoveryName) |
| 483 | + | if err != nil { | |||
| 484 | + | return "", err | |||
| 485 | + | } | |||
| 484 | if !ok { | 486 | if !ok { | ||
| 485 | return "", errors.New("session recovery record not found") | 487 | return "", errors.New("session recovery record not found") | ||
| 486 | } | 488 | } | ||
| @@ -494,6 +496,9 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) { | |||||
| 494 | } | 496 | } | ||
| 495 | return "", errors.New("requested recovery fallback is not available") | 497 | return "", errors.New("requested recovery fallback is not available") | ||
| 496 | case "export": | 498 | case "export": | ||
| 499 | + | if !filepath.IsAbs(argument) { | |||
| 500 | + | return "", command.ErrInvalidArguments | |||
| 501 | + | } | |||
| 497 | file, err := os.OpenFile(argument, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) | 502 | file, err := os.OpenFile(argument, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) | ||
| 498 | if err != nil { | 503 | if err != nil { | ||
| 499 | return "", err | 504 | return "", err | ||
| @@ -520,20 +525,30 @@ func (e controlExec) restoreExplicitRecovery(ctx context.Context) error { | |||||
| 520 | if e.d.catalogue == nil { | 525 | if e.d.catalogue == nil { | ||
| 521 | return errors.New("durable session catalogue unavailable") | 526 | return errors.New("durable session catalogue unavailable") | ||
| 522 | } | 527 | } | ||
| 523 | - | record, ok := e.d.catalogue.Record(e.recoveryName) | 528 | + | record, ok, err := e.d.catalogue.Record(e.recoveryName) |
| 529 | + | if err != nil { | |||
| 530 | + | return err | |||
| 531 | + | } | |||
| 524 | if !ok { | 532 | if !ok { | ||
| 525 | return errors.New("session recovery record not found") | 533 | return errors.New("session recovery record not found") | ||
| 526 | } | 534 | } | ||
| 527 | done := make(chan struct{}) | 535 | done := make(chan struct{}) | ||
| 528 | e.d.mu.Lock() | 536 | e.d.mu.Lock() | ||
| 529 | - | if entry, exists := e.d.stopped[record.Name]; exists { | 537 | + | entry, exists := e.d.stopped[record.Name] |
| 530 | - | entry.record = record | 538 | + | if !exists || entry.incarnation != record.IncarnationID { |
| 531 | - | entry.state = runtimeRestoring | 539 | + | e.d.mu.Unlock() |
| 532 | - | entry.restoreDone = done | 540 | + | return errors.New("session recovery runtime record not found") |
| 533 | - | e.d.stopped[record.Name] = entry | 541 | + | } |
| 534 | - | } | 542 | + | if entry.purging || entry.state != runtimeDegraded { |
| 543 | + | e.d.mu.Unlock() | |||
| 544 | + | return errors.New("session recovery is not available in the current state") | |||
| 545 | + | } | |||
| 546 | + | entry.record = record | |||
| 547 | + | entry.state = runtimeRestoring | |||
| 548 | + | entry.restoreDone = done | |||
| 549 | + | e.d.stopped[record.Name] = entry | |||
| 535 | e.d.mu.Unlock() | 550 | e.d.mu.Unlock() | ||
| 536 | - | err := e.d.restoreRecord(ctx, record) | 551 | + | err = e.d.restoreRecord(ctx, record) |
| 537 | e.d.finishRecordRestore(record, err, done) | 552 | e.d.finishRecordRestore(record, err, done) | ||
| 538 | return err | 553 | return err | ||
| 539 | } | 554 | } | ||
M internal/usecase/daemon/daemon.go +72 -70
| @@ -7,13 +7,13 @@ import ( | |||||
| 7 | "log/slog" | 7 | "log/slog" | ||
| 8 | "math" | 8 | "math" | ||
| 9 | "os" | 9 | "os" | ||
| 10 | + | "reflect" | |||
| 10 | "sort" | 11 | "sort" | ||
| 11 | "sync" | 12 | "sync" | ||
| 12 | "sync/atomic" | 13 | "sync/atomic" | ||
| 13 | "time" | 14 | "time" | ||
| 14 | 15 | ||||
| 15 | "github.com/bnema/vev/internal/domain" | 16 | "github.com/bnema/vev/internal/domain" | ||
| 16 | - | "github.com/bnema/vev/internal/persist" | |||
| 17 | "github.com/bnema/vev/internal/ports" | 17 | "github.com/bnema/vev/internal/ports" | ||
| 18 | "github.com/bnema/vev/internal/usecase/keys" | 18 | "github.com/bnema/vev/internal/usecase/keys" | ||
| 19 | ) | 19 | ) | ||
| @@ -70,6 +70,8 @@ type Daemon struct { | |||||
| 70 | mu sync.Mutex | 70 | mu sync.Mutex | ||
| 71 | sessions map[domain.SessionID]*session | 71 | sessions map[domain.SessionID]*session | ||
| 72 | stopped map[string]stoppedSession | 72 | stopped map[string]stoppedSession | ||
| 73 | + | // creating reserves names while durable creation I/O runs without mu. | |||
| 74 | + | creating map[string]struct{} | |||
| 73 | nextID uint64 | 75 | nextID uint64 | ||
| 74 | // lastAllocatedCreatedAt is the named-session lifecycle timestamp high-water | 76 | // lastAllocatedCreatedAt is the named-session lifecycle timestamp high-water | ||
| 75 | // mark. It is guarded by mu and prevents a wall-clock regression from | 77 | // mark. It is guarded by mu and prevents a wall-clock regression from | ||
| @@ -117,13 +119,11 @@ type Daemon struct { | |||||
| 117 | shell string | 119 | shell string | ||
| 118 | shellArgs []string | 120 | shellArgs []string | ||
| 119 | shellOverride bool | 121 | shellOverride bool | ||
| 120 | - | persist *persist.Persister | |||
| 121 | persistEnabled bool | 122 | persistEnabled bool | ||
| 122 | catalogue ports.Catalogue | 123 | catalogue ports.Catalogue | ||
| 123 | catalogueRecords []domain.CatalogueRecord | 124 | catalogueRecords []domain.CatalogueRecord | ||
| 124 | catalogueRecordsProvided bool | 125 | catalogueRecordsProvided bool | ||
| 125 | - | // snapshotRepository is the sole checkpoint contract. legacySnapshots is | 126 | + | // snapshotRepository is the sole checkpoint contract. |
| 126 | - | // read-only migration input and is never used for new writes. | |||
| 127 | snapshotRepository ports.SnapshotRepository | 127 | snapshotRepository ports.SnapshotRepository | ||
| 128 | checkpointRecovery ports.CheckpointCoordinator | 128 | checkpointRecovery ports.CheckpointCoordinator | ||
| 129 | lifecycleRecovery ports.SessionLifecycleCoordinator | 129 | lifecycleRecovery ports.SessionLifecycleCoordinator | ||
| @@ -136,10 +136,12 @@ type Daemon struct { | |||||
| 136 | // records, so it is a durable writer and is guarded by snapshotWorkerMu with | 136 | // records, so it is a durable writer and is guarded by snapshotWorkerMu with | ||
| 137 | // the other two. | 137 | // the other two. | ||
| 138 | restoreWorkerDone chan struct{} | 138 | restoreWorkerDone chan struct{} | ||
| 139 | - | legacySnapshots ports.LegacySnapshotSource | |||
| 140 | snapsEnabled bool | 139 | snapsEnabled bool | ||
| 141 | noticeStore ports.NoticeStore | 140 | noticeStore ports.NoticeStore | ||
| 142 | snapshotJobs chan *snapshotCapture | 141 | snapshotJobs chan *snapshotCapture | ||
| 142 | + | // snapshotAdmitted contains every capture accepted by either worker queue, | |||
| 143 | + | // including captures buffered in snapshotJobs. Guarded by snapshotWorkerMu. | |||
| 144 | + | snapshotAdmitted map[*snapshotCapture]struct{} | |||
| 143 | // snapshotWake wakes the repository scheduler when a session becomes dirty | 145 | // snapshotWake wakes the repository scheduler when a session becomes dirty | ||
| 144 | // or an attempt completes. It is never closed and producers only send | 146 | // or an attempt completes. It is never closed and producers only send | ||
| 145 | // non-blockingly. | 147 | // non-blockingly. | ||
| @@ -263,16 +265,6 @@ func WithShell(cmd string, args []string) Option { | |||||
| 263 | } | 265 | } | ||
| 264 | } | 266 | } | ||
| 265 | 267 | ||||
| 266 | - | // WithStore enables persisted named session metadata. A nil store keeps the | |||
| 267 | - | // daemon in no-op persistence mode. | |||
| 268 | - | func WithStore(store ports.Store) Option { | |||
| 269 | - | return func(d *Daemon) { | |||
| 270 | - | d.persist = persist.New(store) | |||
| 271 | - | d.catalogue = d.persist | |||
| 272 | - | d.persistEnabled = store != nil | |||
| 273 | - | } | |||
| 274 | - | } | |||
| 275 | - | ||||
| 276 | // WithCatalogue installs the singular catalogue and the strictly opened | 268 | // WithCatalogue installs the singular catalogue and the strictly opened | ||
| 277 | // records that define the daemon's expected-session registry at startup. | 269 | // records that define the daemon's expected-session registry at startup. | ||
| 278 | func WithCatalogue(catalogue ports.Catalogue, records []domain.CatalogueRecord) Option { | 270 | func WithCatalogue(catalogue ports.Catalogue, records []domain.CatalogueRecord) Option { | ||
| @@ -303,26 +295,30 @@ func WithRecoveryCoordinator(coordinator ports.SessionLifecycleCoordinator) Opti | |||||
| 303 | } | 295 | } | ||
| 304 | } | 296 | } | ||
| 305 | 297 | ||||
| 306 | - | // WithSnapshotRepository enables content-addressed incremental snapshots. The | 298 | + | // WithSnapshotRepository enables content-addressed incremental snapshots. |
| 307 | - | // legacy source is read-only migration input and is never used for new writes. | 299 | + | func WithSnapshotRepository(repository ports.SnapshotRepository) Option { |
| 308 | - | func WithSnapshotRepository(repository any, legacy ports.LegacySnapshotSource) Option { | |||
| 309 | return func(d *Daemon) { | 300 | return func(d *Daemon) { | ||
| 310 | - | if repository == nil { | 301 | + | if isNilSnapshotRepository(repository) { |
| 311 | return | 302 | return | ||
| 312 | } | 303 | } | ||
| 313 | - | typed, ok := repository.(ports.SnapshotRepository) | 304 | + | d.snapshotRepository = repository |
| 314 | - | if !ok { | |||
| 315 | - | // Pre-incarnation test repositories are intentionally not admitted to | |||
| 316 | - | // normal runtime paths. They remain accepted by this option only so | |||
| 317 | - | // package-level legacy tests can exercise their isolated helpers. | |||
| 318 | - | return | |||
| 319 | - | } | |||
| 320 | - | d.snapshotRepository = typed | |||
| 321 | - | d.legacySnapshots = legacy | |||
| 322 | d.snapsEnabled = true | 305 | d.snapsEnabled = true | ||
| 323 | } | 306 | } | ||
| 324 | } | 307 | } | ||
| 325 | 308 | ||||
| 309 | + | func isNilSnapshotRepository(repository ports.SnapshotRepository) bool { | |||
| 310 | + | if repository == nil { | |||
| 311 | + | return true | |||
| 312 | + | } | |||
| 313 | + | value := reflect.ValueOf(repository) | |||
| 314 | + | switch value.Kind() { | |||
| 315 | + | case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: | |||
| 316 | + | return value.IsNil() | |||
| 317 | + | default: | |||
| 318 | + | return false | |||
| 319 | + | } | |||
| 320 | + | } | |||
| 321 | + | ||||
| 326 | // WithNoticeStore enables persisting undeliverable notices across daemon | 322 | // WithNoticeStore enables persisting undeliverable notices across daemon | ||
| 327 | // restarts. A nil store keeps the daemon in no-op notice-persistence mode. | 323 | // restarts. A nil store keeps the daemon in no-op notice-persistence mode. | ||
| 328 | func WithNoticeStore(store ports.NoticeStore) Option { | 324 | func WithNoticeStore(store ports.NoticeStore) Option { | ||
| @@ -420,6 +416,9 @@ func (d *Daemon) allocateLifecycleCreatedAtLocked() (int64, error) { | |||||
| 420 | } | 416 | } | ||
| 421 | 417 | ||||
| 422 | func (d *Daemon) nowUnixNano() int64 { | 418 | func (d *Daemon) nowUnixNano() int64 { | ||
| 419 | + | if d == nil || d.clock == nil { | |||
| 420 | + | return time.Now().UnixNano() | |||
| 421 | + | } | |||
| 423 | return d.clock.Now().UnixNano() | 422 | return d.clock.Now().UnixNano() | ||
| 424 | } | 423 | } | ||
| 425 | 424 | ||||
| @@ -434,23 +433,24 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt | |||||
| 434 | shell = "/bin/sh" | 433 | shell = "/bin/sh" | ||
| 435 | } | 434 | } | ||
| 436 | d := &Daemon{ | 435 | d := &Daemon{ | ||
| 437 | - | sessions: make(map[domain.SessionID]*session), | 436 | + | sessions: make(map[domain.SessionID]*session), |
| 438 | - | stopped: make(map[string]stoppedSession), | 437 | + | stopped: make(map[string]stoppedSession), |
| 439 | - | parked: make(map[uint64]*parkedAttachment), | 438 | + | creating: make(map[string]struct{}), |
| 440 | - | ptys: ptys, | 439 | + | parked: make(map[uint64]*parkedAttachment), |
| 441 | - | clock: clock, | 440 | + | ptys: ptys, |
| 442 | - | log: log, | 441 | + | clock: clock, |
| 443 | - | baseEnv: os.Environ(), | 442 | + | log: log, |
| 444 | - | shell: shell, | 443 | + | baseEnv: os.Environ(), |
| 445 | - | persist: persist.New(nil), | 444 | + | shell: shell, |
| 446 | - | dirOrHome: dirOrHome, | 445 | + | dirOrHome: dirOrHome, |
| 447 | - | done: make(chan struct{}), | 446 | + | done: make(chan struct{}), |
| 448 | - | restoreDone: make(chan struct{}), | 447 | + | restoreDone: make(chan struct{}), |
| 449 | - | animWake: make(chan struct{}, 1), | 448 | + | animWake: make(chan struct{}, 1), |
| 450 | - | snapshotJobs: make(chan *snapshotCapture, snapshotQueueCapacity), | 449 | + | snapshotJobs: make(chan *snapshotCapture, snapshotQueueCapacity), |
| 451 | - | snapshotWake: make(chan struct{}, 1), | 450 | + | snapshotAdmitted: make(map[*snapshotCapture]struct{}), |
| 452 | - | notices: newNoticeCenter(), | 451 | + | snapshotWake: make(chan struct{}, 1), |
| 453 | - | resumeParkGrace: defaultResumeParkGrace, | 452 | + | notices: newNoticeCenter(), |
| 453 | + | resumeParkGrace: defaultResumeParkGrace, | |||
| 454 | barScripts: &barScriptState{ | 454 | barScripts: &barScriptState{ | ||
| 455 | cfg: barConfigFromDomain(domain.Defaults().Bar), | 455 | cfg: barConfigFromDomain(domain.Defaults().Bar), | ||
| 456 | outputs: make(map[domain.SessionID]barScriptOutputs), | 456 | outputs: make(map[domain.SessionID]barScriptOutputs), | ||
| @@ -470,9 +470,6 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt | |||||
| 470 | for _, o := range opts { | 470 | for _, o := range opts { | ||
| 471 | o(d) | 471 | o(d) | ||
| 472 | } | 472 | } | ||
| 473 | - | if d.persist == nil { | |||
| 474 | - | d.persist = persist.New(nil) | |||
| 475 | - | } | |||
| 476 | if d.dirOrHome == nil { | 473 | if d.dirOrHome == nil { | ||
| 477 | d.dirOrHome = dirOrHome | 474 | d.dirOrHome = dirOrHome | ||
| 478 | } | 475 | } | ||
| @@ -488,20 +485,6 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt | |||||
| 488 | d.restoreProcessAllowlist.Store(&allow) | 485 | d.restoreProcessAllowlist.Store(&allow) | ||
| 489 | } | 486 | } | ||
| 490 | records := d.catalogueRecords | 487 | records := d.catalogueRecords | ||
| 491 | - | // Only attempt the load when persistence is actually configured | |||
| 492 | - | // (d.persistEnabled tracks the same store-is-non-nil condition | |||
| 493 | - | // persist.Persister uses to distinguish "no store" from "load failed"). | |||
| 494 | - | // Without this gate, the default no-persistence Daemon (persist.New(nil)) | |||
| 495 | - | // would log a false-alarm "loading persisted sessions failed" warning on | |||
| 496 | - | // every construction: nothing failed, persistence simply isn't enabled. | |||
| 497 | - | if !d.catalogueRecordsProvided && d.persistEnabled { | |||
| 498 | - | var err error | |||
| 499 | - | records, err = d.persist.LoadAll() | |||
| 500 | - | if err != nil { | |||
| 501 | - | d.log.Warn("loading persisted sessions failed", "err", err) | |||
| 502 | - | return d | |||
| 503 | - | } | |||
| 504 | - | } | |||
| 505 | var maxSeq uint64 | 488 | var maxSeq uint64 | ||
| 506 | var maxCreatedAt int64 | 489 | var maxCreatedAt int64 | ||
| 507 | hasCreatedAt := false | 490 | hasCreatedAt := false | ||
| @@ -628,8 +611,6 @@ func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error { | |||||
| 628 | var closeErr error | 611 | var closeErr error | ||
| 629 | if d.catalogue != nil { | 612 | if d.catalogue != nil { | ||
| 630 | closeErr = d.catalogue.Close() | 613 | closeErr = d.catalogue.Close() | ||
| 631 | - | } else { | |||
| 632 | - | closeErr = d.persist.Close() | |||
| 633 | } | 614 | } | ||
| 634 | if closeErr != nil { | 615 | if closeErr != nil { | ||
| 635 | d.log.Warn("closing session persister failed", "err", closeErr) | 616 | d.log.Warn("closing session persister failed", "err", closeErr) | ||
| @@ -662,11 +643,23 @@ func (d *Daemon) shutdownAllWithSnapshotDeadline(reason uint8, deadline *snapsho | |||||
| 662 | for _, s := range snapshot { | 643 | for _, s := range snapshot { | ||
| 663 | // Cancellation and PTY closure must not wait behind a teardown owner that | 644 | // Cancellation and PTY closure must not wait behind a teardown owner that | ||
| 664 | // is blocked in snapshot publication or purge work. | 645 | // is blocked in snapshot publication or purge work. | ||
| 646 | + | s.mu.Lock() | |||
| 647 | + | name := s.name | |||
| 648 | + | ephemeral := s.ephemeral | |||
| 649 | + | s.mu.Unlock() | |||
| 665 | s.stopInMemoryLifecycle() | 650 | s.stopInMemoryLifecycle() | ||
| 666 | if err := d.killSessionWithSnapshotDeadline(s, reason, false, deadline); err != nil { | 651 | if err := d.killSessionWithSnapshotDeadline(s, reason, false, deadline); err != nil { | ||
| 667 | checkpointIncomplete = true | 652 | checkpointIncomplete = true | ||
| 668 | d.log.Error("closing session with unpersisted terminal state", "err", err) | 653 | d.log.Error("closing session with unpersisted terminal state", "err", err) | ||
| 669 | } | 654 | } | ||
| 655 | + | if !ephemeral && deadline != nil { | |||
| 656 | + | select { | |||
| 657 | + | case <-deadline.Done(): | |||
| 658 | + | checkpointIncomplete = true | |||
| 659 | + | d.persistShutdownSnapshotFailure(name, context.DeadlineExceeded) | |||
| 660 | + | default: | |||
| 661 | + | } | |||
| 662 | + | } | |||
| 670 | } | 663 | } | ||
| 671 | return checkpointIncomplete | 664 | return checkpointIncomplete | ||
| 672 | } | 665 | } | ||
| @@ -738,6 +731,12 @@ func snapshotStopContext(deadline *snapshotShutdownDeadline) (context.Context, c | |||||
| 738 | if deadline == nil { | 731 | if deadline == nil { | ||
| 739 | return ctx, cancel | 732 | return ctx, cancel | ||
| 740 | } | 733 | } | ||
| 734 | + | select { | |||
| 735 | + | case <-deadline.Done(): | |||
| 736 | + | cancel() | |||
| 737 | + | return ctx, cancel | |||
| 738 | + | default: | |||
| 739 | + | } | |||
| 741 | stop := make(chan struct{}) | 740 | stop := make(chan struct{}) | ||
| 742 | go func() { | 741 | go func() { | ||
| 743 | select { | 742 | select { | ||
| @@ -817,13 +816,16 @@ func (d *Daemon) handleList(tr ports.Transport) { | |||||
| 817 | } | 816 | } | ||
| 818 | state := ports.SessionStopped | 817 | state := ports.SessionStopped | ||
| 819 | if stopped.purging { | 818 | if stopped.purging { | ||
| 819 | + | // Purge is the dominant externally visible state: restoration must | |||
| 820 | + | // never make a deletion-reserved record appear attachable. | |||
| 820 | state = ports.SessionDegraded | 821 | state = ports.SessionDegraded | ||
| 821 | - | } | 822 | + | } else { |
| 822 | - | switch stopped.state { | 823 | + | switch stopped.state { |
| 823 | - | case runtimeRestoring: | 824 | + | case runtimeRestoring: |
| 824 | - | state = ports.SessionRestoring | 825 | + | state = ports.SessionRestoring |
| 825 | - | case runtimeDegraded, runtimeDeleting: | 826 | + | case runtimeDegraded, runtimeDeleting: |
| 826 | - | state = ports.SessionDegraded | 827 | + | state = ports.SessionDegraded |
| 828 | + | } | |||
| 827 | } | 829 | } | ||
| 828 | infos = append(infos, ports.SessionInfo{Name: name, State: state}) | 830 | infos = append(infos, ports.SessionInfo{Name: name, State: state}) | ||
| 829 | } | 831 | } | ||
A internal/usecase/daemon/durable_authority_test.go +49 -0
| @@ -0,0 +1,49 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "crypto/rand" | |||
| 6 | + | ||||
| 7 | + | "github.com/bnema/vev/internal/domain" | |||
| 8 | + | "github.com/bnema/vev/internal/persist" | |||
| 9 | + | "github.com/bnema/vev/internal/ports" | |||
| 10 | + | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | |||
| 11 | + | ) | |||
| 12 | + | ||||
| 13 | + | type testLifecycleRepository struct{ ports.SnapshotRepository } | |||
| 14 | + | ||||
| 15 | + | func (testLifecycleRepository) WriteDeletionTombstone(context.Context, domain.DeletionTombstone) error { | |||
| 16 | + | return nil | |||
| 17 | + | } | |||
| 18 | + | func (testLifecycleRepository) QuarantineDeletionSources(context.Context, domain.DeletionTombstone, bool) error { | |||
| 19 | + | return nil | |||
| 20 | + | } | |||
| 21 | + | func (testLifecycleRepository) DeleteDeletionTombstone(context.Context, domain.IncarnationID) error { | |||
| 22 | + | return nil | |||
| 23 | + | } | |||
| 24 | + | ||||
| 25 | + | // WithStore is package-test composition glue. Production injects the same | |||
| 26 | + | // recovery coordinator explicitly from internal/app. | |||
| 27 | + | func WithStore(store ports.Store) Option { | |||
| 28 | + | return func(d *Daemon) { | |||
| 29 | + | if store == nil { | |||
| 30 | + | return | |||
| 31 | + | } | |||
| 32 | + | catalogue := persist.New(store) | |||
| 33 | + | records, err := catalogue.Records() | |||
| 34 | + | if err != nil { | |||
| 35 | + | return | |||
| 36 | + | } | |||
| 37 | + | d.catalogue = catalogue | |||
| 38 | + | d.persistEnabled = true | |||
| 39 | + | d.catalogueRecords = records | |||
| 40 | + | d.catalogueRecordsProvided = true | |||
| 41 | + | repository := d.snapshotRepository | |||
| 42 | + | if repository == nil { | |||
| 43 | + | repository = testLifecycleRepository{} | |||
| 44 | + | } | |||
| 45 | + | authority := recoveryusecase.NewCoordinator(catalogue, repository, nil, rand.Reader) | |||
| 46 | + | d.lifecycleRecovery = authority | |||
| 47 | + | d.checkpointRecovery = authority | |||
| 48 | + | } | |||
| 49 | + | } | |||
M internal/usecase/daemon/durable_recovery_test.go +126 -24
| @@ -16,6 +16,8 @@ import ( | |||||
| 16 | 16 | ||||
| 17 | "github.com/bnema/vev/internal/domain" | 17 | "github.com/bnema/vev/internal/domain" | ||
| 18 | "github.com/bnema/vev/internal/ports" | 18 | "github.com/bnema/vev/internal/ports" | ||
| 19 | + | portsmocks "github.com/bnema/vev/internal/ports/mocks" | |||
| 20 | + | "github.com/bnema/vev/internal/usecase/command" | |||
| 19 | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | 21 | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | ||
| 20 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | 22 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 21 | ) | 23 | ) | ||
| @@ -35,21 +37,28 @@ func newDurableRecoveryCatalogue(records []domain.CatalogueRecord) *durableRecov | |||||
| 35 | return catalogue | 37 | return catalogue | ||
| 36 | } | 38 | } | ||
| 37 | 39 | ||||
| 38 | - | func (c *durableRecoveryCatalogue) Records() []domain.CatalogueRecord { | 40 | + | func (c *durableRecoveryCatalogue) Records() ([]domain.CatalogueRecord, error) { |
| 39 | c.mu.Lock() | 41 | c.mu.Lock() | ||
| 40 | defer c.mu.Unlock() | 42 | defer c.mu.Unlock() | ||
| 41 | records := make([]domain.CatalogueRecord, 0, len(c.records)) | 43 | records := make([]domain.CatalogueRecord, 0, len(c.records)) | ||
| 42 | for _, record := range c.records { | 44 | for _, record := range c.records { | ||
| 43 | records = append(records, record) | 45 | records = append(records, record) | ||
| 44 | } | 46 | } | ||
| 45 | - | return records | 47 | + | return records, nil |
| 46 | } | 48 | } | ||
| 47 | 49 | ||||
| 48 | - | func (c *durableRecoveryCatalogue) Record(name string) (domain.CatalogueRecord, bool) { | 50 | + | func (c *durableRecoveryCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) { |
| 49 | c.mu.Lock() | 51 | c.mu.Lock() | ||
| 50 | defer c.mu.Unlock() | 52 | defer c.mu.Unlock() | ||
| 51 | record, ok := c.records[name] | 53 | record, ok := c.records[name] | ||
| 52 | - | return record, ok | 54 | + | return record, ok, nil |
| 55 | + | } | |||
| 56 | + | ||||
| 57 | + | func mustDurableRecords(t *testing.T, c *durableRecoveryCatalogue) []domain.CatalogueRecord { | |||
| 58 | + | t.Helper() | |||
| 59 | + | records, err := c.Records() | |||
| 60 | + | require.NoError(t, err) | |||
| 61 | + | return records | |||
| 53 | } | 62 | } | ||
| 54 | 63 | ||||
| 55 | func (c *durableRecoveryCatalogue) Replace(name string, record domain.CatalogueRecord) error { | 64 | func (c *durableRecoveryCatalogue) Replace(name string, record domain.CatalogueRecord) error { | ||
| @@ -111,6 +120,9 @@ type durableRecoveryRepository struct { | |||||
| 111 | func (r *durableRecoveryRepository) LoadCheckpoint(_ context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) { | 120 | func (r *durableRecoveryRepository) LoadCheckpoint(_ context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) { | ||
| 112 | r.mu.Lock() | 121 | r.mu.Lock() | ||
| 113 | defer r.mu.Unlock() | 122 | defer r.mu.Unlock() | ||
| 123 | + | if r.loads == nil { | |||
| 124 | + | r.loads = make(map[string]int) | |||
| 125 | + | } | |||
| 114 | r.loads[name]++ | 126 | r.loads[name]++ | ||
| 115 | if err := r.errors[name]; err != nil { | 127 | if err := r.errors[name]; err != nil { | ||
| 116 | return ports.SnapshotGeneration{}, err | 128 | return ports.SnapshotGeneration{}, err | ||
| @@ -129,6 +141,9 @@ func (r *durableRecoveryRepository) RepairHEAD(_ context.Context, id domain.Inca | |||||
| 129 | if r.repairErr != nil { | 141 | if r.repairErr != nil { | ||
| 130 | return r.repairErr | 142 | return r.repairErr | ||
| 131 | } | 143 | } | ||
| 144 | + | if r.repairs == nil { | |||
| 145 | + | r.repairs = make(map[string]domain.CheckpointRef) | |||
| 146 | + | } | |||
| 132 | r.repairs[id.String()] = ref | 147 | r.repairs[id.String()] = ref | ||
| 133 | return nil | 148 | return nil | ||
| 134 | } | 149 | } | ||
| @@ -169,7 +184,7 @@ func newDurableRecoveryDaemon(t testing.TB, records []domain.CatalogueRecord, re | |||||
| 169 | t.Helper() | 184 | t.Helper() | ||
| 170 | catalogue := newDurableRecoveryCatalogue(records) | 185 | catalogue := newDurableRecoveryCatalogue(records) | ||
| 171 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | 186 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | ||
| 172 | - | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, records), WithSnapshotRepository(repository, nil), WithCheckpointCoordinator(coordinator)) | 187 | + | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, records), WithSnapshotRepository(repository), WithCheckpointCoordinator(coordinator)) |
| 173 | d.serveCtx, d.serveCancel = context.WithCancel(context.Background()) | 188 | d.serveCtx, d.serveCancel = context.WithCancel(context.Background()) | ||
| 174 | t.Cleanup(d.serveCancel) | 189 | t.Cleanup(d.serveCancel) | ||
| 175 | return d, catalogue | 190 | return d, catalogue | ||
| @@ -188,7 +203,7 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) { | |||||
| 188 | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{prior}) | 203 | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{prior}) | ||
| 189 | repository := &durableRecoveryRepository{} | 204 | repository := &durableRecoveryRepository{} | ||
| 190 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | 205 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | ||
| 191 | - | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, []domain.CatalogueRecord{prior}), WithSnapshotRepository(repository, nil), WithCheckpointCoordinator(coordinator)) | 206 | + | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, []domain.CatalogueRecord{prior}), WithSnapshotRepository(repository), WithCheckpointCoordinator(coordinator)) |
| 192 | startSnapshotEncodeWorker(t, d) | 207 | startSnapshotEncodeWorker(t, d) | ||
| 193 | 208 | ||||
| 194 | sess := newSnapshotTestSession(t, prior.Name, false, prior.Cwd) | 209 | sess := newSnapshotTestSession(t, prior.Name, false, prior.Cwd) | ||
| @@ -204,7 +219,7 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) { | |||||
| 204 | 219 | ||||
| 205 | require.True(t, d.scheduleFinalSnapshot(sess)) | 220 | require.True(t, d.scheduleFinalSnapshot(sess)) | ||
| 206 | awaitSnapshotIdle(t, sess) | 221 | awaitSnapshotIdle(t, sess) | ||
| 207 | - | got, ok := catalogue.Record(prior.Name) | 222 | + | got, ok, _ := catalogue.Record(prior.Name) |
| 208 | require.True(t, ok) | 223 | require.True(t, ok) | ||
| 209 | require.Equal(t, prior, got, "a timed-out final checkpoint must not change authoritative bytes") | 224 | require.Equal(t, prior, got, "a timed-out final checkpoint must not change authoritative bytes") | ||
| 210 | require.Equal(t, priorRef, *sess.snapshotPublishedCheckpoint) | 225 | require.Equal(t, priorRef, *sess.snapshotPublishedCheckpoint) | ||
| @@ -214,7 +229,7 @@ func TestCatalogueRestoreIndependent(t *testing.T) { | |||||
| 214 | emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{}) | 229 | emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{}) | ||
| 215 | emptyDone := make(chan struct{}) | 230 | emptyDone := make(chan struct{}) | ||
| 216 | go func() { | 231 | go func() { | ||
| 217 | - | emptyDaemon.restoreCatalogue(context.Background(), emptyCatalogue.Records()) | 232 | + | emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue)) |
| 218 | close(emptyDone) | 233 | close(emptyDone) | ||
| 219 | }() | 234 | }() | ||
| 220 | select { | 235 | select { | ||
| @@ -233,7 +248,7 @@ func TestCatalogueRestoreIndependent(t *testing.T) { | |||||
| 233 | } | 248 | } | ||
| 234 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository) | 249 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository) | ||
| 235 | 250 | ||||
| 236 | - | d.restoreCatalogue(context.Background(), catalogue.Records()) | 251 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) |
| 237 | 252 | ||||
| 238 | require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state) | 253 | require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state) | ||
| 239 | require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state) | 254 | require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state) | ||
| @@ -244,7 +259,7 @@ func TestCatalogueRestoreIndependent(t *testing.T) { | |||||
| 244 | t.Fatalf("restore completion for %q was not closed", name) | 259 | t.Fatalf("restore completion for %q was not closed", name) | ||
| 245 | } | 260 | } | ||
| 246 | } | 261 | } | ||
| 247 | - | degraded, ok := catalogue.Record(broken.Name) | 262 | + | degraded, ok, _ := catalogue.Record(broken.Name) |
| 248 | require.True(t, ok) | 263 | require.True(t, ok) | ||
| 249 | require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState) | 264 | require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState) | ||
| 250 | require.Equal(t, 1, repository.loads[healthy.Name]) | 265 | require.Equal(t, 1, repository.loads[healthy.Name]) | ||
| @@ -265,8 +280,8 @@ func TestCatalogueRestoreIndependent(t *testing.T) { | |||||
| 265 | repairs: make(map[string]domain.CheckpointRef), | 280 | repairs: make(map[string]domain.CheckpointRef), | ||
| 266 | } | 281 | } | ||
| 267 | fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository) | 282 | fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository) | ||
| 268 | - | fallbackDaemon.restoreCatalogue(context.Background(), fallbackCatalogue.Records()) | 283 | + | fallbackDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, fallbackCatalogue)) |
| 269 | - | promoted, ok := fallbackCatalogue.Record(fallback.Name) | 284 | + | promoted, ok, _ := fallbackCatalogue.Record(fallback.Name) |
| 270 | require.True(t, ok) | 285 | require.True(t, ok) | ||
| 271 | require.Equal(t, fallbackRef, *promoted.Committed) | 286 | require.Equal(t, fallbackRef, *promoted.Committed) | ||
| 272 | require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state) | 287 | require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state) | ||
| @@ -291,9 +306,9 @@ func TestCatalogueRestoreKeepsCommittedFallbackHealthyWhenHEADRepairFails(t *tes | |||||
| 291 | } | 306 | } | ||
| 292 | 307 | ||||
| 293 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | 308 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | ||
| 294 | - | d.restoreCatalogue(context.Background(), catalogue.Records()) | 309 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) |
| 295 | 310 | ||||
| 296 | - | committed, ok := catalogue.Record(record.Name) | 311 | + | committed, ok, _ := catalogue.Record(record.Name) |
| 297 | require.True(t, ok) | 312 | require.True(t, ok) | ||
| 298 | require.Equal(t, domain.RecoveryHealthy, committed.RecoveryState) | 313 | require.Equal(t, domain.RecoveryHealthy, committed.RecoveryState) | ||
| 299 | require.Equal(t, fallbackRef, *committed.Committed) | 314 | require.Equal(t, fallbackRef, *committed.Committed) | ||
| @@ -302,7 +317,7 @@ func TestCatalogueRestoreKeepsCommittedFallbackHealthyWhenHEADRepairFails(t *tes | |||||
| 302 | 317 | ||||
| 303 | repository.repairErr = nil | 318 | repository.repairErr = nil | ||
| 304 | next, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{committed}, repository) | 319 | next, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{committed}, repository) | ||
| 305 | - | next.restoreCatalogue(context.Background(), catalogue.Records()) | 320 | + | next.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) |
| 306 | require.Equal(t, runtimeHealthy, next.stopped[record.Name].state) | 321 | require.Equal(t, runtimeHealthy, next.stopped[record.Name].state) | ||
| 307 | require.Equal(t, 2, repository.repairCalls, "the next startup retries the pending HEAD repair") | 322 | require.Equal(t, 2, repository.repairCalls, "the next startup retries the pending HEAD repair") | ||
| 308 | require.Equal(t, fallbackRef, repository.repairs[record.IncarnationID.String()]) | 323 | require.Equal(t, fallbackRef, repository.repairs[record.IncarnationID.String()]) | ||
| @@ -318,7 +333,7 @@ func TestCatalogueRestoreAggregateOver4096(t *testing.T) { | |||||
| 318 | } | 333 | } | ||
| 319 | d, catalogue := newDurableRecoveryDaemon(t, records, repository) | 334 | d, catalogue := newDurableRecoveryDaemon(t, records, repository) | ||
| 320 | 335 | ||||
| 321 | - | d.restoreCatalogue(context.Background(), catalogue.Records()) | 336 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) |
| 322 | 337 | ||||
| 323 | require.Len(t, d.stopped, count) | 338 | require.Len(t, d.stopped, count) | ||
| 324 | for _, record := range records { | 339 | for _, record := range records { | ||
| @@ -334,7 +349,7 @@ func TestCatalogueRestoreSingleSessionOver4096(t *testing.T) { | |||||
| 334 | 349 | ||||
| 335 | // The repository may contain any number of stale entries; restoration has no | 350 | // The repository may contain any number of stale entries; restoration has no | ||
| 336 | // discovery API and therefore performs exactly one direct catalogue lookup. | 351 | // discovery API and therefore performs exactly one direct catalogue lookup. | ||
| 337 | - | d.restoreCatalogue(context.Background(), catalogue.Records()) | 352 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) |
| 338 | 353 | ||||
| 339 | require.Equal(t, runtimeHealthy, d.stopped[record.Name].state) | 354 | require.Equal(t, runtimeHealthy, d.stopped[record.Name].state) | ||
| 340 | require.Equal(t, 1, repository.loads[record.Name]) | 355 | require.Equal(t, 1, repository.loads[record.Name]) | ||
| @@ -347,10 +362,10 @@ func TestCatalogueRestoreIncarnationMismatch(t *testing.T) { | |||||
| 347 | repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: generation}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | 362 | repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: generation}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | ||
| 348 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | 363 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | ||
| 349 | 364 | ||||
| 350 | - | d.restoreCatalogue(context.Background(), catalogue.Records()) | 365 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) |
| 351 | 366 | ||||
| 352 | require.Equal(t, runtimeDegraded, d.stopped[record.Name].state) | 367 | require.Equal(t, runtimeDegraded, d.stopped[record.Name].state) | ||
| 353 | - | got, _ := catalogue.Record(record.Name) | 368 | + | got, _, _ := catalogue.Record(record.Name) |
| 354 | require.Equal(t, domain.RecoveryDegraded, got.RecoveryState) | 369 | require.Equal(t, domain.RecoveryDegraded, got.RecoveryState) | ||
| 355 | } | 370 | } | ||
| 356 | 371 | ||||
| @@ -391,6 +406,59 @@ func (s explicitRecoveryStub) Discard(context.Context, string, string) (domain.C | |||||
| 391 | return s.record, nil | 406 | return s.record, nil | ||
| 392 | } | 407 | } | ||
| 393 | 408 | ||||
| 409 | + | type contextRecoveryStub struct { | |||
| 410 | + | explicitRecoveryStub | |||
| 411 | + | wantKey any | |||
| 412 | + | want any | |||
| 413 | + | seen bool | |||
| 414 | + | err error | |||
| 415 | + | } | |||
| 416 | + | ||||
| 417 | + | func (s *contextRecoveryStub) Retry(ctx context.Context, _ string) error { | |||
| 418 | + | s.seen = ctx.Value(s.wantKey) == s.want | |||
| 419 | + | return s.err | |||
| 420 | + | } | |||
| 421 | + | ||||
| 422 | + | func TestSessionRecoveryUsesIncomingCommandContext(t *testing.T) { | |||
| 423 | + | type contextKey struct{} | |||
| 424 | + | key := contextKey{} | |||
| 425 | + | ctx := context.WithValue(t.Context(), key, "request") | |||
| 426 | + | wantErr := errors.New("stop after context capture") | |||
| 427 | + | stub := &contextRecoveryStub{wantKey: key, want: "request", err: wantErr} | |||
| 428 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 429 | + | d.degradedRecovery = stub | |||
| 430 | + | ||||
| 431 | + | _, err := (controlExec{ctx: ctx, d: d, recoveryName: "work"}).SessionRecovery("retry", "") | |||
| 432 | + | require.ErrorIs(t, err, wantErr) | |||
| 433 | + | require.True(t, stub.seen) | |||
| 434 | + | } | |||
| 435 | + | ||||
| 436 | + | func TestExplicitRecoveryRejectsConcurrentRestore(t *testing.T) { | |||
| 437 | + | record := durableRecoveryRecord(0) | |||
| 438 | + | record.RecoveryState = domain.RecoveryDegraded | |||
| 439 | + | record.DegradedReason = "checkpoint validation failed" | |||
| 440 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | |||
| 441 | + | d.mu.Lock() | |||
| 442 | + | entry := d.stopped[record.Name] | |||
| 443 | + | entry.state = runtimeRestoring | |||
| 444 | + | originalDone := entry.restoreDone | |||
| 445 | + | d.stopped[record.Name] = entry | |||
| 446 | + | d.mu.Unlock() | |||
| 447 | + | ||||
| 448 | + | err := (controlExec{d: d, recoveryName: record.Name}).restoreExplicitRecovery(t.Context()) | |||
| 449 | + | require.ErrorContains(t, err, "current state") | |||
| 450 | + | d.mu.Lock() | |||
| 451 | + | require.Equal(t, originalDone, d.stopped[record.Name].restoreDone) | |||
| 452 | + | d.mu.Unlock() | |||
| 453 | + | } | |||
| 454 | + | ||||
| 455 | + | func TestSessionRecoveryExportRequiresAbsolutePath(t *testing.T) { | |||
| 456 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 457 | + | d.degradedRecovery = explicitRecoveryStub{} | |||
| 458 | + | _, err := (controlExec{ctx: t.Context(), d: d, recoveryName: "work"}).SessionRecovery("export", "relative.snapshot") | |||
| 459 | + | require.ErrorIs(t, err, command.ErrInvalidArguments) | |||
| 460 | + | } | |||
| 461 | + | ||||
| 394 | func TestSessionRecoveryCommand(t *testing.T) { | 462 | func TestSessionRecoveryCommand(t *testing.T) { | ||
| 395 | degraded := durableRecoveryRecord(2) | 463 | degraded := durableRecoveryRecord(2) | ||
| 396 | degraded.RecoveryState = domain.RecoveryDegraded | 464 | degraded.RecoveryState = domain.RecoveryDegraded | ||
| @@ -428,6 +496,20 @@ func TestListShowsDegraded(t *testing.T) { | |||||
| 428 | }, listed.Sessions) | 496 | }, listed.Sessions) | ||
| 429 | } | 497 | } | ||
| 430 | 498 | ||||
| 499 | + | func TestListPurgingDominatesRestoringState(t *testing.T) { | |||
| 500 | + | record := durableRecoveryRecord(0) | |||
| 501 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | |||
| 502 | + | d.mu.Lock() | |||
| 503 | + | entry := d.stopped[record.Name] | |||
| 504 | + | entry.purging = true | |||
| 505 | + | entry.state = runtimeRestoring | |||
| 506 | + | d.stopped[record.Name] = entry | |||
| 507 | + | d.mu.Unlock() | |||
| 508 | + | ||||
| 509 | + | listed := listSessions(t, d) | |||
| 510 | + | require.Equal(t, []ports.SessionInfo{{Name: record.Name, State: ports.SessionDegraded}}, listed.Sessions) | |||
| 511 | + | } | |||
| 512 | + | ||||
| 431 | func TestAttachWaitsForRestore(t *testing.T) { | 513 | func TestAttachWaitsForRestore(t *testing.T) { | ||
| 432 | record := durableRecoveryRecord(0) | 514 | record := durableRecoveryRecord(0) | ||
| 433 | factory := &recoveryCountingPTYFactory{} | 515 | factory := &recoveryCountingPTYFactory{} | ||
| @@ -449,7 +531,9 @@ func TestAttachWaitsForRestore(t *testing.T) { | |||||
| 449 | require.Zero(t, factory.calls.Load()) | 531 | require.Zero(t, factory.calls.Load()) | ||
| 450 | 532 | ||||
| 451 | d.setStoppedRecovery(record, runtimeDegraded) | 533 | d.setStoppedRecovery(record, runtimeDegraded) | ||
| 452 | - | closeRuntimeRestoreDone(d.recordRestoreDone(record.Name)) | 534 | + | d.mu.Lock() |
| 535 | + | closeRuntimeRestoreDoneLocked(d.stopped[record.Name].restoreDone) | |||
| 536 | + | d.mu.Unlock() | |||
| 453 | var protocolError *protoErr | 537 | var protocolError *protoErr | ||
| 454 | require.ErrorAs(t, <-result, &protocolError) | 538 | require.ErrorAs(t, <-result, &protocolError) | ||
| 455 | require.Equal(t, ports.ErrSessionDegraded, protocolError.code) | 539 | require.Equal(t, ports.ErrSessionDegraded, protocolError.code) | ||
| @@ -463,7 +547,7 @@ func TestRestoreCancellationTransitionsBeforeAttachCompletion(t *testing.T) { | |||||
| 463 | restoreCtx, cancelRestore := context.WithCancel(context.Background()) | 547 | restoreCtx, cancelRestore := context.WithCancel(context.Background()) | ||
| 464 | restored := make(chan struct{}) | 548 | restored := make(chan struct{}) | ||
| 465 | go func() { | 549 | go func() { | ||
| 466 | - | d.restoreCatalogue(restoreCtx, catalogue.Records()) | 550 | + | d.restoreCatalogue(restoreCtx, mustDurableRecords(t, catalogue)) |
| 467 | close(restored) | 551 | close(restored) | ||
| 468 | }() | 552 | }() | ||
| 469 | <-repository.started | 553 | <-repository.started | ||
| @@ -488,14 +572,16 @@ func TestRestoreCancellationTransitionsBeforeAttachCompletion(t *testing.T) { | |||||
| 488 | default: | 572 | default: | ||
| 489 | t.Fatal("restore completion was not signaled") | 573 | t.Fatal("restore completion was not signaled") | ||
| 490 | } | 574 | } | ||
| 491 | - | committed, _ := catalogue.Record(record.Name) | 575 | + | committed, _, _ := catalogue.Record(record.Name) |
| 492 | require.Equal(t, entry.record, committed) | 576 | require.Equal(t, entry.record, committed) | ||
| 493 | } | 577 | } | ||
| 494 | 578 | ||||
| 495 | func TestWaitForTargetRestoreRejectsClosedRestoringChannel(t *testing.T) { | 579 | func TestWaitForTargetRestoreRejectsClosedRestoringChannel(t *testing.T) { | ||
| 496 | record := durableRecoveryRecord(0) | 580 | record := durableRecoveryRecord(0) | ||
| 497 | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | 581 | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | ||
| 498 | - | closeRuntimeRestoreDone(d.stopped[record.Name].restoreDone) | 582 | + | d.mu.Lock() |
| 583 | + | closeRuntimeRestoreDoneLocked(d.stopped[record.Name].restoreDone) | |||
| 584 | + | d.mu.Unlock() | |||
| 499 | 585 | ||||
| 500 | result := make(chan error, 1) | 586 | result := make(chan error, 1) | ||
| 501 | go func() { result <- d.waitForTargetRestore(record.Name) }() | 587 | go func() { result <- d.waitForTargetRestore(record.Name) }() | ||
| @@ -552,6 +638,22 @@ func TestAttachRejectsDegraded(t *testing.T) { | |||||
| 552 | require.Zero(t, factory.calls.Load()) | 638 | require.Zero(t, factory.calls.Load()) | ||
| 553 | } | 639 | } | ||
| 554 | 640 | ||||
| 641 | + | func TestRestoreUnavailableReleasesPerRecordWaiters(t *testing.T) { | |||
| 642 | + | record := durableRecoveryRecord(0) | |||
| 643 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, nil) | |||
| 644 | + | d.closeRestoreDone() | |||
| 645 | + | ||||
| 646 | + | d.mu.Lock() | |||
| 647 | + | entry := d.stopped[record.Name] | |||
| 648 | + | d.mu.Unlock() | |||
| 649 | + | require.Equal(t, runtimeDegraded, entry.state) | |||
| 650 | + | select { | |||
| 651 | + | case <-entry.restoreDone: | |||
| 652 | + | default: | |||
| 653 | + | t.Fatal("per-record restore waiter remained blocked when repository was unavailable") | |||
| 654 | + | } | |||
| 655 | + | } | |||
| 656 | + | ||||
| 555 | func TestCatalogueRestoreDegradedVisible(t *testing.T) { | 657 | func TestCatalogueRestoreDegradedVisible(t *testing.T) { | ||
| 556 | record := durableRecoveryRecord(0) | 658 | record := durableRecoveryRecord(0) | ||
| 557 | record.RecoveryState = domain.RecoveryDegraded | 659 | record.RecoveryState = domain.RecoveryDegraded | ||
| @@ -559,7 +661,7 @@ func TestCatalogueRestoreDegradedVisible(t *testing.T) { | |||||
| 559 | repository := &durableRecoveryRepository{generations: make(map[string]ports.SnapshotGeneration), errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | 661 | repository := &durableRecoveryRepository{generations: make(map[string]ports.SnapshotGeneration), errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | ||
| 560 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | 662 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | ||
| 561 | 663 | ||||
| 562 | - | d.restoreCatalogue(context.Background(), catalogue.Records()) | 664 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) |
| 563 | 665 | ||||
| 564 | require.Equal(t, runtimeDegraded, d.stopped[record.Name].state) | 666 | require.Equal(t, runtimeDegraded, d.stopped[record.Name].state) | ||
| 565 | require.Zero(t, repository.loads[record.Name]) | 667 | require.Zero(t, repository.loads[record.Name]) | ||
D internal/usecase/daemon/legacy_snapshot_test.go +0 -8
| @@ -1,8 +0,0 @@ | |||||
| 1 | - | package daemon | |||
| 2 | - | ||||
| 3 | - | import "context" | |||
| 4 | - | ||||
| 5 | - | // importLegacySnapshots is retained only for tests covering the retired | |||
| 6 | - | // pre-migration bridge. Normal runtime legacy decoding lives exclusively in | |||
| 7 | - | // the snapshot migration adapter. | |||
| 8 | - | func (d *Daemon) importLegacySnapshots(context.Context) {} | |||
M internal/usecase/daemon/maintenance.go +33 -8
| @@ -18,6 +18,9 @@ type maintenanceReconciler interface { | |||||
| 18 | Step(context.Context, ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error) | 18 | Step(context.Context, ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error) | ||
| 19 | } | 19 | } | ||
| 20 | 20 | ||||
| 21 | + | // maintenanceDependencies is exclusively owned by runDurableMaintenance's | |||
| 22 | + | // single goroutine after startup. Shutdown joins that goroutine before these | |||
| 23 | + | // cursor and repair sets can be observed or reused. | |||
| 21 | type maintenanceDependencies struct { | 24 | type maintenanceDependencies struct { | ||
| 22 | catalogue ports.Catalogue | 25 | catalogue ports.Catalogue | ||
| 23 | repository ports.SnapshotRepository | 26 | repository ports.SnapshotRepository | ||
| @@ -88,7 +91,13 @@ func (d *Daemon) runDurableMaintenance(ctx context.Context) { | |||||
| 88 | case <-d.restoreDone: | 91 | case <-d.restoreDone: | ||
| 89 | } | 92 | } | ||
| 90 | for { | 93 | for { | ||
| 91 | - | d.runDurableMaintenanceTick(ctx) | 94 | + | incomplete := d.runDurableMaintenanceTick(ctx) |
| 95 | + | if ctx.Err() != nil { | |||
| 96 | + | return | |||
| 97 | + | } | |||
| 98 | + | if incomplete { | |||
| 99 | + | continue | |||
| 100 | + | } | |||
| 92 | timer := d.clock.NewTimer(maintenanceInterval) | 101 | timer := d.clock.NewTimer(maintenanceInterval) | ||
| 93 | select { | 102 | select { | ||
| 94 | case <-ctx.Done(): | 103 | case <-ctx.Done(): | ||
| @@ -99,10 +108,12 @@ func (d *Daemon) runDurableMaintenance(ctx context.Context) { | |||||
| 99 | } | 108 | } | ||
| 100 | } | 109 | } | ||
| 101 | 110 | ||||
| 102 | - | func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) { | 111 | + | // runDurableMaintenanceTick reports whether bounded work remains and should be |
| 112 | + | // continued immediately rather than delayed until the periodic interval. | |||
| 113 | + | func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool { | |||
| 103 | deps := &d.maintenance | 114 | deps := &d.maintenance | ||
| 104 | if deps.catalogue == nil || deps.repository == nil || deps.reconciler == nil { | 115 | if deps.catalogue == nil || deps.repository == nil || deps.reconciler == nil { | ||
| 105 | - | return | 116 | + | return false |
| 106 | } | 117 | } | ||
| 107 | 118 | ||||
| 108 | // Reconcile before collection. Until a complete top-level scan reaches EOF, | 119 | // Reconcile before collection. Until a complete top-level scan reaches EOF, | ||
| @@ -111,11 +122,15 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) { | |||||
| 111 | next, decisions, err := deps.reconciler.Step(ctx, deps.cursor) | 122 | next, decisions, err := deps.reconciler.Step(ctx, deps.cursor) | ||
| 112 | if err != nil { | 123 | if err != nil { | ||
| 113 | d.log.Warn("durable_reconciliation_tick", "action", "scan", "cursor", deps.cursor.DirectoryCookie, "done", false, "err", err) | 124 | d.log.Warn("durable_reconciliation_tick", "action", "scan", "cursor", deps.cursor.DirectoryCookie, "done", false, "err", err) | ||
| 114 | - | return | 125 | + | return false |
| 115 | } | 126 | } | ||
| 116 | deps.cursor = next | 127 | deps.cursor = next | ||
| 117 | for _, decision := range decisions { | 128 | for _, decision := range decisions { | ||
| 118 | - | record, ok := deps.catalogue.Record(decision.Name) | 129 | + | record, ok, err := deps.catalogue.Record(decision.Name) |
| 130 | + | if err != nil { | |||
| 131 | + | d.log.Warn("durable_reconciliation_tick", "action", "catalogue-read", "done", false, "err", err) | |||
| 132 | + | return false | |||
| 133 | + | } | |||
| 119 | if ok && !decision.RetentionResolved { | 134 | if ok && !decision.RetentionResolved { | ||
| 120 | deps.repairUncertain[record.IncarnationID] = struct{}{} | 135 | deps.repairUncertain[record.IncarnationID] = struct{}{} | ||
| 121 | } | 136 | } | ||
| @@ -136,18 +151,27 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) { | |||||
| 136 | } | 151 | } | ||
| 137 | d.log.Info("durable_reconciliation_tick", "action", "scan", "cursor", next.DirectoryCookie, "done", next.DirectoryCookie == 0, "err", nil) | 152 | d.log.Info("durable_reconciliation_tick", "action", "scan", "cursor", next.DirectoryCookie, "done", next.DirectoryCookie == 0, "err", nil) | ||
| 138 | if next.DirectoryCookie != 0 { | 153 | if next.DirectoryCookie != 0 { | ||
| 139 | - | return | 154 | + | return true |
| 140 | } | 155 | } | ||
| 141 | 156 | ||||
| 142 | - | for _, record := range deps.catalogue.Records() { | 157 | + | records, err := deps.catalogue.Records() |
| 158 | + | if err != nil { | |||
| 159 | + | d.log.Warn("durable_maintenance_tick", "action", "catalogue-read", "done", false, "err", err) | |||
| 160 | + | return false | |||
| 161 | + | } | |||
| 162 | + | incomplete := false | |||
| 163 | + | for _, record := range records { | |||
| 143 | if err := ctx.Err(); err != nil { | 164 | if err := ctx.Err(); err != nil { | ||
| 144 | - | return | 165 | + | return false |
| 145 | } | 166 | } | ||
| 146 | _, unresolved := deps.unresolved[record.IncarnationID] | 167 | _, unresolved := deps.unresolved[record.IncarnationID] | ||
| 147 | _, uncertainRepair := deps.repairUncertain[record.IncarnationID] | 168 | _, uncertainRepair := deps.repairUncertain[record.IncarnationID] | ||
| 148 | restoring := d.incarnationRestoring(record.IncarnationID) | 169 | restoring := d.incarnationRestoring(record.IncarnationID) | ||
| 149 | plan := retentionPlan(record, unresolved || uncertainRepair, restoring) | 170 | plan := retentionPlan(record, unresolved || uncertainRepair, restoring) | ||
| 150 | done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes}) | 171 | done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes}) | ||
| 172 | + | if err == nil && !done { | |||
| 173 | + | incomplete = true | |||
| 174 | + | } | |||
| 151 | d.log.Info("durable_maintenance_session", | 175 | d.log.Info("durable_maintenance_session", | ||
| 152 | "incarnation", record.IncarnationID.String(), | 176 | "incarnation", record.IncarnationID.String(), | ||
| 153 | "action", "retention", | 177 | "action", "retention", | ||
| @@ -159,6 +183,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) { | |||||
| 159 | ) | 183 | ) | ||
| 160 | } | 184 | } | ||
| 161 | clear(deps.repairUncertain) | 185 | clear(deps.repairUncertain) | ||
| 186 | + | return incomplete | |||
| 162 | } | 187 | } | ||
| 163 | 188 | ||||
| 164 | func (d *Daemon) incarnationRestoring(id domain.IncarnationID) bool { | 189 | func (d *Daemon) incarnationRestoring(id domain.IncarnationID) bool { | ||
M internal/usecase/daemon/maintenance_test.go +4 -4
| @@ -28,7 +28,7 @@ func (r *cursorMaintenanceReconciler) Step(_ context.Context, cursor ports.Recon | |||||
| 28 | func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) { | 28 | func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) { | ||
| 29 | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh} | 29 | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh} | ||
| 30 | catalogue := portsmocks.NewMockCatalogue(t) | 30 | catalogue := portsmocks.NewMockCatalogue(t) | ||
| 31 | - | catalogue.EXPECT().Records().Return([]domain.CatalogueRecord{record}).Once() | 31 | + | catalogue.EXPECT().Records().Return([]domain.CatalogueRecord{record}, nil).Once() |
| 32 | repository := portsmocks.NewMockSnapshotRepository(t) | 32 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 33 | repository.EXPECT().MaintainSession(mock.Anything, mock.MatchedBy(func(plan ports.RetentionPlan) bool { | 33 | repository.EXPECT().MaintainSession(mock.Anything, mock.MatchedBy(func(plan ports.RetentionPlan) bool { | ||
| 34 | return plan.IncarnationID == record.IncarnationID && !plan.PinAll && len(plan.Keep) == 0 | 34 | return plan.IncarnationID == record.IncarnationID && !plan.PinAll && len(plan.Keep) == 0 | ||
| @@ -37,9 +37,9 @@ func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) { | |||||
| 37 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 37 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 38 | WithDurableMaintenance(catalogue, repository, reconciler, nil)(d) | 38 | WithDurableMaintenance(catalogue, repository, reconciler, nil)(d) | ||
| 39 | 39 | ||||
| 40 | - | d.runDurableMaintenanceTick(context.Background()) | 40 | + | require.True(t, d.runDurableMaintenanceTick(context.Background()), "partial reconciliation must continue immediately") |
| 41 | - | d.runDurableMaintenanceTick(context.Background()) | 41 | + | require.False(t, d.runDurableMaintenanceTick(context.Background())) |
| 42 | - | d.runDurableMaintenanceTick(context.Background()) | 42 | + | require.True(t, d.runDurableMaintenanceTick(context.Background()), "the next partial pass must also continue immediately") |
| 43 | if len(reconciler.seen) != 3 || reconciler.seen[0].DirectoryCookie != 0 || reconciler.seen[1].DirectoryCookie != 7 || reconciler.seen[2].DirectoryCookie != 0 { | 43 | if len(reconciler.seen) != 3 || reconciler.seen[0].DirectoryCookie != 0 || reconciler.seen[1].DirectoryCookie != 7 || reconciler.seen[2].DirectoryCookie != 0 { | ||
| 44 | t.Fatalf("reconciliation cursors = %v, want [0 7 0]", reconciler.seen) | 44 | t.Fatalf("reconciliation cursors = %v, want [0 7 0]", reconciler.seen) | ||
| 45 | } | 45 | } | ||
A internal/usecase/daemon/metadata_order_test.go +285 -0
| @@ -0,0 +1,285 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "errors" | |||
| 5 | + | "sync" | |||
| 6 | + | "testing" | |||
| 7 | + | "time" | |||
| 8 | + | ||||
| 9 | + | "github.com/stretchr/testify/require" | |||
| 10 | + | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 12 | + | "github.com/bnema/vev/internal/ports" | |||
| 13 | + | ) | |||
| 14 | + | ||||
| 15 | + | type blockingMetadataCatalogue struct { | |||
| 16 | + | ports.Catalogue | |||
| 17 | + | ||||
| 18 | + | mu sync.Mutex | |||
| 19 | + | calls int | |||
| 20 | + | firstStarted chan struct{} | |||
| 21 | + | releaseFirst chan struct{} | |||
| 22 | + | firstErr error | |||
| 23 | + | secondErr error | |||
| 24 | + | } | |||
| 25 | + | ||||
| 26 | + | func (c *blockingMetadataCatalogue) UpdateMetadata(update domain.CatalogueMetadataUpdate) error { | |||
| 27 | + | c.mu.Lock() | |||
| 28 | + | c.calls++ | |||
| 29 | + | call := c.calls | |||
| 30 | + | c.mu.Unlock() | |||
| 31 | + | if call == 1 { | |||
| 32 | + | close(c.firstStarted) | |||
| 33 | + | <-c.releaseFirst | |||
| 34 | + | if c.firstErr != nil { | |||
| 35 | + | return c.firstErr | |||
| 36 | + | } | |||
| 37 | + | } | |||
| 38 | + | if call == 2 && c.secondErr != nil { | |||
| 39 | + | return c.secondErr | |||
| 40 | + | } | |||
| 41 | + | return c.Catalogue.UpdateMetadata(update) | |||
| 42 | + | } | |||
| 43 | + | ||||
| 44 | + | func newBlockingCatalogue(sess *session, firstErr error) (*durableRecoveryCatalogue, *blockingMetadataCatalogue) { | |||
| 45 | + | record := sess.persistRecordLocked(sess.createdAt) | |||
| 46 | + | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}) | |||
| 47 | + | return catalogue, &blockingMetadataCatalogue{ | |||
| 48 | + | Catalogue: catalogue, | |||
| 49 | + | firstStarted: make(chan struct{}), | |||
| 50 | + | releaseFirst: make(chan struct{}), | |||
| 51 | + | firstErr: firstErr, | |||
| 52 | + | } | |||
| 53 | + | } | |||
| 54 | + | ||||
| 55 | + | func TestMetadataWritesPreserveLaterMutationWhenFirstWriteIsBlocked(t *testing.T) { | |||
| 56 | + | firstFailure := errors.New("first metadata write failed") | |||
| 57 | + | for _, tt := range []struct { | |||
| 58 | + | name string | |||
| 59 | + | firstErr error | |||
| 60 | + | }{ | |||
| 61 | + | {name: "delayed success"}, | |||
| 62 | + | {name: "delayed failure does not roll back newer state", firstErr: firstFailure}, | |||
| 63 | + | } { | |||
| 64 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 65 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 66 | + | catalogue, blocking := newBlockingCatalogue(sess, tt.firstErr) | |||
| 67 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 68 | + | d.catalogue = blocking | |||
| 69 | + | d.sessions[sess.id] = sess | |||
| 70 | + | ||||
| 71 | + | firstDone := make(chan error, 1) | |||
| 72 | + | go func() { firstDone <- d.renameTab(sess, sess.tabs[0], "first") }() | |||
| 73 | + | select { | |||
| 74 | + | case <-blocking.firstStarted: | |||
| 75 | + | case <-time.After(testWaitTimeout): | |||
| 76 | + | t.Fatal("first metadata write did not block") | |||
| 77 | + | } | |||
| 78 | + | ||||
| 79 | + | secondDone := make(chan error, 1) | |||
| 80 | + | go func() { secondDone <- d.renameTab(sess, sess.tabs[0], "second") }() | |||
| 81 | + | require.Eventually(t, func() bool { | |||
| 82 | + | sess.mu.Lock() | |||
| 83 | + | defer sess.mu.Unlock() | |||
| 84 | + | return sess.tabs[0].name == "second" | |||
| 85 | + | }, testWaitTimeout, time.Millisecond, "later mutation did not proceed while storage was blocked") | |||
| 86 | + | ||||
| 87 | + | close(blocking.releaseFirst) | |||
| 88 | + | firstResult := <-firstDone | |||
| 89 | + | if tt.firstErr == nil { | |||
| 90 | + | require.NoError(t, firstResult) | |||
| 91 | + | } else { | |||
| 92 | + | require.ErrorIs(t, firstResult, tt.firstErr, "an in-memory revision cannot subsume a failed durable write") | |||
| 93 | + | } | |||
| 94 | + | require.NoError(t, <-secondDone) | |||
| 95 | + | ||||
| 96 | + | persisted, ok, err := catalogue.Record("work") | |||
| 97 | + | require.NoError(t, err) | |||
| 98 | + | require.True(t, ok) | |||
| 99 | + | require.Equal(t, []string{"second"}, persisted.TabNames) | |||
| 100 | + | require.Equal(t, "second", sess.tabs[0].name) | |||
| 101 | + | }) | |||
| 102 | + | } | |||
| 103 | + | } | |||
| 104 | + | ||||
| 105 | + | func TestTwoConsecutiveMetadataFailuresAreBothReported(t *testing.T) { | |||
| 106 | + | firstFailure := errors.New("first metadata write failed") | |||
| 107 | + | secondFailure := errors.New("second metadata write failed") | |||
| 108 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 109 | + | catalogue, blocking := newBlockingCatalogue(sess, firstFailure) | |||
| 110 | + | blocking.secondErr = secondFailure | |||
| 111 | + | close(blocking.releaseFirst) | |||
| 112 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 113 | + | d.catalogue = blocking | |||
| 114 | + | d.sessions[sess.id] = sess | |||
| 115 | + | ||||
| 116 | + | require.ErrorIs(t, d.renameTab(sess, sess.tabs[0], "first"), firstFailure) | |||
| 117 | + | require.ErrorIs(t, d.renameTab(sess, sess.tabs[0], "second"), secondFailure) | |||
| 118 | + | ||||
| 119 | + | persisted, ok, err := catalogue.Record("work") | |||
| 120 | + | require.NoError(t, err) | |||
| 121 | + | require.True(t, ok) | |||
| 122 | + | require.Empty(t, persisted.TabNames) | |||
| 123 | + | sess.mu.Lock() | |||
| 124 | + | require.Empty(t, sess.tabs[0].name) | |||
| 125 | + | sess.mu.Unlock() | |||
| 126 | + | } | |||
| 127 | + | ||||
| 128 | + | func TestConsecutiveMetadataFailuresRestoreDurableState(t *testing.T) { | |||
| 129 | + | firstFailure := errors.New("first metadata write failed") | |||
| 130 | + | secondFailure := errors.New("second metadata write failed") | |||
| 131 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 132 | + | catalogue, blocking := newBlockingCatalogue(sess, firstFailure) | |||
| 133 | + | blocking.secondErr = secondFailure | |||
| 134 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 135 | + | d.catalogue = blocking | |||
| 136 | + | d.sessions[sess.id] = sess | |||
| 137 | + | ||||
| 138 | + | firstDone := make(chan error, 1) | |||
| 139 | + | go func() { firstDone <- d.renameTab(sess, sess.tabs[0], "first") }() | |||
| 140 | + | select { | |||
| 141 | + | case <-blocking.firstStarted: | |||
| 142 | + | case <-time.After(testWaitTimeout): | |||
| 143 | + | t.Fatal("first metadata write did not block") | |||
| 144 | + | } | |||
| 145 | + | secondDone := make(chan error, 1) | |||
| 146 | + | go func() { secondDone <- d.renameTab(sess, sess.tabs[0], "second") }() | |||
| 147 | + | require.Eventually(t, func() bool { | |||
| 148 | + | sess.mu.Lock() | |||
| 149 | + | defer sess.mu.Unlock() | |||
| 150 | + | return sess.tabs[0].name == "second" | |||
| 151 | + | }, testWaitTimeout, time.Millisecond) | |||
| 152 | + | ||||
| 153 | + | close(blocking.releaseFirst) | |||
| 154 | + | require.ErrorIs(t, <-firstDone, firstFailure) | |||
| 155 | + | require.ErrorIs(t, <-secondDone, secondFailure) | |||
| 156 | + | ||||
| 157 | + | persisted, ok, err := catalogue.Record("work") | |||
| 158 | + | require.NoError(t, err) | |||
| 159 | + | require.True(t, ok) | |||
| 160 | + | require.Empty(t, persisted.TabNames) | |||
| 161 | + | sess.mu.Lock() | |||
| 162 | + | require.Empty(t, sess.tabs[0].name) | |||
| 163 | + | sess.mu.Unlock() | |||
| 164 | + | } | |||
| 165 | + | ||||
| 166 | + | func TestNewerMetadataFailureDoesNotSkipOlderSnapshot(t *testing.T) { | |||
| 167 | + | failure := errors.New("newer metadata write failed") | |||
| 168 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 169 | + | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{sess.persistRecordLocked(sess.createdAt)}) | |||
| 170 | + | blocking := &blockingMetadataCatalogue{ | |||
| 171 | + | Catalogue: catalogue, | |||
| 172 | + | firstStarted: make(chan struct{}), | |||
| 173 | + | releaseFirst: make(chan struct{}), | |||
| 174 | + | firstErr: failure, | |||
| 175 | + | } | |||
| 176 | + | close(blocking.releaseFirst) | |||
| 177 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 178 | + | d.catalogue = blocking | |||
| 179 | + | d.sessions[sess.id] = sess | |||
| 180 | + | tb := sess.tabs[0] | |||
| 181 | + | ||||
| 182 | + | sess.mu.Lock() | |||
| 183 | + | tb.name = "first" | |||
| 184 | + | firstRecord, firstVersion := sess.nextPersistRecordLocked(sess.createdAt + 1) | |||
| 185 | + | tb.name = "second" | |||
| 186 | + | secondRecord, secondVersion := sess.nextPersistRecordLocked(sess.createdAt + 2) | |||
| 187 | + | sess.mu.Unlock() | |||
| 188 | + | ||||
| 189 | + | secondRollback := func() bool { | |||
| 190 | + | sess.mu.Lock() | |||
| 191 | + | defer sess.mu.Unlock() | |||
| 192 | + | if tb.name != "second" { | |||
| 193 | + | return false | |||
| 194 | + | } | |||
| 195 | + | tb.name = "first" | |||
| 196 | + | return true | |||
| 197 | + | } | |||
| 198 | + | firstRollback := func() bool { | |||
| 199 | + | sess.mu.Lock() | |||
| 200 | + | defer sess.mu.Unlock() | |||
| 201 | + | if tb.name != "first" { | |||
| 202 | + | return false | |||
| 203 | + | } | |||
| 204 | + | tb.name = "" | |||
| 205 | + | return true | |||
| 206 | + | } | |||
| 207 | + | ||||
| 208 | + | require.ErrorIs(t, d.persistSessionMetadata(sess, secondVersion, secondRecord.MetadataUpdate(), secondRollback), failure) | |||
| 209 | + | require.NoError(t, d.persistSessionMetadata(sess, firstVersion, firstRecord.MetadataUpdate(), firstRollback)) | |||
| 210 | + | ||||
| 211 | + | persisted, ok, err := catalogue.Record("work") | |||
| 212 | + | require.NoError(t, err) | |||
| 213 | + | require.True(t, ok) | |||
| 214 | + | require.Equal(t, []string{"first"}, persisted.TabNames) | |||
| 215 | + | sess.mu.Lock() | |||
| 216 | + | require.Equal(t, "first", tb.name) | |||
| 217 | + | sess.mu.Unlock() | |||
| 218 | + | } | |||
| 219 | + | ||||
| 220 | + | func TestLatestMetadataFailureRollsBackRename(t *testing.T) { | |||
| 221 | + | failure := errors.New("metadata write failed") | |||
| 222 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 223 | + | catalogue, blocking := newBlockingCatalogue(sess, failure) | |||
| 224 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 225 | + | d.catalogue = blocking | |||
| 226 | + | d.sessions[sess.id] = sess | |||
| 227 | + | ||||
| 228 | + | done := make(chan error, 1) | |||
| 229 | + | go func() { done <- d.renameTab(sess, sess.tabs[0], "failed") }() | |||
| 230 | + | select { | |||
| 231 | + | case <-blocking.firstStarted: | |||
| 232 | + | case <-time.After(testWaitTimeout): | |||
| 233 | + | t.Fatal("metadata write did not block") | |||
| 234 | + | } | |||
| 235 | + | close(blocking.releaseFirst) | |||
| 236 | + | require.ErrorIs(t, <-done, failure) | |||
| 237 | + | ||||
| 238 | + | persisted, ok, err := catalogue.Record("work") | |||
| 239 | + | require.NoError(t, err) | |||
| 240 | + | require.True(t, ok) | |||
| 241 | + | require.Empty(t, persisted.TabNames) | |||
| 242 | + | require.Empty(t, sess.tabs[0].name) | |||
| 243 | + | } | |||
| 244 | + | ||||
| 245 | + | func TestBlockedCwdWriteCannotOverwriteLaterTabMetadata(t *testing.T) { | |||
| 246 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 247 | + | catalogue, blocking := newBlockingCatalogue(sess, nil) | |||
| 248 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 249 | + | d.catalogue = blocking | |||
| 250 | + | d.sessions[sess.id] = sess | |||
| 251 | + | d.procCwd = func(int) (string, error) { return "/later-cwd", nil } | |||
| 252 | + | ||||
| 253 | + | cwdDone := make(chan struct{}) | |||
| 254 | + | go func() { | |||
| 255 | + | d.refreshSessionCwd(sess) | |||
| 256 | + | close(cwdDone) | |||
| 257 | + | }() | |||
| 258 | + | select { | |||
| 259 | + | case <-blocking.firstStarted: | |||
| 260 | + | case <-time.After(testWaitTimeout): | |||
| 261 | + | t.Fatal("cwd metadata write did not block") | |||
| 262 | + | } | |||
| 263 | + | ||||
| 264 | + | renameDone := make(chan error, 1) | |||
| 265 | + | go func() { renameDone <- d.renameTab(sess, sess.tabs[0], "later-tab") }() | |||
| 266 | + | require.Eventually(t, func() bool { | |||
| 267 | + | sess.mu.Lock() | |||
| 268 | + | defer sess.mu.Unlock() | |||
| 269 | + | return sess.tabs[0].name == "later-tab" | |||
| 270 | + | }, testWaitTimeout, time.Millisecond) | |||
| 271 | + | ||||
| 272 | + | close(blocking.releaseFirst) | |||
| 273 | + | select { | |||
| 274 | + | case <-cwdDone: | |||
| 275 | + | case <-time.After(testWaitTimeout): | |||
| 276 | + | t.Fatal("cwd refresh deadlocked") | |||
| 277 | + | } | |||
| 278 | + | require.NoError(t, <-renameDone) | |||
| 279 | + | ||||
| 280 | + | persisted, ok, err := catalogue.Record("work") | |||
| 281 | + | require.NoError(t, err) | |||
| 282 | + | require.True(t, ok) | |||
| 283 | + | require.Equal(t, "/later-cwd", persisted.Cwd) | |||
| 284 | + | require.Equal(t, []string{"later-tab"}, persisted.TabNames) | |||
| 285 | + | } | |||
M internal/usecase/daemon/performance_benchmark_test.go +3 -3
| @@ -638,7 +638,7 @@ func publishBenchmarkSnapshot(b testing.TB, fixture *performanceFixture, generat | |||||
| 638 | 638 | ||||
| 639 | func mutateBenchmarkVisible(fixture *performanceFixture, operation int) { | 639 | func mutateBenchmarkVisible(fixture *performanceFixture, operation int) { | ||
| 640 | fixture.activePane.mu.Lock() | 640 | fixture.activePane.mu.Lock() | ||
| 641 | - | fixture.activePane.screen.Write([]byte(fmt.Sprintf("\x1b[1;1Hvisible-%08d", operation))) | 641 | + | fixture.activePane.screen.Write(fmt.Appendf(nil, "\x1b[1;1Hvisible-%08d", operation)) |
| 642 | fixture.activePane.mu.Unlock() | 642 | fixture.activePane.mu.Unlock() | ||
| 643 | } | 643 | } | ||
| 644 | 644 | ||||
| @@ -653,7 +653,7 @@ func mutateBenchmarkTail(fixture *performanceFixture, operation int) { | |||||
| 653 | } | 653 | } | ||
| 654 | 654 | ||||
| 655 | func mutateBenchmarkSealedChunk(fixture *performanceFixture, operation int) { | 655 | func mutateBenchmarkSealedChunk(fixture *performanceFixture, operation int) { | ||
| 656 | - | for row := 0; row < 256; row++ { | 656 | + | for row := range 256 { |
| 657 | mutateBenchmarkTail(fixture, operation*256+row) | 657 | mutateBenchmarkTail(fixture, operation*256+row) | ||
| 658 | } | 658 | } | ||
| 659 | } | 659 | } | ||
| @@ -1065,7 +1065,7 @@ func newPerformanceFixtureWithCleanup(t testing.TB, config performanceConfig, re | |||||
| 1065 | liveWrites: [][]byte{[]byte("\x1b[1;1HA\x1b[2;2HA"), []byte("\x1b[1;1HB\x1b[2;2HB")}, | 1065 | liveWrites: [][]byte{[]byte("\x1b[1;1HA\x1b[2;2HA"), []byte("\x1b[1;1HB\x1b[2;2HB")}, | ||
| 1066 | resizeSizes: [2]domain.Size{{Cols: 100, Rows: 30}, config.size}, | 1066 | resizeSizes: [2]domain.Size{{Cols: 100, Rows: 30}, config.size}, | ||
| 1067 | } | 1067 | } | ||
| 1068 | - | WithSnapshotRepository(fixture.snapshots, nil)(d) | 1068 | + | WithSnapshotRepository(fixture.snapshots)(d) |
| 1069 | d.startSnapshotEncodeWorker() | 1069 | d.startSnapshotEncodeWorker() | ||
| 1070 | if registerCleanup { | 1070 | if registerCleanup { | ||
| 1071 | t.Cleanup(d.stopSnapshotEncodeWorker) | 1071 | t.Cleanup(d.stopSnapshotEncodeWorker) | ||
M internal/usecase/daemon/picker_test.go +27 -34
| @@ -25,26 +25,6 @@ import ( | |||||
| 25 | 25 | ||||
| 26 | // --- test doubles ----------------------------------------------------------- | 26 | // --- test doubles ----------------------------------------------------------- | ||
| 27 | 27 | ||||
| 28 | - | // failingDeleteStore is a ports.Store whose Delete always fails, used to | |||
| 29 | - | // exercise the picker's kill-target error path without disturbing the | |||
| 30 | - | // generated mock's default Delete expectation. | |||
| 31 | - | type failingDeleteStore struct{ err error } | |||
| 32 | - | ||||
| 33 | - | func (failingDeleteStore) Get([]byte) ([]byte, bool) { return nil, false } | |||
| 34 | - | func (failingDeleteStore) Set([]byte, []byte) error { return nil } | |||
| 35 | - | func (s failingDeleteStore) Batch(changes []ports.StoreChange) error { | |||
| 36 | - | for _, change := range changes { | |||
| 37 | - | if change.Delete { | |||
| 38 | - | return s.err | |||
| 39 | - | } | |||
| 40 | - | } | |||
| 41 | - | return nil | |||
| 42 | - | } | |||
| 43 | - | func (s failingDeleteStore) Delete([]byte) error { return s.err } | |||
| 44 | - | func (failingDeleteStore) Range(func(k, v []byte) bool) {} | |||
| 45 | - | func (failingDeleteStore) Sync() error { return nil } | |||
| 46 | - | func (failingDeleteStore) Close() error { return nil } | |||
| 47 | - | ||||
| 48 | type refusingSnapshotDeleteRepository struct { | 28 | type refusingSnapshotDeleteRepository struct { | ||
| 49 | noOpSnapshotRepository | 29 | noOpSnapshotRepository | ||
| 50 | err error | 30 | err error | ||
| @@ -277,8 +257,8 @@ func TestPickerWaitsForRestoringTargetBeforeSwitching(t *testing.T) { | |||||
| 277 | d.mu.Lock() | 257 | d.mu.Lock() | ||
| 278 | delete(d.stopped, record.Name) | 258 | delete(d.stopped, record.Name) | ||
| 279 | d.sessions[target.id] = target | 259 | d.sessions[target.id] = target | ||
| 260 | + | closeRuntimeRestoreDoneLocked(done) | |||
| 280 | d.mu.Unlock() | 261 | d.mu.Unlock() | ||
| 281 | - | closeRuntimeRestoreDone(done) | |||
| 282 | 262 | ||||
| 283 | require.NoError(t, <-result) | 263 | require.NoError(t, <-result) | ||
| 284 | require.Same(t, target, ac.currentSession()) | 264 | require.Same(t, target, ac.currentSession()) | ||
| @@ -294,12 +274,12 @@ func TestRestoreCancellationTransitionsBeforePickerCompletion(t *testing.T) { | |||||
| 294 | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}) | 274 | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}) | ||
| 295 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | 275 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | ||
| 296 | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | 276 | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | ||
| 297 | - | WithSnapshotRepository(repository, nil)(d) | 277 | + | WithSnapshotRepository(repository)(d) |
| 298 | WithCheckpointCoordinator(coordinator)(d) | 278 | WithCheckpointCoordinator(coordinator)(d) | ||
| 299 | restoreCtx, cancelRestore := context.WithCancel(context.Background()) | 279 | restoreCtx, cancelRestore := context.WithCancel(context.Background()) | ||
| 300 | restored := make(chan struct{}) | 280 | restored := make(chan struct{}) | ||
| 301 | go func() { | 281 | go func() { | ||
| 302 | - | d.restoreCatalogue(restoreCtx, catalogue.Records()) | 282 | + | d.restoreCatalogue(restoreCtx, mustDurableRecords(t, catalogue)) |
| 303 | close(restored) | 283 | close(restored) | ||
| 304 | }() | 284 | }() | ||
| 305 | <-repository.started | 285 | <-repository.started | ||
| @@ -1150,8 +1130,18 @@ func TestPickerKillActiveSessionSnapshotDeleteRefusalReportsOnceAndKeepsPicker(t | |||||
| 1150 | d, from, ac, sends, releases := newRecentNavigationTestSessions(t) | 1130 | d, from, ac, sends, releases := newRecentNavigationTestSessions(t) | ||
| 1151 | defer releaseAll(releases) | 1131 | defer releaseAll(releases) | ||
| 1152 | cause := errors.New("snapshot delete refused") | 1132 | cause := errors.New("snapshot delete refused") | ||
| 1153 | - | WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause}, nil)(d) | 1133 | + | WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause})(d) |
| 1154 | target := d.sessions[domain.SessionID("recent")] | 1134 | target := d.sessions[domain.SessionID("recent")] | ||
| 1135 | + | store, _ := newMockStore(t) | |||
| 1136 | + | WithStore(store)(d) | |||
| 1137 | + | target.mu.Lock() | |||
| 1138 | + | if target.incarnation == (domain.IncarnationID{}) { | |||
| 1139 | + | target.incarnation = domain.IncarnationID{1} | |||
| 1140 | + | } | |||
| 1141 | + | record := target.persistRecordLocked(1) | |||
| 1142 | + | target.mu.Unlock() | |||
| 1143 | + | record.RecoveryState = domain.RecoveryFresh | |||
| 1144 | + | require.NoError(t, d.catalogue.Create(record)) | |||
| 1155 | 1145 | ||||
| 1156 | d.enterPicker(from, ac) | 1146 | d.enterPicker(from, ac) | ||
| 1157 | awaitFrame(t, sends, ports.MsgOutput) | 1147 | awaitFrame(t, sends, ports.MsgOutput) | ||
| @@ -1184,8 +1174,14 @@ func TestPickerKillStoppedSessionPersistDeleteFailureSurfacesNoticeAndKeepsEntry | |||||
| 1184 | defer release() | 1174 | defer release() | ||
| 1185 | d, from, ac, sends := newManualSessionWithPTYs(t, p) | 1175 | d, from, ac, sends := newManualSessionWithPTYs(t, p) | ||
| 1186 | cause := errors.New("delete failed") | 1176 | cause := errors.New("delete failed") | ||
| 1187 | - | WithStore(failingDeleteStore{err: cause})(d) | 1177 | + | store, state := newMockStore(t) |
| 1188 | - | d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp/stopped", createdAt: 7} | 1178 | + | WithStore(store)(d) |
| 1179 | + | record := domain.CatalogueRecord{Name: "stopped", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/stopped", CreatedAt: 7, RecoveryState: domain.RecoveryFresh} | |||
| 1180 | + | require.NoError(t, d.catalogue.Create(record)) | |||
| 1181 | + | state.mu.Lock() | |||
| 1182 | + | state.deleteErr = func(string) error { return cause } | |||
| 1183 | + | state.mu.Unlock() | |||
| 1184 | + | d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp/stopped", createdAt: 7, incarnation: record.IncarnationID, record: record} | |||
| 1189 | 1185 | ||||
| 1190 | d.enterPicker(from, ac) | 1186 | d.enterPicker(from, ac) | ||
| 1191 | awaitFrame(t, sends, ports.MsgOutput) | 1187 | awaitFrame(t, sends, ports.MsgOutput) | ||
| @@ -1202,12 +1198,9 @@ func TestPickerKillStoppedSessionPersistDeleteFailureSurfacesNoticeAndKeepsEntry | |||||
| 1202 | toasts := awaitToastCount(t, ac, 1) | 1198 | toasts := awaitToastCount(t, ac, 1) | ||
| 1203 | require.Equal(t, domain.NoticePersistDelete, toasts[0].Code) | 1199 | require.Equal(t, domain.NoticePersistDelete, toasts[0].Code) | ||
| 1204 | 1200 | ||||
| 1205 | - | views, _ := d.pickerViews(from) | 1201 | + | d.mu.Lock() |
| 1206 | - | var stillListed bool | 1202 | + | stopped, retained := d.stopped["stopped"] |
| 1207 | - | for _, v := range views { | 1203 | + | d.mu.Unlock() |
| 1208 | - | if v.Stopped && v.Name == "stopped" { | 1204 | + | require.True(t, retained, "failed deletion must retain the reserved name") |
| 1209 | - | stillListed = true | 1205 | + | require.True(t, stopped.purging, "an uncertain deletion stays fenced from restore") |
| 1210 | - | } | |||
| 1211 | - | } | |||
| 1212 | - | require.True(t, stillListed, "entry must remain listed after refreshPicker when the delete failed") | |||
| 1213 | } | 1206 | } | ||
M internal/usecase/daemon/recovery_observability.go +6 -1
| @@ -22,7 +22,12 @@ func (d *Daemon) logSessionDegraded(record domain.CatalogueRecord, reasonCode st | |||||
| 22 | func (d *Daemon) logStartupRecoveryCounts(restoring int) { | 22 | func (d *Daemon) logStartupRecoveryCounts(restoring int) { | ||
| 23 | var records []domain.CatalogueRecord | 23 | var records []domain.CatalogueRecord | ||
| 24 | if d.catalogue != nil { | 24 | if d.catalogue != nil { | ||
| 25 | - | records = d.catalogue.Records() | 25 | + | var err error |
| 26 | + | records, err = d.catalogue.Records() | |||
| 27 | + | if err != nil { | |||
| 28 | + | d.log.Error("daemon_startup_catalogue_read_failed", "err", err) | |||
| 29 | + | return | |||
| 30 | + | } | |||
| 26 | } else { | 31 | } else { | ||
| 27 | d.mu.Lock() | 32 | d.mu.Lock() | ||
| 28 | records = make([]domain.CatalogueRecord, 0, len(d.stopped)) | 33 | records = make([]domain.CatalogueRecord, 0, len(d.stopped)) | ||
M internal/usecase/daemon/render_capture.go +37 -12
| @@ -17,15 +17,21 @@ import ( | |||||
| 17 | // capture/compose/emit transaction holds that attachment's sendMu, so these | 17 | // capture/compose/emit transaction holds that attachment's sendMu, so these | ||
| 18 | // buffers cannot be observed or reused before emission completes. | 18 | // buffers cannot be observed or reused before emission completes. | ||
| 19 | type renderCaptureScratch struct { | 19 | type renderCaptureScratch struct { | ||
| 20 | - | state capturedRenderState | 20 | + | state capturedRenderState |
| 21 | - | panes []capturedPaneRenderState | 21 | + | panes []capturedPaneRenderState |
| 22 | - | placements []layout.Placement | 22 | + | placements []layout.Placement |
| 23 | - | dividers []layout.Divider | 23 | + | dividers []layout.Divider |
| 24 | - | statusTabs []statusTab | 24 | + | layoutTab *tab |
| 25 | - | mru []recentSession | 25 | + | layoutGeneration uint64 |
| 26 | - | ranked []rankedRecent | 26 | + | layoutArea domain.Rect |
| 27 | - | titleIDs []layout.PaneID | 27 | + | layoutFocus layout.PaneID |
| 28 | - | receipts []damageReceipt | 28 | + | layoutFingerprint string |
| 29 | + | layoutValid bool | |||
| 30 | + | statusTabs []statusTab | |||
| 31 | + | mru []recentSession | |||
| 32 | + | ranked []rankedRecent | |||
| 33 | + | titleIDs []layout.PaneID | |||
| 34 | + | receipts []damageReceipt | |||
| 29 | } | 35 | } | ||
| 30 | 36 | ||||
| 31 | type damageReceipt struct { | 37 | type damageReceipt struct { | ||
| @@ -225,9 +231,28 @@ func capturePrimaryRenderState( | |||||
| 225 | 231 | ||||
| 226 | tb.mu.Lock() | 232 | tb.mu.Lock() | ||
| 227 | defer tb.mu.Unlock() | 233 | defer tb.mu.Unlock() | ||
| 228 | - | layoutSnap := solveTabLayoutLocked(tb) | 234 | + | area := domain.Rect{Width: tb.size.Cols, Height: tb.size.Rows} |
| 229 | - | scratch.placements = append(scratch.placements[:0], layoutSnap.placements...) | 235 | + | var focus layout.PaneID |
| 230 | - | scratch.dividers = append(scratch.dividers[:0], layoutSnap.dividers...) | 236 | + | if tb.tree != nil { |
| 237 | + | focus = tb.tree.Focus | |||
| 238 | + | } | |||
| 239 | + | layoutSnap := tabLayoutSnapshot{ | |||
| 240 | + | area: area, focus: focus, placements: scratch.placements, | |||
| 241 | + | dividers: scratch.dividers, fingerprint: scratch.layoutFingerprint, ok: scratch.layoutValid, | |||
| 242 | + | } | |||
| 243 | + | if scratch.layoutTab != tb || scratch.layoutGeneration != tb.layoutGeneration || scratch.layoutArea != area || scratch.layoutFocus != focus { | |||
| 244 | + | layoutSnap = solveTabLayoutLocked(tb) | |||
| 245 | + | scratch.placements = append(scratch.placements[:0], layoutSnap.placements...) | |||
| 246 | + | scratch.dividers = append(scratch.dividers[:0], layoutSnap.dividers...) | |||
| 247 | + | scratch.layoutTab = tb | |||
| 248 | + | scratch.layoutGeneration = tb.layoutGeneration | |||
| 249 | + | scratch.layoutArea = area | |||
| 250 | + | scratch.layoutFocus = focus | |||
| 251 | + | scratch.layoutFingerprint = layoutSnap.fingerprint | |||
| 252 | + | scratch.layoutValid = layoutSnap.ok | |||
| 253 | + | layoutSnap.placements = scratch.placements | |||
| 254 | + | layoutSnap.dividers = scratch.dividers | |||
| 255 | + | } | |||
| 231 | state := &scratch.state | 256 | state := &scratch.state | ||
| 232 | *state = capturedRenderState{ | 257 | *state = capturedRenderState{ | ||
| 233 | attachment: ac, lease: lease, reset: reset, bars: bars, theme: bars.theme, | 258 | attachment: ac, lease: lease, reset: reset, bars: bars, theme: bars.theme, | ||
M internal/usecase/daemon/session.go +263 -126
| @@ -15,7 +15,6 @@ import ( | |||||
| 15 | "time" | 15 | "time" | ||
| 16 | 16 | ||||
| 17 | "github.com/bnema/vev/internal/domain" | 17 | "github.com/bnema/vev/internal/domain" | ||
| 18 | - | "github.com/bnema/vev/internal/persist" | |||
| 19 | "github.com/bnema/vev/internal/ports" | 18 | "github.com/bnema/vev/internal/ports" | ||
| 20 | "github.com/bnema/vev/internal/usecase/layout" | 19 | "github.com/bnema/vev/internal/usecase/layout" | ||
| 21 | ) | 20 | ) | ||
| @@ -47,22 +46,31 @@ type session struct { | |||||
| 47 | // tab, or pane locks. The owner performs teardown without holding teardownMu; | 46 | // tab, or pane locks. The owner performs teardown without holding teardownMu; | ||
| 48 | // duplicate daemon-shutdown callers can therefore bound their ownership wait | 47 | // duplicate daemon-shutdown callers can therefore bound their ownership wait | ||
| 49 | // with Serve's shared snapshot deadline. | 48 | // with Serve's shared snapshot deadline. | ||
| 50 | - | teardownMu sync.Mutex | 49 | + | teardownMu sync.Mutex |
| 51 | - | teardownActive bool | 50 | + | teardownActive bool |
| 52 | - | teardownDone chan struct{} | 51 | + | teardownDone chan struct{} |
| 53 | - | teardownWaiters uint | 52 | + | teardownWaiters uint |
| 54 | - | teardownChanged chan struct{} | 53 | + | teardownChanged chan struct{} |
| 55 | - | lifecycleStopOnce sync.Once | 54 | + | lifecycleStopOnce sync.Once |
| 56 | - | mu sync.Mutex // guards tabs, active, client, clipFiles, and clipboard queue state | 55 | + | mu sync.Mutex // guards tabs, active, client, clipFiles, and clipboard queue state |
| 57 | - | themeMu sync.Mutex | 56 | + | // metadataPersistMu serializes authority writes and post-I/O rollback. State |
| 58 | - | tabs []*tab | 57 | + | // mutation paths must release d.mu and mu before acquiring it. After durable |
| 59 | - | active int | 58 | + | // I/O completes it may acquire d.mu then mu to reconcile failed revisions. |
| 60 | - | client *attachedClient | 59 | + | metadataPersistMu sync.Mutex |
| 61 | - | clipboardQueue []clipboardForward | 60 | + | metadataVersion uint64 // guarded by mu; every durable metadata snapshot receives a revision |
| 62 | - | clipboardWorkerRunning bool | 61 | + | metadataDurableVersion uint64 // guarded by metadataPersistMu |
| 63 | - | renameInProgress bool | 62 | + | metadataLiveVersion uint64 // rollback cursor guarded by metadataPersistMu |
| 64 | - | cwd string | 63 | + | metadataFailedRollbacks map[uint64]func() bool // guarded by metadataPersistMu |
| 65 | - | terminal terminalEnv | 64 | + | themeMu sync.Mutex |
| 65 | + | tabs []*tab | |||
| 66 | + | active int | |||
| 67 | + | client *attachedClient | |||
| 68 | + | clipboardQueue []clipboardForward | |||
| 69 | + | clipboardWorkerRunning bool | |||
| 70 | + | renameInProgress bool | |||
| 71 | + | renameDone chan struct{} | |||
| 72 | + | cwd string | |||
| 73 | + | terminal terminalEnv | |||
| 66 | // env is the authoritative immutable environment snapshot for future PTY children. | 74 | // env is the authoritative immutable environment snapshot for future PTY children. | ||
| 67 | // It is guarded by mu and is always copied on ingress and egress. | 75 | // It is guarded by mu and is always copied on ingress and egress. | ||
| 68 | env []string | 76 | env []string | ||
| @@ -167,12 +175,14 @@ func (d *Daemon) touchMRU(sess *session) { | |||||
| 167 | } | 175 | } | ||
| 168 | seq := d.mruSeq.Add(1) | 176 | seq := d.mruSeq.Add(1) | ||
| 169 | updated := false | 177 | updated := false | ||
| 178 | + | var previous uint64 | |||
| 170 | for { | 179 | for { | ||
| 171 | old := sess.mruAt.Load() | 180 | old := sess.mruAt.Load() | ||
| 172 | if old >= seq { | 181 | if old >= seq { | ||
| 173 | return | 182 | return | ||
| 174 | } | 183 | } | ||
| 175 | if sess.mruAt.CompareAndSwap(old, seq) { | 184 | if sess.mruAt.CompareAndSwap(old, seq) { | ||
| 185 | + | previous = old | |||
| 176 | updated = true | 186 | updated = true | ||
| 177 | break | 187 | break | ||
| 178 | } | 188 | } | ||
| @@ -182,19 +192,49 @@ func (d *Daemon) touchMRU(sess *session) { | |||||
| 182 | } | 192 | } | ||
| 183 | sess.mu.Lock() | 193 | sess.mu.Lock() | ||
| 184 | name := sess.name | 194 | name := sess.name | ||
| 185 | - | incarnation := sess.incarnation | |||
| 186 | ephemeral := sess.ephemeral | 195 | ephemeral := sess.ephemeral | ||
| 196 | + | var record domain.CatalogueRecord | |||
| 197 | + | var version uint64 | |||
| 198 | + | if !ephemeral { | |||
| 199 | + | record, version = sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) | |||
| 200 | + | } | |||
| 187 | sess.mu.Unlock() | 201 | sess.mu.Unlock() | ||
| 188 | if !ephemeral { | 202 | if !ephemeral { | ||
| 189 | - | if err := d.updateCatalogueMetadata(domain.CatalogueMetadataUpdate{Name: name, IncarnationID: incarnation, LastUsedSeq: &seq}); err != nil { | 203 | + | rollback := func() bool { |
| 204 | + | return sess.mruAt.CompareAndSwap(seq, previous) | |||
| 205 | + | } | |||
| 206 | + | if err := d.persistSessionMetadata(sess, version, record.MetadataUpdate(), rollback); err != nil { | |||
| 190 | d.log.Warn("touching persisted session recency failed", "err", err, "session", name) | 207 | d.log.Warn("touching persisted session recency failed", "err", err, "session", name) | ||
| 191 | } | 208 | } | ||
| 192 | } | 209 | } | ||
| 193 | } | 210 | } | ||
| 194 | 211 | ||||
| 195 | func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) { | 212 | func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) { | ||
| 213 | + | // Historical focused tests call this helper directly. Acquire mu for those | |||
| 214 | + | // callers while preserving the production contract that an owning caller | |||
| 215 | + | // receives it still locked on return. | |||
| 216 | + | acquiredMu := d.mu.TryLock() | |||
| 217 | + | if acquiredMu { | |||
| 218 | + | defer d.mu.Unlock() | |||
| 219 | + | } | |||
| 196 | env = copyEnvironment(env) | 220 | env = copyEnvironment(env) | ||
| 221 | + | if _, reserved := d.creating[name]; reserved { | |||
| 222 | + | return nil, errSessionNameInUse | |||
| 223 | + | } | |||
| 224 | + | d.creating[name] = struct{}{} | |||
| 225 | + | defer delete(d.creating, name) | |||
| 197 | stopped, resuming := d.stopped[name] | 226 | stopped, resuming := d.stopped[name] | ||
| 227 | + | var authoritative domain.CatalogueRecord | |||
| 228 | + | var authoritativeExists bool | |||
| 229 | + | if !ephemeral && resuming && d.persistEnabled { | |||
| 230 | + | d.mu.Unlock() | |||
| 231 | + | var err error | |||
| 232 | + | authoritative, authoritativeExists, err = d.catalogueRecord(name) | |||
| 233 | + | d.mu.Lock() | |||
| 234 | + | if err != nil { | |||
| 235 | + | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't read session catalogue", err) | |||
| 236 | + | } | |||
| 237 | + | } | |||
| 198 | var createdAt int64 | 238 | var createdAt int64 | ||
| 199 | var incarnation domain.IncarnationID | 239 | var incarnation domain.IncarnationID | ||
| 200 | if !ephemeral { | 240 | if !ephemeral { | ||
| @@ -203,9 +243,9 @@ func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz | |||||
| 203 | // the persisted identity rather than receive a fresh timestamp. | 243 | // the persisted identity rather than receive a fresh timestamp. | ||
| 204 | createdAt = stopped.createdAt | 244 | createdAt = stopped.createdAt | ||
| 205 | incarnation = stopped.incarnation | 245 | incarnation = stopped.incarnation | ||
| 206 | - | if authoritative, ok := d.catalogueRecord(name); ok { | 246 | + | if authoritativeExists { |
| 207 | - | incarnation = authoritative.IncarnationID | |||
| 208 | createdAt = authoritative.CreatedAt | 247 | createdAt = authoritative.CreatedAt | ||
| 248 | + | incarnation = authoritative.IncarnationID | |||
| 209 | } | 249 | } | ||
| 210 | if createdAt > d.lastAllocatedCreatedAt { | 250 | if createdAt > d.lastAllocatedCreatedAt { | ||
| 211 | d.lastAllocatedCreatedAt = createdAt | 251 | d.lastAllocatedCreatedAt = createdAt | ||
| @@ -263,6 +303,9 @@ func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz | |||||
| 263 | if !ephemeral && resuming { | 303 | if !ephemeral && resuming { | ||
| 264 | lastUsedSeq = stopped.lastUsedSeq | 304 | lastUsedSeq = stopped.lastUsedSeq | ||
| 265 | } | 305 | } | ||
| 306 | + | if lastUsedSeq == 0 { | |||
| 307 | + | lastUsedSeq = d.mruSeq.Add(1) | |||
| 308 | + | } | |||
| 266 | sess := &session{ | 309 | sess := &session{ | ||
| 267 | id: id, | 310 | id: id, | ||
| 268 | name: name, | 311 | name: name, | ||
| @@ -286,34 +329,50 @@ func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz | |||||
| 286 | sess.snapEligible.Store(!ephemeral && name != "") | 329 | sess.snapEligible.Store(!ephemeral && name != "") | ||
| 287 | if !ephemeral { | 330 | if !ephemeral { | ||
| 288 | if d.persistEnabled { | 331 | if d.persistEnabled { | ||
| 289 | - | record := persist.Record{Name: name, IncarnationID: incarnation, Cwd: cwd, CreatedAt: createdAt, UpdatedAt: createdAt, LastUsedSeq: lastUsedSeq, TabNames: names, RecoveryState: domain.RecoveryFresh} | 332 | + | record := domain.CatalogueRecord{Name: name, IncarnationID: incarnation, Cwd: cwd, CreatedAt: createdAt, UpdatedAt: createdAt, LastUsedSeq: lastUsedSeq, TabNames: names, RecoveryState: domain.RecoveryFresh} |
| 333 | + | if d.lifecycleRecovery == nil { | |||
| 334 | + | closeTabs(tabs) | |||
| 335 | + | cancel() | |||
| 336 | + | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", errors.New("durable session authority is not configured")) | |||
| 337 | + | } | |||
| 338 | + | // The name reservation remains visible while durable authority I/O runs, | |||
| 339 | + | // but daemon/session mutexes are released so storage cannot block routing. | |||
| 340 | + | d.mu.Unlock() | |||
| 290 | var err error | 341 | var err error | ||
| 291 | - | if resuming { | 342 | + | if resuming && authoritativeExists { |
| 292 | - | if _, ok := d.catalogueRecord(name); ok { | 343 | + | err = d.updateCatalogueMetadata(record.MetadataUpdate()) |
| 293 | - | err = d.updateCatalogueMetadata(record.MetadataUpdate()) | |||
| 294 | - | } else if d.lifecycleRecovery != nil { | |||
| 295 | - | record, err = d.lifecycleRecovery.Create(context.Background(), record) | |||
| 296 | - | } else { | |||
| 297 | - | err = d.createCatalogueRecord(record) | |||
| 298 | - | } | |||
| 299 | - | } else if d.lifecycleRecovery != nil { | |||
| 300 | - | record, err = d.lifecycleRecovery.Create(context.Background(), record) | |||
| 301 | } else { | 344 | } else { | ||
| 302 | - | err = d.createCatalogueRecord(record) | 345 | + | record, err = d.lifecycleRecovery.Create(d.serveCtx, record) |
| 303 | } | 346 | } | ||
| 347 | + | d.mu.Lock() | |||
| 304 | if err != nil { | 348 | if err != nil { | ||
| 305 | closeTabs(tabs) | 349 | closeTabs(tabs) | ||
| 306 | cancel() | 350 | cancel() | ||
| 307 | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", err) | 351 | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", err) | ||
| 308 | } | 352 | } | ||
| 353 | + | if d.closing { | |||
| 354 | + | // Creation lost its race with shutdown. Roll back a newly committed | |||
| 355 | + | // authority record without holding mu; resumed metadata remains owned | |||
| 356 | + | // by the existing stopped lifecycle. | |||
| 357 | + | if !resuming { | |||
| 358 | + | d.mu.Unlock() | |||
| 359 | + | rollbackCtx, rollbackCancel := context.WithTimeout(context.WithoutCancel(d.serveCtx), snapshotFinalFlushTimeout) | |||
| 360 | + | rollbackErr := d.lifecycleRecovery.Delete(rollbackCtx, name) | |||
| 361 | + | rollbackCancel() | |||
| 362 | + | d.mu.Lock() | |||
| 363 | + | if rollbackErr != nil { | |||
| 364 | + | d.log.Warn("rolling back session creation during shutdown failed", "session", name, "err", rollbackErr) | |||
| 365 | + | } | |||
| 366 | + | } | |||
| 367 | + | closeTabs(tabs) | |||
| 368 | + | cancel() | |||
| 369 | + | return nil, errors.New("daemon is shutting down") | |||
| 370 | + | } | |||
| 309 | sess.incarnation = record.IncarnationID | 371 | sess.incarnation = record.IncarnationID | ||
| 310 | } | 372 | } | ||
| 311 | delete(d.stopped, name) | 373 | delete(d.stopped, name) | ||
| 312 | } | 374 | } | ||
| 313 | d.sessions[id] = sess | 375 | d.sessions[id] = sess | ||
| 314 | - | if lastUsedSeq == 0 { | |||
| 315 | - | d.touchMRU(sess) | |||
| 316 | - | } | |||
| 317 | d.log.Info("session created", "session", name, "id", id, "ephemeral", ephemeral) | 376 | d.log.Info("session created", "session", name, "id", id, "ephemeral", ephemeral) | ||
| 318 | for i, tb := range tabs { | 377 | for i, tb := range tabs { | ||
| 319 | d.log.Info("tab created", "session", name, "tab", i) | 378 | d.log.Info("tab created", "session", name, "tab", i) | ||
| @@ -445,25 +504,41 @@ func (d *Daemon) createTab(sess *session, sz domain.Size) error { | |||||
| 445 | sess.tabs = append(sess.tabs, tb) | 504 | sess.tabs = append(sess.tabs, tb) | ||
| 446 | sess.active = len(sess.tabs) - 1 | 505 | sess.active = len(sess.tabs) - 1 | ||
| 447 | tabIndex := sess.active | 506 | tabIndex := sess.active | ||
| 448 | - | record := sess.persistRecordLocked(time.Now().UnixNano()) | 507 | + | record, metadataVersion := sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) |
| 449 | ephemeral := sess.ephemeral | 508 | ephemeral := sess.ephemeral | ||
| 509 | + | sess.mu.Unlock() | |||
| 510 | + | d.mu.Unlock() | |||
| 450 | if !ephemeral { | 511 | if !ephemeral { | ||
| 451 | - | if err := d.updateCatalogueMetadata(record.MetadataUpdate()); err != nil { | 512 | + | rollback := func() bool { |
| 452 | - | sess.tabs = sess.tabs[:len(sess.tabs)-1] | 513 | + | d.mu.Lock() |
| 453 | - | sess.active = oldActive | 514 | + | defer d.mu.Unlock() |
| 515 | + | if d.sessions[sess.id] != sess { | |||
| 516 | + | return true | |||
| 517 | + | } | |||
| 518 | + | sess.mu.Lock() | |||
| 519 | + | if !slices.Contains(sess.tabs, tb) { | |||
| 520 | + | sess.mu.Unlock() | |||
| 521 | + | return true | |||
| 522 | + | } | |||
| 523 | + | if sess.tabs[len(sess.tabs)-1] != tb { | |||
| 524 | + | sess.mu.Unlock() | |||
| 525 | + | return false | |||
| 526 | + | } | |||
| 527 | + | sess.tabs = slices.DeleteFunc(sess.tabs, func(candidate *tab) bool { return candidate == tb }) | |||
| 528 | + | sess.active = min(oldActive, len(sess.tabs)-1) | |||
| 529 | + | sess.mu.Unlock() | |||
| 454 | if tb.cancel != nil { | 530 | if tb.cancel != nil { | ||
| 455 | tb.cancel() | 531 | tb.cancel() | ||
| 456 | } | 532 | } | ||
| 457 | - | sess.mu.Unlock() | |||
| 458 | - | d.mu.Unlock() | |||
| 459 | _ = pty.Close() | 533 | _ = pty.Close() | ||
| 534 | + | return true | |||
| 535 | + | } | |||
| 536 | + | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |||
| 460 | return err | 537 | return err | ||
| 461 | } | 538 | } | ||
| 462 | } | 539 | } | ||
| 463 | - | sess.mu.Unlock() | |||
| 464 | d.log.Info("tab created", "session", name, "tab", tabIndex) | 540 | d.log.Info("tab created", "session", name, "tab", tabIndex) | ||
| 465 | d.startTabGoroutines(sess, tb) | 541 | d.startTabGoroutines(sess, tb) | ||
| 466 | - | d.mu.Unlock() | |||
| 467 | d.activateTab(sess, tb) | 542 | d.activateTab(sess, tb) | ||
| 468 | markSnapshotDirty(sess) | 543 | markSnapshotDirty(sess) | ||
| 469 | return nil | 544 | return nil | ||
| @@ -686,51 +761,43 @@ func (d *Daemon) renameSession(sess *session, name string) error { | |||||
| 686 | } | 761 | } | ||
| 687 | } | 762 | } | ||
| 688 | lastUsedSeq := sess.mruAt.Load() | 763 | lastUsedSeq := sess.mruAt.Load() | ||
| 689 | - | record := sess.persistRecordLocked(d.nowUnixNano()) | 764 | + | record := sess.persistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) |
| 690 | record.Name = name | 765 | record.Name = name | ||
| 691 | record.CreatedAt = createdAt | 766 | record.CreatedAt = createdAt | ||
| 692 | record.LastUsedSeq = lastUsedSeq | 767 | record.LastUsedSeq = lastUsedSeq | ||
| 693 | sess.renameInProgress = true | 768 | sess.renameInProgress = true | ||
| 769 | + | sess.renameDone = make(chan struct{}) | |||
| 694 | sess.mu.Unlock() | 770 | sess.mu.Unlock() | ||
| 695 | d.mu.Unlock() | 771 | d.mu.Unlock() | ||
| 696 | 772 | ||||
| 697 | rollback := func(err error) error { | 773 | rollback := func(err error) error { | ||
| 698 | sess.mu.Lock() | 774 | sess.mu.Lock() | ||
| 699 | sess.renameInProgress = false | 775 | sess.renameInProgress = false | ||
| 776 | + | if sess.renameDone != nil { | |||
| 777 | + | close(sess.renameDone) | |||
| 778 | + | sess.renameDone = nil | |||
| 779 | + | } | |||
| 700 | sess.mu.Unlock() | 780 | sess.mu.Unlock() | ||
| 701 | return err | 781 | return err | ||
| 702 | } | 782 | } | ||
| 703 | // Durable named renames are one atomic catalogue batch. The incarnation-keyed | 783 | // Durable named renames are one atomic catalogue batch. The incarnation-keyed | ||
| 704 | // snapshot namespace is intentionally untouched. | 784 | // snapshot namespace is intentionally untouched. | ||
| 705 | if (wasEphemeral || oldName != name) && d.persistEnabled { | 785 | if (wasEphemeral || oldName != name) && d.persistEnabled { | ||
| 706 | - | if d.lifecycleRecovery != nil { | 786 | + | if d.lifecycleRecovery == nil { |
| 787 | + | return rollback(errors.New("durable session authority is not configured")) | |||
| 788 | + | } | |||
| 789 | + | { | |||
| 707 | var committed domain.CatalogueRecord | 790 | var committed domain.CatalogueRecord | ||
| 708 | var err error | 791 | var err error | ||
| 709 | if wasEphemeral { | 792 | if wasEphemeral { | ||
| 710 | - | committed, err = d.lifecycleRecovery.Create(context.Background(), record) | 793 | + | committed, err = d.lifecycleRecovery.Create(d.serveCtx, record) |
| 711 | } else { | 794 | } else { | ||
| 712 | - | committed, err = d.lifecycleRecovery.Rename(context.Background(), oldName, name) | 795 | + | committed, err = d.lifecycleRecovery.Rename(d.serveCtx, oldName, name) |
| 713 | } | 796 | } | ||
| 714 | if err != nil { | 797 | if err != nil { | ||
| 715 | return rollback(err) | 798 | return rollback(err) | ||
| 716 | } | 799 | } | ||
| 717 | record = committed | 800 | record = committed | ||
| 718 | - | } else if wasEphemeral { | |||
| 719 | - | if err := d.createCatalogueRecord(record); err != nil { | |||
| 720 | - | return rollback(err) | |||
| 721 | - | } | |||
| 722 | - | } else { | |||
| 723 | - | if current, ok := d.catalogueRecord(oldName); ok { | |||
| 724 | - | current.Name = name | |||
| 725 | - | current.Cwd = record.Cwd | |||
| 726 | - | current.UpdatedAt = record.UpdatedAt | |||
| 727 | - | current.LastUsedSeq = record.LastUsedSeq | |||
| 728 | - | current.TabNames = append([]string(nil), record.TabNames...) | |||
| 729 | - | record = current | |||
| 730 | - | } | |||
| 731 | - | if err := d.catalogue.Rename(oldName, record); err != nil { | |||
| 732 | - | return rollback(err) | |||
| 733 | - | } | |||
| 734 | } | 801 | } | ||
| 735 | } | 802 | } | ||
| 736 | 803 | ||||
| @@ -745,6 +812,10 @@ func (d *Daemon) renameSession(sess *session, name string) error { | |||||
| 745 | sess.incarnation = record.IncarnationID | 812 | sess.incarnation = record.IncarnationID | ||
| 746 | sess.ephemeral = false | 813 | sess.ephemeral = false | ||
| 747 | sess.renameInProgress = false | 814 | sess.renameInProgress = false | ||
| 815 | + | if sess.renameDone != nil { | |||
| 816 | + | close(sess.renameDone) | |||
| 817 | + | sess.renameDone = nil | |||
| 818 | + | } | |||
| 748 | sess.snapEligible.Store(name != "") | 819 | sess.snapEligible.Store(name != "") | ||
| 749 | sess.mu.Unlock() | 820 | sess.mu.Unlock() | ||
| 750 | d.mu.Unlock() | 821 | d.mu.Unlock() | ||
| @@ -774,22 +845,42 @@ func (d *Daemon) renameTab(sess *session, tb *tab, name string) error { | |||||
| 774 | } | 845 | } | ||
| 775 | oldName := tb.name | 846 | oldName := tb.name | ||
| 776 | tb.name = name | 847 | tb.name = name | ||
| 777 | - | record := sess.persistRecordLocked(time.Now().UnixNano()) | 848 | + | record, metadataVersion := sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) |
| 778 | ephemeral := sess.ephemeral | 849 | ephemeral := sess.ephemeral | ||
| 779 | - | if !ephemeral { | |||
| 780 | - | if err := d.updateCatalogueMetadata(record.MetadataUpdate()); err != nil { | |||
| 781 | - | tb.name = oldName | |||
| 782 | - | sess.mu.Unlock() | |||
| 783 | - | d.mu.Unlock() | |||
| 784 | - | return err | |||
| 785 | - | } | |||
| 786 | - | } | |||
| 787 | sess.mu.Unlock() | 850 | sess.mu.Unlock() | ||
| 788 | d.mu.Unlock() | 851 | d.mu.Unlock() | ||
| 852 | + | if ephemeral { | |||
| 853 | + | return nil | |||
| 854 | + | } | |||
| 855 | + | rollback := func() bool { | |||
| 856 | + | d.mu.Lock() | |||
| 857 | + | defer d.mu.Unlock() | |||
| 858 | + | if d.sessions[sess.id] != sess { | |||
| 859 | + | return true | |||
| 860 | + | } | |||
| 861 | + | sess.mu.Lock() | |||
| 862 | + | defer sess.mu.Unlock() | |||
| 863 | + | if !slices.Contains(sess.tabs, tb) { | |||
| 864 | + | return true | |||
| 865 | + | } | |||
| 866 | + | if tb.name != name { | |||
| 867 | + | return false | |||
| 868 | + | } | |||
| 869 | + | tb.name = oldName | |||
| 870 | + | return true | |||
| 871 | + | } | |||
| 872 | + | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |||
| 873 | + | return err | |||
| 874 | + | } | |||
| 789 | return nil | 875 | return nil | ||
| 790 | } | 876 | } | ||
| 791 | 877 | ||||
| 792 | - | func (s *session) persistRecordLocked(updatedAt int64) persist.Record { | 878 | + | func (s *session) nextPersistRecordLocked(updatedAt int64) (domain.CatalogueRecord, uint64) { |
| 879 | + | s.metadataVersion++ | |||
| 880 | + | return s.persistRecordLocked(updatedAt), s.metadataVersion | |||
| 881 | + | } | |||
| 882 | + | ||||
| 883 | + | func (s *session) persistRecordLocked(updatedAt int64) domain.CatalogueRecord { | |||
| 793 | createdAt := s.createdAt | 884 | createdAt := s.createdAt | ||
| 794 | tabNames := make([]string, len(s.tabs)) | 885 | tabNames := make([]string, len(s.tabs)) | ||
| 795 | lastCustom := -1 | 886 | lastCustom := -1 | ||
| @@ -804,7 +895,7 @@ func (s *session) persistRecordLocked(updatedAt int64) persist.Record { | |||||
| 804 | } else { | 895 | } else { | ||
| 805 | tabNames = tabNames[:lastCustom+1] | 896 | tabNames = tabNames[:lastCustom+1] | ||
| 806 | } | 897 | } | ||
| 807 | - | return persist.Record{Name: s.name, IncarnationID: s.incarnation, Cwd: s.cwd, CreatedAt: createdAt, UpdatedAt: updatedAt, LastUsedSeq: s.mruAt.Load(), TabNames: tabNames, RecoveryState: domain.RecoveryFresh} | 898 | + | return domain.CatalogueRecord{Name: s.name, IncarnationID: s.incarnation, Cwd: s.cwd, CreatedAt: createdAt, UpdatedAt: updatedAt, LastUsedSeq: s.mruAt.Load(), TabNames: tabNames, RecoveryState: domain.RecoveryFresh} |
| 808 | } | 899 | } | ||
| 809 | 900 | ||||
| 810 | func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | 901 | func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | ||
| @@ -843,8 +934,10 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | |||||
| 843 | return nil | 934 | return nil | ||
| 844 | } | 935 | } | ||
| 845 | ringing := tb.attention | 936 | ringing := tb.attention | ||
| 937 | + | oldActive := sess.active | |||
| 846 | wasActive := idx == sess.active | 938 | wasActive := idx == sess.active | ||
| 847 | sess.tabs = append(sess.tabs[:idx], sess.tabs[idx+1:]...) | 939 | sess.tabs = append(sess.tabs[:idx], sess.tabs[idx+1:]...) | ||
| 940 | + | tabsAfterClose := len(sess.tabs) | |||
| 848 | if sess.active >= len(sess.tabs) { | 941 | if sess.active >= len(sess.tabs) { | ||
| 849 | sess.active = len(sess.tabs) - 1 | 942 | sess.active = len(sess.tabs) - 1 | ||
| 850 | } else if idx < sess.active { | 943 | } else if idx < sess.active { | ||
| @@ -853,15 +946,34 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | |||||
| 853 | destination := sess.tabs[sess.active] | 946 | destination := sess.tabs[sess.active] | ||
| 854 | ac := sess.client | 947 | ac := sess.client | ||
| 855 | name := sess.name | 948 | name := sess.name | ||
| 856 | - | record := sess.persistRecordLocked(time.Now().UnixNano()) | 949 | + | record, metadataVersion := sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) |
| 857 | ephemeral := sess.ephemeral | 950 | ephemeral := sess.ephemeral | ||
| 951 | + | sess.mu.Unlock() | |||
| 952 | + | d.mu.Unlock() | |||
| 858 | if !ephemeral { | 953 | if !ephemeral { | ||
| 859 | - | if err := d.updateCatalogueMetadata(record.MetadataUpdate()); err != nil { | 954 | + | rollback := func() bool { |
| 955 | + | d.mu.Lock() | |||
| 956 | + | defer d.mu.Unlock() | |||
| 957 | + | if d.sessions[sess.id] != sess { | |||
| 958 | + | return true | |||
| 959 | + | } | |||
| 960 | + | sess.mu.Lock() | |||
| 961 | + | defer sess.mu.Unlock() | |||
| 962 | + | if slices.Contains(sess.tabs, tb) { | |||
| 963 | + | return true | |||
| 964 | + | } | |||
| 965 | + | if len(sess.tabs) != tabsAfterClose { | |||
| 966 | + | return false | |||
| 967 | + | } | |||
| 968 | + | sess.tabs = slices.Insert(sess.tabs, min(idx, len(sess.tabs)), tb) | |||
| 969 | + | sess.active = min(oldActive, len(sess.tabs)-1) | |||
| 970 | + | return true | |||
| 971 | + | } | |||
| 972 | + | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |||
| 860 | d.log.Warn("persisting closed tab failed", "err", err, "session", name) | 973 | d.log.Warn("persisting closed tab failed", "err", err, "session", name) | ||
| 974 | + | return err | |||
| 861 | } | 975 | } | ||
| 862 | } | 976 | } | ||
| 863 | - | sess.mu.Unlock() | |||
| 864 | - | d.mu.Unlock() | |||
| 865 | d.log.Info("tab closed", "session", name) | 977 | d.log.Info("tab closed", "session", name) | ||
| 866 | markSnapshotDirty(sess) | 978 | markSnapshotDirty(sess) | ||
| 867 | 979 | ||||
| @@ -900,60 +1012,48 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | |||||
| 900 | // beginSnapshotPurge commits a logical named-session kill before its live | 1012 | // beginSnapshotPurge commits a logical named-session kill before its live | ||
| 901 | // identity or persisted metadata is removed. It never runs under daemon or | 1013 | // identity or persisted metadata is removed. It never runs under daemon or | ||
| 902 | // session locks. | 1014 | // session locks. | ||
| 903 | - | func (d *Daemon) beginSnapshotPurge(name string, incarnation domain.IncarnationID) error { | 1015 | + | func (d *Daemon) beginSnapshotPurge(_ string, _ domain.IncarnationID) error { |
| 904 | - | if d.lifecycleRecovery != nil { | 1016 | + | if d.persistEnabled && d.lifecycleRecovery == nil { |
| 905 | - | // The lifecycle coordinator performs the complete ordered protocol after | 1017 | + | return errors.New("durable session authority is not configured") |
| 906 | - | // snapshot writers have joined. | |||
| 907 | - | return nil | |||
| 908 | - | } | |||
| 909 | - | if d.snapshotRepository == nil || name == "" { | |||
| 910 | - | return nil | |||
| 911 | } | 1018 | } | ||
| 912 | - | return d.snapshotRepository.WriteDeletionTombstone(context.Background(), domain.DeletionTombstone{Name: name, IncarnationID: incarnation}) | 1019 | + | // The lifecycle coordinator performs the complete ordered protocol after |
| 1020 | + | // snapshot writers have joined. | |||
| 1021 | + | return nil | |||
| 913 | } | 1022 | } | ||
| 914 | 1023 | ||||
| 915 | // finishSnapshotPurge deletes both independently durable snapshot sources, | 1024 | // finishSnapshotPurge deletes both independently durable snapshot sources, | ||
| 916 | // then metadata, and only then clears the tombstone. Any failure deliberately | 1025 | // then metadata, and only then clears the tombstone. Any failure deliberately | ||
| 917 | // leaves the marker in place so startup cannot restore or import the name and | 1026 | // leaves the marker in place so startup cannot restore or import the name and | ||
| 918 | // a later live/offline kill can retry the idempotent source deletes. | 1027 | // a later live/offline kill can retry the idempotent source deletes. | ||
| 919 | - | func (d *Daemon) finishSnapshotPurge(name string, incarnation domain.IncarnationID) error { | 1028 | + | func (d *Daemon) finishSnapshotPurge(ctx context.Context, name string, _ domain.IncarnationID) error { |
| 920 | - | if d.lifecycleRecovery != nil { | 1029 | + | if !d.persistEnabled { |
| 921 | - | return d.lifecycleRecovery.Delete(context.Background(), name) | 1030 | + | return nil |
| 922 | - | } | |||
| 923 | - | if d.snapshotRepository == nil { | |||
| 924 | - | return d.deleteCatalogueRecord(name) | |||
| 925 | } | 1031 | } | ||
| 926 | - | if err := d.snapshotRepository.QuarantineDeletionSources(context.Background(), domain.DeletionTombstone{Name: name, IncarnationID: incarnation}, false); err != nil { | 1032 | + | if d.lifecycleRecovery == nil { |
| 927 | - | return err | 1033 | + | return errors.New("durable session authority is not configured") |
| 928 | } | 1034 | } | ||
| 929 | - | if err := d.deleteCatalogueRecord(name); err != nil { | 1035 | + | if ctx == nil { |
| 930 | - | return err | 1036 | + | ctx = d.serveCtx |
| 931 | } | 1037 | } | ||
| 932 | - | return d.snapshotRepository.DeleteDeletionTombstone(context.Background(), incarnation) | 1038 | + | purgeCtx, cancel := context.WithTimeout(ctx, snapshotFinalFlushTimeout) |
| 1039 | + | defer cancel() | |||
| 1040 | + | return d.lifecycleRecovery.Delete(purgeCtx, name) | |||
| 933 | } | 1041 | } | ||
| 934 | 1042 | ||||
| 935 | // retryStoppedPurge completes a previously closed session's durable purge. | 1043 | // retryStoppedPurge completes a previously closed session's durable purge. | ||
| 936 | // It keeps the stopped record hidden until all sources and metadata cross their | 1044 | // It keeps the stopped record hidden until all sources and metadata cross their | ||
| 937 | // deletion boundaries. | 1045 | // deletion boundaries. | ||
| 938 | func (d *Daemon) retryStoppedPurge(name string) error { | 1046 | func (d *Daemon) retryStoppedPurge(name string) error { | ||
| 1047 | + | return d.retryStoppedPurgeContext(d.serveCtx, name) | |||
| 1048 | + | } | |||
| 1049 | + | ||||
| 1050 | + | func (d *Daemon) retryStoppedPurgeContext(ctx context.Context, name string) error { | |||
| 939 | d.mu.Lock() | 1051 | d.mu.Lock() | ||
| 940 | stopped, ok := d.stopped[name] | 1052 | stopped, ok := d.stopped[name] | ||
| 941 | if !ok { | 1053 | if !ok { | ||
| 942 | d.mu.Unlock() | 1054 | d.mu.Unlock() | ||
| 943 | return nil | 1055 | return nil | ||
| 944 | } | 1056 | } | ||
| 945 | - | // Without a repository there is no restore/import source to fence, so the | |||
| 946 | - | // stopped record retains the metadata-only deletion behavior. | |||
| 947 | - | if d.snapshotRepository == nil { | |||
| 948 | - | d.mu.Unlock() | |||
| 949 | - | if err := d.deleteCatalogueRecord(name); err != nil { | |||
| 950 | - | return err | |||
| 951 | - | } | |||
| 952 | - | d.mu.Lock() | |||
| 953 | - | delete(d.stopped, name) | |||
| 954 | - | d.mu.Unlock() | |||
| 955 | - | return nil | |||
| 956 | - | } | |||
| 957 | stopped.purging = true | 1057 | stopped.purging = true | ||
| 958 | d.stopped[name] = stopped | 1058 | d.stopped[name] = stopped | ||
| 959 | d.mu.Unlock() | 1059 | d.mu.Unlock() | ||
| @@ -961,7 +1061,7 @@ func (d *Daemon) retryStoppedPurge(name string) error { | |||||
| 961 | if err := d.beginSnapshotPurge(name, stopped.incarnation); err != nil { | 1061 | if err := d.beginSnapshotPurge(name, stopped.incarnation); err != nil { | ||
| 962 | return err | 1062 | return err | ||
| 963 | } | 1063 | } | ||
| 964 | - | if err := d.finishSnapshotPurge(name, stopped.incarnation); err != nil { | 1064 | + | if err := d.finishSnapshotPurge(ctx, name, stopped.incarnation); err != nil { |
| 965 | return err | 1065 | return err | ||
| 966 | } | 1066 | } | ||
| 967 | d.mu.Lock() | 1067 | d.mu.Lock() | ||
| @@ -1089,6 +1189,23 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu | |||||
| 1089 | } | 1189 | } | ||
| 1090 | defer sess.finishTeardown() | 1190 | defer sess.finishTeardown() | ||
| 1091 | 1191 | ||||
| 1192 | + | // A rename reserves durable identity while coordinator I/O runs unlocked. | |||
| 1193 | + | // Teardown waits for that reservation before selecting the purge name. | |||
| 1194 | + | sess.mu.Lock() | |||
| 1195 | + | renameDone := sess.renameDone | |||
| 1196 | + | sess.mu.Unlock() | |||
| 1197 | + | if renameDone != nil { | |||
| 1198 | + | if deadline == nil { | |||
| 1199 | + | <-renameDone | |||
| 1200 | + | } else { | |||
| 1201 | + | select { | |||
| 1202 | + | case <-renameDone: | |||
| 1203 | + | case <-deadline.Done(): | |||
| 1204 | + | return context.DeadlineExceeded | |||
| 1205 | + | } | |||
| 1206 | + | } | |||
| 1207 | + | } | |||
| 1208 | + | ||||
| 1092 | d.mu.Lock() | 1209 | d.mu.Lock() | ||
| 1093 | current := d.sessions[sess.id] | 1210 | current := d.sessions[sess.id] | ||
| 1094 | d.mu.Unlock() | 1211 | d.mu.Unlock() | ||
| @@ -1106,13 +1223,14 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu | |||||
| 1106 | sess.mu.Lock() | 1223 | sess.mu.Lock() | ||
| 1107 | isEphemeral := sess.ephemeral | 1224 | isEphemeral := sess.ephemeral | ||
| 1108 | name := sess.name | 1225 | name := sess.name | ||
| 1226 | + | incarnation := sess.incarnation | |||
| 1109 | sess.mu.Unlock() | 1227 | sess.mu.Unlock() | ||
| 1110 | // Commit the tombstone before changing live identity or metadata. A | 1228 | // Commit the tombstone before changing live identity or metadata. A | ||
| 1111 | // repository delete must never race an older publication that could recreate | 1229 | // repository delete must never race an older publication that could recreate | ||
| 1112 | // the just-quarantined name. Do not hold daemon or session locks while | 1230 | // the just-quarantined name. Do not hold daemon or session locks while | ||
| 1113 | // tombstoning or joining: a repository implementation may block on I/O. | 1231 | // tombstoning or joining: a repository implementation may block on I/O. | ||
| 1114 | if purge && !isEphemeral { | 1232 | if purge && !isEphemeral { | ||
| 1115 | - | if err := d.beginSnapshotPurge(name, sess.incarnation); err != nil { | 1233 | + | if err := d.beginSnapshotPurge(name, incarnation); err != nil { |
| 1116 | return err | 1234 | return err | ||
| 1117 | } | 1235 | } | ||
| 1118 | if d.snapshotRepository != nil { | 1236 | if d.snapshotRepository != nil { | ||
| @@ -1188,7 +1306,7 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu | |||||
| 1188 | ephemeral := sess.ephemeral | 1306 | ephemeral := sess.ephemeral | ||
| 1189 | sess.mu.Unlock() | 1307 | sess.mu.Unlock() | ||
| 1190 | if !ephemeral { | 1308 | if !ephemeral { | ||
| 1191 | - | stopped := stoppedSession{name: stoppedName, cwd: stoppedCwd, createdAt: createdAt, incarnation: sess.incarnation, lastUsedSeq: sess.mruAt.Load(), tabNames: tabNames, purging: purge} | 1309 | + | stopped := stoppedSession{name: stoppedName, cwd: stoppedCwd, createdAt: createdAt, incarnation: incarnation, lastUsedSeq: sess.mruAt.Load(), tabNames: tabNames, purging: purge} |
| 1192 | d.stopped[stoppedName] = stopped | 1310 | d.stopped[stoppedName] = stopped | ||
| 1193 | } | 1311 | } | ||
| 1194 | empty := len(d.sessions) == 0 | 1312 | empty := len(d.sessions) == 0 | ||
| @@ -1248,7 +1366,7 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu | |||||
| 1248 | // tombstone and hidden stopped record when any source or metadata operation | 1366 | // tombstone and hidden stopped record when any source or metadata operation | ||
| 1249 | // fails; retryStoppedPurge (including a repeated live kill) resumes safely. | 1367 | // fails; retryStoppedPurge (including a repeated live kill) resumes safely. | ||
| 1250 | if !ephemeral && purge { | 1368 | if !ephemeral && purge { | ||
| 1251 | - | if err := d.finishSnapshotPurge(stoppedName, sess.incarnation); err != nil { | 1369 | + | if err := d.finishSnapshotPurge(d.serveCtx, stoppedName, incarnation); err != nil { |
| 1252 | purgeErr = errors.Join(purgeErr, err) | 1370 | purgeErr = errors.Join(purgeErr, err) | ||
| 1253 | d.log.Warn("finishing session snapshot purge failed", "err", err, "session", stoppedName) | 1371 | d.log.Warn("finishing session snapshot purge failed", "err", err, "session", stoppedName) | ||
| 1254 | } else { | 1372 | } else { | ||
| @@ -1302,8 +1420,11 @@ func (d *Daemon) nameLiveOrStoppedLocked(name string) bool { | |||||
| 1302 | if d.findByNameLocked(name) != nil { | 1420 | if d.findByNameLocked(name) != nil { | ||
| 1303 | return true | 1421 | return true | ||
| 1304 | } | 1422 | } | ||
| 1305 | - | _, ok := d.stopped[name] | 1423 | + | if _, ok := d.stopped[name]; ok { |
| 1306 | - | return ok | 1424 | + | return true |
| 1425 | + | } | |||
| 1426 | + | _, reserved := d.creating[name] | |||
| 1427 | + | return reserved | |||
| 1307 | } | 1428 | } | ||
| 1308 | 1429 | ||||
| 1309 | func (d *Daemon) cwdSampler(ctx context.Context) { | 1430 | func (d *Daemon) cwdSampler(ctx context.Context) { | ||
| @@ -1363,19 +1484,38 @@ func (d *Daemon) refreshSessionCwd(sess *session) { | |||||
| 1363 | d.mu.Unlock() | 1484 | d.mu.Unlock() | ||
| 1364 | return | 1485 | return | ||
| 1365 | } | 1486 | } | ||
| 1487 | + | oldCwd := sess.cwd | |||
| 1366 | sess.cwd = cwd | 1488 | sess.cwd = cwd | ||
| 1367 | name := sess.name | 1489 | name := sess.name | ||
| 1368 | - | incarnation := sess.incarnation | |||
| 1369 | ephemeral := sess.ephemeral | 1490 | ephemeral := sess.ephemeral | ||
| 1491 | + | var record domain.CatalogueRecord | |||
| 1492 | + | var metadataVersion uint64 | |||
| 1493 | + | if !ephemeral { | |||
| 1494 | + | record, metadataVersion = sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) | |||
| 1495 | + | } | |||
| 1370 | sess.mu.Unlock() | 1496 | sess.mu.Unlock() | ||
| 1497 | + | d.mu.Unlock() | |||
| 1371 | if !ephemeral { | 1498 | if !ephemeral { | ||
| 1372 | - | updatedAt := time.Now().UnixNano() | 1499 | + | rollback := func() bool { |
| 1373 | - | if err := d.updateCatalogueMetadata(domain.CatalogueMetadataUpdate{Name: name, IncarnationID: incarnation, Cwd: &cwd, UpdatedAt: &updatedAt}); err != nil { | 1500 | + | d.mu.Lock() |
| 1501 | + | defer d.mu.Unlock() | |||
| 1502 | + | if d.sessions[sess.id] != sess { | |||
| 1503 | + | return true | |||
| 1504 | + | } | |||
| 1505 | + | sess.mu.Lock() | |||
| 1506 | + | defer sess.mu.Unlock() | |||
| 1507 | + | if sess.cwd != cwd { | |||
| 1508 | + | return false | |||
| 1509 | + | } | |||
| 1510 | + | sess.cwd = oldCwd | |||
| 1511 | + | return true | |||
| 1512 | + | } | |||
| 1513 | + | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |||
| 1374 | d.log.Warn("touching persisted session cwd failed", "err", err, "session", name) | 1514 | d.log.Warn("touching persisted session cwd failed", "err", err, "session", name) | ||
| 1515 | + | return | |||
| 1375 | } | 1516 | } | ||
| 1376 | markSnapshotDirty(sess) | 1517 | markSnapshotDirty(sess) | ||
| 1377 | } | 1518 | } | ||
| 1378 | - | d.mu.Unlock() | |||
| 1379 | } | 1519 | } | ||
| 1380 | 1520 | ||||
| 1381 | // childEnv retains the daemon-environment helper for daemon-local legacy callers. | 1521 | // childEnv retains the daemon-environment helper for daemon-local legacy callers. | ||
| @@ -1389,11 +1529,8 @@ func copyEnvironment(env []string) []string { | |||||
| 1389 | } | 1529 | } | ||
| 1390 | 1530 | ||||
| 1391 | func environmentEntry(entry string) (name, value string, ok bool) { | 1531 | func environmentEntry(entry string) (name, value string, ok bool) { | ||
| 1392 | - | i := strings.IndexByte(entry, '=') | 1532 | + | name, value, ok = strings.Cut(entry, "=") |
| 1393 | - | if i < 0 { | 1533 | + | return name, value, ok |
| 1394 | - | return "", "", false | |||
| 1395 | - | } | |||
| 1396 | - | return entry[:i], entry[i+1:], true | |||
| 1397 | } | 1534 | } | ||
| 1398 | 1535 | ||||
| 1399 | func (d *Daemon) ptyCommand(env []string) (string, []string) { | 1536 | func (d *Daemon) ptyCommand(env []string) (string, []string) { | ||
M internal/usecase/daemon/session_purge_control_test.go +5 -20
| @@ -4,40 +4,25 @@ import ( | |||||
| 4 | "errors" | 4 | "errors" | ||
| 5 | "testing" | 5 | "testing" | ||
| 6 | 6 | ||||
| 7 | - | "github.com/stretchr/testify/mock" | |||
| 8 | "github.com/stretchr/testify/require" | 7 | "github.com/stretchr/testify/require" | ||
| 9 | 8 | ||||
| 10 | - | "github.com/bnema/vev/internal/ports" | 9 | + | "github.com/bnema/vev/internal/domain" |
| 11 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | 10 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | ||
| 12 | ) | 11 | ) | ||
| 13 | 12 | ||||
| 14 | - | func TestHandleKillStoppedUsesDurableBothSourcePurge(t *testing.T) { | |||
| 15 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 16 | - | repository := &retryablePurgeRepository{legacyErr: errors.New("legacy delete failed")} | |||
| 17 | - | WithSnapshotRepository(repository, repository)(d) | |||
| 18 | - | d.stopped["work"] = stoppedSession{name: "work"} | |||
| 19 | - | ||||
| 20 | - | transport := portsmocks.NewMockTransport(t) | |||
| 21 | - | transport.EXPECT().Send(mock.Anything).Return(nil).Once() | |||
| 22 | - | transport.EXPECT().Close().Return(nil).Once() | |||
| 23 | - | d.handleKill(transport, ports.Frame{Type: ports.MsgKill, Payload: ports.MarshalKill(ports.Kill{Name: "work"})}) | |||
| 24 | - | ||||
| 25 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy"}, repository.calls) | |||
| 26 | - | require.True(t, repository.tombstoned["work"]) | |||
| 27 | - | require.True(t, d.stopped["work"].purging) | |||
| 28 | - | } | |||
| 29 | - | ||||
| 30 | func TestStoppedPurgeMetadataFailureRemainsFencedForRetry(t *testing.T) { | 13 | func TestStoppedPurgeMetadataFailureRemainsFencedForRetry(t *testing.T) { | ||
| 31 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 14 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 32 | repository := &retryablePurgeRepository{} | 15 | repository := &retryablePurgeRepository{} | ||
| 33 | - | WithSnapshotRepository(repository, repository)(d) | 16 | + | WithSnapshotRepository(repository)(d) |
| 34 | store, state := newMockStore(t) | 17 | store, state := newMockStore(t) | ||
| 35 | WithStore(store)(d) | 18 | WithStore(store)(d) | ||
| 19 | + | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh} | |||
| 20 | + | require.NoError(t, d.catalogue.Create(record)) | |||
| 36 | metadataErr := errors.New("metadata delete failed") | 21 | metadataErr := errors.New("metadata delete failed") | ||
| 37 | state.mu.Lock() | 22 | state.mu.Lock() | ||
| 38 | state.deleteErr = func(string) error { return metadataErr } | 23 | state.deleteErr = func(string) error { return metadataErr } | ||
| 39 | state.mu.Unlock() | 24 | state.mu.Unlock() | ||
| 40 | - | d.stopped["work"] = stoppedSession{name: "work"} | 25 | + | d.stopped["work"] = stoppedSession{name: "work", incarnation: record.IncarnationID, record: record} |
| 41 | 26 | ||||
| 42 | require.ErrorIs(t, d.retryStoppedPurge("work"), metadataErr) | 27 | require.ErrorIs(t, d.retryStoppedPurge("work"), metadataErr) | ||
| 43 | require.True(t, repository.tombstoned["work"]) | 28 | require.True(t, repository.tombstoned["work"]) | ||
M internal/usecase/daemon/session_purge_retry_test.go +20 -4
| @@ -18,12 +18,15 @@ type retryablePurgeRepository struct { | |||||
| 18 | legacyErr error | 18 | legacyErr error | ||
| 19 | tombstoned map[string]bool | 19 | tombstoned map[string]bool | ||
| 20 | calls []string | 20 | calls []string | ||
| 21 | + | deadlineScoped []bool | |||
| 21 | } | 22 | } | ||
| 22 | 23 | ||||
| 23 | func (r *retryablePurgeRepository) Publish(context.Context, ports.SnapshotPublication) error { | 24 | func (r *retryablePurgeRepository) Publish(context.Context, ports.SnapshotPublication) error { | ||
| 24 | return nil | 25 | return nil | ||
| 25 | } | 26 | } | ||
| 26 | - | func (r *retryablePurgeRepository) WriteDeletionTombstone(_ context.Context, tombstone domain.DeletionTombstone) error { | 27 | + | func (r *retryablePurgeRepository) WriteDeletionTombstone(ctx context.Context, tombstone domain.DeletionTombstone) error { |
| 28 | + | _, hasDeadline := ctx.Deadline() | |||
| 29 | + | r.deadlineScoped = append(r.deadlineScoped, hasDeadline) | |||
| 27 | if r.tombstoned == nil { | 30 | if r.tombstoned == nil { | ||
| 28 | r.tombstoned = make(map[string]bool) | 31 | r.tombstoned = make(map[string]bool) | ||
| 29 | } | 32 | } | ||
| @@ -31,14 +34,18 @@ func (r *retryablePurgeRepository) WriteDeletionTombstone(_ context.Context, tom | |||||
| 31 | r.tombstoned[tombstone.Name] = true | 34 | r.tombstoned[tombstone.Name] = true | ||
| 32 | return nil | 35 | return nil | ||
| 33 | } | 36 | } | ||
| 34 | - | func (r *retryablePurgeRepository) QuarantineDeletionSources(_ context.Context, tombstone domain.DeletionTombstone, _ bool) error { | 37 | + | func (r *retryablePurgeRepository) QuarantineDeletionSources(ctx context.Context, tombstone domain.DeletionTombstone, _ bool) error { |
| 38 | + | _, hasDeadline := ctx.Deadline() | |||
| 39 | + | r.deadlineScoped = append(r.deadlineScoped, hasDeadline) | |||
| 35 | r.calls = append(r.calls, "incremental", "legacy") | 40 | r.calls = append(r.calls, "incremental", "legacy") | ||
| 36 | if r.incrementalErr != nil { | 41 | if r.incrementalErr != nil { | ||
| 37 | return r.incrementalErr | 42 | return r.incrementalErr | ||
| 38 | } | 43 | } | ||
| 39 | return r.legacyErr | 44 | return r.legacyErr | ||
| 40 | } | 45 | } | ||
| 41 | - | func (r *retryablePurgeRepository) DeleteDeletionTombstone(_ context.Context, _ domain.IncarnationID) error { | 46 | + | func (r *retryablePurgeRepository) DeleteDeletionTombstone(ctx context.Context, _ domain.IncarnationID) error { |
| 47 | + | _, hasDeadline := ctx.Deadline() | |||
| 48 | + | r.deadlineScoped = append(r.deadlineScoped, hasDeadline) | |||
| 42 | r.calls = append(r.calls, "clear tombstone") | 49 | r.calls = append(r.calls, "clear tombstone") | ||
| 43 | for name := range r.tombstoned { | 50 | for name := range r.tombstoned { | ||
| 44 | delete(r.tombstoned, name) | 51 | delete(r.tombstoned, name) | ||
| @@ -65,8 +72,13 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) { | |||||
| 65 | t.Run(tt.name, func(t *testing.T) { | 72 | t.Run(tt.name, func(t *testing.T) { | ||
| 66 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 73 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 67 | repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy} | 74 | repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy} | ||
| 68 | - | WithSnapshotRepository(repository, repository)(d) | 75 | + | WithSnapshotRepository(repository)(d) |
| 76 | + | store, _ := newMockStore(t) | |||
| 77 | + | WithStore(store)(d) | |||
| 69 | sess := newSnapshotTestSession(t, "work", false, "/work") | 78 | sess := newSnapshotTestSession(t, "work", false, "/work") | ||
| 79 | + | record := sess.persistRecordLocked(1) | |||
| 80 | + | record.RecoveryState = domain.RecoveryFresh | |||
| 81 | + | require.NoError(t, d.catalogue.Create(record)) | |||
| 70 | pty, ok := sess.tabs[0].panes["pane-1"].pty.(*portsmocks.MockPTY) | 82 | pty, ok := sess.tabs[0].panes["pane-1"].pty.(*portsmocks.MockPTY) | ||
| 71 | require.True(t, ok, "snapshot test pane must use MockPTY") | 83 | require.True(t, ok, "snapshot test pane must use MockPTY") | ||
| 72 | pty.EXPECT().Close().Return(nil).Once() | 84 | pty.EXPECT().Close().Return(nil).Once() | ||
| @@ -86,6 +98,10 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) { | |||||
| 86 | require.NoError(t, d.retryStoppedPurge("work")) | 98 | require.NoError(t, d.retryStoppedPurge("work")) | ||
| 87 | require.False(t, repository.tombstoned["work"]) | 99 | require.False(t, repository.tombstoned["work"]) | ||
| 88 | require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls) | 100 | require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls) | ||
| 101 | + | require.NotEmpty(t, repository.deadlineScoped) | |||
| 102 | + | for _, scoped := range repository.deadlineScoped { | |||
| 103 | + | require.True(t, scoped, "every purge repository call must receive a deadline-scoped context") | |||
| 104 | + | } | |||
| 89 | d.mu.Lock() | 105 | d.mu.Lock() | ||
| 90 | _, ok = d.stopped["work"] | 106 | _, ok = d.stopped["work"] | ||
| 91 | d.mu.Unlock() | 107 | d.mu.Unlock() | ||
M internal/usecase/daemon/session_rename_purge_test.go +26 -29
| @@ -1,43 +1,40 @@ | |||||
| 1 | package daemon | 1 | package daemon | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | + | "errors" | |||
| 4 | "testing" | 5 | "testing" | ||
| 5 | 6 | ||||
| 6 | "github.com/stretchr/testify/require" | 7 | "github.com/stretchr/testify/require" | ||
| 7 | 8 | ||||
| 8 | "github.com/bnema/vev/internal/domain" | 9 | "github.com/bnema/vev/internal/domain" | ||
| 10 | + | "github.com/bnema/vev/internal/persist" | |||
| 9 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | 11 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | ||
| 10 | ) | 12 | ) | ||
| 11 | 13 | ||||
| 12 | func TestRenamePreservesIncarnationSnapshotSources(t *testing.T) { | 14 | func TestRenamePreservesIncarnationSnapshotSources(t *testing.T) { | ||
| 13 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 15 | + | legacyDeleteErr := errors.New("legacy delete failed") |
| 14 | - | repository := &retryablePurgeRepository{} | 16 | + | for _, tt := range []struct { |
| 15 | - | WithSnapshotRepository(repository, repository)(d) | 17 | + | name string |
| 16 | - | store, state := newMockStore(t) | 18 | + | legacyErr error |
| 17 | - | WithStore(store)(d) | 19 | + | }{ |
| 18 | - | sess := newSnapshotTestSession(t, "old", false, "/work") | 20 | + | {name: "snapshot sources available"}, |
| 19 | - | d.sessions = map[domain.SessionID]*session{sess.id: sess} | 21 | + | {name: "unrelated legacy source delete would fail", legacyErr: legacyDeleteErr}, |
| 20 | - | require.NoError(t, d.persist.Save(sess.persistRecordLocked(1))) | 22 | + | } { |
| 23 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 24 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 25 | + | repository := &retryablePurgeRepository{legacyErr: tt.legacyErr} | |||
| 26 | + | WithSnapshotRepository(repository)(d) | |||
| 27 | + | store, state := newMockStore(t) | |||
| 28 | + | WithStore(store)(d) | |||
| 29 | + | sess := newSnapshotTestSession(t, "old", false, "/work") | |||
| 30 | + | d.sessions = map[domain.SessionID]*session{sess.id: sess} | |||
| 31 | + | require.NoError(t, d.catalogue.(*persist.Persister).Save(sess.persistRecordLocked(1))) | |||
| 21 | 32 | ||||
| 22 | - | require.NoError(t, d.renameSession(sess, "new")) | 33 | + | require.NoError(t, d.renameSession(sess, "new")) |
| 23 | - | require.Empty(t, repository.calls, "rename must not touch incarnation-keyed snapshot storage") | 34 | + | require.Empty(t, repository.calls, "rename must not invoke source deletion, including a failing legacy source") |
| 24 | - | require.False(t, state.has("old")) | 35 | + | require.False(t, state.has("old")) |
| 25 | - | require.True(t, state.has("new")) | 36 | + | require.True(t, state.has("new")) |
| 26 | - | require.Equal(t, sess.incarnation, state.record(t, "new").IncarnationID) | 37 | + | require.Equal(t, sess.incarnation, state.record(t, "new").IncarnationID) |
| 27 | - | } | 38 | + | }) |
| 28 | - | 39 | + | } | |
| 29 | - | func TestRenameIgnoresUnrelatedLegacySourceFailure(t *testing.T) { | |||
| 30 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 31 | - | repository := &retryablePurgeRepository{} | |||
| 32 | - | WithSnapshotRepository(repository, repository)(d) | |||
| 33 | - | store, state := newMockStore(t) | |||
| 34 | - | WithStore(store)(d) | |||
| 35 | - | sess := newSnapshotTestSession(t, "old", false, "/work") | |||
| 36 | - | d.sessions = map[domain.SessionID]*session{sess.id: sess} | |||
| 37 | - | require.NoError(t, d.persist.Save(sess.persistRecordLocked(1))) | |||
| 38 | - | ||||
| 39 | - | require.NoError(t, d.renameSession(sess, "new")) | |||
| 40 | - | require.Empty(t, repository.calls) | |||
| 41 | - | require.False(t, state.has("old")) | |||
| 42 | - | require.True(t, state.has("new")) | |||
| 43 | } | 40 | } | ||
M internal/usecase/daemon/session_test.go +54 -28
| @@ -806,11 +806,27 @@ func (s newStaticStore) Get(key []byte) ([]byte, bool) { | |||||
| 806 | func (s newStaticStore) Set(_, _ []byte) error { return nil } | 806 | func (s newStaticStore) Set(_, _ []byte) error { return nil } | ||
| 807 | func (s newStaticStore) Delete(_ []byte) error { return nil } | 807 | func (s newStaticStore) Delete(_ []byte) error { return nil } | ||
| 808 | func (s newStaticStore) Batch(changes []ports.StoreChange) error { | 808 | func (s newStaticStore) Batch(changes []ports.StoreChange) error { | ||
| 809 | - | for _, change := range changes { | 809 | + | if len(changes) == 0 { |
| 810 | + | return errors.New("empty batch") | |||
| 811 | + | } | |||
| 812 | + | prepared := make([]ports.StoreChange, len(changes)) | |||
| 813 | + | seen := make(map[string]struct{}, len(changes)) | |||
| 814 | + | for i, change := range changes { | |||
| 815 | + | key := string(change.Key) | |||
| 816 | + | if _, exists := seen[key]; exists { | |||
| 817 | + | return errors.New("duplicate batch key") | |||
| 818 | + | } | |||
| 819 | + | if change.Delete && len(change.Value) != 0 { | |||
| 820 | + | return errors.New("delete batch change has value") | |||
| 821 | + | } | |||
| 822 | + | seen[key] = struct{}{} | |||
| 823 | + | prepared[i] = ports.StoreChange{Key: append([]byte(nil), change.Key...), Value: append([]byte(nil), change.Value...), Delete: change.Delete} | |||
| 824 | + | } | |||
| 825 | + | for _, change := range prepared { | |||
| 810 | if change.Delete { | 826 | if change.Delete { | ||
| 811 | delete(s, string(change.Key)) | 827 | delete(s, string(change.Key)) | ||
| 812 | } else { | 828 | } else { | ||
| 813 | - | s[string(change.Key)] = append([]byte(nil), change.Value...) | 829 | + | s[string(change.Key)] = change.Value |
| 814 | } | 830 | } | ||
| 815 | } | 831 | } | ||
| 816 | return nil | 832 | return nil | ||
| @@ -841,19 +857,26 @@ func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) { | |||||
| 841 | store, _ := newMockStore(t) | 857 | store, _ := newMockStore(t) | ||
| 842 | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | 858 | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | ||
| 843 | incarnation := domain.IncarnationID{1} | 859 | incarnation := domain.IncarnationID{1} | ||
| 844 | - | require.NoError(t, seed.persist.Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh})) | 860 | + | require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh})) |
| 845 | 861 | ||||
| 846 | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | 862 | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | ||
| 847 | d.mu.Lock() | 863 | d.mu.Lock() | ||
| 848 | stopped := d.stopped["work"] | 864 | stopped := d.stopped["work"] | ||
| 849 | d.mu.Unlock() | 865 | d.mu.Unlock() | ||
| 850 | - | require.Equal(t, stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, incarnation: incarnation, lastUsedSeq: 9}, stopped) | 866 | + | require.Equal(t, "work", stopped.name) |
| 867 | + | require.Equal(t, "/tmp/work", stopped.cwd) | |||
| 868 | + | require.Equal(t, int64(7), stopped.createdAt) | |||
| 869 | + | require.Equal(t, incarnation, stopped.incarnation) | |||
| 870 | + | require.Equal(t, uint64(9), stopped.lastUsedSeq) | |||
| 871 | + | require.Equal(t, domain.RecoveryFresh, stopped.record.RecoveryState) | |||
| 872 | + | require.Equal(t, runtimeFresh, stopped.state) | |||
| 873 | + | require.NotNil(t, stopped.restoreDone) | |||
| 851 | require.Equal(t, uint64(9), d.mruSeq.Load()) | 874 | require.Equal(t, uint64(9), d.mruSeq.Load()) | ||
| 852 | } | 875 | } | ||
| 853 | 876 | ||||
| 854 | // TestDaemonNewWithoutPersistenceLogsNoLoadWarning covers the default, | 877 | // TestDaemonNewWithoutPersistenceLogsNoLoadWarning covers the default, | ||
| 855 | // persistence-free construction path (no WithStore/WithCatalogue option): | 878 | // persistence-free construction path (no WithStore/WithCatalogue option): | ||
| 856 | - | // d.persist is the persist.New(nil) sentinel, so LoadAll now returns | 879 | + | // d.catalogue.(*persist.Persister) is the persist.New(nil) sentinel, so LoadAll now returns |
| 857 | // errPersistenceUnavailable instead of an empty slice. That must not surface | 880 | // errPersistenceUnavailable instead of an empty slice. That must not surface | ||
| 858 | // as a "loading persisted sessions failed" warning, since nothing failed — | 881 | // as a "loading persisted sessions failed" warning, since nothing failed — | ||
| 859 | // persistence simply was never configured. | 882 | // persistence simply was never configured. | ||
| @@ -866,20 +889,6 @@ func TestDaemonNewWithoutPersistenceLogsNoLoadWarning(t *testing.T) { | |||||
| 866 | require.NotContains(t, logBuf.String(), "loading persisted sessions failed") | 889 | require.NotContains(t, logBuf.String(), "loading persisted sessions failed") | ||
| 867 | } | 890 | } | ||
| 868 | 891 | ||||
| 869 | - | // TestDaemonNewWithConfiguredStoreStillLogsLoadFailure is the inverse of the | |||
| 870 | - | // above: when persistence IS configured (a real store via WithStore) and the | |||
| 871 | - | // load genuinely fails (an undecodable catalogue record here), the warning | |||
| 872 | - | // must still fire. | |||
| 873 | - | func TestDaemonNewWithConfiguredStoreStillLogsLoadFailure(t *testing.T) { | |||
| 874 | - | store, state := newMockStore(t) | |||
| 875 | - | state.data["bad"] = []byte{0, 1} // not a valid encoded catalogue record | |||
| 876 | - | var logBuf strings.Builder | |||
| 877 | - | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(&logBuf, nil)), WithStore(store)) | |||
| 878 | - | require.NotNil(t, d) | |||
| 879 | - | require.True(t, d.persistEnabled) | |||
| 880 | - | require.Contains(t, logBuf.String(), "loading persisted sessions failed") | |||
| 881 | - | } | |||
| 882 | - | ||||
| 883 | func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) { | 892 | func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) { | ||
| 884 | d := &Daemon{} | 893 | d := &Daemon{} | ||
| 885 | sess := &session{} | 894 | sess := &session{} | ||
| @@ -930,7 +939,7 @@ func TestTouchMRUPersistsNamedButNotEphemeral(t *testing.T) { | |||||
| 930 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 939 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 931 | WithStore(store)(d) | 940 | WithStore(store)(d) | ||
| 932 | named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1, incarnation: domain.IncarnationID{1}} | 941 | named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1, incarnation: domain.IncarnationID{1}} | ||
| 933 | - | require.NoError(t, d.persist.Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) | 942 | + | require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) |
| 934 | 943 | ||||
| 935 | d.touchMRU(named) | 944 | d.touchMRU(named) | ||
| 936 | require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq) | 945 | require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq) | ||
| @@ -1003,7 +1012,7 @@ func TestRenameTabPersistsForNamedSession(t *testing.T) { | |||||
| 1003 | require.NoError(t, err) | 1012 | require.NoError(t, err) | ||
| 1004 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | 1013 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | ||
| 1005 | 1014 | ||||
| 1006 | - | records, err := d.persist.LoadAll() | 1015 | + | records, err := d.catalogue.(*persist.Persister).LoadAll() |
| 1007 | require.NoError(t, err) | 1016 | require.NoError(t, err) | ||
| 1008 | require.Len(t, records, 1) | 1017 | require.Len(t, records, 1) | ||
| 1009 | require.Equal(t, "work", records[0].Name) | 1018 | require.Equal(t, "work", records[0].Name) | ||
| @@ -1013,6 +1022,23 @@ func TestRenameTabPersistsForNamedSession(t *testing.T) { | |||||
| 1013 | require.Equal(t, []string{"shell"}, records[0].TabNames) | 1022 | require.Equal(t, []string{"shell"}, records[0].TabNames) | ||
| 1014 | } | 1023 | } | ||
| 1015 | 1024 | ||||
| 1025 | + | func TestRenameTabDoesNotHoldDaemonOrSessionMutexDuringCatalogueIO(t *testing.T) { | |||
| 1026 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 1027 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 1028 | + | d.sessions[sess.id] = sess | |||
| 1029 | + | catalogue := portsmocks.NewMockCatalogue(t) | |||
| 1030 | + | WithCatalogue(catalogue, nil)(d) | |||
| 1031 | + | catalogue.EXPECT().UpdateMetadata(mock.Anything).RunAndReturn(func(domain.CatalogueMetadataUpdate) error { | |||
| 1032 | + | require.True(t, d.mu.TryLock(), "daemon mutex held during catalogue I/O") | |||
| 1033 | + | d.mu.Unlock() | |||
| 1034 | + | require.True(t, sess.mu.TryLock(), "session mutex held during catalogue I/O") | |||
| 1035 | + | sess.mu.Unlock() | |||
| 1036 | + | return nil | |||
| 1037 | + | }).Once() | |||
| 1038 | + | ||||
| 1039 | + | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | |||
| 1040 | + | } | |||
| 1041 | + | ||||
| 1016 | func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | 1042 | func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | ||
| 1017 | tests := []struct { | 1043 | tests := []struct { | ||
| 1018 | name string | 1044 | name string | ||
| @@ -1045,7 +1071,7 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | |||||
| 1045 | } | 1071 | } | ||
| 1046 | 1072 | ||||
| 1047 | require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false)) | 1073 | require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false)) | ||
| 1048 | - | records, err := d.persist.LoadAll() | 1074 | + | records, err := d.catalogue.(*persist.Persister).LoadAll() |
| 1049 | require.NoError(t, err) | 1075 | require.NoError(t, err) | ||
| 1050 | require.Len(t, records, 1) | 1076 | require.Len(t, records, 1) | ||
| 1051 | require.Equal(t, tt.want, records[0].TabNames) | 1077 | require.Equal(t, tt.want, records[0].TabNames) | ||
| @@ -1090,7 +1116,7 @@ func TestRenameTabDoesNotPersistForEphemeralSession(t *testing.T) { | |||||
| 1090 | require.Equal(t, "shell", sess.tabs[0].name) | 1116 | require.Equal(t, "shell", sess.tabs[0].name) | ||
| 1091 | 1117 | ||||
| 1092 | require.False(t, state.has("0")) | 1118 | require.False(t, state.has("0")) | ||
| 1093 | - | records, err := d.persist.LoadAll() | 1119 | + | records, err := d.catalogue.(*persist.Persister).LoadAll() |
| 1094 | require.NoError(t, err) | 1120 | require.NoError(t, err) | ||
| 1095 | require.Empty(t, records) | 1121 | require.Empty(t, records) | ||
| 1096 | } | 1122 | } | ||
| @@ -1104,7 +1130,7 @@ func TestAttachRestoresPersistedTabNames(t *testing.T) { | |||||
| 1104 | store, _ := newMockStore(t) | 1130 | store, _ := newMockStore(t) | ||
| 1105 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | 1131 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | ||
| 1106 | WithStore(store)(d) | 1132 | WithStore(store)(d) | ||
| 1107 | - | require.NoError(t, d.persist.Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh})) | 1133 | + | require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh})) |
| 1108 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}} | 1134 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}} | ||
| 1109 | tr := portsmocks.NewMockTransport(t) | 1135 | tr := portsmocks.NewMockTransport(t) | ||
| 1110 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | 1136 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | ||
| @@ -1213,8 +1239,8 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T) | |||||
| 1213 | defer release() | 1239 | defer release() | ||
| 1214 | store := portsmocks.NewMockStore(t) | 1240 | store := portsmocks.NewMockStore(t) | ||
| 1215 | var attempted map[string][]byte | 1241 | var attempted map[string][]byte | ||
| 1216 | - | store.EXPECT().Range(mock.Anything).Run(func(func([]byte, []byte) bool) {}).Twice() | 1242 | + | store.EXPECT().Range(mock.Anything).Run(func(func([]byte, []byte) bool) {}).Times(3) |
| 1217 | - | store.EXPECT().Get([]byte("named")).Return(nil, false).Twice() | 1243 | + | store.EXPECT().Get([]byte("named")).Return(nil, false).Times(4) |
| 1218 | store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error { | 1244 | store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error { | ||
| 1219 | attempted = make(map[string][]byte, len(changes)) | 1245 | attempted = make(map[string][]byte, len(changes)) | ||
| 1220 | for _, change := range changes { | 1246 | for _, change := range changes { | ||
| @@ -1318,7 +1344,7 @@ func TestPickerStoppedTargetKillPurges(t *testing.T) { | |||||
| 1318 | store, state := newMockStore(t) | 1344 | store, state := newMockStore(t) | ||
| 1319 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 1345 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 1320 | WithStore(store)(d) | 1346 | WithStore(store)(d) | ||
| 1321 | - | require.NoError(t, d.persist.Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) | 1347 | + | require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) |
| 1322 | d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1} | 1348 | d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1} | ||
| 1323 | require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true})) | 1349 | require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true})) | ||
| 1324 | require.False(t, state.has("old")) | 1350 | require.False(t, state.has("old")) | ||
| @@ -1830,7 +1856,7 @@ func TestCatalogueRecordsConstructExpectedSessionRegistry(t *testing.T) { | |||||
| 1830 | func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) { | 1856 | func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) { | ||
| 1831 | store, _ := newMockStore(t) | 1857 | store, _ := newMockStore(t) | ||
| 1832 | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | 1858 | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | ||
| 1833 | - | require.NoError(t, seed.persist.Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh})) | 1859 | + | require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh})) |
| 1834 | 1860 | ||||
| 1835 | p, release := newBlockingPTY(t) | 1861 | p, release := newBlockingPTY(t) | ||
| 1836 | defer release() | 1862 | defer release() | ||
M internal/usecase/daemon/snapshot.go +6 -0
| @@ -47,6 +47,9 @@ type snapshotCapture struct { | |||||
| 47 | publicationContext context.Context | 47 | publicationContext context.Context | ||
| 48 | sealedRefs map[*vt.HistoryChunk]snapcodec.ObjectRef // set by the single encoder worker | 48 | sealedRefs map[*vt.HistoryChunk]snapcodec.ObjectRef // set by the single encoder worker | ||
| 49 | coordinatorDiscarded bool // guarded by session.snapshotMu | 49 | coordinatorDiscarded bool // guarded by session.snapshotMu | ||
| 50 | + | // normalWorkerAdmitted is set before snapshotJobs admission and immutable | |||
| 51 | + | // until completion; final-queue captures leave it false. | |||
| 52 | + | normalWorkerAdmitted bool | |||
| 50 | finishOnce sync.Once | 53 | finishOnce sync.Once | ||
| 51 | } | 54 | } | ||
| 52 | 55 | ||||
| @@ -98,3 +101,6 @@ const ( | |||||
| 98 | snapshotInterval = 2 * time.Minute | 101 | snapshotInterval = 2 * time.Minute | ||
| 99 | snapshotFinalFlushTimeout = time.Second | 102 | snapshotFinalFlushTimeout = time.Second | ||
| 100 | ) | 103 | ) | ||
| 104 | + | ||||
| 105 | + | // SnapshotShutdownTimeout reports the shared final-checkpoint shutdown budget. | |||
| 106 | + | func SnapshotShutdownTimeout() time.Duration { return snapshotFinalFlushTimeout } | |||
M internal/usecase/daemon/snapshot_coordinator.go +13 -0
| @@ -108,6 +108,14 @@ func (d *Daemon) waitForSnapshotGenerationWithDeadline(sess *session, generation | |||||
| 108 | return d.waitForSnapshotGeneration(sess, generation) | 108 | return d.waitForSnapshotGeneration(sess, generation) | ||
| 109 | } | 109 | } | ||
| 110 | for { | 110 | for { | ||
| 111 | + | // Once the shared deadline is observable, timeout wins even if publication | |||
| 112 | + | // completion becomes ready concurrently. This preserves the timed-out | |||
| 113 | + | // session identity and avoids nondeterministic success after the budget. | |||
| 114 | + | select { | |||
| 115 | + | case <-deadline.Done(): | |||
| 116 | + | return false | |||
| 117 | + | default: | |||
| 118 | + | } | |||
| 111 | sess.snapshotMu.Lock() | 119 | sess.snapshotMu.Lock() | ||
| 112 | published := sess.snapshotPublishedMutationRevision >= generation | 120 | published := sess.snapshotPublishedMutationRevision >= generation | ||
| 113 | changed := sess.snapshotChangeLocked() | 121 | changed := sess.snapshotChangeLocked() | ||
| @@ -207,6 +215,11 @@ func (d *Daemon) finishSnapshotCapture(capture *snapshotCapture, succeeded bool) | |||||
| 207 | capture.session.signalSnapshotChangedLocked() | 215 | capture.session.signalSnapshotChangedLocked() | ||
| 208 | wake := capture.session.snapshotWake | 216 | wake := capture.session.snapshotWake | ||
| 209 | capture.session.snapshotMu.Unlock() | 217 | capture.session.snapshotMu.Unlock() | ||
| 218 | + | if capture.normalWorkerAdmitted { | |||
| 219 | + | d.snapshotWorkerMu.Lock() | |||
| 220 | + | delete(d.snapshotAdmitted, capture) | |||
| 221 | + | d.snapshotWorkerMu.Unlock() | |||
| 222 | + | } | |||
| 210 | if wake != nil { | 223 | if wake != nil { | ||
| 211 | select { | 224 | select { | ||
| 212 | case wake <- struct{}{}: | 225 | case wake <- struct{}{}: | ||
M internal/usecase/daemon/snapshot_generation_test.go +2 -2
| @@ -17,7 +17,7 @@ import ( | |||||
| 17 | func TestSnapshotMultipleMutationsPublishNextRepositoryGeneration(t *testing.T) { | 17 | func TestSnapshotMultipleMutationsPublishNextRepositoryGeneration(t *testing.T) { | ||
| 18 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 18 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 19 | repository := portsmocks.NewMockSnapshotRepository(t) | 19 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 20 | - | WithSnapshotRepository(repository, nil)(d) | 20 | + | WithSnapshotRepository(repository)(d) |
| 21 | startSnapshotEncodeWorker(t, d) | 21 | startSnapshotEncodeWorker(t, d) | ||
| 22 | 22 | ||||
| 23 | publications := make(chan ports.SnapshotPublication, 1) | 23 | publications := make(chan ports.SnapshotPublication, 1) | ||
| @@ -44,7 +44,7 @@ func TestSnapshotMultipleMutationsPublishNextRepositoryGeneration(t *testing.T) | |||||
| 44 | func TestSnapshotRetryKeepsRepositoryGeneration(t *testing.T) { | 44 | func TestSnapshotRetryKeepsRepositoryGeneration(t *testing.T) { | ||
| 45 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 45 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 46 | repository := portsmocks.NewMockSnapshotRepository(t) | 46 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 47 | - | WithSnapshotRepository(repository, nil)(d) | 47 | + | WithSnapshotRepository(repository)(d) |
| 48 | startSnapshotEncodeWorker(t, d) | 48 | startSnapshotEncodeWorker(t, d) | ||
| 49 | 49 | ||||
| 50 | publications := make(chan ports.SnapshotPublication, 2) | 50 | publications := make(chan ports.SnapshotPublication, 2) | ||
D internal/usecase/daemon/snapshot_import_test.go +0 -163
| @@ -1,163 +0,0 @@ | |||||
| 1 | - | package daemon | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "encoding/base64" | |||
| 6 | - | "errors" | |||
| 7 | - | "os" | |||
| 8 | - | "path/filepath" | |||
| 9 | - | "testing" | |||
| 10 | - | ||||
| 11 | - | "github.com/stretchr/testify/require" | |||
| 12 | - | ||||
| 13 | - | "github.com/bnema/vev/internal/ports" | |||
| 14 | - | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |||
| 15 | - | ) | |||
| 16 | - | ||||
| 17 | - | type snapshotAcceptanceLegacySource struct { | |||
| 18 | - | blobs []ports.LegacySnapshot | |||
| 19 | - | loadCalls int | |||
| 20 | - | deleteCalls []string | |||
| 21 | - | deleteErr error | |||
| 22 | - | } | |||
| 23 | - | ||||
| 24 | - | func (s *snapshotAcceptanceLegacySource) LoadLegacy(ctx context.Context) ([]ports.LegacySnapshot, error) { | |||
| 25 | - | if err := ctx.Err(); err != nil { | |||
| 26 | - | return nil, err | |||
| 27 | - | } | |||
| 28 | - | s.loadCalls++ | |||
| 29 | - | out := make([]ports.LegacySnapshot, len(s.blobs)) | |||
| 30 | - | for i, blob := range s.blobs { | |||
| 31 | - | out[i] = ports.LegacySnapshot{Name: blob.Name, Data: append([]byte(nil), blob.Data...)} | |||
| 32 | - | } | |||
| 33 | - | return out, nil | |||
| 34 | - | } | |||
| 35 | - | ||||
| 36 | - | func (s *snapshotAcceptanceLegacySource) DeleteVerifiedLegacy(ctx context.Context, blob ports.LegacySnapshot) error { | |||
| 37 | - | return s.DeleteLegacy(ctx, blob.Name) | |||
| 38 | - | } | |||
| 39 | - | ||||
| 40 | - | func (s *snapshotAcceptanceLegacySource) DeleteLegacy(ctx context.Context, name string) error { | |||
| 41 | - | if err := ctx.Err(); err != nil { | |||
| 42 | - | return err | |||
| 43 | - | } | |||
| 44 | - | s.deleteCalls = append(s.deleteCalls, name) | |||
| 45 | - | if s.deleteErr != nil { | |||
| 46 | - | return s.deleteErr | |||
| 47 | - | } | |||
| 48 | - | for i, blob := range s.blobs { | |||
| 49 | - | if blob.Name == name { | |||
| 50 | - | s.blobs = append(s.blobs[:i], s.blobs[i+1:]...) | |||
| 51 | - | break | |||
| 52 | - | } | |||
| 53 | - | } | |||
| 54 | - | return nil | |||
| 55 | - | } | |||
| 56 | - | ||||
| 57 | - | func legacyAcceptanceSnapshot(t *testing.T) snapcodec.Session { | |||
| 58 | - | t.Helper() | |||
| 59 | - | data, err := os.ReadFile(filepath.Join("testdata", "legacy-v3-empty.b64")) | |||
| 60 | - | require.NoError(t, err) | |||
| 61 | - | encoded, err := base64.StdEncoding.DecodeString(string(data)) | |||
| 62 | - | require.NoError(t, err) | |||
| 63 | - | snapshot, err := snapcodec.Unmarshal(encoded) | |||
| 64 | - | require.NoError(t, err) | |||
| 65 | - | return snapshot | |||
| 66 | - | } | |||
| 67 | - | ||||
| 68 | - | func legacyAcceptanceBlob(t *testing.T, snapshot snapcodec.Session) ports.LegacySnapshot { | |||
| 69 | - | t.Helper() | |||
| 70 | - | fixture := legacyAcceptanceSnapshot(t) | |||
| 71 | - | require.Equal(t, fixture, snapshot, "legacy fixture must be the imported session") | |||
| 72 | - | data, err := os.ReadFile(filepath.Join("testdata", "legacy-v3-empty.b64")) | |||
| 73 | - | require.NoError(t, err) | |||
| 74 | - | encoded, err := base64.StdEncoding.DecodeString(string(data)) | |||
| 75 | - | require.NoError(t, err) | |||
| 76 | - | return ports.LegacySnapshot{Name: snapshot.Name, Data: encoded} | |||
| 77 | - | } | |||
| 78 | - | ||||
| 79 | - | func TestLegacyPublicationMarshalObjectErrorsIdentifyPaneAndStage(t *testing.T) { | |||
| 80 | - | for _, tt := range []struct { | |||
| 81 | - | name string | |||
| 82 | - | mutate func(*snapcodec.Pane) | |||
| 83 | - | stage string | |||
| 84 | - | }{ | |||
| 85 | - | {name: "sealed chunk", mutate: func(pane *snapcodec.Pane) { pane.SealedChunks = [][]byte{nil} }, stage: "sealed chunk 0"}, | |||
| 86 | - | {name: "tail", mutate: func(pane *snapcodec.Pane) { pane.Tail = nil }, stage: "tail"}, | |||
| 87 | - | {name: "visible", mutate: func(pane *snapcodec.Pane) { pane.Visible = nil }, stage: "visible"}, | |||
| 88 | - | } { | |||
| 89 | - | t.Run(tt.name, func(t *testing.T) { | |||
| 90 | - | pane := snapcodec.Pane{ID: "pane-1", StableID: "pane-stable", Tail: []byte("tail"), Visible: []byte("visible")} | |||
| 91 | - | tt.mutate(&pane) | |||
| 92 | - | snapshot := snapcodec.Session{Name: "legacy", Tabs: []snapcodec.Tab{{StableID: "tab-1", Panes: []snapcodec.Pane{pane}}}} | |||
| 93 | - | ||||
| 94 | - | _, err := legacyPublication(snapshot) | |||
| 95 | - | require.Error(t, err) | |||
| 96 | - | require.ErrorContains(t, err, `tab "tab-1"`) | |||
| 97 | - | require.ErrorContains(t, err, `pane "pane-1"`) | |||
| 98 | - | require.ErrorContains(t, err, tt.stage) | |||
| 99 | - | require.ErrorContains(t, err, "marshal object") | |||
| 100 | - | }) | |||
| 101 | - | } | |||
| 102 | - | } | |||
| 103 | - | ||||
| 104 | - | func TestImportLegacySnapshotsIsRetiredRuntimeNoop(t *testing.T) { | |||
| 105 | - | snapshot := legacyAcceptanceSnapshot(t) | |||
| 106 | - | repository := &snapshotAcceptanceRepository{generations: make(map[string]ports.SnapshotGeneration)} | |||
| 107 | - | legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}} | |||
| 108 | - | d := newTestDaemon(t, nil, stubClock{}) | |||
| 109 | - | WithSnapshotRepository(repository, legacy)(d) | |||
| 110 | - | ||||
| 111 | - | d.importLegacySnapshots(context.Background()) | |||
| 112 | - | ||||
| 113 | - | // Migration is adapter-owned. The daemon must never name-scan, publish, or | |||
| 114 | - | // delete a legacy source during normal recovery. | |||
| 115 | - | require.Empty(t, repository.publishes) | |||
| 116 | - | require.Empty(t, legacy.deleteCalls) | |||
| 117 | - | require.Len(t, legacy.blobs, 1) | |||
| 118 | - | } | |||
| 119 | - | ||||
| 120 | - | func TestImportLegacyExistingIncrementalSkipsLegacy(t *testing.T) { | |||
| 121 | - | snapshot := legacyAcceptanceSnapshot(t) | |||
| 122 | - | repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: make(map[string]ports.SnapshotGeneration)} | |||
| 123 | - | legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}} | |||
| 124 | - | d := newTestDaemon(t, nil, stubClock{}) | |||
| 125 | - | WithSnapshotRepository(repository, legacy)(d) | |||
| 126 | - | ||||
| 127 | - | d.importLegacySnapshots(context.Background()) | |||
| 128 | - | ||||
| 129 | - | require.Empty(t, repository.publishes) | |||
| 130 | - | require.Empty(t, legacy.deleteCalls) | |||
| 131 | - | require.Len(t, legacy.blobs, 1) | |||
| 132 | - | } | |||
| 133 | - | ||||
| 134 | - | func TestImportLegacyRetainsSourceUntilExactReloadVerification(t *testing.T) { | |||
| 135 | - | snapshot := legacyAcceptanceSnapshot(t) | |||
| 136 | - | for _, test := range []struct { | |||
| 137 | - | name string | |||
| 138 | - | publish error | |||
| 139 | - | load error | |||
| 140 | - | mutate func(ports.SnapshotGeneration, ports.SnapshotPublication) ports.SnapshotGeneration | |||
| 141 | - | }{ | |||
| 142 | - | {"publish failure", errors.New("publish failed"), nil, nil}, | |||
| 143 | - | {"reload failure", nil, errors.New("reload failed"), nil}, | |||
| 144 | - | {"manifest metadata mismatch", nil, nil, func(g ports.SnapshotGeneration, _ ports.SnapshotPublication) ports.SnapshotGeneration { | |||
| 145 | - | g.Generation++ | |||
| 146 | - | return g | |||
| 147 | - | }}, | |||
| 148 | - | {"manifest bytes mismatch", nil, nil, func(g ports.SnapshotGeneration, _ ports.SnapshotPublication) ports.SnapshotGeneration { | |||
| 149 | - | g.Manifest[len(g.Manifest)-1] ^= 1 | |||
| 150 | - | return g | |||
| 151 | - | }}, | |||
| 152 | - | } { | |||
| 153 | - | t.Run(test.name, func(t *testing.T) { | |||
| 154 | - | repository := &snapshotAcceptanceRepository{generations: make(map[string]ports.SnapshotGeneration), publishErr: test.publish, loadErr: test.load, loadMutate: test.mutate} | |||
| 155 | - | legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}} | |||
| 156 | - | d := newTestDaemon(t, nil, stubClock{}) | |||
| 157 | - | WithSnapshotRepository(repository, legacy)(d) | |||
| 158 | - | d.importLegacySnapshots(context.Background()) | |||
| 159 | - | require.Empty(t, legacy.deleteCalls) | |||
| 160 | - | require.Len(t, legacy.blobs, 1) | |||
| 161 | - | }) | |||
| 162 | - | } | |||
| 163 | - | } | |||
M internal/usecase/daemon/snapshot_queue.go +6 -0
| @@ -107,10 +107,16 @@ func (d *Daemon) enqueueSnapshotCapture(capture *snapshotCapture) bool { | |||||
| 107 | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | 107 | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | ||
| 108 | return false | 108 | return false | ||
| 109 | } | 109 | } | ||
| 110 | + | capture.normalWorkerAdmitted = true | |||
| 110 | select { | 111 | select { | ||
| 111 | case d.snapshotJobs <- capture: | 112 | case d.snapshotJobs <- capture: | ||
| 113 | + | if d.snapshotAdmitted == nil { | |||
| 114 | + | d.snapshotAdmitted = make(map[*snapshotCapture]struct{}) | |||
| 115 | + | } | |||
| 116 | + | d.snapshotAdmitted[capture] = struct{}{} | |||
| 112 | return true | 117 | return true | ||
| 113 | default: | 118 | default: | ||
| 119 | + | capture.normalWorkerAdmitted = false | |||
| 114 | return false | 120 | return false | ||
| 115 | } | 121 | } | ||
| 116 | } | 122 | } | ||
M internal/usecase/daemon/snapshot_restore.go +23 -6
| @@ -3,6 +3,7 @@ package daemon | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | "crypto/rand" | 5 | "crypto/rand" | ||
| 6 | + | "errors" | |||
| 6 | "fmt" | 7 | "fmt" | ||
| 7 | "math" | 8 | "math" | ||
| 8 | 9 | ||||
| @@ -13,6 +14,20 @@ import ( | |||||
| 13 | ) | 14 | ) | ||
| 14 | 15 | ||||
| 15 | func (d *Daemon) closeRestoreDone() { | 16 | func (d *Daemon) closeRestoreDone() { | ||
| 17 | + | d.mu.Lock() | |||
| 18 | + | for name, entry := range d.stopped { | |||
| 19 | + | if entry.state != runtimeRestoring { | |||
| 20 | + | continue | |||
| 21 | + | } | |||
| 22 | + | entry.state = runtimeDegraded | |||
| 23 | + | entry.record.RecoveryState = domain.RecoveryDegraded | |||
| 24 | + | if entry.record.DegradedReason == "" { | |||
| 25 | + | entry.record.DegradedReason = "restore unavailable" | |||
| 26 | + | } | |||
| 27 | + | d.stopped[name] = entry | |||
| 28 | + | closeRuntimeRestoreDoneLocked(entry.restoreDone) | |||
| 29 | + | } | |||
| 30 | + | d.mu.Unlock() | |||
| 16 | d.restoreOnce.Do(func() { close(d.restoreDone) }) | 31 | d.restoreOnce.Do(func() { close(d.restoreDone) }) | ||
| 17 | } | 32 | } | ||
| 18 | 33 | ||||
| @@ -293,13 +308,15 @@ func (d *Daemon) persistAndRegisterRestoredSession(ctx context.Context, sess *se | |||||
| 293 | } | 308 | } | ||
| 294 | } | 309 | } | ||
| 295 | } | 310 | } | ||
| 296 | - | // An existing catalogue checkpoint already authorizes this runtime. Only a | 311 | + | // Restoration is catalogue-authorized. Missing or unreadable authority is a |
| 297 | - | // legacy snapshot without a catalogue record creates fresh metadata here. | 312 | + | // hard failure; migration creates catalogue records before daemon startup. |
| 298 | if d.persistEnabled { | 313 | if d.persistEnabled { | ||
| 299 | - | if _, ok := d.catalogueRecord(sess.name); !ok { | 314 | + | _, ok, err := d.catalogueRecord(sess.name) |
| 300 | - | if err := d.createCatalogueRecord(sess.persistRecordLocked(sess.createdAt)); err != nil { | 315 | + | if err != nil { |
| 301 | - | return false, err | 316 | + | return false, err |
| 302 | - | } | 317 | + | } |
| 318 | + | if !ok { | |||
| 319 | + | return false, errors.New("snapshot: restored session is absent from catalogue") | |||
| 303 | } | 320 | } | ||
| 304 | } | 321 | } | ||
| 305 | d.mu.Lock() | 322 | d.mu.Lock() | ||
R internal/usecase/daemon/snapshot_legacy_import.go → internal/usecase/daemon/snapshot_restore_fixture_test.go +2 -3 renamed, 91% similar
| @@ -8,9 +8,8 @@ import ( | |||||
| 8 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | 8 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 9 | ) | 9 | ) | ||
| 10 | 10 | ||||
| 11 | - | func legacyPublication(snapshot snapcodec.Session) (ports.SnapshotPublication, error) { | 11 | + | func acceptancePublication(snapshot snapcodec.Session) (ports.SnapshotPublication, error) { |
| 12 | - | // Legacy payloads have no durable identity; this test-only conversion uses | 12 | + | // acceptancePublication converts a restore fixture to repository form. |
| 13 | - | // the fixture's stable synthetic identity. | |||
| 14 | incarnation := domain.IncarnationID{1} | 13 | incarnation := domain.IncarnationID{1} | ||
| 15 | manifest := snapcodec.Manifest{Generation: 1, IncarnationID: incarnation, Name: snapshot.Name, CreatedAt: snapshot.CreatedAt, Active: snapshot.Active, Tabs: make([]snapcodec.ManifestTab, 0, len(snapshot.Tabs))} | 14 | manifest := snapcodec.Manifest{Generation: 1, IncarnationID: incarnation, Name: snapshot.Name, CreatedAt: snapshot.CreatedAt, Active: snapshot.Active, Tabs: make([]snapcodec.ManifestTab, 0, len(snapshot.Tabs))} | ||
| 16 | objects := make([]ports.SnapshotObject, 0) | 15 | objects := make([]ports.SnapshotObject, 0) | ||
M internal/usecase/daemon/snapshot_restore_incremental.go +53 -10
| @@ -43,7 +43,12 @@ func (d *Daemon) restoreIncrementalSnapshots(ctx context.Context) { | |||||
| 43 | if d.catalogue == nil { | 43 | if d.catalogue == nil { | ||
| 44 | return | 44 | return | ||
| 45 | } | 45 | } | ||
| 46 | - | d.restoreCatalogue(ctx, d.catalogue.Records()) | 46 | + | records, err := d.catalogue.Records() |
| 47 | + | if err != nil { | |||
| 48 | + | d.log.Error("loading catalogue for snapshot restoration failed", "err", err) | |||
| 49 | + | return | |||
| 50 | + | } | |||
| 51 | + | d.restoreCatalogue(ctx, records) | |||
| 47 | } | 52 | } | ||
| 48 | 53 | ||||
| 49 | // restoreCatalogue gives every expected record an independent bounded job. | 54 | // restoreCatalogue gives every expected record an independent bounded job. | ||
| @@ -95,7 +100,9 @@ func (d *Daemon) recordRestoreDone(name string) chan struct{} { | |||||
| 95 | return d.stopped[name].restoreDone | 100 | return d.stopped[name].restoreDone | ||
| 96 | } | 101 | } | ||
| 97 | 102 | ||||
| 98 | - | func closeRuntimeRestoreDone(done chan struct{}) { | 103 | + | // closeRuntimeRestoreDoneLocked closes a per-record restoration barrier. |
| 104 | + | // Caller must hold d.mu so state transition and waiter release are atomic. | |||
| 105 | + | func closeRuntimeRestoreDoneLocked(done chan struct{}) { | |||
| 99 | if done == nil { | 106 | if done == nil { | ||
| 100 | return | 107 | return | ||
| 101 | } | 108 | } | ||
| @@ -125,8 +132,13 @@ func (d *Daemon) setStoppedRecovery(record domain.CatalogueRecord, state runtime | |||||
| 125 | 132 | ||||
| 126 | func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) { | 133 | func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) { | ||
| 127 | if restoreErr != nil { | 134 | if restoreErr != nil { | ||
| 135 | + | catalogueReadable := true | |||
| 128 | if d.catalogue != nil { | 136 | if d.catalogue != nil { | ||
| 129 | - | if current, ok := d.catalogue.Record(record.Name); ok && current.IncarnationID == record.IncarnationID { | 137 | + | current, ok, readErr := d.catalogue.Record(record.Name) |
| 138 | + | if readErr != nil { | |||
| 139 | + | catalogueReadable = false | |||
| 140 | + | d.log.Error("reading catalogue before degradation failed", "session", record.Name, "err", readErr) | |||
| 141 | + | } else if ok && current.IncarnationID == record.IncarnationID { | |||
| 130 | record = current | 142 | record = current | ||
| 131 | } | 143 | } | ||
| 132 | } | 144 | } | ||
| @@ -135,7 +147,7 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e | |||||
| 135 | if errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded) { | 147 | if errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded) { | ||
| 136 | record.DegradedReason = "restore interrupted" | 148 | record.DegradedReason = "restore interrupted" | ||
| 137 | } | 149 | } | ||
| 138 | - | if d.catalogue != nil { | 150 | + | if d.catalogue != nil && catalogueReadable { |
| 139 | if err := d.catalogue.Replace(record.Name, record); err != nil { | 151 | if err := d.catalogue.Replace(record.Name, record); err != nil { | ||
| 140 | d.log.Error("marking session degraded failed", "session", record.Name, "err", err) | 152 | d.log.Error("marking session degraded failed", "session", record.Name, "err", err) | ||
| 141 | } | 153 | } | ||
| @@ -155,7 +167,7 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e | |||||
| 155 | entry.tabNames = append([]string(nil), record.TabNames...) | 167 | entry.tabNames = append([]string(nil), record.TabNames...) | ||
| 156 | d.stopped[record.Name] = entry | 168 | d.stopped[record.Name] = entry | ||
| 157 | } | 169 | } | ||
| 158 | - | closeRuntimeRestoreDone(done) | 170 | + | closeRuntimeRestoreDoneLocked(done) |
| 159 | } | 171 | } | ||
| 160 | d.mu.Unlock() | 172 | d.mu.Unlock() | ||
| 161 | if restoreErr != nil { | 173 | if restoreErr != nil { | ||
| @@ -197,26 +209,42 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||||
| 197 | var selectedGeneration ports.SnapshotGeneration | 209 | var selectedGeneration ports.SnapshotGeneration | ||
| 198 | var selectedSnapshot snapcodec.Session | 210 | var selectedSnapshot snapcodec.Session | ||
| 199 | selectedIndex := -1 | 211 | selectedIndex := -1 | ||
| 212 | + | selectedFallback := false | |||
| 200 | for i, candidate := range candidates { | 213 | for i, candidate := range candidates { | ||
| 214 | + | fallback := record.Committed == nil || i > 0 | |||
| 201 | generation, err := d.snapshotRepository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, candidate) | 215 | generation, err := d.snapshotRepository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, candidate) | ||
| 202 | if err != nil { | 216 | if err != nil { | ||
| 203 | if ctxErr := ctx.Err(); ctxErr != nil { | 217 | if ctxErr := ctx.Err(); ctxErr != nil { | ||
| 204 | return ctxErr | 218 | return ctxErr | ||
| 205 | } | 219 | } | ||
| 220 | + | d.logRejectedRestoreCandidate(record, candidate, fallback, "load-failed", err) | |||
| 206 | continue | 221 | continue | ||
| 207 | } | 222 | } | ||
| 208 | - | snapshot, err := sessionFromGeneration(generation) | 223 | + | snapshot, decodeErr := sessionFromGeneration(generation) |
| 209 | - | if err != nil || generation.IncarnationID != record.IncarnationID || generation.Name != record.Name || generation.Generation != candidate.Generation { | 224 | + | if decodeErr != nil { |
| 225 | + | d.logRejectedRestoreCandidate(record, candidate, fallback, "decode-failed", decodeErr) | |||
| 210 | continue | 226 | continue | ||
| 211 | } | 227 | } | ||
| 212 | - | selected, selectedGeneration, selectedSnapshot, selectedIndex = candidate, generation, snapshot, i | 228 | + | if generation.IncarnationID != record.IncarnationID { |
| 229 | + | d.logRejectedRestoreCandidate(record, candidate, fallback, "incarnation-mismatch", nil) | |||
| 230 | + | continue | |||
| 231 | + | } | |||
| 232 | + | if generation.Name != record.Name { | |||
| 233 | + | d.logRejectedRestoreCandidate(record, candidate, fallback, "name-mismatch", nil) | |||
| 234 | + | continue | |||
| 235 | + | } | |||
| 236 | + | if generation.Generation != candidate.Generation { | |||
| 237 | + | d.logRejectedRestoreCandidate(record, candidate, fallback, "generation-mismatch", nil) | |||
| 238 | + | continue | |||
| 239 | + | } | |||
| 240 | + | selected, selectedGeneration, selectedSnapshot, selectedIndex, selectedFallback = candidate, generation, snapshot, i, fallback | |||
| 213 | break | 241 | break | ||
| 214 | } | 242 | } | ||
| 215 | if selectedIndex < 0 { | 243 | if selectedIndex < 0 { | ||
| 216 | return errors.New("snapshot: no catalogue checkpoint validated") | 244 | return errors.New("snapshot: no catalogue checkpoint validated") | ||
| 217 | } | 245 | } | ||
| 218 | 246 | ||||
| 219 | - | if selectedIndex > 0 { | 247 | + | if selectedFallback { |
| 220 | if d.checkpointRecovery == nil { | 248 | if d.checkpointRecovery == nil { | ||
| 221 | return errors.New("snapshot: checkpoint coordinator unavailable for fallback promotion") | 249 | return errors.New("snapshot: checkpoint coordinator unavailable for fallback promotion") | ||
| 222 | } | 250 | } | ||
| @@ -244,10 +272,25 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||||
| 244 | return err | 272 | return err | ||
| 245 | } | 273 | } | ||
| 246 | d.setStoppedRecovery(record, runtimeHealthy) | 274 | d.setStoppedRecovery(record, runtimeHealthy) | ||
| 247 | - | d.logSessionRestoreComplete(record, selected.Generation, selectedIndex > 0) | 275 | + | d.logSessionRestoreComplete(record, selected.Generation, selectedFallback) |
| 248 | return nil | 276 | return nil | ||
| 249 | } | 277 | } | ||
| 250 | 278 | ||||
| 279 | + | func (d *Daemon) logRejectedRestoreCandidate(record domain.CatalogueRecord, candidate domain.CheckpointRef, fallback bool, reason string, err error) { | |||
| 280 | + | kind := "committed" | |||
| 281 | + | if fallback { | |||
| 282 | + | kind = "fallback" | |||
| 283 | + | } | |||
| 284 | + | d.log.Warn("snapshot_restore_candidate_rejected", | |||
| 285 | + | "session", record.Name, | |||
| 286 | + | "incarnation", record.IncarnationID.String(), | |||
| 287 | + | "generation", candidate.Generation, | |||
| 288 | + | "candidate", kind, | |||
| 289 | + | "reason_code", reason, | |||
| 290 | + | "err", err, | |||
| 291 | + | ) | |||
| 292 | + | } | |||
| 293 | + | ||||
| 251 | func checkpointCandidates(record domain.CatalogueRecord) []domain.CheckpointRef { | 294 | func checkpointCandidates(record domain.CatalogueRecord) []domain.CheckpointRef { | ||
| 252 | candidates := make([]domain.CheckpointRef, 0, 3) | 295 | candidates := make([]domain.CheckpointRef, 0, 3) | ||
| 253 | if record.Committed != nil { | 296 | if record.Committed != nil { | ||
M internal/usecase/daemon/snapshot_restore_test.go +6 -7
| @@ -175,7 +175,7 @@ func restoreAcceptanceSession(t *testing.T, name string) snapcodec.Session { | |||||
| 175 | 175 | ||||
| 176 | func acceptanceGeneration(t *testing.T, snapshot snapcodec.Session, generation uint64) ports.SnapshotGeneration { | 176 | func acceptanceGeneration(t *testing.T, snapshot snapcodec.Session, generation uint64) ports.SnapshotGeneration { | ||
| 177 | t.Helper() | 177 | t.Helper() | ||
| 178 | - | publication, err := legacyPublication(snapshot) | 178 | + | publication, err := acceptancePublication(snapshot) |
| 179 | require.NoError(t, err) | 179 | require.NoError(t, err) | ||
| 180 | manifest, err := snapcodec.UnmarshalManifest(publication.Manifest) | 180 | manifest, err := snapcodec.UnmarshalManifest(publication.Manifest) | ||
| 181 | require.NoError(t, err) | 181 | require.NoError(t, err) | ||
| @@ -218,13 +218,12 @@ func TestRestoreIncrementalGenerationAcceptance(t *testing.T) { | |||||
| 218 | repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: map[string]ports.SnapshotGeneration{snapshot.Name: generation}} | 218 | repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: map[string]ports.SnapshotGeneration{snapshot.Name: generation}} | ||
| 219 | pty, release := newBlockingPTY(t) | 219 | pty, release := newBlockingPTY(t) | ||
| 220 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | 220 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | ||
| 221 | - | store, _ := newMockStore(t) | |||
| 222 | - | WithStore(store)(d) | |||
| 223 | checkpoint := domain.CheckpointRef{Generation: generation.Generation, ManifestDigest: snapcodec.ManifestDigest(generation.Manifest)} | 221 | checkpoint := domain.CheckpointRef{Generation: generation.Generation, ManifestDigest: snapcodec.ManifestDigest(generation.Manifest)} | ||
| 224 | record := domain.CatalogueRecord{Name: snapshot.Name, IncarnationID: generation.IncarnationID, Cwd: "/snapshot/cwd", CreatedAt: int64(snapshot.CreatedAt), RecoveryState: domain.RecoveryHealthy, Committed: &checkpoint} | 222 | record := domain.CatalogueRecord{Name: snapshot.Name, IncarnationID: generation.IncarnationID, Cwd: "/snapshot/cwd", CreatedAt: int64(snapshot.CreatedAt), RecoveryState: domain.RecoveryHealthy, Committed: &checkpoint} | ||
| 225 | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}) | 223 | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}) | ||
| 226 | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | 224 | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | ||
| 227 | - | WithSnapshotRepository(repository, nil)(d) | 225 | + | WithSnapshotRepository(repository)(d) |
| 226 | + | WithRecoveryCoordinator(recoveryusecase.NewCoordinator(catalogue, repository, nil, nil))(d) | |||
| 228 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | 227 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | ||
| 229 | 228 | ||||
| 230 | d.restoreIncrementalSnapshots(context.Background()) | 229 | d.restoreIncrementalSnapshots(context.Background()) | ||
| @@ -277,7 +276,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { | |||||
| 277 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | 276 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | ||
| 278 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | 277 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | ||
| 279 | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | 278 | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | ||
| 280 | - | WithSnapshotRepository(repository, nil)(d) | 279 | + | WithSnapshotRepository(repository)(d) |
| 281 | WithCheckpointCoordinator(coordinator)(d) | 280 | WithCheckpointCoordinator(coordinator)(d) | ||
| 282 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | 281 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | ||
| 283 | 282 | ||||
| @@ -291,7 +290,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { | |||||
| 291 | updates := catalogue.MetadataUpdates() | 290 | updates := catalogue.MetadataUpdates() | ||
| 292 | require.Len(t, updates, 1, "the authoritative Catalogue port must receive restored-session metadata updates") | 291 | require.Len(t, updates, 1, "the authoritative Catalogue port must receive restored-session metadata updates") | ||
| 293 | require.Equal(t, record.IncarnationID, updates[0].IncarnationID) | 292 | require.Equal(t, record.IncarnationID, updates[0].IncarnationID) | ||
| 294 | - | updated, ok := catalogue.Record(record.Name) | 293 | + | updated, ok, _ := catalogue.Record(record.Name) |
| 295 | require.True(t, ok) | 294 | require.True(t, ok) | ||
| 296 | require.Equal(t, record.IncarnationID, updated.IncarnationID) | 295 | require.Equal(t, record.IncarnationID, updated.IncarnationID) | ||
| 297 | require.Equal(t, record.RecoveryState, updated.RecoveryState) | 296 | require.Equal(t, record.RecoveryState, updated.RecoveryState) | ||
| @@ -310,7 +309,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { | |||||
| 310 | require.Empty(t, failure) | 309 | require.Empty(t, failure) | ||
| 311 | require.Len(t, repository.publishes, 1) | 310 | require.Len(t, repository.publishes, 1) | ||
| 312 | require.Equal(t, record.Committed, repository.publishes[0].ParentCheckpoint) | 311 | require.Equal(t, record.Committed, repository.publishes[0].ParentCheckpoint) | ||
| 313 | - | published, ok := catalogue.Record(record.Name) | 312 | + | published, ok, _ := catalogue.Record(record.Name) |
| 314 | require.True(t, ok) | 313 | require.True(t, ok) | ||
| 315 | require.Equal(t, uint64(10), published.Committed.Generation) | 314 | require.Equal(t, uint64(10), published.Committed.Generation) | ||
| 316 | require.Equal(t, committed, *published.Fallbacks[0]) | 315 | require.Equal(t, committed, *published.Fallbacks[0]) | ||
M internal/usecase/daemon/snapshot_scheduler_test.go +27 -21
| @@ -19,7 +19,7 @@ import ( | |||||
| 19 | func TestSnapshotCoordinatorQuarantineCancelsPublicationBeforeDelete(t *testing.T) { | 19 | func TestSnapshotCoordinatorQuarantineCancelsPublicationBeforeDelete(t *testing.T) { | ||
| 20 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 20 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 21 | repository := portsmocks.NewMockSnapshotRepository(t) | 21 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 22 | - | WithSnapshotRepository(repository, nil)(d) | 22 | + | WithSnapshotRepository(repository)(d) |
| 23 | startSnapshotEncodeWorker(t, d) | 23 | startSnapshotEncodeWorker(t, d) | ||
| 24 | 24 | ||||
| 25 | started := make(chan struct{}) | 25 | started := make(chan struct{}) | ||
| @@ -48,7 +48,7 @@ func TestSnapshotCoordinatorQuarantineCancelsPublicationBeforeDelete(t *testing. | |||||
| 48 | func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) { | 48 | func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) { | ||
| 49 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 49 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 50 | repository := portsmocks.NewMockSnapshotRepository(t) | 50 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 51 | - | WithSnapshotRepository(repository, nil)(d) | 51 | + | WithSnapshotRepository(repository)(d) |
| 52 | startSnapshotEncodeWorker(t, d) | 52 | startSnapshotEncodeWorker(t, d) | ||
| 53 | 53 | ||||
| 54 | started := make(chan struct{}) | 54 | started := make(chan struct{}) | ||
| @@ -84,7 +84,7 @@ func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) { | |||||
| 84 | func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) { | 84 | func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) { | ||
| 85 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 85 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 86 | repository := portsmocks.NewMockSnapshotRepository(t) | 86 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 87 | - | WithSnapshotRepository(repository, nil)(d) | 87 | + | WithSnapshotRepository(repository)(d) |
| 88 | startSnapshotEncodeWorker(t, d) | 88 | startSnapshotEncodeWorker(t, d) | ||
| 89 | 89 | ||||
| 90 | started := make(chan struct{}) | 90 | started := make(chan struct{}) | ||
| @@ -123,7 +123,7 @@ func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) { | |||||
| 123 | func TestForcedSnapshotSchedulesSuccessorForRoutineCaptureQueuedBehindWorker(t *testing.T) { | 123 | func TestForcedSnapshotSchedulesSuccessorForRoutineCaptureQueuedBehindWorker(t *testing.T) { | ||
| 124 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 124 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 125 | repository := portsmocks.NewMockSnapshotRepository(t) | 125 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 126 | - | WithSnapshotRepository(repository, nil)(d) | 126 | + | WithSnapshotRepository(repository)(d) |
| 127 | startSnapshotEncodeWorker(t, d) | 127 | startSnapshotEncodeWorker(t, d) | ||
| 128 | 128 | ||||
| 129 | blockerStarted := make(chan struct{}) | 129 | blockerStarted := make(chan struct{}) | ||
| @@ -164,7 +164,7 @@ func TestForcedSnapshotSchedulesSuccessorForRoutineCaptureQueuedBehindWorker(t * | |||||
| 164 | func TestMultipleForcedSnapshotsCoalesceToOneSuccessor(t *testing.T) { | 164 | func TestMultipleForcedSnapshotsCoalesceToOneSuccessor(t *testing.T) { | ||
| 165 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 165 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 166 | repository := portsmocks.NewMockSnapshotRepository(t) | 166 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 167 | - | WithSnapshotRepository(repository, nil)(d) | 167 | + | WithSnapshotRepository(repository)(d) |
| 168 | startSnapshotEncodeWorker(t, d) | 168 | startSnapshotEncodeWorker(t, d) | ||
| 169 | 169 | ||||
| 170 | started := make(chan struct{}) | 170 | started := make(chan struct{}) | ||
| @@ -207,7 +207,7 @@ func TestForcedSnapshotShutdownTimeoutRetainsRetryableStateAndNotice(t *testing. | |||||
| 207 | clock := newSnapshotDeadlineClock() | 207 | clock := newSnapshotDeadlineClock() | ||
| 208 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 208 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 209 | repository := portsmocks.NewMockSnapshotRepository(t) | 209 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 210 | - | WithSnapshotRepository(repository, nil)(d) | 210 | + | WithSnapshotRepository(repository)(d) |
| 211 | notices := portsmocks.NewMockNoticeStore(t) | 211 | notices := portsmocks.NewMockNoticeStore(t) | ||
| 212 | WithNoticeStore(notices)(d) | 212 | WithNoticeStore(notices)(d) | ||
| 213 | startSnapshotEncodeWorker(t, d) | 213 | startSnapshotEncodeWorker(t, d) | ||
| @@ -290,7 +290,7 @@ func TestServeShutdownDeadlineStillJoinsPaneExitWriter(t *testing.T) { | |||||
| 290 | clock := newServeShutdownClock() | 290 | clock := newServeShutdownClock() | ||
| 291 | d := newTestDaemon(t, newFactory(t, pty), clock) | 291 | d := newTestDaemon(t, newFactory(t, pty), clock) | ||
| 292 | repository := portsmocks.NewMockSnapshotRepository(t) | 292 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 293 | - | WithSnapshotRepository(repository, nil)(d) | 293 | + | WithSnapshotRepository(repository)(d) |
| 294 | 294 | ||||
| 295 | publicationStarted := make(chan struct{}) | 295 | publicationStarted := make(chan struct{}) | ||
| 296 | releasePublication := make(chan struct{}) | 296 | releasePublication := make(chan struct{}) | ||
| @@ -305,7 +305,7 @@ func TestServeShutdownDeadlineStillJoinsPaneExitWriter(t *testing.T) { | |||||
| 305 | 305 | ||||
| 306 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | 306 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | ||
| 307 | listener := serveSnapshotListener(t, tr) | 307 | listener := serveSnapshotListener(t, tr) | ||
| 308 | - | ctx, cancel := context.WithCancel(context.Background()) | 308 | + | ctx, cancel := context.WithCancel(t.Context()) |
| 309 | defer cancel() | 309 | defer cancel() | ||
| 310 | served := make(chan error, 1) | 310 | served := make(chan error, 1) | ||
| 311 | go func() { served <- d.Serve(ctx, listener) }() | 311 | go func() { served <- d.Serve(ctx, listener) }() | ||
| @@ -382,7 +382,7 @@ func awaitTeardownWaiters(t *testing.T, sess *session, want uint) { | |||||
| 382 | func TestConcurrentSessionTeardownPublishesOneFinalSnapshot(t *testing.T) { | 382 | func TestConcurrentSessionTeardownPublishesOneFinalSnapshot(t *testing.T) { | ||
| 383 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 383 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 384 | repository := portsmocks.NewMockSnapshotRepository(t) | 384 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 385 | - | WithSnapshotRepository(repository, nil)(d) | 385 | + | WithSnapshotRepository(repository)(d) |
| 386 | startSnapshotEncodeWorker(t, d) | 386 | startSnapshotEncodeWorker(t, d) | ||
| 387 | 387 | ||||
| 388 | publicationStarted := make(chan struct{}) | 388 | publicationStarted := make(chan struct{}) | ||
| @@ -456,7 +456,7 @@ func TestServeShutdownCheckpointsBeforeStoppingSnapshotWorker(t *testing.T) { | |||||
| 456 | defer releasePTY() | 456 | defer releasePTY() | ||
| 457 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 457 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 458 | repository := portsmocks.NewMockSnapshotRepository(t) | 458 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 459 | - | WithSnapshotRepository(repository, nil)(d) | 459 | + | WithSnapshotRepository(repository)(d) |
| 460 | 460 | ||||
| 461 | published := make(chan ports.SnapshotPublication, 2) | 461 | published := make(chan ports.SnapshotPublication, 2) | ||
| 462 | allowTerminalPublication := make(chan struct{}) | 462 | allowTerminalPublication := make(chan struct{}) | ||
| @@ -470,7 +470,7 @@ func TestServeShutdownCheckpointsBeforeStoppingSnapshotWorker(t *testing.T) { | |||||
| 470 | 470 | ||||
| 471 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | 471 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | ||
| 472 | l := serveSnapshotListener(t, tr) | 472 | l := serveSnapshotListener(t, tr) | ||
| 473 | - | ctx, cancel := context.WithCancel(context.Background()) | 473 | + | ctx, cancel := context.WithCancel(t.Context()) |
| 474 | defer cancel() | 474 | defer cancel() | ||
| 475 | served := make(chan error, 1) | 475 | served := make(chan error, 1) | ||
| 476 | go func() { served <- d.Serve(ctx, l) }() | 476 | go func() { served <- d.Serve(ctx, l) }() | ||
| @@ -500,14 +500,15 @@ func TestServeShutdownDeadlineStillJoinsUncooperativeSnapshotRepository(t *testi | |||||
| 500 | clock := newServeShutdownClock() | 500 | clock := newServeShutdownClock() | ||
| 501 | d := newTestDaemon(t, newFactory(t, p), clock) | 501 | d := newTestDaemon(t, newFactory(t, p), clock) | ||
| 502 | repository := portsmocks.NewMockSnapshotRepository(t) | 502 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 503 | - | WithSnapshotRepository(repository, nil)(d) | 503 | + | WithSnapshotRepository(repository)(d) |
| 504 | noticeStore := portsmocks.NewMockNoticeStore(t) | 504 | noticeStore := portsmocks.NewMockNoticeStore(t) | ||
| 505 | WithNoticeStore(noticeStore)(d) | 505 | WithNoticeStore(noticeStore)(d) | ||
| 506 | noticeStore.EXPECT().Claim().Return(nil, nil).Maybe() | 506 | noticeStore.EXPECT().Claim().Return(nil, nil).Maybe() | ||
| 507 | noticeStore.EXPECT().Ack().Return(nil).Maybe() | 507 | noticeStore.EXPECT().Ack().Return(nil).Maybe() | ||
| 508 | + | noticePersisted := make(chan struct{}) | |||
| 508 | noticeStore.EXPECT().Append(mock.MatchedBy(func(n domain.Notification) bool { | 509 | noticeStore.EXPECT().Append(mock.MatchedBy(func(n domain.Notification) bool { | ||
| 509 | return n.Code == domain.NoticeSnapshotWrite && n.Severity == domain.NoticeError | 510 | return n.Code == domain.NoticeSnapshotWrite && n.Severity == domain.NoticeError | ||
| 510 | - | })).Return(nil).Once() | 511 | + | })).Run(func(domain.Notification) { close(noticePersisted) }).Return(nil).Once() |
| 511 | 512 | ||||
| 512 | firstPublicationCompleted := make(chan struct{}) | 513 | firstPublicationCompleted := make(chan struct{}) | ||
| 513 | terminalPublicationStarted := make(chan struct{}) | 514 | terminalPublicationStarted := make(chan struct{}) | ||
| @@ -526,7 +527,7 @@ func TestServeShutdownDeadlineStillJoinsUncooperativeSnapshotRepository(t *testi | |||||
| 526 | 527 | ||||
| 527 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | 528 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | ||
| 528 | l := serveSnapshotListener(t, tr) | 529 | l := serveSnapshotListener(t, tr) | ||
| 529 | - | ctx, cancel := context.WithCancel(context.Background()) | 530 | + | ctx, cancel := context.WithCancel(t.Context()) |
| 530 | defer cancel() | 531 | defer cancel() | ||
| 531 | served := make(chan error, 1) | 532 | served := make(chan error, 1) | ||
| 532 | go func() { served <- d.Serve(ctx, l) }() | 533 | go func() { served <- d.Serve(ctx, l) }() | ||
| @@ -551,6 +552,11 @@ func TestServeShutdownDeadlineStillJoinsUncooperativeSnapshotRepository(t *testi | |||||
| 551 | d.snapshotWorkerMu.Unlock() | 552 | d.snapshotWorkerMu.Unlock() | ||
| 552 | require.NotNil(t, workerDone) | 553 | require.NotNil(t, workerDone) | ||
| 553 | clock.nextFinalTimer(t).fire() | 554 | clock.nextFinalTimer(t).fire() | ||
| 555 | + | select { | |||
| 556 | + | case <-noticePersisted: | |||
| 557 | + | case <-time.After(testWaitTimeout): | |||
| 558 | + | t.Fatal("shutdown timeout notice was not persisted before the writer join") | |||
| 559 | + | } | |||
| 554 | 560 | ||||
| 555 | select { | 561 | select { | ||
| 556 | case err := <-served: | 562 | case err := <-served: | ||
| @@ -632,7 +638,7 @@ func TestRoutineSnapshotEligibilityStartsAtCompletionAndForcedDoesNotMoveIt(t *t | |||||
| 632 | clock := &snapshotSchedulerClock{now: time.Unix(100, 0)} | 638 | clock := &snapshotSchedulerClock{now: time.Unix(100, 0)} | ||
| 633 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 639 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 634 | repository := portsmocks.NewMockSnapshotRepository(t) | 640 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 635 | - | WithSnapshotRepository(repository, nil)(d) | 641 | + | WithSnapshotRepository(repository)(d) |
| 636 | startSnapshotEncodeWorker(t, d) | 642 | startSnapshotEncodeWorker(t, d) | ||
| 637 | 643 | ||||
| 638 | var publishes atomic.Int32 | 644 | var publishes atomic.Int32 | ||
| @@ -780,7 +786,7 @@ func TestSnapshotCompletionSchedulesRoutineRetriesFromCompletion(t *testing.T) { | |||||
| 780 | clock := newDeterministicSnapshotClock(now) | 786 | clock := newDeterministicSnapshotClock(now) | ||
| 781 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 787 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 782 | repository := portsmocks.NewMockSnapshotRepository(t) | 788 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 783 | - | WithSnapshotRepository(repository, nil)(d) | 789 | + | WithSnapshotRepository(repository)(d) |
| 784 | 790 | ||||
| 785 | for _, tt := range []struct { | 791 | for _, tt := range []struct { | ||
| 786 | name string | 792 | name string | ||
| @@ -813,12 +819,12 @@ func TestSnapshotSchedulerImmediateEligibilityAndStaleCapturesRemainDirty(t *tes | |||||
| 813 | now := time.Unix(100, 0) | 819 | now := time.Unix(100, 0) | ||
| 814 | clock := newDeterministicSnapshotClock(now) | 820 | clock := newDeterministicSnapshotClock(now) | ||
| 815 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 821 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 816 | - | WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t), nil)(d) | 822 | + | WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t))(d) |
| 817 | 823 | ||||
| 818 | t.Run("no prior attempt is immediately eligible", func(t *testing.T) { | 824 | t.Run("no prior attempt is immediately eligible", func(t *testing.T) { | ||
| 819 | published := make(chan ports.SnapshotPublication, 1) | 825 | published := make(chan ports.SnapshotPublication, 1) | ||
| 820 | repository := portsmocks.NewMockSnapshotRepository(t) | 826 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 821 | - | WithSnapshotRepository(repository, nil)(d) | 827 | + | WithSnapshotRepository(repository)(d) |
| 822 | startSnapshotEncodeWorker(t, d) | 828 | startSnapshotEncodeWorker(t, d) | ||
| 823 | repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error { | 829 | repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error { | ||
| 824 | published <- publication | 830 | published <- publication | ||
| @@ -873,7 +879,7 @@ func TestSnapshotWorkerQueueIsBoundedAndForcedCapturesCoalesce(t *testing.T) { | |||||
| 873 | clock := newDeterministicSnapshotClock(now) | 879 | clock := newDeterministicSnapshotClock(now) | ||
| 874 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 880 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 875 | repository := portsmocks.NewMockSnapshotRepository(t) | 881 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 876 | - | WithSnapshotRepository(repository, nil)(d) | 882 | + | WithSnapshotRepository(repository)(d) |
| 877 | startSnapshotEncodeWorker(t, d) | 883 | startSnapshotEncodeWorker(t, d) | ||
| 878 | 884 | ||||
| 879 | started := make(chan struct{}) | 885 | started := make(chan struct{}) | ||
| @@ -931,7 +937,7 @@ func TestSnapshotRepositorySaverUsesEarliestDeadlineAndRecomputes(t *testing.T) | |||||
| 931 | clock := newDeterministicSnapshotClock(now) | 937 | clock := newDeterministicSnapshotClock(now) | ||
| 932 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 938 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 933 | repository := portsmocks.NewMockSnapshotRepository(t) | 939 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 934 | - | WithSnapshotRepository(repository, nil)(d) | 940 | + | WithSnapshotRepository(repository)(d) |
| 935 | startSnapshotEncodeWorker(t, d) | 941 | startSnapshotEncodeWorker(t, d) | ||
| 936 | 942 | ||||
| 937 | published := make(chan ports.SnapshotPublication, 1) | 943 | published := make(chan ports.SnapshotPublication, 1) | ||
| @@ -946,7 +952,7 @@ func TestSnapshotRepositorySaverUsesEarliestDeadlineAndRecomputes(t *testing.T) | |||||
| 946 | later.snapshotNextEligibleAt = now.Add(10 * time.Minute) | 952 | later.snapshotNextEligibleAt = now.Add(10 * time.Minute) | ||
| 947 | d.sessions[later.id] = later | 953 | d.sessions[later.id] = later | ||
| 948 | 954 | ||||
| 949 | - | ctx, cancel := context.WithCancel(context.Background()) | 955 | + | ctx, cancel := context.WithCancel(t.Context()) |
| 950 | done := make(chan struct{}) | 956 | done := make(chan struct{}) | ||
| 951 | go func() { | 957 | go func() { | ||
| 952 | d.snapshotRepositorySaver(ctx) | 958 | d.snapshotRepositorySaver(ctx) | ||
M internal/usecase/daemon/snapshot_test.go +39 -1
| @@ -17,10 +17,45 @@ import ( | |||||
| 17 | 17 | ||||
| 18 | // TestSnapshotWorkerPublishesContentAddressedCapture verifies that all new | 18 | // TestSnapshotWorkerPublishesContentAddressedCapture verifies that all new | ||
| 19 | // checkpoints use the repository publication contract. | 19 | // checkpoints use the repository publication contract. | ||
| 20 | + | func TestWithSnapshotRepositoryRejectsTypedNil(t *testing.T) { | |||
| 21 | + | var repository *snapshotAcceptanceRepository | |||
| 22 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 23 | + | ||||
| 24 | + | WithSnapshotRepository(repository)(d) | |||
| 25 | + | ||||
| 26 | + | require.Nil(t, d.snapshotRepository) | |||
| 27 | + | require.False(t, d.snapsEnabled) | |||
| 28 | + | } | |||
| 29 | + | ||||
| 30 | + | func TestDurableWriterFailureNamesIncludesBufferedCapture(t *testing.T) { | |||
| 31 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 32 | + | capture := &snapshotCapture{name: "work", session: newSnapshotTestSession(t, "work", false, "/work")} | |||
| 33 | + | d.snapshotWorkerMu.Lock() | |||
| 34 | + | d.snapshotAdmitted[capture] = struct{}{} | |||
| 35 | + | d.snapshotJobs <- capture | |||
| 36 | + | d.snapshotWorkerMu.Unlock() | |||
| 37 | + | ||||
| 38 | + | require.Equal(t, []string{"work"}, d.durableWriterFailureNames()) | |||
| 39 | + | } | |||
| 40 | + | ||||
| 41 | + | func TestCheckpointPublicationWithoutCommittedRefFailsCapture(t *testing.T) { | |||
| 42 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 43 | + | WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t))(d) | |||
| 44 | + | coordinator := portsmocks.NewMockCheckpointCoordinator(t) | |||
| 45 | + | WithCheckpointCoordinator(coordinator)(d) | |||
| 46 | + | startSnapshotEncodeWorker(t, d) | |||
| 47 | + | coordinator.EXPECT().PublishCheckpoint(mock.Anything, "work", mock.Anything).Return(domain.CatalogueRecord{Name: "work"}, nil).Once() | |||
| 48 | + | ||||
| 49 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 50 | + | require.True(t, d.captureSession(sess)) | |||
| 51 | + | awaitSnapshotIdle(t, sess) | |||
| 52 | + | require.True(t, sess.snapDirty.Load(), "nil committed publication must remain retryable") | |||
| 53 | + | } | |||
| 54 | + | ||||
| 20 | func TestSnapshotWorkerPublishesContentAddressedCapture(t *testing.T) { | 55 | func TestSnapshotWorkerPublishesContentAddressedCapture(t *testing.T) { | ||
| 21 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 56 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 22 | repository := portsmocks.NewMockSnapshotRepository(t) | 57 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 23 | - | WithSnapshotRepository(repository, nil)(d) | 58 | + | WithSnapshotRepository(repository)(d) |
| 24 | startSnapshotEncodeWorker(t, d) | 59 | startSnapshotEncodeWorker(t, d) | ||
| 25 | 60 | ||||
| 26 | repository.EXPECT().Publish(mock.Anything, mock.MatchedBy(func(p ports.SnapshotPublication) bool { | 61 | repository.EXPECT().Publish(mock.Anything, mock.MatchedBy(func(p ports.SnapshotPublication) bool { | ||
| @@ -57,6 +92,9 @@ func newSnapshotTestSession(t *testing.T, name string, ephemeral bool, cwd strin | |||||
| 57 | // focused test sinks that only need to observe publication or deletion calls. | 92 | // focused test sinks that only need to observe publication or deletion calls. | ||
| 58 | type noOpSnapshotRepository struct{} | 93 | type noOpSnapshotRepository struct{} | ||
| 59 | 94 | ||||
| 95 | + | var _ ports.SnapshotRepository = noOpSnapshotRepository{} | |||
| 96 | + | ||||
| 97 | + | func (noOpSnapshotRepository) Publish(context.Context, ports.SnapshotPublication) error { return nil } | |||
| 60 | func (noOpSnapshotRepository) LoadCheckpoint(context.Context, domain.IncarnationID, string, ports.CheckpointRef) (ports.SnapshotGeneration, error) { | 98 | func (noOpSnapshotRepository) LoadCheckpoint(context.Context, domain.IncarnationID, string, ports.CheckpointRef) (ports.SnapshotGeneration, error) { | ||
| 61 | return ports.SnapshotGeneration{}, errors.New("unused") | 99 | return ports.SnapshotGeneration{}, errors.New("unused") | ||
| 62 | } | 100 | } | ||
M internal/usecase/daemon/snapshot_worker.go +18 -3
| @@ -2,6 +2,7 @@ package daemon | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | + | "errors" | |||
| 5 | "sort" | 6 | "sort" | ||
| 6 | 7 | ||||
| 7 | "github.com/bnema/vev/internal/domain" | 8 | "github.com/bnema/vev/internal/domain" | ||
| @@ -134,8 +135,12 @@ func (d *Daemon) publishSnapshotCapture(workerCtx context.Context, workerID uint | |||||
| 134 | if d.checkpointRecovery != nil { | 135 | if d.checkpointRecovery != nil { | ||
| 135 | var record domain.CatalogueRecord | 136 | var record domain.CatalogueRecord | ||
| 136 | record, err = d.checkpointRecovery.PublishCheckpoint(publicationContext, capture.name, publication) | 137 | record, err = d.checkpointRecovery.PublishCheckpoint(publicationContext, capture.name, publication) | ||
| 137 | - | if err == nil && record.Committed != nil { | 138 | + | if err == nil { |
| 138 | - | capture.checkpoint = *record.Committed | 139 | + | if record.Committed == nil { |
| 140 | + | err = errors.New("snapshot: checkpoint publication returned no committed checkpoint") | |||
| 141 | + | } else { | |||
| 142 | + | capture.checkpoint = *record.Committed | |||
| 143 | + | } | |||
| 139 | } | 144 | } | ||
| 140 | } else { | 145 | } else { | ||
| 141 | err = d.snapshotRepository.Publish(publicationContext, publication) | 146 | err = d.snapshotRepository.Publish(publicationContext, publication) | ||
| @@ -217,8 +222,12 @@ func (d *Daemon) StopDurableWriters(ctx context.Context) []string { | |||||
| 217 | case <-done: | 222 | case <-done: | ||
| 218 | return nil | 223 | return nil | ||
| 219 | case <-ctx.Done(): | 224 | case <-ctx.Done(): | ||
| 225 | + | // Snapshot identity before cancellation can let a cooperative worker clear | |||
| 226 | + | // its in-flight capture. The caller persists timeout notices before the | |||
| 227 | + | // unconditional ownership join. | |||
| 228 | + | names := d.durableWriterFailureNames() | |||
| 220 | cancel() | 229 | cancel() | ||
| 221 | - | return d.durableWriterFailureNames() | 230 | + | return names |
| 222 | } | 231 | } | ||
| 223 | } | 232 | } | ||
| 224 | 233 | ||||
| @@ -267,6 +276,9 @@ func (d *Daemon) requestDurableWriterStop() (context.CancelFunc, <-chan struct{} | |||||
| 267 | return cancel, d.snapshotWorkerDone | 276 | return cancel, d.snapshotWorkerDone | ||
| 268 | } | 277 | } | ||
| 269 | 278 | ||||
| 279 | + | // durableWriterFailureNames includes admitted buffered captures as well as the | |||
| 280 | + | // active and final queues. snapshotAdmitted tracks normal captures from queue | |||
| 281 | + | // admission through completion, so worker dequeue cannot make one disappear. | |||
| 270 | func (d *Daemon) durableWriterFailureNames() []string { | 282 | func (d *Daemon) durableWriterFailureNames() []string { | ||
| 271 | d.snapshotWorkerMu.Lock() | 283 | d.snapshotWorkerMu.Lock() | ||
| 272 | defer d.snapshotWorkerMu.Unlock() | 284 | defer d.snapshotWorkerMu.Unlock() | ||
| @@ -280,6 +292,9 @@ func (d *Daemon) durableWriterFailureNames() []string { | |||||
| 280 | for _, capture := range d.snapshotFinalJobs { | 292 | for _, capture := range d.snapshotFinalJobs { | ||
| 281 | add(capture) | 293 | add(capture) | ||
| 282 | } | 294 | } | ||
| 295 | + | for capture := range d.snapshotAdmitted { | |||
| 296 | + | add(capture) | |||
| 297 | + | } | |||
| 283 | names := make([]string, 0, len(seen)) | 298 | names := make([]string, 0, len(seen)) | ||
| 284 | for name := range seen { | 299 | for name := range seen { | ||
| 285 | names = append(names, name) | 300 | names = append(names, name) | ||
M internal/usecase/recovery/checkpoint.go +12 -15
| @@ -37,7 +37,10 @@ func (c *Coordinator) PublishCheckpoint(ctx context.Context, name string, public | |||||
| 37 | unlock := c.locks.Lock([]string{name}) | 37 | unlock := c.locks.Lock([]string{name}) | ||
| 38 | defer unlock() | 38 | defer unlock() | ||
| 39 | 39 | ||||
| 40 | - | record, ok := c.catalogue.Record(name) | 40 | + | record, ok, err := c.catalogue.Record(name) |
| 41 | + | if err != nil { | |||
| 42 | + | return domain.CatalogueRecord{}, err | |||
| 43 | + | } | |||
| 41 | if !ok { | 44 | if !ok { | ||
| 42 | return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound | 45 | return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound | ||
| 43 | } | 46 | } | ||
| @@ -74,7 +77,10 @@ func (c *Coordinator) PromoteFallback(ctx context.Context, name string, ref doma | |||||
| 74 | } | 77 | } | ||
| 75 | 78 | ||||
| 76 | func (c *Coordinator) promoteFallbackLocked(ctx context.Context, name string, ref domain.CheckpointRef) (ports.FallbackPromotionOutcome, error) { | 79 | func (c *Coordinator) promoteFallbackLocked(ctx context.Context, name string, ref domain.CheckpointRef) (ports.FallbackPromotionOutcome, error) { | ||
| 77 | - | record, ok := c.catalogue.Record(name) | 80 | + | record, ok, err := c.catalogue.Record(name) |
| 81 | + | if err != nil { | |||
| 82 | + | return ports.FallbackPromotionOutcome{}, err | |||
| 83 | + | } | |||
| 78 | if !ok { | 84 | if !ok { | ||
| 79 | return ports.FallbackPromotionOutcome{}, ErrCheckpointRecordNotFound | 85 | return ports.FallbackPromotionOutcome{}, ErrCheckpointRecordNotFound | ||
| 80 | } | 86 | } | ||
| @@ -129,24 +135,15 @@ func (c *Coordinator) validateCheckpoint(ctx context.Context, record domain.Cata | |||||
| 129 | if err != nil { | 135 | if err != nil { | ||
| 130 | return err | 136 | return err | ||
| 131 | } | 137 | } | ||
| 132 | - | if generation.IncarnationID != record.IncarnationID || generation.Name != record.Name || generation.Generation != ref.Generation || | 138 | + | return validateLoadedGeneration(record, ref, generation) |
| 133 | - | sha256.Sum256(generation.Manifest) != ref.ManifestDigest { | |||
| 134 | - | return ErrCheckpointConflict | |||
| 135 | - | } | |||
| 136 | - | return nil | |||
| 137 | } | 139 | } | ||
| 138 | 140 | ||||
| 139 | - | func equalCheckpointRefs(a, b *domain.CheckpointRef) bool { | 141 | + | func equalCheckpointRefs(a, b *domain.CheckpointRef) bool { return a.Equal(b) } |
| 140 | - | if a == nil || b == nil { | |||
| 141 | - | return a == nil && b == nil | |||
| 142 | - | } | |||
| 143 | - | return *a == *b | |||
| 144 | - | } | |||
| 145 | 142 | ||||
| 146 | func copyCheckpointRef(ref *domain.CheckpointRef) *domain.CheckpointRef { | 143 | func copyCheckpointRef(ref *domain.CheckpointRef) *domain.CheckpointRef { | ||
| 147 | if ref == nil { | 144 | if ref == nil { | ||
| 148 | return nil | 145 | return nil | ||
| 149 | } | 146 | } | ||
| 150 | - | copy := *ref | 147 | + | cloned := *ref |
| 151 | - | return © | 148 | + | return &cloned |
| 152 | } | 149 | } | ||
M internal/usecase/recovery/checkpoint_test.go +28 -26
| @@ -20,13 +20,15 @@ type checkpointCatalogue struct { | |||||
| 20 | events *[]string | 20 | events *[]string | ||
| 21 | } | 21 | } | ||
| 22 | 22 | ||||
| 23 | - | func (c *checkpointCatalogue) Records() []domain.CatalogueRecord { | 23 | + | func (c *checkpointCatalogue) Records() ([]domain.CatalogueRecord, error) { |
| 24 | - | return []domain.CatalogueRecord{c.record} | 24 | + | c.mu.Lock() |
| 25 | + | defer c.mu.Unlock() | |||
| 26 | + | return []domain.CatalogueRecord{c.record}, nil | |||
| 25 | } | 27 | } | ||
| 26 | - | func (c *checkpointCatalogue) Record(name string) (domain.CatalogueRecord, bool) { | 28 | + | func (c *checkpointCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) { |
| 27 | c.mu.Lock() | 29 | c.mu.Lock() | ||
| 28 | defer c.mu.Unlock() | 30 | defer c.mu.Unlock() | ||
| 29 | - | return c.record, c.record.Name == name | 31 | + | return c.record, c.record.Name == name, nil |
| 30 | } | 32 | } | ||
| 31 | func (c *checkpointCatalogue) Create(domain.CatalogueRecord) error { return nil } | 33 | func (c *checkpointCatalogue) Create(domain.CatalogueRecord) error { return nil } | ||
| 32 | func (c *checkpointCatalogue) UpdateMetadata(domain.CatalogueMetadataUpdate) error { | 34 | func (c *checkpointCatalogue) UpdateMetadata(domain.CatalogueMetadataUpdate) error { | ||
| @@ -127,33 +129,33 @@ func runCheckpointCommit(t *testing.T, count int) (domain.CatalogueRecord, *chec | |||||
| 127 | catalogue.record, err = coordinator.PublishCheckpoint(context.Background(), "work", checkpointPublication(catalogue.record, generation)) | 129 | catalogue.record, err = coordinator.PublishCheckpoint(context.Background(), "work", checkpointPublication(catalogue.record, generation)) | ||
| 128 | require.NoError(t, err) | 130 | require.NoError(t, err) | ||
| 129 | } | 131 | } | ||
| 130 | - | for i := 0; i < count; i++ { | 132 | + | for i := range count { |
| 131 | require.Equal(t, []string{"repository", "catalogue"}, events[i*2:i*2+2]) | 133 | require.Equal(t, []string{"repository", "catalogue"}, events[i*2:i*2+2]) | ||
| 132 | } | 134 | } | ||
| 133 | return catalogue.record, catalogue, repository | 135 | return catalogue.record, catalogue, repository | ||
| 134 | } | 136 | } | ||
| 135 | 137 | ||||
| 136 | - | func TestCheckpointCommitFirst(t *testing.T) { | 138 | + | func TestCheckpointCommits(t *testing.T) { |
| 137 | - | record, catalogue, _ := runCheckpointCommit(t, 1) | 139 | + | for _, tc := range []struct { |
| 138 | - | require.Equal(t, uint64(1), record.Committed.Generation) | 140 | + | name string |
| 139 | - | require.Empty(t, populatedGenerations(record.Fallbacks)) | 141 | + | count int |
| 140 | - | require.Equal(t, 1, catalogue.replaces) | 142 | + | wantCommitted uint64 |
| 141 | - | } | 143 | + | wantFallbacks []uint64 |
| 142 | - | 144 | + | }{ | |
| 143 | - | func TestCheckpointCommitSecond(t *testing.T) { | 145 | + | {name: "first", count: 1, wantCommitted: 1}, |
| 144 | - | record, catalogue, _ := runCheckpointCommit(t, 2) | 146 | + | {name: "second", count: 2, wantCommitted: 2, wantFallbacks: []uint64{1}}, |
| 145 | - | require.Equal(t, uint64(2), record.Committed.Generation) | 147 | + | {name: "third", count: 3, wantCommitted: 3, wantFallbacks: []uint64{2, 1}}, |
| 146 | - | require.Equal(t, []uint64{1}, populatedGenerations(record.Fallbacks)) | 148 | + | {name: "fourth", count: 4, wantCommitted: 4, wantFallbacks: []uint64{3, 2}}, |
| 147 | - | require.Equal(t, 2, catalogue.replaces) | 149 | + | } { |
| 148 | - | } | 150 | + | t.Run(tc.name, func(t *testing.T) { |
| 149 | - | 151 | + | record, catalogue, _ := runCheckpointCommit(t, tc.count) | |
| 150 | - | func TestCheckpointCommitThird(t *testing.T) { | 152 | + | require.Equal(t, tc.wantCommitted, record.Committed.Generation) |
| 151 | - | record, catalogue, _ := runCheckpointCommit(t, 4) | 153 | + | require.Equal(t, tc.wantFallbacks, populatedGenerations(record.Fallbacks)) |
| 152 | - | require.Equal(t, uint64(4), record.Committed.Generation) | 154 | + | require.Equal(t, domain.RecoveryHealthy, record.RecoveryState) |
| 153 | - | require.Equal(t, []uint64{3, 2}, populatedGenerations(record.Fallbacks)) | 155 | + | require.Empty(t, record.DegradedReason) |
| 154 | - | require.Equal(t, domain.RecoveryHealthy, record.RecoveryState) | 156 | + | require.Equal(t, tc.count, catalogue.replaces) |
| 155 | - | require.Empty(t, record.DegradedReason) | 157 | + | }) |
| 156 | - | require.Equal(t, 4, catalogue.replaces) | 158 | + | } |
| 157 | } | 159 | } | ||
| 158 | 160 | ||||
| 159 | func TestCheckpointCommitThirdPromotesOnlyValidatedDirectFallbacks(t *testing.T) { | 161 | func TestCheckpointCommitThirdPromotesOnlyValidatedDirectFallbacks(t *testing.T) { | ||
M internal/usecase/recovery/coordinator.go +32 -8
| @@ -148,7 +148,9 @@ func (c *Coordinator) Create(ctx context.Context, record domain.CatalogueRecord) | |||||
| 148 | } | 148 | } | ||
| 149 | unlock := c.locks.Lock([]string{record.Name}) | 149 | unlock := c.locks.Lock([]string{record.Name}) | ||
| 150 | defer unlock() | 150 | defer unlock() | ||
| 151 | - | if _, exists := c.catalogue.Record(record.Name); exists { | 151 | + | if _, exists, err := c.catalogue.Record(record.Name); err != nil { |
| 152 | + | return domain.CatalogueRecord{}, err | |||
| 153 | + | } else if exists { | |||
| 152 | return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q already exists", record.Name) | 154 | return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q already exists", record.Name) | ||
| 153 | } | 155 | } | ||
| 154 | id, err := domain.NewIncarnationID(c.random) | 156 | id, err := domain.NewIncarnationID(c.random) | ||
| @@ -180,11 +182,16 @@ func (c *Coordinator) Rename(ctx context.Context, oldName, newName string) (doma | |||||
| 180 | } | 182 | } | ||
| 181 | unlock := c.locks.Lock([]string{oldName, newName}) | 183 | unlock := c.locks.Lock([]string{oldName, newName}) | ||
| 182 | defer unlock() | 184 | defer unlock() | ||
| 183 | - | record, ok := c.catalogue.Record(oldName) | 185 | + | record, ok, err := c.catalogue.Record(oldName) |
| 186 | + | if err != nil { | |||
| 187 | + | return domain.CatalogueRecord{}, err | |||
| 188 | + | } | |||
| 184 | if !ok { | 189 | if !ok { | ||
| 185 | return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q not found", oldName) | 190 | return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q not found", oldName) | ||
| 186 | } | 191 | } | ||
| 187 | - | if existing, exists := c.catalogue.Record(newName); exists && (newName != oldName || existing.IncarnationID != record.IncarnationID) { | 192 | + | if existing, exists, err := c.catalogue.Record(newName); err != nil { |
| 193 | + | return domain.CatalogueRecord{}, err | |||
| 194 | + | } else if exists && (newName != oldName || existing.IncarnationID != record.IncarnationID) { | |||
| 188 | return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q already exists", newName) | 195 | return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q already exists", newName) | ||
| 189 | } | 196 | } | ||
| 190 | record.Name = newName | 197 | record.Name = newName | ||
| @@ -207,7 +214,10 @@ func (c *Coordinator) Delete(ctx context.Context, name string) error { | |||||
| 207 | } | 214 | } | ||
| 208 | unlock := c.locks.Lock([]string{name}) | 215 | unlock := c.locks.Lock([]string{name}) | ||
| 209 | defer unlock() | 216 | defer unlock() | ||
| 210 | - | record, ok := c.catalogue.Record(name) | 217 | + | record, ok, err := c.catalogue.Record(name) |
| 218 | + | if err != nil { | |||
| 219 | + | return err | |||
| 220 | + | } | |||
| 211 | if !ok { | 221 | if !ok { | ||
| 212 | return nil | 222 | return nil | ||
| 213 | } | 223 | } | ||
| @@ -267,13 +277,21 @@ func (c *Coordinator) Recover(ctx context.Context) error { | |||||
| 267 | } | 277 | } | ||
| 268 | } | 278 | } | ||
| 269 | } | 279 | } | ||
| 270 | - | for _, candidate := range c.catalogue.Records() { | 280 | + | records, err := c.catalogue.Records() |
| 281 | + | if err != nil { | |||
| 282 | + | return err | |||
| 283 | + | } | |||
| 284 | + | for _, candidate := range records { | |||
| 271 | if candidate.RecoveryState != domain.RecoveryDeleting { | 285 | if candidate.RecoveryState != domain.RecoveryDeleting { | ||
| 272 | continue | 286 | continue | ||
| 273 | } | 287 | } | ||
| 274 | unlock := c.locks.Lock([]string{candidate.Name}) | 288 | unlock := c.locks.Lock([]string{candidate.Name}) | ||
| 275 | var deleteErr error | 289 | var deleteErr error | ||
| 276 | - | current, ok := c.catalogue.Record(candidate.Name) | 290 | + | current, ok, readErr := c.catalogue.Record(candidate.Name) |
| 291 | + | if readErr != nil { | |||
| 292 | + | unlock() | |||
| 293 | + | return readErr | |||
| 294 | + | } | |||
| 277 | if ok && current.IncarnationID == candidate.IncarnationID && current.RecoveryState == domain.RecoveryDeleting { | 295 | if ok && current.IncarnationID == candidate.IncarnationID && current.RecoveryState == domain.RecoveryDeleting { | ||
| 278 | deleteErr = c.deleteLocked(ctx, current) | 296 | deleteErr = c.deleteLocked(ctx, current) | ||
| 279 | } | 297 | } | ||
| @@ -323,7 +341,10 @@ func (c *Coordinator) fenceSession(name, kind, reason string, cause error) error | |||||
| 323 | 341 | ||||
| 324 | unlock := c.locks.Lock([]string{name}) | 342 | unlock := c.locks.Lock([]string{name}) | ||
| 325 | defer unlock() | 343 | defer unlock() | ||
| 326 | - | record, ok := c.catalogue.Record(name) | 344 | + | record, ok, err := c.catalogue.Record(name) |
| 345 | + | if err != nil { | |||
| 346 | + | return err | |||
| 347 | + | } | |||
| 327 | if !ok || record.RecoveryState != domain.RecoveryHealthy { | 348 | if !ok || record.RecoveryState != domain.RecoveryHealthy { | ||
| 328 | // Fresh records cannot represent degraded state (it requires a committed | 349 | // Fresh records cannot represent degraded state (it requires a committed | ||
| 329 | // checkpoint), and deleting or already degraded records must keep the | 350 | // checkpoint), and deleting or already degraded records must keep the | ||
| @@ -354,7 +375,10 @@ func (c *Coordinator) recoverDeletionTombstone(ctx context.Context, tombstone do | |||||
| 354 | } | 375 | } | ||
| 355 | unlock := c.locks.Lock([]string{tombstone.Name}) | 376 | unlock := c.locks.Lock([]string{tombstone.Name}) | ||
| 356 | defer unlock() | 377 | defer unlock() | ||
| 357 | - | record, exists := c.catalogue.Record(tombstone.Name) | 378 | + | record, exists, err := c.catalogue.Record(tombstone.Name) |
| 379 | + | if err != nil { | |||
| 380 | + | return err | |||
| 381 | + | } | |||
| 358 | includeLegacyName := true | 382 | includeLegacyName := true | ||
| 359 | if exists { | 383 | if exists { | ||
| 360 | switch { | 384 | switch { | ||
M internal/usecase/recovery/discard.go +25 -15
| @@ -39,7 +39,10 @@ func (c *Coordinator) Retry(ctx context.Context, name string) error { | |||||
| 39 | } | 39 | } | ||
| 40 | unlock := c.locks.Lock([]string{name}) | 40 | unlock := c.locks.Lock([]string{name}) | ||
| 41 | defer unlock() | 41 | defer unlock() | ||
| 42 | - | record, ok := c.catalogue.Record(name) | 42 | + | record, ok, err := c.catalogue.Record(name) |
| 43 | + | if err != nil { | |||
| 44 | + | return err | |||
| 45 | + | } | |||
| 43 | if !ok { | 46 | if !ok { | ||
| 44 | return ErrRecoveryRecordNotFound | 47 | return ErrRecoveryRecordNotFound | ||
| 45 | } | 48 | } | ||
| @@ -66,7 +69,10 @@ func (c *Coordinator) RestoreFallback(ctx context.Context, name string, ref doma | |||||
| 66 | } | 69 | } | ||
| 67 | unlock := c.locks.Lock([]string{name}) | 70 | unlock := c.locks.Lock([]string{name}) | ||
| 68 | defer unlock() | 71 | defer unlock() | ||
| 69 | - | record, ok := c.catalogue.Record(name) | 72 | + | record, ok, err := c.catalogue.Record(name) |
| 73 | + | if err != nil { | |||
| 74 | + | return err | |||
| 75 | + | } | |||
| 70 | if !ok { | 76 | if !ok { | ||
| 71 | return ErrRecoveryRecordNotFound | 77 | return ErrRecoveryRecordNotFound | ||
| 72 | } | 78 | } | ||
| @@ -88,7 +94,10 @@ func (c *Coordinator) Export(ctx context.Context, name string, w io.Writer) erro | |||||
| 88 | } | 94 | } | ||
| 89 | unlock := c.locks.Lock([]string{name}) | 95 | unlock := c.locks.Lock([]string{name}) | ||
| 90 | defer unlock() | 96 | defer unlock() | ||
| 91 | - | record, ok := c.catalogue.Record(name) | 97 | + | record, ok, err := c.catalogue.Record(name) |
| 98 | + | if err != nil { | |||
| 99 | + | return err | |||
| 100 | + | } | |||
| 92 | if !ok { | 101 | if !ok { | ||
| 93 | return ErrRecoveryRecordNotFound | 102 | return ErrRecoveryRecordNotFound | ||
| 94 | } | 103 | } | ||
| @@ -167,7 +176,10 @@ func (c *Coordinator) Discard(ctx context.Context, name, reason string) (domain. | |||||
| 167 | } | 176 | } | ||
| 168 | unlock := c.locks.Lock([]string{name}) | 177 | unlock := c.locks.Lock([]string{name}) | ||
| 169 | defer unlock() | 178 | defer unlock() | ||
| 170 | - | record, ok := c.catalogue.Record(name) | 179 | + | record, ok, err := c.catalogue.Record(name) |
| 180 | + | if err != nil { | |||
| 181 | + | return domain.CatalogueRecord{}, err | |||
| 182 | + | } | |||
| 171 | if !ok { | 183 | if !ok { | ||
| 172 | return domain.CatalogueRecord{}, ErrRecoveryRecordNotFound | 184 | return domain.CatalogueRecord{}, ErrRecoveryRecordNotFound | ||
| 173 | } | 185 | } | ||
| @@ -207,9 +219,15 @@ func (c *Coordinator) recoverDiscardLocked(ctx context.Context, intent domain.Di | |||||
| 207 | if err := validateDiscardIntent(intent); err != nil { | 219 | if err := validateDiscardIntent(intent); err != nil { | ||
| 208 | return err | 220 | return err | ||
| 209 | } | 221 | } | ||
| 210 | - | current, ok := c.catalogue.Record(intent.SessionName) | 222 | + | current, ok, err := c.catalogue.Record(intent.SessionName) |
| 223 | + | if err != nil { | |||
| 224 | + | return err | |||
| 225 | + | } | |||
| 211 | if !ok { | 226 | if !ok { | ||
| 212 | - | return fmt.Errorf("%w: missing session %q", ErrDiscardConflict, intent.SessionName) | 227 | + | // Catalogue deletion is terminal for this session identity. There is no |
| 228 | + | // live record to replace or quarantine, so retaining the intent would | |||
| 229 | + | // manufacture a permanent startup conflict. | |||
| 230 | + | return c.journal.DeleteDiscard(ctx, intent.OldIncarnation) | |||
| 213 | } | 231 | } | ||
| 214 | next := freshReplacement(intent) | 232 | next := freshReplacement(intent) | ||
| 215 | if err := next.Validate(); err != nil { | 233 | if err := next.Validate(); err != nil { | ||
| @@ -217,7 +235,7 @@ func (c *Coordinator) recoverDiscardLocked(ctx context.Context, intent domain.Di | |||||
| 217 | } | 235 | } | ||
| 218 | switch current.IncarnationID { | 236 | switch current.IncarnationID { | ||
| 219 | case intent.OldIncarnation: | 237 | case intent.OldIncarnation: | ||
| 220 | - | if !catalogueRecordsEqual(current, intent.OldRecord) { | 238 | + | if !current.Equal(intent.OldRecord) { |
| 221 | return fmt.Errorf("%w: old record changed", ErrDiscardConflict) | 239 | return fmt.Errorf("%w: old record changed", ErrDiscardConflict) | ||
| 222 | } | 240 | } | ||
| 223 | case intent.NewIncarnation: | 241 | case intent.NewIncarnation: | ||
| @@ -275,11 +293,3 @@ func freshReplacement(intent domain.DiscardIntent) domain.CatalogueRecord { | |||||
| 275 | next.DegradedReason = "" | 293 | next.DegradedReason = "" | ||
| 276 | return next | 294 | return next | ||
| 277 | } | 295 | } | ||
| 278 | - | ||||
| 279 | - | func catalogueRecordsEqual(a, b domain.CatalogueRecord) bool { | |||
| 280 | - | return a.Name == b.Name && a.IncarnationID == b.IncarnationID && a.Cwd == b.Cwd && | |||
| 281 | - | a.CreatedAt == b.CreatedAt && a.UpdatedAt == b.UpdatedAt && a.LastUsedSeq == b.LastUsedSeq && | |||
| 282 | - | slices.Equal(a.TabNames, b.TabNames) && a.RecoveryState == b.RecoveryState && | |||
| 283 | - | equalCheckpointRefs(a.Committed, b.Committed) && equalCheckpointRefs(a.Fallbacks[0], b.Fallbacks[0]) && | |||
| 284 | - | equalCheckpointRefs(a.Fallbacks[1], b.Fallbacks[1]) && a.DegradedReason == b.DegradedReason | |||
| 285 | - | } | |||
M internal/usecase/recovery/reconcile.go +26 -5
| @@ -29,23 +29,41 @@ func (r *Reconciler) Step(ctx context.Context, cursor ports.ReconcileCursor) (po | |||||
| 29 | if err := ctx.Err(); err != nil { | 29 | if err := ctx.Err(); err != nil { | ||
| 30 | return cursor, nil, err | 30 | return cursor, nil, err | ||
| 31 | } | 31 | } | ||
| 32 | - | next, findings, err := r.repository.Reconcile(ctx, r.catalogue.Records(), cursor, r.budget) | 32 | + | records, err := r.catalogue.Records() |
| 33 | + | if err != nil { | |||
| 34 | + | return cursor, nil, err | |||
| 35 | + | } | |||
| 36 | + | next, findings, err := r.repository.Reconcile(ctx, records, cursor, r.budget) | |||
| 33 | if err != nil { | 37 | if err != nil { | ||
| 34 | return cursor, nil, err | 38 | return cursor, nil, err | ||
| 35 | } | 39 | } | ||
| 36 | decisions := make([]ports.ReconcileDecision, 0, len(findings)) | 40 | decisions := make([]ports.ReconcileDecision, 0, len(findings)) | ||
| 41 | + | consumed := ports.MaintenanceBudget{} | |||
| 37 | for _, finding := range findings { | 42 | for _, finding := range findings { | ||
| 38 | - | if finding.Consumed.Entries > r.budget.Entries || finding.Consumed.Bytes > r.budget.Bytes { | 43 | + | // Compare against the remainder before adding. This both enforces the |
| 44 | + | // page-wide budget and avoids uint64 wraparound. | |||
| 45 | + | if finding.Consumed.Entries > r.budget.Entries-consumed.Entries || | |||
| 46 | + | finding.Consumed.Bytes > r.budget.Bytes-consumed.Bytes { | |||
| 39 | return cursor, nil, errors.New("recovery: repository exceeded reconciliation budget") | 47 | return cursor, nil, errors.New("recovery: repository exceeded reconciliation budget") | ||
| 40 | } | 48 | } | ||
| 41 | - | record, ok := r.catalogue.Record(finding.Candidate.Name) | 49 | + | consumed.Entries += finding.Consumed.Entries |
| 50 | + | consumed.Bytes += finding.Consumed.Bytes | |||
| 51 | + | record, ok, err := r.catalogue.Record(finding.Candidate.Name) | |||
| 52 | + | if err != nil { | |||
| 53 | + | return cursor, nil, err | |||
| 54 | + | } | |||
| 42 | if !ok { | 55 | if !ok { | ||
| 43 | record = domain.CatalogueRecord{} | 56 | record = domain.CatalogueRecord{} | ||
| 44 | } | 57 | } | ||
| 45 | decision := classifyFinding(record, finding) | 58 | decision := classifyFinding(record, finding) | ||
| 46 | if decision.Kind == ports.ReconcileAdopt { | 59 | if decision.Kind == ports.ReconcileAdopt { | ||
| 47 | if _, err := r.coordinator.PublishReconciledCheckpoint(ctx, finding.Candidate.Name, finding.Candidate, finding.AncestorChain); err != nil { | 60 | if _, err := r.coordinator.PublishReconciledCheckpoint(ctx, finding.Candidate.Name, finding.Candidate, finding.AncestorChain); err != nil { | ||
| 48 | - | return cursor, nil, err | 61 | + | if !errors.Is(err, ErrCheckpointConflict) && !errors.Is(err, ErrCheckpointRecordNotFound) { |
| 62 | + | return cursor, nil, err | |||
| 63 | + | } | |||
| 64 | + | decision.Kind = ports.ReconcileDefer | |||
| 65 | + | decision.ReasonCode = "catalogue-conflict" | |||
| 66 | + | decision.RetentionResolved = false | |||
| 49 | } | 67 | } | ||
| 50 | } | 68 | } | ||
| 51 | decisions = append(decisions, decision) | 69 | decisions = append(decisions, decision) | ||
| @@ -132,7 +150,10 @@ func (c *Coordinator) PublishReconciledCheckpoint(ctx context.Context, name stri | |||||
| 132 | if err := ctx.Err(); err != nil { | 150 | if err := ctx.Err(); err != nil { | ||
| 133 | return domain.CatalogueRecord{}, err | 151 | return domain.CatalogueRecord{}, err | ||
| 134 | } | 152 | } | ||
| 135 | - | record, ok := c.catalogue.Record(name) | 153 | + | record, ok, err := c.catalogue.Record(name) |
| 154 | + | if err != nil { | |||
| 155 | + | return domain.CatalogueRecord{}, err | |||
| 156 | + | } | |||
| 136 | if !ok { | 157 | if !ok { | ||
| 137 | return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound | 158 | return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound | ||
| 138 | } | 159 | } | ||
M internal/usecase/recovery/reconcile_test.go +77 -0
| @@ -24,6 +24,33 @@ type reconcilePage struct { | |||||
| 24 | err error | 24 | err error | ||
| 25 | } | 25 | } | ||
| 26 | 26 | ||||
| 27 | + | type reconcileConflictCatalogue struct { | |||
| 28 | + | *transactionCatalogue | |||
| 29 | + | stale domain.CatalogueRecord | |||
| 30 | + | reads int | |||
| 31 | + | } | |||
| 32 | + | ||||
| 33 | + | func (c *reconcileConflictCatalogue) Records() ([]domain.CatalogueRecord, error) { | |||
| 34 | + | records, err := c.transactionCatalogue.Records() | |||
| 35 | + | if err != nil { | |||
| 36 | + | return nil, err | |||
| 37 | + | } | |||
| 38 | + | for i := range records { | |||
| 39 | + | if records[i].Name == c.stale.Name { | |||
| 40 | + | records[i] = c.stale | |||
| 41 | + | } | |||
| 42 | + | } | |||
| 43 | + | return records, nil | |||
| 44 | + | } | |||
| 45 | + | ||||
| 46 | + | func (c *reconcileConflictCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) { | |||
| 47 | + | if name == c.stale.Name && c.reads == 0 { | |||
| 48 | + | c.reads++ | |||
| 49 | + | return c.stale, true, nil | |||
| 50 | + | } | |||
| 51 | + | return c.transactionCatalogue.Record(name) | |||
| 52 | + | } | |||
| 53 | + | ||||
| 27 | func (r *reconcileRepository) Reconcile(_ context.Context, _ []domain.CatalogueRecord, cursor ports.ReconcileCursor, budget ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error) { | 54 | func (r *reconcileRepository) Reconcile(_ context.Context, _ []domain.CatalogueRecord, cursor ports.ReconcileCursor, budget ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error) { | ||
| 28 | r.seenCursors = append(r.seenCursors, cursor) | 55 | r.seenCursors = append(r.seenCursors, cursor) | ||
| 29 | r.seenBudgets = append(r.seenBudgets, budget) | 56 | r.seenBudgets = append(r.seenBudgets, budget) | ||
| @@ -64,6 +91,56 @@ func TestReconciliationCursor(t *testing.T) { | |||||
| 64 | require.Equal(t, []ports.MaintenanceBudget{{Entries: 2, Bytes: 32}, {Entries: 2, Bytes: 32}}, repository.seenBudgets) | 91 | require.Equal(t, []ports.MaintenanceBudget{{Entries: 2, Bytes: 32}, {Entries: 2, Bytes: 32}}, repository.seenBudgets) | ||
| 65 | } | 92 | } | ||
| 66 | 93 | ||||
| 94 | + | func TestReconciliationRejectsCumulativeAndOverflowingBudgets(t *testing.T) { | |||
| 95 | + | record := healthyReconcileRecord() | |||
| 96 | + | finding := ports.ReconcileFinding{Status: ports.ReconcileBudgetExhausted, Candidate: ports.ReconcileCandidate{Name: record.Name}} | |||
| 97 | + | for _, tc := range []struct { | |||
| 98 | + | name string | |||
| 99 | + | budget ports.MaintenanceBudget | |||
| 100 | + | consumed []ports.MaintenanceBudget | |||
| 101 | + | }{ | |||
| 102 | + | {name: "cumulative entries", budget: ports.MaintenanceBudget{Entries: 3, Bytes: 10}, consumed: []ports.MaintenanceBudget{{Entries: 2, Bytes: 1}, {Entries: 2, Bytes: 1}}}, | |||
| 103 | + | {name: "cumulative bytes", budget: ports.MaintenanceBudget{Entries: 10, Bytes: 3}, consumed: []ports.MaintenanceBudget{{Entries: 1, Bytes: 2}, {Entries: 1, Bytes: 2}}}, | |||
| 104 | + | {name: "overflow safe", budget: ports.MaintenanceBudget{Entries: ^uint64(0), Bytes: ^uint64(0)}, consumed: []ports.MaintenanceBudget{{Entries: ^uint64(0), Bytes: ^uint64(0)}, {Entries: 1, Bytes: 1}}}, | |||
| 105 | + | } { | |||
| 106 | + | t.Run(tc.name, func(t *testing.T) { | |||
| 107 | + | findings := make([]ports.ReconcileFinding, len(tc.consumed)) | |||
| 108 | + | for i, usage := range tc.consumed { | |||
| 109 | + | findings[i] = finding | |||
| 110 | + | findings[i].Consumed = usage | |||
| 111 | + | } | |||
| 112 | + | repository := &reconcileRepository{pages: map[uint64]reconcilePage{0: {findings: findings}}} | |||
| 113 | + | catalogue := &checkpointCatalogue{record: record} | |||
| 114 | + | reconciler := NewReconciler(NewCoordinator(catalogue, repository, nil, nil), catalogue, repository, tc.budget) | |||
| 115 | + | _, _, err := reconciler.Step(context.Background(), ports.ReconcileCursor{}) | |||
| 116 | + | require.EqualError(t, err, "recovery: repository exceeded reconciliation budget") | |||
| 117 | + | }) | |||
| 118 | + | } | |||
| 119 | + | } | |||
| 120 | + | ||||
| 121 | + | func TestReconciliationDefersConflictAndContinues(t *testing.T) { | |||
| 122 | + | first := healthyReconcileRecord() | |||
| 123 | + | first.Name = "first" | |||
| 124 | + | staleRecord := first | |||
| 125 | + | stale := forwardCandidate(staleRecord, 2, staleRecord.Committed) | |||
| 126 | + | first = shiftedCheckpoint(first, domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{9}}) | |||
| 127 | + | second := healthyReconcileRecord() | |||
| 128 | + | second.Name = "second" | |||
| 129 | + | adoptable := forwardCandidate(second, 2, second.Committed) | |||
| 130 | + | catalogue := &reconcileConflictCatalogue{transactionCatalogue: newTransactionCatalogue(first, second), stale: staleRecord} | |||
| 131 | + | repository := &reconcileRepository{pages: map[uint64]reconcilePage{0: {findings: []ports.ReconcileFinding{ | |||
| 132 | + | {Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: stale, Consumed: ports.MaintenanceBudget{Entries: 1, Bytes: 1}}, | |||
| 133 | + | {Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: adoptable, Consumed: ports.MaintenanceBudget{Entries: 1, Bytes: 1}}, | |||
| 134 | + | }}}} | |||
| 135 | + | reconciler := NewReconciler(NewCoordinator(catalogue, repository, nil, nil), catalogue, repository, ports.MaintenanceBudget{Entries: 2, Bytes: 2}) | |||
| 136 | + | ||||
| 137 | + | _, decisions, err := reconciler.Step(context.Background(), ports.ReconcileCursor{}) | |||
| 138 | + | require.NoError(t, err) | |||
| 139 | + | require.Equal(t, []ports.ReconcileDecisionKind{ports.ReconcileDefer, ports.ReconcileAdopt}, []ports.ReconcileDecisionKind{decisions[0].Kind, decisions[1].Kind}) | |||
| 140 | + | require.Equal(t, "catalogue-conflict", decisions[0].ReasonCode) | |||
| 141 | + | require.Equal(t, adoptable.Ref, *catalogue.records[second.Name].Committed) | |||
| 142 | + | } | |||
| 143 | + | ||||
| 67 | func TestForwardOrphanAdoption(t *testing.T) { | 144 | func TestForwardOrphanAdoption(t *testing.T) { | ||
| 68 | for _, tc := range []struct { | 145 | for _, tc := range []struct { | ||
| 69 | name string | 146 | name string | ||
M internal/usecase/recovery/transactions_test.go +21 -10
| @@ -27,20 +27,20 @@ func newTransactionCatalogue(records ...domain.CatalogueRecord) *transactionCata | |||||
| 27 | return catalogue | 27 | return catalogue | ||
| 28 | } | 28 | } | ||
| 29 | 29 | ||||
| 30 | - | func (c *transactionCatalogue) Records() []domain.CatalogueRecord { | 30 | + | func (c *transactionCatalogue) Records() ([]domain.CatalogueRecord, error) { |
| 31 | c.mu.Lock() | 31 | c.mu.Lock() | ||
| 32 | defer c.mu.Unlock() | 32 | defer c.mu.Unlock() | ||
| 33 | out := make([]domain.CatalogueRecord, 0, len(c.records)) | 33 | out := make([]domain.CatalogueRecord, 0, len(c.records)) | ||
| 34 | for _, record := range c.records { | 34 | for _, record := range c.records { | ||
| 35 | out = append(out, record) | 35 | out = append(out, record) | ||
| 36 | } | 36 | } | ||
| 37 | - | return out | 37 | + | return out, nil |
| 38 | } | 38 | } | ||
| 39 | - | func (c *transactionCatalogue) Record(name string) (domain.CatalogueRecord, bool) { | 39 | + | func (c *transactionCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) { |
| 40 | c.mu.Lock() | 40 | c.mu.Lock() | ||
| 41 | defer c.mu.Unlock() | 41 | defer c.mu.Unlock() | ||
| 42 | record, ok := c.records[name] | 42 | record, ok := c.records[name] | ||
| 43 | - | return record, ok | 43 | + | return record, ok, nil |
| 44 | } | 44 | } | ||
| 45 | func (c *transactionCatalogue) Create(record domain.CatalogueRecord) error { | 45 | func (c *transactionCatalogue) Create(record domain.CatalogueRecord) error { | ||
| 46 | c.mu.Lock() | 46 | c.mu.Lock() | ||
| @@ -233,16 +233,13 @@ func TestFallbackPromotion(t *testing.T) { | |||||
| 233 | fallback := *record.Fallbacks[0] | 233 | fallback := *record.Fallbacks[0] | ||
| 234 | catalogue := newTransactionCatalogue(record) | 234 | catalogue := newTransactionCatalogue(record) | ||
| 235 | repository := newTransactionRepository() | 235 | repository := newTransactionRepository() | ||
| 236 | - | repository.generations[fallback] = ports.SnapshotGeneration{ | 236 | + | manifest := []byte("fallback") |
| 237 | - | IncarnationID: record.IncarnationID, Name: record.Name, Generation: fallback.Generation, | 237 | + | fallback.ManifestDigest = checkpointDigest(manifest) |
| 238 | - | Manifest: []byte("fallback"), | |||
| 239 | - | } | |||
| 240 | - | fallback.ManifestDigest = checkpointDigest(repository.generations[*record.Fallbacks[0]].Manifest) | |||
| 241 | record.Fallbacks[0] = &fallback | 238 | record.Fallbacks[0] = &fallback | ||
| 242 | catalogue.records[record.Name] = record | 239 | catalogue.records[record.Name] = record | ||
| 243 | repository.generations[fallback] = ports.SnapshotGeneration{ | 240 | repository.generations[fallback] = ports.SnapshotGeneration{ | ||
| 244 | IncarnationID: record.IncarnationID, Name: record.Name, Generation: fallback.Generation, | 241 | IncarnationID: record.IncarnationID, Name: record.Name, Generation: fallback.Generation, | ||
| 245 | - | Manifest: []byte("fallback"), | 242 | + | Manifest: manifest, |
| 246 | } | 243 | } | ||
| 247 | coordinator := NewCoordinator(catalogue, repository, newTransactionJournal(), nil) | 244 | coordinator := NewCoordinator(catalogue, repository, newTransactionJournal(), nil) | ||
| 248 | 245 | ||||
| @@ -283,6 +280,20 @@ func TestDegradedExportIsReadOnly(t *testing.T) { | |||||
| 283 | 280 | ||||
| 284 | // A third incarnation still fails closed for that session: its intent and | 281 | // A third incarnation still fails closed for that session: its intent and | ||
| 285 | // record are retained untouched, but startup is no longer aborted. | 282 | // record are retained untouched, but startup is no longer aborted. | ||
| 283 | + | func TestDiscardRecoveryRemovesIntentWhenCatalogueRecordIsAbsent(t *testing.T) { | |||
| 284 | + | old := degradedTransactionRecord() | |||
| 285 | + | intent := domain.DiscardIntent{OldRecord: old, OldIncarnation: old.IncarnationID, NewIncarnation: domain.IncarnationID{2}, SessionName: old.Name, Reason: "discard"} | |||
| 286 | + | catalogue := newTransactionCatalogue() | |||
| 287 | + | journal := newTransactionJournal() | |||
| 288 | + | journal.intents[old.IncarnationID] = intent | |||
| 289 | + | coordinator := NewCoordinator(catalogue, newTransactionRepository(), journal, nil) | |||
| 290 | + | ||||
| 291 | + | require.NoError(t, coordinator.Recover(context.Background())) | |||
| 292 | + | require.NotContains(t, journal.intents, old.IncarnationID) | |||
| 293 | + | require.Equal(t, []string{"delete-intent"}, journal.events) | |||
| 294 | + | require.Empty(t, coordinator.Conflicts()) | |||
| 295 | + | } | |||
| 296 | + | ||||
| 286 | func TestDiscardIncarnationConflict(t *testing.T) { | 297 | func TestDiscardIncarnationConflict(t *testing.T) { | ||
| 287 | old := degradedTransactionRecord() | 298 | old := degradedTransactionRecord() | ||
| 288 | intent := domain.DiscardIntent{OldRecord: old, OldIncarnation: old.IncarnationID, NewIncarnation: domain.IncarnationID{2}, SessionName: old.Name, Reason: "discard"} | 299 | intent := domain.DiscardIntent{OldRecord: old, OldIncarnation: old.IncarnationID, NewIncarnation: domain.IncarnationID{2}, SessionName: old.Name, Reason: "discard"} | ||
M internal/usecase/snapshot/manifest_preflight.go +4 -1
| @@ -16,7 +16,10 @@ func preflightManifest(body []byte) error { | |||||
| 16 | if err != nil { | 16 | if err != nil { | ||
| 17 | return err | 17 | return err | ||
| 18 | } | 18 | } | ||
| 19 | - | if generation == 0 || len(r.b) < len(domain.IncarnationID{}) { | 19 | + | if generation == 0 { |
| 20 | + | return fmt.Errorf("%w: manifest generation", ErrInvalidData) | |||
| 21 | + | } | |||
| 22 | + | if len(r.b) < len(domain.IncarnationID{}) { | |||
| 20 | return ErrShortPayload | 23 | return ErrShortPayload | ||
| 21 | } | 24 | } | ||
| 22 | var incarnationID domain.IncarnationID | 25 | var incarnationID domain.IncarnationID | ||
M internal/usecase/snapshot/manifest_preflight_test.go +55 -3
| @@ -40,6 +40,7 @@ func TestManifestEnvelopePreflightParity(t *testing.T) { | |||||
| 40 | if err != nil { | 40 | if err != nil { | ||
| 41 | t.Fatal(err) | 41 | t.Fatal(err) | ||
| 42 | } | 42 | } | ||
| 43 | + | nameOffset, nodeKindOffset := manifestFieldOffsets(t, encoded) | |||
| 43 | mutations := []struct { | 44 | mutations := []struct { | ||
| 44 | name string | 45 | name string | ||
| 45 | edit func([]byte) []byte | 46 | edit func([]byte) []byte | ||
| @@ -51,9 +52,21 @@ func TestManifestEnvelopePreflightParity(t *testing.T) { | |||||
| 51 | {"body length", func(b []byte) []byte { binary.BigEndian.PutUint32(b[8:12], uint32(len(b))); return b }}, | 52 | {"body length", func(b []byte) []byte { binary.BigEndian.PutUint32(b[8:12], uint32(len(b))); return b }}, | ||
| 52 | {"crc", func(b []byte) []byte { b[12] ^= 1; return b }}, | 53 | {"crc", func(b []byte) []byte { b[12] ^= 1; return b }}, | ||
| 53 | {"trailing", func(b []byte) []byte { return append(b, 0) }}, | 54 | {"trailing", func(b []byte) []byte { return append(b, 0) }}, | ||
| 54 | - | {"body name", func(b []byte) []byte { b[26] ^= 1; binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])); return b }}, | 55 | + | {"body name", func(b []byte) []byte { |
| 55 | - | {"body generation", func(b []byte) []byte { b[16] ^= 1; binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])); return b }}, | 56 | + | b[nameOffset] = 0 |
| 56 | - | {"body node kind", func(b []byte) []byte { b[65] = 99; binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])); return b }}, | 57 | + | binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])) |
| 58 | + | return b | |||
| 59 | + | }}, | |||
| 60 | + | {"body generation", func(b []byte) []byte { | |||
| 61 | + | clear(b[manifestHeaderSize : manifestHeaderSize+8]) | |||
| 62 | + | binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])) | |||
| 63 | + | return b | |||
| 64 | + | }}, | |||
| 65 | + | {"body node kind", func(b []byte) []byte { | |||
| 66 | + | b[nodeKindOffset] = 99 | |||
| 67 | + | binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])) | |||
| 68 | + | return b | |||
| 69 | + | }}, | |||
| 57 | } | 70 | } | ||
| 58 | for _, tc := range mutations { | 71 | for _, tc := range mutations { | ||
| 59 | t.Run(tc.name, func(t *testing.T) { | 72 | t.Run(tc.name, func(t *testing.T) { | ||
| @@ -68,6 +81,45 @@ func TestManifestEnvelopePreflightParity(t *testing.T) { | |||||
| 68 | } | 81 | } | ||
| 69 | } | 82 | } | ||
| 70 | 83 | ||||
| 84 | + | func manifestFieldOffsets(t *testing.T, encoded []byte) (nameOffset, nodeKindOffset int) { | |||
| 85 | + | t.Helper() | |||
| 86 | + | r := payloadReader{b: encoded[manifestHeaderSize:]} | |||
| 87 | + | must := func(err error) { | |||
| 88 | + | if err != nil { | |||
| 89 | + | t.Fatal(err) | |||
| 90 | + | } | |||
| 91 | + | } | |||
| 92 | + | _, err := r.getUint64() | |||
| 93 | + | must(err) | |||
| 94 | + | r.b = r.b[len(domain.IncarnationID{}):] | |||
| 95 | + | _, err = r.getUint8() | |||
| 96 | + | must(err) | |||
| 97 | + | nameLengthOffset := len(encoded) - len(r.b) | |||
| 98 | + | name, err := r.getString() | |||
| 99 | + | must(err) | |||
| 100 | + | nameOffset = nameLengthOffset + 2 | |||
| 101 | + | if len(name) == 0 { | |||
| 102 | + | t.Fatal("test manifest has empty name") | |||
| 103 | + | } | |||
| 104 | + | _, err = r.getUint64() | |||
| 105 | + | must(err) | |||
| 106 | + | _, err = r.getUint16() | |||
| 107 | + | must(err) | |||
| 108 | + | _, err = r.getUint16() | |||
| 109 | + | must(err) | |||
| 110 | + | _, err = r.getString() | |||
| 111 | + | must(err) | |||
| 112 | + | _, err = r.getUint16() | |||
| 113 | + | must(err) | |||
| 114 | + | _, err = r.getUint16() | |||
| 115 | + | must(err) | |||
| 116 | + | _, err = r.getUint64() | |||
| 117 | + | must(err) | |||
| 118 | + | _, err = r.getString() | |||
| 119 | + | must(err) | |||
| 120 | + | return nameOffset, len(encoded) - len(r.b) | |||
| 121 | + | } | |||
| 122 | + | ||||
| 71 | func TestManifestObjectRefKinds(t *testing.T) { | 123 | func TestManifestObjectRefKinds(t *testing.T) { | ||
| 72 | for _, tc := range []struct { | 124 | for _, tc := range []struct { | ||
| 73 | name string | 125 | name string | ||
M pkg/kv/codec.go +12 -7
| @@ -19,8 +19,9 @@ const ( | |||||
| 19 | // place by Open; nothing ever writes this framing again. | 19 | // place by Open; nothing ever writes this framing again. | ||
| 20 | legacyHeaderLen = 8 | 20 | legacyHeaderLen = 8 | ||
| 21 | 21 | ||||
| 22 | - | // Current framing. Every file starts with fileMagic plus a uint16 format | 22 | + | // Current framing. Every file starts with fileMagic, a uint16 format |
| 23 | - | // version, and every record carries a self-verifying header: | 23 | + | // version, and CRC32 over that six-byte identity. Every record carries a |
| 24 | + | // self-verifying header: | |||
| 24 | // | 25 | // | ||
| 25 | // [0:4] payload length, big endian | 26 | // [0:4] payload length, big endian | ||
| 26 | // [4:8] CRC32(payload) | 27 | // [4:8] CRC32(payload) | ||
| @@ -29,10 +30,13 @@ const ( | |||||
| 29 | // The header checksum is what makes the length trustworthy: a corrupt | 30 | // The header checksum is what makes the length trustworthy: a corrupt | ||
| 30 | // length now fails closed instead of masquerading as a torn tail and | 31 | // length now fails closed instead of masquerading as a torn tail and | ||
| 31 | // silently truncating every record that follows it. | 32 | // silently truncating every record that follows it. | ||
| 32 | - | recordHeaderLen = 12 | 33 | + | recordHeaderLen = 12 |
| 33 | - | fileMagicLen = 4 | 34 | + | fileMagicLen = 4 |
| 34 | - | fileHeaderLen = fileMagicLen + 2 | 35 | + | fileHeaderV1Len = fileMagicLen + 2 |
| 35 | - | formatVersion = 1 | 36 | + | fileHeaderBodyLen = fileHeaderV1Len |
| 37 | + | fileHeaderLen = fileHeaderBodyLen + 4 | |||
| 38 | + | formatVersionV1 = 1 | |||
| 39 | + | formatVersion = 2 | |||
| 36 | 40 | ||||
| 37 | // maxPayloadLen bounds any allocation driven by a length field. Catalogue | 41 | // maxPayloadLen bounds any allocation driven by a length field. Catalogue | ||
| 38 | // records and batches are orders of magnitude smaller than this. | 42 | // records and batches are orders of magnitude smaller than this. | ||
| @@ -145,7 +149,8 @@ func encodeBatch(changes []BatchChange) ([]byte, error) { | |||||
| 145 | func fileHeader() []byte { | 149 | func fileHeader() []byte { | ||
| 146 | buf := make([]byte, fileHeaderLen) | 150 | buf := make([]byte, fileHeaderLen) | ||
| 147 | copy(buf, fileMagic[:]) | 151 | copy(buf, fileMagic[:]) | ||
| 148 | - | binary.BigEndian.PutUint16(buf[fileMagicLen:], formatVersion) | 152 | + | binary.BigEndian.PutUint16(buf[fileMagicLen:fileHeaderBodyLen], formatVersion) |
| 153 | + | binary.BigEndian.PutUint32(buf[fileHeaderBodyLen:], crc32.ChecksumIEEE(buf[:fileHeaderBodyLen])) | |||
| 149 | return buf | 154 | return buf | ||
| 150 | } | 155 | } | ||
| 151 | 156 | ||||
M pkg/kv/kv.go +100 -48
| @@ -36,16 +36,30 @@ const ( | |||||
| 36 | // formatLegacy is a file written before the record header carried its own | 36 | // formatLegacy is a file written before the record header carried its own | ||
| 37 | // checksum. It is read with the original semantics and upgraded at Open. | 37 | // checksum. It is read with the original semantics and upgraded at Open. | ||
| 38 | formatLegacy walFormat = iota | 38 | formatLegacy walFormat = iota | ||
| 39 | + | // formatCurrentV1 is the released VEVK v1 framing with its six-byte file | |||
| 40 | + | // header. Its record headers are identical to the current format. | |||
| 41 | + | formatCurrentV1 | |||
| 39 | formatCurrent | 42 | formatCurrent | ||
| 40 | ) | 43 | ) | ||
| 41 | 44 | ||||
| 42 | func (f walFormat) headerLen() int64 { | 45 | func (f walFormat) headerLen() int64 { | ||
| 43 | - | if f == formatCurrent { | 46 | + | if f != formatLegacy { |
| 44 | return recordHeaderLen | 47 | return recordHeaderLen | ||
| 45 | } | 48 | } | ||
| 46 | return legacyHeaderLen | 49 | return legacyHeaderLen | ||
| 47 | } | 50 | } | ||
| 48 | 51 | ||||
| 52 | + | func (f walFormat) fileHeaderLen() int64 { | |||
| 53 | + | switch f { | |||
| 54 | + | case formatCurrentV1: | |||
| 55 | + | return fileHeaderV1Len | |||
| 56 | + | case formatCurrent: | |||
| 57 | + | return fileHeaderLen | |||
| 58 | + | default: | |||
| 59 | + | return 0 | |||
| 60 | + | } | |||
| 61 | + | } | |||
| 62 | + | ||||
| 49 | type headerStatus uint8 | 63 | type headerStatus uint8 | ||
| 50 | 64 | ||||
| 51 | const ( | 65 | const ( | ||
| @@ -466,10 +480,7 @@ func replayFile(f *os.File) (ReplayResult, error) { | |||||
| 466 | if err != nil { | 480 | if err != nil { | ||
| 467 | return ReplayResult{}, err | 481 | return ReplayResult{}, err | ||
| 468 | } | 482 | } | ||
| 469 | - | result := ReplayResult{Data: make(map[string][]byte), EntrySize: make(map[string]int64), Format: format} | 483 | + | result := ReplayResult{Data: make(map[string][]byte), EntrySize: make(map[string]int64), Format: format, LastGood: format.fileHeaderLen()} |
| 470 | - | if format == formatCurrent { | |||
| 471 | - | result.LastGood = fileHeaderLen | |||
| 472 | - | } | |||
| 473 | for { | 484 | for { | ||
| 474 | header, status, err := readRecordHeader(f, format, fileSize, result.LastGood) | 485 | header, status, err := readRecordHeader(f, format, fileSize, result.LastGood) | ||
| 475 | if err != nil { | 486 | if err != nil { | ||
| @@ -501,7 +512,7 @@ func replayFile(f *os.File) (ReplayResult, error) { | |||||
| 501 | records = []record{rec} | 512 | records = []record{rec} | ||
| 502 | } | 513 | } | ||
| 503 | if err != nil { | 514 | if err != nil { | ||
| 504 | - | return ReplayResult{}, fmt.Errorf("%w: invalid record at offset %d: %v", ErrCorruptWAL, result.LastGood, err) | 515 | + | return ReplayResult{}, fmt.Errorf("%w: invalid record at offset %d: %w", ErrCorruptWAL, result.LastGood, err) |
| 505 | } | 516 | } | ||
| 506 | size := format.headerLen() + int64(len(payload)) | 517 | size := format.headerLen() + int64(len(payload)) | ||
| 507 | applyRecords(result.Data, result.EntrySize, &result.Live, records, size) | 518 | applyRecords(result.Data, result.EntrySize, &result.Live, records, size) | ||
| @@ -513,23 +524,60 @@ func replayFile(f *os.File) (ReplayResult, error) { | |||||
| 513 | // readFileFormat consumes the file header when one is present and leaves the | 524 | // readFileFormat consumes the file header when one is present and leaves the | ||
| 514 | // reader positioned at the first record either way. | 525 | // reader positioned at the first record either way. | ||
| 515 | func readFileFormat(f io.ReadSeeker, fileSize int64) (walFormat, error) { | 526 | func readFileFormat(f io.ReadSeeker, fileSize int64) (walFormat, error) { | ||
| 516 | - | if fileSize < fileHeaderLen { | 527 | + | if fileSize == 0 { |
| 517 | return formatLegacy, nil | 528 | return formatLegacy, nil | ||
| 518 | } | 529 | } | ||
| 519 | - | header := make([]byte, fileHeaderLen) | 530 | + | probeLen := min(fileSize, int64(fileHeaderLen)) |
| 520 | - | if _, err := io.ReadFull(f, header); err != nil { | 531 | + | probe := make([]byte, probeLen) |
| 532 | + | if _, err := io.ReadFull(f, probe); err != nil { | |||
| 521 | return formatLegacy, err | 533 | return formatLegacy, err | ||
| 522 | } | 534 | } | ||
| 523 | - | if !bytes.Equal(header[:fileMagicLen], fileMagic[:]) { | 535 | + | if fileSize < fileMagicLen { |
| 536 | + | if bytes.Equal(probe, fileMagic[:len(probe)]) { | |||
| 537 | + | return formatLegacy, fmt.Errorf("%w: incomplete VEVK header", ErrCorruptWAL) | |||
| 538 | + | } | |||
| 524 | if _, err := f.Seek(0, io.SeekStart); err != nil { | 539 | if _, err := f.Seek(0, io.SeekStart); err != nil { | ||
| 525 | return formatLegacy, err | 540 | return formatLegacy, err | ||
| 526 | } | 541 | } | ||
| 527 | return formatLegacy, nil | 542 | return formatLegacy, nil | ||
| 528 | } | 543 | } | ||
| 529 | - | if version := binary.BigEndian.Uint16(header[fileMagicLen:]); version != formatVersion { | 544 | + | if !bytes.Equal(probe[:fileMagicLen], fileMagic[:]) { |
| 545 | + | // A complete v2 identity checksum authenticates the intended magic as | |||
| 546 | + | // well as the version. Detect it before legacy replay so damage to the | |||
| 547 | + | // first magic byte cannot masquerade as a small legacy length and be | |||
| 548 | + | // truncated or compacted away. | |||
| 549 | + | if len(probe) == fileHeaderLen && | |||
| 550 | + | binary.BigEndian.Uint16(probe[fileMagicLen:fileHeaderBodyLen]) == formatVersion && | |||
| 551 | + | binary.BigEndian.Uint32(probe[fileHeaderBodyLen:]) == binary.BigEndian.Uint32(fileHeader()[fileHeaderBodyLen:]) { | |||
| 552 | + | return formatLegacy, fmt.Errorf("%w: damaged VEVK v2 magic", ErrCorruptWAL) | |||
| 553 | + | } | |||
| 554 | + | if _, err := f.Seek(0, io.SeekStart); err != nil { | |||
| 555 | + | return formatLegacy, err | |||
| 556 | + | } | |||
| 557 | + | return formatLegacy, nil | |||
| 558 | + | } | |||
| 559 | + | if fileSize < fileHeaderV1Len { | |||
| 560 | + | return formatLegacy, fmt.Errorf("%w: incomplete VEVK header", ErrCorruptWAL) | |||
| 561 | + | } | |||
| 562 | + | prefix := probe[:fileHeaderV1Len] | |||
| 563 | + | if _, err := f.Seek(fileHeaderV1Len, io.SeekStart); err != nil { | |||
| 564 | + | return formatLegacy, err | |||
| 565 | + | } | |||
| 566 | + | switch version := binary.BigEndian.Uint16(prefix[fileMagicLen:]); version { | |||
| 567 | + | case formatVersionV1: | |||
| 568 | + | return formatCurrentV1, nil | |||
| 569 | + | case formatVersion: | |||
| 570 | + | checksum := make([]byte, fileHeaderLen-fileHeaderV1Len) | |||
| 571 | + | if _, err := io.ReadFull(f, checksum); err != nil { | |||
| 572 | + | return formatLegacy, fmt.Errorf("%w: incomplete VEVK v2 header", ErrCorruptWAL) | |||
| 573 | + | } | |||
| 574 | + | if crc32.ChecksumIEEE(prefix) != binary.BigEndian.Uint32(checksum) { | |||
| 575 | + | return formatLegacy, fmt.Errorf("%w: file header checksum mismatch", ErrCorruptWAL) | |||
| 576 | + | } | |||
| 577 | + | return formatCurrent, nil | |||
| 578 | + | default: | |||
| 530 | return formatLegacy, fmt.Errorf("%w: unsupported format version %d", ErrCorruptWAL, version) | 579 | return formatLegacy, fmt.Errorf("%w: unsupported format version %d", ErrCorruptWAL, version) | ||
| 531 | } | 580 | } | ||
| 532 | - | return formatCurrent, nil | |||
| 533 | } | 581 | } | ||
| 534 | 582 | ||||
| 535 | type recordHeader struct { | 583 | type recordHeader struct { | ||
| @@ -554,15 +602,18 @@ func readRecordHeader(f io.Reader, format walFormat, fileSize, offset int64) (re | |||||
| 554 | payloadLen: binary.BigEndian.Uint32(buf[0:4]), | 602 | payloadLen: binary.BigEndian.Uint32(buf[0:4]), | ||
| 555 | payloadCRC: binary.BigEndian.Uint32(buf[4:8]), | 603 | payloadCRC: binary.BigEndian.Uint32(buf[4:8]), | ||
| 556 | } | 604 | } | ||
| 557 | - | if format == formatCurrent { | 605 | + | if format != formatLegacy { |
| 558 | if crc32.ChecksumIEEE(buf[0:8]) != binary.BigEndian.Uint32(buf[8:recordHeaderLen]) { | 606 | if crc32.ChecksumIEEE(buf[0:8]) != binary.BigEndian.Uint32(buf[8:recordHeaderLen]) { | ||
| 559 | return recordHeader{}, headerOK, fmt.Errorf("%w: record header CRC mismatch at offset %d", ErrCorruptWAL, offset) | 607 | return recordHeader{}, headerOK, fmt.Errorf("%w: record header CRC mismatch at offset %d", ErrCorruptWAL, offset) | ||
| 560 | } | 608 | } | ||
| 561 | // The length is now trustworthy, so an implausible one is corruption | 609 | // The length is now trustworthy, so an implausible one is corruption | ||
| 562 | // rather than a short tail. Checked before any allocation. | 610 | // rather than a short tail. Checked before any allocation. | ||
| 563 | - | if header.payloadLen > maxPayloadLen { | 611 | + | } |
| 564 | - | return recordHeader{}, headerOK, fmt.Errorf("%w: record length %d exceeds maximum at offset %d", ErrCorruptWAL, header.payloadLen, offset) | 612 | + | // Every framing is allocation-bounded. In legacy framing an oversized |
| 565 | - | } | 613 | + | // declaration remains corruption rather than driving an attacker-sized |
| 614 | + | // allocation; ordinary short tails retain their released truncation semantics. | |||
| 615 | + | if header.payloadLen > maxPayloadLen { | |||
| 616 | + | return recordHeader{}, headerOK, fmt.Errorf("%w: record length %d exceeds maximum at offset %d", ErrCorruptWAL, header.payloadLen, offset) | |||
| 566 | } | 617 | } | ||
| 567 | if int64(header.payloadLen) > fileSize-offset-format.headerLen() { | 618 | if int64(header.payloadLen) > fileSize-offset-format.headerLen() { | ||
| 568 | return recordHeader{}, headerTorn, nil | 619 | return recordHeader{}, headerTorn, nil | ||
| @@ -577,9 +628,9 @@ func applyRecords(data map[string][]byte, sizes map[string]int64, live *int64, r | |||||
| 577 | setCount++ | 628 | setCount++ | ||
| 578 | } | 629 | } | ||
| 579 | } | 630 | } | ||
| 580 | - | setSize := size | 631 | + | setSize, remainder := size, int64(0) |
| 581 | if setCount > 1 { | 632 | if setCount > 1 { | ||
| 582 | - | setSize = size / setCount | 633 | + | setSize, remainder = size/setCount, size%setCount |
| 583 | } | 634 | } | ||
| 584 | for _, rec := range records { | 635 | for _, rec := range records { | ||
| 585 | key := string(rec.key) | 636 | key := string(rec.key) | ||
| @@ -587,9 +638,14 @@ func applyRecords(data map[string][]byte, sizes map[string]int64, live *int64, r | |||||
| 587 | *live -= old | 638 | *live -= old | ||
| 588 | } | 639 | } | ||
| 589 | if rec.op == opSet { | 640 | if rec.op == opSet { | ||
| 641 | + | entrySize := setSize | |||
| 642 | + | if remainder > 0 { | |||
| 643 | + | entrySize++ | |||
| 644 | + | remainder-- | |||
| 645 | + | } | |||
| 590 | data[key] = append([]byte(nil), rec.value...) | 646 | data[key] = append([]byte(nil), rec.value...) | ||
| 591 | - | sizes[key] = setSize | 647 | + | sizes[key] = entrySize |
| 592 | - | *live += setSize | 648 | + | *live += entrySize |
| 593 | } else { | 649 | } else { | ||
| 594 | delete(data, key) | 650 | delete(data, key) | ||
| 595 | delete(sizes, key) | 651 | delete(sizes, key) | ||
| @@ -598,7 +654,7 @@ func applyRecords(data map[string][]byte, sizes map[string]int64, live *int64, r | |||||
| 598 | } | 654 | } | ||
| 599 | 655 | ||||
| 600 | func recoverCompaction(path string, hooks fsHooks) error { | 656 | func recoverCompaction(path string, hooks fsHooks) error { | ||
| 601 | - | _, currentExists, currentErr := candidate(path) | 657 | + | currentResult, currentExists, currentErr := candidate(path) |
| 602 | nextResult, nextExists, nextErr := candidate(path + ".next") | 658 | nextResult, nextExists, nextErr := candidate(path + ".next") | ||
| 603 | _, prevExists, prevErr := candidate(path + ".prev") | 659 | _, prevExists, prevErr := candidate(path + ".prev") | ||
| 604 | if nextErr == nil && nextResult.TornTail { | 660 | if nextErr == nil && nextResult.TornTail { | ||
| @@ -606,6 +662,12 @@ func recoverCompaction(path string, hooks fsHooks) error { | |||||
| 606 | } | 662 | } | ||
| 607 | dir := filepath.Dir(path) | 663 | dir := filepath.Dir(path) | ||
| 608 | if currentExists && currentErr == nil { | 664 | if currentExists && currentErr == nil { | ||
| 665 | + | if currentResult.TornTail && prevExists { | |||
| 666 | + | // A predecessor proves compaction was in flight. The torn current is | |||
| 667 | + | // ambiguous, so preserve both candidates and fail closed rather than | |||
| 668 | + | // truncating current and deleting the last complete predecessor. | |||
| 669 | + | return fmt.Errorf("%w: torn current with predecessor", ErrCorruptWAL) | |||
| 670 | + | } | |||
| 609 | if nextExists { | 671 | if nextExists { | ||
| 610 | if err := hooks.Remove(path + ".next"); err != nil { | 672 | if err := hooks.Remove(path + ".next"); err != nil { | ||
| 611 | return err | 673 | return err | ||
| @@ -625,33 +687,23 @@ func recoverCompaction(path string, hooks fsHooks) error { | |||||
| 625 | return nil | 687 | return nil | ||
| 626 | } | 688 | } | ||
| 627 | if currentExists && currentErr != nil { | 689 | if currentExists && currentErr != nil { | ||
| 628 | - | if !prevExists || prevErr != nil { | 690 | + | // Corruption does not prove which bytes are disposable. Preserve every |
| 629 | - | return fmt.Errorf("%w: no valid current or predecessor", ErrCorruptWAL) | 691 | + | // candidate for diagnosis/recovery instead of deleting current or a valid |
| 630 | - | } | 692 | + | // predecessor and fail closed. |
| 631 | - | if err := hooks.Remove(path); err != nil { | 693 | + | return fmt.Errorf("%w: invalid current candidate", ErrCorruptWAL) |
| 632 | - | return err | 694 | + | } |
| 633 | - | } | 695 | + | if !currentExists && (nextExists || prevExists) { |
| 634 | - | if err := hooks.Rename(path+".prev", path); err != nil { | 696 | + | // A complete .next is the newest fully published compaction candidate and |
| 635 | - | return err | 697 | + | // wins even when .prev exists but is corrupt. Otherwise a complete .prev |
| 636 | - | } | 698 | + | // is the only safe rollback. Invalid candidates are never selected. |
| 637 | - | if err := hooks.SyncDir(dir); err != nil { | 699 | + | source := "" |
| 638 | - | return err | |||
| 639 | - | } | |||
| 640 | - | if nextExists { | |||
| 641 | - | if err := hooks.Remove(path + ".next"); err != nil { | |||
| 642 | - | return err | |||
| 643 | - | } | |||
| 644 | - | return hooks.SyncDir(dir) | |||
| 645 | - | } | |||
| 646 | - | return nil | |||
| 647 | - | } | |||
| 648 | - | if !currentExists && prevExists { | |||
| 649 | - | if prevErr != nil { | |||
| 650 | - | return fmt.Errorf("%w: invalid predecessor", ErrCorruptWAL) | |||
| 651 | - | } | |||
| 652 | - | source := path + ".prev" | |||
| 653 | if nextExists && nextErr == nil { | 700 | if nextExists && nextErr == nil { | ||
| 654 | source = path + ".next" | 701 | source = path + ".next" | ||
| 702 | + | } else if prevExists && prevErr == nil { | |||
| 703 | + | source = path + ".prev" | |||
| 704 | + | } | |||
| 705 | + | if source == "" { | |||
| 706 | + | return fmt.Errorf("%w: no valid current, successor, or predecessor", ErrCorruptWAL) | |||
| 655 | } | 707 | } | ||
| 656 | if err := hooks.Rename(source, path); err != nil { | 708 | if err := hooks.Rename(source, path); err != nil { | ||
| 657 | return err | 709 | return err | ||
| @@ -668,8 +720,8 @@ func recoverCompaction(path string, hooks fsHooks) error { | |||||
| 668 | } | 720 | } | ||
| 669 | return hooks.SyncDir(dir) | 721 | return hooks.SyncDir(dir) | ||
| 670 | } | 722 | } | ||
| 671 | - | if nextExists || (currentExists && currentErr != nil) || (prevExists && prevErr != nil) { | 723 | + | if currentExists && currentErr != nil { |
| 672 | - | return fmt.Errorf("%w: no valid current or predecessor", ErrCorruptWAL) | 724 | + | return fmt.Errorf("%w: no valid current", ErrCorruptWAL) |
| 673 | } | 725 | } | ||
| 674 | return nil | 726 | return nil | ||
| 675 | } | 727 | } | ||
M pkg/kv/kv_test.go +168 -19
| @@ -4,7 +4,9 @@ import ( | |||||
| 4 | "bytes" | 4 | "bytes" | ||
| 5 | "encoding/binary" | 5 | "encoding/binary" | ||
| 6 | "errors" | 6 | "errors" | ||
| 7 | + | "fmt" | |||
| 7 | "hash/crc32" | 8 | "hash/crc32" | ||
| 9 | + | "maps" | |||
| 8 | "os" | 10 | "os" | ||
| 9 | "path/filepath" | 11 | "path/filepath" | ||
| 10 | "reflect" | 12 | "reflect" | ||
| @@ -26,6 +28,17 @@ func currentFile(records ...[]byte) []byte { | |||||
| 26 | return buf | 28 | return buf | ||
| 27 | } | 29 | } | ||
| 28 | 30 | ||||
| 31 | + | // v1File is the previously released VEVK framing: magic and uint16 version, | |||
| 32 | + | // followed directly by the same self-checksummed records used today. | |||
| 33 | + | func v1File(records ...[]byte) []byte { | |||
| 34 | + | buf := append([]byte(nil), fileMagic[:]...) | |||
| 35 | + | buf = binary.BigEndian.AppendUint16(buf, formatVersionV1) | |||
| 36 | + | for _, record := range records { | |||
| 37 | + | buf = append(buf, record...) | |||
| 38 | + | } | |||
| 39 | + | return buf | |||
| 40 | + | } | |||
| 41 | + | ||||
| 29 | // legacyFile reproduces the framing written by the released version: no file | 42 | // legacyFile reproduces the framing written by the released version: no file | ||
| 30 | // header, and a record header of length plus payload CRC only. Compatibility | 43 | // header, and a record header of length plus payload CRC only. Compatibility | ||
| 31 | // with these files is a hard requirement, so the fixture stays in the tests. | 44 | // with these files is a hard requirement, so the fixture stays in the tests. | ||
| @@ -378,6 +391,34 @@ func TestRecoverCompactionCleansStaleNextRegardlessOfPrev(t *testing.T) { | |||||
| 378 | } | 391 | } | ||
| 379 | } | 392 | } | ||
| 380 | 393 | ||||
| 394 | + | func TestReplayPreservesWrappedDecodeCause(t *testing.T) { | |||
| 395 | + | path := privatePath(t) | |||
| 396 | + | writeFile(t, path, currentFile(rawRecord([]byte{0xff, 0, 0}))) | |||
| 397 | + | f, err := os.Open(path) | |||
| 398 | + | if err != nil { | |||
| 399 | + | t.Fatal(err) | |||
| 400 | + | } | |||
| 401 | + | defer func() { _ = f.Close() }() | |||
| 402 | + | _, err = replayFile(f) | |||
| 403 | + | if !errors.Is(err, ErrCorruptWAL) || !errors.Is(err, errBadRecord) { | |||
| 404 | + | t.Fatalf("replayFile error = %v, want wrapped ErrCorruptWAL and errBadRecord", err) | |||
| 405 | + | } | |||
| 406 | + | } | |||
| 407 | + | ||||
| 408 | + | func TestBatchLiveSizeAccountsForRemainderExactly(t *testing.T) { | |||
| 409 | + | data := map[string][]byte{} | |||
| 410 | + | sizes := map[string]int64{} | |||
| 411 | + | var live int64 | |||
| 412 | + | applyRecords(data, sizes, &live, []record{ | |||
| 413 | + | {op: opSet, key: []byte("a"), value: []byte("one")}, | |||
| 414 | + | {op: opSet, key: []byte("b"), value: []byte("two")}, | |||
| 415 | + | {op: opSet, key: []byte("c"), value: []byte("three")}, | |||
| 416 | + | }, 10) | |||
| 417 | + | if live != 10 || sizes["a"]+sizes["b"]+sizes["c"] != 10 { | |||
| 418 | + | t.Fatalf("live sizes = %v, live = %d, want exact total 10", sizes, live) | |||
| 419 | + | } | |||
| 420 | + | } | |||
| 421 | + | ||||
| 381 | func TestDecodeBatchRejectsImpossibleCountBeforeAllocating(t *testing.T) { | 422 | func TestDecodeBatchRejectsImpossibleCountBeforeAllocating(t *testing.T) { | ||
| 382 | payload := []byte{opBatch, 0xff, 0xff, 0xff, 0xff} | 423 | payload := []byte{opBatch, 0xff, 0xff, 0xff, 0xff} | ||
| 383 | allocs := testing.AllocsPerRun(10, func() { | 424 | allocs := testing.AllocsPerRun(10, func() { | ||
| @@ -429,14 +470,17 @@ func TestCompactionRecoveryMatrix(t *testing.T) { | |||||
| 429 | name string | 470 | name string | ||
| 430 | current, next, prev []byte | 471 | current, next, prev []byte | ||
| 431 | want string | 472 | want string | ||
| 473 | + | wantErr bool | |||
| 432 | }{ | 474 | }{ | ||
| 433 | {name: "current-alone", current: oldFile, want: "old"}, | 475 | {name: "current-alone", current: oldFile, want: "old"}, | ||
| 434 | {name: "current-next", current: oldFile, next: newFile, want: "old"}, | 476 | {name: "current-next", current: oldFile, next: newFile, want: "old"}, | ||
| 435 | {name: "prev-next", prev: oldFile, next: newFile, want: "new"}, | 477 | {name: "prev-next", prev: oldFile, next: newFile, want: "new"}, | ||
| 436 | - | {name: "prev-invalid-next", prev: oldFile, next: currentFile(rawRecord([]byte{0xff, 0, 0})), want: "old"}, | 478 | + | {name: "next-alone", next: newFile, want: "new"}, |
| 479 | + | {name: "invalid-prev-valid-next", prev: currentFile(rawRecord([]byte{0xff, 0, 0})), next: newFile, want: "new"}, | |||
| 480 | + | {name: "valid-prev-invalid-next", prev: oldFile, next: currentFile(rawRecord([]byte{0xff, 0, 0})), want: "old"}, | |||
| 437 | {name: "prev-torn-next", prev: oldFile, next: newFile[:len(newFile)-1], want: "old"}, | 481 | {name: "prev-torn-next", prev: oldFile, next: newFile[:len(newFile)-1], want: "old"}, | ||
| 438 | {name: "current-prev", current: newFile, prev: oldFile, want: "new"}, | 482 | {name: "current-prev", current: newFile, prev: oldFile, want: "new"}, | ||
| 439 | - | {name: "invalid-current-prev", current: currentFile(rawRecord([]byte{0xff, 0, 0})), prev: oldFile, want: "old"}, | 483 | + | {name: "invalid-current-prev", current: currentFile(rawRecord([]byte{0xff, 0, 0})), prev: oldFile, wantErr: true}, |
| 440 | {name: "prev-alone", prev: oldFile, want: "old"}, | 484 | {name: "prev-alone", prev: oldFile, want: "old"}, | ||
| 441 | } | 485 | } | ||
| 442 | for _, tt := range tests { | 486 | for _, tt := range tests { | ||
| @@ -452,7 +496,18 @@ func TestCompactionRecoveryMatrix(t *testing.T) { | |||||
| 452 | } | 496 | } | ||
| 453 | } | 497 | } | ||
| 454 | } | 498 | } | ||
| 455 | - | if err := recoverCompaction(path, defaultFSHooks()); err != nil { | 499 | + | before := candidateBytes(t, path) |
| 500 | + | err := recoverCompaction(path, defaultFSHooks()) | |||
| 501 | + | if tt.wantErr { | |||
| 502 | + | if !errors.Is(err, ErrCorruptWAL) { | |||
| 503 | + | t.Fatalf("recoverCompaction error = %v, want ErrCorruptWAL", err) | |||
| 504 | + | } | |||
| 505 | + | if got := candidateBytes(t, path); !reflect.DeepEqual(got, before) { | |||
| 506 | + | t.Fatalf("uncertain candidates changed: got %#v, want %#v", got, before) | |||
| 507 | + | } | |||
| 508 | + | return | |||
| 509 | + | } | |||
| 510 | + | if err != nil { | |||
| 456 | t.Fatal(err) | 511 | t.Fatal(err) | ||
| 457 | } | 512 | } | ||
| 458 | got, err := Replay(path) | 513 | got, err := Replay(path) | ||
| @@ -701,25 +756,85 @@ func rawBatchPayload(changes []BatchChange) []byte { | |||||
| 701 | return payload | 756 | return payload | ||
| 702 | } | 757 | } | ||
| 703 | 758 | ||||
| 704 | - | // An oversized length must never drive an allocation. In a legacy file the | 759 | + | // An oversized length must never drive an allocation. Legacy framing is |
| 705 | - | // length is unprotected, so it stays a torn tail; in the current format the | 760 | + | // accepted only after complete strict replay, while the current header checksum |
| 706 | - | // header checksum proves the length was written that way, which is corruption. | 761 | + | // proves an oversized length was written that way; both fail closed. |
| 762 | + | func TestCorruptedCurrentHeaderNeverFallsBackToLegacyOrDeletesPredecessor(t *testing.T) { | |||
| 763 | + | record, _ := encodeRecord(opSet, []byte("version"), []byte("current")) | |||
| 764 | + | previous, _ := encodeRecord(opSet, []byte("version"), []byte("previous")) | |||
| 765 | + | ||||
| 766 | + | tests := []struct { | |||
| 767 | + | name string | |||
| 768 | + | current []byte | |||
| 769 | + | standalone bool | |||
| 770 | + | }{ | |||
| 771 | + | {name: "torn file header", current: append([]byte(nil), fileHeader()[:fileHeaderLen-1]...), standalone: true}, | |||
| 772 | + | {name: "torn current record", current: currentFile(record)[:fileHeaderLen+recordHeaderLen+1]}, | |||
| 773 | + | } | |||
| 774 | + | for i := range fileMagicLen { | |||
| 775 | + | corrupted := currentFile(record) | |||
| 776 | + | corrupted[i] = 0 | |||
| 777 | + | tests = append(tests, struct { | |||
| 778 | + | name string | |||
| 779 | + | current []byte | |||
| 780 | + | standalone bool | |||
| 781 | + | }{name: fmt.Sprintf("corrupted magic byte %d", i), current: corrupted, standalone: true}) | |||
| 782 | + | } | |||
| 783 | + | for length := 1; length < fileHeaderLen; length++ { | |||
| 784 | + | tests = append(tests, struct { | |||
| 785 | + | name string | |||
| 786 | + | current []byte | |||
| 787 | + | standalone bool | |||
| 788 | + | }{name: fmt.Sprintf("torn header prefix %d", length), current: append([]byte(nil), fileHeader()[:length]...), standalone: true}) | |||
| 789 | + | } | |||
| 790 | + | ||||
| 791 | + | for _, tt := range tests { | |||
| 792 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 793 | + | for _, withCandidates := range []bool{false, true} { | |||
| 794 | + | if !withCandidates && !tt.standalone { | |||
| 795 | + | continue | |||
| 796 | + | } | |||
| 797 | + | name := "standalone" | |||
| 798 | + | if withCandidates { | |||
| 799 | + | name = "with candidates" | |||
| 800 | + | } | |||
| 801 | + | t.Run(name, func(t *testing.T) { | |||
| 802 | + | path := privatePath(t) | |||
| 803 | + | writeFile(t, path, tt.current) | |||
| 804 | + | if withCandidates { | |||
| 805 | + | if err := os.WriteFile(path+".prev", currentFile(previous), 0o600); err != nil { | |||
| 806 | + | t.Fatal(err) | |||
| 807 | + | } | |||
| 808 | + | if err := os.WriteFile(path+".next", currentFile(record), 0o600); err != nil { | |||
| 809 | + | t.Fatal(err) | |||
| 810 | + | } | |||
| 811 | + | } | |||
| 812 | + | before := candidateBytes(t, path) | |||
| 813 | + | if _, err := Open(path); !errors.Is(err, ErrCorruptWAL) { | |||
| 814 | + | t.Fatalf("Open error = %v, want ErrCorruptWAL", err) | |||
| 815 | + | } | |||
| 816 | + | if got := candidateBytes(t, path); !reflect.DeepEqual(got, before) { | |||
| 817 | + | t.Fatalf("candidates changed: got %#v, want %#v", got, before) | |||
| 818 | + | } | |||
| 819 | + | }) | |||
| 820 | + | } | |||
| 821 | + | }) | |||
| 822 | + | } | |||
| 823 | + | } | |||
| 824 | + | ||||
| 707 | func TestReplayRejectsOversizedLengthWithoutAllocating(t *testing.T) { | 825 | func TestReplayRejectsOversizedLengthWithoutAllocating(t *testing.T) { | ||
| 708 | first, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | 826 | first, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | ||
| 709 | legacyHeader := make([]byte, legacyHeaderLen) | 827 | legacyHeader := make([]byte, legacyHeaderLen) | ||
| 710 | binary.BigEndian.PutUint32(legacyHeader[0:4], 1<<31) | 828 | binary.BigEndian.PutUint32(legacyHeader[0:4], 1<<31) | ||
| 711 | 829 | ||||
| 712 | - | t.Run("legacy length is a torn tail", func(t *testing.T) { | 830 | + | t.Run("legacy incomplete record fails closed", func(t *testing.T) { |
| 713 | path := privatePath(t) | 831 | path := privatePath(t) | ||
| 714 | - | writeFile(t, path, append(legacyFile(first), legacyHeader...)) | 832 | + | wal := append(legacyFile(first), legacyHeader...) |
| 715 | - | s, err := Open(path) | 833 | + | writeFile(t, path, wal) |
| 716 | - | if err != nil { | 834 | + | if _, err := Open(path); !errors.Is(err, ErrCorruptWAL) { |
| 717 | - | t.Fatal(err) | 835 | + | t.Fatalf("Open error = %v, want ErrCorruptWAL", err) |
| 718 | - | } | |||
| 719 | - | defer closeStore(t, s) | |||
| 720 | - | if got, ok := s.Get([]byte("a")); !ok || string(got) != "one" { | |||
| 721 | - | t.Fatalf("good record missing before bad length: %q %v", got, ok) | |||
| 722 | } | 836 | } | ||
| 837 | + | requireFileBytes(t, path, wal) | |||
| 723 | }) | 838 | }) | ||
| 724 | 839 | ||||
| 725 | t.Run("current oversized length fails closed", func(t *testing.T) { | 840 | t.Run("current oversized length fails closed", func(t *testing.T) { | ||
| @@ -851,13 +966,49 @@ func TestCurrentFormatReplayDecisionTable(t *testing.T) { | |||||
| 851 | 966 | ||||
| 852 | // A file written by the released version must keep working: it opens with the | 967 | // A file written by the released version must keep working: it opens with the | ||
| 853 | // original semantics, is upgraded in place, and reopens identically. | 968 | // original semantics, is upgraded in place, and reopens identically. | ||
| 969 | + | func TestVEVKV1FileOpensAndUpgradesInPlace(t *testing.T) { | |||
| 970 | + | setA, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | |||
| 971 | + | setB, _ := encodeRecord(opSet, []byte("b"), bytes.Repeat([]byte("x"), 1<<20)) | |||
| 972 | + | for _, tc := range []struct { | |||
| 973 | + | name string | |||
| 974 | + | wal []byte | |||
| 975 | + | }{ | |||
| 976 | + | {name: "ordinary catalogue payload", wal: v1File(setA)}, | |||
| 977 | + | {name: "prior large catalogue payload", wal: v1File(setB)}, | |||
| 978 | + | } { | |||
| 979 | + | t.Run(tc.name, func(t *testing.T) { | |||
| 980 | + | path := privatePath(t) | |||
| 981 | + | writeFile(t, path, tc.wal) | |||
| 982 | + | store, err := Open(path) | |||
| 983 | + | if err != nil { | |||
| 984 | + | t.Fatal(err) | |||
| 985 | + | } | |||
| 986 | + | closeStore(t, store) | |||
| 987 | + | upgraded, err := os.ReadFile(path) | |||
| 988 | + | if err != nil { | |||
| 989 | + | t.Fatal(err) | |||
| 990 | + | } | |||
| 991 | + | if !bytes.Equal(upgraded[:fileHeaderLen], fileHeader()) { | |||
| 992 | + | t.Fatalf("upgraded header = %x, want %x", upgraded[:fileHeaderLen], fileHeader()) | |||
| 993 | + | } | |||
| 994 | + | }) | |||
| 995 | + | } | |||
| 996 | + | } | |||
| 997 | + | ||||
| 998 | + | func TestVEVKV1RejectsOversizedRecordBeforeAllocation(t *testing.T) { | |||
| 999 | + | path := privatePath(t) | |||
| 1000 | + | writeFile(t, path, v1File(oversizedRecordHeader(maxPayloadLen+1))) | |||
| 1001 | + | if _, err := Open(path); !errors.Is(err, ErrCorruptWAL) { | |||
| 1002 | + | t.Fatalf("Open error = %v, want ErrCorruptWAL", err) | |||
| 1003 | + | } | |||
| 1004 | + | } | |||
| 1005 | + | ||||
| 854 | func TestLegacyFileOpensAndUpgradesInPlace(t *testing.T) { | 1006 | func TestLegacyFileOpensAndUpgradesInPlace(t *testing.T) { | ||
| 855 | setA, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | 1007 | setA, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | ||
| 856 | setB, _ := encodeRecord(opSet, []byte("b"), []byte("two")) | 1008 | setB, _ := encodeRecord(opSet, []byte("b"), []byte("two")) | ||
| 857 | replaceA, _ := encodeRecord(opSet, []byte("a"), []byte("uno")) | 1009 | replaceA, _ := encodeRecord(opSet, []byte("a"), []byte("uno")) | ||
| 858 | delB, _ := encodeRecord(opDel, []byte("b"), nil) | 1010 | delB, _ := encodeRecord(opDel, []byte("b"), nil) | ||
| 859 | torn, _ := encodeRecord(opSet, []byte("c"), []byte("three")) | 1011 | torn, _ := encodeRecord(opSet, []byte("c"), []byte("three")) | ||
| 860 | - | ||||
| 861 | tests := []struct { | 1012 | tests := []struct { | ||
| 862 | name string | 1013 | name string | ||
| 863 | wal []byte | 1014 | wal []byte | ||
| @@ -896,9 +1047,7 @@ func TestLegacyFileOpensAndUpgradesInPlace(t *testing.T) { | |||||
| 896 | } | 1047 | } | ||
| 897 | defer closeStore(t, reopened) | 1048 | defer closeStore(t, reopened) | ||
| 898 | want := map[string]string{"added": "after-upgrade"} | 1049 | want := map[string]string{"added": "after-upgrade"} | ||
| 899 | - | for k, v := range tt.want { | 1050 | + | maps.Copy(want, tt.want) |
| 900 | - | want[k] = v | |||
| 901 | - | } | |||
| 902 | requireContents(t, reopened, want) | 1051 | requireContents(t, reopened, want) | ||
| 903 | replayed, err := Replay(path) | 1052 | replayed, err := Replay(path) | ||
| 904 | if err != nil { | 1053 | if err != nil { | ||