fix(recovery): harden durable session invariants
- commit
- 979dbfb6387143dee313544090da2238ce9508b9
- author
- brice · · 2026-07-26 07:25 UTC
- parent
- a6f1d55
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
80 files · +2738 -1260
M internal/adapters/noticefile/noticefile.go +2 -7
| @@ -311,13 +311,8 @@ func appendNoticeFragment(line, fragment []byte) ([]byte, bool) { | |||
| 311 | 311 | } | |
| 312 | 312 | required := len(line) + len(fragment) | |
| 313 | 313 | if required > cap(line) { | |
| 314 | - | capacity := cap(line) * 2 | |
| 315 | - | if capacity < noticeReadBufferSize { | |
| 316 | - | capacity = noticeReadBufferSize | |
| 317 | - | } | |
| 318 | - | if capacity > maxNoticeRecordSize { | |
| 319 | - | capacity = maxNoticeRecordSize | |
| 320 | - | } | |
| 314 | + | capacity := max(cap(line)*2, noticeReadBufferSize) | |
| 315 | + | capacity = min(capacity, maxNoticeRecordSize) | |
| 321 | 316 | grown := make([]byte, len(line), capacity) | |
| 322 | 317 | copy(grown, line) | |
| 323 | 318 | line = grown | |
M internal/adapters/recoveryfs/journal.go +26 -12
| @@ -104,16 +104,33 @@ func (j *Journal) ListDiscards(ctx context.Context) ([]domain.DiscardIntent, err | |||
| 104 | 104 | if err != nil { | |
| 105 | 105 | return nil, err | |
| 106 | 106 | } | |
| 107 | - | entries, readErr := dir.ReadDir(maxDiscardIntents + 1) | |
| 108 | - | closeErr := dir.Close() | |
| 109 | - | if readErr != nil && !errors.Is(readErr, io.EOF) { | |
| 110 | - | return nil, readErr | |
| 111 | - | } | |
| 112 | - | if closeErr != nil { | |
| 113 | - | return nil, closeErr | |
| 107 | + | entries := make([]os.DirEntry, 0, maxDiscardIntents) | |
| 108 | + | for { | |
| 109 | + | batch, readErr := dir.ReadDir(maxDiscardIntents + 1) | |
| 110 | + | for _, entry := range batch { | |
| 111 | + | if strings.HasPrefix(entry.Name(), ".intent-") && !entry.IsDir() { | |
| 112 | + | continue | |
| 113 | + | } | |
| 114 | + | if entry.IsDir() || !strings.HasPrefix(entry.Name(), "discard-") { | |
| 115 | + | _ = dir.Close() | |
| 116 | + | return nil, errors.New("recovery journal: invalid transaction entry") | |
| 117 | + | } | |
| 118 | + | entries = append(entries, entry) | |
| 119 | + | if len(entries) > maxDiscardIntents { | |
| 120 | + | _ = dir.Close() | |
| 121 | + | return nil, errors.New("recovery journal: transaction entry limit exceeded") | |
| 122 | + | } | |
| 123 | + | } | |
| 124 | + | if errors.Is(readErr, io.EOF) { | |
| 125 | + | break | |
| 126 | + | } | |
| 127 | + | if readErr != nil { | |
| 128 | + | _ = dir.Close() | |
| 129 | + | return nil, readErr | |
| 130 | + | } | |
| 114 | 131 | } | |
| 115 | - | if len(entries) > maxDiscardIntents { | |
| 116 | - | return nil, errors.New("recovery journal: transaction entry limit exceeded") | |
| 132 | + | if err := dir.Close(); err != nil { | |
| 133 | + | return nil, err | |
| 117 | 134 | } | |
| 118 | 135 | sort.Slice(entries, func(a, b int) bool { return entries[a].Name() < entries[b].Name() }) | |
| 119 | 136 | out := make([]domain.DiscardIntent, 0, len(entries)) | |
| @@ -122,9 +139,6 @@ func (j *Journal) ListDiscards(ctx context.Context) ([]domain.DiscardIntent, err | |||
| 122 | 139 | if err := ctx.Err(); err != nil { | |
| 123 | 140 | return nil, err | |
| 124 | 141 | } | |
| 125 | - | if entry.IsDir() || !strings.HasPrefix(entry.Name(), "discard-") { | |
| 126 | - | return nil, errors.New("recovery journal: invalid transaction entry") | |
| 127 | - | } | |
| 128 | 142 | data, err := readFileBounded(filepath.Join(j.dir, entry.Name()), maxDiscardIntentSize) | |
| 129 | 143 | if err != nil { | |
| 130 | 144 | return nil, err | |
M internal/adapters/recoveryfs/journal_test.go +13 -0
| @@ -38,6 +38,7 @@ func TestDiscardCrashMatrix(t *testing.T) { | |||
| 38 | 38 | path := j.path(old) | |
| 39 | 39 | encoded, err := os.ReadFile(path) | |
| 40 | 40 | require.NoError(t, err) | |
| 41 | + | t.Cleanup(func() { require.NoError(t, os.WriteFile(path, encoded, 0o600)) }) | |
| 41 | 42 | require.NoError(t, os.WriteFile(path, append(encoded, 0), 0o600)) | |
| 42 | 43 | _, err = j.ListDiscards(ctx) | |
| 43 | 44 | require.Error(t, err) | |
| @@ -77,6 +78,18 @@ func TestRecoveryJournalRoundTripAndIdempotentDelete(t *testing.T) { | |||
| 77 | 78 | require.Empty(t, got) | |
| 78 | 79 | } | |
| 79 | 80 | ||
| 81 | + | func TestRecoveryJournalIgnoresAtomicLeftoversWhenCountingDiscards(t *testing.T) { | |
| 82 | + | j := New(t.TempDir()) | |
| 83 | + | require.NoError(t, os.MkdirAll(j.dir, 0o700)) | |
| 84 | + | for i := 0; i < maxDiscardIntents+1; i++ { | |
| 85 | + | require.NoError(t, os.WriteFile(filepath.Join(j.dir, ".intent-leftover-"+string(rune(0x1000+i))), nil, 0o600)) | |
| 86 | + | } | |
| 87 | + | ||
| 88 | + | got, err := j.ListDiscards(context.Background()) | |
| 89 | + | require.NoError(t, err) | |
| 90 | + | require.Empty(t, got) | |
| 91 | + | } | |
| 92 | + | ||
| 80 | 93 | func TestRecoveryJournalRejectsMalformedIntent(t *testing.T) { | |
| 81 | 94 | j := New(t.TempDir()) | |
| 82 | 95 | require.Error(t, j.SaveDiscard(context.Background(), domain.DiscardIntent{})) | |
M internal/adapters/snapshot/atomic.go +4 -3
| @@ -13,15 +13,16 @@ import ( | |||
| 13 | 13 | ) | |
| 14 | 14 | ||
| 15 | 15 | func (r *Repository) ensureSession(id domain.IncarnationID) error { | |
| 16 | + | sessionPath := r.sessionPath(id) | |
| 16 | 17 | for _, directory := range []struct { | |
| 17 | 18 | path string | |
| 18 | 19 | phase string | |
| 19 | 20 | }{ | |
| 20 | 21 | {r.dir, "repository"}, | |
| 21 | 22 | {filepath.Join(r.dir, repositorySessionsDir), "sessions"}, | |
| 22 | - | {r.sessionPath(id), "session"}, | |
| 23 | - | {filepath.Join(r.sessionPath(id), repositoryObjectsDir), "objects"}, | |
| 24 | - | {filepath.Join(r.sessionPath(id), repositoryGenerations), "generations"}, | |
| 23 | + | {sessionPath, "session"}, | |
| 24 | + | {filepath.Join(sessionPath, repositoryObjectsDir), "objects"}, | |
| 25 | + | {filepath.Join(sessionPath, repositoryGenerations), "generations"}, | |
| 25 | 26 | } { | |
| 26 | 27 | if err := r.ensurePrivateDirectoryPhase(directory.path, directory.phase); err != nil { | |
| 27 | 28 | return fmt.Errorf("create snapshot repository directory: %w", err) | |
M internal/adapters/snapshot/legacy_load_test.go +0 -4
| @@ -69,7 +69,6 @@ func (r *Repository) Load(ctx context.Context, name string) (ports.SnapshotGener | |||
| 69 | 69 | ||
| 70 | 70 | func (r *Repository) load(ctx context.Context, name, key string) (ports.SnapshotGeneration, error) { | |
| 71 | 71 | preferred, preferredDigest, err := r.readLegacyHead(key) | |
| 72 | - | skipped := errors.Is(err, ErrInvalidHEAD) | |
| 73 | 72 | if err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, ErrInvalidHEAD) { | |
| 74 | 73 | return ports.SnapshotGeneration{}, fmt.Errorf("read snapshot HEAD: %w", err) | |
| 75 | 74 | } | |
| @@ -78,7 +77,6 @@ func (r *Repository) load(ctx context.Context, name, key string) (ports.Snapshot | |||
| 78 | 77 | if err == nil && sha256.Sum256(got.Manifest) == preferredDigest { | |
| 79 | 78 | return got, nil | |
| 80 | 79 | } | |
| 81 | - | skipped = true | |
| 82 | 80 | } | |
| 83 | 81 | ||
| 84 | 82 | budget := maxDirectoryTraversalEntries | |
| @@ -89,10 +87,8 @@ func (r *Repository) load(ctx context.Context, name, key string) (ports.Snapshot | |||
| 89 | 87 | for _, generation := range candidates { | |
| 90 | 88 | got, err := r.loadGeneration(ctx, name, key, generation) | |
| 91 | 89 | if err != nil { | |
| 92 | - | skipped = true | |
| 93 | 90 | continue | |
| 94 | 91 | } | |
| 95 | - | _ = skipped | |
| 96 | 92 | return got, nil | |
| 97 | 93 | } | |
| 98 | 94 | return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name) | |
M internal/adapters/snapshot/load_bounds_test.go +10 -7
| @@ -15,7 +15,7 @@ func TestRepositoryListStopsAtTraversalBudget(t *testing.T) { | |||
| 15 | 15 | if err := os.MkdirAll(sessions, 0o700); err != nil { | |
| 16 | 16 | t.Fatal(err) | |
| 17 | 17 | } | |
| 18 | - | for i := 0; i < maxDirectoryTraversalEntries+1; i++ { | |
| 18 | + | for i := range maxDirectoryTraversalEntries + 1 { | |
| 19 | 19 | if err := os.Mkdir(filepath.Join(sessions, fmt.Sprintf("hostile-%05d", i)), 0o700); err != nil { | |
| 20 | 20 | t.Fatal(err) | |
| 21 | 21 | } | |
| @@ -34,7 +34,8 @@ func TestRepositoryLoadStopsAtGenerationTraversalBudget(t *testing.T) { | |||
| 34 | 34 | if err := os.MkdirAll(generations, 0o700); err != nil { | |
| 35 | 35 | t.Fatal(err) | |
| 36 | 36 | } | |
| 37 | - | for generation := uint64(1); generation <= uint64(maxDirectoryTraversalEntries+1); generation++ { | |
| 37 | + | for i := range maxDirectoryTraversalEntries + 1 { | |
| 38 | + | generation := uint64(i + 1) | |
| 38 | 39 | if err := os.WriteFile(filepath.Join(generations, generationFilename(generation)), nil, 0o600); err != nil { | |
| 39 | 40 | t.Fatal(err) | |
| 40 | 41 | } | |
| @@ -48,8 +49,9 @@ func TestRepositoryLoadStopsAtGenerationTraversalBudget(t *testing.T) { | |||
| 48 | 49 | ||
| 49 | 50 | func TestRepositoryLoadFallsBackNewestToOldestWithoutGenerationEnumeration(t *testing.T) { | |
| 50 | 51 | repo := NewRepository(privateDir(t)) | |
| 51 | - | for generation := uint64(1); generation <= 3; generation++ { | |
| 52 | - | if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation)))); err != nil { | |
| 52 | + | for i := range 3 { | |
| 53 | + | generation := uint64(i + 1) | |
| 54 | + | if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))); err != nil { | |
| 53 | 55 | t.Fatal(err) | |
| 54 | 56 | } | |
| 55 | 57 | } | |
| @@ -70,12 +72,13 @@ func TestRepositoryLoadFallsBackNewestToOldestWithoutGenerationEnumeration(t *te | |||
| 70 | 72 | ||
| 71 | 73 | func TestRepositoryMaintainDoesNotCollectBeyondRetainedMetadataBudget(t *testing.T) { | |
| 72 | 74 | repo := NewRepository(privateDir(t)) | |
| 73 | - | for generation := uint64(1); generation <= uint64(maxMaintenanceMarkedGenerations+1); generation++ { | |
| 74 | - | if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, []byte(fmt.Sprintf("state-%d", generation)))); err != nil { | |
| 75 | + | for i := range maxMaintenanceMarkedGenerations + 1 { | |
| 76 | + | generation := uint64(i + 1) | |
| 77 | + | if err := repo.Publish(context.Background(), repositoryPublicationAfter(t, repo, "named", generation, fmt.Appendf(nil, "state-%d", generation))); err != nil { | |
| 75 | 78 | t.Fatal(err) | |
| 76 | 79 | } | |
| 77 | 80 | } | |
| 78 | - | for pass := 0; pass < maxMaintenanceMarkedGenerations*3; pass++ { | |
| 81 | + | for pass := range maxMaintenanceMarkedGenerations * 3 { | |
| 79 | 82 | if err := repo.Maintain(context.Background()); err != nil { | |
| 80 | 83 | t.Fatal(err) | |
| 81 | 84 | } | |
M internal/adapters/snapshot/maintenance_delete.go +1 -22
| @@ -644,28 +644,7 @@ func (r *Repository) SaveQuarantineDescriptor(ctx context.Context, descriptor do | |||
| 644 | 644 | ||
| 645 | 645 | func quarantineDescriptorsEqual(a, b domain.QuarantineDescriptor) bool { | |
| 646 | 646 | return a.OldIncarnation == b.OldIncarnation && a.ReplacementIncarnation == b.ReplacementIncarnation && | |
| 647 | - | a.SessionName == b.SessionName && a.Reason == b.Reason && catalogueRecordsEqual(a.OldRecord, b.OldRecord) | |
| 648 | - | } | |
| 649 | - | ||
| 650 | - | func catalogueRecordsEqual(a, b domain.CatalogueRecord) bool { | |
| 651 | - | if a.Name != b.Name || a.IncarnationID != b.IncarnationID || a.Cwd != b.Cwd || a.CreatedAt != b.CreatedAt || | |
| 652 | - | a.UpdatedAt != b.UpdatedAt || a.LastUsedSeq != b.LastUsedSeq || a.RecoveryState != b.RecoveryState || | |
| 653 | - | a.DegradedReason != b.DegradedReason || len(a.TabNames) != len(b.TabNames) { | |
| 654 | - | return false | |
| 655 | - | } | |
| 656 | - | for i := range a.TabNames { | |
| 657 | - | if a.TabNames[i] != b.TabNames[i] { | |
| 658 | - | return false | |
| 659 | - | } | |
| 660 | - | } | |
| 661 | - | return checkpointRefsEqual(a.Committed, b.Committed) && checkpointRefsEqual(a.Fallbacks[0], b.Fallbacks[0]) && checkpointRefsEqual(a.Fallbacks[1], b.Fallbacks[1]) | |
| 662 | - | } | |
| 663 | - | ||
| 664 | - | func checkpointRefsEqual(a, b *domain.CheckpointRef) bool { | |
| 665 | - | if a == nil || b == nil { | |
| 666 | - | return a == nil && b == nil | |
| 667 | - | } | |
| 668 | - | return *a == *b | |
| 647 | + | a.SessionName == b.SessionName && a.Reason == b.Reason && a.OldRecord.Equal(b.OldRecord) | |
| 669 | 648 | } | |
| 670 | 649 | ||
| 671 | 650 | func (r *Repository) QuarantineIncarnation(ctx context.Context, id domain.IncarnationID) error { | |
M internal/adapters/snapshot/maintenance_gc.go +88 -61
| @@ -31,6 +31,37 @@ type manifestMaintenance struct { | |||
| 31 | 31 | complete bool | |
| 32 | 32 | } | |
| 33 | 33 | ||
| 34 | + | type retentionEntryKind uint8 | |
| 35 | + | ||
| 36 | + | const ( | |
| 37 | + | retentionManifest retentionEntryKind = iota + 1 | |
| 38 | + | retentionObject | |
| 39 | + | ) | |
| 40 | + | ||
| 41 | + | type retentionEntryPolicy struct { | |
| 42 | + | kind retentionEntryKind | |
| 43 | + | generations map[uint64]ports.SnapshotDigest | |
| 44 | + | objects map[ports.SnapshotDigest]struct{} | |
| 45 | + | } | |
| 46 | + | ||
| 47 | + | func (p retentionEntryPolicy) delete(entry maintenanceDirEntry) bool { | |
| 48 | + | if entry.isDir { | |
| 49 | + | return false | |
| 50 | + | } | |
| 51 | + | switch p.kind { | |
| 52 | + | case retentionManifest: | |
| 53 | + | generation, canonical := parseGenerationFilename(entry.name) | |
| 54 | + | _, retained := p.generations[generation] | |
| 55 | + | return canonical && !retained | |
| 56 | + | case retentionObject: | |
| 57 | + | digest, canonical := parseObjectDigest(entry.name) | |
| 58 | + | _, retained := p.objects[digest] | |
| 59 | + | return canonical && !retained | |
| 60 | + | default: | |
| 61 | + | return false | |
| 62 | + | } | |
| 63 | + | } | |
| 64 | + | ||
| 34 | 65 | type sessionMaintenance struct { | |
| 35 | 66 | lock *sessionMutex | |
| 36 | 67 | token string | |
| @@ -370,6 +401,44 @@ func keepSet(plan ports.RetentionPlan) map[uint64]ports.SnapshotDigest { | |||
| 370 | 401 | ||
| 371 | 402 | // MaintainSession incrementally validates and retains only catalogue-indexed | |
| 372 | 403 | // checkpoints. Every incarnation owns its budget and continuation state. | |
| 404 | + | // processRetentionEntries charges stat-reported size for every entry, requeues | |
| 405 | + | // the unadmitted suffix, and removes only canonical entries rejected by policy. | |
| 406 | + | func (r *Repository) processRetentionEntries(ctx context.Context, dir, purpose string, entries []maintenanceDirEntry, budget *ports.MaintenanceBudget, consumedBefore bool, policy retentionEntryPolicy) (bool, error) { | |
| 407 | + | for i, entry := range entries { | |
| 408 | + | if err := ctx.Err(); err != nil { | |
| 409 | + | return false, err | |
| 410 | + | } | |
| 411 | + | path := filepath.Join(dir, entry.name) | |
| 412 | + | info, statErr := r.stat(path) | |
| 413 | + | if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { | |
| 414 | + | return false, statErr | |
| 415 | + | } | |
| 416 | + | size := uint64(0) | |
| 417 | + | if statErr == nil && info.Size() > 0 { | |
| 418 | + | size = uint64(info.Size()) | |
| 419 | + | } | |
| 420 | + | if budget.Entries == 0 || size > budget.Bytes { | |
| 421 | + | r.requeueMaintenanceEntries(dir, purpose, entries[i:]) | |
| 422 | + | if !consumedBefore && i == 0 { | |
| 423 | + | return false, ErrMaintenanceBudgetTooSmall | |
| 424 | + | } | |
| 425 | + | return false, nil | |
| 426 | + | } | |
| 427 | + | budget.Entries-- | |
| 428 | + | budget.Bytes -= size | |
| 429 | + | if !policy.delete(entry) { | |
| 430 | + | continue | |
| 431 | + | } | |
| 432 | + | if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { | |
| 433 | + | return false, err | |
| 434 | + | } | |
| 435 | + | if err := r.syncDirectory(dir); err != nil { | |
| 436 | + | return false, err | |
| 437 | + | } | |
| 438 | + | } | |
| 439 | + | return true, nil | |
| 440 | + | } | |
| 441 | + | ||
| 373 | 442 | func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPlan, budget ports.MaintenanceBudget) (done bool, err error) { | |
| 374 | 443 | requested := budget | |
| 375 | 444 | if err := ctx.Err(); err != nil { | |
| @@ -459,34 +528,14 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||
| 459 | 528 | if err != nil { | |
| 460 | 529 | return false, err | |
| 461 | 530 | } | |
| 462 | - | for i, entry := range entries { | |
| 463 | - | path := r.manifestPath(plan.IncarnationID, 0) | |
| 464 | - | path = filepath.Join(filepath.Dir(path), entry.name) | |
| 465 | - | info, statErr := r.stat(path) | |
| 466 | - | if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { | |
| 467 | - | return false, statErr | |
| 468 | - | } | |
| 469 | - | size := uint64(0) | |
| 470 | - | if statErr == nil && info.Size() > 0 { | |
| 471 | - | size = uint64(info.Size()) | |
| 472 | - | } | |
| 473 | - | if budget.Entries == 0 || size > budget.Bytes { | |
| 474 | - | r.requeueMaintenanceEntries(filepath.Dir(path), purpose, entries[i:]) | |
| 475 | - | return false, nil | |
| 476 | - | } | |
| 477 | - | budget.Entries-- | |
| 478 | - | budget.Bytes -= size | |
| 479 | - | generation, canonical := parseGenerationFilename(entry.name) | |
| 480 | - | if !entry.isDir && canonical { | |
| 481 | - | if _, retained := keep[generation]; !retained { | |
| 482 | - | if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { | |
| 483 | - | return false, err | |
| 484 | - | } | |
| 485 | - | if err := r.syncDirectory(filepath.Dir(path)); err != nil { | |
| 486 | - | return false, err | |
| 487 | - | } | |
| 488 | - | } | |
| 489 | - | } | |
| 531 | + | dir := filepath.Join(r.sessionPath(plan.IncarnationID), repositoryGenerations) | |
| 532 | + | consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes | |
| 533 | + | admitted, err := r.processRetentionEntries(ctx, dir, purpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionManifest, generations: keep}) | |
| 534 | + | if err != nil { | |
| 535 | + | return false, err | |
| 536 | + | } | |
| 537 | + | if !admitted { | |
| 538 | + | return false, nil | |
| 490 | 539 | } | |
| 491 | 540 | state.manifestsDone = done | |
| 492 | 541 | if len(entries) == 0 && !done { | |
| @@ -511,33 +560,13 @@ func (r *Repository) MaintainSession(ctx context.Context, plan ports.RetentionPl | |||
| 511 | 560 | if err != nil { | |
| 512 | 561 | return false, err | |
| 513 | 562 | } | |
| 514 | - | for i, entry := range entries { | |
| 515 | - | path := filepath.Join(dir, entry.name) | |
| 516 | - | info, statErr := r.stat(path) | |
| 517 | - | if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { | |
| 518 | - | return false, statErr | |
| 519 | - | } | |
| 520 | - | size := uint64(0) | |
| 521 | - | if statErr == nil && info.Size() > 0 { | |
| 522 | - | size = uint64(info.Size()) | |
| 523 | - | } | |
| 524 | - | if budget.Entries == 0 || size > budget.Bytes { | |
| 525 | - | r.requeueMaintenanceEntries(dir, objectPurpose, entries[i:]) | |
| 526 | - | return false, nil | |
| 527 | - | } | |
| 528 | - | budget.Entries-- | |
| 529 | - | budget.Bytes -= size | |
| 530 | - | digest, canonical := parseObjectDigest(entry.name) | |
| 531 | - | if !entry.isDir && canonical { | |
| 532 | - | if _, retained := state.references[digest]; !retained { | |
| 533 | - | if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { | |
| 534 | - | return false, err | |
| 535 | - | } | |
| 536 | - | if err := r.syncDirectory(dir); err != nil { | |
| 537 | - | return false, err | |
| 538 | - | } | |
| 539 | - | } | |
| 540 | - | } | |
| 563 | + | consumedBefore := requested.Entries != budget.Entries || requested.Bytes != budget.Bytes | |
| 564 | + | admitted, err := r.processRetentionEntries(ctx, dir, objectPurpose, entries, &budget, consumedBefore, retentionEntryPolicy{kind: retentionObject, objects: state.references}) | |
| 565 | + | if err != nil { | |
| 566 | + | return false, err | |
| 567 | + | } | |
| 568 | + | if !admitted { | |
| 569 | + | return false, nil | |
| 541 | 570 | } | |
| 542 | 571 | if done { | |
| 543 | 572 | state.objectShard = "" | |
| @@ -587,15 +616,15 @@ func retentionToken(plan ports.RetentionPlan) string { | |||
| 587 | 616 | return b.String() | |
| 588 | 617 | } | |
| 589 | 618 | ||
| 590 | - | // validateMaintenanceCheckpoint admits every payload by stat-reported size | |
| 591 | - | // before reading it. It validates directly instead of calling LoadCheckpoint, | |
| 592 | - | // which would materialize a complete generation before maintenance could | |
| 593 | - | // enforce its byte budget. | |
| 594 | 619 | type validatedMaintenanceCheckpoint struct { | |
| 595 | 620 | refs map[ports.SnapshotDigest]codec.ObjectRef | |
| 596 | 621 | parent *domain.CheckpointRef | |
| 597 | 622 | } | |
| 598 | 623 | ||
| 624 | + | // validateMaintenanceCheckpoint admits every payload by stat-reported size | |
| 625 | + | // before reading it. It validates directly instead of calling LoadCheckpoint, | |
| 626 | + | // which would materialize a complete generation before maintenance could | |
| 627 | + | // enforce its byte budget. | |
| 599 | 628 | func (r *Repository) validateMaintenanceCheckpoint(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef, byteBudget uint64) (validatedMaintenanceCheckpoint, uint64, bool, error) { | |
| 600 | 629 | manifestPath := r.manifestPath(id, ref.Generation) | |
| 601 | 630 | info, err := r.stat(manifestPath) | |
| @@ -692,8 +721,6 @@ func (r *Repository) Reconcile(ctx context.Context, records []domain.CatalogueRe | |||
| 692 | 721 | byIncarnation[record.IncarnationID] = record | |
| 693 | 722 | } | |
| 694 | 723 | ||
| 695 | - | r.maintenanceMu.Lock() | |
| 696 | - | defer r.maintenanceMu.Unlock() | |
| 697 | 724 | root := filepath.Join(r.dir, repositorySessionsDir) | |
| 698 | 725 | state := &maintenanceCursor{offset: offset} | |
| 699 | 726 | findingCapacity := maintenanceBatch | |
M internal/adapters/snapshot/maintenance_test.go +88 -16
| @@ -1,6 +1,7 @@ | |||
| 1 | 1 | package snapshot | |
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | + | "cmp" | |
| 4 | 5 | "context" | |
| 5 | 6 | "crypto/sha256" | |
| 6 | 7 | "encoding/binary" | |
| @@ -11,8 +12,10 @@ import ( | |||
| 11 | 12 | "path/filepath" | |
| 12 | 13 | "slices" | |
| 13 | 14 | "strings" | |
| 15 | + | "sync" | |
| 14 | 16 | "syscall" | |
| 15 | 17 | "testing" | |
| 18 | + | "time" | |
| 16 | 19 | "unsafe" | |
| 17 | 20 | ||
| 18 | 21 | "github.com/bnema/vev/internal/domain" | |
| @@ -1043,21 +1046,17 @@ func TestRetentionSlots(t *testing.T) { | |||
| 1043 | 1046 | } | |
| 1044 | 1047 | ||
| 1045 | 1048 | func TestUnresolvedDataPinned(t *testing.T) { | |
| 1046 | - | for _, state := range []string{"degraded", "deleting", "migration", "transaction"} { | |
| 1047 | - | t.Run(state, func(t *testing.T) { | |
| 1048 | - | repo := NewRepository(privateDir(t)) | |
| 1049 | - | publications := publishMaintenanceGenerations(t, repo, state, 4) | |
| 1050 | - | done, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: publications[0].IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | |
| 1051 | - | if err != nil { | |
| 1052 | - | t.Fatal(err) | |
| 1053 | - | } | |
| 1054 | - | if !done { | |
| 1055 | - | t.Fatal("pinned maintenance did not complete without traversing") | |
| 1056 | - | } | |
| 1057 | - | if got := remainingMaintenanceGenerations(t, repo, publications[0].IncarnationID); !slices.Equal(got, []uint64{4, 3, 2, 1}) { | |
| 1058 | - | t.Fatalf("pinned generations = %v", got) | |
| 1059 | - | } | |
| 1060 | - | }) | |
| 1049 | + | repo := NewRepository(privateDir(t)) | |
| 1050 | + | publications := publishMaintenanceGenerations(t, repo, "unresolved", 4) | |
| 1051 | + | done, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: publications[0].IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | |
| 1052 | + | if err != nil { | |
| 1053 | + | t.Fatal(err) | |
| 1054 | + | } | |
| 1055 | + | if !done { | |
| 1056 | + | t.Fatal("pinned maintenance did not complete without traversing") | |
| 1057 | + | } | |
| 1058 | + | if got := remainingMaintenanceGenerations(t, repo, publications[0].IncarnationID); !slices.Equal(got, []uint64{4, 3, 2, 1}) { | |
| 1059 | + | t.Fatalf("pinned generations = %v", got) | |
| 1061 | 1060 | } | |
| 1062 | 1061 | } | |
| 1063 | 1062 | ||
| @@ -1094,6 +1093,79 @@ func TestMaintenanceBudgetAdmitsPayloadsBeforeRead(t *testing.T) { | |||
| 1094 | 1093 | } | |
| 1095 | 1094 | } | |
| 1096 | 1095 | ||
| 1096 | + | func TestMaintainSessionDistinguishesInitialOversizeFromResumableExhaustion(t *testing.T) { | |
| 1097 | + | repo := NewRepository(privateDir(t)) | |
| 1098 | + | publications := publishMaintenanceGenerations(t, repo, "budget-semantics", 2) | |
| 1099 | + | plan := ports.RetentionPlan{IncarnationID: publications[0].IncarnationID} | |
| 1100 | + | firstInfo, err := os.Stat(repo.manifestPath(plan.IncarnationID, 1)) | |
| 1101 | + | if err != nil { | |
| 1102 | + | t.Fatal(err) | |
| 1103 | + | } | |
| 1104 | + | secondInfo, err := os.Stat(repo.manifestPath(plan.IncarnationID, 2)) | |
| 1105 | + | if err != nil { | |
| 1106 | + | t.Fatal(err) | |
| 1107 | + | } | |
| 1108 | + | firstSize, secondSize := uint64(firstInfo.Size()), uint64(secondInfo.Size()) | |
| 1109 | + | ||
| 1110 | + | done, err := repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 2, Bytes: min(firstSize, secondSize) - 1}) | |
| 1111 | + | if done || !errors.Is(err, ErrMaintenanceBudgetTooSmall) { | |
| 1112 | + | t.Fatalf("initial oversized item: done=%v err=%v", done, err) | |
| 1113 | + | } | |
| 1114 | + | ||
| 1115 | + | done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 2, Bytes: firstSize + secondSize - 1}) | |
| 1116 | + | if err != nil || done { | |
| 1117 | + | t.Fatalf("mid-pass exhaustion must be resumable: done=%v err=%v", done, err) | |
| 1118 | + | } | |
| 1119 | + | done, err = repo.MaintainSession(context.Background(), plan, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20}) | |
| 1120 | + | if err != nil || !done { | |
| 1121 | + | t.Fatalf("resumed maintenance: done=%v err=%v", done, err) | |
| 1122 | + | } | |
| 1123 | + | } | |
| 1124 | + | ||
| 1125 | + | func TestReconcileDoesNotHoldMaintenanceLockDuringPayloadRead(t *testing.T) { | |
| 1126 | + | repo := NewRepository(privateDir(t)) | |
| 1127 | + | publications := publishMaintenanceGenerations(t, repo, "reconcile-lock", 2) | |
| 1128 | + | committed := checkpointRefForPublication(publications[0]) | |
| 1129 | + | record := domain.CatalogueRecord{Name: "reconcile-lock", IncarnationID: publications[0].IncarnationID, RecoveryState: domain.RecoveryHealthy, Committed: &committed} | |
| 1130 | + | ||
| 1131 | + | entered := make(chan struct{}) | |
| 1132 | + | release := make(chan struct{}) | |
| 1133 | + | var releaseOnce sync.Once | |
| 1134 | + | defer releaseOnce.Do(func() { close(release) }) | |
| 1135 | + | repo.hooks.beforeMaintenancePayloadRead = func(string) { | |
| 1136 | + | select { | |
| 1137 | + | case <-entered: | |
| 1138 | + | default: | |
| 1139 | + | close(entered) | |
| 1140 | + | } | |
| 1141 | + | <-release | |
| 1142 | + | } | |
| 1143 | + | reconcileDone := make(chan error, 1) | |
| 1144 | + | go func() { | |
| 1145 | + | _, _, err := repo.Reconcile(context.Background(), []domain.CatalogueRecord{record}, ports.ReconcileCursor{}, ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20}) | |
| 1146 | + | reconcileDone <- err | |
| 1147 | + | }() | |
| 1148 | + | <-entered | |
| 1149 | + | ||
| 1150 | + | pinDone := make(chan error, 1) | |
| 1151 | + | go func() { | |
| 1152 | + | _, err := repo.MaintainSession(context.Background(), ports.RetentionPlan{IncarnationID: record.IncarnationID, PinAll: true}, ports.MaintenanceBudget{Entries: 1, Bytes: 1}) | |
| 1153 | + | pinDone <- err | |
| 1154 | + | }() | |
| 1155 | + | select { | |
| 1156 | + | case err := <-pinDone: | |
| 1157 | + | if err != nil { | |
| 1158 | + | t.Fatal(err) | |
| 1159 | + | } | |
| 1160 | + | case <-time.After(time.Second): | |
| 1161 | + | t.Fatal("maintenance lock remained held during reconciliation payload I/O") | |
| 1162 | + | } | |
| 1163 | + | releaseOnce.Do(func() { close(release) }) | |
| 1164 | + | if err := <-reconcileDone; err != nil { | |
| 1165 | + | t.Fatal(err) | |
| 1166 | + | } | |
| 1167 | + | } | |
| 1168 | + | ||
| 1097 | 1169 | func TestPerSessionBudgetIsolation(t *testing.T) { | |
| 1098 | 1170 | repo := NewRepository(privateDir(t)) | |
| 1099 | 1171 | large := publishMaintenanceGenerations(t, repo, "large", 4) | |
| @@ -1149,6 +1221,6 @@ func remainingMaintenanceGenerations(t *testing.T, repo *Repository, id domain.I | |||
| 1149 | 1221 | generations = append(generations, generation) | |
| 1150 | 1222 | } | |
| 1151 | 1223 | } | |
| 1152 | - | slices.SortFunc(generations, func(a, b uint64) int { return int(b) - int(a) }) | |
| 1224 | + | slices.SortFunc(generations, func(a, b uint64) int { return cmp.Compare(b, a) }) | |
| 1153 | 1225 | return generations | |
| 1154 | 1226 | } | |
M internal/adapters/snapshot/migration_v0.go +6 -3
| @@ -49,7 +49,7 @@ func decodeManifestV1(data []byte) (legacyManifestV1, error) { | |||
| 49 | 49 | objects := make([]legacyObjectV1, 0) | |
| 50 | 50 | add := func(ref codec.ObjectRef) error { | |
| 51 | 51 | if _, ok := seen[ref.Digest]; ok { | |
| 52 | - | return errors.New("snapshot: duplicate legacy object digest") | |
| 52 | + | return nil | |
| 53 | 53 | } | |
| 54 | 54 | seen[ref.Digest] = struct{}{} | |
| 55 | 55 | objects = append(objects, legacyObjectV1{Kind: uint8(ref.Kind), Digest: ref.Digest, Size: uint64(ref.Size)}) | |
| @@ -209,7 +209,7 @@ func uncertainLegacyError(operation string, cause error) error { | |||
| 209 | 209 | if cause == nil { | |
| 210 | 210 | return fmt.Errorf("%w: %s", ports.ErrLegacySnapshotUncertain, operation) | |
| 211 | 211 | } | |
| 212 | - | return fmt.Errorf("%w: %s: %v", ports.ErrLegacySnapshotUncertain, operation, cause) | |
| 212 | + | return fmt.Errorf("%w: %s: %w", ports.ErrLegacySnapshotUncertain, operation, cause) | |
| 213 | 213 | } | |
| 214 | 214 | ||
| 215 | 215 | // MigrateV1Checkpoint is the migration-only admission path for an authoritative | |
| @@ -244,7 +244,10 @@ func (r *Repository) MigrateV1Checkpoint(ctx context.Context, req ports.Snapshot | |||
| 244 | 244 | if manifest.Name != req.LegacyName || manifest.Generation != req.LegacyRef.Generation { | |
| 245 | 245 | return domain.CheckpointRef{}, uncertainLegacyError("manifest identity mismatch", nil) | |
| 246 | 246 | } | |
| 247 | - | key, _ := incarnationKey(req.IncarnationID) | |
| 247 | + | key, err := incarnationKey(req.IncarnationID) | |
| 248 | + | if err != nil { | |
| 249 | + | return domain.CheckpointRef{}, err | |
| 250 | + | } | |
| 248 | 251 | lock := r.lockSession(key) | |
| 249 | 252 | defer r.unlockSession(lock) | |
| 250 | 253 | if err := r.ensureSession(req.IncarnationID); err != nil { | |
M internal/adapters/snapshot/migration_v0_test.go +39 -3
| @@ -4,6 +4,7 @@ import ( | |||
| 4 | 4 | "context" | |
| 5 | 5 | "crypto/sha256" | |
| 6 | 6 | "encoding/binary" | |
| 7 | + | "errors" | |
| 7 | 8 | "fmt" | |
| 8 | 9 | "hash/crc32" | |
| 9 | 10 | "os" | |
| @@ -37,10 +38,45 @@ func TestLegacyManifestV1(t *testing.T) { | |||
| 37 | 38 | require.NoError(t, err) | |
| 38 | 39 | require.Equal(t, "work", got.Name) | |
| 39 | 40 | require.Equal(t, uint64(7), got.Generation) | |
| 40 | - | for _, invalid := range [][]byte{encoded[:5], append(append([]byte(nil), encoded...), 0)} { | |
| 41 | - | _, err := decodeManifestV1(invalid) | |
| 42 | - | require.Error(t, err) | |
| 41 | + | for size := range len(encoded) { | |
| 42 | + | _, err := decodeManifestV1(encoded[:size]) | |
| 43 | + | require.Errorf(t, err, "prefix length %d", size) | |
| 43 | 44 | } | |
| 45 | + | _, err = decodeManifestV1(append(append([]byte(nil), encoded...), 0)) | |
| 46 | + | require.Error(t, err) | |
| 47 | + | } | |
| 48 | + | ||
| 49 | + | func TestLegacyManifestV1DeduplicatesObjectsPreservingFirst(t *testing.T) { | |
| 50 | + | chunk, err := codec.MarshalObject(codec.HistoryChunk, []byte("chunk")) | |
| 51 | + | require.NoError(t, err) | |
| 52 | + | tail, err := codec.MarshalObject(codec.HistoryTail, []byte("tail")) | |
| 53 | + | require.NoError(t, err) | |
| 54 | + | visible, err := codec.MarshalObject(codec.Visible, []byte("visible")) | |
| 55 | + | require.NoError(t, err) | |
| 56 | + | chunkRef := codec.ObjectRef{Kind: codec.HistoryChunk, Digest: chunk.Digest, Size: uint32(len(chunk.Data))} | |
| 57 | + | manifest := codec.Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "work", Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Sealed: []codec.ObjectRef{chunkRef, chunkRef}, Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}} | |
| 58 | + | modern, err := codec.MarshalManifest(manifest) | |
| 59 | + | require.NoError(t, err) | |
| 60 | + | body := append([]byte(nil), modern[legacyManifestHeaderSize:]...) | |
| 61 | + | body = append(body[:8], body[25:]...) | |
| 62 | + | legacy := make([]byte, legacyManifestHeaderSize, legacyManifestHeaderSize+len(body)) | |
| 63 | + | copy(legacy, "VEVM") | |
| 64 | + | binary.BigEndian.PutUint16(legacy[4:6], 1) | |
| 65 | + | binary.BigEndian.PutUint32(legacy[8:12], uint32(len(body))) | |
| 66 | + | binary.BigEndian.PutUint32(legacy[12:16], crc32.ChecksumIEEE(body)) | |
| 67 | + | legacy = append(legacy, body...) | |
| 68 | + | ||
| 69 | + | decoded, err := decodeManifestV1(legacy) | |
| 70 | + | require.NoError(t, err) | |
| 71 | + | require.Len(t, decoded.Objects, 3) | |
| 72 | + | require.Equal(t, chunk.Digest, decoded.Objects[0].Digest) | |
| 73 | + | } | |
| 74 | + | ||
| 75 | + | func TestUncertainLegacyErrorWrapsSentinelAndCause(t *testing.T) { | |
| 76 | + | cause := errors.New("legacy cause") | |
| 77 | + | err := uncertainLegacyError("read", cause) | |
| 78 | + | require.ErrorIs(t, err, ports.ErrLegacySnapshotUncertain) | |
| 79 | + | require.ErrorIs(t, err, cause) | |
| 44 | 80 | } | |
| 45 | 81 | ||
| 46 | 82 | func TestHasLegacyState(t *testing.T) { | |
M internal/adapters/snapshot/publish_efficiency_test.go +7 -8
| @@ -120,11 +120,11 @@ func TestRepositoryPublishVerifiesNecessaryExistingObjectOnce(t *testing.T) { | |||
| 120 | 120 | } | |
| 121 | 121 | second := repositoryPublicationAfter(t, repo, "named", 2, []byte("second")) | |
| 122 | 122 | newTail := second.Objects[0] | |
| 123 | - | key := legacyIncarnationID(second.Name).String() | |
| 124 | - | if err := os.MkdirAll(filepath.Dir(repo.legacyObjectPath(key, newTail.Digest)), 0o700); err != nil { | |
| 123 | + | path := repo.objectPath(second.IncarnationID, newTail.Digest) | |
| 124 | + | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |
| 125 | 125 | t.Fatal(err) | |
| 126 | 126 | } | |
| 127 | - | if err := os.WriteFile(repo.legacyObjectPath(key, newTail.Digest), newTail.Data, 0o600); err != nil { | |
| 127 | + | if err := os.WriteFile(path, newTail.Data, 0o600); err != nil { | |
| 128 | 128 | t.Fatal(err) | |
| 129 | 129 | } | |
| 130 | 130 | ||
| @@ -178,7 +178,6 @@ func largeIncrementalPublications(t testing.TB, name string, count int) (ports.S | |||
| 178 | 178 | ||
| 179 | 179 | func seedCompletePublication(t testing.TB, repo *Repository, publication ports.SnapshotPublication) { | |
| 180 | 180 | t.Helper() | |
| 181 | - | key := legacyIncarnationID(publication.Name).String() | |
| 182 | 181 | if err := repo.ensureSession(publication.IncarnationID); err != nil { | |
| 183 | 182 | t.Fatal(err) | |
| 184 | 183 | } | |
| @@ -187,17 +186,17 @@ func seedCompletePublication(t testing.TB, repo *Repository, publication ports.S | |||
| 187 | 186 | t.Fatal(err) | |
| 188 | 187 | } | |
| 189 | 188 | for _, object := range publication.Objects { | |
| 190 | - | if err := os.MkdirAll(filepath.Dir(repo.legacyObjectPath(key, object.Digest)), 0o700); err != nil { | |
| 189 | + | if err := os.MkdirAll(filepath.Dir(repo.objectPath(publication.IncarnationID, object.Digest)), 0o700); err != nil { | |
| 191 | 190 | t.Fatal(err) | |
| 192 | 191 | } | |
| 193 | - | if err := os.WriteFile(repo.legacyObjectPath(key, object.Digest), object.Data, 0o600); err != nil { | |
| 192 | + | if err := os.WriteFile(repo.objectPath(publication.IncarnationID, object.Digest), object.Data, 0o600); err != nil { | |
| 194 | 193 | t.Fatal(err) | |
| 195 | 194 | } | |
| 196 | 195 | } | |
| 197 | - | if err := os.WriteFile(repo.legacyManifestPath(key, publication.Generation), publication.Manifest, 0o600); err != nil { | |
| 196 | + | if err := os.WriteFile(repo.manifestPath(publication.IncarnationID, publication.Generation), publication.Manifest, 0o600); err != nil { | |
| 198 | 197 | t.Fatal(err) | |
| 199 | 198 | } | |
| 200 | - | if err := os.WriteFile(repo.legacyHeadPath(key), marshalHead(publication.Generation, sha256.Sum256(publication.Manifest)), 0o600); err != nil { | |
| 199 | + | if err := os.WriteFile(repo.headPath(publication.IncarnationID), marshalHead(publication.Generation, sha256.Sum256(publication.Manifest)), 0o600); err != nil { | |
| 201 | 200 | t.Fatal(err) | |
| 202 | 201 | } | |
| 203 | 202 | if refs := manifestRefs(manifest); refs == nil { | |
M internal/adapters/snapshot/reconciliation_test.go +2 -2
| @@ -2,13 +2,13 @@ package snapshot | |||
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | - | "os" | |
| 6 | 5 | "path/filepath" | |
| 7 | 6 | "testing" | |
| 8 | 7 | ||
| 9 | 8 | "github.com/bnema/vev/internal/domain" | |
| 10 | 9 | "github.com/bnema/vev/internal/ports" | |
| 11 | 10 | codec "github.com/bnema/vev/internal/usecase/snapshot" | |
| 11 | + | "github.com/bnema/vev/pkg/safedir" | |
| 12 | 12 | "github.com/stretchr/testify/require" | |
| 13 | 13 | ) | |
| 14 | 14 | ||
| @@ -108,7 +108,7 @@ func TestReconciliationCursor(t *testing.T) { | |||
| 108 | 108 | sessions := filepath.Join(repo.dir, repositorySessionsDir) | |
| 109 | 109 | for i := byte(20); i < 26; i++ { | |
| 110 | 110 | id := domain.IncarnationID{i} | |
| 111 | - | require.NoError(t, os.Mkdir(filepath.Join(sessions, id.String()), 0o700)) | |
| 111 | + | require.NoError(t, safedir.EnsurePrivate(filepath.Join(sessions, id.String()))) | |
| 112 | 112 | } | |
| 113 | 113 | ||
| 114 | 114 | cursor := ports.ReconcileCursor{} | |
M internal/adapters/snapshot/recovery_crash_test.go +16 -7
| @@ -13,6 +13,7 @@ import ( | |||
| 13 | 13 | "github.com/bnema/vev/internal/persist" | |
| 14 | 14 | "github.com/bnema/vev/internal/ports" | |
| 15 | 15 | "github.com/bnema/vev/internal/usecase/recovery" | |
| 16 | + | "github.com/bnema/vev/pkg/safedir" | |
| 16 | 17 | "github.com/stretchr/testify/require" | |
| 17 | 18 | ) | |
| 18 | 19 | ||
| @@ -57,7 +58,7 @@ func TestDeletionRestartsAtInternalQuarantineBoundaries(t *testing.T) { | |||
| 57 | 58 | record := domain.CatalogueRecord{Name: "work", IncarnationID: oldID, RecoveryState: domain.RecoveryFresh} | |
| 58 | 59 | require.NoError(t, catalogue.Create(record)) | |
| 59 | 60 | sessionPath := repository.sessionPath(oldID) | |
| 60 | - | require.NoError(t, os.MkdirAll(sessionPath, 0o700)) | |
| 61 | + | require.NoError(t, safedir.EnsurePrivate(sessionPath)) | |
| 61 | 62 | require.NoError(t, os.WriteFile(filepath.Join(sessionPath, "durable"), []byte("old namespace"), 0o600)) | |
| 62 | 63 | require.NoError(t, os.WriteFile(filepath.Join(h.snapshotDir, filenameForName(record.Name)), []byte("old legacy"), 0o600)) | |
| 63 | 64 | tc.inject(repository) | |
| @@ -69,7 +70,8 @@ func TestDeletionRestartsAtInternalQuarantineBoundaries(t *testing.T) { | |||
| 69 | 70 | // Simulate a new process: none of the adapter or coordinator state survives. | |
| 70 | 71 | catalogue, repository, journal = h.open(t) | |
| 71 | 72 | require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx)) | |
| 72 | - | _, exists := catalogue.Record(record.Name) | |
| 73 | + | _, exists, err := catalogue.Record(record.Name) | |
| 74 | + | require.NoError(t, err) | |
| 73 | 75 | require.False(t, exists) | |
| 74 | 76 | require.NoDirExists(t, repository.sessionPath(oldID)) | |
| 75 | 77 | require.Equal(t, []byte("old namespace"), mustReadCrashFile(t, filepath.Join(h.snapshotDir, "quarantine", oldID.String(), "snapshot", "durable"))) | |
| @@ -84,13 +86,14 @@ func TestDeletionRestartsAtInternalQuarantineBoundaries(t *testing.T) { | |||
| 84 | 86 | require.NoError(t, err) | |
| 85 | 87 | require.Equal(t, newID, created.IncarnationID) | |
| 86 | 88 | newPath := repository.sessionPath(newID) | |
| 87 | - | require.NoError(t, os.MkdirAll(newPath, 0o700)) | |
| 89 | + | require.NoError(t, safedir.EnsurePrivate(newPath)) | |
| 88 | 90 | require.NoError(t, os.WriteFile(filepath.Join(newPath, "durable"), []byte("new namespace"), 0o600)) | |
| 89 | 91 | require.NoError(t, catalogue.Close()) | |
| 90 | 92 | ||
| 91 | 93 | catalogue, repository, journal = h.open(t) | |
| 92 | 94 | require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx)) | |
| 93 | - | got, exists := catalogue.Record(record.Name) | |
| 95 | + | got, exists, err := catalogue.Record(record.Name) | |
| 96 | + | require.NoError(t, err) | |
| 94 | 97 | require.True(t, exists) | |
| 95 | 98 | require.Equal(t, newID, got.IncarnationID) | |
| 96 | 99 | require.Equal(t, []byte("new namespace"), mustReadCrashFile(t, filepath.Join(repository.sessionPath(newID), "durable"))) | |
| @@ -114,20 +117,26 @@ func TestCreatePreCommitRestartLeavesNoAdoptableNamespace(t *testing.T) { | |||
| 114 | 117 | interruptedID := domain.IncarnationID{3} | |
| 115 | 118 | _, err := recovery.NewCoordinator(rejectCreateCatalogue{Catalogue: catalogue}, repository, journal, bytes.NewReader(interruptedID[:])).Create(ctx, domain.CatalogueRecord{Name: "work"}) | |
| 116 | 119 | require.ErrorIs(t, err, errDeletionBoundaryCrash) | |
| 117 | - | require.Empty(t, catalogue.Records()) | |
| 120 | + | records, err := catalogue.Records() | |
| 121 | + | require.NoError(t, err) | |
| 122 | + | require.Empty(t, records) | |
| 118 | 123 | require.NoDirExists(t, filepath.Join(h.snapshotDir, repositorySessionsDir)) | |
| 119 | 124 | require.NoError(t, catalogue.Close()) | |
| 120 | 125 | ||
| 121 | 126 | catalogue, repository, journal = h.open(t) | |
| 122 | 127 | require.NoError(t, recovery.NewCoordinator(catalogue, repository, journal, nil).Recover(ctx)) | |
| 123 | - | require.Empty(t, catalogue.Records()) | |
| 128 | + | records, err = catalogue.Records() | |
| 129 | + | require.NoError(t, err) | |
| 130 | + | require.Empty(t, records) | |
| 124 | 131 | require.NoDirExists(t, repository.sessionPath(interruptedID)) | |
| 125 | 132 | ||
| 126 | 133 | retryID := domain.IncarnationID{4} | |
| 127 | 134 | created, err := recovery.NewCoordinator(catalogue, repository, journal, bytes.NewReader(retryID[:])).Create(ctx, domain.CatalogueRecord{Name: "work"}) | |
| 128 | 135 | require.NoError(t, err) | |
| 129 | 136 | require.Equal(t, retryID, created.IncarnationID) | |
| 130 | - | require.Len(t, catalogue.Records(), 1) | |
| 137 | + | records, err = catalogue.Records() | |
| 138 | + | require.NoError(t, err) | |
| 139 | + | require.Len(t, records, 1) | |
| 131 | 140 | require.NoDirExists(t, repository.sessionPath(interruptedID)) | |
| 132 | 141 | require.NoError(t, catalogue.Close()) | |
| 133 | 142 | } | |
M internal/adapters/snapshot/recovery_observability_test.go +58 -20
| @@ -14,25 +14,63 @@ import ( | |||
| 14 | 14 | ) | |
| 15 | 15 | ||
| 16 | 16 | func TestGCBudgetDiagnostic(t *testing.T) { | |
| 17 | - | var buffer bytes.Buffer | |
| 18 | - | log := slog.New(slog.NewJSONHandler(&buffer, nil)) | |
| 19 | - | repository := NewRepository(t.TempDir(), log) | |
| 20 | - | id := domain.IncarnationID{1} | |
| 21 | - | done, err := repository.MaintainSession(context.Background(), ports.RetentionPlan{ | |
| 22 | - | IncarnationID: id, | |
| 23 | - | Keep: []ports.CheckpointRef{{Generation: 3}, {Generation: 2}, {Generation: 1}}, | |
| 24 | - | }, ports.MaintenanceBudget{}) | |
| 25 | - | require.NoError(t, err) | |
| 26 | - | require.False(t, done) | |
| 17 | + | for _, tc := range []struct { | |
| 18 | + | name string | |
| 19 | + | arrange func(*testing.T, *Repository) ports.RetentionPlan | |
| 20 | + | budget ports.MaintenanceBudget | |
| 21 | + | wantDone bool | |
| 22 | + | wantExhausted bool | |
| 23 | + | wantConsumption bool | |
| 24 | + | }{ | |
| 25 | + | { | |
| 26 | + | name: "exhausted", | |
| 27 | + | arrange: func(_ *testing.T, _ *Repository) ports.RetentionPlan { | |
| 28 | + | return ports.RetentionPlan{IncarnationID: domain.IncarnationID{1}, Keep: []ports.CheckpointRef{{Generation: 3}, {Generation: 2}, {Generation: 1}}} | |
| 29 | + | }, | |
| 30 | + | wantExhausted: true, | |
| 31 | + | }, | |
| 32 | + | { | |
| 33 | + | name: "sufficient", | |
| 34 | + | arrange: func(t *testing.T, repository *Repository) ports.RetentionPlan { | |
| 35 | + | publication := publishMaintenanceGenerations(t, repository, "diagnostic", 1)[0] | |
| 36 | + | return ports.RetentionPlan{IncarnationID: publication.IncarnationID} | |
| 37 | + | }, | |
| 38 | + | budget: ports.MaintenanceBudget{Entries: 64, Bytes: 8 << 20}, | |
| 39 | + | wantDone: true, | |
| 40 | + | wantConsumption: true, | |
| 41 | + | }, | |
| 42 | + | } { | |
| 43 | + | t.Run(tc.name, func(t *testing.T) { | |
| 44 | + | var buffer bytes.Buffer | |
| 45 | + | log := slog.New(slog.NewJSONHandler(&buffer, nil)) | |
| 46 | + | repository := NewRepository(privateDir(t), log) | |
| 47 | + | plan := tc.arrange(t, repository) | |
| 48 | + | buffer.Reset() | |
| 27 | 49 | ||
| 28 | - | scanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes())) | |
| 29 | - | require.True(t, scanner.Scan()) | |
| 30 | - | var entry map[string]any | |
| 31 | - | require.NoError(t, json.Unmarshal(scanner.Bytes(), &entry)) | |
| 32 | - | require.NoError(t, scanner.Err()) | |
| 33 | - | require.Equal(t, "snapshot_maintenance_progress", entry["msg"]) | |
| 34 | - | require.Equal(t, id.String(), entry["incarnation"]) | |
| 35 | - | require.EqualValues(t, 3, entry["retained"]) | |
| 36 | - | require.Contains(t, entry, "cursor") | |
| 37 | - | require.Equal(t, true, entry["budget_exhausted"]) | |
| 50 | + | done, err := repository.MaintainSession(context.Background(), plan, tc.budget) | |
| 51 | + | require.NoError(t, err) | |
| 52 | + | require.Equal(t, tc.wantDone, done) | |
| 53 | + | ||
| 54 | + | scanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes())) | |
| 55 | + | require.True(t, scanner.Scan()) | |
| 56 | + | var entry map[string]any | |
| 57 | + | require.NoError(t, json.Unmarshal(scanner.Bytes(), &entry)) | |
| 58 | + | require.NoError(t, scanner.Err()) | |
| 59 | + | require.Equal(t, "snapshot_maintenance_progress", entry["msg"]) | |
| 60 | + | require.Equal(t, plan.IncarnationID.String(), entry["incarnation"]) | |
| 61 | + | require.Contains(t, entry, "cursor") | |
| 62 | + | require.Equal(t, tc.wantExhausted, entry["budget_exhausted"]) | |
| 63 | + | consumedEntries, entriesOK := entry["consumed_entries"].(float64) | |
| 64 | + | consumedBytes, bytesOK := entry["consumed_bytes"].(float64) | |
| 65 | + | require.True(t, entriesOK) | |
| 66 | + | require.True(t, bytesOK) | |
| 67 | + | if tc.wantConsumption { | |
| 68 | + | require.Positive(t, consumedEntries) | |
| 69 | + | require.Positive(t, consumedBytes) | |
| 70 | + | } else { | |
| 71 | + | require.Zero(t, consumedEntries) | |
| 72 | + | require.Zero(t, consumedBytes) | |
| 73 | + | } | |
| 74 | + | }) | |
| 75 | + | } | |
| 38 | 76 | } | |
M internal/app/daemon_integration_test.go +27 -18
| @@ -453,15 +453,13 @@ func TestMultipleClientsOneLifecycleOwner(t *testing.T) { | |||
| 453 | 453 | errs := make(chan error, clients) | |
| 454 | 454 | var wg sync.WaitGroup | |
| 455 | 455 | for range clients { | |
| 456 | - | wg.Add(1) | |
| 457 | - | go func() { | |
| 458 | - | defer wg.Done() | |
| 456 | + | wg.Go(func() { | |
| 459 | 457 | _, err := ensureDaemonWithLifecycle(context.Background(), dir, dial, func() error { | |
| 460 | 458 | spawns.Add(1) | |
| 461 | 459 | return nil | |
| 462 | 460 | }, cfg) | |
| 463 | 461 | errs <- err | |
| 464 | - | }() | |
| 462 | + | }) | |
| 465 | 463 | } | |
| 466 | 464 | require.Never(t, func() bool { return spawns.Load() != 0 }, 50*time.Millisecond, time.Millisecond) | |
| 467 | 465 | close(ready) | |
| @@ -537,7 +535,7 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) { | |||
| 537 | 535 | closed: make(chan struct{}), | |
| 538 | 536 | } | |
| 539 | 537 | repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots")) | |
| 540 | - | shutdownClock := newLifecycleShutdownClock() | |
| 538 | + | shutdownClock := newLifecycleShutdownClock(t) | |
| 541 | 539 | reconciler := &lifecycleBlockingReconciler{ | |
| 542 | 540 | entered: make(chan struct{}), | |
| 543 | 541 | release: make(chan struct{}), | |
| @@ -564,7 +562,7 @@ func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) { | |||
| 564 | 562 | pty.NewFactory(), | |
| 565 | 563 | shutdownClock, | |
| 566 | 564 | discardLog(), | |
| 567 | - | daemon.WithSnapshotRepository(repository, nil), | |
| 565 | + | daemon.WithSnapshotRepository(repository), | |
| 568 | 566 | daemon.WithDurableMaintenance(catalogue, repository, reconciler, nil), | |
| 569 | 567 | daemon.WithCatalogue(catalogue, nil), | |
| 570 | 568 | ) | |
| @@ -679,7 +677,7 @@ func TestLifecycleOwnershipOutlivesRestorationWriter(t *testing.T) { | |||
| 679 | 677 | releaseRestore := func() { releaseOnce.Do(func() { close(blocking.release) }) } | |
| 680 | 678 | t.Cleanup(releaseRestore) | |
| 681 | 679 | ||
| 682 | - | shutdownClock := newLifecycleShutdownClock() | |
| 680 | + | shutdownClock := newLifecycleShutdownClock(t) | |
| 683 | 681 | ctx, cancel := context.WithCancel(context.Background()) | |
| 684 | 682 | t.Cleanup(cancel) | |
| 685 | 683 | listenerReady := make(chan struct{}) | |
| @@ -707,7 +705,7 @@ func TestLifecycleOwnershipOutlivesRestorationWriter(t *testing.T) { | |||
| 707 | 705 | discardLog(), | |
| 708 | 706 | daemon.WithShell("/bin/cat", nil), | |
| 709 | 707 | daemon.WithCatalogue(opened.Catalogue, opened.Records), | |
| 710 | - | daemon.WithSnapshotRepository(blocking, nil), | |
| 708 | + | daemon.WithSnapshotRepository(blocking), | |
| 711 | 709 | ) | |
| 712 | 710 | return d.Serve(ctx, listener) | |
| 713 | 711 | }) | |
| @@ -764,7 +762,7 @@ func publishRestorableCheckpoint(t *testing.T, stateDir string, repository *snap | |||
| 764 | 762 | discardLog(), | |
| 765 | 763 | daemon.WithShell("/bin/cat", nil), | |
| 766 | 764 | daemon.WithCatalogue(opened.Catalogue, opened.Records), | |
| 767 | - | daemon.WithSnapshotRepository(repository, nil), | |
| 765 | + | daemon.WithSnapshotRepository(repository), | |
| 768 | 766 | daemon.WithRecoveryCoordinator(coordinator), | |
| 769 | 767 | ) | |
| 770 | 768 | served := make(chan error, 1) | |
| @@ -773,7 +771,7 @@ func publishRestorableCheckpoint(t *testing.T, stateDir string, repository *snap | |||
| 773 | 771 | tr, _ := attach(t, runtimeDir, ports.IntentNew, name, domain.Size{Cols: 80, Rows: 24}) | |
| 774 | 772 | require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgInput, Payload: ports.MarshalInput(ports.Input{Data: []byte("checkpoint me\n")})})) | |
| 775 | 773 | require.Eventually(t, func() bool { | |
| 776 | - | record, ok := opened.Catalogue.Record(name) | |
| 774 | + | record, ok, _ := opened.Catalogue.Record(name) | |
| 777 | 775 | return ok && record.RecoveryState == domain.RecoveryHealthy && record.Committed != nil | |
| 778 | 776 | }, 5*time.Second, 10*time.Millisecond, "session never committed a checkpoint") | |
| 779 | 777 | require.NoError(t, tr.Close()) | |
| @@ -826,7 +824,7 @@ func runBlockedSnapshotWriterShutdown(t *testing.T) blockedSnapshotWriterShutdow | |||
| 826 | 824 | t.Helper() | |
| 827 | 825 | runtimeDir := filepath.Join(t.TempDir(), "runtime") | |
| 828 | 826 | stateDir := filepath.Join(t.TempDir(), "state") | |
| 829 | - | shutdownClock := newLifecycleShutdownClock() | |
| 827 | + | shutdownClock := newLifecycleShutdownClock(t) | |
| 830 | 828 | repository := newLifecycleBlockingRepository() | |
| 831 | 829 | ctx, cancel := context.WithCancel(context.Background()) | |
| 832 | 830 | t.Cleanup(cancel) | |
| @@ -849,7 +847,7 @@ func runBlockedSnapshotWriterShutdown(t *testing.T) blockedSnapshotWriterShutdow | |||
| 849 | 847 | shutdownClock, | |
| 850 | 848 | discardLog(), | |
| 851 | 849 | daemon.WithShell("/bin/cat", nil), | |
| 852 | - | daemon.WithSnapshotRepository(repository, nil), | |
| 850 | + | daemon.WithSnapshotRepository(repository), | |
| 853 | 851 | ) | |
| 854 | 852 | return d.Serve(ctx, observed) | |
| 855 | 853 | }) | |
| @@ -913,21 +911,28 @@ func (r *lifecycleBlockingRepository) Publish(context.Context, ports.SnapshotPub | |||
| 913 | 911 | } | |
| 914 | 912 | ||
| 915 | 913 | type lifecycleShutdownClock struct { | |
| 914 | + | t *testing.T | |
| 916 | 915 | base ports.Clock | |
| 917 | 916 | timers chan *lifecycleManualTimer | |
| 918 | 917 | } | |
| 919 | 918 | ||
| 920 | - | func newLifecycleShutdownClock() *lifecycleShutdownClock { | |
| 921 | - | return &lifecycleShutdownClock{base: clock.New(), timers: make(chan *lifecycleManualTimer, 4)} | |
| 919 | + | func newLifecycleShutdownClock(t *testing.T) *lifecycleShutdownClock { | |
| 920 | + | t.Helper() | |
| 921 | + | return &lifecycleShutdownClock{t: t, base: clock.New(), timers: make(chan *lifecycleManualTimer, 4)} | |
| 922 | 922 | } | |
| 923 | 923 | ||
| 924 | 924 | func (c *lifecycleShutdownClock) Now() time.Time { return c.base.Now() } | |
| 925 | 925 | func (c *lifecycleShutdownClock) NewTimer(delay time.Duration) ports.Timer { | |
| 926 | - | if delay != time.Second { | |
| 926 | + | if delay != daemon.SnapshotShutdownTimeout() { | |
| 927 | 927 | return c.base.NewTimer(delay) | |
| 928 | 928 | } | |
| 929 | - | timer := &lifecycleManualTimer{ch: make(chan time.Time, 1)} | |
| 930 | - | c.timers <- timer | |
| 929 | + | timer := &lifecycleManualTimer{t: c.t, ch: make(chan time.Time, 1)} | |
| 930 | + | select { | |
| 931 | + | case c.timers <- timer: | |
| 932 | + | default: | |
| 933 | + | c.t.Errorf("unexpected additional snapshot shutdown timer with delay %s", delay) | |
| 934 | + | timer.fire() | |
| 935 | + | } | |
| 931 | 936 | return timer | |
| 932 | 937 | } | |
| 933 | 938 | ||
| @@ -943,12 +948,16 @@ func (c *lifecycleShutdownClock) nextTimer(t *testing.T) *lifecycleManualTimer { | |||
| 943 | 948 | } | |
| 944 | 949 | ||
| 945 | 950 | type lifecycleManualTimer struct { | |
| 951 | + | t *testing.T | |
| 946 | 952 | ch chan time.Time | |
| 947 | 953 | stopped atomic.Bool | |
| 948 | 954 | } | |
| 949 | 955 | ||
| 950 | 956 | func (t *lifecycleManualTimer) C() <-chan time.Time { return t.ch } | |
| 951 | - | func (t *lifecycleManualTimer) Reset(time.Duration) bool { | |
| 957 | + | func (t *lifecycleManualTimer) Reset(delay time.Duration) bool { | |
| 958 | + | if delay != daemon.SnapshotShutdownTimeout() { | |
| 959 | + | t.t.Errorf("snapshot shutdown timer reset with delay %s, want %s", delay, daemon.SnapshotShutdownTimeout()) | |
| 960 | + | } | |
| 952 | 961 | t.stopped.Store(false) | |
| 953 | 962 | return false | |
| 954 | 963 | } | |
M internal/app/durable_crash_matrix_test.go +7 -7
| @@ -132,7 +132,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||
| 132 | 132 | require.NoError(t, catalogue.Close()) | |
| 133 | 133 | ||
| 134 | 134 | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 135 | - | record, exists := catalogue.Record("work") | |
| 135 | + | record, exists, _ := catalogue.Record("work") | |
| 136 | 136 | require.True(t, exists) | |
| 137 | 137 | require.Equal(t, domain.IncarnationID{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, record.IncarnationID) | |
| 138 | 138 | require.Equal(t, domain.RecoveryFresh, record.RecoveryState) | |
| @@ -160,7 +160,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||
| 160 | 160 | ||
| 161 | 161 | ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(publication.Manifest)} | |
| 162 | 162 | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 163 | - | stored, exists := catalogue.Record(record.Name) | |
| 163 | + | stored, exists, _ := catalogue.Record(record.Name) | |
| 164 | 164 | require.True(t, exists) | |
| 165 | 165 | if boundaryName == "snapshot-publish" { | |
| 166 | 166 | require.Equal(t, domain.RecoveryFresh, stored.RecoveryState) | |
| @@ -189,8 +189,8 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||
| 189 | 189 | require.NoError(t, catalogue.Close()) | |
| 190 | 190 | ||
| 191 | 191 | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 192 | - | _, oldExists := catalogue.Record("old") | |
| 193 | - | renamed, newExists := catalogue.Record("new") | |
| 192 | + | _, oldExists, _ := catalogue.Record("old") | |
| 193 | + | renamed, newExists, _ := catalogue.Record("new") | |
| 194 | 194 | require.False(t, oldExists) | |
| 195 | 195 | require.True(t, newExists) | |
| 196 | 196 | require.Equal(t, record.IncarnationID, renamed.IncarnationID) | |
| @@ -228,7 +228,7 @@ func TestDurableTransactionCrashMatrix(t *testing.T) { | |||
| 228 | 228 | require.NoError(t, catalogue.Close()) | |
| 229 | 229 | ||
| 230 | 230 | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 231 | - | _, exists := catalogue.Record(record.Name) | |
| 231 | + | _, exists, _ := catalogue.Record(record.Name) | |
| 232 | 232 | require.False(t, exists) | |
| 233 | 233 | assertNoTombstones(t, repository) | |
| 234 | 234 | require.NoDirExists(t, filepath.Join(h.snapshotDir, "sessions", record.IncarnationID.String())) | |
| @@ -275,7 +275,7 @@ func TestDurableDiscardCrashMatrix(t *testing.T) { | |||
| 275 | 275 | require.NoError(t, catalogue.Close()) | |
| 276 | 276 | ||
| 277 | 277 | h.recoverTwice(t, func(catalogue *persist.Persister, _ *snapshot.Repository, journal *recoveryfs.Journal) { | |
| 278 | - | got, exists := catalogue.Record(old.Name) | |
| 278 | + | got, exists, _ := catalogue.Record(old.Name) | |
| 279 | 279 | require.True(t, exists) | |
| 280 | 280 | require.Equal(t, newID, got.IncarnationID) | |
| 281 | 281 | require.Equal(t, domain.RecoveryFresh, got.RecoveryState) | |
| @@ -312,7 +312,7 @@ func TestDeletionRecoveryProtectsReusedNameFilesystemBytes(t *testing.T) { | |||
| 312 | 312 | require.NoError(t, catalogue.Close()) | |
| 313 | 313 | ||
| 314 | 314 | h.recoverTwice(t, func(catalogue *persist.Persister, repository *snapshot.Repository, _ *recoveryfs.Journal) { | |
| 315 | - | got, exists := catalogue.Record("work") | |
| 315 | + | got, exists, _ := catalogue.Record("work") | |
| 316 | 316 | require.True(t, exists) | |
| 317 | 317 | require.Equal(t, replacement, got) | |
| 318 | 318 | require.Equal(t, replacementBytes, readTree(t, replacementPath)) | |
M internal/app/offline_kill_retry_test.go +39 -46
| @@ -14,50 +14,43 @@ import ( | |||
| 14 | 14 | ) | |
| 15 | 15 | ||
| 16 | 16 | func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) { | |
| 17 | - | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | |
| 18 | - | t.Setenv("XDG_STATE_HOME", stateRoot) | |
| 19 | - | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | |
| 20 | - | ||
| 21 | - | id := domain.IncarnationID{1} | |
| 22 | - | p := newTestPersister(t, filepath.Join(stateRoot, "vev")) | |
| 23 | - | now := time.Now().UnixNano() | |
| 24 | - | require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: id, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh})) | |
| 25 | - | require.NoError(t, p.Close()) | |
| 26 | - | source := filepath.Join(stateRoot, "vev", "snapshots", "sessions", id.String()) | |
| 27 | - | require.NoError(t, os.MkdirAll(source, 0o700)) | |
| 28 | - | require.NoError(t, os.WriteFile(filepath.Join(source, "sentinel"), []byte("durable"), 0o600)) | |
| 29 | - | ||
| 30 | - | require.NoError(t, runKill(context.Background(), "named", false, false)) | |
| 31 | - | require.NoFileExists(t, filepath.Join(source, "sentinel")) | |
| 32 | - | require.FileExists(t, filepath.Join(stateRoot, "vev", "snapshots", "quarantine", id.String(), "snapshot", "sentinel")) | |
| 33 | - | } | |
| 34 | - | ||
| 35 | - | // TestRunKillOfflineFindsSessionWhenOnlyPrevCandidateExists covers the same | |
| 36 | - | // bug class as persist.LoadCatalogueReadOnly's fresh-install regression: a | |
| 37 | - | // crash mid-compaction can leave sessions.kv absent with only a .prev (or | |
| 38 | - | // .next) candidate on disk. `vev kill <name>` must still find and delete the | |
| 39 | - | // session rather than reporting "no such session" from a bare os.Stat | |
| 40 | - | // precheck on the fixed sessions.kv path. | |
| 41 | - | func TestRunKillOfflineFindsSessionWhenOnlyPrevCandidateExists(t *testing.T) { | |
| 42 | - | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | |
| 43 | - | t.Setenv("XDG_STATE_HOME", stateRoot) | |
| 44 | - | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | |
| 45 | - | ||
| 46 | - | id := domain.IncarnationID{1} | |
| 47 | - | p := newTestPersister(t, filepath.Join(stateRoot, "vev")) | |
| 48 | - | now := time.Now().UnixNano() | |
| 49 | - | require.NoError(t, p.Save(persist.Record{Name: "crashed", IncarnationID: id, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh})) | |
| 50 | - | require.NoError(t, p.Close()) | |
| 51 | - | source := filepath.Join(stateRoot, "vev", "snapshots", "sessions", id.String()) | |
| 52 | - | require.NoError(t, os.MkdirAll(source, 0o700)) | |
| 53 | - | require.NoError(t, os.WriteFile(filepath.Join(source, "sentinel"), []byte("durable"), 0o600)) | |
| 54 | - | ||
| 55 | - | // Simulate a crash mid-compaction: current renamed away to .prev, no | |
| 56 | - | // sessions.kv on disk at all. | |
| 57 | - | storePath := persist.StorePath(filepath.Join(stateRoot, "vev")) | |
| 58 | - | require.NoError(t, os.Rename(storePath, storePath+".prev")) | |
| 59 | - | ||
| 60 | - | require.NoError(t, runKill(context.Background(), "crashed", false, false)) | |
| 61 | - | require.NoFileExists(t, filepath.Join(source, "sentinel")) | |
| 62 | - | require.FileExists(t, filepath.Join(stateRoot, "vev", "snapshots", "quarantine", id.String(), "snapshot", "sentinel")) | |
| 17 | + | tests := []struct { | |
| 18 | + | name string | |
| 19 | + | crashHook func(*testing.T, string) | |
| 20 | + | }{ | |
| 21 | + | {name: "normal"}, | |
| 22 | + | { | |
| 23 | + | name: ".prev crash hook", | |
| 24 | + | crashHook: func(t *testing.T, stateDir string) { | |
| 25 | + | t.Helper() | |
| 26 | + | storePath := persist.StorePath(stateDir) | |
| 27 | + | require.NoError(t, os.Rename(storePath, storePath+".prev")) | |
| 28 | + | }, | |
| 29 | + | }, | |
| 30 | + | } | |
| 31 | + | ||
| 32 | + | for _, tt := range tests { | |
| 33 | + | t.Run(tt.name, func(t *testing.T) { | |
| 34 | + | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | |
| 35 | + | t.Setenv("XDG_STATE_HOME", stateRoot) | |
| 36 | + | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | |
| 37 | + | stateDir := filepath.Join(stateRoot, "vev") | |
| 38 | + | ||
| 39 | + | id := domain.IncarnationID{1} | |
| 40 | + | p := newTestPersister(t, stateDir) | |
| 41 | + | now := time.Now().UnixNano() | |
| 42 | + | require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: id, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now, RecoveryState: domain.RecoveryFresh})) | |
| 43 | + | require.NoError(t, p.Close()) | |
| 44 | + | source := filepath.Join(stateDir, "snapshots", "sessions", id.String()) | |
| 45 | + | require.NoError(t, os.MkdirAll(source, 0o700)) | |
| 46 | + | require.NoError(t, os.WriteFile(filepath.Join(source, "sentinel"), []byte("durable"), 0o600)) | |
| 47 | + | if tt.crashHook != nil { | |
| 48 | + | tt.crashHook(t, stateDir) | |
| 49 | + | } | |
| 50 | + | ||
| 51 | + | require.NoError(t, runKill(context.Background(), "named", false, false)) | |
| 52 | + | require.NoFileExists(t, filepath.Join(source, "sentinel")) | |
| 53 | + | require.FileExists(t, filepath.Join(stateDir, "snapshots", "quarantine", id.String(), "snapshot", "sentinel")) | |
| 54 | + | }) | |
| 55 | + | } | |
| 63 | 56 | } | |
M internal/app/run.go +29 -8
| @@ -583,18 +583,22 @@ type fencedRecoveryIdentity struct { | |||
| 583 | 583 | ||
| 584 | 584 | // fencedRecoveries resolves each conflict against the post-recovery catalogue | |
| 585 | 585 | // so a fenced session is reported with the incarnation it actually holds now. | |
| 586 | - | func fencedRecoveries(conflicts []recovery.RecoveryConflict, catalogue ports.Catalogue) []fencedRecoveryIdentity { | |
| 586 | + | func fencedRecoveries(conflicts []recovery.RecoveryConflict, catalogue ports.Catalogue) ([]fencedRecoveryIdentity, error) { | |
| 587 | 587 | fenced := make([]fencedRecoveryIdentity, 0, len(conflicts)) | |
| 588 | 588 | for _, conflict := range conflicts { | |
| 589 | 589 | identity := fencedRecoveryIdentity{name: conflict.Session, kind: conflict.Kind, reasonCode: fencedRecoveryReasonCode(conflict.Err)} | |
| 590 | 590 | if catalogue != nil { | |
| 591 | - | if record, ok := catalogue.Record(conflict.Session); ok { | |
| 591 | + | record, ok, err := catalogue.Record(conflict.Session) | |
| 592 | + | if err != nil { | |
| 593 | + | return nil, err | |
| 594 | + | } | |
| 595 | + | if ok { | |
| 592 | 596 | identity.id = record.IncarnationID | |
| 593 | 597 | } | |
| 594 | 598 | } | |
| 595 | 599 | fenced = append(fenced, identity) | |
| 596 | 600 | } | |
| 597 | - | return fenced | |
| 601 | + | return fenced, nil | |
| 598 | 602 | } | |
| 599 | 603 | ||
| 600 | 604 | func fencedRecoveryReasonCode(err error) string { | |
| @@ -674,6 +678,13 @@ func runDaemonOwned(ctx context.Context) (retErr error) { | |||
| 674 | 678 | return runDaemonOwnedWithLogger(ctx, log) | |
| 675 | 679 | } | |
| 676 | 680 | ||
| 681 | + | func recoverAuthoritativeCatalogue(ctx context.Context, coordinator *recovery.Coordinator, catalogue ports.Catalogue) ([]domain.CatalogueRecord, error) { | |
| 682 | + | if err := coordinator.Recover(ctx); err != nil { | |
| 683 | + | return nil, err | |
| 684 | + | } | |
| 685 | + | return catalogue.Records() | |
| 686 | + | } | |
| 687 | + | ||
| 677 | 688 | func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr error) { | |
| 678 | 689 | clk := clock.New() | |
| 679 | 690 | observer, observerCloser, err := newPerformanceTrace(clk) | |
| @@ -710,7 +721,7 @@ func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr err | |||
| 710 | 721 | daemonOpts = append(daemonOpts, daemon.WithBarScriptCommandRunner(shellcmd.New())) | |
| 711 | 722 | daemonOpts = append(daemonOpts, daemon.WithProcessInspector(platform.NewProcessInspector()), daemon.WithDirOrHome(platform.DirOrHome)) | |
| 712 | 723 | snapshotRepository := snapshotadapter.NewRepository(snapshotDir(), log) | |
| 713 | - | daemonOpts = append(daemonOpts, daemon.WithSnapshotRepository(snapshotRepository, snapshotRepository)) | |
| 724 | + | daemonOpts = append(daemonOpts, daemon.WithSnapshotRepository(snapshotRepository)) | |
| 714 | 725 | noticeStore := noticefile.New(platform.StateDir()) | |
| 715 | 726 | daemonOpts = append(daemonOpts, daemon.WithNoticeStore(noticeStore)) | |
| 716 | 727 | storePath := persist.StorePath(platform.StateDir()) | |
| @@ -736,13 +747,21 @@ func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr err | |||
| 736 | 747 | return errors.Join(fmt.Errorf("vev: inspect durable session transactions: %w", err), opened.Catalogue.Close()) | |
| 737 | 748 | } | |
| 738 | 749 | coordinator := recovery.NewCoordinator(opened.Catalogue, snapshotRepository, journal, rand.Reader) | |
| 739 | - | if err := coordinator.Recover(ctx); err != nil { | |
| 740 | - | return errors.Join(fmt.Errorf("vev: recover durable session transactions: %w", err), opened.Catalogue.Close()) | |
| 750 | + | // Recovery may roll catalogue changes forward, so reload the authoritative | |
| 751 | + | // snapshot before constructing runtime reservations. A failed reload aborts | |
| 752 | + | // startup rather than treating durable sessions as absent. | |
| 753 | + | recoveredRecords, err := recoverAuthoritativeCatalogue(ctx, coordinator, opened.Catalogue) | |
| 754 | + | if err != nil { | |
| 755 | + | return errors.Join(fmt.Errorf("vev: recover and reload durable session transactions: %w", err), opened.Catalogue.Close()) | |
| 741 | 756 | } | |
| 757 | + | opened.Records = recoveredRecords | |
| 742 | 758 | // Recovery fences a session instead of failing startup when only that | |
| 743 | 759 | // session's durable state is self-inconsistent. The daemon then comes up | |
| 744 | 760 | // looking healthy, so every fenced session must be reported and persisted. | |
| 745 | - | fenced := fencedRecoveries(coordinator.Conflicts(), opened.Catalogue) | |
| 761 | + | fenced, err := fencedRecoveries(coordinator.Conflicts(), opened.Catalogue) | |
| 762 | + | if err != nil { | |
| 763 | + | return errors.Join(fmt.Errorf("vev: read fenced recovery identities: %w", err), opened.Catalogue.Close()) | |
| 764 | + | } | |
| 746 | 765 | logFencedRecoveries(log, fenced) | |
| 747 | 766 | if err := persistTransactionRecoveryNotices(noticeStore, pendingRecoveries, clk.Now()); err != nil { | |
| 748 | 767 | log.Error("transaction_recovery_notice_failed", "err", err) | |
| @@ -1558,7 +1577,9 @@ func runOfflineNamedKill(ctx context.Context, name string) (retErr error) { | |||
| 1558 | 1577 | } | |
| 1559 | 1578 | defer func() { retErr = errors.Join(retErr, opened.Catalogue.Close()) }() | |
| 1560 | 1579 | ||
| 1561 | - | if _, ok := opened.Catalogue.Record(name); !ok { | |
| 1580 | + | if _, ok, err := opened.Catalogue.Record(name); err != nil { | |
| 1581 | + | return fmt.Errorf("vev: reading stored session: %w", err) | |
| 1582 | + | } else if !ok { | |
| 1562 | 1583 | return fmt.Errorf("vev: no such session: %s", name) | |
| 1563 | 1584 | } | |
| 1564 | 1585 | coordinator := recovery.NewCoordinator(opened.Catalogue, repository, recoveryfs.New(platform.StateDir()), rand.Reader) | |
M internal/app/run_test.go +52 -19
| @@ -19,6 +19,7 @@ import ( | |||
| 19 | 19 | "time" | |
| 20 | 20 | ||
| 21 | 21 | "github.com/bnema/vev/internal/adapters/lifecycle" | |
| 22 | + | "github.com/bnema/vev/internal/adapters/noticefile" | |
| 22 | 23 | "github.com/bnema/vev/internal/adapters/recoveryfs" | |
| 23 | 24 | "github.com/bnema/vev/internal/adapters/snapshot" | |
| 24 | 25 | "github.com/bnema/vev/internal/domain" | |
| @@ -110,9 +111,6 @@ func TestRecoveryObservability(t *testing.T) { | |||
| 110 | 111 | require.Equal(t, "clean", recovered["outcome"]) | |
| 111 | 112 | require.NotContains(t, eventNames(entries), "interrupted_transaction_fenced") | |
| 112 | 113 | ||
| 113 | - | logBuffer.Reset() | |
| 114 | - | log.Error("catalogue_validation_failed", "path", "/state/vev/sessions.kv", "reason_code", "corrupt") | |
| 115 | - | require.NotContains(t, eventNames(decodeJSONLogs(t, logBuffer.Bytes())), "daemon_startup_complete") | |
| 116 | 114 | require.NotContains(t, fmt.Sprint(requireEvent(t, entries, "interrupted_transaction_recovery_complete")), "terminal contents") | |
| 117 | 115 | require.Equal(t, domain.NoticeSnapshotRestore, recoveryNotice.Code) | |
| 118 | 116 | require.Equal(t, domain.SessionID(domain.IncarnationID{9}.String()), recoveryNotice.SessionID) | |
| @@ -122,6 +120,27 @@ func TestRecoveryObservability(t *testing.T) { | |||
| 122 | 120 | // A session-scoped conflict no longer aborts startup, so the daemon comes up | |
| 123 | 121 | // looking healthy. Startup must therefore name the fenced session in the log, | |
| 124 | 122 | // persist a notice for it, and never report the recovery as clean. | |
| 123 | + | func TestStartupReloadsCatalogueAfterRolledForwardDeletion(t *testing.T) { | |
| 124 | + | ctx := t.Context() | |
| 125 | + | stateDir := filepath.Join(t.TempDir(), "vev") | |
| 126 | + | catalogue := newTestPersister(t, stateDir) | |
| 127 | + | t.Cleanup(func() { _ = catalogue.Close() }) | |
| 128 | + | repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots")) | |
| 129 | + | record := domain.CatalogueRecord{ | |
| 130 | + | Name: "work", IncarnationID: domain.IncarnationID{9}, Cwd: t.TempDir(), | |
| 131 | + | CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryDeleting, | |
| 132 | + | } | |
| 133 | + | require.NoError(t, catalogue.Save(record)) | |
| 134 | + | coordinator := recovery.NewCoordinator(catalogue, repository, recoveryfs.New(stateDir), rand.Reader) | |
| 135 | + | ||
| 136 | + | records, err := recoverAuthoritativeCatalogue(ctx, coordinator, catalogue) | |
| 137 | + | require.NoError(t, err) | |
| 138 | + | require.Empty(t, records, "daemon startup must not retain the pre-recovery reservation") | |
| 139 | + | _, exists, err := catalogue.Record(record.Name) | |
| 140 | + | require.NoError(t, err) | |
| 141 | + | require.False(t, exists, "rolled-forward deletion must remain absent in authoritative state") | |
| 142 | + | } | |
| 143 | + | ||
| 125 | 144 | func TestFencedRecoveryObservability(t *testing.T) { | |
| 126 | 145 | ctx := context.Background() | |
| 127 | 146 | stateDir := filepath.Join(t.TempDir(), "vev") | |
| @@ -139,7 +158,8 @@ func TestFencedRecoveryObservability(t *testing.T) { | |||
| 139 | 158 | ||
| 140 | 159 | coordinator := recovery.NewCoordinator(catalogue, repository, recoveryfs.New(stateDir), rand.Reader) | |
| 141 | 160 | require.NoError(t, coordinator.Recover(ctx), "a session-scoped conflict must not abort startup") | |
| 142 | - | fenced := fencedRecoveries(coordinator.Conflicts(), catalogue) | |
| 161 | + | fenced, err := fencedRecoveries(coordinator.Conflicts(), catalogue) | |
| 162 | + | require.NoError(t, err) | |
| 143 | 163 | require.Equal(t, []fencedRecoveryIdentity{{ | |
| 144 | 164 | name: live.Name, id: live.IncarnationID, kind: "deletion-tombstone", reasonCode: "deletion-tombstone-conflict", | |
| 145 | 165 | }}, fenced) | |
| @@ -148,7 +168,9 @@ func TestFencedRecoveryObservability(t *testing.T) { | |||
| 148 | 168 | log := slog.New(slog.NewJSONHandler(&logBuffer, nil)) | |
| 149 | 169 | logFencedRecoveries(log, fenced) | |
| 150 | 170 | logTransactionRecovery(log, 1, len(fenced)) | |
| 151 | - | logStartupRecoveryCounts(log, catalogue.Records(), 0) | |
| 171 | + | records, err := catalogue.Records() | |
| 172 | + | require.NoError(t, err) | |
| 173 | + | logStartupRecoveryCounts(log, records, 0) | |
| 152 | 174 | ||
| 153 | 175 | entries := decodeJSONLogs(t, logBuffer.Bytes()) | |
| 154 | 176 | event := requireEvent(t, entries, "interrupted_transaction_fenced") | |
| @@ -163,17 +185,22 @@ func TestFencedRecoveryObservability(t *testing.T) { | |||
| 163 | 185 | require.Equal(t, "fenced", recovered["outcome"]) | |
| 164 | 186 | require.Contains(t, eventNames(entries), "daemon_startup_complete") | |
| 165 | 187 | ||
| 166 | - | store := portsmocks.NewMockNoticeStore(t) | |
| 167 | - | var notice domain.Notification | |
| 168 | - | store.EXPECT().Append(mock.Anything).Run(func(n domain.Notification) { notice = n }).Return(nil).Once() | |
| 188 | + | store := noticefile.New(stateDir) | |
| 169 | 189 | require.NoError(t, persistFencedRecoveryNotices(store, fenced, time.Unix(1, 0))) | |
| 190 | + | reopened := noticefile.New(stateDir) | |
| 191 | + | pending, err := reopened.Claim() | |
| 192 | + | require.NoError(t, err) | |
| 193 | + | require.Len(t, pending, 1) | |
| 194 | + | notice := pending[0] | |
| 170 | 195 | require.Equal(t, domain.NoticeSnapshotRestore, notice.Code) | |
| 171 | 196 | require.Equal(t, domain.NoticeWarn, notice.Severity) | |
| 172 | 197 | require.Equal(t, domain.SessionID(live.IncarnationID.String()), notice.SessionID) | |
| 173 | 198 | require.Contains(t, notice.Details, "session="+live.Name) | |
| 174 | 199 | require.Contains(t, notice.Details, "reason_code=deletion-tombstone-conflict") | |
| 200 | + | require.NoError(t, reopened.Ack()) | |
| 175 | 201 | ||
| 176 | - | degraded, ok := catalogue.Record(live.Name) | |
| 202 | + | degraded, ok, err := catalogue.Record(live.Name) | |
| 203 | + | require.NoError(t, err) | |
| 177 | 204 | require.True(t, ok) | |
| 178 | 205 | require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState) | |
| 179 | 206 | } | |
| @@ -444,19 +471,25 @@ func TestLifecycleOwnershipPrecedesDaemonStartup(t *testing.T) { | |||
| 444 | 471 | stateDir := filepath.Join(stateRoot, "vev") | |
| 445 | 472 | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | |
| 446 | 473 | t.Setenv("XDG_STATE_HOME", stateRoot) | |
| 447 | - | ctx, cancel := context.WithCancel(context.Background()) | |
| 448 | - | defer cancel() | |
| 474 | + | ctx, cancel := context.WithCancel(t.Context()) | |
| 475 | + | result := make(chan error, 1) | |
| 449 | 476 | go func() { | |
| 450 | - | for { | |
| 451 | - | if _, err := os.Stat(filepath.Join(runtimeDir, "daemon.sock")); err == nil { | |
| 452 | - | cancel() | |
| 453 | - | return | |
| 454 | - | } | |
| 455 | - | time.Sleep(time.Millisecond) | |
| 456 | - | } | |
| 477 | + | result <- runWithLifecycleOwner(ctx, runtimeDir, stateDir, runDaemonOwned) | |
| 457 | 478 | }() | |
| 479 | + | t.Cleanup(func() { | |
| 480 | + | cancel() | |
| 481 | + | select { | |
| 482 | + | case err := <-result: | |
| 483 | + | require.NoError(t, err) | |
| 484 | + | case <-time.After(5 * time.Second): | |
| 485 | + | t.Error("daemon lifecycle did not stop after cancellation") | |
| 486 | + | } | |
| 487 | + | }) | |
| 458 | 488 | ||
| 459 | - | require.NoError(t, runWithLifecycleOwner(ctx, runtimeDir, stateDir, runDaemonOwned)) | |
| 489 | + | require.Eventually(t, func() bool { | |
| 490 | + | _, err := os.Stat(filepath.Join(runtimeDir, "daemon.sock")) | |
| 491 | + | return err == nil | |
| 492 | + | }, 5*time.Second, time.Millisecond, "daemon socket was not published") | |
| 460 | 493 | require.FileExists(t, persist.StorePath(stateDir)) | |
| 461 | 494 | }) | |
| 462 | 495 | ||
M internal/app/spawn.go +45 -49
| @@ -67,25 +67,29 @@ var defaultBackoff = backoffConfig{ | |||
| 67 | 67 | total: 5 * time.Second, | |
| 68 | 68 | } | |
| 69 | 69 | ||
| 70 | - | func waitForLifecycleAvailability(ctx context.Context, runtimeDir string, cfg backoffConfig) (lifecycleOwnership, error) { | |
| 70 | + | func retryAttempts[T any](ctx context.Context, cfg backoffConfig, attempt func() (T, bool, error)) (T, error) { | |
| 71 | 71 | deadline := time.Now().Add(cfg.total) | |
| 72 | 72 | backoff := cfg.initial | |
| 73 | 73 | for { | |
| 74 | 74 | if err := ctx.Err(); err != nil { | |
| 75 | - | return nil, err | |
| 75 | + | var zero T | |
| 76 | + | return zero, err | |
| 76 | 77 | } | |
| 77 | - | owner, err := daemonLifecycleProbe.TryAcquire(runtimeDir) | |
| 78 | - | if err == nil { | |
| 79 | - | return owner, nil | |
| 78 | + | result, done, err := attempt() | |
| 79 | + | if err != nil { | |
| 80 | + | var zero T | |
| 81 | + | return zero, err | |
| 80 | 82 | } | |
| 81 | - | if !errors.Is(err, lifecycle.ErrBusy) { | |
| 82 | - | return nil, err | |
| 83 | + | if done { | |
| 84 | + | return result, nil | |
| 83 | 85 | } | |
| 84 | 86 | if !time.Now().Before(deadline) { | |
| 85 | - | return nil, ErrDaemonUnreachable | |
| 87 | + | var zero T | |
| 88 | + | return zero, ErrDaemonUnreachable | |
| 86 | 89 | } | |
| 87 | 90 | if err := waitBackoff(ctx, backoff); err != nil { | |
| 88 | - | return nil, err | |
| 91 | + | var zero T | |
| 92 | + | return zero, err | |
| 89 | 93 | } | |
| 90 | 94 | if backoff *= 2; backoff > cfg.max { | |
| 91 | 95 | backoff = cfg.max | |
| @@ -93,33 +97,39 @@ func waitForLifecycleAvailability(ctx context.Context, runtimeDir string, cfg ba | |||
| 93 | 97 | } | |
| 94 | 98 | } | |
| 95 | 99 | ||
| 96 | - | func waitForDaemonOrLifecycle(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, lifecycleOwnership, error) { | |
| 97 | - | deadline := time.Now().Add(cfg.total) | |
| 98 | - | backoff := cfg.initial | |
| 99 | - | for { | |
| 100 | - | if err := ctx.Err(); err != nil { | |
| 101 | - | return nil, nil, err | |
| 100 | + | func waitForLifecycleAvailability(ctx context.Context, runtimeDir string, cfg backoffConfig) (lifecycleOwnership, error) { | |
| 101 | + | return retryAttempts(ctx, cfg, func() (lifecycleOwnership, bool, error) { | |
| 102 | + | owner, err := daemonLifecycleProbe.TryAcquire(runtimeDir) | |
| 103 | + | if err == nil { | |
| 104 | + | return owner, true, nil | |
| 105 | + | } | |
| 106 | + | if !errors.Is(err, lifecycle.ErrBusy) { | |
| 107 | + | return nil, false, err | |
| 102 | 108 | } | |
| 109 | + | return nil, false, nil | |
| 110 | + | }) | |
| 111 | + | } | |
| 112 | + | ||
| 113 | + | type daemonOrLifecycle struct { | |
| 114 | + | transport ports.Transport | |
| 115 | + | owner lifecycleOwnership | |
| 116 | + | } | |
| 117 | + | ||
| 118 | + | func waitForDaemonOrLifecycle(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, lifecycleOwnership, error) { | |
| 119 | + | result, err := retryAttempts(ctx, cfg, func() (daemonOrLifecycle, bool, error) { | |
| 103 | 120 | if transport, err := dial(ctx, dir); err == nil { | |
| 104 | - | return transport, nil, nil | |
| 121 | + | return daemonOrLifecycle{transport: transport}, true, nil | |
| 105 | 122 | } | |
| 106 | 123 | owner, err := daemonLifecycleProbe.TryAcquire(dir) | |
| 107 | 124 | if err == nil { | |
| 108 | - | return nil, owner, nil | |
| 125 | + | return daemonOrLifecycle{owner: owner}, true, nil | |
| 109 | 126 | } | |
| 110 | 127 | if !errors.Is(err, lifecycle.ErrBusy) { | |
| 111 | - | return nil, nil, err | |
| 128 | + | return daemonOrLifecycle{}, false, err | |
| 112 | 129 | } | |
| 113 | - | if !time.Now().Before(deadline) { | |
| 114 | - | return nil, nil, ErrDaemonUnreachable | |
| 115 | - | } | |
| 116 | - | if err := waitBackoff(ctx, backoff); err != nil { | |
| 117 | - | return nil, nil, err | |
| 118 | - | } | |
| 119 | - | if backoff *= 2; backoff > cfg.max { | |
| 120 | - | backoff = cfg.max | |
| 121 | - | } | |
| 122 | - | } | |
| 130 | + | return daemonOrLifecycle{}, false, nil | |
| 131 | + | }) | |
| 132 | + | return result.transport, result.owner, err | |
| 123 | 133 | } | |
| 124 | 134 | ||
| 125 | 135 | func waitBackoff(ctx context.Context, duration time.Duration) error { | |
| @@ -224,28 +234,14 @@ func acquireSpawnLock(dir string) (release func(), acquired bool, err error) { | |||
| 224 | 234 | // retryDial dials repeatedly with exponential backoff until the daemon | |
| 225 | 235 | // answers, the context is cancelled, or the total budget is exhausted. | |
| 226 | 236 | func retryDial(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, error) { | |
| 227 | - | deadline := time.Now().Add(cfg.total) | |
| 228 | - | backoff := cfg.initial | |
| 229 | - | for { | |
| 230 | - | if err := ctx.Err(); err != nil { | |
| 231 | - | return nil, err | |
| 232 | - | } | |
| 233 | - | if t, err := dial(ctx, dir); err == nil { | |
| 234 | - | return t, nil | |
| 235 | - | } | |
| 236 | - | if !time.Now().Before(deadline) { | |
| 237 | - | slog.Error("daemon did not become reachable before retry budget expired", "socket_dir", dir, "budget", cfg.total) | |
| 238 | - | return nil, ErrDaemonUnreachable | |
| 239 | - | } | |
| 240 | - | select { | |
| 241 | - | case <-ctx.Done(): | |
| 242 | - | return nil, ctx.Err() | |
| 243 | - | case <-time.After(backoff): | |
| 244 | - | } | |
| 245 | - | if backoff *= 2; backoff > cfg.max { | |
| 246 | - | backoff = cfg.max | |
| 247 | - | } | |
| 237 | + | transport, err := retryAttempts(ctx, cfg, func() (ports.Transport, bool, error) { | |
| 238 | + | transport, err := dial(ctx, dir) | |
| 239 | + | return transport, err == nil, nil | |
| 240 | + | }) | |
| 241 | + | if errors.Is(err, ErrDaemonUnreachable) { | |
| 242 | + | slog.Error("daemon did not become reachable before retry budget expired", "socket_dir", dir, "budget", cfg.total) | |
| 248 | 243 | } | |
| 244 | + | return transport, err | |
| 249 | 245 | } | |
| 250 | 246 | ||
| 251 | 247 | // realDial is the production dialer. | |
M internal/app/spawn_test.go +15 -0
| @@ -258,6 +258,21 @@ func TestAcquireSpawnLockTakesOverStaleLock(t *testing.T) { | |||
| 258 | 258 | release() | |
| 259 | 259 | } | |
| 260 | 260 | ||
| 261 | + | func TestRetryAttemptsChecksOperationErrorBeforeDeadline(t *testing.T) { | |
| 262 | + | attemptErr := errors.New("attempt failed") | |
| 263 | + | calls := 0 | |
| 264 | + | _, err := retryAttempts(context.Background(), backoffConfig{}, func() (struct{}, bool, error) { | |
| 265 | + | calls++ | |
| 266 | + | return struct{}{}, false, attemptErr | |
| 267 | + | }) | |
| 268 | + | if !errors.Is(err, attemptErr) { | |
| 269 | + | t.Fatalf("retryAttempts error = %v, want %v", err, attemptErr) | |
| 270 | + | } | |
| 271 | + | if calls != 1 { | |
| 272 | + | t.Fatalf("attempt called %d times, want 1", calls) | |
| 273 | + | } | |
| 274 | + | } | |
| 275 | + | ||
| 261 | 276 | func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) { | |
| 262 | 277 | want := portsmocks.NewMockTransport(t) | |
| 263 | 278 | var calls atomic.Int32 | |
M internal/domain/recovery.go +19 -0
| @@ -5,6 +5,7 @@ import ( | |||
| 5 | 5 | "errors" | |
| 6 | 6 | "fmt" | |
| 7 | 7 | "io" | |
| 8 | + | "slices" | |
| 8 | 9 | "strings" | |
| 9 | 10 | ) | |
| 10 | 11 | ||
| @@ -44,6 +45,24 @@ type CatalogueRecord struct { | |||
| 44 | 45 | ||
| 45 | 46 | // CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue | |
| 46 | 47 | // incarnation without carrying authority-owned recovery or checkpoint fields. | |
| 48 | + | // Equal reports canonical value equality between checkpoint references. | |
| 49 | + | func (r *CheckpointRef) Equal(other *CheckpointRef) bool { | |
| 50 | + | if r == nil || other == nil { | |
| 51 | + | return r == nil && other == nil | |
| 52 | + | } | |
| 53 | + | return *r == *other | |
| 54 | + | } | |
| 55 | + | ||
| 56 | + | // Equal reports canonical value equality, including ordered tab names and | |
| 57 | + | // pointed-to checkpoint values rather than pointer identity. | |
| 58 | + | func (r CatalogueRecord) Equal(other CatalogueRecord) bool { | |
| 59 | + | return r.Name == other.Name && r.IncarnationID == other.IncarnationID && r.Cwd == other.Cwd && | |
| 60 | + | r.CreatedAt == other.CreatedAt && r.UpdatedAt == other.UpdatedAt && r.LastUsedSeq == other.LastUsedSeq && | |
| 61 | + | slices.Equal(r.TabNames, other.TabNames) && r.RecoveryState == other.RecoveryState && | |
| 62 | + | r.Committed.Equal(other.Committed) && r.Fallbacks[0].Equal(other.Fallbacks[0]) && | |
| 63 | + | r.Fallbacks[1].Equal(other.Fallbacks[1]) && r.DegradedReason == other.DegradedReason | |
| 64 | + | } | |
| 65 | + | ||
| 47 | 66 | type CatalogueMetadataUpdate struct { | |
| 48 | 67 | Name string | |
| 49 | 68 | IncarnationID IncarnationID | |
M internal/domain/recovery_test.go +22 -1
| @@ -22,7 +22,12 @@ func TestIncarnationIDText(t *testing.T) { | |||
| 22 | 22 | var decoded IncarnationID | |
| 23 | 23 | require.NoError(t, decoded.UnmarshalText(text)) | |
| 24 | 24 | require.Equal(t, id, decoded) | |
| 25 | - | require.Error(t, decoded.UnmarshalText([]byte("AB"))) | |
| 25 | + | require.Error(t, decoded.UnmarshalText([]byte("AB0000000000000000000000000000CD"))) | |
| 26 | + | ||
| 27 | + | _, err = (IncarnationID{}).MarshalText() | |
| 28 | + | require.Error(t, err) | |
| 29 | + | require.Error(t, decoded.UnmarshalText([]byte("00000000000000000000000000000000"))) | |
| 30 | + | require.Equal(t, id, decoded, "rejected zero text must not mutate the receiver") | |
| 26 | 31 | } | |
| 27 | 32 | ||
| 28 | 33 | func TestCatalogueRecordValidate(t *testing.T) { | |
| @@ -50,6 +55,7 @@ func TestCatalogueRecordValidate(t *testing.T) { | |||
| 50 | 55 | {name: "zero checkpoint generation", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{ManifestDigest: [32]byte{1}}}}, | |
| 51 | 56 | {name: "zero checkpoint digest", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 7}}}, | |
| 52 | 57 | {name: "second fallback without first", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: validRef, Fallbacks: [2]*CheckpointRef{nil, {Generation: 6, ManifestDigest: [32]byte{2}}}}}, | |
| 58 | + | {name: "fallback newer than committed", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, RecoveryState: RecoveryHealthy, Committed: &CheckpointRef{Generation: 6, ManifestDigest: [32]byte{1}}, Fallbacks: [2]*CheckpointRef{{Generation: 7, ManifestDigest: [32]byte{2}}}}}, | |
| 53 | 59 | } | |
| 54 | 60 | for _, tt := range tests { | |
| 55 | 61 | t.Run(tt.name, func(t *testing.T) { | |
| @@ -58,6 +64,21 @@ func TestCatalogueRecordValidate(t *testing.T) { | |||
| 58 | 64 | } | |
| 59 | 65 | } | |
| 60 | 66 | ||
| 67 | + | func TestCatalogueRecordEqualUsesCheckpointValuesAndOrderedTabs(t *testing.T) { | |
| 68 | + | committed := CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}} | |
| 69 | + | left := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, TabNames: []string{"shell", "logs"}, RecoveryState: RecoveryHealthy, Committed: &committed} | |
| 70 | + | right := left | |
| 71 | + | right.TabNames = append([]string(nil), left.TabNames...) | |
| 72 | + | copy := committed | |
| 73 | + | right.Committed = © | |
| 74 | + | require.True(t, left.Equal(right)) | |
| 75 | + | ||
| 76 | + | right.TabNames[0], right.TabNames[1] = right.TabNames[1], right.TabNames[0] | |
| 77 | + | require.False(t, left.Equal(right)) | |
| 78 | + | require.True(t, (*CheckpointRef)(nil).Equal(nil)) | |
| 79 | + | require.False(t, left.Committed.Equal(nil)) | |
| 80 | + | } | |
| 81 | + | ||
| 61 | 82 | func TestDeletionTombstoneValidate(t *testing.T) { | |
| 62 | 83 | tests := []struct { | |
| 63 | 84 | name string | |
M internal/persist/catalogue_test.go +31 -3
| @@ -40,11 +40,29 @@ func testCatalogueRecordRoundTrip(t *testing.T) { | |||
| 40 | 40 | got, err := decodeRecordValue(record.Name, encoded) | |
| 41 | 41 | require.NoError(t, err) | |
| 42 | 42 | require.Equal(t, record, got) | |
| 43 | + | for prefix := range len(encoded) { | |
| 44 | + | _, err := decodeRecordValue(record.Name, encoded[:prefix]) | |
| 45 | + | require.Error(t, err, "prefix length %d", prefix) | |
| 46 | + | } | |
| 43 | 47 | require.Error(t, func() error { _, err := decodeRecordValue(record.Name, append(encoded, 0)); return err }()) | |
| 44 | 48 | }) | |
| 45 | 49 | } | |
| 46 | 50 | } | |
| 47 | 51 | ||
| 52 | + | func TestDecodeRecordValueWrapsMalformedAndValidationErrors(t *testing.T) { | |
| 53 | + | encoded, err := encodeRecordValue(validRecord("work", 1)) | |
| 54 | + | require.NoError(t, err) | |
| 55 | + | const recoveryStateOffset = 4 + 2 + 16 + 4 + 8 + 8 + 8 + 4 | |
| 56 | + | encoded[recoveryStateOffset] = 99 | |
| 57 | + | ||
| 58 | + | _, err = decodeRecordValue("work", encoded) | |
| 59 | + | require.ErrorIs(t, err, errMalformedRecord) | |
| 60 | + | joined, ok := err.(interface{ Unwrap() []error }) | |
| 61 | + | require.True(t, ok) | |
| 62 | + | require.Len(t, joined.Unwrap(), 2) | |
| 63 | + | require.ErrorContains(t, joined.Unwrap()[1], "invalid recovery state") | |
| 64 | + | } | |
| 65 | + | ||
| 48 | 66 | func testCatalogueDuplicateIncarnations(t *testing.T) { | |
| 49 | 67 | id := domain.IncarnationID{1} | |
| 50 | 68 | require.Error(t, validateUniqueIncarnations([]domain.CatalogueRecord{ | |
| @@ -110,7 +128,8 @@ func testCatalogueMetadataUpdatePreservesAuthority(t *testing.T) { | |||
| 110 | 128 | next := domain.CatalogueRecord{Name: "work", IncarnationID: original.IncarnationID, Cwd: "/new", UpdatedAt: 22, LastUsedSeq: 23, TabNames: []string{"editor", "logs"}} | |
| 111 | 129 | update := next.MetadataUpdate() | |
| 112 | 130 | require.NoError(t, p.UpdateMetadata(update)) | |
| 113 | - | got, ok := p.Record("work") | |
| 131 | + | got, ok, err := p.Record("work") | |
| 132 | + | require.NoError(t, err) | |
| 114 | 133 | require.True(t, ok) | |
| 115 | 134 | expected := original | |
| 116 | 135 | expected.Cwd, expected.UpdatedAt, expected.LastUsedSeq, expected.TabNames = next.Cwd, next.UpdatedAt, next.LastUsedSeq, next.TabNames | |
| @@ -121,14 +140,16 @@ func testCatalogueMetadataUpdatePreservesAuthority(t *testing.T) { | |||
| 121 | 140 | Name: "work", IncarnationID: original.IncarnationID, LastUsedSeq: &lastUsedSeq, | |
| 122 | 141 | })) | |
| 123 | 142 | expected.LastUsedSeq = lastUsedSeq | |
| 124 | - | partiallyUpdated, ok := p.Record("work") | |
| 143 | + | partiallyUpdated, ok, err := p.Record("work") | |
| 144 | + | require.NoError(t, err) | |
| 125 | 145 | require.True(t, ok) | |
| 126 | 146 | require.Equal(t, expected, partiallyUpdated, "a partial metadata update must not clear other mutable or authority-owned fields") | |
| 127 | 147 | ||
| 128 | 148 | stale := update | |
| 129 | 149 | stale.IncarnationID = domain.IncarnationID{2} | |
| 130 | 150 | require.Error(t, p.UpdateMetadata(stale)) | |
| 131 | - | unchanged, ok := p.Record("work") | |
| 151 | + | unchanged, ok, err := p.Record("work") | |
| 152 | + | require.NoError(t, err) | |
| 132 | 153 | require.True(t, ok) | |
| 133 | 154 | require.Equal(t, expected, unchanged) | |
| 134 | 155 | } | |
| @@ -201,6 +222,13 @@ func testCatalogueLoadReadOnlyRecoversCrashStates(t *testing.T) { | |||
| 201 | 222 | writeCandidate(t, StorePath(dir)+".next", want) | |
| 202 | 223 | }, | |
| 203 | 224 | }, | |
| 225 | + | { | |
| 226 | + | // Crash before a new installation publishes .next as current. | |
| 227 | + | name: "next-only-recovers-from-next", | |
| 228 | + | write: func(t *testing.T, dir string, want domain.CatalogueRecord) { | |
| 229 | + | writeCandidate(t, StorePath(dir)+".next", want) | |
| 230 | + | }, | |
| 231 | + | }, | |
| 204 | 232 | { | |
| 205 | 233 | // Crash leaves only a predecessor behind; no current, no successor. | |
| 206 | 234 | name: "prev-only-recovers-from-prev", | |
M internal/persist/migration.go +21 -5
| @@ -6,6 +6,7 @@ import ( | |||
| 6 | 6 | "errors" | |
| 7 | 7 | "fmt" | |
| 8 | 8 | "io" | |
| 9 | + | "log/slog" | |
| 9 | 10 | "os" | |
| 10 | 11 | "path/filepath" | |
| 11 | 12 | "sort" | |
| @@ -37,6 +38,8 @@ type OpenDeps struct { | |||
| 37 | 38 | StateDir string | |
| 38 | 39 | Random io.Reader | |
| 39 | 40 | SnapshotMigration ports.SnapshotMigration | |
| 41 | + | // Logger receives full adapter causes; durable degraded reasons are sanitized. | |
| 42 | + | Logger *slog.Logger | |
| 40 | 43 | // Fault is a test-only crash seam called after named durable boundaries. | |
| 41 | 44 | Fault func(string) error | |
| 42 | 45 | } | |
| @@ -65,6 +68,9 @@ const ( | |||
| 65 | 68 | catalogueFormatName = "durable-session-v1.format" | |
| 66 | 69 | migrationRecordMagic = "VEVJ" | |
| 67 | 70 | migrationRecordVersion uint16 = 1 | |
| 71 | + | ||
| 72 | + | migrationDegradedLegacySnapshotUncertain = "legacy_snapshot_uncertain" | |
| 73 | + | migrationDegradedReasonMaxBytes = 64 | |
| 68 | 74 | ) | |
| 69 | 75 | ||
| 70 | 76 | func OpenOrMigrate(ctx context.Context, deps OpenDeps) (OpenResult, error) { | |
| @@ -300,11 +306,12 @@ func resumeLegacyMigration(ctx context.Context, deps OpenDeps) (OpenResult, erro | |||
| 300 | 306 | if !errors.Is(err, ports.ErrLegacySnapshotUncertain) { | |
| 301 | 307 | return OpenResult{}, fmt.Errorf("persist: migrate legacy snapshot %q: %w", legacy.Name, err) | |
| 302 | 308 | } | |
| 309 | + | migrationLogger(deps).Error("legacy snapshot migration uncertain", "session", legacy.Name, "err", err) | |
| 303 | 310 | if ref.Generation != 0 && ref.ManifestDigest != ([32]byte{}) { | |
| 304 | 311 | legacyRef := ref | |
| 305 | 312 | record.Validated[legacy.Name] = &legacyRef | |
| 306 | 313 | } | |
| 307 | - | record.Degraded[legacy.Name] = err.Error() | |
| 314 | + | record.Degraded[legacy.Name] = migrationDegradedLegacySnapshotUncertain | |
| 308 | 315 | } | |
| 309 | 316 | if err := writeMigrationRecord(intent, record); err != nil { | |
| 310 | 317 | return OpenResult{}, err | |
| @@ -374,6 +381,13 @@ func resumeLegacyMigration(ctx context.Context, deps OpenDeps) (OpenResult, erro | |||
| 374 | 381 | return OpenResult{Catalogue: catalogue, Records: opened, Migrated: true}, nil | |
| 375 | 382 | } | |
| 376 | 383 | ||
| 384 | + | func migrationLogger(deps OpenDeps) *slog.Logger { | |
| 385 | + | if deps.Logger != nil { | |
| 386 | + | return deps.Logger | |
| 387 | + | } | |
| 388 | + | return slog.Default() | |
| 389 | + | } | |
| 390 | + | ||
| 377 | 391 | func migrationBoundary(deps OpenDeps, name string) error { | |
| 378 | 392 | if deps.Fault != nil { | |
| 379 | 393 | return deps.Fault(name) | |
| @@ -609,10 +623,12 @@ func catalogueRecordsEqual(a, b []domain.CatalogueRecord) bool { | |||
| 609 | 623 | if len(a) != len(b) { | |
| 610 | 624 | return false | |
| 611 | 625 | } | |
| 612 | - | sort.Slice(a, func(i, j int) bool { return a[i].Name < a[j].Name }) | |
| 613 | - | sort.Slice(b, func(i, j int) bool { return b[i].Name < b[j].Name }) | |
| 614 | - | for i := range a { | |
| 615 | - | if a[i].Name != b[i].Name || a[i].IncarnationID != b[i].IncarnationID || a[i].RecoveryState != b[i].RecoveryState { | |
| 626 | + | left := append([]domain.CatalogueRecord(nil), a...) | |
| 627 | + | right := append([]domain.CatalogueRecord(nil), b...) | |
| 628 | + | sort.Slice(left, func(i, j int) bool { return left[i].Name < left[j].Name }) | |
| 629 | + | sort.Slice(right, func(i, j int) bool { return right[i].Name < right[j].Name }) | |
| 630 | + | for i := range left { | |
| 631 | + | if !left[i].Equal(right[i]) { | |
| 616 | 632 | return false | |
| 617 | 633 | } | |
| 618 | 634 | } | |
M internal/persist/migration_test.go +67 -15
| @@ -1,24 +1,29 @@ | |||
| 1 | 1 | package persist | |
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | + | "bytes" | |
| 4 | 5 | "context" | |
| 5 | 6 | "crypto/sha256" | |
| 6 | 7 | "encoding/binary" | |
| 7 | 8 | "errors" | |
| 8 | 9 | "io" | |
| 10 | + | "log/slog" | |
| 9 | 11 | "os" | |
| 10 | 12 | "path/filepath" | |
| 11 | 13 | "strconv" | |
| 14 | + | "strings" | |
| 12 | 15 | "testing" | |
| 13 | 16 | ||
| 14 | 17 | "github.com/bnema/vev/internal/domain" | |
| 15 | 18 | "github.com/bnema/vev/internal/ports" | |
| 16 | 19 | "github.com/bnema/vev/pkg/kv" | |
| 20 | + | "github.com/bnema/vev/pkg/safedir" | |
| 17 | 21 | "github.com/stretchr/testify/require" | |
| 18 | 22 | ) | |
| 19 | 23 | ||
| 20 | 24 | type migrationStub struct { | |
| 21 | 25 | legacy bool | |
| 26 | + | probePaths []string | |
| 22 | 27 | probeErr error | |
| 23 | 28 | heads map[string]domain.CheckpointRef | |
| 24 | 29 | migrateErr map[string]error | |
| @@ -26,7 +31,19 @@ type migrationStub struct { | |||
| 26 | 31 | sources map[string]string | |
| 27 | 32 | } | |
| 28 | 33 | ||
| 29 | - | func (s *migrationStub) HasLegacyState(context.Context) (bool, error) { return s.legacy, s.probeErr } | |
| 34 | + | func (s *migrationStub) HasLegacyState(context.Context) (bool, error) { | |
| 35 | + | if s.probeErr != nil || s.legacy { | |
| 36 | + | return s.legacy, s.probeErr | |
| 37 | + | } | |
| 38 | + | for _, path := range s.probePaths { | |
| 39 | + | if _, err := os.Stat(path); err == nil { | |
| 40 | + | return true, nil | |
| 41 | + | } else if !errors.Is(err, os.ErrNotExist) { | |
| 42 | + | return false, err | |
| 43 | + | } | |
| 44 | + | } | |
| 45 | + | return false, nil | |
| 46 | + | } | |
| 30 | 47 | func (s *migrationStub) ReadLegacyHEAD(_ context.Context, name string) (domain.CheckpointRef, error) { | |
| 31 | 48 | if source := s.sources[name]; source != "" { | |
| 32 | 49 | if _, err := os.Stat(source); err != nil { | |
| @@ -74,7 +91,7 @@ func encodeLegacyRecordV0(r legacyCatalogueRecordV0) []byte { | |||
| 74 | 91 | func writeLegacyCatalogue(t *testing.T, dir string, records ...legacyCatalogueRecordV0) string { | |
| 75 | 92 | t.Helper() | |
| 76 | 93 | require.NoError(t, os.Chmod(dir, 0o700)) | |
| 77 | - | require.NoError(t, os.MkdirAll(dir, 0o700)) | |
| 94 | + | require.NoError(t, safedir.EnsurePrivate(dir)) | |
| 78 | 95 | store, err := kv.Open(StorePath(dir)) | |
| 79 | 96 | require.NoError(t, err) | |
| 80 | 97 | for _, record := range records { | |
| @@ -198,7 +215,7 @@ func TestMigrationResumeMatrix(t *testing.T) { | |||
| 198 | 215 | } | |
| 199 | 216 | uncertain := recordByName(t, result.Records, "uncertain") | |
| 200 | 217 | require.Equal(t, domain.RecoveryDegraded, uncertain.RecoveryState) | |
| 201 | - | require.Contains(t, uncertain.DegradedReason, "corrupt legacy manifest") | |
| 218 | + | require.Equal(t, migrationDegradedLegacySnapshotUncertain, uncertain.DegradedReason) | |
| 202 | 219 | require.NoError(t, result.Catalogue.Close()) | |
| 203 | 220 | ||
| 204 | 221 | reopened, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: io.LimitReader(&countingReader{}, 0), SnapshotMigration: restarted}) | |
| @@ -219,12 +236,18 @@ func TestMigrationPreservesUncertain(t *testing.T) { | |||
| 219 | 236 | dir := t.TempDir() | |
| 220 | 237 | legacy := writeLegacyCatalogue(t, dir, legacyCatalogueRecordV0{Name: "broken", Cwd: "/tmp"}) | |
| 221 | 238 | ref := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}} | |
| 222 | - | result, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"broken": ref}, migrateErr: map[string]error{"broken": errors.Join(ports.ErrLegacySnapshotUncertain, errors.New("invalid legacy manifest"))}}}) | |
| 239 | + | adapterCause := strings.Repeat("sensitive-adapter-detail", 64) | |
| 240 | + | var logs bytes.Buffer | |
| 241 | + | logger := slog.New(slog.NewTextHandler(&logs, nil)) | |
| 242 | + | result, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: &migrationStub{heads: map[string]domain.CheckpointRef{"broken": ref}, migrateErr: map[string]error{"broken": errors.Join(ports.ErrLegacySnapshotUncertain, errors.New(adapterCause))}}, Logger: logger}) | |
| 223 | 243 | require.NoError(t, err) | |
| 224 | 244 | require.NoError(t, result.Catalogue.Close()) | |
| 225 | 245 | require.FileExists(t, legacy) | |
| 226 | 246 | require.Equal(t, domain.RecoveryDegraded, result.Records[0].RecoveryState) | |
| 227 | - | require.NotEmpty(t, result.Records[0].DegradedReason) | |
| 247 | + | require.Equal(t, migrationDegradedLegacySnapshotUncertain, result.Records[0].DegradedReason) | |
| 248 | + | require.LessOrEqual(t, len(result.Records[0].DegradedReason), migrationDegradedReasonMaxBytes) | |
| 249 | + | require.NotContains(t, result.Records[0].DegradedReason, adapterCause) | |
| 250 | + | require.Contains(t, logs.String(), adapterCause, "the full adapter cause must remain diagnosable") | |
| 228 | 251 | ||
| 229 | 252 | failClosedDir := t.TempDir() | |
| 230 | 253 | writeLegacyCatalogue(t, failClosedDir, legacyCatalogueRecordV0{Name: "work", Cwd: "/tmp"}) | |
| @@ -240,23 +263,36 @@ func TestMigrationPreservesUncertain(t *testing.T) { | |||
| 240 | 263 | func TestOpenOrMigrateLegacyProbe(t *testing.T) { | |
| 241 | 264 | probeErr := errors.New("probe uncertain") | |
| 242 | 265 | for _, tc := range []struct { | |
| 243 | - | name string | |
| 244 | - | stub *migrationStub | |
| 245 | - | wantNew bool | |
| 246 | - | wantErr error | |
| 266 | + | name string | |
| 267 | + | stub *migrationStub | |
| 268 | + | arrange func(t *testing.T, dir string, stub *migrationStub) | |
| 269 | + | wantNew bool | |
| 270 | + | wantLegacy bool | |
| 271 | + | wantErr error | |
| 247 | 272 | }{ | |
| 248 | - | {"proven absent", &migrationStub{}, true, nil}, | |
| 249 | - | {"legacy blob", &migrationStub{legacy: true}, false, nil}, | |
| 250 | - | {"incremental namespace", &migrationStub{legacy: true}, false, nil}, | |
| 251 | - | {"probe error", &migrationStub{probeErr: probeErr}, false, probeErr}, | |
| 273 | + | {name: "proven absent", stub: &migrationStub{}, wantNew: true}, | |
| 274 | + | {name: "legacy blob", stub: &migrationStub{}, wantLegacy: true, arrange: func(t *testing.T, dir string, stub *migrationStub) { | |
| 275 | + | path := filepath.Join(dir, "legacy.snapshot") | |
| 276 | + | require.NoError(t, os.WriteFile(path, []byte("legacy"), 0o600)) | |
| 277 | + | stub.probePaths = []string{path} | |
| 278 | + | }}, | |
| 279 | + | {name: "incremental namespace", stub: &migrationStub{}, wantLegacy: true, arrange: func(t *testing.T, dir string, stub *migrationStub) { | |
| 280 | + | path := filepath.Join(dir, "snapshots") | |
| 281 | + | require.NoError(t, safedir.EnsurePrivate(path)) | |
| 282 | + | stub.probePaths = []string{path} | |
| 283 | + | }}, | |
| 284 | + | {name: "probe error", stub: &migrationStub{probeErr: probeErr}, wantErr: probeErr}, | |
| 252 | 285 | } { | |
| 253 | 286 | t.Run(tc.name, func(t *testing.T) { | |
| 254 | 287 | dir := t.TempDir() | |
| 255 | 288 | require.NoError(t, os.Chmod(dir, 0o700)) | |
| 289 | + | if tc.arrange != nil { | |
| 290 | + | tc.arrange(t, dir, tc.stub) | |
| 291 | + | } | |
| 256 | 292 | result, err := OpenOrMigrate(context.Background(), OpenDeps{StateDir: dir, Random: &countingReader{}, SnapshotMigration: tc.stub}) | |
| 257 | 293 | if tc.wantErr != nil { | |
| 258 | 294 | require.ErrorIs(t, err, tc.wantErr) | |
| 259 | - | } else if tc.stub.legacy { | |
| 295 | + | } else if tc.wantLegacy { | |
| 260 | 296 | require.Error(t, err) | |
| 261 | 297 | } else { | |
| 262 | 298 | require.NoError(t, err) | |
| @@ -270,7 +306,7 @@ func TestOpenOrMigrateLegacyProbe(t *testing.T) { | |||
| 270 | 306 | require.NoError(t, reopened.Catalogue.Close()) | |
| 271 | 307 | } | |
| 272 | 308 | } | |
| 273 | - | if tc.stub.legacy || tc.wantErr != nil { | |
| 309 | + | if tc.wantLegacy || tc.wantErr != nil { | |
| 274 | 310 | _, statErr := os.Stat(StorePath(dir)) | |
| 275 | 311 | require.ErrorIs(t, statErr, os.ErrNotExist) | |
| 276 | 312 | } | |
| @@ -278,6 +314,22 @@ func TestOpenOrMigrateLegacyProbe(t *testing.T) { | |||
| 278 | 314 | } | |
| 279 | 315 | } | |
| 280 | 316 | ||
| 317 | + | func TestCatalogueRecordsEqualIsCompleteAndNonMutating(t *testing.T) { | |
| 318 | + | committed := &domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}} | |
| 319 | + | left := []domain.CatalogueRecord{ | |
| 320 | + | {Name: "z", IncarnationID: domain.IncarnationID{1}, Cwd: "/old", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3, TabNames: []string{"shell"}, RecoveryState: domain.RecoveryHealthy, Committed: committed}, | |
| 321 | + | validRecord("a", 2), | |
| 322 | + | } | |
| 323 | + | right := []domain.CatalogueRecord{left[1], left[0]} | |
| 324 | + | leftOrder, rightOrder := left[0].Name, right[0].Name | |
| 325 | + | require.True(t, catalogueRecordsEqual(left, right)) | |
| 326 | + | require.Equal(t, leftOrder, left[0].Name) | |
| 327 | + | require.Equal(t, rightOrder, right[0].Name) | |
| 328 | + | ||
| 329 | + | right[1].Cwd = "/different" | |
| 330 | + | require.False(t, catalogueRecordsEqual(left, right), "installed catalogue validation must compare every record field") | |
| 331 | + | } | |
| 332 | + | ||
| 281 | 333 | func TestAtomicMigrationWriteRemovesTempAfterRenameFailure(t *testing.T) { | |
| 282 | 334 | dir := t.TempDir() | |
| 283 | 335 | target := filepath.Join(dir, "target") | |
M internal/persist/persist.go +13 -55
| @@ -3,8 +3,6 @@ package persist | |||
| 3 | 3 | import ( | |
| 4 | 4 | "errors" | |
| 5 | 5 | "fmt" | |
| 6 | - | "log/slog" | |
| 7 | - | "os" | |
| 8 | 6 | "path/filepath" | |
| 9 | 7 | "sort" | |
| 10 | 8 | "sync" | |
| @@ -23,25 +21,6 @@ func StorePath(dir string) string { return filepath.Join(dir, filename) } | |||
| 23 | 21 | type Persister struct { | |
| 24 | 22 | store ports.Store | |
| 25 | 23 | mu sync.Mutex | |
| 26 | - | log *slog.Logger | |
| 27 | - | } | |
| 28 | - | ||
| 29 | - | // Option configures a Persister at construction time. | |
| 30 | - | type Option func(*Persister) | |
| 31 | - | ||
| 32 | - | // WithLogger directs diagnostics for failures that ports.Catalogue has no | |
| 33 | - | // error return for (see Records) to log instead of the default logger. | |
| 34 | - | func WithLogger(log *slog.Logger) Option { | |
| 35 | - | return func(p *Persister) { p.log = log } | |
| 36 | - | } | |
| 37 | - | ||
| 38 | - | // logger returns a nil-safe logger, defaulting to slog.Default() when no | |
| 39 | - | // logger was configured. | |
| 40 | - | func (p *Persister) logger() *slog.Logger { | |
| 41 | - | if p != nil && p.log != nil { | |
| 42 | - | return p.log | |
| 43 | - | } | |
| 44 | - | return slog.Default() | |
| 45 | 24 | } | |
| 46 | 25 | ||
| 47 | 26 | // KVStore adapts the reusable WAL implementation to the persistence port. | |
| @@ -76,28 +55,15 @@ func Open(dir string) (*Persister, error) { | |||
| 76 | 55 | p, _, err := openCurrentCatalogue(dir, false) | |
| 77 | 56 | return p, err | |
| 78 | 57 | } | |
| 79 | - | func New(store ports.Store, opts ...Option) *Persister { | |
| 80 | - | p := &Persister{store: store} | |
| 81 | - | for _, opt := range opts { | |
| 82 | - | opt(p) | |
| 83 | - | } | |
| 84 | - | return p | |
| 85 | - | } | |
| 58 | + | func New(store ports.Store) *Persister { return &Persister{store: store} } | |
| 86 | 59 | ||
| 87 | 60 | func openCurrentCatalogue(dir string, createProvenEmpty bool) (*Persister, []domain.CatalogueRecord, error) { | |
| 88 | 61 | path := StorePath(dir) | |
| 89 | - | absent := true | |
| 90 | - | for _, candidate := range []string{path, path + ".next", path + ".prev"} { | |
| 91 | - | _, err := os.Stat(candidate) | |
| 92 | - | if err == nil { | |
| 93 | - | absent = false | |
| 94 | - | continue | |
| 95 | - | } | |
| 96 | - | if !errors.Is(err, os.ErrNotExist) { | |
| 97 | - | return nil, nil, err | |
| 98 | - | } | |
| 62 | + | present, err := catalogueCandidatesPresent(path) | |
| 63 | + | if err != nil { | |
| 64 | + | return nil, nil, err | |
| 99 | 65 | } | |
| 100 | - | if absent && !createProvenEmpty { | |
| 66 | + | if !present && !createProvenEmpty { | |
| 101 | 67 | return nil, nil, fmt.Errorf("%w: no catalogue candidates", errPersistenceUnavailable) | |
| 102 | 68 | } | |
| 103 | 69 | store, err := OpenStore(path) | |
| @@ -148,30 +114,22 @@ func (p *Persister) Create(record domain.CatalogueRecord) error { | |||
| 148 | 114 | return p.applyLocked(map[string]*domain.CatalogueRecord{record.Name: &record}) | |
| 149 | 115 | } | |
| 150 | 116 | ||
| 151 | - | // Records satisfies ports.Catalogue, whose signature has no error return. A | |
| 152 | - | // load failure (unavailable store, or a malformed record surfaced by | |
| 153 | - | // decodeAll) must not silently look like an empty catalogue, so it is logged | |
| 154 | - | // here instead of being swallowed. | |
| 155 | - | func (p *Persister) Records() []domain.CatalogueRecord { | |
| 156 | - | records, err := p.LoadCatalogue() | |
| 157 | - | if err != nil { | |
| 158 | - | p.logger().Error("persist: loading catalogue records failed", "err", err) | |
| 159 | - | return []domain.CatalogueRecord{} | |
| 160 | - | } | |
| 161 | - | return records | |
| 162 | - | } | |
| 163 | - | func (p *Persister) Record(name string) (domain.CatalogueRecord, bool) { | |
| 117 | + | func (p *Persister) Records() ([]domain.CatalogueRecord, error) { return p.LoadCatalogue() } | |
| 118 | + | func (p *Persister) Record(name string) (domain.CatalogueRecord, bool, error) { | |
| 164 | 119 | if p == nil || p.store == nil { | |
| 165 | - | return domain.CatalogueRecord{}, false | |
| 120 | + | return domain.CatalogueRecord{}, false, errPersistenceUnavailable | |
| 166 | 121 | } | |
| 167 | 122 | p.mu.Lock() | |
| 168 | 123 | defer p.mu.Unlock() | |
| 169 | 124 | value, ok := p.store.Get([]byte(name)) | |
| 170 | 125 | if !ok { | |
| 171 | - | return domain.CatalogueRecord{}, false | |
| 126 | + | return domain.CatalogueRecord{}, false, nil | |
| 172 | 127 | } | |
| 173 | 128 | record, err := decodeRecordValue(name, value) | |
| 174 | - | return record, err == nil | |
| 129 | + | if err != nil { | |
| 130 | + | return domain.CatalogueRecord{}, false, err | |
| 131 | + | } | |
| 132 | + | return record, true, nil | |
| 175 | 133 | } | |
| 176 | 134 | func (p *Persister) Apply(records map[string]*domain.CatalogueRecord) error { | |
| 177 | 135 | if p == nil { | |
M internal/persist/persist_test.go +22 -47
| @@ -1,9 +1,7 @@ | |||
| 1 | 1 | package persist | |
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | - | "bytes" | |
| 5 | 4 | "errors" | |
| 6 | - | "log/slog" | |
| 7 | 5 | "path/filepath" | |
| 8 | 6 | "testing" | |
| 9 | 7 | ||
| @@ -51,9 +49,17 @@ func TestCatalogueBatchSyncBehavior(t *testing.T) { | |||
| 51 | 49 | p := New(store) | |
| 52 | 50 | one := validRecord("one", 1) | |
| 53 | 51 | require.NoError(t, p.Replace("one", one)) | |
| 52 | + | require.Contains(t, state, "one") | |
| 54 | 53 | renamed := one | |
| 55 | 54 | renamed.Name = "renamed" | |
| 55 | + | renamed.Cwd = "/renamed" | |
| 56 | 56 | require.NoError(t, p.Rename("one", renamed)) | |
| 57 | + | require.NotContains(t, state, "one") | |
| 58 | + | encoded, ok := state["renamed"] | |
| 59 | + | require.True(t, ok) | |
| 60 | + | got, err := decodeRecordValue("renamed", encoded) | |
| 61 | + | require.NoError(t, err) | |
| 62 | + | require.Equal(t, renamed, got) | |
| 57 | 63 | } | |
| 58 | 64 | ||
| 59 | 65 | func TestCatalogueBatchFailureDoesNotSync(t *testing.T) { | |
| @@ -80,7 +86,9 @@ func TestCatalogueNilPersisterIsNoOp(t *testing.T) { | |||
| 80 | 86 | require.NoError(t, err) | |
| 81 | 87 | require.Empty(t, records) | |
| 82 | 88 | require.NoError(t, p.Close()) | |
| 83 | - | require.Empty(t, p.Records()) | |
| 89 | + | got, err := p.Records() | |
| 90 | + | require.NoError(t, err) | |
| 91 | + | require.Empty(t, got) | |
| 84 | 92 | } | |
| 85 | 93 | ||
| 86 | 94 | // TestCatalogueUnavailableStoreErrors covers a non-nil Persister with no | |
| @@ -102,51 +110,18 @@ func TestCatalogueUnavailableStoreErrors(t *testing.T) { | |||
| 102 | 110 | records, err := p.LoadAll() | |
| 103 | 111 | require.ErrorIs(t, err, errPersistenceUnavailable) | |
| 104 | 112 | require.Empty(t, records) | |
| 105 | - | require.Empty(t, p.Records()) | |
| 113 | + | got, err := p.Records() | |
| 114 | + | require.ErrorIs(t, err, errPersistenceUnavailable) | |
| 115 | + | require.Empty(t, got) | |
| 106 | 116 | require.NoError(t, p.Close()) | |
| 107 | 117 | } | |
| 108 | 118 | ||
| 109 | - | // TestCatalogueRecordsMakesLoadFailureObservable covers Records(), which | |
| 110 | - | // ports.Catalogue.Records() gives no error return for: a load failure must | |
| 111 | - | // not silently look like an empty catalogue to every caller. Records() | |
| 112 | - | // degrades to an empty slice but must emit the underlying error through its | |
| 113 | - | // (nil-safe, optional) logger rather than swallowing it outright. | |
| 114 | - | func TestCatalogueRecordsMakesLoadFailureObservable(t *testing.T) { | |
| 115 | - | tests := []struct { | |
| 116 | - | name string | |
| 117 | - | build func(t *testing.T, log *slog.Logger) *Persister | |
| 118 | - | }{ | |
| 119 | - | { | |
| 120 | - | name: "unavailable-store", | |
| 121 | - | build: func(t *testing.T, log *slog.Logger) *Persister { | |
| 122 | - | return New(nil, WithLogger(log)) | |
| 123 | - | }, | |
| 124 | - | }, | |
| 125 | - | { | |
| 126 | - | name: "malformed-record", | |
| 127 | - | build: func(t *testing.T, log *slog.Logger) *Persister { | |
| 128 | - | store := portsmocks.NewMockStore(t) | |
| 129 | - | store.EXPECT().Range(mock.Anything).Run(func(fn func([]byte, []byte) bool) { | |
| 130 | - | fn([]byte("bad"), []byte{0, 1}) | |
| 131 | - | }) | |
| 132 | - | return New(store, WithLogger(log)) | |
| 133 | - | }, | |
| 134 | - | }, | |
| 135 | - | } | |
| 136 | - | for _, tt := range tests { | |
| 137 | - | t.Run(tt.name, func(t *testing.T) { | |
| 138 | - | var buf bytes.Buffer | |
| 139 | - | p := tt.build(t, slog.New(slog.NewTextHandler(&buf, nil))) | |
| 140 | - | require.Empty(t, p.Records()) | |
| 141 | - | require.Contains(t, buf.String(), "loading catalogue records failed") | |
| 142 | - | }) | |
| 143 | - | } | |
| 144 | - | } | |
| 145 | - | ||
| 146 | - | // TestCatalogueRecordsDefaultsToSlogDefaultLogger covers the nil-safe logger | |
| 147 | - | // fallback: a Persister built without WithLogger must not panic when a load | |
| 148 | - | // failure needs to be reported. | |
| 149 | - | func TestCatalogueRecordsDefaultsToSlogDefaultLogger(t *testing.T) { | |
| 150 | - | p := New(nil) | |
| 151 | - | require.NotPanics(t, func() { require.Empty(t, p.Records()) }) | |
| 119 | + | func TestCatalogueRecordsPropagatesMalformedRecord(t *testing.T) { | |
| 120 | + | store := portsmocks.NewMockStore(t) | |
| 121 | + | store.EXPECT().Range(mock.Anything).Run(func(fn func([]byte, []byte) bool) { | |
| 122 | + | fn([]byte("bad"), []byte{0, 1}) | |
| 123 | + | }) | |
| 124 | + | records, err := New(store).Records() | |
| 125 | + | require.Error(t, err) | |
| 126 | + | require.Empty(t, records) | |
| 152 | 127 | } | |
M internal/persist/record.go +6 -4
| @@ -26,7 +26,11 @@ func encodeRecordValue(record domain.CatalogueRecord) ([]byte, error) { | |||
| 26 | 26 | buf = append(buf, catalogueMagic[:]...) | |
| 27 | 27 | buf = binary.BigEndian.AppendUint16(buf, catalogueRecordVersion) | |
| 28 | 28 | buf = append(buf, record.IncarnationID[:]...) | |
| 29 | - | buf = appendString(buf, record.Cwd) | |
| 29 | + | var err error | |
| 30 | + | buf, err = appendCheckedString(buf, record.Cwd) | |
| 31 | + | if err != nil { | |
| 32 | + | return nil, err | |
| 33 | + | } | |
| 30 | 34 | buf = binary.BigEndian.AppendUint64(buf, uint64(record.CreatedAt)) | |
| 31 | 35 | buf = binary.BigEndian.AppendUint64(buf, uint64(record.UpdatedAt)) | |
| 32 | 36 | buf = binary.BigEndian.AppendUint64(buf, record.LastUsedSeq) | |
| @@ -35,7 +39,6 @@ func encodeRecordValue(record domain.CatalogueRecord) ([]byte, error) { | |||
| 35 | 39 | } | |
| 36 | 40 | buf = binary.BigEndian.AppendUint32(buf, uint32(len(record.TabNames))) | |
| 37 | 41 | for _, name := range record.TabNames { | |
| 38 | - | var err error | |
| 39 | 42 | buf, err = appendCheckedString(buf, name) | |
| 40 | 43 | if err != nil { | |
| 41 | 44 | return nil, err | |
| @@ -45,7 +48,6 @@ func encodeRecordValue(record domain.CatalogueRecord) ([]byte, error) { | |||
| 45 | 48 | buf = appendRef(buf, record.Committed) | |
| 46 | 49 | buf = appendRef(buf, record.Fallbacks[0]) | |
| 47 | 50 | buf = appendRef(buf, record.Fallbacks[1]) | |
| 48 | - | var err error | |
| 49 | 51 | buf, err = appendCheckedString(buf, record.DegradedReason) | |
| 50 | 52 | return buf, err | |
| 51 | 53 | } | |
| @@ -118,7 +120,7 @@ func decodeRecordValue(name string, value []byte) (domain.CatalogueRecord, error | |||
| 118 | 120 | return domain.CatalogueRecord{}, errMalformedRecord | |
| 119 | 121 | } | |
| 120 | 122 | if err := record.Validate(); err != nil { | |
| 121 | - | return domain.CatalogueRecord{}, fmt.Errorf("%w: %v", errMalformedRecord, err) | |
| 123 | + | return domain.CatalogueRecord{}, errors.Join(errMalformedRecord, fmt.Errorf("persist: invalid catalogue record: %w", err)) | |
| 122 | 124 | } | |
| 123 | 125 | return record, nil | |
| 124 | 126 | } | |
M internal/ports/mocks/recovery_mock.go +26 -11
| @@ -1428,7 +1428,7 @@ func (_c *MockCatalogue_Delete_Call) RunAndReturn(run func(s string) error) *Moc | |||
| 1428 | 1428 | } | |
| 1429 | 1429 | ||
| 1430 | 1430 | // Record provides a mock function for the type MockCatalogue | |
| 1431 | - | func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool) { | |
| 1431 | + | func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool, error) { | |
| 1432 | 1432 | ret := _mock.Called(s) | |
| 1433 | 1433 | ||
| 1434 | 1434 | if len(ret) == 0 { | |
| @@ -1437,7 +1437,8 @@ func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool) { | |||
| 1437 | 1437 | ||
| 1438 | 1438 | var r0 domain.CatalogueRecord | |
| 1439 | 1439 | var r1 bool | |
| 1440 | - | if returnFunc, ok := ret.Get(0).(func(string) (domain.CatalogueRecord, bool)); ok { | |
| 1440 | + | var r2 error | |
| 1441 | + | if returnFunc, ok := ret.Get(0).(func(string) (domain.CatalogueRecord, bool, error)); ok { | |
| 1441 | 1442 | return returnFunc(s) | |
| 1442 | 1443 | } | |
| 1443 | 1444 | if returnFunc, ok := ret.Get(0).(func(string) domain.CatalogueRecord); ok { | |
| @@ -1450,7 +1451,12 @@ func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool) { | |||
| 1450 | 1451 | } else { | |
| 1451 | 1452 | r1 = ret.Get(1).(bool) | |
| 1452 | 1453 | } | |
| 1453 | - | return r0, r1 | |
| 1454 | + | if returnFunc, ok := ret.Get(2).(func(string) error); ok { | |
| 1455 | + | r2 = returnFunc(s) | |
| 1456 | + | } else { | |
| 1457 | + | r2 = ret.Error(2) | |
| 1458 | + | } | |
| 1459 | + | return r0, r1, r2 | |
| 1454 | 1460 | } | |
| 1455 | 1461 | ||
| 1456 | 1462 | // MockCatalogue_Record_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Record' | |
| @@ -1477,18 +1483,18 @@ func (_c *MockCatalogue_Record_Call) Run(run func(s string)) *MockCatalogue_Reco | |||
| 1477 | 1483 | return _c | |
| 1478 | 1484 | } | |
| 1479 | 1485 | ||
| 1480 | - | func (_c *MockCatalogue_Record_Call) Return(catalogueRecord domain.CatalogueRecord, b bool) *MockCatalogue_Record_Call { | |
| 1481 | - | _c.Call.Return(catalogueRecord, b) | |
| 1486 | + | func (_c *MockCatalogue_Record_Call) Return(catalogueRecord domain.CatalogueRecord, b bool, err error) *MockCatalogue_Record_Call { | |
| 1487 | + | _c.Call.Return(catalogueRecord, b, err) | |
| 1482 | 1488 | return _c | |
| 1483 | 1489 | } | |
| 1484 | 1490 | ||
| 1485 | - | func (_c *MockCatalogue_Record_Call) RunAndReturn(run func(s string) (domain.CatalogueRecord, bool)) *MockCatalogue_Record_Call { | |
| 1491 | + | func (_c *MockCatalogue_Record_Call) RunAndReturn(run func(s string) (domain.CatalogueRecord, bool, error)) *MockCatalogue_Record_Call { | |
| 1486 | 1492 | _c.Call.Return(run) | |
| 1487 | 1493 | return _c | |
| 1488 | 1494 | } | |
| 1489 | 1495 | ||
| 1490 | 1496 | // Records provides a mock function for the type MockCatalogue | |
| 1491 | - | func (_mock *MockCatalogue) Records() []domain.CatalogueRecord { | |
| 1497 | + | func (_mock *MockCatalogue) Records() ([]domain.CatalogueRecord, error) { | |
| 1492 | 1498 | ret := _mock.Called() | |
| 1493 | 1499 | ||
| 1494 | 1500 | if len(ret) == 0 { | |
| @@ -1496,6 +1502,10 @@ func (_mock *MockCatalogue) Records() []domain.CatalogueRecord { | |||
| 1496 | 1502 | } | |
| 1497 | 1503 | ||
| 1498 | 1504 | var r0 []domain.CatalogueRecord | |
| 1505 | + | var r1 error | |
| 1506 | + | if returnFunc, ok := ret.Get(0).(func() ([]domain.CatalogueRecord, error)); ok { | |
| 1507 | + | return returnFunc() | |
| 1508 | + | } | |
| 1499 | 1509 | if returnFunc, ok := ret.Get(0).(func() []domain.CatalogueRecord); ok { | |
| 1500 | 1510 | r0 = returnFunc() | |
| 1501 | 1511 | } else { | |
| @@ -1503,7 +1513,12 @@ func (_mock *MockCatalogue) Records() []domain.CatalogueRecord { | |||
| 1503 | 1513 | r0 = ret.Get(0).([]domain.CatalogueRecord) | |
| 1504 | 1514 | } | |
| 1505 | 1515 | } | |
| 1506 | - | return r0 | |
| 1516 | + | if returnFunc, ok := ret.Get(1).(func() error); ok { | |
| 1517 | + | r1 = returnFunc() | |
| 1518 | + | } else { | |
| 1519 | + | r1 = ret.Error(1) | |
| 1520 | + | } | |
| 1521 | + | return r0, r1 | |
| 1507 | 1522 | } | |
| 1508 | 1523 | ||
| 1509 | 1524 | // MockCatalogue_Records_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Records' | |
| @@ -1523,12 +1538,12 @@ func (_c *MockCatalogue_Records_Call) Run(run func()) *MockCatalogue_Records_Cal | |||
| 1523 | 1538 | return _c | |
| 1524 | 1539 | } | |
| 1525 | 1540 | ||
| 1526 | - | func (_c *MockCatalogue_Records_Call) Return(catalogueRecords []domain.CatalogueRecord) *MockCatalogue_Records_Call { | |
| 1527 | - | _c.Call.Return(catalogueRecords) | |
| 1541 | + | func (_c *MockCatalogue_Records_Call) Return(catalogueRecords []domain.CatalogueRecord, err error) *MockCatalogue_Records_Call { | |
| 1542 | + | _c.Call.Return(catalogueRecords, err) | |
| 1528 | 1543 | return _c | |
| 1529 | 1544 | } | |
| 1530 | 1545 | ||
| 1531 | - | func (_c *MockCatalogue_Records_Call) RunAndReturn(run func() []domain.CatalogueRecord) *MockCatalogue_Records_Call { | |
| 1546 | + | func (_c *MockCatalogue_Records_Call) RunAndReturn(run func() ([]domain.CatalogueRecord, error)) *MockCatalogue_Records_Call { | |
| 1532 | 1547 | _c.Call.Return(run) | |
| 1533 | 1548 | return _c | |
| 1534 | 1549 | } | |
M internal/ports/recovery.go +2 -2
| @@ -157,8 +157,8 @@ type DegradedRecoveryCoordinator interface { | |||
| 157 | 157 | } | |
| 158 | 158 | ||
| 159 | 159 | type Catalogue interface { | |
| 160 | - | Records() []domain.CatalogueRecord | |
| 161 | - | Record(string) (domain.CatalogueRecord, bool) | |
| 160 | + | Records() ([]domain.CatalogueRecord, error) | |
| 161 | + | Record(string) (domain.CatalogueRecord, bool, error) | |
| 162 | 162 | Create(domain.CatalogueRecord) error | |
| 163 | 163 | UpdateMetadata(domain.CatalogueMetadataUpdate) error | |
| 164 | 164 | Replace(string, domain.CatalogueRecord) error | |
M internal/ports/store.go +5 -0
| @@ -21,6 +21,11 @@ type Store interface { | |||
| 21 | 21 | Get(key []byte) ([]byte, bool) | |
| 22 | 22 | Set(key, val []byte) error | |
| 23 | 23 | Delete(key []byte) error | |
| 24 | + | // Batch applies all changes atomically to in-memory state: an error leaves | |
| 25 | + | // every prior value unchanged. Implementations repair or discard any torn | |
| 26 | + | // WAL batch during replay so no partial batch becomes visible after restart. | |
| 27 | + | // Batch does not retain StoreChange key or value slices; the caller owns and | |
| 28 | + | // may mutate them after Batch returns. | |
| 24 | 29 | Batch([]StoreChange) error | |
| 25 | 30 | // Range iterates key/value pairs; fn returning false stops iteration early. | |
| 26 | 31 | // Each key and value is a stable copy owned by the caller, which may retain | |
M internal/ports/wire_test.go +21 -6
| @@ -744,12 +744,27 @@ func TestKillGoldenAndRoundTrip(t *testing.T) { | |||
| 744 | 744 | } | |
| 745 | 745 | ||
| 746 | 746 | func TestSessionInfoRecoveryState(t *testing.T) { | |
| 747 | - | payload := MarshalSessions(Sessions{Sessions: []SessionInfo{{Name: "broken", State: SessionDegraded}}}) | |
| 748 | - | got, err := UnmarshalSessions(payload) | |
| 749 | - | require.NoError(t, err) | |
| 750 | - | require.Equal(t, SessionDegraded, got.Sessions[0].State) | |
| 751 | - | assertAllPrefixesFail(t, payload, UnmarshalSessions) | |
| 752 | - | assertTrailingGarbageFails(t, payload, UnmarshalSessions) | |
| 747 | + | tests := []struct { | |
| 748 | + | name string | |
| 749 | + | state SessionState | |
| 750 | + | want []byte | |
| 751 | + | }{ | |
| 752 | + | {name: "running", state: SessionRunning, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1}}, | |
| 753 | + | {name: "stopped", state: SessionStopped, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2}}, | |
| 754 | + | {name: "restoring", state: SessionRestoring, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3}}, | |
| 755 | + | {name: "degraded", state: SessionDegraded, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4}}, | |
| 756 | + | } | |
| 757 | + | for _, tt := range tests { | |
| 758 | + | t.Run(tt.name, func(t *testing.T) { | |
| 759 | + | payload := MarshalSessions(Sessions{Sessions: []SessionInfo{{State: tt.state}}}) | |
| 760 | + | require.Equal(t, tt.want, payload) | |
| 761 | + | got, err := UnmarshalSessions(payload) | |
| 762 | + | require.NoError(t, err) | |
| 763 | + | require.Equal(t, tt.state, got.Sessions[0].State) | |
| 764 | + | assertAllPrefixesFail(t, payload, UnmarshalSessions) | |
| 765 | + | assertTrailingGarbageFails(t, payload, UnmarshalSessions) | |
| 766 | + | }) | |
| 767 | + | } | |
| 753 | 768 | } | |
| 754 | 769 | ||
| 755 | 770 | func TestSessionsGoldenAndRoundTrip(t *testing.T) { | |
M internal/usecase/command/command.go +12 -3
| @@ -8,9 +8,9 @@ import ( | |||
| 8 | 8 | // ErrInvalidArguments reports arguments that do not have the exact command format. | |
| 9 | 9 | var ErrInvalidArguments = errors.New("invalid command arguments") | |
| 10 | 10 | ||
| 11 | - | // ParsePositiveDecimal accepts exactly one base-10 positive decimal value. | |
| 11 | + | // ParsePositiveUint64 accepts exactly one base-10 positive decimal value. | |
| 12 | 12 | // It deliberately rejects signs, whitespace, zero, and non-canonical forms. | |
| 13 | - | func ParsePositiveDecimal(args []string) (int, error) { | |
| 13 | + | func ParsePositiveUint64(args []string) (uint64, error) { | |
| 14 | 14 | if len(args) != 1 || args[0] == "" || (len(args[0]) > 1 && args[0][0] == '0') { | |
| 15 | 15 | return 0, ErrInvalidArguments | |
| 16 | 16 | } | |
| @@ -19,13 +19,22 @@ func ParsePositiveDecimal(args []string) (int, error) { | |||
| 19 | 19 | return 0, ErrInvalidArguments | |
| 20 | 20 | } | |
| 21 | 21 | } | |
| 22 | - | n, err := strconv.Atoi(args[0]) | |
| 22 | + | n, err := strconv.ParseUint(args[0], 10, 64) | |
| 23 | 23 | if err != nil || n < 1 { | |
| 24 | 24 | return 0, ErrInvalidArguments | |
| 25 | 25 | } | |
| 26 | 26 | return n, nil | |
| 27 | 27 | } | |
| 28 | 28 | ||
| 29 | + | // ParsePositiveDecimal parses a positive decimal that must fit the platform int. | |
| 30 | + | func ParsePositiveDecimal(args []string) (int, error) { | |
| 31 | + | n, err := ParsePositiveUint64(args) | |
| 32 | + | if err != nil || n > uint64(^uint(0)>>1) { | |
| 33 | + | return 0, ErrInvalidArguments | |
| 34 | + | } | |
| 35 | + | return int(n), nil | |
| 36 | + | } | |
| 37 | + | ||
| 29 | 38 | // Context is the set of application actions available to commands. | |
| 30 | 39 | type Context interface { | |
| 31 | 40 | CreateTab() error | |
M internal/usecase/command/command_test.go +11 -0
| @@ -5,6 +5,17 @@ import ( | |||
| 5 | 5 | "testing" | |
| 6 | 6 | ) | |
| 7 | 7 | ||
| 8 | + | func TestParsePositiveUint64(t *testing.T) { | |
| 9 | + | const maxUint64 = "18446744073709551615" | |
| 10 | + | got, err := ParsePositiveUint64([]string{maxUint64}) | |
| 11 | + | if err != nil || got != ^uint64(0) { | |
| 12 | + | t.Fatalf("ParsePositiveUint64(max) = %d, %v", got, err) | |
| 13 | + | } | |
| 14 | + | if _, err := ParsePositiveUint64([]string{"18446744073709551616"}); !errors.Is(err, ErrInvalidArguments) { | |
| 15 | + | t.Fatalf("ParsePositiveUint64(overflow) error = %v", err) | |
| 16 | + | } | |
| 17 | + | } | |
| 18 | + | ||
| 8 | 19 | func TestParsePositiveDecimal(t *testing.T) { | |
| 9 | 20 | for _, tt := range []struct { | |
| 10 | 21 | name string | |
M internal/usecase/command/registry.go +6 -3
| @@ -1,6 +1,9 @@ | |||
| 1 | 1 | package command | |
| 2 | 2 | ||
| 3 | - | import "strings" | |
| 3 | + | import ( | |
| 4 | + | "path/filepath" | |
| 5 | + | "strings" | |
| 6 | + | ) | |
| 4 | 7 | ||
| 5 | 8 | // Registry returns all palette and control commands in stable display order. | |
| 6 | 9 | func Registry() []Command { | |
| @@ -101,11 +104,11 @@ func sessionRecoveryCommand() Command { | |||
| 101 | 104 | case len(args) == 1 && (args[0] == "retry" || args[0] == "discard"): | |
| 102 | 105 | action = args[0] | |
| 103 | 106 | case len(args) == 2 && args[0] == "restore": | |
| 104 | - | if _, err := ParsePositiveDecimal(args[1:]); err != nil { | |
| 107 | + | if _, err := ParsePositiveUint64(args[1:]); err != nil { | |
| 105 | 108 | return ControlResult{}, ErrInvalidArguments | |
| 106 | 109 | } | |
| 107 | 110 | action, argument = args[0], args[1] | |
| 108 | - | case len(args) == 2 && args[0] == "export" && args[1] != "": | |
| 111 | + | case len(args) == 2 && args[0] == "export" && filepath.IsAbs(args[1]): | |
| 109 | 112 | action, argument = args[0], args[1] | |
| 110 | 113 | default: | |
| 111 | 114 | return ControlResult{}, ErrInvalidArguments | |
M internal/usecase/command/registry_test.go +22 -6
| @@ -117,16 +117,29 @@ func TestSessionRecoveryCommand(t *testing.T) { | |||
| 117 | 117 | }{ | |
| 118 | 118 | {args: []string{"retry"}, wantCall: "session-recovery:retry:"}, | |
| 119 | 119 | {args: []string{"restore", "7"}, wantCall: "session-recovery:restore:7"}, | |
| 120 | + | {args: []string{"restore", "18446744073709551615"}, wantCall: "session-recovery:restore:18446744073709551615"}, | |
| 120 | 121 | {args: []string{"export", "/tmp/export"}, wantCall: "session-recovery:export:/tmp/export"}, | |
| 121 | 122 | {args: []string{"discard"}, wantCall: "session-recovery:discard:"}, | |
| 123 | + | {args: nil, wantErr: ErrInvalidArguments}, | |
| 124 | + | {args: []string{"retry", "extra"}, wantErr: ErrInvalidArguments}, | |
| 125 | + | {args: []string{"restore"}, wantErr: ErrInvalidArguments}, | |
| 122 | 126 | {args: []string{"restore", "0"}, wantErr: ErrInvalidArguments}, | |
| 127 | + | {args: []string{"restore", "01"}, wantErr: ErrInvalidArguments}, | |
| 128 | + | {args: []string{"restore", "18446744073709551616"}, wantErr: ErrInvalidArguments}, | |
| 129 | + | {args: []string{"export", "relative/path"}, wantErr: ErrInvalidArguments}, | |
| 130 | + | {args: []string{"export"}, wantErr: ErrInvalidArguments}, | |
| 131 | + | {args: []string{"discard", "extra"}, wantErr: ErrInvalidArguments}, | |
| 123 | 132 | {args: []string{"unknown"}, wantErr: ErrInvalidArguments}, | |
| 124 | 133 | } | |
| 125 | 134 | for _, test := range tests { | |
| 126 | - | ctx := &controlSpy{} | |
| 127 | - | _, err := cmd.Control(ctx, test.args, ControlOptions{}) | |
| 128 | - | if !errors.Is(err, test.wantErr) || ctx.call != test.wantCall { | |
| 129 | - | t.Errorf("args %v: call/error = %q/%v, want %q/%v", test.args, ctx.call, err, test.wantCall, test.wantErr) | |
| 135 | + | ctx := &controlSpy{recoveryOutput: "recovery output"} | |
| 136 | + | got, err := cmd.Control(ctx, test.args, ControlOptions{}) | |
| 137 | + | wantOutput := "" | |
| 138 | + | if test.wantCall != "" { | |
| 139 | + | wantOutput = ctx.recoveryOutput | |
| 140 | + | } | |
| 141 | + | if !errors.Is(err, test.wantErr) || ctx.call != test.wantCall || got.Output != wantOutput { | |
| 142 | + | t.Errorf("args %v: call/output/error = %q/%q/%v, want %q/%q/%v", test.args, ctx.call, got.Output, err, test.wantCall, wantOutput, test.wantErr) | |
| 130 | 143 | } | |
| 131 | 144 | } | |
| 132 | 145 | } | |
| @@ -175,7 +188,10 @@ func TestControlHandlersValidateAndDelegate(t *testing.T) { | |||
| 175 | 188 | } | |
| 176 | 189 | } | |
| 177 | 190 | ||
| 178 | - | type controlSpy struct{ call string } | |
| 191 | + | type controlSpy struct { | |
| 192 | + | call string | |
| 193 | + | recoveryOutput string | |
| 194 | + | } | |
| 179 | 195 | ||
| 180 | 196 | func (s *controlSpy) record(call string) error { s.call = call; return nil } | |
| 181 | 197 | func (s *controlSpy) CreateTab() error { return s.record("new-tab") } | |
| @@ -205,7 +221,7 @@ func (s *controlSpy) Toast(level, message string) error { | |||
| 205 | 221 | return s.record("toast:" + level + ":" + message) | |
| 206 | 222 | } | |
| 207 | 223 | func (s *controlSpy) SessionRecovery(action, argument string) (string, error) { | |
| 208 | - | return "", s.record("session-recovery:" + action + ":" + argument) | |
| 224 | + | return s.recoveryOutput, s.record("session-recovery:" + action + ":" + argument) | |
| 209 | 225 | } | |
| 210 | 226 | func (s *controlSpy) ListSessions(json bool) (string, error) { | |
| 211 | 227 | _ = s.record("list-sessions:" + strconv.FormatBool(json)) | |
M internal/usecase/daemon/catalogue.go +56 -11
| @@ -7,30 +7,75 @@ import ( | |||
| 7 | 7 | // Catalogue helpers keep persistence optional for ephemeral and focused unit | |
| 8 | 8 | // tests while ensuring every durable named-session mutation uses the authority | |
| 9 | 9 | // port when one is configured. | |
| 10 | - | func (d *Daemon) catalogueRecord(name string) (domain.CatalogueRecord, bool) { | |
| 10 | + | func (d *Daemon) catalogueRecord(name string) (domain.CatalogueRecord, bool, error) { | |
| 11 | 11 | if d == nil || d.catalogue == nil { | |
| 12 | - | return domain.CatalogueRecord{}, false | |
| 12 | + | return domain.CatalogueRecord{}, false, nil | |
| 13 | 13 | } | |
| 14 | 14 | return d.catalogue.Record(name) | |
| 15 | 15 | } | |
| 16 | 16 | ||
| 17 | - | func (d *Daemon) createCatalogueRecord(record domain.CatalogueRecord) error { | |
| 17 | + | func (d *Daemon) updateCatalogueMetadata(update domain.CatalogueMetadataUpdate) error { | |
| 18 | 18 | if d == nil || d.catalogue == nil { | |
| 19 | 19 | return nil | |
| 20 | 20 | } | |
| 21 | - | return d.catalogue.Create(record) | |
| 21 | + | return d.catalogue.UpdateMetadata(update) | |
| 22 | 22 | } | |
| 23 | 23 | ||
| 24 | - | func (d *Daemon) updateCatalogueMetadata(update domain.CatalogueMetadataUpdate) error { | |
| 25 | - | if d == nil || d.catalogue == nil { | |
| 24 | + | // persistSessionMetadata serializes authority writes without holding daemon or | |
| 25 | + | // session state locks during I/O. Only a successfully committed newer revision | |
| 26 | + | // subsumes an older caller. Failed revisions retain their rollback until later | |
| 27 | + | // failed mutations have unwound, keeping live metadata at the durable boundary. | |
| 28 | + | func (d *Daemon) persistSessionMetadata(sess *session, version uint64, update domain.CatalogueMetadataUpdate, rollback func() bool) error { | |
| 29 | + | if sess == nil { | |
| 26 | 30 | return nil | |
| 27 | 31 | } | |
| 28 | - | return d.catalogue.UpdateMetadata(update) | |
| 29 | - | } | |
| 32 | + | sess.metadataPersistMu.Lock() | |
| 33 | + | defer sess.metadataPersistMu.Unlock() | |
| 30 | 34 | ||
| 31 | - | func (d *Daemon) deleteCatalogueRecord(name string) error { | |
| 32 | - | if d == nil || d.catalogue == nil { | |
| 35 | + | if sess.metadataDurableVersion >= version { | |
| 33 | 36 | return nil | |
| 34 | 37 | } | |
| 35 | - | return d.catalogue.Delete(name) | |
| 38 | + | if err := d.updateCatalogueMetadata(update); err != nil { | |
| 39 | + | if sess.metadataFailedRollbacks == nil { | |
| 40 | + | sess.metadataFailedRollbacks = make(map[uint64]func() bool) | |
| 41 | + | } | |
| 42 | + | sess.metadataFailedRollbacks[version] = rollback | |
| 43 | + | // A later mutation may have been reserved while authority I/O blocked. | |
| 44 | + | // It must settle before an older failed mutation can be unwound. | |
| 45 | + | sess.mu.Lock() | |
| 46 | + | current := sess.metadataVersion | |
| 47 | + | sess.mu.Unlock() | |
| 48 | + | if current > sess.metadataLiveVersion { | |
| 49 | + | sess.metadataLiveVersion = current | |
| 50 | + | } | |
| 51 | + | sess.reconcileFailedMetadataLocked() | |
| 52 | + | return err | |
| 53 | + | } | |
| 54 | + | ||
| 55 | + | sess.metadataDurableVersion = version | |
| 56 | + | for failedVersion := range sess.metadataFailedRollbacks { | |
| 57 | + | if failedVersion <= version { | |
| 58 | + | delete(sess.metadataFailedRollbacks, failedVersion) | |
| 59 | + | } | |
| 60 | + | } | |
| 61 | + | return nil | |
| 62 | + | } | |
| 63 | + | ||
| 64 | + | // reconcileFailedMetadataLocked unwinds only a contiguous failed suffix. An | |
| 65 | + | // older failure remains pending while a newer mutation is still writing. | |
| 66 | + | // Rollbacks run only after authority I/O has returned and acquire state locks in | |
| 67 | + | // the daemon-before-session order. | |
| 68 | + | func (s *session) reconcileFailedMetadataLocked() { | |
| 69 | + | for s.metadataLiveVersion > s.metadataDurableVersion { | |
| 70 | + | version := s.metadataLiveVersion | |
| 71 | + | rollback, failed := s.metadataFailedRollbacks[version] | |
| 72 | + | if !failed { | |
| 73 | + | return | |
| 74 | + | } | |
| 75 | + | if rollback != nil && !rollback() { | |
| 76 | + | return | |
| 77 | + | } | |
| 78 | + | delete(s.metadataFailedRollbacks, version) | |
| 79 | + | s.metadataLiveVersion-- | |
| 80 | + | } | |
| 36 | 81 | } | |
M internal/usecase/daemon/control.go +37 -22
| @@ -6,6 +6,8 @@ import ( | |||
| 6 | 6 | "errors" | |
| 7 | 7 | "fmt" | |
| 8 | 8 | "os" | |
| 9 | + | "path/filepath" | |
| 10 | + | "slices" | |
| 9 | 11 | "sort" | |
| 10 | 12 | "strconv" | |
| 11 | 13 | "strings" | |
| @@ -30,14 +32,14 @@ func (d *Daemon) handleCommand(tr ports.Transport, f ports.Frame) { | |||
| 30 | 32 | _ = tr.Send(frameCommandResult(ports.CommandResult{Code: ports.ErrInternal, Text: "malformed command request"})) | |
| 31 | 33 | return | |
| 32 | 34 | } | |
| 33 | - | _ = tr.Send(frameCommandResult(d.dispatchCommand(request))) | |
| 35 | + | _ = tr.Send(frameCommandResult(d.dispatchCommand(d.serveCtx, request))) | |
| 34 | 36 | } | |
| 35 | 37 | ||
| 36 | 38 | func frameCommandResult(result ports.CommandResult) ports.Frame { | |
| 37 | 39 | return ports.Frame{Type: ports.MsgCommandResult, Payload: ports.MarshalCommandResult(result)} | |
| 38 | 40 | } | |
| 39 | 41 | ||
| 40 | - | func (d *Daemon) dispatchCommand(request ports.CommandRequest) ports.CommandResult { | |
| 42 | + | func (d *Daemon) dispatchCommand(ctx context.Context, request ports.CommandRequest) ports.CommandResult { | |
| 41 | 43 | cmd, ok := command.BySlug(request.Slug) | |
| 42 | 44 | if !ok { | |
| 43 | 45 | return commandFailure(ports.ErrUnknownCommand, "unknown command: "+request.Slug) | |
| @@ -46,7 +48,7 @@ func (d *Daemon) dispatchCommand(request ports.CommandRequest) ports.CommandResu | |||
| 46 | 48 | return commandFailure(ports.ErrNotScriptable, request.Slug+" requires an attached client") | |
| 47 | 49 | } | |
| 48 | 50 | if cmd.Target == command.TargetNone && !request.Self { | |
| 49 | - | return d.runControl(cmd, controlExec{d: d, recoveryName: request.TargetSession}, request) | |
| 51 | + | return d.runControl(cmd, controlExec{ctx: ctx, d: d, recoveryName: request.TargetSession}, request) | |
| 50 | 52 | } | |
| 51 | 53 | ||
| 52 | 54 | sess, code, text := d.resolveTargetSession(request) | |
| @@ -60,7 +62,7 @@ func (d *Daemon) dispatchCommand(request ports.CommandRequest) ports.CommandResu | |||
| 60 | 62 | if code != 0 { | |
| 61 | 63 | return commandFailure(code, text) | |
| 62 | 64 | } | |
| 63 | - | return d.runControl(cmd, controlExec{d: d, sess: sess, tab: tb, target: daemonActionTarget{session: sess, tab: tb, pane: pane}}, request) | |
| 65 | + | return d.runControl(cmd, controlExec{ctx: ctx, d: d, sess: sess, tab: tb, target: daemonActionTarget{session: sess, tab: tb, pane: pane}}, request) | |
| 64 | 66 | } | |
| 65 | 67 | ||
| 66 | 68 | func (d *Daemon) runControl(cmd command.Command, exec controlExec, request ports.CommandRequest) ports.CommandResult { | |
| @@ -309,14 +311,7 @@ func (d *Daemon) hasDaemonActionPaneTarget(target daemonActionTarget) bool { | |||
| 309 | 311 | } | |
| 310 | 312 | target.session.mu.Lock() | |
| 311 | 313 | defer target.session.mu.Unlock() | |
| 312 | - | foundTab := false | |
| 313 | - | for _, tb := range target.session.tabs { | |
| 314 | - | if tb == target.tab { | |
| 315 | - | foundTab = true | |
| 316 | - | break | |
| 317 | - | } | |
| 318 | - | } | |
| 319 | - | if !foundTab { | |
| 314 | + | if !slices.Contains(target.session.tabs, target.tab) { | |
| 320 | 315 | return false | |
| 321 | 316 | } | |
| 322 | 317 | target.tab.mu.Lock() | |
| @@ -335,6 +330,7 @@ func (a daemonActions) switchRelative(sess *session, delta int) error { | |||
| 335 | 330 | // controlExec implements command.ControlContext against resolved daemon-owned | |
| 336 | 331 | // targets. It deliberately contains no attached-client state. | |
| 337 | 332 | type controlExec struct { | |
| 333 | + | ctx context.Context | |
| 338 | 334 | d *Daemon | |
| 339 | 335 | sess *session | |
| 340 | 336 | tab *tab | |
| @@ -465,7 +461,10 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) { | |||
| 465 | 461 | if e.d == nil || e.d.degradedRecovery == nil || e.recoveryName == "" { | |
| 466 | 462 | return "", command.ErrInvalidArguments | |
| 467 | 463 | } | |
| 468 | - | ctx := context.Background() | |
| 464 | + | ctx := e.ctx | |
| 465 | + | if ctx == nil { | |
| 466 | + | ctx = e.d.serveCtx | |
| 467 | + | } | |
| 469 | 468 | switch action { | |
| 470 | 469 | case "retry": | |
| 471 | 470 | if err := e.d.degradedRecovery.Retry(ctx, e.recoveryName); err != nil { | |
| @@ -480,7 +479,10 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) { | |||
| 480 | 479 | if err != nil || generation == 0 { | |
| 481 | 480 | return "", command.ErrInvalidArguments | |
| 482 | 481 | } | |
| 483 | - | record, ok := e.d.catalogue.Record(e.recoveryName) | |
| 482 | + | record, ok, err := e.d.catalogue.Record(e.recoveryName) | |
| 483 | + | if err != nil { | |
| 484 | + | return "", err | |
| 485 | + | } | |
| 484 | 486 | if !ok { | |
| 485 | 487 | return "", errors.New("session recovery record not found") | |
| 486 | 488 | } | |
| @@ -494,6 +496,9 @@ func (e controlExec) SessionRecovery(action, argument string) (string, error) { | |||
| 494 | 496 | } | |
| 495 | 497 | return "", errors.New("requested recovery fallback is not available") | |
| 496 | 498 | case "export": | |
| 499 | + | if !filepath.IsAbs(argument) { | |
| 500 | + | return "", command.ErrInvalidArguments | |
| 501 | + | } | |
| 497 | 502 | file, err := os.OpenFile(argument, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) | |
| 498 | 503 | if err != nil { | |
| 499 | 504 | return "", err | |
| @@ -520,20 +525,30 @@ func (e controlExec) restoreExplicitRecovery(ctx context.Context) error { | |||
| 520 | 525 | if e.d.catalogue == nil { | |
| 521 | 526 | return errors.New("durable session catalogue unavailable") | |
| 522 | 527 | } | |
| 523 | - | record, ok := e.d.catalogue.Record(e.recoveryName) | |
| 528 | + | record, ok, err := e.d.catalogue.Record(e.recoveryName) | |
| 529 | + | if err != nil { | |
| 530 | + | return err | |
| 531 | + | } | |
| 524 | 532 | if !ok { | |
| 525 | 533 | return errors.New("session recovery record not found") | |
| 526 | 534 | } | |
| 527 | 535 | done := make(chan struct{}) | |
| 528 | 536 | e.d.mu.Lock() | |
| 529 | - | if entry, exists := e.d.stopped[record.Name]; exists { | |
| 530 | - | entry.record = record | |
| 531 | - | entry.state = runtimeRestoring | |
| 532 | - | entry.restoreDone = done | |
| 533 | - | e.d.stopped[record.Name] = entry | |
| 534 | - | } | |
| 537 | + | entry, exists := e.d.stopped[record.Name] | |
| 538 | + | if !exists || entry.incarnation != record.IncarnationID { | |
| 539 | + | e.d.mu.Unlock() | |
| 540 | + | return errors.New("session recovery runtime record not found") | |
| 541 | + | } | |
| 542 | + | if entry.purging || entry.state != runtimeDegraded { | |
| 543 | + | e.d.mu.Unlock() | |
| 544 | + | return errors.New("session recovery is not available in the current state") | |
| 545 | + | } | |
| 546 | + | entry.record = record | |
| 547 | + | entry.state = runtimeRestoring | |
| 548 | + | entry.restoreDone = done | |
| 549 | + | e.d.stopped[record.Name] = entry | |
| 535 | 550 | e.d.mu.Unlock() | |
| 536 | - | err := e.d.restoreRecord(ctx, record) | |
| 551 | + | err = e.d.restoreRecord(ctx, record) | |
| 537 | 552 | e.d.finishRecordRestore(record, err, done) | |
| 538 | 553 | return err | |
| 539 | 554 | } | |
M internal/usecase/daemon/daemon.go +72 -70
| @@ -7,13 +7,13 @@ import ( | |||
| 7 | 7 | "log/slog" | |
| 8 | 8 | "math" | |
| 9 | 9 | "os" | |
| 10 | + | "reflect" | |
| 10 | 11 | "sort" | |
| 11 | 12 | "sync" | |
| 12 | 13 | "sync/atomic" | |
| 13 | 14 | "time" | |
| 14 | 15 | ||
| 15 | 16 | "github.com/bnema/vev/internal/domain" | |
| 16 | - | "github.com/bnema/vev/internal/persist" | |
| 17 | 17 | "github.com/bnema/vev/internal/ports" | |
| 18 | 18 | "github.com/bnema/vev/internal/usecase/keys" | |
| 19 | 19 | ) | |
| @@ -70,6 +70,8 @@ type Daemon struct { | |||
| 70 | 70 | mu sync.Mutex | |
| 71 | 71 | sessions map[domain.SessionID]*session | |
| 72 | 72 | stopped map[string]stoppedSession | |
| 73 | + | // creating reserves names while durable creation I/O runs without mu. | |
| 74 | + | creating map[string]struct{} | |
| 73 | 75 | nextID uint64 | |
| 74 | 76 | // lastAllocatedCreatedAt is the named-session lifecycle timestamp high-water | |
| 75 | 77 | // mark. It is guarded by mu and prevents a wall-clock regression from | |
| @@ -117,13 +119,11 @@ type Daemon struct { | |||
| 117 | 119 | shell string | |
| 118 | 120 | shellArgs []string | |
| 119 | 121 | shellOverride bool | |
| 120 | - | persist *persist.Persister | |
| 121 | 122 | persistEnabled bool | |
| 122 | 123 | catalogue ports.Catalogue | |
| 123 | 124 | catalogueRecords []domain.CatalogueRecord | |
| 124 | 125 | catalogueRecordsProvided bool | |
| 125 | - | // snapshotRepository is the sole checkpoint contract. legacySnapshots is | |
| 126 | - | // read-only migration input and is never used for new writes. | |
| 126 | + | // snapshotRepository is the sole checkpoint contract. | |
| 127 | 127 | snapshotRepository ports.SnapshotRepository | |
| 128 | 128 | checkpointRecovery ports.CheckpointCoordinator | |
| 129 | 129 | lifecycleRecovery ports.SessionLifecycleCoordinator | |
| @@ -136,10 +136,12 @@ type Daemon struct { | |||
| 136 | 136 | // records, so it is a durable writer and is guarded by snapshotWorkerMu with | |
| 137 | 137 | // the other two. | |
| 138 | 138 | restoreWorkerDone chan struct{} | |
| 139 | - | legacySnapshots ports.LegacySnapshotSource | |
| 140 | 139 | snapsEnabled bool | |
| 141 | 140 | noticeStore ports.NoticeStore | |
| 142 | 141 | snapshotJobs chan *snapshotCapture | |
| 142 | + | // snapshotAdmitted contains every capture accepted by either worker queue, | |
| 143 | + | // including captures buffered in snapshotJobs. Guarded by snapshotWorkerMu. | |
| 144 | + | snapshotAdmitted map[*snapshotCapture]struct{} | |
| 143 | 145 | // snapshotWake wakes the repository scheduler when a session becomes dirty | |
| 144 | 146 | // or an attempt completes. It is never closed and producers only send | |
| 145 | 147 | // non-blockingly. | |
| @@ -263,16 +265,6 @@ func WithShell(cmd string, args []string) Option { | |||
| 263 | 265 | } | |
| 264 | 266 | } | |
| 265 | 267 | ||
| 266 | - | // WithStore enables persisted named session metadata. A nil store keeps the | |
| 267 | - | // daemon in no-op persistence mode. | |
| 268 | - | func WithStore(store ports.Store) Option { | |
| 269 | - | return func(d *Daemon) { | |
| 270 | - | d.persist = persist.New(store) | |
| 271 | - | d.catalogue = d.persist | |
| 272 | - | d.persistEnabled = store != nil | |
| 273 | - | } | |
| 274 | - | } | |
| 275 | - | ||
| 276 | 268 | // WithCatalogue installs the singular catalogue and the strictly opened | |
| 277 | 269 | // records that define the daemon's expected-session registry at startup. | |
| 278 | 270 | func WithCatalogue(catalogue ports.Catalogue, records []domain.CatalogueRecord) Option { | |
| @@ -303,26 +295,30 @@ func WithRecoveryCoordinator(coordinator ports.SessionLifecycleCoordinator) Opti | |||
| 303 | 295 | } | |
| 304 | 296 | } | |
| 305 | 297 | ||
| 306 | - | // WithSnapshotRepository enables content-addressed incremental snapshots. The | |
| 307 | - | // legacy source is read-only migration input and is never used for new writes. | |
| 308 | - | func WithSnapshotRepository(repository any, legacy ports.LegacySnapshotSource) Option { | |
| 298 | + | // WithSnapshotRepository enables content-addressed incremental snapshots. | |
| 299 | + | func WithSnapshotRepository(repository ports.SnapshotRepository) Option { | |
| 309 | 300 | return func(d *Daemon) { | |
| 310 | - | if repository == nil { | |
| 301 | + | if isNilSnapshotRepository(repository) { | |
| 311 | 302 | return | |
| 312 | 303 | } | |
| 313 | - | typed, ok := repository.(ports.SnapshotRepository) | |
| 314 | - | if !ok { | |
| 315 | - | // Pre-incarnation test repositories are intentionally not admitted to | |
| 316 | - | // normal runtime paths. They remain accepted by this option only so | |
| 317 | - | // package-level legacy tests can exercise their isolated helpers. | |
| 318 | - | return | |
| 319 | - | } | |
| 320 | - | d.snapshotRepository = typed | |
| 321 | - | d.legacySnapshots = legacy | |
| 304 | + | d.snapshotRepository = repository | |
| 322 | 305 | d.snapsEnabled = true | |
| 323 | 306 | } | |
| 324 | 307 | } | |
| 325 | 308 | ||
| 309 | + | func isNilSnapshotRepository(repository ports.SnapshotRepository) bool { | |
| 310 | + | if repository == nil { | |
| 311 | + | return true | |
| 312 | + | } | |
| 313 | + | value := reflect.ValueOf(repository) | |
| 314 | + | switch value.Kind() { | |
| 315 | + | case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: | |
| 316 | + | return value.IsNil() | |
| 317 | + | default: | |
| 318 | + | return false | |
| 319 | + | } | |
| 320 | + | } | |
| 321 | + | ||
| 326 | 322 | // WithNoticeStore enables persisting undeliverable notices across daemon | |
| 327 | 323 | // restarts. A nil store keeps the daemon in no-op notice-persistence mode. | |
| 328 | 324 | func WithNoticeStore(store ports.NoticeStore) Option { | |
| @@ -420,6 +416,9 @@ func (d *Daemon) allocateLifecycleCreatedAtLocked() (int64, error) { | |||
| 420 | 416 | } | |
| 421 | 417 | ||
| 422 | 418 | func (d *Daemon) nowUnixNano() int64 { | |
| 419 | + | if d == nil || d.clock == nil { | |
| 420 | + | return time.Now().UnixNano() | |
| 421 | + | } | |
| 423 | 422 | return d.clock.Now().UnixNano() | |
| 424 | 423 | } | |
| 425 | 424 | ||
| @@ -434,23 +433,24 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt | |||
| 434 | 433 | shell = "/bin/sh" | |
| 435 | 434 | } | |
| 436 | 435 | d := &Daemon{ | |
| 437 | - | sessions: make(map[domain.SessionID]*session), | |
| 438 | - | stopped: make(map[string]stoppedSession), | |
| 439 | - | parked: make(map[uint64]*parkedAttachment), | |
| 440 | - | ptys: ptys, | |
| 441 | - | clock: clock, | |
| 442 | - | log: log, | |
| 443 | - | baseEnv: os.Environ(), | |
| 444 | - | shell: shell, | |
| 445 | - | persist: persist.New(nil), | |
| 446 | - | dirOrHome: dirOrHome, | |
| 447 | - | done: make(chan struct{}), | |
| 448 | - | restoreDone: make(chan struct{}), | |
| 449 | - | animWake: make(chan struct{}, 1), | |
| 450 | - | snapshotJobs: make(chan *snapshotCapture, snapshotQueueCapacity), | |
| 451 | - | snapshotWake: make(chan struct{}, 1), | |
| 452 | - | notices: newNoticeCenter(), | |
| 453 | - | resumeParkGrace: defaultResumeParkGrace, | |
| 436 | + | sessions: make(map[domain.SessionID]*session), | |
| 437 | + | stopped: make(map[string]stoppedSession), | |
| 438 | + | creating: make(map[string]struct{}), | |
| 439 | + | parked: make(map[uint64]*parkedAttachment), | |
| 440 | + | ptys: ptys, | |
| 441 | + | clock: clock, | |
| 442 | + | log: log, | |
| 443 | + | baseEnv: os.Environ(), | |
| 444 | + | shell: shell, | |
| 445 | + | dirOrHome: dirOrHome, | |
| 446 | + | done: make(chan struct{}), | |
| 447 | + | restoreDone: make(chan struct{}), | |
| 448 | + | animWake: make(chan struct{}, 1), | |
| 449 | + | snapshotJobs: make(chan *snapshotCapture, snapshotQueueCapacity), | |
| 450 | + | snapshotAdmitted: make(map[*snapshotCapture]struct{}), | |
| 451 | + | snapshotWake: make(chan struct{}, 1), | |
| 452 | + | notices: newNoticeCenter(), | |
| 453 | + | resumeParkGrace: defaultResumeParkGrace, | |
| 454 | 454 | barScripts: &barScriptState{ | |
| 455 | 455 | cfg: barConfigFromDomain(domain.Defaults().Bar), | |
| 456 | 456 | outputs: make(map[domain.SessionID]barScriptOutputs), | |
| @@ -470,9 +470,6 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt | |||
| 470 | 470 | for _, o := range opts { | |
| 471 | 471 | o(d) | |
| 472 | 472 | } | |
| 473 | - | if d.persist == nil { | |
| 474 | - | d.persist = persist.New(nil) | |
| 475 | - | } | |
| 476 | 473 | if d.dirOrHome == nil { | |
| 477 | 474 | d.dirOrHome = dirOrHome | |
| 478 | 475 | } | |
| @@ -488,20 +485,6 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt | |||
| 488 | 485 | d.restoreProcessAllowlist.Store(&allow) | |
| 489 | 486 | } | |
| 490 | 487 | records := d.catalogueRecords | |
| 491 | - | // Only attempt the load when persistence is actually configured | |
| 492 | - | // (d.persistEnabled tracks the same store-is-non-nil condition | |
| 493 | - | // persist.Persister uses to distinguish "no store" from "load failed"). | |
| 494 | - | // Without this gate, the default no-persistence Daemon (persist.New(nil)) | |
| 495 | - | // would log a false-alarm "loading persisted sessions failed" warning on | |
| 496 | - | // every construction: nothing failed, persistence simply isn't enabled. | |
| 497 | - | if !d.catalogueRecordsProvided && d.persistEnabled { | |
| 498 | - | var err error | |
| 499 | - | records, err = d.persist.LoadAll() | |
| 500 | - | if err != nil { | |
| 501 | - | d.log.Warn("loading persisted sessions failed", "err", err) | |
| 502 | - | return d | |
| 503 | - | } | |
| 504 | - | } | |
| 505 | 488 | var maxSeq uint64 | |
| 506 | 489 | var maxCreatedAt int64 | |
| 507 | 490 | hasCreatedAt := false | |
| @@ -628,8 +611,6 @@ func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error { | |||
| 628 | 611 | var closeErr error | |
| 629 | 612 | if d.catalogue != nil { | |
| 630 | 613 | closeErr = d.catalogue.Close() | |
| 631 | - | } else { | |
| 632 | - | closeErr = d.persist.Close() | |
| 633 | 614 | } | |
| 634 | 615 | if closeErr != nil { | |
| 635 | 616 | d.log.Warn("closing session persister failed", "err", closeErr) | |
| @@ -662,11 +643,23 @@ func (d *Daemon) shutdownAllWithSnapshotDeadline(reason uint8, deadline *snapsho | |||
| 662 | 643 | for _, s := range snapshot { | |
| 663 | 644 | // Cancellation and PTY closure must not wait behind a teardown owner that | |
| 664 | 645 | // is blocked in snapshot publication or purge work. | |
| 646 | + | s.mu.Lock() | |
| 647 | + | name := s.name | |
| 648 | + | ephemeral := s.ephemeral | |
| 649 | + | s.mu.Unlock() | |
| 665 | 650 | s.stopInMemoryLifecycle() | |
| 666 | 651 | if err := d.killSessionWithSnapshotDeadline(s, reason, false, deadline); err != nil { | |
| 667 | 652 | checkpointIncomplete = true | |
| 668 | 653 | d.log.Error("closing session with unpersisted terminal state", "err", err) | |
| 669 | 654 | } | |
| 655 | + | if !ephemeral && deadline != nil { | |
| 656 | + | select { | |
| 657 | + | case <-deadline.Done(): | |
| 658 | + | checkpointIncomplete = true | |
| 659 | + | d.persistShutdownSnapshotFailure(name, context.DeadlineExceeded) | |
| 660 | + | default: | |
| 661 | + | } | |
| 662 | + | } | |
| 670 | 663 | } | |
| 671 | 664 | return checkpointIncomplete | |
| 672 | 665 | } | |
| @@ -738,6 +731,12 @@ func snapshotStopContext(deadline *snapshotShutdownDeadline) (context.Context, c | |||
| 738 | 731 | if deadline == nil { | |
| 739 | 732 | return ctx, cancel | |
| 740 | 733 | } | |
| 734 | + | select { | |
| 735 | + | case <-deadline.Done(): | |
| 736 | + | cancel() | |
| 737 | + | return ctx, cancel | |
| 738 | + | default: | |
| 739 | + | } | |
| 741 | 740 | stop := make(chan struct{}) | |
| 742 | 741 | go func() { | |
| 743 | 742 | select { | |
| @@ -817,13 +816,16 @@ func (d *Daemon) handleList(tr ports.Transport) { | |||
| 817 | 816 | } | |
| 818 | 817 | state := ports.SessionStopped | |
| 819 | 818 | if stopped.purging { | |
| 819 | + | // Purge is the dominant externally visible state: restoration must | |
| 820 | + | // never make a deletion-reserved record appear attachable. | |
| 820 | 821 | state = ports.SessionDegraded | |
| 821 | - | } | |
| 822 | - | switch stopped.state { | |
| 823 | - | case runtimeRestoring: | |
| 824 | - | state = ports.SessionRestoring | |
| 825 | - | case runtimeDegraded, runtimeDeleting: | |
| 826 | - | state = ports.SessionDegraded | |
| 822 | + | } else { | |
| 823 | + | switch stopped.state { | |
| 824 | + | case runtimeRestoring: | |
| 825 | + | state = ports.SessionRestoring | |
| 826 | + | case runtimeDegraded, runtimeDeleting: | |
| 827 | + | state = ports.SessionDegraded | |
| 828 | + | } | |
| 827 | 829 | } | |
| 828 | 830 | infos = append(infos, ports.SessionInfo{Name: name, State: state}) | |
| 829 | 831 | } | |
A internal/usecase/daemon/durable_authority_test.go +49 -0
| @@ -0,0 +1,49 @@ | |||
| 1 | + | package daemon | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "crypto/rand" | |
| 6 | + | ||
| 7 | + | "github.com/bnema/vev/internal/domain" | |
| 8 | + | "github.com/bnema/vev/internal/persist" | |
| 9 | + | "github.com/bnema/vev/internal/ports" | |
| 10 | + | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | |
| 11 | + | ) | |
| 12 | + | ||
| 13 | + | type testLifecycleRepository struct{ ports.SnapshotRepository } | |
| 14 | + | ||
| 15 | + | func (testLifecycleRepository) WriteDeletionTombstone(context.Context, domain.DeletionTombstone) error { | |
| 16 | + | return nil | |
| 17 | + | } | |
| 18 | + | func (testLifecycleRepository) QuarantineDeletionSources(context.Context, domain.DeletionTombstone, bool) error { | |
| 19 | + | return nil | |
| 20 | + | } | |
| 21 | + | func (testLifecycleRepository) DeleteDeletionTombstone(context.Context, domain.IncarnationID) error { | |
| 22 | + | return nil | |
| 23 | + | } | |
| 24 | + | ||
| 25 | + | // WithStore is package-test composition glue. Production injects the same | |
| 26 | + | // recovery coordinator explicitly from internal/app. | |
| 27 | + | func WithStore(store ports.Store) Option { | |
| 28 | + | return func(d *Daemon) { | |
| 29 | + | if store == nil { | |
| 30 | + | return | |
| 31 | + | } | |
| 32 | + | catalogue := persist.New(store) | |
| 33 | + | records, err := catalogue.Records() | |
| 34 | + | if err != nil { | |
| 35 | + | return | |
| 36 | + | } | |
| 37 | + | d.catalogue = catalogue | |
| 38 | + | d.persistEnabled = true | |
| 39 | + | d.catalogueRecords = records | |
| 40 | + | d.catalogueRecordsProvided = true | |
| 41 | + | repository := d.snapshotRepository | |
| 42 | + | if repository == nil { | |
| 43 | + | repository = testLifecycleRepository{} | |
| 44 | + | } | |
| 45 | + | authority := recoveryusecase.NewCoordinator(catalogue, repository, nil, rand.Reader) | |
| 46 | + | d.lifecycleRecovery = authority | |
| 47 | + | d.checkpointRecovery = authority | |
| 48 | + | } | |
| 49 | + | } | |
M internal/usecase/daemon/durable_recovery_test.go +126 -24
| @@ -16,6 +16,8 @@ import ( | |||
| 16 | 16 | ||
| 17 | 17 | "github.com/bnema/vev/internal/domain" | |
| 18 | 18 | "github.com/bnema/vev/internal/ports" | |
| 19 | + | portsmocks "github.com/bnema/vev/internal/ports/mocks" | |
| 20 | + | "github.com/bnema/vev/internal/usecase/command" | |
| 19 | 21 | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | |
| 20 | 22 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |
| 21 | 23 | ) | |
| @@ -35,21 +37,28 @@ func newDurableRecoveryCatalogue(records []domain.CatalogueRecord) *durableRecov | |||
| 35 | 37 | return catalogue | |
| 36 | 38 | } | |
| 37 | 39 | ||
| 38 | - | func (c *durableRecoveryCatalogue) Records() []domain.CatalogueRecord { | |
| 40 | + | func (c *durableRecoveryCatalogue) Records() ([]domain.CatalogueRecord, error) { | |
| 39 | 41 | c.mu.Lock() | |
| 40 | 42 | defer c.mu.Unlock() | |
| 41 | 43 | records := make([]domain.CatalogueRecord, 0, len(c.records)) | |
| 42 | 44 | for _, record := range c.records { | |
| 43 | 45 | records = append(records, record) | |
| 44 | 46 | } | |
| 45 | - | return records | |
| 47 | + | return records, nil | |
| 46 | 48 | } | |
| 47 | 49 | ||
| 48 | - | func (c *durableRecoveryCatalogue) Record(name string) (domain.CatalogueRecord, bool) { | |
| 50 | + | func (c *durableRecoveryCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) { | |
| 49 | 51 | c.mu.Lock() | |
| 50 | 52 | defer c.mu.Unlock() | |
| 51 | 53 | record, ok := c.records[name] | |
| 52 | - | return record, ok | |
| 54 | + | return record, ok, nil | |
| 55 | + | } | |
| 56 | + | ||
| 57 | + | func mustDurableRecords(t *testing.T, c *durableRecoveryCatalogue) []domain.CatalogueRecord { | |
| 58 | + | t.Helper() | |
| 59 | + | records, err := c.Records() | |
| 60 | + | require.NoError(t, err) | |
| 61 | + | return records | |
| 53 | 62 | } | |
| 54 | 63 | ||
| 55 | 64 | func (c *durableRecoveryCatalogue) Replace(name string, record domain.CatalogueRecord) error { | |
| @@ -111,6 +120,9 @@ type durableRecoveryRepository struct { | |||
| 111 | 120 | func (r *durableRecoveryRepository) LoadCheckpoint(_ context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) { | |
| 112 | 121 | r.mu.Lock() | |
| 113 | 122 | defer r.mu.Unlock() | |
| 123 | + | if r.loads == nil { | |
| 124 | + | r.loads = make(map[string]int) | |
| 125 | + | } | |
| 114 | 126 | r.loads[name]++ | |
| 115 | 127 | if err := r.errors[name]; err != nil { | |
| 116 | 128 | return ports.SnapshotGeneration{}, err | |
| @@ -129,6 +141,9 @@ func (r *durableRecoveryRepository) RepairHEAD(_ context.Context, id domain.Inca | |||
| 129 | 141 | if r.repairErr != nil { | |
| 130 | 142 | return r.repairErr | |
| 131 | 143 | } | |
| 144 | + | if r.repairs == nil { | |
| 145 | + | r.repairs = make(map[string]domain.CheckpointRef) | |
| 146 | + | } | |
| 132 | 147 | r.repairs[id.String()] = ref | |
| 133 | 148 | return nil | |
| 134 | 149 | } | |
| @@ -169,7 +184,7 @@ func newDurableRecoveryDaemon(t testing.TB, records []domain.CatalogueRecord, re | |||
| 169 | 184 | t.Helper() | |
| 170 | 185 | catalogue := newDurableRecoveryCatalogue(records) | |
| 171 | 186 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | |
| 172 | - | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, records), WithSnapshotRepository(repository, nil), WithCheckpointCoordinator(coordinator)) | |
| 187 | + | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, records), WithSnapshotRepository(repository), WithCheckpointCoordinator(coordinator)) | |
| 173 | 188 | d.serveCtx, d.serveCancel = context.WithCancel(context.Background()) | |
| 174 | 189 | t.Cleanup(d.serveCancel) | |
| 175 | 190 | return d, catalogue | |
| @@ -188,7 +203,7 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) { | |||
| 188 | 203 | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{prior}) | |
| 189 | 204 | repository := &durableRecoveryRepository{} | |
| 190 | 205 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | |
| 191 | - | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, []domain.CatalogueRecord{prior}), WithSnapshotRepository(repository, nil), WithCheckpointCoordinator(coordinator)) | |
| 206 | + | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, []domain.CatalogueRecord{prior}), WithSnapshotRepository(repository), WithCheckpointCoordinator(coordinator)) | |
| 192 | 207 | startSnapshotEncodeWorker(t, d) | |
| 193 | 208 | ||
| 194 | 209 | sess := newSnapshotTestSession(t, prior.Name, false, prior.Cwd) | |
| @@ -204,7 +219,7 @@ func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) { | |||
| 204 | 219 | ||
| 205 | 220 | require.True(t, d.scheduleFinalSnapshot(sess)) | |
| 206 | 221 | awaitSnapshotIdle(t, sess) | |
| 207 | - | got, ok := catalogue.Record(prior.Name) | |
| 222 | + | got, ok, _ := catalogue.Record(prior.Name) | |
| 208 | 223 | require.True(t, ok) | |
| 209 | 224 | require.Equal(t, prior, got, "a timed-out final checkpoint must not change authoritative bytes") | |
| 210 | 225 | require.Equal(t, priorRef, *sess.snapshotPublishedCheckpoint) | |
| @@ -214,7 +229,7 @@ func TestCatalogueRestoreIndependent(t *testing.T) { | |||
| 214 | 229 | emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{}) | |
| 215 | 230 | emptyDone := make(chan struct{}) | |
| 216 | 231 | go func() { | |
| 217 | - | emptyDaemon.restoreCatalogue(context.Background(), emptyCatalogue.Records()) | |
| 232 | + | emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue)) | |
| 218 | 233 | close(emptyDone) | |
| 219 | 234 | }() | |
| 220 | 235 | select { | |
| @@ -233,7 +248,7 @@ func TestCatalogueRestoreIndependent(t *testing.T) { | |||
| 233 | 248 | } | |
| 234 | 249 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository) | |
| 235 | 250 | ||
| 236 | - | d.restoreCatalogue(context.Background(), catalogue.Records()) | |
| 251 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 237 | 252 | ||
| 238 | 253 | require.Equal(t, runtimeHealthy, d.stopped[healthy.Name].state) | |
| 239 | 254 | require.Equal(t, runtimeDegraded, d.stopped[broken.Name].state) | |
| @@ -244,7 +259,7 @@ func TestCatalogueRestoreIndependent(t *testing.T) { | |||
| 244 | 259 | t.Fatalf("restore completion for %q was not closed", name) | |
| 245 | 260 | } | |
| 246 | 261 | } | |
| 247 | - | degraded, ok := catalogue.Record(broken.Name) | |
| 262 | + | degraded, ok, _ := catalogue.Record(broken.Name) | |
| 248 | 263 | require.True(t, ok) | |
| 249 | 264 | require.Equal(t, domain.RecoveryDegraded, degraded.RecoveryState) | |
| 250 | 265 | require.Equal(t, 1, repository.loads[healthy.Name]) | |
| @@ -265,8 +280,8 @@ func TestCatalogueRestoreIndependent(t *testing.T) { | |||
| 265 | 280 | repairs: make(map[string]domain.CheckpointRef), | |
| 266 | 281 | } | |
| 267 | 282 | fallbackDaemon, fallbackCatalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fallback}, fallbackRepository) | |
| 268 | - | fallbackDaemon.restoreCatalogue(context.Background(), fallbackCatalogue.Records()) | |
| 269 | - | promoted, ok := fallbackCatalogue.Record(fallback.Name) | |
| 283 | + | fallbackDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, fallbackCatalogue)) | |
| 284 | + | promoted, ok, _ := fallbackCatalogue.Record(fallback.Name) | |
| 270 | 285 | require.True(t, ok) | |
| 271 | 286 | require.Equal(t, fallbackRef, *promoted.Committed) | |
| 272 | 287 | require.Equal(t, runtimeHealthy, fallbackDaemon.stopped[fallback.Name].state) | |
| @@ -291,9 +306,9 @@ func TestCatalogueRestoreKeepsCommittedFallbackHealthyWhenHEADRepairFails(t *tes | |||
| 291 | 306 | } | |
| 292 | 307 | ||
| 293 | 308 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 294 | - | d.restoreCatalogue(context.Background(), catalogue.Records()) | |
| 309 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 295 | 310 | ||
| 296 | - | committed, ok := catalogue.Record(record.Name) | |
| 311 | + | committed, ok, _ := catalogue.Record(record.Name) | |
| 297 | 312 | require.True(t, ok) | |
| 298 | 313 | require.Equal(t, domain.RecoveryHealthy, committed.RecoveryState) | |
| 299 | 314 | require.Equal(t, fallbackRef, *committed.Committed) | |
| @@ -302,7 +317,7 @@ func TestCatalogueRestoreKeepsCommittedFallbackHealthyWhenHEADRepairFails(t *tes | |||
| 302 | 317 | ||
| 303 | 318 | repository.repairErr = nil | |
| 304 | 319 | next, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{committed}, repository) | |
| 305 | - | next.restoreCatalogue(context.Background(), catalogue.Records()) | |
| 320 | + | next.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 306 | 321 | require.Equal(t, runtimeHealthy, next.stopped[record.Name].state) | |
| 307 | 322 | require.Equal(t, 2, repository.repairCalls, "the next startup retries the pending HEAD repair") | |
| 308 | 323 | require.Equal(t, fallbackRef, repository.repairs[record.IncarnationID.String()]) | |
| @@ -318,7 +333,7 @@ func TestCatalogueRestoreAggregateOver4096(t *testing.T) { | |||
| 318 | 333 | } | |
| 319 | 334 | d, catalogue := newDurableRecoveryDaemon(t, records, repository) | |
| 320 | 335 | ||
| 321 | - | d.restoreCatalogue(context.Background(), catalogue.Records()) | |
| 336 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 322 | 337 | ||
| 323 | 338 | require.Len(t, d.stopped, count) | |
| 324 | 339 | for _, record := range records { | |
| @@ -334,7 +349,7 @@ func TestCatalogueRestoreSingleSessionOver4096(t *testing.T) { | |||
| 334 | 349 | ||
| 335 | 350 | // The repository may contain any number of stale entries; restoration has no | |
| 336 | 351 | // discovery API and therefore performs exactly one direct catalogue lookup. | |
| 337 | - | d.restoreCatalogue(context.Background(), catalogue.Records()) | |
| 352 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 338 | 353 | ||
| 339 | 354 | require.Equal(t, runtimeHealthy, d.stopped[record.Name].state) | |
| 340 | 355 | require.Equal(t, 1, repository.loads[record.Name]) | |
| @@ -347,10 +362,10 @@ func TestCatalogueRestoreIncarnationMismatch(t *testing.T) { | |||
| 347 | 362 | repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: generation}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | |
| 348 | 363 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 349 | 364 | ||
| 350 | - | d.restoreCatalogue(context.Background(), catalogue.Records()) | |
| 365 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 351 | 366 | ||
| 352 | 367 | require.Equal(t, runtimeDegraded, d.stopped[record.Name].state) | |
| 353 | - | got, _ := catalogue.Record(record.Name) | |
| 368 | + | got, _, _ := catalogue.Record(record.Name) | |
| 354 | 369 | require.Equal(t, domain.RecoveryDegraded, got.RecoveryState) | |
| 355 | 370 | } | |
| 356 | 371 | ||
| @@ -391,6 +406,59 @@ func (s explicitRecoveryStub) Discard(context.Context, string, string) (domain.C | |||
| 391 | 406 | return s.record, nil | |
| 392 | 407 | } | |
| 393 | 408 | ||
| 409 | + | type contextRecoveryStub struct { | |
| 410 | + | explicitRecoveryStub | |
| 411 | + | wantKey any | |
| 412 | + | want any | |
| 413 | + | seen bool | |
| 414 | + | err error | |
| 415 | + | } | |
| 416 | + | ||
| 417 | + | func (s *contextRecoveryStub) Retry(ctx context.Context, _ string) error { | |
| 418 | + | s.seen = ctx.Value(s.wantKey) == s.want | |
| 419 | + | return s.err | |
| 420 | + | } | |
| 421 | + | ||
| 422 | + | func TestSessionRecoveryUsesIncomingCommandContext(t *testing.T) { | |
| 423 | + | type contextKey struct{} | |
| 424 | + | key := contextKey{} | |
| 425 | + | ctx := context.WithValue(t.Context(), key, "request") | |
| 426 | + | wantErr := errors.New("stop after context capture") | |
| 427 | + | stub := &contextRecoveryStub{wantKey: key, want: "request", err: wantErr} | |
| 428 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 429 | + | d.degradedRecovery = stub | |
| 430 | + | ||
| 431 | + | _, err := (controlExec{ctx: ctx, d: d, recoveryName: "work"}).SessionRecovery("retry", "") | |
| 432 | + | require.ErrorIs(t, err, wantErr) | |
| 433 | + | require.True(t, stub.seen) | |
| 434 | + | } | |
| 435 | + | ||
| 436 | + | func TestExplicitRecoveryRejectsConcurrentRestore(t *testing.T) { | |
| 437 | + | record := durableRecoveryRecord(0) | |
| 438 | + | record.RecoveryState = domain.RecoveryDegraded | |
| 439 | + | record.DegradedReason = "checkpoint validation failed" | |
| 440 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | |
| 441 | + | d.mu.Lock() | |
| 442 | + | entry := d.stopped[record.Name] | |
| 443 | + | entry.state = runtimeRestoring | |
| 444 | + | originalDone := entry.restoreDone | |
| 445 | + | d.stopped[record.Name] = entry | |
| 446 | + | d.mu.Unlock() | |
| 447 | + | ||
| 448 | + | err := (controlExec{d: d, recoveryName: record.Name}).restoreExplicitRecovery(t.Context()) | |
| 449 | + | require.ErrorContains(t, err, "current state") | |
| 450 | + | d.mu.Lock() | |
| 451 | + | require.Equal(t, originalDone, d.stopped[record.Name].restoreDone) | |
| 452 | + | d.mu.Unlock() | |
| 453 | + | } | |
| 454 | + | ||
| 455 | + | func TestSessionRecoveryExportRequiresAbsolutePath(t *testing.T) { | |
| 456 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 457 | + | d.degradedRecovery = explicitRecoveryStub{} | |
| 458 | + | _, err := (controlExec{ctx: t.Context(), d: d, recoveryName: "work"}).SessionRecovery("export", "relative.snapshot") | |
| 459 | + | require.ErrorIs(t, err, command.ErrInvalidArguments) | |
| 460 | + | } | |
| 461 | + | ||
| 394 | 462 | func TestSessionRecoveryCommand(t *testing.T) { | |
| 395 | 463 | degraded := durableRecoveryRecord(2) | |
| 396 | 464 | degraded.RecoveryState = domain.RecoveryDegraded | |
| @@ -428,6 +496,20 @@ func TestListShowsDegraded(t *testing.T) { | |||
| 428 | 496 | }, listed.Sessions) | |
| 429 | 497 | } | |
| 430 | 498 | ||
| 499 | + | func TestListPurgingDominatesRestoringState(t *testing.T) { | |
| 500 | + | record := durableRecoveryRecord(0) | |
| 501 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | |
| 502 | + | d.mu.Lock() | |
| 503 | + | entry := d.stopped[record.Name] | |
| 504 | + | entry.purging = true | |
| 505 | + | entry.state = runtimeRestoring | |
| 506 | + | d.stopped[record.Name] = entry | |
| 507 | + | d.mu.Unlock() | |
| 508 | + | ||
| 509 | + | listed := listSessions(t, d) | |
| 510 | + | require.Equal(t, []ports.SessionInfo{{Name: record.Name, State: ports.SessionDegraded}}, listed.Sessions) | |
| 511 | + | } | |
| 512 | + | ||
| 431 | 513 | func TestAttachWaitsForRestore(t *testing.T) { | |
| 432 | 514 | record := durableRecoveryRecord(0) | |
| 433 | 515 | factory := &recoveryCountingPTYFactory{} | |
| @@ -449,7 +531,9 @@ func TestAttachWaitsForRestore(t *testing.T) { | |||
| 449 | 531 | require.Zero(t, factory.calls.Load()) | |
| 450 | 532 | ||
| 451 | 533 | d.setStoppedRecovery(record, runtimeDegraded) | |
| 452 | - | closeRuntimeRestoreDone(d.recordRestoreDone(record.Name)) | |
| 534 | + | d.mu.Lock() | |
| 535 | + | closeRuntimeRestoreDoneLocked(d.stopped[record.Name].restoreDone) | |
| 536 | + | d.mu.Unlock() | |
| 453 | 537 | var protocolError *protoErr | |
| 454 | 538 | require.ErrorAs(t, <-result, &protocolError) | |
| 455 | 539 | require.Equal(t, ports.ErrSessionDegraded, protocolError.code) | |
| @@ -463,7 +547,7 @@ func TestRestoreCancellationTransitionsBeforeAttachCompletion(t *testing.T) { | |||
| 463 | 547 | restoreCtx, cancelRestore := context.WithCancel(context.Background()) | |
| 464 | 548 | restored := make(chan struct{}) | |
| 465 | 549 | go func() { | |
| 466 | - | d.restoreCatalogue(restoreCtx, catalogue.Records()) | |
| 550 | + | d.restoreCatalogue(restoreCtx, mustDurableRecords(t, catalogue)) | |
| 467 | 551 | close(restored) | |
| 468 | 552 | }() | |
| 469 | 553 | <-repository.started | |
| @@ -488,14 +572,16 @@ func TestRestoreCancellationTransitionsBeforeAttachCompletion(t *testing.T) { | |||
| 488 | 572 | default: | |
| 489 | 573 | t.Fatal("restore completion was not signaled") | |
| 490 | 574 | } | |
| 491 | - | committed, _ := catalogue.Record(record.Name) | |
| 575 | + | committed, _, _ := catalogue.Record(record.Name) | |
| 492 | 576 | require.Equal(t, entry.record, committed) | |
| 493 | 577 | } | |
| 494 | 578 | ||
| 495 | 579 | func TestWaitForTargetRestoreRejectsClosedRestoringChannel(t *testing.T) { | |
| 496 | 580 | record := durableRecoveryRecord(0) | |
| 497 | 581 | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | |
| 498 | - | closeRuntimeRestoreDone(d.stopped[record.Name].restoreDone) | |
| 582 | + | d.mu.Lock() | |
| 583 | + | closeRuntimeRestoreDoneLocked(d.stopped[record.Name].restoreDone) | |
| 584 | + | d.mu.Unlock() | |
| 499 | 585 | ||
| 500 | 586 | result := make(chan error, 1) | |
| 501 | 587 | go func() { result <- d.waitForTargetRestore(record.Name) }() | |
| @@ -552,6 +638,22 @@ func TestAttachRejectsDegraded(t *testing.T) { | |||
| 552 | 638 | require.Zero(t, factory.calls.Load()) | |
| 553 | 639 | } | |
| 554 | 640 | ||
| 641 | + | func TestRestoreUnavailableReleasesPerRecordWaiters(t *testing.T) { | |
| 642 | + | record := durableRecoveryRecord(0) | |
| 643 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, nil) | |
| 644 | + | d.closeRestoreDone() | |
| 645 | + | ||
| 646 | + | d.mu.Lock() | |
| 647 | + | entry := d.stopped[record.Name] | |
| 648 | + | d.mu.Unlock() | |
| 649 | + | require.Equal(t, runtimeDegraded, entry.state) | |
| 650 | + | select { | |
| 651 | + | case <-entry.restoreDone: | |
| 652 | + | default: | |
| 653 | + | t.Fatal("per-record restore waiter remained blocked when repository was unavailable") | |
| 654 | + | } | |
| 655 | + | } | |
| 656 | + | ||
| 555 | 657 | func TestCatalogueRestoreDegradedVisible(t *testing.T) { | |
| 556 | 658 | record := durableRecoveryRecord(0) | |
| 557 | 659 | record.RecoveryState = domain.RecoveryDegraded | |
| @@ -559,7 +661,7 @@ func TestCatalogueRestoreDegradedVisible(t *testing.T) { | |||
| 559 | 661 | repository := &durableRecoveryRepository{generations: make(map[string]ports.SnapshotGeneration), errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | |
| 560 | 662 | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |
| 561 | 663 | ||
| 562 | - | d.restoreCatalogue(context.Background(), catalogue.Records()) | |
| 664 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |
| 563 | 665 | ||
| 564 | 666 | require.Equal(t, runtimeDegraded, d.stopped[record.Name].state) | |
| 565 | 667 | require.Zero(t, repository.loads[record.Name]) | |
D internal/usecase/daemon/legacy_snapshot_test.go +0 -8
| @@ -1,8 +0,0 @@ | |||
| 1 | - | package daemon | |
| 2 | - | ||
| 3 | - | import "context" | |
| 4 | - | ||
| 5 | - | // importLegacySnapshots is retained only for tests covering the retired | |
| 6 | - | // pre-migration bridge. Normal runtime legacy decoding lives exclusively in | |
| 7 | - | // the snapshot migration adapter. | |
| 8 | - | func (d *Daemon) importLegacySnapshots(context.Context) {} | |
M internal/usecase/daemon/maintenance.go +33 -8
| @@ -18,6 +18,9 @@ type maintenanceReconciler interface { | |||
| 18 | 18 | Step(context.Context, ports.ReconcileCursor) (ports.ReconcileCursor, []ports.ReconcileDecision, error) | |
| 19 | 19 | } | |
| 20 | 20 | ||
| 21 | + | // maintenanceDependencies is exclusively owned by runDurableMaintenance's | |
| 22 | + | // single goroutine after startup. Shutdown joins that goroutine before these | |
| 23 | + | // cursor and repair sets can be observed or reused. | |
| 21 | 24 | type maintenanceDependencies struct { | |
| 22 | 25 | catalogue ports.Catalogue | |
| 23 | 26 | repository ports.SnapshotRepository | |
| @@ -88,7 +91,13 @@ func (d *Daemon) runDurableMaintenance(ctx context.Context) { | |||
| 88 | 91 | case <-d.restoreDone: | |
| 89 | 92 | } | |
| 90 | 93 | for { | |
| 91 | - | d.runDurableMaintenanceTick(ctx) | |
| 94 | + | incomplete := d.runDurableMaintenanceTick(ctx) | |
| 95 | + | if ctx.Err() != nil { | |
| 96 | + | return | |
| 97 | + | } | |
| 98 | + | if incomplete { | |
| 99 | + | continue | |
| 100 | + | } | |
| 92 | 101 | timer := d.clock.NewTimer(maintenanceInterval) | |
| 93 | 102 | select { | |
| 94 | 103 | case <-ctx.Done(): | |
| @@ -99,10 +108,12 @@ func (d *Daemon) runDurableMaintenance(ctx context.Context) { | |||
| 99 | 108 | } | |
| 100 | 109 | } | |
| 101 | 110 | ||
| 102 | - | func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) { | |
| 111 | + | // runDurableMaintenanceTick reports whether bounded work remains and should be | |
| 112 | + | // continued immediately rather than delayed until the periodic interval. | |
| 113 | + | func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) bool { | |
| 103 | 114 | deps := &d.maintenance | |
| 104 | 115 | if deps.catalogue == nil || deps.repository == nil || deps.reconciler == nil { | |
| 105 | - | return | |
| 116 | + | return false | |
| 106 | 117 | } | |
| 107 | 118 | ||
| 108 | 119 | // Reconcile before collection. Until a complete top-level scan reaches EOF, | |
| @@ -111,11 +122,15 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) { | |||
| 111 | 122 | next, decisions, err := deps.reconciler.Step(ctx, deps.cursor) | |
| 112 | 123 | if err != nil { | |
| 113 | 124 | d.log.Warn("durable_reconciliation_tick", "action", "scan", "cursor", deps.cursor.DirectoryCookie, "done", false, "err", err) | |
| 114 | - | return | |
| 125 | + | return false | |
| 115 | 126 | } | |
| 116 | 127 | deps.cursor = next | |
| 117 | 128 | for _, decision := range decisions { | |
| 118 | - | record, ok := deps.catalogue.Record(decision.Name) | |
| 129 | + | record, ok, err := deps.catalogue.Record(decision.Name) | |
| 130 | + | if err != nil { | |
| 131 | + | d.log.Warn("durable_reconciliation_tick", "action", "catalogue-read", "done", false, "err", err) | |
| 132 | + | return false | |
| 133 | + | } | |
| 119 | 134 | if ok && !decision.RetentionResolved { | |
| 120 | 135 | deps.repairUncertain[record.IncarnationID] = struct{}{} | |
| 121 | 136 | } | |
| @@ -136,18 +151,27 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) { | |||
| 136 | 151 | } | |
| 137 | 152 | d.log.Info("durable_reconciliation_tick", "action", "scan", "cursor", next.DirectoryCookie, "done", next.DirectoryCookie == 0, "err", nil) | |
| 138 | 153 | if next.DirectoryCookie != 0 { | |
| 139 | - | return | |
| 154 | + | return true | |
| 140 | 155 | } | |
| 141 | 156 | ||
| 142 | - | for _, record := range deps.catalogue.Records() { | |
| 157 | + | records, err := deps.catalogue.Records() | |
| 158 | + | if err != nil { | |
| 159 | + | d.log.Warn("durable_maintenance_tick", "action", "catalogue-read", "done", false, "err", err) | |
| 160 | + | return false | |
| 161 | + | } | |
| 162 | + | incomplete := false | |
| 163 | + | for _, record := range records { | |
| 143 | 164 | if err := ctx.Err(); err != nil { | |
| 144 | - | return | |
| 165 | + | return false | |
| 145 | 166 | } | |
| 146 | 167 | _, unresolved := deps.unresolved[record.IncarnationID] | |
| 147 | 168 | _, uncertainRepair := deps.repairUncertain[record.IncarnationID] | |
| 148 | 169 | restoring := d.incarnationRestoring(record.IncarnationID) | |
| 149 | 170 | plan := retentionPlan(record, unresolved || uncertainRepair, restoring) | |
| 150 | 171 | done, err := deps.repository.MaintainSession(ctx, plan, ports.MaintenanceBudget{Entries: maintenanceEntries, Bytes: maintenanceBytes}) | |
| 172 | + | if err == nil && !done { | |
| 173 | + | incomplete = true | |
| 174 | + | } | |
| 151 | 175 | d.log.Info("durable_maintenance_session", | |
| 152 | 176 | "incarnation", record.IncarnationID.String(), | |
| 153 | 177 | "action", "retention", | |
| @@ -159,6 +183,7 @@ func (d *Daemon) runDurableMaintenanceTick(ctx context.Context) { | |||
| 159 | 183 | ) | |
| 160 | 184 | } | |
| 161 | 185 | clear(deps.repairUncertain) | |
| 186 | + | return incomplete | |
| 162 | 187 | } | |
| 163 | 188 | ||
| 164 | 189 | func (d *Daemon) incarnationRestoring(id domain.IncarnationID) bool { | |
M internal/usecase/daemon/maintenance_test.go +4 -4
| @@ -28,7 +28,7 @@ func (r *cursorMaintenanceReconciler) Step(_ context.Context, cursor ports.Recon | |||
| 28 | 28 | func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) { | |
| 29 | 29 | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh} | |
| 30 | 30 | catalogue := portsmocks.NewMockCatalogue(t) | |
| 31 | - | catalogue.EXPECT().Records().Return([]domain.CatalogueRecord{record}).Once() | |
| 31 | + | catalogue.EXPECT().Records().Return([]domain.CatalogueRecord{record}, nil).Once() | |
| 32 | 32 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 33 | 33 | repository.EXPECT().MaintainSession(mock.Anything, mock.MatchedBy(func(plan ports.RetentionPlan) bool { | |
| 34 | 34 | return plan.IncarnationID == record.IncarnationID && !plan.PinAll && len(plan.Keep) == 0 | |
| @@ -37,9 +37,9 @@ func TestProductionMaintenanceUsesCatalogueAndResumesCursor(t *testing.T) { | |||
| 37 | 37 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 38 | 38 | WithDurableMaintenance(catalogue, repository, reconciler, nil)(d) | |
| 39 | 39 | ||
| 40 | - | d.runDurableMaintenanceTick(context.Background()) | |
| 41 | - | d.runDurableMaintenanceTick(context.Background()) | |
| 42 | - | d.runDurableMaintenanceTick(context.Background()) | |
| 40 | + | require.True(t, d.runDurableMaintenanceTick(context.Background()), "partial reconciliation must continue immediately") | |
| 41 | + | require.False(t, d.runDurableMaintenanceTick(context.Background())) | |
| 42 | + | require.True(t, d.runDurableMaintenanceTick(context.Background()), "the next partial pass must also continue immediately") | |
| 43 | 43 | if len(reconciler.seen) != 3 || reconciler.seen[0].DirectoryCookie != 0 || reconciler.seen[1].DirectoryCookie != 7 || reconciler.seen[2].DirectoryCookie != 0 { | |
| 44 | 44 | t.Fatalf("reconciliation cursors = %v, want [0 7 0]", reconciler.seen) | |
| 45 | 45 | } | |
A internal/usecase/daemon/metadata_order_test.go +285 -0
| @@ -0,0 +1,285 @@ | |||
| 1 | + | package daemon | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "errors" | |
| 5 | + | "sync" | |
| 6 | + | "testing" | |
| 7 | + | "time" | |
| 8 | + | ||
| 9 | + | "github.com/stretchr/testify/require" | |
| 10 | + | ||
| 11 | + | "github.com/bnema/vev/internal/domain" | |
| 12 | + | "github.com/bnema/vev/internal/ports" | |
| 13 | + | ) | |
| 14 | + | ||
| 15 | + | type blockingMetadataCatalogue struct { | |
| 16 | + | ports.Catalogue | |
| 17 | + | ||
| 18 | + | mu sync.Mutex | |
| 19 | + | calls int | |
| 20 | + | firstStarted chan struct{} | |
| 21 | + | releaseFirst chan struct{} | |
| 22 | + | firstErr error | |
| 23 | + | secondErr error | |
| 24 | + | } | |
| 25 | + | ||
| 26 | + | func (c *blockingMetadataCatalogue) UpdateMetadata(update domain.CatalogueMetadataUpdate) error { | |
| 27 | + | c.mu.Lock() | |
| 28 | + | c.calls++ | |
| 29 | + | call := c.calls | |
| 30 | + | c.mu.Unlock() | |
| 31 | + | if call == 1 { | |
| 32 | + | close(c.firstStarted) | |
| 33 | + | <-c.releaseFirst | |
| 34 | + | if c.firstErr != nil { | |
| 35 | + | return c.firstErr | |
| 36 | + | } | |
| 37 | + | } | |
| 38 | + | if call == 2 && c.secondErr != nil { | |
| 39 | + | return c.secondErr | |
| 40 | + | } | |
| 41 | + | return c.Catalogue.UpdateMetadata(update) | |
| 42 | + | } | |
| 43 | + | ||
| 44 | + | func newBlockingCatalogue(sess *session, firstErr error) (*durableRecoveryCatalogue, *blockingMetadataCatalogue) { | |
| 45 | + | record := sess.persistRecordLocked(sess.createdAt) | |
| 46 | + | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}) | |
| 47 | + | return catalogue, &blockingMetadataCatalogue{ | |
| 48 | + | Catalogue: catalogue, | |
| 49 | + | firstStarted: make(chan struct{}), | |
| 50 | + | releaseFirst: make(chan struct{}), | |
| 51 | + | firstErr: firstErr, | |
| 52 | + | } | |
| 53 | + | } | |
| 54 | + | ||
| 55 | + | func TestMetadataWritesPreserveLaterMutationWhenFirstWriteIsBlocked(t *testing.T) { | |
| 56 | + | firstFailure := errors.New("first metadata write failed") | |
| 57 | + | for _, tt := range []struct { | |
| 58 | + | name string | |
| 59 | + | firstErr error | |
| 60 | + | }{ | |
| 61 | + | {name: "delayed success"}, | |
| 62 | + | {name: "delayed failure does not roll back newer state", firstErr: firstFailure}, | |
| 63 | + | } { | |
| 64 | + | t.Run(tt.name, func(t *testing.T) { | |
| 65 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |
| 66 | + | catalogue, blocking := newBlockingCatalogue(sess, tt.firstErr) | |
| 67 | + | d := newTestDaemon(t, nil, stubClock{}) | |
| 68 | + | d.catalogue = blocking | |
| 69 | + | d.sessions[sess.id] = sess | |
| 70 | + | ||
| 71 | + | firstDone := make(chan error, 1) | |
| 72 | + | go func() { firstDone <- d.renameTab(sess, sess.tabs[0], "first") }() | |
| 73 | + | select { | |
| 74 | + | case <-blocking.firstStarted: | |
| 75 | + | case <-time.After(testWaitTimeout): | |
| 76 | + | t.Fatal("first metadata write did not block") | |
| 77 | + | } | |
| 78 | + | ||
| 79 | + | secondDone := make(chan error, 1) | |
| 80 | + | go func() { secondDone <- d.renameTab(sess, sess.tabs[0], "second") }() | |
| 81 | + | require.Eventually(t, func() bool { | |
| 82 | + | sess.mu.Lock() | |
| 83 | + | defer sess.mu.Unlock() | |
| 84 | + | return sess.tabs[0].name == "second" | |
| 85 | + | }, testWaitTimeout, time.Millisecond, "later mutation did not proceed while storage was blocked") | |
| 86 | + | ||
| 87 | + | close(blocking.releaseFirst) | |
| 88 | + | firstResult := <-firstDone | |
| 89 | + | if tt.firstErr == nil { | |
| 90 | + | require.NoError(t, firstResult) | |
| 91 | + | } else { | |
| 92 | + | require.ErrorIs(t, firstResult, tt.firstErr, "an in-memory revision cannot subsume a failed durable write") | |
| 93 | + | } | |
| 94 | + | require.NoError(t, <-secondDone) | |
| 95 | + | ||
| 96 | + | persisted, ok, err := catalogue.Record("work") | |
| 97 | + | require.NoError(t, err) | |
| 98 | + | require.True(t, ok) | |
| 99 | + | require.Equal(t, []string{"second"}, persisted.TabNames) | |
| 100 | + | require.Equal(t, "second", sess.tabs[0].name) | |
| 101 | + | }) | |
| 102 | + | } | |
| 103 | + | } | |
| 104 | + | ||
| 105 | + | func TestTwoConsecutiveMetadataFailuresAreBothReported(t *testing.T) { | |
| 106 | + | firstFailure := errors.New("first metadata write failed") | |
| 107 | + | secondFailure := errors.New("second metadata write failed") | |
| 108 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |
| 109 | + | catalogue, blocking := newBlockingCatalogue(sess, firstFailure) | |
| 110 | + | blocking.secondErr = secondFailure | |
| 111 | + | close(blocking.releaseFirst) | |
| 112 | + | d := newTestDaemon(t, nil, stubClock{}) | |
| 113 | + | d.catalogue = blocking | |
| 114 | + | d.sessions[sess.id] = sess | |
| 115 | + | ||
| 116 | + | require.ErrorIs(t, d.renameTab(sess, sess.tabs[0], "first"), firstFailure) | |
| 117 | + | require.ErrorIs(t, d.renameTab(sess, sess.tabs[0], "second"), secondFailure) | |
| 118 | + | ||
| 119 | + | persisted, ok, err := catalogue.Record("work") | |
| 120 | + | require.NoError(t, err) | |
| 121 | + | require.True(t, ok) | |
| 122 | + | require.Empty(t, persisted.TabNames) | |
| 123 | + | sess.mu.Lock() | |
| 124 | + | require.Empty(t, sess.tabs[0].name) | |
| 125 | + | sess.mu.Unlock() | |
| 126 | + | } | |
| 127 | + | ||
| 128 | + | func TestConsecutiveMetadataFailuresRestoreDurableState(t *testing.T) { | |
| 129 | + | firstFailure := errors.New("first metadata write failed") | |
| 130 | + | secondFailure := errors.New("second metadata write failed") | |
| 131 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |
| 132 | + | catalogue, blocking := newBlockingCatalogue(sess, firstFailure) | |
| 133 | + | blocking.secondErr = secondFailure | |
| 134 | + | d := newTestDaemon(t, nil, stubClock{}) | |
| 135 | + | d.catalogue = blocking | |
| 136 | + | d.sessions[sess.id] = sess | |
| 137 | + | ||
| 138 | + | firstDone := make(chan error, 1) | |
| 139 | + | go func() { firstDone <- d.renameTab(sess, sess.tabs[0], "first") }() | |
| 140 | + | select { | |
| 141 | + | case <-blocking.firstStarted: | |
| 142 | + | case <-time.After(testWaitTimeout): | |
| 143 | + | t.Fatal("first metadata write did not block") | |
| 144 | + | } | |
| 145 | + | secondDone := make(chan error, 1) | |
| 146 | + | go func() { secondDone <- d.renameTab(sess, sess.tabs[0], "second") }() | |
| 147 | + | require.Eventually(t, func() bool { | |
| 148 | + | sess.mu.Lock() | |
| 149 | + | defer sess.mu.Unlock() | |
| 150 | + | return sess.tabs[0].name == "second" | |
| 151 | + | }, testWaitTimeout, time.Millisecond) | |
| 152 | + | ||
| 153 | + | close(blocking.releaseFirst) | |
| 154 | + | require.ErrorIs(t, <-firstDone, firstFailure) | |
| 155 | + | require.ErrorIs(t, <-secondDone, secondFailure) | |
| 156 | + | ||
| 157 | + | persisted, ok, err := catalogue.Record("work") | |
| 158 | + | require.NoError(t, err) | |
| 159 | + | require.True(t, ok) | |
| 160 | + | require.Empty(t, persisted.TabNames) | |
| 161 | + | sess.mu.Lock() | |
| 162 | + | require.Empty(t, sess.tabs[0].name) | |
| 163 | + | sess.mu.Unlock() | |
| 164 | + | } | |
| 165 | + | ||
| 166 | + | func TestNewerMetadataFailureDoesNotSkipOlderSnapshot(t *testing.T) { | |
| 167 | + | failure := errors.New("newer metadata write failed") | |
| 168 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |
| 169 | + | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{sess.persistRecordLocked(sess.createdAt)}) | |
| 170 | + | blocking := &blockingMetadataCatalogue{ | |
| 171 | + | Catalogue: catalogue, | |
| 172 | + | firstStarted: make(chan struct{}), | |
| 173 | + | releaseFirst: make(chan struct{}), | |
| 174 | + | firstErr: failure, | |
| 175 | + | } | |
| 176 | + | close(blocking.releaseFirst) | |
| 177 | + | d := newTestDaemon(t, nil, stubClock{}) | |
| 178 | + | d.catalogue = blocking | |
| 179 | + | d.sessions[sess.id] = sess | |
| 180 | + | tb := sess.tabs[0] | |
| 181 | + | ||
| 182 | + | sess.mu.Lock() | |
| 183 | + | tb.name = "first" | |
| 184 | + | firstRecord, firstVersion := sess.nextPersistRecordLocked(sess.createdAt + 1) | |
| 185 | + | tb.name = "second" | |
| 186 | + | secondRecord, secondVersion := sess.nextPersistRecordLocked(sess.createdAt + 2) | |
| 187 | + | sess.mu.Unlock() | |
| 188 | + | ||
| 189 | + | secondRollback := func() bool { | |
| 190 | + | sess.mu.Lock() | |
| 191 | + | defer sess.mu.Unlock() | |
| 192 | + | if tb.name != "second" { | |
| 193 | + | return false | |
| 194 | + | } | |
| 195 | + | tb.name = "first" | |
| 196 | + | return true | |
| 197 | + | } | |
| 198 | + | firstRollback := func() bool { | |
| 199 | + | sess.mu.Lock() | |
| 200 | + | defer sess.mu.Unlock() | |
| 201 | + | if tb.name != "first" { | |
| 202 | + | return false | |
| 203 | + | } | |
| 204 | + | tb.name = "" | |
| 205 | + | return true | |
| 206 | + | } | |
| 207 | + | ||
| 208 | + | require.ErrorIs(t, d.persistSessionMetadata(sess, secondVersion, secondRecord.MetadataUpdate(), secondRollback), failure) | |
| 209 | + | require.NoError(t, d.persistSessionMetadata(sess, firstVersion, firstRecord.MetadataUpdate(), firstRollback)) | |
| 210 | + | ||
| 211 | + | persisted, ok, err := catalogue.Record("work") | |
| 212 | + | require.NoError(t, err) | |
| 213 | + | require.True(t, ok) | |
| 214 | + | require.Equal(t, []string{"first"}, persisted.TabNames) | |
| 215 | + | sess.mu.Lock() | |
| 216 | + | require.Equal(t, "first", tb.name) | |
| 217 | + | sess.mu.Unlock() | |
| 218 | + | } | |
| 219 | + | ||
| 220 | + | func TestLatestMetadataFailureRollsBackRename(t *testing.T) { | |
| 221 | + | failure := errors.New("metadata write failed") | |
| 222 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |
| 223 | + | catalogue, blocking := newBlockingCatalogue(sess, failure) | |
| 224 | + | d := newTestDaemon(t, nil, stubClock{}) | |
| 225 | + | d.catalogue = blocking | |
| 226 | + | d.sessions[sess.id] = sess | |
| 227 | + | ||
| 228 | + | done := make(chan error, 1) | |
| 229 | + | go func() { done <- d.renameTab(sess, sess.tabs[0], "failed") }() | |
| 230 | + | select { | |
| 231 | + | case <-blocking.firstStarted: | |
| 232 | + | case <-time.After(testWaitTimeout): | |
| 233 | + | t.Fatal("metadata write did not block") | |
| 234 | + | } | |
| 235 | + | close(blocking.releaseFirst) | |
| 236 | + | require.ErrorIs(t, <-done, failure) | |
| 237 | + | ||
| 238 | + | persisted, ok, err := catalogue.Record("work") | |
| 239 | + | require.NoError(t, err) | |
| 240 | + | require.True(t, ok) | |
| 241 | + | require.Empty(t, persisted.TabNames) | |
| 242 | + | require.Empty(t, sess.tabs[0].name) | |
| 243 | + | } | |
| 244 | + | ||
| 245 | + | func TestBlockedCwdWriteCannotOverwriteLaterTabMetadata(t *testing.T) { | |
| 246 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |
| 247 | + | catalogue, blocking := newBlockingCatalogue(sess, nil) | |
| 248 | + | d := newTestDaemon(t, nil, stubClock{}) | |
| 249 | + | d.catalogue = blocking | |
| 250 | + | d.sessions[sess.id] = sess | |
| 251 | + | d.procCwd = func(int) (string, error) { return "/later-cwd", nil } | |
| 252 | + | ||
| 253 | + | cwdDone := make(chan struct{}) | |
| 254 | + | go func() { | |
| 255 | + | d.refreshSessionCwd(sess) | |
| 256 | + | close(cwdDone) | |
| 257 | + | }() | |
| 258 | + | select { | |
| 259 | + | case <-blocking.firstStarted: | |
| 260 | + | case <-time.After(testWaitTimeout): | |
| 261 | + | t.Fatal("cwd metadata write did not block") | |
| 262 | + | } | |
| 263 | + | ||
| 264 | + | renameDone := make(chan error, 1) | |
| 265 | + | go func() { renameDone <- d.renameTab(sess, sess.tabs[0], "later-tab") }() | |
| 266 | + | require.Eventually(t, func() bool { | |
| 267 | + | sess.mu.Lock() | |
| 268 | + | defer sess.mu.Unlock() | |
| 269 | + | return sess.tabs[0].name == "later-tab" | |
| 270 | + | }, testWaitTimeout, time.Millisecond) | |
| 271 | + | ||
| 272 | + | close(blocking.releaseFirst) | |
| 273 | + | select { | |
| 274 | + | case <-cwdDone: | |
| 275 | + | case <-time.After(testWaitTimeout): | |
| 276 | + | t.Fatal("cwd refresh deadlocked") | |
| 277 | + | } | |
| 278 | + | require.NoError(t, <-renameDone) | |
| 279 | + | ||
| 280 | + | persisted, ok, err := catalogue.Record("work") | |
| 281 | + | require.NoError(t, err) | |
| 282 | + | require.True(t, ok) | |
| 283 | + | require.Equal(t, "/later-cwd", persisted.Cwd) | |
| 284 | + | require.Equal(t, []string{"later-tab"}, persisted.TabNames) | |
| 285 | + | } | |
M internal/usecase/daemon/performance_benchmark_test.go +3 -3
| @@ -638,7 +638,7 @@ func publishBenchmarkSnapshot(b testing.TB, fixture *performanceFixture, generat | |||
| 638 | 638 | ||
| 639 | 639 | func mutateBenchmarkVisible(fixture *performanceFixture, operation int) { | |
| 640 | 640 | fixture.activePane.mu.Lock() | |
| 641 | - | fixture.activePane.screen.Write([]byte(fmt.Sprintf("\x1b[1;1Hvisible-%08d", operation))) | |
| 641 | + | fixture.activePane.screen.Write(fmt.Appendf(nil, "\x1b[1;1Hvisible-%08d", operation)) | |
| 642 | 642 | fixture.activePane.mu.Unlock() | |
| 643 | 643 | } | |
| 644 | 644 | ||
| @@ -653,7 +653,7 @@ func mutateBenchmarkTail(fixture *performanceFixture, operation int) { | |||
| 653 | 653 | } | |
| 654 | 654 | ||
| 655 | 655 | func mutateBenchmarkSealedChunk(fixture *performanceFixture, operation int) { | |
| 656 | - | for row := 0; row < 256; row++ { | |
| 656 | + | for row := range 256 { | |
| 657 | 657 | mutateBenchmarkTail(fixture, operation*256+row) | |
| 658 | 658 | } | |
| 659 | 659 | } | |
| @@ -1065,7 +1065,7 @@ func newPerformanceFixtureWithCleanup(t testing.TB, config performanceConfig, re | |||
| 1065 | 1065 | liveWrites: [][]byte{[]byte("\x1b[1;1HA\x1b[2;2HA"), []byte("\x1b[1;1HB\x1b[2;2HB")}, | |
| 1066 | 1066 | resizeSizes: [2]domain.Size{{Cols: 100, Rows: 30}, config.size}, | |
| 1067 | 1067 | } | |
| 1068 | - | WithSnapshotRepository(fixture.snapshots, nil)(d) | |
| 1068 | + | WithSnapshotRepository(fixture.snapshots)(d) | |
| 1069 | 1069 | d.startSnapshotEncodeWorker() | |
| 1070 | 1070 | if registerCleanup { | |
| 1071 | 1071 | t.Cleanup(d.stopSnapshotEncodeWorker) | |
M internal/usecase/daemon/picker_test.go +27 -34
| @@ -25,26 +25,6 @@ import ( | |||
| 25 | 25 | ||
| 26 | 26 | // --- test doubles ----------------------------------------------------------- | |
| 27 | 27 | ||
| 28 | - | // failingDeleteStore is a ports.Store whose Delete always fails, used to | |
| 29 | - | // exercise the picker's kill-target error path without disturbing the | |
| 30 | - | // generated mock's default Delete expectation. | |
| 31 | - | type failingDeleteStore struct{ err error } | |
| 32 | - | ||
| 33 | - | func (failingDeleteStore) Get([]byte) ([]byte, bool) { return nil, false } | |
| 34 | - | func (failingDeleteStore) Set([]byte, []byte) error { return nil } | |
| 35 | - | func (s failingDeleteStore) Batch(changes []ports.StoreChange) error { | |
| 36 | - | for _, change := range changes { | |
| 37 | - | if change.Delete { | |
| 38 | - | return s.err | |
| 39 | - | } | |
| 40 | - | } | |
| 41 | - | return nil | |
| 42 | - | } | |
| 43 | - | func (s failingDeleteStore) Delete([]byte) error { return s.err } | |
| 44 | - | func (failingDeleteStore) Range(func(k, v []byte) bool) {} | |
| 45 | - | func (failingDeleteStore) Sync() error { return nil } | |
| 46 | - | func (failingDeleteStore) Close() error { return nil } | |
| 47 | - | ||
| 48 | 28 | type refusingSnapshotDeleteRepository struct { | |
| 49 | 29 | noOpSnapshotRepository | |
| 50 | 30 | err error | |
| @@ -277,8 +257,8 @@ func TestPickerWaitsForRestoringTargetBeforeSwitching(t *testing.T) { | |||
| 277 | 257 | d.mu.Lock() | |
| 278 | 258 | delete(d.stopped, record.Name) | |
| 279 | 259 | d.sessions[target.id] = target | |
| 260 | + | closeRuntimeRestoreDoneLocked(done) | |
| 280 | 261 | d.mu.Unlock() | |
| 281 | - | closeRuntimeRestoreDone(done) | |
| 282 | 262 | ||
| 283 | 263 | require.NoError(t, <-result) | |
| 284 | 264 | require.Same(t, target, ac.currentSession()) | |
| @@ -294,12 +274,12 @@ func TestRestoreCancellationTransitionsBeforePickerCompletion(t *testing.T) { | |||
| 294 | 274 | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}) | |
| 295 | 275 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | |
| 296 | 276 | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | |
| 297 | - | WithSnapshotRepository(repository, nil)(d) | |
| 277 | + | WithSnapshotRepository(repository)(d) | |
| 298 | 278 | WithCheckpointCoordinator(coordinator)(d) | |
| 299 | 279 | restoreCtx, cancelRestore := context.WithCancel(context.Background()) | |
| 300 | 280 | restored := make(chan struct{}) | |
| 301 | 281 | go func() { | |
| 302 | - | d.restoreCatalogue(restoreCtx, catalogue.Records()) | |
| 282 | + | d.restoreCatalogue(restoreCtx, mustDurableRecords(t, catalogue)) | |
| 303 | 283 | close(restored) | |
| 304 | 284 | }() | |
| 305 | 285 | <-repository.started | |
| @@ -1150,8 +1130,18 @@ func TestPickerKillActiveSessionSnapshotDeleteRefusalReportsOnceAndKeepsPicker(t | |||
| 1150 | 1130 | d, from, ac, sends, releases := newRecentNavigationTestSessions(t) | |
| 1151 | 1131 | defer releaseAll(releases) | |
| 1152 | 1132 | cause := errors.New("snapshot delete refused") | |
| 1153 | - | WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause}, nil)(d) | |
| 1133 | + | WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause})(d) | |
| 1154 | 1134 | target := d.sessions[domain.SessionID("recent")] | |
| 1135 | + | store, _ := newMockStore(t) | |
| 1136 | + | WithStore(store)(d) | |
| 1137 | + | target.mu.Lock() | |
| 1138 | + | if target.incarnation == (domain.IncarnationID{}) { | |
| 1139 | + | target.incarnation = domain.IncarnationID{1} | |
| 1140 | + | } | |
| 1141 | + | record := target.persistRecordLocked(1) | |
| 1142 | + | target.mu.Unlock() | |
| 1143 | + | record.RecoveryState = domain.RecoveryFresh | |
| 1144 | + | require.NoError(t, d.catalogue.Create(record)) | |
| 1155 | 1145 | ||
| 1156 | 1146 | d.enterPicker(from, ac) | |
| 1157 | 1147 | awaitFrame(t, sends, ports.MsgOutput) | |
| @@ -1184,8 +1174,14 @@ func TestPickerKillStoppedSessionPersistDeleteFailureSurfacesNoticeAndKeepsEntry | |||
| 1184 | 1174 | defer release() | |
| 1185 | 1175 | d, from, ac, sends := newManualSessionWithPTYs(t, p) | |
| 1186 | 1176 | cause := errors.New("delete failed") | |
| 1187 | - | WithStore(failingDeleteStore{err: cause})(d) | |
| 1188 | - | d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp/stopped", createdAt: 7} | |
| 1177 | + | store, state := newMockStore(t) | |
| 1178 | + | WithStore(store)(d) | |
| 1179 | + | record := domain.CatalogueRecord{Name: "stopped", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/stopped", CreatedAt: 7, RecoveryState: domain.RecoveryFresh} | |
| 1180 | + | require.NoError(t, d.catalogue.Create(record)) | |
| 1181 | + | state.mu.Lock() | |
| 1182 | + | state.deleteErr = func(string) error { return cause } | |
| 1183 | + | state.mu.Unlock() | |
| 1184 | + | d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp/stopped", createdAt: 7, incarnation: record.IncarnationID, record: record} | |
| 1189 | 1185 | ||
| 1190 | 1186 | d.enterPicker(from, ac) | |
| 1191 | 1187 | awaitFrame(t, sends, ports.MsgOutput) | |
| @@ -1202,12 +1198,9 @@ func TestPickerKillStoppedSessionPersistDeleteFailureSurfacesNoticeAndKeepsEntry | |||
| 1202 | 1198 | toasts := awaitToastCount(t, ac, 1) | |
| 1203 | 1199 | require.Equal(t, domain.NoticePersistDelete, toasts[0].Code) | |
| 1204 | 1200 | ||
| 1205 | - | views, _ := d.pickerViews(from) | |
| 1206 | - | var stillListed bool | |
| 1207 | - | for _, v := range views { | |
| 1208 | - | if v.Stopped && v.Name == "stopped" { | |
| 1209 | - | stillListed = true | |
| 1210 | - | } | |
| 1211 | - | } | |
| 1212 | - | require.True(t, stillListed, "entry must remain listed after refreshPicker when the delete failed") | |
| 1201 | + | d.mu.Lock() | |
| 1202 | + | stopped, retained := d.stopped["stopped"] | |
| 1203 | + | d.mu.Unlock() | |
| 1204 | + | require.True(t, retained, "failed deletion must retain the reserved name") | |
| 1205 | + | require.True(t, stopped.purging, "an uncertain deletion stays fenced from restore") | |
| 1213 | 1206 | } | |
M internal/usecase/daemon/recovery_observability.go +6 -1
| @@ -22,7 +22,12 @@ func (d *Daemon) logSessionDegraded(record domain.CatalogueRecord, reasonCode st | |||
| 22 | 22 | func (d *Daemon) logStartupRecoveryCounts(restoring int) { | |
| 23 | 23 | var records []domain.CatalogueRecord | |
| 24 | 24 | if d.catalogue != nil { | |
| 25 | - | records = d.catalogue.Records() | |
| 25 | + | var err error | |
| 26 | + | records, err = d.catalogue.Records() | |
| 27 | + | if err != nil { | |
| 28 | + | d.log.Error("daemon_startup_catalogue_read_failed", "err", err) | |
| 29 | + | return | |
| 30 | + | } | |
| 26 | 31 | } else { | |
| 27 | 32 | d.mu.Lock() | |
| 28 | 33 | records = make([]domain.CatalogueRecord, 0, len(d.stopped)) | |
M internal/usecase/daemon/render_capture.go +37 -12
| @@ -17,15 +17,21 @@ import ( | |||
| 17 | 17 | // capture/compose/emit transaction holds that attachment's sendMu, so these | |
| 18 | 18 | // buffers cannot be observed or reused before emission completes. | |
| 19 | 19 | type renderCaptureScratch struct { | |
| 20 | - | state capturedRenderState | |
| 21 | - | panes []capturedPaneRenderState | |
| 22 | - | placements []layout.Placement | |
| 23 | - | dividers []layout.Divider | |
| 24 | - | statusTabs []statusTab | |
| 25 | - | mru []recentSession | |
| 26 | - | ranked []rankedRecent | |
| 27 | - | titleIDs []layout.PaneID | |
| 28 | - | receipts []damageReceipt | |
| 20 | + | state capturedRenderState | |
| 21 | + | panes []capturedPaneRenderState | |
| 22 | + | placements []layout.Placement | |
| 23 | + | dividers []layout.Divider | |
| 24 | + | layoutTab *tab | |
| 25 | + | layoutGeneration uint64 | |
| 26 | + | layoutArea domain.Rect | |
| 27 | + | layoutFocus layout.PaneID | |
| 28 | + | layoutFingerprint string | |
| 29 | + | layoutValid bool | |
| 30 | + | statusTabs []statusTab | |
| 31 | + | mru []recentSession | |
| 32 | + | ranked []rankedRecent | |
| 33 | + | titleIDs []layout.PaneID | |
| 34 | + | receipts []damageReceipt | |
| 29 | 35 | } | |
| 30 | 36 | ||
| 31 | 37 | type damageReceipt struct { | |
| @@ -225,9 +231,28 @@ func capturePrimaryRenderState( | |||
| 225 | 231 | ||
| 226 | 232 | tb.mu.Lock() | |
| 227 | 233 | defer tb.mu.Unlock() | |
| 228 | - | layoutSnap := solveTabLayoutLocked(tb) | |
| 229 | - | scratch.placements = append(scratch.placements[:0], layoutSnap.placements...) | |
| 230 | - | scratch.dividers = append(scratch.dividers[:0], layoutSnap.dividers...) | |
| 234 | + | area := domain.Rect{Width: tb.size.Cols, Height: tb.size.Rows} | |
| 235 | + | var focus layout.PaneID | |
| 236 | + | if tb.tree != nil { | |
| 237 | + | focus = tb.tree.Focus | |
| 238 | + | } | |
| 239 | + | layoutSnap := tabLayoutSnapshot{ | |
| 240 | + | area: area, focus: focus, placements: scratch.placements, | |
| 241 | + | dividers: scratch.dividers, fingerprint: scratch.layoutFingerprint, ok: scratch.layoutValid, | |
| 242 | + | } | |
| 243 | + | if scratch.layoutTab != tb || scratch.layoutGeneration != tb.layoutGeneration || scratch.layoutArea != area || scratch.layoutFocus != focus { | |
| 244 | + | layoutSnap = solveTabLayoutLocked(tb) | |
| 245 | + | scratch.placements = append(scratch.placements[:0], layoutSnap.placements...) | |
| 246 | + | scratch.dividers = append(scratch.dividers[:0], layoutSnap.dividers...) | |
| 247 | + | scratch.layoutTab = tb | |
| 248 | + | scratch.layoutGeneration = tb.layoutGeneration | |
| 249 | + | scratch.layoutArea = area | |
| 250 | + | scratch.layoutFocus = focus | |
| 251 | + | scratch.layoutFingerprint = layoutSnap.fingerprint | |
| 252 | + | scratch.layoutValid = layoutSnap.ok | |
| 253 | + | layoutSnap.placements = scratch.placements | |
| 254 | + | layoutSnap.dividers = scratch.dividers | |
| 255 | + | } | |
| 231 | 256 | state := &scratch.state | |
| 232 | 257 | *state = capturedRenderState{ | |
| 233 | 258 | attachment: ac, lease: lease, reset: reset, bars: bars, theme: bars.theme, | |
M internal/usecase/daemon/session.go +263 -126
| @@ -15,7 +15,6 @@ import ( | |||
| 15 | 15 | "time" | |
| 16 | 16 | ||
| 17 | 17 | "github.com/bnema/vev/internal/domain" | |
| 18 | - | "github.com/bnema/vev/internal/persist" | |
| 19 | 18 | "github.com/bnema/vev/internal/ports" | |
| 20 | 19 | "github.com/bnema/vev/internal/usecase/layout" | |
| 21 | 20 | ) | |
| @@ -47,22 +46,31 @@ type session struct { | |||
| 47 | 46 | // tab, or pane locks. The owner performs teardown without holding teardownMu; | |
| 48 | 47 | // duplicate daemon-shutdown callers can therefore bound their ownership wait | |
| 49 | 48 | // with Serve's shared snapshot deadline. | |
| 50 | - | teardownMu sync.Mutex | |
| 51 | - | teardownActive bool | |
| 52 | - | teardownDone chan struct{} | |
| 53 | - | teardownWaiters uint | |
| 54 | - | teardownChanged chan struct{} | |
| 55 | - | lifecycleStopOnce sync.Once | |
| 56 | - | mu sync.Mutex // guards tabs, active, client, clipFiles, and clipboard queue state | |
| 57 | - | themeMu sync.Mutex | |
| 58 | - | tabs []*tab | |
| 59 | - | active int | |
| 60 | - | client *attachedClient | |
| 61 | - | clipboardQueue []clipboardForward | |
| 62 | - | clipboardWorkerRunning bool | |
| 63 | - | renameInProgress bool | |
| 64 | - | cwd string | |
| 65 | - | terminal terminalEnv | |
| 49 | + | teardownMu sync.Mutex | |
| 50 | + | teardownActive bool | |
| 51 | + | teardownDone chan struct{} | |
| 52 | + | teardownWaiters uint | |
| 53 | + | teardownChanged chan struct{} | |
| 54 | + | lifecycleStopOnce sync.Once | |
| 55 | + | mu sync.Mutex // guards tabs, active, client, clipFiles, and clipboard queue state | |
| 56 | + | // metadataPersistMu serializes authority writes and post-I/O rollback. State | |
| 57 | + | // mutation paths must release d.mu and mu before acquiring it. After durable | |
| 58 | + | // I/O completes it may acquire d.mu then mu to reconcile failed revisions. | |
| 59 | + | metadataPersistMu sync.Mutex | |
| 60 | + | metadataVersion uint64 // guarded by mu; every durable metadata snapshot receives a revision | |
| 61 | + | metadataDurableVersion uint64 // guarded by metadataPersistMu | |
| 62 | + | metadataLiveVersion uint64 // rollback cursor guarded by metadataPersistMu | |
| 63 | + | metadataFailedRollbacks map[uint64]func() bool // guarded by metadataPersistMu | |
| 64 | + | themeMu sync.Mutex | |
| 65 | + | tabs []*tab | |
| 66 | + | active int | |
| 67 | + | client *attachedClient | |
| 68 | + | clipboardQueue []clipboardForward | |
| 69 | + | clipboardWorkerRunning bool | |
| 70 | + | renameInProgress bool | |
| 71 | + | renameDone chan struct{} | |
| 72 | + | cwd string | |
| 73 | + | terminal terminalEnv | |
| 66 | 74 | // env is the authoritative immutable environment snapshot for future PTY children. | |
| 67 | 75 | // It is guarded by mu and is always copied on ingress and egress. | |
| 68 | 76 | env []string | |
| @@ -167,12 +175,14 @@ func (d *Daemon) touchMRU(sess *session) { | |||
| 167 | 175 | } | |
| 168 | 176 | seq := d.mruSeq.Add(1) | |
| 169 | 177 | updated := false | |
| 178 | + | var previous uint64 | |
| 170 | 179 | for { | |
| 171 | 180 | old := sess.mruAt.Load() | |
| 172 | 181 | if old >= seq { | |
| 173 | 182 | return | |
| 174 | 183 | } | |
| 175 | 184 | if sess.mruAt.CompareAndSwap(old, seq) { | |
| 185 | + | previous = old | |
| 176 | 186 | updated = true | |
| 177 | 187 | break | |
| 178 | 188 | } | |
| @@ -182,19 +192,49 @@ func (d *Daemon) touchMRU(sess *session) { | |||
| 182 | 192 | } | |
| 183 | 193 | sess.mu.Lock() | |
| 184 | 194 | name := sess.name | |
| 185 | - | incarnation := sess.incarnation | |
| 186 | 195 | ephemeral := sess.ephemeral | |
| 196 | + | var record domain.CatalogueRecord | |
| 197 | + | var version uint64 | |
| 198 | + | if !ephemeral { | |
| 199 | + | record, version = sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) | |
| 200 | + | } | |
| 187 | 201 | sess.mu.Unlock() | |
| 188 | 202 | if !ephemeral { | |
| 189 | - | if err := d.updateCatalogueMetadata(domain.CatalogueMetadataUpdate{Name: name, IncarnationID: incarnation, LastUsedSeq: &seq}); err != nil { | |
| 203 | + | rollback := func() bool { | |
| 204 | + | return sess.mruAt.CompareAndSwap(seq, previous) | |
| 205 | + | } | |
| 206 | + | if err := d.persistSessionMetadata(sess, version, record.MetadataUpdate(), rollback); err != nil { | |
| 190 | 207 | d.log.Warn("touching persisted session recency failed", "err", err, "session", name) | |
| 191 | 208 | } | |
| 192 | 209 | } | |
| 193 | 210 | } | |
| 194 | 211 | ||
| 195 | 212 | func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) { | |
| 213 | + | // Historical focused tests call this helper directly. Acquire mu for those | |
| 214 | + | // callers while preserving the production contract that an owning caller | |
| 215 | + | // receives it still locked on return. | |
| 216 | + | acquiredMu := d.mu.TryLock() | |
| 217 | + | if acquiredMu { | |
| 218 | + | defer d.mu.Unlock() | |
| 219 | + | } | |
| 196 | 220 | env = copyEnvironment(env) | |
| 221 | + | if _, reserved := d.creating[name]; reserved { | |
| 222 | + | return nil, errSessionNameInUse | |
| 223 | + | } | |
| 224 | + | d.creating[name] = struct{}{} | |
| 225 | + | defer delete(d.creating, name) | |
| 197 | 226 | stopped, resuming := d.stopped[name] | |
| 227 | + | var authoritative domain.CatalogueRecord | |
| 228 | + | var authoritativeExists bool | |
| 229 | + | if !ephemeral && resuming && d.persistEnabled { | |
| 230 | + | d.mu.Unlock() | |
| 231 | + | var err error | |
| 232 | + | authoritative, authoritativeExists, err = d.catalogueRecord(name) | |
| 233 | + | d.mu.Lock() | |
| 234 | + | if err != nil { | |
| 235 | + | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't read session catalogue", err) | |
| 236 | + | } | |
| 237 | + | } | |
| 198 | 238 | var createdAt int64 | |
| 199 | 239 | var incarnation domain.IncarnationID | |
| 200 | 240 | if !ephemeral { | |
| @@ -203,9 +243,9 @@ func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz | |||
| 203 | 243 | // the persisted identity rather than receive a fresh timestamp. | |
| 204 | 244 | createdAt = stopped.createdAt | |
| 205 | 245 | incarnation = stopped.incarnation | |
| 206 | - | if authoritative, ok := d.catalogueRecord(name); ok { | |
| 207 | - | incarnation = authoritative.IncarnationID | |
| 246 | + | if authoritativeExists { | |
| 208 | 247 | createdAt = authoritative.CreatedAt | |
| 248 | + | incarnation = authoritative.IncarnationID | |
| 209 | 249 | } | |
| 210 | 250 | if createdAt > d.lastAllocatedCreatedAt { | |
| 211 | 251 | d.lastAllocatedCreatedAt = createdAt | |
| @@ -263,6 +303,9 @@ func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz | |||
| 263 | 303 | if !ephemeral && resuming { | |
| 264 | 304 | lastUsedSeq = stopped.lastUsedSeq | |
| 265 | 305 | } | |
| 306 | + | if lastUsedSeq == 0 { | |
| 307 | + | lastUsedSeq = d.mruSeq.Add(1) | |
| 308 | + | } | |
| 266 | 309 | sess := &session{ | |
| 267 | 310 | id: id, | |
| 268 | 311 | name: name, | |
| @@ -286,34 +329,50 @@ func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz | |||
| 286 | 329 | sess.snapEligible.Store(!ephemeral && name != "") | |
| 287 | 330 | if !ephemeral { | |
| 288 | 331 | if d.persistEnabled { | |
| 289 | - | record := persist.Record{Name: name, IncarnationID: incarnation, Cwd: cwd, CreatedAt: createdAt, UpdatedAt: createdAt, LastUsedSeq: lastUsedSeq, TabNames: names, RecoveryState: domain.RecoveryFresh} | |
| 332 | + | record := domain.CatalogueRecord{Name: name, IncarnationID: incarnation, Cwd: cwd, CreatedAt: createdAt, UpdatedAt: createdAt, LastUsedSeq: lastUsedSeq, TabNames: names, RecoveryState: domain.RecoveryFresh} | |
| 333 | + | if d.lifecycleRecovery == nil { | |
| 334 | + | closeTabs(tabs) | |
| 335 | + | cancel() | |
| 336 | + | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", errors.New("durable session authority is not configured")) | |
| 337 | + | } | |
| 338 | + | // The name reservation remains visible while durable authority I/O runs, | |
| 339 | + | // but daemon/session mutexes are released so storage cannot block routing. | |
| 340 | + | d.mu.Unlock() | |
| 290 | 341 | var err error | |
| 291 | - | if resuming { | |
| 292 | - | if _, ok := d.catalogueRecord(name); ok { | |
| 293 | - | err = d.updateCatalogueMetadata(record.MetadataUpdate()) | |
| 294 | - | } else if d.lifecycleRecovery != nil { | |
| 295 | - | record, err = d.lifecycleRecovery.Create(context.Background(), record) | |
| 296 | - | } else { | |
| 297 | - | err = d.createCatalogueRecord(record) | |
| 298 | - | } | |
| 299 | - | } else if d.lifecycleRecovery != nil { | |
| 300 | - | record, err = d.lifecycleRecovery.Create(context.Background(), record) | |
| 342 | + | if resuming && authoritativeExists { | |
| 343 | + | err = d.updateCatalogueMetadata(record.MetadataUpdate()) | |
| 301 | 344 | } else { | |
| 302 | - | err = d.createCatalogueRecord(record) | |
| 345 | + | record, err = d.lifecycleRecovery.Create(d.serveCtx, record) | |
| 303 | 346 | } | |
| 347 | + | d.mu.Lock() | |
| 304 | 348 | if err != nil { | |
| 305 | 349 | closeTabs(tabs) | |
| 306 | 350 | cancel() | |
| 307 | 351 | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", err) | |
| 308 | 352 | } | |
| 353 | + | if d.closing { | |
| 354 | + | // Creation lost its race with shutdown. Roll back a newly committed | |
| 355 | + | // authority record without holding mu; resumed metadata remains owned | |
| 356 | + | // by the existing stopped lifecycle. | |
| 357 | + | if !resuming { | |
| 358 | + | d.mu.Unlock() | |
| 359 | + | rollbackCtx, rollbackCancel := context.WithTimeout(context.WithoutCancel(d.serveCtx), snapshotFinalFlushTimeout) | |
| 360 | + | rollbackErr := d.lifecycleRecovery.Delete(rollbackCtx, name) | |
| 361 | + | rollbackCancel() | |
| 362 | + | d.mu.Lock() | |
| 363 | + | if rollbackErr != nil { | |
| 364 | + | d.log.Warn("rolling back session creation during shutdown failed", "session", name, "err", rollbackErr) | |
| 365 | + | } | |
| 366 | + | } | |
| 367 | + | closeTabs(tabs) | |
| 368 | + | cancel() | |
| 369 | + | return nil, errors.New("daemon is shutting down") | |
| 370 | + | } | |
| 309 | 371 | sess.incarnation = record.IncarnationID | |
| 310 | 372 | } | |
| 311 | 373 | delete(d.stopped, name) | |
| 312 | 374 | } | |
| 313 | 375 | d.sessions[id] = sess | |
| 314 | - | if lastUsedSeq == 0 { | |
| 315 | - | d.touchMRU(sess) | |
| 316 | - | } | |
| 317 | 376 | d.log.Info("session created", "session", name, "id", id, "ephemeral", ephemeral) | |
| 318 | 377 | for i, tb := range tabs { | |
| 319 | 378 | d.log.Info("tab created", "session", name, "tab", i) | |
| @@ -445,25 +504,41 @@ func (d *Daemon) createTab(sess *session, sz domain.Size) error { | |||
| 445 | 504 | sess.tabs = append(sess.tabs, tb) | |
| 446 | 505 | sess.active = len(sess.tabs) - 1 | |
| 447 | 506 | tabIndex := sess.active | |
| 448 | - | record := sess.persistRecordLocked(time.Now().UnixNano()) | |
| 507 | + | record, metadataVersion := sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) | |
| 449 | 508 | ephemeral := sess.ephemeral | |
| 509 | + | sess.mu.Unlock() | |
| 510 | + | d.mu.Unlock() | |
| 450 | 511 | if !ephemeral { | |
| 451 | - | if err := d.updateCatalogueMetadata(record.MetadataUpdate()); err != nil { | |
| 452 | - | sess.tabs = sess.tabs[:len(sess.tabs)-1] | |
| 453 | - | sess.active = oldActive | |
| 512 | + | rollback := func() bool { | |
| 513 | + | d.mu.Lock() | |
| 514 | + | defer d.mu.Unlock() | |
| 515 | + | if d.sessions[sess.id] != sess { | |
| 516 | + | return true | |
| 517 | + | } | |
| 518 | + | sess.mu.Lock() | |
| 519 | + | if !slices.Contains(sess.tabs, tb) { | |
| 520 | + | sess.mu.Unlock() | |
| 521 | + | return true | |
| 522 | + | } | |
| 523 | + | if sess.tabs[len(sess.tabs)-1] != tb { | |
| 524 | + | sess.mu.Unlock() | |
| 525 | + | return false | |
| 526 | + | } | |
| 527 | + | sess.tabs = slices.DeleteFunc(sess.tabs, func(candidate *tab) bool { return candidate == tb }) | |
| 528 | + | sess.active = min(oldActive, len(sess.tabs)-1) | |
| 529 | + | sess.mu.Unlock() | |
| 454 | 530 | if tb.cancel != nil { | |
| 455 | 531 | tb.cancel() | |
| 456 | 532 | } | |
| 457 | - | sess.mu.Unlock() | |
| 458 | - | d.mu.Unlock() | |
| 459 | 533 | _ = pty.Close() | |
| 534 | + | return true | |
| 535 | + | } | |
| 536 | + | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |
| 460 | 537 | return err | |
| 461 | 538 | } | |
| 462 | 539 | } | |
| 463 | - | sess.mu.Unlock() | |
| 464 | 540 | d.log.Info("tab created", "session", name, "tab", tabIndex) | |
| 465 | 541 | d.startTabGoroutines(sess, tb) | |
| 466 | - | d.mu.Unlock() | |
| 467 | 542 | d.activateTab(sess, tb) | |
| 468 | 543 | markSnapshotDirty(sess) | |
| 469 | 544 | return nil | |
| @@ -686,51 +761,43 @@ func (d *Daemon) renameSession(sess *session, name string) error { | |||
| 686 | 761 | } | |
| 687 | 762 | } | |
| 688 | 763 | lastUsedSeq := sess.mruAt.Load() | |
| 689 | - | record := sess.persistRecordLocked(d.nowUnixNano()) | |
| 764 | + | record := sess.persistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) | |
| 690 | 765 | record.Name = name | |
| 691 | 766 | record.CreatedAt = createdAt | |
| 692 | 767 | record.LastUsedSeq = lastUsedSeq | |
| 693 | 768 | sess.renameInProgress = true | |
| 769 | + | sess.renameDone = make(chan struct{}) | |
| 694 | 770 | sess.mu.Unlock() | |
| 695 | 771 | d.mu.Unlock() | |
| 696 | 772 | ||
| 697 | 773 | rollback := func(err error) error { | |
| 698 | 774 | sess.mu.Lock() | |
| 699 | 775 | sess.renameInProgress = false | |
| 776 | + | if sess.renameDone != nil { | |
| 777 | + | close(sess.renameDone) | |
| 778 | + | sess.renameDone = nil | |
| 779 | + | } | |
| 700 | 780 | sess.mu.Unlock() | |
| 701 | 781 | return err | |
| 702 | 782 | } | |
| 703 | 783 | // Durable named renames are one atomic catalogue batch. The incarnation-keyed | |
| 704 | 784 | // snapshot namespace is intentionally untouched. | |
| 705 | 785 | if (wasEphemeral || oldName != name) && d.persistEnabled { | |
| 706 | - | if d.lifecycleRecovery != nil { | |
| 786 | + | if d.lifecycleRecovery == nil { | |
| 787 | + | return rollback(errors.New("durable session authority is not configured")) | |
| 788 | + | } | |
| 789 | + | { | |
| 707 | 790 | var committed domain.CatalogueRecord | |
| 708 | 791 | var err error | |
| 709 | 792 | if wasEphemeral { | |
| 710 | - | committed, err = d.lifecycleRecovery.Create(context.Background(), record) | |
| 793 | + | committed, err = d.lifecycleRecovery.Create(d.serveCtx, record) | |
| 711 | 794 | } else { | |
| 712 | - | committed, err = d.lifecycleRecovery.Rename(context.Background(), oldName, name) | |
| 795 | + | committed, err = d.lifecycleRecovery.Rename(d.serveCtx, oldName, name) | |
| 713 | 796 | } | |
| 714 | 797 | if err != nil { | |
| 715 | 798 | return rollback(err) | |
| 716 | 799 | } | |
| 717 | 800 | record = committed | |
| 718 | - | } else if wasEphemeral { | |
| 719 | - | if err := d.createCatalogueRecord(record); err != nil { | |
| 720 | - | return rollback(err) | |
| 721 | - | } | |
| 722 | - | } else { | |
| 723 | - | if current, ok := d.catalogueRecord(oldName); ok { | |
| 724 | - | current.Name = name | |
| 725 | - | current.Cwd = record.Cwd | |
| 726 | - | current.UpdatedAt = record.UpdatedAt | |
| 727 | - | current.LastUsedSeq = record.LastUsedSeq | |
| 728 | - | current.TabNames = append([]string(nil), record.TabNames...) | |
| 729 | - | record = current | |
| 730 | - | } | |
| 731 | - | if err := d.catalogue.Rename(oldName, record); err != nil { | |
| 732 | - | return rollback(err) | |
| 733 | - | } | |
| 734 | 801 | } | |
| 735 | 802 | } | |
| 736 | 803 | ||
| @@ -745,6 +812,10 @@ func (d *Daemon) renameSession(sess *session, name string) error { | |||
| 745 | 812 | sess.incarnation = record.IncarnationID | |
| 746 | 813 | sess.ephemeral = false | |
| 747 | 814 | sess.renameInProgress = false | |
| 815 | + | if sess.renameDone != nil { | |
| 816 | + | close(sess.renameDone) | |
| 817 | + | sess.renameDone = nil | |
| 818 | + | } | |
| 748 | 819 | sess.snapEligible.Store(name != "") | |
| 749 | 820 | sess.mu.Unlock() | |
| 750 | 821 | d.mu.Unlock() | |
| @@ -774,22 +845,42 @@ func (d *Daemon) renameTab(sess *session, tb *tab, name string) error { | |||
| 774 | 845 | } | |
| 775 | 846 | oldName := tb.name | |
| 776 | 847 | tb.name = name | |
| 777 | - | record := sess.persistRecordLocked(time.Now().UnixNano()) | |
| 848 | + | record, metadataVersion := sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) | |
| 778 | 849 | ephemeral := sess.ephemeral | |
| 779 | - | if !ephemeral { | |
| 780 | - | if err := d.updateCatalogueMetadata(record.MetadataUpdate()); err != nil { | |
| 781 | - | tb.name = oldName | |
| 782 | - | sess.mu.Unlock() | |
| 783 | - | d.mu.Unlock() | |
| 784 | - | return err | |
| 785 | - | } | |
| 786 | - | } | |
| 787 | 850 | sess.mu.Unlock() | |
| 788 | 851 | d.mu.Unlock() | |
| 852 | + | if ephemeral { | |
| 853 | + | return nil | |
| 854 | + | } | |
| 855 | + | rollback := func() bool { | |
| 856 | + | d.mu.Lock() | |
| 857 | + | defer d.mu.Unlock() | |
| 858 | + | if d.sessions[sess.id] != sess { | |
| 859 | + | return true | |
| 860 | + | } | |
| 861 | + | sess.mu.Lock() | |
| 862 | + | defer sess.mu.Unlock() | |
| 863 | + | if !slices.Contains(sess.tabs, tb) { | |
| 864 | + | return true | |
| 865 | + | } | |
| 866 | + | if tb.name != name { | |
| 867 | + | return false | |
| 868 | + | } | |
| 869 | + | tb.name = oldName | |
| 870 | + | return true | |
| 871 | + | } | |
| 872 | + | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |
| 873 | + | return err | |
| 874 | + | } | |
| 789 | 875 | return nil | |
| 790 | 876 | } | |
| 791 | 877 | ||
| 792 | - | func (s *session) persistRecordLocked(updatedAt int64) persist.Record { | |
| 878 | + | func (s *session) nextPersistRecordLocked(updatedAt int64) (domain.CatalogueRecord, uint64) { | |
| 879 | + | s.metadataVersion++ | |
| 880 | + | return s.persistRecordLocked(updatedAt), s.metadataVersion | |
| 881 | + | } | |
| 882 | + | ||
| 883 | + | func (s *session) persistRecordLocked(updatedAt int64) domain.CatalogueRecord { | |
| 793 | 884 | createdAt := s.createdAt | |
| 794 | 885 | tabNames := make([]string, len(s.tabs)) | |
| 795 | 886 | lastCustom := -1 | |
| @@ -804,7 +895,7 @@ func (s *session) persistRecordLocked(updatedAt int64) persist.Record { | |||
| 804 | 895 | } else { | |
| 805 | 896 | tabNames = tabNames[:lastCustom+1] | |
| 806 | 897 | } | |
| 807 | - | return persist.Record{Name: s.name, IncarnationID: s.incarnation, Cwd: s.cwd, CreatedAt: createdAt, UpdatedAt: updatedAt, LastUsedSeq: s.mruAt.Load(), TabNames: tabNames, RecoveryState: domain.RecoveryFresh} | |
| 898 | + | return domain.CatalogueRecord{Name: s.name, IncarnationID: s.incarnation, Cwd: s.cwd, CreatedAt: createdAt, UpdatedAt: updatedAt, LastUsedSeq: s.mruAt.Load(), TabNames: tabNames, RecoveryState: domain.RecoveryFresh} | |
| 808 | 899 | } | |
| 809 | 900 | ||
| 810 | 901 | func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | |
| @@ -843,8 +934,10 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | |||
| 843 | 934 | return nil | |
| 844 | 935 | } | |
| 845 | 936 | ringing := tb.attention | |
| 937 | + | oldActive := sess.active | |
| 846 | 938 | wasActive := idx == sess.active | |
| 847 | 939 | sess.tabs = append(sess.tabs[:idx], sess.tabs[idx+1:]...) | |
| 940 | + | tabsAfterClose := len(sess.tabs) | |
| 848 | 941 | if sess.active >= len(sess.tabs) { | |
| 849 | 942 | sess.active = len(sess.tabs) - 1 | |
| 850 | 943 | } else if idx < sess.active { | |
| @@ -853,15 +946,34 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | |||
| 853 | 946 | destination := sess.tabs[sess.active] | |
| 854 | 947 | ac := sess.client | |
| 855 | 948 | name := sess.name | |
| 856 | - | record := sess.persistRecordLocked(time.Now().UnixNano()) | |
| 949 | + | record, metadataVersion := sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) | |
| 857 | 950 | ephemeral := sess.ephemeral | |
| 951 | + | sess.mu.Unlock() | |
| 952 | + | d.mu.Unlock() | |
| 858 | 953 | if !ephemeral { | |
| 859 | - | if err := d.updateCatalogueMetadata(record.MetadataUpdate()); err != nil { | |
| 954 | + | rollback := func() bool { | |
| 955 | + | d.mu.Lock() | |
| 956 | + | defer d.mu.Unlock() | |
| 957 | + | if d.sessions[sess.id] != sess { | |
| 958 | + | return true | |
| 959 | + | } | |
| 960 | + | sess.mu.Lock() | |
| 961 | + | defer sess.mu.Unlock() | |
| 962 | + | if slices.Contains(sess.tabs, tb) { | |
| 963 | + | return true | |
| 964 | + | } | |
| 965 | + | if len(sess.tabs) != tabsAfterClose { | |
| 966 | + | return false | |
| 967 | + | } | |
| 968 | + | sess.tabs = slices.Insert(sess.tabs, min(idx, len(sess.tabs)), tb) | |
| 969 | + | sess.active = min(oldActive, len(sess.tabs)-1) | |
| 970 | + | return true | |
| 971 | + | } | |
| 972 | + | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |
| 860 | 973 | d.log.Warn("persisting closed tab failed", "err", err, "session", name) | |
| 974 | + | return err | |
| 861 | 975 | } | |
| 862 | 976 | } | |
| 863 | - | sess.mu.Unlock() | |
| 864 | - | d.mu.Unlock() | |
| 865 | 977 | d.log.Info("tab closed", "session", name) | |
| 866 | 978 | markSnapshotDirty(sess) | |
| 867 | 979 | ||
| @@ -900,60 +1012,48 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | |||
| 900 | 1012 | // beginSnapshotPurge commits a logical named-session kill before its live | |
| 901 | 1013 | // identity or persisted metadata is removed. It never runs under daemon or | |
| 902 | 1014 | // session locks. | |
| 903 | - | func (d *Daemon) beginSnapshotPurge(name string, incarnation domain.IncarnationID) error { | |
| 904 | - | if d.lifecycleRecovery != nil { | |
| 905 | - | // The lifecycle coordinator performs the complete ordered protocol after | |
| 906 | - | // snapshot writers have joined. | |
| 907 | - | return nil | |
| 908 | - | } | |
| 909 | - | if d.snapshotRepository == nil || name == "" { | |
| 910 | - | return nil | |
| 1015 | + | func (d *Daemon) beginSnapshotPurge(_ string, _ domain.IncarnationID) error { | |
| 1016 | + | if d.persistEnabled && d.lifecycleRecovery == nil { | |
| 1017 | + | return errors.New("durable session authority is not configured") | |
| 911 | 1018 | } | |
| 912 | - | return d.snapshotRepository.WriteDeletionTombstone(context.Background(), domain.DeletionTombstone{Name: name, IncarnationID: incarnation}) | |
| 1019 | + | // The lifecycle coordinator performs the complete ordered protocol after | |
| 1020 | + | // snapshot writers have joined. | |
| 1021 | + | return nil | |
| 913 | 1022 | } | |
| 914 | 1023 | ||
| 915 | 1024 | // finishSnapshotPurge deletes both independently durable snapshot sources, | |
| 916 | 1025 | // then metadata, and only then clears the tombstone. Any failure deliberately | |
| 917 | 1026 | // leaves the marker in place so startup cannot restore or import the name and | |
| 918 | 1027 | // a later live/offline kill can retry the idempotent source deletes. | |
| 919 | - | func (d *Daemon) finishSnapshotPurge(name string, incarnation domain.IncarnationID) error { | |
| 920 | - | if d.lifecycleRecovery != nil { | |
| 921 | - | return d.lifecycleRecovery.Delete(context.Background(), name) | |
| 922 | - | } | |
| 923 | - | if d.snapshotRepository == nil { | |
| 924 | - | return d.deleteCatalogueRecord(name) | |
| 1028 | + | func (d *Daemon) finishSnapshotPurge(ctx context.Context, name string, _ domain.IncarnationID) error { | |
| 1029 | + | if !d.persistEnabled { | |
| 1030 | + | return nil | |
| 925 | 1031 | } | |
| 926 | - | if err := d.snapshotRepository.QuarantineDeletionSources(context.Background(), domain.DeletionTombstone{Name: name, IncarnationID: incarnation}, false); err != nil { | |
| 927 | - | return err | |
| 1032 | + | if d.lifecycleRecovery == nil { | |
| 1033 | + | return errors.New("durable session authority is not configured") | |
| 928 | 1034 | } | |
| 929 | - | if err := d.deleteCatalogueRecord(name); err != nil { | |
| 930 | - | return err | |
| 1035 | + | if ctx == nil { | |
| 1036 | + | ctx = d.serveCtx | |
| 931 | 1037 | } | |
| 932 | - | return d.snapshotRepository.DeleteDeletionTombstone(context.Background(), incarnation) | |
| 1038 | + | purgeCtx, cancel := context.WithTimeout(ctx, snapshotFinalFlushTimeout) | |
| 1039 | + | defer cancel() | |
| 1040 | + | return d.lifecycleRecovery.Delete(purgeCtx, name) | |
| 933 | 1041 | } | |
| 934 | 1042 | ||
| 935 | 1043 | // retryStoppedPurge completes a previously closed session's durable purge. | |
| 936 | 1044 | // It keeps the stopped record hidden until all sources and metadata cross their | |
| 937 | 1045 | // deletion boundaries. | |
| 938 | 1046 | func (d *Daemon) retryStoppedPurge(name string) error { | |
| 1047 | + | return d.retryStoppedPurgeContext(d.serveCtx, name) | |
| 1048 | + | } | |
| 1049 | + | ||
| 1050 | + | func (d *Daemon) retryStoppedPurgeContext(ctx context.Context, name string) error { | |
| 939 | 1051 | d.mu.Lock() | |
| 940 | 1052 | stopped, ok := d.stopped[name] | |
| 941 | 1053 | if !ok { | |
| 942 | 1054 | d.mu.Unlock() | |
| 943 | 1055 | return nil | |
| 944 | 1056 | } | |
| 945 | - | // Without a repository there is no restore/import source to fence, so the | |
| 946 | - | // stopped record retains the metadata-only deletion behavior. | |
| 947 | - | if d.snapshotRepository == nil { | |
| 948 | - | d.mu.Unlock() | |
| 949 | - | if err := d.deleteCatalogueRecord(name); err != nil { | |
| 950 | - | return err | |
| 951 | - | } | |
| 952 | - | d.mu.Lock() | |
| 953 | - | delete(d.stopped, name) | |
| 954 | - | d.mu.Unlock() | |
| 955 | - | return nil | |
| 956 | - | } | |
| 957 | 1057 | stopped.purging = true | |
| 958 | 1058 | d.stopped[name] = stopped | |
| 959 | 1059 | d.mu.Unlock() | |
| @@ -961,7 +1061,7 @@ func (d *Daemon) retryStoppedPurge(name string) error { | |||
| 961 | 1061 | if err := d.beginSnapshotPurge(name, stopped.incarnation); err != nil { | |
| 962 | 1062 | return err | |
| 963 | 1063 | } | |
| 964 | - | if err := d.finishSnapshotPurge(name, stopped.incarnation); err != nil { | |
| 1064 | + | if err := d.finishSnapshotPurge(ctx, name, stopped.incarnation); err != nil { | |
| 965 | 1065 | return err | |
| 966 | 1066 | } | |
| 967 | 1067 | d.mu.Lock() | |
| @@ -1089,6 +1189,23 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu | |||
| 1089 | 1189 | } | |
| 1090 | 1190 | defer sess.finishTeardown() | |
| 1091 | 1191 | ||
| 1192 | + | // A rename reserves durable identity while coordinator I/O runs unlocked. | |
| 1193 | + | // Teardown waits for that reservation before selecting the purge name. | |
| 1194 | + | sess.mu.Lock() | |
| 1195 | + | renameDone := sess.renameDone | |
| 1196 | + | sess.mu.Unlock() | |
| 1197 | + | if renameDone != nil { | |
| 1198 | + | if deadline == nil { | |
| 1199 | + | <-renameDone | |
| 1200 | + | } else { | |
| 1201 | + | select { | |
| 1202 | + | case <-renameDone: | |
| 1203 | + | case <-deadline.Done(): | |
| 1204 | + | return context.DeadlineExceeded | |
| 1205 | + | } | |
| 1206 | + | } | |
| 1207 | + | } | |
| 1208 | + | ||
| 1092 | 1209 | d.mu.Lock() | |
| 1093 | 1210 | current := d.sessions[sess.id] | |
| 1094 | 1211 | d.mu.Unlock() | |
| @@ -1106,13 +1223,14 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu | |||
| 1106 | 1223 | sess.mu.Lock() | |
| 1107 | 1224 | isEphemeral := sess.ephemeral | |
| 1108 | 1225 | name := sess.name | |
| 1226 | + | incarnation := sess.incarnation | |
| 1109 | 1227 | sess.mu.Unlock() | |
| 1110 | 1228 | // Commit the tombstone before changing live identity or metadata. A | |
| 1111 | 1229 | // repository delete must never race an older publication that could recreate | |
| 1112 | 1230 | // the just-quarantined name. Do not hold daemon or session locks while | |
| 1113 | 1231 | // tombstoning or joining: a repository implementation may block on I/O. | |
| 1114 | 1232 | if purge && !isEphemeral { | |
| 1115 | - | if err := d.beginSnapshotPurge(name, sess.incarnation); err != nil { | |
| 1233 | + | if err := d.beginSnapshotPurge(name, incarnation); err != nil { | |
| 1116 | 1234 | return err | |
| 1117 | 1235 | } | |
| 1118 | 1236 | if d.snapshotRepository != nil { | |
| @@ -1188,7 +1306,7 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu | |||
| 1188 | 1306 | ephemeral := sess.ephemeral | |
| 1189 | 1307 | sess.mu.Unlock() | |
| 1190 | 1308 | if !ephemeral { | |
| 1191 | - | stopped := stoppedSession{name: stoppedName, cwd: stoppedCwd, createdAt: createdAt, incarnation: sess.incarnation, lastUsedSeq: sess.mruAt.Load(), tabNames: tabNames, purging: purge} | |
| 1309 | + | stopped := stoppedSession{name: stoppedName, cwd: stoppedCwd, createdAt: createdAt, incarnation: incarnation, lastUsedSeq: sess.mruAt.Load(), tabNames: tabNames, purging: purge} | |
| 1192 | 1310 | d.stopped[stoppedName] = stopped | |
| 1193 | 1311 | } | |
| 1194 | 1312 | empty := len(d.sessions) == 0 | |
| @@ -1248,7 +1366,7 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu | |||
| 1248 | 1366 | // tombstone and hidden stopped record when any source or metadata operation | |
| 1249 | 1367 | // fails; retryStoppedPurge (including a repeated live kill) resumes safely. | |
| 1250 | 1368 | if !ephemeral && purge { | |
| 1251 | - | if err := d.finishSnapshotPurge(stoppedName, sess.incarnation); err != nil { | |
| 1369 | + | if err := d.finishSnapshotPurge(d.serveCtx, stoppedName, incarnation); err != nil { | |
| 1252 | 1370 | purgeErr = errors.Join(purgeErr, err) | |
| 1253 | 1371 | d.log.Warn("finishing session snapshot purge failed", "err", err, "session", stoppedName) | |
| 1254 | 1372 | } else { | |
| @@ -1302,8 +1420,11 @@ func (d *Daemon) nameLiveOrStoppedLocked(name string) bool { | |||
| 1302 | 1420 | if d.findByNameLocked(name) != nil { | |
| 1303 | 1421 | return true | |
| 1304 | 1422 | } | |
| 1305 | - | _, ok := d.stopped[name] | |
| 1306 | - | return ok | |
| 1423 | + | if _, ok := d.stopped[name]; ok { | |
| 1424 | + | return true | |
| 1425 | + | } | |
| 1426 | + | _, reserved := d.creating[name] | |
| 1427 | + | return reserved | |
| 1307 | 1428 | } | |
| 1308 | 1429 | ||
| 1309 | 1430 | func (d *Daemon) cwdSampler(ctx context.Context) { | |
| @@ -1363,19 +1484,38 @@ func (d *Daemon) refreshSessionCwd(sess *session) { | |||
| 1363 | 1484 | d.mu.Unlock() | |
| 1364 | 1485 | return | |
| 1365 | 1486 | } | |
| 1487 | + | oldCwd := sess.cwd | |
| 1366 | 1488 | sess.cwd = cwd | |
| 1367 | 1489 | name := sess.name | |
| 1368 | - | incarnation := sess.incarnation | |
| 1369 | 1490 | ephemeral := sess.ephemeral | |
| 1491 | + | var record domain.CatalogueRecord | |
| 1492 | + | var metadataVersion uint64 | |
| 1493 | + | if !ephemeral { | |
| 1494 | + | record, metadataVersion = sess.nextPersistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) | |
| 1495 | + | } | |
| 1370 | 1496 | sess.mu.Unlock() | |
| 1497 | + | d.mu.Unlock() | |
| 1371 | 1498 | if !ephemeral { | |
| 1372 | - | updatedAt := time.Now().UnixNano() | |
| 1373 | - | if err := d.updateCatalogueMetadata(domain.CatalogueMetadataUpdate{Name: name, IncarnationID: incarnation, Cwd: &cwd, UpdatedAt: &updatedAt}); err != nil { | |
| 1499 | + | rollback := func() bool { | |
| 1500 | + | d.mu.Lock() | |
| 1501 | + | defer d.mu.Unlock() | |
| 1502 | + | if d.sessions[sess.id] != sess { | |
| 1503 | + | return true | |
| 1504 | + | } | |
| 1505 | + | sess.mu.Lock() | |
| 1506 | + | defer sess.mu.Unlock() | |
| 1507 | + | if sess.cwd != cwd { | |
| 1508 | + | return false | |
| 1509 | + | } | |
| 1510 | + | sess.cwd = oldCwd | |
| 1511 | + | return true | |
| 1512 | + | } | |
| 1513 | + | if err := d.persistSessionMetadata(sess, metadataVersion, record.MetadataUpdate(), rollback); err != nil { | |
| 1374 | 1514 | d.log.Warn("touching persisted session cwd failed", "err", err, "session", name) | |
| 1515 | + | return | |
| 1375 | 1516 | } | |
| 1376 | 1517 | markSnapshotDirty(sess) | |
| 1377 | 1518 | } | |
| 1378 | - | d.mu.Unlock() | |
| 1379 | 1519 | } | |
| 1380 | 1520 | ||
| 1381 | 1521 | // childEnv retains the daemon-environment helper for daemon-local legacy callers. | |
| @@ -1389,11 +1529,8 @@ func copyEnvironment(env []string) []string { | |||
| 1389 | 1529 | } | |
| 1390 | 1530 | ||
| 1391 | 1531 | func environmentEntry(entry string) (name, value string, ok bool) { | |
| 1392 | - | i := strings.IndexByte(entry, '=') | |
| 1393 | - | if i < 0 { | |
| 1394 | - | return "", "", false | |
| 1395 | - | } | |
| 1396 | - | return entry[:i], entry[i+1:], true | |
| 1532 | + | name, value, ok = strings.Cut(entry, "=") | |
| 1533 | + | return name, value, ok | |
| 1397 | 1534 | } | |
| 1398 | 1535 | ||
| 1399 | 1536 | func (d *Daemon) ptyCommand(env []string) (string, []string) { | |
M internal/usecase/daemon/session_purge_control_test.go +5 -20
| @@ -4,40 +4,25 @@ import ( | |||
| 4 | 4 | "errors" | |
| 5 | 5 | "testing" | |
| 6 | 6 | ||
| 7 | - | "github.com/stretchr/testify/mock" | |
| 8 | 7 | "github.com/stretchr/testify/require" | |
| 9 | 8 | ||
| 10 | - | "github.com/bnema/vev/internal/ports" | |
| 9 | + | "github.com/bnema/vev/internal/domain" | |
| 11 | 10 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | |
| 12 | 11 | ) | |
| 13 | 12 | ||
| 14 | - | func TestHandleKillStoppedUsesDurableBothSourcePurge(t *testing.T) { | |
| 15 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 16 | - | repository := &retryablePurgeRepository{legacyErr: errors.New("legacy delete failed")} | |
| 17 | - | WithSnapshotRepository(repository, repository)(d) | |
| 18 | - | d.stopped["work"] = stoppedSession{name: "work"} | |
| 19 | - | ||
| 20 | - | transport := portsmocks.NewMockTransport(t) | |
| 21 | - | transport.EXPECT().Send(mock.Anything).Return(nil).Once() | |
| 22 | - | transport.EXPECT().Close().Return(nil).Once() | |
| 23 | - | d.handleKill(transport, ports.Frame{Type: ports.MsgKill, Payload: ports.MarshalKill(ports.Kill{Name: "work"})}) | |
| 24 | - | ||
| 25 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy"}, repository.calls) | |
| 26 | - | require.True(t, repository.tombstoned["work"]) | |
| 27 | - | require.True(t, d.stopped["work"].purging) | |
| 28 | - | } | |
| 29 | - | ||
| 30 | 13 | func TestStoppedPurgeMetadataFailureRemainsFencedForRetry(t *testing.T) { | |
| 31 | 14 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 32 | 15 | repository := &retryablePurgeRepository{} | |
| 33 | - | WithSnapshotRepository(repository, repository)(d) | |
| 16 | + | WithSnapshotRepository(repository)(d) | |
| 34 | 17 | store, state := newMockStore(t) | |
| 35 | 18 | WithStore(store)(d) | |
| 19 | + | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, RecoveryState: domain.RecoveryFresh} | |
| 20 | + | require.NoError(t, d.catalogue.Create(record)) | |
| 36 | 21 | metadataErr := errors.New("metadata delete failed") | |
| 37 | 22 | state.mu.Lock() | |
| 38 | 23 | state.deleteErr = func(string) error { return metadataErr } | |
| 39 | 24 | state.mu.Unlock() | |
| 40 | - | d.stopped["work"] = stoppedSession{name: "work"} | |
| 25 | + | d.stopped["work"] = stoppedSession{name: "work", incarnation: record.IncarnationID, record: record} | |
| 41 | 26 | ||
| 42 | 27 | require.ErrorIs(t, d.retryStoppedPurge("work"), metadataErr) | |
| 43 | 28 | require.True(t, repository.tombstoned["work"]) | |
M internal/usecase/daemon/session_purge_retry_test.go +20 -4
| @@ -18,12 +18,15 @@ type retryablePurgeRepository struct { | |||
| 18 | 18 | legacyErr error | |
| 19 | 19 | tombstoned map[string]bool | |
| 20 | 20 | calls []string | |
| 21 | + | deadlineScoped []bool | |
| 21 | 22 | } | |
| 22 | 23 | ||
| 23 | 24 | func (r *retryablePurgeRepository) Publish(context.Context, ports.SnapshotPublication) error { | |
| 24 | 25 | return nil | |
| 25 | 26 | } | |
| 26 | - | func (r *retryablePurgeRepository) WriteDeletionTombstone(_ context.Context, tombstone domain.DeletionTombstone) error { | |
| 27 | + | func (r *retryablePurgeRepository) WriteDeletionTombstone(ctx context.Context, tombstone domain.DeletionTombstone) error { | |
| 28 | + | _, hasDeadline := ctx.Deadline() | |
| 29 | + | r.deadlineScoped = append(r.deadlineScoped, hasDeadline) | |
| 27 | 30 | if r.tombstoned == nil { | |
| 28 | 31 | r.tombstoned = make(map[string]bool) | |
| 29 | 32 | } | |
| @@ -31,14 +34,18 @@ func (r *retryablePurgeRepository) WriteDeletionTombstone(_ context.Context, tom | |||
| 31 | 34 | r.tombstoned[tombstone.Name] = true | |
| 32 | 35 | return nil | |
| 33 | 36 | } | |
| 34 | - | func (r *retryablePurgeRepository) QuarantineDeletionSources(_ context.Context, tombstone domain.DeletionTombstone, _ bool) error { | |
| 37 | + | func (r *retryablePurgeRepository) QuarantineDeletionSources(ctx context.Context, tombstone domain.DeletionTombstone, _ bool) error { | |
| 38 | + | _, hasDeadline := ctx.Deadline() | |
| 39 | + | r.deadlineScoped = append(r.deadlineScoped, hasDeadline) | |
| 35 | 40 | r.calls = append(r.calls, "incremental", "legacy") | |
| 36 | 41 | if r.incrementalErr != nil { | |
| 37 | 42 | return r.incrementalErr | |
| 38 | 43 | } | |
| 39 | 44 | return r.legacyErr | |
| 40 | 45 | } | |
| 41 | - | func (r *retryablePurgeRepository) DeleteDeletionTombstone(_ context.Context, _ domain.IncarnationID) error { | |
| 46 | + | func (r *retryablePurgeRepository) DeleteDeletionTombstone(ctx context.Context, _ domain.IncarnationID) error { | |
| 47 | + | _, hasDeadline := ctx.Deadline() | |
| 48 | + | r.deadlineScoped = append(r.deadlineScoped, hasDeadline) | |
| 42 | 49 | r.calls = append(r.calls, "clear tombstone") | |
| 43 | 50 | for name := range r.tombstoned { | |
| 44 | 51 | delete(r.tombstoned, name) | |
| @@ -65,8 +72,13 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) { | |||
| 65 | 72 | t.Run(tt.name, func(t *testing.T) { | |
| 66 | 73 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 67 | 74 | repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy} | |
| 68 | - | WithSnapshotRepository(repository, repository)(d) | |
| 75 | + | WithSnapshotRepository(repository)(d) | |
| 76 | + | store, _ := newMockStore(t) | |
| 77 | + | WithStore(store)(d) | |
| 69 | 78 | sess := newSnapshotTestSession(t, "work", false, "/work") | |
| 79 | + | record := sess.persistRecordLocked(1) | |
| 80 | + | record.RecoveryState = domain.RecoveryFresh | |
| 81 | + | require.NoError(t, d.catalogue.Create(record)) | |
| 70 | 82 | pty, ok := sess.tabs[0].panes["pane-1"].pty.(*portsmocks.MockPTY) | |
| 71 | 83 | require.True(t, ok, "snapshot test pane must use MockPTY") | |
| 72 | 84 | pty.EXPECT().Close().Return(nil).Once() | |
| @@ -86,6 +98,10 @@ func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) { | |||
| 86 | 98 | require.NoError(t, d.retryStoppedPurge("work")) | |
| 87 | 99 | require.False(t, repository.tombstoned["work"]) | |
| 88 | 100 | require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls) | |
| 101 | + | require.NotEmpty(t, repository.deadlineScoped) | |
| 102 | + | for _, scoped := range repository.deadlineScoped { | |
| 103 | + | require.True(t, scoped, "every purge repository call must receive a deadline-scoped context") | |
| 104 | + | } | |
| 89 | 105 | d.mu.Lock() | |
| 90 | 106 | _, ok = d.stopped["work"] | |
| 91 | 107 | d.mu.Unlock() | |
M internal/usecase/daemon/session_rename_purge_test.go +26 -29
| @@ -1,43 +1,40 @@ | |||
| 1 | 1 | package daemon | |
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | + | "errors" | |
| 4 | 5 | "testing" | |
| 5 | 6 | ||
| 6 | 7 | "github.com/stretchr/testify/require" | |
| 7 | 8 | ||
| 8 | 9 | "github.com/bnema/vev/internal/domain" | |
| 10 | + | "github.com/bnema/vev/internal/persist" | |
| 9 | 11 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | |
| 10 | 12 | ) | |
| 11 | 13 | ||
| 12 | 14 | func TestRenamePreservesIncarnationSnapshotSources(t *testing.T) { | |
| 13 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 14 | - | repository := &retryablePurgeRepository{} | |
| 15 | - | WithSnapshotRepository(repository, repository)(d) | |
| 16 | - | store, state := newMockStore(t) | |
| 17 | - | WithStore(store)(d) | |
| 18 | - | sess := newSnapshotTestSession(t, "old", false, "/work") | |
| 19 | - | d.sessions = map[domain.SessionID]*session{sess.id: sess} | |
| 20 | - | require.NoError(t, d.persist.Save(sess.persistRecordLocked(1))) | |
| 15 | + | legacyDeleteErr := errors.New("legacy delete failed") | |
| 16 | + | for _, tt := range []struct { | |
| 17 | + | name string | |
| 18 | + | legacyErr error | |
| 19 | + | }{ | |
| 20 | + | {name: "snapshot sources available"}, | |
| 21 | + | {name: "unrelated legacy source delete would fail", legacyErr: legacyDeleteErr}, | |
| 22 | + | } { | |
| 23 | + | t.Run(tt.name, func(t *testing.T) { | |
| 24 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 25 | + | repository := &retryablePurgeRepository{legacyErr: tt.legacyErr} | |
| 26 | + | WithSnapshotRepository(repository)(d) | |
| 27 | + | store, state := newMockStore(t) | |
| 28 | + | WithStore(store)(d) | |
| 29 | + | sess := newSnapshotTestSession(t, "old", false, "/work") | |
| 30 | + | d.sessions = map[domain.SessionID]*session{sess.id: sess} | |
| 31 | + | require.NoError(t, d.catalogue.(*persist.Persister).Save(sess.persistRecordLocked(1))) | |
| 21 | 32 | ||
| 22 | - | require.NoError(t, d.renameSession(sess, "new")) | |
| 23 | - | require.Empty(t, repository.calls, "rename must not touch incarnation-keyed snapshot storage") | |
| 24 | - | require.False(t, state.has("old")) | |
| 25 | - | require.True(t, state.has("new")) | |
| 26 | - | require.Equal(t, sess.incarnation, state.record(t, "new").IncarnationID) | |
| 27 | - | } | |
| 28 | - | ||
| 29 | - | func TestRenameIgnoresUnrelatedLegacySourceFailure(t *testing.T) { | |
| 30 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 31 | - | repository := &retryablePurgeRepository{} | |
| 32 | - | WithSnapshotRepository(repository, repository)(d) | |
| 33 | - | store, state := newMockStore(t) | |
| 34 | - | WithStore(store)(d) | |
| 35 | - | sess := newSnapshotTestSession(t, "old", false, "/work") | |
| 36 | - | d.sessions = map[domain.SessionID]*session{sess.id: sess} | |
| 37 | - | require.NoError(t, d.persist.Save(sess.persistRecordLocked(1))) | |
| 38 | - | ||
| 39 | - | require.NoError(t, d.renameSession(sess, "new")) | |
| 40 | - | require.Empty(t, repository.calls) | |
| 41 | - | require.False(t, state.has("old")) | |
| 42 | - | require.True(t, state.has("new")) | |
| 33 | + | require.NoError(t, d.renameSession(sess, "new")) | |
| 34 | + | require.Empty(t, repository.calls, "rename must not invoke source deletion, including a failing legacy source") | |
| 35 | + | require.False(t, state.has("old")) | |
| 36 | + | require.True(t, state.has("new")) | |
| 37 | + | require.Equal(t, sess.incarnation, state.record(t, "new").IncarnationID) | |
| 38 | + | }) | |
| 39 | + | } | |
| 43 | 40 | } | |
M internal/usecase/daemon/session_test.go +54 -28
| @@ -806,11 +806,27 @@ func (s newStaticStore) Get(key []byte) ([]byte, bool) { | |||
| 806 | 806 | func (s newStaticStore) Set(_, _ []byte) error { return nil } | |
| 807 | 807 | func (s newStaticStore) Delete(_ []byte) error { return nil } | |
| 808 | 808 | func (s newStaticStore) Batch(changes []ports.StoreChange) error { | |
| 809 | - | for _, change := range changes { | |
| 809 | + | if len(changes) == 0 { | |
| 810 | + | return errors.New("empty batch") | |
| 811 | + | } | |
| 812 | + | prepared := make([]ports.StoreChange, len(changes)) | |
| 813 | + | seen := make(map[string]struct{}, len(changes)) | |
| 814 | + | for i, change := range changes { | |
| 815 | + | key := string(change.Key) | |
| 816 | + | if _, exists := seen[key]; exists { | |
| 817 | + | return errors.New("duplicate batch key") | |
| 818 | + | } | |
| 819 | + | if change.Delete && len(change.Value) != 0 { | |
| 820 | + | return errors.New("delete batch change has value") | |
| 821 | + | } | |
| 822 | + | seen[key] = struct{}{} | |
| 823 | + | prepared[i] = ports.StoreChange{Key: append([]byte(nil), change.Key...), Value: append([]byte(nil), change.Value...), Delete: change.Delete} | |
| 824 | + | } | |
| 825 | + | for _, change := range prepared { | |
| 810 | 826 | if change.Delete { | |
| 811 | 827 | delete(s, string(change.Key)) | |
| 812 | 828 | } else { | |
| 813 | - | s[string(change.Key)] = append([]byte(nil), change.Value...) | |
| 829 | + | s[string(change.Key)] = change.Value | |
| 814 | 830 | } | |
| 815 | 831 | } | |
| 816 | 832 | return nil | |
| @@ -841,19 +857,26 @@ func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) { | |||
| 841 | 857 | store, _ := newMockStore(t) | |
| 842 | 858 | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | |
| 843 | 859 | incarnation := domain.IncarnationID{1} | |
| 844 | - | require.NoError(t, seed.persist.Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh})) | |
| 860 | + | require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9, RecoveryState: domain.RecoveryFresh})) | |
| 845 | 861 | ||
| 846 | 862 | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | |
| 847 | 863 | d.mu.Lock() | |
| 848 | 864 | stopped := d.stopped["work"] | |
| 849 | 865 | d.mu.Unlock() | |
| 850 | - | require.Equal(t, stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, incarnation: incarnation, lastUsedSeq: 9}, stopped) | |
| 866 | + | require.Equal(t, "work", stopped.name) | |
| 867 | + | require.Equal(t, "/tmp/work", stopped.cwd) | |
| 868 | + | require.Equal(t, int64(7), stopped.createdAt) | |
| 869 | + | require.Equal(t, incarnation, stopped.incarnation) | |
| 870 | + | require.Equal(t, uint64(9), stopped.lastUsedSeq) | |
| 871 | + | require.Equal(t, domain.RecoveryFresh, stopped.record.RecoveryState) | |
| 872 | + | require.Equal(t, runtimeFresh, stopped.state) | |
| 873 | + | require.NotNil(t, stopped.restoreDone) | |
| 851 | 874 | require.Equal(t, uint64(9), d.mruSeq.Load()) | |
| 852 | 875 | } | |
| 853 | 876 | ||
| 854 | 877 | // TestDaemonNewWithoutPersistenceLogsNoLoadWarning covers the default, | |
| 855 | 878 | // persistence-free construction path (no WithStore/WithCatalogue option): | |
| 856 | - | // d.persist is the persist.New(nil) sentinel, so LoadAll now returns | |
| 879 | + | // d.catalogue.(*persist.Persister) is the persist.New(nil) sentinel, so LoadAll now returns | |
| 857 | 880 | // errPersistenceUnavailable instead of an empty slice. That must not surface | |
| 858 | 881 | // as a "loading persisted sessions failed" warning, since nothing failed — | |
| 859 | 882 | // persistence simply was never configured. | |
| @@ -866,20 +889,6 @@ func TestDaemonNewWithoutPersistenceLogsNoLoadWarning(t *testing.T) { | |||
| 866 | 889 | require.NotContains(t, logBuf.String(), "loading persisted sessions failed") | |
| 867 | 890 | } | |
| 868 | 891 | ||
| 869 | - | // TestDaemonNewWithConfiguredStoreStillLogsLoadFailure is the inverse of the | |
| 870 | - | // above: when persistence IS configured (a real store via WithStore) and the | |
| 871 | - | // load genuinely fails (an undecodable catalogue record here), the warning | |
| 872 | - | // must still fire. | |
| 873 | - | func TestDaemonNewWithConfiguredStoreStillLogsLoadFailure(t *testing.T) { | |
| 874 | - | store, state := newMockStore(t) | |
| 875 | - | state.data["bad"] = []byte{0, 1} // not a valid encoded catalogue record | |
| 876 | - | var logBuf strings.Builder | |
| 877 | - | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(&logBuf, nil)), WithStore(store)) | |
| 878 | - | require.NotNil(t, d) | |
| 879 | - | require.True(t, d.persistEnabled) | |
| 880 | - | require.Contains(t, logBuf.String(), "loading persisted sessions failed") | |
| 881 | - | } | |
| 882 | - | ||
| 883 | 892 | func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) { | |
| 884 | 893 | d := &Daemon{} | |
| 885 | 894 | sess := &session{} | |
| @@ -930,7 +939,7 @@ func TestTouchMRUPersistsNamedButNotEphemeral(t *testing.T) { | |||
| 930 | 939 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 931 | 940 | WithStore(store)(d) | |
| 932 | 941 | named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1, incarnation: domain.IncarnationID{1}} | |
| 933 | - | require.NoError(t, d.persist.Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) | |
| 942 | + | require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) | |
| 934 | 943 | ||
| 935 | 944 | d.touchMRU(named) | |
| 936 | 945 | require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq) | |
| @@ -1003,7 +1012,7 @@ func TestRenameTabPersistsForNamedSession(t *testing.T) { | |||
| 1003 | 1012 | require.NoError(t, err) | |
| 1004 | 1013 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | |
| 1005 | 1014 | ||
| 1006 | - | records, err := d.persist.LoadAll() | |
| 1015 | + | records, err := d.catalogue.(*persist.Persister).LoadAll() | |
| 1007 | 1016 | require.NoError(t, err) | |
| 1008 | 1017 | require.Len(t, records, 1) | |
| 1009 | 1018 | require.Equal(t, "work", records[0].Name) | |
| @@ -1013,6 +1022,23 @@ func TestRenameTabPersistsForNamedSession(t *testing.T) { | |||
| 1013 | 1022 | require.Equal(t, []string{"shell"}, records[0].TabNames) | |
| 1014 | 1023 | } | |
| 1015 | 1024 | ||
| 1025 | + | func TestRenameTabDoesNotHoldDaemonOrSessionMutexDuringCatalogueIO(t *testing.T) { | |
| 1026 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 1027 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |
| 1028 | + | d.sessions[sess.id] = sess | |
| 1029 | + | catalogue := portsmocks.NewMockCatalogue(t) | |
| 1030 | + | WithCatalogue(catalogue, nil)(d) | |
| 1031 | + | catalogue.EXPECT().UpdateMetadata(mock.Anything).RunAndReturn(func(domain.CatalogueMetadataUpdate) error { | |
| 1032 | + | require.True(t, d.mu.TryLock(), "daemon mutex held during catalogue I/O") | |
| 1033 | + | d.mu.Unlock() | |
| 1034 | + | require.True(t, sess.mu.TryLock(), "session mutex held during catalogue I/O") | |
| 1035 | + | sess.mu.Unlock() | |
| 1036 | + | return nil | |
| 1037 | + | }).Once() | |
| 1038 | + | ||
| 1039 | + | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | |
| 1040 | + | } | |
| 1041 | + | ||
| 1016 | 1042 | func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | |
| 1017 | 1043 | tests := []struct { | |
| 1018 | 1044 | name string | |
| @@ -1045,7 +1071,7 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | |||
| 1045 | 1071 | } | |
| 1046 | 1072 | ||
| 1047 | 1073 | require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false)) | |
| 1048 | - | records, err := d.persist.LoadAll() | |
| 1074 | + | records, err := d.catalogue.(*persist.Persister).LoadAll() | |
| 1049 | 1075 | require.NoError(t, err) | |
| 1050 | 1076 | require.Len(t, records, 1) | |
| 1051 | 1077 | require.Equal(t, tt.want, records[0].TabNames) | |
| @@ -1090,7 +1116,7 @@ func TestRenameTabDoesNotPersistForEphemeralSession(t *testing.T) { | |||
| 1090 | 1116 | require.Equal(t, "shell", sess.tabs[0].name) | |
| 1091 | 1117 | ||
| 1092 | 1118 | require.False(t, state.has("0")) | |
| 1093 | - | records, err := d.persist.LoadAll() | |
| 1119 | + | records, err := d.catalogue.(*persist.Persister).LoadAll() | |
| 1094 | 1120 | require.NoError(t, err) | |
| 1095 | 1121 | require.Empty(t, records) | |
| 1096 | 1122 | } | |
| @@ -1104,7 +1130,7 @@ func TestAttachRestoresPersistedTabNames(t *testing.T) { | |||
| 1104 | 1130 | store, _ := newMockStore(t) | |
| 1105 | 1131 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | |
| 1106 | 1132 | WithStore(store)(d) | |
| 1107 | - | require.NoError(t, d.persist.Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh})) | |
| 1133 | + | require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}, RecoveryState: domain.RecoveryFresh})) | |
| 1108 | 1134 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}} | |
| 1109 | 1135 | tr := portsmocks.NewMockTransport(t) | |
| 1110 | 1136 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | |
| @@ -1213,8 +1239,8 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T) | |||
| 1213 | 1239 | defer release() | |
| 1214 | 1240 | store := portsmocks.NewMockStore(t) | |
| 1215 | 1241 | var attempted map[string][]byte | |
| 1216 | - | store.EXPECT().Range(mock.Anything).Run(func(func([]byte, []byte) bool) {}).Twice() | |
| 1217 | - | store.EXPECT().Get([]byte("named")).Return(nil, false).Twice() | |
| 1242 | + | store.EXPECT().Range(mock.Anything).Run(func(func([]byte, []byte) bool) {}).Times(3) | |
| 1243 | + | store.EXPECT().Get([]byte("named")).Return(nil, false).Times(4) | |
| 1218 | 1244 | store.EXPECT().Batch(mock.Anything).RunAndReturn(func(changes []ports.StoreChange) error { | |
| 1219 | 1245 | attempted = make(map[string][]byte, len(changes)) | |
| 1220 | 1246 | for _, change := range changes { | |
| @@ -1318,7 +1344,7 @@ func TestPickerStoppedTargetKillPurges(t *testing.T) { | |||
| 1318 | 1344 | store, state := newMockStore(t) | |
| 1319 | 1345 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 1320 | 1346 | WithStore(store)(d) | |
| 1321 | - | require.NoError(t, d.persist.Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) | |
| 1347 | + | require.NoError(t, d.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1, RecoveryState: domain.RecoveryFresh})) | |
| 1322 | 1348 | d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1} | |
| 1323 | 1349 | require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true})) | |
| 1324 | 1350 | require.False(t, state.has("old")) | |
| @@ -1830,7 +1856,7 @@ func TestCatalogueRecordsConstructExpectedSessionRegistry(t *testing.T) { | |||
| 1830 | 1856 | func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) { | |
| 1831 | 1857 | store, _ := newMockStore(t) | |
| 1832 | 1858 | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | |
| 1833 | - | require.NoError(t, seed.persist.Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh})) | |
| 1859 | + | require.NoError(t, seed.catalogue.(*persist.Persister).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900, RecoveryState: domain.RecoveryFresh})) | |
| 1834 | 1860 | ||
| 1835 | 1861 | p, release := newBlockingPTY(t) | |
| 1836 | 1862 | defer release() | |
M internal/usecase/daemon/snapshot.go +6 -0
| @@ -47,6 +47,9 @@ type snapshotCapture struct { | |||
| 47 | 47 | publicationContext context.Context | |
| 48 | 48 | sealedRefs map[*vt.HistoryChunk]snapcodec.ObjectRef // set by the single encoder worker | |
| 49 | 49 | coordinatorDiscarded bool // guarded by session.snapshotMu | |
| 50 | + | // normalWorkerAdmitted is set before snapshotJobs admission and immutable | |
| 51 | + | // until completion; final-queue captures leave it false. | |
| 52 | + | normalWorkerAdmitted bool | |
| 50 | 53 | finishOnce sync.Once | |
| 51 | 54 | } | |
| 52 | 55 | ||
| @@ -98,3 +101,6 @@ const ( | |||
| 98 | 101 | snapshotInterval = 2 * time.Minute | |
| 99 | 102 | snapshotFinalFlushTimeout = time.Second | |
| 100 | 103 | ) | |
| 104 | + | ||
| 105 | + | // SnapshotShutdownTimeout reports the shared final-checkpoint shutdown budget. | |
| 106 | + | func SnapshotShutdownTimeout() time.Duration { return snapshotFinalFlushTimeout } | |
M internal/usecase/daemon/snapshot_coordinator.go +13 -0
| @@ -108,6 +108,14 @@ func (d *Daemon) waitForSnapshotGenerationWithDeadline(sess *session, generation | |||
| 108 | 108 | return d.waitForSnapshotGeneration(sess, generation) | |
| 109 | 109 | } | |
| 110 | 110 | for { | |
| 111 | + | // Once the shared deadline is observable, timeout wins even if publication | |
| 112 | + | // completion becomes ready concurrently. This preserves the timed-out | |
| 113 | + | // session identity and avoids nondeterministic success after the budget. | |
| 114 | + | select { | |
| 115 | + | case <-deadline.Done(): | |
| 116 | + | return false | |
| 117 | + | default: | |
| 118 | + | } | |
| 111 | 119 | sess.snapshotMu.Lock() | |
| 112 | 120 | published := sess.snapshotPublishedMutationRevision >= generation | |
| 113 | 121 | changed := sess.snapshotChangeLocked() | |
| @@ -207,6 +215,11 @@ func (d *Daemon) finishSnapshotCapture(capture *snapshotCapture, succeeded bool) | |||
| 207 | 215 | capture.session.signalSnapshotChangedLocked() | |
| 208 | 216 | wake := capture.session.snapshotWake | |
| 209 | 217 | capture.session.snapshotMu.Unlock() | |
| 218 | + | if capture.normalWorkerAdmitted { | |
| 219 | + | d.snapshotWorkerMu.Lock() | |
| 220 | + | delete(d.snapshotAdmitted, capture) | |
| 221 | + | d.snapshotWorkerMu.Unlock() | |
| 222 | + | } | |
| 210 | 223 | if wake != nil { | |
| 211 | 224 | select { | |
| 212 | 225 | case wake <- struct{}{}: | |
M internal/usecase/daemon/snapshot_generation_test.go +2 -2
| @@ -17,7 +17,7 @@ import ( | |||
| 17 | 17 | func TestSnapshotMultipleMutationsPublishNextRepositoryGeneration(t *testing.T) { | |
| 18 | 18 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 19 | 19 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 20 | - | WithSnapshotRepository(repository, nil)(d) | |
| 20 | + | WithSnapshotRepository(repository)(d) | |
| 21 | 21 | startSnapshotEncodeWorker(t, d) | |
| 22 | 22 | ||
| 23 | 23 | publications := make(chan ports.SnapshotPublication, 1) | |
| @@ -44,7 +44,7 @@ func TestSnapshotMultipleMutationsPublishNextRepositoryGeneration(t *testing.T) | |||
| 44 | 44 | func TestSnapshotRetryKeepsRepositoryGeneration(t *testing.T) { | |
| 45 | 45 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 46 | 46 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 47 | - | WithSnapshotRepository(repository, nil)(d) | |
| 47 | + | WithSnapshotRepository(repository)(d) | |
| 48 | 48 | startSnapshotEncodeWorker(t, d) | |
| 49 | 49 | ||
| 50 | 50 | publications := make(chan ports.SnapshotPublication, 2) | |
D internal/usecase/daemon/snapshot_import_test.go +0 -163
| @@ -1,163 +0,0 @@ | |||
| 1 | - | package daemon | |
| 2 | - | ||
| 3 | - | import ( | |
| 4 | - | "context" | |
| 5 | - | "encoding/base64" | |
| 6 | - | "errors" | |
| 7 | - | "os" | |
| 8 | - | "path/filepath" | |
| 9 | - | "testing" | |
| 10 | - | ||
| 11 | - | "github.com/stretchr/testify/require" | |
| 12 | - | ||
| 13 | - | "github.com/bnema/vev/internal/ports" | |
| 14 | - | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |
| 15 | - | ) | |
| 16 | - | ||
| 17 | - | type snapshotAcceptanceLegacySource struct { | |
| 18 | - | blobs []ports.LegacySnapshot | |
| 19 | - | loadCalls int | |
| 20 | - | deleteCalls []string | |
| 21 | - | deleteErr error | |
| 22 | - | } | |
| 23 | - | ||
| 24 | - | func (s *snapshotAcceptanceLegacySource) LoadLegacy(ctx context.Context) ([]ports.LegacySnapshot, error) { | |
| 25 | - | if err := ctx.Err(); err != nil { | |
| 26 | - | return nil, err | |
| 27 | - | } | |
| 28 | - | s.loadCalls++ | |
| 29 | - | out := make([]ports.LegacySnapshot, len(s.blobs)) | |
| 30 | - | for i, blob := range s.blobs { | |
| 31 | - | out[i] = ports.LegacySnapshot{Name: blob.Name, Data: append([]byte(nil), blob.Data...)} | |
| 32 | - | } | |
| 33 | - | return out, nil | |
| 34 | - | } | |
| 35 | - | ||
| 36 | - | func (s *snapshotAcceptanceLegacySource) DeleteVerifiedLegacy(ctx context.Context, blob ports.LegacySnapshot) error { | |
| 37 | - | return s.DeleteLegacy(ctx, blob.Name) | |
| 38 | - | } | |
| 39 | - | ||
| 40 | - | func (s *snapshotAcceptanceLegacySource) DeleteLegacy(ctx context.Context, name string) error { | |
| 41 | - | if err := ctx.Err(); err != nil { | |
| 42 | - | return err | |
| 43 | - | } | |
| 44 | - | s.deleteCalls = append(s.deleteCalls, name) | |
| 45 | - | if s.deleteErr != nil { | |
| 46 | - | return s.deleteErr | |
| 47 | - | } | |
| 48 | - | for i, blob := range s.blobs { | |
| 49 | - | if blob.Name == name { | |
| 50 | - | s.blobs = append(s.blobs[:i], s.blobs[i+1:]...) | |
| 51 | - | break | |
| 52 | - | } | |
| 53 | - | } | |
| 54 | - | return nil | |
| 55 | - | } | |
| 56 | - | ||
| 57 | - | func legacyAcceptanceSnapshot(t *testing.T) snapcodec.Session { | |
| 58 | - | t.Helper() | |
| 59 | - | data, err := os.ReadFile(filepath.Join("testdata", "legacy-v3-empty.b64")) | |
| 60 | - | require.NoError(t, err) | |
| 61 | - | encoded, err := base64.StdEncoding.DecodeString(string(data)) | |
| 62 | - | require.NoError(t, err) | |
| 63 | - | snapshot, err := snapcodec.Unmarshal(encoded) | |
| 64 | - | require.NoError(t, err) | |
| 65 | - | return snapshot | |
| 66 | - | } | |
| 67 | - | ||
| 68 | - | func legacyAcceptanceBlob(t *testing.T, snapshot snapcodec.Session) ports.LegacySnapshot { | |
| 69 | - | t.Helper() | |
| 70 | - | fixture := legacyAcceptanceSnapshot(t) | |
| 71 | - | require.Equal(t, fixture, snapshot, "legacy fixture must be the imported session") | |
| 72 | - | data, err := os.ReadFile(filepath.Join("testdata", "legacy-v3-empty.b64")) | |
| 73 | - | require.NoError(t, err) | |
| 74 | - | encoded, err := base64.StdEncoding.DecodeString(string(data)) | |
| 75 | - | require.NoError(t, err) | |
| 76 | - | return ports.LegacySnapshot{Name: snapshot.Name, Data: encoded} | |
| 77 | - | } | |
| 78 | - | ||
| 79 | - | func TestLegacyPublicationMarshalObjectErrorsIdentifyPaneAndStage(t *testing.T) { | |
| 80 | - | for _, tt := range []struct { | |
| 81 | - | name string | |
| 82 | - | mutate func(*snapcodec.Pane) | |
| 83 | - | stage string | |
| 84 | - | }{ | |
| 85 | - | {name: "sealed chunk", mutate: func(pane *snapcodec.Pane) { pane.SealedChunks = [][]byte{nil} }, stage: "sealed chunk 0"}, | |
| 86 | - | {name: "tail", mutate: func(pane *snapcodec.Pane) { pane.Tail = nil }, stage: "tail"}, | |
| 87 | - | {name: "visible", mutate: func(pane *snapcodec.Pane) { pane.Visible = nil }, stage: "visible"}, | |
| 88 | - | } { | |
| 89 | - | t.Run(tt.name, func(t *testing.T) { | |
| 90 | - | pane := snapcodec.Pane{ID: "pane-1", StableID: "pane-stable", Tail: []byte("tail"), Visible: []byte("visible")} | |
| 91 | - | tt.mutate(&pane) | |
| 92 | - | snapshot := snapcodec.Session{Name: "legacy", Tabs: []snapcodec.Tab{{StableID: "tab-1", Panes: []snapcodec.Pane{pane}}}} | |
| 93 | - | ||
| 94 | - | _, err := legacyPublication(snapshot) | |
| 95 | - | require.Error(t, err) | |
| 96 | - | require.ErrorContains(t, err, `tab "tab-1"`) | |
| 97 | - | require.ErrorContains(t, err, `pane "pane-1"`) | |
| 98 | - | require.ErrorContains(t, err, tt.stage) | |
| 99 | - | require.ErrorContains(t, err, "marshal object") | |
| 100 | - | }) | |
| 101 | - | } | |
| 102 | - | } | |
| 103 | - | ||
| 104 | - | func TestImportLegacySnapshotsIsRetiredRuntimeNoop(t *testing.T) { | |
| 105 | - | snapshot := legacyAcceptanceSnapshot(t) | |
| 106 | - | repository := &snapshotAcceptanceRepository{generations: make(map[string]ports.SnapshotGeneration)} | |
| 107 | - | legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}} | |
| 108 | - | d := newTestDaemon(t, nil, stubClock{}) | |
| 109 | - | WithSnapshotRepository(repository, legacy)(d) | |
| 110 | - | ||
| 111 | - | d.importLegacySnapshots(context.Background()) | |
| 112 | - | ||
| 113 | - | // Migration is adapter-owned. The daemon must never name-scan, publish, or | |
| 114 | - | // delete a legacy source during normal recovery. | |
| 115 | - | require.Empty(t, repository.publishes) | |
| 116 | - | require.Empty(t, legacy.deleteCalls) | |
| 117 | - | require.Len(t, legacy.blobs, 1) | |
| 118 | - | } | |
| 119 | - | ||
| 120 | - | func TestImportLegacyExistingIncrementalSkipsLegacy(t *testing.T) { | |
| 121 | - | snapshot := legacyAcceptanceSnapshot(t) | |
| 122 | - | repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: make(map[string]ports.SnapshotGeneration)} | |
| 123 | - | legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}} | |
| 124 | - | d := newTestDaemon(t, nil, stubClock{}) | |
| 125 | - | WithSnapshotRepository(repository, legacy)(d) | |
| 126 | - | ||
| 127 | - | d.importLegacySnapshots(context.Background()) | |
| 128 | - | ||
| 129 | - | require.Empty(t, repository.publishes) | |
| 130 | - | require.Empty(t, legacy.deleteCalls) | |
| 131 | - | require.Len(t, legacy.blobs, 1) | |
| 132 | - | } | |
| 133 | - | ||
| 134 | - | func TestImportLegacyRetainsSourceUntilExactReloadVerification(t *testing.T) { | |
| 135 | - | snapshot := legacyAcceptanceSnapshot(t) | |
| 136 | - | for _, test := range []struct { | |
| 137 | - | name string | |
| 138 | - | publish error | |
| 139 | - | load error | |
| 140 | - | mutate func(ports.SnapshotGeneration, ports.SnapshotPublication) ports.SnapshotGeneration | |
| 141 | - | }{ | |
| 142 | - | {"publish failure", errors.New("publish failed"), nil, nil}, | |
| 143 | - | {"reload failure", nil, errors.New("reload failed"), nil}, | |
| 144 | - | {"manifest metadata mismatch", nil, nil, func(g ports.SnapshotGeneration, _ ports.SnapshotPublication) ports.SnapshotGeneration { | |
| 145 | - | g.Generation++ | |
| 146 | - | return g | |
| 147 | - | }}, | |
| 148 | - | {"manifest bytes mismatch", nil, nil, func(g ports.SnapshotGeneration, _ ports.SnapshotPublication) ports.SnapshotGeneration { | |
| 149 | - | g.Manifest[len(g.Manifest)-1] ^= 1 | |
| 150 | - | return g | |
| 151 | - | }}, | |
| 152 | - | } { | |
| 153 | - | t.Run(test.name, func(t *testing.T) { | |
| 154 | - | repository := &snapshotAcceptanceRepository{generations: make(map[string]ports.SnapshotGeneration), publishErr: test.publish, loadErr: test.load, loadMutate: test.mutate} | |
| 155 | - | legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}} | |
| 156 | - | d := newTestDaemon(t, nil, stubClock{}) | |
| 157 | - | WithSnapshotRepository(repository, legacy)(d) | |
| 158 | - | d.importLegacySnapshots(context.Background()) | |
| 159 | - | require.Empty(t, legacy.deleteCalls) | |
| 160 | - | require.Len(t, legacy.blobs, 1) | |
| 161 | - | }) | |
| 162 | - | } | |
| 163 | - | } | |
M internal/usecase/daemon/snapshot_queue.go +6 -0
| @@ -107,10 +107,16 @@ func (d *Daemon) enqueueSnapshotCapture(capture *snapshotCapture) bool { | |||
| 107 | 107 | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | |
| 108 | 108 | return false | |
| 109 | 109 | } | |
| 110 | + | capture.normalWorkerAdmitted = true | |
| 110 | 111 | select { | |
| 111 | 112 | case d.snapshotJobs <- capture: | |
| 113 | + | if d.snapshotAdmitted == nil { | |
| 114 | + | d.snapshotAdmitted = make(map[*snapshotCapture]struct{}) | |
| 115 | + | } | |
| 116 | + | d.snapshotAdmitted[capture] = struct{}{} | |
| 112 | 117 | return true | |
| 113 | 118 | default: | |
| 119 | + | capture.normalWorkerAdmitted = false | |
| 114 | 120 | return false | |
| 115 | 121 | } | |
| 116 | 122 | } | |
M internal/usecase/daemon/snapshot_restore.go +23 -6
| @@ -3,6 +3,7 @@ package daemon | |||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | 5 | "crypto/rand" | |
| 6 | + | "errors" | |
| 6 | 7 | "fmt" | |
| 7 | 8 | "math" | |
| 8 | 9 | ||
| @@ -13,6 +14,20 @@ import ( | |||
| 13 | 14 | ) | |
| 14 | 15 | ||
| 15 | 16 | func (d *Daemon) closeRestoreDone() { | |
| 17 | + | d.mu.Lock() | |
| 18 | + | for name, entry := range d.stopped { | |
| 19 | + | if entry.state != runtimeRestoring { | |
| 20 | + | continue | |
| 21 | + | } | |
| 22 | + | entry.state = runtimeDegraded | |
| 23 | + | entry.record.RecoveryState = domain.RecoveryDegraded | |
| 24 | + | if entry.record.DegradedReason == "" { | |
| 25 | + | entry.record.DegradedReason = "restore unavailable" | |
| 26 | + | } | |
| 27 | + | d.stopped[name] = entry | |
| 28 | + | closeRuntimeRestoreDoneLocked(entry.restoreDone) | |
| 29 | + | } | |
| 30 | + | d.mu.Unlock() | |
| 16 | 31 | d.restoreOnce.Do(func() { close(d.restoreDone) }) | |
| 17 | 32 | } | |
| 18 | 33 | ||
| @@ -293,13 +308,15 @@ func (d *Daemon) persistAndRegisterRestoredSession(ctx context.Context, sess *se | |||
| 293 | 308 | } | |
| 294 | 309 | } | |
| 295 | 310 | } | |
| 296 | - | // An existing catalogue checkpoint already authorizes this runtime. Only a | |
| 297 | - | // legacy snapshot without a catalogue record creates fresh metadata here. | |
| 311 | + | // Restoration is catalogue-authorized. Missing or unreadable authority is a | |
| 312 | + | // hard failure; migration creates catalogue records before daemon startup. | |
| 298 | 313 | if d.persistEnabled { | |
| 299 | - | if _, ok := d.catalogueRecord(sess.name); !ok { | |
| 300 | - | if err := d.createCatalogueRecord(sess.persistRecordLocked(sess.createdAt)); err != nil { | |
| 301 | - | return false, err | |
| 302 | - | } | |
| 314 | + | _, ok, err := d.catalogueRecord(sess.name) | |
| 315 | + | if err != nil { | |
| 316 | + | return false, err | |
| 317 | + | } | |
| 318 | + | if !ok { | |
| 319 | + | return false, errors.New("snapshot: restored session is absent from catalogue") | |
| 303 | 320 | } | |
| 304 | 321 | } | |
| 305 | 322 | d.mu.Lock() | |
R internal/usecase/daemon/snapshot_legacy_import.go → internal/usecase/daemon/snapshot_restore_fixture_test.go +2 -3 renamed, 91% similar
| @@ -8,9 +8,8 @@ import ( | |||
| 8 | 8 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |
| 9 | 9 | ) | |
| 10 | 10 | ||
| 11 | - | func legacyPublication(snapshot snapcodec.Session) (ports.SnapshotPublication, error) { | |
| 12 | - | // Legacy payloads have no durable identity; this test-only conversion uses | |
| 13 | - | // the fixture's stable synthetic identity. | |
| 11 | + | func acceptancePublication(snapshot snapcodec.Session) (ports.SnapshotPublication, error) { | |
| 12 | + | // acceptancePublication converts a restore fixture to repository form. | |
| 14 | 13 | incarnation := domain.IncarnationID{1} | |
| 15 | 14 | manifest := snapcodec.Manifest{Generation: 1, IncarnationID: incarnation, Name: snapshot.Name, CreatedAt: snapshot.CreatedAt, Active: snapshot.Active, Tabs: make([]snapcodec.ManifestTab, 0, len(snapshot.Tabs))} | |
| 16 | 15 | objects := make([]ports.SnapshotObject, 0) | |
M internal/usecase/daemon/snapshot_restore_incremental.go +53 -10
| @@ -43,7 +43,12 @@ func (d *Daemon) restoreIncrementalSnapshots(ctx context.Context) { | |||
| 43 | 43 | if d.catalogue == nil { | |
| 44 | 44 | return | |
| 45 | 45 | } | |
| 46 | - | d.restoreCatalogue(ctx, d.catalogue.Records()) | |
| 46 | + | records, err := d.catalogue.Records() | |
| 47 | + | if err != nil { | |
| 48 | + | d.log.Error("loading catalogue for snapshot restoration failed", "err", err) | |
| 49 | + | return | |
| 50 | + | } | |
| 51 | + | d.restoreCatalogue(ctx, records) | |
| 47 | 52 | } | |
| 48 | 53 | ||
| 49 | 54 | // restoreCatalogue gives every expected record an independent bounded job. | |
| @@ -95,7 +100,9 @@ func (d *Daemon) recordRestoreDone(name string) chan struct{} { | |||
| 95 | 100 | return d.stopped[name].restoreDone | |
| 96 | 101 | } | |
| 97 | 102 | ||
| 98 | - | func closeRuntimeRestoreDone(done chan struct{}) { | |
| 103 | + | // closeRuntimeRestoreDoneLocked closes a per-record restoration barrier. | |
| 104 | + | // Caller must hold d.mu so state transition and waiter release are atomic. | |
| 105 | + | func closeRuntimeRestoreDoneLocked(done chan struct{}) { | |
| 99 | 106 | if done == nil { | |
| 100 | 107 | return | |
| 101 | 108 | } | |
| @@ -125,8 +132,13 @@ func (d *Daemon) setStoppedRecovery(record domain.CatalogueRecord, state runtime | |||
| 125 | 132 | ||
| 126 | 133 | func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) { | |
| 127 | 134 | if restoreErr != nil { | |
| 135 | + | catalogueReadable := true | |
| 128 | 136 | if d.catalogue != nil { | |
| 129 | - | if current, ok := d.catalogue.Record(record.Name); ok && current.IncarnationID == record.IncarnationID { | |
| 137 | + | current, ok, readErr := d.catalogue.Record(record.Name) | |
| 138 | + | if readErr != nil { | |
| 139 | + | catalogueReadable = false | |
| 140 | + | d.log.Error("reading catalogue before degradation failed", "session", record.Name, "err", readErr) | |
| 141 | + | } else if ok && current.IncarnationID == record.IncarnationID { | |
| 130 | 142 | record = current | |
| 131 | 143 | } | |
| 132 | 144 | } | |
| @@ -135,7 +147,7 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e | |||
| 135 | 147 | if errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded) { | |
| 136 | 148 | record.DegradedReason = "restore interrupted" | |
| 137 | 149 | } | |
| 138 | - | if d.catalogue != nil { | |
| 150 | + | if d.catalogue != nil && catalogueReadable { | |
| 139 | 151 | if err := d.catalogue.Replace(record.Name, record); err != nil { | |
| 140 | 152 | d.log.Error("marking session degraded failed", "session", record.Name, "err", err) | |
| 141 | 153 | } | |
| @@ -155,7 +167,7 @@ func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr e | |||
| 155 | 167 | entry.tabNames = append([]string(nil), record.TabNames...) | |
| 156 | 168 | d.stopped[record.Name] = entry | |
| 157 | 169 | } | |
| 158 | - | closeRuntimeRestoreDone(done) | |
| 170 | + | closeRuntimeRestoreDoneLocked(done) | |
| 159 | 171 | } | |
| 160 | 172 | d.mu.Unlock() | |
| 161 | 173 | if restoreErr != nil { | |
| @@ -197,26 +209,42 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||
| 197 | 209 | var selectedGeneration ports.SnapshotGeneration | |
| 198 | 210 | var selectedSnapshot snapcodec.Session | |
| 199 | 211 | selectedIndex := -1 | |
| 212 | + | selectedFallback := false | |
| 200 | 213 | for i, candidate := range candidates { | |
| 214 | + | fallback := record.Committed == nil || i > 0 | |
| 201 | 215 | generation, err := d.snapshotRepository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, candidate) | |
| 202 | 216 | if err != nil { | |
| 203 | 217 | if ctxErr := ctx.Err(); ctxErr != nil { | |
| 204 | 218 | return ctxErr | |
| 205 | 219 | } | |
| 220 | + | d.logRejectedRestoreCandidate(record, candidate, fallback, "load-failed", err) | |
| 206 | 221 | continue | |
| 207 | 222 | } | |
| 208 | - | snapshot, err := sessionFromGeneration(generation) | |
| 209 | - | if err != nil || generation.IncarnationID != record.IncarnationID || generation.Name != record.Name || generation.Generation != candidate.Generation { | |
| 223 | + | snapshot, decodeErr := sessionFromGeneration(generation) | |
| 224 | + | if decodeErr != nil { | |
| 225 | + | d.logRejectedRestoreCandidate(record, candidate, fallback, "decode-failed", decodeErr) | |
| 210 | 226 | continue | |
| 211 | 227 | } | |
| 212 | - | selected, selectedGeneration, selectedSnapshot, selectedIndex = candidate, generation, snapshot, i | |
| 228 | + | if generation.IncarnationID != record.IncarnationID { | |
| 229 | + | d.logRejectedRestoreCandidate(record, candidate, fallback, "incarnation-mismatch", nil) | |
| 230 | + | continue | |
| 231 | + | } | |
| 232 | + | if generation.Name != record.Name { | |
| 233 | + | d.logRejectedRestoreCandidate(record, candidate, fallback, "name-mismatch", nil) | |
| 234 | + | continue | |
| 235 | + | } | |
| 236 | + | if generation.Generation != candidate.Generation { | |
| 237 | + | d.logRejectedRestoreCandidate(record, candidate, fallback, "generation-mismatch", nil) | |
| 238 | + | continue | |
| 239 | + | } | |
| 240 | + | selected, selectedGeneration, selectedSnapshot, selectedIndex, selectedFallback = candidate, generation, snapshot, i, fallback | |
| 213 | 241 | break | |
| 214 | 242 | } | |
| 215 | 243 | if selectedIndex < 0 { | |
| 216 | 244 | return errors.New("snapshot: no catalogue checkpoint validated") | |
| 217 | 245 | } | |
| 218 | 246 | ||
| 219 | - | if selectedIndex > 0 { | |
| 247 | + | if selectedFallback { | |
| 220 | 248 | if d.checkpointRecovery == nil { | |
| 221 | 249 | return errors.New("snapshot: checkpoint coordinator unavailable for fallback promotion") | |
| 222 | 250 | } | |
| @@ -244,10 +272,25 @@ func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecor | |||
| 244 | 272 | return err | |
| 245 | 273 | } | |
| 246 | 274 | d.setStoppedRecovery(record, runtimeHealthy) | |
| 247 | - | d.logSessionRestoreComplete(record, selected.Generation, selectedIndex > 0) | |
| 275 | + | d.logSessionRestoreComplete(record, selected.Generation, selectedFallback) | |
| 248 | 276 | return nil | |
| 249 | 277 | } | |
| 250 | 278 | ||
| 279 | + | func (d *Daemon) logRejectedRestoreCandidate(record domain.CatalogueRecord, candidate domain.CheckpointRef, fallback bool, reason string, err error) { | |
| 280 | + | kind := "committed" | |
| 281 | + | if fallback { | |
| 282 | + | kind = "fallback" | |
| 283 | + | } | |
| 284 | + | d.log.Warn("snapshot_restore_candidate_rejected", | |
| 285 | + | "session", record.Name, | |
| 286 | + | "incarnation", record.IncarnationID.String(), | |
| 287 | + | "generation", candidate.Generation, | |
| 288 | + | "candidate", kind, | |
| 289 | + | "reason_code", reason, | |
| 290 | + | "err", err, | |
| 291 | + | ) | |
| 292 | + | } | |
| 293 | + | ||
| 251 | 294 | func checkpointCandidates(record domain.CatalogueRecord) []domain.CheckpointRef { | |
| 252 | 295 | candidates := make([]domain.CheckpointRef, 0, 3) | |
| 253 | 296 | if record.Committed != nil { | |
M internal/usecase/daemon/snapshot_restore_test.go +6 -7
| @@ -175,7 +175,7 @@ func restoreAcceptanceSession(t *testing.T, name string) snapcodec.Session { | |||
| 175 | 175 | ||
| 176 | 176 | func acceptanceGeneration(t *testing.T, snapshot snapcodec.Session, generation uint64) ports.SnapshotGeneration { | |
| 177 | 177 | t.Helper() | |
| 178 | - | publication, err := legacyPublication(snapshot) | |
| 178 | + | publication, err := acceptancePublication(snapshot) | |
| 179 | 179 | require.NoError(t, err) | |
| 180 | 180 | manifest, err := snapcodec.UnmarshalManifest(publication.Manifest) | |
| 181 | 181 | require.NoError(t, err) | |
| @@ -218,13 +218,12 @@ func TestRestoreIncrementalGenerationAcceptance(t *testing.T) { | |||
| 218 | 218 | repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: map[string]ports.SnapshotGeneration{snapshot.Name: generation}} | |
| 219 | 219 | pty, release := newBlockingPTY(t) | |
| 220 | 220 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | |
| 221 | - | store, _ := newMockStore(t) | |
| 222 | - | WithStore(store)(d) | |
| 223 | 221 | checkpoint := domain.CheckpointRef{Generation: generation.Generation, ManifestDigest: snapcodec.ManifestDigest(generation.Manifest)} | |
| 224 | 222 | record := domain.CatalogueRecord{Name: snapshot.Name, IncarnationID: generation.IncarnationID, Cwd: "/snapshot/cwd", CreatedAt: int64(snapshot.CreatedAt), RecoveryState: domain.RecoveryHealthy, Committed: &checkpoint} | |
| 225 | 223 | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}) | |
| 226 | 224 | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | |
| 227 | - | WithSnapshotRepository(repository, nil)(d) | |
| 225 | + | WithSnapshotRepository(repository)(d) | |
| 226 | + | WithRecoveryCoordinator(recoveryusecase.NewCoordinator(catalogue, repository, nil, nil))(d) | |
| 228 | 227 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | |
| 229 | 228 | ||
| 230 | 229 | d.restoreIncrementalSnapshots(context.Background()) | |
| @@ -277,7 +276,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { | |||
| 277 | 276 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | |
| 278 | 277 | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil, nil) | |
| 279 | 278 | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | |
| 280 | - | WithSnapshotRepository(repository, nil)(d) | |
| 279 | + | WithSnapshotRepository(repository)(d) | |
| 281 | 280 | WithCheckpointCoordinator(coordinator)(d) | |
| 282 | 281 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | |
| 283 | 282 | ||
| @@ -291,7 +290,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { | |||
| 291 | 290 | updates := catalogue.MetadataUpdates() | |
| 292 | 291 | require.Len(t, updates, 1, "the authoritative Catalogue port must receive restored-session metadata updates") | |
| 293 | 292 | require.Equal(t, record.IncarnationID, updates[0].IncarnationID) | |
| 294 | - | updated, ok := catalogue.Record(record.Name) | |
| 293 | + | updated, ok, _ := catalogue.Record(record.Name) | |
| 295 | 294 | require.True(t, ok) | |
| 296 | 295 | require.Equal(t, record.IncarnationID, updated.IncarnationID) | |
| 297 | 296 | require.Equal(t, record.RecoveryState, updated.RecoveryState) | |
| @@ -310,7 +309,7 @@ func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { | |||
| 310 | 309 | require.Empty(t, failure) | |
| 311 | 310 | require.Len(t, repository.publishes, 1) | |
| 312 | 311 | require.Equal(t, record.Committed, repository.publishes[0].ParentCheckpoint) | |
| 313 | - | published, ok := catalogue.Record(record.Name) | |
| 312 | + | published, ok, _ := catalogue.Record(record.Name) | |
| 314 | 313 | require.True(t, ok) | |
| 315 | 314 | require.Equal(t, uint64(10), published.Committed.Generation) | |
| 316 | 315 | require.Equal(t, committed, *published.Fallbacks[0]) | |
M internal/usecase/daemon/snapshot_scheduler_test.go +27 -21
| @@ -19,7 +19,7 @@ import ( | |||
| 19 | 19 | func TestSnapshotCoordinatorQuarantineCancelsPublicationBeforeDelete(t *testing.T) { | |
| 20 | 20 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 21 | 21 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 22 | - | WithSnapshotRepository(repository, nil)(d) | |
| 22 | + | WithSnapshotRepository(repository)(d) | |
| 23 | 23 | startSnapshotEncodeWorker(t, d) | |
| 24 | 24 | ||
| 25 | 25 | started := make(chan struct{}) | |
| @@ -48,7 +48,7 @@ func TestSnapshotCoordinatorQuarantineCancelsPublicationBeforeDelete(t *testing. | |||
| 48 | 48 | func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) { | |
| 49 | 49 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 50 | 50 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 51 | - | WithSnapshotRepository(repository, nil)(d) | |
| 51 | + | WithSnapshotRepository(repository)(d) | |
| 52 | 52 | startSnapshotEncodeWorker(t, d) | |
| 53 | 53 | ||
| 54 | 54 | started := make(chan struct{}) | |
| @@ -84,7 +84,7 @@ func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) { | |||
| 84 | 84 | func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) { | |
| 85 | 85 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 86 | 86 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 87 | - | WithSnapshotRepository(repository, nil)(d) | |
| 87 | + | WithSnapshotRepository(repository)(d) | |
| 88 | 88 | startSnapshotEncodeWorker(t, d) | |
| 89 | 89 | ||
| 90 | 90 | started := make(chan struct{}) | |
| @@ -123,7 +123,7 @@ func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) { | |||
| 123 | 123 | func TestForcedSnapshotSchedulesSuccessorForRoutineCaptureQueuedBehindWorker(t *testing.T) { | |
| 124 | 124 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 125 | 125 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 126 | - | WithSnapshotRepository(repository, nil)(d) | |
| 126 | + | WithSnapshotRepository(repository)(d) | |
| 127 | 127 | startSnapshotEncodeWorker(t, d) | |
| 128 | 128 | ||
| 129 | 129 | blockerStarted := make(chan struct{}) | |
| @@ -164,7 +164,7 @@ func TestForcedSnapshotSchedulesSuccessorForRoutineCaptureQueuedBehindWorker(t * | |||
| 164 | 164 | func TestMultipleForcedSnapshotsCoalesceToOneSuccessor(t *testing.T) { | |
| 165 | 165 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 166 | 166 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 167 | - | WithSnapshotRepository(repository, nil)(d) | |
| 167 | + | WithSnapshotRepository(repository)(d) | |
| 168 | 168 | startSnapshotEncodeWorker(t, d) | |
| 169 | 169 | ||
| 170 | 170 | started := make(chan struct{}) | |
| @@ -207,7 +207,7 @@ func TestForcedSnapshotShutdownTimeoutRetainsRetryableStateAndNotice(t *testing. | |||
| 207 | 207 | clock := newSnapshotDeadlineClock() | |
| 208 | 208 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | |
| 209 | 209 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 210 | - | WithSnapshotRepository(repository, nil)(d) | |
| 210 | + | WithSnapshotRepository(repository)(d) | |
| 211 | 211 | notices := portsmocks.NewMockNoticeStore(t) | |
| 212 | 212 | WithNoticeStore(notices)(d) | |
| 213 | 213 | startSnapshotEncodeWorker(t, d) | |
| @@ -290,7 +290,7 @@ func TestServeShutdownDeadlineStillJoinsPaneExitWriter(t *testing.T) { | |||
| 290 | 290 | clock := newServeShutdownClock() | |
| 291 | 291 | d := newTestDaemon(t, newFactory(t, pty), clock) | |
| 292 | 292 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 293 | - | WithSnapshotRepository(repository, nil)(d) | |
| 293 | + | WithSnapshotRepository(repository)(d) | |
| 294 | 294 | ||
| 295 | 295 | publicationStarted := make(chan struct{}) | |
| 296 | 296 | releasePublication := make(chan struct{}) | |
| @@ -305,7 +305,7 @@ func TestServeShutdownDeadlineStillJoinsPaneExitWriter(t *testing.T) { | |||
| 305 | 305 | ||
| 306 | 306 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | |
| 307 | 307 | listener := serveSnapshotListener(t, tr) | |
| 308 | - | ctx, cancel := context.WithCancel(context.Background()) | |
| 308 | + | ctx, cancel := context.WithCancel(t.Context()) | |
| 309 | 309 | defer cancel() | |
| 310 | 310 | served := make(chan error, 1) | |
| 311 | 311 | go func() { served <- d.Serve(ctx, listener) }() | |
| @@ -382,7 +382,7 @@ func awaitTeardownWaiters(t *testing.T, sess *session, want uint) { | |||
| 382 | 382 | func TestConcurrentSessionTeardownPublishesOneFinalSnapshot(t *testing.T) { | |
| 383 | 383 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 384 | 384 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 385 | - | WithSnapshotRepository(repository, nil)(d) | |
| 385 | + | WithSnapshotRepository(repository)(d) | |
| 386 | 386 | startSnapshotEncodeWorker(t, d) | |
| 387 | 387 | ||
| 388 | 388 | publicationStarted := make(chan struct{}) | |
| @@ -456,7 +456,7 @@ func TestServeShutdownCheckpointsBeforeStoppingSnapshotWorker(t *testing.T) { | |||
| 456 | 456 | defer releasePTY() | |
| 457 | 457 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | |
| 458 | 458 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 459 | - | WithSnapshotRepository(repository, nil)(d) | |
| 459 | + | WithSnapshotRepository(repository)(d) | |
| 460 | 460 | ||
| 461 | 461 | published := make(chan ports.SnapshotPublication, 2) | |
| 462 | 462 | allowTerminalPublication := make(chan struct{}) | |
| @@ -470,7 +470,7 @@ func TestServeShutdownCheckpointsBeforeStoppingSnapshotWorker(t *testing.T) { | |||
| 470 | 470 | ||
| 471 | 471 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | |
| 472 | 472 | l := serveSnapshotListener(t, tr) | |
| 473 | - | ctx, cancel := context.WithCancel(context.Background()) | |
| 473 | + | ctx, cancel := context.WithCancel(t.Context()) | |
| 474 | 474 | defer cancel() | |
| 475 | 475 | served := make(chan error, 1) | |
| 476 | 476 | go func() { served <- d.Serve(ctx, l) }() | |
| @@ -500,14 +500,15 @@ func TestServeShutdownDeadlineStillJoinsUncooperativeSnapshotRepository(t *testi | |||
| 500 | 500 | clock := newServeShutdownClock() | |
| 501 | 501 | d := newTestDaemon(t, newFactory(t, p), clock) | |
| 502 | 502 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 503 | - | WithSnapshotRepository(repository, nil)(d) | |
| 503 | + | WithSnapshotRepository(repository)(d) | |
| 504 | 504 | noticeStore := portsmocks.NewMockNoticeStore(t) | |
| 505 | 505 | WithNoticeStore(noticeStore)(d) | |
| 506 | 506 | noticeStore.EXPECT().Claim().Return(nil, nil).Maybe() | |
| 507 | 507 | noticeStore.EXPECT().Ack().Return(nil).Maybe() | |
| 508 | + | noticePersisted := make(chan struct{}) | |
| 508 | 509 | noticeStore.EXPECT().Append(mock.MatchedBy(func(n domain.Notification) bool { | |
| 509 | 510 | return n.Code == domain.NoticeSnapshotWrite && n.Severity == domain.NoticeError | |
| 510 | - | })).Return(nil).Once() | |
| 511 | + | })).Run(func(domain.Notification) { close(noticePersisted) }).Return(nil).Once() | |
| 511 | 512 | ||
| 512 | 513 | firstPublicationCompleted := make(chan struct{}) | |
| 513 | 514 | terminalPublicationStarted := make(chan struct{}) | |
| @@ -526,7 +527,7 @@ func TestServeShutdownDeadlineStillJoinsUncooperativeSnapshotRepository(t *testi | |||
| 526 | 527 | ||
| 527 | 528 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | |
| 528 | 529 | l := serveSnapshotListener(t, tr) | |
| 529 | - | ctx, cancel := context.WithCancel(context.Background()) | |
| 530 | + | ctx, cancel := context.WithCancel(t.Context()) | |
| 530 | 531 | defer cancel() | |
| 531 | 532 | served := make(chan error, 1) | |
| 532 | 533 | go func() { served <- d.Serve(ctx, l) }() | |
| @@ -551,6 +552,11 @@ func TestServeShutdownDeadlineStillJoinsUncooperativeSnapshotRepository(t *testi | |||
| 551 | 552 | d.snapshotWorkerMu.Unlock() | |
| 552 | 553 | require.NotNil(t, workerDone) | |
| 553 | 554 | clock.nextFinalTimer(t).fire() | |
| 555 | + | select { | |
| 556 | + | case <-noticePersisted: | |
| 557 | + | case <-time.After(testWaitTimeout): | |
| 558 | + | t.Fatal("shutdown timeout notice was not persisted before the writer join") | |
| 559 | + | } | |
| 554 | 560 | ||
| 555 | 561 | select { | |
| 556 | 562 | case err := <-served: | |
| @@ -632,7 +638,7 @@ func TestRoutineSnapshotEligibilityStartsAtCompletionAndForcedDoesNotMoveIt(t *t | |||
| 632 | 638 | clock := &snapshotSchedulerClock{now: time.Unix(100, 0)} | |
| 633 | 639 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | |
| 634 | 640 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 635 | - | WithSnapshotRepository(repository, nil)(d) | |
| 641 | + | WithSnapshotRepository(repository)(d) | |
| 636 | 642 | startSnapshotEncodeWorker(t, d) | |
| 637 | 643 | ||
| 638 | 644 | var publishes atomic.Int32 | |
| @@ -780,7 +786,7 @@ func TestSnapshotCompletionSchedulesRoutineRetriesFromCompletion(t *testing.T) { | |||
| 780 | 786 | clock := newDeterministicSnapshotClock(now) | |
| 781 | 787 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | |
| 782 | 788 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 783 | - | WithSnapshotRepository(repository, nil)(d) | |
| 789 | + | WithSnapshotRepository(repository)(d) | |
| 784 | 790 | ||
| 785 | 791 | for _, tt := range []struct { | |
| 786 | 792 | name string | |
| @@ -813,12 +819,12 @@ func TestSnapshotSchedulerImmediateEligibilityAndStaleCapturesRemainDirty(t *tes | |||
| 813 | 819 | now := time.Unix(100, 0) | |
| 814 | 820 | clock := newDeterministicSnapshotClock(now) | |
| 815 | 821 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | |
| 816 | - | WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t), nil)(d) | |
| 822 | + | WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t))(d) | |
| 817 | 823 | ||
| 818 | 824 | t.Run("no prior attempt is immediately eligible", func(t *testing.T) { | |
| 819 | 825 | published := make(chan ports.SnapshotPublication, 1) | |
| 820 | 826 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 821 | - | WithSnapshotRepository(repository, nil)(d) | |
| 827 | + | WithSnapshotRepository(repository)(d) | |
| 822 | 828 | startSnapshotEncodeWorker(t, d) | |
| 823 | 829 | repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error { | |
| 824 | 830 | published <- publication | |
| @@ -873,7 +879,7 @@ func TestSnapshotWorkerQueueIsBoundedAndForcedCapturesCoalesce(t *testing.T) { | |||
| 873 | 879 | clock := newDeterministicSnapshotClock(now) | |
| 874 | 880 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | |
| 875 | 881 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 876 | - | WithSnapshotRepository(repository, nil)(d) | |
| 882 | + | WithSnapshotRepository(repository)(d) | |
| 877 | 883 | startSnapshotEncodeWorker(t, d) | |
| 878 | 884 | ||
| 879 | 885 | started := make(chan struct{}) | |
| @@ -931,7 +937,7 @@ func TestSnapshotRepositorySaverUsesEarliestDeadlineAndRecomputes(t *testing.T) | |||
| 931 | 937 | clock := newDeterministicSnapshotClock(now) | |
| 932 | 938 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | |
| 933 | 939 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 934 | - | WithSnapshotRepository(repository, nil)(d) | |
| 940 | + | WithSnapshotRepository(repository)(d) | |
| 935 | 941 | startSnapshotEncodeWorker(t, d) | |
| 936 | 942 | ||
| 937 | 943 | published := make(chan ports.SnapshotPublication, 1) | |
| @@ -946,7 +952,7 @@ func TestSnapshotRepositorySaverUsesEarliestDeadlineAndRecomputes(t *testing.T) | |||
| 946 | 952 | later.snapshotNextEligibleAt = now.Add(10 * time.Minute) | |
| 947 | 953 | d.sessions[later.id] = later | |
| 948 | 954 | ||
| 949 | - | ctx, cancel := context.WithCancel(context.Background()) | |
| 955 | + | ctx, cancel := context.WithCancel(t.Context()) | |
| 950 | 956 | done := make(chan struct{}) | |
| 951 | 957 | go func() { | |
| 952 | 958 | d.snapshotRepositorySaver(ctx) | |
M internal/usecase/daemon/snapshot_test.go +39 -1
| @@ -17,10 +17,45 @@ import ( | |||
| 17 | 17 | ||
| 18 | 18 | // TestSnapshotWorkerPublishesContentAddressedCapture verifies that all new | |
| 19 | 19 | // checkpoints use the repository publication contract. | |
| 20 | + | func TestWithSnapshotRepositoryRejectsTypedNil(t *testing.T) { | |
| 21 | + | var repository *snapshotAcceptanceRepository | |
| 22 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 23 | + | ||
| 24 | + | WithSnapshotRepository(repository)(d) | |
| 25 | + | ||
| 26 | + | require.Nil(t, d.snapshotRepository) | |
| 27 | + | require.False(t, d.snapsEnabled) | |
| 28 | + | } | |
| 29 | + | ||
| 30 | + | func TestDurableWriterFailureNamesIncludesBufferedCapture(t *testing.T) { | |
| 31 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 32 | + | capture := &snapshotCapture{name: "work", session: newSnapshotTestSession(t, "work", false, "/work")} | |
| 33 | + | d.snapshotWorkerMu.Lock() | |
| 34 | + | d.snapshotAdmitted[capture] = struct{}{} | |
| 35 | + | d.snapshotJobs <- capture | |
| 36 | + | d.snapshotWorkerMu.Unlock() | |
| 37 | + | ||
| 38 | + | require.Equal(t, []string{"work"}, d.durableWriterFailureNames()) | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | func TestCheckpointPublicationWithoutCommittedRefFailsCapture(t *testing.T) { | |
| 42 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 43 | + | WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t))(d) | |
| 44 | + | coordinator := portsmocks.NewMockCheckpointCoordinator(t) | |
| 45 | + | WithCheckpointCoordinator(coordinator)(d) | |
| 46 | + | startSnapshotEncodeWorker(t, d) | |
| 47 | + | coordinator.EXPECT().PublishCheckpoint(mock.Anything, "work", mock.Anything).Return(domain.CatalogueRecord{Name: "work"}, nil).Once() | |
| 48 | + | ||
| 49 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |
| 50 | + | require.True(t, d.captureSession(sess)) | |
| 51 | + | awaitSnapshotIdle(t, sess) | |
| 52 | + | require.True(t, sess.snapDirty.Load(), "nil committed publication must remain retryable") | |
| 53 | + | } | |
| 54 | + | ||
| 20 | 55 | func TestSnapshotWorkerPublishesContentAddressedCapture(t *testing.T) { | |
| 21 | 56 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |
| 22 | 57 | repository := portsmocks.NewMockSnapshotRepository(t) | |
| 23 | - | WithSnapshotRepository(repository, nil)(d) | |
| 58 | + | WithSnapshotRepository(repository)(d) | |
| 24 | 59 | startSnapshotEncodeWorker(t, d) | |
| 25 | 60 | ||
| 26 | 61 | repository.EXPECT().Publish(mock.Anything, mock.MatchedBy(func(p ports.SnapshotPublication) bool { | |
| @@ -57,6 +92,9 @@ func newSnapshotTestSession(t *testing.T, name string, ephemeral bool, cwd strin | |||
| 57 | 92 | // focused test sinks that only need to observe publication or deletion calls. | |
| 58 | 93 | type noOpSnapshotRepository struct{} | |
| 59 | 94 | ||
| 95 | + | var _ ports.SnapshotRepository = noOpSnapshotRepository{} | |
| 96 | + | ||
| 97 | + | func (noOpSnapshotRepository) Publish(context.Context, ports.SnapshotPublication) error { return nil } | |
| 60 | 98 | func (noOpSnapshotRepository) LoadCheckpoint(context.Context, domain.IncarnationID, string, ports.CheckpointRef) (ports.SnapshotGeneration, error) { | |
| 61 | 99 | return ports.SnapshotGeneration{}, errors.New("unused") | |
| 62 | 100 | } | |
M internal/usecase/daemon/snapshot_worker.go +18 -3
| @@ -2,6 +2,7 @@ package daemon | |||
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | + | "errors" | |
| 5 | 6 | "sort" | |
| 6 | 7 | ||
| 7 | 8 | "github.com/bnema/vev/internal/domain" | |
| @@ -134,8 +135,12 @@ func (d *Daemon) publishSnapshotCapture(workerCtx context.Context, workerID uint | |||
| 134 | 135 | if d.checkpointRecovery != nil { | |
| 135 | 136 | var record domain.CatalogueRecord | |
| 136 | 137 | record, err = d.checkpointRecovery.PublishCheckpoint(publicationContext, capture.name, publication) | |
| 137 | - | if err == nil && record.Committed != nil { | |
| 138 | - | capture.checkpoint = *record.Committed | |
| 138 | + | if err == nil { | |
| 139 | + | if record.Committed == nil { | |
| 140 | + | err = errors.New("snapshot: checkpoint publication returned no committed checkpoint") | |
| 141 | + | } else { | |
| 142 | + | capture.checkpoint = *record.Committed | |
| 143 | + | } | |
| 139 | 144 | } | |
| 140 | 145 | } else { | |
| 141 | 146 | err = d.snapshotRepository.Publish(publicationContext, publication) | |
| @@ -217,8 +222,12 @@ func (d *Daemon) StopDurableWriters(ctx context.Context) []string { | |||
| 217 | 222 | case <-done: | |
| 218 | 223 | return nil | |
| 219 | 224 | case <-ctx.Done(): | |
| 225 | + | // Snapshot identity before cancellation can let a cooperative worker clear | |
| 226 | + | // its in-flight capture. The caller persists timeout notices before the | |
| 227 | + | // unconditional ownership join. | |
| 228 | + | names := d.durableWriterFailureNames() | |
| 220 | 229 | cancel() | |
| 221 | - | return d.durableWriterFailureNames() | |
| 230 | + | return names | |
| 222 | 231 | } | |
| 223 | 232 | } | |
| 224 | 233 | ||
| @@ -267,6 +276,9 @@ func (d *Daemon) requestDurableWriterStop() (context.CancelFunc, <-chan struct{} | |||
| 267 | 276 | return cancel, d.snapshotWorkerDone | |
| 268 | 277 | } | |
| 269 | 278 | ||
| 279 | + | // durableWriterFailureNames includes admitted buffered captures as well as the | |
| 280 | + | // active and final queues. snapshotAdmitted tracks normal captures from queue | |
| 281 | + | // admission through completion, so worker dequeue cannot make one disappear. | |
| 270 | 282 | func (d *Daemon) durableWriterFailureNames() []string { | |
| 271 | 283 | d.snapshotWorkerMu.Lock() | |
| 272 | 284 | defer d.snapshotWorkerMu.Unlock() | |
| @@ -280,6 +292,9 @@ func (d *Daemon) durableWriterFailureNames() []string { | |||
| 280 | 292 | for _, capture := range d.snapshotFinalJobs { | |
| 281 | 293 | add(capture) | |
| 282 | 294 | } | |
| 295 | + | for capture := range d.snapshotAdmitted { | |
| 296 | + | add(capture) | |
| 297 | + | } | |
| 283 | 298 | names := make([]string, 0, len(seen)) | |
| 284 | 299 | for name := range seen { | |
| 285 | 300 | names = append(names, name) | |
M internal/usecase/recovery/checkpoint.go +12 -15
| @@ -37,7 +37,10 @@ func (c *Coordinator) PublishCheckpoint(ctx context.Context, name string, public | |||
| 37 | 37 | unlock := c.locks.Lock([]string{name}) | |
| 38 | 38 | defer unlock() | |
| 39 | 39 | ||
| 40 | - | record, ok := c.catalogue.Record(name) | |
| 40 | + | record, ok, err := c.catalogue.Record(name) | |
| 41 | + | if err != nil { | |
| 42 | + | return domain.CatalogueRecord{}, err | |
| 43 | + | } | |
| 41 | 44 | if !ok { | |
| 42 | 45 | return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound | |
| 43 | 46 | } | |
| @@ -74,7 +77,10 @@ func (c *Coordinator) PromoteFallback(ctx context.Context, name string, ref doma | |||
| 74 | 77 | } | |
| 75 | 78 | ||
| 76 | 79 | func (c *Coordinator) promoteFallbackLocked(ctx context.Context, name string, ref domain.CheckpointRef) (ports.FallbackPromotionOutcome, error) { | |
| 77 | - | record, ok := c.catalogue.Record(name) | |
| 80 | + | record, ok, err := c.catalogue.Record(name) | |
| 81 | + | if err != nil { | |
| 82 | + | return ports.FallbackPromotionOutcome{}, err | |
| 83 | + | } | |
| 78 | 84 | if !ok { | |
| 79 | 85 | return ports.FallbackPromotionOutcome{}, ErrCheckpointRecordNotFound | |
| 80 | 86 | } | |
| @@ -129,24 +135,15 @@ func (c *Coordinator) validateCheckpoint(ctx context.Context, record domain.Cata | |||
| 129 | 135 | if err != nil { | |
| 130 | 136 | return err | |
| 131 | 137 | } | |
| 132 | - | if generation.IncarnationID != record.IncarnationID || generation.Name != record.Name || generation.Generation != ref.Generation || | |
| 133 | - | sha256.Sum256(generation.Manifest) != ref.ManifestDigest { | |
| 134 | - | return ErrCheckpointConflict | |
| 135 | - | } | |
| 136 | - | return nil | |
| 138 | + | return validateLoadedGeneration(record, ref, generation) | |
| 137 | 139 | } | |
| 138 | 140 | ||
| 139 | - | func equalCheckpointRefs(a, b *domain.CheckpointRef) bool { | |
| 140 | - | if a == nil || b == nil { | |
| 141 | - | return a == nil && b == nil | |
| 142 | - | } | |
| 143 | - | return *a == *b | |
| 144 | - | } | |
| 141 | + | func equalCheckpointRefs(a, b *domain.CheckpointRef) bool { return a.Equal(b) } | |
| 145 | 142 | ||
| 146 | 143 | func copyCheckpointRef(ref *domain.CheckpointRef) *domain.CheckpointRef { | |
| 147 | 144 | if ref == nil { | |
| 148 | 145 | return nil | |
| 149 | 146 | } | |
| 150 | - | copy := *ref | |
| 151 | - | return © | |
| 147 | + | cloned := *ref | |
| 148 | + | return &cloned | |
| 152 | 149 | } | |
M internal/usecase/recovery/checkpoint_test.go +28 -26
| @@ -20,13 +20,15 @@ type checkpointCatalogue struct { | |||
| 20 | 20 | events *[]string | |
| 21 | 21 | } | |
| 22 | 22 | ||
| 23 | - | func (c *checkpointCatalogue) Records() []domain.CatalogueRecord { | |
| 24 | - | return []domain.CatalogueRecord{c.record} | |
| 23 | + | func (c *checkpointCatalogue) Records() ([]domain.CatalogueRecord, error) { | |
| 24 | + | c.mu.Lock() | |
| 25 | + | defer c.mu.Unlock() | |
| 26 | + | return []domain.CatalogueRecord{c.record}, nil | |
| 25 | 27 | } | |
| 26 | - | func (c *checkpointCatalogue) Record(name string) (domain.CatalogueRecord, bool) { | |
| 28 | + | func (c *checkpointCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) { | |
| 27 | 29 | c.mu.Lock() | |
| 28 | 30 | defer c.mu.Unlock() | |
| 29 | - | return c.record, c.record.Name == name | |
| 31 | + | return c.record, c.record.Name == name, nil | |
| 30 | 32 | } | |
| 31 | 33 | func (c *checkpointCatalogue) Create(domain.CatalogueRecord) error { return nil } | |
| 32 | 34 | func (c *checkpointCatalogue) UpdateMetadata(domain.CatalogueMetadataUpdate) error { | |
| @@ -127,33 +129,33 @@ func runCheckpointCommit(t *testing.T, count int) (domain.CatalogueRecord, *chec | |||
| 127 | 129 | catalogue.record, err = coordinator.PublishCheckpoint(context.Background(), "work", checkpointPublication(catalogue.record, generation)) | |
| 128 | 130 | require.NoError(t, err) | |
| 129 | 131 | } | |
| 130 | - | for i := 0; i < count; i++ { | |
| 132 | + | for i := range count { | |
| 131 | 133 | require.Equal(t, []string{"repository", "catalogue"}, events[i*2:i*2+2]) | |
| 132 | 134 | } | |
| 133 | 135 | return catalogue.record, catalogue, repository | |
| 134 | 136 | } | |
| 135 | 137 | ||
| 136 | - | func TestCheckpointCommitFirst(t *testing.T) { | |
| 137 | - | record, catalogue, _ := runCheckpointCommit(t, 1) | |
| 138 | - | require.Equal(t, uint64(1), record.Committed.Generation) | |
| 139 | - | require.Empty(t, populatedGenerations(record.Fallbacks)) | |
| 140 | - | require.Equal(t, 1, catalogue.replaces) | |
| 141 | - | } | |
| 142 | - | ||
| 143 | - | func TestCheckpointCommitSecond(t *testing.T) { | |
| 144 | - | record, catalogue, _ := runCheckpointCommit(t, 2) | |
| 145 | - | require.Equal(t, uint64(2), record.Committed.Generation) | |
| 146 | - | require.Equal(t, []uint64{1}, populatedGenerations(record.Fallbacks)) | |
| 147 | - | require.Equal(t, 2, catalogue.replaces) | |
| 148 | - | } | |
| 149 | - | ||
| 150 | - | func TestCheckpointCommitThird(t *testing.T) { | |
| 151 | - | record, catalogue, _ := runCheckpointCommit(t, 4) | |
| 152 | - | require.Equal(t, uint64(4), record.Committed.Generation) | |
| 153 | - | require.Equal(t, []uint64{3, 2}, populatedGenerations(record.Fallbacks)) | |
| 154 | - | require.Equal(t, domain.RecoveryHealthy, record.RecoveryState) | |
| 155 | - | require.Empty(t, record.DegradedReason) | |
| 156 | - | require.Equal(t, 4, catalogue.replaces) | |
| 138 | + | func TestCheckpointCommits(t *testing.T) { | |
| 139 | + | for _, tc := range []struct { | |
| 140 | + | name string | |
| 141 | + | count int | |
| 142 | + | wantCommitted uint64 | |
| 143 | + | wantFallbacks []uint64 | |
| 144 | + | }{ | |
| 145 | + | {name: "first", count: 1, wantCommitted: 1}, | |
| 146 | + | {name: "second", count: 2, wantCommitted: 2, wantFallbacks: []uint64{1}}, | |
| 147 | + | {name: "third", count: 3, wantCommitted: 3, wantFallbacks: []uint64{2, 1}}, | |
| 148 | + | {name: "fourth", count: 4, wantCommitted: 4, wantFallbacks: []uint64{3, 2}}, | |
| 149 | + | } { | |
| 150 | + | t.Run(tc.name, func(t *testing.T) { | |
| 151 | + | record, catalogue, _ := runCheckpointCommit(t, tc.count) | |
| 152 | + | require.Equal(t, tc.wantCommitted, record.Committed.Generation) | |
| 153 | + | require.Equal(t, tc.wantFallbacks, populatedGenerations(record.Fallbacks)) | |
| 154 | + | require.Equal(t, domain.RecoveryHealthy, record.RecoveryState) | |
| 155 | + | require.Empty(t, record.DegradedReason) | |
| 156 | + | require.Equal(t, tc.count, catalogue.replaces) | |
| 157 | + | }) | |
| 158 | + | } | |
| 157 | 159 | } | |
| 158 | 160 | ||
| 159 | 161 | func TestCheckpointCommitThirdPromotesOnlyValidatedDirectFallbacks(t *testing.T) { | |
M internal/usecase/recovery/coordinator.go +32 -8
| @@ -148,7 +148,9 @@ func (c *Coordinator) Create(ctx context.Context, record domain.CatalogueRecord) | |||
| 148 | 148 | } | |
| 149 | 149 | unlock := c.locks.Lock([]string{record.Name}) | |
| 150 | 150 | defer unlock() | |
| 151 | - | if _, exists := c.catalogue.Record(record.Name); exists { | |
| 151 | + | if _, exists, err := c.catalogue.Record(record.Name); err != nil { | |
| 152 | + | return domain.CatalogueRecord{}, err | |
| 153 | + | } else if exists { | |
| 152 | 154 | return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q already exists", record.Name) | |
| 153 | 155 | } | |
| 154 | 156 | id, err := domain.NewIncarnationID(c.random) | |
| @@ -180,11 +182,16 @@ func (c *Coordinator) Rename(ctx context.Context, oldName, newName string) (doma | |||
| 180 | 182 | } | |
| 181 | 183 | unlock := c.locks.Lock([]string{oldName, newName}) | |
| 182 | 184 | defer unlock() | |
| 183 | - | record, ok := c.catalogue.Record(oldName) | |
| 185 | + | record, ok, err := c.catalogue.Record(oldName) | |
| 186 | + | if err != nil { | |
| 187 | + | return domain.CatalogueRecord{}, err | |
| 188 | + | } | |
| 184 | 189 | if !ok { | |
| 185 | 190 | return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q not found", oldName) | |
| 186 | 191 | } | |
| 187 | - | if existing, exists := c.catalogue.Record(newName); exists && (newName != oldName || existing.IncarnationID != record.IncarnationID) { | |
| 192 | + | if existing, exists, err := c.catalogue.Record(newName); err != nil { | |
| 193 | + | return domain.CatalogueRecord{}, err | |
| 194 | + | } else if exists && (newName != oldName || existing.IncarnationID != record.IncarnationID) { | |
| 188 | 195 | return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q already exists", newName) | |
| 189 | 196 | } | |
| 190 | 197 | record.Name = newName | |
| @@ -207,7 +214,10 @@ func (c *Coordinator) Delete(ctx context.Context, name string) error { | |||
| 207 | 214 | } | |
| 208 | 215 | unlock := c.locks.Lock([]string{name}) | |
| 209 | 216 | defer unlock() | |
| 210 | - | record, ok := c.catalogue.Record(name) | |
| 217 | + | record, ok, err := c.catalogue.Record(name) | |
| 218 | + | if err != nil { | |
| 219 | + | return err | |
| 220 | + | } | |
| 211 | 221 | if !ok { | |
| 212 | 222 | return nil | |
| 213 | 223 | } | |
| @@ -267,13 +277,21 @@ func (c *Coordinator) Recover(ctx context.Context) error { | |||
| 267 | 277 | } | |
| 268 | 278 | } | |
| 269 | 279 | } | |
| 270 | - | for _, candidate := range c.catalogue.Records() { | |
| 280 | + | records, err := c.catalogue.Records() | |
| 281 | + | if err != nil { | |
| 282 | + | return err | |
| 283 | + | } | |
| 284 | + | for _, candidate := range records { | |
| 271 | 285 | if candidate.RecoveryState != domain.RecoveryDeleting { | |
| 272 | 286 | continue | |
| 273 | 287 | } | |
| 274 | 288 | unlock := c.locks.Lock([]string{candidate.Name}) | |
| 275 | 289 | var deleteErr error | |
| 276 | - | current, ok := c.catalogue.Record(candidate.Name) | |
| 290 | + | current, ok, readErr := c.catalogue.Record(candidate.Name) | |
| 291 | + | if readErr != nil { | |
| 292 | + | unlock() | |
| 293 | + | return readErr | |
| 294 | + | } | |
| 277 | 295 | if ok && current.IncarnationID == candidate.IncarnationID && current.RecoveryState == domain.RecoveryDeleting { | |
| 278 | 296 | deleteErr = c.deleteLocked(ctx, current) | |
| 279 | 297 | } | |
| @@ -323,7 +341,10 @@ func (c *Coordinator) fenceSession(name, kind, reason string, cause error) error | |||
| 323 | 341 | ||
| 324 | 342 | unlock := c.locks.Lock([]string{name}) | |
| 325 | 343 | defer unlock() | |
| 326 | - | record, ok := c.catalogue.Record(name) | |
| 344 | + | record, ok, err := c.catalogue.Record(name) | |
| 345 | + | if err != nil { | |
| 346 | + | return err | |
| 347 | + | } | |
| 327 | 348 | if !ok || record.RecoveryState != domain.RecoveryHealthy { | |
| 328 | 349 | // Fresh records cannot represent degraded state (it requires a committed | |
| 329 | 350 | // checkpoint), and deleting or already degraded records must keep the | |
| @@ -354,7 +375,10 @@ func (c *Coordinator) recoverDeletionTombstone(ctx context.Context, tombstone do | |||
| 354 | 375 | } | |
| 355 | 376 | unlock := c.locks.Lock([]string{tombstone.Name}) | |
| 356 | 377 | defer unlock() | |
| 357 | - | record, exists := c.catalogue.Record(tombstone.Name) | |
| 378 | + | record, exists, err := c.catalogue.Record(tombstone.Name) | |
| 379 | + | if err != nil { | |
| 380 | + | return err | |
| 381 | + | } | |
| 358 | 382 | includeLegacyName := true | |
| 359 | 383 | if exists { | |
| 360 | 384 | switch { | |
M internal/usecase/recovery/discard.go +25 -15
| @@ -39,7 +39,10 @@ func (c *Coordinator) Retry(ctx context.Context, name string) error { | |||
| 39 | 39 | } | |
| 40 | 40 | unlock := c.locks.Lock([]string{name}) | |
| 41 | 41 | defer unlock() | |
| 42 | - | record, ok := c.catalogue.Record(name) | |
| 42 | + | record, ok, err := c.catalogue.Record(name) | |
| 43 | + | if err != nil { | |
| 44 | + | return err | |
| 45 | + | } | |
| 43 | 46 | if !ok { | |
| 44 | 47 | return ErrRecoveryRecordNotFound | |
| 45 | 48 | } | |
| @@ -66,7 +69,10 @@ func (c *Coordinator) RestoreFallback(ctx context.Context, name string, ref doma | |||
| 66 | 69 | } | |
| 67 | 70 | unlock := c.locks.Lock([]string{name}) | |
| 68 | 71 | defer unlock() | |
| 69 | - | record, ok := c.catalogue.Record(name) | |
| 72 | + | record, ok, err := c.catalogue.Record(name) | |
| 73 | + | if err != nil { | |
| 74 | + | return err | |
| 75 | + | } | |
| 70 | 76 | if !ok { | |
| 71 | 77 | return ErrRecoveryRecordNotFound | |
| 72 | 78 | } | |
| @@ -88,7 +94,10 @@ func (c *Coordinator) Export(ctx context.Context, name string, w io.Writer) erro | |||
| 88 | 94 | } | |
| 89 | 95 | unlock := c.locks.Lock([]string{name}) | |
| 90 | 96 | defer unlock() | |
| 91 | - | record, ok := c.catalogue.Record(name) | |
| 97 | + | record, ok, err := c.catalogue.Record(name) | |
| 98 | + | if err != nil { | |
| 99 | + | return err | |
| 100 | + | } | |
| 92 | 101 | if !ok { | |
| 93 | 102 | return ErrRecoveryRecordNotFound | |
| 94 | 103 | } | |
| @@ -167,7 +176,10 @@ func (c *Coordinator) Discard(ctx context.Context, name, reason string) (domain. | |||
| 167 | 176 | } | |
| 168 | 177 | unlock := c.locks.Lock([]string{name}) | |
| 169 | 178 | defer unlock() | |
| 170 | - | record, ok := c.catalogue.Record(name) | |
| 179 | + | record, ok, err := c.catalogue.Record(name) | |
| 180 | + | if err != nil { | |
| 181 | + | return domain.CatalogueRecord{}, err | |
| 182 | + | } | |
| 171 | 183 | if !ok { | |
| 172 | 184 | return domain.CatalogueRecord{}, ErrRecoveryRecordNotFound | |
| 173 | 185 | } | |
| @@ -207,9 +219,15 @@ func (c *Coordinator) recoverDiscardLocked(ctx context.Context, intent domain.Di | |||
| 207 | 219 | if err := validateDiscardIntent(intent); err != nil { | |
| 208 | 220 | return err | |
| 209 | 221 | } | |
| 210 | - | current, ok := c.catalogue.Record(intent.SessionName) | |
| 222 | + | current, ok, err := c.catalogue.Record(intent.SessionName) | |
| 223 | + | if err != nil { | |
| 224 | + | return err | |
| 225 | + | } | |
| 211 | 226 | if !ok { | |
| 212 | - | return fmt.Errorf("%w: missing session %q", ErrDiscardConflict, intent.SessionName) | |
| 227 | + | // Catalogue deletion is terminal for this session identity. There is no | |
| 228 | + | // live record to replace or quarantine, so retaining the intent would | |
| 229 | + | // manufacture a permanent startup conflict. | |
| 230 | + | return c.journal.DeleteDiscard(ctx, intent.OldIncarnation) | |
| 213 | 231 | } | |
| 214 | 232 | next := freshReplacement(intent) | |
| 215 | 233 | if err := next.Validate(); err != nil { | |
| @@ -217,7 +235,7 @@ func (c *Coordinator) recoverDiscardLocked(ctx context.Context, intent domain.Di | |||
| 217 | 235 | } | |
| 218 | 236 | switch current.IncarnationID { | |
| 219 | 237 | case intent.OldIncarnation: | |
| 220 | - | if !catalogueRecordsEqual(current, intent.OldRecord) { | |
| 238 | + | if !current.Equal(intent.OldRecord) { | |
| 221 | 239 | return fmt.Errorf("%w: old record changed", ErrDiscardConflict) | |
| 222 | 240 | } | |
| 223 | 241 | case intent.NewIncarnation: | |
| @@ -275,11 +293,3 @@ func freshReplacement(intent domain.DiscardIntent) domain.CatalogueRecord { | |||
| 275 | 293 | next.DegradedReason = "" | |
| 276 | 294 | return next | |
| 277 | 295 | } | |
| 278 | - | ||
| 279 | - | func catalogueRecordsEqual(a, b domain.CatalogueRecord) bool { | |
| 280 | - | return a.Name == b.Name && a.IncarnationID == b.IncarnationID && a.Cwd == b.Cwd && | |
| 281 | - | a.CreatedAt == b.CreatedAt && a.UpdatedAt == b.UpdatedAt && a.LastUsedSeq == b.LastUsedSeq && | |
| 282 | - | slices.Equal(a.TabNames, b.TabNames) && a.RecoveryState == b.RecoveryState && | |
| 283 | - | equalCheckpointRefs(a.Committed, b.Committed) && equalCheckpointRefs(a.Fallbacks[0], b.Fallbacks[0]) && | |
| 284 | - | equalCheckpointRefs(a.Fallbacks[1], b.Fallbacks[1]) && a.DegradedReason == b.DegradedReason | |
| 285 | - | } | |
M internal/usecase/recovery/reconcile.go +26 -5
| @@ -29,23 +29,41 @@ func (r *Reconciler) Step(ctx context.Context, cursor ports.ReconcileCursor) (po | |||
| 29 | 29 | if err := ctx.Err(); err != nil { | |
| 30 | 30 | return cursor, nil, err | |
| 31 | 31 | } | |
| 32 | - | next, findings, err := r.repository.Reconcile(ctx, r.catalogue.Records(), cursor, r.budget) | |
| 32 | + | records, err := r.catalogue.Records() | |
| 33 | + | if err != nil { | |
| 34 | + | return cursor, nil, err | |
| 35 | + | } | |
| 36 | + | next, findings, err := r.repository.Reconcile(ctx, records, cursor, r.budget) | |
| 33 | 37 | if err != nil { | |
| 34 | 38 | return cursor, nil, err | |
| 35 | 39 | } | |
| 36 | 40 | decisions := make([]ports.ReconcileDecision, 0, len(findings)) | |
| 41 | + | consumed := ports.MaintenanceBudget{} | |
| 37 | 42 | for _, finding := range findings { | |
| 38 | - | if finding.Consumed.Entries > r.budget.Entries || finding.Consumed.Bytes > r.budget.Bytes { | |
| 43 | + | // Compare against the remainder before adding. This both enforces the | |
| 44 | + | // page-wide budget and avoids uint64 wraparound. | |
| 45 | + | if finding.Consumed.Entries > r.budget.Entries-consumed.Entries || | |
| 46 | + | finding.Consumed.Bytes > r.budget.Bytes-consumed.Bytes { | |
| 39 | 47 | return cursor, nil, errors.New("recovery: repository exceeded reconciliation budget") | |
| 40 | 48 | } | |
| 41 | - | record, ok := r.catalogue.Record(finding.Candidate.Name) | |
| 49 | + | consumed.Entries += finding.Consumed.Entries | |
| 50 | + | consumed.Bytes += finding.Consumed.Bytes | |
| 51 | + | record, ok, err := r.catalogue.Record(finding.Candidate.Name) | |
| 52 | + | if err != nil { | |
| 53 | + | return cursor, nil, err | |
| 54 | + | } | |
| 42 | 55 | if !ok { | |
| 43 | 56 | record = domain.CatalogueRecord{} | |
| 44 | 57 | } | |
| 45 | 58 | decision := classifyFinding(record, finding) | |
| 46 | 59 | if decision.Kind == ports.ReconcileAdopt { | |
| 47 | 60 | if _, err := r.coordinator.PublishReconciledCheckpoint(ctx, finding.Candidate.Name, finding.Candidate, finding.AncestorChain); err != nil { | |
| 48 | - | return cursor, nil, err | |
| 61 | + | if !errors.Is(err, ErrCheckpointConflict) && !errors.Is(err, ErrCheckpointRecordNotFound) { | |
| 62 | + | return cursor, nil, err | |
| 63 | + | } | |
| 64 | + | decision.Kind = ports.ReconcileDefer | |
| 65 | + | decision.ReasonCode = "catalogue-conflict" | |
| 66 | + | decision.RetentionResolved = false | |
| 49 | 67 | } | |
| 50 | 68 | } | |
| 51 | 69 | decisions = append(decisions, decision) | |
| @@ -132,7 +150,10 @@ func (c *Coordinator) PublishReconciledCheckpoint(ctx context.Context, name stri | |||
| 132 | 150 | if err := ctx.Err(); err != nil { | |
| 133 | 151 | return domain.CatalogueRecord{}, err | |
| 134 | 152 | } | |
| 135 | - | record, ok := c.catalogue.Record(name) | |
| 153 | + | record, ok, err := c.catalogue.Record(name) | |
| 154 | + | if err != nil { | |
| 155 | + | return domain.CatalogueRecord{}, err | |
| 156 | + | } | |
| 136 | 157 | if !ok { | |
| 137 | 158 | return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound | |
| 138 | 159 | } | |
M internal/usecase/recovery/reconcile_test.go +77 -0
| @@ -24,6 +24,33 @@ type reconcilePage struct { | |||
| 24 | 24 | err error | |
| 25 | 25 | } | |
| 26 | 26 | ||
| 27 | + | type reconcileConflictCatalogue struct { | |
| 28 | + | *transactionCatalogue | |
| 29 | + | stale domain.CatalogueRecord | |
| 30 | + | reads int | |
| 31 | + | } | |
| 32 | + | ||
| 33 | + | func (c *reconcileConflictCatalogue) Records() ([]domain.CatalogueRecord, error) { | |
| 34 | + | records, err := c.transactionCatalogue.Records() | |
| 35 | + | if err != nil { | |
| 36 | + | return nil, err | |
| 37 | + | } | |
| 38 | + | for i := range records { | |
| 39 | + | if records[i].Name == c.stale.Name { | |
| 40 | + | records[i] = c.stale | |
| 41 | + | } | |
| 42 | + | } | |
| 43 | + | return records, nil | |
| 44 | + | } | |
| 45 | + | ||
| 46 | + | func (c *reconcileConflictCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) { | |
| 47 | + | if name == c.stale.Name && c.reads == 0 { | |
| 48 | + | c.reads++ | |
| 49 | + | return c.stale, true, nil | |
| 50 | + | } | |
| 51 | + | return c.transactionCatalogue.Record(name) | |
| 52 | + | } | |
| 53 | + | ||
| 27 | 54 | func (r *reconcileRepository) Reconcile(_ context.Context, _ []domain.CatalogueRecord, cursor ports.ReconcileCursor, budget ports.MaintenanceBudget) (ports.ReconcileCursor, []ports.ReconcileFinding, error) { | |
| 28 | 55 | r.seenCursors = append(r.seenCursors, cursor) | |
| 29 | 56 | r.seenBudgets = append(r.seenBudgets, budget) | |
| @@ -64,6 +91,56 @@ func TestReconciliationCursor(t *testing.T) { | |||
| 64 | 91 | require.Equal(t, []ports.MaintenanceBudget{{Entries: 2, Bytes: 32}, {Entries: 2, Bytes: 32}}, repository.seenBudgets) | |
| 65 | 92 | } | |
| 66 | 93 | ||
| 94 | + | func TestReconciliationRejectsCumulativeAndOverflowingBudgets(t *testing.T) { | |
| 95 | + | record := healthyReconcileRecord() | |
| 96 | + | finding := ports.ReconcileFinding{Status: ports.ReconcileBudgetExhausted, Candidate: ports.ReconcileCandidate{Name: record.Name}} | |
| 97 | + | for _, tc := range []struct { | |
| 98 | + | name string | |
| 99 | + | budget ports.MaintenanceBudget | |
| 100 | + | consumed []ports.MaintenanceBudget | |
| 101 | + | }{ | |
| 102 | + | {name: "cumulative entries", budget: ports.MaintenanceBudget{Entries: 3, Bytes: 10}, consumed: []ports.MaintenanceBudget{{Entries: 2, Bytes: 1}, {Entries: 2, Bytes: 1}}}, | |
| 103 | + | {name: "cumulative bytes", budget: ports.MaintenanceBudget{Entries: 10, Bytes: 3}, consumed: []ports.MaintenanceBudget{{Entries: 1, Bytes: 2}, {Entries: 1, Bytes: 2}}}, | |
| 104 | + | {name: "overflow safe", budget: ports.MaintenanceBudget{Entries: ^uint64(0), Bytes: ^uint64(0)}, consumed: []ports.MaintenanceBudget{{Entries: ^uint64(0), Bytes: ^uint64(0)}, {Entries: 1, Bytes: 1}}}, | |
| 105 | + | } { | |
| 106 | + | t.Run(tc.name, func(t *testing.T) { | |
| 107 | + | findings := make([]ports.ReconcileFinding, len(tc.consumed)) | |
| 108 | + | for i, usage := range tc.consumed { | |
| 109 | + | findings[i] = finding | |
| 110 | + | findings[i].Consumed = usage | |
| 111 | + | } | |
| 112 | + | repository := &reconcileRepository{pages: map[uint64]reconcilePage{0: {findings: findings}}} | |
| 113 | + | catalogue := &checkpointCatalogue{record: record} | |
| 114 | + | reconciler := NewReconciler(NewCoordinator(catalogue, repository, nil, nil), catalogue, repository, tc.budget) | |
| 115 | + | _, _, err := reconciler.Step(context.Background(), ports.ReconcileCursor{}) | |
| 116 | + | require.EqualError(t, err, "recovery: repository exceeded reconciliation budget") | |
| 117 | + | }) | |
| 118 | + | } | |
| 119 | + | } | |
| 120 | + | ||
| 121 | + | func TestReconciliationDefersConflictAndContinues(t *testing.T) { | |
| 122 | + | first := healthyReconcileRecord() | |
| 123 | + | first.Name = "first" | |
| 124 | + | staleRecord := first | |
| 125 | + | stale := forwardCandidate(staleRecord, 2, staleRecord.Committed) | |
| 126 | + | first = shiftedCheckpoint(first, domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{9}}) | |
| 127 | + | second := healthyReconcileRecord() | |
| 128 | + | second.Name = "second" | |
| 129 | + | adoptable := forwardCandidate(second, 2, second.Committed) | |
| 130 | + | catalogue := &reconcileConflictCatalogue{transactionCatalogue: newTransactionCatalogue(first, second), stale: staleRecord} | |
| 131 | + | repository := &reconcileRepository{pages: map[uint64]reconcilePage{0: {findings: []ports.ReconcileFinding{ | |
| 132 | + | {Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: stale, Consumed: ports.MaintenanceBudget{Entries: 1, Bytes: 1}}, | |
| 133 | + | {Kind: ports.ReconcileForwardOrphan, Status: ports.ReconcileValidated, Candidate: adoptable, Consumed: ports.MaintenanceBudget{Entries: 1, Bytes: 1}}, | |
| 134 | + | }}}} | |
| 135 | + | reconciler := NewReconciler(NewCoordinator(catalogue, repository, nil, nil), catalogue, repository, ports.MaintenanceBudget{Entries: 2, Bytes: 2}) | |
| 136 | + | ||
| 137 | + | _, decisions, err := reconciler.Step(context.Background(), ports.ReconcileCursor{}) | |
| 138 | + | require.NoError(t, err) | |
| 139 | + | require.Equal(t, []ports.ReconcileDecisionKind{ports.ReconcileDefer, ports.ReconcileAdopt}, []ports.ReconcileDecisionKind{decisions[0].Kind, decisions[1].Kind}) | |
| 140 | + | require.Equal(t, "catalogue-conflict", decisions[0].ReasonCode) | |
| 141 | + | require.Equal(t, adoptable.Ref, *catalogue.records[second.Name].Committed) | |
| 142 | + | } | |
| 143 | + | ||
| 67 | 144 | func TestForwardOrphanAdoption(t *testing.T) { | |
| 68 | 145 | for _, tc := range []struct { | |
| 69 | 146 | name string | |
M internal/usecase/recovery/transactions_test.go +21 -10
| @@ -27,20 +27,20 @@ func newTransactionCatalogue(records ...domain.CatalogueRecord) *transactionCata | |||
| 27 | 27 | return catalogue | |
| 28 | 28 | } | |
| 29 | 29 | ||
| 30 | - | func (c *transactionCatalogue) Records() []domain.CatalogueRecord { | |
| 30 | + | func (c *transactionCatalogue) Records() ([]domain.CatalogueRecord, error) { | |
| 31 | 31 | c.mu.Lock() | |
| 32 | 32 | defer c.mu.Unlock() | |
| 33 | 33 | out := make([]domain.CatalogueRecord, 0, len(c.records)) | |
| 34 | 34 | for _, record := range c.records { | |
| 35 | 35 | out = append(out, record) | |
| 36 | 36 | } | |
| 37 | - | return out | |
| 37 | + | return out, nil | |
| 38 | 38 | } | |
| 39 | - | func (c *transactionCatalogue) Record(name string) (domain.CatalogueRecord, bool) { | |
| 39 | + | func (c *transactionCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) { | |
| 40 | 40 | c.mu.Lock() | |
| 41 | 41 | defer c.mu.Unlock() | |
| 42 | 42 | record, ok := c.records[name] | |
| 43 | - | return record, ok | |
| 43 | + | return record, ok, nil | |
| 44 | 44 | } | |
| 45 | 45 | func (c *transactionCatalogue) Create(record domain.CatalogueRecord) error { | |
| 46 | 46 | c.mu.Lock() | |
| @@ -233,16 +233,13 @@ func TestFallbackPromotion(t *testing.T) { | |||
| 233 | 233 | fallback := *record.Fallbacks[0] | |
| 234 | 234 | catalogue := newTransactionCatalogue(record) | |
| 235 | 235 | repository := newTransactionRepository() | |
| 236 | - | repository.generations[fallback] = ports.SnapshotGeneration{ | |
| 237 | - | IncarnationID: record.IncarnationID, Name: record.Name, Generation: fallback.Generation, | |
| 238 | - | Manifest: []byte("fallback"), | |
| 239 | - | } | |
| 240 | - | fallback.ManifestDigest = checkpointDigest(repository.generations[*record.Fallbacks[0]].Manifest) | |
| 236 | + | manifest := []byte("fallback") | |
| 237 | + | fallback.ManifestDigest = checkpointDigest(manifest) | |
| 241 | 238 | record.Fallbacks[0] = &fallback | |
| 242 | 239 | catalogue.records[record.Name] = record | |
| 243 | 240 | repository.generations[fallback] = ports.SnapshotGeneration{ | |
| 244 | 241 | IncarnationID: record.IncarnationID, Name: record.Name, Generation: fallback.Generation, | |
| 245 | - | Manifest: []byte("fallback"), | |
| 242 | + | Manifest: manifest, | |
| 246 | 243 | } | |
| 247 | 244 | coordinator := NewCoordinator(catalogue, repository, newTransactionJournal(), nil) | |
| 248 | 245 | ||
| @@ -283,6 +280,20 @@ func TestDegradedExportIsReadOnly(t *testing.T) { | |||
| 283 | 280 | ||
| 284 | 281 | // A third incarnation still fails closed for that session: its intent and | |
| 285 | 282 | // record are retained untouched, but startup is no longer aborted. | |
| 283 | + | func TestDiscardRecoveryRemovesIntentWhenCatalogueRecordIsAbsent(t *testing.T) { | |
| 284 | + | old := degradedTransactionRecord() | |
| 285 | + | intent := domain.DiscardIntent{OldRecord: old, OldIncarnation: old.IncarnationID, NewIncarnation: domain.IncarnationID{2}, SessionName: old.Name, Reason: "discard"} | |
| 286 | + | catalogue := newTransactionCatalogue() | |
| 287 | + | journal := newTransactionJournal() | |
| 288 | + | journal.intents[old.IncarnationID] = intent | |
| 289 | + | coordinator := NewCoordinator(catalogue, newTransactionRepository(), journal, nil) | |
| 290 | + | ||
| 291 | + | require.NoError(t, coordinator.Recover(context.Background())) | |
| 292 | + | require.NotContains(t, journal.intents, old.IncarnationID) | |
| 293 | + | require.Equal(t, []string{"delete-intent"}, journal.events) | |
| 294 | + | require.Empty(t, coordinator.Conflicts()) | |
| 295 | + | } | |
| 296 | + | ||
| 286 | 297 | func TestDiscardIncarnationConflict(t *testing.T) { | |
| 287 | 298 | old := degradedTransactionRecord() | |
| 288 | 299 | intent := domain.DiscardIntent{OldRecord: old, OldIncarnation: old.IncarnationID, NewIncarnation: domain.IncarnationID{2}, SessionName: old.Name, Reason: "discard"} | |
M internal/usecase/snapshot/manifest_preflight.go +4 -1
| @@ -16,7 +16,10 @@ func preflightManifest(body []byte) error { | |||
| 16 | 16 | if err != nil { | |
| 17 | 17 | return err | |
| 18 | 18 | } | |
| 19 | - | if generation == 0 || len(r.b) < len(domain.IncarnationID{}) { | |
| 19 | + | if generation == 0 { | |
| 20 | + | return fmt.Errorf("%w: manifest generation", ErrInvalidData) | |
| 21 | + | } | |
| 22 | + | if len(r.b) < len(domain.IncarnationID{}) { | |
| 20 | 23 | return ErrShortPayload | |
| 21 | 24 | } | |
| 22 | 25 | var incarnationID domain.IncarnationID | |
M internal/usecase/snapshot/manifest_preflight_test.go +55 -3
| @@ -40,6 +40,7 @@ func TestManifestEnvelopePreflightParity(t *testing.T) { | |||
| 40 | 40 | if err != nil { | |
| 41 | 41 | t.Fatal(err) | |
| 42 | 42 | } | |
| 43 | + | nameOffset, nodeKindOffset := manifestFieldOffsets(t, encoded) | |
| 43 | 44 | mutations := []struct { | |
| 44 | 45 | name string | |
| 45 | 46 | edit func([]byte) []byte | |
| @@ -51,9 +52,21 @@ func TestManifestEnvelopePreflightParity(t *testing.T) { | |||
| 51 | 52 | {"body length", func(b []byte) []byte { binary.BigEndian.PutUint32(b[8:12], uint32(len(b))); return b }}, | |
| 52 | 53 | {"crc", func(b []byte) []byte { b[12] ^= 1; return b }}, | |
| 53 | 54 | {"trailing", func(b []byte) []byte { return append(b, 0) }}, | |
| 54 | - | {"body name", func(b []byte) []byte { b[26] ^= 1; binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])); return b }}, | |
| 55 | - | {"body generation", func(b []byte) []byte { b[16] ^= 1; binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])); return b }}, | |
| 56 | - | {"body node kind", func(b []byte) []byte { b[65] = 99; binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])); return b }}, | |
| 55 | + | {"body name", func(b []byte) []byte { | |
| 56 | + | b[nameOffset] = 0 | |
| 57 | + | binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])) | |
| 58 | + | return b | |
| 59 | + | }}, | |
| 60 | + | {"body generation", func(b []byte) []byte { | |
| 61 | + | clear(b[manifestHeaderSize : manifestHeaderSize+8]) | |
| 62 | + | binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])) | |
| 63 | + | return b | |
| 64 | + | }}, | |
| 65 | + | {"body node kind", func(b []byte) []byte { | |
| 66 | + | b[nodeKindOffset] = 99 | |
| 67 | + | binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])) | |
| 68 | + | return b | |
| 69 | + | }}, | |
| 57 | 70 | } | |
| 58 | 71 | for _, tc := range mutations { | |
| 59 | 72 | t.Run(tc.name, func(t *testing.T) { | |
| @@ -68,6 +81,45 @@ func TestManifestEnvelopePreflightParity(t *testing.T) { | |||
| 68 | 81 | } | |
| 69 | 82 | } | |
| 70 | 83 | ||
| 84 | + | func manifestFieldOffsets(t *testing.T, encoded []byte) (nameOffset, nodeKindOffset int) { | |
| 85 | + | t.Helper() | |
| 86 | + | r := payloadReader{b: encoded[manifestHeaderSize:]} | |
| 87 | + | must := func(err error) { | |
| 88 | + | if err != nil { | |
| 89 | + | t.Fatal(err) | |
| 90 | + | } | |
| 91 | + | } | |
| 92 | + | _, err := r.getUint64() | |
| 93 | + | must(err) | |
| 94 | + | r.b = r.b[len(domain.IncarnationID{}):] | |
| 95 | + | _, err = r.getUint8() | |
| 96 | + | must(err) | |
| 97 | + | nameLengthOffset := len(encoded) - len(r.b) | |
| 98 | + | name, err := r.getString() | |
| 99 | + | must(err) | |
| 100 | + | nameOffset = nameLengthOffset + 2 | |
| 101 | + | if len(name) == 0 { | |
| 102 | + | t.Fatal("test manifest has empty name") | |
| 103 | + | } | |
| 104 | + | _, err = r.getUint64() | |
| 105 | + | must(err) | |
| 106 | + | _, err = r.getUint16() | |
| 107 | + | must(err) | |
| 108 | + | _, err = r.getUint16() | |
| 109 | + | must(err) | |
| 110 | + | _, err = r.getString() | |
| 111 | + | must(err) | |
| 112 | + | _, err = r.getUint16() | |
| 113 | + | must(err) | |
| 114 | + | _, err = r.getUint16() | |
| 115 | + | must(err) | |
| 116 | + | _, err = r.getUint64() | |
| 117 | + | must(err) | |
| 118 | + | _, err = r.getString() | |
| 119 | + | must(err) | |
| 120 | + | return nameOffset, len(encoded) - len(r.b) | |
| 121 | + | } | |
| 122 | + | ||
| 71 | 123 | func TestManifestObjectRefKinds(t *testing.T) { | |
| 72 | 124 | for _, tc := range []struct { | |
| 73 | 125 | name string | |
M pkg/kv/codec.go +12 -7
| @@ -19,8 +19,9 @@ const ( | |||
| 19 | 19 | // place by Open; nothing ever writes this framing again. | |
| 20 | 20 | legacyHeaderLen = 8 | |
| 21 | 21 | ||
| 22 | - | // Current framing. Every file starts with fileMagic plus a uint16 format | |
| 23 | - | // version, and every record carries a self-verifying header: | |
| 22 | + | // Current framing. Every file starts with fileMagic, a uint16 format | |
| 23 | + | // version, and CRC32 over that six-byte identity. Every record carries a | |
| 24 | + | // self-verifying header: | |
| 24 | 25 | // | |
| 25 | 26 | // [0:4] payload length, big endian | |
| 26 | 27 | // [4:8] CRC32(payload) | |
| @@ -29,10 +30,13 @@ const ( | |||
| 29 | 30 | // The header checksum is what makes the length trustworthy: a corrupt | |
| 30 | 31 | // length now fails closed instead of masquerading as a torn tail and | |
| 31 | 32 | // silently truncating every record that follows it. | |
| 32 | - | recordHeaderLen = 12 | |
| 33 | - | fileMagicLen = 4 | |
| 34 | - | fileHeaderLen = fileMagicLen + 2 | |
| 35 | - | formatVersion = 1 | |
| 33 | + | recordHeaderLen = 12 | |
| 34 | + | fileMagicLen = 4 | |
| 35 | + | fileHeaderV1Len = fileMagicLen + 2 | |
| 36 | + | fileHeaderBodyLen = fileHeaderV1Len | |
| 37 | + | fileHeaderLen = fileHeaderBodyLen + 4 | |
| 38 | + | formatVersionV1 = 1 | |
| 39 | + | formatVersion = 2 | |
| 36 | 40 | ||
| 37 | 41 | // maxPayloadLen bounds any allocation driven by a length field. Catalogue | |
| 38 | 42 | // records and batches are orders of magnitude smaller than this. | |
| @@ -145,7 +149,8 @@ func encodeBatch(changes []BatchChange) ([]byte, error) { | |||
| 145 | 149 | func fileHeader() []byte { | |
| 146 | 150 | buf := make([]byte, fileHeaderLen) | |
| 147 | 151 | copy(buf, fileMagic[:]) | |
| 148 | - | binary.BigEndian.PutUint16(buf[fileMagicLen:], formatVersion) | |
| 152 | + | binary.BigEndian.PutUint16(buf[fileMagicLen:fileHeaderBodyLen], formatVersion) | |
| 153 | + | binary.BigEndian.PutUint32(buf[fileHeaderBodyLen:], crc32.ChecksumIEEE(buf[:fileHeaderBodyLen])) | |
| 149 | 154 | return buf | |
| 150 | 155 | } | |
| 151 | 156 | ||
M pkg/kv/kv.go +100 -48
| @@ -36,16 +36,30 @@ const ( | |||
| 36 | 36 | // formatLegacy is a file written before the record header carried its own | |
| 37 | 37 | // checksum. It is read with the original semantics and upgraded at Open. | |
| 38 | 38 | formatLegacy walFormat = iota | |
| 39 | + | // formatCurrentV1 is the released VEVK v1 framing with its six-byte file | |
| 40 | + | // header. Its record headers are identical to the current format. | |
| 41 | + | formatCurrentV1 | |
| 39 | 42 | formatCurrent | |
| 40 | 43 | ) | |
| 41 | 44 | ||
| 42 | 45 | func (f walFormat) headerLen() int64 { | |
| 43 | - | if f == formatCurrent { | |
| 46 | + | if f != formatLegacy { | |
| 44 | 47 | return recordHeaderLen | |
| 45 | 48 | } | |
| 46 | 49 | return legacyHeaderLen | |
| 47 | 50 | } | |
| 48 | 51 | ||
| 52 | + | func (f walFormat) fileHeaderLen() int64 { | |
| 53 | + | switch f { | |
| 54 | + | case formatCurrentV1: | |
| 55 | + | return fileHeaderV1Len | |
| 56 | + | case formatCurrent: | |
| 57 | + | return fileHeaderLen | |
| 58 | + | default: | |
| 59 | + | return 0 | |
| 60 | + | } | |
| 61 | + | } | |
| 62 | + | ||
| 49 | 63 | type headerStatus uint8 | |
| 50 | 64 | ||
| 51 | 65 | const ( | |
| @@ -466,10 +480,7 @@ func replayFile(f *os.File) (ReplayResult, error) { | |||
| 466 | 480 | if err != nil { | |
| 467 | 481 | return ReplayResult{}, err | |
| 468 | 482 | } | |
| 469 | - | result := ReplayResult{Data: make(map[string][]byte), EntrySize: make(map[string]int64), Format: format} | |
| 470 | - | if format == formatCurrent { | |
| 471 | - | result.LastGood = fileHeaderLen | |
| 472 | - | } | |
| 483 | + | result := ReplayResult{Data: make(map[string][]byte), EntrySize: make(map[string]int64), Format: format, LastGood: format.fileHeaderLen()} | |
| 473 | 484 | for { | |
| 474 | 485 | header, status, err := readRecordHeader(f, format, fileSize, result.LastGood) | |
| 475 | 486 | if err != nil { | |
| @@ -501,7 +512,7 @@ func replayFile(f *os.File) (ReplayResult, error) { | |||
| 501 | 512 | records = []record{rec} | |
| 502 | 513 | } | |
| 503 | 514 | if err != nil { | |
| 504 | - | return ReplayResult{}, fmt.Errorf("%w: invalid record at offset %d: %v", ErrCorruptWAL, result.LastGood, err) | |
| 515 | + | return ReplayResult{}, fmt.Errorf("%w: invalid record at offset %d: %w", ErrCorruptWAL, result.LastGood, err) | |
| 505 | 516 | } | |
| 506 | 517 | size := format.headerLen() + int64(len(payload)) | |
| 507 | 518 | applyRecords(result.Data, result.EntrySize, &result.Live, records, size) | |
| @@ -513,23 +524,60 @@ func replayFile(f *os.File) (ReplayResult, error) { | |||
| 513 | 524 | // readFileFormat consumes the file header when one is present and leaves the | |
| 514 | 525 | // reader positioned at the first record either way. | |
| 515 | 526 | func readFileFormat(f io.ReadSeeker, fileSize int64) (walFormat, error) { | |
| 516 | - | if fileSize < fileHeaderLen { | |
| 527 | + | if fileSize == 0 { | |
| 517 | 528 | return formatLegacy, nil | |
| 518 | 529 | } | |
| 519 | - | header := make([]byte, fileHeaderLen) | |
| 520 | - | if _, err := io.ReadFull(f, header); err != nil { | |
| 530 | + | probeLen := min(fileSize, int64(fileHeaderLen)) | |
| 531 | + | probe := make([]byte, probeLen) | |
| 532 | + | if _, err := io.ReadFull(f, probe); err != nil { | |
| 521 | 533 | return formatLegacy, err | |
| 522 | 534 | } | |
| 523 | - | if !bytes.Equal(header[:fileMagicLen], fileMagic[:]) { | |
| 535 | + | if fileSize < fileMagicLen { | |
| 536 | + | if bytes.Equal(probe, fileMagic[:len(probe)]) { | |
| 537 | + | return formatLegacy, fmt.Errorf("%w: incomplete VEVK header", ErrCorruptWAL) | |
| 538 | + | } | |
| 524 | 539 | if _, err := f.Seek(0, io.SeekStart); err != nil { | |
| 525 | 540 | return formatLegacy, err | |
| 526 | 541 | } | |
| 527 | 542 | return formatLegacy, nil | |
| 528 | 543 | } | |
| 529 | - | if version := binary.BigEndian.Uint16(header[fileMagicLen:]); version != formatVersion { | |
| 544 | + | if !bytes.Equal(probe[:fileMagicLen], fileMagic[:]) { | |
| 545 | + | // A complete v2 identity checksum authenticates the intended magic as | |
| 546 | + | // well as the version. Detect it before legacy replay so damage to the | |
| 547 | + | // first magic byte cannot masquerade as a small legacy length and be | |
| 548 | + | // truncated or compacted away. | |
| 549 | + | if len(probe) == fileHeaderLen && | |
| 550 | + | binary.BigEndian.Uint16(probe[fileMagicLen:fileHeaderBodyLen]) == formatVersion && | |
| 551 | + | binary.BigEndian.Uint32(probe[fileHeaderBodyLen:]) == binary.BigEndian.Uint32(fileHeader()[fileHeaderBodyLen:]) { | |
| 552 | + | return formatLegacy, fmt.Errorf("%w: damaged VEVK v2 magic", ErrCorruptWAL) | |
| 553 | + | } | |
| 554 | + | if _, err := f.Seek(0, io.SeekStart); err != nil { | |
| 555 | + | return formatLegacy, err | |
| 556 | + | } | |
| 557 | + | return formatLegacy, nil | |
| 558 | + | } | |
| 559 | + | if fileSize < fileHeaderV1Len { | |
| 560 | + | return formatLegacy, fmt.Errorf("%w: incomplete VEVK header", ErrCorruptWAL) | |
| 561 | + | } | |
| 562 | + | prefix := probe[:fileHeaderV1Len] | |
| 563 | + | if _, err := f.Seek(fileHeaderV1Len, io.SeekStart); err != nil { | |
| 564 | + | return formatLegacy, err | |
| 565 | + | } | |
| 566 | + | switch version := binary.BigEndian.Uint16(prefix[fileMagicLen:]); version { | |
| 567 | + | case formatVersionV1: | |
| 568 | + | return formatCurrentV1, nil | |
| 569 | + | case formatVersion: | |
| 570 | + | checksum := make([]byte, fileHeaderLen-fileHeaderV1Len) | |
| 571 | + | if _, err := io.ReadFull(f, checksum); err != nil { | |
| 572 | + | return formatLegacy, fmt.Errorf("%w: incomplete VEVK v2 header", ErrCorruptWAL) | |
| 573 | + | } | |
| 574 | + | if crc32.ChecksumIEEE(prefix) != binary.BigEndian.Uint32(checksum) { | |
| 575 | + | return formatLegacy, fmt.Errorf("%w: file header checksum mismatch", ErrCorruptWAL) | |
| 576 | + | } | |
| 577 | + | return formatCurrent, nil | |
| 578 | + | default: | |
| 530 | 579 | return formatLegacy, fmt.Errorf("%w: unsupported format version %d", ErrCorruptWAL, version) | |
| 531 | 580 | } | |
| 532 | - | return formatCurrent, nil | |
| 533 | 581 | } | |
| 534 | 582 | ||
| 535 | 583 | type recordHeader struct { | |
| @@ -554,15 +602,18 @@ func readRecordHeader(f io.Reader, format walFormat, fileSize, offset int64) (re | |||
| 554 | 602 | payloadLen: binary.BigEndian.Uint32(buf[0:4]), | |
| 555 | 603 | payloadCRC: binary.BigEndian.Uint32(buf[4:8]), | |
| 556 | 604 | } | |
| 557 | - | if format == formatCurrent { | |
| 605 | + | if format != formatLegacy { | |
| 558 | 606 | if crc32.ChecksumIEEE(buf[0:8]) != binary.BigEndian.Uint32(buf[8:recordHeaderLen]) { | |
| 559 | 607 | return recordHeader{}, headerOK, fmt.Errorf("%w: record header CRC mismatch at offset %d", ErrCorruptWAL, offset) | |
| 560 | 608 | } | |
| 561 | 609 | // The length is now trustworthy, so an implausible one is corruption | |
| 562 | 610 | // rather than a short tail. Checked before any allocation. | |
| 563 | - | if header.payloadLen > maxPayloadLen { | |
| 564 | - | return recordHeader{}, headerOK, fmt.Errorf("%w: record length %d exceeds maximum at offset %d", ErrCorruptWAL, header.payloadLen, offset) | |
| 565 | - | } | |
| 611 | + | } | |
| 612 | + | // Every framing is allocation-bounded. In legacy framing an oversized | |
| 613 | + | // declaration remains corruption rather than driving an attacker-sized | |
| 614 | + | // allocation; ordinary short tails retain their released truncation semantics. | |
| 615 | + | if header.payloadLen > maxPayloadLen { | |
| 616 | + | return recordHeader{}, headerOK, fmt.Errorf("%w: record length %d exceeds maximum at offset %d", ErrCorruptWAL, header.payloadLen, offset) | |
| 566 | 617 | } | |
| 567 | 618 | if int64(header.payloadLen) > fileSize-offset-format.headerLen() { | |
| 568 | 619 | return recordHeader{}, headerTorn, nil | |
| @@ -577,9 +628,9 @@ func applyRecords(data map[string][]byte, sizes map[string]int64, live *int64, r | |||
| 577 | 628 | setCount++ | |
| 578 | 629 | } | |
| 579 | 630 | } | |
| 580 | - | setSize := size | |
| 631 | + | setSize, remainder := size, int64(0) | |
| 581 | 632 | if setCount > 1 { | |
| 582 | - | setSize = size / setCount | |
| 633 | + | setSize, remainder = size/setCount, size%setCount | |
| 583 | 634 | } | |
| 584 | 635 | for _, rec := range records { | |
| 585 | 636 | key := string(rec.key) | |
| @@ -587,9 +638,14 @@ func applyRecords(data map[string][]byte, sizes map[string]int64, live *int64, r | |||
| 587 | 638 | *live -= old | |
| 588 | 639 | } | |
| 589 | 640 | if rec.op == opSet { | |
| 641 | + | entrySize := setSize | |
| 642 | + | if remainder > 0 { | |
| 643 | + | entrySize++ | |
| 644 | + | remainder-- | |
| 645 | + | } | |
| 590 | 646 | data[key] = append([]byte(nil), rec.value...) | |
| 591 | - | sizes[key] = setSize | |
| 592 | - | *live += setSize | |
| 647 | + | sizes[key] = entrySize | |
| 648 | + | *live += entrySize | |
| 593 | 649 | } else { | |
| 594 | 650 | delete(data, key) | |
| 595 | 651 | delete(sizes, key) | |
| @@ -598,7 +654,7 @@ func applyRecords(data map[string][]byte, sizes map[string]int64, live *int64, r | |||
| 598 | 654 | } | |
| 599 | 655 | ||
| 600 | 656 | func recoverCompaction(path string, hooks fsHooks) error { | |
| 601 | - | _, currentExists, currentErr := candidate(path) | |
| 657 | + | currentResult, currentExists, currentErr := candidate(path) | |
| 602 | 658 | nextResult, nextExists, nextErr := candidate(path + ".next") | |
| 603 | 659 | _, prevExists, prevErr := candidate(path + ".prev") | |
| 604 | 660 | if nextErr == nil && nextResult.TornTail { | |
| @@ -606,6 +662,12 @@ func recoverCompaction(path string, hooks fsHooks) error { | |||
| 606 | 662 | } | |
| 607 | 663 | dir := filepath.Dir(path) | |
| 608 | 664 | if currentExists && currentErr == nil { | |
| 665 | + | if currentResult.TornTail && prevExists { | |
| 666 | + | // A predecessor proves compaction was in flight. The torn current is | |
| 667 | + | // ambiguous, so preserve both candidates and fail closed rather than | |
| 668 | + | // truncating current and deleting the last complete predecessor. | |
| 669 | + | return fmt.Errorf("%w: torn current with predecessor", ErrCorruptWAL) | |
| 670 | + | } | |
| 609 | 671 | if nextExists { | |
| 610 | 672 | if err := hooks.Remove(path + ".next"); err != nil { | |
| 611 | 673 | return err | |
| @@ -625,33 +687,23 @@ func recoverCompaction(path string, hooks fsHooks) error { | |||
| 625 | 687 | return nil | |
| 626 | 688 | } | |
| 627 | 689 | if currentExists && currentErr != nil { | |
| 628 | - | if !prevExists || prevErr != nil { | |
| 629 | - | return fmt.Errorf("%w: no valid current or predecessor", ErrCorruptWAL) | |
| 630 | - | } | |
| 631 | - | if err := hooks.Remove(path); err != nil { | |
| 632 | - | return err | |
| 633 | - | } | |
| 634 | - | if err := hooks.Rename(path+".prev", path); err != nil { | |
| 635 | - | return err | |
| 636 | - | } | |
| 637 | - | if err := hooks.SyncDir(dir); err != nil { | |
| 638 | - | return err | |
| 639 | - | } | |
| 640 | - | if nextExists { | |
| 641 | - | if err := hooks.Remove(path + ".next"); err != nil { | |
| 642 | - | return err | |
| 643 | - | } | |
| 644 | - | return hooks.SyncDir(dir) | |
| 645 | - | } | |
| 646 | - | return nil | |
| 647 | - | } | |
| 648 | - | if !currentExists && prevExists { | |
| 649 | - | if prevErr != nil { | |
| 650 | - | return fmt.Errorf("%w: invalid predecessor", ErrCorruptWAL) | |
| 651 | - | } | |
| 652 | - | source := path + ".prev" | |
| 690 | + | // Corruption does not prove which bytes are disposable. Preserve every | |
| 691 | + | // candidate for diagnosis/recovery instead of deleting current or a valid | |
| 692 | + | // predecessor and fail closed. | |
| 693 | + | return fmt.Errorf("%w: invalid current candidate", ErrCorruptWAL) | |
| 694 | + | } | |
| 695 | + | if !currentExists && (nextExists || prevExists) { | |
| 696 | + | // A complete .next is the newest fully published compaction candidate and | |
| 697 | + | // wins even when .prev exists but is corrupt. Otherwise a complete .prev | |
| 698 | + | // is the only safe rollback. Invalid candidates are never selected. | |
| 699 | + | source := "" | |
| 653 | 700 | if nextExists && nextErr == nil { | |
| 654 | 701 | source = path + ".next" | |
| 702 | + | } else if prevExists && prevErr == nil { | |
| 703 | + | source = path + ".prev" | |
| 704 | + | } | |
| 705 | + | if source == "" { | |
| 706 | + | return fmt.Errorf("%w: no valid current, successor, or predecessor", ErrCorruptWAL) | |
| 655 | 707 | } | |
| 656 | 708 | if err := hooks.Rename(source, path); err != nil { | |
| 657 | 709 | return err | |
| @@ -668,8 +720,8 @@ func recoverCompaction(path string, hooks fsHooks) error { | |||
| 668 | 720 | } | |
| 669 | 721 | return hooks.SyncDir(dir) | |
| 670 | 722 | } | |
| 671 | - | if nextExists || (currentExists && currentErr != nil) || (prevExists && prevErr != nil) { | |
| 672 | - | return fmt.Errorf("%w: no valid current or predecessor", ErrCorruptWAL) | |
| 723 | + | if currentExists && currentErr != nil { | |
| 724 | + | return fmt.Errorf("%w: no valid current", ErrCorruptWAL) | |
| 673 | 725 | } | |
| 674 | 726 | return nil | |
| 675 | 727 | } | |
M pkg/kv/kv_test.go +168 -19
| @@ -4,7 +4,9 @@ import ( | |||
| 4 | 4 | "bytes" | |
| 5 | 5 | "encoding/binary" | |
| 6 | 6 | "errors" | |
| 7 | + | "fmt" | |
| 7 | 8 | "hash/crc32" | |
| 9 | + | "maps" | |
| 8 | 10 | "os" | |
| 9 | 11 | "path/filepath" | |
| 10 | 12 | "reflect" | |
| @@ -26,6 +28,17 @@ func currentFile(records ...[]byte) []byte { | |||
| 26 | 28 | return buf | |
| 27 | 29 | } | |
| 28 | 30 | ||
| 31 | + | // v1File is the previously released VEVK framing: magic and uint16 version, | |
| 32 | + | // followed directly by the same self-checksummed records used today. | |
| 33 | + | func v1File(records ...[]byte) []byte { | |
| 34 | + | buf := append([]byte(nil), fileMagic[:]...) | |
| 35 | + | buf = binary.BigEndian.AppendUint16(buf, formatVersionV1) | |
| 36 | + | for _, record := range records { | |
| 37 | + | buf = append(buf, record...) | |
| 38 | + | } | |
| 39 | + | return buf | |
| 40 | + | } | |
| 41 | + | ||
| 29 | 42 | // legacyFile reproduces the framing written by the released version: no file | |
| 30 | 43 | // header, and a record header of length plus payload CRC only. Compatibility | |
| 31 | 44 | // with these files is a hard requirement, so the fixture stays in the tests. | |
| @@ -378,6 +391,34 @@ func TestRecoverCompactionCleansStaleNextRegardlessOfPrev(t *testing.T) { | |||
| 378 | 391 | } | |
| 379 | 392 | } | |
| 380 | 393 | ||
| 394 | + | func TestReplayPreservesWrappedDecodeCause(t *testing.T) { | |
| 395 | + | path := privatePath(t) | |
| 396 | + | writeFile(t, path, currentFile(rawRecord([]byte{0xff, 0, 0}))) | |
| 397 | + | f, err := os.Open(path) | |
| 398 | + | if err != nil { | |
| 399 | + | t.Fatal(err) | |
| 400 | + | } | |
| 401 | + | defer func() { _ = f.Close() }() | |
| 402 | + | _, err = replayFile(f) | |
| 403 | + | if !errors.Is(err, ErrCorruptWAL) || !errors.Is(err, errBadRecord) { | |
| 404 | + | t.Fatalf("replayFile error = %v, want wrapped ErrCorruptWAL and errBadRecord", err) | |
| 405 | + | } | |
| 406 | + | } | |
| 407 | + | ||
| 408 | + | func TestBatchLiveSizeAccountsForRemainderExactly(t *testing.T) { | |
| 409 | + | data := map[string][]byte{} | |
| 410 | + | sizes := map[string]int64{} | |
| 411 | + | var live int64 | |
| 412 | + | applyRecords(data, sizes, &live, []record{ | |
| 413 | + | {op: opSet, key: []byte("a"), value: []byte("one")}, | |
| 414 | + | {op: opSet, key: []byte("b"), value: []byte("two")}, | |
| 415 | + | {op: opSet, key: []byte("c"), value: []byte("three")}, | |
| 416 | + | }, 10) | |
| 417 | + | if live != 10 || sizes["a"]+sizes["b"]+sizes["c"] != 10 { | |
| 418 | + | t.Fatalf("live sizes = %v, live = %d, want exact total 10", sizes, live) | |
| 419 | + | } | |
| 420 | + | } | |
| 421 | + | ||
| 381 | 422 | func TestDecodeBatchRejectsImpossibleCountBeforeAllocating(t *testing.T) { | |
| 382 | 423 | payload := []byte{opBatch, 0xff, 0xff, 0xff, 0xff} | |
| 383 | 424 | allocs := testing.AllocsPerRun(10, func() { | |
| @@ -429,14 +470,17 @@ func TestCompactionRecoveryMatrix(t *testing.T) { | |||
| 429 | 470 | name string | |
| 430 | 471 | current, next, prev []byte | |
| 431 | 472 | want string | |
| 473 | + | wantErr bool | |
| 432 | 474 | }{ | |
| 433 | 475 | {name: "current-alone", current: oldFile, want: "old"}, | |
| 434 | 476 | {name: "current-next", current: oldFile, next: newFile, want: "old"}, | |
| 435 | 477 | {name: "prev-next", prev: oldFile, next: newFile, want: "new"}, | |
| 436 | - | {name: "prev-invalid-next", prev: oldFile, next: currentFile(rawRecord([]byte{0xff, 0, 0})), want: "old"}, | |
| 478 | + | {name: "next-alone", next: newFile, want: "new"}, | |
| 479 | + | {name: "invalid-prev-valid-next", prev: currentFile(rawRecord([]byte{0xff, 0, 0})), next: newFile, want: "new"}, | |
| 480 | + | {name: "valid-prev-invalid-next", prev: oldFile, next: currentFile(rawRecord([]byte{0xff, 0, 0})), want: "old"}, | |
| 437 | 481 | {name: "prev-torn-next", prev: oldFile, next: newFile[:len(newFile)-1], want: "old"}, | |
| 438 | 482 | {name: "current-prev", current: newFile, prev: oldFile, want: "new"}, | |
| 439 | - | {name: "invalid-current-prev", current: currentFile(rawRecord([]byte{0xff, 0, 0})), prev: oldFile, want: "old"}, | |
| 483 | + | {name: "invalid-current-prev", current: currentFile(rawRecord([]byte{0xff, 0, 0})), prev: oldFile, wantErr: true}, | |
| 440 | 484 | {name: "prev-alone", prev: oldFile, want: "old"}, | |
| 441 | 485 | } | |
| 442 | 486 | for _, tt := range tests { | |
| @@ -452,7 +496,18 @@ func TestCompactionRecoveryMatrix(t *testing.T) { | |||
| 452 | 496 | } | |
| 453 | 497 | } | |
| 454 | 498 | } | |
| 455 | - | if err := recoverCompaction(path, defaultFSHooks()); err != nil { | |
| 499 | + | before := candidateBytes(t, path) | |
| 500 | + | err := recoverCompaction(path, defaultFSHooks()) | |
| 501 | + | if tt.wantErr { | |
| 502 | + | if !errors.Is(err, ErrCorruptWAL) { | |
| 503 | + | t.Fatalf("recoverCompaction error = %v, want ErrCorruptWAL", err) | |
| 504 | + | } | |
| 505 | + | if got := candidateBytes(t, path); !reflect.DeepEqual(got, before) { | |
| 506 | + | t.Fatalf("uncertain candidates changed: got %#v, want %#v", got, before) | |
| 507 | + | } | |
| 508 | + | return | |
| 509 | + | } | |
| 510 | + | if err != nil { | |
| 456 | 511 | t.Fatal(err) | |
| 457 | 512 | } | |
| 458 | 513 | got, err := Replay(path) | |
| @@ -701,25 +756,85 @@ func rawBatchPayload(changes []BatchChange) []byte { | |||
| 701 | 756 | return payload | |
| 702 | 757 | } | |
| 703 | 758 | ||
| 704 | - | // An oversized length must never drive an allocation. In a legacy file the | |
| 705 | - | // length is unprotected, so it stays a torn tail; in the current format the | |
| 706 | - | // header checksum proves the length was written that way, which is corruption. | |
| 759 | + | // An oversized length must never drive an allocation. Legacy framing is | |
| 760 | + | // accepted only after complete strict replay, while the current header checksum | |
| 761 | + | // proves an oversized length was written that way; both fail closed. | |
| 762 | + | func TestCorruptedCurrentHeaderNeverFallsBackToLegacyOrDeletesPredecessor(t *testing.T) { | |
| 763 | + | record, _ := encodeRecord(opSet, []byte("version"), []byte("current")) | |
| 764 | + | previous, _ := encodeRecord(opSet, []byte("version"), []byte("previous")) | |
| 765 | + | ||
| 766 | + | tests := []struct { | |
| 767 | + | name string | |
| 768 | + | current []byte | |
| 769 | + | standalone bool | |
| 770 | + | }{ | |
| 771 | + | {name: "torn file header", current: append([]byte(nil), fileHeader()[:fileHeaderLen-1]...), standalone: true}, | |
| 772 | + | {name: "torn current record", current: currentFile(record)[:fileHeaderLen+recordHeaderLen+1]}, | |
| 773 | + | } | |
| 774 | + | for i := range fileMagicLen { | |
| 775 | + | corrupted := currentFile(record) | |
| 776 | + | corrupted[i] = 0 | |
| 777 | + | tests = append(tests, struct { | |
| 778 | + | name string | |
| 779 | + | current []byte | |
| 780 | + | standalone bool | |
| 781 | + | }{name: fmt.Sprintf("corrupted magic byte %d", i), current: corrupted, standalone: true}) | |
| 782 | + | } | |
| 783 | + | for length := 1; length < fileHeaderLen; length++ { | |
| 784 | + | tests = append(tests, struct { | |
| 785 | + | name string | |
| 786 | + | current []byte | |
| 787 | + | standalone bool | |
| 788 | + | }{name: fmt.Sprintf("torn header prefix %d", length), current: append([]byte(nil), fileHeader()[:length]...), standalone: true}) | |
| 789 | + | } | |
| 790 | + | ||
| 791 | + | for _, tt := range tests { | |
| 792 | + | t.Run(tt.name, func(t *testing.T) { | |
| 793 | + | for _, withCandidates := range []bool{false, true} { | |
| 794 | + | if !withCandidates && !tt.standalone { | |
| 795 | + | continue | |
| 796 | + | } | |
| 797 | + | name := "standalone" | |
| 798 | + | if withCandidates { | |
| 799 | + | name = "with candidates" | |
| 800 | + | } | |
| 801 | + | t.Run(name, func(t *testing.T) { | |
| 802 | + | path := privatePath(t) | |
| 803 | + | writeFile(t, path, tt.current) | |
| 804 | + | if withCandidates { | |
| 805 | + | if err := os.WriteFile(path+".prev", currentFile(previous), 0o600); err != nil { | |
| 806 | + | t.Fatal(err) | |
| 807 | + | } | |
| 808 | + | if err := os.WriteFile(path+".next", currentFile(record), 0o600); err != nil { | |
| 809 | + | t.Fatal(err) | |
| 810 | + | } | |
| 811 | + | } | |
| 812 | + | before := candidateBytes(t, path) | |
| 813 | + | if _, err := Open(path); !errors.Is(err, ErrCorruptWAL) { | |
| 814 | + | t.Fatalf("Open error = %v, want ErrCorruptWAL", err) | |
| 815 | + | } | |
| 816 | + | if got := candidateBytes(t, path); !reflect.DeepEqual(got, before) { | |
| 817 | + | t.Fatalf("candidates changed: got %#v, want %#v", got, before) | |
| 818 | + | } | |
| 819 | + | }) | |
| 820 | + | } | |
| 821 | + | }) | |
| 822 | + | } | |
| 823 | + | } | |
| 824 | + | ||
| 707 | 825 | func TestReplayRejectsOversizedLengthWithoutAllocating(t *testing.T) { | |
| 708 | 826 | first, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | |
| 709 | 827 | legacyHeader := make([]byte, legacyHeaderLen) | |
| 710 | 828 | binary.BigEndian.PutUint32(legacyHeader[0:4], 1<<31) | |
| 711 | 829 | ||
| 712 | - | t.Run("legacy length is a torn tail", func(t *testing.T) { | |
| 830 | + | t.Run("legacy incomplete record fails closed", func(t *testing.T) { | |
| 713 | 831 | path := privatePath(t) | |
| 714 | - | writeFile(t, path, append(legacyFile(first), legacyHeader...)) | |
| 715 | - | s, err := Open(path) | |
| 716 | - | if err != nil { | |
| 717 | - | t.Fatal(err) | |
| 718 | - | } | |
| 719 | - | defer closeStore(t, s) | |
| 720 | - | if got, ok := s.Get([]byte("a")); !ok || string(got) != "one" { | |
| 721 | - | t.Fatalf("good record missing before bad length: %q %v", got, ok) | |
| 832 | + | wal := append(legacyFile(first), legacyHeader...) | |
| 833 | + | writeFile(t, path, wal) | |
| 834 | + | if _, err := Open(path); !errors.Is(err, ErrCorruptWAL) { | |
| 835 | + | t.Fatalf("Open error = %v, want ErrCorruptWAL", err) | |
| 722 | 836 | } | |
| 837 | + | requireFileBytes(t, path, wal) | |
| 723 | 838 | }) | |
| 724 | 839 | ||
| 725 | 840 | t.Run("current oversized length fails closed", func(t *testing.T) { | |
| @@ -851,13 +966,49 @@ func TestCurrentFormatReplayDecisionTable(t *testing.T) { | |||
| 851 | 966 | ||
| 852 | 967 | // A file written by the released version must keep working: it opens with the | |
| 853 | 968 | // original semantics, is upgraded in place, and reopens identically. | |
| 969 | + | func TestVEVKV1FileOpensAndUpgradesInPlace(t *testing.T) { | |
| 970 | + | setA, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | |
| 971 | + | setB, _ := encodeRecord(opSet, []byte("b"), bytes.Repeat([]byte("x"), 1<<20)) | |
| 972 | + | for _, tc := range []struct { | |
| 973 | + | name string | |
| 974 | + | wal []byte | |
| 975 | + | }{ | |
| 976 | + | {name: "ordinary catalogue payload", wal: v1File(setA)}, | |
| 977 | + | {name: "prior large catalogue payload", wal: v1File(setB)}, | |
| 978 | + | } { | |
| 979 | + | t.Run(tc.name, func(t *testing.T) { | |
| 980 | + | path := privatePath(t) | |
| 981 | + | writeFile(t, path, tc.wal) | |
| 982 | + | store, err := Open(path) | |
| 983 | + | if err != nil { | |
| 984 | + | t.Fatal(err) | |
| 985 | + | } | |
| 986 | + | closeStore(t, store) | |
| 987 | + | upgraded, err := os.ReadFile(path) | |
| 988 | + | if err != nil { | |
| 989 | + | t.Fatal(err) | |
| 990 | + | } | |
| 991 | + | if !bytes.Equal(upgraded[:fileHeaderLen], fileHeader()) { | |
| 992 | + | t.Fatalf("upgraded header = %x, want %x", upgraded[:fileHeaderLen], fileHeader()) | |
| 993 | + | } | |
| 994 | + | }) | |
| 995 | + | } | |
| 996 | + | } | |
| 997 | + | ||
| 998 | + | func TestVEVKV1RejectsOversizedRecordBeforeAllocation(t *testing.T) { | |
| 999 | + | path := privatePath(t) | |
| 1000 | + | writeFile(t, path, v1File(oversizedRecordHeader(maxPayloadLen+1))) | |
| 1001 | + | if _, err := Open(path); !errors.Is(err, ErrCorruptWAL) { | |
| 1002 | + | t.Fatalf("Open error = %v, want ErrCorruptWAL", err) | |
| 1003 | + | } | |
| 1004 | + | } | |
| 1005 | + | ||
| 854 | 1006 | func TestLegacyFileOpensAndUpgradesInPlace(t *testing.T) { | |
| 855 | 1007 | setA, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | |
| 856 | 1008 | setB, _ := encodeRecord(opSet, []byte("b"), []byte("two")) | |
| 857 | 1009 | replaceA, _ := encodeRecord(opSet, []byte("a"), []byte("uno")) | |
| 858 | 1010 | delB, _ := encodeRecord(opDel, []byte("b"), nil) | |
| 859 | 1011 | torn, _ := encodeRecord(opSet, []byte("c"), []byte("three")) | |
| 860 | - | ||
| 861 | 1012 | tests := []struct { | |
| 862 | 1013 | name string | |
| 863 | 1014 | wal []byte | |
| @@ -896,9 +1047,7 @@ func TestLegacyFileOpensAndUpgradesInPlace(t *testing.T) { | |||
| 896 | 1047 | } | |
| 897 | 1048 | defer closeStore(t, reopened) | |
| 898 | 1049 | want := map[string]string{"added": "after-upgrade"} | |
| 899 | - | for k, v := range tt.want { | |
| 900 | - | want[k] = v | |
| 901 | - | } | |
| 1050 | + | maps.Copy(want, tt.want) | |
| 902 | 1051 | requireContents(t, reopened, want) | |
| 903 | 1052 | replayed, err := Replay(path) | |
| 904 | 1053 | if err != nil { | |