feat(snapshot): add durable repository primitives
- commit
- e144d99c31d1e6b3fdd01628b8bc4acb24a42e57
- author
- brice · · 2026-07-21 19:52 UTC
- committer
- brice · · 2026-07-21 19:53 UTC
- parent
- c34959e
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
1 file · +629 -0
A internal/adapters/snapshot/repository.go +629 -0
| @@ -0,0 +1,629 @@ | |||
| 1 | + | package snapshot | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "crypto/sha256" | |
| 6 | + | "encoding/binary" | |
| 7 | + | "encoding/hex" | |
| 8 | + | "errors" | |
| 9 | + | "fmt" | |
| 10 | + | "io" | |
| 11 | + | "os" | |
| 12 | + | "path/filepath" | |
| 13 | + | "sort" | |
| 14 | + | "strconv" | |
| 15 | + | "strings" | |
| 16 | + | "sync" | |
| 17 | + | ||
| 18 | + | "github.com/bnema/vev/internal/ports" | |
| 19 | + | codec "github.com/bnema/vev/internal/usecase/snapshot" | |
| 20 | + | "github.com/bnema/vev/pkg/safedir" | |
| 21 | + | ) | |
| 22 | + | ||
| 23 | + | const ( | |
| 24 | + | repositorySessionsDir = "sessions" | |
| 25 | + | repositoryObjectsDir = "objects" | |
| 26 | + | repositoryGenerations = "generations" | |
| 27 | + | repositoryHead = "HEAD" | |
| 28 | + | generationWidth = 20 | |
| 29 | + | maxRepositoryRead = maxSnapshotFileSize | |
| 30 | + | ) | |
| 31 | + | ||
| 32 | + | // Repository is the crash-safe, content-addressed session snapshot store. | |
| 33 | + | // Store remains available separately for the one-way legacy bridge. | |
| 34 | + | type Repository struct { | |
| 35 | + | dir string | |
| 36 | + | legacy *Store | |
| 37 | + | locks sync.Map // map[string]*sync.Mutex | |
| 38 | + | hooks repositoryHooks | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | type repositoryHooks struct { | |
| 42 | + | beforeBlobWrite func(string) error | |
| 43 | + | beforeManifestWrite func(string) error | |
| 44 | + | beforeHeadWrite func(string) error | |
| 45 | + | writeFile func(string, []byte) error | |
| 46 | + | } | |
| 47 | + | ||
| 48 | + | var _ ports.SnapshotRepository = (*Repository)(nil) | |
| 49 | + | var _ ports.LegacySnapshotSource = (*Repository)(nil) | |
| 50 | + | ||
| 51 | + | // NewRepository creates a repository rooted at dir. It does not create files | |
| 52 | + | // until the first publication, so merely constructing it is side-effect free. | |
| 53 | + | func NewRepository(dir string) *Repository { return &Repository{dir: dir, legacy: NewStore(dir)} } | |
| 54 | + | ||
| 55 | + | func (r *Repository) Publish(ctx context.Context, publication ports.SnapshotPublication) error { | |
| 56 | + | if err := ctx.Err(); err != nil { | |
| 57 | + | return err | |
| 58 | + | } | |
| 59 | + | key := sessionKey(publication.Name) | |
| 60 | + | lock := r.sessionLock(key) | |
| 61 | + | lock.Lock() | |
| 62 | + | defer lock.Unlock() | |
| 63 | + | ||
| 64 | + | manifest, refs, supplied, err := validatePublication(publication) | |
| 65 | + | if err != nil { | |
| 66 | + | return err | |
| 67 | + | } | |
| 68 | + | _ = manifest | |
| 69 | + | if err := r.ensureSession(key); err != nil { | |
| 70 | + | return err | |
| 71 | + | } | |
| 72 | + | ||
| 73 | + | current, currentManifest, err := r.currentGeneration(ctx, publication.Name, key) | |
| 74 | + | if err != nil { | |
| 75 | + | return err | |
| 76 | + | } | |
| 77 | + | if publication.Generation < current || publication.Generation > current+1 { | |
| 78 | + | return fmt.Errorf("snapshot generation %d, current %d: immutable conflict", publication.Generation, current) | |
| 79 | + | } | |
| 80 | + | if publication.Generation == current { | |
| 81 | + | if !equalBytes(currentManifest, publication.Manifest) { | |
| 82 | + | return fmt.Errorf("snapshot generation %d: immutable conflict", publication.Generation) | |
| 83 | + | } | |
| 84 | + | return nil // exact replay; validation above also rejects altered objects. | |
| 85 | + | } | |
| 86 | + | ||
| 87 | + | for digest, ref := range refs { | |
| 88 | + | if err := ctx.Err(); err != nil { | |
| 89 | + | return err | |
| 90 | + | } | |
| 91 | + | path := r.objectPath(key, digest) | |
| 92 | + | exists, err := verifyObjectFile(path, digest, ref) | |
| 93 | + | if err != nil { | |
| 94 | + | return fmt.Errorf("verify object %x: %w", digest, err) | |
| 95 | + | } | |
| 96 | + | if exists { | |
| 97 | + | continue | |
| 98 | + | } | |
| 99 | + | object, supplied := supplied[digest] | |
| 100 | + | if !supplied { | |
| 101 | + | return fmt.Errorf("missing referenced object %x", digest) | |
| 102 | + | } | |
| 103 | + | if r.hooks.beforeBlobWrite != nil { | |
| 104 | + | if err := r.hooks.beforeBlobWrite(path); err != nil { | |
| 105 | + | return err | |
| 106 | + | } | |
| 107 | + | } | |
| 108 | + | if err := r.atomicWrite(path, object.Data); err != nil { | |
| 109 | + | return fmt.Errorf("write object: %w", err) | |
| 110 | + | } | |
| 111 | + | } | |
| 112 | + | ||
| 113 | + | manifestPath := r.manifestPath(key, publication.Generation) | |
| 114 | + | if existing, exists, err := readOptionalBounded(manifestPath); err != nil { | |
| 115 | + | return err | |
| 116 | + | } else if exists { | |
| 117 | + | if !equalBytes(existing, publication.Manifest) { | |
| 118 | + | return fmt.Errorf("manifest generation %d: immutable conflict", publication.Generation) | |
| 119 | + | } | |
| 120 | + | } else { | |
| 121 | + | if r.hooks.beforeManifestWrite != nil { | |
| 122 | + | if err := r.hooks.beforeManifestWrite(manifestPath); err != nil { | |
| 123 | + | return err | |
| 124 | + | } | |
| 125 | + | } | |
| 126 | + | if err := r.atomicWrite(manifestPath, publication.Manifest); err != nil { | |
| 127 | + | return fmt.Errorf("write manifest: %w", err) | |
| 128 | + | } | |
| 129 | + | } | |
| 130 | + | if r.hooks.beforeHeadWrite != nil { | |
| 131 | + | if err := r.hooks.beforeHeadWrite(r.headPath(key)); err != nil { | |
| 132 | + | return err | |
| 133 | + | } | |
| 134 | + | } | |
| 135 | + | if err := r.atomicWrite(r.headPath(key), marshalHead(publication.Generation, sha256.Sum256(publication.Manifest))); err != nil { | |
| 136 | + | return fmt.Errorf("write HEAD: %w", err) | |
| 137 | + | } | |
| 138 | + | return nil | |
| 139 | + | } | |
| 140 | + | ||
| 141 | + | func (r *Repository) List(ctx context.Context) ([]string, error) { | |
| 142 | + | if err := ctx.Err(); err != nil { | |
| 143 | + | return nil, err | |
| 144 | + | } | |
| 145 | + | entries, err := os.ReadDir(filepath.Join(r.dir, repositorySessionsDir)) | |
| 146 | + | if errors.Is(err, os.ErrNotExist) { | |
| 147 | + | return nil, nil | |
| 148 | + | } | |
| 149 | + | if err != nil { | |
| 150 | + | return nil, fmt.Errorf("read snapshot sessions: %w", err) | |
| 151 | + | } | |
| 152 | + | out := make([]string, 0, len(entries)) | |
| 153 | + | for _, entry := range entries { | |
| 154 | + | if err := ctx.Err(); err != nil { | |
| 155 | + | return nil, err | |
| 156 | + | } | |
| 157 | + | if !entry.IsDir() || !canonicalSessionKey(entry.Name()) { | |
| 158 | + | continue | |
| 159 | + | } | |
| 160 | + | // Loading is intentionally authoritative: a directory is listed only | |
| 161 | + | // when its complete newest valid generation can be restored. Unsafe | |
| 162 | + | // keys are not reversible, so obtain their name from a validated VEVM. | |
| 163 | + | numbers, err := r.generationNumbers(entry.Name()) | |
| 164 | + | if err != nil { | |
| 165 | + | continue | |
| 166 | + | } | |
| 167 | + | for _, number := range numbers { | |
| 168 | + | data, err := readBounded(r.manifestPath(entry.Name(), number)) | |
| 169 | + | if err != nil { | |
| 170 | + | continue | |
| 171 | + | } | |
| 172 | + | manifest, err := codec.UnmarshalManifest(data) | |
| 173 | + | if err != nil || sessionKey(manifest.Name) != entry.Name() { | |
| 174 | + | continue | |
| 175 | + | } | |
| 176 | + | generation, err := r.loadGeneration(ctx, manifest.Name, entry.Name(), number) | |
| 177 | + | if err == nil { | |
| 178 | + | out = append(out, generation.Name) | |
| 179 | + | break | |
| 180 | + | } | |
| 181 | + | } | |
| 182 | + | } | |
| 183 | + | sort.Strings(out) | |
| 184 | + | return out, nil | |
| 185 | + | } | |
| 186 | + | ||
| 187 | + | func (r *Repository) Load(ctx context.Context, name string) (ports.SnapshotGeneration, error) { | |
| 188 | + | if err := ctx.Err(); err != nil { | |
| 189 | + | return ports.SnapshotGeneration{}, err | |
| 190 | + | } | |
| 191 | + | key := sessionKey(name) | |
| 192 | + | // HEAD is only a hint. A torn/corrupt newest generation must never hide an | |
| 193 | + | // older complete generation. | |
| 194 | + | preferred, preferredDigest, err := r.readHead(key) | |
| 195 | + | if err != nil && !errors.Is(err, os.ErrNotExist) { | |
| 196 | + | preferred = 0 | |
| 197 | + | } | |
| 198 | + | candidates, err := r.generationNumbers(key) | |
| 199 | + | if err != nil { | |
| 200 | + | return ports.SnapshotGeneration{}, err | |
| 201 | + | } | |
| 202 | + | if preferred != 0 { | |
| 203 | + | candidates = append([]uint64{preferred}, removeGeneration(candidates, preferred)...) | |
| 204 | + | } | |
| 205 | + | skipped := false | |
| 206 | + | for _, generation := range candidates { | |
| 207 | + | if err := ctx.Err(); err != nil { | |
| 208 | + | return ports.SnapshotGeneration{}, err | |
| 209 | + | } | |
| 210 | + | got, err := r.loadGeneration(ctx, name, key, generation) | |
| 211 | + | if err != nil || (generation == preferred && sha256.Sum256(got.Manifest) != preferredDigest) { | |
| 212 | + | skipped = true | |
| 213 | + | continue | |
| 214 | + | } | |
| 215 | + | got.Fallback = skipped || (preferred != 0 && generation != preferred) | |
| 216 | + | return got, nil | |
| 217 | + | } | |
| 218 | + | return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name) | |
| 219 | + | } | |
| 220 | + | ||
| 221 | + | func (r *Repository) Delete(ctx context.Context, name string) error { | |
| 222 | + | if err := ctx.Err(); err != nil { | |
| 223 | + | return err | |
| 224 | + | } | |
| 225 | + | key := sessionKey(name) | |
| 226 | + | lock := r.sessionLock(key) | |
| 227 | + | lock.Lock() | |
| 228 | + | defer lock.Unlock() | |
| 229 | + | err := os.RemoveAll(r.sessionPath(key)) | |
| 230 | + | if err != nil { | |
| 231 | + | return fmt.Errorf("delete snapshot session: %w", err) | |
| 232 | + | } | |
| 233 | + | if _, err := os.Stat(r.dir); err == nil { | |
| 234 | + | return syncDirectory(r.dir) | |
| 235 | + | } | |
| 236 | + | return nil | |
| 237 | + | } | |
| 238 | + | ||
| 239 | + | // Maintain removes abandoned temporary and quarantine files. Immutable data is | |
| 240 | + | // deliberately never garbage-collected here because older generations need it. | |
| 241 | + | func (r *Repository) Maintain(ctx context.Context) error { | |
| 242 | + | if err := ctx.Err(); err != nil { | |
| 243 | + | return err | |
| 244 | + | } | |
| 245 | + | return filepath.WalkDir(r.dir, func(path string, entry os.DirEntry, err error) error { | |
| 246 | + | if err != nil { | |
| 247 | + | if errors.Is(err, os.ErrNotExist) { | |
| 248 | + | return nil | |
| 249 | + | } | |
| 250 | + | return err | |
| 251 | + | } | |
| 252 | + | if err := ctx.Err(); err != nil { | |
| 253 | + | return err | |
| 254 | + | } | |
| 255 | + | if entry.IsDir() { | |
| 256 | + | return nil | |
| 257 | + | } | |
| 258 | + | base := entry.Name() | |
| 259 | + | if strings.HasPrefix(base, ".tmp-") || strings.HasPrefix(base, ".quarantine-") { | |
| 260 | + | return os.Remove(path) | |
| 261 | + | } | |
| 262 | + | return nil | |
| 263 | + | }) | |
| 264 | + | } | |
| 265 | + | ||
| 266 | + | func (r *Repository) LoadLegacy(ctx context.Context) ([]ports.LegacySnapshot, error) { | |
| 267 | + | if err := ctx.Err(); err != nil { | |
| 268 | + | return nil, err | |
| 269 | + | } | |
| 270 | + | blobs, err := r.legacy.Load() | |
| 271 | + | if err != nil { | |
| 272 | + | return nil, err | |
| 273 | + | } | |
| 274 | + | out := make([]ports.LegacySnapshot, len(blobs)) | |
| 275 | + | for i, blob := range blobs { | |
| 276 | + | out[i] = ports.LegacySnapshot{Name: blob.Name, Data: append([]byte(nil), blob.Data...)} | |
| 277 | + | } | |
| 278 | + | return out, nil | |
| 279 | + | } | |
| 280 | + | func (r *Repository) DeleteLegacy(ctx context.Context, name string) error { | |
| 281 | + | if err := ctx.Err(); err != nil { | |
| 282 | + | return err | |
| 283 | + | } | |
| 284 | + | return r.legacy.Delete(name) | |
| 285 | + | } | |
| 286 | + | ||
| 287 | + | func validatePublication(p ports.SnapshotPublication) (codec.Manifest, map[ports.SnapshotDigest]codec.ObjectRef, map[ports.SnapshotDigest]ports.SnapshotObject, error) { | |
| 288 | + | manifest, err := codec.UnmarshalManifest(p.Manifest) | |
| 289 | + | if err != nil { | |
| 290 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("invalid manifest: %w", err) | |
| 291 | + | } | |
| 292 | + | if manifest.Name != p.Name || manifest.Generation != p.Generation || p.Generation == 0 { | |
| 293 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("manifest name or generation does not match publication") | |
| 294 | + | } | |
| 295 | + | refs := manifestRefs(manifest) | |
| 296 | + | if refs == nil { | |
| 297 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("conflicting manifest references") | |
| 298 | + | } | |
| 299 | + | objects := make(map[ports.SnapshotDigest]ports.SnapshotObject, len(p.Objects)) | |
| 300 | + | for _, object := range p.Objects { | |
| 301 | + | if sha256.Sum256(object.Data) != object.Digest { | |
| 302 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("object digest mismatch") | |
| 303 | + | } | |
| 304 | + | ref, used := refs[object.Digest] | |
| 305 | + | if !used { | |
| 306 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("unreferenced object") | |
| 307 | + | } | |
| 308 | + | kind, payload, err := codec.PreflightObject(object.Data) | |
| 309 | + | if err != nil || kind != ref.Kind || len(object.Data) != int(ref.Size) || len(payload) == 0 { | |
| 310 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("invalid object envelope") | |
| 311 | + | } | |
| 312 | + | if old, duplicate := objects[object.Digest]; duplicate && !equalBytes(old.Data, object.Data) { | |
| 313 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("conflicting object") | |
| 314 | + | } | |
| 315 | + | objects[object.Digest] = ports.SnapshotObject{Digest: object.Digest, Data: append([]byte(nil), object.Data...)} | |
| 316 | + | } | |
| 317 | + | return manifest, refs, objects, nil | |
| 318 | + | } | |
| 319 | + | ||
| 320 | + | func manifestRefs(manifest codec.Manifest) map[ports.SnapshotDigest]codec.ObjectRef { | |
| 321 | + | refs := make(map[ports.SnapshotDigest]codec.ObjectRef) | |
| 322 | + | add := func(ref codec.ObjectRef) bool { | |
| 323 | + | if old, ok := refs[ref.Digest]; ok && old != ref { | |
| 324 | + | return false | |
| 325 | + | } | |
| 326 | + | refs[ref.Digest] = ref | |
| 327 | + | return true | |
| 328 | + | } | |
| 329 | + | for _, tab := range manifest.Tabs { | |
| 330 | + | for _, pane := range tab.Panes { | |
| 331 | + | for _, ref := range pane.Sealed { | |
| 332 | + | if !add(ref) { | |
| 333 | + | return nil | |
| 334 | + | } | |
| 335 | + | } | |
| 336 | + | if !add(pane.Tail) || !add(pane.Visible) { | |
| 337 | + | return nil | |
| 338 | + | } | |
| 339 | + | } | |
| 340 | + | } | |
| 341 | + | return refs | |
| 342 | + | } | |
| 343 | + | ||
| 344 | + | func (r *Repository) currentGeneration(ctx context.Context, name, key string) (uint64, []byte, error) { | |
| 345 | + | generation, digest, err := r.readHead(key) | |
| 346 | + | if err == nil { | |
| 347 | + | got, err := r.loadGeneration(ctx, name, key, generation) | |
| 348 | + | if err == nil && sha256.Sum256(got.Manifest) == digest { | |
| 349 | + | return generation, got.Manifest, nil | |
| 350 | + | } | |
| 351 | + | } | |
| 352 | + | candidates, err := r.generationNumbers(key) | |
| 353 | + | if err != nil { | |
| 354 | + | return 0, nil, err | |
| 355 | + | } | |
| 356 | + | for _, candidate := range candidates { | |
| 357 | + | if got, err := r.loadGeneration(ctx, name, key, candidate); err == nil { | |
| 358 | + | return candidate, got.Manifest, nil | |
| 359 | + | } | |
| 360 | + | } | |
| 361 | + | return 0, nil, nil | |
| 362 | + | } | |
| 363 | + | ||
| 364 | + | func (r *Repository) loadGeneration(ctx context.Context, name, key string, generation uint64) (ports.SnapshotGeneration, error) { | |
| 365 | + | data, err := readBounded(r.manifestPath(key, generation)) | |
| 366 | + | if err != nil { | |
| 367 | + | return ports.SnapshotGeneration{}, err | |
| 368 | + | } | |
| 369 | + | manifest, err := codec.UnmarshalManifest(data) | |
| 370 | + | if err != nil || manifest.Name != name || manifest.Generation != generation { | |
| 371 | + | return ports.SnapshotGeneration{}, fmt.Errorf("invalid manifest") | |
| 372 | + | } | |
| 373 | + | refs := manifestRefs(manifest) | |
| 374 | + | if refs == nil { | |
| 375 | + | return ports.SnapshotGeneration{}, fmt.Errorf("conflicting manifest references") | |
| 376 | + | } | |
| 377 | + | objects := make(map[ports.SnapshotDigest][]byte, len(refs)) | |
| 378 | + | total := len(data) | |
| 379 | + | for digest, ref := range refs { | |
| 380 | + | if err := ctx.Err(); err != nil { | |
| 381 | + | return ports.SnapshotGeneration{}, err | |
| 382 | + | } | |
| 383 | + | // Ref sizes were validated by the VEVM preflight. Check the aggregate | |
| 384 | + | // before allocating another object so hostile repositories cannot exceed | |
| 385 | + | // the restore budget transiently. | |
| 386 | + | if uint64(ref.Size) > uint64(maxRepositoryRead-total) { | |
| 387 | + | return ports.SnapshotGeneration{}, fmt.Errorf("snapshot generation too large") | |
| 388 | + | } | |
| 389 | + | object, err := readBounded(r.objectPath(key, digest)) | |
| 390 | + | if err != nil { | |
| 391 | + | return ports.SnapshotGeneration{}, err | |
| 392 | + | } | |
| 393 | + | total += len(object) | |
| 394 | + | if sha256.Sum256(object) != digest || !validObject(object, ref) { | |
| 395 | + | return ports.SnapshotGeneration{}, fmt.Errorf("invalid object") | |
| 396 | + | } | |
| 397 | + | objects[digest] = append([]byte(nil), object...) | |
| 398 | + | } | |
| 399 | + | return ports.SnapshotGeneration{Name: manifest.Name, Generation: generation, Manifest: append([]byte(nil), data...), Objects: objects}, nil | |
| 400 | + | } | |
| 401 | + | ||
| 402 | + | func validObject(data []byte, ref codec.ObjectRef) bool { | |
| 403 | + | kind, payload, err := codec.PreflightObject(data) | |
| 404 | + | return err == nil && kind == ref.Kind && len(payload) > 0 && len(data) == int(ref.Size) | |
| 405 | + | } | |
| 406 | + | func verifyObjectFile(path string, digest ports.SnapshotDigest, ref codec.ObjectRef) (bool, error) { | |
| 407 | + | data, exists, err := readOptionalBounded(path) | |
| 408 | + | if err != nil || !exists { | |
| 409 | + | return exists, err | |
| 410 | + | } | |
| 411 | + | return true, func() error { | |
| 412 | + | if sha256.Sum256(data) != digest || !validObject(data, ref) { | |
| 413 | + | return fmt.Errorf("existing immutable object is invalid") | |
| 414 | + | } | |
| 415 | + | return nil | |
| 416 | + | }() | |
| 417 | + | } | |
| 418 | + | ||
| 419 | + | func (r *Repository) ensureSession(key string) error { | |
| 420 | + | for _, dir := range []string{r.dir, filepath.Join(r.dir, repositorySessionsDir), r.sessionPath(key), filepath.Join(r.sessionPath(key), repositoryObjectsDir), filepath.Join(r.sessionPath(key), repositoryGenerations)} { | |
| 421 | + | if err := safedir.EnsurePrivate(dir); err != nil { | |
| 422 | + | return fmt.Errorf("create snapshot repository directory: %w", err) | |
| 423 | + | } | |
| 424 | + | } | |
| 425 | + | return nil | |
| 426 | + | } | |
| 427 | + | func (r *Repository) sessionPath(key string) string { | |
| 428 | + | return filepath.Join(r.dir, repositorySessionsDir, key) | |
| 429 | + | } | |
| 430 | + | func (r *Repository) objectPath(key string, digest ports.SnapshotDigest) string { | |
| 431 | + | hexDigest := hex.EncodeToString(digest[:]) | |
| 432 | + | return filepath.Join(r.sessionPath(key), repositoryObjectsDir, hexDigest[:2], hexDigest) | |
| 433 | + | } | |
| 434 | + | func (r *Repository) manifestPath(key string, generation uint64) string { | |
| 435 | + | return filepath.Join(r.sessionPath(key), repositoryGenerations, generationFilename(generation)) | |
| 436 | + | } | |
| 437 | + | func (r *Repository) headPath(key string) string { | |
| 438 | + | return filepath.Join(r.sessionPath(key), repositoryHead) | |
| 439 | + | } | |
| 440 | + | func (r *Repository) sessionLock(key string) *sync.Mutex { | |
| 441 | + | lock, _ := r.locks.LoadOrStore(key, &sync.Mutex{}) | |
| 442 | + | return lock.(*sync.Mutex) | |
| 443 | + | } | |
| 444 | + | ||
| 445 | + | func sessionKey(name string) string { | |
| 446 | + | if safeNameRE.MatchString(name) && name != "." && name != ".." { | |
| 447 | + | return name | |
| 448 | + | } | |
| 449 | + | sum := sha256.Sum256([]byte(name)) | |
| 450 | + | return "@" + hex.EncodeToString(sum[:]) | |
| 451 | + | } | |
| 452 | + | func canonicalSessionKey(key string) bool { | |
| 453 | + | if safeNameRE.MatchString(key) && key != "." && key != ".." { | |
| 454 | + | return true | |
| 455 | + | } | |
| 456 | + | if len(key) != 65 || key[0] != '@' { | |
| 457 | + | return false | |
| 458 | + | } | |
| 459 | + | _, err := hex.DecodeString(key[1:]) | |
| 460 | + | return err == nil && strings.ToLower(key[1:]) == key[1:] | |
| 461 | + | } | |
| 462 | + | func nameFromSafeKey(key string) string { | |
| 463 | + | if strings.HasPrefix(key, "@") { | |
| 464 | + | return "" | |
| 465 | + | } | |
| 466 | + | return key | |
| 467 | + | } | |
| 468 | + | func generationFilename(generation uint64) string { | |
| 469 | + | return fmt.Sprintf("%0*d.manifest", generationWidth, generation) | |
| 470 | + | } | |
| 471 | + | func parseGenerationFilename(name string) (uint64, bool) { | |
| 472 | + | if !strings.HasSuffix(name, ".manifest") || len(name) != generationWidth+len(".manifest") { | |
| 473 | + | return 0, false | |
| 474 | + | } | |
| 475 | + | n := strings.TrimSuffix(name, ".manifest") | |
| 476 | + | if strings.HasPrefix(n, "0") && n != fmt.Sprintf("%0*d", generationWidth, 0) { /* zero padding required, strconv below verifies width */ | |
| 477 | + | } | |
| 478 | + | generation, err := strconv.ParseUint(n, 10, 64) | |
| 479 | + | return generation, err == nil && generationFilename(generation) == name && generation != 0 | |
| 480 | + | } | |
| 481 | + | ||
| 482 | + | func (r *Repository) generationNumbers(key string) ([]uint64, error) { | |
| 483 | + | entries, err := os.ReadDir(filepath.Join(r.sessionPath(key), repositoryGenerations)) | |
| 484 | + | if errors.Is(err, os.ErrNotExist) { | |
| 485 | + | return nil, nil | |
| 486 | + | } | |
| 487 | + | if err != nil { | |
| 488 | + | return nil, err | |
| 489 | + | } | |
| 490 | + | numbers := make([]uint64, 0, len(entries)) | |
| 491 | + | for _, entry := range entries { | |
| 492 | + | if entry.IsDir() { | |
| 493 | + | continue | |
| 494 | + | } | |
| 495 | + | if generation, ok := parseGenerationFilename(entry.Name()); ok { | |
| 496 | + | numbers = append(numbers, generation) | |
| 497 | + | } | |
| 498 | + | } | |
| 499 | + | sort.Slice(numbers, func(i, j int) bool { return numbers[i] > numbers[j] }) | |
| 500 | + | return numbers, nil | |
| 501 | + | } | |
| 502 | + | func removeGeneration(numbers []uint64, generation uint64) []uint64 { | |
| 503 | + | out := numbers[:0] | |
| 504 | + | for _, n := range numbers { | |
| 505 | + | if n != generation { | |
| 506 | + | out = append(out, n) | |
| 507 | + | } | |
| 508 | + | } | |
| 509 | + | return out | |
| 510 | + | } | |
| 511 | + | ||
| 512 | + | func marshalHead(generation uint64, digest ports.SnapshotDigest) []byte { | |
| 513 | + | out := make([]byte, 4+8+sha256.Size) | |
| 514 | + | copy(out, "VEVH") | |
| 515 | + | binary.BigEndian.PutUint64(out[4:12], generation) | |
| 516 | + | copy(out[12:], digest[:]) | |
| 517 | + | return out | |
| 518 | + | } | |
| 519 | + | func (r *Repository) readHead(key string) (uint64, ports.SnapshotDigest, error) { | |
| 520 | + | data, err := readBounded(r.headPath(key)) | |
| 521 | + | if err != nil { | |
| 522 | + | return 0, ports.SnapshotDigest{}, err | |
| 523 | + | } | |
| 524 | + | if len(data) != 4+8+sha256.Size || string(data[:4]) != "VEVH" { | |
| 525 | + | return 0, ports.SnapshotDigest{}, fmt.Errorf("invalid HEAD") | |
| 526 | + | } | |
| 527 | + | generation := binary.BigEndian.Uint64(data[4:12]) | |
| 528 | + | var digest ports.SnapshotDigest | |
| 529 | + | copy(digest[:], data[12:]) | |
| 530 | + | if generation == 0 { | |
| 531 | + | return 0, ports.SnapshotDigest{}, fmt.Errorf("invalid HEAD") | |
| 532 | + | } | |
| 533 | + | return generation, digest, nil | |
| 534 | + | } | |
| 535 | + | ||
| 536 | + | func (r *Repository) atomicWrite(path string, data []byte) error { | |
| 537 | + | if r.hooks.writeFile != nil { | |
| 538 | + | return r.hooks.writeFile(path, append([]byte(nil), data...)) | |
| 539 | + | } | |
| 540 | + | return atomicWriteFile(path, data) | |
| 541 | + | } | |
| 542 | + | func atomicWriteFile(path string, data []byte) error { | |
| 543 | + | dir := filepath.Dir(path) | |
| 544 | + | if err := safedir.EnsurePrivate(dir); err != nil { | |
| 545 | + | return err | |
| 546 | + | } | |
| 547 | + | temp, err := os.CreateTemp(dir, ".tmp-") | |
| 548 | + | if err != nil { | |
| 549 | + | return err | |
| 550 | + | } | |
| 551 | + | tempPath := temp.Name() | |
| 552 | + | cleanup := true | |
| 553 | + | defer func() { | |
| 554 | + | if cleanup { | |
| 555 | + | _ = os.Remove(tempPath) | |
| 556 | + | } | |
| 557 | + | }() | |
| 558 | + | if err := temp.Chmod(0o600); err != nil { | |
| 559 | + | _ = temp.Close() | |
| 560 | + | return err | |
| 561 | + | } | |
| 562 | + | if _, err := temp.Write(data); err != nil { | |
| 563 | + | _ = temp.Close() | |
| 564 | + | return err | |
| 565 | + | } | |
| 566 | + | if err := temp.Sync(); err != nil { | |
| 567 | + | _ = temp.Close() | |
| 568 | + | return err | |
| 569 | + | } | |
| 570 | + | if err := temp.Close(); err != nil { | |
| 571 | + | return err | |
| 572 | + | } | |
| 573 | + | if err := os.Rename(tempPath, path); err != nil { | |
| 574 | + | return err | |
| 575 | + | } | |
| 576 | + | cleanup = false | |
| 577 | + | return syncDirectory(dir) | |
| 578 | + | } | |
| 579 | + | func syncDirectory(dir string) error { | |
| 580 | + | f, err := os.Open(dir) | |
| 581 | + | if err != nil { | |
| 582 | + | return err | |
| 583 | + | } | |
| 584 | + | defer f.Close() | |
| 585 | + | return f.Sync() | |
| 586 | + | } | |
| 587 | + | func readOptionalBounded(path string) ([]byte, bool, error) { | |
| 588 | + | data, err := readBounded(path) | |
| 589 | + | if errors.Is(err, os.ErrNotExist) { | |
| 590 | + | return nil, false, nil | |
| 591 | + | } | |
| 592 | + | return data, err == nil, err | |
| 593 | + | } | |
| 594 | + | func readBounded(path string) ([]byte, error) { | |
| 595 | + | info, err := os.Lstat(path) | |
| 596 | + | if err != nil { | |
| 597 | + | return nil, err | |
| 598 | + | } | |
| 599 | + | if !info.Mode().IsRegular() { | |
| 600 | + | return nil, fmt.Errorf("not a regular file") | |
| 601 | + | } | |
| 602 | + | if info.Size() < 0 || info.Size() > maxRepositoryRead { | |
| 603 | + | return nil, fmt.Errorf("snapshot file too large") | |
| 604 | + | } | |
| 605 | + | f, err := os.Open(path) | |
| 606 | + | if err != nil { | |
| 607 | + | return nil, err | |
| 608 | + | } | |
| 609 | + | defer f.Close() | |
| 610 | + | data, err := io.ReadAll(io.LimitReader(f, maxRepositoryRead+1)) | |
| 611 | + | if err != nil { | |
| 612 | + | return nil, err | |
| 613 | + | } | |
| 614 | + | if len(data) > maxRepositoryRead { | |
| 615 | + | return nil, fmt.Errorf("snapshot file too large") | |
| 616 | + | } | |
| 617 | + | return data, nil | |
| 618 | + | } | |
| 619 | + | func equalBytes(a, b []byte) bool { | |
| 620 | + | if len(a) != len(b) { | |
| 621 | + | return false | |
| 622 | + | } | |
| 623 | + | for i := range a { | |
| 624 | + | if a[i] != b[i] { | |
| 625 | + | return false | |
| 626 | + | } | |
| 627 | + | } | |
| 628 | + | return true | |
| 629 | + | } | |
| @@ -0,0 +1,629 @@ | |||||
| 1 | + | package snapshot | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "crypto/sha256" | |||
| 6 | + | "encoding/binary" | |||
| 7 | + | "encoding/hex" | |||
| 8 | + | "errors" | |||
| 9 | + | "fmt" | |||
| 10 | + | "io" | |||
| 11 | + | "os" | |||
| 12 | + | "path/filepath" | |||
| 13 | + | "sort" | |||
| 14 | + | "strconv" | |||
| 15 | + | "strings" | |||
| 16 | + | "sync" | |||
| 17 | + | ||||
| 18 | + | "github.com/bnema/vev/internal/ports" | |||
| 19 | + | codec "github.com/bnema/vev/internal/usecase/snapshot" | |||
| 20 | + | "github.com/bnema/vev/pkg/safedir" | |||
| 21 | + | ) | |||
| 22 | + | ||||
| 23 | + | const ( | |||
| 24 | + | repositorySessionsDir = "sessions" | |||
| 25 | + | repositoryObjectsDir = "objects" | |||
| 26 | + | repositoryGenerations = "generations" | |||
| 27 | + | repositoryHead = "HEAD" | |||
| 28 | + | generationWidth = 20 | |||
| 29 | + | maxRepositoryRead = maxSnapshotFileSize | |||
| 30 | + | ) | |||
| 31 | + | ||||
| 32 | + | // Repository is the crash-safe, content-addressed session snapshot store. | |||
| 33 | + | // Store remains available separately for the one-way legacy bridge. | |||
| 34 | + | type Repository struct { | |||
| 35 | + | dir string | |||
| 36 | + | legacy *Store | |||
| 37 | + | locks sync.Map // map[string]*sync.Mutex | |||
| 38 | + | hooks repositoryHooks | |||
| 39 | + | } | |||
| 40 | + | ||||
| 41 | + | type repositoryHooks struct { | |||
| 42 | + | beforeBlobWrite func(string) error | |||
| 43 | + | beforeManifestWrite func(string) error | |||
| 44 | + | beforeHeadWrite func(string) error | |||
| 45 | + | writeFile func(string, []byte) error | |||
| 46 | + | } | |||
| 47 | + | ||||
| 48 | + | var _ ports.SnapshotRepository = (*Repository)(nil) | |||
| 49 | + | var _ ports.LegacySnapshotSource = (*Repository)(nil) | |||
| 50 | + | ||||
| 51 | + | // NewRepository creates a repository rooted at dir. It does not create files | |||
| 52 | + | // until the first publication, so merely constructing it is side-effect free. | |||
| 53 | + | func NewRepository(dir string) *Repository { return &Repository{dir: dir, legacy: NewStore(dir)} } | |||
| 54 | + | ||||
| 55 | + | func (r *Repository) Publish(ctx context.Context, publication ports.SnapshotPublication) error { | |||
| 56 | + | if err := ctx.Err(); err != nil { | |||
| 57 | + | return err | |||
| 58 | + | } | |||
| 59 | + | key := sessionKey(publication.Name) | |||
| 60 | + | lock := r.sessionLock(key) | |||
| 61 | + | lock.Lock() | |||
| 62 | + | defer lock.Unlock() | |||
| 63 | + | ||||
| 64 | + | manifest, refs, supplied, err := validatePublication(publication) | |||
| 65 | + | if err != nil { | |||
| 66 | + | return err | |||
| 67 | + | } | |||
| 68 | + | _ = manifest | |||
| 69 | + | if err := r.ensureSession(key); err != nil { | |||
| 70 | + | return err | |||
| 71 | + | } | |||
| 72 | + | ||||
| 73 | + | current, currentManifest, err := r.currentGeneration(ctx, publication.Name, key) | |||
| 74 | + | if err != nil { | |||
| 75 | + | return err | |||
| 76 | + | } | |||
| 77 | + | if publication.Generation < current || publication.Generation > current+1 { | |||
| 78 | + | return fmt.Errorf("snapshot generation %d, current %d: immutable conflict", publication.Generation, current) | |||
| 79 | + | } | |||
| 80 | + | if publication.Generation == current { | |||
| 81 | + | if !equalBytes(currentManifest, publication.Manifest) { | |||
| 82 | + | return fmt.Errorf("snapshot generation %d: immutable conflict", publication.Generation) | |||
| 83 | + | } | |||
| 84 | + | return nil // exact replay; validation above also rejects altered objects. | |||
| 85 | + | } | |||
| 86 | + | ||||
| 87 | + | for digest, ref := range refs { | |||
| 88 | + | if err := ctx.Err(); err != nil { | |||
| 89 | + | return err | |||
| 90 | + | } | |||
| 91 | + | path := r.objectPath(key, digest) | |||
| 92 | + | exists, err := verifyObjectFile(path, digest, ref) | |||
| 93 | + | if err != nil { | |||
| 94 | + | return fmt.Errorf("verify object %x: %w", digest, err) | |||
| 95 | + | } | |||
| 96 | + | if exists { | |||
| 97 | + | continue | |||
| 98 | + | } | |||
| 99 | + | object, supplied := supplied[digest] | |||
| 100 | + | if !supplied { | |||
| 101 | + | return fmt.Errorf("missing referenced object %x", digest) | |||
| 102 | + | } | |||
| 103 | + | if r.hooks.beforeBlobWrite != nil { | |||
| 104 | + | if err := r.hooks.beforeBlobWrite(path); err != nil { | |||
| 105 | + | return err | |||
| 106 | + | } | |||
| 107 | + | } | |||
| 108 | + | if err := r.atomicWrite(path, object.Data); err != nil { | |||
| 109 | + | return fmt.Errorf("write object: %w", err) | |||
| 110 | + | } | |||
| 111 | + | } | |||
| 112 | + | ||||
| 113 | + | manifestPath := r.manifestPath(key, publication.Generation) | |||
| 114 | + | if existing, exists, err := readOptionalBounded(manifestPath); err != nil { | |||
| 115 | + | return err | |||
| 116 | + | } else if exists { | |||
| 117 | + | if !equalBytes(existing, publication.Manifest) { | |||
| 118 | + | return fmt.Errorf("manifest generation %d: immutable conflict", publication.Generation) | |||
| 119 | + | } | |||
| 120 | + | } else { | |||
| 121 | + | if r.hooks.beforeManifestWrite != nil { | |||
| 122 | + | if err := r.hooks.beforeManifestWrite(manifestPath); err != nil { | |||
| 123 | + | return err | |||
| 124 | + | } | |||
| 125 | + | } | |||
| 126 | + | if err := r.atomicWrite(manifestPath, publication.Manifest); err != nil { | |||
| 127 | + | return fmt.Errorf("write manifest: %w", err) | |||
| 128 | + | } | |||
| 129 | + | } | |||
| 130 | + | if r.hooks.beforeHeadWrite != nil { | |||
| 131 | + | if err := r.hooks.beforeHeadWrite(r.headPath(key)); err != nil { | |||
| 132 | + | return err | |||
| 133 | + | } | |||
| 134 | + | } | |||
| 135 | + | if err := r.atomicWrite(r.headPath(key), marshalHead(publication.Generation, sha256.Sum256(publication.Manifest))); err != nil { | |||
| 136 | + | return fmt.Errorf("write HEAD: %w", err) | |||
| 137 | + | } | |||
| 138 | + | return nil | |||
| 139 | + | } | |||
| 140 | + | ||||
| 141 | + | func (r *Repository) List(ctx context.Context) ([]string, error) { | |||
| 142 | + | if err := ctx.Err(); err != nil { | |||
| 143 | + | return nil, err | |||
| 144 | + | } | |||
| 145 | + | entries, err := os.ReadDir(filepath.Join(r.dir, repositorySessionsDir)) | |||
| 146 | + | if errors.Is(err, os.ErrNotExist) { | |||
| 147 | + | return nil, nil | |||
| 148 | + | } | |||
| 149 | + | if err != nil { | |||
| 150 | + | return nil, fmt.Errorf("read snapshot sessions: %w", err) | |||
| 151 | + | } | |||
| 152 | + | out := make([]string, 0, len(entries)) | |||
| 153 | + | for _, entry := range entries { | |||
| 154 | + | if err := ctx.Err(); err != nil { | |||
| 155 | + | return nil, err | |||
| 156 | + | } | |||
| 157 | + | if !entry.IsDir() || !canonicalSessionKey(entry.Name()) { | |||
| 158 | + | continue | |||
| 159 | + | } | |||
| 160 | + | // Loading is intentionally authoritative: a directory is listed only | |||
| 161 | + | // when its complete newest valid generation can be restored. Unsafe | |||
| 162 | + | // keys are not reversible, so obtain their name from a validated VEVM. | |||
| 163 | + | numbers, err := r.generationNumbers(entry.Name()) | |||
| 164 | + | if err != nil { | |||
| 165 | + | continue | |||
| 166 | + | } | |||
| 167 | + | for _, number := range numbers { | |||
| 168 | + | data, err := readBounded(r.manifestPath(entry.Name(), number)) | |||
| 169 | + | if err != nil { | |||
| 170 | + | continue | |||
| 171 | + | } | |||
| 172 | + | manifest, err := codec.UnmarshalManifest(data) | |||
| 173 | + | if err != nil || sessionKey(manifest.Name) != entry.Name() { | |||
| 174 | + | continue | |||
| 175 | + | } | |||
| 176 | + | generation, err := r.loadGeneration(ctx, manifest.Name, entry.Name(), number) | |||
| 177 | + | if err == nil { | |||
| 178 | + | out = append(out, generation.Name) | |||
| 179 | + | break | |||
| 180 | + | } | |||
| 181 | + | } | |||
| 182 | + | } | |||
| 183 | + | sort.Strings(out) | |||
| 184 | + | return out, nil | |||
| 185 | + | } | |||
| 186 | + | ||||
| 187 | + | func (r *Repository) Load(ctx context.Context, name string) (ports.SnapshotGeneration, error) { | |||
| 188 | + | if err := ctx.Err(); err != nil { | |||
| 189 | + | return ports.SnapshotGeneration{}, err | |||
| 190 | + | } | |||
| 191 | + | key := sessionKey(name) | |||
| 192 | + | // HEAD is only a hint. A torn/corrupt newest generation must never hide an | |||
| 193 | + | // older complete generation. | |||
| 194 | + | preferred, preferredDigest, err := r.readHead(key) | |||
| 195 | + | if err != nil && !errors.Is(err, os.ErrNotExist) { | |||
| 196 | + | preferred = 0 | |||
| 197 | + | } | |||
| 198 | + | candidates, err := r.generationNumbers(key) | |||
| 199 | + | if err != nil { | |||
| 200 | + | return ports.SnapshotGeneration{}, err | |||
| 201 | + | } | |||
| 202 | + | if preferred != 0 { | |||
| 203 | + | candidates = append([]uint64{preferred}, removeGeneration(candidates, preferred)...) | |||
| 204 | + | } | |||
| 205 | + | skipped := false | |||
| 206 | + | for _, generation := range candidates { | |||
| 207 | + | if err := ctx.Err(); err != nil { | |||
| 208 | + | return ports.SnapshotGeneration{}, err | |||
| 209 | + | } | |||
| 210 | + | got, err := r.loadGeneration(ctx, name, key, generation) | |||
| 211 | + | if err != nil || (generation == preferred && sha256.Sum256(got.Manifest) != preferredDigest) { | |||
| 212 | + | skipped = true | |||
| 213 | + | continue | |||
| 214 | + | } | |||
| 215 | + | got.Fallback = skipped || (preferred != 0 && generation != preferred) | |||
| 216 | + | return got, nil | |||
| 217 | + | } | |||
| 218 | + | return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name) | |||
| 219 | + | } | |||
| 220 | + | ||||
| 221 | + | func (r *Repository) Delete(ctx context.Context, name string) error { | |||
| 222 | + | if err := ctx.Err(); err != nil { | |||
| 223 | + | return err | |||
| 224 | + | } | |||
| 225 | + | key := sessionKey(name) | |||
| 226 | + | lock := r.sessionLock(key) | |||
| 227 | + | lock.Lock() | |||
| 228 | + | defer lock.Unlock() | |||
| 229 | + | err := os.RemoveAll(r.sessionPath(key)) | |||
| 230 | + | if err != nil { | |||
| 231 | + | return fmt.Errorf("delete snapshot session: %w", err) | |||
| 232 | + | } | |||
| 233 | + | if _, err := os.Stat(r.dir); err == nil { | |||
| 234 | + | return syncDirectory(r.dir) | |||
| 235 | + | } | |||
| 236 | + | return nil | |||
| 237 | + | } | |||
| 238 | + | ||||
| 239 | + | // Maintain removes abandoned temporary and quarantine files. Immutable data is | |||
| 240 | + | // deliberately never garbage-collected here because older generations need it. | |||
| 241 | + | func (r *Repository) Maintain(ctx context.Context) error { | |||
| 242 | + | if err := ctx.Err(); err != nil { | |||
| 243 | + | return err | |||
| 244 | + | } | |||
| 245 | + | return filepath.WalkDir(r.dir, func(path string, entry os.DirEntry, err error) error { | |||
| 246 | + | if err != nil { | |||
| 247 | + | if errors.Is(err, os.ErrNotExist) { | |||
| 248 | + | return nil | |||
| 249 | + | } | |||
| 250 | + | return err | |||
| 251 | + | } | |||
| 252 | + | if err := ctx.Err(); err != nil { | |||
| 253 | + | return err | |||
| 254 | + | } | |||
| 255 | + | if entry.IsDir() { | |||
| 256 | + | return nil | |||
| 257 | + | } | |||
| 258 | + | base := entry.Name() | |||
| 259 | + | if strings.HasPrefix(base, ".tmp-") || strings.HasPrefix(base, ".quarantine-") { | |||
| 260 | + | return os.Remove(path) | |||
| 261 | + | } | |||
| 262 | + | return nil | |||
| 263 | + | }) | |||
| 264 | + | } | |||
| 265 | + | ||||
| 266 | + | func (r *Repository) LoadLegacy(ctx context.Context) ([]ports.LegacySnapshot, error) { | |||
| 267 | + | if err := ctx.Err(); err != nil { | |||
| 268 | + | return nil, err | |||
| 269 | + | } | |||
| 270 | + | blobs, err := r.legacy.Load() | |||
| 271 | + | if err != nil { | |||
| 272 | + | return nil, err | |||
| 273 | + | } | |||
| 274 | + | out := make([]ports.LegacySnapshot, len(blobs)) | |||
| 275 | + | for i, blob := range blobs { | |||
| 276 | + | out[i] = ports.LegacySnapshot{Name: blob.Name, Data: append([]byte(nil), blob.Data...)} | |||
| 277 | + | } | |||
| 278 | + | return out, nil | |||
| 279 | + | } | |||
| 280 | + | func (r *Repository) DeleteLegacy(ctx context.Context, name string) error { | |||
| 281 | + | if err := ctx.Err(); err != nil { | |||
| 282 | + | return err | |||
| 283 | + | } | |||
| 284 | + | return r.legacy.Delete(name) | |||
| 285 | + | } | |||
| 286 | + | ||||
| 287 | + | func validatePublication(p ports.SnapshotPublication) (codec.Manifest, map[ports.SnapshotDigest]codec.ObjectRef, map[ports.SnapshotDigest]ports.SnapshotObject, error) { | |||
| 288 | + | manifest, err := codec.UnmarshalManifest(p.Manifest) | |||
| 289 | + | if err != nil { | |||
| 290 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("invalid manifest: %w", err) | |||
| 291 | + | } | |||
| 292 | + | if manifest.Name != p.Name || manifest.Generation != p.Generation || p.Generation == 0 { | |||
| 293 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("manifest name or generation does not match publication") | |||
| 294 | + | } | |||
| 295 | + | refs := manifestRefs(manifest) | |||
| 296 | + | if refs == nil { | |||
| 297 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("conflicting manifest references") | |||
| 298 | + | } | |||
| 299 | + | objects := make(map[ports.SnapshotDigest]ports.SnapshotObject, len(p.Objects)) | |||
| 300 | + | for _, object := range p.Objects { | |||
| 301 | + | if sha256.Sum256(object.Data) != object.Digest { | |||
| 302 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("object digest mismatch") | |||
| 303 | + | } | |||
| 304 | + | ref, used := refs[object.Digest] | |||
| 305 | + | if !used { | |||
| 306 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("unreferenced object") | |||
| 307 | + | } | |||
| 308 | + | kind, payload, err := codec.PreflightObject(object.Data) | |||
| 309 | + | if err != nil || kind != ref.Kind || len(object.Data) != int(ref.Size) || len(payload) == 0 { | |||
| 310 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("invalid object envelope") | |||
| 311 | + | } | |||
| 312 | + | if old, duplicate := objects[object.Digest]; duplicate && !equalBytes(old.Data, object.Data) { | |||
| 313 | + | return codec.Manifest{}, nil, nil, fmt.Errorf("conflicting object") | |||
| 314 | + | } | |||
| 315 | + | objects[object.Digest] = ports.SnapshotObject{Digest: object.Digest, Data: append([]byte(nil), object.Data...)} | |||
| 316 | + | } | |||
| 317 | + | return manifest, refs, objects, nil | |||
| 318 | + | } | |||
| 319 | + | ||||
| 320 | + | func manifestRefs(manifest codec.Manifest) map[ports.SnapshotDigest]codec.ObjectRef { | |||
| 321 | + | refs := make(map[ports.SnapshotDigest]codec.ObjectRef) | |||
| 322 | + | add := func(ref codec.ObjectRef) bool { | |||
| 323 | + | if old, ok := refs[ref.Digest]; ok && old != ref { | |||
| 324 | + | return false | |||
| 325 | + | } | |||
| 326 | + | refs[ref.Digest] = ref | |||
| 327 | + | return true | |||
| 328 | + | } | |||
| 329 | + | for _, tab := range manifest.Tabs { | |||
| 330 | + | for _, pane := range tab.Panes { | |||
| 331 | + | for _, ref := range pane.Sealed { | |||
| 332 | + | if !add(ref) { | |||
| 333 | + | return nil | |||
| 334 | + | } | |||
| 335 | + | } | |||
| 336 | + | if !add(pane.Tail) || !add(pane.Visible) { | |||
| 337 | + | return nil | |||
| 338 | + | } | |||
| 339 | + | } | |||
| 340 | + | } | |||
| 341 | + | return refs | |||
| 342 | + | } | |||
| 343 | + | ||||
| 344 | + | func (r *Repository) currentGeneration(ctx context.Context, name, key string) (uint64, []byte, error) { | |||
| 345 | + | generation, digest, err := r.readHead(key) | |||
| 346 | + | if err == nil { | |||
| 347 | + | got, err := r.loadGeneration(ctx, name, key, generation) | |||
| 348 | + | if err == nil && sha256.Sum256(got.Manifest) == digest { | |||
| 349 | + | return generation, got.Manifest, nil | |||
| 350 | + | } | |||
| 351 | + | } | |||
| 352 | + | candidates, err := r.generationNumbers(key) | |||
| 353 | + | if err != nil { | |||
| 354 | + | return 0, nil, err | |||
| 355 | + | } | |||
| 356 | + | for _, candidate := range candidates { | |||
| 357 | + | if got, err := r.loadGeneration(ctx, name, key, candidate); err == nil { | |||
| 358 | + | return candidate, got.Manifest, nil | |||
| 359 | + | } | |||
| 360 | + | } | |||
| 361 | + | return 0, nil, nil | |||
| 362 | + | } | |||
| 363 | + | ||||
| 364 | + | func (r *Repository) loadGeneration(ctx context.Context, name, key string, generation uint64) (ports.SnapshotGeneration, error) { | |||
| 365 | + | data, err := readBounded(r.manifestPath(key, generation)) | |||
| 366 | + | if err != nil { | |||
| 367 | + | return ports.SnapshotGeneration{}, err | |||
| 368 | + | } | |||
| 369 | + | manifest, err := codec.UnmarshalManifest(data) | |||
| 370 | + | if err != nil || manifest.Name != name || manifest.Generation != generation { | |||
| 371 | + | return ports.SnapshotGeneration{}, fmt.Errorf("invalid manifest") | |||
| 372 | + | } | |||
| 373 | + | refs := manifestRefs(manifest) | |||
| 374 | + | if refs == nil { | |||
| 375 | + | return ports.SnapshotGeneration{}, fmt.Errorf("conflicting manifest references") | |||
| 376 | + | } | |||
| 377 | + | objects := make(map[ports.SnapshotDigest][]byte, len(refs)) | |||
| 378 | + | total := len(data) | |||
| 379 | + | for digest, ref := range refs { | |||
| 380 | + | if err := ctx.Err(); err != nil { | |||
| 381 | + | return ports.SnapshotGeneration{}, err | |||
| 382 | + | } | |||
| 383 | + | // Ref sizes were validated by the VEVM preflight. Check the aggregate | |||
| 384 | + | // before allocating another object so hostile repositories cannot exceed | |||
| 385 | + | // the restore budget transiently. | |||
| 386 | + | if uint64(ref.Size) > uint64(maxRepositoryRead-total) { | |||
| 387 | + | return ports.SnapshotGeneration{}, fmt.Errorf("snapshot generation too large") | |||
| 388 | + | } | |||
| 389 | + | object, err := readBounded(r.objectPath(key, digest)) | |||
| 390 | + | if err != nil { | |||
| 391 | + | return ports.SnapshotGeneration{}, err | |||
| 392 | + | } | |||
| 393 | + | total += len(object) | |||
| 394 | + | if sha256.Sum256(object) != digest || !validObject(object, ref) { | |||
| 395 | + | return ports.SnapshotGeneration{}, fmt.Errorf("invalid object") | |||
| 396 | + | } | |||
| 397 | + | objects[digest] = append([]byte(nil), object...) | |||
| 398 | + | } | |||
| 399 | + | return ports.SnapshotGeneration{Name: manifest.Name, Generation: generation, Manifest: append([]byte(nil), data...), Objects: objects}, nil | |||
| 400 | + | } | |||
| 401 | + | ||||
| 402 | + | func validObject(data []byte, ref codec.ObjectRef) bool { | |||
| 403 | + | kind, payload, err := codec.PreflightObject(data) | |||
| 404 | + | return err == nil && kind == ref.Kind && len(payload) > 0 && len(data) == int(ref.Size) | |||
| 405 | + | } | |||
| 406 | + | func verifyObjectFile(path string, digest ports.SnapshotDigest, ref codec.ObjectRef) (bool, error) { | |||
| 407 | + | data, exists, err := readOptionalBounded(path) | |||
| 408 | + | if err != nil || !exists { | |||
| 409 | + | return exists, err | |||
| 410 | + | } | |||
| 411 | + | return true, func() error { | |||
| 412 | + | if sha256.Sum256(data) != digest || !validObject(data, ref) { | |||
| 413 | + | return fmt.Errorf("existing immutable object is invalid") | |||
| 414 | + | } | |||
| 415 | + | return nil | |||
| 416 | + | }() | |||
| 417 | + | } | |||
| 418 | + | ||||
| 419 | + | func (r *Repository) ensureSession(key string) error { | |||
| 420 | + | for _, dir := range []string{r.dir, filepath.Join(r.dir, repositorySessionsDir), r.sessionPath(key), filepath.Join(r.sessionPath(key), repositoryObjectsDir), filepath.Join(r.sessionPath(key), repositoryGenerations)} { | |||
| 421 | + | if err := safedir.EnsurePrivate(dir); err != nil { | |||
| 422 | + | return fmt.Errorf("create snapshot repository directory: %w", err) | |||
| 423 | + | } | |||
| 424 | + | } | |||
| 425 | + | return nil | |||
| 426 | + | } | |||
| 427 | + | func (r *Repository) sessionPath(key string) string { | |||
| 428 | + | return filepath.Join(r.dir, repositorySessionsDir, key) | |||
| 429 | + | } | |||
| 430 | + | func (r *Repository) objectPath(key string, digest ports.SnapshotDigest) string { | |||
| 431 | + | hexDigest := hex.EncodeToString(digest[:]) | |||
| 432 | + | return filepath.Join(r.sessionPath(key), repositoryObjectsDir, hexDigest[:2], hexDigest) | |||
| 433 | + | } | |||
| 434 | + | func (r *Repository) manifestPath(key string, generation uint64) string { | |||
| 435 | + | return filepath.Join(r.sessionPath(key), repositoryGenerations, generationFilename(generation)) | |||
| 436 | + | } | |||
| 437 | + | func (r *Repository) headPath(key string) string { | |||
| 438 | + | return filepath.Join(r.sessionPath(key), repositoryHead) | |||
| 439 | + | } | |||
| 440 | + | func (r *Repository) sessionLock(key string) *sync.Mutex { | |||
| 441 | + | lock, _ := r.locks.LoadOrStore(key, &sync.Mutex{}) | |||
| 442 | + | return lock.(*sync.Mutex) | |||
| 443 | + | } | |||
| 444 | + | ||||
| 445 | + | func sessionKey(name string) string { | |||
| 446 | + | if safeNameRE.MatchString(name) && name != "." && name != ".." { | |||
| 447 | + | return name | |||
| 448 | + | } | |||
| 449 | + | sum := sha256.Sum256([]byte(name)) | |||
| 450 | + | return "@" + hex.EncodeToString(sum[:]) | |||
| 451 | + | } | |||
| 452 | + | func canonicalSessionKey(key string) bool { | |||
| 453 | + | if safeNameRE.MatchString(key) && key != "." && key != ".." { | |||
| 454 | + | return true | |||
| 455 | + | } | |||
| 456 | + | if len(key) != 65 || key[0] != '@' { | |||
| 457 | + | return false | |||
| 458 | + | } | |||
| 459 | + | _, err := hex.DecodeString(key[1:]) | |||
| 460 | + | return err == nil && strings.ToLower(key[1:]) == key[1:] | |||
| 461 | + | } | |||
| 462 | + | func nameFromSafeKey(key string) string { | |||
| 463 | + | if strings.HasPrefix(key, "@") { | |||
| 464 | + | return "" | |||
| 465 | + | } | |||
| 466 | + | return key | |||
| 467 | + | } | |||
| 468 | + | func generationFilename(generation uint64) string { | |||
| 469 | + | return fmt.Sprintf("%0*d.manifest", generationWidth, generation) | |||
| 470 | + | } | |||
| 471 | + | func parseGenerationFilename(name string) (uint64, bool) { | |||
| 472 | + | if !strings.HasSuffix(name, ".manifest") || len(name) != generationWidth+len(".manifest") { | |||
| 473 | + | return 0, false | |||
| 474 | + | } | |||
| 475 | + | n := strings.TrimSuffix(name, ".manifest") | |||
| 476 | + | if strings.HasPrefix(n, "0") && n != fmt.Sprintf("%0*d", generationWidth, 0) { /* zero padding required, strconv below verifies width */ | |||
| 477 | + | } | |||
| 478 | + | generation, err := strconv.ParseUint(n, 10, 64) | |||
| 479 | + | return generation, err == nil && generationFilename(generation) == name && generation != 0 | |||
| 480 | + | } | |||
| 481 | + | ||||
| 482 | + | func (r *Repository) generationNumbers(key string) ([]uint64, error) { | |||
| 483 | + | entries, err := os.ReadDir(filepath.Join(r.sessionPath(key), repositoryGenerations)) | |||
| 484 | + | if errors.Is(err, os.ErrNotExist) { | |||
| 485 | + | return nil, nil | |||
| 486 | + | } | |||
| 487 | + | if err != nil { | |||
| 488 | + | return nil, err | |||
| 489 | + | } | |||
| 490 | + | numbers := make([]uint64, 0, len(entries)) | |||
| 491 | + | for _, entry := range entries { | |||
| 492 | + | if entry.IsDir() { | |||
| 493 | + | continue | |||
| 494 | + | } | |||
| 495 | + | if generation, ok := parseGenerationFilename(entry.Name()); ok { | |||
| 496 | + | numbers = append(numbers, generation) | |||
| 497 | + | } | |||
| 498 | + | } | |||
| 499 | + | sort.Slice(numbers, func(i, j int) bool { return numbers[i] > numbers[j] }) | |||
| 500 | + | return numbers, nil | |||
| 501 | + | } | |||
| 502 | + | func removeGeneration(numbers []uint64, generation uint64) []uint64 { | |||
| 503 | + | out := numbers[:0] | |||
| 504 | + | for _, n := range numbers { | |||
| 505 | + | if n != generation { | |||
| 506 | + | out = append(out, n) | |||
| 507 | + | } | |||
| 508 | + | } | |||
| 509 | + | return out | |||
| 510 | + | } | |||
| 511 | + | ||||
| 512 | + | func marshalHead(generation uint64, digest ports.SnapshotDigest) []byte { | |||
| 513 | + | out := make([]byte, 4+8+sha256.Size) | |||
| 514 | + | copy(out, "VEVH") | |||
| 515 | + | binary.BigEndian.PutUint64(out[4:12], generation) | |||
| 516 | + | copy(out[12:], digest[:]) | |||
| 517 | + | return out | |||
| 518 | + | } | |||
| 519 | + | func (r *Repository) readHead(key string) (uint64, ports.SnapshotDigest, error) { | |||
| 520 | + | data, err := readBounded(r.headPath(key)) | |||
| 521 | + | if err != nil { | |||
| 522 | + | return 0, ports.SnapshotDigest{}, err | |||
| 523 | + | } | |||
| 524 | + | if len(data) != 4+8+sha256.Size || string(data[:4]) != "VEVH" { | |||
| 525 | + | return 0, ports.SnapshotDigest{}, fmt.Errorf("invalid HEAD") | |||
| 526 | + | } | |||
| 527 | + | generation := binary.BigEndian.Uint64(data[4:12]) | |||
| 528 | + | var digest ports.SnapshotDigest | |||
| 529 | + | copy(digest[:], data[12:]) | |||
| 530 | + | if generation == 0 { | |||
| 531 | + | return 0, ports.SnapshotDigest{}, fmt.Errorf("invalid HEAD") | |||
| 532 | + | } | |||
| 533 | + | return generation, digest, nil | |||
| 534 | + | } | |||
| 535 | + | ||||
| 536 | + | func (r *Repository) atomicWrite(path string, data []byte) error { | |||
| 537 | + | if r.hooks.writeFile != nil { | |||
| 538 | + | return r.hooks.writeFile(path, append([]byte(nil), data...)) | |||
| 539 | + | } | |||
| 540 | + | return atomicWriteFile(path, data) | |||
| 541 | + | } | |||
| 542 | + | func atomicWriteFile(path string, data []byte) error { | |||
| 543 | + | dir := filepath.Dir(path) | |||
| 544 | + | if err := safedir.EnsurePrivate(dir); err != nil { | |||
| 545 | + | return err | |||
| 546 | + | } | |||
| 547 | + | temp, err := os.CreateTemp(dir, ".tmp-") | |||
| 548 | + | if err != nil { | |||
| 549 | + | return err | |||
| 550 | + | } | |||
| 551 | + | tempPath := temp.Name() | |||
| 552 | + | cleanup := true | |||
| 553 | + | defer func() { | |||
| 554 | + | if cleanup { | |||
| 555 | + | _ = os.Remove(tempPath) | |||
| 556 | + | } | |||
| 557 | + | }() | |||
| 558 | + | if err := temp.Chmod(0o600); err != nil { | |||
| 559 | + | _ = temp.Close() | |||
| 560 | + | return err | |||
| 561 | + | } | |||
| 562 | + | if _, err := temp.Write(data); err != nil { | |||
| 563 | + | _ = temp.Close() | |||
| 564 | + | return err | |||
| 565 | + | } | |||
| 566 | + | if err := temp.Sync(); err != nil { | |||
| 567 | + | _ = temp.Close() | |||
| 568 | + | return err | |||
| 569 | + | } | |||
| 570 | + | if err := temp.Close(); err != nil { | |||
| 571 | + | return err | |||
| 572 | + | } | |||
| 573 | + | if err := os.Rename(tempPath, path); err != nil { | |||
| 574 | + | return err | |||
| 575 | + | } | |||
| 576 | + | cleanup = false | |||
| 577 | + | return syncDirectory(dir) | |||
| 578 | + | } | |||
| 579 | + | func syncDirectory(dir string) error { | |||
| 580 | + | f, err := os.Open(dir) | |||
| 581 | + | if err != nil { | |||
| 582 | + | return err | |||
| 583 | + | } | |||
| 584 | + | defer f.Close() | |||
| 585 | + | return f.Sync() | |||
| 586 | + | } | |||
| 587 | + | func readOptionalBounded(path string) ([]byte, bool, error) { | |||
| 588 | + | data, err := readBounded(path) | |||
| 589 | + | if errors.Is(err, os.ErrNotExist) { | |||
| 590 | + | return nil, false, nil | |||
| 591 | + | } | |||
| 592 | + | return data, err == nil, err | |||
| 593 | + | } | |||
| 594 | + | func readBounded(path string) ([]byte, error) { | |||
| 595 | + | info, err := os.Lstat(path) | |||
| 596 | + | if err != nil { | |||
| 597 | + | return nil, err | |||
| 598 | + | } | |||
| 599 | + | if !info.Mode().IsRegular() { | |||
| 600 | + | return nil, fmt.Errorf("not a regular file") | |||
| 601 | + | } | |||
| 602 | + | if info.Size() < 0 || info.Size() > maxRepositoryRead { | |||
| 603 | + | return nil, fmt.Errorf("snapshot file too large") | |||
| 604 | + | } | |||
| 605 | + | f, err := os.Open(path) | |||
| 606 | + | if err != nil { | |||
| 607 | + | return nil, err | |||
| 608 | + | } | |||
| 609 | + | defer f.Close() | |||
| 610 | + | data, err := io.ReadAll(io.LimitReader(f, maxRepositoryRead+1)) | |||
| 611 | + | if err != nil { | |||
| 612 | + | return nil, err | |||
| 613 | + | } | |||
| 614 | + | if len(data) > maxRepositoryRead { | |||
| 615 | + | return nil, fmt.Errorf("snapshot file too large") | |||
| 616 | + | } | |||
| 617 | + | return data, nil | |||
| 618 | + | } | |||
| 619 | + | func equalBytes(a, b []byte) bool { | |||
| 620 | + | if len(a) != len(b) { | |||
| 621 | + | return false | |||
| 622 | + | } | |||
| 623 | + | for i := range a { | |||
| 624 | + | if a[i] != b[i] { | |||
| 625 | + | return false | |||
| 626 | + | } | |||
| 627 | + | } | |||
| 628 | + | return true | |||
| 629 | + | } | |||