// Package filesystem implements storage adapters using the local filesystem. package filesystem import ( "crypto/sha256" "crypto/sha512" "encoding/hex" "fmt" "hash" "io" "os" "path/filepath" "strings" "sync" "time" "github.com/bnema/zerowrap" "github.com/google/uuid" "github.com/bnema/gordon/pkg/validation" ) // BlobStorage implements the BlobStorage interface using the local filesystem. type BlobStorage struct { rootDir string log zerowrap.Logger uploadLocks sync.Map } // NewBlobStorage creates a new filesystem blob storage instance. func NewBlobStorage(rootDir string, log zerowrap.Logger) (*BlobStorage, error) { // Create directory structure if it doesn't exist dirs := []string{ filepath.Join(rootDir, "blobs"), filepath.Join(rootDir, "uploads"), } for _, dir := range dirs { if err := os.MkdirAll(dir, 0750); err != nil { return nil, fmt.Errorf("failed to create directory %s: %w", dir, err) } } log.Info(). Str(zerowrap.FieldLayer, "adapter"). Str(zerowrap.FieldAdapter, "filesystem"). Str("root_dir", rootDir). Msg("blob storage initialized") return &BlobStorage{ rootDir: rootDir, log: log, }, nil } // GetBlob retrieves a blob by digest. func (s *BlobStorage) GetBlob(digest string) (io.ReadCloser, error) { blobPath, err := s.getBlobPath(digest) if err != nil { return nil, fmt.Errorf("invalid path: %w", err) } file, err := os.Open(blobPath) if err != nil { if os.IsNotExist(err) { return nil, fmt.Errorf("blob not found: %s", digest) } return nil, fmt.Errorf("failed to open blob: %w", err) } return file, nil } // GetBlobPath returns the filesystem path to a blob. func (s *BlobStorage) GetBlobPath(digest string) (string, error) { path, err := s.getBlobPath(digest) if err != nil { return "", fmt.Errorf("invalid path: %w", err) } if _, err := os.Stat(path); err != nil { if os.IsNotExist(err) { return "", fmt.Errorf("blob not found: %s", digest) } return "", err } return path, nil } // PutBlob stores a blob with the given digest. func (s *BlobStorage) PutBlob(digest string, data io.Reader, size int64) error { blobPath, err := s.getBlobPath(digest) if err != nil { return fmt.Errorf("invalid path: %w", err) } // Create directory if it doesn't exist if err := os.MkdirAll(filepath.Dir(blobPath), 0750); err != nil { return fmt.Errorf("failed to create blob directory: %w", err) } // Create temporary file first tmpPath := blobPath + ".tmp" file, err := os.Create(tmpPath) if err != nil { return fmt.Errorf("failed to create temporary blob file: %w", err) } defer file.Close() // Copy data to file written, err := io.Copy(file, data) if err != nil { os.Remove(tmpPath) return fmt.Errorf("failed to write blob data: %w", err) } // Verify size if provided if size > 0 && written != size { os.Remove(tmpPath) return fmt.Errorf("blob size mismatch: expected %d, got %d", size, written) } // Move to final location if err := os.Rename(tmpPath, blobPath); err != nil { os.Remove(tmpPath) return fmt.Errorf("failed to move blob to final location: %w", err) } s.log.Info(). Str(zerowrap.FieldLayer, "adapter"). Str(zerowrap.FieldAdapter, "filesystem"). Str("digest", digest). Int64(zerowrap.FieldSize, written). Msg("blob stored") return nil } // DeleteBlob removes a blob by digest. func (s *BlobStorage) DeleteBlob(digest string) error { blobPath, err := s.getBlobPath(digest) if err != nil { return fmt.Errorf("invalid path: %w", err) } if err := os.Remove(blobPath); err != nil { if os.IsNotExist(err) { return fmt.Errorf("blob not found: %s", digest) } return fmt.Errorf("failed to delete blob: %w", err) } s.log.Info(). Str(zerowrap.FieldLayer, "adapter"). Str(zerowrap.FieldAdapter, "filesystem"). Str("digest", digest). Msg("blob deleted") return nil } // BlobExists checks if a blob exists. func (s *BlobStorage) BlobExists(digest string) bool { blobPath, err := s.getBlobPath(digest) if err != nil { return false // Invalid path means blob doesn't exist } _, err = os.Stat(blobPath) return err == nil } // ListBlobs returns all blob digests. func (s *BlobStorage) ListBlobs() ([]string, error) { blobsDir := filepath.Join(s.rootDir, "blobs") if _, err := os.Stat(blobsDir); err != nil { if os.IsNotExist(err) { return []string{}, nil } return nil, fmt.Errorf("failed to access blobs directory: %w", err) } digests := make([]string, 0) err := filepath.Walk(blobsDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() { return nil } relPath, err := filepath.Rel(blobsDir, path) if err != nil { return err } parts := strings.Split(relPath, string(filepath.Separator)) if len(parts) < 2 { return nil } digest := parts[0] + ":" + parts[len(parts)-1] if err := validation.ValidateDigest(digest); err != nil { return nil } digests = append(digests, digest) return nil }) if err != nil { return nil, fmt.Errorf("failed to list blobs: %w", err) } return digests, nil } // StartBlobUpload starts a new blob upload and returns the upload UUID. func (s *BlobStorage) StartBlobUpload(name string) (string, error) { uploadID := uuid.New().String() uploadPath, err := s.getUploadPath(uploadID) if err != nil { return "", fmt.Errorf("invalid upload path: %w", err) } // Create directory if it doesn't exist if err := os.MkdirAll(filepath.Dir(uploadPath), 0750); err != nil { return "", fmt.Errorf("failed to create upload directory: %w", err) } // Create empty upload file file, err := os.Create(uploadPath) if err != nil { return "", fmt.Errorf("failed to create upload file: %w", err) } file.Close() s.log.Info(). Str(zerowrap.FieldLayer, "adapter"). Str(zerowrap.FieldAdapter, "filesystem"). Str("uuid", uploadID). Str("name", name). Msg("blob upload started") return uploadID, nil } // AppendBlobChunk appends data to an in-progress upload. func (s *BlobStorage) AppendBlobChunk(name, uuid string, data io.Reader) (int64, error) { uploadPath, err := s.getUploadPath(uuid) if err != nil { return 0, fmt.Errorf("invalid upload path: %w", err) } mu := s.getUploadLock(uuid) mu.Lock() defer mu.Unlock() file, err := os.OpenFile(uploadPath, os.O_WRONLY|os.O_APPEND, 0600) if err != nil { if os.IsNotExist(err) { s.cleanupUploadLock(uuid) return 0, fmt.Errorf("upload not found: %s", uuid) } return 0, fmt.Errorf("failed to open upload file: %w", err) } defer file.Close() written, err := io.Copy(file, data) if err != nil { return 0, fmt.Errorf("failed to write chunk to upload file: %w", err) } fi, err := file.Stat() if err != nil { return 0, fmt.Errorf("failed to get file info: %w", err) } s.log.Debug(). Str(zerowrap.FieldLayer, "adapter"). Str(zerowrap.FieldAdapter, "filesystem"). Str("uuid", uuid). Str("name", name). Int64("chunk_size", written). Int64("total_size", fi.Size()). Msg("appended chunk to blob upload") return fi.Size(), nil } // GetBlobUpload returns a writer for the upload. // The returned WriteCloser holds the per-upload lock; callers must Close it to release the lock. func (s *BlobStorage) GetBlobUpload(uuid string) (io.WriteCloser, error) { uploadPath, err := s.getUploadPath(uuid) if err != nil { return nil, fmt.Errorf("invalid upload path: %w", err) } mu := s.getUploadLock(uuid) mu.Lock() file, err := os.OpenFile(uploadPath, os.O_WRONLY|os.O_APPEND, 0600) if err != nil { mu.Unlock() if os.IsNotExist(err) { s.cleanupUploadLock(uuid) return nil, fmt.Errorf("upload not found: %s", uuid) } return nil, fmt.Errorf("failed to open upload file: %w", err) } return &lockedWriteCloser{file: file, mu: mu}, nil } // lockedWriteCloser wraps an io.WriteCloser and releases a mutex on Close. type lockedWriteCloser struct { file io.WriteCloser mu *sync.Mutex closeOnce sync.Once closeErr error } func (lw *lockedWriteCloser) Write(p []byte) (int, error) { return lw.file.Write(p) } func (lw *lockedWriteCloser) Close() error { lw.closeOnce.Do(func() { defer lw.mu.Unlock() lw.closeErr = lw.file.Close() }) return lw.closeErr } // FinishBlobUpload completes an upload and moves it to blob storage. func (s *BlobStorage) FinishBlobUpload(uuid, digest string) error { uploadPath, err := s.getUploadPath(uuid) if err != nil { return fmt.Errorf("invalid upload path: %w", err) } mu := s.getUploadLock(uuid) mu.Lock() var finalized bool defer func() { mu.Unlock() if finalized { s.cleanupUploadLock(uuid) } }() blobPath, err := s.getBlobPath(digest) if err != nil { return fmt.Errorf("invalid blob path: %w", err) } // Verify digest before moving to ensure integrity if err := s.verifyUploadDigest(uploadPath, digest); err != nil { return fmt.Errorf("digest verification failed: %w", err) } // Create blob directory if it doesn't exist if err := os.MkdirAll(filepath.Dir(blobPath), 0750); err != nil { return fmt.Errorf("failed to create blob directory: %w", err) } // Move upload file to blob location if err := os.Rename(uploadPath, blobPath); err != nil { return fmt.Errorf("failed to move upload to blob location: %w", err) } finalized = true s.log.Info(). Str(zerowrap.FieldLayer, "adapter"). Str(zerowrap.FieldAdapter, "filesystem"). Str("uuid", uuid). Str("digest", digest). Msg("blob upload finished") return nil } // CancelBlobUpload cancels an in-progress upload. func (s *BlobStorage) CancelBlobUpload(uuid string) error { uploadPath, err := s.getUploadPath(uuid) if err != nil { return fmt.Errorf("invalid upload path: %w", err) } mu := s.getUploadLock(uuid) mu.Lock() var finalized bool defer func() { mu.Unlock() if finalized { s.cleanupUploadLock(uuid) } }() if err := os.Remove(uploadPath); err != nil { if os.IsNotExist(err) { s.cleanupUploadLock(uuid) return fmt.Errorf("upload not found: %s", uuid) } return fmt.Errorf("failed to cancel upload: %w", err) } finalized = true s.log.Info(). Str(zerowrap.FieldLayer, "adapter"). Str(zerowrap.FieldAdapter, "filesystem"). Str("uuid", uuid). Msg("blob upload cancelled") return nil } // CleanupStaleUploads removes upload files older than maxAge. func (s *BlobStorage) CleanupStaleUploads(maxAge time.Duration) (int, int64, error) { uploadsDir := filepath.Join(s.rootDir, "uploads") entries, err := os.ReadDir(uploadsDir) if err != nil { if os.IsNotExist(err) { return 0, 0, nil } return 0, 0, fmt.Errorf("failed to read uploads directory: %w", err) } cutoff := time.Now().Add(-maxAge) var removed int var bytesReclaimed int64 for _, entry := range entries { if entry.IsDir() { continue } info, err := entry.Info() if err != nil { s.log.Warn().Err(err).Str("file", entry.Name()).Msg("failed to get file info for stale upload") continue } if info.ModTime().Before(cutoff) { path := filepath.Join(uploadsDir, entry.Name()) size := info.Size() mu := s.getUploadLock(entry.Name()) mu.Lock() if err := os.Remove(path); err != nil { mu.Unlock() s.log.Warn().Err(err).Str("file", entry.Name()).Msg("failed to remove stale upload") continue } mu.Unlock() s.cleanupUploadLock(entry.Name()) removed++ bytesReclaimed += size s.log.Info(). Str(zerowrap.FieldLayer, "adapter"). Str(zerowrap.FieldAdapter, "filesystem"). Str("uuid", entry.Name()). Int64("size", size). Msg("removed stale upload") } } return removed, bytesReclaimed, nil } // Helper methods for path generation with security validation func (s *BlobStorage) getBlobPath(digest string) (string, error) { // Validate digest to prevent path traversal (defense in depth) if err := validation.ValidateDigest(digest); err != nil { return "", fmt.Errorf("invalid digest: %w", err) } // Split digest into directory structure (e.g., sha256:abc123... -> sha256/ab/abc123...) parts := strings.SplitN(digest, ":", 2) if len(parts) != 2 { return "", fmt.Errorf("invalid digest format: missing algorithm separator") } algorithm := parts[0] hashPart := parts[1] var path string // Create two-level directory structure for better performance if len(hashPart) >= 2 { path = filepath.Join(s.rootDir, "blobs", algorithm, hashPart[:2], hashPart) } else { path = filepath.Join(s.rootDir, "blobs", algorithm, hashPart) } // Verify the path stays within root directory if err := validation.ValidatePathWithinRoot(s.rootDir, path); err != nil { return "", fmt.Errorf("path validation failed: %w", err) } return path, nil } func (s *BlobStorage) getUploadPath(uuid string) (string, error) { // Validate UUID to prevent path traversal if err := validation.ValidateUUID(uuid); err != nil { return "", fmt.Errorf("invalid UUID: %w", err) } path := filepath.Join(s.rootDir, "uploads", uuid) // Verify the path stays within root directory if err := validation.ValidatePathWithinRoot(s.rootDir, path); err != nil { return "", fmt.Errorf("path validation failed: %w", err) } return path, nil } func (s *BlobStorage) getUploadLock(uuid string) *sync.Mutex { v, _ := s.uploadLocks.LoadOrStore(uuid, &sync.Mutex{}) return v.(*sync.Mutex) } func (s *BlobStorage) cleanupUploadLock(uuid string) { s.uploadLocks.Delete(uuid) } // verifyUploadDigest computes and verifies the digest of an uploaded file. func (s *BlobStorage) verifyUploadDigest(uploadPath, expectedDigest string) error { parts := strings.SplitN(expectedDigest, ":", 2) if len(parts) != 2 { return fmt.Errorf("invalid digest format") } algorithm := parts[0] expectedHash := parts[1] var hasher hash.Hash switch algorithm { case "sha256": hasher = sha256.New() case "sha512": hasher = sha512.New() default: return fmt.Errorf("unsupported hash algorithm: %s", algorithm) } file, err := os.Open(uploadPath) if err != nil { return fmt.Errorf("failed to open upload file: %w", err) } defer file.Close() if _, err := io.Copy(hasher, file); err != nil { return fmt.Errorf("failed to compute digest: %w", err) } actualHash := hex.EncodeToString(hasher.Sum(nil)) if actualHash != expectedHash { return fmt.Errorf("digest mismatch: expected %s, got %s", expectedHash, actualHash) } return nil }