1package filtering
2
3import (
4 "context"
5 "encoding/json"
6 "fmt"
7 "io"
8 "net/http"
9 "os"
10 pathpkg "path"
11 "path/filepath"
12 "slices"
13 "strings"
14 "time"
15
16 "github.com/bnema/dumber/internal/logging"
17)
18
19const (
20 cacheDirPerm = 0o755
21 manifestFilePerm = 0o644
22
23 defaultMaxManifestBytes int64 = 1 << 20 // 1 MiB
24 defaultMaxFilterFiles = 128
25 defaultMaxFilterFileBytes int64 = 64 << 20 // 64 MiB
26 defaultMaxTotalFilterBytes int64 = 512 << 20 // 512 MiB
27)
28
29// downloadLimits bounds remote filter metadata and payloads.
30type downloadLimits struct {
31 maxManifestBytes int64
32 maxFilterFiles int
33 maxFilterFileBytes int64
34 maxTotalFilterBytes int64
35}
36
37var defaultDownloadLimits = downloadLimits{
38 maxManifestBytes: defaultMaxManifestBytes,
39 maxFilterFiles: defaultMaxFilterFiles,
40 maxFilterFileBytes: defaultMaxFilterFileBytes,
41 maxTotalFilterBytes: defaultMaxTotalFilterBytes,
42}
43
44// Downloader handles downloading filter files from GitHub releases.
45type Downloader struct {
46 baseURL string
47 cacheDir string
48 httpClient *http.Client
49 limits downloadLimits
50}
51
52// NewDownloader creates a new Downloader.
53// cacheDir is where downloaded JSON files will be stored.
54func NewDownloader(cacheDir string) *Downloader {
55 return &Downloader{
56 baseURL: GitHubReleaseURL,
57 cacheDir: cacheDir,
58 httpClient: &http.Client{
59 Timeout: 60 * time.Second,
60 },
61 limits: defaultDownloadLimits,
62 }
63}
64
65// GetCachedManifest reads the locally cached manifest.json.
66// Returns nil if no cached manifest exists.
67func (d *Downloader) GetCachedManifest() (*Manifest, error) {
68 manifestPath := filepath.Join(d.cacheDir, FilterFiles.Manifest)
69 data, err := readLimitedFile(manifestPath, d.limits.maxManifestBytes, "cached manifest")
70 if err != nil {
71 if os.IsNotExist(err) {
72 return nil, nil
73 }
74 return nil, err
75 }
76
77 var manifest Manifest
78 if err := json.Unmarshal(data, &manifest); err != nil {
79 return nil, fmt.Errorf("failed to parse cached manifest: %w", err)
80 }
81 if err := d.validateManifest(&manifest); err != nil {
82 return nil, fmt.Errorf("invalid cached manifest: %w", err)
83 }
84 return &manifest, nil
85}
86
87// FetchManifest downloads the latest manifest.json from GitHub.
88func (d *Downloader) FetchManifest(ctx context.Context) (*Manifest, error) {
89 log := logging.FromContext(ctx).With().
90 Str("component", "filter-downloader").
91 Logger()
92
93 url := fmt.Sprintf("%s/%s", d.baseURL, FilterFiles.Manifest)
94 log.Debug().Str("url", url).Msg("fetching manifest")
95
96 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
97 if err != nil {
98 return nil, fmt.Errorf("failed to create request: %w", err)
99 }
100
101 resp, err := d.httpClient.Do(req)
102 if err != nil {
103 return nil, fmt.Errorf("failed to fetch manifest: %w", err)
104 }
105 defer func() {
106 if closeErr := resp.Body.Close(); closeErr != nil {
107 log.Debug().Err(closeErr).Msg("failed to close manifest response body")
108 }
109 }()
110
111 if resp.StatusCode != http.StatusOK {
112 return nil, fmt.Errorf("manifest fetch failed with status %d", resp.StatusCode)
113 }
114
115 data, err := readLimitedResponseBody(resp, d.limits.maxManifestBytes, "manifest")
116 if err != nil {
117 return nil, fmt.Errorf("failed to read manifest body: %w", err)
118 }
119
120 var manifest Manifest
121 if err := json.Unmarshal(data, &manifest); err != nil {
122 return nil, fmt.Errorf("failed to parse manifest: %w", err)
123 }
124 if err := d.validateManifest(&manifest); err != nil {
125 return nil, fmt.Errorf("invalid manifest: %w", err)
126 }
127
128 log.Debug().Str("version", manifest.Version).Msg("fetched manifest")
129 return &manifest, nil
130}
131
132// DownloadProgress reports download progress.
133type DownloadProgress struct {
134 File string
135 Current int
136 Total int
137 BytesTotal int64
138}
139
140// DownloadFilters fetches the manifest to discover which filter files exist,
141// then downloads them. Returns the paths to the downloaded files.
142func (d *Downloader) DownloadFilters(ctx context.Context, onProgress func(DownloadProgress)) ([]string, error) {
143 log := logging.FromContext(ctx).With().
144 Str("component", "filter-downloader").
145 Logger()
146
147 // Ensure cache directory exists
148 if err := os.MkdirAll(d.cacheDir, cacheDirPerm); err != nil {
149 return nil, fmt.Errorf("failed to create cache dir: %w", err)
150 }
151
152 // Fetch manifest to discover which combined files to download
153 manifest, err := d.FetchManifest(ctx)
154 if err != nil {
155 return nil, fmt.Errorf("failed to fetch manifest: %w", err)
156 }
157 if len(manifest.Combined.Files) == 0 {
158 return nil, fmt.Errorf("manifest contains no combined filter files")
159 }
160
161 log.Info().
162 Int("files", len(manifest.Combined.Files)).
163 Int("total_rules", manifest.Combined.TotalRules).
164 Str("version", manifest.Version).
165 Msg("manifest lists filter files")
166
167 files := manifest.Combined.Files
168 paths := make([]string, 0, len(files))
169 var totalBytes int64
170
171 for i, filename := range files {
172 select {
173 case <-ctx.Done():
174 return nil, ctx.Err()
175 default:
176 }
177
178 if onProgress != nil {
179 onProgress(DownloadProgress{
180 File: filename,
181 Current: i + 1,
182 Total: len(files),
183 BytesTotal: totalBytes,
184 })
185 }
186
187 remainingBytes := d.limits.maxTotalFilterBytes - totalBytes
188 if remainingBytes <= 0 {
189 return nil, fmt.Errorf("filter downloads exceed total size limit of %d bytes", d.limits.maxTotalFilterBytes)
190 }
191
192 path, bytesDownloaded, err := d.downloadFile(ctx, filename, minInt64(d.limits.maxFilterFileBytes, remainingBytes))
193 if err != nil {
194 log.Error().Err(err).Str("file", filename).Msg("failed to download filter file")
195 return nil, err
196 }
197
198 totalBytes += bytesDownloaded
199 paths = append(paths, path)
200 log.Debug().Str("file", filename).Int64("bytes", bytesDownloaded).Msg("downloaded filter file")
201 }
202
203 // Cache the manifest we already fetched
204 if err := d.cacheManifest(manifest); err != nil {
205 log.Warn().Err(err).Msg("failed to cache manifest")
206 } else if err := d.pruneObsoleteCachedFilterFiles(files); err != nil {
207 log.Warn().Err(err).Msg("failed to prune obsolete cached filter files")
208 }
209
210 log.Info().Int("files", len(paths)).Int64("total_bytes", totalBytes).Msg("filter download complete")
211 return paths, nil
212}
213
214func (d *Downloader) pruneObsoleteCachedFilterFiles(currentFiles []string) error {
215 current := make(map[string]struct{}, len(currentFiles))
216 for _, filename := range currentFiles {
217 _, safeFilename, err := d.cachePathForFilterFile(filename)
218 if err != nil {
219 return err
220 }
221 current[safeFilename] = struct{}{}
222 }
223
224 entries, err := os.ReadDir(d.cacheDir)
225 if err != nil {
226 return err
227 }
228 for _, entry := range entries {
229 if entry.IsDir() {
230 continue
231 }
232 name := entry.Name()
233 if !strings.HasPrefix(name, "combined-part") || !strings.HasSuffix(name, ".json") {
234 continue
235 }
236 if _, ok := current[name]; ok {
237 continue
238 }
239 if err := os.Remove(filepath.Join(d.cacheDir, name)); err != nil {
240 return fmt.Errorf("remove obsolete cached filter %s: %w", name, err)
241 }
242 }
243 return nil
244}
245
246// downloadFile downloads a single file and returns its local path.
247func (d *Downloader) downloadFile(ctx context.Context, filename string, maxBytes int64) (string, int64, error) {
248 localPath, safeFilename, err := d.cachePathForFilterFile(filename)
249 if err != nil {
250 return "", 0, err
251 }
252
253 url := fmt.Sprintf("%s/%s", d.baseURL, safeFilename)
254
255 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
256 if err != nil {
257 return "", 0, fmt.Errorf("failed to create request: %w", err)
258 }
259
260 resp, err := d.httpClient.Do(req)
261 if err != nil {
262 return "", 0, fmt.Errorf("failed to download %s: %w", filename, err)
263 }
264 defer func() {
265 if closeErr := resp.Body.Close(); closeErr != nil {
266 logging.FromContext(ctx).Debug().Err(closeErr).Str("file", filename).Msg("failed to close filter response body")
267 }
268 }()
269
270 if resp.StatusCode != http.StatusOK {
271 return "", 0, fmt.Errorf("download %s failed with status %d", filename, resp.StatusCode)
272 }
273
274 if resp.ContentLength > maxBytes {
275 return "", 0, fmt.Errorf("filter file %q too large: content length %d exceeds limit %d", safeFilename, resp.ContentLength, maxBytes)
276 }
277
278 // Write to temp file in cacheDir, then atomic rename into destination.
279 tmpFile, err := os.CreateTemp(d.cacheDir, filepath.Base(safeFilename)+".*.tmp")
280 if err != nil {
281 return "", 0, fmt.Errorf("failed to create temp file: %w", err)
282 }
283 tmpPath := tmpFile.Name()
284
285 bytesWritten, err := copyLimited(tmpFile, resp.Body, maxBytes, fmt.Sprintf("filter file %q", safeFilename))
286 if err != nil {
287 _ = tmpFile.Close()
288 _ = os.Remove(tmpPath)
289 return "", 0, fmt.Errorf("failed to write %s: %w", safeFilename, err)
290 }
291 if err := tmpFile.Close(); err != nil {
292 _ = os.Remove(tmpPath)
293 return "", 0, fmt.Errorf("failed to close temp file: %w", err)
294 }
295
296 if err := d.renameTempFile(tmpPath, safeFilename); err != nil {
297 _ = os.Remove(tmpPath)
298 return "", 0, err
299 }
300
301 return localPath, bytesWritten, nil
302}
303
304func (d *Downloader) cachePathForFilterFile(filename string) (string, string, error) {
305 safeFilename, err := validateFilterFilename(filename)
306 if err != nil {
307 return "", "", err
308 }
309
310 cacheDirAbs, err := filepath.Abs(d.cacheDir)
311 if err != nil {
312 return "", "", fmt.Errorf("failed to resolve cache dir: %w", err)
313 }
314 localPath := filepath.Join(d.cacheDir, filepath.FromSlash(safeFilename))
315 targetAbs, err := filepath.Abs(localPath)
316 if err != nil {
317 return "", "", fmt.Errorf("failed to resolve cache path: %w", err)
318 }
319 if err := ensurePathInside(cacheDirAbs, targetAbs); err != nil {
320 return "", "", fmt.Errorf("invalid filter filename %q: %w", filename, err)
321 }
322
323 return localPath, safeFilename, nil
324}
325
326func validateFilterFilename(filename string) (string, error) {
327 if filename == "" {
328 return "", fmt.Errorf("invalid filter filename: empty")
329 }
330 if strings.TrimSpace(filename) != filename {
331 return "", fmt.Errorf("invalid filter filename %q: surrounding whitespace is not allowed", filename)
332 }
333 if filepath.IsAbs(filename) || strings.HasPrefix(filename, "/") {
334 return "", fmt.Errorf("invalid filter filename %q: absolute paths are not allowed", filename)
335 }
336 if strings.Contains(filename, "\\") {
337 return "", fmt.Errorf("invalid filter filename %q: backslashes are not allowed", filename)
338 }
339 if strings.ContainsAny(filename, "?#") {
340 return "", fmt.Errorf("invalid filter filename %q: URL control characters are not allowed", filename)
341 }
342
343 parts := strings.Split(filename, "/")
344 for _, part := range parts {
345 if part == "" || part == "." || part == ".." {
346 return "", fmt.Errorf("invalid filter filename %q: path traversal components are not allowed", filename)
347 }
348 }
349
350 clean := pathpkg.Clean(filename)
351 if pathpkg.Ext(clean) != ".json" {
352 return "", fmt.Errorf("invalid filter filename %q: expected .json extension", filename)
353 }
354
355 return clean, nil
356}
357
358func ensurePathInside(baseAbs, targetAbs string) error {
359 rel, err := filepath.Rel(baseAbs, targetAbs)
360 if err != nil {
361 return fmt.Errorf("failed to compare with cache dir: %w", err)
362 }
363 if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
364 return fmt.Errorf("resolved path escapes cache dir")
365 }
366 return nil
367}
368
369func (d *Downloader) renameTempFile(tmpPath, safeFilename string) error {
370 relTarget := filepath.FromSlash(safeFilename)
371 parentDir := filepath.Dir(relTarget)
372
373 root, err := os.OpenRoot(d.cacheDir)
374 if err != nil {
375 return fmt.Errorf("failed to open cache root: %w", err)
376 }
377 defer func() { _ = root.Close() }()
378
379 if parentDir != "." {
380 if err := root.MkdirAll(parentDir, cacheDirPerm); err != nil {
381 return fmt.Errorf("failed to create target dir: %w", err)
382 }
383 }
384
385 if err := root.Rename(filepath.Base(tmpPath), relTarget); err != nil {
386 return fmt.Errorf("failed to rename temp file: %w", err)
387 }
388
389 return nil
390}
391
392// cacheManifest writes an already-fetched manifest to disk.
393func (d *Downloader) cacheManifest(manifest *Manifest) error {
394 if err := d.validateManifest(manifest); err != nil {
395 return fmt.Errorf("invalid manifest: %w", err)
396 }
397
398 data, err := json.MarshalIndent(manifest, "", " ")
399 if err != nil {
400 return fmt.Errorf("failed to marshal manifest: %w", err)
401 }
402
403 path := filepath.Join(d.cacheDir, FilterFiles.Manifest)
404 if err := os.WriteFile(path, data, manifestFilePerm); err != nil {
405 return fmt.Errorf("failed to write manifest: %w", err)
406 }
407
408 return nil
409}
410
411// GetCachedFilterPaths returns paths to cached filter JSON files.
412// Reads the cached manifest to discover which files should exist.
413// Returns nil if the manifest is missing or any filter file is missing.
414func (d *Downloader) GetCachedFilterPaths() []string {
415 manifest, err := d.GetCachedManifest()
416 if err != nil || manifest == nil || len(manifest.Combined.Files) == 0 {
417 return nil
418 }
419 paths := make([]string, 0, len(manifest.Combined.Files))
420 for _, filename := range manifest.Combined.Files {
421 path, safeFilename, err := d.cachePathForFilterFile(filename)
422 if err != nil {
423 return nil
424 }
425 if !d.hasCachedFilterFile(safeFilename) {
426 return nil
427 }
428 paths = append(paths, path)
429 }
430 return paths
431}
432
433// HasCachedFilters checks if all filter JSON files are cached.
434func (d *Downloader) HasCachedFilters() bool {
435 return d.GetCachedFilterPaths() != nil
436}
437
438// NeedsUpdate checks if filters need to be updated based on manifest version.
439func (d *Downloader) NeedsUpdate(ctx context.Context) (bool, error) {
440 log := logging.FromContext(ctx).With().
441 Str("component", "filter-downloader").
442 Logger()
443
444 cached, err := d.GetCachedManifest()
445 if err != nil {
446 return true, nil // If we can't read cached, assume we need update
447 }
448 if cached == nil {
449 log.Debug().Msg("no cached manifest, update needed")
450 return true, nil
451 }
452
453 latest, err := d.FetchManifest(ctx)
454 if err != nil {
455 return false, err // Can't check, return error
456 }
457
458 needsUpdate := !sameManifestRelease(cached, latest)
459 if needsUpdate {
460 log.Info().
461 Str("cached_version", cached.Version).
462 Str("latest_version", latest.Version).
463 Msg("filter update available")
464 } else {
465 log.Debug().Str("version", cached.Version).Msg("filters up to date")
466 }
467
468 return needsUpdate, nil
469}
470
471func sameManifestRelease(cached, latest *Manifest) bool {
472 if cached == nil || latest == nil {
473 return false
474 }
475 return cached.Version == latest.Version &&
476 cached.GeneratedAt.Equal(latest.GeneratedAt) &&
477 cached.Combined.TotalRules == latest.Combined.TotalRules &&
478 slices.Equal(cached.Combined.Files, latest.Combined.Files)
479}
480
481// ClearCache removes all cached filter files.
482func (d *Downloader) ClearCache() error {
483 return os.RemoveAll(d.cacheDir)
484}
485
486func (d *Downloader) hasCachedFilterFile(safeFilename string) bool {
487 root, err := os.OpenRoot(d.cacheDir)
488 if err != nil {
489 return false
490 }
491 defer func() { _ = root.Close() }()
492
493 info, err := root.Stat(filepath.FromSlash(safeFilename))
494 return err == nil && !info.IsDir()
495}
496
497func (d *Downloader) validateManifest(manifest *Manifest) error {
498 if manifest == nil {
499 return fmt.Errorf("manifest is nil")
500 }
501 if len(manifest.Combined.Files) > d.limits.maxFilterFiles {
502 return fmt.Errorf("manifest lists %d combined filter files, limit is %d", len(manifest.Combined.Files), d.limits.maxFilterFiles)
503 }
504 for _, filename := range manifest.Combined.Files {
505 if _, _, err := d.cachePathForFilterFile(filename); err != nil {
506 return err
507 }
508 }
509 return nil
510}
511
512func readLimitedFile(filename string, limit int64, label string) ([]byte, error) {
513 file, err := os.Open(filename)
514 if err != nil {
515 return nil, err
516 }
517 defer func() { _ = file.Close() }()
518
519 info, err := file.Stat()
520 if err != nil {
521 return nil, err
522 }
523 if info.Size() > limit {
524 return nil, fmt.Errorf("%s too large: size %d exceeds limit %d", label, info.Size(), limit)
525 }
526
527 return readLimitedReader(file, limit, label)
528}
529
530func readLimitedResponseBody(resp *http.Response, limit int64, label string) ([]byte, error) {
531 if resp.ContentLength > limit {
532 return nil, fmt.Errorf("%s too large: content length %d exceeds limit %d", label, resp.ContentLength, limit)
533 }
534 return readLimitedReader(resp.Body, limit, label)
535}
536
537func readLimitedReader(r io.Reader, limit int64, label string) ([]byte, error) {
538 if limit <= 0 {
539 return nil, fmt.Errorf("%s size limit must be positive", label)
540 }
541
542 data, err := io.ReadAll(io.LimitReader(r, limit+1))
543 if err != nil {
544 return nil, err
545 }
546 if int64(len(data)) > limit {
547 return nil, fmt.Errorf("%s too large: exceeds limit %d bytes", label, limit)
548 }
549 return data, nil
550}
551
552func copyLimited(dst io.Writer, src io.Reader, limit int64, label string) (int64, error) {
553 if limit <= 0 {
554 return 0, fmt.Errorf("%s size limit must be positive", label)
555 }
556
557 written, err := io.Copy(dst, io.LimitReader(src, limit+1))
558 if err != nil {
559 return written, err
560 }
561 if written > limit {
562 return written, fmt.Errorf("%s too large: exceeds limit %d bytes", label, limit)
563 }
564 return written, nil
565}
566
567func minInt64(a, b int64) int64 {
568 if a < b {
569 return a
570 }
571 return b
572}
573
574// IsCacheStale checks if the cached manifest is older than maxAge.
575// Returns true if cache is stale or doesn't exist.
576func (d *Downloader) IsCacheStale(maxAge time.Duration) bool {
577 path := filepath.Join(d.cacheDir, FilterFiles.Manifest)
578 info, err := os.Stat(path)
579 if err != nil {
580 return true
581 }
582 return time.Since(info.ModTime()) > maxAge
583}