1package filtering
2
3import (
4 "context"
5 "encoding/json"
6 "fmt"
7 "io"
8 "net/http"
9 "os"
10 "path/filepath"
11 "time"
12
13 "github.com/bnema/dumber/internal/logging"
14)
15
16const (
17 cacheDirPerm = 0o755
18 manifestFilePerm = 0o644
19)
20
21// Downloader handles downloading filter files from GitHub releases.
22type Downloader struct {
23 baseURL string
24 cacheDir string
25 httpClient *http.Client
26}
27
28// NewDownloader creates a new Downloader.
29// cacheDir is where downloaded JSON files will be stored.
30func NewDownloader(cacheDir string) *Downloader {
31 return &Downloader{
32 baseURL: GitHubReleaseURL,
33 cacheDir: cacheDir,
34 httpClient: &http.Client{
35 Timeout: 60 * time.Second,
36 },
37 }
38}
39
40// GetCachedManifest reads the locally cached manifest.json.
41// Returns nil if no cached manifest exists.
42func (d *Downloader) GetCachedManifest() (*Manifest, error) {
43 path := filepath.Join(d.cacheDir, FilterFiles.Manifest)
44 data, err := os.ReadFile(path)
45 if err != nil {
46 if os.IsNotExist(err) {
47 return nil, nil
48 }
49 return nil, err
50 }
51
52 var manifest Manifest
53 if err := json.Unmarshal(data, &manifest); err != nil {
54 return nil, fmt.Errorf("failed to parse cached manifest: %w", err)
55 }
56 return &manifest, nil
57}
58
59// FetchManifest downloads the latest manifest.json from GitHub.
60func (d *Downloader) FetchManifest(ctx context.Context) (*Manifest, error) {
61 log := logging.FromContext(ctx).With().
62 Str("component", "filter-downloader").
63 Logger()
64
65 url := fmt.Sprintf("%s/%s", d.baseURL, FilterFiles.Manifest)
66 log.Debug().Str("url", url).Msg("fetching manifest")
67
68 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
69 if err != nil {
70 return nil, fmt.Errorf("failed to create request: %w", err)
71 }
72
73 resp, err := d.httpClient.Do(req)
74 if err != nil {
75 return nil, fmt.Errorf("failed to fetch manifest: %w", err)
76 }
77 defer func() {
78 if closeErr := resp.Body.Close(); closeErr != nil {
79 log.Debug().Err(closeErr).Msg("failed to close manifest response body")
80 }
81 }()
82
83 if resp.StatusCode != http.StatusOK {
84 return nil, fmt.Errorf("manifest fetch failed with status %d", resp.StatusCode)
85 }
86
87 data, err := io.ReadAll(resp.Body)
88 if err != nil {
89 return nil, fmt.Errorf("failed to read manifest body: %w", err)
90 }
91
92 var manifest Manifest
93 if err := json.Unmarshal(data, &manifest); err != nil {
94 return nil, fmt.Errorf("failed to parse manifest: %w", err)
95 }
96
97 log.Debug().Str("version", manifest.Version).Msg("fetched manifest")
98 return &manifest, nil
99}
100
101// DownloadProgress reports download progress.
102type DownloadProgress struct {
103 File string
104 Current int
105 Total int
106 BytesTotal int64
107}
108
109// DownloadFilters fetches the manifest to discover which filter files exist,
110// then downloads them. Returns the paths to the downloaded files.
111func (d *Downloader) DownloadFilters(ctx context.Context, onProgress func(DownloadProgress)) ([]string, error) {
112 log := logging.FromContext(ctx).With().
113 Str("component", "filter-downloader").
114 Logger()
115
116 // Ensure cache directory exists
117 if err := os.MkdirAll(d.cacheDir, cacheDirPerm); err != nil {
118 return nil, fmt.Errorf("failed to create cache dir: %w", err)
119 }
120
121 // Fetch manifest to discover which combined files to download
122 manifest, err := d.FetchManifest(ctx)
123 if err != nil {
124 return nil, fmt.Errorf("failed to fetch manifest: %w", err)
125 }
126 if len(manifest.Combined.Files) == 0 {
127 return nil, fmt.Errorf("manifest contains no combined filter files")
128 }
129
130 log.Info().
131 Int("files", len(manifest.Combined.Files)).
132 Int("total_rules", manifest.Combined.TotalRules).
133 Str("version", manifest.Version).
134 Msg("manifest lists filter files")
135
136 files := manifest.Combined.Files
137 paths := make([]string, 0, len(files))
138 var totalBytes int64
139
140 for i, filename := range files {
141 select {
142 case <-ctx.Done():
143 return nil, ctx.Err()
144 default:
145 }
146
147 if onProgress != nil {
148 onProgress(DownloadProgress{
149 File: filename,
150 Current: i + 1,
151 Total: len(files),
152 BytesTotal: totalBytes,
153 })
154 }
155
156 path, bytesDownloaded, err := d.downloadFile(ctx, filename)
157 if err != nil {
158 log.Error().Err(err).Str("file", filename).Msg("failed to download filter file")
159 return nil, err
160 }
161
162 totalBytes += bytesDownloaded
163 paths = append(paths, path)
164 log.Debug().Str("file", filename).Int64("bytes", bytesDownloaded).Msg("downloaded filter file")
165 }
166
167 // Cache the manifest we already fetched
168 if err := d.cacheManifest(manifest); err != nil {
169 log.Warn().Err(err).Msg("failed to cache manifest")
170 }
171
172 log.Info().Int("files", len(paths)).Int64("total_bytes", totalBytes).Msg("filter download complete")
173 return paths, nil
174}
175
176// downloadFile downloads a single file and returns its local path.
177func (d *Downloader) downloadFile(ctx context.Context, filename string) (string, int64, error) {
178 url := fmt.Sprintf("%s/%s", d.baseURL, filename)
179 localPath := filepath.Join(d.cacheDir, filename)
180
181 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
182 if err != nil {
183 return "", 0, fmt.Errorf("failed to create request: %w", err)
184 }
185
186 resp, err := d.httpClient.Do(req)
187 if err != nil {
188 return "", 0, fmt.Errorf("failed to download %s: %w", filename, err)
189 }
190 defer func() {
191 if closeErr := resp.Body.Close(); closeErr != nil {
192 logging.FromContext(ctx).Debug().Err(closeErr).Str("file", filename).Msg("failed to close filter response body")
193 }
194 }()
195
196 if resp.StatusCode != http.StatusOK {
197 return "", 0, fmt.Errorf("download %s failed with status %d", filename, resp.StatusCode)
198 }
199
200 // Write to temp file in cacheDir, then atomic rename into destination.
201 tmpFile, err := os.CreateTemp(d.cacheDir, filepath.Base(filename)+".*.tmp")
202 if err != nil {
203 return "", 0, fmt.Errorf("failed to create temp file: %w", err)
204 }
205 tmpPath := tmpFile.Name()
206
207 bytesWritten, err := io.Copy(tmpFile, resp.Body)
208 if err != nil {
209 _ = tmpFile.Close()
210 _ = os.Remove(tmpPath)
211 return "", 0, fmt.Errorf("failed to write %s: %w", filename, err)
212 }
213 if err := tmpFile.Close(); err != nil {
214 _ = os.Remove(tmpPath)
215 return "", 0, fmt.Errorf("failed to close temp file: %w", err)
216 }
217
218 if err := d.renameTempFile(tmpPath, localPath); err != nil {
219 _ = os.Remove(tmpPath)
220 return "", 0, err
221 }
222
223 return localPath, bytesWritten, nil
224}
225
226func (*Downloader) renameTempFile(tmpPath, localPath string) error {
227 parentDir := filepath.Dir(localPath)
228
229 if err := os.MkdirAll(parentDir, cacheDirPerm); err != nil {
230 return fmt.Errorf("failed to create target dir: %w", err)
231 }
232
233 if err := os.Rename(tmpPath, localPath); err != nil {
234 return fmt.Errorf("failed to rename temp file: %w", err)
235 }
236
237 return nil
238}
239
240// cacheManifest writes an already-fetched manifest to disk.
241func (d *Downloader) cacheManifest(manifest *Manifest) error {
242 data, err := json.MarshalIndent(manifest, "", " ")
243 if err != nil {
244 return fmt.Errorf("failed to marshal manifest: %w", err)
245 }
246
247 path := filepath.Join(d.cacheDir, FilterFiles.Manifest)
248 if err := os.WriteFile(path, data, manifestFilePerm); err != nil {
249 return fmt.Errorf("failed to write manifest: %w", err)
250 }
251
252 return nil
253}
254
255// GetCachedFilterPaths returns paths to cached filter JSON files.
256// Reads the cached manifest to discover which files should exist.
257// Returns nil if the manifest is missing or any filter file is missing.
258func (d *Downloader) GetCachedFilterPaths() []string {
259 manifest, err := d.GetCachedManifest()
260 if err != nil || manifest == nil || len(manifest.Combined.Files) == 0 {
261 return nil
262 }
263 paths := make([]string, 0, len(manifest.Combined.Files))
264 for _, filename := range manifest.Combined.Files {
265 path := filepath.Join(d.cacheDir, filename)
266 if _, err := os.Stat(path); err != nil {
267 return nil
268 }
269 paths = append(paths, path)
270 }
271 return paths
272}
273
274// HasCachedFilters checks if all filter JSON files are cached.
275func (d *Downloader) HasCachedFilters() bool {
276 return d.GetCachedFilterPaths() != nil
277}
278
279// NeedsUpdate checks if filters need to be updated based on manifest version.
280func (d *Downloader) NeedsUpdate(ctx context.Context) (bool, error) {
281 log := logging.FromContext(ctx).With().
282 Str("component", "filter-downloader").
283 Logger()
284
285 cached, err := d.GetCachedManifest()
286 if err != nil {
287 return true, nil // If we can't read cached, assume we need update
288 }
289 if cached == nil {
290 log.Debug().Msg("no cached manifest, update needed")
291 return true, nil
292 }
293
294 latest, err := d.FetchManifest(ctx)
295 if err != nil {
296 return false, err // Can't check, return error
297 }
298
299 needsUpdate := cached.Version != latest.Version
300 if needsUpdate {
301 log.Info().
302 Str("cached_version", cached.Version).
303 Str("latest_version", latest.Version).
304 Msg("filter update available")
305 } else {
306 log.Debug().Str("version", cached.Version).Msg("filters up to date")
307 }
308
309 return needsUpdate, nil
310}
311
312// ClearCache removes all cached filter files.
313func (d *Downloader) ClearCache() error {
314 return os.RemoveAll(d.cacheDir)
315}
316
317// IsCacheStale checks if the cached manifest is older than maxAge.
318// Returns true if cache is stale or doesn't exist.
319func (d *Downloader) IsCacheStale(maxAge time.Duration) bool {
320 path := filepath.Join(d.cacheDir, FilterFiles.Manifest)
321 info, err := os.Stat(path)
322 if err != nil {
323 return true
324 }
325 return time.Since(info.ModTime()) > maxAge
326}