detached 7505dfa

feat(filtering): add downloader, store, and filter state types

commit
7505dfa8ebe957b7d16add671317a2ac8288461f
author
brice · · 2025-12-18 22:00 UTC
committer
brice · · 2025-12-18 22:52 UTC
parent
ded032c
signature
unsigned
3 files · +598 -0
Unified
UnifiedSplit
A internal/infrastructure/filtering/downloader.go +272 -0
@@ -0,0 +1,272 @@
1+package filtering
2+
3+import (
4+ "context"
5+ "encoding/json"
6+ "fmt"
7+ "io"
8+ "net/http"
9+ "os"
10+ "path/filepath"
11+ "time"
12+
13+ "github.com/bnema/dumber/internal/logging"
14+)
15+
16+// Downloader handles downloading filter files from GitHub releases.
17+type Downloader struct {
18+ baseURL string
19+ cacheDir string
20+ httpClient *http.Client
21+}
22+
23+// NewDownloader creates a new Downloader.
24+// cacheDir is where downloaded JSON files will be stored.
25+func NewDownloader(cacheDir string) *Downloader {
26+ return &Downloader{
27+ baseURL: GitHubReleaseURL,
28+ cacheDir: cacheDir,
29+ httpClient: &http.Client{
30+ Timeout: 60 * time.Second,
31+ },
32+ }
33+}
34+
35+// GetCachedManifest reads the locally cached manifest.json.
36+// Returns nil if no cached manifest exists.
37+func (d *Downloader) GetCachedManifest() (*Manifest, error) {
38+ path := filepath.Join(d.cacheDir, FilterFiles.Manifest)
39+ data, err := os.ReadFile(path)
40+ if err != nil {
41+ if os.IsNotExist(err) {
42+ return nil, nil
43+ }
44+ return nil, err
45+ }
46+
47+ var manifest Manifest
48+ if err := json.Unmarshal(data, &manifest); err != nil {
49+ return nil, fmt.Errorf("failed to parse cached manifest: %w", err)
50+ }
51+ return &manifest, nil
52+}
53+
54+// FetchManifest downloads the latest manifest.json from GitHub.
55+func (d *Downloader) FetchManifest(ctx context.Context) (*Manifest, error) {
56+ log := logging.FromContext(ctx).With().
57+ Str("component", "filter-downloader").
58+ Logger()
59+
60+ url := fmt.Sprintf("%s/%s", d.baseURL, FilterFiles.Manifest)
61+ log.Debug().Str("url", url).Msg("fetching manifest")
62+
63+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
64+ if err != nil {
65+ return nil, fmt.Errorf("failed to create request: %w", err)
66+ }
67+
68+ resp, err := d.httpClient.Do(req)
69+ if err != nil {
70+ return nil, fmt.Errorf("failed to fetch manifest: %w", err)
71+ }
72+ defer resp.Body.Close()
73+
74+ if resp.StatusCode != http.StatusOK {
75+ return nil, fmt.Errorf("manifest fetch failed with status %d", resp.StatusCode)
76+ }
77+
78+ data, err := io.ReadAll(resp.Body)
79+ if err != nil {
80+ return nil, fmt.Errorf("failed to read manifest body: %w", err)
81+ }
82+
83+ var manifest Manifest
84+ if err := json.Unmarshal(data, &manifest); err != nil {
85+ return nil, fmt.Errorf("failed to parse manifest: %w", err)
86+ }
87+
88+ log.Debug().Str("version", manifest.Version).Msg("fetched manifest")
89+ return &manifest, nil
90+}
91+
92+// DownloadProgress reports download progress.
93+type DownloadProgress struct {
94+ File string
95+ Current int
96+ Total int
97+ BytesTotal int64
98+}
99+
100+// DownloadFilters downloads filter JSON files from GitHub releases.
101+// Returns the paths to the downloaded files.
102+func (d *Downloader) DownloadFilters(ctx context.Context, onProgress func(DownloadProgress)) ([]string, error) {
103+ log := logging.FromContext(ctx).With().
104+ Str("component", "filter-downloader").
105+ Logger()
106+
107+ // Ensure cache directory exists
108+ if err := os.MkdirAll(d.cacheDir, 0o755); err != nil {
109+ return nil, fmt.Errorf("failed to create cache dir: %w", err)
110+ }
111+
112+ files := FilterFiles.Combined
113+ paths := make([]string, 0, len(files))
114+ var totalBytes int64
115+
116+ for i, filename := range files {
117+ select {
118+ case <-ctx.Done():
119+ return nil, ctx.Err()
120+ default:
121+ }
122+
123+ if onProgress != nil {
124+ onProgress(DownloadProgress{
125+ File: filename,
126+ Current: i + 1,
127+ Total: len(files),
128+ BytesTotal: totalBytes,
129+ })
130+ }
131+
132+ path, bytesDownloaded, err := d.downloadFile(ctx, filename)
133+ if err != nil {
134+ log.Error().Err(err).Str("file", filename).Msg("failed to download filter file")
135+ return nil, err
136+ }
137+
138+ totalBytes += bytesDownloaded
139+ paths = append(paths, path)
140+ log.Debug().Str("file", filename).Int64("bytes", bytesDownloaded).Msg("downloaded filter file")
141+ }
142+
143+ // Also download and cache the manifest
144+ if err := d.downloadAndCacheManifest(ctx); err != nil {
145+ log.Warn().Err(err).Msg("failed to cache manifest")
146+ // Non-fatal, continue
147+ }
148+
149+ log.Info().Int("files", len(paths)).Int64("total_bytes", totalBytes).Msg("filter download complete")
150+ return paths, nil
151+}
152+
153+// downloadFile downloads a single file and returns its local path.
154+func (d *Downloader) downloadFile(ctx context.Context, filename string) (string, int64, error) {
155+ url := fmt.Sprintf("%s/%s", d.baseURL, filename)
156+ localPath := filepath.Join(d.cacheDir, filename)
157+
158+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
159+ if err != nil {
160+ return "", 0, fmt.Errorf("failed to create request: %w", err)
161+ }
162+
163+ resp, err := d.httpClient.Do(req)
164+ if err != nil {
165+ return "", 0, fmt.Errorf("failed to download %s: %w", filename, err)
166+ }
167+ defer resp.Body.Close()
168+
169+ if resp.StatusCode != http.StatusOK {
170+ return "", 0, fmt.Errorf("download %s failed with status %d", filename, resp.StatusCode)
171+ }
172+
173+ // Write to temp file first, then atomic rename
174+ tmpPath := localPath + ".tmp"
175+ file, err := os.Create(tmpPath)
176+ if err != nil {
177+ return "", 0, fmt.Errorf("failed to create temp file: %w", err)
178+ }
179+
180+ bytesWritten, err := io.Copy(file, resp.Body)
181+ if err != nil {
182+ file.Close()
183+ os.Remove(tmpPath)
184+ return "", 0, fmt.Errorf("failed to write %s: %w", filename, err)
185+ }
186+ file.Close()
187+
188+ // Atomic rename
189+ if err := os.Rename(tmpPath, localPath); err != nil {
190+ os.Remove(tmpPath)
191+ return "", 0, fmt.Errorf("failed to rename temp file: %w", err)
192+ }
193+
194+ return localPath, bytesWritten, nil
195+}
196+
197+// downloadAndCacheManifest downloads and caches the manifest.
198+func (d *Downloader) downloadAndCacheManifest(ctx context.Context) error {
199+ manifest, err := d.FetchManifest(ctx)
200+ if err != nil {
201+ return err
202+ }
203+
204+ data, err := json.MarshalIndent(manifest, "", " ")
205+ if err != nil {
206+ return fmt.Errorf("failed to marshal manifest: %w", err)
207+ }
208+
209+ path := filepath.Join(d.cacheDir, FilterFiles.Manifest)
210+ if err := os.WriteFile(path, data, 0o644); err != nil {
211+ return fmt.Errorf("failed to write manifest: %w", err)
212+ }
213+
214+ return nil
215+}
216+
217+// GetCachedFilterPaths returns paths to cached filter JSON files.
218+// Returns nil if any files are missing.
219+func (d *Downloader) GetCachedFilterPaths() []string {
220+ paths := make([]string, 0, len(FilterFiles.Combined))
221+ for _, filename := range FilterFiles.Combined {
222+ path := filepath.Join(d.cacheDir, filename)
223+ if _, err := os.Stat(path); err != nil {
224+ return nil
225+ }
226+ paths = append(paths, path)
227+ }
228+ return paths
229+}
230+
231+// HasCachedFilters checks if all filter JSON files are cached.
232+func (d *Downloader) HasCachedFilters() bool {
233+ return d.GetCachedFilterPaths() != nil
234+}
235+
236+// NeedsUpdate checks if filters need to be updated based on manifest version.
237+func (d *Downloader) NeedsUpdate(ctx context.Context) (bool, error) {
238+ log := logging.FromContext(ctx).With().
239+ Str("component", "filter-downloader").
240+ Logger()
241+
242+ cached, err := d.GetCachedManifest()
243+ if err != nil {
244+ return true, nil // If we can't read cached, assume we need update
245+ }
246+ if cached == nil {
247+ log.Debug().Msg("no cached manifest, update needed")
248+ return true, nil
249+ }
250+
251+ latest, err := d.FetchManifest(ctx)
252+ if err != nil {
253+ return false, err // Can't check, return error
254+ }
255+
256+ needsUpdate := cached.Version != latest.Version
257+ if needsUpdate {
258+ log.Info().
259+ Str("cached_version", cached.Version).
260+ Str("latest_version", latest.Version).
261+ Msg("filter update available")
262+ } else {
263+ log.Debug().Str("version", cached.Version).Msg("filters up to date")
264+ }
265+
266+ return needsUpdate, nil
267+}
268+
269+// ClearCache removes all cached filter files.
270+func (d *Downloader) ClearCache() error {
271+ return os.RemoveAll(d.cacheDir)
272+}
@@ -0,0 +1,272 @@
1+package filtering
2+
3+import (
4+ "context"
5+ "encoding/json"
6+ "fmt"
7+ "io"
8+ "net/http"
9+ "os"
10+ "path/filepath"
11+ "time"
12+
13+ "github.com/bnema/dumber/internal/logging"
14+)
15+
16+// Downloader handles downloading filter files from GitHub releases.
17+type Downloader struct {
18+ baseURL string
19+ cacheDir string
20+ httpClient *http.Client
21+}
22+
23+// NewDownloader creates a new Downloader.
24+// cacheDir is where downloaded JSON files will be stored.
25+func NewDownloader(cacheDir string) *Downloader {
26+ return &Downloader{
27+ baseURL: GitHubReleaseURL,
28+ cacheDir: cacheDir,
29+ httpClient: &http.Client{
30+ Timeout: 60 * time.Second,
31+ },
32+ }
33+}
34+
35+// GetCachedManifest reads the locally cached manifest.json.
36+// Returns nil if no cached manifest exists.
37+func (d *Downloader) GetCachedManifest() (*Manifest, error) {
38+ path := filepath.Join(d.cacheDir, FilterFiles.Manifest)
39+ data, err := os.ReadFile(path)
40+ if err != nil {
41+ if os.IsNotExist(err) {
42+ return nil, nil
43+ }
44+ return nil, err
45+ }
46+
47+ var manifest Manifest
48+ if err := json.Unmarshal(data, &manifest); err != nil {
49+ return nil, fmt.Errorf("failed to parse cached manifest: %w", err)
50+ }
51+ return &manifest, nil
52+}
53+
54+// FetchManifest downloads the latest manifest.json from GitHub.
55+func (d *Downloader) FetchManifest(ctx context.Context) (*Manifest, error) {
56+ log := logging.FromContext(ctx).With().
57+ Str("component", "filter-downloader").
58+ Logger()
59+
60+ url := fmt.Sprintf("%s/%s", d.baseURL, FilterFiles.Manifest)
61+ log.Debug().Str("url", url).Msg("fetching manifest")
62+
63+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
64+ if err != nil {
65+ return nil, fmt.Errorf("failed to create request: %w", err)
66+ }
67+
68+ resp, err := d.httpClient.Do(req)
69+ if err != nil {
70+ return nil, fmt.Errorf("failed to fetch manifest: %w", err)
71+ }
72+ defer resp.Body.Close()
73+
74+ if resp.StatusCode != http.StatusOK {
75+ return nil, fmt.Errorf("manifest fetch failed with status %d", resp.StatusCode)
76+ }
77+
78+ data, err := io.ReadAll(resp.Body)
79+ if err != nil {
80+ return nil, fmt.Errorf("failed to read manifest body: %w", err)
81+ }
82+
83+ var manifest Manifest
84+ if err := json.Unmarshal(data, &manifest); err != nil {
85+ return nil, fmt.Errorf("failed to parse manifest: %w", err)
86+ }
87+
88+ log.Debug().Str("version", manifest.Version).Msg("fetched manifest")
89+ return &manifest, nil
90+}
91+
92+// DownloadProgress reports download progress.
93+type DownloadProgress struct {
94+ File string
95+ Current int
96+ Total int
97+ BytesTotal int64
98+}
99+
100+// DownloadFilters downloads filter JSON files from GitHub releases.
101+// Returns the paths to the downloaded files.
102+func (d *Downloader) DownloadFilters(ctx context.Context, onProgress func(DownloadProgress)) ([]string, error) {
103+ log := logging.FromContext(ctx).With().
104+ Str("component", "filter-downloader").
105+ Logger()
106+
107+ // Ensure cache directory exists
108+ if err := os.MkdirAll(d.cacheDir, 0o755); err != nil {
109+ return nil, fmt.Errorf("failed to create cache dir: %w", err)
110+ }
111+
112+ files := FilterFiles.Combined
113+ paths := make([]string, 0, len(files))
114+ var totalBytes int64
115+
116+ for i, filename := range files {
117+ select {
118+ case <-ctx.Done():
119+ return nil, ctx.Err()
120+ default:
121+ }
122+
123+ if onProgress != nil {
124+ onProgress(DownloadProgress{
125+ File: filename,
126+ Current: i + 1,
127+ Total: len(files),
128+ BytesTotal: totalBytes,
129+ })
130+ }
131+
132+ path, bytesDownloaded, err := d.downloadFile(ctx, filename)
133+ if err != nil {
134+ log.Error().Err(err).Str("file", filename).Msg("failed to download filter file")
135+ return nil, err
136+ }
137+
138+ totalBytes += bytesDownloaded
139+ paths = append(paths, path)
140+ log.Debug().Str("file", filename).Int64("bytes", bytesDownloaded).Msg("downloaded filter file")
141+ }
142+
143+ // Also download and cache the manifest
144+ if err := d.downloadAndCacheManifest(ctx); err != nil {
145+ log.Warn().Err(err).Msg("failed to cache manifest")
146+ // Non-fatal, continue
147+ }
148+
149+ log.Info().Int("files", len(paths)).Int64("total_bytes", totalBytes).Msg("filter download complete")
150+ return paths, nil
151+}
152+
153+// downloadFile downloads a single file and returns its local path.
154+func (d *Downloader) downloadFile(ctx context.Context, filename string) (string, int64, error) {
155+ url := fmt.Sprintf("%s/%s", d.baseURL, filename)
156+ localPath := filepath.Join(d.cacheDir, filename)
157+
158+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
159+ if err != nil {
160+ return "", 0, fmt.Errorf("failed to create request: %w", err)
161+ }
162+
163+ resp, err := d.httpClient.Do(req)
164+ if err != nil {
165+ return "", 0, fmt.Errorf("failed to download %s: %w", filename, err)
166+ }
167+ defer resp.Body.Close()
168+
169+ if resp.StatusCode != http.StatusOK {
170+ return "", 0, fmt.Errorf("download %s failed with status %d", filename, resp.StatusCode)
171+ }
172+
173+ // Write to temp file first, then atomic rename
174+ tmpPath := localPath + ".tmp"
175+ file, err := os.Create(tmpPath)
176+ if err != nil {
177+ return "", 0, fmt.Errorf("failed to create temp file: %w", err)
178+ }
179+
180+ bytesWritten, err := io.Copy(file, resp.Body)
181+ if err != nil {
182+ file.Close()
183+ os.Remove(tmpPath)
184+ return "", 0, fmt.Errorf("failed to write %s: %w", filename, err)
185+ }
186+ file.Close()
187+
188+ // Atomic rename
189+ if err := os.Rename(tmpPath, localPath); err != nil {
190+ os.Remove(tmpPath)
191+ return "", 0, fmt.Errorf("failed to rename temp file: %w", err)
192+ }
193+
194+ return localPath, bytesWritten, nil
195+}
196+
197+// downloadAndCacheManifest downloads and caches the manifest.
198+func (d *Downloader) downloadAndCacheManifest(ctx context.Context) error {
199+ manifest, err := d.FetchManifest(ctx)
200+ if err != nil {
201+ return err
202+ }
203+
204+ data, err := json.MarshalIndent(manifest, "", " ")
205+ if err != nil {
206+ return fmt.Errorf("failed to marshal manifest: %w", err)
207+ }
208+
209+ path := filepath.Join(d.cacheDir, FilterFiles.Manifest)
210+ if err := os.WriteFile(path, data, 0o644); err != nil {
211+ return fmt.Errorf("failed to write manifest: %w", err)
212+ }
213+
214+ return nil
215+}
216+
217+// GetCachedFilterPaths returns paths to cached filter JSON files.
218+// Returns nil if any files are missing.
219+func (d *Downloader) GetCachedFilterPaths() []string {
220+ paths := make([]string, 0, len(FilterFiles.Combined))
221+ for _, filename := range FilterFiles.Combined {
222+ path := filepath.Join(d.cacheDir, filename)
223+ if _, err := os.Stat(path); err != nil {
224+ return nil
225+ }
226+ paths = append(paths, path)
227+ }
228+ return paths
229+}
230+
231+// HasCachedFilters checks if all filter JSON files are cached.
232+func (d *Downloader) HasCachedFilters() bool {
233+ return d.GetCachedFilterPaths() != nil
234+}
235+
236+// NeedsUpdate checks if filters need to be updated based on manifest version.
237+func (d *Downloader) NeedsUpdate(ctx context.Context) (bool, error) {
238+ log := logging.FromContext(ctx).With().
239+ Str("component", "filter-downloader").
240+ Logger()
241+
242+ cached, err := d.GetCachedManifest()
243+ if err != nil {
244+ return true, nil // If we can't read cached, assume we need update
245+ }
246+ if cached == nil {
247+ log.Debug().Msg("no cached manifest, update needed")
248+ return true, nil
249+ }
250+
251+ latest, err := d.FetchManifest(ctx)
252+ if err != nil {
253+ return false, err // Can't check, return error
254+ }
255+
256+ needsUpdate := cached.Version != latest.Version
257+ if needsUpdate {
258+ log.Info().
259+ Str("cached_version", cached.Version).
260+ Str("latest_version", latest.Version).
261+ Msg("filter update available")
262+ } else {
263+ log.Debug().Str("version", cached.Version).Msg("filters up to date")
264+ }
265+
266+ return needsUpdate, nil
267+}
268+
269+// ClearCache removes all cached filter files.
270+func (d *Downloader) ClearCache() error {
271+ return os.RemoveAll(d.cacheDir)
272+}
A internal/infrastructure/filtering/store.go +257 -0
@@ -0,0 +1,257 @@
1+package filtering
2+
3+import (
4+ "context"
5+ "fmt"
6+ "sync"
7+
8+ "github.com/bnema/dumber/internal/logging"
9+ "github.com/bnema/puregotk-webkit/webkit"
10+ "github.com/jwijenbergh/puregotk/v4/gio"
11+)
12+
13+// Store wraps webkit.UserContentFilterStore with Go-friendly async operations.
14+type Store struct {
15+ inner *webkit.UserContentFilterStore
16+ path string
17+ mu sync.Mutex
18+}
19+
20+// NewStore creates a new Store at the given path.
21+// The path is where WebKit will store compiled filter bytecode.
22+func NewStore(storagePath string) *Store {
23+ inner := webkit.NewUserContentFilterStore(storagePath)
24+ if inner == nil {
25+ return nil
26+ }
27+ return &Store{
28+ inner: inner,
29+ path: storagePath,
30+ }
31+}
32+
33+// Path returns the storage path for compiled filters.
34+func (s *Store) Path() string {
35+ return s.path
36+}
37+
38+// CompileResult holds the result of a filter compilation.
39+type CompileResult struct {
40+ Filter *webkit.UserContentFilter
41+ Err error
42+}
43+
44+// Compile compiles a JSON filter file and stores it with the given identifier.
45+// This is an async operation that may take several seconds for large filter sets.
46+// The JSON file must be in Safari Content Blocker format.
47+func (s *Store) Compile(ctx context.Context, identifier string, jsonPath string) (*webkit.UserContentFilter, error) {
48+ log := logging.FromContext(ctx).With().
49+ Str("component", "filter-store").
50+ Str("identifier", identifier).
51+ Str("json_path", jsonPath).
52+ Logger()
53+
54+ s.mu.Lock()
55+ defer s.mu.Unlock()
56+
57+ // Create GFile from path
58+ file := gio.FileNewForPath(jsonPath)
59+ if file == nil {
60+ return nil, fmt.Errorf("failed to create GFile for path: %s", jsonPath)
61+ }
62+
63+ log.Debug().Msg("starting filter compilation")
64+
65+ // Use a channel to receive the async result
66+ resultCh := make(chan CompileResult, 1)
67+
68+ // Create async callback
69+ cb := gio.AsyncReadyCallback(func(_ uintptr, resPtr uintptr, _ uintptr) {
70+ if resPtr == 0 {
71+ resultCh <- CompileResult{Err: fmt.Errorf("nil async result")}
72+ return
73+ }
74+
75+ res := &gio.AsyncResultBase{Ptr: resPtr}
76+ filter, err := s.inner.SaveFromFileFinish(res)
77+ resultCh <- CompileResult{Filter: filter, Err: err}
78+ })
79+
80+ // Start async compilation
81+ s.inner.SaveFromFile(identifier, file, nil, &cb, 0)
82+
83+ // Wait for result or context cancellation
84+ select {
85+ case result := <-resultCh:
86+ if result.Err != nil {
87+ log.Error().Err(result.Err).Msg("filter compilation failed")
88+ return nil, result.Err
89+ }
90+ log.Info().Msg("filter compilation completed")
91+ return result.Filter, nil
92+ case <-ctx.Done():
93+ log.Warn().Msg("filter compilation cancelled")
94+ return nil, ctx.Err()
95+ }
96+}
97+
98+// LoadResult holds the result of loading a compiled filter.
99+type LoadResult struct {
100+ Filter *webkit.UserContentFilter
101+ Err error
102+}
103+
104+// Load loads a previously compiled filter by its identifier.
105+// This is fast (~50ms) as it loads pre-compiled bytecode.
106+func (s *Store) Load(ctx context.Context, identifier string) (*webkit.UserContentFilter, error) {
107+ log := logging.FromContext(ctx).With().
108+ Str("component", "filter-store").
109+ Str("identifier", identifier).
110+ Logger()
111+
112+ s.mu.Lock()
113+ defer s.mu.Unlock()
114+
115+ log.Debug().Msg("loading compiled filter")
116+
117+ // Use a channel to receive the async result
118+ resultCh := make(chan LoadResult, 1)
119+
120+ // Create async callback
121+ cb := gio.AsyncReadyCallback(func(_ uintptr, resPtr uintptr, _ uintptr) {
122+ if resPtr == 0 {
123+ resultCh <- LoadResult{Err: fmt.Errorf("nil async result")}
124+ return
125+ }
126+
127+ res := &gio.AsyncResultBase{Ptr: resPtr}
128+ filter, err := s.inner.LoadFinish(res)
129+ resultCh <- LoadResult{Filter: filter, Err: err}
130+ })
131+
132+ // Start async load
133+ s.inner.Load(identifier, nil, &cb, 0)
134+
135+ // Wait for result or context cancellation
136+ select {
137+ case result := <-resultCh:
138+ if result.Err != nil {
139+ log.Debug().Err(result.Err).Msg("filter load failed (may not exist)")
140+ return nil, result.Err
141+ }
142+ log.Debug().Msg("filter loaded successfully")
143+ return result.Filter, nil
144+ case <-ctx.Done():
145+ log.Warn().Msg("filter load cancelled")
146+ return nil, ctx.Err()
147+ }
148+}
149+
150+// FetchIdentifiersResult holds the result of fetching filter identifiers.
151+type FetchIdentifiersResult struct {
152+ Identifiers []string
153+ Err error
154+}
155+
156+// FetchIdentifiers returns all stored filter identifiers.
157+func (s *Store) FetchIdentifiers(ctx context.Context) ([]string, error) {
158+ log := logging.FromContext(ctx).With().
159+ Str("component", "filter-store").
160+ Logger()
161+
162+ s.mu.Lock()
163+ defer s.mu.Unlock()
164+
165+ log.Debug().Msg("fetching filter identifiers")
166+
167+ // Use a channel to receive the async result
168+ resultCh := make(chan FetchIdentifiersResult, 1)
169+
170+ // Create async callback
171+ cb := gio.AsyncReadyCallback(func(_ uintptr, resPtr uintptr, _ uintptr) {
172+ if resPtr == 0 {
173+ resultCh <- FetchIdentifiersResult{Err: fmt.Errorf("nil async result")}
174+ return
175+ }
176+
177+ res := &gio.AsyncResultBase{Ptr: resPtr}
178+ identifiers := s.inner.FetchIdentifiersFinish(res)
179+ resultCh <- FetchIdentifiersResult{Identifiers: identifiers}
180+ })
181+
182+ // Start async fetch
183+ s.inner.FetchIdentifiers(nil, &cb, 0)
184+
185+ // Wait for result or context cancellation
186+ select {
187+ case result := <-resultCh:
188+ if result.Err != nil {
189+ log.Error().Err(result.Err).Msg("failed to fetch identifiers")
190+ return nil, result.Err
191+ }
192+ log.Debug().Int("count", len(result.Identifiers)).Msg("fetched filter identifiers")
193+ return result.Identifiers, nil
194+ case <-ctx.Done():
195+ log.Warn().Msg("fetch identifiers cancelled")
196+ return nil, ctx.Err()
197+ }
198+}
199+
200+// Remove removes a compiled filter by its identifier.
201+func (s *Store) Remove(ctx context.Context, identifier string) error {
202+ log := logging.FromContext(ctx).With().
203+ Str("component", "filter-store").
204+ Str("identifier", identifier).
205+ Logger()
206+
207+ s.mu.Lock()
208+ defer s.mu.Unlock()
209+
210+ log.Debug().Msg("removing filter")
211+
212+ // Use a channel to receive the async result
213+ resultCh := make(chan error, 1)
214+
215+ // Create async callback
216+ cb := gio.AsyncReadyCallback(func(_ uintptr, resPtr uintptr, _ uintptr) {
217+ if resPtr == 0 {
218+ resultCh <- fmt.Errorf("nil async result")
219+ return
220+ }
221+
222+ res := &gio.AsyncResultBase{Ptr: resPtr}
223+ _, err := s.inner.RemoveFinish(res)
224+ resultCh <- err
225+ })
226+
227+ // Start async remove
228+ s.inner.Remove(identifier, nil, &cb, 0)
229+
230+ // Wait for result or context cancellation
231+ select {
232+ case err := <-resultCh:
233+ if err != nil {
234+ log.Error().Err(err).Msg("failed to remove filter")
235+ return err
236+ }
237+ log.Info().Msg("filter removed")
238+ return nil
239+ case <-ctx.Done():
240+ log.Warn().Msg("remove filter cancelled")
241+ return ctx.Err()
242+ }
243+}
244+
245+// HasCompiledFilter checks if a compiled filter exists for the given identifier.
246+func (s *Store) HasCompiledFilter(ctx context.Context, identifier string) bool {
247+ identifiers, err := s.FetchIdentifiers(ctx)
248+ if err != nil {
249+ return false
250+ }
251+ for _, id := range identifiers {
252+ if id == identifier {
253+ return true
254+ }
255+ }
256+ return false
257+}
@@ -0,0 +1,257 @@
1+package filtering
2+
3+import (
4+ "context"
5+ "fmt"
6+ "sync"
7+
8+ "github.com/bnema/dumber/internal/logging"
9+ "github.com/bnema/puregotk-webkit/webkit"
10+ "github.com/jwijenbergh/puregotk/v4/gio"
11+)
12+
13+// Store wraps webkit.UserContentFilterStore with Go-friendly async operations.
14+type Store struct {
15+ inner *webkit.UserContentFilterStore
16+ path string
17+ mu sync.Mutex
18+}
19+
20+// NewStore creates a new Store at the given path.
21+// The path is where WebKit will store compiled filter bytecode.
22+func NewStore(storagePath string) *Store {
23+ inner := webkit.NewUserContentFilterStore(storagePath)
24+ if inner == nil {
25+ return nil
26+ }
27+ return &Store{
28+ inner: inner,
29+ path: storagePath,
30+ }
31+}
32+
33+// Path returns the storage path for compiled filters.
34+func (s *Store) Path() string {
35+ return s.path
36+}
37+
38+// CompileResult holds the result of a filter compilation.
39+type CompileResult struct {
40+ Filter *webkit.UserContentFilter
41+ Err error
42+}
43+
44+// Compile compiles a JSON filter file and stores it with the given identifier.
45+// This is an async operation that may take several seconds for large filter sets.
46+// The JSON file must be in Safari Content Blocker format.
47+func (s *Store) Compile(ctx context.Context, identifier string, jsonPath string) (*webkit.UserContentFilter, error) {
48+ log := logging.FromContext(ctx).With().
49+ Str("component", "filter-store").
50+ Str("identifier", identifier).
51+ Str("json_path", jsonPath).
52+ Logger()
53+
54+ s.mu.Lock()
55+ defer s.mu.Unlock()
56+
57+ // Create GFile from path
58+ file := gio.FileNewForPath(jsonPath)
59+ if file == nil {
60+ return nil, fmt.Errorf("failed to create GFile for path: %s", jsonPath)
61+ }
62+
63+ log.Debug().Msg("starting filter compilation")
64+
65+ // Use a channel to receive the async result
66+ resultCh := make(chan CompileResult, 1)
67+
68+ // Create async callback
69+ cb := gio.AsyncReadyCallback(func(_ uintptr, resPtr uintptr, _ uintptr) {
70+ if resPtr == 0 {
71+ resultCh <- CompileResult{Err: fmt.Errorf("nil async result")}
72+ return
73+ }
74+
75+ res := &gio.AsyncResultBase{Ptr: resPtr}
76+ filter, err := s.inner.SaveFromFileFinish(res)
77+ resultCh <- CompileResult{Filter: filter, Err: err}
78+ })
79+
80+ // Start async compilation
81+ s.inner.SaveFromFile(identifier, file, nil, &cb, 0)
82+
83+ // Wait for result or context cancellation
84+ select {
85+ case result := <-resultCh:
86+ if result.Err != nil {
87+ log.Error().Err(result.Err).Msg("filter compilation failed")
88+ return nil, result.Err
89+ }
90+ log.Info().Msg("filter compilation completed")
91+ return result.Filter, nil
92+ case <-ctx.Done():
93+ log.Warn().Msg("filter compilation cancelled")
94+ return nil, ctx.Err()
95+ }
96+}
97+
98+// LoadResult holds the result of loading a compiled filter.
99+type LoadResult struct {
100+ Filter *webkit.UserContentFilter
101+ Err error
102+}
103+
104+// Load loads a previously compiled filter by its identifier.
105+// This is fast (~50ms) as it loads pre-compiled bytecode.
106+func (s *Store) Load(ctx context.Context, identifier string) (*webkit.UserContentFilter, error) {
107+ log := logging.FromContext(ctx).With().
108+ Str("component", "filter-store").
109+ Str("identifier", identifier).
110+ Logger()
111+
112+ s.mu.Lock()
113+ defer s.mu.Unlock()
114+
115+ log.Debug().Msg("loading compiled filter")
116+
117+ // Use a channel to receive the async result
118+ resultCh := make(chan LoadResult, 1)
119+
120+ // Create async callback
121+ cb := gio.AsyncReadyCallback(func(_ uintptr, resPtr uintptr, _ uintptr) {
122+ if resPtr == 0 {
123+ resultCh <- LoadResult{Err: fmt.Errorf("nil async result")}
124+ return
125+ }
126+
127+ res := &gio.AsyncResultBase{Ptr: resPtr}
128+ filter, err := s.inner.LoadFinish(res)
129+ resultCh <- LoadResult{Filter: filter, Err: err}
130+ })
131+
132+ // Start async load
133+ s.inner.Load(identifier, nil, &cb, 0)
134+
135+ // Wait for result or context cancellation
136+ select {
137+ case result := <-resultCh:
138+ if result.Err != nil {
139+ log.Debug().Err(result.Err).Msg("filter load failed (may not exist)")
140+ return nil, result.Err
141+ }
142+ log.Debug().Msg("filter loaded successfully")
143+ return result.Filter, nil
144+ case <-ctx.Done():
145+ log.Warn().Msg("filter load cancelled")
146+ return nil, ctx.Err()
147+ }
148+}
149+
150+// FetchIdentifiersResult holds the result of fetching filter identifiers.
151+type FetchIdentifiersResult struct {
152+ Identifiers []string
153+ Err error
154+}
155+
156+// FetchIdentifiers returns all stored filter identifiers.
157+func (s *Store) FetchIdentifiers(ctx context.Context) ([]string, error) {
158+ log := logging.FromContext(ctx).With().
159+ Str("component", "filter-store").
160+ Logger()
161+
162+ s.mu.Lock()
163+ defer s.mu.Unlock()
164+
165+ log.Debug().Msg("fetching filter identifiers")
166+
167+ // Use a channel to receive the async result
168+ resultCh := make(chan FetchIdentifiersResult, 1)
169+
170+ // Create async callback
171+ cb := gio.AsyncReadyCallback(func(_ uintptr, resPtr uintptr, _ uintptr) {
172+ if resPtr == 0 {
173+ resultCh <- FetchIdentifiersResult{Err: fmt.Errorf("nil async result")}
174+ return
175+ }
176+
177+ res := &gio.AsyncResultBase{Ptr: resPtr}
178+ identifiers := s.inner.FetchIdentifiersFinish(res)
179+ resultCh <- FetchIdentifiersResult{Identifiers: identifiers}
180+ })
181+
182+ // Start async fetch
183+ s.inner.FetchIdentifiers(nil, &cb, 0)
184+
185+ // Wait for result or context cancellation
186+ select {
187+ case result := <-resultCh:
188+ if result.Err != nil {
189+ log.Error().Err(result.Err).Msg("failed to fetch identifiers")
190+ return nil, result.Err
191+ }
192+ log.Debug().Int("count", len(result.Identifiers)).Msg("fetched filter identifiers")
193+ return result.Identifiers, nil
194+ case <-ctx.Done():
195+ log.Warn().Msg("fetch identifiers cancelled")
196+ return nil, ctx.Err()
197+ }
198+}
199+
200+// Remove removes a compiled filter by its identifier.
201+func (s *Store) Remove(ctx context.Context, identifier string) error {
202+ log := logging.FromContext(ctx).With().
203+ Str("component", "filter-store").
204+ Str("identifier", identifier).
205+ Logger()
206+
207+ s.mu.Lock()
208+ defer s.mu.Unlock()
209+
210+ log.Debug().Msg("removing filter")
211+
212+ // Use a channel to receive the async result
213+ resultCh := make(chan error, 1)
214+
215+ // Create async callback
216+ cb := gio.AsyncReadyCallback(func(_ uintptr, resPtr uintptr, _ uintptr) {
217+ if resPtr == 0 {
218+ resultCh <- fmt.Errorf("nil async result")
219+ return
220+ }
221+
222+ res := &gio.AsyncResultBase{Ptr: resPtr}
223+ _, err := s.inner.RemoveFinish(res)
224+ resultCh <- err
225+ })
226+
227+ // Start async remove
228+ s.inner.Remove(identifier, nil, &cb, 0)
229+
230+ // Wait for result or context cancellation
231+ select {
232+ case err := <-resultCh:
233+ if err != nil {
234+ log.Error().Err(err).Msg("failed to remove filter")
235+ return err
236+ }
237+ log.Info().Msg("filter removed")
238+ return nil
239+ case <-ctx.Done():
240+ log.Warn().Msg("remove filter cancelled")
241+ return ctx.Err()
242+ }
243+}
244+
245+// HasCompiledFilter checks if a compiled filter exists for the given identifier.
246+func (s *Store) HasCompiledFilter(ctx context.Context, identifier string) bool {
247+ identifiers, err := s.FetchIdentifiers(ctx)
248+ if err != nil {
249+ return false
250+ }
251+ for _, id := range identifiers {
252+ if id == identifier {
253+ return true
254+ }
255+ }
256+ return false
257+}
A internal/infrastructure/filtering/types.go +69 -0
@@ -0,0 +1,69 @@
1+package filtering
2+
3+import "time"
4+
5+// FilterState represents the current state of the content filter system.
6+type FilterState string
7+
8+const (
9+ // StateUninitialized means the filter manager has not been initialized yet.
10+ StateUninitialized FilterState = "uninitialized"
11+ // StateLoading means filters are being downloaded or compiled.
12+ StateLoading FilterState = "loading"
13+ // StateActive means filters are loaded and active.
14+ StateActive FilterState = "active"
15+ // StateDisabled means filtering is disabled by configuration.
16+ StateDisabled FilterState = "disabled"
17+ // StateError means an error occurred during filter loading.
18+ StateError FilterState = "error"
19+)
20+
21+// FilterStatus represents the current status of the content filter system.
22+// Used for UI toast notifications and status reporting.
23+type FilterStatus struct {
24+ State FilterState
25+ Message string
26+ Version string
27+}
28+
29+// Manifest represents the manifest.json from ublock-webkit-filters releases.
30+// It contains metadata about the filter lists and their rule counts.
31+type Manifest struct {
32+ Version string `json:"version"`
33+ GeneratedAt time.Time `json:"generated_at"`
34+ Lists map[string]ListInfo `json:"lists"`
35+ Combined CombinedInfo `json:"combined"`
36+}
37+
38+// ListInfo contains metadata for a single filter list.
39+type ListInfo struct {
40+ Name string `json:"name"`
41+ SourceURL string `json:"source_url"`
42+ RulesCount int `json:"rules_count"`
43+ SkippedCount int `json:"skipped_count"`
44+}
45+
46+// CombinedInfo contains metadata for the combined (merged) filter set.
47+type CombinedInfo struct {
48+ TotalRules int `json:"total_rules"`
49+ Files []string `json:"files"`
50+}
51+
52+// FilterFiles defines the files to download from GitHub releases.
53+var FilterFiles = struct {
54+ Manifest string
55+ Combined []string
56+}{
57+ Manifest: "manifest.json",
58+ Combined: []string{
59+ "combined-part1.json",
60+ "combined-part2.json",
61+ "combined-part3.json",
62+ },
63+}
64+
65+// FilterIdentifier is the identifier used when storing/loading compiled filters.
66+const FilterIdentifier = "ublock-combined"
67+
68+// GitHubReleaseURL is the base URL for downloading filter files.
69+const GitHubReleaseURL = "https://github.com/bnema/ublock-webkit-filters/releases/latest/download"
@@ -0,0 +1,69 @@
1+package filtering
2+
3+import "time"
4+
5+// FilterState represents the current state of the content filter system.
6+type FilterState string
7+
8+const (
9+ // StateUninitialized means the filter manager has not been initialized yet.
10+ StateUninitialized FilterState = "uninitialized"
11+ // StateLoading means filters are being downloaded or compiled.
12+ StateLoading FilterState = "loading"
13+ // StateActive means filters are loaded and active.
14+ StateActive FilterState = "active"
15+ // StateDisabled means filtering is disabled by configuration.
16+ StateDisabled FilterState = "disabled"
17+ // StateError means an error occurred during filter loading.
18+ StateError FilterState = "error"
19+)
20+
21+// FilterStatus represents the current status of the content filter system.
22+// Used for UI toast notifications and status reporting.
23+type FilterStatus struct {
24+ State FilterState
25+ Message string
26+ Version string
27+}
28+
29+// Manifest represents the manifest.json from ublock-webkit-filters releases.
30+// It contains metadata about the filter lists and their rule counts.
31+type Manifest struct {
32+ Version string `json:"version"`
33+ GeneratedAt time.Time `json:"generated_at"`
34+ Lists map[string]ListInfo `json:"lists"`
35+ Combined CombinedInfo `json:"combined"`
36+}
37+
38+// ListInfo contains metadata for a single filter list.
39+type ListInfo struct {
40+ Name string `json:"name"`
41+ SourceURL string `json:"source_url"`
42+ RulesCount int `json:"rules_count"`
43+ SkippedCount int `json:"skipped_count"`
44+}
45+
46+// CombinedInfo contains metadata for the combined (merged) filter set.
47+type CombinedInfo struct {
48+ TotalRules int `json:"total_rules"`
49+ Files []string `json:"files"`
50+}
51+
52+// FilterFiles defines the files to download from GitHub releases.
53+var FilterFiles = struct {
54+ Manifest string
55+ Combined []string
56+}{
57+ Manifest: "manifest.json",
58+ Combined: []string{
59+ "combined-part1.json",
60+ "combined-part2.json",
61+ "combined-part3.json",
62+ },
63+}
64+
65+// FilterIdentifier is the identifier used when storing/loading compiled filters.
66+const FilterIdentifier = "ublock-combined"
67+
68+// GitHubReleaseURL is the base URL for downloading filter files.
69+const GitHubReleaseURL = "https://github.com/bnema/ublock-webkit-filters/releases/latest/download"

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 8

Tags 42

↑↓ moveenter openesc close