1// Package filtering provides filter management and compilation for content blocking.
2package filtering
3
4import (
5 "bufio"
6 "context"
7 "encoding/json"
8 "fmt"
9 "net/http"
10 "sync"
11 "time"
12
13 "github.com/bnema/dumber/internal/filtering/converter"
14 "github.com/bnema/dumber/internal/filtering/cosmetic"
15 "github.com/bnema/dumber/internal/logging"
16)
17
18const (
19 defaultUpdateHours = 24 // Default update interval in hours
20 httpTimeoutSeconds = 30 // HTTP client timeout in seconds
21 contextCheckInterval = 1000 // Check context cancellation every N lines
22)
23
24// FilterReadyCallback is called when filters are loaded/updated
25type FilterReadyCallback func()
26
27// FilterManager manages filter loading, compilation, and updates
28type FilterManager struct {
29 store FilterStore
30 compiler FilterCompiler
31 cosmeticInjector *cosmetic.CosmeticInjector
32 updateInterval time.Duration
33 compileMutex sync.RWMutex
34 compiled *CompiledFilters
35 onFiltersReady FilterReadyCallback
36}
37
38// FilterStore defines the interface for filter storage operations
39type FilterStore interface {
40 LoadCached() (*CompiledFilters, error)
41 SaveCache(filters *CompiledFilters) error
42 GetCacheInfo() (bool, time.Time, error)
43}
44
45// FilterCompiler defines the interface for filter compilation
46type FilterCompiler interface {
47 CompileFromSources(ctx context.Context, sources []string) (*CompiledFilters, error)
48 CompileFromData(data []byte) (*CompiledFilters, error)
49}
50
51// CosmeticInjector defines the interface for cosmetic filter injection
52type CosmeticInjector interface {
53 InjectRules(rules map[string][]string) error
54 GetScriptForDomain(domain string) string
55}
56
57// CompiledFilters represents the compiled filter data
58type CompiledFilters struct {
59 NetworkRules []converter.WebKitRule `json:"network_rules"`
60 CosmeticRules map[string][]string `json:"cosmetic_rules"`
61 GenericHiding []string `json:"generic_hiding"`
62 CompiledAt time.Time `json:"compiled_at"`
63 Version string `json:"version"`
64 mu sync.RWMutex
65}
66
67// NewFilterManager creates a new filter manager
68func NewFilterManager(store FilterStore, compiler FilterCompiler) *FilterManager {
69 return &FilterManager{
70 store: store,
71 compiler: compiler,
72 cosmeticInjector: cosmetic.NewCosmeticInjector(),
73 updateInterval: defaultUpdateHours * time.Hour, // Default update interval
74 compiled: &CompiledFilters{},
75 onFiltersReady: nil,
76 }
77}
78
79// SetFiltersReadyCallback sets a callback to be called when filters are loaded/updated
80func (fm *FilterManager) SetFiltersReadyCallback(callback FilterReadyCallback) {
81 fm.onFiltersReady = callback
82}
83
84// InitAsync initializes filter loading in background
85func (fm *FilterManager) InitAsync(ctx context.Context) error {
86 // Load pre-compiled filters in background
87 go fm.loadCompiledFilters(ctx)
88
89 // Start periodic update checker
90 go fm.startUpdateLoop(ctx)
91
92 // Note: compileIfNeeded removed to prevent race condition
93 // Cache age checking is now handled in loadCompiledFilters
94
95 return nil
96}
97
98func (fm *FilterManager) loadCompiledFilters(ctx context.Context) {
99 // Try to load from cache first
100 if cached, err := fm.store.LoadCached(); err == nil {
101 logging.Info(fmt.Sprintf("[filtering] Successfully loaded cache with %d network rules, %d cosmetic domains",
102 len(cached.NetworkRules), len(cached.CosmeticRules)))
103 fm.applyFilters(cached)
104
105 // Check if cache needs update
106 if exists, lastModified, _ := fm.store.GetCacheInfo(); exists {
107 if time.Since(lastModified) > fm.updateInterval {
108 // Schedule background update without clearing current filters
109 logging.Info("[filtering] Cache is outdated, scheduling background update")
110 go fm.updateFiltersInBackground(ctx)
111 }
112 }
113 return
114 } else {
115 logging.Error(fmt.Sprintf("[filtering] Failed to load cache: %v", err))
116 }
117
118 // Only compile if no cache exists
119 logging.Info("[filtering] No cache found, compiling from sources")
120 fm.compileFromSources(ctx)
121}
122
123func (fm *FilterManager) compileIfNeeded(ctx context.Context) {
124 // Check if cache exists and is recent
125 exists, lastModified, err := fm.store.GetCacheInfo()
126 if err != nil {
127 fm.compileFromSources(ctx)
128 return
129 }
130
131 // If cache is older than update interval, recompile
132 if !exists || time.Since(lastModified) > fm.updateInterval {
133 fm.compileFromSources(ctx)
134 }
135}
136
137func (fm *FilterManager) compileFromSources(ctx context.Context) {
138 sources := []string{
139 "https://easylist.to/easylist/easylist.txt",
140 "https://easylist.to/easylist/easyprivacy.txt",
141 "https://easylist-downloads.adblockplus.org/liste_fr.txt", // For French sites like lesnumeriques.com
142 }
143
144 var wg sync.WaitGroup
145 results := make(chan *CompiledFilters, len(sources))
146
147 for _, url := range sources {
148 wg.Add(1)
149 go func(u string) {
150 defer wg.Done()
151 if filters, err := fm.downloadAndCompile(ctx, u); err == nil {
152 results <- filters
153 } else {
154 logging.Error(fmt.Sprintf("[filtering] Failed to download/compile %s: %v", u, err))
155 }
156 }(url)
157 }
158
159 // Wait for all compilations
160 go func() {
161 wg.Wait()
162 close(results)
163 }()
164
165 // Merge results as they come in
166 merged := NewCompiledFilters()
167 hasContent := false
168
169 for compiled := range results {
170 if compiled != nil && len(compiled.NetworkRules) > 0 {
171 hasContent = true
172 merged.Merge(compiled)
173 // Apply incrementally for faster initial blocking
174 logging.Info(fmt.Sprintf("[filtering] Applying incremental filters: %d network rules", len(merged.NetworkRules)))
175 fm.applyFilters(merged)
176 }
177 }
178
179 // Only save/apply final result if we got actual content
180 if hasContent {
181 logging.Info(fmt.Sprintf("[filtering] Successfully compiled %d network rules from sources", len(merged.NetworkRules)))
182
183 // Cache the final result
184 if err := fm.store.SaveCache(merged); err != nil {
185 logging.Error(fmt.Sprintf("Failed to save filter cache: %v", err))
186 }
187 } else {
188 logging.Warn("[filtering] No network rules were successfully compiled from any source - keeping existing filters if any")
189 }
190}
191
192func (fm *FilterManager) downloadAndCompile(ctx context.Context, url string) (*CompiledFilters, error) {
193 // Create HTTP client with timeout
194 client := &http.Client{
195 Timeout: httpTimeoutSeconds * time.Second,
196 }
197
198 // Create request with context
199 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
200 if err != nil {
201 return nil, fmt.Errorf("failed to create request for %s: %w", url, err)
202 }
203
204 // Set User-Agent to identify as browser
205 req.Header.Set("User-Agent", "Dumber Browser/1.0")
206
207 // Download filter list
208 resp, err := client.Do(req)
209 if err != nil {
210 return nil, fmt.Errorf("failed to download %s: %w", url, ErrNetworkError)
211 }
212 defer func() {
213 if closeErr := resp.Body.Close(); closeErr != nil {
214 logging.Error(fmt.Sprintf("Failed to close response body for %s: %v", url, closeErr))
215 }
216 }()
217
218 if resp.StatusCode != http.StatusOK {
219 return nil, fmt.Errorf("HTTP %d downloading %s", resp.StatusCode, url)
220 }
221
222 // Create filter converter
223 filterConverter := converter.NewFilterConverter()
224 compiled := NewCompiledFilters()
225
226 // Parse line by line for memory efficiency
227 scanner := bufio.NewScanner(resp.Body)
228 lineCount := 0
229
230 for scanner.Scan() {
231 line := scanner.Text()
232 lineCount++
233
234 // Check context cancellation periodically
235 if lineCount%contextCheckInterval == 0 {
236 select {
237 case <-ctx.Done():
238 return nil, ctx.Err()
239 default:
240 }
241 }
242
243 // Convert each line
244 if err := filterConverter.ConvertEasyListLine(line); err != nil {
245 logging.Debug(fmt.Sprintf("Failed to convert filter line: %s, error: %v", line, err))
246 continue
247 }
248 }
249
250 if err := scanner.Err(); err != nil {
251 return nil, fmt.Errorf("error reading filter list %s: %w", url, err)
252 }
253
254 // Get compiled rules from converter
255 compiled.NetworkRules = filterConverter.GetNetworkRules()
256 compiled.CosmeticRules = filterConverter.GetCosmeticRules()
257 compiled.GenericHiding = filterConverter.GetGenericHiding()
258 compiled.CompiledAt = time.Now()
259 compiled.Version = fmt.Sprintf("downloaded-%d", time.Now().Unix())
260
261 // Log parsing results
262 logging.Info(fmt.Sprintf("[filtering] Parsed %s: %d lines, %d network rules, %d cosmetic domains, %d generic hiding rules",
263 url, lineCount, len(compiled.NetworkRules), len(compiled.CosmeticRules), len(compiled.GenericHiding)))
264
265 return compiled, nil
266}
267
268func (fm *FilterManager) applyFilters(filters *CompiledFilters) {
269 fm.compileMutex.Lock()
270 defer fm.compileMutex.Unlock()
271
272 // Add uBlock Origin style scriptlets to handle broken page loading
273
274 // Instead of blocking completely, add exception rules for critical ad scripts that break page loading
275 // This allows the page to load while neutralizing tracking
276 antiBreakageRules := []converter.WebKitRule{
277 {
278 Trigger: converter.Trigger{
279 URLFilter: "googlesyndication.com",
280 ResourceType: []string{converter.ResourceTypeScript},
281 },
282 Action: converter.Action{
283 Type: converter.ActionTypeBlock,
284 },
285 },
286 {
287 Trigger: converter.Trigger{
288 URLFilter: "doubleclick.net",
289 },
290 Action: converter.Action{
291 Type: converter.ActionTypeBlock,
292 },
293 },
294 {
295 Trigger: converter.Trigger{
296 URLFilter: "googletagservices.com/tag/js/gpt.js",
297 ResourceType: []string{converter.ResourceTypeScript},
298 },
299 Action: converter.Action{
300 Type: converter.ActionTypeBlock,
301 },
302 },
303 }
304 filters.NetworkRules = append(filters.NetworkRules, antiBreakageRules...)
305
306 // Add generic cosmetic rules for common ad loading indicators and infinite loaders
307 antiLoaderSelectors := []string{
308 // Generic ad loader patterns (work across all sites)
309 ".ad-loading", ".ad-placeholder", ".loading-ad", ".ad-spinner",
310 "[class*='loading'][class*='ad']", "[id*='loading'][id*='ad']",
311 ".spinner[class*='ad']", "[data-loading*='ad']",
312 // Generic infinite loader patterns
313 ".infinite-loader", ".loading-infinite", "[data-loading='infinite']",
314 ".loader[style*='infinite']", ".spinner[data-infinite='true']",
315 // Generic advertising loader patterns (multi-language)
316 ".advertisement-loading", ".ads-loading", ".publicity-loading",
317 }
318
319 // Add generic hiding rules for loader prevention
320 filters.GenericHiding = append(filters.GenericHiding, antiLoaderSelectors...)
321
322 logging.Info(fmt.Sprintf("[filtering] Applying filters: %d network rules (including test rule), %d cosmetic domains, %d generic hiding rules",
323 len(filters.NetworkRules), len(filters.CosmeticRules), len(filters.GenericHiding)))
324
325 fm.compiled = filters
326
327 // Inject cosmetic rules
328 if err := fm.cosmeticInjector.InjectRules(filters.CosmeticRules); err != nil {
329 logging.Error(fmt.Sprintf("Failed to inject cosmetic rules: %v", err))
330 }
331
332 // Notify that filters are ready
333 if fm.onFiltersReady != nil {
334 go fm.onFiltersReady()
335 }
336}
337
338func (fm *FilterManager) startUpdateLoop(ctx context.Context) {
339 ticker := time.NewTicker(fm.updateInterval)
340 defer ticker.Stop()
341
342 for {
343 select {
344 case <-ctx.Done():
345 return
346 case <-ticker.C:
347 // Check if cache needs update and schedule background update
348 if exists, lastModified, _ := fm.store.GetCacheInfo(); exists {
349 if time.Since(lastModified) > fm.updateInterval {
350 logging.Info("[filtering] Periodic update triggered")
351 go fm.updateFiltersInBackground(ctx)
352 }
353 }
354 }
355 }
356}
357
358// updateFiltersInBackground updates filters without clearing existing ones
359func (fm *FilterManager) updateFiltersInBackground(ctx context.Context) {
360 logging.Info("[filtering] Starting background filter update")
361
362 // Compile new filters
363 sources := []string{
364 "https://easylist.to/easylist/easylist.txt",
365 "https://easylist.to/easylist/easyprivacy.txt",
366 "https://easylist-downloads.adblockplus.org/liste_fr.txt", // For French sites like lesnumeriques.com
367 }
368
369 var wg sync.WaitGroup
370 results := make(chan *CompiledFilters, len(sources))
371
372 for _, url := range sources {
373 wg.Add(1)
374 go func(u string) {
375 defer wg.Done()
376 if filters, err := fm.downloadAndCompile(ctx, u); err == nil {
377 results <- filters
378 } else {
379 logging.Warn(fmt.Sprintf("[filtering] Failed to update from %s: %v", u, err))
380 }
381 }(url)
382 }
383
384 // Wait for all compilations
385 go func() {
386 wg.Wait()
387 close(results)
388 }()
389
390 // Merge results
391 merged := NewCompiledFilters()
392 hasContent := false
393
394 for compiled := range results {
395 if compiled != nil && len(compiled.NetworkRules) > 0 {
396 hasContent = true
397 merged.Merge(compiled)
398 }
399 }
400
401 // Only apply and save if we got actual content
402 if hasContent {
403 logging.Info(fmt.Sprintf("[filtering] Background update succeeded with %d network rules", len(merged.NetworkRules)))
404 fm.applyFilters(merged)
405
406 if err := fm.store.SaveCache(merged); err != nil {
407 logging.Error(fmt.Sprintf("Failed to save updated filter cache: %v", err))
408 }
409 } else {
410 logging.Warn("[filtering] Background update produced no filters, keeping existing filters")
411 }
412}
413
414// GetNetworkFilters returns the compiled network filters for WebKit
415func (fm *FilterManager) GetNetworkFilters() ([]byte, error) {
416 fm.compileMutex.RLock()
417 defer fm.compileMutex.RUnlock()
418
419 if fm.compiled == nil {
420 return nil, ErrFiltersNotReady
421 }
422
423 json, err := fm.compiled.ToJSON()
424 if err != nil {
425 return nil, err
426 }
427
428 return json, nil
429}
430
431// GetCosmeticRulesForDomain returns cosmetic rules for a specific domain
432func (fm *FilterManager) GetCosmeticRulesForDomain(domain string) []string {
433 fm.compileMutex.RLock()
434 defer fm.compileMutex.RUnlock()
435
436 if fm.compiled == nil {
437 return nil
438 }
439
440 return fm.compiled.GetRulesForDomain(domain)
441}
442
443// GetCosmeticScript returns the cosmetic filtering script
444func (fm *FilterManager) GetCosmeticScript() string {
445 return fm.cosmeticInjector.GetScriptForDomain("")
446}
447
448// GetCosmeticScriptForDomain returns the cosmetic filtering script for a specific domain
449func (fm *FilterManager) GetCosmeticScriptForDomain(domain string) string {
450 return fm.cosmeticInjector.GetScriptForDomain(domain)
451}
452
453// UpdateCosmeticRules adds new cosmetic rules for a domain
454func (fm *FilterManager) UpdateCosmeticRules(domain string, selectors []string) {
455 fm.cosmeticInjector.UpdateRulesForDomain(domain, selectors)
456}
457
458// GetCosmeticUpdateScript returns JavaScript to update cosmetic rules dynamically
459func (fm *FilterManager) GetCosmeticUpdateScript(selectors []string) string {
460 return fm.cosmeticInjector.GetUpdateScript(selectors)
461}
462
463// GetCosmeticCleanupScript returns JavaScript to cleanup cosmetic filtering
464func (fm *FilterManager) GetCosmeticCleanupScript() string {
465 return fm.cosmeticInjector.GetCleanupScript()
466}
467
468// NewCompiledFilters creates a new CompiledFilters instance
469func NewCompiledFilters() *CompiledFilters {
470 return &CompiledFilters{
471 NetworkRules: make([]converter.WebKitRule, 0),
472 CosmeticRules: make(map[string][]string),
473 GenericHiding: make([]string, 0),
474 CompiledAt: time.Now(),
475 }
476}
477
478// Merge combines two CompiledFilters instances
479func (cf *CompiledFilters) Merge(other *CompiledFilters) {
480 cf.mu.Lock()
481 defer cf.mu.Unlock()
482
483 if other == nil {
484 return
485 }
486
487 other.mu.RLock()
488 defer other.mu.RUnlock()
489
490 // Merge network rules
491 cf.NetworkRules = append(cf.NetworkRules, other.NetworkRules...)
492
493 // Merge cosmetic rules
494 for domain, rules := range other.CosmeticRules {
495 cf.CosmeticRules[domain] = append(cf.CosmeticRules[domain], rules...)
496 }
497
498 // Merge generic hiding rules
499 cf.GenericHiding = append(cf.GenericHiding, other.GenericHiding...)
500}
501
502// GetRulesForDomain returns cosmetic rules for a specific domain
503func (cf *CompiledFilters) GetRulesForDomain(domain string) []string {
504 cf.mu.RLock()
505 defer cf.mu.RUnlock()
506
507 rules := make([]string, 0)
508
509 // Add generic hiding rules
510 rules = append(rules, cf.GenericHiding...)
511
512 // Add domain-specific rules
513 if domainRules, exists := cf.CosmeticRules[domain]; exists {
514 rules = append(rules, domainRules...)
515 }
516
517 return rules
518}
519
520// ToJSON converts network rules to JSON format
521func (cf *CompiledFilters) ToJSON() ([]byte, error) {
522 cf.mu.RLock()
523 defer cf.mu.RUnlock()
524
525 return json.Marshal(cf.NetworkRules)
526}