1// Package filtering provides filter management and compilation for content blocking.
2package filtering
3
4import (
5 "bufio"
6 "context"
7 "encoding/json"
8 "fmt"
9 "net/http"
10 "strings"
11 "sync"
12 "time"
13
14 "github.com/bnema/dumber/internal/config"
15 "github.com/bnema/dumber/internal/filtering/converter"
16 "github.com/bnema/dumber/internal/filtering/cosmetic"
17 "github.com/bnema/dumber/internal/logging"
18)
19
20const (
21 defaultUpdateHours = 24 // Default update interval in hours
22 httpTimeoutSeconds = 30 // HTTP client timeout in seconds
23 contextCheckInterval = 1000 // Check context cancellation every N lines
24)
25
26// FilterReadyCallback is called when filters are loaded/updated
27type FilterReadyCallback func()
28
29// FilterManager manages filter loading, compilation, and updates
30type FilterManager struct {
31 store FilterStore
32 compiler FilterCompiler
33 cosmeticInjector *cosmetic.CosmeticInjector
34 updateInterval time.Duration
35 compileMutex sync.RWMutex
36 compiled *CompiledFilters
37 onFiltersReady FilterReadyCallback
38 // Cached whitelist rules to avoid recomputing on every filter application
39 cachedWhitelistRules []converter.WebKitRule
40 cachedWhitelistHash string
41}
42
43// FilterStore defines the interface for filter storage operations
44type FilterStore interface {
45 LoadCached() (*CompiledFilters, error)
46 SaveCache(filters *CompiledFilters) error
47 GetCacheInfo() (bool, time.Time, error)
48}
49
50// FilterCompiler defines the interface for filter compilation
51type FilterCompiler interface {
52 CompileFromSources(ctx context.Context, sources []string) (*CompiledFilters, error)
53 CompileFromData(data []byte) (*CompiledFilters, error)
54}
55
56// CosmeticInjector defines the interface for cosmetic filter injection
57type CosmeticInjector interface {
58 InjectRules(rules map[string][]string) error
59 GetScriptForDomain(domain string) string
60}
61
62// CompiledFilters represents the compiled filter data
63type CompiledFilters struct {
64 NetworkRules []converter.WebKitRule `json:"network_rules"`
65 CosmeticRules map[string][]string `json:"cosmetic_rules"`
66 GenericHiding []string `json:"generic_hiding"`
67 CompiledAt time.Time `json:"compiled_at"`
68 Version string `json:"version"`
69 mu sync.RWMutex
70}
71
72// NewFilterManager creates a new filter manager
73func NewFilterManager(store FilterStore, compiler FilterCompiler) *FilterManager {
74 return &FilterManager{
75 store: store,
76 compiler: compiler,
77 cosmeticInjector: cosmetic.NewCosmeticInjector(),
78 updateInterval: defaultUpdateHours * time.Hour, // Default update interval
79 compiled: &CompiledFilters{},
80 onFiltersReady: nil,
81 }
82}
83
84// updateWhitelistCache rebuilds the cached whitelist rules from config if needed
85func (fm *FilterManager) updateWhitelistCache(whitelist []string) {
86 // Create a hash of the whitelist to detect changes
87 currentHash := strings.Join(whitelist, "|")
88
89 // Only rebuild if whitelist changed
90 if fm.cachedWhitelistHash == currentHash && fm.cachedWhitelistRules != nil {
91 return
92 }
93
94 // Rebuild cache with pre-allocation
95 rules := make([]converter.WebKitRule, 0, len(whitelist))
96 for _, domain := range whitelist {
97 // Create a regex pattern that matches all URLs containing this domain
98 // Escape special regex characters in the domain
99 escapedDomain := strings.ReplaceAll(domain, ".", "\\.")
100 // Match any URL containing this domain (with protocol and optional path)
101 urlPattern := ".*" + escapedDomain
102
103 rules = append(rules, converter.WebKitRule{
104 Trigger: converter.Trigger{
105 URLFilter: urlPattern,
106 },
107 Action: converter.Action{
108 Type: converter.ActionTypeIgnorePreviousRules,
109 },
110 })
111 logging.Info(fmt.Sprintf("[filtering] Created whitelist rule: %s → pattern: %s", domain, urlPattern))
112 }
113
114 fm.cachedWhitelistRules = rules
115 fm.cachedWhitelistHash = currentHash
116 logging.Info(fmt.Sprintf("[filtering] Whitelist cache updated with %d domains", len(whitelist)))
117}
118
119// SetFiltersReadyCallback sets a callback to be called when filters are loaded/updated
120func (fm *FilterManager) SetFiltersReadyCallback(callback FilterReadyCallback) {
121 fm.onFiltersReady = callback
122}
123
124// InitAsync initializes filter loading in background
125func (fm *FilterManager) InitAsync(ctx context.Context) error {
126 // Load pre-compiled filters in background
127 go fm.loadCompiledFilters(ctx)
128
129 // Start periodic update checker
130 go fm.startUpdateLoop(ctx)
131
132 // Note: compileIfNeeded removed to prevent race condition
133 // Cache age checking is now handled in loadCompiledFilters
134
135 return nil
136}
137
138func (fm *FilterManager) loadCompiledFilters(ctx context.Context) {
139 // Try to load from cache first
140 if cached, err := fm.store.LoadCached(); err == nil {
141 logging.Info(fmt.Sprintf("[filtering] Successfully loaded cache with %d network rules, %d cosmetic domains",
142 len(cached.NetworkRules), len(cached.CosmeticRules)))
143 fm.applyFilters(cached)
144
145 // Check if cache needs update
146 if exists, lastModified, _ := fm.store.GetCacheInfo(); exists {
147 if time.Since(lastModified) > fm.updateInterval {
148 // Schedule background update without clearing current filters
149 logging.Info("[filtering] Cache is outdated, scheduling background update")
150 go fm.updateFiltersInBackground(ctx)
151 }
152 }
153 return
154 } else {
155 logging.Error(fmt.Sprintf("[filtering] Failed to load cache: %v", err))
156 }
157
158 // Only compile if no cache exists
159 logging.Info("[filtering] No cache found, compiling from sources")
160 fm.compileFromSources(ctx)
161}
162
163func (fm *FilterManager) compileIfNeeded(ctx context.Context) {
164 // Check if cache exists and is recent
165 exists, lastModified, err := fm.store.GetCacheInfo()
166 if err != nil {
167 fm.compileFromSources(ctx)
168 return
169 }
170
171 // If cache is older than update interval, recompile
172 if !exists || time.Since(lastModified) > fm.updateInterval {
173 fm.compileFromSources(ctx)
174 }
175}
176
177func (fm *FilterManager) compileFromSources(ctx context.Context) {
178 sources := []string{
179 "https://easylist.to/easylist/easylist.txt",
180 "https://easylist.to/easylist/easyprivacy.txt",
181 "https://easylist-downloads.adblockplus.org/liste_fr.txt", // For French sites like lesnumeriques.com
182 }
183
184 var wg sync.WaitGroup
185 results := make(chan *CompiledFilters, len(sources))
186
187 for _, url := range sources {
188 wg.Add(1)
189 go func(u string) {
190 defer wg.Done()
191 if filters, err := fm.downloadAndCompile(ctx, u); err == nil {
192 results <- filters
193 } else {
194 logging.Error(fmt.Sprintf("[filtering] Failed to download/compile %s: %v", u, err))
195 }
196 }(url)
197 }
198
199 // Wait for all compilations
200 go func() {
201 wg.Wait()
202 close(results)
203 }()
204
205 // Merge results as they come in
206 merged := NewCompiledFilters()
207 hasContent := false
208
209 for compiled := range results {
210 if compiled != nil && len(compiled.NetworkRules) > 0 {
211 hasContent = true
212 merged.Merge(compiled)
213 // Apply incrementally for faster initial blocking
214 logging.Info(fmt.Sprintf("[filtering] Applying incremental filters: %d network rules", len(merged.NetworkRules)))
215 fm.applyFilters(merged)
216 }
217 }
218
219 // Only save/apply final result if we got actual content
220 if hasContent {
221 logging.Info(fmt.Sprintf("[filtering] Successfully compiled %d network rules from sources", len(merged.NetworkRules)))
222
223 // Cache the final result
224 if err := fm.store.SaveCache(merged); err != nil {
225 logging.Error(fmt.Sprintf("Failed to save filter cache: %v", err))
226 }
227 } else {
228 logging.Warn("[filtering] No network rules were successfully compiled from any source - keeping existing filters if any")
229 }
230}
231
232func (fm *FilterManager) downloadAndCompile(ctx context.Context, url string) (*CompiledFilters, error) {
233 // Create HTTP client with timeout
234 client := &http.Client{
235 Timeout: httpTimeoutSeconds * time.Second,
236 }
237
238 // Create request with context
239 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
240 if err != nil {
241 return nil, fmt.Errorf("failed to create request for %s: %w", url, err)
242 }
243
244 // Set User-Agent to identify as browser
245 req.Header.Set("User-Agent", "Dumber Browser/1.0")
246
247 // Download filter list
248 resp, err := client.Do(req)
249 if err != nil {
250 return nil, fmt.Errorf("failed to download %s: %w", url, ErrNetworkError)
251 }
252 defer func() {
253 if closeErr := resp.Body.Close(); closeErr != nil {
254 logging.Error(fmt.Sprintf("Failed to close response body for %s: %v", url, closeErr))
255 }
256 }()
257
258 if resp.StatusCode != http.StatusOK {
259 return nil, fmt.Errorf("HTTP %d downloading %s", resp.StatusCode, url)
260 }
261
262 // Create filter converter
263 filterConverter := converter.NewFilterConverter()
264 compiled := NewCompiledFilters()
265
266 // Parse line by line for memory efficiency
267 scanner := bufio.NewScanner(resp.Body)
268 lineCount := 0
269
270 for scanner.Scan() {
271 line := scanner.Text()
272 lineCount++
273
274 // Check context cancellation periodically
275 if lineCount%contextCheckInterval == 0 {
276 select {
277 case <-ctx.Done():
278 return nil, ctx.Err()
279 default:
280 }
281 }
282
283 // Convert each line
284 if err := filterConverter.ConvertEasyListLine(line); err != nil {
285 logging.Debug(fmt.Sprintf("Failed to convert filter line: %s, error: %v", line, err))
286 continue
287 }
288 }
289
290 if err := scanner.Err(); err != nil {
291 return nil, fmt.Errorf("error reading filter list %s: %w", url, err)
292 }
293
294 // Get compiled rules from converter
295 compiled.NetworkRules = filterConverter.GetNetworkRules()
296 compiled.CosmeticRules = filterConverter.GetCosmeticRules()
297 compiled.GenericHiding = filterConverter.GetGenericHiding()
298 compiled.CompiledAt = time.Now()
299 compiled.Version = fmt.Sprintf("downloaded-%d", time.Now().Unix())
300
301 // Log parsing results
302 logging.Info(fmt.Sprintf("[filtering] Parsed %s: %d lines, %d network rules, %d cosmetic domains, %d generic hiding rules",
303 url, lineCount, len(compiled.NetworkRules), len(compiled.CosmeticRules), len(compiled.GenericHiding)))
304
305 return compiled, nil
306}
307
308func (fm *FilterManager) applyFilters(filters *CompiledFilters) {
309 fm.compileMutex.Lock()
310 defer fm.compileMutex.Unlock()
311
312 // Add whitelist rules from config (ignore-previous-rules for whitelisted domains)
313 // Update cache if whitelist changed, then use cached rules
314 cfg := config.Get()
315
316 // Only apply filters if content filtering is enabled
317 if !cfg.ContentFiltering.Enabled {
318 logging.Info("[filtering] Content filtering is DISABLED in config - no ad blocking active")
319 return
320 }
321
322 fm.updateWhitelistCache(cfg.ContentFiltering.Whitelist)
323 if len(fm.cachedWhitelistRules) > 0 {
324 filters.NetworkRules = append(filters.NetworkRules, fm.cachedWhitelistRules...)
325 logging.Info(fmt.Sprintf("[filtering] Applied %d whitelist rules (ignore-previous-rules)", len(fm.cachedWhitelistRules)))
326 }
327
328 // Add uBlock Origin style scriptlets to handle broken page loading
329
330 // Instead of blocking completely, add exception rules for critical ad scripts that break page loading
331 // This allows the page to load while neutralizing tracking
332 antiBreakageRules := []converter.WebKitRule{
333 {
334 Trigger: converter.Trigger{
335 URLFilter: "googlesyndication.com",
336 ResourceType: []string{converter.ResourceTypeScript},
337 },
338 Action: converter.Action{
339 Type: converter.ActionTypeBlock,
340 },
341 },
342 {
343 Trigger: converter.Trigger{
344 URLFilter: "doubleclick.net",
345 },
346 Action: converter.Action{
347 Type: converter.ActionTypeBlock,
348 },
349 },
350 {
351 Trigger: converter.Trigger{
352 URLFilter: "googletagservices.com/tag/js/gpt.js",
353 ResourceType: []string{converter.ResourceTypeScript},
354 },
355 Action: converter.Action{
356 Type: converter.ActionTypeBlock,
357 },
358 },
359 }
360 filters.NetworkRules = append(filters.NetworkRules, antiBreakageRules...)
361
362 // Add generic cosmetic rules for common ad loading indicators and infinite loaders
363 antiLoaderSelectors := []string{
364 // Generic ad loader patterns (work across all sites)
365 ".ad-loading", ".ad-placeholder", ".loading-ad", ".ad-spinner",
366 "[class*='loading'][class*='ad']", "[id*='loading'][id*='ad']",
367 ".spinner[class*='ad']", "[data-loading*='ad']",
368 // Generic infinite loader patterns
369 ".infinite-loader", ".loading-infinite", "[data-loading='infinite']",
370 ".loader[style*='infinite']", ".spinner[data-infinite='true']",
371 // Generic advertising loader patterns (multi-language)
372 ".advertisement-loading", ".ads-loading", ".publicity-loading",
373 }
374
375 // Add generic hiding rules for loader prevention
376 filters.GenericHiding = append(filters.GenericHiding, antiLoaderSelectors...)
377
378 logging.Info(fmt.Sprintf("[filtering] Applying filters: %d network rules (including test rule), %d cosmetic domains, %d generic hiding rules",
379 len(filters.NetworkRules), len(filters.CosmeticRules), len(filters.GenericHiding)))
380
381 fm.compiled = filters
382
383 // Inject cosmetic rules
384 if err := fm.cosmeticInjector.InjectRules(filters.CosmeticRules); err != nil {
385 logging.Error(fmt.Sprintf("Failed to inject cosmetic rules: %v", err))
386 }
387
388 // Notify that filters are ready
389 if fm.onFiltersReady != nil {
390 go fm.onFiltersReady()
391 }
392}
393
394func (fm *FilterManager) startUpdateLoop(ctx context.Context) {
395 ticker := time.NewTicker(fm.updateInterval)
396 defer ticker.Stop()
397
398 for {
399 select {
400 case <-ctx.Done():
401 return
402 case <-ticker.C:
403 // Check if cache needs update and schedule background update
404 if exists, lastModified, _ := fm.store.GetCacheInfo(); exists {
405 if time.Since(lastModified) > fm.updateInterval {
406 logging.Info("[filtering] Periodic update triggered")
407 go fm.updateFiltersInBackground(ctx)
408 }
409 }
410 }
411 }
412}
413
414// updateFiltersInBackground updates filters without clearing existing ones
415func (fm *FilterManager) updateFiltersInBackground(ctx context.Context) {
416 logging.Info("[filtering] Starting background filter update")
417
418 // Compile new filters
419 sources := []string{
420 "https://easylist.to/easylist/easylist.txt",
421 "https://easylist.to/easylist/easyprivacy.txt",
422 "https://easylist-downloads.adblockplus.org/liste_fr.txt", // For French sites like lesnumeriques.com
423 }
424
425 var wg sync.WaitGroup
426 results := make(chan *CompiledFilters, len(sources))
427
428 for _, url := range sources {
429 wg.Add(1)
430 go func(u string) {
431 defer wg.Done()
432 if filters, err := fm.downloadAndCompile(ctx, u); err == nil {
433 results <- filters
434 } else {
435 logging.Warn(fmt.Sprintf("[filtering] Failed to update from %s: %v", u, err))
436 }
437 }(url)
438 }
439
440 // Wait for all compilations
441 go func() {
442 wg.Wait()
443 close(results)
444 }()
445
446 // Merge results
447 merged := NewCompiledFilters()
448 hasContent := false
449
450 for compiled := range results {
451 if compiled != nil && len(compiled.NetworkRules) > 0 {
452 hasContent = true
453 merged.Merge(compiled)
454 }
455 }
456
457 // Only apply and save if we got actual content
458 if hasContent {
459 logging.Info(fmt.Sprintf("[filtering] Background update succeeded with %d network rules", len(merged.NetworkRules)))
460 fm.applyFilters(merged)
461
462 if err := fm.store.SaveCache(merged); err != nil {
463 logging.Error(fmt.Sprintf("Failed to save updated filter cache: %v", err))
464 }
465 } else {
466 logging.Warn("[filtering] Background update produced no filters, keeping existing filters")
467 }
468}
469
470// GetNetworkFilters returns the compiled network filters for WebKit
471func (fm *FilterManager) GetNetworkFilters() ([]byte, error) {
472 fm.compileMutex.RLock()
473 defer fm.compileMutex.RUnlock()
474
475 if fm.compiled == nil {
476 return nil, ErrFiltersNotReady
477 }
478
479 json, err := fm.compiled.ToJSON()
480 if err != nil {
481 return nil, err
482 }
483
484 return json, nil
485}
486
487// GetCosmeticRulesForDomain returns cosmetic rules for a specific domain
488func (fm *FilterManager) GetCosmeticRulesForDomain(domain string) []string {
489 fm.compileMutex.RLock()
490 defer fm.compileMutex.RUnlock()
491
492 if fm.compiled == nil {
493 return nil
494 }
495
496 return fm.compiled.GetRulesForDomain(domain)
497}
498
499// GetCosmeticScript returns the cosmetic filtering script
500func (fm *FilterManager) GetCosmeticScript() string {
501 return fm.cosmeticInjector.GetScriptForDomain("")
502}
503
504// GetCosmeticScriptForDomain returns the cosmetic filtering script for a specific domain
505func (fm *FilterManager) GetCosmeticScriptForDomain(domain string) string {
506 return fm.cosmeticInjector.GetScriptForDomain(domain)
507}
508
509// UpdateCosmeticRules adds new cosmetic rules for a domain
510func (fm *FilterManager) UpdateCosmeticRules(domain string, selectors []string) {
511 fm.cosmeticInjector.UpdateRulesForDomain(domain, selectors)
512}
513
514// GetCosmeticUpdateScript returns JavaScript to update cosmetic rules dynamically
515func (fm *FilterManager) GetCosmeticUpdateScript(selectors []string) string {
516 return fm.cosmeticInjector.GetUpdateScript(selectors)
517}
518
519// GetCosmeticCleanupScript returns JavaScript to cleanup cosmetic filtering
520func (fm *FilterManager) GetCosmeticCleanupScript() string {
521 return fm.cosmeticInjector.GetCleanupScript()
522}
523
524// NewCompiledFilters creates a new CompiledFilters instance
525func NewCompiledFilters() *CompiledFilters {
526 return &CompiledFilters{
527 NetworkRules: make([]converter.WebKitRule, 0),
528 CosmeticRules: make(map[string][]string),
529 GenericHiding: make([]string, 0),
530 CompiledAt: time.Now(),
531 }
532}
533
534// Merge combines two CompiledFilters instances
535func (cf *CompiledFilters) Merge(other *CompiledFilters) {
536 cf.mu.Lock()
537 defer cf.mu.Unlock()
538
539 if other == nil {
540 return
541 }
542
543 other.mu.RLock()
544 defer other.mu.RUnlock()
545
546 // Merge network rules
547 cf.NetworkRules = append(cf.NetworkRules, other.NetworkRules...)
548
549 // Merge cosmetic rules
550 for domain, rules := range other.CosmeticRules {
551 cf.CosmeticRules[domain] = append(cf.CosmeticRules[domain], rules...)
552 }
553
554 // Merge generic hiding rules
555 cf.GenericHiding = append(cf.GenericHiding, other.GenericHiding...)
556}
557
558// GetRulesForDomain returns cosmetic rules for a specific domain
559func (cf *CompiledFilters) GetRulesForDomain(domain string) []string {
560 cf.mu.RLock()
561 defer cf.mu.RUnlock()
562
563 rules := make([]string, 0)
564
565 // Add generic hiding rules
566 rules = append(rules, cf.GenericHiding...)
567
568 // Add domain-specific rules
569 if domainRules, exists := cf.CosmeticRules[domain]; exists {
570 rules = append(rules, domainRules...)
571 }
572
573 return rules
574}
575
576// ToJSON converts network rules to JSON format
577func (cf *CompiledFilters) ToJSON() ([]byte, error) {
578 cf.mu.RLock()
579 defer cf.mu.RUnlock()
580
581 return json.Marshal(cf.NetworkRules)
582}