1// Package filtering provides filter management and compilation for content blocking.
2package filtering
3
4import (
5 "bufio"
6 "context"
7 "encoding/json"
8 "fmt"
9 "net/http"
10 "strings"
11 "sync"
12 "time"
13
14 "github.com/bnema/dumber/internal/config"
15 "github.com/bnema/dumber/internal/filtering/converter"
16 "github.com/bnema/dumber/internal/filtering/cosmetic"
17 "github.com/bnema/dumber/internal/logging"
18)
19
20const (
21 defaultUpdateHours = 24 // Default update interval in hours
22 httpTimeoutSeconds = 30 // HTTP client timeout in seconds
23 contextCheckInterval = 1000 // Check context cancellation every N lines
24)
25
26// FilterReadyCallback is called when filters are loaded/updated
27type FilterReadyCallback func()
28
29// FilterManager manages filter loading, compilation, and updates
30type FilterManager struct {
31 store FilterStore
32 compiler FilterCompiler
33 cosmeticInjector *cosmetic.CosmeticInjector
34 updater *FilterUpdater
35 updateInterval time.Duration
36 compileMutex sync.RWMutex
37 compiled *CompiledFilters
38 onFiltersReady FilterReadyCallback
39 // Cached whitelist rules to avoid recomputing on every filter application
40 cachedWhitelistRules []converter.WebKitRule
41 cachedWhitelistHash string
42}
43
44// FilterStore defines the interface for filter storage operations
45type FilterStore interface {
46 LoadCached() (*CompiledFilters, error)
47 SaveCache(filters *CompiledFilters) error
48 GetCacheInfo() (bool, time.Time, error)
49 GetSourceVersion(url string) string
50 SetSourceVersion(url string, version string) error
51 GetLastCheckTime() time.Time
52}
53
54// FilterCompiler defines the interface for filter compilation
55type FilterCompiler interface {
56 CompileFromSources(ctx context.Context, sources []string) (*CompiledFilters, error)
57 CompileFromData(data []byte) (*CompiledFilters, error)
58}
59
60// CosmeticInjector defines the interface for cosmetic filter injection
61type CosmeticInjector interface {
62 InjectRules(rules map[string][]string) error
63 GetScriptForDomain(domain string) string
64}
65
66// CompiledFilters represents the compiled filter data
67type CompiledFilters struct {
68 NetworkRules []converter.WebKitRule `json:"network_rules"`
69 CosmeticRules map[string][]string `json:"cosmetic_rules"`
70 GenericHiding []string `json:"generic_hiding"`
71 CompiledAt time.Time `json:"compiled_at"`
72 Version string `json:"version"`
73 mu sync.RWMutex
74}
75
76// NewFilterManager creates a new filter manager
77func NewFilterManager(store FilterStore, compiler FilterCompiler) *FilterManager {
78 fm := &FilterManager{
79 store: store,
80 compiler: compiler,
81 cosmeticInjector: cosmetic.NewCosmeticInjector(),
82 updateInterval: defaultUpdateHours * time.Hour, // Default update interval
83 compiled: &CompiledFilters{},
84 onFiltersReady: nil,
85 }
86 // Initialize updater with reference to manager
87 fm.updater = NewFilterUpdater(fm)
88 return fm
89}
90
91// updateWhitelistCache rebuilds the cached whitelist rules from config if needed
92func (fm *FilterManager) updateWhitelistCache(whitelist []string) {
93 // Create a hash of the whitelist to detect changes
94 currentHash := strings.Join(whitelist, "|")
95
96 // Only rebuild if whitelist changed
97 if fm.cachedWhitelistHash == currentHash && fm.cachedWhitelistRules != nil {
98 return
99 }
100
101 // Rebuild cache with pre-allocation
102 rules := make([]converter.WebKitRule, 0, len(whitelist))
103 for _, domain := range whitelist {
104 // Create a regex pattern that matches all URLs containing this domain
105 // Escape special regex characters in the domain
106 escapedDomain := strings.ReplaceAll(domain, ".", "\\.")
107 // Match any URL containing this domain (with protocol and optional path)
108 urlPattern := ".*" + escapedDomain
109
110 rules = append(rules, converter.WebKitRule{
111 Trigger: converter.Trigger{
112 URLFilter: urlPattern,
113 },
114 Action: converter.Action{
115 Type: converter.ActionTypeIgnorePreviousRules,
116 },
117 })
118 logging.Info(fmt.Sprintf("[filtering] Created whitelist rule: %s → pattern: %s", domain, urlPattern))
119 }
120
121 fm.cachedWhitelistRules = rules
122 fm.cachedWhitelistHash = currentHash
123 logging.Info(fmt.Sprintf("[filtering] Whitelist cache updated with %d domains", len(whitelist)))
124}
125
126// SetFiltersReadyCallback sets a callback to be called when filters are loaded/updated
127func (fm *FilterManager) SetFiltersReadyCallback(callback FilterReadyCallback) {
128 fm.onFiltersReady = callback
129}
130
131// InitAsync initializes filter loading in background
132func (fm *FilterManager) InitAsync(ctx context.Context) error {
133 // Load pre-compiled filters in background
134 go fm.loadCompiledFilters(ctx)
135
136 // Start periodic update checker
137 go fm.startUpdateLoop(ctx)
138
139 // Note: compileIfNeeded removed to prevent race condition
140 // Cache age checking is now handled in loadCompiledFilters
141
142 return nil
143}
144
145func (fm *FilterManager) loadCompiledFilters(ctx context.Context) {
146 // Try to load from cache first
147 if cached, err := fm.store.LoadCached(); err == nil {
148 logging.Info(fmt.Sprintf("[filtering] Successfully loaded cache with %d network rules, %d cosmetic domains",
149 len(cached.NetworkRules), len(cached.CosmeticRules)))
150 fm.applyFilters(cached)
151
152 // Check if cache needs update
153 if exists, lastModified, _ := fm.store.GetCacheInfo(); exists {
154 if time.Since(lastModified) > fm.updateInterval {
155 // Schedule background update without clearing current filters
156 logging.Info("[filtering] Cache is outdated, scheduling background update")
157 go fm.updateFiltersInBackground(ctx)
158 }
159 }
160 return
161 } else {
162 logging.Error(fmt.Sprintf("[filtering] Failed to load cache: %v", err))
163 }
164
165 // Only compile if no cache exists
166 logging.Info("[filtering] No cache found, compiling from sources")
167 fm.compileFromSources(ctx)
168}
169
170func (fm *FilterManager) compileIfNeeded(ctx context.Context) {
171 // Check if cache exists and is recent
172 exists, lastModified, err := fm.store.GetCacheInfo()
173 if err != nil {
174 fm.compileFromSources(ctx)
175 return
176 }
177
178 // If cache is older than update interval, recompile
179 if !exists || time.Since(lastModified) > fm.updateInterval {
180 fm.compileFromSources(ctx)
181 }
182}
183
184func (fm *FilterManager) compileFromSources(ctx context.Context) {
185 sources := config.Get().ContentFiltering.FilterLists
186 if len(sources) == 0 {
187 logging.Warn("[filtering] No filter lists configured")
188 return
189 }
190
191 var wg sync.WaitGroup
192 results := make(chan *CompiledFilters, len(sources))
193
194 for _, url := range sources {
195 wg.Add(1)
196 go func(u string) {
197 defer wg.Done()
198 if filters, err := fm.downloadAndCompile(ctx, u); err == nil {
199 results <- filters
200 } else {
201 logging.Error(fmt.Sprintf("[filtering] Failed to download/compile %s: %v", u, err))
202 }
203 }(url)
204 }
205
206 // Wait for all compilations
207 go func() {
208 wg.Wait()
209 close(results)
210 }()
211
212 // Merge results as they come in
213 merged := NewCompiledFilters()
214 hasContent := false
215
216 for compiled := range results {
217 if compiled != nil && len(compiled.NetworkRules) > 0 {
218 hasContent = true
219 merged.Merge(compiled)
220 // Apply incrementally for faster initial blocking
221 logging.Info(fmt.Sprintf("[filtering] Applying incremental filters: %d network rules", len(merged.NetworkRules)))
222 fm.applyFilters(merged)
223 }
224 }
225
226 // Only save/apply final result if we got actual content
227 if hasContent {
228 logging.Info(fmt.Sprintf("[filtering] Successfully compiled %d network rules from sources", len(merged.NetworkRules)))
229
230 // Cache the final result
231 if err := fm.store.SaveCache(merged); err != nil {
232 logging.Error(fmt.Sprintf("Failed to save filter cache: %v", err))
233 }
234 } else {
235 logging.Warn("[filtering] No network rules were successfully compiled from any source - keeping existing filters if any")
236 }
237}
238
239func (fm *FilterManager) downloadAndCompile(ctx context.Context, url string) (*CompiledFilters, error) {
240 // Create HTTP client with timeout
241 client := &http.Client{
242 Timeout: httpTimeoutSeconds * time.Second,
243 }
244
245 // Create request with context
246 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
247 if err != nil {
248 return nil, fmt.Errorf("failed to create request for %s: %w", url, err)
249 }
250
251 // Set User-Agent to identify as browser
252 req.Header.Set("User-Agent", "Dumber Browser/1.0")
253
254 // Download filter list
255 resp, err := client.Do(req)
256 if err != nil {
257 return nil, fmt.Errorf("failed to download %s: %w", url, ErrNetworkError)
258 }
259 defer func() {
260 if closeErr := resp.Body.Close(); closeErr != nil {
261 logging.Error(fmt.Sprintf("Failed to close response body for %s: %v", url, closeErr))
262 }
263 }()
264
265 if resp.StatusCode != http.StatusOK {
266 return nil, fmt.Errorf("HTTP %d downloading %s", resp.StatusCode, url)
267 }
268
269 // Create filter converter
270 filterConverter := converter.NewFilterConverter()
271 compiled := NewCompiledFilters()
272
273 // Parse line by line for memory efficiency
274 scanner := bufio.NewScanner(resp.Body)
275 lineCount := 0
276
277 for scanner.Scan() {
278 line := scanner.Text()
279 lineCount++
280
281 // Check context cancellation periodically
282 if lineCount%contextCheckInterval == 0 {
283 select {
284 case <-ctx.Done():
285 return nil, ctx.Err()
286 default:
287 }
288 }
289
290 // Convert each line
291 if err := filterConverter.ConvertEasyListLine(line); err != nil {
292 logging.Debug(fmt.Sprintf("Failed to convert filter line: %s, error: %v", line, err))
293 continue
294 }
295 }
296
297 if err := scanner.Err(); err != nil {
298 return nil, fmt.Errorf("error reading filter list %s: %w", url, err)
299 }
300
301 // Get compiled rules from converter
302 compiled.NetworkRules = filterConverter.GetNetworkRules()
303 compiled.CosmeticRules = filterConverter.GetCosmeticRules()
304 compiled.GenericHiding = filterConverter.GetGenericHiding()
305 compiled.CompiledAt = time.Now()
306 compiled.Version = fmt.Sprintf("downloaded-%d", time.Now().Unix())
307
308 // Log parsing results
309 logging.Info(fmt.Sprintf("[filtering] Parsed %s: %d lines, %d network rules, %d cosmetic domains, %d generic hiding rules",
310 url, lineCount, len(compiled.NetworkRules), len(compiled.CosmeticRules), len(compiled.GenericHiding)))
311
312 return compiled, nil
313}
314
315func (fm *FilterManager) applyFilters(filters *CompiledFilters) {
316 fm.compileMutex.Lock()
317 defer fm.compileMutex.Unlock()
318
319 // Add whitelist rules from config (ignore-previous-rules for whitelisted domains)
320 // Update cache if whitelist changed, then use cached rules
321 cfg := config.Get()
322
323 // Only apply filters if content filtering is enabled
324 if !cfg.ContentFiltering.Enabled {
325 logging.Info("[filtering] Content filtering is DISABLED in config - no ad blocking active")
326 return
327 }
328
329 fm.updateWhitelistCache(cfg.ContentFiltering.Whitelist)
330 if len(fm.cachedWhitelistRules) > 0 {
331 filters.NetworkRules = append(filters.NetworkRules, fm.cachedWhitelistRules...)
332 logging.Info(fmt.Sprintf("[filtering] Applied %d whitelist rules (ignore-previous-rules)", len(fm.cachedWhitelistRules)))
333 }
334
335 // Add uBlock Origin style scriptlets to handle broken page loading
336
337 // Instead of blocking completely, add exception rules for critical ad scripts that break page loading
338 // This allows the page to load while neutralizing tracking
339 antiBreakageRules := []converter.WebKitRule{
340 {
341 Trigger: converter.Trigger{
342 URLFilter: "googlesyndication.com",
343 ResourceType: []string{converter.ResourceTypeScript},
344 },
345 Action: converter.Action{
346 Type: converter.ActionTypeBlock,
347 },
348 },
349 {
350 Trigger: converter.Trigger{
351 URLFilter: "doubleclick.net",
352 },
353 Action: converter.Action{
354 Type: converter.ActionTypeBlock,
355 },
356 },
357 {
358 Trigger: converter.Trigger{
359 URLFilter: "googletagservices.com/tag/js/gpt.js",
360 ResourceType: []string{converter.ResourceTypeScript},
361 },
362 Action: converter.Action{
363 Type: converter.ActionTypeBlock,
364 },
365 },
366 }
367 filters.NetworkRules = append(filters.NetworkRules, antiBreakageRules...)
368
369 // Add generic cosmetic rules for common ad loading indicators and infinite loaders
370 antiLoaderSelectors := []string{
371 // Generic ad loader patterns (work across all sites)
372 ".ad-loading", ".ad-placeholder", ".loading-ad", ".ad-spinner",
373 "[class*='loading'][class*='ad']", "[id*='loading'][id*='ad']",
374 ".spinner[class*='ad']", "[data-loading*='ad']",
375 // Generic infinite loader patterns
376 ".infinite-loader", ".loading-infinite", "[data-loading='infinite']",
377 ".loader[style*='infinite']", ".spinner[data-infinite='true']",
378 // Generic advertising loader patterns (multi-language)
379 ".advertisement-loading", ".ads-loading", ".publicity-loading",
380 }
381
382 // Add generic hiding rules for loader prevention
383 filters.GenericHiding = append(filters.GenericHiding, antiLoaderSelectors...)
384
385 logging.Info(fmt.Sprintf("[filtering] Applying filters: %d network rules (including test rule), %d cosmetic domains, %d generic hiding rules",
386 len(filters.NetworkRules), len(filters.CosmeticRules), len(filters.GenericHiding)))
387
388 fm.compiled = filters
389
390 // Inject cosmetic rules (domain-specific + generic hiding under empty key)
391 cosmeticRulesWithGeneric := make(map[string][]string)
392 for domain, selectors := range filters.CosmeticRules {
393 cosmeticRulesWithGeneric[domain] = selectors
394 }
395 // Add generic hiding rules under empty string key (applies to all domains)
396 if len(filters.GenericHiding) > 0 {
397 cosmeticRulesWithGeneric[""] = filters.GenericHiding
398 logging.Info(fmt.Sprintf("[filtering] Adding %d generic hiding rules to cosmetic injector", len(filters.GenericHiding)))
399 }
400 logging.Info(fmt.Sprintf("[filtering] Injecting cosmetic rules: %d domains + %d generic rules",
401 len(filters.CosmeticRules), len(filters.GenericHiding)))
402 if err := fm.cosmeticInjector.InjectRules(cosmeticRulesWithGeneric); err != nil {
403 logging.Error(fmt.Sprintf("Failed to inject cosmetic rules: %v", err))
404 }
405
406 // Notify that filters are ready
407 if fm.onFiltersReady != nil {
408 go fm.onFiltersReady()
409 }
410}
411
412func (fm *FilterManager) startUpdateLoop(ctx context.Context) {
413 // Wait 2 minutes after startup before first version check
414 // This ensures browser startup is not impacted
415 startupDelay := 2 * time.Minute
416 logging.Debug("[filtering] Waiting 2 minutes before first filter update check")
417
418 select {
419 case <-time.After(startupDelay):
420 // Startup delay complete, proceed with first check
421 case <-ctx.Done():
422 return
423 }
424
425 // Perform initial version check
426 logging.Info("[filtering] Performing initial filter version check")
427 if fm.updater != nil {
428 go fm.updater.CheckAndUpdate(ctx)
429 }
430
431 // Then check periodically (every 24 hours)
432 ticker := time.NewTicker(fm.updateInterval)
433 defer ticker.Stop()
434
435 for {
436 select {
437 case <-ctx.Done():
438 return
439 case <-ticker.C:
440 logging.Info("[filtering] Periodic filter version check")
441 if fm.updater != nil {
442 go fm.updater.CheckAndUpdate(ctx)
443 }
444 }
445 }
446}
447
448// updateFiltersInBackground updates filters without clearing existing ones
449func (fm *FilterManager) updateFiltersInBackground(ctx context.Context) {
450 logging.Info("[filtering] Starting background filter update")
451
452 sources := config.Get().ContentFiltering.FilterLists
453 if len(sources) == 0 {
454 logging.Warn("[filtering] No filter lists configured")
455 return
456 }
457
458 var wg sync.WaitGroup
459 results := make(chan *CompiledFilters, len(sources))
460
461 for _, url := range sources {
462 wg.Add(1)
463 go func(u string) {
464 defer wg.Done()
465 if filters, err := fm.downloadAndCompile(ctx, u); err == nil {
466 results <- filters
467 } else {
468 logging.Warn(fmt.Sprintf("[filtering] Failed to update from %s: %v", u, err))
469 }
470 }(url)
471 }
472
473 // Wait for all compilations
474 go func() {
475 wg.Wait()
476 close(results)
477 }()
478
479 // Merge results
480 merged := NewCompiledFilters()
481 hasContent := false
482
483 for compiled := range results {
484 if compiled != nil && len(compiled.NetworkRules) > 0 {
485 hasContent = true
486 merged.Merge(compiled)
487 }
488 }
489
490 // Only apply and save if we got actual content
491 if hasContent {
492 logging.Info(fmt.Sprintf("[filtering] Background update succeeded with %d network rules", len(merged.NetworkRules)))
493 fm.applyFilters(merged)
494
495 if err := fm.store.SaveCache(merged); err != nil {
496 logging.Error(fmt.Sprintf("Failed to save updated filter cache: %v", err))
497 }
498 } else {
499 logging.Warn("[filtering] Background update produced no filters, keeping existing filters")
500 }
501}
502
503// GetNetworkFilters returns the compiled network filters for WebKit
504func (fm *FilterManager) GetNetworkFilters() ([]byte, error) {
505 fm.compileMutex.RLock()
506 defer fm.compileMutex.RUnlock()
507
508 if fm.compiled == nil {
509 return nil, ErrFiltersNotReady
510 }
511
512 json, err := fm.compiled.ToJSON()
513 if err != nil {
514 return nil, err
515 }
516
517 return json, nil
518}
519
520// GetCosmeticRulesForDomain returns cosmetic rules for a specific domain
521func (fm *FilterManager) GetCosmeticRulesForDomain(domain string) []string {
522 fm.compileMutex.RLock()
523 defer fm.compileMutex.RUnlock()
524
525 if fm.compiled == nil {
526 return nil
527 }
528
529 return fm.compiled.GetRulesForDomain(domain)
530}
531
532// GetCosmeticScript returns the cosmetic filtering script
533func (fm *FilterManager) GetCosmeticScript() string {
534 return fm.cosmeticInjector.GetScriptForDomain("")
535}
536
537// GetCosmeticScriptForDomain returns the cosmetic filtering script for a specific domain
538func (fm *FilterManager) GetCosmeticScriptForDomain(domain string) string {
539 return fm.cosmeticInjector.GetScriptForDomain(domain)
540}
541
542// UpdateCosmeticRules adds new cosmetic rules for a domain
543func (fm *FilterManager) UpdateCosmeticRules(domain string, selectors []string) {
544 fm.cosmeticInjector.UpdateRulesForDomain(domain, selectors)
545}
546
547// GetCosmeticUpdateScript returns JavaScript to update cosmetic rules dynamically
548func (fm *FilterManager) GetCosmeticUpdateScript(selectors []string) string {
549 return fm.cosmeticInjector.GetUpdateScript(selectors)
550}
551
552// GetCosmeticCleanupScript returns JavaScript to cleanup cosmetic filtering
553func (fm *FilterManager) GetCosmeticCleanupScript() string {
554 return fm.cosmeticInjector.GetCleanupScript()
555}
556
557// NewCompiledFilters creates a new CompiledFilters instance
558func NewCompiledFilters() *CompiledFilters {
559 return &CompiledFilters{
560 NetworkRules: make([]converter.WebKitRule, 0),
561 CosmeticRules: make(map[string][]string),
562 GenericHiding: make([]string, 0),
563 CompiledAt: time.Now(),
564 }
565}
566
567// Merge combines two CompiledFilters instances
568func (cf *CompiledFilters) Merge(other *CompiledFilters) {
569 cf.mu.Lock()
570 defer cf.mu.Unlock()
571
572 if other == nil {
573 return
574 }
575
576 other.mu.RLock()
577 defer other.mu.RUnlock()
578
579 // Merge network rules
580 cf.NetworkRules = append(cf.NetworkRules, other.NetworkRules...)
581
582 // Merge cosmetic rules
583 for domain, rules := range other.CosmeticRules {
584 cf.CosmeticRules[domain] = append(cf.CosmeticRules[domain], rules...)
585 }
586
587 // Merge generic hiding rules
588 cf.GenericHiding = append(cf.GenericHiding, other.GenericHiding...)
589}
590
591// GetRulesForDomain returns cosmetic rules for a specific domain
592func (cf *CompiledFilters) GetRulesForDomain(domain string) []string {
593 cf.mu.RLock()
594 defer cf.mu.RUnlock()
595
596 rules := make([]string, 0)
597
598 // Add generic hiding rules
599 rules = append(rules, cf.GenericHiding...)
600
601 // Add domain-specific rules
602 if domainRules, exists := cf.CosmeticRules[domain]; exists {
603 rules = append(rules, domainRules...)
604 }
605
606 return rules
607}
608
609// ToJSON converts network rules to JSON format
610func (cf *CompiledFilters) ToJSON() ([]byte, error) {
611 cf.mu.RLock()
612 defer cf.mu.RUnlock()
613
614 return json.Marshal(cf.NetworkRules)
615}