1package filtering
2
3import (
4 "context"
5 "crypto/sha256"
6 "fmt"
7 "net/http"
8 "strings"
9 "sync"
10 "time"
11
12 "github.com/bnema/dumber/internal/config"
13 "github.com/bnema/dumber/internal/logging"
14)
15
16const (
17 updateTimeoutSeconds = 60 // HTTP timeout for update checks
18 maxConcurrentUpdates = 4 // Maximum concurrent filter list downloads
19)
20
21// FilterUpdater manages differential filter list updates
22type FilterUpdater struct {
23 manager *FilterManager
24 httpClient *http.Client
25 diffEngine *DiffEngine
26 updateMutex sync.Mutex
27}
28
29// FilterUpdate represents an update to a filter list
30type FilterUpdate struct {
31 ListID string
32 URL string
33 Previous []byte
34 Current []byte
35 Hash string
36 Version string // Version extracted from content or HTTP headers
37 Compiled *CompiledFilters
38}
39
40// FilterListInfo stores metadata about a filter list
41type FilterListInfo struct {
42 URL string
43 LastModified time.Time
44 ETag string
45 ContentHash string
46 Size int64
47}
48
49// DiffEngine handles differential updates between filter lists
50type DiffEngine struct {
51 // We'll implement a simple line-based diff for filter lists
52}
53
54// NewFilterUpdater creates a new filter updater
55func NewFilterUpdater(manager *FilterManager) *FilterUpdater {
56 return &FilterUpdater{
57 manager: manager,
58 httpClient: &http.Client{
59 Timeout: updateTimeoutSeconds * time.Second,
60 },
61 diffEngine: &DiffEngine{},
62 }
63}
64
65// CheckAndUpdate checks for filter list updates and applies them
66func (fu *FilterUpdater) CheckAndUpdate(ctx context.Context) error {
67 fu.updateMutex.Lock()
68 defer fu.updateMutex.Unlock()
69
70 // Skip if we checked within the last 24 hours
71 lastCheck := fu.manager.store.GetLastCheckTime()
72 if !lastCheck.IsZero() && time.Since(lastCheck) < 24*time.Hour {
73 logging.Info(fmt.Sprintf("[filtering] Skipping update check, last check was %v ago", time.Since(lastCheck).Round(time.Minute)))
74 return nil
75 }
76
77 startTime := time.Now()
78 logging.Info("[filtering] Starting filter list update check...")
79
80 sources := config.Get().ContentFiltering.FilterLists
81 if len(sources) == 0 {
82 logging.Warn("[filtering] No filter lists configured")
83 return nil
84 }
85
86 // Check for updates concurrently
87 updates := make(chan *FilterUpdate, len(sources))
88 var wg sync.WaitGroup
89
90 // Limit concurrent updates
91 semaphore := make(chan struct{}, maxConcurrentUpdates)
92
93 for _, url := range sources {
94 wg.Add(1)
95 go func(u string) {
96 defer wg.Done()
97
98 // Acquire semaphore
99 semaphore <- struct{}{}
100 defer func() { <-semaphore }()
101
102 update, err := fu.checkSingleUpdate(ctx, u)
103 if err != nil {
104 logging.Error(fmt.Sprintf("Failed to check update for %s: %v", u, err))
105 return
106 }
107
108 if update != nil {
109 updates <- update
110 }
111 }(url)
112 }
113
114 // Wait for all checks to complete
115 go func() {
116 wg.Wait()
117 close(updates)
118 }()
119
120 // Apply updates as they come in
121 updateCount := 0
122 for update := range updates {
123 if err := fu.applyUpdate(update); err != nil {
124 logging.Error(fmt.Sprintf("Failed to apply update for %s: %v", update.URL, err))
125 continue
126 }
127 updateCount++
128 }
129
130 elapsed := time.Since(startTime)
131 if updateCount > 0 {
132 logging.Info(fmt.Sprintf("[filtering] Update complete: applied %d filter list updates in %v", updateCount, elapsed.Round(time.Millisecond)))
133 } else {
134 logging.Info(fmt.Sprintf("[filtering] Update complete: all filters up to date (checked in %v)", elapsed.Round(time.Millisecond)))
135 }
136
137 return nil
138}
139
140// checkSingleUpdate checks if a single filter list needs updating
141func (fu *FilterUpdater) checkSingleUpdate(ctx context.Context, url string) (*FilterUpdate, error) {
142 // Check version using lightweight Range request
143 if !fu.needsUpdate(ctx, url) {
144 return nil, nil // No update needed
145 }
146
147 // Download the full content
148 return fu.downloadUpdate(ctx, url, "", "")
149}
150
151// fetchRemoteVersion fetches version info using HEAD request (ETag or Last-Modified)
152func (fu *FilterUpdater) fetchRemoteVersion(ctx context.Context, url string) (string, error) {
153 req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
154 if err != nil {
155 return "", fmt.Errorf("failed to create request: %w", err)
156 }
157 req.Header.Set("User-Agent", "Dumber Browser/1.0 Filter Updater")
158
159 resp, err := fu.httpClient.Do(req)
160 if err != nil {
161 return "", fmt.Errorf("HEAD request failed: %w", err)
162 }
163 defer func() {
164 if closeErr := resp.Body.Close(); closeErr != nil {
165 logging.Warn(fmt.Sprintf("warning: failed to close response body: %v", closeErr))
166 }
167 }()
168
169 if resp.StatusCode != http.StatusOK {
170 return "", fmt.Errorf("HEAD request returned %d", resp.StatusCode)
171 }
172
173 // Use ETag or Last-Modified as version identifier
174 if etag := resp.Header.Get("ETag"); etag != "" {
175 return etag, nil
176 }
177 if lastMod := resp.Header.Get("Last-Modified"); lastMod != "" {
178 return lastMod, nil
179 }
180
181 return "", fmt.Errorf("no ETag or Last-Modified header")
182}
183
184// needsUpdate determines if a filter list needs updating based on version comparison
185func (fu *FilterUpdater) needsUpdate(ctx context.Context, url string) bool {
186 // Extract short name for logging
187 shortName := url[strings.LastIndex(url, "/")+1:]
188
189 remoteVersion, err := fu.fetchRemoteVersion(ctx, url)
190 if err != nil {
191 logging.Info(fmt.Sprintf("[filtering] %s: fetch failed, will update (%v)", shortName, err))
192 return true
193 }
194
195 storedVersion := fu.manager.store.GetSourceVersion(url)
196 if storedVersion == "" {
197 logging.Info(fmt.Sprintf("[filtering] %s: no stored version, will update", shortName))
198 return true
199 }
200
201 needsUpdate := remoteVersion != storedVersion
202 if needsUpdate {
203 logging.Info(fmt.Sprintf("[filtering] %s: new version %s (was %s)", shortName, remoteVersion, storedVersion))
204 } else {
205 logging.Info(fmt.Sprintf("[filtering] %s: up to date (%s)", shortName, storedVersion))
206 }
207
208 return needsUpdate
209}
210
211// downloadUpdate downloads the updated filter list and creates a diff
212func (fu *FilterUpdater) downloadUpdate(ctx context.Context, url, lastModified, etag string) (*FilterUpdate, error) {
213 // Download current content
214 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
215 if err != nil {
216 return nil, fmt.Errorf("failed to create GET request: %w", err)
217 }
218
219 req.Header.Set("User-Agent", "Dumber Browser/1.0 Filter Updater")
220
221 resp, err := fu.httpClient.Do(req)
222 if err != nil {
223 return nil, fmt.Errorf("GET request failed: %w", err)
224 }
225 defer func() {
226 if err := resp.Body.Close(); err != nil {
227 logging.Warn(fmt.Sprintf("warning: failed to close response body: %v", err))
228 }
229 }()
230
231 if resp.StatusCode != http.StatusOK {
232 return nil, fmt.Errorf("GET request returned %d", resp.StatusCode)
233 }
234
235 // Read content (handle ContentLength=-1 when server uses chunked encoding)
236 initialCap := resp.ContentLength
237 if initialCap < 0 {
238 initialCap = 64 * 1024 // 64KB initial capacity
239 }
240 currentContent := make([]byte, 0, initialCap)
241 buffer := make([]byte, 8192)
242
243 for {
244 select {
245 case <-ctx.Done():
246 return nil, ctx.Err()
247 default:
248 }
249
250 n, err := resp.Body.Read(buffer)
251 if n > 0 {
252 currentContent = append(currentContent, buffer[:n]...)
253 }
254 if err != nil {
255 break
256 }
257 }
258
259 // Calculate content hash
260 hash := fmt.Sprintf("%x", sha256.Sum256(currentContent))
261
262 // Fetch version with HEAD request (same method as fetchRemoteVersion)
263 // This ensures consistency - HEAD may return different ETag than GET
264 version, _ := fu.fetchRemoteVersion(ctx, url)
265
266 // Create update object
267 update := &FilterUpdate{
268 ListID: fu.getListID(url),
269 URL: url,
270 Current: currentContent,
271 Hash: hash,
272 Version: version,
273 }
274
275 // Get previous version for diffing (if available)
276 previous := fu.getPreviousContent(update.ListID)
277 if previous != nil {
278 update.Previous = previous
279
280 // Check if content actually changed
281 if fu.contentEqual(previous, currentContent) {
282 logging.Debug(fmt.Sprintf("No content changes in %s", url))
283 return nil, nil
284 }
285 }
286
287 // Compile the differential update
288 compiled, err := fu.compileDiff(update)
289 if err != nil {
290 return nil, fmt.Errorf("failed to compile diff: %w", err)
291 }
292
293 update.Compiled = compiled
294 return update, nil
295}
296
297// compileDiff compiles only the changed parts of a filter list
298func (fu *FilterUpdater) compileDiff(update *FilterUpdate) (*CompiledFilters, error) {
299 if update.Previous == nil {
300 // No previous version, compile everything
301 return fu.compileComplete(update.Current)
302 }
303
304 // Find differences between old and new content
305 added, removed := fu.diffEngine.Diff(update.Previous, update.Current)
306
307 compiled := NewCompiledFilters()
308
309 // Process added lines
310 for _, line := range added {
311 rule, err := fu.parseFilterLine(line)
312 if err != nil {
313 logging.Debug(fmt.Sprintf("Failed to parse added line: %s, error: %v", line, err))
314 continue
315 }
316 if rule != nil {
317 fu.addRuleToCompiled(compiled, rule)
318 }
319 }
320
321 // Process removed lines (mark for removal)
322 for _, line := range removed {
323 rule, err := fu.parseFilterLine(line)
324 if err != nil {
325 logging.Debug(fmt.Sprintf("Failed to parse removed line: %s, error: %v", line, err))
326 continue
327 }
328 if rule != nil {
329 fu.removeRuleFromCompiled(compiled, rule)
330 }
331 }
332
333 compiled.Version = fmt.Sprintf("diff-update-%d", time.Now().Unix())
334 compiled.CompiledAt = time.Now()
335
336 logging.Info(fmt.Sprintf("Compiled differential update: %d rules added/removed for %s",
337 len(added)+len(removed), update.URL))
338
339 return compiled, nil
340}
341
342// compileComplete compiles a complete filter list
343func (fu *FilterUpdater) compileComplete(content []byte) (*CompiledFilters, error) {
344 // Use existing manager's compiler functionality
345 if fu.manager.compiler != nil {
346 return fu.manager.compiler.CompileFromData(content)
347 }
348
349 // Fallback: manual compilation
350 return fu.compileManually(content)
351}
352
353// compileManually manually compiles filter content when no compiler is available
354func (fu *FilterUpdater) compileManually(content []byte) (*CompiledFilters, error) {
355 compiled := NewCompiledFilters()
356
357 lines := strings.Split(string(content), "\n")
358 for _, line := range lines {
359 rule, err := fu.parseFilterLine(strings.TrimSpace(line))
360 if err != nil {
361 continue // Skip invalid lines
362 }
363 if rule != nil {
364 fu.addRuleToCompiled(compiled, rule)
365 }
366 }
367
368 compiled.Version = fmt.Sprintf("manual-compile-%d", time.Now().Unix())
369 compiled.CompiledAt = time.Now()
370
371 return compiled, nil
372}
373
374// parseFilterLine parses a single filter line (simplified implementation)
375func (fu *FilterUpdater) parseFilterLine(line string) (interface{}, error) {
376 line = strings.TrimSpace(line)
377
378 // Skip empty lines and comments
379 if line == "" || strings.HasPrefix(line, "!") {
380 return nil, nil
381 }
382
383 // This is a simplified parser - in reality, you'd use the converter package
384 return map[string]string{
385 "type": "filter",
386 "line": line,
387 }, nil
388}
389
390// addRuleToCompiled adds a rule to compiled filters
391func (fu *FilterUpdater) addRuleToCompiled(compiled *CompiledFilters, rule interface{}) {
392 // Simplified implementation - in reality, this would properly categorize rules
393 ruleMap, ok := rule.(map[string]string)
394 if !ok {
395 return
396 }
397
398 line := ruleMap["line"]
399 if strings.Contains(line, "##") {
400 // Cosmetic filter
401 parts := strings.SplitN(line, "##", 2)
402 if len(parts) == 2 {
403 domain := parts[0]
404 selector := parts[1]
405
406 if domain == "" {
407 compiled.GenericHiding = append(compiled.GenericHiding, selector)
408 } else {
409 if compiled.CosmeticRules == nil {
410 compiled.CosmeticRules = make(map[string][]string)
411 }
412 compiled.CosmeticRules[domain] = append(compiled.CosmeticRules[domain], selector)
413 }
414 }
415 }
416 // Network filters would be handled here too
417}
418
419// removeRuleFromCompiled marks a rule for removal
420func (fu *FilterUpdater) removeRuleFromCompiled(compiled *CompiledFilters, rule interface{}) {
421 // Implementation would mark rules for removal
422 // For now, we'll just log the removal
423 ruleMap, ok := rule.(map[string]string)
424 if ok {
425 logging.Debug(fmt.Sprintf("Marking rule for removal: %s", ruleMap["line"]))
426 }
427}
428
429// applyUpdate applies a compiled filter update to the manager
430func (fu *FilterUpdater) applyUpdate(update *FilterUpdate) error {
431 if update.Compiled == nil {
432 return fmt.Errorf("no compiled filters in update")
433 }
434
435 // Apply the update to the filter manager
436 fu.manager.compileMutex.Lock()
437 defer fu.manager.compileMutex.Unlock()
438
439 // Merge the update with existing filters
440 if fu.manager.compiled == nil {
441 fu.manager.compiled = update.Compiled
442 } else {
443 fu.manager.compiled.Merge(update.Compiled)
444 }
445
446 // Update cosmetic rules
447 if err := fu.manager.cosmeticInjector.InjectRules(update.Compiled.CosmeticRules); err != nil {
448 return fmt.Errorf("failed to inject cosmetic rules: %w", err)
449 }
450
451 // Store the updated content for future diffs
452 fu.storePreviousContent(update.ListID, update.Current)
453
454 // Store the version for future update checks
455 if update.Version != "" {
456 if err := fu.manager.store.SetSourceVersion(update.URL, update.Version); err != nil {
457 logging.Warn(fmt.Sprintf("Failed to store version for %s: %v", update.URL, err))
458 }
459 }
460
461 logging.Info(fmt.Sprintf("Applied filter update for %s", update.URL))
462 return nil
463}
464
465
466// getListID generates a unique ID for a filter list URL
467func (fu *FilterUpdater) getListID(url string) string {
468 hash := sha256.Sum256([]byte(url))
469 return fmt.Sprintf("%x", hash)[:16]
470}
471
472// getPreviousContent retrieves previously stored filter list content
473func (fu *FilterUpdater) getPreviousContent(listID string) []byte {
474 // This would typically read from persistent storage
475 // For now, return nil (no previous content)
476 return nil
477}
478
479// storePreviousContent stores filter list content for future diff operations
480func (fu *FilterUpdater) storePreviousContent(listID string, content []byte) {
481 // This would typically write to persistent storage
482 // For now, we'll just log the action
483 logging.Debug(fmt.Sprintf("Storing %d bytes for list %s", len(content), listID))
484}
485
486// contentEqual checks if two byte slices contain the same content
487func (fu *FilterUpdater) contentEqual(a, b []byte) bool {
488 if len(a) != len(b) {
489 return false
490 }
491
492 hashA := sha256.Sum256(a)
493 hashB := sha256.Sum256(b)
494
495 return hashA == hashB
496}
497
498// Diff finds added and removed lines between two versions
499func (de *DiffEngine) Diff(previous, current []byte) (added []string, removed []string) {
500 prevLines := strings.Split(string(previous), "\n")
501 currLines := strings.Split(string(current), "\n")
502
503 // Create sets for efficient lookup
504 prevSet := make(map[string]bool)
505 currSet := make(map[string]bool)
506
507 for _, line := range prevLines {
508 line = strings.TrimSpace(line)
509 if line != "" {
510 prevSet[line] = true
511 }
512 }
513
514 for _, line := range currLines {
515 line = strings.TrimSpace(line)
516 if line != "" {
517 currSet[line] = true
518 }
519 }
520
521 // Find added lines (in current but not in previous)
522 for line := range currSet {
523 if !prevSet[line] {
524 added = append(added, line)
525 }
526 }
527
528 // Find removed lines (in previous but not in current)
529 for line := range prevSet {
530 if !currSet[line] {
531 removed = append(removed, line)
532 }
533 }
534
535 return added, removed
536}