1package filtering
2
3import (
4 "context"
5 "crypto/sha256"
6 "fmt"
7 "log"
8 "net/http"
9 "strings"
10 "sync"
11 "time"
12
13 "github.com/bnema/dumber/internal/logging"
14)
15
16const (
17 updateTimeoutSeconds = 60 // HTTP timeout for update checks
18 maxConcurrentUpdates = 4 // Maximum concurrent filter list downloads
19)
20
21// FilterUpdater manages differential filter list updates
22type FilterUpdater struct {
23 manager *FilterManager
24 httpClient *http.Client
25 diffEngine *DiffEngine
26 updateMutex sync.Mutex
27}
28
29// FilterUpdate represents an update to a filter list
30type FilterUpdate struct {
31 ListID string
32 URL string
33 Previous []byte
34 Current []byte
35 Hash string
36 Compiled *CompiledFilters
37}
38
39// FilterListInfo stores metadata about a filter list
40type FilterListInfo struct {
41 URL string
42 LastModified time.Time
43 ETag string
44 ContentHash string
45 Size int64
46}
47
48// DiffEngine handles differential updates between filter lists
49type DiffEngine struct {
50 // We'll implement a simple line-based diff for filter lists
51}
52
53// NewFilterUpdater creates a new filter updater
54func NewFilterUpdater(manager *FilterManager) *FilterUpdater {
55 return &FilterUpdater{
56 manager: manager,
57 httpClient: &http.Client{
58 Timeout: updateTimeoutSeconds * time.Second,
59 },
60 diffEngine: &DiffEngine{},
61 }
62}
63
64// CheckAndUpdate checks for filter list updates and applies them
65func (fu *FilterUpdater) CheckAndUpdate(ctx context.Context) error {
66 fu.updateMutex.Lock()
67 defer fu.updateMutex.Unlock()
68
69 sources := []string{
70 "https://easylist.to/easylist/easylist.txt",
71 "https://easylist.to/easylist/easyprivacy.txt",
72 }
73
74 // Check for updates concurrently
75 updates := make(chan *FilterUpdate, len(sources))
76 var wg sync.WaitGroup
77
78 // Limit concurrent updates
79 semaphore := make(chan struct{}, maxConcurrentUpdates)
80
81 for _, url := range sources {
82 wg.Add(1)
83 go func(u string) {
84 defer wg.Done()
85
86 // Acquire semaphore
87 semaphore <- struct{}{}
88 defer func() { <-semaphore }()
89
90 update, err := fu.checkSingleUpdate(ctx, u)
91 if err != nil {
92 logging.Error(fmt.Sprintf("Failed to check update for %s: %v", u, err))
93 return
94 }
95
96 if update != nil {
97 updates <- update
98 }
99 }(url)
100 }
101
102 // Wait for all checks to complete
103 go func() {
104 wg.Wait()
105 close(updates)
106 }()
107
108 // Apply updates as they come in
109 updateCount := 0
110 for update := range updates {
111 if err := fu.applyUpdate(update); err != nil {
112 logging.Error(fmt.Sprintf("Failed to apply update for %s: %v", update.URL, err))
113 continue
114 }
115 updateCount++
116 }
117
118 if updateCount > 0 {
119 logging.Info(fmt.Sprintf("Applied %d filter list updates", updateCount))
120 } else {
121 logging.Debug("No filter list updates available")
122 }
123
124 return nil
125}
126
127// checkSingleUpdate checks if a single filter list needs updating
128func (fu *FilterUpdater) checkSingleUpdate(ctx context.Context, url string) (*FilterUpdate, error) {
129 // Create HEAD request to check if content has changed
130 req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
131 if err != nil {
132 return nil, fmt.Errorf("failed to create HEAD request: %w", err)
133 }
134
135 // Add headers to support conditional requests
136 req.Header.Set("User-Agent", "Dumber Browser/1.0 Filter Updater")
137
138 resp, err := fu.httpClient.Do(req)
139 if err != nil {
140 return nil, fmt.Errorf("HEAD request failed: %w", err)
141 }
142 defer func() {
143 if err := resp.Body.Close(); err != nil {
144 log.Printf("warning: failed to close response body: %v", err)
145 }
146 }()
147
148 if resp.StatusCode != http.StatusOK {
149 return nil, fmt.Errorf("HEAD request returned %d", resp.StatusCode)
150 }
151
152 // Get metadata from response headers
153 lastModified := resp.Header.Get("Last-Modified")
154 etag := resp.Header.Get("ETag")
155 contentLength := resp.ContentLength
156
157 // Check if we need to download full content
158 needsUpdate := fu.needsUpdate(url, lastModified, etag, contentLength)
159 if !needsUpdate {
160 return nil, nil // No update needed
161 }
162
163 // Download the full content
164 return fu.downloadUpdate(ctx, url, lastModified, etag)
165}
166
167// needsUpdate determines if a filter list needs updating based on metadata
168func (fu *FilterUpdater) needsUpdate(url, lastModified, etag string, contentLength int64) bool {
169 // For now, we'll implement a simple time-based check
170 // In a full implementation, this would check against stored metadata
171
172 // Always check for updates if we don't have cached metadata
173 // In practice, this would compare with stored filter list metadata
174 return true
175}
176
177// downloadUpdate downloads the updated filter list and creates a diff
178func (fu *FilterUpdater) downloadUpdate(ctx context.Context, url, lastModified, etag string) (*FilterUpdate, error) {
179 // Download current content
180 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
181 if err != nil {
182 return nil, fmt.Errorf("failed to create GET request: %w", err)
183 }
184
185 req.Header.Set("User-Agent", "Dumber Browser/1.0 Filter Updater")
186
187 resp, err := fu.httpClient.Do(req)
188 if err != nil {
189 return nil, fmt.Errorf("GET request failed: %w", err)
190 }
191 defer func() {
192 if err := resp.Body.Close(); err != nil {
193 log.Printf("warning: failed to close response body: %v", err)
194 }
195 }()
196
197 if resp.StatusCode != http.StatusOK {
198 return nil, fmt.Errorf("GET request returned %d", resp.StatusCode)
199 }
200
201 // Read content
202 currentContent := make([]byte, 0, resp.ContentLength)
203 buffer := make([]byte, 8192)
204
205 for {
206 select {
207 case <-ctx.Done():
208 return nil, ctx.Err()
209 default:
210 }
211
212 n, err := resp.Body.Read(buffer)
213 if n > 0 {
214 currentContent = append(currentContent, buffer[:n]...)
215 }
216 if err != nil {
217 break
218 }
219 }
220
221 // Calculate content hash
222 hash := fmt.Sprintf("%x", sha256.Sum256(currentContent))
223
224 // Create update object
225 update := &FilterUpdate{
226 ListID: fu.getListID(url),
227 URL: url,
228 Current: currentContent,
229 Hash: hash,
230 }
231
232 // Get previous version for diffing (if available)
233 previous := fu.getPreviousContent(update.ListID)
234 if previous != nil {
235 update.Previous = previous
236
237 // Check if content actually changed
238 if fu.contentEqual(previous, currentContent) {
239 logging.Debug(fmt.Sprintf("No content changes in %s", url))
240 return nil, nil
241 }
242 }
243
244 // Compile the differential update
245 compiled, err := fu.compileDiff(update)
246 if err != nil {
247 return nil, fmt.Errorf("failed to compile diff: %w", err)
248 }
249
250 update.Compiled = compiled
251 return update, nil
252}
253
254// compileDiff compiles only the changed parts of a filter list
255func (fu *FilterUpdater) compileDiff(update *FilterUpdate) (*CompiledFilters, error) {
256 if update.Previous == nil {
257 // No previous version, compile everything
258 return fu.compileComplete(update.Current)
259 }
260
261 // Find differences between old and new content
262 added, removed := fu.diffEngine.Diff(update.Previous, update.Current)
263
264 compiled := NewCompiledFilters()
265
266 // Process added lines
267 for _, line := range added {
268 rule, err := fu.parseFilterLine(line)
269 if err != nil {
270 logging.Debug(fmt.Sprintf("Failed to parse added line: %s, error: %v", line, err))
271 continue
272 }
273 if rule != nil {
274 fu.addRuleToCompiled(compiled, rule)
275 }
276 }
277
278 // Process removed lines (mark for removal)
279 for _, line := range removed {
280 rule, err := fu.parseFilterLine(line)
281 if err != nil {
282 logging.Debug(fmt.Sprintf("Failed to parse removed line: %s, error: %v", line, err))
283 continue
284 }
285 if rule != nil {
286 fu.removeRuleFromCompiled(compiled, rule)
287 }
288 }
289
290 compiled.Version = fmt.Sprintf("diff-update-%d", time.Now().Unix())
291 compiled.CompiledAt = time.Now()
292
293 logging.Info(fmt.Sprintf("Compiled differential update: %d rules added/removed for %s",
294 len(added)+len(removed), update.URL))
295
296 return compiled, nil
297}
298
299// compileComplete compiles a complete filter list
300func (fu *FilterUpdater) compileComplete(content []byte) (*CompiledFilters, error) {
301 // Use existing manager's compiler functionality
302 if fu.manager.compiler != nil {
303 return fu.manager.compiler.CompileFromData(content)
304 }
305
306 // Fallback: manual compilation
307 return fu.compileManually(content)
308}
309
310// compileManually manually compiles filter content when no compiler is available
311func (fu *FilterUpdater) compileManually(content []byte) (*CompiledFilters, error) {
312 compiled := NewCompiledFilters()
313
314 lines := strings.Split(string(content), "\n")
315 for _, line := range lines {
316 rule, err := fu.parseFilterLine(strings.TrimSpace(line))
317 if err != nil {
318 continue // Skip invalid lines
319 }
320 if rule != nil {
321 fu.addRuleToCompiled(compiled, rule)
322 }
323 }
324
325 compiled.Version = fmt.Sprintf("manual-compile-%d", time.Now().Unix())
326 compiled.CompiledAt = time.Now()
327
328 return compiled, nil
329}
330
331// parseFilterLine parses a single filter line (simplified implementation)
332func (fu *FilterUpdater) parseFilterLine(line string) (interface{}, error) {
333 line = strings.TrimSpace(line)
334
335 // Skip empty lines and comments
336 if line == "" || strings.HasPrefix(line, "!") {
337 return nil, nil
338 }
339
340 // This is a simplified parser - in reality, you'd use the converter package
341 return map[string]string{
342 "type": "filter",
343 "line": line,
344 }, nil
345}
346
347// addRuleToCompiled adds a rule to compiled filters
348func (fu *FilterUpdater) addRuleToCompiled(compiled *CompiledFilters, rule interface{}) {
349 // Simplified implementation - in reality, this would properly categorize rules
350 ruleMap, ok := rule.(map[string]string)
351 if !ok {
352 return
353 }
354
355 line := ruleMap["line"]
356 if strings.Contains(line, "##") {
357 // Cosmetic filter
358 parts := strings.SplitN(line, "##", 2)
359 if len(parts) == 2 {
360 domain := parts[0]
361 selector := parts[1]
362
363 if domain == "" {
364 compiled.GenericHiding = append(compiled.GenericHiding, selector)
365 } else {
366 if compiled.CosmeticRules == nil {
367 compiled.CosmeticRules = make(map[string][]string)
368 }
369 compiled.CosmeticRules[domain] = append(compiled.CosmeticRules[domain], selector)
370 }
371 }
372 }
373 // Network filters would be handled here too
374}
375
376// removeRuleFromCompiled marks a rule for removal
377func (fu *FilterUpdater) removeRuleFromCompiled(compiled *CompiledFilters, rule interface{}) {
378 // Implementation would mark rules for removal
379 // For now, we'll just log the removal
380 ruleMap, ok := rule.(map[string]string)
381 if ok {
382 logging.Debug(fmt.Sprintf("Marking rule for removal: %s", ruleMap["line"]))
383 }
384}
385
386// applyUpdate applies a compiled filter update to the manager
387func (fu *FilterUpdater) applyUpdate(update *FilterUpdate) error {
388 if update.Compiled == nil {
389 return fmt.Errorf("no compiled filters in update")
390 }
391
392 // Apply the update to the filter manager
393 fu.manager.compileMutex.Lock()
394 defer fu.manager.compileMutex.Unlock()
395
396 // Merge the update with existing filters
397 if fu.manager.compiled == nil {
398 fu.manager.compiled = update.Compiled
399 } else {
400 fu.manager.compiled.Merge(update.Compiled)
401 }
402
403 // Update cosmetic rules
404 if err := fu.manager.cosmeticInjector.InjectRules(update.Compiled.CosmeticRules); err != nil {
405 return fmt.Errorf("failed to inject cosmetic rules: %w", err)
406 }
407
408 // Store the updated content for future diffs
409 fu.storePreviousContent(update.ListID, update.Current)
410
411 logging.Info(fmt.Sprintf("Applied filter update for %s", update.URL))
412 return nil
413}
414
415// getListID generates a unique ID for a filter list URL
416func (fu *FilterUpdater) getListID(url string) string {
417 hash := sha256.Sum256([]byte(url))
418 return fmt.Sprintf("%x", hash)[:16]
419}
420
421// getPreviousContent retrieves previously stored filter list content
422func (fu *FilterUpdater) getPreviousContent(listID string) []byte {
423 // This would typically read from persistent storage
424 // For now, return nil (no previous content)
425 return nil
426}
427
428// storePreviousContent stores filter list content for future diff operations
429func (fu *FilterUpdater) storePreviousContent(listID string, content []byte) {
430 // This would typically write to persistent storage
431 // For now, we'll just log the action
432 logging.Debug(fmt.Sprintf("Storing %d bytes for list %s", len(content), listID))
433}
434
435// contentEqual checks if two byte slices contain the same content
436func (fu *FilterUpdater) contentEqual(a, b []byte) bool {
437 if len(a) != len(b) {
438 return false
439 }
440
441 hashA := sha256.Sum256(a)
442 hashB := sha256.Sum256(b)
443
444 return hashA == hashB
445}
446
447// Diff finds added and removed lines between two versions
448func (de *DiffEngine) Diff(previous, current []byte) (added []string, removed []string) {
449 prevLines := strings.Split(string(previous), "\n")
450 currLines := strings.Split(string(current), "\n")
451
452 // Create sets for efficient lookup
453 prevSet := make(map[string]bool)
454 currSet := make(map[string]bool)
455
456 for _, line := range prevLines {
457 line = strings.TrimSpace(line)
458 if line != "" {
459 prevSet[line] = true
460 }
461 }
462
463 for _, line := range currLines {
464 line = strings.TrimSpace(line)
465 if line != "" {
466 currSet[line] = true
467 }
468 }
469
470 // Find added lines (in current but not in previous)
471 for line := range currSet {
472 if !prevSet[line] {
473 added = append(added, line)
474 }
475 }
476
477 // Find removed lines (in previous but not in current)
478 for line := range prevSet {
479 if !currSet[line] {
480 removed = append(removed, line)
481 }
482 }
483
484 return added, removed
485}