1package filtering
2
3import (
4 "context"
5 "crypto/sha256"
6 "fmt"
7 "net/http"
8 "strings"
9 "sync"
10 "time"
11
12 "github.com/bnema/dumber/internal/logging"
13)
14
15const (
16 updateTimeoutSeconds = 60 // HTTP timeout for update checks
17 maxConcurrentUpdates = 4 // Maximum concurrent filter list downloads
18)
19
20// FilterUpdater manages differential filter list updates
21type FilterUpdater struct {
22 manager *FilterManager
23 httpClient *http.Client
24 diffEngine *DiffEngine
25 updateMutex sync.Mutex
26}
27
28// FilterUpdate represents an update to a filter list
29type FilterUpdate struct {
30 ListID string
31 URL string
32 Previous []byte
33 Current []byte
34 Hash string
35 Compiled *CompiledFilters
36}
37
38// FilterListInfo stores metadata about a filter list
39type FilterListInfo struct {
40 URL string
41 LastModified time.Time
42 ETag string
43 ContentHash string
44 Size int64
45}
46
47// DiffEngine handles differential updates between filter lists
48type DiffEngine struct {
49 // We'll implement a simple line-based diff for filter lists
50}
51
52// NewFilterUpdater creates a new filter updater
53func NewFilterUpdater(manager *FilterManager) *FilterUpdater {
54 return &FilterUpdater{
55 manager: manager,
56 httpClient: &http.Client{
57 Timeout: updateTimeoutSeconds * time.Second,
58 },
59 diffEngine: &DiffEngine{},
60 }
61}
62
63// CheckAndUpdate checks for filter list updates and applies them
64func (fu *FilterUpdater) CheckAndUpdate(ctx context.Context) error {
65 fu.updateMutex.Lock()
66 defer fu.updateMutex.Unlock()
67
68 sources := []string{
69 "https://easylist.to/easylist/easylist.txt",
70 "https://easylist.to/easylist/easyprivacy.txt",
71 }
72
73 // Check for updates concurrently
74 updates := make(chan *FilterUpdate, len(sources))
75 var wg sync.WaitGroup
76
77 // Limit concurrent updates
78 semaphore := make(chan struct{}, maxConcurrentUpdates)
79
80 for _, url := range sources {
81 wg.Add(1)
82 go func(u string) {
83 defer wg.Done()
84
85 // Acquire semaphore
86 semaphore <- struct{}{}
87 defer func() { <-semaphore }()
88
89 update, err := fu.checkSingleUpdate(ctx, u)
90 if err != nil {
91 logging.Error(fmt.Sprintf("Failed to check update for %s: %v", u, err))
92 return
93 }
94
95 if update != nil {
96 updates <- update
97 }
98 }(url)
99 }
100
101 // Wait for all checks to complete
102 go func() {
103 wg.Wait()
104 close(updates)
105 }()
106
107 // Apply updates as they come in
108 updateCount := 0
109 for update := range updates {
110 if err := fu.applyUpdate(update); err != nil {
111 logging.Error(fmt.Sprintf("Failed to apply update for %s: %v", update.URL, err))
112 continue
113 }
114 updateCount++
115 }
116
117 if updateCount > 0 {
118 logging.Info(fmt.Sprintf("Applied %d filter list updates", updateCount))
119 } else {
120 logging.Debug("No filter list updates available")
121 }
122
123 return nil
124}
125
126// checkSingleUpdate checks if a single filter list needs updating
127func (fu *FilterUpdater) checkSingleUpdate(ctx context.Context, url string) (*FilterUpdate, error) {
128 // Create HEAD request to check if content has changed
129 req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
130 if err != nil {
131 return nil, fmt.Errorf("failed to create HEAD request: %w", err)
132 }
133
134 // Add headers to support conditional requests
135 req.Header.Set("User-Agent", "Dumber Browser/1.0 Filter Updater")
136
137 resp, err := fu.httpClient.Do(req)
138 if err != nil {
139 return nil, fmt.Errorf("HEAD request failed: %w", err)
140 }
141 defer resp.Body.Close()
142
143 if resp.StatusCode != http.StatusOK {
144 return nil, fmt.Errorf("HEAD request returned %d", resp.StatusCode)
145 }
146
147 // Get metadata from response headers
148 lastModified := resp.Header.Get("Last-Modified")
149 etag := resp.Header.Get("ETag")
150 contentLength := resp.ContentLength
151
152 // Check if we need to download full content
153 needsUpdate := fu.needsUpdate(url, lastModified, etag, contentLength)
154 if !needsUpdate {
155 return nil, nil // No update needed
156 }
157
158 // Download the full content
159 return fu.downloadUpdate(ctx, url, lastModified, etag)
160}
161
162// needsUpdate determines if a filter list needs updating based on metadata
163func (fu *FilterUpdater) needsUpdate(url, lastModified, etag string, contentLength int64) bool {
164 // For now, we'll implement a simple time-based check
165 // In a full implementation, this would check against stored metadata
166
167 // Always check for updates if we don't have cached metadata
168 // In practice, this would compare with stored filter list metadata
169 return true
170}
171
172// downloadUpdate downloads the updated filter list and creates a diff
173func (fu *FilterUpdater) downloadUpdate(ctx context.Context, url, lastModified, etag string) (*FilterUpdate, error) {
174 // Download current content
175 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
176 if err != nil {
177 return nil, fmt.Errorf("failed to create GET request: %w", err)
178 }
179
180 req.Header.Set("User-Agent", "Dumber Browser/1.0 Filter Updater")
181
182 resp, err := fu.httpClient.Do(req)
183 if err != nil {
184 return nil, fmt.Errorf("GET request failed: %w", err)
185 }
186 defer resp.Body.Close()
187
188 if resp.StatusCode != http.StatusOK {
189 return nil, fmt.Errorf("GET request returned %d", resp.StatusCode)
190 }
191
192 // Read content
193 currentContent := make([]byte, 0, resp.ContentLength)
194 buffer := make([]byte, 8192)
195
196 for {
197 select {
198 case <-ctx.Done():
199 return nil, ctx.Err()
200 default:
201 }
202
203 n, err := resp.Body.Read(buffer)
204 if n > 0 {
205 currentContent = append(currentContent, buffer[:n]...)
206 }
207 if err != nil {
208 break
209 }
210 }
211
212 // Calculate content hash
213 hash := fmt.Sprintf("%x", sha256.Sum256(currentContent))
214
215 // Create update object
216 update := &FilterUpdate{
217 ListID: fu.getListID(url),
218 URL: url,
219 Current: currentContent,
220 Hash: hash,
221 }
222
223 // Get previous version for diffing (if available)
224 previous := fu.getPreviousContent(update.ListID)
225 if previous != nil {
226 update.Previous = previous
227
228 // Check if content actually changed
229 if fu.contentEqual(previous, currentContent) {
230 logging.Debug(fmt.Sprintf("No content changes in %s", url))
231 return nil, nil
232 }
233 }
234
235 // Compile the differential update
236 compiled, err := fu.compileDiff(update)
237 if err != nil {
238 return nil, fmt.Errorf("failed to compile diff: %w", err)
239 }
240
241 update.Compiled = compiled
242 return update, nil
243}
244
245// compileDiff compiles only the changed parts of a filter list
246func (fu *FilterUpdater) compileDiff(update *FilterUpdate) (*CompiledFilters, error) {
247 if update.Previous == nil {
248 // No previous version, compile everything
249 return fu.compileComplete(update.Current)
250 }
251
252 // Find differences between old and new content
253 added, removed := fu.diffEngine.Diff(update.Previous, update.Current)
254
255 compiled := NewCompiledFilters()
256
257 // Process added lines
258 for _, line := range added {
259 rule, err := fu.parseFilterLine(line)
260 if err != nil {
261 logging.Debug(fmt.Sprintf("Failed to parse added line: %s, error: %v", line, err))
262 continue
263 }
264 if rule != nil {
265 fu.addRuleToCompiled(compiled, rule)
266 }
267 }
268
269 // Process removed lines (mark for removal)
270 for _, line := range removed {
271 rule, err := fu.parseFilterLine(line)
272 if err != nil {
273 logging.Debug(fmt.Sprintf("Failed to parse removed line: %s, error: %v", line, err))
274 continue
275 }
276 if rule != nil {
277 fu.removeRuleFromCompiled(compiled, rule)
278 }
279 }
280
281 compiled.Version = fmt.Sprintf("diff-update-%d", time.Now().Unix())
282 compiled.CompiledAt = time.Now()
283
284 logging.Info(fmt.Sprintf("Compiled differential update: %d rules added/removed for %s",
285 len(added)+len(removed), update.URL))
286
287 return compiled, nil
288}
289
290// compileComplete compiles a complete filter list
291func (fu *FilterUpdater) compileComplete(content []byte) (*CompiledFilters, error) {
292 // Use existing manager's compiler functionality
293 if fu.manager.compiler != nil {
294 return fu.manager.compiler.CompileFromData(content)
295 }
296
297 // Fallback: manual compilation
298 return fu.compileManually(content)
299}
300
301// compileManually manually compiles filter content when no compiler is available
302func (fu *FilterUpdater) compileManually(content []byte) (*CompiledFilters, error) {
303 compiled := NewCompiledFilters()
304
305 lines := strings.Split(string(content), "\n")
306 for _, line := range lines {
307 rule, err := fu.parseFilterLine(strings.TrimSpace(line))
308 if err != nil {
309 continue // Skip invalid lines
310 }
311 if rule != nil {
312 fu.addRuleToCompiled(compiled, rule)
313 }
314 }
315
316 compiled.Version = fmt.Sprintf("manual-compile-%d", time.Now().Unix())
317 compiled.CompiledAt = time.Now()
318
319 return compiled, nil
320}
321
322// parseFilterLine parses a single filter line (simplified implementation)
323func (fu *FilterUpdater) parseFilterLine(line string) (interface{}, error) {
324 line = strings.TrimSpace(line)
325
326 // Skip empty lines and comments
327 if line == "" || strings.HasPrefix(line, "!") {
328 return nil, nil
329 }
330
331 // This is a simplified parser - in reality, you'd use the converter package
332 return map[string]string{
333 "type": "filter",
334 "line": line,
335 }, nil
336}
337
338// addRuleToCompiled adds a rule to compiled filters
339func (fu *FilterUpdater) addRuleToCompiled(compiled *CompiledFilters, rule interface{}) {
340 // Simplified implementation - in reality, this would properly categorize rules
341 ruleMap, ok := rule.(map[string]string)
342 if !ok {
343 return
344 }
345
346 line := ruleMap["line"]
347 if strings.Contains(line, "##") {
348 // Cosmetic filter
349 parts := strings.SplitN(line, "##", 2)
350 if len(parts) == 2 {
351 domain := parts[0]
352 selector := parts[1]
353
354 if domain == "" {
355 compiled.GenericHiding = append(compiled.GenericHiding, selector)
356 } else {
357 if compiled.CosmeticRules == nil {
358 compiled.CosmeticRules = make(map[string][]string)
359 }
360 compiled.CosmeticRules[domain] = append(compiled.CosmeticRules[domain], selector)
361 }
362 }
363 }
364 // Network filters would be handled here too
365}
366
367// removeRuleFromCompiled marks a rule for removal
368func (fu *FilterUpdater) removeRuleFromCompiled(compiled *CompiledFilters, rule interface{}) {
369 // Implementation would mark rules for removal
370 // For now, we'll just log the removal
371 ruleMap, ok := rule.(map[string]string)
372 if ok {
373 logging.Debug(fmt.Sprintf("Marking rule for removal: %s", ruleMap["line"]))
374 }
375}
376
377// applyUpdate applies a compiled filter update to the manager
378func (fu *FilterUpdater) applyUpdate(update *FilterUpdate) error {
379 if update.Compiled == nil {
380 return fmt.Errorf("no compiled filters in update")
381 }
382
383 // Apply the update to the filter manager
384 fu.manager.compileMutex.Lock()
385 defer fu.manager.compileMutex.Unlock()
386
387 // Merge the update with existing filters
388 if fu.manager.compiled == nil {
389 fu.manager.compiled = update.Compiled
390 } else {
391 fu.manager.compiled.Merge(update.Compiled)
392 }
393
394 // Update cosmetic rules
395 if err := fu.manager.cosmeticInjector.InjectRules(update.Compiled.CosmeticRules); err != nil {
396 return fmt.Errorf("failed to inject cosmetic rules: %w", err)
397 }
398
399 // Store the updated content for future diffs
400 fu.storePreviousContent(update.ListID, update.Current)
401
402 logging.Info(fmt.Sprintf("Applied filter update for %s", update.URL))
403 return nil
404}
405
406// getListID generates a unique ID for a filter list URL
407func (fu *FilterUpdater) getListID(url string) string {
408 hash := sha256.Sum256([]byte(url))
409 return fmt.Sprintf("%x", hash)[:16]
410}
411
412// getPreviousContent retrieves previously stored filter list content
413func (fu *FilterUpdater) getPreviousContent(listID string) []byte {
414 // This would typically read from persistent storage
415 // For now, return nil (no previous content)
416 return nil
417}
418
419// storePreviousContent stores filter list content for future diff operations
420func (fu *FilterUpdater) storePreviousContent(listID string, content []byte) {
421 // This would typically write to persistent storage
422 // For now, we'll just log the action
423 logging.Debug(fmt.Sprintf("Storing %d bytes for list %s", len(content), listID))
424}
425
426// contentEqual checks if two byte slices contain the same content
427func (fu *FilterUpdater) contentEqual(a, b []byte) bool {
428 if len(a) != len(b) {
429 return false
430 }
431
432 hashA := sha256.Sum256(a)
433 hashB := sha256.Sum256(b)
434
435 return hashA == hashB
436}
437
438// Diff finds added and removed lines between two versions
439func (de *DiffEngine) Diff(previous, current []byte) (added []string, removed []string) {
440 prevLines := strings.Split(string(previous), "\n")
441 currLines := strings.Split(string(current), "\n")
442
443 // Create sets for efficient lookup
444 prevSet := make(map[string]bool)
445 currSet := make(map[string]bool)
446
447 for _, line := range prevLines {
448 line = strings.TrimSpace(line)
449 if line != "" {
450 prevSet[line] = true
451 }
452 }
453
454 for _, line := range currLines {
455 line = strings.TrimSpace(line)
456 if line != "" {
457 currSet[line] = true
458 }
459 }
460
461 // Find added lines (in current but not in previous)
462 for line := range currSet {
463 if !prevSet[line] {
464 added = append(added, line)
465 }
466 }
467
468 // Find removed lines (in previous but not in current)
469 for line := range prevSet {
470 if !currSet[line] {
471 removed = append(removed, line)
472 }
473 }
474
475 return added, removed
476}