detached 61444b7

feat(filtering): add async auto-update with configurable filter lists

commit
61444b70bd68d3e2401e61768303231f35b3c4be
author
bnema · · 2025-11-29 13:24 UTC
committer
bnema · · 2025-11-29 13:24 UTC
parent
c874c3b
signature
GPG Unverified2B82207D370127A0 The key is unknown
6 files · +197 -54
Unified
UnifiedSplit
M go.mod +3 -0
@@ -11,6 +11,7 @@ require (
1111 github.com/ncruces/go-sqlite3 v0.29.0
1212 github.com/spf13/cobra v1.10.1
1313 github.com/spf13/viper v1.21.0
14+ github.com/stretchr/testify v1.11.1
1415 go.uber.org/mock v0.6.0
1516 golang.org/x/sync v0.17.0
1617 golang.org/x/sys v0.36.0
@@ -20,12 +21,14 @@ require (
2021 github.com/KarpelesLab/weak v0.1.1 // indirect
2122 github.com/bahlo/generic-list-go v0.2.0 // indirect
2223 github.com/buger/jsonparser v1.1.1 // indirect
24+ github.com/davecgh/go-spew v1.1.1 // indirect
2325 github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
2426 github.com/google/go-cmp v0.7.0 // indirect
2527 github.com/inconshreveable/mousetrap v1.1.0 // indirect
2628 github.com/mailru/easyjson v0.7.7 // indirect
2729 github.com/ncruces/julianday v1.0.0 // indirect
2830 github.com/pelletier/go-toml/v2 v2.2.4 // indirect
31+ github.com/pmezard/go-difflib v1.0.0 // indirect
2932 github.com/rogpeppe/go-internal v1.12.0 // indirect
3033 github.com/sagikazarmark/locafero v0.11.0 // indirect
3134 github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
@@ -11,6 +11,7 @@ require (
11 github.com/ncruces/go-sqlite3 v0.29.011 github.com/ncruces/go-sqlite3 v0.29.0
12 github.com/spf13/cobra v1.10.112 github.com/spf13/cobra v1.10.1
13 github.com/spf13/viper v1.21.013 github.com/spf13/viper v1.21.0
14+ github.com/stretchr/testify v1.11.1
14 go.uber.org/mock v0.6.015 go.uber.org/mock v0.6.0
15 golang.org/x/sync v0.17.016 golang.org/x/sync v0.17.0
16 golang.org/x/sys v0.36.017 golang.org/x/sys v0.36.0
@@ -20,12 +21,14 @@ require (
20 github.com/KarpelesLab/weak v0.1.1 // indirect21 github.com/KarpelesLab/weak v0.1.1 // indirect
21 github.com/bahlo/generic-list-go v0.2.0 // indirect22 github.com/bahlo/generic-list-go v0.2.0 // indirect
22 github.com/buger/jsonparser v1.1.1 // indirect23 github.com/buger/jsonparser v1.1.1 // indirect
24+ github.com/davecgh/go-spew v1.1.1 // indirect
23 github.com/go-viper/mapstructure/v2 v2.4.0 // indirect25 github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
24 github.com/google/go-cmp v0.7.0 // indirect26 github.com/google/go-cmp v0.7.0 // indirect
25 github.com/inconshreveable/mousetrap v1.1.0 // indirect27 github.com/inconshreveable/mousetrap v1.1.0 // indirect
26 github.com/mailru/easyjson v0.7.7 // indirect28 github.com/mailru/easyjson v0.7.7 // indirect
27 github.com/ncruces/julianday v1.0.0 // indirect29 github.com/ncruces/julianday v1.0.0 // indirect
28 github.com/pelletier/go-toml/v2 v2.2.4 // indirect30 github.com/pelletier/go-toml/v2 v2.2.4 // indirect
31+ github.com/pmezard/go-difflib v1.0.0 // indirect
29 github.com/rogpeppe/go-internal v1.12.0 // indirect32 github.com/rogpeppe/go-internal v1.12.0 // indirect
30 github.com/sagikazarmark/locafero v0.11.0 // indirect33 github.com/sagikazarmark/locafero v0.11.0 // indirect
31 github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect34 github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
M internal/config/config.go +3 -0
@@ -183,6 +183,8 @@ type ContentFilteringConfig struct {
183183 Enabled bool `mapstructure:"enabled" yaml:"enabled"`
184184 // Whitelist domains that should skip ad blocking (e.g. Twitch breaks with bot detection)
185185 Whitelist []string `mapstructure:"whitelist" yaml:"whitelist"`
186+ // FilterLists URLs of filter lists to use (EasyList, uBlock, etc.)
187+ FilterLists []string `mapstructure:"filter_lists" yaml:"filter_lists"`
186188 }
187189
188190 // OmniboxConfig holds omnibox behavior preferences
@@ -763,6 +765,7 @@ func (m *Manager) setDefaults() {
763765 // Content filtering
764766 m.viper.SetDefault("content_filtering.enabled", defaults.ContentFiltering.Enabled)
765767 m.viper.SetDefault("content_filtering.whitelist", defaults.ContentFiltering.Whitelist)
768+ m.viper.SetDefault("content_filtering.filter_lists", defaults.ContentFiltering.FilterLists)
766769
767770 // Omnibox defaults
768771 m.viper.SetDefault("omnibox.initial_behavior", defaults.Omnibox.InitialBehavior)
@@ -183,6 +183,8 @@ type ContentFilteringConfig struct {
183 Enabled bool `mapstructure:"enabled" yaml:"enabled"`183 Enabled bool `mapstructure:"enabled" yaml:"enabled"`
184 // Whitelist domains that should skip ad blocking (e.g. Twitch breaks with bot detection)184 // Whitelist domains that should skip ad blocking (e.g. Twitch breaks with bot detection)
185 Whitelist []string `mapstructure:"whitelist" yaml:"whitelist"`185 Whitelist []string `mapstructure:"whitelist" yaml:"whitelist"`
186+ // FilterLists URLs of filter lists to use (EasyList, uBlock, etc.)
187+ FilterLists []string `mapstructure:"filter_lists" yaml:"filter_lists"`
186 }188 }
187 189
188 // OmniboxConfig holds omnibox behavior preferences190 // OmniboxConfig holds omnibox behavior preferences
@@ -763,6 +765,7 @@ func (m *Manager) setDefaults() {
763 // Content filtering765 // Content filtering
764 m.viper.SetDefault("content_filtering.enabled", defaults.ContentFiltering.Enabled)766 m.viper.SetDefault("content_filtering.enabled", defaults.ContentFiltering.Enabled)
765 m.viper.SetDefault("content_filtering.whitelist", defaults.ContentFiltering.Whitelist)767 m.viper.SetDefault("content_filtering.whitelist", defaults.ContentFiltering.Whitelist)
768+ m.viper.SetDefault("content_filtering.filter_lists", defaults.ContentFiltering.FilterLists)
766 769
767 // Omnibox defaults770 // Omnibox defaults
768 m.viper.SetDefault("omnibox.initial_behavior", defaults.Omnibox.InitialBehavior)771 m.viper.SetDefault("omnibox.initial_behavior", defaults.Omnibox.InitialBehavior)
M internal/config/defaults.go +9 -0
@@ -219,6 +219,15 @@ func DefaultConfig() *Config {
219219 "passport.twitch.tv", // Auth subdomain
220220 "gql.twitch.tv", // GraphQL API
221221 },
222+ FilterLists: []string{
223+ // Core blocking
224+ "https://easylist.to/easylist/easylist.txt", // Ads
225+ "https://easylist.to/easylist/easyprivacy.txt", // Tracking
226+ // uBlock extras
227+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt", // uBO optimizations
228+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/annoyances.txt", // Cookie banners, popups
229+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/quick-fixes.txt", // Site fixes
230+ },
222231 },
223232 Omnibox: OmniboxConfig{
224233 InitialBehavior: defaultOmniboxInitialBehavior,
@@ -219,6 +219,15 @@ func DefaultConfig() *Config {
219 "passport.twitch.tv", // Auth subdomain219 "passport.twitch.tv", // Auth subdomain
220 "gql.twitch.tv", // GraphQL API220 "gql.twitch.tv", // GraphQL API
221 },221 },
222+ FilterLists: []string{
223+ // Core blocking
224+ "https://easylist.to/easylist/easylist.txt", // Ads
225+ "https://easylist.to/easylist/easyprivacy.txt", // Tracking
226+ // uBlock extras
227+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt", // uBO optimizations
228+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/annoyances.txt", // Cookie banners, popups
229+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/quick-fixes.txt", // Site fixes
230+ },
222 },231 },
223 Omnibox: OmniboxConfig{232 Omnibox: OmniboxConfig{
224 InitialBehavior: defaultOmniboxInitialBehavior,233 InitialBehavior: defaultOmniboxInitialBehavior,
M internal/filtering/manager.go +37 -16
@@ -31,6 +31,7 @@ type FilterManager struct {
3131 store FilterStore
3232 compiler FilterCompiler
3333 cosmeticInjector *cosmetic.CosmeticInjector
34+ updater *FilterUpdater
3435 updateInterval time.Duration
3536 compileMutex sync.RWMutex
3637 compiled *CompiledFilters
@@ -45,6 +46,8 @@ type FilterStore interface {
4546 LoadCached() (*CompiledFilters, error)
4647 SaveCache(filters *CompiledFilters) error
4748 GetCacheInfo() (bool, time.Time, error)
49+ GetSourceVersion(url string) string
50+ SetSourceVersion(url string, version string) error
4851 }
4952
5053 // FilterCompiler defines the interface for filter compilation
@@ -71,7 +74,7 @@ type CompiledFilters struct {
7174
7275 // NewFilterManager creates a new filter manager
7376 func NewFilterManager(store FilterStore, compiler FilterCompiler) *FilterManager {
74- return &FilterManager{
77+ fm := &FilterManager{
7578 store: store,
7679 compiler: compiler,
7780 cosmeticInjector: cosmetic.NewCosmeticInjector(),
@@ -79,6 +82,9 @@ func NewFilterManager(store FilterStore, compiler FilterCompiler) *FilterManager
7982 compiled: &CompiledFilters{},
8083 onFiltersReady: nil,
8184 }
85+ // Initialize updater with reference to manager
86+ fm.updater = NewFilterUpdater(fm)
87+ return fm
8288 }
8389
8490 // updateWhitelistCache rebuilds the cached whitelist rules from config if needed
@@ -175,10 +181,10 @@ func (fm *FilterManager) compileIfNeeded(ctx context.Context) {
175181 }
176182
177183 func (fm *FilterManager) compileFromSources(ctx context.Context) {
178- sources := []string{
179- "https://easylist.to/easylist/easylist.txt",
180- "https://easylist.to/easylist/easyprivacy.txt",
181- "https://easylist-downloads.adblockplus.org/liste_fr.txt", // For French sites like lesnumeriques.com
184+ sources := config.Get().ContentFiltering.FilterLists
185+ if len(sources) == 0 {
186+ logging.Warn("[filtering] No filter lists configured")
187+ return
182188 }
183189
184190 var wg sync.WaitGroup
@@ -403,6 +409,25 @@ func (fm *FilterManager) applyFilters(filters *CompiledFilters) {
403409 }
404410
405411 func (fm *FilterManager) startUpdateLoop(ctx context.Context) {
412+ // Wait 2 minutes after startup before first version check
413+ // This ensures browser startup is not impacted
414+ startupDelay := 2 * time.Minute
415+ logging.Debug("[filtering] Waiting 2 minutes before first filter update check")
416+
417+ select {
418+ case <-time.After(startupDelay):
419+ // Startup delay complete, proceed with first check
420+ case <-ctx.Done():
421+ return
422+ }
423+
424+ // Perform initial version check
425+ logging.Info("[filtering] Performing initial filter version check")
426+ if fm.updater != nil {
427+ go fm.updater.CheckAndUpdate(ctx)
428+ }
429+
430+ // Then check periodically (every 24 hours)
406431 ticker := time.NewTicker(fm.updateInterval)
407432 defer ticker.Stop()
408433
@@ -411,12 +436,9 @@ func (fm *FilterManager) startUpdateLoop(ctx context.Context) {
411436 case <-ctx.Done():
412437 return
413438 case <-ticker.C:
414- // Check if cache needs update and schedule background update
415- if exists, lastModified, _ := fm.store.GetCacheInfo(); exists {
416- if time.Since(lastModified) > fm.updateInterval {
417- logging.Info("[filtering] Periodic update triggered")
418- go fm.updateFiltersInBackground(ctx)
419- }
439+ logging.Info("[filtering] Periodic filter version check")
440+ if fm.updater != nil {
441+ go fm.updater.CheckAndUpdate(ctx)
420442 }
421443 }
422444 }
@@ -426,11 +448,10 @@ func (fm *FilterManager) startUpdateLoop(ctx context.Context) {
426448 func (fm *FilterManager) updateFiltersInBackground(ctx context.Context) {
427449 logging.Info("[filtering] Starting background filter update")
428450
429- // Compile new filters
430- sources := []string{
431- "https://easylist.to/easylist/easylist.txt",
432- "https://easylist.to/easylist/easyprivacy.txt",
433- "https://easylist-downloads.adblockplus.org/liste_fr.txt", // For French sites like lesnumeriques.com
451+ sources := config.Get().ContentFiltering.FilterLists
452+ if len(sources) == 0 {
453+ logging.Warn("[filtering] No filter lists configured")
454+ return
434455 }
435456
436457 var wg sync.WaitGroup
@@ -31,6 +31,7 @@ type FilterManager struct {
31 store FilterStore31 store FilterStore
32 compiler FilterCompiler32 compiler FilterCompiler
33 cosmeticInjector *cosmetic.CosmeticInjector33 cosmeticInjector *cosmetic.CosmeticInjector
34+ updater *FilterUpdater
34 updateInterval time.Duration35 updateInterval time.Duration
35 compileMutex sync.RWMutex36 compileMutex sync.RWMutex
36 compiled *CompiledFilters37 compiled *CompiledFilters
@@ -45,6 +46,8 @@ type FilterStore interface {
45 LoadCached() (*CompiledFilters, error)46 LoadCached() (*CompiledFilters, error)
46 SaveCache(filters *CompiledFilters) error47 SaveCache(filters *CompiledFilters) error
47 GetCacheInfo() (bool, time.Time, error)48 GetCacheInfo() (bool, time.Time, error)
49+ GetSourceVersion(url string) string
50+ SetSourceVersion(url string, version string) error
48 }51 }
49 52
50 // FilterCompiler defines the interface for filter compilation53 // FilterCompiler defines the interface for filter compilation
@@ -71,7 +74,7 @@ type CompiledFilters struct {
71 74
72 // NewFilterManager creates a new filter manager75 // NewFilterManager creates a new filter manager
73 func NewFilterManager(store FilterStore, compiler FilterCompiler) *FilterManager {76 func NewFilterManager(store FilterStore, compiler FilterCompiler) *FilterManager {
74- return &FilterManager{77+ fm := &FilterManager{
75 store: store,78 store: store,
76 compiler: compiler,79 compiler: compiler,
77 cosmeticInjector: cosmetic.NewCosmeticInjector(),80 cosmeticInjector: cosmetic.NewCosmeticInjector(),
@@ -79,6 +82,9 @@ func NewFilterManager(store FilterStore, compiler FilterCompiler) *FilterManager
79 compiled: &CompiledFilters{},82 compiled: &CompiledFilters{},
80 onFiltersReady: nil,83 onFiltersReady: nil,
81 }84 }
85+ // Initialize updater with reference to manager
86+ fm.updater = NewFilterUpdater(fm)
87+ return fm
82 }88 }
83 89
84 // updateWhitelistCache rebuilds the cached whitelist rules from config if needed90 // updateWhitelistCache rebuilds the cached whitelist rules from config if needed
@@ -175,10 +181,10 @@ func (fm *FilterManager) compileIfNeeded(ctx context.Context) {
175 }181 }
176 182
177 func (fm *FilterManager) compileFromSources(ctx context.Context) {183 func (fm *FilterManager) compileFromSources(ctx context.Context) {
178- sources := []string{184+ sources := config.Get().ContentFiltering.FilterLists
179- "https://easylist.to/easylist/easylist.txt",185+ if len(sources) == 0 {
180- "https://easylist.to/easylist/easyprivacy.txt",186+ logging.Warn("[filtering] No filter lists configured")
181- "https://easylist-downloads.adblockplus.org/liste_fr.txt", // For French sites like lesnumeriques.com187+ return
182 }188 }
183 189
184 var wg sync.WaitGroup190 var wg sync.WaitGroup
@@ -403,6 +409,25 @@ func (fm *FilterManager) applyFilters(filters *CompiledFilters) {
403 }409 }
404 410
405 func (fm *FilterManager) startUpdateLoop(ctx context.Context) {411 func (fm *FilterManager) startUpdateLoop(ctx context.Context) {
412+ // Wait 2 minutes after startup before first version check
413+ // This ensures browser startup is not impacted
414+ startupDelay := 2 * time.Minute
415+ logging.Debug("[filtering] Waiting 2 minutes before first filter update check")
416+
417+ select {
418+ case <-time.After(startupDelay):
419+ // Startup delay complete, proceed with first check
420+ case <-ctx.Done():
421+ return
422+ }
423+
424+ // Perform initial version check
425+ logging.Info("[filtering] Performing initial filter version check")
426+ if fm.updater != nil {
427+ go fm.updater.CheckAndUpdate(ctx)
428+ }
429+
430+ // Then check periodically (every 24 hours)
406 ticker := time.NewTicker(fm.updateInterval)431 ticker := time.NewTicker(fm.updateInterval)
407 defer ticker.Stop()432 defer ticker.Stop()
408 433
@@ -411,12 +436,9 @@ func (fm *FilterManager) startUpdateLoop(ctx context.Context) {
411 case <-ctx.Done():436 case <-ctx.Done():
412 return437 return
413 case <-ticker.C:438 case <-ticker.C:
414- // Check if cache needs update and schedule background update439+ logging.Info("[filtering] Periodic filter version check")
415- if exists, lastModified, _ := fm.store.GetCacheInfo(); exists {440+ if fm.updater != nil {
416- if time.Since(lastModified) > fm.updateInterval {441+ go fm.updater.CheckAndUpdate(ctx)
417- logging.Info("[filtering] Periodic update triggered")
418- go fm.updateFiltersInBackground(ctx)
419- }
420 }442 }
421 }443 }
422 }444 }
@@ -426,11 +448,10 @@ func (fm *FilterManager) startUpdateLoop(ctx context.Context) {
426 func (fm *FilterManager) updateFiltersInBackground(ctx context.Context) {448 func (fm *FilterManager) updateFiltersInBackground(ctx context.Context) {
427 logging.Info("[filtering] Starting background filter update")449 logging.Info("[filtering] Starting background filter update")
428 450
429- // Compile new filters451+ sources := config.Get().ContentFiltering.FilterLists
430- sources := []string{452+ if len(sources) == 0 {
431- "https://easylist.to/easylist/easylist.txt",453+ logging.Warn("[filtering] No filter lists configured")
432- "https://easylist.to/easylist/easyprivacy.txt",454+ return
433- "https://easylist-downloads.adblockplus.org/liste_fr.txt", // For French sites like lesnumeriques.com
434 }455 }
435 456
436 var wg sync.WaitGroup457 var wg sync.WaitGroup
M internal/filtering/store.go +47 -5
@@ -26,11 +26,13 @@ type FileFilterStore struct {
2626
2727 // CacheMetadata stores information about the cache
2828 type CacheMetadata struct {
29- Version int `json:"version"`
30- CreatedAt time.Time `json:"created_at"`
31- LastUsed time.Time `json:"last_used"`
32- DataHash string `json:"data_hash"`
33- FilterHashes []string `json:"filter_hashes"` // Hashes of source filter lists
29+ Version int `json:"version"`
30+ CreatedAt time.Time `json:"created_at"`
31+ LastUsed time.Time `json:"last_used"`
32+ DataHash string `json:"data_hash"`
33+ FilterHashes []string `json:"filter_hashes"` // Hashes of source filter lists
34+ SourceVersions map[string]string `json:"source_versions"` // URL -> filter list version
35+ LastCheckTime time.Time `json:"last_check_time"` // Last version check time
3436 }
3537
3638 // NewFileFilterStore creates a new file-based filter store
@@ -206,3 +208,43 @@ func (fs *FileFilterStore) saveMetadata(metadata *CacheMetadata) error {
206208
207209 return nil
208210 }
211+
212+// GetSourceVersion returns the stored version for a filter list URL
213+func (fs *FileFilterStore) GetSourceVersion(url string) string {
214+ metadata, err := fs.loadMetadata()
215+ if err != nil {
216+ return ""
217+ }
218+ if metadata.SourceVersions == nil {
219+ return ""
220+ }
221+ return metadata.SourceVersions[url]
222+}
223+
224+// SetSourceVersion stores the version for a filter list URL
225+func (fs *FileFilterStore) SetSourceVersion(url string, version string) error {
226+ metadata, err := fs.loadMetadata()
227+ if err != nil {
228+ // Create new metadata if none exists
229+ metadata = &CacheMetadata{
230+ Version: cacheVersion,
231+ CreatedAt: time.Now(),
232+ SourceVersions: make(map[string]string),
233+ }
234+ }
235+ if metadata.SourceVersions == nil {
236+ metadata.SourceVersions = make(map[string]string)
237+ }
238+ metadata.SourceVersions[url] = version
239+ metadata.LastCheckTime = time.Now()
240+ return fs.saveMetadata(metadata)
241+}
242+
243+// GetLastCheckTime returns when versions were last checked
244+func (fs *FileFilterStore) GetLastCheckTime() time.Time {
245+ metadata, err := fs.loadMetadata()
246+ if err != nil {
247+ return time.Time{}
248+ }
249+ return metadata.LastCheckTime
250+}
@@ -26,11 +26,13 @@ type FileFilterStore struct {
26 26
27 // CacheMetadata stores information about the cache27 // CacheMetadata stores information about the cache
28 type CacheMetadata struct {28 type CacheMetadata struct {
29- Version int `json:"version"`29+ Version int `json:"version"`
30- CreatedAt time.Time `json:"created_at"`30+ CreatedAt time.Time `json:"created_at"`
31- LastUsed time.Time `json:"last_used"`31+ LastUsed time.Time `json:"last_used"`
32- DataHash string `json:"data_hash"`32+ DataHash string `json:"data_hash"`
33- FilterHashes []string `json:"filter_hashes"` // Hashes of source filter lists33+ FilterHashes []string `json:"filter_hashes"` // Hashes of source filter lists
34+ SourceVersions map[string]string `json:"source_versions"` // URL -> filter list version
35+ LastCheckTime time.Time `json:"last_check_time"` // Last version check time
34 }36 }
35 37
36 // NewFileFilterStore creates a new file-based filter store38 // NewFileFilterStore creates a new file-based filter store
@@ -206,3 +208,43 @@ func (fs *FileFilterStore) saveMetadata(metadata *CacheMetadata) error {
206 208
207 return nil209 return nil
208 }210 }
211+
212+// GetSourceVersion returns the stored version for a filter list URL
213+func (fs *FileFilterStore) GetSourceVersion(url string) string {
214+ metadata, err := fs.loadMetadata()
215+ if err != nil {
216+ return ""
217+ }
218+ if metadata.SourceVersions == nil {
219+ return ""
220+ }
221+ return metadata.SourceVersions[url]
222+}
223+
224+// SetSourceVersion stores the version for a filter list URL
225+func (fs *FileFilterStore) SetSourceVersion(url string, version string) error {
226+ metadata, err := fs.loadMetadata()
227+ if err != nil {
228+ // Create new metadata if none exists
229+ metadata = &CacheMetadata{
230+ Version: cacheVersion,
231+ CreatedAt: time.Now(),
232+ SourceVersions: make(map[string]string),
233+ }
234+ }
235+ if metadata.SourceVersions == nil {
236+ metadata.SourceVersions = make(map[string]string)
237+ }
238+ metadata.SourceVersions[url] = version
239+ metadata.LastCheckTime = time.Now()
240+ return fs.saveMetadata(metadata)
241+}
242+
243+// GetLastCheckTime returns when versions were last checked
244+func (fs *FileFilterStore) GetLastCheckTime() time.Time {
245+ metadata, err := fs.loadMetadata()
246+ if err != nil {
247+ return time.Time{}
248+ }
249+ return metadata.LastCheckTime
250+}
M internal/filtering/updater.go +98 -33
@@ -1,14 +1,18 @@
11 package filtering
22
33 import (
4+ "bufio"
5+ "bytes"
46 "context"
57 "crypto/sha256"
68 "fmt"
9+ "io"
710 "net/http"
811 "strings"
912 "sync"
1013 "time"
1114
15+ "github.com/bnema/dumber/internal/config"
1216 "github.com/bnema/dumber/internal/logging"
1317 )
1418
@@ -65,9 +69,13 @@ func (fu *FilterUpdater) CheckAndUpdate(ctx context.Context) error {
6569 fu.updateMutex.Lock()
6670 defer fu.updateMutex.Unlock()
6771
68- sources := []string{
69- "https://easylist.to/easylist/easylist.txt",
70- "https://easylist.to/easylist/easyprivacy.txt",
72+ startTime := time.Now()
73+ logging.Info("[filtering] Starting filter list update check...")
74+
75+ sources := config.Get().ContentFiltering.FilterLists
76+ if len(sources) == 0 {
77+ logging.Warn("[filtering] No filter lists configured")
78+ return nil
7179 }
7280
7381 // Check for updates concurrently
@@ -114,10 +122,11 @@ func (fu *FilterUpdater) CheckAndUpdate(ctx context.Context) error {
114122 updateCount++
115123 }
116124
125+ elapsed := time.Since(startTime)
117126 if updateCount > 0 {
118- logging.Info(fmt.Sprintf("Applied %d filter list updates", updateCount))
127+ logging.Info(fmt.Sprintf("[filtering] Update complete: applied %d filter list updates in %v", updateCount, elapsed.Round(time.Millisecond)))
119128 } else {
120- logging.Debug("No filter list updates available")
129+ logging.Info(fmt.Sprintf("[filtering] Update complete: all filters up to date (checked in %v)", elapsed.Round(time.Millisecond)))
121130 }
122131
123132 return nil
@@ -125,52 +134,84 @@ func (fu *FilterUpdater) CheckAndUpdate(ctx context.Context) error {
125134
126135 // checkSingleUpdate checks if a single filter list needs updating
127136 func (fu *FilterUpdater) checkSingleUpdate(ctx context.Context, url string) (*FilterUpdate, error) {
128- // Create HEAD request to check if content has changed
129- req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
137+ // Check version using lightweight Range request
138+ if !fu.needsUpdate(ctx, url) {
139+ return nil, nil // No update needed
140+ }
141+
142+ // Download the full content
143+ return fu.downloadUpdate(ctx, url, "", "")
144+}
145+
146+// fetchRemoteVersion fetches only the header of a filter list to extract version
147+// Uses HTTP Range request to minimize bandwidth (fetches ~1KB instead of ~100KB+)
148+func (fu *FilterUpdater) fetchRemoteVersion(ctx context.Context, url string) (string, error) {
149+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
130150 if err != nil {
131- return nil, fmt.Errorf("failed to create HEAD request: %w", err)
151+ return "", fmt.Errorf("failed to create request: %w", err)
132152 }
133153
134- // Add headers to support conditional requests
154+ // Request only first 1024 bytes (enough to get version header)
155+ req.Header.Set("Range", "bytes=0-1023")
135156 req.Header.Set("User-Agent", "Dumber Browser/1.0 Filter Updater")
136157
137158 resp, err := fu.httpClient.Do(req)
138159 if err != nil {
139- return nil, fmt.Errorf("HEAD request failed: %w", err)
160+ return "", fmt.Errorf("range request failed: %w", err)
140161 }
141162 defer func() {
142- if err := resp.Body.Close(); err != nil {
143- logging.Warn(fmt.Sprintf("warning: failed to close response body: %v", err))
163+ if closeErr := resp.Body.Close(); closeErr != nil {
164+ logging.Warn(fmt.Sprintf("warning: failed to close response body: %v", closeErr))
144165 }
145166 }()
146167
147- if resp.StatusCode != http.StatusOK {
148- return nil, fmt.Errorf("HEAD request returned %d", resp.StatusCode)
168+ // Accept both 200 (server ignores Range) and 206 (Partial Content)
169+ if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
170+ return "", fmt.Errorf("request returned %d", resp.StatusCode)
149171 }
150172
151- // Get metadata from response headers
152- lastModified := resp.Header.Get("Last-Modified")
153- etag := resp.Header.Get("ETag")
154- contentLength := resp.ContentLength
173+ // Read the response (limited to 1KB even if server ignores Range)
174+ data, err := io.ReadAll(io.LimitReader(resp.Body, 1024))
175+ if err != nil {
176+ return "", fmt.Errorf("failed to read response: %w", err)
177+ }
155178
156- // Check if we need to download full content
157- needsUpdate := fu.needsUpdate(url, lastModified, etag, contentLength)
158- if !needsUpdate {
159- return nil, nil // No update needed
179+ // Parse version from header (format: "! Version: 202511291248")
180+ scanner := bufio.NewScanner(bytes.NewReader(data))
181+ for scanner.Scan() {
182+ line := scanner.Text()
183+ if strings.HasPrefix(line, "! Version:") {
184+ version := strings.TrimSpace(strings.TrimPrefix(line, "! Version:"))
185+ return version, nil
186+ }
160187 }
161188
162- // Download the full content
163- return fu.downloadUpdate(ctx, url, lastModified, etag)
189+ return "", fmt.Errorf("version header not found in filter list")
164190 }
165191
166-// needsUpdate determines if a filter list needs updating based on metadata
167-func (fu *FilterUpdater) needsUpdate(url, lastModified, etag string, contentLength int64) bool {
168- // For now, we'll implement a simple time-based check
169- // In a full implementation, this would check against stored metadata
192+// needsUpdate determines if a filter list needs updating based on version comparison
193+func (fu *FilterUpdater) needsUpdate(ctx context.Context, url string) bool {
194+ remoteVersion, err := fu.fetchRemoteVersion(ctx, url)
195+ if err != nil {
196+ logging.Debug(fmt.Sprintf("Failed to fetch remote version for %s: %v", url, err))
197+ // On error, assume update needed to be safe
198+ return true
199+ }
200+
201+ storedVersion := fu.manager.store.GetSourceVersion(url)
202+ if storedVersion == "" {
203+ logging.Debug(fmt.Sprintf("No stored version for %s, update needed", url))
204+ return true
205+ }
170206
171- // Always check for updates if we don't have cached metadata
172- // In practice, this would compare with stored filter list metadata
173- return true
207+ needsUpdate := remoteVersion != storedVersion
208+ if needsUpdate {
209+ logging.Info(fmt.Sprintf("Filter %s has new version: %s (stored: %s)", url, remoteVersion, storedVersion))
210+ } else {
211+ logging.Debug(fmt.Sprintf("Filter %s is up to date: %s", url, remoteVersion))
212+ }
213+
214+ return needsUpdate
174215 }
175216
176217 // downloadUpdate downloads the updated filter list and creates a diff
@@ -197,8 +238,12 @@ func (fu *FilterUpdater) downloadUpdate(ctx context.Context, url, lastModified,
197238 return nil, fmt.Errorf("GET request returned %d", resp.StatusCode)
198239 }
199240
200- // Read content
201- currentContent := make([]byte, 0, resp.ContentLength)
241+ // Read content (handle ContentLength=-1 when server uses chunked encoding)
242+ initialCap := resp.ContentLength
243+ if initialCap < 0 {
244+ initialCap = 64 * 1024 // 64KB initial capacity
245+ }
246+ currentContent := make([]byte, 0, initialCap)
202247 buffer := make([]byte, 8192)
203248
204249 for {
@@ -407,10 +452,30 @@ func (fu *FilterUpdater) applyUpdate(update *FilterUpdate) error {
407452 // Store the updated content for future diffs
408453 fu.storePreviousContent(update.ListID, update.Current)
409454
455+ // Store the version for future update checks
456+ if version := fu.extractVersion(update.Current); version != "" {
457+ if err := fu.manager.store.SetSourceVersion(update.URL, version); err != nil {
458+ logging.Warn(fmt.Sprintf("Failed to store version for %s: %v", update.URL, err))
459+ }
460+ }
461+
410462 logging.Info(fmt.Sprintf("Applied filter update for %s", update.URL))
411463 return nil
412464 }
413465
466+// extractVersion extracts the version from filter list content
467+func (fu *FilterUpdater) extractVersion(content []byte) string {
468+ scanner := bufio.NewScanner(bytes.NewReader(content))
469+ // Only scan first 20 lines for version header
470+ for i := 0; i < 20 && scanner.Scan(); i++ {
471+ line := scanner.Text()
472+ if strings.HasPrefix(line, "! Version:") {
473+ return strings.TrimSpace(strings.TrimPrefix(line, "! Version:"))
474+ }
475+ }
476+ return ""
477+}
478+
414479 // getListID generates a unique ID for a filter list URL
415480 func (fu *FilterUpdater) getListID(url string) string {
416481 hash := sha256.Sum256([]byte(url))
@@ -1,14 +1,18 @@
1 package filtering1 package filtering
2 2
3 import (3 import (
4+ "bufio"
5+ "bytes"
4 "context"6 "context"
5 "crypto/sha256"7 "crypto/sha256"
6 "fmt"8 "fmt"
9+ "io"
7 "net/http"10 "net/http"
8 "strings"11 "strings"
9 "sync"12 "sync"
10 "time"13 "time"
11 14
15+ "github.com/bnema/dumber/internal/config"
12 "github.com/bnema/dumber/internal/logging"16 "github.com/bnema/dumber/internal/logging"
13 )17 )
14 18
@@ -65,9 +69,13 @@ func (fu *FilterUpdater) CheckAndUpdate(ctx context.Context) error {
65 fu.updateMutex.Lock()69 fu.updateMutex.Lock()
66 defer fu.updateMutex.Unlock()70 defer fu.updateMutex.Unlock()
67 71
68- sources := []string{72+ startTime := time.Now()
69- "https://easylist.to/easylist/easylist.txt",73+ logging.Info("[filtering] Starting filter list update check...")
70- "https://easylist.to/easylist/easyprivacy.txt",74+
75+ sources := config.Get().ContentFiltering.FilterLists
76+ if len(sources) == 0 {
77+ logging.Warn("[filtering] No filter lists configured")
78+ return nil
71 }79 }
72 80
73 // Check for updates concurrently81 // Check for updates concurrently
@@ -114,10 +122,11 @@ func (fu *FilterUpdater) CheckAndUpdate(ctx context.Context) error {
114 updateCount++122 updateCount++
115 }123 }
116 124
125+ elapsed := time.Since(startTime)
117 if updateCount > 0 {126 if updateCount > 0 {
118- logging.Info(fmt.Sprintf("Applied %d filter list updates", updateCount))127+ logging.Info(fmt.Sprintf("[filtering] Update complete: applied %d filter list updates in %v", updateCount, elapsed.Round(time.Millisecond)))
119 } else {128 } else {
120- logging.Debug("No filter list updates available")129+ logging.Info(fmt.Sprintf("[filtering] Update complete: all filters up to date (checked in %v)", elapsed.Round(time.Millisecond)))
121 }130 }
122 131
123 return nil132 return nil
@@ -125,52 +134,84 @@ func (fu *FilterUpdater) CheckAndUpdate(ctx context.Context) error {
125 134
126 // checkSingleUpdate checks if a single filter list needs updating135 // checkSingleUpdate checks if a single filter list needs updating
127 func (fu *FilterUpdater) checkSingleUpdate(ctx context.Context, url string) (*FilterUpdate, error) {136 func (fu *FilterUpdater) checkSingleUpdate(ctx context.Context, url string) (*FilterUpdate, error) {
128- // Create HEAD request to check if content has changed137+ // Check version using lightweight Range request
129- req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)138+ if !fu.needsUpdate(ctx, url) {
139+ return nil, nil // No update needed
140+ }
141+
142+ // Download the full content
143+ return fu.downloadUpdate(ctx, url, "", "")
144+}
145+
146+// fetchRemoteVersion fetches only the header of a filter list to extract version
147+// Uses HTTP Range request to minimize bandwidth (fetches ~1KB instead of ~100KB+)
148+func (fu *FilterUpdater) fetchRemoteVersion(ctx context.Context, url string) (string, error) {
149+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
130 if err != nil {150 if err != nil {
131- return nil, fmt.Errorf("failed to create HEAD request: %w", err)151+ return "", fmt.Errorf("failed to create request: %w", err)
132 }152 }
133 153
134- // Add headers to support conditional requests154+ // Request only first 1024 bytes (enough to get version header)
155+ req.Header.Set("Range", "bytes=0-1023")
135 req.Header.Set("User-Agent", "Dumber Browser/1.0 Filter Updater")156 req.Header.Set("User-Agent", "Dumber Browser/1.0 Filter Updater")
136 157
137 resp, err := fu.httpClient.Do(req)158 resp, err := fu.httpClient.Do(req)
138 if err != nil {159 if err != nil {
139- return nil, fmt.Errorf("HEAD request failed: %w", err)160+ return "", fmt.Errorf("range request failed: %w", err)
140 }161 }
141 defer func() {162 defer func() {
142- if err := resp.Body.Close(); err != nil {163+ if closeErr := resp.Body.Close(); closeErr != nil {
143- logging.Warn(fmt.Sprintf("warning: failed to close response body: %v", err))164+ logging.Warn(fmt.Sprintf("warning: failed to close response body: %v", closeErr))
144 }165 }
145 }()166 }()
146 167
147- if resp.StatusCode != http.StatusOK {168+ // Accept both 200 (server ignores Range) and 206 (Partial Content)
148- return nil, fmt.Errorf("HEAD request returned %d", resp.StatusCode)169+ if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
170+ return "", fmt.Errorf("request returned %d", resp.StatusCode)
149 }171 }
150 172
151- // Get metadata from response headers173+ // Read the response (limited to 1KB even if server ignores Range)
152- lastModified := resp.Header.Get("Last-Modified")174+ data, err := io.ReadAll(io.LimitReader(resp.Body, 1024))
153- etag := resp.Header.Get("ETag")175+ if err != nil {
154- contentLength := resp.ContentLength176+ return "", fmt.Errorf("failed to read response: %w", err)
177+ }
155 178
156- // Check if we need to download full content179+ // Parse version from header (format: "! Version: 202511291248")
157- needsUpdate := fu.needsUpdate(url, lastModified, etag, contentLength)180+ scanner := bufio.NewScanner(bytes.NewReader(data))
158- if !needsUpdate {181+ for scanner.Scan() {
159- return nil, nil // No update needed182+ line := scanner.Text()
183+ if strings.HasPrefix(line, "! Version:") {
184+ version := strings.TrimSpace(strings.TrimPrefix(line, "! Version:"))
185+ return version, nil
186+ }
160 }187 }
161 188
162- // Download the full content189+ return "", fmt.Errorf("version header not found in filter list")
163- return fu.downloadUpdate(ctx, url, lastModified, etag)
164 }190 }
165 191
166-// needsUpdate determines if a filter list needs updating based on metadata192+// needsUpdate determines if a filter list needs updating based on version comparison
167-func (fu *FilterUpdater) needsUpdate(url, lastModified, etag string, contentLength int64) bool {193+func (fu *FilterUpdater) needsUpdate(ctx context.Context, url string) bool {
168- // For now, we'll implement a simple time-based check194+ remoteVersion, err := fu.fetchRemoteVersion(ctx, url)
169- // In a full implementation, this would check against stored metadata195+ if err != nil {
196+ logging.Debug(fmt.Sprintf("Failed to fetch remote version for %s: %v", url, err))
197+ // On error, assume update needed to be safe
198+ return true
199+ }
200+
201+ storedVersion := fu.manager.store.GetSourceVersion(url)
202+ if storedVersion == "" {
203+ logging.Debug(fmt.Sprintf("No stored version for %s, update needed", url))
204+ return true
205+ }
170 206
171- // Always check for updates if we don't have cached metadata207+ needsUpdate := remoteVersion != storedVersion
172- // In practice, this would compare with stored filter list metadata208+ if needsUpdate {
173- return true209+ logging.Info(fmt.Sprintf("Filter %s has new version: %s (stored: %s)", url, remoteVersion, storedVersion))
210+ } else {
211+ logging.Debug(fmt.Sprintf("Filter %s is up to date: %s", url, remoteVersion))
212+ }
213+
214+ return needsUpdate
174 }215 }
175 216
176 // downloadUpdate downloads the updated filter list and creates a diff217 // downloadUpdate downloads the updated filter list and creates a diff
@@ -197,8 +238,12 @@ func (fu *FilterUpdater) downloadUpdate(ctx context.Context, url, lastModified,
197 return nil, fmt.Errorf("GET request returned %d", resp.StatusCode)238 return nil, fmt.Errorf("GET request returned %d", resp.StatusCode)
198 }239 }
199 240
200- // Read content241+ // Read content (handle ContentLength=-1 when server uses chunked encoding)
201- currentContent := make([]byte, 0, resp.ContentLength)242+ initialCap := resp.ContentLength
243+ if initialCap < 0 {
244+ initialCap = 64 * 1024 // 64KB initial capacity
245+ }
246+ currentContent := make([]byte, 0, initialCap)
202 buffer := make([]byte, 8192)247 buffer := make([]byte, 8192)
203 248
204 for {249 for {
@@ -407,10 +452,30 @@ func (fu *FilterUpdater) applyUpdate(update *FilterUpdate) error {
407 // Store the updated content for future diffs452 // Store the updated content for future diffs
408 fu.storePreviousContent(update.ListID, update.Current)453 fu.storePreviousContent(update.ListID, update.Current)
409 454
455+ // Store the version for future update checks
456+ if version := fu.extractVersion(update.Current); version != "" {
457+ if err := fu.manager.store.SetSourceVersion(update.URL, version); err != nil {
458+ logging.Warn(fmt.Sprintf("Failed to store version for %s: %v", update.URL, err))
459+ }
460+ }
461+
410 logging.Info(fmt.Sprintf("Applied filter update for %s", update.URL))462 logging.Info(fmt.Sprintf("Applied filter update for %s", update.URL))
411 return nil463 return nil
412 }464 }
413 465
466+// extractVersion extracts the version from filter list content
467+func (fu *FilterUpdater) extractVersion(content []byte) string {
468+ scanner := bufio.NewScanner(bytes.NewReader(content))
469+ // Only scan first 20 lines for version header
470+ for i := 0; i < 20 && scanner.Scan(); i++ {
471+ line := scanner.Text()
472+ if strings.HasPrefix(line, "! Version:") {
473+ return strings.TrimSpace(strings.TrimPrefix(line, "! Version:"))
474+ }
475+ }
476+ return ""
477+}
478+
414 // getListID generates a unique ID for a filter list URL479 // getListID generates a unique ID for a filter list URL
415 func (fu *FilterUpdater) getListID(url string) string {480 func (fu *FilterUpdater) getListID(url string) string {
416 hash := sha256.Sum256([]byte(url))481 hash := sha256.Sum256([]byte(url))

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 8

Tags 42

↑↓ moveenter openesc close