fix: use manifest to discover filter files instead of hardcoded list

Fetch manifest.json first to get the list of combined filter files
dynamically. This handles changes in the upstream ublock-webkit-filters
repo where the number of split files varies based on entries-per-file
limits (currently 150k). Also read cached manifest for GetCachedFilterPaths
instead of the removed hardcoded list.

commit: 0d670e06df6fcc628c2e9e0ada5b7eabf8b5e91d
author: brice · 7mo ago · 2026-03-27 06:03 UTC
parent: 2158d6c
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

2 files · +31 -23

M internal/infrastructure/filtering/downloader.go +30 -16


@@ -106,8 +106,8 @@ type DownloadProgress struct {
106	106		BytesTotal int64
107	107		}
108	108
109		-	// DownloadFilters downloads filter JSON files from GitHub releases.
110		-	// Returns the paths to the downloaded files.
	109	+	// DownloadFilters fetches the manifest to discover which filter files exist,
	110	+	// then downloads them. Returns the paths to the downloaded files.
111	111		func (d *Downloader) DownloadFilters(ctx context.Context, onProgress func(DownloadProgress)) ([]string, error) {
112	112		log := logging.FromContext(ctx).With().
113	113		Str("component", "filter-downloader").

@@ -118,7 +118,22 @@ func (d *Downloader) DownloadFilters(ctx context.Context, onProgress func(Downlo
118	118		return nil, fmt.Errorf("failed to create cache dir: %w", err)
119	119		}
120	120
121		-	files := FilterFiles.Combined
	121	+	// Fetch manifest to discover which combined files to download
	122	+	manifest, err := d.FetchManifest(ctx)
	123	+	if err != nil {
	124	+	return nil, fmt.Errorf("failed to fetch manifest: %w", err)
	125	+	}
	126	+	if len(manifest.Combined.Files) == 0 {
	127	+	return nil, fmt.Errorf("manifest contains no combined filter files")
	128	+	}
	129	+
	130	+	log.Info().
	131	+	Int("files", len(manifest.Combined.Files)).
	132	+	Int("total_rules", manifest.Combined.TotalRules).
	133	+	Str("version", manifest.Version).
	134	+	Msg("manifest lists filter files")
	135	+
	136	+	files := manifest.Combined.Files
122	137		paths := make([]string, 0, len(files))
123	138		var totalBytes int64
124	139

@@ -149,10 +164,9 @@ func (d *Downloader) DownloadFilters(ctx context.Context, onProgress func(Downlo
149	164		log.Debug().Str("file", filename).Int64("bytes", bytesDownloaded).Msg("downloaded filter file")
150	165		}
151	166
152		-	// Also download and cache the manifest
153		-	if err := d.downloadAndCacheManifest(ctx); err != nil {
	167	+	// Cache the manifest we already fetched
	168	+	if err := d.cacheManifest(manifest); err != nil {
154	169		log.Warn().Err(err).Msg("failed to cache manifest")
155		-	// Non-fatal, continue
156	170		}
157	171
158	172		log.Info().Int("files", len(paths)).Int64("total_bytes", totalBytes).Msg("filter download complete")

@@ -223,13 +237,8 @@ func (*Downloader) renameTempFile(tmpPath, localPath string) error {
223	237		return nil
224	238		}
225	239
226		-	// downloadAndCacheManifest downloads and caches the manifest.
227		-	func (d *Downloader) downloadAndCacheManifest(ctx context.Context) error {
228		-	manifest, err := d.FetchManifest(ctx)
229		-	if err != nil {
230		-	return err
231		-	}
232		-
	240	+	// cacheManifest writes an already-fetched manifest to disk.
	241	+	func (d Downloader) cacheManifest(manifest Manifest) error {
233	242		data, err := json.MarshalIndent(manifest, "", " ")
234	243		if err != nil {
235	244		return fmt.Errorf("failed to marshal manifest: %w", err)

@@ -244,10 +253,15 @@ func (d *Downloader) downloadAndCacheManifest(ctx context.Context) error {
244	253		}
245	254
246	255		// GetCachedFilterPaths returns paths to cached filter JSON files.
247		-	// Returns nil if any files are missing.
	256	+	// Reads the cached manifest to discover which files should exist.
	257	+	// Returns nil if the manifest is missing or any filter file is missing.
248	258		func (d *Downloader) GetCachedFilterPaths() []string {
249		-	paths := make([]string, 0, len(FilterFiles.Combined))
250		-	for _, filename := range FilterFiles.Combined {
	259	+	manifest, err := d.GetCachedManifest()
	260	+	if err != nil \|\| manifest == nil \|\| len(manifest.Combined.Files) == 0 {
	261	+	return nil
	262	+	}
	263	+	paths := make([]string, 0, len(manifest.Combined.Files))
	264	+	for _, filename := range manifest.Combined.Files {
251	265		path := filepath.Join(d.cacheDir, filename)
252	266		if _, err := os.Stat(path); err != nil {
253	267		return nil

 	BytesTotal int64
 }
-// DownloadFilters downloads filter JSON files from GitHub releases.
+// DownloadFilters fetches the manifest to discover which filter files exist,
-// Returns the paths to the downloaded files.
+// then downloads them. Returns the paths to the downloaded files.
 func (d *Downloader) DownloadFilters(ctx context.Context, onProgress func(DownloadProgress)) ([]string, error) {
 	log := logging.FromContext(ctx).With().
 		Str("component", "filter-downloader").
 		return nil, fmt.Errorf("failed to create cache dir: %w", err)
 	}
-	files := FilterFiles.Combined
+	// Fetch manifest to discover which combined files to download
+	manifest, err := d.FetchManifest(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch manifest: %w", err)
+	}
+	if len(manifest.Combined.Files) == 0 {
+		return nil, fmt.Errorf("manifest contains no combined filter files")
+	}
+	log.Info().
+		Int("files", len(manifest.Combined.Files)).
+		Int("total_rules", manifest.Combined.TotalRules).
+		Str("version", manifest.Version).
+		Msg("manifest lists filter files")
+	files := manifest.Combined.Files
 	paths := make([]string, 0, len(files))
 	var totalBytes int64
 		log.Debug().Str("file", filename).Int64("bytes", bytesDownloaded).Msg("downloaded filter file")
 	}
-	// Also download and cache the manifest
+	// Cache the manifest we already fetched
-	if err := d.downloadAndCacheManifest(ctx); err != nil {
+	if err := d.cacheManifest(manifest); err != nil {
 		log.Warn().Err(err).Msg("failed to cache manifest")
-		// Non-fatal, continue
 	}
 	log.Info().Int("files", len(paths)).Int64("total_bytes", totalBytes).Msg("filter download complete")
 	return nil
 }
-// downloadAndCacheManifest downloads and caches the manifest.
+// cacheManifest writes an already-fetched manifest to disk.
-func (d *Downloader) downloadAndCacheManifest(ctx context.Context) error {
+func (d *Downloader) cacheManifest(manifest *Manifest) error {
-	manifest, err := d.FetchManifest(ctx)
-	if err != nil {
-		return err
-	}
 	data, err := json.MarshalIndent(manifest, "", "  ")
 	if err != nil {
 		return fmt.Errorf("failed to marshal manifest: %w", err)
 }
 // GetCachedFilterPaths returns paths to cached filter JSON files.
-// Returns nil if any files are missing.
+// Reads the cached manifest to discover which files should exist.
+// Returns nil if the manifest is missing or any filter file is missing.
 func (d *Downloader) GetCachedFilterPaths() []string {
-	paths := make([]string, 0, len(FilterFiles.Combined))
+	manifest, err := d.GetCachedManifest()
-	for _, filename := range FilterFiles.Combined {
+	if err != nil || manifest == nil || len(manifest.Combined.Files) == 0 {
+		return nil
+	}
+	paths := make([]string, 0, len(manifest.Combined.Files))
+	for _, filename := range manifest.Combined.Files {
 		path := filepath.Join(d.cacheDir, filename)
 		if _, err := os.Stat(path); err != nil {
 			return nil

M internal/infrastructure/filtering/types.go +1 -7


@@ -48,17 +48,11 @@ type CombinedInfo struct {
48	48		Files []string `json:"files"`
49	49		}
50	50
51		-	// FilterFiles defines the files to download from GitHub releases.
	51	+	// FilterFiles defines the well-known file names for GitHub releases.
52	52		var FilterFiles = struct {
53	53		Manifest string
54		-	Combined []string
55	54		}{
56	55		Manifest: "manifest.json",
57		-	Combined: []string{
58		-	"combined-part1.json",
59		-	"combined-part2.json",
60		-	"combined-part3.json",
61		-	},
62	56		}
63	57
64	58		// FilterIdentifier is the identifier used when storing/loading compiled filters.

 	Files      []string `json:"files"`
 }
-// FilterFiles defines the files to download from GitHub releases.
+// FilterFiles defines the well-known file names for GitHub releases.
 var FilterFiles = struct {
 	Manifest string
-	Combined []string
 }{
 	Manifest: "manifest.json",
-	Combined: []string{
-		"combined-part1.json",
-		"combined-part2.json",
-		"combined-part3.json",
-	},
 }
 // FilterIdentifier is the identifier used when storing/loading compiled filters.