1package filtering
2
3import (
4 "context"
5 "encoding/json"
6 "io"
7 "net/http"
8 "net/http/httptest"
9 "os"
10 "path/filepath"
11 "strings"
12 "testing"
13 "time"
14
15 "github.com/stretchr/testify/require"
16)
17
18func TestDownloader_DownloadFile_RecoversWhenTargetDirDeletedConcurrently(t *testing.T) {
19 t.Parallel()
20
21 cacheDir := filepath.Join(t.TempDir(), "cache")
22 filename := filepath.Join("nested", "combined-part1.json")
23 localPath := filepath.Join(cacheDir, filename)
24 targetDir := filepath.Dir(localPath)
25
26 require.NoError(t, os.MkdirAll(targetDir, cacheDirPerm))
27
28 downloadStarted := make(chan struct{})
29 allowResponse := make(chan struct{})
30
31 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
32 if r.URL.Path == "/"+filename {
33 close(downloadStarted)
34 <-allowResponse
35 _, _ = io.WriteString(w, `[{"trigger":{"url-filter":"test"},"action":{"type":"block"}}]`)
36 return
37 }
38 http.NotFound(w, r)
39 }))
40 defer server.Close()
41
42 d := NewDownloader(cacheDir)
43 d.baseURL = server.URL
44
45 type result struct {
46 path string
47 n int64
48 err error
49 }
50 resultCh := make(chan result, 1)
51 go func() {
52 path, n, err := d.downloadFile(context.Background(), filename, d.limits.maxFilterFileBytes)
53 resultCh <- result{path: path, n: n, err: err}
54 }()
55
56 <-downloadStarted
57 require.NoError(t, os.RemoveAll(targetDir))
58 close(allowResponse)
59
60 res := <-resultCh
61 require.NoError(t, res.err)
62 require.Equal(t, localPath, res.path)
63 require.Positive(t, res.n)
64
65 content, err := os.ReadFile(localPath)
66 require.NoError(t, err)
67 require.Contains(t, string(content), "url-filter")
68}
69
70func TestDownloader_DownloadFilters_RejectsUnsafeManifestFilenames(t *testing.T) {
71 cacheDir := filepath.Join(t.TempDir(), "cache")
72 requested := make([]string, 0, 1)
73
74 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
75 requested = append(requested, r.URL.Path)
76 if r.URL.Path != "/"+FilterFiles.Manifest {
77 http.NotFound(w, r)
78 return
79 }
80 _, _ = io.WriteString(w, `{"version":"test","combined":{"total_rules":1,"files":["../escape.json"]}}`)
81 }))
82 defer server.Close()
83
84 d := NewDownloader(cacheDir)
85 d.baseURL = server.URL
86
87 paths, err := d.DownloadFilters(context.Background(), nil)
88 require.Error(t, err)
89 require.Nil(t, paths)
90 require.Contains(t, err.Error(), "invalid manifest")
91 require.Equal(t, []string{"/" + FilterFiles.Manifest}, requested)
92}
93
94func TestDownloader_DownloadFilters_AllowsSafeNestedManifestFilenames(t *testing.T) {
95 cacheDir := filepath.Join(t.TempDir(), "cache")
96 filename := "nested/combined-part1.json"
97 body := `[{"trigger":{"url-filter":"test"},"action":{"type":"block"}}]`
98
99 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
100 switch r.URL.Path {
101 case "/" + FilterFiles.Manifest:
102 _, _ = io.WriteString(w, `{"version":"test","combined":{"total_rules":1,"files":["`+filename+`"]}}`)
103 case "/" + filename:
104 _, _ = io.WriteString(w, body)
105 default:
106 http.NotFound(w, r)
107 }
108 }))
109 defer server.Close()
110
111 d := NewDownloader(cacheDir)
112 d.baseURL = server.URL
113
114 paths, err := d.DownloadFilters(context.Background(), nil)
115 require.NoError(t, err)
116
117 wantPath := filepath.Join(cacheDir, "nested", "combined-part1.json")
118 require.Equal(t, []string{wantPath}, paths)
119 content, err := os.ReadFile(wantPath)
120 require.NoError(t, err)
121 require.Equal(t, body, string(content))
122}
123
124func TestDownloader_DownloadFilters_PrunesObsoleteCachedCombinedParts(t *testing.T) {
125 cacheDir := filepath.Join(t.TempDir(), "cache")
126 require.NoError(t, os.MkdirAll(cacheDir, cacheDirPerm))
127 require.NoError(t, os.WriteFile(filepath.Join(cacheDir, "combined-part3.json"), []byte(`[]`), manifestFilePerm))
128 require.NoError(t, os.WriteFile(filepath.Join(cacheDir, "other.json"), []byte(`[]`), manifestFilePerm))
129
130 body := `[{"trigger":{"url-filter":"test"},"action":{"type":"block"}}]`
131 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
132 switch r.URL.Path {
133 case "/" + FilterFiles.Manifest:
134 _, _ = io.WriteString(w, `{"version":"test","combined":{"total_rules":2,"files":["combined-part1.json","combined-part2.json"]}}`)
135 case "/combined-part1.json", "/combined-part2.json":
136 _, _ = io.WriteString(w, body)
137 default:
138 http.NotFound(w, r)
139 }
140 }))
141 defer server.Close()
142
143 d := NewDownloader(cacheDir)
144 d.baseURL = server.URL
145
146 paths, err := d.DownloadFilters(context.Background(), nil)
147 require.NoError(t, err)
148 require.Len(t, paths, 2)
149 require.FileExists(t, filepath.Join(cacheDir, "combined-part1.json"))
150 require.FileExists(t, filepath.Join(cacheDir, "combined-part2.json"))
151 require.NoFileExists(t, filepath.Join(cacheDir, "combined-part3.json"))
152 require.FileExists(t, filepath.Join(cacheDir, "other.json"))
153}
154
155func TestDownloader_FetchManifest_RejectsOversizedManifest(t *testing.T) {
156 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
157 if r.URL.Path != "/"+FilterFiles.Manifest {
158 http.NotFound(w, r)
159 return
160 }
161 _, _ = io.WriteString(w, strings.Repeat("x", 17))
162 }))
163 defer server.Close()
164
165 d := NewDownloader(t.TempDir())
166 d.baseURL = server.URL
167 d.limits.maxManifestBytes = 16
168
169 manifest, err := d.FetchManifest(context.Background())
170 require.Error(t, err)
171 require.Nil(t, manifest)
172 require.Contains(t, err.Error(), "manifest too large")
173}
174
175func TestDownloader_FetchManifest_RejectsTooManyFiles(t *testing.T) {
176 payload, err := json.Marshal(Manifest{
177 Version: "test",
178 Combined: CombinedInfo{
179 TotalRules: 3,
180 Files: []string{"part1.json", "part2.json", "part3.json"},
181 },
182 })
183 require.NoError(t, err)
184
185 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
186 if r.URL.Path != "/"+FilterFiles.Manifest {
187 http.NotFound(w, r)
188 return
189 }
190 _, _ = w.Write(payload)
191 }))
192 defer server.Close()
193
194 d := NewDownloader(t.TempDir())
195 d.baseURL = server.URL
196 d.limits.maxFilterFiles = 2
197
198 manifest, err := d.FetchManifest(context.Background())
199 require.Error(t, err)
200 require.Nil(t, manifest)
201 require.Contains(t, err.Error(), "limit is 2")
202}
203
204func TestDownloader_NeedsUpdate_DetectsSameVersionManifestMetadataChange(t *testing.T) {
205 cacheDir := filepath.Join(t.TempDir(), "cache")
206 require.NoError(t, os.MkdirAll(cacheDir, cacheDirPerm))
207
208 cachedGeneratedAt := time.Date(2026, 3, 28, 4, 0, 0, 0, time.UTC)
209 latestGeneratedAt := time.Date(2026, 3, 28, 5, 0, 0, 0, time.UTC)
210 cachedManifest := Manifest{
211 Version: "2026.03.28",
212 GeneratedAt: cachedGeneratedAt,
213 Combined: CombinedInfo{
214 TotalRules: 3,
215 Files: []string{"combined-part1.json", "combined-part2.json", "combined-part3.json"},
216 },
217 }
218 cachedPayload, err := json.Marshal(cachedManifest)
219 require.NoError(t, err)
220 require.NoError(t, os.WriteFile(filepath.Join(cacheDir, FilterFiles.Manifest), cachedPayload, manifestFilePerm))
221
222 latestManifest := Manifest{
223 Version: "2026.03.28",
224 GeneratedAt: latestGeneratedAt,
225 Combined: CombinedInfo{
226 TotalRules: 2,
227 Files: []string{"combined-part1.json", "combined-part2.json"},
228 },
229 }
230 latestPayload, err := json.Marshal(latestManifest)
231 require.NoError(t, err)
232
233 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
234 if r.URL.Path != "/"+FilterFiles.Manifest {
235 http.NotFound(w, r)
236 return
237 }
238 _, _ = w.Write(latestPayload)
239 }))
240 defer server.Close()
241
242 d := NewDownloader(cacheDir)
243 d.baseURL = server.URL
244
245 needsUpdate, err := d.NeedsUpdate(context.Background())
246 require.NoError(t, err)
247 require.True(t, needsUpdate)
248}
249
250func TestDownloader_DownloadFilters_RejectsOversizedFilterFile(t *testing.T) {
251 cacheDir := filepath.Join(t.TempDir(), "cache")
252 filename := "combined.json"
253 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
254 switch r.URL.Path {
255 case "/" + FilterFiles.Manifest:
256 _, _ = io.WriteString(w, `{"version":"test","combined":{"total_rules":1,"files":["combined.json"]}}`)
257 case "/" + filename:
258 _, _ = io.WriteString(w, "12345")
259 default:
260 http.NotFound(w, r)
261 }
262 }))
263 defer server.Close()
264
265 d := NewDownloader(cacheDir)
266 d.baseURL = server.URL
267 d.limits.maxFilterFileBytes = 4
268 d.limits.maxTotalFilterBytes = 32
269
270 paths, err := d.DownloadFilters(context.Background(), nil)
271 require.Error(t, err)
272 require.Nil(t, paths)
273 require.Contains(t, err.Error(), "too large")
274 require.NoFileExists(t, filepath.Join(cacheDir, filename))
275}
276
277func TestDownloader_DownloadFilters_EnforcesTotalDownloadLimit(t *testing.T) {
278 cacheDir := filepath.Join(t.TempDir(), "cache")
279 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
280 switch r.URL.Path {
281 case "/" + FilterFiles.Manifest:
282 _, _ = io.WriteString(w, `{"version":"test","combined":{"total_rules":2,"files":["part1.json","part2.json"]}}`)
283 case "/part1.json", "/part2.json":
284 _, _ = io.WriteString(w, "12345")
285 default:
286 http.NotFound(w, r)
287 }
288 }))
289 defer server.Close()
290
291 d := NewDownloader(cacheDir)
292 d.baseURL = server.URL
293 d.limits.maxFilterFileBytes = 8
294 d.limits.maxTotalFilterBytes = 8
295
296 paths, err := d.DownloadFilters(context.Background(), nil)
297 require.Error(t, err)
298 require.Nil(t, paths)
299 require.Contains(t, err.Error(), "too large")
300 require.FileExists(t, filepath.Join(cacheDir, "part1.json"))
301 require.NoFileExists(t, filepath.Join(cacheDir, "part2.json"))
302}
303
304func TestDownloader_DownloadFilters_RejectsSymlinkEscape(t *testing.T) {
305 cacheDir := filepath.Join(t.TempDir(), "cache")
306 outsideDir := filepath.Join(t.TempDir(), "outside")
307 require.NoError(t, os.MkdirAll(cacheDir, cacheDirPerm))
308 require.NoError(t, os.MkdirAll(outsideDir, cacheDirPerm))
309 require.NoError(t, os.Symlink(outsideDir, filepath.Join(cacheDir, "nested")))
310
311 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
312 switch r.URL.Path {
313 case "/" + FilterFiles.Manifest:
314 _, _ = io.WriteString(w, `{"version":"test","combined":{"total_rules":1,"files":["nested/combined.json"]}}`)
315 case "/nested/combined.json":
316 _, _ = io.WriteString(w, `[{"trigger":{"url-filter":"test"},"action":{"type":"block"}}]`)
317 default:
318 http.NotFound(w, r)
319 }
320 }))
321 defer server.Close()
322
323 d := NewDownloader(cacheDir)
324 d.baseURL = server.URL
325
326 paths, err := d.DownloadFilters(context.Background(), nil)
327 require.Error(t, err)
328 require.Nil(t, paths)
329 require.Contains(t, err.Error(), "failed to create target dir")
330 require.NoFileExists(t, filepath.Join(outsideDir, "combined.json"))
331}
332
333func TestDownloader_GetCachedFilterPaths_RejectsSymlinkEscape(t *testing.T) {
334 cacheDir := filepath.Join(t.TempDir(), "cache")
335 outsideDir := filepath.Join(t.TempDir(), "outside")
336 require.NoError(t, os.MkdirAll(cacheDir, cacheDirPerm))
337 require.NoError(t, os.MkdirAll(outsideDir, cacheDirPerm))
338 require.NoError(t, os.Symlink(outsideDir, filepath.Join(cacheDir, "nested")))
339 require.NoError(t, os.WriteFile(filepath.Join(outsideDir, "combined.json"), []byte("[]"), manifestFilePerm))
340 require.NoError(t, os.WriteFile(
341 filepath.Join(cacheDir, FilterFiles.Manifest),
342 []byte(`{"version":"test","combined":{"total_rules":1,"files":["nested/combined.json"]}}`),
343 manifestFilePerm,
344 ))
345
346 d := NewDownloader(cacheDir)
347 require.Nil(t, d.GetCachedFilterPaths())
348}
349
350func TestDownloader_RenameTempFile_RequiresDirectoryParent(t *testing.T) {
351 t.Parallel()
352
353 cacheDir := filepath.Join(t.TempDir(), "cache")
354 require.NoError(t, os.MkdirAll(cacheDir, cacheDirPerm))
355
356 tmpFile := filepath.Join(cacheDir, "payload.tmp")
357 require.NoError(t, os.WriteFile(tmpFile, []byte("payload"), manifestFilePerm))
358
359 parentAsFile := filepath.Join(cacheDir, "target")
360 require.NoError(t, os.WriteFile(parentAsFile, []byte("not-a-directory"), manifestFilePerm))
361
362 d := NewDownloader(cacheDir)
363
364 err := d.renameTempFile(tmpFile, "target/filter.json")
365 require.Error(t, err)
366 require.Contains(t, err.Error(), "failed to create target dir")
367}