1// SPDX-License-Identifier: MIT
2
3package git
4
5import (
6 "bytes"
7 "compress/zlib"
8 "errors"
9 "fmt"
10 "io"
11 "math/rand/v2"
12 "os"
13 "path/filepath"
14 "testing"
15)
16
17// inflateAll reads a whole zlib stream with the wrapper, reusing z.
18func inflateAll(z *zlibReader, src []byte) ([]byte, error) {
19 br := bytes.NewReader(src)
20 if err := z.Reset(br); err != nil {
21 return nil, err
22 }
23 return io.ReadAll(z)
24}
25
26// stdInflate is the reference: compress/zlib on the same bytes.
27func stdInflate(src []byte) ([]byte, error) {
28 zr, err := zlib.NewReader(bytes.NewReader(src))
29 if err != nil {
30 return nil, err
31 }
32 return io.ReadAll(zr)
33}
34
35func zlibLevel(b []byte, level int) []byte {
36 var out bytes.Buffer
37 w, _ := zlib.NewWriterLevel(&out, level)
38 _, _ = w.Write(b)
39 _ = w.Close()
40 return out.Bytes()
41}
42
43// zlibSamples are streams of every block type: stored, fixed and dynamic
44// Huffman, empty, and several blocks.
45func zlibSamples() map[string][]byte {
46 rng := rand.New(rand.NewPCG(1, 2))
47 random := make([]byte, 1200)
48 for i := range random {
49 random[i] = byte(rng.Uint32())
50 }
51 text := textLines(60, 3)
52 return map[string][]byte{
53 "empty": zlibLevel(nil, zlib.DefaultCompression),
54 "tiny": zlibLevel([]byte("a"), zlib.DefaultCompression),
55 "text": zlibLevel(text, zlib.DefaultCompression),
56 "random": zlibLevel(random, zlib.DefaultCompression),
57 "stored": zlibLevel(text, zlib.NoCompression),
58 "huffman": zlibLevel(text, zlib.HuffmanOnly),
59 "fast": zlibLevel(text, zlib.BestSpeed),
60 "big": zlibLevel(bytes.Repeat(text, 3), zlib.BestCompression),
61 }
62}
63
64func TestZlibReaderMatchesStdlib(t *testing.T) {
65 var z zlibReader // one reader for every stream: Reset must clear all state
66 for name, src := range zlibSamples() {
67 want, werr := stdInflate(src)
68 got, err := inflateAll(&z, src)
69 if werr != nil || err != nil || !bytes.Equal(got, want) {
70 t.Errorf("%s: got %d bytes (%v), stdlib %d bytes (%v)", name, len(got), err, len(want), werr)
71 }
72 // EOF is sticky and the checksum was verified.
73 if n, err := z.Read(make([]byte, 1)); n != 0 || err != io.EOF {
74 t.Errorf("%s: Read after the end = %d, %v", name, n, err)
75 }
76 }
77}
78
79func TestZlibReaderHeader(t *testing.T) {
80 good := zlibLevel([]byte("hello"), zlib.DefaultCompression)
81 mk := func(cmf, flg byte) []byte { return append([]byte{cmf, flg}, good[2:]...) }
82 // A valid FCHECK for the given CMF and FLG bits.
83 fix := func(cmf, flg byte) []byte {
84 flg &^= 0x1f
85 flg |= byte(31 - (uint(cmf)<<8|uint(flg))%31)
86 return mk(cmf, flg)
87 }
88 var z zlibReader
89 for _, c := range []struct {
90 name string
91 src []byte
92 want error // nil: accepted
93 }{
94 {"valid", good, nil},
95 {"window 4 KiB", fix(0x28, 0), nil},
96 {"bad check", mk(good[0], good[1]+1), errZlibHeader},
97 {"not deflate", fix(0x79, 0), errZlibHeader},
98 {"window over 32 KiB", fix(0x88, 0), errZlibHeader},
99 {"preset dictionary", fix(0x78, 0x20), errZlibDict},
100 {"empty", nil, io.ErrUnexpectedEOF},
101 {"one byte", good[:1], io.ErrUnexpectedEOF},
102 } {
103 err := z.Reset(bytes.NewReader(c.src))
104 if !errors.Is(err, c.want) && (err != nil || c.want != nil) {
105 t.Errorf("%s: Reset = %v, want %v", c.name, err, c.want)
106 }
107 }
108 // A reader that failed its header recovers on the next Reset.
109 if got, err := inflateAll(&z, good); err != nil || string(got) != "hello" {
110 t.Fatalf("after errors: %q, %v", got, err)
111 }
112}
113
114// Every byte of the stream matters: a bit flipped in the deflate data or in
115// the trailer never yields wrong content without an error. The only flips
116// that go unnoticed are those that leave the content as it was (the unused
117// bits after the last block).
118func TestZlibReaderDetectsCorruption(t *testing.T) {
119 var z zlibReader
120 for name, src := range zlibSamples() {
121 want, err := stdInflate(src)
122 if err != nil {
123 t.Fatal(err)
124 }
125 detected := 0
126 for i := range src {
127 for bit := range 8 {
128 bad := bytes.Clone(src)
129 bad[i] ^= 1 << bit
130 got, err := inflateAll(&z, bad)
131 switch {
132 case err != nil:
133 detected++
134 case !bytes.Equal(got, want):
135 t.Fatalf("%s: bit %d of byte %d/%d changed the content without an error", name, bit, i, len(src))
136 }
137 }
138 }
139 // The 4 trailer bytes are always checked.
140 for i := len(src) - 4; i < len(src); i++ {
141 bad := bytes.Clone(src)
142 bad[i] ^= 0x01
143 if _, err := inflateAll(&z, bad); !errors.Is(err, errZlibChecksum) {
144 t.Errorf("%s: flipped trailer byte %d: %v, want a checksum error", name, i, err)
145 }
146 }
147 if total := len(src) * 8; detected < total*9/10 {
148 t.Errorf("%s: only %d of %d flips detected", name, detected, total)
149 }
150 // A truncated trailer or stream is an error too.
151 for _, n := range []int{len(src) - 1, len(src) - 4, len(src) - 5} {
152 if n < 2 {
153 continue
154 }
155 if _, err := inflateAll(&z, src[:n]); err == nil {
156 t.Errorf("%s: stream cut at %d of %d accepted", name, n, len(src))
157 }
158 }
159 }
160}
161
162// The wrapper answers like compress/zlib on any input, except that it
163// refuses a preset dictionary.
164func FuzzZlibReader(f *testing.F) {
165 for _, src := range zlibSamples() {
166 f.Add(src)
167 if len(src) > 8 {
168 bad := bytes.Clone(src)
169 bad[len(src)-2] ^= 0x40
170 f.Add(bad)
171 f.Add(src[:len(src)-3])
172 }
173 }
174 f.Add([]byte{0x78, 0x20, 0, 0, 0, 1, 0x03, 0x00, 0, 0, 0, 1})
175 f.Add([]byte{0x78})
176 f.Add([]byte{})
177 var z zlibReader
178 warm := zlibLevel([]byte("warm up the reused state"), zlib.DefaultCompression)
179 f.Fuzz(func(t *testing.T, src []byte) {
180 if _, err := inflateAll(&z, warm); err != nil { // dirty state from a good stream
181 t.Fatal(err)
182 }
183 got, err := inflateAll(&z, src)
184 if len(src) >= 2 && src[1]&0x20 != 0 && errors.Is(err, errZlibDict) {
185 return
186 }
187 want, werr := stdInflate(src)
188 if (err == nil) != (werr == nil) {
189 t.Fatalf("wrapper: %v, stdlib: %v", err, werr)
190 }
191 if err == nil && !bytes.Equal(got, want) {
192 t.Fatalf("content differs: %d bytes against %d", len(got), len(want))
193 }
194 })
195}
196
197// flipByteAt flips a bit of the byte at off (from the end when negative).
198func flipByteAt(t *testing.T, path string, off int64) {
199 t.Helper()
200 b, err := os.ReadFile(path)
201 if err != nil {
202 t.Fatal(err)
203 }
204 if off < 0 {
205 off += int64(len(b))
206 }
207 b[off] ^= 0x01
208 if err := os.Chmod(path, 0o644); err != nil {
209 t.Fatal(err)
210 }
211 if err := os.WriteFile(path, b, 0o644); err != nil {
212 t.Fatal(err)
213 }
214}
215
216// corruptObject flips a bit in the deflate data or in the adler32 trailer of
217// the loose or packed object id (packed: repack first).
218func corruptObject(t *testing.T, f *fx, id ObjectID, packed bool, where string) {
219 t.Helper()
220 var file string
221 var entryEnd int64 = -1
222 if packed {
223 f.git("repack", "-adf")
224 f.git("prune-packed")
225 matches, _ := filepath.Glob(filepath.Join(f.dir, "objects/pack/*.pack"))
226 if len(matches) != 1 {
227 t.Fatalf("packs: %v", matches)
228 }
229 file = matches[0]
230 // The blob's entry ends where the next one starts, or at
231 // the pack trailer.
232 r := f.open()
233 p := r.snap.Load().packs[0]
234 pos, ok := p.idx.find(&id)
235 if !ok {
236 t.Fatal("blob not in the pack")
237 }
238 off, _ := p.idx.offset(pos)
239 next := uint64(len(p.data) - packTrailerLen)
240 for i := range p.idx.n {
241 if o, _ := p.idx.offset(i); o > off && o < next {
242 next = o
243 }
244 }
245 entryEnd = int64(next)
246 _ = r.Close()
247 } else {
248 f.git("prune-packed")
249 file = filepath.Join(f.dir, "objects", id.String()[:2], id.String()[2:])
250 }
251 switch {
252 case where == "trailer" && packed:
253 flipByteAt(t, file, entryEnd-1)
254 case where == "trailer":
255 flipByteAt(t, file, -1)
256 case packed:
257 flipByteAt(t, file, entryEnd/2+6) // inside the deflate data
258 default:
259 st, _ := os.Stat(file)
260 flipByteAt(t, file, st.Size()/2)
261 }
262}
263
264// A flipped byte in the compressed content or in the adler32 trailer of a
265// loose or a packed object is ErrCorrupt on every way to read it, since the
266// checksum is the only check of the inflated content.
267func TestCorruptedObjectsAreErrCorrupt(t *testing.T) {
268 content := textLines(400, 9)
269 for _, packed := range []bool{false, true} {
270 for _, where := range []string{"trailer", "deflate"} {
271 t.Run(fmt.Sprintf("packed=%v/%s", packed, where), func(t *testing.T) {
272 f := newFx(t)
273 f.write("f.txt", content, 0o644)
274 f.commit("one")
275 id := mustID(t, f.git("rev-parse", "HEAD:f.txt"))
276 corruptObject(t, f, id, packed, where)
277 r := f.open()
278 if _, _, err := r.ReadObject(id, nil); !errors.Is(err, ErrCorrupt) {
279 t.Errorf("ReadObject = %v, want ErrCorrupt", err)
280 }
281 _, rc, err := r.OpenBlob(id)
282 if err == nil {
283 _, err = io.Copy(io.Discard, rc)
284 _ = rc.Close()
285 }
286 if !errors.Is(err, ErrCorrupt) {
287 t.Errorf("OpenBlob = %v, want ErrCorrupt", err)
288 }
289 })
290 }
291 }
292}
293
294// A corrupt adler32 trailer is reported by readers that stop at the declared
295// size too (io.ReadFull, io.CopyN), not only by those that read to EOF, and
296// the last chunk is not returned along with a nil error.
297func TestOpenBlobTrailerCorruptionOnExactReads(t *testing.T) {
298 for _, packed := range []bool{false, true} {
299 for _, size := range []int{0, 1, 30, 20000} {
300 t.Run(fmt.Sprintf("packed=%v/size=%d", packed, size), func(t *testing.T) {
301 f := newFx(t)
302 content := textLines(size/16+1, 4)[:size]
303 f.write("f.txt", content, 0o644)
304 f.commit("one")
305 id := mustID(t, f.git("rev-parse", "HEAD:f.txt"))
306 corruptObject(t, f, id, packed, "trailer")
307 r := f.open()
308 open := func() io.ReadCloser {
309 n, rc, err := r.OpenBlob(id)
310 if err != nil || n != int64(size) {
311 t.Fatalf("OpenBlob = %d %v", n, err)
312 }
313 return rc
314 }
315 if size > 0 {
316 rc := open()
317 buf := make([]byte, size)
318 if n, err := io.ReadFull(rc, buf); !errors.Is(err, ErrCorrupt) {
319 t.Errorf("io.ReadFull = %d %v, want ErrCorrupt", n, err)
320 }
321 _ = rc.Close()
322
323 rc = open()
324 if n, err := io.CopyN(io.Discard, rc, int64(size)); !errors.Is(err, ErrCorrupt) {
325 t.Errorf("io.CopyN = %d %v, want ErrCorrupt", n, err)
326 }
327 _ = rc.Close()
328
329 rc = open()
330 if _, err := io.Copy(io.Discard, rc); !errors.Is(err, ErrCorrupt) {
331 t.Errorf("io.Copy = %v, want ErrCorrupt", err)
332 }
333 // The error is sticky.
334 if _, err := rc.Read(make([]byte, 1)); !errors.Is(err, ErrCorrupt) {
335 t.Errorf("Read after the error = %v, want ErrCorrupt", err)
336 }
337 _ = rc.Close()
338
339 // One byte at a time: the last Read fails.
340 rc = open()
341 var err error
342 one := make([]byte, 1)
343 for i := 0; i < size && err == nil; i++ {
344 _, err = rc.Read(one)
345 }
346 if !errors.Is(err, ErrCorrupt) {
347 t.Errorf("byte-wise read = %v, want ErrCorrupt", err)
348 }
349 _ = rc.Close()
350 }
351 if size == 0 {
352 // Nothing is read from an empty blob: Close reports it.
353 rc := open()
354 if err := rc.Close(); !errors.Is(err, ErrCorrupt) {
355 t.Errorf("Close = %v, want ErrCorrupt", err)
356 }
357 }
358 })
359 }
360 }
361}
362
363// An intact blob read to exactly its size is returned whole and Close is nil,
364// and an early Close is not an error.
365func TestOpenBlobExactReadsIntact(t *testing.T) {
366 for _, packed := range []bool{false, true} {
367 f := newFx(t)
368 content := textLines(1500, 5)
369 f.write("f.txt", content, 0o644)
370 f.write("empty", nil, 0o644)
371 f.commit("one")
372 if packed {
373 f.git("repack", "-adf")
374 }
375 r := f.open()
376 id := mustID(t, f.git("rev-parse", "HEAD:f.txt"))
377 n, rc, err := r.OpenBlob(id)
378 if err != nil {
379 t.Fatal(err)
380 }
381 buf := make([]byte, n)
382 if _, err := io.ReadFull(rc, buf); err != nil || !bytes.Equal(buf, content) {
383 t.Fatalf("packed=%v: ReadFull: %v", packed, err)
384 }
385 if err := rc.Close(); err != nil {
386 t.Errorf("Close = %v", err)
387 }
388 _, rc, _ = r.OpenBlob(id)
389 if _, err := io.CopyN(io.Discard, rc, 10); err != nil {
390 t.Fatal(err)
391 }
392 if err := rc.Close(); err != nil {
393 t.Errorf("early Close = %v", err)
394 }
395 _, rc, err = r.OpenBlob(mustID(t, f.git("rev-parse", "HEAD:empty")))
396 if err != nil {
397 t.Fatal(err)
398 }
399 if err := rc.Close(); err != nil {
400 t.Errorf("Close of an intact empty blob = %v", err)
401 }
402 }
403}