detached edd9b2d

test(graph): git conformance, synthetic corruption, fuzz seeds and benchmarks for commit-graph

commit
edd9b2dc0368e134b8ff3115c477dbafbc968b6f
author
brice · · 2026-10-10 20:00 UTC
parent
5d84510
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
11 files · +1795 -0
Unified
UnifiedSplit
A bloom_test.go +546 -0
@@ -0,0 +1,546 @@
1+// SPDX-License-Identifier: MIT
2+
3+package git
4+
5+import (
6+ "encoding/binary"
7+ "fmt"
8+ "os"
9+ "path/filepath"
10+ "strconv"
11+ "strings"
12+ "testing"
13+)
14+
15+func TestMurmur3Vectors(t *testing.T) {
16+ // Reference values of MurmurHash3_x86_32 (same vectors as git t0095).
17+ for _, c := range []struct {
18+ in string
19+ seed uint32
20+ want uint32
21+ }{
22+ {"", 0, 0},
23+ {"", 1, 0x514e28b7},
24+ {"", 0xffffffff, 0x81f16f39},
25+ {"test", 0, 0xba6bd213},
26+ {"test", 0x9747b28c, 0x704b81dc},
27+ {"Hello, world!", 0, 0xc0363e43},
28+ {"Hello, world!", 0x9747b28c, 0x24884cba},
29+ {"The quick brown fox jumps over the lazy dog", 0, 0x2e4ff723},
30+ {"The quick brown fox jumps over the lazy dog", 0x9747b28c, 0x2fa826cd},
31+ {"aaaa", 0x9747b28c, 0x5a97808a},
32+ {"aaa", 0x9747b28c, 0x283e0130},
33+ {"aa", 0x9747b28c, 0x5d211726},
34+ {"a", 0x9747b28c, 0x7fa09ea6},
35+ } {
36+ if got := murmur3Seeded(c.seed, []byte(c.in), false); got != c.want {
37+ t.Errorf("murmur3(%q, %#x) = %#x, want %#x", c.in, c.seed, got, c.want)
38+ }
39+ // ASCII input is identical in both versions.
40+ if got := murmur3Seeded(c.seed, []byte(c.in), true); got != c.want {
41+ t.Errorf("signed murmur3(%q, %#x) = %#x, want %#x", c.in, c.seed, got, c.want)
42+ }
43+ }
44+}
45+
46+func TestBloomKeyVersions(t *testing.T) {
47+ a := newBloomKey([]byte("src/main.go"))
48+ if a.v1 != a.v2 {
49+ t.Error("ASCII path must hash identically in v1 and v2")
50+ }
51+ b := newBloomKey([]byte("caf\xc3\xa9/men\xc3\xbc.txt"))
52+ if b.v1 == b.v2 {
53+ t.Error("high-bit path must hash differently in v1 and v2")
54+ }
55+ if newBloomKeyString("x") != newBloomKey([]byte("x")) {
56+ t.Error("string and bytes keys differ")
57+ }
58+}
59+
60+// bloomPaths returns the keys git inserts for a changed path: the path and
61+// every leading directory.
62+func bloomPaths(p string) []string {
63+ out := []string{p}
64+ for i := len(p) - 1; i > 0; i-- {
65+ if p[i] == '/' {
66+ out = append(out, p[:i])
67+ }
68+ }
69+ return out
70+}
71+
72+// changedPaths lists the paths that differ between c and its first parent
73+// (the empty tree for a root commit), as git sees them.
74+func changedPaths(t testing.TB, dir string, c *refCommit) []string {
75+ t.Helper()
76+ var out string
77+ if len(c.parents) == 0 {
78+ out = graphGit(t, dir, 0, "diff-tree", "-r", "-z", "--root", "--name-only", "--no-commit-id", fmt.Sprintf("%x", c.id))
79+ } else {
80+ out = graphGit(t, dir, 0, "diff-tree", "-r", "-z", "--name-only", "--no-commit-id", fmt.Sprintf("%x", c.parents[0]), fmt.Sprintf("%x", c.id))
81+ }
82+ var paths []string
83+ for _, p := range strings.Split(out, "\x00") {
84+ if p != "" {
85+ paths = append(paths, p)
86+ }
87+ }
88+ return paths
89+}
90+
91+// checkBloomAgainstGit verifies the Bloom filters of every commit of g for
92+// the changed paths reported by git: no false negatives for the path or any
93+// leading directory, and unchanged paths mostly rejected.
94+func checkBloomAgainstGit(t *testing.T, dir string, g *commitGraph, cs []refCommit, c gcase) {
95+ t.Helper()
96+ checkBloomSample(t, dir, g, cs, c.bloom, c.bloomVer, 1)
97+}
98+
99+// checkBloomSample is checkBloomAgainstGit for every step-th commit. wantVer
100+// is the BDAT version expected when bloom is true.
101+func checkBloomSample(t *testing.T, dir string, g *commitGraph, cs []refCommit, bloom bool, wantVer uint32, step int) {
102+ t.Helper()
103+ var checked, queries, unchangedQueries, falsePositives, unknown int
104+ for i := 0; i < len(cs); i += step {
105+ cm := &cs[i]
106+ pos, ok := g.lookup(cm.id)
107+ if !ok {
108+ t.Fatalf("lookup(%x)", cm.id)
109+ }
110+ paths := changedPaths(t, dir, cm)
111+ if !bloom {
112+ if g.hasBloom(pos) {
113+ t.Fatalf("commit %x has a Bloom filter in a graph written without --changed-paths", cm.id)
114+ }
115+ for _, p := range paths {
116+ if maybe, ok := g.maybeChanged(pos, []byte(p)); ok || maybe {
117+ t.Fatalf("maybeChanged = %v,%v without filters", maybe, ok)
118+ }
119+ }
120+ continue
121+ }
122+ l, _ := g.layerOf(pos)
123+ if !l.hasBloom || l.bloomVersion != wantVer {
124+ t.Fatalf("layer bloom = %v version %d, want version %d", l.hasBloom, l.bloomVersion, wantVer)
125+ }
126+ if len(paths) > 512 {
127+ if _, ok := g.maybeChanged(pos, []byte(paths[0])); ok {
128+ t.Fatalf("commit with %d changes should be unknown", len(paths))
129+ }
130+ continue
131+ }
132+ set := map[string]bool{}
133+ for _, p := range paths {
134+ for _, k := range bloomPaths(p) {
135+ set[k] = true
136+ }
137+ }
138+ if len(paths) == 0 {
139+ // git writes the one-byte all-zero filter for "nothing changed".
140+ if maybe, ok := g.maybeChanged(pos, []byte("README")); !ok || maybe {
141+ t.Fatalf("empty change set: maybe=%v ok=%v", maybe, ok)
142+ }
143+ }
144+ for k := range set {
145+ key := newBloomKey([]byte(k))
146+ maybe, ok := g.maybeChangedKey(pos, &key)
147+ if !ok {
148+ unknown++
149+ continue
150+ }
151+ if !maybe {
152+ t.Fatalf("commit %x: false negative for %q (changed: %q)", cm.id, k, paths)
153+ }
154+ queries++
155+ }
156+ // Unchanged paths: names that cannot be in the set.
157+ for j := range 40 {
158+ p := fmt.Sprintf("no/such/dir%d/file-%d.txt", j, j*7+i)
159+ if set[p] {
160+ continue
161+ }
162+ for _, q := range bloomPaths(p) {
163+ if set[q] {
164+ continue
165+ }
166+ key := newBloomKey([]byte(q))
167+ maybe, ok := g.maybeChangedKey(pos, &key)
168+ if !ok {
169+ continue
170+ }
171+ unchangedQueries++
172+ if maybe {
173+ falsePositives++
174+ }
175+ }
176+ }
177+ checked++
178+ }
179+ if !bloom {
180+ return
181+ }
182+ if checked == 0 || queries == 0 {
183+ t.Fatalf("no Bloom queries ran (checked %d commits)", checked)
184+ }
185+ rate := float64(falsePositives) / float64(max(unchangedQueries, 1))
186+ t.Logf("bloom v%d: %d commits, %d positive queries (%d unknown), false positives %d/%d = %.2f%%",
187+ wantVer, checked, queries, unknown, falsePositives, unchangedQueries, rate*100)
188+ if rate > 0.10 {
189+ t.Errorf("false positive rate %.1f%% is too high for 10 bits per entry", rate*100)
190+ }
191+}
192+
193+// With a v1 graph, the v2 hash must not be used for high-bit paths (that is
194+// the bug v2 fixes); with v2, both agree on being negative-free.
195+func TestBloomHighBitVersions(t *testing.T) {
196+ for _, ver := range []uint32{1, 2} {
197+ t.Run(fmt.Sprintf("v%d", ver), func(t *testing.T) {
198+ r := newGTRepo(t)
199+ for i := range 30 {
200+ p := fmt.Sprintf("d\xe9\xff%d/\xc3\xa9\xfe%d.txt", i, i)
201+ r.commit(p, "x\n")
202+ }
203+ r.git("-c", fmt.Sprintf("commitGraph.changedPathsVersion=%d", ver),
204+ "commit-graph", "write", "--reachable", "--changed-paths")
205+ g, err := openCommitGraph(r.objects())
206+ if err != nil {
207+ t.Fatal(err)
208+ }
209+ defer func() { _ = g.close() }()
210+ cs := gitCommits(t, r.dir)
211+ wrong := 0
212+ for _, cm := range cs {
213+ pos, _ := g.lookup(cm.id)
214+ for _, p := range changedPaths(t, r.dir, &cm) {
215+ for _, k := range bloomPaths(p) {
216+ if maybe, ok := g.maybeChanged(pos, []byte(k)); !ok || !maybe {
217+ t.Fatalf("v%d: %q not found (maybe=%v ok=%v)", ver, k, maybe, ok)
218+ }
219+ // Probe with the other version's hash.
220+ key := newBloomKey([]byte(k))
221+ other := key
222+ other.v1, other.v2 = key.v2, key.v1
223+ if maybe, ok := g.maybeChangedKey(pos, &other); ok && !maybe {
224+ wrong++
225+ }
226+ }
227+ }
228+ }
229+ if wrong == 0 {
230+ t.Errorf("using the other hash version never failed: v%d not distinguished", ver)
231+ }
232+ })
233+ }
234+}
235+
236+// ---- synthetic filters ----------------------------------------------------
237+
238+// synthBloom builds a graph with the given filters (nil entry: empty).
239+func synthBloom(t testing.TB, version, hashes, bitsPer uint32, filters [][]byte) *commitGraph {
240+ n := len(filters)
241+ s := &synthLayer{ids: synthIDs(n)}
242+ hdr := binary.BigEndian.AppendUint32(nil, version)
243+ hdr = binary.BigEndian.AppendUint32(hdr, hashes)
244+ hdr = binary.BigEndian.AppendUint32(hdr, bitsPer)
245+ var total uint32
246+ s.bdat = hdr
247+ for _, f := range filters {
248+ total += uint32(len(f))
249+ s.bidx = append(s.bidx, total)
250+ s.bdat = append(s.bdat, f...)
251+ }
252+ return mustParseSynth(t, s.bytes())
253+}
254+
255+// setBloomBits sets the bits of path in filter f.
256+func setBloomBits(f []byte, hashes uint32, k bloomKey, v1 bool) {
257+ h := k.v2
258+ if v1 {
259+ h = k.v1
260+ }
261+ for i := uint32(0); i < hashes; i++ {
262+ bit := uint64(h[0]+i*h[1]) % uint64(len(f)*8)
263+ f[bit>>3] |= 1 << (bit & 7)
264+ }
265+}
266+
267+func TestBloomSynthetic(t *testing.T) {
268+ const hashes = 7
269+ f := make([]byte, 16)
270+ setBloomBits(f, hashes, newBloomKey([]byte("a/b/c.go")), false)
271+ setBloomBits(f, hashes, newBloomKey([]byte("a/b")), false)
272+ setBloomBits(f, hashes, newBloomKey([]byte("a")), false)
273+ filters := [][]byte{
274+ f,
275+ {0x00}, // nothing changed
276+ {0xff}, // too many changes
277+ nil, // no data
278+ f[:8], // different size: still a valid filter
279+ {0x01, 0x02}, // random garbage filter
280+ }
281+ g := synthBloom(t, 2, hashes, 10, filters)
282+ defer func() { _ = g.close() }()
283+ if maybe, ok := g.maybeChanged(0, []byte("a/b/c.go")); !maybe || !ok {
284+ t.Errorf("filter 0: %v %v", maybe, ok)
285+ }
286+ if maybe, ok := g.maybeChanged(0, []byte("zzz")); maybe || !ok {
287+ t.Errorf("filter 0 unrelated: %v %v", maybe, ok)
288+ }
289+ if maybe, ok := g.maybeChanged(1, []byte("a")); maybe || !ok {
290+ t.Errorf("zero filter: %v %v", maybe, ok)
291+ }
292+ if _, ok := g.maybeChanged(2, []byte("a")); ok {
293+ t.Error("0xff filter must be unknown")
294+ }
295+ if _, ok := g.maybeChanged(3, []byte("a")); ok {
296+ t.Error("empty filter must be unknown")
297+ }
298+ if _, ok := g.maybeChanged(g.numCommits(), []byte("a")); ok {
299+ t.Error("bad position must be unknown")
300+ }
301+ if g.hasBloom(2) || !g.hasBloom(1) {
302+ t.Error("hasBloom")
303+ }
304+}
305+
306+func TestBloomTruncated(t *testing.T) {
307+ // BIDX claims more bytes than BDAT holds: unknown, no panic.
308+ s := &synthLayer{ids: synthIDs(3)}
309+ hdr := binary.BigEndian.AppendUint32(nil, 2)
310+ hdr = binary.BigEndian.AppendUint32(hdr, 7)
311+ hdr = binary.BigEndian.AppendUint32(hdr, 10)
312+ s.bdat = append(hdr, 1, 2, 3, 4)
313+ s.bidx = []uint32{4, 4000, 0xffffffff}
314+ g := mustParseSynth(t, s.bytes())
315+ defer func() { _ = g.close() }()
316+ for pos := range uint32(3) {
317+ if _, ok := g.maybeChanged(pos, []byte("x")); pos > 0 && ok {
318+ t.Errorf("pos %d: truncated filter reported known", pos)
319+ }
320+ }
321+ // Non-monotonic offsets.
322+ s.bidx = []uint32{4, 2, 3}
323+ g2 := mustParseSynth(t, s.bytes())
324+ for pos := range uint32(3) {
325+ g2.maybeChanged(pos, []byte("x"))
326+ }
327+}
328+
329+func TestBloomIgnoredSettings(t *testing.T) {
330+ for _, c := range []struct{ version, hashes uint32 }{{3, 7}, {0, 7}, {2, 0}, {2, bloomMaxHashes + 1}, {2, 1 << 31}} {
331+ g := synthBloom(t, c.version, c.hashes, 10, [][]byte{{0x00}, {0x00}})
332+ if g.hasBloom(0) {
333+ t.Errorf("version %d hashes %d should be ignored", c.version, c.hashes)
334+ }
335+ if _, ok := g.maybeChanged(0, []byte("a")); ok {
336+ t.Errorf("version %d hashes %d should be unknown", c.version, c.hashes)
337+ }
338+ }
339+ // BIDX without BDAT, BDAT too short.
340+ s := &synthLayer{ids: synthIDs(2), bidx: []uint32{1, 2}, bdat: []byte{0, 0, 0, 2}}
341+ if _, err := parseGraphLayer(s.bytes()); err == nil {
342+ t.Error("short BDAT accepted")
343+ }
344+}
345+
346+// ---- allocation and fuzz --------------------------------------------------
347+
348+func TestBloomNoAllocs(t *testing.T) {
349+ const hashes = 7
350+ f := make([]byte, 16)
351+ setBloomBits(f, hashes, newBloomKey([]byte("a/b")), false)
352+ g := synthBloom(t, 2, hashes, 10, [][]byte{f, f})
353+ path := []byte("a/b/ünï.go")
354+ var sink bool
355+ if n := testing.AllocsPerRun(100, func() {
356+ k := newBloomKey(path)
357+ m, _ := g.maybeChangedKey(0, &k)
358+ m2, _ := g.maybeChanged(1, path)
359+ sink = sink || m || m2
360+ }); n != 0 {
361+ t.Errorf("allocs = %v", n)
362+ }
363+ _ = sink
364+}
365+
366+func FuzzBloomLookup(f *testing.F) {
367+ f.Add([]byte{0x00}, []byte("a"), uint32(2), uint32(7))
368+ f.Add([]byte{0xff}, []byte("a/b"), uint32(1), uint32(7))
369+ f.Add([]byte{0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0}, []byte("caf\xc3\xa9/x"), uint32(1), uint32(3))
370+ f.Add([]byte{}, []byte{}, uint32(2), uint32(7))
371+ f.Add([]byte{0x80, 0x01}, []byte("\xff\xfe\xfd\xfc\xfb"), uint32(9), uint32(1<<31))
372+ f.Fuzz(func(t *testing.T, filter, path []byte, version, hashes uint32) {
373+ hdr := binary.BigEndian.AppendUint32(nil, version)
374+ hdr = binary.BigEndian.AppendUint32(hdr, hashes)
375+ hdr = binary.BigEndian.AppendUint32(hdr, 10)
376+ s := &synthLayer{ids: synthIDs(2)}
377+ s.bdat = append(hdr, filter...)
378+ // Second filter claims the rest; the first is cut anywhere.
379+ s.bidx = []uint32{uint32(len(filter) / 2), uint32(len(filter))}
380+ l, err := parseGraphLayer(s.bytes())
381+ if err != nil {
382+ return
383+ }
384+ g := &commitGraph{layers: []graphLayer{*l}, total: l.n}
385+ k := newBloomKey(path)
386+ for pos := range uint32(3) {
387+ maybe, ok := g.maybeChangedKey(pos, &k)
388+ if maybe && !ok {
389+ t.Fatal("maybe without ok")
390+ }
391+ }
392+ // A path inserted into a valid filter is always found.
393+ half := len(filter) / 2
394+ if (version == 1 || version == 2) && hashes >= 1 && hashes <= bloomMaxHashes && len(filter)-half >= 1 {
395+ second := append([]byte(nil), filter[half:]...)
396+ setBloomBits(second, hashes, k, version == 1)
397+ s.bdat = append(append(hdr[:12:12], filter[:half]...), second...)
398+ l2, err := parseGraphLayer(s.bytes())
399+ if err != nil {
400+ t.Fatal(err)
401+ }
402+ g2 := &commitGraph{layers: []graphLayer{*l2}, total: l2.n}
403+ if maybe, ok := g2.maybeChangedKey(1, &k); ok && !maybe {
404+ t.Fatal("false negative after insertion")
405+ }
406+ }
407+ })
408+}
409+
410+// ---- benchmarks -----------------------------------------------------------
411+
412+func BenchmarkBloomMaybeChanged(b *testing.B) {
413+ g, _ := benchGraph(b)
414+ n := g.numCommits()
415+ key := newBloomKey([]byte("pkg3/sub1/g1234.go"))
416+ b.ReportAllocs()
417+ var pos uint32
418+ var hits int
419+ for b.Loop() {
420+ if maybe, ok := g.maybeChangedKey(pos, &key); ok && maybe {
421+ hits++
422+ }
423+ if pos++; pos == n {
424+ pos = 0
425+ }
426+ }
427+ _ = hits
428+}
429+
430+func BenchmarkBloomMaybeChangedPath(b *testing.B) {
431+ g, _ := benchGraph(b)
432+ n := g.numCommits()
433+ path := []byte("pkg3/sub1/g1234.go")
434+ b.ReportAllocs()
435+ var pos uint32
436+ for b.Loop() {
437+ g.maybeChanged(pos, path)
438+ if pos++; pos == n {
439+ pos = 0
440+ }
441+ }
442+}
443+
444+func BenchmarkBloomNewKey(b *testing.B) {
445+ path := []byte("internal/adapters/gitcli/repos.go")
446+ b.ReportAllocs()
447+ for b.Loop() {
448+ _ = newBloomKey(path)
449+ }
450+}
451+
452+// ---- real repository ------------------------------------------------------
453+
454+// TestGraphBenchRepo checks a large real repository (ERMITE_GIT_BENCH_REPO).
455+// The repository is never modified: its objects are copied into a bare clone
456+// where the split graph is written.
457+func TestGraphBenchRepo(t *testing.T) {
458+ src := os.Getenv("ERMITE_GIT_BENCH_REPO")
459+ if src == "" {
460+ t.Skip("ERMITE_GIT_BENCH_REPO not set")
461+ }
462+ requireGit(t)
463+ for _, tc := range []struct {
464+ ver uint32
465+ multi bool
466+ }{{2, false}, {1, false}, {2, true}, {1, true}} {
467+ ver, multi := tc.ver, tc.multi
468+ t.Run(fmt.Sprintf("bloom-v%d-multi=%v", ver, multi), func(t *testing.T) {
469+ dst := filepath.Join(t.TempDir(), "clone.git")
470+ graphGit(t, "", 0, "clone", "--bare", "--no-local", "-q", src, dst)
471+ cfg := fmt.Sprintf("commitGraph.changedPathsVersion=%d", ver)
472+ if multi {
473+ // Several layers, like successive pushes would produce.
474+ all := strings.Fields(graphGit(t, dst, 0, "rev-list", "--all"))
475+ for _, back := range []int{5000, 2500, 500} {
476+ tip := all[min(back, len(all)-1)]
477+ graphGitStdin(t, dst, tip+"\n", "-c", cfg,
478+ "commit-graph", "write", "--stdin-commits", "--changed-paths", "--split=no-merge")
479+ }
480+ graphGit(t, dst, 0, "-c", cfg, "commit-graph", "write", "--reachable", "--changed-paths", "--split=no-merge")
481+ } else {
482+ graphGit(t, dst, 0, "-c", cfg, "commit-graph", "write", "--reachable", "--changed-paths", "--split")
483+ }
484+ g, err := openCommitGraph(filepath.Join(dst, "objects"))
485+ if err != nil {
486+ t.Fatal(err)
487+ }
488+ defer func() { _ = g.close() }()
489+ out := graphGit(t, dst, 0, "log", "--all", "--format=%H %T %P|%ct")
490+ var cs []refCommit
491+ for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
492+ left, ct, _ := strings.Cut(line, "|")
493+ f := strings.Fields(left)
494+ c := refCommit{id: graphTestHex(f[0]), tree: graphTestHex(f[1])}
495+ for _, p := range f[2:] {
496+ c.parents = append(c.parents, graphTestHex(p))
497+ }
498+ c.ctime, _ = strconv.ParseInt(ct, 10, 64)
499+ cs = append(cs, c)
500+ }
501+ if int(g.numCommits()) != len(cs) {
502+ t.Fatalf("numCommits = %d, git log has %d", g.numCommits(), len(cs))
503+ }
504+ t.Logf("%d commits, %d layers, generation v2 = %v", len(cs), g.numLayers(), g.hasGenerationV2())
505+ if multi && g.numLayers() < 4 {
506+ t.Fatalf("expected a multi-layer chain, got %d layers", g.numLayers())
507+ }
508+ var buf [16]uint32
509+ for i := range cs {
510+ c := &cs[i]
511+ pos, ok := g.lookup(c.id)
512+ if !ok {
513+ t.Fatalf("lookup(%x)", c.id)
514+ }
515+ if id, _ := g.commitID(pos); id != c.id {
516+ t.Fatalf("commitID(%d)", pos)
517+ }
518+ if tr, _ := g.rootTree(pos); tr != c.tree {
519+ t.Fatalf("rootTree of %x = %x, want %x", c.id, tr, c.tree)
520+ }
521+ if ct, _ := g.commitTime(pos); ct != c.ctime {
522+ t.Fatalf("commitTime of %x = %d, want %d", c.id, ct, c.ctime)
523+ }
524+ ps, err := g.appendParents(buf[:0], pos)
525+ if err != nil || len(ps) != len(c.parents) {
526+ t.Fatalf("parents of %x: %v, %v; want %d", c.id, ps, err, len(c.parents))
527+ }
528+ for j, p := range ps {
529+ if pid, _ := g.commitID(p); pid != c.parents[j] {
530+ t.Fatalf("parent %d of %x", j, c.id)
531+ }
532+ }
533+ gen, err := g.generation(pos)
534+ if err != nil {
535+ t.Fatal(err)
536+ }
537+ for _, p := range ps {
538+ if pg, _ := g.generation(p); pg >= gen {
539+ t.Fatalf("generation(%x) = %d not above parent %d", c.id, gen, pg)
540+ }
541+ }
542+ }
543+ checkBloomSample(t, dst, g, cs, true, ver, 25)
544+ })
545+ }
546+}
@@ -0,0 +1,546 @@
1+// SPDX-License-Identifier: MIT
2+
3+package git
4+
5+import (
6+ "encoding/binary"
7+ "fmt"
8+ "os"
9+ "path/filepath"
10+ "strconv"
11+ "strings"
12+ "testing"
13+)
14+
15+func TestMurmur3Vectors(t *testing.T) {
16+ // Reference values of MurmurHash3_x86_32 (same vectors as git t0095).
17+ for _, c := range []struct {
18+ in string
19+ seed uint32
20+ want uint32
21+ }{
22+ {"", 0, 0},
23+ {"", 1, 0x514e28b7},
24+ {"", 0xffffffff, 0x81f16f39},
25+ {"test", 0, 0xba6bd213},
26+ {"test", 0x9747b28c, 0x704b81dc},
27+ {"Hello, world!", 0, 0xc0363e43},
28+ {"Hello, world!", 0x9747b28c, 0x24884cba},
29+ {"The quick brown fox jumps over the lazy dog", 0, 0x2e4ff723},
30+ {"The quick brown fox jumps over the lazy dog", 0x9747b28c, 0x2fa826cd},
31+ {"aaaa", 0x9747b28c, 0x5a97808a},
32+ {"aaa", 0x9747b28c, 0x283e0130},
33+ {"aa", 0x9747b28c, 0x5d211726},
34+ {"a", 0x9747b28c, 0x7fa09ea6},
35+ } {
36+ if got := murmur3Seeded(c.seed, []byte(c.in), false); got != c.want {
37+ t.Errorf("murmur3(%q, %#x) = %#x, want %#x", c.in, c.seed, got, c.want)
38+ }
39+ // ASCII input is identical in both versions.
40+ if got := murmur3Seeded(c.seed, []byte(c.in), true); got != c.want {
41+ t.Errorf("signed murmur3(%q, %#x) = %#x, want %#x", c.in, c.seed, got, c.want)
42+ }
43+ }
44+}
45+
46+func TestBloomKeyVersions(t *testing.T) {
47+ a := newBloomKey([]byte("src/main.go"))
48+ if a.v1 != a.v2 {
49+ t.Error("ASCII path must hash identically in v1 and v2")
50+ }
51+ b := newBloomKey([]byte("caf\xc3\xa9/men\xc3\xbc.txt"))
52+ if b.v1 == b.v2 {
53+ t.Error("high-bit path must hash differently in v1 and v2")
54+ }
55+ if newBloomKeyString("x") != newBloomKey([]byte("x")) {
56+ t.Error("string and bytes keys differ")
57+ }
58+}
59+
60+// bloomPaths returns the keys git inserts for a changed path: the path and
61+// every leading directory.
62+func bloomPaths(p string) []string {
63+ out := []string{p}
64+ for i := len(p) - 1; i > 0; i-- {
65+ if p[i] == '/' {
66+ out = append(out, p[:i])
67+ }
68+ }
69+ return out
70+}
71+
72+// changedPaths lists the paths that differ between c and its first parent
73+// (the empty tree for a root commit), as git sees them.
74+func changedPaths(t testing.TB, dir string, c *refCommit) []string {
75+ t.Helper()
76+ var out string
77+ if len(c.parents) == 0 {
78+ out = graphGit(t, dir, 0, "diff-tree", "-r", "-z", "--root", "--name-only", "--no-commit-id", fmt.Sprintf("%x", c.id))
79+ } else {
80+ out = graphGit(t, dir, 0, "diff-tree", "-r", "-z", "--name-only", "--no-commit-id", fmt.Sprintf("%x", c.parents[0]), fmt.Sprintf("%x", c.id))
81+ }
82+ var paths []string
83+ for _, p := range strings.Split(out, "\x00") {
84+ if p != "" {
85+ paths = append(paths, p)
86+ }
87+ }
88+ return paths
89+}
90+
91+// checkBloomAgainstGit verifies the Bloom filters of every commit of g for
92+// the changed paths reported by git: no false negatives for the path or any
93+// leading directory, and unchanged paths mostly rejected.
94+func checkBloomAgainstGit(t *testing.T, dir string, g *commitGraph, cs []refCommit, c gcase) {
95+ t.Helper()
96+ checkBloomSample(t, dir, g, cs, c.bloom, c.bloomVer, 1)
97+}
98+
99+// checkBloomSample is checkBloomAgainstGit for every step-th commit. wantVer
100+// is the BDAT version expected when bloom is true.
101+func checkBloomSample(t *testing.T, dir string, g *commitGraph, cs []refCommit, bloom bool, wantVer uint32, step int) {
102+ t.Helper()
103+ var checked, queries, unchangedQueries, falsePositives, unknown int
104+ for i := 0; i < len(cs); i += step {
105+ cm := &cs[i]
106+ pos, ok := g.lookup(cm.id)
107+ if !ok {
108+ t.Fatalf("lookup(%x)", cm.id)
109+ }
110+ paths := changedPaths(t, dir, cm)
111+ if !bloom {
112+ if g.hasBloom(pos) {
113+ t.Fatalf("commit %x has a Bloom filter in a graph written without --changed-paths", cm.id)
114+ }
115+ for _, p := range paths {
116+ if maybe, ok := g.maybeChanged(pos, []byte(p)); ok || maybe {
117+ t.Fatalf("maybeChanged = %v,%v without filters", maybe, ok)
118+ }
119+ }
120+ continue
121+ }
122+ l, _ := g.layerOf(pos)
123+ if !l.hasBloom || l.bloomVersion != wantVer {
124+ t.Fatalf("layer bloom = %v version %d, want version %d", l.hasBloom, l.bloomVersion, wantVer)
125+ }
126+ if len(paths) > 512 {
127+ if _, ok := g.maybeChanged(pos, []byte(paths[0])); ok {
128+ t.Fatalf("commit with %d changes should be unknown", len(paths))
129+ }
130+ continue
131+ }
132+ set := map[string]bool{}
133+ for _, p := range paths {
134+ for _, k := range bloomPaths(p) {
135+ set[k] = true
136+ }
137+ }
138+ if len(paths) == 0 {
139+ // git writes the one-byte all-zero filter for "nothing changed".
140+ if maybe, ok := g.maybeChanged(pos, []byte("README")); !ok || maybe {
141+ t.Fatalf("empty change set: maybe=%v ok=%v", maybe, ok)
142+ }
143+ }
144+ for k := range set {
145+ key := newBloomKey([]byte(k))
146+ maybe, ok := g.maybeChangedKey(pos, &key)
147+ if !ok {
148+ unknown++
149+ continue
150+ }
151+ if !maybe {
152+ t.Fatalf("commit %x: false negative for %q (changed: %q)", cm.id, k, paths)
153+ }
154+ queries++
155+ }
156+ // Unchanged paths: names that cannot be in the set.
157+ for j := range 40 {
158+ p := fmt.Sprintf("no/such/dir%d/file-%d.txt", j, j*7+i)
159+ if set[p] {
160+ continue
161+ }
162+ for _, q := range bloomPaths(p) {
163+ if set[q] {
164+ continue
165+ }
166+ key := newBloomKey([]byte(q))
167+ maybe, ok := g.maybeChangedKey(pos, &key)
168+ if !ok {
169+ continue
170+ }
171+ unchangedQueries++
172+ if maybe {
173+ falsePositives++
174+ }
175+ }
176+ }
177+ checked++
178+ }
179+ if !bloom {
180+ return
181+ }
182+ if checked == 0 || queries == 0 {
183+ t.Fatalf("no Bloom queries ran (checked %d commits)", checked)
184+ }
185+ rate := float64(falsePositives) / float64(max(unchangedQueries, 1))
186+ t.Logf("bloom v%d: %d commits, %d positive queries (%d unknown), false positives %d/%d = %.2f%%",
187+ wantVer, checked, queries, unknown, falsePositives, unchangedQueries, rate*100)
188+ if rate > 0.10 {
189+ t.Errorf("false positive rate %.1f%% is too high for 10 bits per entry", rate*100)
190+ }
191+}
192+
193+// With a v1 graph, the v2 hash must not be used for high-bit paths (that is
194+// the bug v2 fixes); with v2, both agree on being negative-free.
195+func TestBloomHighBitVersions(t *testing.T) {
196+ for _, ver := range []uint32{1, 2} {
197+ t.Run(fmt.Sprintf("v%d", ver), func(t *testing.T) {
198+ r := newGTRepo(t)
199+ for i := range 30 {
200+ p := fmt.Sprintf("d\xe9\xff%d/\xc3\xa9\xfe%d.txt", i, i)
201+ r.commit(p, "x\n")
202+ }
203+ r.git("-c", fmt.Sprintf("commitGraph.changedPathsVersion=%d", ver),
204+ "commit-graph", "write", "--reachable", "--changed-paths")
205+ g, err := openCommitGraph(r.objects())
206+ if err != nil {
207+ t.Fatal(err)
208+ }
209+ defer func() { _ = g.close() }()
210+ cs := gitCommits(t, r.dir)
211+ wrong := 0
212+ for _, cm := range cs {
213+ pos, _ := g.lookup(cm.id)
214+ for _, p := range changedPaths(t, r.dir, &cm) {
215+ for _, k := range bloomPaths(p) {
216+ if maybe, ok := g.maybeChanged(pos, []byte(k)); !ok || !maybe {
217+ t.Fatalf("v%d: %q not found (maybe=%v ok=%v)", ver, k, maybe, ok)
218+ }
219+ // Probe with the other version's hash.
220+ key := newBloomKey([]byte(k))
221+ other := key
222+ other.v1, other.v2 = key.v2, key.v1
223+ if maybe, ok := g.maybeChangedKey(pos, &other); ok && !maybe {
224+ wrong++
225+ }
226+ }
227+ }
228+ }
229+ if wrong == 0 {
230+ t.Errorf("using the other hash version never failed: v%d not distinguished", ver)
231+ }
232+ })
233+ }
234+}
235+
236+// ---- synthetic filters ----------------------------------------------------
237+
238+// synthBloom builds a graph with the given filters (nil entry: empty).
239+func synthBloom(t testing.TB, version, hashes, bitsPer uint32, filters [][]byte) *commitGraph {
240+ n := len(filters)
241+ s := &synthLayer{ids: synthIDs(n)}
242+ hdr := binary.BigEndian.AppendUint32(nil, version)
243+ hdr = binary.BigEndian.AppendUint32(hdr, hashes)
244+ hdr = binary.BigEndian.AppendUint32(hdr, bitsPer)
245+ var total uint32
246+ s.bdat = hdr
247+ for _, f := range filters {
248+ total += uint32(len(f))
249+ s.bidx = append(s.bidx, total)
250+ s.bdat = append(s.bdat, f...)
251+ }
252+ return mustParseSynth(t, s.bytes())
253+}
254+
255+// setBloomBits sets the bits of path in filter f.
256+func setBloomBits(f []byte, hashes uint32, k bloomKey, v1 bool) {
257+ h := k.v2
258+ if v1 {
259+ h = k.v1
260+ }
261+ for i := uint32(0); i < hashes; i++ {
262+ bit := uint64(h[0]+i*h[1]) % uint64(len(f)*8)
263+ f[bit>>3] |= 1 << (bit & 7)
264+ }
265+}
266+
267+func TestBloomSynthetic(t *testing.T) {
268+ const hashes = 7
269+ f := make([]byte, 16)
270+ setBloomBits(f, hashes, newBloomKey([]byte("a/b/c.go")), false)
271+ setBloomBits(f, hashes, newBloomKey([]byte("a/b")), false)
272+ setBloomBits(f, hashes, newBloomKey([]byte("a")), false)
273+ filters := [][]byte{
274+ f,
275+ {0x00}, // nothing changed
276+ {0xff}, // too many changes
277+ nil, // no data
278+ f[:8], // different size: still a valid filter
279+ {0x01, 0x02}, // random garbage filter
280+ }
281+ g := synthBloom(t, 2, hashes, 10, filters)
282+ defer func() { _ = g.close() }()
283+ if maybe, ok := g.maybeChanged(0, []byte("a/b/c.go")); !maybe || !ok {
284+ t.Errorf("filter 0: %v %v", maybe, ok)
285+ }
286+ if maybe, ok := g.maybeChanged(0, []byte("zzz")); maybe || !ok {
287+ t.Errorf("filter 0 unrelated: %v %v", maybe, ok)
288+ }
289+ if maybe, ok := g.maybeChanged(1, []byte("a")); maybe || !ok {
290+ t.Errorf("zero filter: %v %v", maybe, ok)
291+ }
292+ if _, ok := g.maybeChanged(2, []byte("a")); ok {
293+ t.Error("0xff filter must be unknown")
294+ }
295+ if _, ok := g.maybeChanged(3, []byte("a")); ok {
296+ t.Error("empty filter must be unknown")
297+ }
298+ if _, ok := g.maybeChanged(g.numCommits(), []byte("a")); ok {
299+ t.Error("bad position must be unknown")
300+ }
301+ if g.hasBloom(2) || !g.hasBloom(1) {
302+ t.Error("hasBloom")
303+ }
304+}
305+
306+func TestBloomTruncated(t *testing.T) {
307+ // BIDX claims more bytes than BDAT holds: unknown, no panic.
308+ s := &synthLayer{ids: synthIDs(3)}
309+ hdr := binary.BigEndian.AppendUint32(nil, 2)
310+ hdr = binary.BigEndian.AppendUint32(hdr, 7)
311+ hdr = binary.BigEndian.AppendUint32(hdr, 10)
312+ s.bdat = append(hdr, 1, 2, 3, 4)
313+ s.bidx = []uint32{4, 4000, 0xffffffff}
314+ g := mustParseSynth(t, s.bytes())
315+ defer func() { _ = g.close() }()
316+ for pos := range uint32(3) {
317+ if _, ok := g.maybeChanged(pos, []byte("x")); pos > 0 && ok {
318+ t.Errorf("pos %d: truncated filter reported known", pos)
319+ }
320+ }
321+ // Non-monotonic offsets.
322+ s.bidx = []uint32{4, 2, 3}
323+ g2 := mustParseSynth(t, s.bytes())
324+ for pos := range uint32(3) {
325+ g2.maybeChanged(pos, []byte("x"))
326+ }
327+}
328+
329+func TestBloomIgnoredSettings(t *testing.T) {
330+ for _, c := range []struct{ version, hashes uint32 }{{3, 7}, {0, 7}, {2, 0}, {2, bloomMaxHashes + 1}, {2, 1 << 31}} {
331+ g := synthBloom(t, c.version, c.hashes, 10, [][]byte{{0x00}, {0x00}})
332+ if g.hasBloom(0) {
333+ t.Errorf("version %d hashes %d should be ignored", c.version, c.hashes)
334+ }
335+ if _, ok := g.maybeChanged(0, []byte("a")); ok {
336+ t.Errorf("version %d hashes %d should be unknown", c.version, c.hashes)
337+ }
338+ }
339+ // BIDX without BDAT, BDAT too short.
340+ s := &synthLayer{ids: synthIDs(2), bidx: []uint32{1, 2}, bdat: []byte{0, 0, 0, 2}}
341+ if _, err := parseGraphLayer(s.bytes()); err == nil {
342+ t.Error("short BDAT accepted")
343+ }
344+}
345+
346+// ---- allocation and fuzz --------------------------------------------------
347+
348+func TestBloomNoAllocs(t *testing.T) {
349+ const hashes = 7
350+ f := make([]byte, 16)
351+ setBloomBits(f, hashes, newBloomKey([]byte("a/b")), false)
352+ g := synthBloom(t, 2, hashes, 10, [][]byte{f, f})
353+ path := []byte("a/b/ünï.go")
354+ var sink bool
355+ if n := testing.AllocsPerRun(100, func() {
356+ k := newBloomKey(path)
357+ m, _ := g.maybeChangedKey(0, &k)
358+ m2, _ := g.maybeChanged(1, path)
359+ sink = sink || m || m2
360+ }); n != 0 {
361+ t.Errorf("allocs = %v", n)
362+ }
363+ _ = sink
364+}
365+
366+func FuzzBloomLookup(f *testing.F) {
367+ f.Add([]byte{0x00}, []byte("a"), uint32(2), uint32(7))
368+ f.Add([]byte{0xff}, []byte("a/b"), uint32(1), uint32(7))
369+ f.Add([]byte{0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0}, []byte("caf\xc3\xa9/x"), uint32(1), uint32(3))
370+ f.Add([]byte{}, []byte{}, uint32(2), uint32(7))
371+ f.Add([]byte{0x80, 0x01}, []byte("\xff\xfe\xfd\xfc\xfb"), uint32(9), uint32(1<<31))
372+ f.Fuzz(func(t *testing.T, filter, path []byte, version, hashes uint32) {
373+ hdr := binary.BigEndian.AppendUint32(nil, version)
374+ hdr = binary.BigEndian.AppendUint32(hdr, hashes)
375+ hdr = binary.BigEndian.AppendUint32(hdr, 10)
376+ s := &synthLayer{ids: synthIDs(2)}
377+ s.bdat = append(hdr, filter...)
378+ // Second filter claims the rest; the first is cut anywhere.
379+ s.bidx = []uint32{uint32(len(filter) / 2), uint32(len(filter))}
380+ l, err := parseGraphLayer(s.bytes())
381+ if err != nil {
382+ return
383+ }
384+ g := &commitGraph{layers: []graphLayer{*l}, total: l.n}
385+ k := newBloomKey(path)
386+ for pos := range uint32(3) {
387+ maybe, ok := g.maybeChangedKey(pos, &k)
388+ if maybe && !ok {
389+ t.Fatal("maybe without ok")
390+ }
391+ }
392+ // A path inserted into a valid filter is always found.
393+ half := len(filter) / 2
394+ if (version == 1 || version == 2) && hashes >= 1 && hashes <= bloomMaxHashes && len(filter)-half >= 1 {
395+ second := append([]byte(nil), filter[half:]...)
396+ setBloomBits(second, hashes, k, version == 1)
397+ s.bdat = append(append(hdr[:12:12], filter[:half]...), second...)
398+ l2, err := parseGraphLayer(s.bytes())
399+ if err != nil {
400+ t.Fatal(err)
401+ }
402+ g2 := &commitGraph{layers: []graphLayer{*l2}, total: l2.n}
403+ if maybe, ok := g2.maybeChangedKey(1, &k); ok && !maybe {
404+ t.Fatal("false negative after insertion")
405+ }
406+ }
407+ })
408+}
409+
410+// ---- benchmarks -----------------------------------------------------------
411+
412+func BenchmarkBloomMaybeChanged(b *testing.B) {
413+ g, _ := benchGraph(b)
414+ n := g.numCommits()
415+ key := newBloomKey([]byte("pkg3/sub1/g1234.go"))
416+ b.ReportAllocs()
417+ var pos uint32
418+ var hits int
419+ for b.Loop() {
420+ if maybe, ok := g.maybeChangedKey(pos, &key); ok && maybe {
421+ hits++
422+ }
423+ if pos++; pos == n {
424+ pos = 0
425+ }
426+ }
427+ _ = hits
428+}
429+
430+func BenchmarkBloomMaybeChangedPath(b *testing.B) {
431+ g, _ := benchGraph(b)
432+ n := g.numCommits()
433+ path := []byte("pkg3/sub1/g1234.go")
434+ b.ReportAllocs()
435+ var pos uint32
436+ for b.Loop() {
437+ g.maybeChanged(pos, path)
438+ if pos++; pos == n {
439+ pos = 0
440+ }
441+ }
442+}
443+
444+func BenchmarkBloomNewKey(b *testing.B) {
445+ path := []byte("internal/adapters/gitcli/repos.go")
446+ b.ReportAllocs()
447+ for b.Loop() {
448+ _ = newBloomKey(path)
449+ }
450+}
451+
452+// ---- real repository ------------------------------------------------------
453+
454+// TestGraphBenchRepo checks a large real repository (ERMITE_GIT_BENCH_REPO).
455+// The repository is never modified: its objects are copied into a bare clone
456+// where the split graph is written.
457+func TestGraphBenchRepo(t *testing.T) {
458+ src := os.Getenv("ERMITE_GIT_BENCH_REPO")
459+ if src == "" {
460+ t.Skip("ERMITE_GIT_BENCH_REPO not set")
461+ }
462+ requireGit(t)
463+ for _, tc := range []struct {
464+ ver uint32
465+ multi bool
466+ }{{2, false}, {1, false}, {2, true}, {1, true}} {
467+ ver, multi := tc.ver, tc.multi
468+ t.Run(fmt.Sprintf("bloom-v%d-multi=%v", ver, multi), func(t *testing.T) {
469+ dst := filepath.Join(t.TempDir(), "clone.git")
470+ graphGit(t, "", 0, "clone", "--bare", "--no-local", "-q", src, dst)
471+ cfg := fmt.Sprintf("commitGraph.changedPathsVersion=%d", ver)
472+ if multi {
473+ // Several layers, like successive pushes would produce.
474+ all := strings.Fields(graphGit(t, dst, 0, "rev-list", "--all"))
475+ for _, back := range []int{5000, 2500, 500} {
476+ tip := all[min(back, len(all)-1)]
477+ graphGitStdin(t, dst, tip+"\n", "-c", cfg,
478+ "commit-graph", "write", "--stdin-commits", "--changed-paths", "--split=no-merge")
479+ }
480+ graphGit(t, dst, 0, "-c", cfg, "commit-graph", "write", "--reachable", "--changed-paths", "--split=no-merge")
481+ } else {
482+ graphGit(t, dst, 0, "-c", cfg, "commit-graph", "write", "--reachable", "--changed-paths", "--split")
483+ }
484+ g, err := openCommitGraph(filepath.Join(dst, "objects"))
485+ if err != nil {
486+ t.Fatal(err)
487+ }
488+ defer func() { _ = g.close() }()
489+ out := graphGit(t, dst, 0, "log", "--all", "--format=%H %T %P|%ct")
490+ var cs []refCommit
491+ for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
492+ left, ct, _ := strings.Cut(line, "|")
493+ f := strings.Fields(left)
494+ c := refCommit{id: graphTestHex(f[0]), tree: graphTestHex(f[1])}
495+ for _, p := range f[2:] {
496+ c.parents = append(c.parents, graphTestHex(p))
497+ }
498+ c.ctime, _ = strconv.ParseInt(ct, 10, 64)
499+ cs = append(cs, c)
500+ }
501+ if int(g.numCommits()) != len(cs) {
502+ t.Fatalf("numCommits = %d, git log has %d", g.numCommits(), len(cs))
503+ }
504+ t.Logf("%d commits, %d layers, generation v2 = %v", len(cs), g.numLayers(), g.hasGenerationV2())
505+ if multi && g.numLayers() < 4 {
506+ t.Fatalf("expected a multi-layer chain, got %d layers", g.numLayers())
507+ }
508+ var buf [16]uint32
509+ for i := range cs {
510+ c := &cs[i]
511+ pos, ok := g.lookup(c.id)
512+ if !ok {
513+ t.Fatalf("lookup(%x)", c.id)
514+ }
515+ if id, _ := g.commitID(pos); id != c.id {
516+ t.Fatalf("commitID(%d)", pos)
517+ }
518+ if tr, _ := g.rootTree(pos); tr != c.tree {
519+ t.Fatalf("rootTree of %x = %x, want %x", c.id, tr, c.tree)
520+ }
521+ if ct, _ := g.commitTime(pos); ct != c.ctime {
522+ t.Fatalf("commitTime of %x = %d, want %d", c.id, ct, c.ctime)
523+ }
524+ ps, err := g.appendParents(buf[:0], pos)
525+ if err != nil || len(ps) != len(c.parents) {
526+ t.Fatalf("parents of %x: %v, %v; want %d", c.id, ps, err, len(c.parents))
527+ }
528+ for j, p := range ps {
529+ if pid, _ := g.commitID(p); pid != c.parents[j] {
530+ t.Fatalf("parent %d of %x", j, c.id)
531+ }
532+ }
533+ gen, err := g.generation(pos)
534+ if err != nil {
535+ t.Fatal(err)
536+ }
537+ for _, p := range ps {
538+ if pg, _ := g.generation(p); pg >= gen {
539+ t.Fatalf("generation(%x) = %d not above parent %d", c.id, gen, pg)
540+ }
541+ }
542+ }
543+ checkBloomSample(t, dst, g, cs, true, ver, 25)
544+ })
545+ }
546+}
A graph_test.go +1249 -0
@@ -0,0 +1,1249 @@
1+// SPDX-License-Identifier: MIT
2+
3+package git
4+
5+import (
6+ "bufio"
7+ "bytes"
8+ "encoding/binary"
9+ "encoding/hex"
10+ "errors"
11+ "fmt"
12+ "os"
13+ "os/exec"
14+ "path/filepath"
15+ "sort"
16+ "strconv"
17+ "strings"
18+ "testing"
19+)
20+
21+// oid is the raw object id used by the graph code until the core's ObjectID
22+// is merged.
23+type oid = [20]byte
24+
25+// graphTestHex parses a 40-digit hex string.
26+func graphTestHex(s string) oid {
27+ var id oid
28+ if _, err := hex.Decode(id[:], []byte(s)); err != nil || len(s) != 40 {
29+ panic("graphTestHex: bad id " + strconv.Quote(s))
30+ }
31+ return id
32+}
33+
34+// ---- git fixtures ---------------------------------------------------------
35+
36+func requireGit(t testing.TB) {
37+ t.Helper()
38+ if _, err := exec.LookPath("git"); err != nil {
39+ t.Skip("git binary not available")
40+ }
41+}
42+
43+func graphGitEnv(date int64) []string {
44+ env := append(os.Environ(),
45+ "GIT_CONFIG_GLOBAL=/dev/null",
46+ "GIT_CONFIG_NOSYSTEM=1",
47+ "GIT_TERMINAL_PROMPT=0",
48+ "GIT_AUTHOR_NAME=T", "GIT_AUTHOR_EMAIL=t@example.com",
49+ "GIT_COMMITTER_NAME=T", "GIT_COMMITTER_EMAIL=t@example.com",
50+ )
51+ if date != 0 {
52+ d := strconv.FormatInt(date, 10) + " +0000"
53+ env = append(env, "GIT_AUTHOR_DATE="+d, "GIT_COMMITTER_DATE="+d)
54+ }
55+ return env
56+}
57+
58+func graphGit(t testing.TB, dir string, date int64, args ...string) string {
59+ t.Helper()
60+ return graphGitIn(t, dir, date, "", args...)
61+}
62+
63+func graphGitStdin(t testing.TB, dir, stdin string, args ...string) string {
64+ t.Helper()
65+ return graphGitIn(t, dir, 0, stdin, args...)
66+}
67+
68+func graphGitIn(t testing.TB, dir string, date int64, stdin string, args ...string) string {
69+ t.Helper()
70+ cmd := exec.Command("git", args...)
71+ cmd.Stdin = strings.NewReader(stdin)
72+ cmd.Dir = dir
73+ cmd.Env = graphGitEnv(date)
74+ var stderr bytes.Buffer
75+ cmd.Stderr = &stderr
76+ out, err := cmd.Output()
77+ if err != nil {
78+ t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, stderr.String())
79+ }
80+ return string(out)
81+}
82+
83+// gtRepo is a scratch repository with a deterministic clock.
84+type gtRepo struct {
85+ t testing.TB
86+ dir string
87+ clock int64
88+}
89+
90+func newGTRepo(t testing.TB) *gtRepo {
91+ t.Helper()
92+ requireGit(t)
93+ r := &gtRepo{t: t, dir: t.TempDir(), clock: 1_700_000_000}
94+ r.git("init", "-q", "-b", "main")
95+ return r
96+}
97+
98+func (r *gtRepo) git(args ...string) string {
99+ r.t.Helper()
100+ return graphGit(r.t, r.dir, r.clock, args...)
101+}
102+
103+func (r *gtRepo) objects() string { return filepath.Join(r.dir, ".git", "objects") }
104+
105+func (r *gtRepo) write(path, content string) {
106+ r.t.Helper()
107+ full := filepath.Join(r.dir, path)
108+ if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
109+ r.t.Fatal(err)
110+ }
111+ if err := os.WriteFile(full, []byte(content), 0o644); err != nil {
112+ r.t.Fatal(err)
113+ }
114+}
115+
116+func (r *gtRepo) tick() int64 {
117+ r.clock += 60
118+ return r.clock
119+}
120+
121+// commit writes files (path, content pairs) and commits them.
122+func (r *gtRepo) commit(files ...string) {
123+ r.t.Helper()
124+ for i := 0; i+1 < len(files); i += 2 {
125+ r.write(files[i], files[i+1])
126+ }
127+ r.tick()
128+ r.git("add", "-A")
129+ r.git("commit", "-q", "-m", fmt.Sprintf("c%d", r.clock))
130+}
131+
132+// commitAt commits with an explicit date (to build clock skew).
133+func (r *gtRepo) commitAt(date int64, files ...string) {
134+ r.t.Helper()
135+ for i := 0; i+1 < len(files); i += 2 {
136+ r.write(files[i], files[i+1])
137+ }
138+ r.git("add", "-A")
139+ graphGit(r.t, r.dir, date, "commit", "-q", "-m", fmt.Sprintf("c%d", date))
140+}
141+
142+func (r *gtRepo) remove(path string) {
143+ r.t.Helper()
144+ r.tick()
145+ r.git("rm", "-q", "-r", path)
146+ r.git("commit", "-q", "-m", "rm "+path)
147+}
148+
149+// stageOne: linear history with nested dirs, deletes, non-ASCII and
150+// high-bit paths, an empty commit, clock skew and ordinary merges.
151+func (r *gtRepo) stageOne() {
152+ r.commit("README", "r\n", "src/a.go", "a\n", "src/pkg/b.go", "b\n")
153+ r.commit("src/a.go", "a2\n", "docs/guide.md", "g\n")
154+ r.commit("café/menü.txt", "x\n", "\xe9\xe8/\xff\xfe.txt", "latin1\n", "日本語/ファイル.txt", "j\n")
155+ r.commit("src/pkg/deep/er/c.go", "c\n")
156+ r.tick()
157+ r.git("commit", "-q", "--allow-empty", "-m", "empty")
158+ r.remove("docs")
159+ // clock skew: a child older than its parent.
160+ r.commitAt(r.clock-5000, "src/skew.go", "s\n")
161+ r.commit("src/a.go", "a3\n")
162+
163+ r.git("checkout", "-q", "-b", "topic")
164+ r.commit("topic/one.txt", "1\n")
165+ r.commit("topic/two.txt", "2\n", "src/pkg/b.go", "b2\n")
166+ r.git("checkout", "-q", "main")
167+ r.commit("main-only.txt", "m\n")
168+ r.tick()
169+ r.git("merge", "-q", "--no-ff", "-m", "merge topic", "topic")
170+ r.commit("README", "r2\n")
171+}
172+
173+// stageTwo: an octopus merge (5 parents) and a merge of an older branch.
174+func (r *gtRepo) stageTwo() {
175+ for i := 1; i <= 4; i++ {
176+ r.git("checkout", "-q", "-b", fmt.Sprintf("b%d", i), "main")
177+ r.commit(fmt.Sprintf("oct/b%d.txt", i), fmt.Sprintf("%d\n", i), "ünï/b"+strconv.Itoa(i), "x\n")
178+ r.commit(fmt.Sprintf("oct/b%d.txt", i), fmt.Sprintf("%d%d\n", i, i))
179+ }
180+ r.git("checkout", "-q", "main")
181+ r.commit("after.txt", "a\n")
182+ r.tick()
183+ r.git("merge", "-q", "--no-ff", "-m", "octopus", "b1", "b2", "b3", "b4")
184+ r.commit("src/a.go", "a4\n")
185+ r.tick()
186+ r.git("merge", "-q", "--no-ff", "-m", "merge topic again", "topic")
187+}
188+
189+// stageThree: more linear commits, a merge whose parents are in base layers.
190+func (r *gtRepo) stageThree() {
191+ r.git("checkout", "-q", "-b", "late", "topic")
192+ r.commit("late/x.txt", "x\n")
193+ r.git("checkout", "-q", "main")
194+ r.commit("src/pkg/b.go", "b3\n")
195+ r.tick()
196+ r.git("merge", "-q", "--no-ff", "-m", "merge late", "late")
197+ for i := range 5 {
198+ r.commit(fmt.Sprintf("tail/%d.txt", i), "t\n")
199+ }
200+}
201+
202+type gcase struct {
203+ name string
204+ split bool
205+ bloom bool
206+ cfg []string // -c key=value before the subcommand
207+ bloomVer uint32 // expected BDAT version when bloom
208+ wantGenV2 bool
209+}
210+
211+// writeGraph runs git commit-graph write. git leaves its files read-only.
212+func (r *gtRepo) writeGraph(c gcase) {
213+ r.t.Helper()
214+ args := []string{}
215+ for _, kv := range c.cfg {
216+ args = append(args, "-c", kv)
217+ }
218+ args = append(args, "commit-graph", "write", "--reachable")
219+ if c.bloom {
220+ args = append(args, "--changed-paths")
221+ } else {
222+ args = append(args, "--no-changed-paths")
223+ }
224+ if c.split {
225+ args = append(args, "--split=no-merge")
226+ }
227+ r.git(args...)
228+}
229+
230+// buildCase creates the three-stage history, writing a graph after each.
231+func buildCase(t *testing.T, c gcase) *gtRepo {
232+ r := newGTRepo(t)
233+ r.stageOne()
234+ r.writeGraph(c)
235+ r.stageTwo()
236+ r.writeGraph(c)
237+ r.stageThree()
238+ r.writeGraph(c)
239+ r.git("commit-graph", "verify")
240+ return r
241+}
242+
243+// ---- reference data from git ---------------------------------------------
244+
245+type refCommit struct {
246+ id oid
247+ tree oid
248+ parents []oid
249+ ctime int64
250+}
251+
252+func gitCommits(t testing.TB, dir string) []refCommit {
253+ t.Helper()
254+ out := graphGit(t, dir, 0, "log", "--all", "--format=%H %T %P|%ct")
255+ var cs []refCommit
256+ for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
257+ left, ct, ok := strings.Cut(line, "|")
258+ if !ok {
259+ t.Fatalf("bad log line %q", line)
260+ }
261+ f := strings.Fields(left)
262+ c := refCommit{id: graphTestHex(f[0]), tree: graphTestHex(f[1])}
263+ for _, p := range f[2:] {
264+ c.parents = append(c.parents, graphTestHex(p))
265+ }
266+ c.ctime, _ = strconv.ParseInt(ct, 10, 64)
267+ cs = append(cs, c)
268+ }
269+ return cs
270+}
271+
272+// checkGraphAgainstGit compares every commit of g with git's view.
273+func checkGraphAgainstGit(t *testing.T, dir string, g *commitGraph, wantMerges bool) []refCommit {
274+ t.Helper()
275+ cs := gitCommits(t, dir)
276+ if int(g.numCommits()) != len(cs) {
277+ t.Fatalf("numCommits = %d, git has %d commits", g.numCommits(), len(cs))
278+ }
279+ byID := make(map[oid]*refCommit, len(cs))
280+ for i := range cs {
281+ byID[cs[i].id] = &cs[i]
282+ }
283+ var buf [8]uint32
284+ sawOctopus, sawMerge := false, false
285+ for i := range cs {
286+ c := &cs[i]
287+ pos, ok := g.lookup(c.id)
288+ if !ok {
289+ t.Fatalf("lookup(%x) not found", c.id)
290+ }
291+ if id, err := g.commitID(pos); err != nil || id != c.id {
292+ t.Fatalf("commitID(%d) = %x, %v; want %x", pos, id, err, c.id)
293+ }
294+ if tree, err := g.rootTree(pos); err != nil || tree != c.tree {
295+ t.Fatalf("rootTree(%d) = %x, %v; want %x", pos, tree, err, c.tree)
296+ }
297+ if ct, err := g.commitTime(pos); err != nil || ct != c.ctime {
298+ t.Fatalf("commitTime(%d) = %d, %v; want %d", pos, ct, err, c.ctime)
299+ }
300+ ps, err := g.appendParents(buf[:0], pos)
301+ if err != nil {
302+ t.Fatalf("appendParents(%d): %v", pos, err)
303+ }
304+ if len(ps) != len(c.parents) {
305+ t.Fatalf("commit %x: %d parents, git has %d", c.id, len(ps), len(c.parents))
306+ }
307+ for j, p := range ps {
308+ pid, err := g.commitID(p)
309+ if err != nil || pid != c.parents[j] {
310+ t.Fatalf("commit %x parent %d = %x, %v; want %x", c.id, j, pid, err, c.parents[j])
311+ }
312+ }
313+ sawMerge = sawMerge || len(ps) == 2
314+ sawOctopus = sawOctopus || len(ps) > 2
315+ }
316+ if wantMerges && (!sawMerge || !sawOctopus) {
317+ t.Errorf("fixture lacks merge (%v) or octopus (%v)", sawMerge, sawOctopus)
318+ }
319+ checkGeneration(t, g, cs, byID, wantMerges)
320+ return cs
321+}
322+
323+// checkGeneration verifies the topological level exactly (1 + max parents)
324+// and, when present, the corrected commit date (max(date, max parents + 1)).
325+func checkGeneration(t *testing.T, g *commitGraph, cs []refCommit, byID map[oid]*refCommit, wantSkew bool) {
326+ t.Helper()
327+ level := func(id oid) uint32 {
328+ pos, _ := g.lookup(id)
329+ v, err := g.topoLevel(pos)
330+ if err != nil {
331+ t.Fatal(err)
332+ }
333+ return v
334+ }
335+ gen := func(id oid) uint64 {
336+ pos, _ := g.lookup(id)
337+ v, err := g.generation(pos)
338+ if err != nil {
339+ t.Fatal(err)
340+ }
341+ return v
342+ }
343+ skewed := false
344+ for i := range cs {
345+ c := &cs[i]
346+ wantLevel := uint32(1)
347+ wantGen := uint64(c.ctime)
348+ for _, p := range c.parents {
349+ if l := level(p) + 1; l > wantLevel {
350+ wantLevel = l
351+ }
352+ if v := gen(p) + 1; g.hasGenerationV2() && v > wantGen {
353+ wantGen = v
354+ }
355+ }
356+ if got := level(c.id); got != wantLevel {
357+ t.Fatalf("topoLevel(%x) = %d, want %d", c.id, got, wantLevel)
358+ }
359+ if g.hasGenerationV2() {
360+ got := gen(c.id)
361+ if got != wantGen {
362+ t.Fatalf("generation(%x) = %d, want corrected date %d", c.id, got, wantGen)
363+ }
364+ skewed = skewed || got != uint64(c.ctime)
365+ } else if got := gen(c.id); got != uint64(wantLevel) {
366+ t.Fatalf("generation(%x) = %d, want level %d", c.id, got, wantLevel)
367+ }
368+ for _, p := range c.parents {
369+ if gen(p) >= gen(c.id) {
370+ t.Fatalf("generation not increasing from parent %x to %x", p, c.id)
371+ }
372+ }
373+ }
374+ if wantSkew && g.hasGenerationV2() && !skewed {
375+ t.Error("clock skew fixture produced no corrected date different from the commit time")
376+ }
377+ _ = byID
378+}
379+
380+// ---- conformance ----------------------------------------------------------
381+
382+func graphCases() []gcase {
383+ return []gcase{
384+ {name: "single", wantGenV2: true},
385+ {name: "single-bloom", bloom: true, bloomVer: 1, wantGenV2: true},
386+ {name: "split", split: true, wantGenV2: true},
387+ {name: "split-bloom", split: true, bloom: true, bloomVer: 1, wantGenV2: true},
388+ {name: "split-bloom-v1", split: true, bloom: true, bloomVer: 1, wantGenV2: true,
389+ cfg: []string{"commitGraph.changedPathsVersion=1"}},
390+ {name: "split-bloom-v2-explicit", split: true, bloom: true, bloomVer: 2, wantGenV2: true,
391+ cfg: []string{"commitGraph.changedPathsVersion=2"}},
392+ {name: "split-gen1", split: true, bloom: true, bloomVer: 1,
393+ cfg: []string{"commitGraph.generationVersion=1"}},
394+ {name: "single-gen1", cfg: []string{"commitGraph.generationVersion=1"}},
395+ }
396+}
397+
398+func TestGraphConformance(t *testing.T) {
399+ for _, c := range graphCases() {
400+ t.Run(c.name, func(t *testing.T) {
401+ r := buildCase(t, c)
402+ g, err := openCommitGraph(r.objects())
403+ if err != nil {
404+ t.Fatal(err)
405+ }
406+ defer func() { _ = g.close() }()
407+
408+ wantLayers := 1
409+ if c.split {
410+ wantLayers = 3
411+ }
412+ if g.numLayers() != wantLayers {
413+ t.Fatalf("layers = %d, want %d", g.numLayers(), wantLayers)
414+ }
415+ if g.hasGenerationV2() != c.wantGenV2 {
416+ t.Fatalf("hasGenerationV2 = %v, want %v", g.hasGenerationV2(), c.wantGenV2)
417+ }
418+ cs := checkGraphAgainstGit(t, r.dir, g, true)
419+
420+ // Positions are dense and round-trip, and absent ids miss.
421+ for pos := uint32(0); pos < g.numCommits(); pos++ {
422+ id, err := g.commitID(pos)
423+ if err != nil {
424+ t.Fatal(err)
425+ }
426+ if p, ok := g.lookup(id); !ok || p != pos {
427+ t.Fatalf("lookup(commitID(%d)) = %d, %v", pos, p, ok)
428+ }
429+ }
430+ for i := range cs {
431+ miss := cs[i].id
432+ miss[19] ^= 0x5a
433+ if _, ok := g.lookup(miss); ok {
434+ if _, real := byIDHas(cs, miss); !real {
435+ t.Fatalf("lookup found absent id %x", miss)
436+ }
437+ }
438+ }
439+ if _, err := g.commitID(g.numCommits()); !errors.Is(err, errGraphPosition) {
440+ t.Fatalf("commitID(numCommits) err = %v", err)
441+ }
442+ if _, err := g.appendParents(nil, g.numCommits()); !errors.Is(err, errGraphPosition) {
443+ t.Fatalf("appendParents(numCommits) err = %v", err)
444+ }
445+
446+ checkBloomAgainstGit(t, r.dir, g, cs, c)
447+ })
448+ }
449+}
450+
451+func byIDHas(cs []refCommit, id oid) (int, bool) {
452+ for i := range cs {
453+ if cs[i].id == id {
454+ return i, true
455+ }
456+ }
457+ return 0, false
458+}
459+
460+func TestGraphNone(t *testing.T) {
461+ r := newGTRepo(t)
462+ r.commit("a", "a\n")
463+ if _, err := openCommitGraph(r.objects()); !errors.Is(err, errNoGraph) {
464+ t.Fatalf("err = %v, want errNoGraph", err)
465+ }
466+}
467+
468+// A split chain written with the default merge strategy still reads fine.
469+func TestGraphSplitDefaultMerge(t *testing.T) {
470+ r := newGTRepo(t)
471+ for i := range 4 {
472+ r.commit(fmt.Sprintf("f%d", i), "x\n")
473+ r.git("commit-graph", "write", "--reachable", "--split", "--changed-paths")
474+ }
475+ g, err := openCommitGraph(r.objects())
476+ if err != nil {
477+ t.Fatal(err)
478+ }
479+ defer func() { _ = g.close() }()
480+ checkGraphAgainstGit(t, r.dir, g, false)
481+}
482+
483+func TestGraphRefcount(t *testing.T) {
484+ r := newGTRepo(t)
485+ r.commit("a", "a\n")
486+ r.git("commit-graph", "write", "--reachable")
487+ g, err := openCommitGraph(r.objects())
488+ if err != nil {
489+ t.Fatal(err)
490+ }
491+ if !g.retain() {
492+ t.Fatal("retain failed on open graph")
493+ }
494+ if err := g.close(); err != nil { // owner drops its reference
495+ t.Fatal(err)
496+ }
497+ // Still usable: the reader holds a reference.
498+ if g.numCommits() != 1 {
499+ t.Fatal("numCommits")
500+ }
501+ if _, err := g.commitID(0); err != nil {
502+ t.Fatal(err)
503+ }
504+ if err := g.release(); err != nil {
505+ t.Fatal(err)
506+ }
507+ if g.retain() {
508+ t.Fatal("retain succeeded on closed graph")
509+ }
510+ if err := g.release(); err == nil {
511+ t.Fatal("over-release not reported")
512+ }
513+}
514+
515+// A layer removed or rewritten while a graph is open must not affect it.
516+func TestGraphSurvivesRewrite(t *testing.T) {
517+ r := newGTRepo(t)
518+ r.stageOne()
519+ c := gcase{split: true, bloom: true}
520+ r.writeGraph(c)
521+ g, err := openCommitGraph(r.objects())
522+ if err != nil {
523+ t.Fatal(err)
524+ }
525+ defer func() { _ = g.close() }()
526+ before := g.numCommits()
527+ r.stageTwo()
528+ r.git("commit-graph", "write", "--reachable", "--changed-paths") // replaces the chain
529+ if st := statGraph(r.objects()); !st.single.exists || st.chain.exists {
530+ t.Fatalf("unexpected stamp %+v", st)
531+ }
532+ if g.numCommits() != before {
533+ t.Fatal("numCommits changed")
534+ }
535+ for pos := uint32(0); pos < before; pos++ {
536+ if _, err := g.commitID(pos); err != nil {
537+ t.Fatal(err)
538+ }
539+ }
540+}
541+
542+func TestGraphStamp(t *testing.T) {
543+ r := newGTRepo(t)
544+ r.commit("a", "a\n")
545+ s0 := statGraph(r.objects())
546+ if s0.single.exists || s0.chain.exists {
547+ t.Fatalf("stamp = %+v", s0)
548+ }
549+ r.git("commit-graph", "write", "--reachable", "--split")
550+ s1 := statGraph(r.objects())
551+ if !s1.chain.exists || s1 == s0 {
552+ t.Fatalf("stamp = %+v", s1)
553+ }
554+ r.commit("b", "b\n")
555+ r.git("commit-graph", "write", "--reachable", "--split=no-merge")
556+ if s2 := statGraph(r.objects()); s2 == s1 {
557+ t.Fatal("stamp did not change after a new layer")
558+ }
559+}
560+
561+// ---- synthetic graph writer ----------------------------------------------
562+
563+// synthLayer builds a commit-graph layer byte for byte, for cases git does
564+// not produce (generation overflow, wide octopus, corruption).
565+type synthLayer struct {
566+ ids []oid // must be sorted
567+ trees []oid
568+ parents [][]uint32 // global positions
569+ times []int64
570+ levels []uint32
571+ gda2 []uint32 // nil: no chunk
572+ gdo2 []uint64
573+ bidx []uint32
574+ bdat []byte // including the 12-byte header
575+ base []oid // base graph checksums
576+ skipEdge bool
577+}
578+
579+func (s *synthLayer) bytes() []byte {
580+ n := len(s.ids)
581+ var oidl, cdat, edge, gda2, gdo2, bidx, base bytes.Buffer
582+ fan := make([]uint32, 256)
583+ for _, id := range s.ids {
584+ fan[id[0]]++
585+ oidl.Write(id[:])
586+ }
587+ for i := 1; i < 256; i++ {
588+ fan[i] += fan[i-1]
589+ }
590+ var fanout bytes.Buffer
591+ for _, v := range fan {
592+ _ = binary.Write(&fanout, binary.BigEndian, v)
593+ }
594+ be := binary.BigEndian
595+ put32 := func(b *bytes.Buffer, v uint32) { _ = binary.Write(b, be, v) }
596+ for i := range n {
597+ var tree oid
598+ if i < len(s.trees) {
599+ tree = s.trees[i]
600+ }
601+ cdat.Write(tree[:])
602+ var ps []uint32
603+ if i < len(s.parents) {
604+ ps = s.parents[i]
605+ }
606+ p1, p2 := uint32(graphParentNone), uint32(graphParentNone)
607+ if len(ps) > 0 {
608+ p1 = ps[0]
609+ }
610+ switch {
611+ case len(ps) == 2:
612+ p2 = ps[1]
613+ case len(ps) > 2:
614+ p2 = graphParentExtra | uint32(edge.Len()/4)
615+ for j, p := range ps[1:] {
616+ if j == len(ps)-2 {
617+ p |= graphEdgeLast
618+ }
619+ put32(&edge, p)
620+ }
621+ }
622+ put32(&cdat, p1)
623+ put32(&cdat, p2)
624+ var tm int64
625+ if i < len(s.times) {
626+ tm = s.times[i]
627+ }
628+ var lv uint32 = 1
629+ if i < len(s.levels) {
630+ lv = s.levels[i]
631+ }
632+ put32(&cdat, lv<<2|uint32(tm>>32)&3)
633+ put32(&cdat, uint32(tm))
634+ }
635+ for _, v := range s.gda2 {
636+ put32(&gda2, v)
637+ }
638+ for _, v := range s.gdo2 {
639+ _ = binary.Write(&gdo2, be, v)
640+ }
641+ for _, v := range s.bidx {
642+ put32(&bidx, v)
643+ }
644+ for _, b := range s.base {
645+ base.Write(b[:])
646+ }
647+
648+ type chunk struct {
649+ id uint32
650+ data []byte
651+ }
652+ chunks := []chunk{{chunkOIDF, fanout.Bytes()}, {chunkOIDL, oidl.Bytes()}, {chunkCDAT, cdat.Bytes()}}
653+ if edge.Len() > 0 && !s.skipEdge {
654+ chunks = append(chunks, chunk{chunkEDGE, edge.Bytes()})
655+ }
656+ if s.gda2 != nil {
657+ chunks = append(chunks, chunk{chunkGDA2, gda2.Bytes()})
658+ }
659+ if s.gdo2 != nil {
660+ chunks = append(chunks, chunk{chunkGDO2, gdo2.Bytes()})
661+ }
662+ if s.bdat != nil {
663+ chunks = append(chunks, chunk{chunkBIDX, bidx.Bytes()}, chunk{chunkBDAT, s.bdat})
664+ }
665+ if len(s.base) > 0 {
666+ chunks = append(chunks, chunk{chunkBASE, base.Bytes()})
667+ }
668+
669+ var out bytes.Buffer
670+ out.WriteString("CGPH")
671+ out.Write([]byte{1, 1, byte(len(chunks)), byte(len(s.base))})
672+ off := uint64(graphHeaderSize + (len(chunks)+1)*graphChunkEntrySize)
673+ for _, c := range chunks {
674+ put32(&out, c.id)
675+ _ = binary.Write(&out, be, off)
676+ off += uint64(len(c.data))
677+ }
678+ put32(&out, 0)
679+ _ = binary.Write(&out, be, off)
680+ for _, c := range chunks {
681+ out.Write(c.data)
682+ }
683+ out.Write(make([]byte, graphHashSize))
684+ return out.Bytes()
685+}
686+
687+func synthIDs(n int) []oid {
688+ ids := make([]oid, n)
689+ for i := range ids {
690+ ids[i][0] = byte(i * 251 / max(n, 1))
691+ binary.BigEndian.PutUint32(ids[i][16:], uint32(i))
692+ }
693+ sort.Slice(ids, func(a, b int) bool { return bytes.Compare(ids[a][:], ids[b][:]) < 0 })
694+ return ids
695+}
696+
697+func mustParseSynth(t testing.TB, b []byte) *commitGraph {
698+ t.Helper()
699+ l, err := parseGraphLayer(b)
700+ if err != nil {
701+ t.Fatal(err)
702+ }
703+ g, err := assembleGraph([]*graphLayer{l}, nil)
704+ if err != nil {
705+ t.Fatal(err)
706+ }
707+ return g
708+}
709+
710+func TestGraphSyntheticOctopusAndOverflow(t *testing.T) {
711+ const n = 12
712+ s := &synthLayer{ids: synthIDs(n)}
713+ s.parents = make([][]uint32, n)
714+ s.times = make([]int64, n)
715+ s.gda2 = make([]uint32, n)
716+ for i := range n {
717+ s.times[i] = 1_000_000 + int64(i)
718+ s.gda2[i] = uint32(i)
719+ }
720+ s.parents[11] = []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10} // 11 parents
721+ s.parents[10] = []uint32{9}
722+ s.parents[9] = []uint32{8, 7}
723+ s.parents[8] = []uint32{7, 6, 5}
724+ // Overflow: commit 3 has a 40-bit offset, and a time over 32 bits.
725+ s.gda2[3] = graphGenOverflow | 1
726+ s.gdo2 = []uint64{7, 1 << 40}
727+ s.times[4] = 1<<33 + 5
728+
729+ g := mustParseSynth(t, s.bytes())
730+ defer func() { _ = g.close() }()
731+ var buf [4]uint32
732+ for pos := range uint32(n) {
733+ ps, err := g.appendParents(buf[:0], pos)
734+ if err != nil {
735+ t.Fatal(err)
736+ }
737+ want := s.parents[pos]
738+ if len(ps) != len(want) {
739+ t.Fatalf("pos %d: parents %v, want %v", pos, ps, want)
740+ }
741+ for i := range ps {
742+ if ps[i] != want[i] {
743+ t.Fatalf("pos %d: parents %v, want %v", pos, ps, want)
744+ }
745+ }
746+ }
747+ if !g.hasGenerationV2() {
748+ t.Fatal("expected generation v2")
749+ }
750+ if v, err := g.generation(3); err != nil || v != uint64(s.times[3])+1<<40 {
751+ t.Fatalf("overflow generation = %d, %v", v, err)
752+ }
753+ if v, err := g.generation(2); err != nil || v != uint64(s.times[2])+2 {
754+ t.Fatalf("generation(2) = %d, %v", v, err)
755+ }
756+ if v, err := g.commitTime(4); err != nil || v != 1<<33+5 {
757+ t.Fatalf("34-bit time = %d, %v", v, err)
758+ }
759+}
760+
761+func TestGraphSyntheticCorruption(t *testing.T) {
762+ const n = 6
763+ mk := func(f func(s *synthLayer)) *synthLayer {
764+ s := &synthLayer{ids: synthIDs(n), parents: make([][]uint32, n), times: make([]int64, n)}
765+ s.parents[5] = []uint32{0, 1, 2}
766+ if f != nil {
767+ f(s)
768+ }
769+ return s
770+ }
771+ t.Run("ok", func(t *testing.T) {
772+ g := mustParseSynth(t, mk(nil).bytes())
773+ _ = g.close()
774+ })
775+ t.Run("parent out of range", func(t *testing.T) {
776+ g := mustParseSynth(t, mk(func(s *synthLayer) { s.parents[2] = []uint32{9} }).bytes())
777+ if _, err := g.appendParents(nil, 2); !errors.Is(err, errGraphCorrupt) {
778+ t.Fatalf("err = %v", err)
779+ }
780+ // A parent equal to the commit's own layer end is rejected, a
781+ // parent in a later position of the same layer is accepted (the
782+ // graph is not required to be topologically ordered by id).
783+ if _, err := g.appendParents(nil, 5); err != nil {
784+ t.Fatal(err)
785+ }
786+ })
787+ t.Run("edge list overrun", func(t *testing.T) {
788+ g := mustParseSynth(t, mk(func(s *synthLayer) {
789+ s.parents[5] = []uint32{0, 1, 2}
790+ }).bytes())
791+ l := &g.layers[0]
792+ // Clear the "last" bit of the final EDGE entry.
793+ last := len(l.edge) - 4
794+ mutable := append([]byte(nil), l.edge...)
795+ mutable[last] &^= 0x80
796+ l.edge = mutable
797+ if _, err := g.appendParents(nil, 5); !errors.Is(err, errGraphCorrupt) {
798+ t.Fatalf("err = %v", err)
799+ }
800+ })
801+ t.Run("missing edge chunk", func(t *testing.T) {
802+ g := mustParseSynth(t, mk(func(s *synthLayer) { s.skipEdge = true }).bytes())
803+ if _, err := g.appendParents(nil, 5); !errors.Is(err, errGraphCorrupt) {
804+ t.Fatalf("err = %v", err)
805+ }
806+ })
807+ t.Run("overflow index out of range", func(t *testing.T) {
808+ g := mustParseSynth(t, mk(func(s *synthLayer) {
809+ s.gda2 = make([]uint32, n)
810+ s.gda2[1] = graphGenOverflow | 3
811+ s.gdo2 = []uint64{1}
812+ }).bytes())
813+ if _, err := g.generation(1); !errors.Is(err, errGraphCorrupt) {
814+ t.Fatalf("err = %v", err)
815+ }
816+ if _, err := g.generation(0); err != nil {
817+ t.Fatal(err)
818+ }
819+ })
820+ t.Run("second parent without first", func(t *testing.T) {
821+ g := mustParseSynth(t, mk(nil).bytes())
822+ l := &g.layers[0]
823+ rec := append([]byte(nil), l.cdat...)
824+ binary.BigEndian.PutUint32(rec[graphHashSize:], graphParentNone)
825+ binary.BigEndian.PutUint32(rec[graphHashSize+4:], 1)
826+ l.cdat = rec
827+ if _, err := g.appendParents(nil, 0); !errors.Is(err, errGraphCorrupt) {
828+ t.Fatalf("err = %v", err)
829+ }
830+ })
831+}
832+
833+func TestGraphParseErrors(t *testing.T) {
834+ good := (&synthLayer{ids: synthIDs(4)}).bytes()
835+ if _, err := parseGraphLayer(good); err != nil {
836+ t.Fatal(err)
837+ }
838+ mut := func(f func(b []byte) []byte) []byte { return f(append([]byte(nil), good...)) }
839+ cases := map[string][]byte{
840+ "empty": nil,
841+ "short": good[:20],
842+ "signature": mut(func(b []byte) []byte { b[0] = 'X'; return b }),
843+ "hash version": mut(func(b []byte) []byte { b[5] = 9; return b }),
844+ "chunk count": mut(func(b []byte) []byte { b[6] = 255; return b }),
845+ "offset past": mut(func(b []byte) []byte { binary.BigEndian.PutUint64(b[12:], 1<<62); return b }),
846+ "offset behind": mut(func(b []byte) []byte { binary.BigEndian.PutUint64(b[12:], 1); return b }),
847+ "dup chunk": mut(func(b []byte) []byte { copy(b[8+12:], b[8:12]); return b }),
848+ "fanout order": mut(func(b []byte) []byte { binary.BigEndian.PutUint32(b[(8+5*12)+4:], 99); return b }),
849+ "truncated": good[:len(good)-30],
850+ }
851+ for name, b := range cases {
852+ if _, err := parseGraphLayer(b); !errors.Is(err, errGraphCorrupt) {
853+ t.Errorf("%s: err = %v, want corrupt", name, err)
854+ }
855+ }
856+ sha256 := mut(func(b []byte) []byte { b[5] = 2; return b })
857+ if _, err := parseGraphLayer(sha256); !errors.Is(err, errGraphUnsupported) {
858+ t.Errorf("sha256: err = %v", err)
859+ }
860+ v2 := mut(func(b []byte) []byte { b[4] = 2; return b })
861+ if _, err := parseGraphLayer(v2); !errors.Is(err, errGraphUnsupported) {
862+ t.Errorf("version 2: err = %v", err)
863+ }
864+ // Every truncation and every single-byte corruption of the header and
865+ // chunk table must be handled without panicking.
866+ for i := range good {
867+ _, _ = parseGraphLayer(good[:i])
868+ }
869+ for i := range min(len(good), 8+6*12) {
870+ for _, v := range []byte{0, 1, 0x7f, 0x80, 0xff} {
871+ b := append([]byte(nil), good...)
872+ b[i] = v
873+ exerciseGraphBytes(b)
874+ }
875+ }
876+}
877+
878+func TestGraphChainParse(t *testing.T) {
879+ h1 := strings.Repeat("ab", 20)
880+ h2 := strings.Repeat("01", 20)
881+ names, err := parseGraphChain([]byte(h1 + "\n" + h2 + "\n"))
882+ if err != nil || len(names) != 2 || names[1][0] != 1 {
883+ t.Fatalf("names = %v, err = %v", names, err)
884+ }
885+ for _, bad := range []string{h1, h1 + "\r\n", strings.Repeat("zz", 20) + "\n", h1 + "\n" + "x\n"} {
886+ if _, err := parseGraphChain([]byte(bad)); !errors.Is(err, errGraphCorrupt) {
887+ t.Errorf("%q: err = %v", bad, err)
888+ }
889+ }
890+ if _, err := parseGraphChain(nil); !errors.Is(err, errNoGraph) {
891+ t.Errorf("empty chain: err = %v", err)
892+ }
893+}
894+
895+func TestGraphChainMismatch(t *testing.T) {
896+ r := newGTRepo(t)
897+ r.commit("a", "a\n")
898+ r.git("commit-graph", "write", "--reachable", "--split=no-merge")
899+ r.commit("b", "b\n")
900+ r.git("commit-graph", "write", "--reachable", "--split=no-merge")
901+ chainPath := filepath.Join(r.objects(), "info", "commit-graphs", "commit-graph-chain")
902+ raw, err := os.ReadFile(chainPath)
903+ if err != nil {
904+ t.Fatal(err)
905+ }
906+ lines := strings.Fields(string(raw))
907+ if len(lines) != 2 {
908+ t.Fatalf("chain = %q", raw)
909+ }
910+ g, err := openCommitGraph(r.objects())
911+ if err != nil {
912+ t.Fatal(err)
913+ }
914+ _ = g.close()
915+
916+ // Reverse: layer 0 would be the top layer, which declares a base.
917+ if err := os.Chmod(chainPath, 0o644); err != nil {
918+ t.Fatal(err)
919+ }
920+ if err := os.WriteFile(chainPath, []byte(lines[1]+"\n"+lines[0]+"\n"), 0o644); err != nil {
921+ t.Fatal(err)
922+ }
923+ if _, err := openCommitGraph(r.objects()); !errors.Is(err, errGraphCorrupt) {
924+ t.Fatalf("reversed chain: err = %v", err)
925+ }
926+ // Top layer alone declares one base but the chain has none.
927+ if err := os.WriteFile(chainPath, []byte(lines[1]+"\n"), 0o644); err != nil {
928+ t.Fatal(err)
929+ }
930+ if _, err := openCommitGraph(r.objects()); !errors.Is(err, errGraphCorrupt) {
931+ t.Fatalf("truncated chain: err = %v", err)
932+ }
933+ // A missing layer file is an error, not a panic.
934+ if err := os.WriteFile(chainPath, []byte(strings.Repeat("0", 40)+"\n"), 0o644); err != nil {
935+ t.Fatal(err)
936+ }
937+ if _, err := openCommitGraph(r.objects()); err == nil {
938+ t.Fatal("missing layer accepted")
939+ }
940+ // Two layers whose base list names the wrong checksum.
941+ if err := os.WriteFile(chainPath, []byte(lines[0]+"\n"+lines[0]+"\n"), 0o644); err != nil {
942+ t.Fatal(err)
943+ }
944+ if _, err := openCommitGraph(r.objects()); !errors.Is(err, errGraphCorrupt) {
945+ t.Fatalf("duplicated layer: err = %v", err)
946+ }
947+}
948+
949+func TestGraphCorruptFiles(t *testing.T) {
950+ r := newGTRepo(t)
951+ r.commit("a", "a\n")
952+ r.git("commit-graph", "write", "--reachable", "--changed-paths")
953+ path := filepath.Join(r.objects(), "info", "commit-graph")
954+ good, err := os.ReadFile(path)
955+ if err != nil {
956+ t.Fatal(err)
957+ }
958+ if err := os.Chmod(path, 0o644); err != nil {
959+ t.Fatal(err)
960+ }
961+ for _, b := range [][]byte{good[:10], {}, []byte("CGPH"), bytes.Repeat([]byte{0xff}, 200)} {
962+ if err := os.WriteFile(path, b, 0o644); err != nil {
963+ t.Fatal(err)
964+ }
965+ if _, err := openCommitGraph(r.objects()); err == nil {
966+ t.Fatalf("accepted %d corrupt bytes", len(b))
967+ }
968+ }
969+ // A directory in place of the file must not panic or be mapped.
970+ _ = os.Remove(path)
971+ if err := os.Mkdir(path, 0o755); err != nil {
972+ t.Fatal(err)
973+ }
974+ if _, err := openCommitGraph(r.objects()); err == nil {
975+ t.Fatal("directory accepted")
976+ }
977+}
978+
979+// ---- fuzzing --------------------------------------------------------------
980+
981+// exerciseGraphBytes runs every accessor over a layer parsed from data, alone
982+// and stacked on itself, and must never panic.
983+func exerciseGraphBytes(data []byte) {
984+ l, err := parseGraphLayer(data)
985+ if err != nil {
986+ return
987+ }
988+ one := *l
989+ one.numBase = 0
990+ two := one
991+ two.base = one.n
992+ for _, g := range []*commitGraph{
993+ {layers: []graphLayer{one}, total: one.n, genV2: one.gda2 != nil},
994+ {layers: []graphLayer{one, two}, total: 2 * one.n, genV2: one.gda2 != nil},
995+ } {
996+ var buf [4]uint32
997+ n := min(g.total, 256)
998+ for pos := range n {
999+ id, _ := g.commitID(pos)
1000+ g.lookup(id)
1001+ _, _ = g.rootTree(pos)
1002+ _, _ = g.commitTime(pos)
1003+ _, _ = g.topoLevel(pos)
1004+ _, _ = g.generation(pos)
1005+ _, _ = g.appendParents(buf[:0], pos)
1006+ g.maybeChanged(pos, []byte("src/main.go"))
1007+ g.hasBloom(pos)
1008+ }
1009+ _, _ = g.commitID(g.total)
1010+ _, _ = g.appendParents(nil, g.total+1)
1011+ g.maybeChanged(g.total, nil)
1012+ g.lookup(oid{0xff, 0xff})
1013+ g.lookup(oid{})
1014+ }
1015+}
1016+
1017+func graphSeedFiles(tb testing.TB) map[string][]byte {
1018+ tb.Helper()
1019+ entries, err := os.ReadDir(filepath.Join("testdata", "graph"))
1020+ if err != nil {
1021+ tb.Fatal(err)
1022+ }
1023+ seeds := map[string][]byte{}
1024+ for _, e := range entries {
1025+ if !strings.HasSuffix(e.Name(), ".graph") {
1026+ continue
1027+ }
1028+ b, err := os.ReadFile(filepath.Join("testdata", "graph", e.Name()))
1029+ if err != nil {
1030+ tb.Fatal(err)
1031+ }
1032+ seeds[e.Name()] = b
1033+ }
1034+ if len(seeds) == 0 {
1035+ tb.Fatal("no fuzz seeds in testdata/graph")
1036+ }
1037+ return seeds
1038+}
1039+
1040+func FuzzGraphFile(f *testing.F) {
1041+ for _, b := range graphSeedFiles(f) {
1042+ f.Add(b)
1043+ f.Add(b[:len(b)/2])
1044+ }
1045+ f.Fuzz(func(t *testing.T, data []byte) { exerciseGraphBytes(data) })
1046+}
1047+
1048+func FuzzGraphChain(f *testing.F) {
1049+ f.Add([]byte(strings.Repeat("ab", 20) + "\n"))
1050+ f.Add([]byte(strings.Repeat("ab", 20) + "\n" + strings.Repeat("cd", 20) + "\n"))
1051+ f.Add([]byte("\n"))
1052+ f.Fuzz(func(t *testing.T, data []byte) {
1053+ names, err := parseGraphChain(data)
1054+ if err == nil && len(names)*41 != len(data) {
1055+ t.Fatalf("parsed %d names from %d bytes", len(names), len(data))
1056+ }
1057+ })
1058+}
1059+
1060+// TestGraphSeedsParse checks the committed seeds stay valid files.
1061+func TestGraphSeedsParse(t *testing.T) {
1062+ for name, b := range graphSeedFiles(t) {
1063+ l, err := parseGraphLayer(b)
1064+ if err != nil {
1065+ t.Errorf("%s: %v", name, err)
1066+ continue
1067+ }
1068+ exerciseGraphBytes(b)
1069+ _ = l
1070+ }
1071+}
1072+
1073+// TestWriteGraphSeeds regenerates testdata/graph with the git binary:
1074+//
1075+// ERMITE_GIT_UPDATE_SEEDS=1 go test -run TestWriteGraphSeeds
1076+func TestWriteGraphSeeds(t *testing.T) {
1077+ if os.Getenv("ERMITE_GIT_UPDATE_SEEDS") == "" {
1078+ t.Skip("set ERMITE_GIT_UPDATE_SEEDS=1 to regenerate testdata/graph")
1079+ }
1080+ out := filepath.Join("testdata", "graph")
1081+ if err := os.MkdirAll(out, 0o755); err != nil {
1082+ t.Fatal(err)
1083+ }
1084+ save := func(name string, b []byte) {
1085+ if err := os.WriteFile(filepath.Join(out, name), b, 0o644); err != nil {
1086+ t.Fatal(err)
1087+ }
1088+ }
1089+ copyLayers := func(prefix string, r *gtRepo) {
1090+ dir := filepath.Join(r.objects(), "info")
1091+ if b, err := os.ReadFile(filepath.Join(dir, "commit-graph")); err == nil {
1092+ save(prefix+".graph", b)
1093+ return
1094+ }
1095+ chain, err := os.ReadFile(filepath.Join(dir, "commit-graphs", "commit-graph-chain"))
1096+ if err != nil {
1097+ t.Fatal(err)
1098+ }
1099+ for i, h := range strings.Fields(string(chain)) {
1100+ b, err := os.ReadFile(filepath.Join(dir, "commit-graphs", "graph-"+h+".graph"))
1101+ if err != nil {
1102+ t.Fatal(err)
1103+ }
1104+ save(fmt.Sprintf("%s-layer%d.graph", prefix, i), b)
1105+ }
1106+ }
1107+ for _, c := range []gcase{
1108+ {name: "single-nobloom"},
1109+ {name: "single-bloom-v2", bloom: true, cfg: []string{"commitGraph.changedPathsVersion=2"}},
1110+ {name: "split-bloom-v2", split: true, bloom: true, cfg: []string{"commitGraph.changedPathsVersion=2"}},
1111+ {name: "split-bloom-v1", split: true, bloom: true, cfg: []string{"commitGraph.changedPathsVersion=1"}},
1112+ {name: "split-gen1", split: true, cfg: []string{"commitGraph.generationVersion=1"}},
1113+ } {
1114+ r := newGTRepo(t)
1115+ r.stageOne()
1116+ r.writeGraph(c)
1117+ r.stageTwo()
1118+ r.writeGraph(c)
1119+ copyLayers(c.name, r)
1120+ }
1121+ // Synthetic: wide octopus, overflow generation, Bloom edge cases.
1122+ s := &synthLayer{ids: synthIDs(12), parents: make([][]uint32, 12), times: make([]int64, 12)}
1123+ s.parents[11] = []uint32{0, 1, 2, 3, 4, 5}
1124+ s.gda2 = make([]uint32, 12)
1125+ s.gda2[3] = graphGenOverflow
1126+ s.gdo2 = []uint64{1 << 40}
1127+ hdr := binary.BigEndian.AppendUint32(nil, 2)
1128+ hdr = binary.BigEndian.AppendUint32(hdr, 7)
1129+ hdr = binary.BigEndian.AppendUint32(hdr, 10)
1130+ s.bdat = append(hdr, 0x00, 0xff, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0)
1131+ s.bidx = []uint32{1, 2, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10}
1132+ save("synthetic.graph", s.bytes())
1133+}
1134+
1135+// ---- benchmarks -----------------------------------------------------------
1136+
1137+// fastImportRepo creates a bare repo with n commits (a merge every 50) using
1138+// git fast-import, and writes its split commit-graph.
1139+func fastImportRepo(tb testing.TB, n int) string {
1140+ tb.Helper()
1141+ requireGit(tb)
1142+ dir := tb.TempDir()
1143+ graphGit(tb, dir, 0, "init", "-q", "--bare", "-b", "main")
1144+ cmd := exec.Command("git", "fast-import", "--quiet")
1145+ cmd.Dir = dir
1146+ cmd.Env = graphGitEnv(0)
1147+ var stderr bytes.Buffer
1148+ cmd.Stderr = &stderr
1149+ in, err := cmd.StdinPipe()
1150+ if err != nil {
1151+ tb.Fatal(err)
1152+ }
1153+ if err := cmd.Start(); err != nil {
1154+ tb.Fatal(err)
1155+ }
1156+ w := bufio.NewWriter(in)
1157+ for i := 1; i <= n; i++ {
1158+ msg := "commit " + strconv.Itoa(i)
1159+ _, _ = fmt.Fprintf(w, "commit refs/heads/main\nmark :%d\ncommitter T <t@example.com> %d +0000\ndata %d\n%s\n", i, 1_600_000_000+i*30, len(msg), msg)
1160+ if i > 1 {
1161+ _, _ = fmt.Fprintf(w, "from :%d\n", i-1)
1162+ }
1163+ if i > 60 && i%50 == 0 {
1164+ _, _ = fmt.Fprintf(w, "merge :%d\n", i-30)
1165+ }
1166+ for _, p := range []string{
1167+ fmt.Sprintf("d%d/f%d.txt", i%17, i%5),
1168+ fmt.Sprintf("pkg%d/sub%d/g%d.go", i%7, i%3, i),
1169+ } {
1170+ body := "v" + strconv.Itoa(i) + "\n"
1171+ _, _ = fmt.Fprintf(w, "M 644 inline %s\ndata %d\n%s", p, len(body), body)
1172+ }
1173+ _, _ = w.WriteString("\n")
1174+ }
1175+ if err := w.Flush(); err != nil {
1176+ tb.Fatal(err)
1177+ }
1178+ _ = in.Close()
1179+ if err := cmd.Wait(); err != nil {
1180+ tb.Fatalf("fast-import: %v\n%s", err, stderr.String())
1181+ }
1182+ graphGit(tb, dir, 0, "commit-graph", "write", "--reachable", "--changed-paths", "--split")
1183+ return dir
1184+}
1185+
1186+func benchGraph(b *testing.B) (*commitGraph, []oid) {
1187+ dir := fastImportRepo(b, 5000)
1188+ g, err := openCommitGraph(filepath.Join(dir, "objects"))
1189+ if err != nil {
1190+ b.Fatal(err)
1191+ }
1192+ b.Cleanup(func() { _ = g.close() })
1193+ ids := make([]oid, g.numCommits())
1194+ for i := range ids {
1195+ ids[i], _ = g.commitID(uint32(i))
1196+ }
1197+ return g, ids
1198+}
1199+
1200+func BenchmarkGraphLookup(b *testing.B) {
1201+ g, ids := benchGraph(b)
1202+ b.ReportAllocs()
1203+ i := 0
1204+ for b.Loop() {
1205+ if _, ok := g.lookup(ids[i]); !ok {
1206+ b.Fatal("miss")
1207+ }
1208+ if i++; i == len(ids) {
1209+ i = 0
1210+ }
1211+ }
1212+}
1213+
1214+func BenchmarkGraphParents(b *testing.B) {
1215+ g, _ := benchGraph(b)
1216+ var buf [8]uint32
1217+ n := g.numCommits()
1218+ b.ReportAllocs()
1219+ var pos uint32
1220+ for b.Loop() {
1221+ if _, err := g.appendParents(buf[:0], pos); err != nil {
1222+ b.Fatal(err)
1223+ }
1224+ if pos++; pos == n {
1225+ pos = 0
1226+ }
1227+ }
1228+}
1229+
1230+func BenchmarkGraphCommitData(b *testing.B) {
1231+ g, _ := benchGraph(b)
1232+ n := g.numCommits()
1233+ b.ReportAllocs()
1234+ var pos uint32
1235+ for b.Loop() {
1236+ if _, err := g.generation(pos); err != nil {
1237+ b.Fatal(err)
1238+ }
1239+ if _, err := g.commitTime(pos); err != nil {
1240+ b.Fatal(err)
1241+ }
1242+ if _, err := g.rootTree(pos); err != nil {
1243+ b.Fatal(err)
1244+ }
1245+ if pos++; pos == n {
1246+ pos = 0
1247+ }
1248+ }
1249+}
@@ -0,0 +1,1249 @@
1+// SPDX-License-Identifier: MIT
2+
3+package git
4+
5+import (
6+ "bufio"
7+ "bytes"
8+ "encoding/binary"
9+ "encoding/hex"
10+ "errors"
11+ "fmt"
12+ "os"
13+ "os/exec"
14+ "path/filepath"
15+ "sort"
16+ "strconv"
17+ "strings"
18+ "testing"
19+)
20+
21+// oid is the raw object id used by the graph code until the core's ObjectID
22+// is merged.
23+type oid = [20]byte
24+
25+// graphTestHex parses a 40-digit hex string.
26+func graphTestHex(s string) oid {
27+ var id oid
28+ if _, err := hex.Decode(id[:], []byte(s)); err != nil || len(s) != 40 {
29+ panic("graphTestHex: bad id " + strconv.Quote(s))
30+ }
31+ return id
32+}
33+
34+// ---- git fixtures ---------------------------------------------------------
35+
36+func requireGit(t testing.TB) {
37+ t.Helper()
38+ if _, err := exec.LookPath("git"); err != nil {
39+ t.Skip("git binary not available")
40+ }
41+}
42+
43+func graphGitEnv(date int64) []string {
44+ env := append(os.Environ(),
45+ "GIT_CONFIG_GLOBAL=/dev/null",
46+ "GIT_CONFIG_NOSYSTEM=1",
47+ "GIT_TERMINAL_PROMPT=0",
48+ "GIT_AUTHOR_NAME=T", "GIT_AUTHOR_EMAIL=t@example.com",
49+ "GIT_COMMITTER_NAME=T", "GIT_COMMITTER_EMAIL=t@example.com",
50+ )
51+ if date != 0 {
52+ d := strconv.FormatInt(date, 10) + " +0000"
53+ env = append(env, "GIT_AUTHOR_DATE="+d, "GIT_COMMITTER_DATE="+d)
54+ }
55+ return env
56+}
57+
58+func graphGit(t testing.TB, dir string, date int64, args ...string) string {
59+ t.Helper()
60+ return graphGitIn(t, dir, date, "", args...)
61+}
62+
63+func graphGitStdin(t testing.TB, dir, stdin string, args ...string) string {
64+ t.Helper()
65+ return graphGitIn(t, dir, 0, stdin, args...)
66+}
67+
68+func graphGitIn(t testing.TB, dir string, date int64, stdin string, args ...string) string {
69+ t.Helper()
70+ cmd := exec.Command("git", args...)
71+ cmd.Stdin = strings.NewReader(stdin)
72+ cmd.Dir = dir
73+ cmd.Env = graphGitEnv(date)
74+ var stderr bytes.Buffer
75+ cmd.Stderr = &stderr
76+ out, err := cmd.Output()
77+ if err != nil {
78+ t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, stderr.String())
79+ }
80+ return string(out)
81+}
82+
83+// gtRepo is a scratch repository with a deterministic clock.
84+type gtRepo struct {
85+ t testing.TB
86+ dir string
87+ clock int64
88+}
89+
90+func newGTRepo(t testing.TB) *gtRepo {
91+ t.Helper()
92+ requireGit(t)
93+ r := &gtRepo{t: t, dir: t.TempDir(), clock: 1_700_000_000}
94+ r.git("init", "-q", "-b", "main")
95+ return r
96+}
97+
98+func (r *gtRepo) git(args ...string) string {
99+ r.t.Helper()
100+ return graphGit(r.t, r.dir, r.clock, args...)
101+}
102+
103+func (r *gtRepo) objects() string { return filepath.Join(r.dir, ".git", "objects") }
104+
105+func (r *gtRepo) write(path, content string) {
106+ r.t.Helper()
107+ full := filepath.Join(r.dir, path)
108+ if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
109+ r.t.Fatal(err)
110+ }
111+ if err := os.WriteFile(full, []byte(content), 0o644); err != nil {
112+ r.t.Fatal(err)
113+ }
114+}
115+
116+func (r *gtRepo) tick() int64 {
117+ r.clock += 60
118+ return r.clock
119+}
120+
121+// commit writes files (path, content pairs) and commits them.
122+func (r *gtRepo) commit(files ...string) {
123+ r.t.Helper()
124+ for i := 0; i+1 < len(files); i += 2 {
125+ r.write(files[i], files[i+1])
126+ }
127+ r.tick()
128+ r.git("add", "-A")
129+ r.git("commit", "-q", "-m", fmt.Sprintf("c%d", r.clock))
130+}
131+
132+// commitAt commits with an explicit date (to build clock skew).
133+func (r *gtRepo) commitAt(date int64, files ...string) {
134+ r.t.Helper()
135+ for i := 0; i+1 < len(files); i += 2 {
136+ r.write(files[i], files[i+1])
137+ }
138+ r.git("add", "-A")
139+ graphGit(r.t, r.dir, date, "commit", "-q", "-m", fmt.Sprintf("c%d", date))
140+}
141+
142+func (r *gtRepo) remove(path string) {
143+ r.t.Helper()
144+ r.tick()
145+ r.git("rm", "-q", "-r", path)
146+ r.git("commit", "-q", "-m", "rm "+path)
147+}
148+
149+// stageOne: linear history with nested dirs, deletes, non-ASCII and
150+// high-bit paths, an empty commit, clock skew and ordinary merges.
151+func (r *gtRepo) stageOne() {
152+ r.commit("README", "r\n", "src/a.go", "a\n", "src/pkg/b.go", "b\n")
153+ r.commit("src/a.go", "a2\n", "docs/guide.md", "g\n")
154+ r.commit("café/menü.txt", "x\n", "\xe9\xe8/\xff\xfe.txt", "latin1\n", "日本語/ファイル.txt", "j\n")
155+ r.commit("src/pkg/deep/er/c.go", "c\n")
156+ r.tick()
157+ r.git("commit", "-q", "--allow-empty", "-m", "empty")
158+ r.remove("docs")
159+ // clock skew: a child older than its parent.
160+ r.commitAt(r.clock-5000, "src/skew.go", "s\n")
161+ r.commit("src/a.go", "a3\n")
162+
163+ r.git("checkout", "-q", "-b", "topic")
164+ r.commit("topic/one.txt", "1\n")
165+ r.commit("topic/two.txt", "2\n", "src/pkg/b.go", "b2\n")
166+ r.git("checkout", "-q", "main")
167+ r.commit("main-only.txt", "m\n")
168+ r.tick()
169+ r.git("merge", "-q", "--no-ff", "-m", "merge topic", "topic")
170+ r.commit("README", "r2\n")
171+}
172+
173+// stageTwo: an octopus merge (5 parents) and a merge of an older branch.
174+func (r *gtRepo) stageTwo() {
175+ for i := 1; i <= 4; i++ {
176+ r.git("checkout", "-q", "-b", fmt.Sprintf("b%d", i), "main")
177+ r.commit(fmt.Sprintf("oct/b%d.txt", i), fmt.Sprintf("%d\n", i), "ünï/b"+strconv.Itoa(i), "x\n")
178+ r.commit(fmt.Sprintf("oct/b%d.txt", i), fmt.Sprintf("%d%d\n", i, i))
179+ }
180+ r.git("checkout", "-q", "main")
181+ r.commit("after.txt", "a\n")
182+ r.tick()
183+ r.git("merge", "-q", "--no-ff", "-m", "octopus", "b1", "b2", "b3", "b4")
184+ r.commit("src/a.go", "a4\n")
185+ r.tick()
186+ r.git("merge", "-q", "--no-ff", "-m", "merge topic again", "topic")
187+}
188+
189+// stageThree: more linear commits, a merge whose parents are in base layers.
190+func (r *gtRepo) stageThree() {
191+ r.git("checkout", "-q", "-b", "late", "topic")
192+ r.commit("late/x.txt", "x\n")
193+ r.git("checkout", "-q", "main")
194+ r.commit("src/pkg/b.go", "b3\n")
195+ r.tick()
196+ r.git("merge", "-q", "--no-ff", "-m", "merge late", "late")
197+ for i := range 5 {
198+ r.commit(fmt.Sprintf("tail/%d.txt", i), "t\n")
199+ }
200+}
201+
202+type gcase struct {
203+ name string
204+ split bool
205+ bloom bool
206+ cfg []string // -c key=value before the subcommand
207+ bloomVer uint32 // expected BDAT version when bloom
208+ wantGenV2 bool
209+}
210+
211+// writeGraph runs git commit-graph write. git leaves its files read-only.
212+func (r *gtRepo) writeGraph(c gcase) {
213+ r.t.Helper()
214+ args := []string{}
215+ for _, kv := range c.cfg {
216+ args = append(args, "-c", kv)
217+ }
218+ args = append(args, "commit-graph", "write", "--reachable")
219+ if c.bloom {
220+ args = append(args, "--changed-paths")
221+ } else {
222+ args = append(args, "--no-changed-paths")
223+ }
224+ if c.split {
225+ args = append(args, "--split=no-merge")
226+ }
227+ r.git(args...)
228+}
229+
230+// buildCase creates the three-stage history, writing a graph after each.
231+func buildCase(t *testing.T, c gcase) *gtRepo {
232+ r := newGTRepo(t)
233+ r.stageOne()
234+ r.writeGraph(c)
235+ r.stageTwo()
236+ r.writeGraph(c)
237+ r.stageThree()
238+ r.writeGraph(c)
239+ r.git("commit-graph", "verify")
240+ return r
241+}
242+
243+// ---- reference data from git ---------------------------------------------
244+
245+type refCommit struct {
246+ id oid
247+ tree oid
248+ parents []oid
249+ ctime int64
250+}
251+
252+func gitCommits(t testing.TB, dir string) []refCommit {
253+ t.Helper()
254+ out := graphGit(t, dir, 0, "log", "--all", "--format=%H %T %P|%ct")
255+ var cs []refCommit
256+ for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
257+ left, ct, ok := strings.Cut(line, "|")
258+ if !ok {
259+ t.Fatalf("bad log line %q", line)
260+ }
261+ f := strings.Fields(left)
262+ c := refCommit{id: graphTestHex(f[0]), tree: graphTestHex(f[1])}
263+ for _, p := range f[2:] {
264+ c.parents = append(c.parents, graphTestHex(p))
265+ }
266+ c.ctime, _ = strconv.ParseInt(ct, 10, 64)
267+ cs = append(cs, c)
268+ }
269+ return cs
270+}
271+
272+// checkGraphAgainstGit compares every commit of g with git's view.
273+func checkGraphAgainstGit(t *testing.T, dir string, g *commitGraph, wantMerges bool) []refCommit {
274+ t.Helper()
275+ cs := gitCommits(t, dir)
276+ if int(g.numCommits()) != len(cs) {
277+ t.Fatalf("numCommits = %d, git has %d commits", g.numCommits(), len(cs))
278+ }
279+ byID := make(map[oid]*refCommit, len(cs))
280+ for i := range cs {
281+ byID[cs[i].id] = &cs[i]
282+ }
283+ var buf [8]uint32
284+ sawOctopus, sawMerge := false, false
285+ for i := range cs {
286+ c := &cs[i]
287+ pos, ok := g.lookup(c.id)
288+ if !ok {
289+ t.Fatalf("lookup(%x) not found", c.id)
290+ }
291+ if id, err := g.commitID(pos); err != nil || id != c.id {
292+ t.Fatalf("commitID(%d) = %x, %v; want %x", pos, id, err, c.id)
293+ }
294+ if tree, err := g.rootTree(pos); err != nil || tree != c.tree {
295+ t.Fatalf("rootTree(%d) = %x, %v; want %x", pos, tree, err, c.tree)
296+ }
297+ if ct, err := g.commitTime(pos); err != nil || ct != c.ctime {
298+ t.Fatalf("commitTime(%d) = %d, %v; want %d", pos, ct, err, c.ctime)
299+ }
300+ ps, err := g.appendParents(buf[:0], pos)
301+ if err != nil {
302+ t.Fatalf("appendParents(%d): %v", pos, err)
303+ }
304+ if len(ps) != len(c.parents) {
305+ t.Fatalf("commit %x: %d parents, git has %d", c.id, len(ps), len(c.parents))
306+ }
307+ for j, p := range ps {
308+ pid, err := g.commitID(p)
309+ if err != nil || pid != c.parents[j] {
310+ t.Fatalf("commit %x parent %d = %x, %v; want %x", c.id, j, pid, err, c.parents[j])
311+ }
312+ }
313+ sawMerge = sawMerge || len(ps) == 2
314+ sawOctopus = sawOctopus || len(ps) > 2
315+ }
316+ if wantMerges && (!sawMerge || !sawOctopus) {
317+ t.Errorf("fixture lacks merge (%v) or octopus (%v)", sawMerge, sawOctopus)
318+ }
319+ checkGeneration(t, g, cs, byID, wantMerges)
320+ return cs
321+}
322+
323+// checkGeneration verifies the topological level exactly (1 + max parents)
324+// and, when present, the corrected commit date (max(date, max parents + 1)).
325+func checkGeneration(t *testing.T, g *commitGraph, cs []refCommit, byID map[oid]*refCommit, wantSkew bool) {
326+ t.Helper()
327+ level := func(id oid) uint32 {
328+ pos, _ := g.lookup(id)
329+ v, err := g.topoLevel(pos)
330+ if err != nil {
331+ t.Fatal(err)
332+ }
333+ return v
334+ }
335+ gen := func(id oid) uint64 {
336+ pos, _ := g.lookup(id)
337+ v, err := g.generation(pos)
338+ if err != nil {
339+ t.Fatal(err)
340+ }
341+ return v
342+ }
343+ skewed := false
344+ for i := range cs {
345+ c := &cs[i]
346+ wantLevel := uint32(1)
347+ wantGen := uint64(c.ctime)
348+ for _, p := range c.parents {
349+ if l := level(p) + 1; l > wantLevel {
350+ wantLevel = l
351+ }
352+ if v := gen(p) + 1; g.hasGenerationV2() && v > wantGen {
353+ wantGen = v
354+ }
355+ }
356+ if got := level(c.id); got != wantLevel {
357+ t.Fatalf("topoLevel(%x) = %d, want %d", c.id, got, wantLevel)
358+ }
359+ if g.hasGenerationV2() {
360+ got := gen(c.id)
361+ if got != wantGen {
362+ t.Fatalf("generation(%x) = %d, want corrected date %d", c.id, got, wantGen)
363+ }
364+ skewed = skewed || got != uint64(c.ctime)
365+ } else if got := gen(c.id); got != uint64(wantLevel) {
366+ t.Fatalf("generation(%x) = %d, want level %d", c.id, got, wantLevel)
367+ }
368+ for _, p := range c.parents {
369+ if gen(p) >= gen(c.id) {
370+ t.Fatalf("generation not increasing from parent %x to %x", p, c.id)
371+ }
372+ }
373+ }
374+ if wantSkew && g.hasGenerationV2() && !skewed {
375+ t.Error("clock skew fixture produced no corrected date different from the commit time")
376+ }
377+ _ = byID
378+}
379+
380+// ---- conformance ----------------------------------------------------------
381+
382+func graphCases() []gcase {
383+ return []gcase{
384+ {name: "single", wantGenV2: true},
385+ {name: "single-bloom", bloom: true, bloomVer: 1, wantGenV2: true},
386+ {name: "split", split: true, wantGenV2: true},
387+ {name: "split-bloom", split: true, bloom: true, bloomVer: 1, wantGenV2: true},
388+ {name: "split-bloom-v1", split: true, bloom: true, bloomVer: 1, wantGenV2: true,
389+ cfg: []string{"commitGraph.changedPathsVersion=1"}},
390+ {name: "split-bloom-v2-explicit", split: true, bloom: true, bloomVer: 2, wantGenV2: true,
391+ cfg: []string{"commitGraph.changedPathsVersion=2"}},
392+ {name: "split-gen1", split: true, bloom: true, bloomVer: 1,
393+ cfg: []string{"commitGraph.generationVersion=1"}},
394+ {name: "single-gen1", cfg: []string{"commitGraph.generationVersion=1"}},
395+ }
396+}
397+
398+func TestGraphConformance(t *testing.T) {
399+ for _, c := range graphCases() {
400+ t.Run(c.name, func(t *testing.T) {
401+ r := buildCase(t, c)
402+ g, err := openCommitGraph(r.objects())
403+ if err != nil {
404+ t.Fatal(err)
405+ }
406+ defer func() { _ = g.close() }()
407+
408+ wantLayers := 1
409+ if c.split {
410+ wantLayers = 3
411+ }
412+ if g.numLayers() != wantLayers {
413+ t.Fatalf("layers = %d, want %d", g.numLayers(), wantLayers)
414+ }
415+ if g.hasGenerationV2() != c.wantGenV2 {
416+ t.Fatalf("hasGenerationV2 = %v, want %v", g.hasGenerationV2(), c.wantGenV2)
417+ }
418+ cs := checkGraphAgainstGit(t, r.dir, g, true)
419+
420+ // Positions are dense and round-trip, and absent ids miss.
421+ for pos := uint32(0); pos < g.numCommits(); pos++ {
422+ id, err := g.commitID(pos)
423+ if err != nil {
424+ t.Fatal(err)
425+ }
426+ if p, ok := g.lookup(id); !ok || p != pos {
427+ t.Fatalf("lookup(commitID(%d)) = %d, %v", pos, p, ok)
428+ }
429+ }
430+ for i := range cs {
431+ miss := cs[i].id
432+ miss[19] ^= 0x5a
433+ if _, ok := g.lookup(miss); ok {
434+ if _, real := byIDHas(cs, miss); !real {
435+ t.Fatalf("lookup found absent id %x", miss)
436+ }
437+ }
438+ }
439+ if _, err := g.commitID(g.numCommits()); !errors.Is(err, errGraphPosition) {
440+ t.Fatalf("commitID(numCommits) err = %v", err)
441+ }
442+ if _, err := g.appendParents(nil, g.numCommits()); !errors.Is(err, errGraphPosition) {
443+ t.Fatalf("appendParents(numCommits) err = %v", err)
444+ }
445+
446+ checkBloomAgainstGit(t, r.dir, g, cs, c)
447+ })
448+ }
449+}
450+
451+func byIDHas(cs []refCommit, id oid) (int, bool) {
452+ for i := range cs {
453+ if cs[i].id == id {
454+ return i, true
455+ }
456+ }
457+ return 0, false
458+}
459+
460+func TestGraphNone(t *testing.T) {
461+ r := newGTRepo(t)
462+ r.commit("a", "a\n")
463+ if _, err := openCommitGraph(r.objects()); !errors.Is(err, errNoGraph) {
464+ t.Fatalf("err = %v, want errNoGraph", err)
465+ }
466+}
467+
468+// A split chain written with the default merge strategy still reads fine.
469+func TestGraphSplitDefaultMerge(t *testing.T) {
470+ r := newGTRepo(t)
471+ for i := range 4 {
472+ r.commit(fmt.Sprintf("f%d", i), "x\n")
473+ r.git("commit-graph", "write", "--reachable", "--split", "--changed-paths")
474+ }
475+ g, err := openCommitGraph(r.objects())
476+ if err != nil {
477+ t.Fatal(err)
478+ }
479+ defer func() { _ = g.close() }()
480+ checkGraphAgainstGit(t, r.dir, g, false)
481+}
482+
483+func TestGraphRefcount(t *testing.T) {
484+ r := newGTRepo(t)
485+ r.commit("a", "a\n")
486+ r.git("commit-graph", "write", "--reachable")
487+ g, err := openCommitGraph(r.objects())
488+ if err != nil {
489+ t.Fatal(err)
490+ }
491+ if !g.retain() {
492+ t.Fatal("retain failed on open graph")
493+ }
494+ if err := g.close(); err != nil { // owner drops its reference
495+ t.Fatal(err)
496+ }
497+ // Still usable: the reader holds a reference.
498+ if g.numCommits() != 1 {
499+ t.Fatal("numCommits")
500+ }
501+ if _, err := g.commitID(0); err != nil {
502+ t.Fatal(err)
503+ }
504+ if err := g.release(); err != nil {
505+ t.Fatal(err)
506+ }
507+ if g.retain() {
508+ t.Fatal("retain succeeded on closed graph")
509+ }
510+ if err := g.release(); err == nil {
511+ t.Fatal("over-release not reported")
512+ }
513+}
514+
515+// A layer removed or rewritten while a graph is open must not affect it.
516+func TestGraphSurvivesRewrite(t *testing.T) {
517+ r := newGTRepo(t)
518+ r.stageOne()
519+ c := gcase{split: true, bloom: true}
520+ r.writeGraph(c)
521+ g, err := openCommitGraph(r.objects())
522+ if err != nil {
523+ t.Fatal(err)
524+ }
525+ defer func() { _ = g.close() }()
526+ before := g.numCommits()
527+ r.stageTwo()
528+ r.git("commit-graph", "write", "--reachable", "--changed-paths") // replaces the chain
529+ if st := statGraph(r.objects()); !st.single.exists || st.chain.exists {
530+ t.Fatalf("unexpected stamp %+v", st)
531+ }
532+ if g.numCommits() != before {
533+ t.Fatal("numCommits changed")
534+ }
535+ for pos := uint32(0); pos < before; pos++ {
536+ if _, err := g.commitID(pos); err != nil {
537+ t.Fatal(err)
538+ }
539+ }
540+}
541+
542+func TestGraphStamp(t *testing.T) {
543+ r := newGTRepo(t)
544+ r.commit("a", "a\n")
545+ s0 := statGraph(r.objects())
546+ if s0.single.exists || s0.chain.exists {
547+ t.Fatalf("stamp = %+v", s0)
548+ }
549+ r.git("commit-graph", "write", "--reachable", "--split")
550+ s1 := statGraph(r.objects())
551+ if !s1.chain.exists || s1 == s0 {
552+ t.Fatalf("stamp = %+v", s1)
553+ }
554+ r.commit("b", "b\n")
555+ r.git("commit-graph", "write", "--reachable", "--split=no-merge")
556+ if s2 := statGraph(r.objects()); s2 == s1 {
557+ t.Fatal("stamp did not change after a new layer")
558+ }
559+}
560+
561+// ---- synthetic graph writer ----------------------------------------------
562+
563+// synthLayer builds a commit-graph layer byte for byte, for cases git does
564+// not produce (generation overflow, wide octopus, corruption).
565+type synthLayer struct {
566+ ids []oid // must be sorted
567+ trees []oid
568+ parents [][]uint32 // global positions
569+ times []int64
570+ levels []uint32
571+ gda2 []uint32 // nil: no chunk
572+ gdo2 []uint64
573+ bidx []uint32
574+ bdat []byte // including the 12-byte header
575+ base []oid // base graph checksums
576+ skipEdge bool
577+}
578+
579+func (s *synthLayer) bytes() []byte {
580+ n := len(s.ids)
581+ var oidl, cdat, edge, gda2, gdo2, bidx, base bytes.Buffer
582+ fan := make([]uint32, 256)
583+ for _, id := range s.ids {
584+ fan[id[0]]++
585+ oidl.Write(id[:])
586+ }
587+ for i := 1; i < 256; i++ {
588+ fan[i] += fan[i-1]
589+ }
590+ var fanout bytes.Buffer
591+ for _, v := range fan {
592+ _ = binary.Write(&fanout, binary.BigEndian, v)
593+ }
594+ be := binary.BigEndian
595+ put32 := func(b *bytes.Buffer, v uint32) { _ = binary.Write(b, be, v) }
596+ for i := range n {
597+ var tree oid
598+ if i < len(s.trees) {
599+ tree = s.trees[i]
600+ }
601+ cdat.Write(tree[:])
602+ var ps []uint32
603+ if i < len(s.parents) {
604+ ps = s.parents[i]
605+ }
606+ p1, p2 := uint32(graphParentNone), uint32(graphParentNone)
607+ if len(ps) > 0 {
608+ p1 = ps[0]
609+ }
610+ switch {
611+ case len(ps) == 2:
612+ p2 = ps[1]
613+ case len(ps) > 2:
614+ p2 = graphParentExtra | uint32(edge.Len()/4)
615+ for j, p := range ps[1:] {
616+ if j == len(ps)-2 {
617+ p |= graphEdgeLast
618+ }
619+ put32(&edge, p)
620+ }
621+ }
622+ put32(&cdat, p1)
623+ put32(&cdat, p2)
624+ var tm int64
625+ if i < len(s.times) {
626+ tm = s.times[i]
627+ }
628+ var lv uint32 = 1
629+ if i < len(s.levels) {
630+ lv = s.levels[i]
631+ }
632+ put32(&cdat, lv<<2|uint32(tm>>32)&3)
633+ put32(&cdat, uint32(tm))
634+ }
635+ for _, v := range s.gda2 {
636+ put32(&gda2, v)
637+ }
638+ for _, v := range s.gdo2 {
639+ _ = binary.Write(&gdo2, be, v)
640+ }
641+ for _, v := range s.bidx {
642+ put32(&bidx, v)
643+ }
644+ for _, b := range s.base {
645+ base.Write(b[:])
646+ }
647+
648+ type chunk struct {
649+ id uint32
650+ data []byte
651+ }
652+ chunks := []chunk{{chunkOIDF, fanout.Bytes()}, {chunkOIDL, oidl.Bytes()}, {chunkCDAT, cdat.Bytes()}}
653+ if edge.Len() > 0 && !s.skipEdge {
654+ chunks = append(chunks, chunk{chunkEDGE, edge.Bytes()})
655+ }
656+ if s.gda2 != nil {
657+ chunks = append(chunks, chunk{chunkGDA2, gda2.Bytes()})
658+ }
659+ if s.gdo2 != nil {
660+ chunks = append(chunks, chunk{chunkGDO2, gdo2.Bytes()})
661+ }
662+ if s.bdat != nil {
663+ chunks = append(chunks, chunk{chunkBIDX, bidx.Bytes()}, chunk{chunkBDAT, s.bdat})
664+ }
665+ if len(s.base) > 0 {
666+ chunks = append(chunks, chunk{chunkBASE, base.Bytes()})
667+ }
668+
669+ var out bytes.Buffer
670+ out.WriteString("CGPH")
671+ out.Write([]byte{1, 1, byte(len(chunks)), byte(len(s.base))})
672+ off := uint64(graphHeaderSize + (len(chunks)+1)*graphChunkEntrySize)
673+ for _, c := range chunks {
674+ put32(&out, c.id)
675+ _ = binary.Write(&out, be, off)
676+ off += uint64(len(c.data))
677+ }
678+ put32(&out, 0)
679+ _ = binary.Write(&out, be, off)
680+ for _, c := range chunks {
681+ out.Write(c.data)
682+ }
683+ out.Write(make([]byte, graphHashSize))
684+ return out.Bytes()
685+}
686+
687+func synthIDs(n int) []oid {
688+ ids := make([]oid, n)
689+ for i := range ids {
690+ ids[i][0] = byte(i * 251 / max(n, 1))
691+ binary.BigEndian.PutUint32(ids[i][16:], uint32(i))
692+ }
693+ sort.Slice(ids, func(a, b int) bool { return bytes.Compare(ids[a][:], ids[b][:]) < 0 })
694+ return ids
695+}
696+
697+func mustParseSynth(t testing.TB, b []byte) *commitGraph {
698+ t.Helper()
699+ l, err := parseGraphLayer(b)
700+ if err != nil {
701+ t.Fatal(err)
702+ }
703+ g, err := assembleGraph([]*graphLayer{l}, nil)
704+ if err != nil {
705+ t.Fatal(err)
706+ }
707+ return g
708+}
709+
710+func TestGraphSyntheticOctopusAndOverflow(t *testing.T) {
711+ const n = 12
712+ s := &synthLayer{ids: synthIDs(n)}
713+ s.parents = make([][]uint32, n)
714+ s.times = make([]int64, n)
715+ s.gda2 = make([]uint32, n)
716+ for i := range n {
717+ s.times[i] = 1_000_000 + int64(i)
718+ s.gda2[i] = uint32(i)
719+ }
720+ s.parents[11] = []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10} // 11 parents
721+ s.parents[10] = []uint32{9}
722+ s.parents[9] = []uint32{8, 7}
723+ s.parents[8] = []uint32{7, 6, 5}
724+ // Overflow: commit 3 has a 40-bit offset, and a time over 32 bits.
725+ s.gda2[3] = graphGenOverflow | 1
726+ s.gdo2 = []uint64{7, 1 << 40}
727+ s.times[4] = 1<<33 + 5
728+
729+ g := mustParseSynth(t, s.bytes())
730+ defer func() { _ = g.close() }()
731+ var buf [4]uint32
732+ for pos := range uint32(n) {
733+ ps, err := g.appendParents(buf[:0], pos)
734+ if err != nil {
735+ t.Fatal(err)
736+ }
737+ want := s.parents[pos]
738+ if len(ps) != len(want) {
739+ t.Fatalf("pos %d: parents %v, want %v", pos, ps, want)
740+ }
741+ for i := range ps {
742+ if ps[i] != want[i] {
743+ t.Fatalf("pos %d: parents %v, want %v", pos, ps, want)
744+ }
745+ }
746+ }
747+ if !g.hasGenerationV2() {
748+ t.Fatal("expected generation v2")
749+ }
750+ if v, err := g.generation(3); err != nil || v != uint64(s.times[3])+1<<40 {
751+ t.Fatalf("overflow generation = %d, %v", v, err)
752+ }
753+ if v, err := g.generation(2); err != nil || v != uint64(s.times[2])+2 {
754+ t.Fatalf("generation(2) = %d, %v", v, err)
755+ }
756+ if v, err := g.commitTime(4); err != nil || v != 1<<33+5 {
757+ t.Fatalf("34-bit time = %d, %v", v, err)
758+ }
759+}
760+
761+func TestGraphSyntheticCorruption(t *testing.T) {
762+ const n = 6
763+ mk := func(f func(s *synthLayer)) *synthLayer {
764+ s := &synthLayer{ids: synthIDs(n), parents: make([][]uint32, n), times: make([]int64, n)}
765+ s.parents[5] = []uint32{0, 1, 2}
766+ if f != nil {
767+ f(s)
768+ }
769+ return s
770+ }
771+ t.Run("ok", func(t *testing.T) {
772+ g := mustParseSynth(t, mk(nil).bytes())
773+ _ = g.close()
774+ })
775+ t.Run("parent out of range", func(t *testing.T) {
776+ g := mustParseSynth(t, mk(func(s *synthLayer) { s.parents[2] = []uint32{9} }).bytes())
777+ if _, err := g.appendParents(nil, 2); !errors.Is(err, errGraphCorrupt) {
778+ t.Fatalf("err = %v", err)
779+ }
780+ // A parent equal to the commit's own layer end is rejected, a
781+ // parent in a later position of the same layer is accepted (the
782+ // graph is not required to be topologically ordered by id).
783+ if _, err := g.appendParents(nil, 5); err != nil {
784+ t.Fatal(err)
785+ }
786+ })
787+ t.Run("edge list overrun", func(t *testing.T) {
788+ g := mustParseSynth(t, mk(func(s *synthLayer) {
789+ s.parents[5] = []uint32{0, 1, 2}
790+ }).bytes())
791+ l := &g.layers[0]
792+ // Clear the "last" bit of the final EDGE entry.
793+ last := len(l.edge) - 4
794+ mutable := append([]byte(nil), l.edge...)
795+ mutable[last] &^= 0x80
796+ l.edge = mutable
797+ if _, err := g.appendParents(nil, 5); !errors.Is(err, errGraphCorrupt) {
798+ t.Fatalf("err = %v", err)
799+ }
800+ })
801+ t.Run("missing edge chunk", func(t *testing.T) {
802+ g := mustParseSynth(t, mk(func(s *synthLayer) { s.skipEdge = true }).bytes())
803+ if _, err := g.appendParents(nil, 5); !errors.Is(err, errGraphCorrupt) {
804+ t.Fatalf("err = %v", err)
805+ }
806+ })
807+ t.Run("overflow index out of range", func(t *testing.T) {
808+ g := mustParseSynth(t, mk(func(s *synthLayer) {
809+ s.gda2 = make([]uint32, n)
810+ s.gda2[1] = graphGenOverflow | 3
811+ s.gdo2 = []uint64{1}
812+ }).bytes())
813+ if _, err := g.generation(1); !errors.Is(err, errGraphCorrupt) {
814+ t.Fatalf("err = %v", err)
815+ }
816+ if _, err := g.generation(0); err != nil {
817+ t.Fatal(err)
818+ }
819+ })
820+ t.Run("second parent without first", func(t *testing.T) {
821+ g := mustParseSynth(t, mk(nil).bytes())
822+ l := &g.layers[0]
823+ rec := append([]byte(nil), l.cdat...)
824+ binary.BigEndian.PutUint32(rec[graphHashSize:], graphParentNone)
825+ binary.BigEndian.PutUint32(rec[graphHashSize+4:], 1)
826+ l.cdat = rec
827+ if _, err := g.appendParents(nil, 0); !errors.Is(err, errGraphCorrupt) {
828+ t.Fatalf("err = %v", err)
829+ }
830+ })
831+}
832+
833+func TestGraphParseErrors(t *testing.T) {
834+ good := (&synthLayer{ids: synthIDs(4)}).bytes()
835+ if _, err := parseGraphLayer(good); err != nil {
836+ t.Fatal(err)
837+ }
838+ mut := func(f func(b []byte) []byte) []byte { return f(append([]byte(nil), good...)) }
839+ cases := map[string][]byte{
840+ "empty": nil,
841+ "short": good[:20],
842+ "signature": mut(func(b []byte) []byte { b[0] = 'X'; return b }),
843+ "hash version": mut(func(b []byte) []byte { b[5] = 9; return b }),
844+ "chunk count": mut(func(b []byte) []byte { b[6] = 255; return b }),
845+ "offset past": mut(func(b []byte) []byte { binary.BigEndian.PutUint64(b[12:], 1<<62); return b }),
846+ "offset behind": mut(func(b []byte) []byte { binary.BigEndian.PutUint64(b[12:], 1); return b }),
847+ "dup chunk": mut(func(b []byte) []byte { copy(b[8+12:], b[8:12]); return b }),
848+ "fanout order": mut(func(b []byte) []byte { binary.BigEndian.PutUint32(b[(8+5*12)+4:], 99); return b }),
849+ "truncated": good[:len(good)-30],
850+ }
851+ for name, b := range cases {
852+ if _, err := parseGraphLayer(b); !errors.Is(err, errGraphCorrupt) {
853+ t.Errorf("%s: err = %v, want corrupt", name, err)
854+ }
855+ }
856+ sha256 := mut(func(b []byte) []byte { b[5] = 2; return b })
857+ if _, err := parseGraphLayer(sha256); !errors.Is(err, errGraphUnsupported) {
858+ t.Errorf("sha256: err = %v", err)
859+ }
860+ v2 := mut(func(b []byte) []byte { b[4] = 2; return b })
861+ if _, err := parseGraphLayer(v2); !errors.Is(err, errGraphUnsupported) {
862+ t.Errorf("version 2: err = %v", err)
863+ }
864+ // Every truncation and every single-byte corruption of the header and
865+ // chunk table must be handled without panicking.
866+ for i := range good {
867+ _, _ = parseGraphLayer(good[:i])
868+ }
869+ for i := range min(len(good), 8+6*12) {
870+ for _, v := range []byte{0, 1, 0x7f, 0x80, 0xff} {
871+ b := append([]byte(nil), good...)
872+ b[i] = v
873+ exerciseGraphBytes(b)
874+ }
875+ }
876+}
877+
878+func TestGraphChainParse(t *testing.T) {
879+ h1 := strings.Repeat("ab", 20)
880+ h2 := strings.Repeat("01", 20)
881+ names, err := parseGraphChain([]byte(h1 + "\n" + h2 + "\n"))
882+ if err != nil || len(names) != 2 || names[1][0] != 1 {
883+ t.Fatalf("names = %v, err = %v", names, err)
884+ }
885+ for _, bad := range []string{h1, h1 + "\r\n", strings.Repeat("zz", 20) + "\n", h1 + "\n" + "x\n"} {
886+ if _, err := parseGraphChain([]byte(bad)); !errors.Is(err, errGraphCorrupt) {
887+ t.Errorf("%q: err = %v", bad, err)
888+ }
889+ }
890+ if _, err := parseGraphChain(nil); !errors.Is(err, errNoGraph) {
891+ t.Errorf("empty chain: err = %v", err)
892+ }
893+}
894+
895+func TestGraphChainMismatch(t *testing.T) {
896+ r := newGTRepo(t)
897+ r.commit("a", "a\n")
898+ r.git("commit-graph", "write", "--reachable", "--split=no-merge")
899+ r.commit("b", "b\n")
900+ r.git("commit-graph", "write", "--reachable", "--split=no-merge")
901+ chainPath := filepath.Join(r.objects(), "info", "commit-graphs", "commit-graph-chain")
902+ raw, err := os.ReadFile(chainPath)
903+ if err != nil {
904+ t.Fatal(err)
905+ }
906+ lines := strings.Fields(string(raw))
907+ if len(lines) != 2 {
908+ t.Fatalf("chain = %q", raw)
909+ }
910+ g, err := openCommitGraph(r.objects())
911+ if err != nil {
912+ t.Fatal(err)
913+ }
914+ _ = g.close()
915+
916+ // Reverse: layer 0 would be the top layer, which declares a base.
917+ if err := os.Chmod(chainPath, 0o644); err != nil {
918+ t.Fatal(err)
919+ }
920+ if err := os.WriteFile(chainPath, []byte(lines[1]+"\n"+lines[0]+"\n"), 0o644); err != nil {
921+ t.Fatal(err)
922+ }
923+ if _, err := openCommitGraph(r.objects()); !errors.Is(err, errGraphCorrupt) {
924+ t.Fatalf("reversed chain: err = %v", err)
925+ }
926+ // Top layer alone declares one base but the chain has none.
927+ if err := os.WriteFile(chainPath, []byte(lines[1]+"\n"), 0o644); err != nil {
928+ t.Fatal(err)
929+ }
930+ if _, err := openCommitGraph(r.objects()); !errors.Is(err, errGraphCorrupt) {
931+ t.Fatalf("truncated chain: err = %v", err)
932+ }
933+ // A missing layer file is an error, not a panic.
934+ if err := os.WriteFile(chainPath, []byte(strings.Repeat("0", 40)+"\n"), 0o644); err != nil {
935+ t.Fatal(err)
936+ }
937+ if _, err := openCommitGraph(r.objects()); err == nil {
938+ t.Fatal("missing layer accepted")
939+ }
940+ // Two layers whose base list names the wrong checksum.
941+ if err := os.WriteFile(chainPath, []byte(lines[0]+"\n"+lines[0]+"\n"), 0o644); err != nil {
942+ t.Fatal(err)
943+ }
944+ if _, err := openCommitGraph(r.objects()); !errors.Is(err, errGraphCorrupt) {
945+ t.Fatalf("duplicated layer: err = %v", err)
946+ }
947+}
948+
949+func TestGraphCorruptFiles(t *testing.T) {
950+ r := newGTRepo(t)
951+ r.commit("a", "a\n")
952+ r.git("commit-graph", "write", "--reachable", "--changed-paths")
953+ path := filepath.Join(r.objects(), "info", "commit-graph")
954+ good, err := os.ReadFile(path)
955+ if err != nil {
956+ t.Fatal(err)
957+ }
958+ if err := os.Chmod(path, 0o644); err != nil {
959+ t.Fatal(err)
960+ }
961+ for _, b := range [][]byte{good[:10], {}, []byte("CGPH"), bytes.Repeat([]byte{0xff}, 200)} {
962+ if err := os.WriteFile(path, b, 0o644); err != nil {
963+ t.Fatal(err)
964+ }
965+ if _, err := openCommitGraph(r.objects()); err == nil {
966+ t.Fatalf("accepted %d corrupt bytes", len(b))
967+ }
968+ }
969+ // A directory in place of the file must not panic or be mapped.
970+ _ = os.Remove(path)
971+ if err := os.Mkdir(path, 0o755); err != nil {
972+ t.Fatal(err)
973+ }
974+ if _, err := openCommitGraph(r.objects()); err == nil {
975+ t.Fatal("directory accepted")
976+ }
977+}
978+
979+// ---- fuzzing --------------------------------------------------------------
980+
981+// exerciseGraphBytes runs every accessor over a layer parsed from data, alone
982+// and stacked on itself, and must never panic.
983+func exerciseGraphBytes(data []byte) {
984+ l, err := parseGraphLayer(data)
985+ if err != nil {
986+ return
987+ }
988+ one := *l
989+ one.numBase = 0
990+ two := one
991+ two.base = one.n
992+ for _, g := range []*commitGraph{
993+ {layers: []graphLayer{one}, total: one.n, genV2: one.gda2 != nil},
994+ {layers: []graphLayer{one, two}, total: 2 * one.n, genV2: one.gda2 != nil},
995+ } {
996+ var buf [4]uint32
997+ n := min(g.total, 256)
998+ for pos := range n {
999+ id, _ := g.commitID(pos)
1000+ g.lookup(id)
1001+ _, _ = g.rootTree(pos)
1002+ _, _ = g.commitTime(pos)
1003+ _, _ = g.topoLevel(pos)
1004+ _, _ = g.generation(pos)
1005+ _, _ = g.appendParents(buf[:0], pos)
1006+ g.maybeChanged(pos, []byte("src/main.go"))
1007+ g.hasBloom(pos)
1008+ }
1009+ _, _ = g.commitID(g.total)
1010+ _, _ = g.appendParents(nil, g.total+1)
1011+ g.maybeChanged(g.total, nil)
1012+ g.lookup(oid{0xff, 0xff})
1013+ g.lookup(oid{})
1014+ }
1015+}
1016+
1017+func graphSeedFiles(tb testing.TB) map[string][]byte {
1018+ tb.Helper()
1019+ entries, err := os.ReadDir(filepath.Join("testdata", "graph"))
1020+ if err != nil {
1021+ tb.Fatal(err)
1022+ }
1023+ seeds := map[string][]byte{}
1024+ for _, e := range entries {
1025+ if !strings.HasSuffix(e.Name(), ".graph") {
1026+ continue
1027+ }
1028+ b, err := os.ReadFile(filepath.Join("testdata", "graph", e.Name()))
1029+ if err != nil {
1030+ tb.Fatal(err)
1031+ }
1032+ seeds[e.Name()] = b
1033+ }
1034+ if len(seeds) == 0 {
1035+ tb.Fatal("no fuzz seeds in testdata/graph")
1036+ }
1037+ return seeds
1038+}
1039+
1040+func FuzzGraphFile(f *testing.F) {
1041+ for _, b := range graphSeedFiles(f) {
1042+ f.Add(b)
1043+ f.Add(b[:len(b)/2])
1044+ }
1045+ f.Fuzz(func(t *testing.T, data []byte) { exerciseGraphBytes(data) })
1046+}
1047+
1048+func FuzzGraphChain(f *testing.F) {
1049+ f.Add([]byte(strings.Repeat("ab", 20) + "\n"))
1050+ f.Add([]byte(strings.Repeat("ab", 20) + "\n" + strings.Repeat("cd", 20) + "\n"))
1051+ f.Add([]byte("\n"))
1052+ f.Fuzz(func(t *testing.T, data []byte) {
1053+ names, err := parseGraphChain(data)
1054+ if err == nil && len(names)*41 != len(data) {
1055+ t.Fatalf("parsed %d names from %d bytes", len(names), len(data))
1056+ }
1057+ })
1058+}
1059+
1060+// TestGraphSeedsParse checks the committed seeds stay valid files.
1061+func TestGraphSeedsParse(t *testing.T) {
1062+ for name, b := range graphSeedFiles(t) {
1063+ l, err := parseGraphLayer(b)
1064+ if err != nil {
1065+ t.Errorf("%s: %v", name, err)
1066+ continue
1067+ }
1068+ exerciseGraphBytes(b)
1069+ _ = l
1070+ }
1071+}
1072+
1073+// TestWriteGraphSeeds regenerates testdata/graph with the git binary:
1074+//
1075+// ERMITE_GIT_UPDATE_SEEDS=1 go test -run TestWriteGraphSeeds
1076+func TestWriteGraphSeeds(t *testing.T) {
1077+ if os.Getenv("ERMITE_GIT_UPDATE_SEEDS") == "" {
1078+ t.Skip("set ERMITE_GIT_UPDATE_SEEDS=1 to regenerate testdata/graph")
1079+ }
1080+ out := filepath.Join("testdata", "graph")
1081+ if err := os.MkdirAll(out, 0o755); err != nil {
1082+ t.Fatal(err)
1083+ }
1084+ save := func(name string, b []byte) {
1085+ if err := os.WriteFile(filepath.Join(out, name), b, 0o644); err != nil {
1086+ t.Fatal(err)
1087+ }
1088+ }
1089+ copyLayers := func(prefix string, r *gtRepo) {
1090+ dir := filepath.Join(r.objects(), "info")
1091+ if b, err := os.ReadFile(filepath.Join(dir, "commit-graph")); err == nil {
1092+ save(prefix+".graph", b)
1093+ return
1094+ }
1095+ chain, err := os.ReadFile(filepath.Join(dir, "commit-graphs", "commit-graph-chain"))
1096+ if err != nil {
1097+ t.Fatal(err)
1098+ }
1099+ for i, h := range strings.Fields(string(chain)) {
1100+ b, err := os.ReadFile(filepath.Join(dir, "commit-graphs", "graph-"+h+".graph"))
1101+ if err != nil {
1102+ t.Fatal(err)
1103+ }
1104+ save(fmt.Sprintf("%s-layer%d.graph", prefix, i), b)
1105+ }
1106+ }
1107+ for _, c := range []gcase{
1108+ {name: "single-nobloom"},
1109+ {name: "single-bloom-v2", bloom: true, cfg: []string{"commitGraph.changedPathsVersion=2"}},
1110+ {name: "split-bloom-v2", split: true, bloom: true, cfg: []string{"commitGraph.changedPathsVersion=2"}},
1111+ {name: "split-bloom-v1", split: true, bloom: true, cfg: []string{"commitGraph.changedPathsVersion=1"}},
1112+ {name: "split-gen1", split: true, cfg: []string{"commitGraph.generationVersion=1"}},
1113+ } {
1114+ r := newGTRepo(t)
1115+ r.stageOne()
1116+ r.writeGraph(c)
1117+ r.stageTwo()
1118+ r.writeGraph(c)
1119+ copyLayers(c.name, r)
1120+ }
1121+ // Synthetic: wide octopus, overflow generation, Bloom edge cases.
1122+ s := &synthLayer{ids: synthIDs(12), parents: make([][]uint32, 12), times: make([]int64, 12)}
1123+ s.parents[11] = []uint32{0, 1, 2, 3, 4, 5}
1124+ s.gda2 = make([]uint32, 12)
1125+ s.gda2[3] = graphGenOverflow
1126+ s.gdo2 = []uint64{1 << 40}
1127+ hdr := binary.BigEndian.AppendUint32(nil, 2)
1128+ hdr = binary.BigEndian.AppendUint32(hdr, 7)
1129+ hdr = binary.BigEndian.AppendUint32(hdr, 10)
1130+ s.bdat = append(hdr, 0x00, 0xff, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0)
1131+ s.bidx = []uint32{1, 2, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10}
1132+ save("synthetic.graph", s.bytes())
1133+}
1134+
1135+// ---- benchmarks -----------------------------------------------------------
1136+
1137+// fastImportRepo creates a bare repo with n commits (a merge every 50) using
1138+// git fast-import, and writes its split commit-graph.
1139+func fastImportRepo(tb testing.TB, n int) string {
1140+ tb.Helper()
1141+ requireGit(tb)
1142+ dir := tb.TempDir()
1143+ graphGit(tb, dir, 0, "init", "-q", "--bare", "-b", "main")
1144+ cmd := exec.Command("git", "fast-import", "--quiet")
1145+ cmd.Dir = dir
1146+ cmd.Env = graphGitEnv(0)
1147+ var stderr bytes.Buffer
1148+ cmd.Stderr = &stderr
1149+ in, err := cmd.StdinPipe()
1150+ if err != nil {
1151+ tb.Fatal(err)
1152+ }
1153+ if err := cmd.Start(); err != nil {
1154+ tb.Fatal(err)
1155+ }
1156+ w := bufio.NewWriter(in)
1157+ for i := 1; i <= n; i++ {
1158+ msg := "commit " + strconv.Itoa(i)
1159+ _, _ = fmt.Fprintf(w, "commit refs/heads/main\nmark :%d\ncommitter T <t@example.com> %d +0000\ndata %d\n%s\n", i, 1_600_000_000+i*30, len(msg), msg)
1160+ if i > 1 {
1161+ _, _ = fmt.Fprintf(w, "from :%d\n", i-1)
1162+ }
1163+ if i > 60 && i%50 == 0 {
1164+ _, _ = fmt.Fprintf(w, "merge :%d\n", i-30)
1165+ }
1166+ for _, p := range []string{
1167+ fmt.Sprintf("d%d/f%d.txt", i%17, i%5),
1168+ fmt.Sprintf("pkg%d/sub%d/g%d.go", i%7, i%3, i),
1169+ } {
1170+ body := "v" + strconv.Itoa(i) + "\n"
1171+ _, _ = fmt.Fprintf(w, "M 644 inline %s\ndata %d\n%s", p, len(body), body)
1172+ }
1173+ _, _ = w.WriteString("\n")
1174+ }
1175+ if err := w.Flush(); err != nil {
1176+ tb.Fatal(err)
1177+ }
1178+ _ = in.Close()
1179+ if err := cmd.Wait(); err != nil {
1180+ tb.Fatalf("fast-import: %v\n%s", err, stderr.String())
1181+ }
1182+ graphGit(tb, dir, 0, "commit-graph", "write", "--reachable", "--changed-paths", "--split")
1183+ return dir
1184+}
1185+
1186+func benchGraph(b *testing.B) (*commitGraph, []oid) {
1187+ dir := fastImportRepo(b, 5000)
1188+ g, err := openCommitGraph(filepath.Join(dir, "objects"))
1189+ if err != nil {
1190+ b.Fatal(err)
1191+ }
1192+ b.Cleanup(func() { _ = g.close() })
1193+ ids := make([]oid, g.numCommits())
1194+ for i := range ids {
1195+ ids[i], _ = g.commitID(uint32(i))
1196+ }
1197+ return g, ids
1198+}
1199+
1200+func BenchmarkGraphLookup(b *testing.B) {
1201+ g, ids := benchGraph(b)
1202+ b.ReportAllocs()
1203+ i := 0
1204+ for b.Loop() {
1205+ if _, ok := g.lookup(ids[i]); !ok {
1206+ b.Fatal("miss")
1207+ }
1208+ if i++; i == len(ids) {
1209+ i = 0
1210+ }
1211+ }
1212+}
1213+
1214+func BenchmarkGraphParents(b *testing.B) {
1215+ g, _ := benchGraph(b)
1216+ var buf [8]uint32
1217+ n := g.numCommits()
1218+ b.ReportAllocs()
1219+ var pos uint32
1220+ for b.Loop() {
1221+ if _, err := g.appendParents(buf[:0], pos); err != nil {
1222+ b.Fatal(err)
1223+ }
1224+ if pos++; pos == n {
1225+ pos = 0
1226+ }
1227+ }
1228+}
1229+
1230+func BenchmarkGraphCommitData(b *testing.B) {
1231+ g, _ := benchGraph(b)
1232+ n := g.numCommits()
1233+ b.ReportAllocs()
1234+ var pos uint32
1235+ for b.Loop() {
1236+ if _, err := g.generation(pos); err != nil {
1237+ b.Fatal(err)
1238+ }
1239+ if _, err := g.commitTime(pos); err != nil {
1240+ b.Fatal(err)
1241+ }
1242+ if _, err := g.rootTree(pos); err != nil {
1243+ b.Fatal(err)
1244+ }
1245+ if pos++; pos == n {
1246+ pos = 0
1247+ }
1248+ }
1249+}
A testdata/graph/single-bloom-v2.graph +0 -0 binary file: no text diff

binary file: no text diff

A testdata/graph/single-nobloom.graph +0 -0 binary file: no text diff

binary file: no text diff

A testdata/graph/split-bloom-v1-layer0.graph +0 -0 binary file: no text diff

binary file: no text diff

A testdata/graph/split-bloom-v1-layer1.graph +0 -0 binary file: no text diff

binary file: no text diff

A testdata/graph/split-bloom-v2-layer0.graph +0 -0 binary file: no text diff

binary file: no text diff

A testdata/graph/split-bloom-v2-layer1.graph +0 -0 binary file: no text diff

binary file: no text diff

A testdata/graph/split-gen1-layer0.graph +0 -0 binary file: no text diff

binary file: no text diff

A testdata/graph/split-gen1-layer1.graph +0 -0 binary file: no text diff

binary file: no text diff

A testdata/graph/synthetic.graph +0 -0 binary file: no text diff

binary file: no text diff

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 1

Tags 2

↑↓ moveenter openesc close