perf(git): allocation-free lookup caches; add benchmarks, race test and README

commit: f0b398e72fcfc73d6f9c3deb3119fa217bd8dab5
author: brice · 11d ago · 2026-10-10 21:23 UTC
parent: e685703
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

6 files · +389 -26

M README.md +14 -1


@@ -2,7 +2,7 @@
2	2
3	3		A fast, read-only Git repository reader for Go: refs, objects, packfiles, trees and history, using commit-graph and changed-path Bloom filters. Standard library only.
4	4
5		-	Status: early development, no stable API. Phase A (this tree) reads refs, loose and packed objects, trees, blobs and commits; history walks come next.
	5	+	Status: early development, no stable API. Phase A reads refs, loose and packed objects, trees, blobs and commits; phase B adds the commit-graph, changed-path Bloom filters and history walks.
6	6
7	7		```go
8	8		import git "bnema.dev/ermite-git"

@@ -24,6 +24,19 @@ _ = repo.ReadTree(c.Tree, func(e git.Entry) bool {
24	24		})
25	25		```
26	26
	27	+	History in the order of `git log`, with the same path simplification as `git log -- path`:
	28	+
	29	+	```go
	30	+	for c, err := range repo.Walk(ctx, commitID, git.WalkOptions{Path: "pkg/gui", Skip: 100, Limit: 50}) {
	31	+	if err != nil {
	32	+	return err
	33	+	}
	34	+	fmt.Printf("%s\n", c.ID) // c is reused: valid until the next iteration
	35	+	}
	36	+	// the last commit of every entry of a directory, in one walk
	37	+	ids, err := repo.LastCommits(ctx, commitID, "pkg", []string{"gui", "commands"})
	38	+	```
	39	+
27	40		A `Repo` is safe for concurrent use. Call `Refresh` after the repository changed (a push, a repack) to see new refs and packs; reads in flight keep working while packs are replaced.
28	41
29	42		Repositories the package does not read are rejected by `Open` with `ErrUnsupported`: SHA-256, reftable, other `extensions.*`, shallow, grafts, replace refs, alternates, partial clones, incremental or stale multi-pack-index.

 A fast, read-only Git repository reader for Go: refs, objects, packfiles, trees and history, using commit-graph and changed-path Bloom filters. Standard library only.
-Status: early development, no stable API. Phase A (this tree) reads refs, loose and packed objects, trees, blobs and commits; history walks come next.
+Status: early development, no stable API. Phase A reads refs, loose and packed objects, trees, blobs and commits; phase B adds the commit-graph, changed-path Bloom filters and history walks.
 ```go
 import git "bnema.dev/ermite-git"
 })
 ```
+History in the order of `git log`, with the same path simplification as `git log -- path`:
+```go
+for c, err := range repo.Walk(ctx, commitID, git.WalkOptions{Path: "pkg/gui", Skip: 100, Limit: 50}) {
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%s\n", c.ID) // c is reused: valid until the next iteration
+}
+// the last commit of every entry of a directory, in one walk
+ids, err := repo.LastCommits(ctx, commitID, "pkg", []string{"gui", "commands"})
+```
 A `Repo` is safe for concurrent use. Call `Refresh` after the repository changed (a push, a repack) to see new refs and packs; reads in flight keep working while packs are replaced.
 Repositories the package does not read are rejected by `Open` with `ErrUnsupported`: SHA-256, reftable, other `extensions.*`, shallow, grafts, replace refs, alternates, partial clones, incremental or stale multi-pack-index.

M lastcommit.go +54 -12


@@ -6,6 +6,7 @@ import (
6	6		"context"
7	7		"fmt"
8	8		"strings"
	9	+	"unsafe"
9	10		)
10	11
11	12		// LastCommits finds, in one walk, the last commit that touched each entry of

@@ -75,8 +76,8 @@ func (r *Repo) LastCommits(ctx context.Context, start ObjectID, dir string, name
75	76		s.key = newBloomKey([]byte(spec.String()))
76	77		s.useBloom = s.key.usable
77	78		}
78		-	s.maxTrees = max(64, (16<<20)/(len(uniq)*28+64))
79		-	s.cache = make(map[ObjectID][]pathEntry, 256)
	79	+	// The cache holds at most about 1 MiB of entries, 16 trees at least.
	80	+	s.slots = min(256, max(16, (1<<20)/(len(uniq)*int(unsafe.Sizeof(pathEntry{})))))
80	81		s.absent = make([]pathEntry, len(uniq))
81	82		if err := s.run(); err != nil {
82	83		return nil, err

@@ -118,15 +119,24 @@ type lastState struct {
118	119		found []ObjectID
119	120		remaining int
120	121
121		-	cache map[ObjectID][]pathEntry // entries of the names, by directory tree
122		-	maxTrees int
123		-	absent []pathEntry // all names missing
124		-	buf []byte
	122	+	// A cache of the entries of the names in directory trees, 4-way
	123	+	// associative, backed by one slab so that it never allocates per tree.
	124	+	slots int
	125	+	cache []entrySlot
	126	+	slab []pathEntry
	127	+	tick uint64 // counts steps; the slots used by the current step are kept
	128	+	absent []pathEntry // all names missing
	129	+	buf []byte
125	130
126	131		// scratch
127	132		ks []int8
128	133		}
129	134
	135	+	type entrySlot struct {
	136	+	tree ObjectID
	137	+	used uint64 // step that last used the slot, 0 when empty
	138	+	}
	139	+
130	140		// dirOf finds the directory in a root tree.
131	141		func (s *lastState) dirOf(root ObjectID) (ObjectID, bool, error) {
132	142		if len(s.d.comps) == 0 {

@@ -145,8 +155,23 @@ func (s *lastState) entries(dir ObjectID, ok bool) ([]pathEntry, error) {
145	155		if !ok \|\| dir == emptyTreeID {
146	156		return s.absent, nil
147	157		}
148		-	if e, hit := s.cache[dir]; hit {
149		-	return e, nil
	158	+	if s.cache == nil {
	159	+	s.cache = make([]entrySlot, s.slots)
	160	+	s.slab = make([]pathEntry, s.slots*s.n)
	161	+	}
	162	+	base := (int(dir[0]) \| int(dir[1])<<8) % s.slots
	163	+	victim := -1
	164	+	for j := 0; j < 4 && j < s.slots; j++ {
	165	+	k := (base + j) % s.slots
	166	+	c := &s.cache[k]
	167	+	if c.used != 0 && c.tree == dir {
	168	+	c.used = s.tick
	169	+	return s.slab[ks.n : (k+1)s.n], nil
	170	+	}
	171	+	// Slots used by this step still back slices in use: keep them.
	172	+	if c.used != s.tick && (victim < 0 \|\| c.used < s.cache[victim].used) {
	173	+	victim = k
	174	+	}
150	175		}
151	176		typ, data, err := s.w.r.ReadObject(dir, s.buf)
152	177		if err != nil {

@@ -156,7 +181,13 @@ func (s *lastState) entries(dir ObjectID, ok bool) ([]pathEntry, error) {
156	181		if typ != TypeTree {
157	182		return nil, wrongType(dir, typ, TypeTree)
158	183		}
159		-	ents := make([]pathEntry, s.n)
	184	+	var ents []pathEntry
	185	+	if victim >= 0 {
	186	+	ents = s.slab[victims.n : (victim+1)s.n]
	187	+	clear(ents)
	188	+	} else {
	189	+	ents = make([]pathEntry, s.n) // every slot is in use by this step
	190	+	}
160	191		if err := ParseTree(data, func(e Entry) bool {
161	192		if i, hit := s.idx[string(e.Name)]; hit {
162	193		ents[i] = pathEntry{id: e.ID, mode: e.Mode, found: true}

@@ -165,10 +196,9 @@ func (s *lastState) entries(dir ObjectID, ok bool) ([]pathEntry, error) {
165	196		}); err != nil {
166	197		return nil, wrapObj(dir, err)
167	198		}
168		-	if len(s.cache) >= s.maxTrees {
169		-	clear(s.cache)
	199	+	if victim >= 0 {
	200	+	s.cache[victim] = entrySlot{tree: dir, used: s.tick}
170	201		}
171		-	s.cache[dir] = ents
172	202		return ents, nil
173	203		}
174	204

@@ -202,6 +232,7 @@ func (s *lastState) run() error {
202	232		// off from g.
203	233		func (s lastState) step(g lastGroup) ([]*lastGroup, error) {
204	234		w := s.w
	235	+	s.tick++
205	236		n := g.pop().n
206	237		tree, parents, err := w.load(n, w.parents[:0])
207	238		w.parents = parents

@@ -295,6 +326,17 @@ func (s lastState) step(g lastGroup) ([]*lastGroup, error) {
295	326		if len(live) == 0 {
296	327		return nil, nil
297	328		}
	329	+	// All the names follow the same parent: no split.
	330	+	same := true
	331	+	for _, i := range live[1:] {
	332	+	if ks[i] != ks[live[0]] {
	333	+	same = false
	334	+	break
	335	+	}
	336	+	}
	337	+	if same {
	338	+	return nil, s.follow(g, parents[ks[live[0]]])
	339	+	}
298	340		// Split the group by the parent each remaining name follows.
299	341		var extra []*lastGroup
300	342		first := true

 	"context"
 	"fmt"
 	"strings"
+	"unsafe"
 )
 // LastCommits finds, in one walk, the last commit that touched each entry of
 		s.key = newBloomKey([]byte(spec.String()))
 		s.useBloom = s.key.usable
 	}
-	s.maxTrees = max(64, (16<<20)/(len(uniq)*28+64))
+	// The cache holds at most about 1 MiB of entries, 16 trees at least.
-	s.cache = make(map[ObjectID][]pathEntry, 256)
+	s.slots = min(256, max(16, (1<<20)/(len(uniq)*int(unsafe.Sizeof(pathEntry{})))))
 	s.absent = make([]pathEntry, len(uniq))
 	if err := s.run(); err != nil {
 		return nil, err
 	found     []ObjectID
 	remaining int
-	cache    map[ObjectID][]pathEntry // entries of the names, by directory tree
+	// A cache of the entries of the names in directory trees, 4-way
-	maxTrees int
+	// associative, backed by one slab so that it never allocates per tree.
-	absent   []pathEntry // all names missing
+	slots  int
-	buf      []byte
+	cache  []entrySlot
+	slab   []pathEntry
+	tick   uint64      // counts steps; the slots used by the current step are kept
+	absent []pathEntry // all names missing
+	buf    []byte
 	// scratch
 	ks []int8
 }
+type entrySlot struct {
+	tree ObjectID
+	used uint64 // step that last used the slot, 0 when empty
+}
 // dirOf finds the directory in a root tree.
 func (s *lastState) dirOf(root ObjectID) (ObjectID, bool, error) {
 	if len(s.d.comps) == 0 {
 	if !ok || dir == emptyTreeID {
 		return s.absent, nil
 	}
-	if e, hit := s.cache[dir]; hit {
+	if s.cache == nil {
-		return e, nil
+		s.cache = make([]entrySlot, s.slots)
+		s.slab = make([]pathEntry, s.slots*s.n)
+	}
+	base := (int(dir[0]) | int(dir[1])<<8) % s.slots
+	victim := -1
+	for j := 0; j < 4 && j < s.slots; j++ {
+		k := (base + j) % s.slots
+		c := &s.cache[k]
+		if c.used != 0 && c.tree == dir {
+			c.used = s.tick
+			return s.slab[k*s.n : (k+1)*s.n], nil
+		}
+		// Slots used by this step still back slices in use: keep them.
+		if c.used != s.tick && (victim < 0 || c.used < s.cache[victim].used) {
+			victim = k
+		}
 	}
 	typ, data, err := s.w.r.ReadObject(dir, s.buf)
 	if err != nil {
 	if typ != TypeTree {
 		return nil, wrongType(dir, typ, TypeTree)
 	}
-	ents := make([]pathEntry, s.n)
+	var ents []pathEntry
+	if victim >= 0 {
+		ents = s.slab[victim*s.n : (victim+1)*s.n]
+		clear(ents)
+	} else {
+		ents = make([]pathEntry, s.n) // every slot is in use by this step
+	}
 	if err := ParseTree(data, func(e Entry) bool {
 		if i, hit := s.idx[string(e.Name)]; hit {
 			ents[i] = pathEntry{id: e.ID, mode: e.Mode, found: true}
 	}); err != nil {
 		return nil, wrapObj(dir, err)
 	}
-	if len(s.cache) >= s.maxTrees {
+	if victim >= 0 {
-		clear(s.cache)
+		s.cache[victim] = entrySlot{tree: dir, used: s.tick}
 	}
-	s.cache[dir] = ents
 	return ents, nil
 }
 // off from g.
 func (s *lastState) step(g *lastGroup) ([]*lastGroup, error) {
 	w := s.w
+	s.tick++
 	n := g.pop().n
 	tree, parents, err := w.load(n, w.parents[:0])
 	w.parents = parents
 	if len(live) == 0 {
 		return nil, nil
 	}
+	// All the names follow the same parent: no split.
+	same := true
+	for _, i := range live[1:] {
+		if ks[i] != ks[live[0]] {
+			same = false
+			break
+		}
+	}
+	if same {
+		return nil, s.follow(g, parents[ks[live[0]]])
+	}
 	// Split the group by the parent each remaining name follows.
 	var extra []*lastGroup
 	first := true

M treediff.go +19 -9


@@ -91,7 +91,16 @@ type lookupKey struct {
91	91		level uint16
92	92		}
93	93
94		-	const maxLookupCache = 1 << 14
	94	+	// lookupSlots is the size of the direct-mapped cache of lookups. A walk
	95	+	// along a chain asks again for the trees it has just read, so a small table
	96	+	// is enough, and it never allocates after it is created.
	97	+	const lookupSlots = 256
	98	+
	99	+	type lookupSlot struct {
	100	+	key lookupKey
	101	+	e pathEntry
	102	+	ok bool
	103	+	}
95	104
96	105		// pathDiffer answers TREESAME questions for one path. It is not safe for
97	106		// concurrent use.

@@ -100,13 +109,13 @@ type pathDiffer struct {
100	109		comps []string
101	110		dirOnly bool
102	111		names [][]byte
103		-	cache map[lookupKey]pathEntry
	112	+	cache []lookupSlot // allocated on first use
104	113		buf []byte
105	114		}
106	115
107	116		func newPathDiffer(r Repo, spec pathSpec) pathDiffer {
108	117		comps := spec.comps
109		-	d := &pathDiffer{r: r, comps: comps, dirOnly: spec.dirOnly, cache: make(map[lookupKey]pathEntry, 256)}
	118	+	d := &pathDiffer{r: r, comps: comps, dirOnly: spec.dirOnly}
110	119		d.names = make([][]byte, len(comps))
111	120		for i, c := range comps {
112	121		d.names[i] = []byte(c)

@@ -120,8 +129,12 @@ func (d *pathDiffer) lookup(tree ObjectID, level int) (pathEntry, error) {
120	129		return pathEntry{}, nil
121	130		}
122	131		k := lookupKey{tree, uint16(level)}
123		-	if e, ok := d.cache[k]; ok {
124		-	return e, nil
	132	+	if d.cache == nil {
	133	+	d.cache = make([]lookupSlot, lookupSlots)
	134	+	}
	135	+	slot := &d.cache[(uint(tree[0])\|uint(tree[1])<<8+uint(level))%lookupSlots]
	136	+	if slot.ok && slot.key == k {
	137	+	return slot.e, nil
125	138		}
126	139		typ, data, err := d.r.ReadObject(tree, d.buf)
127	140		if err != nil {

@@ -135,10 +148,7 @@ func (d *pathDiffer) lookup(tree ObjectID, level int) (pathEntry, error) {
135	148		if err != nil {
136	149		return pathEntry{}, wrapObj(tree, err)
137	150		}
138		-	if len(d.cache) >= maxLookupCache {
139		-	clear(d.cache)
140		-	}
141		-	d.cache[k] = e
	151	+	*slot = lookupSlot{key: k, e: e, ok: true}
142	152		return e, nil
143	153		}
144	154

 	level uint16
 }
-const maxLookupCache = 1 << 14
+// lookupSlots is the size of the direct-mapped cache of lookups. A walk
+// along a chain asks again for the trees it has just read, so a small table
+// is enough, and it never allocates after it is created.
+const lookupSlots = 256
+type lookupSlot struct {
+	key lookupKey
+	e   pathEntry
+	ok  bool
+}
 // pathDiffer answers TREESAME questions for one path. It is not safe for
 // concurrent use.
 	comps   []string
 	dirOnly bool
 	names   [][]byte
-	cache   map[lookupKey]pathEntry
+	cache   []lookupSlot // allocated on first use
 	buf     []byte
 }
 func newPathDiffer(r *Repo, spec pathSpec) *pathDiffer {
 	comps := spec.comps
-	d := &pathDiffer{r: r, comps: comps, dirOnly: spec.dirOnly, cache: make(map[lookupKey]pathEntry, 256)}
+	d := &pathDiffer{r: r, comps: comps, dirOnly: spec.dirOnly}
 	d.names = make([][]byte, len(comps))
 	for i, c := range comps {
 		d.names[i] = []byte(c)
 		return pathEntry{}, nil
 	}
 	k := lookupKey{tree, uint16(level)}
-	if e, ok := d.cache[k]; ok {
+	if d.cache == nil {
-		return e, nil
+		d.cache = make([]lookupSlot, lookupSlots)
+	}
+	slot := &d.cache[(uint(tree[0])|uint(tree[1])<<8+uint(level))%lookupSlots]
+	if slot.ok && slot.key == k {
+		return slot.e, nil
 	}
 	typ, data, err := d.r.ReadObject(tree, d.buf)
 	if err != nil {
 	if err != nil {
 		return pathEntry{}, wrapObj(tree, err)
 	}
-	if len(d.cache) >= maxLookupCache {
+	*slot = lookupSlot{key: k, e: e, ok: true}
-		clear(d.cache)
-	}
-	d.cache[k] = e
 	return e, nil
 }

A walkbench_test.go +146 -0

@@ -0,0 +1,146 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package git
	4	+
	5	+	import (
	6	+	"bufio"
	7	+	"context"
	8	+	"io"
	9	+	"os/exec"
	10	+	"path"
	11	+	"testing"
	12	+	)
	13	+
	14	+	// gitLogCount runs git log like the gitcli adapter does and reads every
	15	+	// line of the output.
	16	+	func gitLogCount(b testing.B, x realWalk, args ...string) int {
	17	+	cmd := exec.Command("git", append([]string{"-C", x.dir}, args...)...)
	18	+	cmd.Env = gitEnv()
	19	+	out, err := cmd.StdoutPipe()
	20	+	if err != nil {
	21	+	b.Fatal(err)
	22	+	}
	23	+	if err := cmd.Start(); err != nil {
	24	+	b.Fatal(err)
	25	+	}
	26	+	n := 0
	27	+	br := bufio.NewReader(out)
	28	+	for {
	29	+	_, err := br.ReadSlice('\n')
	30	+	if err == io.EOF {
	31	+	break
	32	+	}
	33	+	if err != nil && err != bufio.ErrBufferFull {
	34	+	b.Fatal(err)
	35	+	}
	36	+	n++
	37	+	}
	38	+	if err := cmd.Wait(); err != nil {
	39	+	b.Fatal(err)
	40	+	}
	41	+	return n
	42	+	}
	43	+
	44	+	// gitcliFormat is the format the gitcli adapter of ermite asks git log for.
	45	+	const gitcliFormat = "%H%x00%P%x00%an%x00%ae%x00%at%x00%cn%x00%ce%x00%ct%x00%B"
	46	+
	47	+	// benchWalk compares Walk with git log in two shapes: ids only (git answers
	48	+	// from the commit-graph without reading commits) and complete commits, which
	49	+	// is what the gitcli adapter asks for (git has to read every commit).
	50	+	func benchWalk(b testing.B, x realWalk, opts WalkOptions, gitArgs ...string) {
	51	+	run := func(b *testing.B, o WalkOptions) {
	52	+	b.ReportAllocs()
	53	+	var n int
	54	+	for i := 0; i < b.N; i++ {
	55	+	n = 0
	56	+	for c, err := range x.r.Walk(context.Background(), x.head, o) {
	57	+	if err != nil {
	58	+	b.Fatal(err)
	59	+	}
	60	+	if !o.IDOnly {
	61	+	n += len(c.Message) // the commit is really parsed
	62	+	}
	63	+	n++
	64	+	}
	65	+	}
	66	+	_ = n
	67	+	}
	68	+	for _, shape := range []struct {
	69	+	name string
	70	+	idOnly bool
	71	+	format string
	72	+	}{{"ids", true, "--format=%H"}, {"full", false, "-z --format=" + gitcliFormat}} {
	73	+	b.Run(shape.name+"/ermite", func(b *testing.B) {
	74	+	o := opts
	75	+	o.IDOnly = shape.idOnly
	76	+	run(b, o)
	77	+	})
	78	+	b.Run(shape.name+"/git", func(b *testing.B) {
	79	+	b.ReportAllocs()
	80	+	args := []string{"log"}
	81	+	if shape.idOnly {
	82	+	args = append(args, shape.format)
	83	+	} else {
	84	+	args = append(args, "-z", "--format="+gitcliFormat)
	85	+	}
	86	+	args = append(args, gitArgs...)
	87	+	for i := 0; i < b.N; i++ {
	88	+	gitLogCount(b, x, args...)
	89	+	}
	90	+	})
	91	+	}
	92	+	}
	93	+
	94	+	func BenchmarkWalk(b *testing.B) {
	95	+	x := getRealWalk(b)
	96	+	head := x.head.String()
	97	+	b.Run("full-history", func(b *testing.B) { benchWalk(b, x, WalkOptions{}, head) })
	98	+	b.Run("skip5000-limit100", func(b *testing.B) {
	99	+	benchWalk(b, x, WalkOptions{Skip: 5000, Limit: 100}, "--skip=5000", "-n", "100", head)
	100	+	})
	101	+	b.Run("file", func(b *testing.B) {
	102	+	benchWalk(b, x, WalkOptions{Path: "pkg/gui/gui.go"}, head, "--", "pkg/gui/gui.go")
	103	+	})
	104	+	b.Run("dir", func(b *testing.B) {
	105	+	benchWalk(b, x, WalkOptions{Path: "pkg/commands"}, head, "--", "pkg/commands")
	106	+	})
	107	+	b.Run("rare-file", func(b *testing.B) {
	108	+	benchWalk(b, x, WalkOptions{Path: "go.mod"}, head, "--", "go.mod")
	109	+	})
	110	+	b.Run("page-100-of-file", func(b *testing.B) {
	111	+	benchWalk(b, x, WalkOptions{Path: "pkg/gui/gui.go", Limit: 100}, "-n", "100", head, "--", "pkg/gui/gui.go")
	112	+	})
	113	+	}
	114	+
	115	+	func BenchmarkLastCommits(b *testing.B) {
	116	+	x := getRealWalk(b)
	117	+	head := x.head.String()
	118	+	for _, dir := range []string{"", "pkg/gui"} {
	119	+	spec := head
	120	+	if dir != "" {
	121	+	spec += ":" + dir
	122	+	}
	123	+	names := lines(x.git(b, "ls-tree", "--name-only", spec))
	124	+	name := dir
	125	+	if name == "" {
	126	+	name = "root"
	127	+	}
	128	+	b.Run(name+"/ermite", func(b *testing.B) {
	129	+	b.ReportAllocs()
	130	+	for i := 0; i < b.N; i++ {
	131	+	if _, err := x.r.LastCommits(context.Background(), x.head, dir, names); err != nil {
	132	+	b.Fatal(err)
	133	+	}
	134	+	}
	135	+	b.ReportMetric(float64(len(names)), "entries")
	136	+	})
	137	+	b.Run(name+"/git-n-calls", func(b *testing.B) {
	138	+	b.ReportAllocs()
	139	+	for i := 0; i < b.N; i++ {
	140	+	for _, n := range names {
	141	+	gitLogCount(b, x, "log", "-n", "1", "--format=%H", head, "--", path.Join(dir, n))
	142	+	}
	143	+	}
	144	+	})
	145	+	}
	146	+	}

@@ -0,0 +1,146 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package git
	4	+
	5	+	import (
	6	+	"bufio"
	7	+	"context"
	8	+	"io"
	9	+	"os/exec"
	10	+	"path"
	11	+	"testing"
	12	+	)
	13	+
	14	+	// gitLogCount runs git log like the gitcli adapter does and reads every
	15	+	// line of the output.
	16	+	func gitLogCount(b testing.B, x realWalk, args ...string) int {
	17	+	cmd := exec.Command("git", append([]string{"-C", x.dir}, args...)...)
	18	+	cmd.Env = gitEnv()
	19	+	out, err := cmd.StdoutPipe()
	20	+	if err != nil {
	21	+	b.Fatal(err)
	22	+	}
	23	+	if err := cmd.Start(); err != nil {
	24	+	b.Fatal(err)
	25	+	}
	26	+	n := 0
	27	+	br := bufio.NewReader(out)
	28	+	for {
	29	+	_, err := br.ReadSlice('\n')
	30	+	if err == io.EOF {
	31	+	break
	32	+	}
	33	+	if err != nil && err != bufio.ErrBufferFull {
	34	+	b.Fatal(err)
	35	+	}
	36	+	n++
	37	+	}
	38	+	if err := cmd.Wait(); err != nil {
	39	+	b.Fatal(err)
	40	+	}
	41	+	return n
	42	+	}
	43	+
	44	+	// gitcliFormat is the format the gitcli adapter of ermite asks git log for.
	45	+	const gitcliFormat = "%H%x00%P%x00%an%x00%ae%x00%at%x00%cn%x00%ce%x00%ct%x00%B"
	46	+
	47	+	// benchWalk compares Walk with git log in two shapes: ids only (git answers
	48	+	// from the commit-graph without reading commits) and complete commits, which
	49	+	// is what the gitcli adapter asks for (git has to read every commit).
	50	+	func benchWalk(b testing.B, x realWalk, opts WalkOptions, gitArgs ...string) {
	51	+	run := func(b *testing.B, o WalkOptions) {
	52	+	b.ReportAllocs()
	53	+	var n int
	54	+	for i := 0; i < b.N; i++ {
	55	+	n = 0
	56	+	for c, err := range x.r.Walk(context.Background(), x.head, o) {
	57	+	if err != nil {
	58	+	b.Fatal(err)
	59	+	}
	60	+	if !o.IDOnly {
	61	+	n += len(c.Message) // the commit is really parsed
	62	+	}
	63	+	n++
	64	+	}
	65	+	}
	66	+	_ = n
	67	+	}
	68	+	for _, shape := range []struct {
	69	+	name string
	70	+	idOnly bool
	71	+	format string
	72	+	}{{"ids", true, "--format=%H"}, {"full", false, "-z --format=" + gitcliFormat}} {
	73	+	b.Run(shape.name+"/ermite", func(b *testing.B) {
	74	+	o := opts
	75	+	o.IDOnly = shape.idOnly
	76	+	run(b, o)
	77	+	})
	78	+	b.Run(shape.name+"/git", func(b *testing.B) {
	79	+	b.ReportAllocs()
	80	+	args := []string{"log"}
	81	+	if shape.idOnly {
	82	+	args = append(args, shape.format)
	83	+	} else {
	84	+	args = append(args, "-z", "--format="+gitcliFormat)
	85	+	}
	86	+	args = append(args, gitArgs...)
	87	+	for i := 0; i < b.N; i++ {
	88	+	gitLogCount(b, x, args...)
	89	+	}
	90	+	})
	91	+	}
	92	+	}
	93	+
	94	+	func BenchmarkWalk(b *testing.B) {
	95	+	x := getRealWalk(b)
	96	+	head := x.head.String()
	97	+	b.Run("full-history", func(b *testing.B) { benchWalk(b, x, WalkOptions{}, head) })
	98	+	b.Run("skip5000-limit100", func(b *testing.B) {
	99	+	benchWalk(b, x, WalkOptions{Skip: 5000, Limit: 100}, "--skip=5000", "-n", "100", head)
	100	+	})
	101	+	b.Run("file", func(b *testing.B) {
	102	+	benchWalk(b, x, WalkOptions{Path: "pkg/gui/gui.go"}, head, "--", "pkg/gui/gui.go")
	103	+	})
	104	+	b.Run("dir", func(b *testing.B) {
	105	+	benchWalk(b, x, WalkOptions{Path: "pkg/commands"}, head, "--", "pkg/commands")
	106	+	})
	107	+	b.Run("rare-file", func(b *testing.B) {
	108	+	benchWalk(b, x, WalkOptions{Path: "go.mod"}, head, "--", "go.mod")
	109	+	})
	110	+	b.Run("page-100-of-file", func(b *testing.B) {
	111	+	benchWalk(b, x, WalkOptions{Path: "pkg/gui/gui.go", Limit: 100}, "-n", "100", head, "--", "pkg/gui/gui.go")
	112	+	})
	113	+	}
	114	+
	115	+	func BenchmarkLastCommits(b *testing.B) {
	116	+	x := getRealWalk(b)
	117	+	head := x.head.String()
	118	+	for _, dir := range []string{"", "pkg/gui"} {
	119	+	spec := head
	120	+	if dir != "" {
	121	+	spec += ":" + dir
	122	+	}
	123	+	names := lines(x.git(b, "ls-tree", "--name-only", spec))
	124	+	name := dir
	125	+	if name == "" {
	126	+	name = "root"
	127	+	}
	128	+	b.Run(name+"/ermite", func(b *testing.B) {
	129	+	b.ReportAllocs()
	130	+	for i := 0; i < b.N; i++ {
	131	+	if _, err := x.r.LastCommits(context.Background(), x.head, dir, names); err != nil {
	132	+	b.Fatal(err)
	133	+	}
	134	+	}
	135	+	b.ReportMetric(float64(len(names)), "entries")
	136	+	})
	137	+	b.Run(name+"/git-n-calls", func(b *testing.B) {
	138	+	b.ReportAllocs()
	139	+	for i := 0; i < b.N; i++ {
	140	+	for _, n := range names {
	141	+	gitLogCount(b, x, "log", "-n", "1", "--format=%H", head, "--", path.Join(dir, n))
	142	+	}
	143	+	}
	144	+	})
	145	+	}
	146	+	}

A walkrace_test.go +155 -0

@@ -0,0 +1,155 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package git
	4	+
	5	+	import (
	6	+	"bytes"
	7	+	"context"
	8	+	"fmt"
	9	+	"os/exec"
	10	+	"slices"
	11	+	"strings"
	12	+	"sync"
	13	+	"sync/atomic"
	14	+	"testing"
	15	+	"time"
	16	+	)
	17	+
	18	+	// gitQuiet runs git in the fixture without touching the testing.T, so it can
	19	+	// be used from goroutines.
	20	+	func (w *wfix) gitQuiet(stdin string, args ...string) error {
	21	+	cmd := exec.Command("git", args...)
	22	+	cmd.Dir = w.dir
	23	+	cmd.Env = append(gitEnv(), "GIT_DIR="+w.dir)
	24	+	cmd.Stdin = strings.NewReader(stdin)
	25	+	var errb bytes.Buffer
	26	+	cmd.Stderr = &errb
	27	+	if err := cmd.Run(); err != nil {
	28	+	return fmt.Errorf("git %s: %v: %s", strings.Join(args, " "), err, errb.String())
	29	+	}
	30	+	return nil
	31	+	}
	32	+
	33	+	// TestWalkConcurrentRefresh runs walks and LastCommits while the repository
	34	+	// changes under them: Refresh, new commits arriving by fast-import (a push:
	35	+	// new pack, new graph layer), rewrites of the commit-graph (with and without
	36	+	// Bloom filters, split and single, or removed) and repacks. Every walk starts
	37	+	// from a fixed tip and must give the same answer as before the changes.
	38	+	func TestWalkConcurrentRefresh(t *testing.T) {
	39	+	requireGit(t)
	40	+	cs := randomHistory(21, 120)
	41	+	w := buildWFix(t, cs)
	42	+	w.setGraph(wgraphs[4])
	43	+	tipIdx := len(cs) - 1
	44	+	tip := mustID(t, w.ids[tipIdx])
	45	+	paths := []string{"", "a", "d", "d/e/z", "q/r"}
	46	+	want := map[string][]string{}
	47	+	for _, p := range paths {
	48	+	want[p] = w.gitLog(tipIdx, 0, 0, p)
	49	+	}
	50	+	names := []string{"a", "b", "d", "m", "q", "nope"}
	51	+	wantLast := w.lastOracle(tipIdx, "", names)
	52	+
	53	+	r, err := Open(w.dir)
	54	+	if err != nil {
	55	+	t.Fatal(err)
	56	+	}
	57	+	defer func() { _ = r.Close() }()
	58	+
	59	+	dur := 3 * time.Second
	60	+	if testing.Short() {
	61	+	dur = time.Second
	62	+	}
	63	+	deadline := time.Now().Add(dur)
	64	+	var stop atomic.Bool
	65	+	var wg sync.WaitGroup
	66	+	errc := make(chan error, 64)
	67	+	fail := func(err error) {
	68	+	select {
	69	+	case errc <- err:
	70	+	default:
	71	+	}
	72	+	stop.Store(true)
	73	+	}
	74	+
	75	+	for g := 0; g < 6; g++ {
	76	+	wg.Add(1)
	77	+	go func(g int) {
	78	+	defer wg.Done()
	79	+	for i := 0; !stop.Load(); i++ {
	80	+	p := paths[(g+i)%len(paths)]
	81	+	var got []string
	82	+	for c, err := range r.Walk(context.Background(), tip, WalkOptions{Path: p, IDOnly: i%2 == 0}) {
	83	+	if err != nil {
	84	+	fail(fmt.Errorf("walk %q: %w", p, err))
	85	+	return
	86	+	}
	87	+	got = append(got, c.ID.String())
	88	+	}
	89	+	if !slices.Equal(got, want[p]) {
	90	+	fail(fmt.Errorf("walk %q: wrong history (%d commits, want %d)", p, len(got), len(want[p])))
	91	+	return
	92	+	}
	93	+	if i%3 == 0 {
	94	+	res, err := r.LastCommits(context.Background(), tip, "", names)
	95	+	if err != nil {
	96	+	fail(fmt.Errorf("LastCommits: %w", err))
	97	+	return
	98	+	}
	99	+	if !slices.Equal(res, wantLast) {
	100	+	fail(fmt.Errorf("LastCommits: wrong answer"))
	101	+	return
	102	+	}
	103	+	}
	104	+	}
	105	+	}(g)
	106	+	}
	107	+
	108	+	wg.Add(1)
	109	+	go func() { // Refresh as fast as possible
	110	+	defer wg.Done()
	111	+	for !stop.Load() {
	112	+	if err := r.Refresh(); err != nil {
	113	+	fail(fmt.Errorf("Refresh: %w", err))
	114	+	return
	115	+	}
	116	+	time.Sleep(time.Millisecond)
	117	+	}
	118	+	}()
	119	+
	120	+	wg.Add(1)
	121	+	go func() { // the writer
	122	+	defer wg.Done()
	123	+	defer stop.Store(true)
	124	+	var in bytes.Buffer
	125	+	for _, id := range w.ids {
	126	+	fmt.Fprintln(&in, id)
	127	+	}
	128	+	for i := 0; time.Now().Before(deadline) && !stop.Load(); i++ {
	129	+	var err error
	130	+	switch i % 6 {
	131	+	case 0: // a push: a new commit on top of the tip, a new pack
	132	+	err = w.gitQuiet(fmt.Sprintf("commit refs/heads/push%d\ncommitter T <t@example.com> %d +0000\ndata 1\nx\nfrom %s\n\n", i, 1_700_000_000+i, w.ids[tipIdx]), "fast-import", "--quiet")
	133	+	case 1:
	134	+	err = w.gitQuiet(in.String(), "commit-graph", "write", "--stdin-commits", "--split", "--changed-paths")
	135	+	case 2:
	136	+	err = w.gitQuiet("", "repack", "-adq")
	137	+	case 3:
	138	+	err = w.gitQuiet(in.String(), "commit-graph", "write", "--stdin-commits", "--no-changed-paths")
	139	+	case 4:
	140	+	err = w.gitQuiet(in.String(), "commit-graph", "write", "--stdin-commits", "--split=replace", "--changed-paths")
	141	+	case 5:
	142	+	err = w.gitQuiet("", "commit-graph", "write", "--reachable", "--split=no-merge", "--changed-paths")
	143	+	}
	144	+	if err != nil {
	145	+	fail(err)
	146	+	return
	147	+	}
	148	+	}
	149	+	}()
	150	+	wg.Wait()
	151	+	close(errc)
	152	+	for err := range errc {
	153	+	t.Error(err)
	154	+	}
	155	+	}

@@ -0,0 +1,155 @@
	1	+	// SPDX-License-Identifier: MIT
	2	+
	3	+	package git
	4	+
	5	+	import (
	6	+	"bytes"
	7	+	"context"
	8	+	"fmt"
	9	+	"os/exec"
	10	+	"slices"
	11	+	"strings"
	12	+	"sync"
	13	+	"sync/atomic"
	14	+	"testing"
	15	+	"time"
	16	+	)
	17	+
	18	+	// gitQuiet runs git in the fixture without touching the testing.T, so it can
	19	+	// be used from goroutines.
	20	+	func (w *wfix) gitQuiet(stdin string, args ...string) error {
	21	+	cmd := exec.Command("git", args...)
	22	+	cmd.Dir = w.dir
	23	+	cmd.Env = append(gitEnv(), "GIT_DIR="+w.dir)
	24	+	cmd.Stdin = strings.NewReader(stdin)
	25	+	var errb bytes.Buffer
	26	+	cmd.Stderr = &errb
	27	+	if err := cmd.Run(); err != nil {
	28	+	return fmt.Errorf("git %s: %v: %s", strings.Join(args, " "), err, errb.String())
	29	+	}
	30	+	return nil
	31	+	}
	32	+
	33	+	// TestWalkConcurrentRefresh runs walks and LastCommits while the repository
	34	+	// changes under them: Refresh, new commits arriving by fast-import (a push:
	35	+	// new pack, new graph layer), rewrites of the commit-graph (with and without
	36	+	// Bloom filters, split and single, or removed) and repacks. Every walk starts
	37	+	// from a fixed tip and must give the same answer as before the changes.
	38	+	func TestWalkConcurrentRefresh(t *testing.T) {
	39	+	requireGit(t)
	40	+	cs := randomHistory(21, 120)
	41	+	w := buildWFix(t, cs)
	42	+	w.setGraph(wgraphs[4])
	43	+	tipIdx := len(cs) - 1
	44	+	tip := mustID(t, w.ids[tipIdx])
	45	+	paths := []string{"", "a", "d", "d/e/z", "q/r"}
	46	+	want := map[string][]string{}
	47	+	for _, p := range paths {
	48	+	want[p] = w.gitLog(tipIdx, 0, 0, p)
	49	+	}
	50	+	names := []string{"a", "b", "d", "m", "q", "nope"}
	51	+	wantLast := w.lastOracle(tipIdx, "", names)
	52	+
	53	+	r, err := Open(w.dir)
	54	+	if err != nil {
	55	+	t.Fatal(err)
	56	+	}
	57	+	defer func() { _ = r.Close() }()
	58	+
	59	+	dur := 3 * time.Second
	60	+	if testing.Short() {
	61	+	dur = time.Second
	62	+	}
	63	+	deadline := time.Now().Add(dur)
	64	+	var stop atomic.Bool
	65	+	var wg sync.WaitGroup
	66	+	errc := make(chan error, 64)
	67	+	fail := func(err error) {
	68	+	select {
	69	+	case errc <- err:
	70	+	default:
	71	+	}
	72	+	stop.Store(true)
	73	+	}
	74	+
	75	+	for g := 0; g < 6; g++ {
	76	+	wg.Add(1)
	77	+	go func(g int) {
	78	+	defer wg.Done()
	79	+	for i := 0; !stop.Load(); i++ {
	80	+	p := paths[(g+i)%len(paths)]
	81	+	var got []string
	82	+	for c, err := range r.Walk(context.Background(), tip, WalkOptions{Path: p, IDOnly: i%2 == 0}) {
	83	+	if err != nil {
	84	+	fail(fmt.Errorf("walk %q: %w", p, err))
	85	+	return
	86	+	}
	87	+	got = append(got, c.ID.String())
	88	+	}
	89	+	if !slices.Equal(got, want[p]) {
	90	+	fail(fmt.Errorf("walk %q: wrong history (%d commits, want %d)", p, len(got), len(want[p])))
	91	+	return
	92	+	}
	93	+	if i%3 == 0 {
	94	+	res, err := r.LastCommits(context.Background(), tip, "", names)
	95	+	if err != nil {
	96	+	fail(fmt.Errorf("LastCommits: %w", err))
	97	+	return
	98	+	}
	99	+	if !slices.Equal(res, wantLast) {
	100	+	fail(fmt.Errorf("LastCommits: wrong answer"))
	101	+	return
	102	+	}
	103	+	}
	104	+	}
	105	+	}(g)
	106	+	}
	107	+
	108	+	wg.Add(1)
	109	+	go func() { // Refresh as fast as possible
	110	+	defer wg.Done()
	111	+	for !stop.Load() {
	112	+	if err := r.Refresh(); err != nil {
	113	+	fail(fmt.Errorf("Refresh: %w", err))
	114	+	return
	115	+	}
	116	+	time.Sleep(time.Millisecond)
	117	+	}
	118	+	}()
	119	+
	120	+	wg.Add(1)
	121	+	go func() { // the writer
	122	+	defer wg.Done()
	123	+	defer stop.Store(true)
	124	+	var in bytes.Buffer
	125	+	for _, id := range w.ids {
	126	+	fmt.Fprintln(&in, id)
	127	+	}
	128	+	for i := 0; time.Now().Before(deadline) && !stop.Load(); i++ {
	129	+	var err error
	130	+	switch i % 6 {
	131	+	case 0: // a push: a new commit on top of the tip, a new pack
	132	+	err = w.gitQuiet(fmt.Sprintf("commit refs/heads/push%d\ncommitter T <t@example.com> %d +0000\ndata 1\nx\nfrom %s\n\n", i, 1_700_000_000+i, w.ids[tipIdx]), "fast-import", "--quiet")
	133	+	case 1:
	134	+	err = w.gitQuiet(in.String(), "commit-graph", "write", "--stdin-commits", "--split", "--changed-paths")
	135	+	case 2:
	136	+	err = w.gitQuiet("", "repack", "-adq")
	137	+	case 3:
	138	+	err = w.gitQuiet(in.String(), "commit-graph", "write", "--stdin-commits", "--no-changed-paths")
	139	+	case 4:
	140	+	err = w.gitQuiet(in.String(), "commit-graph", "write", "--stdin-commits", "--split=replace", "--changed-paths")
	141	+	case 5:
	142	+	err = w.gitQuiet("", "commit-graph", "write", "--reachable", "--split=no-merge", "--changed-paths")
	143	+	}
	144	+	if err != nil {
	145	+	fail(err)
	146	+	return
	147	+	}
	148	+	}
	149	+	}()
	150	+	wg.Wait()
	151	+	close(errc)
	152	+	for err := range errc {
	153	+	t.Error(err)
	154	+	}
	155	+	}

M walkreal_test.go +1 -4


@@ -174,10 +174,7 @@ func TestLastCommitsRealRepository(t *testing.T) {
174	174		if dir != "" {
175	175		spec += ":" + dir
176	176		}
177		-	var names []string
178		-	for _, l := range lines(x.git(t, "ls-tree", "--name-only", spec)) {
179		-	names = append(names, l)
180		-	}
	177	+	names := lines(x.git(t, "ls-tree", "--name-only", spec))
181	178		if len(names) > 120 {
182	179		names = names[:120]
183	180		}

 		if dir != "" {
 			spec += ":" + dir
 		}
-		var names []string
+		names := lines(x.git(t, "ls-tree", "--name-only", spec))
-		for _, l := range lines(x.git(t, "ls-tree", "--name-only", spec)) {
-			names = append(names, l)
-		}
 		if len(names) > 120 {
 			names = names[:120]
 		}