detached 5d84510

feat(graph): commit-graph split chain and changed-path Bloom readers

commit
5d84510493c7a5d7a1f68fa737343aa38e6c0759
author
brice · · 2026-10-10 20:00 UTC
parent
f886b9a
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
2 files · +882 -0
Unified
UnifiedSplit
A bloom.go +184 -0
@@ -0,0 +1,184 @@
1+// SPDX-License-Identifier: MIT
2+
3+package git
4+
5+import (
6+ "encoding/binary"
7+)
8+
9+// Changed-path Bloom filters (BIDX and BDAT chunks of the commit-graph).
10+//
11+// For each commit the filter holds the paths that differ between the commit
12+// and its first parent (or the empty tree for a root commit). Git inserts
13+// every changed path together with each of its leading directories, without
14+// a trailing slash: changing "a/b/c" adds "a/b/c", "a/b" and "a". Looking
15+// up a path therefore only needs the key of that path, whether it is a file
16+// or a directory.
17+//
18+// Hashing mirrors git's bloom.c. A key is two 32-bit murmur3 hashes of the
19+// path with fixed seeds, h0 and h1; the i-th probe is h0 + i*h1 (mod 2^32),
20+// taken modulo the number of bits of the filter. Bit n of the filter is
21+// bit (n & 7) of byte n/8.
22+//
23+// BDAT header version 1 is the original hash, which sign-extends bytes >=
24+// 0x80 when git was built with a signed char (x86); version 2 fixes it with
25+// unsigned arithmetic. Both are supported; the version is read per layer
26+// from BDAT. For paths made of bytes < 0x80 the two are identical.
27+//
28+// A filter of one byte 0x00 means "no changed path" and one byte 0xff means
29+// "too many changed paths (more than 512)": the latter is reported as
30+// unknown. An empty or out-of-bounds filter is unknown as well.
31+
32+const (
33+ bloomHeaderSize = 12 // version, number of hashes, bits per entry
34+ bloomMaxHashes = 256 // sanity bound on the probe count read from disk
35+ bloomSeed0 = 0x293ae76f
36+ bloomSeed1 = 0x7e646e2c
37+)
38+
39+// bloomKey holds the precomputed hashes of a path for both hash versions. It
40+// is a small value; build it once per path and reuse it for many commits.
41+type bloomKey struct {
42+ v1, v2 [2]uint32
43+}
44+
45+// newBloomKey computes the key of path. It does not allocate.
46+func newBloomKey(path []byte) bloomKey {
47+ k := bloomKey{v2: [2]uint32{
48+ murmur3Seeded(bloomSeed0, path, false),
49+ murmur3Seeded(bloomSeed1, path, false),
50+ }}
51+ k.v1 = k.v2
52+ for _, b := range path {
53+ if b >= 0x80 {
54+ k.v1 = [2]uint32{
55+ murmur3Seeded(bloomSeed0, path, true),
56+ murmur3Seeded(bloomSeed1, path, true),
57+ }
58+ break
59+ }
60+ }
61+ return k
62+}
63+
64+// newBloomKeyString is newBloomKey for a string path.
65+func newBloomKeyString(path string) bloomKey {
66+ return newBloomKey([]byte(path))
67+}
68+
69+func rotl32(x uint32, r uint) uint32 { return x<<r | x>>(32-r) }
70+
71+// murmur3Seeded is MurmurHash3 x86 32-bit as implemented by git. With
72+// signedChars it reproduces git's version 1 behaviour on platforms where
73+// char is signed (x86): every byte >= 0x80, in full blocks and in the tail, is
74+// sign-extended before being shifted and OR-ed, which corrupts the bits of
75+// the neighbouring bytes of the block.
76+func murmur3Seeded(seed uint32, data []byte, signedChars bool) uint32 {
77+ const (
78+ c1 = 0xcc9e2d51
79+ c2 = 0x1b873593
80+ )
81+ ext := func(b byte) uint32 {
82+ if signedChars {
83+ return uint32(int32(int8(b)))
84+ }
85+ return uint32(b)
86+ }
87+ n4 := len(data) / 4
88+ for i := 0; i < n4; i++ {
89+ d := data[4*i : 4*i+4]
90+ k := ext(d[0]) | ext(d[1])<<8 | ext(d[2])<<16 | ext(d[3])<<24
91+ k *= c1
92+ k = rotl32(k, 15)
93+ k *= c2
94+ seed ^= k
95+ seed = rotl32(seed, 13)*5 + 0xe6546b64
96+ }
97+ tail := data[4*n4:]
98+ var k uint32
99+ switch len(tail) {
100+ case 3:
101+ k ^= ext(tail[2]) << 16
102+ fallthrough
103+ case 2:
104+ k ^= ext(tail[1]) << 8
105+ fallthrough
106+ case 1:
107+ k ^= ext(tail[0])
108+ k *= c1
109+ k = rotl32(k, 15)
110+ k *= c2
111+ seed ^= k
112+ }
113+ seed ^= uint32(len(data))
114+ seed ^= seed >> 16
115+ seed *= 0x85ebca6b
116+ seed ^= seed >> 13
117+ seed *= 0xc2b2ae35
118+ seed ^= seed >> 16
119+ return seed
120+}
121+
122+// bloomFilter returns the filter bytes of the commit at pos. ok is false when
123+// the layer has no usable Bloom data, or the filter is empty, truncated or
124+// the "too many changes" marker.
125+func (g *commitGraph) bloomFilter(pos uint32) (l *graphLayer, filter []byte, ok bool) {
126+ l, idx := g.layerOf(pos)
127+ if l == nil || !l.hasBloom {
128+ return nil, nil, false
129+ }
130+ var start uint32
131+ if idx > 0 {
132+ start = binary.BigEndian.Uint32(l.bidx[(idx-1)*4:])
133+ }
134+ end := binary.BigEndian.Uint32(l.bidx[idx*4:])
135+ data := l.bdat[bloomHeaderSize:]
136+ if end <= start || uint64(end) > uint64(len(data)) {
137+ return l, nil, false
138+ }
139+ filter = data[start:end]
140+ if len(filter) == 1 && filter[0] == 0xff {
141+ return l, nil, false
142+ }
143+ return l, filter, true
144+}
145+
146+// maybeChangedKey reports whether the path of k may have changed in the
147+// commit at pos relative to its first parent. maybe=false is definitive (the
148+// path did not change); maybe=true may be a false positive. ok=false means
149+// the answer is unknown (no filter, too large, truncated, bad position) and
150+// the caller must diff the trees.
151+func (g *commitGraph) maybeChangedKey(pos uint32, k *bloomKey) (maybe, ok bool) {
152+ l, f, ok := g.bloomFilter(pos)
153+ if !ok {
154+ return false, false
155+ }
156+ h := &k.v2
157+ if l.bloomVersion == 1 {
158+ h = &k.v1
159+ }
160+ mod := uint64(len(f)) * 8
161+ var h0, h1 = h[0], h[1]
162+ for i := uint32(0); i < l.bloomHashes; i++ {
163+ bit := uint64(h0+i*h1) % mod
164+ if f[bit>>3]&(1<<(bit&7)) == 0 {
165+ return false, true
166+ }
167+ }
168+ return true, true
169+}
170+
171+// maybeChanged is maybeChangedKey for a path given as bytes, without a
172+// trailing slash, relative to the repository root. Use newBloomKey when
173+// testing the same path against many commits.
174+func (g *commitGraph) maybeChanged(pos uint32, path []byte) (maybe, ok bool) {
175+ k := newBloomKey(path)
176+ return g.maybeChangedKey(pos, &k)
177+}
178+
179+// hasBloom reports whether the commit at pos has a usable changed-path
180+// filter.
181+func (g *commitGraph) hasBloom(pos uint32) bool {
182+ _, _, ok := g.bloomFilter(pos)
183+ return ok
184+}
@@ -0,0 +1,184 @@
1+// SPDX-License-Identifier: MIT
2+
3+package git
4+
5+import (
6+ "encoding/binary"
7+)
8+
9+// Changed-path Bloom filters (BIDX and BDAT chunks of the commit-graph).
10+//
11+// For each commit the filter holds the paths that differ between the commit
12+// and its first parent (or the empty tree for a root commit). Git inserts
13+// every changed path together with each of its leading directories, without
14+// a trailing slash: changing "a/b/c" adds "a/b/c", "a/b" and "a". Looking
15+// up a path therefore only needs the key of that path, whether it is a file
16+// or a directory.
17+//
18+// Hashing mirrors git's bloom.c. A key is two 32-bit murmur3 hashes of the
19+// path with fixed seeds, h0 and h1; the i-th probe is h0 + i*h1 (mod 2^32),
20+// taken modulo the number of bits of the filter. Bit n of the filter is
21+// bit (n & 7) of byte n/8.
22+//
23+// BDAT header version 1 is the original hash, which sign-extends bytes >=
24+// 0x80 when git was built with a signed char (x86); version 2 fixes it with
25+// unsigned arithmetic. Both are supported; the version is read per layer
26+// from BDAT. For paths made of bytes < 0x80 the two are identical.
27+//
28+// A filter of one byte 0x00 means "no changed path" and one byte 0xff means
29+// "too many changed paths (more than 512)": the latter is reported as
30+// unknown. An empty or out-of-bounds filter is unknown as well.
31+
32+const (
33+ bloomHeaderSize = 12 // version, number of hashes, bits per entry
34+ bloomMaxHashes = 256 // sanity bound on the probe count read from disk
35+ bloomSeed0 = 0x293ae76f
36+ bloomSeed1 = 0x7e646e2c
37+)
38+
39+// bloomKey holds the precomputed hashes of a path for both hash versions. It
40+// is a small value; build it once per path and reuse it for many commits.
41+type bloomKey struct {
42+ v1, v2 [2]uint32
43+}
44+
45+// newBloomKey computes the key of path. It does not allocate.
46+func newBloomKey(path []byte) bloomKey {
47+ k := bloomKey{v2: [2]uint32{
48+ murmur3Seeded(bloomSeed0, path, false),
49+ murmur3Seeded(bloomSeed1, path, false),
50+ }}
51+ k.v1 = k.v2
52+ for _, b := range path {
53+ if b >= 0x80 {
54+ k.v1 = [2]uint32{
55+ murmur3Seeded(bloomSeed0, path, true),
56+ murmur3Seeded(bloomSeed1, path, true),
57+ }
58+ break
59+ }
60+ }
61+ return k
62+}
63+
64+// newBloomKeyString is newBloomKey for a string path.
65+func newBloomKeyString(path string) bloomKey {
66+ return newBloomKey([]byte(path))
67+}
68+
69+func rotl32(x uint32, r uint) uint32 { return x<<r | x>>(32-r) }
70+
71+// murmur3Seeded is MurmurHash3 x86 32-bit as implemented by git. With
72+// signedChars it reproduces git's version 1 behaviour on platforms where
73+// char is signed (x86): every byte >= 0x80, in full blocks and in the tail, is
74+// sign-extended before being shifted and OR-ed, which corrupts the bits of
75+// the neighbouring bytes of the block.
76+func murmur3Seeded(seed uint32, data []byte, signedChars bool) uint32 {
77+ const (
78+ c1 = 0xcc9e2d51
79+ c2 = 0x1b873593
80+ )
81+ ext := func(b byte) uint32 {
82+ if signedChars {
83+ return uint32(int32(int8(b)))
84+ }
85+ return uint32(b)
86+ }
87+ n4 := len(data) / 4
88+ for i := 0; i < n4; i++ {
89+ d := data[4*i : 4*i+4]
90+ k := ext(d[0]) | ext(d[1])<<8 | ext(d[2])<<16 | ext(d[3])<<24
91+ k *= c1
92+ k = rotl32(k, 15)
93+ k *= c2
94+ seed ^= k
95+ seed = rotl32(seed, 13)*5 + 0xe6546b64
96+ }
97+ tail := data[4*n4:]
98+ var k uint32
99+ switch len(tail) {
100+ case 3:
101+ k ^= ext(tail[2]) << 16
102+ fallthrough
103+ case 2:
104+ k ^= ext(tail[1]) << 8
105+ fallthrough
106+ case 1:
107+ k ^= ext(tail[0])
108+ k *= c1
109+ k = rotl32(k, 15)
110+ k *= c2
111+ seed ^= k
112+ }
113+ seed ^= uint32(len(data))
114+ seed ^= seed >> 16
115+ seed *= 0x85ebca6b
116+ seed ^= seed >> 13
117+ seed *= 0xc2b2ae35
118+ seed ^= seed >> 16
119+ return seed
120+}
121+
122+// bloomFilter returns the filter bytes of the commit at pos. ok is false when
123+// the layer has no usable Bloom data, or the filter is empty, truncated or
124+// the "too many changes" marker.
125+func (g *commitGraph) bloomFilter(pos uint32) (l *graphLayer, filter []byte, ok bool) {
126+ l, idx := g.layerOf(pos)
127+ if l == nil || !l.hasBloom {
128+ return nil, nil, false
129+ }
130+ var start uint32
131+ if idx > 0 {
132+ start = binary.BigEndian.Uint32(l.bidx[(idx-1)*4:])
133+ }
134+ end := binary.BigEndian.Uint32(l.bidx[idx*4:])
135+ data := l.bdat[bloomHeaderSize:]
136+ if end <= start || uint64(end) > uint64(len(data)) {
137+ return l, nil, false
138+ }
139+ filter = data[start:end]
140+ if len(filter) == 1 && filter[0] == 0xff {
141+ return l, nil, false
142+ }
143+ return l, filter, true
144+}
145+
146+// maybeChangedKey reports whether the path of k may have changed in the
147+// commit at pos relative to its first parent. maybe=false is definitive (the
148+// path did not change); maybe=true may be a false positive. ok=false means
149+// the answer is unknown (no filter, too large, truncated, bad position) and
150+// the caller must diff the trees.
151+func (g *commitGraph) maybeChangedKey(pos uint32, k *bloomKey) (maybe, ok bool) {
152+ l, f, ok := g.bloomFilter(pos)
153+ if !ok {
154+ return false, false
155+ }
156+ h := &k.v2
157+ if l.bloomVersion == 1 {
158+ h = &k.v1
159+ }
160+ mod := uint64(len(f)) * 8
161+ var h0, h1 = h[0], h[1]
162+ for i := uint32(0); i < l.bloomHashes; i++ {
163+ bit := uint64(h0+i*h1) % mod
164+ if f[bit>>3]&(1<<(bit&7)) == 0 {
165+ return false, true
166+ }
167+ }
168+ return true, true
169+}
170+
171+// maybeChanged is maybeChangedKey for a path given as bytes, without a
172+// trailing slash, relative to the repository root. Use newBloomKey when
173+// testing the same path against many commits.
174+func (g *commitGraph) maybeChanged(pos uint32, path []byte) (maybe, ok bool) {
175+ k := newBloomKey(path)
176+ return g.maybeChangedKey(pos, &k)
177+}
178+
179+// hasBloom reports whether the commit at pos has a usable changed-path
180+// filter.
181+func (g *commitGraph) hasBloom(pos uint32) bool {
182+ _, _, ok := g.bloomFilter(pos)
183+ return ok
184+}
A graph.go +698 -0
@@ -0,0 +1,698 @@
1+// SPDX-License-Identifier: MIT
2+
3+package git
4+
5+import (
6+ "bytes"
7+ "encoding/binary"
8+ "encoding/hex"
9+ "errors"
10+ "fmt"
11+ "io"
12+ "io/fs"
13+ "math"
14+ "os"
15+ "path/filepath"
16+ "sync/atomic"
17+ "syscall"
18+)
19+
20+// This file reads Git commit-graph files (gitformat-commit-graph): a single
21+// objects/info/commit-graph or a split chain
22+// objects/info/commit-graphs/commit-graph-chain plus graph-<hash>.graph
23+// layers. Files are mmapped read-only and never written.
24+//
25+// Lifecycle. openCommitGraph returns a graph holding one reference (the
26+// owner's). A reader that may outlive a concurrent reopen calls retain
27+// (which fails once the graph is closed) and release when done. The owner
28+// calls close when it replaces the graph. The mappings are unmapped when the
29+// last reference is dropped; no accessor may be used after that. Layers are
30+// immutable files named after their checksum, so a concurrent
31+// "git commit-graph write" never changes a mapped file; it can only delete
32+// it, which does not affect an existing mapping.
33+//
34+// Positions are global across the chain: the commits of the base layer come
35+// first, then those of the next layer, and so on. Within a layer commits are
36+// sorted by object id.
37+//
38+// Every offset read from a file is bounds-checked. Structural problems found
39+// when opening return an error wrapping errGraphCorrupt; per-commit problems
40+// (edge list, overflow generation) are reported by the accessor that meets
41+// them. No accessor panics on malformed input.
42+
43+var (
44+ // errGraphCorrupt wraps every error caused by malformed graph content.
45+ // After merging with the core it becomes an alias of (or is wrapped
46+ // with) ErrCorrupt.
47+ errGraphCorrupt = errors.New("corrupt commit-graph")
48+ // errGraphUnsupported wraps valid graphs this reader does not handle
49+ // (SHA-256, unknown format version).
50+ errGraphUnsupported = errors.New("unsupported commit-graph")
51+ // errNoGraph means the repository has no commit-graph.
52+ errNoGraph = errors.New("no commit-graph")
53+ // errGraphPosition is returned for a position >= numCommits.
54+ errGraphPosition = errors.New("commit-graph position out of range")
55+)
56+
57+func graphCorruptf(format string, args ...any) error {
58+ return fmt.Errorf("%w: %s", errGraphCorrupt, fmt.Sprintf(format, args...))
59+}
60+
61+const (
62+ graphHashSize = 20
63+ graphHeaderSize = 8
64+ graphChunkEntrySize = 12
65+ graphCommitSize = graphHashSize + 16 // tree, 2 parents, level+time
66+ graphFanoutSize = 256 * 4
67+
68+ graphParentNone = 0x70000000 // no parent in this slot
69+ graphParentExtra = 0x80000000 // second slot: index into the EDGE chunk
70+ graphEdgeLast = 0x80000000 // EDGE entry: last parent of the commit
71+ graphPosMask = 0x7fffffff
72+ graphGenOverflow = 0x80000000 // GDA2 entry: index into GDO2
73+
74+ graphMaxLayers = 256 // the base count is a single byte
75+ graphMaxParents = 1 << 13
76+ graphMaxChain = graphMaxLayers * (2*graphHashSize + 1)
77+
78+ chunkOIDF = 'O'<<24 | 'I'<<16 | 'D'<<8 | 'F'
79+ chunkOIDL = 'O'<<24 | 'I'<<16 | 'D'<<8 | 'L'
80+ chunkCDAT = 'C'<<24 | 'D'<<16 | 'A'<<8 | 'T'
81+ chunkEDGE = 'E'<<24 | 'D'<<16 | 'G'<<8 | 'E'
82+ chunkGDA2 = 'G'<<24 | 'D'<<16 | 'A'<<8 | '2'
83+ chunkGDO2 = 'G'<<24 | 'D'<<16 | 'O'<<8 | '2'
84+ chunkBIDX = 'B'<<24 | 'I'<<16 | 'D'<<8 | 'X'
85+ chunkBDAT = 'B'<<24 | 'D'<<16 | 'A'<<8 | 'T'
86+ chunkBASE = 'B'<<24 | 'A'<<16 | 'S'<<8 | 'E'
87+)
88+
89+// graphLayer is one commit-graph file.
90+type graphLayer struct {
91+ data []byte
92+ numBase int // base graphs declared in the header
93+ base uint32 // global position of the first commit
94+ n uint32 // commits in this layer
95+ fanout []byte
96+ oidl []byte
97+ cdat []byte
98+ edge []byte
99+ gda2 []byte
100+ gdo2 []byte
101+ bidx []byte
102+ bdat []byte
103+ baseList []byte
104+
105+ // Changed-path Bloom filter settings, valid when hasBloom.
106+ hasBloom bool
107+ bloomVersion uint32
108+ bloomHashes uint32
109+ bloomBits uint32
110+}
111+
112+// commitGraph is a (possibly split) commit-graph.
113+type commitGraph struct {
114+ layers []graphLayer // base first
115+ total uint32
116+ genV2 bool // every layer has GDA2: generation() is a corrected date
117+
118+ refs atomic.Int64
119+ unmaps [][]byte
120+}
121+
122+// graphMmap maps a file read-only. The returned slice must be released with
123+// syscall.Munmap.
124+func graphMmap(path string) ([]byte, error) {
125+ f, err := os.Open(path)
126+ if err != nil {
127+ return nil, err
128+ }
129+ defer func() { _ = f.Close() }()
130+ st, err := f.Stat()
131+ if err != nil {
132+ return nil, err
133+ }
134+ if !st.Mode().IsRegular() {
135+ return nil, graphCorruptf("%s: not a regular file", filepath.Base(path))
136+ }
137+ size := st.Size()
138+ if size < graphHeaderSize+graphChunkEntrySize+graphHashSize {
139+ return nil, graphCorruptf("%s: file too short (%d bytes)", filepath.Base(path), size)
140+ }
141+ if size > math.MaxInt32 {
142+ return nil, graphCorruptf("%s: file too large (%d bytes)", filepath.Base(path), size)
143+ }
144+ data, err := syscall.Mmap(int(f.Fd()), 0, int(size), syscall.PROT_READ, syscall.MAP_SHARED)
145+ if err != nil {
146+ return nil, fmt.Errorf("mmap %s: %w", filepath.Base(path), err)
147+ }
148+ _ = syscall.Madvise(data, syscall.MADV_RANDOM)
149+ return data, nil
150+}
151+
152+// openCommitGraph opens the commit-graph of the objects directory objectsDir:
153+// objects/info/commit-graph if present, else the split chain. It returns
154+// errNoGraph when there is none. The result holds one reference.
155+func openCommitGraph(objectsDir string) (*commitGraph, error) {
156+ infoDir := filepath.Join(objectsDir, "info")
157+ data, err := graphMmap(filepath.Join(infoDir, "commit-graph"))
158+ switch {
159+ case err == nil:
160+ l, perr := parseGraphLayer(data)
161+ if perr != nil {
162+ _ = syscall.Munmap(data)
163+ return nil, fmt.Errorf("commit-graph: %w", perr)
164+ }
165+ g, aerr := assembleGraph([]*graphLayer{l}, nil)
166+ if aerr != nil {
167+ _ = syscall.Munmap(data)
168+ return nil, fmt.Errorf("commit-graph: %w", aerr)
169+ }
170+ g.unmaps = [][]byte{data}
171+ return g, nil
172+ case errors.Is(err, fs.ErrNotExist):
173+ return openCommitGraphChain(infoDir)
174+ default:
175+ return nil, fmt.Errorf("commit-graph: %w", err)
176+ }
177+}
178+
179+func openCommitGraphChain(infoDir string) (*commitGraph, error) {
180+ dir := filepath.Join(infoDir, "commit-graphs")
181+ names, err := readGraphChain(filepath.Join(dir, "commit-graph-chain"))
182+ if err != nil {
183+ return nil, err
184+ }
185+ layers := make([]*graphLayer, 0, len(names))
186+ maps := make([][]byte, 0, len(names))
187+ cleanup := func() {
188+ for _, m := range maps {
189+ _ = syscall.Munmap(m)
190+ }
191+ }
192+ for _, name := range names {
193+ file := "graph-" + hex.EncodeToString(name[:]) + ".graph"
194+ data, err := graphMmap(filepath.Join(dir, file))
195+ if err != nil {
196+ cleanup()
197+ return nil, fmt.Errorf("commit-graph chain layer: %w", err)
198+ }
199+ maps = append(maps, data)
200+ l, err := parseGraphLayer(data)
201+ if err != nil {
202+ cleanup()
203+ return nil, fmt.Errorf("commit-graph layer %s: %w", file, err)
204+ }
205+ layers = append(layers, l)
206+ }
207+ g, err := assembleGraph(layers, names)
208+ if err != nil {
209+ cleanup()
210+ return nil, fmt.Errorf("commit-graph chain: %w", err)
211+ }
212+ g.unmaps = maps
213+ return g, nil
214+}
215+
216+// readGraphChain reads commit-graph-chain.
217+func readGraphChain(path string) ([][graphHashSize]byte, error) {
218+ f, err := os.Open(path)
219+ if err != nil {
220+ if errors.Is(err, fs.ErrNotExist) {
221+ return nil, errNoGraph
222+ }
223+ return nil, fmt.Errorf("commit-graph chain: %w", err)
224+ }
225+ defer func() { _ = f.Close() }()
226+ raw, err := io.ReadAll(io.LimitReader(f, graphMaxChain+1))
227+ if err != nil {
228+ return nil, fmt.Errorf("commit-graph chain: %w", err)
229+ }
230+ return parseGraphChain(raw)
231+}
232+
233+// parseGraphChain parses the content of commit-graph-chain: one lowercase
234+// hex checksum per line, base layer first.
235+func parseGraphChain(raw []byte) ([][graphHashSize]byte, error) {
236+ if len(raw) == 0 {
237+ return nil, errNoGraph
238+ }
239+ const line = 2*graphHashSize + 1
240+ if len(raw) > graphMaxChain || len(raw)%line != 0 {
241+ return nil, graphCorruptf("commit-graph-chain: bad size %d", len(raw))
242+ }
243+ names := make([][graphHashSize]byte, len(raw)/line)
244+ for i := range names {
245+ row := raw[i*line : (i+1)*line]
246+ if row[line-1] != '\n' {
247+ return nil, graphCorruptf("commit-graph-chain: line %d not terminated", i+1)
248+ }
249+ if _, err := hex.Decode(names[i][:], row[:line-1]); err != nil {
250+ return nil, graphCorruptf("commit-graph-chain: line %d: %v", i+1, err)
251+ }
252+ }
253+ return names, nil
254+}
255+
256+// parseGraphLayer validates the header, the chunk table and the sizes of the
257+// chunks it understands. It does not verify the checksum nor the content of
258+// individual records.
259+func parseGraphLayer(data []byte) (*graphLayer, error) {
260+ if len(data) < graphHeaderSize+graphChunkEntrySize+graphHashSize {
261+ return nil, graphCorruptf("file too short (%d bytes)", len(data))
262+ }
263+ if string(data[:4]) != "CGPH" {
264+ return nil, graphCorruptf("bad signature %q", data[:4])
265+ }
266+ if data[4] != 1 {
267+ return nil, fmt.Errorf("%w: version %d", errGraphUnsupported, data[4])
268+ }
269+ switch data[5] {
270+ case 1:
271+ case 2:
272+ return nil, fmt.Errorf("%w: SHA-256", errGraphUnsupported)
273+ default:
274+ return nil, graphCorruptf("unknown hash version %d", data[5])
275+ }
276+ nChunks := int(data[6])
277+ l := &graphLayer{data: data, numBase: int(data[7])}
278+
279+ tableEnd := uint64(graphHeaderSize + (nChunks+1)*graphChunkEntrySize)
280+ bodyEnd := uint64(len(data) - graphHashSize)
281+ if tableEnd > bodyEnd {
282+ return nil, graphCorruptf("chunk table exceeds file")
283+ }
284+
285+ var ids [256]uint32
286+ prevOff := tableEnd
287+ for i := 0; i <= nChunks; i++ {
288+ ent := data[graphHeaderSize+i*graphChunkEntrySize:]
289+ id := binary.BigEndian.Uint32(ent)
290+ off := binary.BigEndian.Uint64(ent[4:])
291+ if (id == 0) != (i == nChunks) {
292+ return nil, graphCorruptf("chunk table: bad terminator at entry %d", i)
293+ }
294+ if off < prevOff || off > bodyEnd {
295+ return nil, graphCorruptf("chunk table: bad offset %d for entry %d", off, i)
296+ }
297+ if i > 0 {
298+ // Close the previous chunk: it spans [its offset, this offset).
299+ if err := l.setChunk(ids[i-1], data[prevOff:off]); err != nil {
300+ return nil, err
301+ }
302+ }
303+ for j := 0; j < i; j++ {
304+ if ids[j] == id {
305+ return nil, graphCorruptf("chunk table: duplicate chunk %08x", id)
306+ }
307+ }
308+ ids[i] = id
309+ prevOff = off
310+ }
311+
312+ if err := l.validate(); err != nil {
313+ return nil, err
314+ }
315+ return l, nil
316+}
317+
318+// setChunk records chunk id. Unknown chunks (and the obsolete GDAT/GDOV) are
319+// ignored.
320+func (l *graphLayer) setChunk(id uint32, b []byte) error {
321+ switch id {
322+ case chunkOIDF:
323+ l.fanout = b
324+ case chunkOIDL:
325+ l.oidl = b
326+ case chunkCDAT:
327+ l.cdat = b
328+ case chunkEDGE:
329+ l.edge = b
330+ case chunkGDA2:
331+ l.gda2 = b
332+ case chunkGDO2:
333+ l.gdo2 = b
334+ case chunkBIDX:
335+ l.bidx = b
336+ case chunkBDAT:
337+ l.bdat = b
338+ case chunkBASE:
339+ l.baseList = b
340+ }
341+ return nil
342+}
343+
344+func (l *graphLayer) validate() error {
345+ if len(l.fanout) != graphFanoutSize {
346+ return graphCorruptf("OIDF chunk missing or has size %d", len(l.fanout))
347+ }
348+ var prev uint32
349+ for i := 0; i < 256; i++ {
350+ v := binary.BigEndian.Uint32(l.fanout[i*4:])
351+ if v < prev {
352+ return graphCorruptf("OIDF chunk not monotonic at %d", i)
353+ }
354+ prev = v
355+ }
356+ if prev >= graphParentNone {
357+ return graphCorruptf("too many commits (%d)", prev)
358+ }
359+ l.n = prev
360+ n := uint64(prev)
361+ if l.oidl == nil || uint64(len(l.oidl)) != n*graphHashSize {
362+ return graphCorruptf("OIDL chunk missing or has size %d for %d commits", len(l.oidl), n)
363+ }
364+ if l.cdat == nil || uint64(len(l.cdat)) != n*graphCommitSize {
365+ return graphCorruptf("CDAT chunk missing or has size %d for %d commits", len(l.cdat), n)
366+ }
367+ if len(l.edge)%4 != 0 {
368+ return graphCorruptf("EDGE chunk size %d not a multiple of 4", len(l.edge))
369+ }
370+ if l.gda2 != nil && uint64(len(l.gda2)) != n*4 {
371+ return graphCorruptf("GDA2 chunk has size %d for %d commits", len(l.gda2), n)
372+ }
373+ if len(l.gdo2)%8 != 0 {
374+ return graphCorruptf("GDO2 chunk size %d not a multiple of 8", len(l.gdo2))
375+ }
376+ if l.numBase > 0 && uint64(len(l.baseList)) != uint64(l.numBase)*graphHashSize {
377+ return graphCorruptf("BASE chunk has size %d for %d base graphs", len(l.baseList), l.numBase)
378+ }
379+ if l.numBase == 0 && len(l.baseList) != 0 {
380+ return graphCorruptf("BASE chunk present without base graphs")
381+ }
382+ // Bloom filters need both chunks; with only one of them git ignores it.
383+ if l.bidx != nil && l.bdat != nil {
384+ if uint64(len(l.bidx)) != n*4 {
385+ return graphCorruptf("BIDX chunk has size %d for %d commits", len(l.bidx), n)
386+ }
387+ if len(l.bdat) < bloomHeaderSize {
388+ return graphCorruptf("BDAT chunk too short (%d bytes)", len(l.bdat))
389+ }
390+ l.bloomVersion = binary.BigEndian.Uint32(l.bdat)
391+ l.bloomHashes = binary.BigEndian.Uint32(l.bdat[4:])
392+ l.bloomBits = binary.BigEndian.Uint32(l.bdat[8:])
393+ // Unknown versions are ignored like git does; an absurd hash
394+ // count would only make queries slow.
395+ l.hasBloom = (l.bloomVersion == 1 || l.bloomVersion == 2) &&
396+ l.bloomHashes >= 1 && l.bloomHashes <= bloomMaxHashes
397+ }
398+ return nil
399+}
400+
401+// assembleGraph builds a graph from validated layers, base first. names are
402+// the chain entries (checksums) of the layers, nil for a single file.
403+func assembleGraph(layers []*graphLayer, names [][graphHashSize]byte) (*commitGraph, error) {
404+ if len(layers) == 0 {
405+ return nil, errNoGraph
406+ }
407+ if len(layers) > graphMaxLayers {
408+ return nil, graphCorruptf("too many layers (%d)", len(layers))
409+ }
410+ if names != nil && len(names) != len(layers) {
411+ return nil, graphCorruptf("%d layers for %d chain entries", len(layers), len(names))
412+ }
413+ g := &commitGraph{layers: make([]graphLayer, len(layers)), genV2: true}
414+ var total uint64
415+ for k, l := range layers {
416+ if l.numBase != k {
417+ return nil, graphCorruptf("layer %d declares %d base graphs", k, l.numBase)
418+ }
419+ for i := 0; i < k; i++ {
420+ if !bytes.Equal(l.baseList[i*graphHashSize:(i+1)*graphHashSize], names[i][:]) {
421+ return nil, graphCorruptf("layer %d: base graph %d does not match the chain", k, i)
422+ }
423+ }
424+ l.base = uint32(total)
425+ total += uint64(l.n)
426+ if total >= graphParentNone {
427+ return nil, graphCorruptf("too many commits in chain")
428+ }
429+ if l.gda2 == nil {
430+ g.genV2 = false
431+ }
432+ g.layers[k] = *l
433+ }
434+ g.total = uint32(total)
435+ g.refs.Store(1)
436+ return g, nil
437+}
438+
439+// retain adds a reference. It reports false when the graph is already closed.
440+func (g *commitGraph) retain() bool {
441+ for {
442+ n := g.refs.Load()
443+ if n <= 0 {
444+ return false
445+ }
446+ if g.refs.CompareAndSwap(n, n+1) {
447+ return true
448+ }
449+ }
450+}
451+
452+// release drops a reference; the last one unmaps the files.
453+func (g *commitGraph) release() error {
454+ switch n := g.refs.Add(-1); {
455+ case n > 0:
456+ return nil
457+ case n == 0:
458+ var first error
459+ for _, m := range g.unmaps {
460+ if err := syscall.Munmap(m); err != nil && first == nil {
461+ first = err
462+ }
463+ }
464+ g.unmaps = nil
465+ return first
466+ default:
467+ g.refs.Add(1)
468+ return errors.New("commit-graph released too many times")
469+ }
470+}
471+
472+// close drops the owner's reference.
473+func (g *commitGraph) close() error { return g.release() }
474+
475+// numCommits is the number of commits in the whole chain.
476+func (g *commitGraph) numCommits() uint32 { return g.total }
477+
478+// numLayers is the number of files in the chain.
479+func (g *commitGraph) numLayers() int { return len(g.layers) }
480+
481+// hasGenerationV2 reports whether generation returns corrected commit dates
482+// (GDA2 present in every layer, as git requires) rather than topological
483+// levels. The two must not be compared with each other.
484+func (g *commitGraph) hasGenerationV2() bool { return g.genV2 }
485+
486+// layerOf finds the layer holding the global position pos and the index of
487+// the commit inside it.
488+func (g *commitGraph) layerOf(pos uint32) (*graphLayer, uint32) {
489+ if pos >= g.total {
490+ return nil, 0
491+ }
492+ for i := len(g.layers) - 1; i >= 0; i-- {
493+ if l := &g.layers[i]; pos >= l.base {
494+ return l, pos - l.base
495+ }
496+ }
497+ return nil, 0
498+}
499+
500+// find looks up id in one layer using the fanout table.
501+func (l *graphLayer) find(id *[graphHashSize]byte) (uint32, bool) {
502+ b := int(id[0])
503+ var lo uint32
504+ if b > 0 {
505+ lo = binary.BigEndian.Uint32(l.fanout[(b-1)*4:])
506+ }
507+ hi := binary.BigEndian.Uint32(l.fanout[b*4:]) // <= l.n, checked when opening
508+ for lo < hi {
509+ mid := lo + (hi-lo)/2
510+ off := int(mid) * graphHashSize
511+ switch c := bytes.Compare(l.oidl[off:off+graphHashSize], id[:]); {
512+ case c == 0:
513+ return mid, true
514+ case c < 0:
515+ lo = mid + 1
516+ default:
517+ hi = mid
518+ }
519+ }
520+ return 0, false
521+}
522+
523+// lookup returns the global position of commit id. It does not allocate.
524+func (g *commitGraph) lookup(id [graphHashSize]byte) (uint32, bool) {
525+ for i := len(g.layers) - 1; i >= 0; i-- {
526+ l := &g.layers[i]
527+ if idx, ok := l.find(&id); ok {
528+ return l.base + idx, true
529+ }
530+ }
531+ return 0, false
532+}
533+
534+// commitID returns the object id at pos.
535+func (g *commitGraph) commitID(pos uint32) (id [graphHashSize]byte, err error) {
536+ l, idx := g.layerOf(pos)
537+ if l == nil {
538+ return id, errGraphPosition
539+ }
540+ copy(id[:], l.oidl[int(idx)*graphHashSize:])
541+ return id, nil
542+}
543+
544+// record returns the CDAT record of pos.
545+func (g *commitGraph) record(pos uint32) (*graphLayer, []byte, error) {
546+ l, idx := g.layerOf(pos)
547+ if l == nil {
548+ return nil, nil, errGraphPosition
549+ }
550+ off := int(idx) * graphCommitSize
551+ return l, l.cdat[off : off+graphCommitSize], nil
552+}
553+
554+// rootTree returns the root tree id of the commit at pos.
555+func (g *commitGraph) rootTree(pos uint32) (id [graphHashSize]byte, err error) {
556+ _, rec, err := g.record(pos)
557+ if err != nil {
558+ return id, err
559+ }
560+ copy(id[:], rec)
561+ return id, nil
562+}
563+
564+// commitTime returns the commit time in seconds since the epoch (34 bits).
565+func (g *commitGraph) commitTime(pos uint32) (int64, error) {
566+ _, rec, err := g.record(pos)
567+ if err != nil {
568+ return 0, err
569+ }
570+ return graphRecordTime(rec), nil
571+}
572+
573+func graphRecordTime(rec []byte) int64 {
574+ hi := uint64(binary.BigEndian.Uint32(rec[graphHashSize+8:]) & 3)
575+ lo := uint64(binary.BigEndian.Uint32(rec[graphHashSize+12:]))
576+ return int64(hi<<32 | lo)
577+}
578+
579+// topoLevel returns the topological level (generation number v1) of pos;
580+// 0 means "not computed".
581+func (g *commitGraph) topoLevel(pos uint32) (uint32, error) {
582+ _, rec, err := g.record(pos)
583+ if err != nil {
584+ return 0, err
585+ }
586+ return binary.BigEndian.Uint32(rec[graphHashSize+8:]) >> 2, nil
587+}
588+
589+// generation returns the corrected commit date of pos when
590+// hasGenerationV2, else its topological level. In both cases a commit has a
591+// strictly greater generation than each of its parents.
592+func (g *commitGraph) generation(pos uint32) (uint64, error) {
593+ l, rec, err := g.record(pos)
594+ if err != nil {
595+ return 0, err
596+ }
597+ if !g.genV2 {
598+ return uint64(binary.BigEndian.Uint32(rec[graphHashSize+8:]) >> 2), nil
599+ }
600+ _, idx := g.layerOf(pos)
601+ v := binary.BigEndian.Uint32(l.gda2[int(idx)*4:])
602+ offset := uint64(v)
603+ if v&graphGenOverflow != 0 {
604+ oi := uint64(v & graphPosMask)
605+ if (oi+1)*8 > uint64(len(l.gdo2)) {
606+ return 0, graphCorruptf("GDA2 overflow index %d beyond GDO2 (%d bytes)", oi, len(l.gdo2))
607+ }
608+ offset = binary.BigEndian.Uint64(l.gdo2[oi*8:])
609+ if offset >= 1<<62 {
610+ return 0, graphCorruptf("GDO2 offset %d out of range", offset)
611+ }
612+ }
613+ return uint64(graphRecordTime(rec)) + offset, nil
614+}
615+
616+// appendParents appends the parent positions of pos, in commit order, to dst
617+// and returns it. It allocates only if dst has to grow (octopus merges).
618+func (g *commitGraph) appendParents(dst []uint32, pos uint32) ([]uint32, error) {
619+ l, rec, err := g.record(pos)
620+ if err != nil {
621+ return dst, err
622+ }
623+ // Parents are in this layer or in a lower one.
624+ limit := l.base + l.n
625+ p1 := binary.BigEndian.Uint32(rec[graphHashSize:])
626+ p2 := binary.BigEndian.Uint32(rec[graphHashSize+4:])
627+ if p1 == graphParentNone {
628+ if p2 != graphParentNone {
629+ return dst, graphCorruptf("commit %d has a second parent but no first", pos)
630+ }
631+ return dst, nil
632+ }
633+ if p1 >= limit {
634+ return dst, graphCorruptf("commit %d: parent %d out of range", pos, p1)
635+ }
636+ dst = append(dst, p1)
637+ switch {
638+ case p2 == graphParentNone:
639+ return dst, nil
640+ case p2&graphParentExtra == 0:
641+ if p2 >= limit {
642+ return dst, graphCorruptf("commit %d: parent %d out of range", pos, p2)
643+ }
644+ return append(dst, p2), nil
645+ }
646+ count := 1
647+ for e := uint64(p2 & graphPosMask); ; e++ {
648+ if (e+1)*4 > uint64(len(l.edge)) {
649+ return dst, graphCorruptf("commit %d: EDGE list overruns the chunk", pos)
650+ }
651+ v := binary.BigEndian.Uint32(l.edge[e*4:])
652+ p := v & graphPosMask
653+ if p >= limit {
654+ return dst, graphCorruptf("commit %d: parent %d out of range", pos, p)
655+ }
656+ if count++; count > graphMaxParents {
657+ return dst, graphCorruptf("commit %d: too many parents", pos)
658+ }
659+ dst = append(dst, p)
660+ if v&graphEdgeLast != 0 {
661+ return dst, nil
662+ }
663+ }
664+}
665+
666+// graphFileStamp identifies a version of a file for change detection.
667+type graphFileStamp struct {
668+ exists bool
669+ size int64
670+ mtime int64
671+ ino uint64
672+}
673+
674+// graphStamp identifies the on-disk commit-graph state: the single file and
675+// the chain file (layers are immutable and named by checksum).
676+type graphStamp struct{ single, chain graphFileStamp }
677+
678+func statGraphFile(path string) graphFileStamp {
679+ st, err := os.Stat(path)
680+ if err != nil {
681+ return graphFileStamp{}
682+ }
683+ s := graphFileStamp{exists: true, size: st.Size(), mtime: st.ModTime().UnixNano()}
684+ if sys, ok := st.Sys().(*syscall.Stat_t); ok {
685+ s.ino = uint64(sys.Ino) //nolint:unconvert // width differs across platforms
686+ }
687+ return s
688+}
689+
690+// statGraph returns the stamp of the commit-graph files of objectsDir.
691+// Comparing two stamps tells whether openCommitGraph may give another result.
692+func statGraph(objectsDir string) graphStamp {
693+ info := filepath.Join(objectsDir, "info")
694+ return graphStamp{
695+ single: statGraphFile(filepath.Join(info, "commit-graph")),
696+ chain: statGraphFile(filepath.Join(info, "commit-graphs", "commit-graph-chain")),
697+ }
698+}
@@ -0,0 +1,698 @@
1+// SPDX-License-Identifier: MIT
2+
3+package git
4+
5+import (
6+ "bytes"
7+ "encoding/binary"
8+ "encoding/hex"
9+ "errors"
10+ "fmt"
11+ "io"
12+ "io/fs"
13+ "math"
14+ "os"
15+ "path/filepath"
16+ "sync/atomic"
17+ "syscall"
18+)
19+
20+// This file reads Git commit-graph files (gitformat-commit-graph): a single
21+// objects/info/commit-graph or a split chain
22+// objects/info/commit-graphs/commit-graph-chain plus graph-<hash>.graph
23+// layers. Files are mmapped read-only and never written.
24+//
25+// Lifecycle. openCommitGraph returns a graph holding one reference (the
26+// owner's). A reader that may outlive a concurrent reopen calls retain
27+// (which fails once the graph is closed) and release when done. The owner
28+// calls close when it replaces the graph. The mappings are unmapped when the
29+// last reference is dropped; no accessor may be used after that. Layers are
30+// immutable files named after their checksum, so a concurrent
31+// "git commit-graph write" never changes a mapped file; it can only delete
32+// it, which does not affect an existing mapping.
33+//
34+// Positions are global across the chain: the commits of the base layer come
35+// first, then those of the next layer, and so on. Within a layer commits are
36+// sorted by object id.
37+//
38+// Every offset read from a file is bounds-checked. Structural problems found
39+// when opening return an error wrapping errGraphCorrupt; per-commit problems
40+// (edge list, overflow generation) are reported by the accessor that meets
41+// them. No accessor panics on malformed input.
42+
43+var (
44+ // errGraphCorrupt wraps every error caused by malformed graph content.
45+ // After merging with the core it becomes an alias of (or is wrapped
46+ // with) ErrCorrupt.
47+ errGraphCorrupt = errors.New("corrupt commit-graph")
48+ // errGraphUnsupported wraps valid graphs this reader does not handle
49+ // (SHA-256, unknown format version).
50+ errGraphUnsupported = errors.New("unsupported commit-graph")
51+ // errNoGraph means the repository has no commit-graph.
52+ errNoGraph = errors.New("no commit-graph")
53+ // errGraphPosition is returned for a position >= numCommits.
54+ errGraphPosition = errors.New("commit-graph position out of range")
55+)
56+
57+func graphCorruptf(format string, args ...any) error {
58+ return fmt.Errorf("%w: %s", errGraphCorrupt, fmt.Sprintf(format, args...))
59+}
60+
61+const (
62+ graphHashSize = 20
63+ graphHeaderSize = 8
64+ graphChunkEntrySize = 12
65+ graphCommitSize = graphHashSize + 16 // tree, 2 parents, level+time
66+ graphFanoutSize = 256 * 4
67+
68+ graphParentNone = 0x70000000 // no parent in this slot
69+ graphParentExtra = 0x80000000 // second slot: index into the EDGE chunk
70+ graphEdgeLast = 0x80000000 // EDGE entry: last parent of the commit
71+ graphPosMask = 0x7fffffff
72+ graphGenOverflow = 0x80000000 // GDA2 entry: index into GDO2
73+
74+ graphMaxLayers = 256 // the base count is a single byte
75+ graphMaxParents = 1 << 13
76+ graphMaxChain = graphMaxLayers * (2*graphHashSize + 1)
77+
78+ chunkOIDF = 'O'<<24 | 'I'<<16 | 'D'<<8 | 'F'
79+ chunkOIDL = 'O'<<24 | 'I'<<16 | 'D'<<8 | 'L'
80+ chunkCDAT = 'C'<<24 | 'D'<<16 | 'A'<<8 | 'T'
81+ chunkEDGE = 'E'<<24 | 'D'<<16 | 'G'<<8 | 'E'
82+ chunkGDA2 = 'G'<<24 | 'D'<<16 | 'A'<<8 | '2'
83+ chunkGDO2 = 'G'<<24 | 'D'<<16 | 'O'<<8 | '2'
84+ chunkBIDX = 'B'<<24 | 'I'<<16 | 'D'<<8 | 'X'
85+ chunkBDAT = 'B'<<24 | 'D'<<16 | 'A'<<8 | 'T'
86+ chunkBASE = 'B'<<24 | 'A'<<16 | 'S'<<8 | 'E'
87+)
88+
89+// graphLayer is one commit-graph file.
90+type graphLayer struct {
91+ data []byte
92+ numBase int // base graphs declared in the header
93+ base uint32 // global position of the first commit
94+ n uint32 // commits in this layer
95+ fanout []byte
96+ oidl []byte
97+ cdat []byte
98+ edge []byte
99+ gda2 []byte
100+ gdo2 []byte
101+ bidx []byte
102+ bdat []byte
103+ baseList []byte
104+
105+ // Changed-path Bloom filter settings, valid when hasBloom.
106+ hasBloom bool
107+ bloomVersion uint32
108+ bloomHashes uint32
109+ bloomBits uint32
110+}
111+
112+// commitGraph is a (possibly split) commit-graph.
113+type commitGraph struct {
114+ layers []graphLayer // base first
115+ total uint32
116+ genV2 bool // every layer has GDA2: generation() is a corrected date
117+
118+ refs atomic.Int64
119+ unmaps [][]byte
120+}
121+
122+// graphMmap maps a file read-only. The returned slice must be released with
123+// syscall.Munmap.
124+func graphMmap(path string) ([]byte, error) {
125+ f, err := os.Open(path)
126+ if err != nil {
127+ return nil, err
128+ }
129+ defer func() { _ = f.Close() }()
130+ st, err := f.Stat()
131+ if err != nil {
132+ return nil, err
133+ }
134+ if !st.Mode().IsRegular() {
135+ return nil, graphCorruptf("%s: not a regular file", filepath.Base(path))
136+ }
137+ size := st.Size()
138+ if size < graphHeaderSize+graphChunkEntrySize+graphHashSize {
139+ return nil, graphCorruptf("%s: file too short (%d bytes)", filepath.Base(path), size)
140+ }
141+ if size > math.MaxInt32 {
142+ return nil, graphCorruptf("%s: file too large (%d bytes)", filepath.Base(path), size)
143+ }
144+ data, err := syscall.Mmap(int(f.Fd()), 0, int(size), syscall.PROT_READ, syscall.MAP_SHARED)
145+ if err != nil {
146+ return nil, fmt.Errorf("mmap %s: %w", filepath.Base(path), err)
147+ }
148+ _ = syscall.Madvise(data, syscall.MADV_RANDOM)
149+ return data, nil
150+}
151+
152+// openCommitGraph opens the commit-graph of the objects directory objectsDir:
153+// objects/info/commit-graph if present, else the split chain. It returns
154+// errNoGraph when there is none. The result holds one reference.
155+func openCommitGraph(objectsDir string) (*commitGraph, error) {
156+ infoDir := filepath.Join(objectsDir, "info")
157+ data, err := graphMmap(filepath.Join(infoDir, "commit-graph"))
158+ switch {
159+ case err == nil:
160+ l, perr := parseGraphLayer(data)
161+ if perr != nil {
162+ _ = syscall.Munmap(data)
163+ return nil, fmt.Errorf("commit-graph: %w", perr)
164+ }
165+ g, aerr := assembleGraph([]*graphLayer{l}, nil)
166+ if aerr != nil {
167+ _ = syscall.Munmap(data)
168+ return nil, fmt.Errorf("commit-graph: %w", aerr)
169+ }
170+ g.unmaps = [][]byte{data}
171+ return g, nil
172+ case errors.Is(err, fs.ErrNotExist):
173+ return openCommitGraphChain(infoDir)
174+ default:
175+ return nil, fmt.Errorf("commit-graph: %w", err)
176+ }
177+}
178+
179+func openCommitGraphChain(infoDir string) (*commitGraph, error) {
180+ dir := filepath.Join(infoDir, "commit-graphs")
181+ names, err := readGraphChain(filepath.Join(dir, "commit-graph-chain"))
182+ if err != nil {
183+ return nil, err
184+ }
185+ layers := make([]*graphLayer, 0, len(names))
186+ maps := make([][]byte, 0, len(names))
187+ cleanup := func() {
188+ for _, m := range maps {
189+ _ = syscall.Munmap(m)
190+ }
191+ }
192+ for _, name := range names {
193+ file := "graph-" + hex.EncodeToString(name[:]) + ".graph"
194+ data, err := graphMmap(filepath.Join(dir, file))
195+ if err != nil {
196+ cleanup()
197+ return nil, fmt.Errorf("commit-graph chain layer: %w", err)
198+ }
199+ maps = append(maps, data)
200+ l, err := parseGraphLayer(data)
201+ if err != nil {
202+ cleanup()
203+ return nil, fmt.Errorf("commit-graph layer %s: %w", file, err)
204+ }
205+ layers = append(layers, l)
206+ }
207+ g, err := assembleGraph(layers, names)
208+ if err != nil {
209+ cleanup()
210+ return nil, fmt.Errorf("commit-graph chain: %w", err)
211+ }
212+ g.unmaps = maps
213+ return g, nil
214+}
215+
216+// readGraphChain reads commit-graph-chain.
217+func readGraphChain(path string) ([][graphHashSize]byte, error) {
218+ f, err := os.Open(path)
219+ if err != nil {
220+ if errors.Is(err, fs.ErrNotExist) {
221+ return nil, errNoGraph
222+ }
223+ return nil, fmt.Errorf("commit-graph chain: %w", err)
224+ }
225+ defer func() { _ = f.Close() }()
226+ raw, err := io.ReadAll(io.LimitReader(f, graphMaxChain+1))
227+ if err != nil {
228+ return nil, fmt.Errorf("commit-graph chain: %w", err)
229+ }
230+ return parseGraphChain(raw)
231+}
232+
233+// parseGraphChain parses the content of commit-graph-chain: one lowercase
234+// hex checksum per line, base layer first.
235+func parseGraphChain(raw []byte) ([][graphHashSize]byte, error) {
236+ if len(raw) == 0 {
237+ return nil, errNoGraph
238+ }
239+ const line = 2*graphHashSize + 1
240+ if len(raw) > graphMaxChain || len(raw)%line != 0 {
241+ return nil, graphCorruptf("commit-graph-chain: bad size %d", len(raw))
242+ }
243+ names := make([][graphHashSize]byte, len(raw)/line)
244+ for i := range names {
245+ row := raw[i*line : (i+1)*line]
246+ if row[line-1] != '\n' {
247+ return nil, graphCorruptf("commit-graph-chain: line %d not terminated", i+1)
248+ }
249+ if _, err := hex.Decode(names[i][:], row[:line-1]); err != nil {
250+ return nil, graphCorruptf("commit-graph-chain: line %d: %v", i+1, err)
251+ }
252+ }
253+ return names, nil
254+}
255+
256+// parseGraphLayer validates the header, the chunk table and the sizes of the
257+// chunks it understands. It does not verify the checksum nor the content of
258+// individual records.
259+func parseGraphLayer(data []byte) (*graphLayer, error) {
260+ if len(data) < graphHeaderSize+graphChunkEntrySize+graphHashSize {
261+ return nil, graphCorruptf("file too short (%d bytes)", len(data))
262+ }
263+ if string(data[:4]) != "CGPH" {
264+ return nil, graphCorruptf("bad signature %q", data[:4])
265+ }
266+ if data[4] != 1 {
267+ return nil, fmt.Errorf("%w: version %d", errGraphUnsupported, data[4])
268+ }
269+ switch data[5] {
270+ case 1:
271+ case 2:
272+ return nil, fmt.Errorf("%w: SHA-256", errGraphUnsupported)
273+ default:
274+ return nil, graphCorruptf("unknown hash version %d", data[5])
275+ }
276+ nChunks := int(data[6])
277+ l := &graphLayer{data: data, numBase: int(data[7])}
278+
279+ tableEnd := uint64(graphHeaderSize + (nChunks+1)*graphChunkEntrySize)
280+ bodyEnd := uint64(len(data) - graphHashSize)
281+ if tableEnd > bodyEnd {
282+ return nil, graphCorruptf("chunk table exceeds file")
283+ }
284+
285+ var ids [256]uint32
286+ prevOff := tableEnd
287+ for i := 0; i <= nChunks; i++ {
288+ ent := data[graphHeaderSize+i*graphChunkEntrySize:]
289+ id := binary.BigEndian.Uint32(ent)
290+ off := binary.BigEndian.Uint64(ent[4:])
291+ if (id == 0) != (i == nChunks) {
292+ return nil, graphCorruptf("chunk table: bad terminator at entry %d", i)
293+ }
294+ if off < prevOff || off > bodyEnd {
295+ return nil, graphCorruptf("chunk table: bad offset %d for entry %d", off, i)
296+ }
297+ if i > 0 {
298+ // Close the previous chunk: it spans [its offset, this offset).
299+ if err := l.setChunk(ids[i-1], data[prevOff:off]); err != nil {
300+ return nil, err
301+ }
302+ }
303+ for j := 0; j < i; j++ {
304+ if ids[j] == id {
305+ return nil, graphCorruptf("chunk table: duplicate chunk %08x", id)
306+ }
307+ }
308+ ids[i] = id
309+ prevOff = off
310+ }
311+
312+ if err := l.validate(); err != nil {
313+ return nil, err
314+ }
315+ return l, nil
316+}
317+
318+// setChunk records chunk id. Unknown chunks (and the obsolete GDAT/GDOV) are
319+// ignored.
320+func (l *graphLayer) setChunk(id uint32, b []byte) error {
321+ switch id {
322+ case chunkOIDF:
323+ l.fanout = b
324+ case chunkOIDL:
325+ l.oidl = b
326+ case chunkCDAT:
327+ l.cdat = b
328+ case chunkEDGE:
329+ l.edge = b
330+ case chunkGDA2:
331+ l.gda2 = b
332+ case chunkGDO2:
333+ l.gdo2 = b
334+ case chunkBIDX:
335+ l.bidx = b
336+ case chunkBDAT:
337+ l.bdat = b
338+ case chunkBASE:
339+ l.baseList = b
340+ }
341+ return nil
342+}
343+
344+func (l *graphLayer) validate() error {
345+ if len(l.fanout) != graphFanoutSize {
346+ return graphCorruptf("OIDF chunk missing or has size %d", len(l.fanout))
347+ }
348+ var prev uint32
349+ for i := 0; i < 256; i++ {
350+ v := binary.BigEndian.Uint32(l.fanout[i*4:])
351+ if v < prev {
352+ return graphCorruptf("OIDF chunk not monotonic at %d", i)
353+ }
354+ prev = v
355+ }
356+ if prev >= graphParentNone {
357+ return graphCorruptf("too many commits (%d)", prev)
358+ }
359+ l.n = prev
360+ n := uint64(prev)
361+ if l.oidl == nil || uint64(len(l.oidl)) != n*graphHashSize {
362+ return graphCorruptf("OIDL chunk missing or has size %d for %d commits", len(l.oidl), n)
363+ }
364+ if l.cdat == nil || uint64(len(l.cdat)) != n*graphCommitSize {
365+ return graphCorruptf("CDAT chunk missing or has size %d for %d commits", len(l.cdat), n)
366+ }
367+ if len(l.edge)%4 != 0 {
368+ return graphCorruptf("EDGE chunk size %d not a multiple of 4", len(l.edge))
369+ }
370+ if l.gda2 != nil && uint64(len(l.gda2)) != n*4 {
371+ return graphCorruptf("GDA2 chunk has size %d for %d commits", len(l.gda2), n)
372+ }
373+ if len(l.gdo2)%8 != 0 {
374+ return graphCorruptf("GDO2 chunk size %d not a multiple of 8", len(l.gdo2))
375+ }
376+ if l.numBase > 0 && uint64(len(l.baseList)) != uint64(l.numBase)*graphHashSize {
377+ return graphCorruptf("BASE chunk has size %d for %d base graphs", len(l.baseList), l.numBase)
378+ }
379+ if l.numBase == 0 && len(l.baseList) != 0 {
380+ return graphCorruptf("BASE chunk present without base graphs")
381+ }
382+ // Bloom filters need both chunks; with only one of them git ignores it.
383+ if l.bidx != nil && l.bdat != nil {
384+ if uint64(len(l.bidx)) != n*4 {
385+ return graphCorruptf("BIDX chunk has size %d for %d commits", len(l.bidx), n)
386+ }
387+ if len(l.bdat) < bloomHeaderSize {
388+ return graphCorruptf("BDAT chunk too short (%d bytes)", len(l.bdat))
389+ }
390+ l.bloomVersion = binary.BigEndian.Uint32(l.bdat)
391+ l.bloomHashes = binary.BigEndian.Uint32(l.bdat[4:])
392+ l.bloomBits = binary.BigEndian.Uint32(l.bdat[8:])
393+ // Unknown versions are ignored like git does; an absurd hash
394+ // count would only make queries slow.
395+ l.hasBloom = (l.bloomVersion == 1 || l.bloomVersion == 2) &&
396+ l.bloomHashes >= 1 && l.bloomHashes <= bloomMaxHashes
397+ }
398+ return nil
399+}
400+
401+// assembleGraph builds a graph from validated layers, base first. names are
402+// the chain entries (checksums) of the layers, nil for a single file.
403+func assembleGraph(layers []*graphLayer, names [][graphHashSize]byte) (*commitGraph, error) {
404+ if len(layers) == 0 {
405+ return nil, errNoGraph
406+ }
407+ if len(layers) > graphMaxLayers {
408+ return nil, graphCorruptf("too many layers (%d)", len(layers))
409+ }
410+ if names != nil && len(names) != len(layers) {
411+ return nil, graphCorruptf("%d layers for %d chain entries", len(layers), len(names))
412+ }
413+ g := &commitGraph{layers: make([]graphLayer, len(layers)), genV2: true}
414+ var total uint64
415+ for k, l := range layers {
416+ if l.numBase != k {
417+ return nil, graphCorruptf("layer %d declares %d base graphs", k, l.numBase)
418+ }
419+ for i := 0; i < k; i++ {
420+ if !bytes.Equal(l.baseList[i*graphHashSize:(i+1)*graphHashSize], names[i][:]) {
421+ return nil, graphCorruptf("layer %d: base graph %d does not match the chain", k, i)
422+ }
423+ }
424+ l.base = uint32(total)
425+ total += uint64(l.n)
426+ if total >= graphParentNone {
427+ return nil, graphCorruptf("too many commits in chain")
428+ }
429+ if l.gda2 == nil {
430+ g.genV2 = false
431+ }
432+ g.layers[k] = *l
433+ }
434+ g.total = uint32(total)
435+ g.refs.Store(1)
436+ return g, nil
437+}
438+
439+// retain adds a reference. It reports false when the graph is already closed.
440+func (g *commitGraph) retain() bool {
441+ for {
442+ n := g.refs.Load()
443+ if n <= 0 {
444+ return false
445+ }
446+ if g.refs.CompareAndSwap(n, n+1) {
447+ return true
448+ }
449+ }
450+}
451+
452+// release drops a reference; the last one unmaps the files.
453+func (g *commitGraph) release() error {
454+ switch n := g.refs.Add(-1); {
455+ case n > 0:
456+ return nil
457+ case n == 0:
458+ var first error
459+ for _, m := range g.unmaps {
460+ if err := syscall.Munmap(m); err != nil && first == nil {
461+ first = err
462+ }
463+ }
464+ g.unmaps = nil
465+ return first
466+ default:
467+ g.refs.Add(1)
468+ return errors.New("commit-graph released too many times")
469+ }
470+}
471+
472+// close drops the owner's reference.
473+func (g *commitGraph) close() error { return g.release() }
474+
475+// numCommits is the number of commits in the whole chain.
476+func (g *commitGraph) numCommits() uint32 { return g.total }
477+
478+// numLayers is the number of files in the chain.
479+func (g *commitGraph) numLayers() int { return len(g.layers) }
480+
481+// hasGenerationV2 reports whether generation returns corrected commit dates
482+// (GDA2 present in every layer, as git requires) rather than topological
483+// levels. The two must not be compared with each other.
484+func (g *commitGraph) hasGenerationV2() bool { return g.genV2 }
485+
486+// layerOf finds the layer holding the global position pos and the index of
487+// the commit inside it.
488+func (g *commitGraph) layerOf(pos uint32) (*graphLayer, uint32) {
489+ if pos >= g.total {
490+ return nil, 0
491+ }
492+ for i := len(g.layers) - 1; i >= 0; i-- {
493+ if l := &g.layers[i]; pos >= l.base {
494+ return l, pos - l.base
495+ }
496+ }
497+ return nil, 0
498+}
499+
500+// find looks up id in one layer using the fanout table.
501+func (l *graphLayer) find(id *[graphHashSize]byte) (uint32, bool) {
502+ b := int(id[0])
503+ var lo uint32
504+ if b > 0 {
505+ lo = binary.BigEndian.Uint32(l.fanout[(b-1)*4:])
506+ }
507+ hi := binary.BigEndian.Uint32(l.fanout[b*4:]) // <= l.n, checked when opening
508+ for lo < hi {
509+ mid := lo + (hi-lo)/2
510+ off := int(mid) * graphHashSize
511+ switch c := bytes.Compare(l.oidl[off:off+graphHashSize], id[:]); {
512+ case c == 0:
513+ return mid, true
514+ case c < 0:
515+ lo = mid + 1
516+ default:
517+ hi = mid
518+ }
519+ }
520+ return 0, false
521+}
522+
523+// lookup returns the global position of commit id. It does not allocate.
524+func (g *commitGraph) lookup(id [graphHashSize]byte) (uint32, bool) {
525+ for i := len(g.layers) - 1; i >= 0; i-- {
526+ l := &g.layers[i]
527+ if idx, ok := l.find(&id); ok {
528+ return l.base + idx, true
529+ }
530+ }
531+ return 0, false
532+}
533+
534+// commitID returns the object id at pos.
535+func (g *commitGraph) commitID(pos uint32) (id [graphHashSize]byte, err error) {
536+ l, idx := g.layerOf(pos)
537+ if l == nil {
538+ return id, errGraphPosition
539+ }
540+ copy(id[:], l.oidl[int(idx)*graphHashSize:])
541+ return id, nil
542+}
543+
544+// record returns the CDAT record of pos.
545+func (g *commitGraph) record(pos uint32) (*graphLayer, []byte, error) {
546+ l, idx := g.layerOf(pos)
547+ if l == nil {
548+ return nil, nil, errGraphPosition
549+ }
550+ off := int(idx) * graphCommitSize
551+ return l, l.cdat[off : off+graphCommitSize], nil
552+}
553+
554+// rootTree returns the root tree id of the commit at pos.
555+func (g *commitGraph) rootTree(pos uint32) (id [graphHashSize]byte, err error) {
556+ _, rec, err := g.record(pos)
557+ if err != nil {
558+ return id, err
559+ }
560+ copy(id[:], rec)
561+ return id, nil
562+}
563+
564+// commitTime returns the commit time in seconds since the epoch (34 bits).
565+func (g *commitGraph) commitTime(pos uint32) (int64, error) {
566+ _, rec, err := g.record(pos)
567+ if err != nil {
568+ return 0, err
569+ }
570+ return graphRecordTime(rec), nil
571+}
572+
573+func graphRecordTime(rec []byte) int64 {
574+ hi := uint64(binary.BigEndian.Uint32(rec[graphHashSize+8:]) & 3)
575+ lo := uint64(binary.BigEndian.Uint32(rec[graphHashSize+12:]))
576+ return int64(hi<<32 | lo)
577+}
578+
579+// topoLevel returns the topological level (generation number v1) of pos;
580+// 0 means "not computed".
581+func (g *commitGraph) topoLevel(pos uint32) (uint32, error) {
582+ _, rec, err := g.record(pos)
583+ if err != nil {
584+ return 0, err
585+ }
586+ return binary.BigEndian.Uint32(rec[graphHashSize+8:]) >> 2, nil
587+}
588+
589+// generation returns the corrected commit date of pos when
590+// hasGenerationV2, else its topological level. In both cases a commit has a
591+// strictly greater generation than each of its parents.
592+func (g *commitGraph) generation(pos uint32) (uint64, error) {
593+ l, rec, err := g.record(pos)
594+ if err != nil {
595+ return 0, err
596+ }
597+ if !g.genV2 {
598+ return uint64(binary.BigEndian.Uint32(rec[graphHashSize+8:]) >> 2), nil
599+ }
600+ _, idx := g.layerOf(pos)
601+ v := binary.BigEndian.Uint32(l.gda2[int(idx)*4:])
602+ offset := uint64(v)
603+ if v&graphGenOverflow != 0 {
604+ oi := uint64(v & graphPosMask)
605+ if (oi+1)*8 > uint64(len(l.gdo2)) {
606+ return 0, graphCorruptf("GDA2 overflow index %d beyond GDO2 (%d bytes)", oi, len(l.gdo2))
607+ }
608+ offset = binary.BigEndian.Uint64(l.gdo2[oi*8:])
609+ if offset >= 1<<62 {
610+ return 0, graphCorruptf("GDO2 offset %d out of range", offset)
611+ }
612+ }
613+ return uint64(graphRecordTime(rec)) + offset, nil
614+}
615+
616+// appendParents appends the parent positions of pos, in commit order, to dst
617+// and returns it. It allocates only if dst has to grow (octopus merges).
618+func (g *commitGraph) appendParents(dst []uint32, pos uint32) ([]uint32, error) {
619+ l, rec, err := g.record(pos)
620+ if err != nil {
621+ return dst, err
622+ }
623+ // Parents are in this layer or in a lower one.
624+ limit := l.base + l.n
625+ p1 := binary.BigEndian.Uint32(rec[graphHashSize:])
626+ p2 := binary.BigEndian.Uint32(rec[graphHashSize+4:])
627+ if p1 == graphParentNone {
628+ if p2 != graphParentNone {
629+ return dst, graphCorruptf("commit %d has a second parent but no first", pos)
630+ }
631+ return dst, nil
632+ }
633+ if p1 >= limit {
634+ return dst, graphCorruptf("commit %d: parent %d out of range", pos, p1)
635+ }
636+ dst = append(dst, p1)
637+ switch {
638+ case p2 == graphParentNone:
639+ return dst, nil
640+ case p2&graphParentExtra == 0:
641+ if p2 >= limit {
642+ return dst, graphCorruptf("commit %d: parent %d out of range", pos, p2)
643+ }
644+ return append(dst, p2), nil
645+ }
646+ count := 1
647+ for e := uint64(p2 & graphPosMask); ; e++ {
648+ if (e+1)*4 > uint64(len(l.edge)) {
649+ return dst, graphCorruptf("commit %d: EDGE list overruns the chunk", pos)
650+ }
651+ v := binary.BigEndian.Uint32(l.edge[e*4:])
652+ p := v & graphPosMask
653+ if p >= limit {
654+ return dst, graphCorruptf("commit %d: parent %d out of range", pos, p)
655+ }
656+ if count++; count > graphMaxParents {
657+ return dst, graphCorruptf("commit %d: too many parents", pos)
658+ }
659+ dst = append(dst, p)
660+ if v&graphEdgeLast != 0 {
661+ return dst, nil
662+ }
663+ }
664+}
665+
666+// graphFileStamp identifies a version of a file for change detection.
667+type graphFileStamp struct {
668+ exists bool
669+ size int64
670+ mtime int64
671+ ino uint64
672+}
673+
674+// graphStamp identifies the on-disk commit-graph state: the single file and
675+// the chain file (layers are immutable and named by checksum).
676+type graphStamp struct{ single, chain graphFileStamp }
677+
678+func statGraphFile(path string) graphFileStamp {
679+ st, err := os.Stat(path)
680+ if err != nil {
681+ return graphFileStamp{}
682+ }
683+ s := graphFileStamp{exists: true, size: st.Size(), mtime: st.ModTime().UnixNano()}
684+ if sys, ok := st.Sys().(*syscall.Stat_t); ok {
685+ s.ino = uint64(sys.Ino) //nolint:unconvert // width differs across platforms
686+ }
687+ return s
688+}
689+
690+// statGraph returns the stamp of the commit-graph files of objectsDir.
691+// Comparing two stamps tells whether openCommitGraph may give another result.
692+func statGraph(objectsDir string) graphStamp {
693+ info := filepath.Join(objectsDir, "info")
694+ return graphStamp{
695+ single: statGraphFile(filepath.Join(info, "commit-graph")),
696+ chain: statGraphFile(filepath.Join(info, "commit-graphs", "commit-graph-chain")),
697+ }
698+}

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 1

Tags 2

↑↓ moveenter openesc close