1// SPDX-License-Identifier: MIT
2
3package git
4
5import (
6 "encoding/binary"
7 "errors"
8 "io/fs"
9 "os"
10 "slices"
11 "strings"
12 "sync/atomic"
13 "time"
14)
15
16// fileSig identifies a version of a file or directory by stat data, and
17// remembers when that data was read.
18type fileSig struct {
19 exists bool
20 size int64
21 mtime int64 // unix nanoseconds
22 ino uint64
23 read int64 // unix nanoseconds: when the file was stat'ed or read
24}
25
26// sigOf returns the signature of fi, which was obtained at read (unix
27// nanoseconds). Take read before reading the content: a change that happens
28// later is then never mistaken for the version that was read.
29func sigOf(fi os.FileInfo, read int64) fileSig {
30 return fileSig{exists: true, size: fi.Size(), mtime: fi.ModTime().UnixNano(), ino: fileIno(fi), read: read}
31}
32
33func statSig(path string) (fileSig, error) {
34 read := time.Now().UnixNano()
35 fi, err := os.Stat(path)
36 if errors.Is(err, fs.ErrNotExist) {
37 return fileSig{read: read}, nil
38 }
39 if err != nil {
40 return fileSig{}, err
41 }
42 return sigOf(fi, read), nil
43}
44
45// racyWindow is how long after its modification time a file may still change
46// without its stat data changing, because of coarse file system clocks.
47const racyWindow = 2 * time.Second
48
49// sameStat reports whether a and b have the same stat data, whenever they
50// were read.
51func (a fileSig) sameStat(b fileSig) bool {
52 return a.exists == b.exists && a.size == b.size && a.mtime == b.mtime && a.ino == b.ino
53}
54
55// same reports whether b is the same version as a and can be trusted to be
56// unchanged. This is Git's racy rule: a was read at a.read, and a write that
57// follows within the clock granularity of the previous modification leaves
58// the stat data as it was. So the stat data only proves that nothing
59// changed when the file was read more than racyWindow after its mtime.
60func (a fileSig) same(b fileSig) bool {
61 if !a.sameStat(b) {
62 return false
63 }
64 return !a.exists || a.read-a.mtime > int64(racyWindow)
65}
66
67func ignoreMissing(err error) error {
68 if errors.Is(err, fs.ErrNotExist) {
69 return nil
70 }
71 return err
72}
73
74// snapshot is an immutable list of packs, shared by readers. It is released
75// by each reader and by the Repo; the last release drops its packs.
76type snapshot struct {
77 refs atomic.Int32
78 packs []*pack // newest first
79 dirSig fileSig // of objects/pack when the list was built
80}
81
82func newSnapshot(packs []*pack) *snapshot {
83 s := &snapshot{packs: packs}
84 s.refs.Store(1)
85 return s
86}
87
88func (s *snapshot) release() {
89 if s.refs.Add(-1) == 0 {
90 for _, p := range s.packs {
91 p.release()
92 }
93 }
94}
95
96// find looks id up in the packs.
97func (s *snapshot) find(id *ObjectID) (*pack, uint64, error) {
98 for _, p := range s.packs {
99 if pos, ok := p.idx.find(id); ok {
100 off, err := p.idx.offset(pos)
101 if err != nil {
102 return nil, 0, wrapPack(p.name, err)
103 }
104 return p, off, nil
105 }
106 }
107 return nil, 0, nil
108}
109
110// acquire returns the current snapshot, which the caller must release.
111func (r *Repo) acquire() (*snapshot, error) {
112 for {
113 s := r.snap.Load()
114 if s == nil {
115 return nil, ErrClosed
116 }
117 n := s.refs.Load()
118 if n > 0 && s.refs.CompareAndSwap(n, n+1) {
119 return s, nil
120 }
121 }
122}
123
124func (r *Repo) packDir() string { return r.dir + "/objects/pack" }
125
126// reloadPacks rescans objects/pack and installs a new snapshot when the
127// directory changed, or always with force. It reports whether the list of
128// packs changed.
129func (r *Repo) reloadPacks(force bool) (bool, error) {
130 r.mu.Lock()
131 defer r.mu.Unlock()
132 if r.closed {
133 return false, ErrClosed
134 }
135 cur := r.snap.Load()
136 dsig, err := r.paths.packDir.sig()
137 if err != nil {
138 return false, err
139 }
140 if !force && cur.dirSig.same(dsig) {
141 return false, nil
142 }
143 dir := r.packDir()
144 entries, err := os.ReadDir(dir)
145 if err != nil && !errors.Is(err, fs.ErrNotExist) {
146 return false, err
147 }
148 var idxNames []string
149 hasMidx := false
150 for _, e := range entries {
151 n := e.Name()
152 switch {
153 case n == "multi-pack-index":
154 hasMidx = true
155 case n == "multi-pack-index.d":
156 return false, unsupported("incremental multi-pack-index (objects/pack/multi-pack-index.d)")
157 case strings.HasPrefix(n, "pack-") && strings.HasSuffix(n, ".promisor"):
158 return false, unsupported("partial clone (%s)", n)
159 case strings.HasPrefix(n, "pack-") && strings.HasSuffix(n, ".idx"):
160 idxNames = append(idxNames, strings.TrimSuffix(n, ".idx"))
161 }
162 }
163 if hasMidx {
164 if err := checkMidx(dir+"/multi-pack-index", idxNames); err != nil {
165 return false, err
166 }
167 }
168
169 old := make(map[string]*pack, len(cur.packs))
170 for _, p := range cur.packs {
171 old[p.name] = p
172 }
173 var packs []*pack
174 changed := len(idxNames) != len(cur.packs)
175 for _, name := range idxNames {
176 isig, err := statSig(dir + "/" + name + ".idx")
177 if err != nil {
178 return false, err
179 }
180 if p := old[name]; p != nil && p.sig.sameStat(isig) {
181 p.retain()
182 packs = append(packs, p)
183 continue
184 }
185 p, err := openPack(dir, name, r.nextPack.Add(1), isig)
186 if err != nil {
187 for _, q := range packs {
188 q.release()
189 }
190 return false, err
191 }
192 if p == nil { // replaced while we looked
193 changed = true
194 continue
195 }
196 changed = true
197 packs = append(packs, p)
198 }
199 if len(packs) != len(cur.packs) {
200 changed = true
201 }
202 slices.SortStableFunc(packs, func(a, b *pack) int {
203 switch {
204 case a.sig.mtime > b.sig.mtime:
205 return -1
206 case a.sig.mtime < b.sig.mtime:
207 return 1
208 }
209 return strings.Compare(a.name, b.name)
210 })
211 if !changed {
212 // Same packs: keep the current snapshot (and its readers) unless the
213 // directory stat data moved on.
214 for _, p := range packs {
215 p.release()
216 }
217 if cur.dirSig.sameStat(dsig) {
218 // The listing above was made after dsig was taken: record it, so
219 // that a directory that was racy when first read stops being
220 // rescanned once its window has passed. dirSig is only used
221 // with r.mu held.
222 cur.dirSig = dsig
223 return false, nil
224 }
225 packs = cur.packs
226 for _, p := range packs {
227 p.retain()
228 }
229 }
230 ns := newSnapshot(packs)
231 ns.dirSig = dsig
232 r.snap.Store(ns)
233 cur.release()
234 return changed, nil
235}
236
237const (
238 midxHeaderLen = 12
239 midxChunkPNAM = 0x504e414d
240 maxMidxNames = 4 << 20
241)
242
243// checkMidx accepts a multi-pack-index only when every pack it lists has
244// its own .idx file: the midx is then redundant and ignored.
245func checkMidx(path string, idxNames []string) error {
246 f, err := os.Open(path)
247 if err != nil {
248 return ignoreMissing(err)
249 }
250 defer func() { _ = f.Close() }()
251 var hdr [midxHeaderLen]byte
252 if _, err := f.ReadAt(hdr[:], 0); err != nil || string(hdr[:4]) != "MIDX" {
253 return unsupported("unreadable multi-pack-index")
254 }
255 if hdr[4] != 1 || hdr[5] != 1 {
256 return unsupported("multi-pack-index version %d, object format %d", hdr[4], hdr[5])
257 }
258 chunks := int(hdr[6])
259 if hdr[7] != 0 {
260 return unsupported("incremental multi-pack-index")
261 }
262 nPacks := binary.BigEndian.Uint32(hdr[8:])
263 table := make([]byte, (chunks+1)*12)
264 if _, err := f.ReadAt(table, midxHeaderLen); err != nil {
265 return unsupported("multi-pack-index chunk table: %v", err)
266 }
267 var start, end uint64
268 for i := 0; i < chunks; i++ {
269 if binary.BigEndian.Uint32(table[i*12:]) == midxChunkPNAM {
270 start = binary.BigEndian.Uint64(table[i*12+4:])
271 end = binary.BigEndian.Uint64(table[(i+1)*12+4:])
272 break
273 }
274 }
275 if end <= start || end-start > maxMidxNames {
276 return unsupported("multi-pack-index without a usable pack name chunk")
277 }
278 names := make([]byte, end-start)
279 if _, err := f.ReadAt(names, int64(start)); err != nil {
280 return unsupported("multi-pack-index pack names: %v", err)
281 }
282 have := make(map[string]bool, len(idxNames))
283 for _, n := range idxNames {
284 have[n] = true
285 }
286 listed := uint32(0)
287 for len(names) > 0 && listed < nPacks {
288 i := 0
289 for i < len(names) && names[i] != 0 {
290 i++
291 }
292 if i == len(names) {
293 break
294 }
295 name := string(names[:i])
296 names = names[i+1:]
297 listed++
298 if name == "" { // padding
299 continue
300 }
301 name = strings.TrimSuffix(strings.TrimSuffix(name, ".idx"), ".pack")
302 if !have[name] {
303 return unsupported("multi-pack-index lists %s, which has no .idx file", truncate(name))
304 }
305 }
306 return nil
307}