1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3// Package ermitegit implements ports.RepoReader over the ermite-git library.
4//
5// The adapter wraps the gitcli reader. Each method has a route, a constant
6// below: the methods that match gitcli in the shared conformance table and
7// beat it in the benchmarks are served by ermite-git, the others (and
8// GetCommit) by the wrapped reader. A repository that ermite-git cannot read
9// (ErrUnsupported: SHA-256, reftable, alternates, replace refs...) is served
10// by the wrapped reader for every method.
11//
12// Any other error of the library is returned, never hidden by a fallback.
13// The exceptions are decisions about the data, not failures: a commit whose
14// message is not in UTF-8 (git converts it, the library does not) and an
15// object larger than the library's in-memory bound are read by the wrapped
16// reader.
17//
18// Freshness: a read that begins after a push has returned sees the push,
19// whichever path made it, because the library reads refs from disk on each
20// call (loose refs always, packed-refs after a stat) and rescans the packs
21// when an object is missing (TestFreshness proves it with no Refresh at all).
22// The adapter still calls Refresh to release replaced packs, pick up a new
23// commit-graph and notice a repository that stopped being supported: at once
24// when ermite itself wrote (the push and the maintenance tell it through
25// Refresh; a merge request merge waits for its maintenance), and as a safety
26// net at most once per Config.RefreshEvery for each repository while it is
27// read, by one reader at a time. A Refresh stats the files of the repository,
28// too much to pay on every read of a method that itself costs 2 µs.
29//
30// Memory: the cache of open repositories is bounded (LRU) and drops the ones
31// idle longer than Config.IdleTTL; each repository holds at most
32// Config.DeltaCacheSize of delta bases and reads objects up to
33// Config.MaxObjectSize into memory. gitcli's Trash tells the adapter to drop
34// a deleted repository (see Forget).
35package ermitegit
36
37import (
38 "context"
39 "errors"
40 "fmt"
41 "io"
42 "sync"
43 "time"
44
45 git "bnema.dev/ermite-git"
46 "github.com/bnema/zerowrap"
47
48 "bnema.dev/ermite/internal/domain"
49 "bnema.dev/ermite/internal/ports"
50)
51
52var (
53 _ ports.RepoReader = (*Adapter)(nil)
54 _ ports.RepoFreshener = (*Adapter)(nil)
55)
56
57// ErrClosed is returned by reads after Close.
58var ErrClosed = errors.New("ermitegit: reader is closed")
59
60// Backend says which reader serves a method.
61type Backend uint8
62
63// Backends.
64const (
65 // CLI is the git binary, through the gitcli reader.
66 CLI Backend = iota
67 // ErmiteGit is the ermite-git library.
68 ErmiteGit
69)
70
71// String returns "cli" or "ermite-git".
72func (b Backend) String() string {
73 if b == ErmiteGit {
74 return "ermite-git"
75 }
76 return "cli"
77}
78
79// Routes says which reader serves each method. GetCommit is not here: it is
80// always served by the wrapped reader.
81type Routes struct {
82 ListRefs Backend
83 // ResolveRevision is a revision that is a name or a full id.
84 ResolveRevision Backend
85 // ResolveRevisionAbbrev is a revision that is an abbreviated id.
86 ResolveRevisionAbbrev Backend
87 ResolveRefPath Backend
88 // GetTree is a directory listing without last commits.
89 GetTree Backend
90 // GetTreeLastCommits is a listing with the last commit of each entry.
91 GetTreeLastCommits Backend
92 OpenBlob Backend
93 // ListCommits is a page of history of a revision.
94 ListCommits Backend
95 // ListCommitsPath is a page of the history of a file or a directory.
96 ListCommitsPath Backend
97 // ListPaths is the recursive list of the file paths of a revision.
98 ListPaths Backend
99}
100
101// AllRoutes returns routes that send every method to b. The tests and the
102// benchmarks use it to exercise a backend whatever the production routes are.
103func AllRoutes(b Backend) Routes {
104 return Routes{b, b, b, b, b, b, b, b, b, b}
105}
106
107// Config configures an Adapter.
108type Config struct {
109 // Fallback serves the methods that are not routed to ermite-git and the
110 // repositories ermite-git cannot read. Required.
111 Fallback ports.RepoReader
112 // RepoDir returns the directory of a repository. It does not check that it
113 // exists. Required.
114 RepoDir func(domain.RepoName) string
115 // RefreshEvery is the longest a repository goes without a Refresh of the
116 // library while it is read. Zero means DefaultRefreshEvery; a negative
117 // value refreshes before every read. It does not affect what a read
118 // sees: see the package comment.
119 RefreshEvery time.Duration
120 // MaxRepos bounds the repositories kept open; zero means DefaultMaxRepos.
121 MaxRepos int
122 // IdleTTL is how long a repository may go unused before it is closed.
123 // Zero means DefaultIdleTTL; a negative value keeps repositories open
124 // until evicted.
125 IdleTTL time.Duration
126 // DeltaCacheSize is the budget in bytes of the cache of delta bases of
127 // each open repository. Zero means DefaultDeltaCacheSize; a negative value
128 // disables the cache.
129 DeltaCacheSize int64
130 // MaxObjectSize is the size in bytes above which an object that must be
131 // held in memory is read by the wrapped reader instead. Zero means
132 // DefaultMaxObjectSize; a negative value means the library default
133 // (128 MiB).
134 MaxObjectSize int64
135 // Routes overrides DefaultRoutes when not nil.
136 Routes *Routes
137}
138
139// Adapter is a ports.RepoReader that routes each method to ermite-git or to
140// the wrapped reader.
141type Adapter struct {
142 cli ports.RepoReader
143 dirOf func(domain.RepoName) string
144 routes Routes
145 cache *cache
146
147 logged sync.Map // dir -> struct{}: the unsupported repositories already logged
148}
149
150// New builds an adapter.
151func New(cfg Config) *Adapter {
152 routes := DefaultRoutes()
153 if cfg.Routes != nil {
154 routes = *cfg.Routes
155 }
156 refreshEvery := cfg.RefreshEvery
157 if refreshEvery == 0 {
158 refreshEvery = DefaultRefreshEvery
159 }
160 idleTTL := cfg.IdleTTL
161 if idleTTL == 0 {
162 idleTTL = DefaultIdleTTL
163 }
164 deltaCache := cfg.DeltaCacheSize
165 if deltaCache == 0 {
166 deltaCache = DefaultDeltaCacheSize
167 }
168 maxObject := cfg.MaxObjectSize
169 if maxObject == 0 {
170 maxObject = DefaultMaxObjectSize
171 }
172 a := &Adapter{cli: cfg.Fallback, dirOf: cfg.RepoDir, routes: routes}
173 a.cache = newCache(cacheConfig{
174 max: cfg.MaxRepos, refreshEvery: refreshEvery, idleTTL: idleTTL,
175 openOpts: []git.Option{git.WithDeltaCacheSize(deltaCache), git.WithMaxObjectSize(maxObject)},
176 onUnsupport: a.logUnsupported,
177 })
178 return a
179}
180
181// Close closes the open repositories. Reads after Close fail with ErrClosed;
182// blob streams already returned stay valid until they are closed. The wrapped
183// reader is not closed.
184func (a *Adapter) Close() { a.cache.close() }
185
186// Forget drops the open repository of dir, which was moved away or deleted.
187func (a *Adapter) Forget(dir string) {
188 a.cache.forget(dir)
189 a.logged.Delete(dir)
190}
191
192// Refresh tells the adapter that ermite wrote to the repository (a push, the
193// maintenance): the library refreshes it now, if it is open, instead of at the
194// next read after RefreshEvery. A repository that is not open has nothing to
195// refresh. A failure is logged, not returned: reads do not depend on it.
196func (a *Adapter) Refresh(ctx context.Context, repo domain.RepoName) {
197 if err := a.cache.refreshNow(ctx, repo, a.dirOf(repo)); err != nil {
198 log := zerowrap.FromCtx(ctx)
199 log.Warn().Err(err).Str("repo", repo.String()).Msg("refresh of the open repository failed")
200 }
201}
202
203// OpenRepos returns the number of repositories in the cache, for tests.
204func (a *Adapter) OpenRepos() int { return a.cache.size() }
205
206// logUnsupported logs, once per directory, that a repository is served by git.
207func (a *Adapter) logUnsupported(ctx context.Context, dir string, name domain.RepoName, cause error) {
208 if _, seen := a.logged.LoadOrStore(dir, struct{}{}); seen {
209 return
210 }
211 log := zerowrap.FromCtx(ctx)
212 log.Info().Str("repo", name.String()).Str("reason", cause.Error()).Msg("repository is not readable by ermite-git, served by git")
213}
214
215// errFallback makes serve hand the call to the wrapped reader.
216var errFallback = errors.New("ermitegit: served by git")
217
218// serve runs fn on the open repository, or cli when the repository is
219// unsupported, when fn asks for it, or when the library refuses on a bound.
220func serve[T any](ctx context.Context, a *Adapter, repo domain.RepoName, fn func(*git.Repo) (T, error), cli func() (T, error)) (T, error) {
221 var zero T
222 if err := ctx.Err(); err != nil {
223 return zero, err
224 }
225 h, err := a.cache.get(ctx, repo, a.dirOf(repo))
226 if err != nil {
227 return zero, err
228 }
229 defer h.release()
230 if h.repo == nil {
231 return cli()
232 }
233 v, err := fn(h.repo)
234 switch {
235 case err == nil:
236 return v, nil
237 case errors.Is(err, errFallback), errors.Is(err, git.ErrTooLarge):
238 return cli()
239 case errors.Is(err, git.ErrUnsupported):
240 h.fallBack(ctx, repo, err)
241 return cli()
242 }
243 return zero, mapErr(ctx, err)
244}
245
246// mapErr turns the errors of the library into the ones of the port. A
247// cancelled context stays itself.
248func mapErr(ctx context.Context, err error) error {
249 switch {
250 case ctx.Err() != nil && errors.Is(err, ctx.Err()):
251 return err
252 case errors.Is(err, domain.ErrNotFound), errors.Is(err, domain.ErrNotFile), errors.Is(err, domain.ErrNotDirectory):
253 return err // already one of the port's
254 case errors.Is(err, git.ErrNotFound), errors.Is(err, git.ErrAmbiguous):
255 // gitcli reads an ambiguous abbreviation as a missing object.
256 return fmt.Errorf("%w: %w", domain.ErrNotFound, err)
257 case errors.Is(err, git.ErrInvalidPath):
258 return fmt.Errorf("%w: %w", domain.ErrInvalidPath, err)
259 case errors.Is(err, git.ErrInvalidID):
260 return fmt.Errorf("%w: %w", domain.ErrInvalidObjectID, err)
261 case errors.Is(err, git.ErrClosed):
262 return fmt.Errorf("%w: %w", ErrClosed, err)
263 }
264 return err
265}
266
267// ListRefs lists HEAD, the branches and the tags.
268func (a *Adapter) ListRefs(ctx context.Context, repo domain.RepoName) ([]domain.Ref, error) {
269 cli := func() ([]domain.Ref, error) { return a.cli.ListRefs(ctx, repo) }
270 if a.routes.ListRefs == CLI {
271 return cli()
272 }
273 return serve(ctx, a, repo, func(r *git.Repo) ([]domain.Ref, error) { return listRefs(r) }, cli)
274}
275
276// ResolveRevision turns a revision into a commit.
277//
278//hexcheck:ignore suspicious-business-logic-in-adapter routes a call by a code constant; rules and limits come from domain and ports
279func (a *Adapter) ResolveRevision(ctx context.Context, repo domain.RepoName, rev domain.Revision) (domain.ResolvedRevision, error) {
280 cli := func() (domain.ResolvedRevision, error) { return a.cli.ResolveRevision(ctx, repo, rev) }
281 route := a.routes.ResolveRevision
282 if rev.IsAbbrevID() {
283 route = a.routes.ResolveRevisionAbbrev
284 }
285 if route == CLI {
286 return cli()
287 }
288 return serve(ctx, a, repo, func(r *git.Repo) (domain.ResolvedRevision, error) {
289 res, err := resolveRevision(r, rev)
290 if err != nil {
291 return domain.ResolvedRevision{}, fmt.Errorf("resolve %q in %q: %w", rev, repo, err)
292 }
293 return res.domain()
294 }, cli)
295}
296
297// ResolveRefPath splits a path into a revision and the rest.
298//
299//hexcheck:ignore suspicious-business-logic-in-adapter routes a call by a code constant; rules and limits come from domain and ports
300func (a *Adapter) ResolveRefPath(ctx context.Context, repo domain.RepoName, path string) (domain.RefPath, error) {
301 cli := func() (domain.RefPath, error) { return a.cli.ResolveRefPath(ctx, repo, path) }
302 if a.routes.ResolveRefPath == CLI {
303 return cli()
304 }
305 return serve(ctx, a, repo, func(r *git.Repo) (domain.RefPath, error) {
306 out, err := domain.SplitRefPath(path, func(rev domain.Revision) (domain.ResolvedRevision, error) {
307 res, err := resolveRevision(r, rev)
308 if err != nil {
309 return domain.ResolvedRevision{}, mapErr(ctx, err)
310 }
311 return res.domain()
312 })
313 if err != nil {
314 return domain.RefPath{}, fmt.Errorf("resolve path %q in %q: %w", path, repo, err)
315 }
316 return out, nil
317 }, cli)
318}
319
320// ListPaths lists the file and symbolic link paths of a revision.
321func (a *Adapter) ListPaths(ctx context.Context, repo domain.RepoName, rev domain.Revision, maxPaths, maxBytes int) (domain.PathList, error) {
322 cli := func() (domain.PathList, error) { return a.cli.ListPaths(ctx, repo, rev, maxPaths, maxBytes) }
323 if a.routes.ListPaths == CLI {
324 return cli()
325 }
326 return serve(ctx, a, repo, func(r *git.Repo) (domain.PathList, error) {
327 list, err := listPaths(ctx, r, rev, maxPaths, maxBytes)
328 if err != nil {
329 return domain.PathList{}, fmt.Errorf("paths of %q: %w", repo, err)
330 }
331 return list, nil
332 }, cli)
333}
334
335// GetTree lists a directory.
336func (a *Adapter) GetTree(ctx context.Context, repo domain.RepoName, rev domain.Revision, path domain.TreePath, includeLastCommit bool) (domain.Tree, error) {
337 cli := func() (domain.Tree, error) { return a.cli.GetTree(ctx, repo, rev, path, includeLastCommit) }
338 route := a.routes.GetTree
339 if includeLastCommit {
340 route = a.routes.GetTreeLastCommits
341 }
342 if route == CLI {
343 return cli()
344 }
345 return serve(ctx, a, repo, func(r *git.Repo) (domain.Tree, error) {
346 tree, err := getTree(ctx, r, rev, path, includeLastCommit)
347 if err != nil {
348 return domain.Tree{}, fmt.Errorf("tree %q of %q: %w", path, repo, err)
349 }
350 return tree, nil
351 }, cli)
352}
353
354// OpenBlob describes a file and returns its content as a stream.
355func (a *Adapter) OpenBlob(ctx context.Context, repo domain.RepoName, rev domain.Revision, path domain.TreePath) (domain.BlobInfo, io.ReadCloser, error) {
356 type result struct {
357 info domain.BlobInfo
358 rc io.ReadCloser
359 }
360 cli := func() (result, error) {
361 info, rc, err := a.cli.OpenBlob(ctx, repo, rev, path)
362 return result{info, rc}, err
363 }
364 if a.routes.OpenBlob == CLI {
365 res, err := cli()
366 return res.info, res.rc, err
367 }
368 res, err := serve(ctx, a, repo, func(r *git.Repo) (result, error) {
369 info, rc, err := openBlob(ctx, r, rev, path)
370 if err != nil {
371 return result{}, fmt.Errorf("blob %q of %q: %w", path, repo, err)
372 }
373 return result{info, rc}, nil
374 }, cli)
375 return res.info, res.rc, err
376}
377
378// ListCommits returns a page of history.
379//
380//hexcheck:ignore suspicious-business-logic-in-adapter routes a call by a code constant; rules and limits come from domain and ports
381func (a *Adapter) ListCommits(ctx context.Context, repo domain.RepoName, q ports.CommitQuery) (ports.CommitPage, error) {
382 if err := q.Validate(); err != nil {
383 return ports.CommitPage{}, err
384 }
385 cli := func() (ports.CommitPage, error) { return a.cli.ListCommits(ctx, repo, q) }
386 route := a.routes.ListCommits
387 if !q.Path.IsRoot() {
388 route = a.routes.ListCommitsPath
389 }
390 if route == CLI {
391 return cli()
392 }
393 return serve(ctx, a, repo, func(r *git.Repo) (ports.CommitPage, error) {
394 page, err := listCommits(ctx, r, q)
395 if err != nil {
396 return ports.CommitPage{}, fmt.Errorf("history of %q: %w", repo, err)
397 }
398 return page, nil
399 }, cli)
400}
401
402// GetCommit is served by the wrapped reader: the diff stays on git.
403func (a *Adapter) GetCommit(ctx context.Context, repo domain.RepoName, rev domain.Revision, limits domain.DiffLimits) (domain.CommitDetail, error) {
404 return a.cli.GetCommit(ctx, repo, rev, limits)
405}