1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3// Package ermitegit implements ports.RepoReader over the ermite-git library.
4//
5// The adapter wraps the gitcli reader. Each method has a route, a constant
6// below: the methods that match gitcli in the shared conformance table and
7// beat it in the benchmarks are served by ermite-git, the others (and
8// GetCommit) by the wrapped reader. A repository that ermite-git cannot read
9// (ErrUnsupported: SHA-256, reftable, alternates, replace refs...) is served
10// by the wrapped reader for every method.
11//
12// Any other error of the library is returned, never hidden by a fallback.
13// The exceptions are decisions about the data, not failures: a commit whose
14// message is not in UTF-8 (git converts it, the library does not) and an
15// object larger than the library's in-memory bound are read by the wrapped
16// reader.
17//
18// Freshness: a read that begins after a push has returned sees the push,
19// whichever path made it, because the library reads refs from disk on each
20// call (loose refs always, packed-refs after a stat) and rescans the packs
21// when an object is missing (TestFreshness proves it with no Refresh at all).
22// The adapter still calls Refresh to release replaced packs, pick up a new
23// commit-graph and notice a repository that stopped being supported: at once
24// when ermite itself wrote (the push and the maintenance tell it through
25// Refresh; a merge request merge waits for its maintenance), and as a safety
26// net at most once per Config.RefreshEvery for each repository while it is
27// read, by one reader at a time. A Refresh stats the files of the repository,
28// too much to pay on every read of a method that itself costs 2 µs.
29//
30// Memory: the cache of open repositories is bounded (LRU) and drops the ones
31// idle longer than Config.IdleTTL; each repository holds at most
32// Config.DeltaCacheSize of delta bases and reads objects up to
33// Config.MaxObjectSize into memory. gitcli's Trash tells the adapter to drop
34// a deleted repository (see Forget).
35package ermitegit
36
37import (
38 "context"
39 "errors"
40 "fmt"
41 "io"
42 "sync"
43 "time"
44
45 git "bnema.dev/ermite-git"
46 "github.com/bnema/zerowrap"
47
48 "bnema.dev/ermite/internal/domain"
49 "bnema.dev/ermite/internal/ports"
50)
51
52var (
53 _ ports.RepoReader = (*Adapter)(nil)
54 _ ports.RepoFreshener = (*Adapter)(nil)
55)
56
57// ErrClosed is returned by reads after Close.
58var ErrClosed = errors.New("ermitegit: reader is closed")
59
60// Backend says which reader serves a method.
61type Backend uint8
62
63// Backends.
64const (
65 // CLI is the git binary, through the gitcli reader.
66 CLI Backend = iota
67 // ErmiteGit is the ermite-git library.
68 ErmiteGit
69)
70
71// String returns "cli" or "ermite-git".
72func (b Backend) String() string {
73 if b == ErmiteGit {
74 return "ermite-git"
75 }
76 return "cli"
77}
78
79// Routes says which reader serves each method. GetCommit is not here: it is
80// always served by the wrapped reader.
81type Routes struct {
82 ListRefs Backend
83 // ResolveRevision is a revision that is a name or a full id.
84 ResolveRevision Backend
85 // ResolveRevisionAbbrev is a revision that is an abbreviated id.
86 ResolveRevisionAbbrev Backend
87 ResolveRefPath Backend
88 // GetTree is a directory listing without last commits.
89 GetTree Backend
90 // GetTreeLastCommits is a listing with the last commit of each entry.
91 GetTreeLastCommits Backend
92 OpenBlob Backend
93 // ListCommits is a page of history of a revision.
94 ListCommits Backend
95 // ListCommitsPath is a page of the history of a file or a directory.
96 ListCommitsPath Backend
97}
98
99// AllRoutes returns routes that send every method to b. The tests and the
100// benchmarks use it to exercise a backend whatever the production routes are.
101func AllRoutes(b Backend) Routes {
102 return Routes{b, b, b, b, b, b, b, b, b}
103}
104
105// Config configures an Adapter.
106type Config struct {
107 // Fallback serves the methods that are not routed to ermite-git and the
108 // repositories ermite-git cannot read. Required.
109 Fallback ports.RepoReader
110 // RepoDir returns the directory of a repository. It does not check that it
111 // exists. Required.
112 RepoDir func(domain.RepoName) string
113 // RefreshEvery is the longest a repository goes without a Refresh of the
114 // library while it is read. Zero means DefaultRefreshEvery; a negative
115 // value refreshes before every read. It does not affect what a read
116 // sees: see the package comment.
117 RefreshEvery time.Duration
118 // MaxRepos bounds the repositories kept open; zero means DefaultMaxRepos.
119 MaxRepos int
120 // IdleTTL is how long a repository may go unused before it is closed.
121 // Zero means DefaultIdleTTL; a negative value keeps repositories open
122 // until evicted.
123 IdleTTL time.Duration
124 // DeltaCacheSize is the budget in bytes of the cache of delta bases of
125 // each open repository. Zero means DefaultDeltaCacheSize; a negative value
126 // disables the cache.
127 DeltaCacheSize int64
128 // MaxObjectSize is the size in bytes above which an object that must be
129 // held in memory is read by the wrapped reader instead. Zero means
130 // DefaultMaxObjectSize; a negative value means the library default
131 // (128 MiB).
132 MaxObjectSize int64
133 // Routes overrides DefaultRoutes when not nil.
134 Routes *Routes
135}
136
137// Adapter is a ports.RepoReader that routes each method to ermite-git or to
138// the wrapped reader.
139type Adapter struct {
140 cli ports.RepoReader
141 dirOf func(domain.RepoName) string
142 routes Routes
143 cache *cache
144
145 logged sync.Map // dir -> struct{}: the unsupported repositories already logged
146}
147
148// New builds an adapter.
149func New(cfg Config) *Adapter {
150 routes := DefaultRoutes()
151 if cfg.Routes != nil {
152 routes = *cfg.Routes
153 }
154 refreshEvery := cfg.RefreshEvery
155 if refreshEvery == 0 {
156 refreshEvery = DefaultRefreshEvery
157 }
158 idleTTL := cfg.IdleTTL
159 if idleTTL == 0 {
160 idleTTL = DefaultIdleTTL
161 }
162 deltaCache := cfg.DeltaCacheSize
163 if deltaCache == 0 {
164 deltaCache = DefaultDeltaCacheSize
165 }
166 maxObject := cfg.MaxObjectSize
167 if maxObject == 0 {
168 maxObject = DefaultMaxObjectSize
169 }
170 a := &Adapter{cli: cfg.Fallback, dirOf: cfg.RepoDir, routes: routes}
171 a.cache = newCache(cacheConfig{
172 max: cfg.MaxRepos, refreshEvery: refreshEvery, idleTTL: idleTTL,
173 openOpts: []git.Option{git.WithDeltaCacheSize(deltaCache), git.WithMaxObjectSize(maxObject)},
174 onUnsupport: a.logUnsupported,
175 })
176 return a
177}
178
179// Close closes the open repositories. Reads after Close fail with ErrClosed;
180// blob streams already returned stay valid until they are closed. The wrapped
181// reader is not closed.
182func (a *Adapter) Close() { a.cache.close() }
183
184// Forget drops the open repository of dir, which was moved away or deleted.
185func (a *Adapter) Forget(dir string) {
186 a.cache.forget(dir)
187 a.logged.Delete(dir)
188}
189
190// Refresh tells the adapter that ermite wrote to the repository (a push, the
191// maintenance): the library refreshes it now, if it is open, instead of at the
192// next read after RefreshEvery. A repository that is not open has nothing to
193// refresh. A failure is logged, not returned: reads do not depend on it.
194func (a *Adapter) Refresh(ctx context.Context, repo domain.RepoName) {
195 if err := a.cache.refreshNow(ctx, repo, a.dirOf(repo)); err != nil {
196 log := zerowrap.FromCtx(ctx)
197 log.Warn().Err(err).Str("repo", repo.String()).Msg("refresh of the open repository failed")
198 }
199}
200
201// OpenRepos returns the number of repositories in the cache, for tests.
202func (a *Adapter) OpenRepos() int { return a.cache.size() }
203
204// logUnsupported logs, once per directory, that a repository is served by git.
205func (a *Adapter) logUnsupported(ctx context.Context, dir string, name domain.RepoName, cause error) {
206 if _, seen := a.logged.LoadOrStore(dir, struct{}{}); seen {
207 return
208 }
209 log := zerowrap.FromCtx(ctx)
210 log.Info().Str("repo", name.String()).Str("reason", cause.Error()).Msg("repository is not readable by ermite-git, served by git")
211}
212
213// errFallback makes serve hand the call to the wrapped reader.
214var errFallback = errors.New("ermitegit: served by git")
215
216// serve runs fn on the open repository, or cli when the repository is
217// unsupported, when fn asks for it, or when the library refuses on a bound.
218func serve[T any](ctx context.Context, a *Adapter, repo domain.RepoName, fn func(*git.Repo) (T, error), cli func() (T, error)) (T, error) {
219 var zero T
220 if err := ctx.Err(); err != nil {
221 return zero, err
222 }
223 h, err := a.cache.get(ctx, repo, a.dirOf(repo))
224 if err != nil {
225 return zero, err
226 }
227 defer h.release()
228 if h.repo == nil {
229 return cli()
230 }
231 v, err := fn(h.repo)
232 switch {
233 case err == nil:
234 return v, nil
235 case errors.Is(err, errFallback), errors.Is(err, git.ErrTooLarge):
236 return cli()
237 case errors.Is(err, git.ErrUnsupported):
238 h.fallBack(ctx, repo, err)
239 return cli()
240 }
241 return zero, mapErr(ctx, err)
242}
243
244// mapErr turns the errors of the library into the ones of the port. A
245// cancelled context stays itself.
246func mapErr(ctx context.Context, err error) error {
247 switch {
248 case ctx.Err() != nil && errors.Is(err, ctx.Err()):
249 return err
250 case errors.Is(err, domain.ErrNotFound), errors.Is(err, domain.ErrNotFile), errors.Is(err, domain.ErrNotDirectory):
251 return err // already one of the port's
252 case errors.Is(err, git.ErrNotFound), errors.Is(err, git.ErrAmbiguous):
253 // gitcli reads an ambiguous abbreviation as a missing object.
254 return fmt.Errorf("%w: %w", domain.ErrNotFound, err)
255 case errors.Is(err, git.ErrInvalidPath):
256 return fmt.Errorf("%w: %w", domain.ErrInvalidPath, err)
257 case errors.Is(err, git.ErrInvalidID):
258 return fmt.Errorf("%w: %w", domain.ErrInvalidObjectID, err)
259 case errors.Is(err, git.ErrClosed):
260 return fmt.Errorf("%w: %w", ErrClosed, err)
261 }
262 return err
263}
264
265// ListRefs lists HEAD, the branches and the tags.
266func (a *Adapter) ListRefs(ctx context.Context, repo domain.RepoName) ([]domain.Ref, error) {
267 cli := func() ([]domain.Ref, error) { return a.cli.ListRefs(ctx, repo) }
268 if a.routes.ListRefs == CLI {
269 return cli()
270 }
271 return serve(ctx, a, repo, func(r *git.Repo) ([]domain.Ref, error) { return listRefs(r) }, cli)
272}
273
274// ResolveRevision turns a revision into a commit.
275//
276//hexcheck:ignore suspicious-business-logic-in-adapter routes a call by a code constant; rules and limits come from domain and ports
277func (a *Adapter) ResolveRevision(ctx context.Context, repo domain.RepoName, rev domain.Revision) (domain.ResolvedRevision, error) {
278 cli := func() (domain.ResolvedRevision, error) { return a.cli.ResolveRevision(ctx, repo, rev) }
279 route := a.routes.ResolveRevision
280 if rev.IsAbbrevID() {
281 route = a.routes.ResolveRevisionAbbrev
282 }
283 if route == CLI {
284 return cli()
285 }
286 return serve(ctx, a, repo, func(r *git.Repo) (domain.ResolvedRevision, error) {
287 res, err := resolveRevision(r, rev)
288 if err != nil {
289 return domain.ResolvedRevision{}, fmt.Errorf("resolve %q in %q: %w", rev, repo, err)
290 }
291 return res.domain()
292 }, cli)
293}
294
295// ResolveRefPath splits a path into a revision and the rest.
296//
297//hexcheck:ignore suspicious-business-logic-in-adapter routes a call by a code constant; rules and limits come from domain and ports
298func (a *Adapter) ResolveRefPath(ctx context.Context, repo domain.RepoName, path string) (domain.RefPath, error) {
299 cli := func() (domain.RefPath, error) { return a.cli.ResolveRefPath(ctx, repo, path) }
300 if a.routes.ResolveRefPath == CLI {
301 return cli()
302 }
303 return serve(ctx, a, repo, func(r *git.Repo) (domain.RefPath, error) {
304 out, err := domain.SplitRefPath(path, func(rev domain.Revision) (domain.ResolvedRevision, error) {
305 res, err := resolveRevision(r, rev)
306 if err != nil {
307 return domain.ResolvedRevision{}, mapErr(ctx, err)
308 }
309 return res.domain()
310 })
311 if err != nil {
312 return domain.RefPath{}, fmt.Errorf("resolve path %q in %q: %w", path, repo, err)
313 }
314 return out, nil
315 }, cli)
316}
317
318// GetTree lists a directory.
319func (a *Adapter) GetTree(ctx context.Context, repo domain.RepoName, rev domain.Revision, path domain.TreePath, includeLastCommit bool) (domain.Tree, error) {
320 cli := func() (domain.Tree, error) { return a.cli.GetTree(ctx, repo, rev, path, includeLastCommit) }
321 route := a.routes.GetTree
322 if includeLastCommit {
323 route = a.routes.GetTreeLastCommits
324 }
325 if route == CLI {
326 return cli()
327 }
328 return serve(ctx, a, repo, func(r *git.Repo) (domain.Tree, error) {
329 tree, err := getTree(ctx, r, rev, path, includeLastCommit)
330 if err != nil {
331 return domain.Tree{}, fmt.Errorf("tree %q of %q: %w", path, repo, err)
332 }
333 return tree, nil
334 }, cli)
335}
336
337// OpenBlob describes a file and returns its content as a stream.
338func (a *Adapter) OpenBlob(ctx context.Context, repo domain.RepoName, rev domain.Revision, path domain.TreePath) (domain.BlobInfo, io.ReadCloser, error) {
339 type result struct {
340 info domain.BlobInfo
341 rc io.ReadCloser
342 }
343 cli := func() (result, error) {
344 info, rc, err := a.cli.OpenBlob(ctx, repo, rev, path)
345 return result{info, rc}, err
346 }
347 if a.routes.OpenBlob == CLI {
348 res, err := cli()
349 return res.info, res.rc, err
350 }
351 res, err := serve(ctx, a, repo, func(r *git.Repo) (result, error) {
352 info, rc, err := openBlob(ctx, r, rev, path)
353 if err != nil {
354 return result{}, fmt.Errorf("blob %q of %q: %w", path, repo, err)
355 }
356 return result{info, rc}, nil
357 }, cli)
358 return res.info, res.rc, err
359}
360
361// ListCommits returns a page of history.
362//
363//hexcheck:ignore suspicious-business-logic-in-adapter routes a call by a code constant; rules and limits come from domain and ports
364func (a *Adapter) ListCommits(ctx context.Context, repo domain.RepoName, q ports.CommitQuery) (ports.CommitPage, error) {
365 if err := q.Validate(); err != nil {
366 return ports.CommitPage{}, err
367 }
368 cli := func() (ports.CommitPage, error) { return a.cli.ListCommits(ctx, repo, q) }
369 route := a.routes.ListCommits
370 if !q.Path.IsRoot() {
371 route = a.routes.ListCommitsPath
372 }
373 if route == CLI {
374 return cli()
375 }
376 return serve(ctx, a, repo, func(r *git.Repo) (ports.CommitPage, error) {
377 page, err := listCommits(ctx, r, q)
378 if err != nil {
379 return ports.CommitPage{}, fmt.Errorf("history of %q: %w", repo, err)
380 }
381 return page, nil
382 }, cli)
383}
384
385// GetCommit is served by the wrapped reader: the diff stays on git.
386func (a *Adapter) GetCommit(ctx context.Context, repo domain.RepoName, rev domain.Revision, limits domain.DiffLimits) (domain.CommitDetail, error) {
387 return a.cli.GetCommit(ctx, repo, rev, limits)
388}