detached c33e43c

perf(browse): route warm reads to ermite-git and cache immutable last-commit results

- ListRefs, ResolveRevision, ResolveRefPath and tree last-commit listings
  (root and subdirectory) now go to ermite-git; routing weighs warm time and
  bytes instead of cold allocation counts
- app.Browse caches last-commit tree listings and single-commit path lookups
  keyed by repo, full commit id and path, in a bounded LRU (256 entries, 8 MiB)
commit
c33e43c9e86c0234ed243b028e7bd62c10935077
author
brice · · 2026-10-11 18:40 UTC
parent
e42fec5
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
5 files · +577 -47
M internal/adapters/ermitegit/ermitegit.go +11 -4
@@ -87,9 +87,13 @@ type Routes struct {
8787 ResolveRefPath Backend
8888 // GetTree is a directory listing without last commits.
8989 GetTree Backend
90- // GetTreeLastCommits is a listing with the last commit of each entry.
90+ // GetTreeLastCommits is the listing of the root with the last commit of
91+ // each entry.
9192 GetTreeLastCommits Backend
92- OpenBlob Backend
93+ // GetTreeSubdirLastCommits is the same for a subdirectory, which costs a
94+ // much longer history walk (195 ms with git on lazygit).
95+ GetTreeSubdirLastCommits Backend
96+ OpenBlob Backend
9397 // ListCommits is a page of history of a revision.
9498 ListCommits Backend
9599 // ListCommitsPath is a page of the history of a file or a directory.
@@ -99,7 +103,7 @@ type Routes struct {
99103 // AllRoutes returns routes that send every method to b. The tests and the
100104 // benchmarks use it to exercise a backend whatever the production routes are.
101105 func AllRoutes(b Backend) Routes {
102- return Routes{b, b, b, b, b, b, b, b, b}
106+ return Routes{b, b, b, b, b, b, b, b, b, b}
103107 }
104108
105109 // Config configures an Adapter.
@@ -320,7 +324,10 @@ func (a *Adapter) GetTree(ctx context.Context, repo domain.RepoName, rev domain.
320324 cli := func() (domain.Tree, error) { return a.cli.GetTree(ctx, repo, rev, path, includeLastCommit) }
321325 route := a.routes.GetTree
322326 if includeLastCommit {
323- route = a.routes.GetTreeLastCommits
327+ route = a.routes.GetTreeSubdirLastCommits
328+ if path.IsRoot() {
329+ route = a.routes.GetTreeLastCommits
330+ }
324331 }
325332 if route == CLI {
326333 return cli()
@@ -87,9 +87,13 @@ type Routes struct {
87 ResolveRefPath Backend87 ResolveRefPath Backend
88 // GetTree is a directory listing without last commits.88 // GetTree is a directory listing without last commits.
89 GetTree Backend89 GetTree Backend
90- // GetTreeLastCommits is a listing with the last commit of each entry.90+ // GetTreeLastCommits is the listing of the root with the last commit of
91+ // each entry.
91 GetTreeLastCommits Backend92 GetTreeLastCommits Backend
92- OpenBlob Backend93+ // GetTreeSubdirLastCommits is the same for a subdirectory, which costs a
94+ // much longer history walk (195 ms with git on lazygit).
95+ GetTreeSubdirLastCommits Backend
96+ OpenBlob Backend
93 // ListCommits is a page of history of a revision.97 // ListCommits is a page of history of a revision.
94 ListCommits Backend98 ListCommits Backend
95 // ListCommitsPath is a page of the history of a file or a directory.99 // ListCommitsPath is a page of the history of a file or a directory.
@@ -99,7 +103,7 @@ type Routes struct {
99 // AllRoutes returns routes that send every method to b. The tests and the103 // AllRoutes returns routes that send every method to b. The tests and the
100 // benchmarks use it to exercise a backend whatever the production routes are.104 // benchmarks use it to exercise a backend whatever the production routes are.
101 func AllRoutes(b Backend) Routes {105 func AllRoutes(b Backend) Routes {
102- return Routes{b, b, b, b, b, b, b, b, b}106+ return Routes{b, b, b, b, b, b, b, b, b, b}
103 }107 }
104 108
105 // Config configures an Adapter.109 // Config configures an Adapter.
@@ -320,7 +324,10 @@ func (a *Adapter) GetTree(ctx context.Context, repo domain.RepoName, rev domain.
320 cli := func() (domain.Tree, error) { return a.cli.GetTree(ctx, repo, rev, path, includeLastCommit) }324 cli := func() (domain.Tree, error) { return a.cli.GetTree(ctx, repo, rev, path, includeLastCommit) }
321 route := a.routes.GetTree325 route := a.routes.GetTree
322 if includeLastCommit {326 if includeLastCommit {
323- route = a.routes.GetTreeLastCommits327+ route = a.routes.GetTreeSubdirLastCommits
328+ if path.IsRoot() {
329+ route = a.routes.GetTreeLastCommits
330+ }
324 }331 }
325 if route == CLI {332 if route == CLI {
326 return cli()333 return cli()
M internal/adapters/ermitegit/routes.go +51 -39
@@ -4,15 +4,19 @@ package ermitegit
44
55 // DefaultRoutes returns the routes of production.
66 //
7-// The rule: a method is routed to ermite-git only when it passes the shared
7+// The rule: a method is routed to ermite-git when it passes the shared
88 // conformance table (TestConformance, TestCompareWithCLI) and its benchmark
9-// beats the CLI adapter in time, in bytes AND in allocations per call, warm
10-// AND cold, on the generated fixture AND on lazygit (BenchmarkRepoReader,
11-// ERMITE_GIT_BENCH_REPO). Where a method has several shapes, every shape must
12-// win: the constants GetTree, GetTreeLastCommits, ResolveRevisionAbbrev and
13-// ListCommitsPath cover directories of any size and files of any history. A
14-// method that does not win stays on the CLI. GetCommit is not routable: the
15-// diff stays on git.
9+// beats the CLI adapter WARM in time AND in bytes per call, on the generated
10+// fixture AND on lazygit (BenchmarkRepoReader, ERMITE_GIT_BENCH_REPO), and its
11+// COLD run does not lose in time by more than the open itself (about 65 µs,
12+// paid once per open, see below). Allocation counts are reported but do not
13+// decide: the repository cache (cache.go) keeps up to 64 repositories open for
14+// ten minutes, so nearly every request is warm, and the CLI's own cost (a
15+// process per call, absent from the Go counters) is not in its numbers. Where
16+// a method has several shapes that behave differently, each has its own
17+// constant: GetTree, GetTreeLastCommits (root), GetTreeSubdirLastCommits,
18+// ResolveRevisionAbbrev and ListCommitsPath. A method that does not win stays
19+// on the CLI. GetCommit is not routable: the diff stays on git.
1620 //
1721 // Warm is a repository that is open; cold closes it before every call, the
1822 // price of the first read after the idle TTL, an eviction or a restart (a
@@ -92,39 +96,47 @@ package ermitegit
9296 // ListCommits/dirpath lazygit warm 6.7ms 127.8K 403 2.0ms 50.2K 181 win
9397 // ListCommits/dirpath lazygit cold 6.7ms 127.8K 403 3.3ms 887.1K 2019 loses ba
9498 //
95-// What decides the routes, with ermite-git 5228465 (zlib reader with a reused
96-// adler32, header-size cache, Refresh and LookupRefID without allocations):
99+// The "verdict" column is the strict reading, every line of the table won;
100+// the routes follow the rule above. What decides them, with ermite-git 5228465
101+// (zlib reader with a reused adler32, header-size cache, Refresh and
102+// LookupRefID without allocations):
97103 //
98-// - The warm runs all win but GetTree of a large directory on lazygit (time
99-// is won since the zlib change, bytes are not) and OpenBlob, whose
100-// allocations come from the inflate of a small blob and whose time is
101-// lost for a 10 MiB blob (inflate speed: 52 ms against 21 ms).
102-// - The cold runs lose in allocations for every method that does little
103-// work after the open (ListRefs, GetTree, ResolveRefPath: 150 to 200
104-// against 60 to 140), and in time too for the ones that cost a few
105-// microseconds warm (ResolveRevision: 65 µs against 9 to 63 µs).
106-// - Only the history of a revision (ListCommits page 1 and deep) wins every
107-// line, so it is the only method routed. GetTree dir+last also wins every
108-// line, but GetTreeLastCommits covers GetTree root+last too, which loses
109-// cold allocations on lazygit (4204 against 3073): it is the nearest
110-// miss, and the first to flip once opening a repository allocates less.
111-// - OpenBlob by size: below 1 MiB the small blob loses cold and, on lazygit,
112-// warm bytes and allocations; above, it loses time. No size makes it win,
113-// so there is no size-based route.
114-//
115-// Once opening a repository allocates less, re-run the benchmark and flip the
116-// constants it wins: ListRefs, ResolveRefPath and ResolveRevision win warm by 4
117-// to 60 times in time.
104+// - ListRefs (every repository page, through loadRepo): 905 µs against 24 µs
105+// warm, 120 KiB against 19 KiB (fixture), 960 against 16 µs and 108 KiB
106+// against 2 KiB (lazygit). Cold it is still 4 to 10 times faster
107+// (93 µs); only its allocation count loses. Routed.
108+// - ResolveRevision (name, full id, abbreviation) and ResolveRefPath: 4 to 8
109+// times faster warm, with fewer bytes. Cold they lose 45 µs of time to the
110+// open, but the ListRefs of the same page has opened the repository
111+// already, so the open is paid by a call that wins it back. Routed.
112+// - GetTree with last commits: root listing 4.1 ms against 68 µs on the
113+// fixture, 20.5 ms against 6.2 ms on lazygit (cold 8.7 ms; its 4,206
114+// allocations against 3,082 are the only line lost); a subdirectory of
115+// lazygit 195-199 ms against 8-9 ms warm, 15-17 ms cold, 20 MiB against
116+// 1.5 MiB. They are two constants so that either can be flipped back on
117+// its own. Both routed.
118+// - GetTree without last commits wins warm in time everywhere (13 µs against
119+// 0.7 ms at the root), but a large directory of lazygit loses warm bytes
120+// (325 KiB against 275 KiB) and, cold, 4.0 ms against 1.4 ms in time. It
121+// stays on the CLI.
122+// - ListCommits of a revision wins every line and stays routed.
123+// ListCommitsPath wins warm and cold in time (lazygit file: 12.9 ms against
124+// 6.0 ms warm), only its cold bytes lose (2.4 MiB): it is a candidate, not
125+// flipped here, to be decided with its own conformance review.
126+// - OpenBlob by size: below 1 MiB the small blob loses warm bytes and
127+// allocations on lazygit; above, it loses time (inflate speed: 52 ms
128+// against 21 ms). No size makes it win, so there is no size-based route.
118129 func DefaultRoutes() Routes {
119130 return Routes{
120- ListRefs: CLI,
121- ResolveRevision: CLI,
122- ResolveRevisionAbbrev: CLI,
123- ResolveRefPath: CLI,
124- GetTree: CLI,
125- GetTreeLastCommits: CLI,
126- OpenBlob: CLI,
127- ListCommits: ErmiteGit,
128- ListCommitsPath: CLI,
131+ ListRefs: ErmiteGit,
132+ ResolveRevision: ErmiteGit,
133+ ResolveRevisionAbbrev: ErmiteGit,
134+ ResolveRefPath: ErmiteGit,
135+ GetTree: CLI,
136+ GetTreeLastCommits: ErmiteGit,
137+ GetTreeSubdirLastCommits: ErmiteGit,
138+ OpenBlob: CLI,
139+ ListCommits: ErmiteGit,
140+ ListCommitsPath: CLI,
129141 }
130142 }
@@ -4,15 +4,19 @@ package ermitegit
4 4
5 // DefaultRoutes returns the routes of production.5 // DefaultRoutes returns the routes of production.
6 //6 //
7-// The rule: a method is routed to ermite-git only when it passes the shared7+// The rule: a method is routed to ermite-git when it passes the shared
8 // conformance table (TestConformance, TestCompareWithCLI) and its benchmark8 // conformance table (TestConformance, TestCompareWithCLI) and its benchmark
9-// beats the CLI adapter in time, in bytes AND in allocations per call, warm9+// beats the CLI adapter WARM in time AND in bytes per call, on the generated
10-// AND cold, on the generated fixture AND on lazygit (BenchmarkRepoReader,10+// fixture AND on lazygit (BenchmarkRepoReader, ERMITE_GIT_BENCH_REPO), and its
11-// ERMITE_GIT_BENCH_REPO). Where a method has several shapes, every shape must11+// COLD run does not lose in time by more than the open itself (about 65 µs,
12-// win: the constants GetTree, GetTreeLastCommits, ResolveRevisionAbbrev and12+// paid once per open, see below). Allocation counts are reported but do not
13-// ListCommitsPath cover directories of any size and files of any history. A13+// decide: the repository cache (cache.go) keeps up to 64 repositories open for
14-// method that does not win stays on the CLI. GetCommit is not routable: the14+// ten minutes, so nearly every request is warm, and the CLI's own cost (a
15-// diff stays on git.15+// process per call, absent from the Go counters) is not in its numbers. Where
16+// a method has several shapes that behave differently, each has its own
17+// constant: GetTree, GetTreeLastCommits (root), GetTreeSubdirLastCommits,
18+// ResolveRevisionAbbrev and ListCommitsPath. A method that does not win stays
19+// on the CLI. GetCommit is not routable: the diff stays on git.
16 //20 //
17 // Warm is a repository that is open; cold closes it before every call, the21 // Warm is a repository that is open; cold closes it before every call, the
18 // price of the first read after the idle TTL, an eviction or a restart (a22 // price of the first read after the idle TTL, an eviction or a restart (a
@@ -92,39 +96,47 @@ package ermitegit
92 // ListCommits/dirpath lazygit warm 6.7ms 127.8K 403 2.0ms 50.2K 181 win96 // ListCommits/dirpath lazygit warm 6.7ms 127.8K 403 2.0ms 50.2K 181 win
93 // ListCommits/dirpath lazygit cold 6.7ms 127.8K 403 3.3ms 887.1K 2019 loses ba97 // ListCommits/dirpath lazygit cold 6.7ms 127.8K 403 3.3ms 887.1K 2019 loses ba
94 //98 //
95-// What decides the routes, with ermite-git 5228465 (zlib reader with a reused99+// The "verdict" column is the strict reading, every line of the table won;
96-// adler32, header-size cache, Refresh and LookupRefID without allocations):100+// the routes follow the rule above. What decides them, with ermite-git 5228465
101+// (zlib reader with a reused adler32, header-size cache, Refresh and
102+// LookupRefID without allocations):
97 //103 //
98-// - The warm runs all win but GetTree of a large directory on lazygit (time104+// - ListRefs (every repository page, through loadRepo): 905 µs against 24 µs
99-// is won since the zlib change, bytes are not) and OpenBlob, whose105+// warm, 120 KiB against 19 KiB (fixture), 960 against 16 µs and 108 KiB
100-// allocations come from the inflate of a small blob and whose time is106+// against 2 KiB (lazygit). Cold it is still 4 to 10 times faster
101-// lost for a 10 MiB blob (inflate speed: 52 ms against 21 ms).107+// (93 µs); only its allocation count loses. Routed.
102-// - The cold runs lose in allocations for every method that does little108+// - ResolveRevision (name, full id, abbreviation) and ResolveRefPath: 4 to 8
103-// work after the open (ListRefs, GetTree, ResolveRefPath: 150 to 200109+// times faster warm, with fewer bytes. Cold they lose 45 µs of time to the
104-// against 60 to 140), and in time too for the ones that cost a few110+// open, but the ListRefs of the same page has opened the repository
105-// microseconds warm (ResolveRevision: 65 µs against 9 to 63 µs).111+// already, so the open is paid by a call that wins it back. Routed.
106-// - Only the history of a revision (ListCommits page 1 and deep) wins every112+// - GetTree with last commits: root listing 4.1 ms against 68 µs on the
107-// line, so it is the only method routed. GetTree dir+last also wins every113+// fixture, 20.5 ms against 6.2 ms on lazygit (cold 8.7 ms; its 4,206
108-// line, but GetTreeLastCommits covers GetTree root+last too, which loses114+// allocations against 3,082 are the only line lost); a subdirectory of
109-// cold allocations on lazygit (4204 against 3073): it is the nearest115+// lazygit 195-199 ms against 8-9 ms warm, 15-17 ms cold, 20 MiB against
110-// miss, and the first to flip once opening a repository allocates less.116+// 1.5 MiB. They are two constants so that either can be flipped back on
111-// - OpenBlob by size: below 1 MiB the small blob loses cold and, on lazygit,117+// its own. Both routed.
112-// warm bytes and allocations; above, it loses time. No size makes it win,118+// - GetTree without last commits wins warm in time everywhere (13 µs against
113-// so there is no size-based route.119+// 0.7 ms at the root), but a large directory of lazygit loses warm bytes
114-//120+// (325 KiB against 275 KiB) and, cold, 4.0 ms against 1.4 ms in time. It
115-// Once opening a repository allocates less, re-run the benchmark and flip the121+// stays on the CLI.
116-// constants it wins: ListRefs, ResolveRefPath and ResolveRevision win warm by 4122+// - ListCommits of a revision wins every line and stays routed.
117-// to 60 times in time.123+// ListCommitsPath wins warm and cold in time (lazygit file: 12.9 ms against
124+// 6.0 ms warm), only its cold bytes lose (2.4 MiB): it is a candidate, not
125+// flipped here, to be decided with its own conformance review.
126+// - OpenBlob by size: below 1 MiB the small blob loses warm bytes and
127+// allocations on lazygit; above, it loses time (inflate speed: 52 ms
128+// against 21 ms). No size makes it win, so there is no size-based route.
118 func DefaultRoutes() Routes {129 func DefaultRoutes() Routes {
119 return Routes{130 return Routes{
120- ListRefs: CLI,131+ ListRefs: ErmiteGit,
121- ResolveRevision: CLI,132+ ResolveRevision: ErmiteGit,
122- ResolveRevisionAbbrev: CLI,133+ ResolveRevisionAbbrev: ErmiteGit,
123- ResolveRefPath: CLI,134+ ResolveRefPath: ErmiteGit,
124- GetTree: CLI,135+ GetTree: CLI,
125- GetTreeLastCommits: CLI,136+ GetTreeLastCommits: ErmiteGit,
126- OpenBlob: CLI,137+ GetTreeSubdirLastCommits: ErmiteGit,
127- ListCommits: ErmiteGit,138+ OpenBlob: CLI,
128- ListCommitsPath: CLI,139+ ListCommits: ErmiteGit,
140+ ListCommitsPath: CLI,
129 }141 }
130 }142 }
M internal/app/browse.go +56 -4
@@ -24,11 +24,29 @@ import (
2424 type Browse struct {
2525 meta ports.RepoMetaStore
2626 reader ports.RepoReader
27+ // history keeps the answers about a commit designated by its full id (a
28+ // tree with the last commit of its entries, the latest commit of a path):
29+ // they never change, and computing them costs a history walk. Nil
30+ // disables it.
31+ history *historyCache
2732 }
2833
29-// NewBrowse builds the read use cases over their ports.
34+// NewBrowse builds the read use cases over their ports, with a history cache
35+// of the default bounds.
3036 func NewBrowse(meta ports.RepoMetaStore, reader ports.RepoReader) *Browse {
31- return &Browse{meta: meta, reader: reader}
37+ return &Browse{
38+ meta: meta, reader: reader,
39+ history: newHistoryCache(DefaultHistoryCacheEntries, DefaultHistoryCacheBytes),
40+ }
41+}
42+
43+// WithHistoryCache replaces the bounds of the history cache: at most
44+// maxEntries answers and about maxBytes of memory, least recently used first
45+// out. A bound that is not positive disables the cache. Call it before the
46+// first request.
47+func (b *Browse) WithHistoryCache(maxEntries int, maxBytes int64) *Browse {
48+ b.history = newHistoryCache(maxEntries, maxBytes)
49+ return b
3250 }
3351
3452 // authorize returns nil if p may read the repository.
@@ -171,7 +189,22 @@ func (b *Browse) GetTree(ctx context.Context, p domain.Principal, repo domain.Re
171189 if err := b.authorize(ctx, p, repo); err != nil {
172190 return domain.Tree{}, err
173191 }
174- return b.reader.GetTree(ctx, repo, rev, path, includeLastCommit)
192+ // The last commits are the costly part (a history walk per entry) and
193+ // depend only on the commit and the path: cache them once the revision is
194+ // a full commit id. A name or HEAD can move, so it is never a key.
195+ if !includeLastCommit || !rev.IsObjectID() {
196+ return b.reader.GetTree(ctx, repo, rev, path, includeLastCommit)
197+ }
198+ key := historyKey{repo: repo, rev: rev.String(), path: path.String(), kind: kindTreeLast}
199+ if e, ok := b.history.get(key); ok {
200+ return cloneTree(e.tree), nil
201+ }
202+ tree, err := b.reader.GetTree(ctx, repo, rev, path, true)
203+ if err != nil {
204+ return domain.Tree{}, err
205+ }
206+ b.history.put(treeEntry(key, tree))
207+ return tree, nil
175208 }
176209
177210 // OpenBlob returns the description of a file and a reader of its content. The
@@ -224,7 +257,7 @@ func (b *Browse) ListCommits(ctx context.Context, p domain.Principal, repo domai
224257 if err := b.authorize(ctx, p, repo); err != nil {
225258 return CommitPage{}, err
226259 }
227- res, err := b.reader.ListCommits(ctx, repo, q)
260+ res, err := b.listCommits(ctx, repo, q)
228261 if err != nil {
229262 return CommitPage{}, err
230263 }
@@ -235,6 +268,25 @@ func (b *Browse) ListCommits(ctx context.Context, p domain.Principal, repo domai
235268 return out, nil
236269 }
237270
271+// listCommits asks the reader, through the history cache for the first
272+// commit of a path at a full commit id (the "last change" of a file): that
273+// answer is immutable.
274+func (b *Browse) listCommits(ctx context.Context, repo domain.RepoName, q ports.CommitQuery) (ports.CommitPage, error) {
275+ if q.Limit != 1 || q.Skip != 0 || !q.Revision.IsObjectID() {
276+ return b.reader.ListCommits(ctx, repo, q)
277+ }
278+ key := historyKey{repo: repo, rev: q.Revision.String(), path: q.Path.String(), kind: kindPage, limit: q.Limit}
279+ if e, ok := b.history.get(key); ok {
280+ return clonePage(e.page), nil
281+ }
282+ res, err := b.reader.ListCommits(ctx, repo, q)
283+ if err != nil {
284+ return ports.CommitPage{}, err
285+ }
286+ b.history.put(pageEntry(key, res))
287+ return res, nil
288+}
289+
238290 func pathDigest(path domain.TreePath) string {
239291 sum := sha256.Sum256([]byte(path.String()))
240292 return hex.EncodeToString(sum[:6])
@@ -24,11 +24,29 @@ import (
24 type Browse struct {24 type Browse struct {
25 meta ports.RepoMetaStore25 meta ports.RepoMetaStore
26 reader ports.RepoReader26 reader ports.RepoReader
27+ // history keeps the answers about a commit designated by its full id (a
28+ // tree with the last commit of its entries, the latest commit of a path):
29+ // they never change, and computing them costs a history walk. Nil
30+ // disables it.
31+ history *historyCache
27 }32 }
28 33
29-// NewBrowse builds the read use cases over their ports.34+// NewBrowse builds the read use cases over their ports, with a history cache
35+// of the default bounds.
30 func NewBrowse(meta ports.RepoMetaStore, reader ports.RepoReader) *Browse {36 func NewBrowse(meta ports.RepoMetaStore, reader ports.RepoReader) *Browse {
31- return &Browse{meta: meta, reader: reader}37+ return &Browse{
38+ meta: meta, reader: reader,
39+ history: newHistoryCache(DefaultHistoryCacheEntries, DefaultHistoryCacheBytes),
40+ }
41+}
42+
43+// WithHistoryCache replaces the bounds of the history cache: at most
44+// maxEntries answers and about maxBytes of memory, least recently used first
45+// out. A bound that is not positive disables the cache. Call it before the
46+// first request.
47+func (b *Browse) WithHistoryCache(maxEntries int, maxBytes int64) *Browse {
48+ b.history = newHistoryCache(maxEntries, maxBytes)
49+ return b
32 }50 }
33 51
34 // authorize returns nil if p may read the repository.52 // authorize returns nil if p may read the repository.
@@ -171,7 +189,22 @@ func (b *Browse) GetTree(ctx context.Context, p domain.Principal, repo domain.Re
171 if err := b.authorize(ctx, p, repo); err != nil {189 if err := b.authorize(ctx, p, repo); err != nil {
172 return domain.Tree{}, err190 return domain.Tree{}, err
173 }191 }
174- return b.reader.GetTree(ctx, repo, rev, path, includeLastCommit)192+ // The last commits are the costly part (a history walk per entry) and
193+ // depend only on the commit and the path: cache them once the revision is
194+ // a full commit id. A name or HEAD can move, so it is never a key.
195+ if !includeLastCommit || !rev.IsObjectID() {
196+ return b.reader.GetTree(ctx, repo, rev, path, includeLastCommit)
197+ }
198+ key := historyKey{repo: repo, rev: rev.String(), path: path.String(), kind: kindTreeLast}
199+ if e, ok := b.history.get(key); ok {
200+ return cloneTree(e.tree), nil
201+ }
202+ tree, err := b.reader.GetTree(ctx, repo, rev, path, true)
203+ if err != nil {
204+ return domain.Tree{}, err
205+ }
206+ b.history.put(treeEntry(key, tree))
207+ return tree, nil
175 }208 }
176 209
177 // OpenBlob returns the description of a file and a reader of its content. The210 // OpenBlob returns the description of a file and a reader of its content. The
@@ -224,7 +257,7 @@ func (b *Browse) ListCommits(ctx context.Context, p domain.Principal, repo domai
224 if err := b.authorize(ctx, p, repo); err != nil {257 if err := b.authorize(ctx, p, repo); err != nil {
225 return CommitPage{}, err258 return CommitPage{}, err
226 }259 }
227- res, err := b.reader.ListCommits(ctx, repo, q)260+ res, err := b.listCommits(ctx, repo, q)
228 if err != nil {261 if err != nil {
229 return CommitPage{}, err262 return CommitPage{}, err
230 }263 }
@@ -235,6 +268,25 @@ func (b *Browse) ListCommits(ctx context.Context, p domain.Principal, repo domai
235 return out, nil268 return out, nil
236 }269 }
237 270
271+// listCommits asks the reader, through the history cache for the first
272+// commit of a path at a full commit id (the "last change" of a file): that
273+// answer is immutable.
274+func (b *Browse) listCommits(ctx context.Context, repo domain.RepoName, q ports.CommitQuery) (ports.CommitPage, error) {
275+ if q.Limit != 1 || q.Skip != 0 || !q.Revision.IsObjectID() {
276+ return b.reader.ListCommits(ctx, repo, q)
277+ }
278+ key := historyKey{repo: repo, rev: q.Revision.String(), path: q.Path.String(), kind: kindPage, limit: q.Limit}
279+ if e, ok := b.history.get(key); ok {
280+ return clonePage(e.page), nil
281+ }
282+ res, err := b.reader.ListCommits(ctx, repo, q)
283+ if err != nil {
284+ return ports.CommitPage{}, err
285+ }
286+ b.history.put(pageEntry(key, res))
287+ return res, nil
288+}
289+
238 func pathDigest(path domain.TreePath) string {290 func pathDigest(path domain.TreePath) string {
239 sum := sha256.Sum256([]byte(path.String()))291 sum := sha256.Sum256([]byte(path.String()))
240 return hex.EncodeToString(sum[:6])292 return hex.EncodeToString(sum[:6])
A internal/app/browsecache.go +163 -0
@@ -0,0 +1,163 @@
1+// SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+package app
4+
5+import (
6+ "container/list"
7+ "slices"
8+ "sync"
9+
10+ "bnema.dev/ermite/internal/domain"
11+ "bnema.dev/ermite/internal/ports"
12+)
13+
14+// Default bounds of the history cache of Browse.
15+const (
16+ // DefaultHistoryCacheEntries bounds the cached answers.
17+ DefaultHistoryCacheEntries = 256
18+ // DefaultHistoryCacheBytes bounds their estimated size.
19+ DefaultHistoryCacheBytes = 8 << 20
20+)
21+
22+// historyKey names an answer that depends only on immutable data: a commit
23+// designated by its full id, the path inside it, and for a page of history
24+// where it starts and how long it is. The repository is part of the key so
25+// that an answer is never served across a name.
26+type historyKey struct {
27+ repo domain.RepoName
28+ rev string // full commit id
29+ path string
30+ kind historyKind
31+ limit int
32+}
33+
34+type historyKind uint8
35+
36+const (
37+ kindTreeLast historyKind = iota // a directory with the last commit of its entries
38+ kindPage // the first page of the history of a path
39+)
40+
41+// historyCache is a small LRU of answers about commits, bounded by entry count
42+// and by estimated bytes. The answers are immutable (a commit id fixes its
43+// whole history), so there is no invalidation: entries leave by eviction only.
44+// It is safe for concurrent use. Values are stored and returned as copies, so
45+// a caller cannot alter what another one gets.
46+type historyCache struct {
47+ maxEntries int
48+ maxBytes int64
49+
50+ mu sync.Mutex
51+ order *list.List // front is the most recently used
52+ items map[historyKey]*list.Element
53+ bytes int64
54+}
55+
56+type historyEntry struct {
57+ key historyKey
58+ tree domain.Tree
59+ page ports.CommitPage
60+ size int64
61+}
62+
63+// newHistoryCache returns nil (no cache) unless both bounds are positive.
64+func newHistoryCache(maxEntries int, maxBytes int64) *historyCache {
65+ if maxEntries <= 0 || maxBytes <= 0 {
66+ return nil
67+ }
68+ return &historyCache{
69+ maxEntries: maxEntries, maxBytes: maxBytes,
70+ order: list.New(), items: map[historyKey]*list.Element{},
71+ }
72+}
73+
74+// get returns the entry of key and marks it recently used.
75+func (c *historyCache) get(key historyKey) (*historyEntry, bool) {
76+ if c == nil {
77+ return nil, false
78+ }
79+ c.mu.Lock()
80+ defer c.mu.Unlock()
81+ el, ok := c.items[key]
82+ if !ok {
83+ return nil, false
84+ }
85+ c.order.MoveToFront(el)
86+ return el.Value.(*historyEntry), true
87+}
88+
89+// put stores e, then evicts the least recently used entries until both bounds
90+// hold. An entry larger than the byte bound is not stored.
91+func (c *historyCache) put(e *historyEntry) {
92+ if c == nil || e.size > c.maxBytes {
93+ return
94+ }
95+ c.mu.Lock()
96+ defer c.mu.Unlock()
97+ if el, ok := c.items[e.key]; ok {
98+ c.bytes -= el.Value.(*historyEntry).size
99+ c.order.Remove(el)
100+ }
101+ c.items[e.key] = c.order.PushFront(e)
102+ c.bytes += e.size
103+ for c.order.Len() > c.maxEntries || c.bytes > c.maxBytes {
104+ oldest := c.order.Back()
105+ old := oldest.Value.(*historyEntry)
106+ c.order.Remove(oldest)
107+ delete(c.items, old.key)
108+ c.bytes -= old.size
109+ }
110+}
111+
112+// treeEntry builds the entry of a tree; the tree is copied.
113+func treeEntry(key historyKey, t domain.Tree) *historyEntry {
114+ t = cloneTree(t)
115+ size := int64(256 + len(t.Entries)*160)
116+ for _, e := range t.Entries {
117+ size += int64(len(e.Name) + len(e.Path))
118+ if e.LastCommit != nil {
119+ size += commitSize(*e.LastCommit)
120+ }
121+ }
122+ return &historyEntry{key: key, tree: t, size: size}
123+}
124+
125+// pageEntry builds the entry of a page of history; the page is copied.
126+func pageEntry(key historyKey, p ports.CommitPage) *historyEntry {
127+ p = clonePage(p)
128+ size := int64(128)
129+ for _, c := range p.Commits {
130+ size += commitSize(c)
131+ }
132+ return &historyEntry{key: key, page: p, size: size}
133+}
134+
135+func commitSize(c domain.Commit) int64 {
136+ n := 384 + len(c.Parents)*48 + len(c.Summary) + len(c.Message)
137+ n += len(c.Author.Name) + len(c.Author.Email) + len(c.Committer.Name) + len(c.Committer.Email)
138+ return int64(n)
139+}
140+
141+func cloneCommit(c domain.Commit) domain.Commit {
142+ c.Parents = slices.Clone(c.Parents)
143+ return c
144+}
145+
146+func cloneTree(t domain.Tree) domain.Tree {
147+ t.Entries = slices.Clone(t.Entries)
148+ for i := range t.Entries {
149+ if lc := t.Entries[i].LastCommit; lc != nil {
150+ c := cloneCommit(*lc)
151+ t.Entries[i].LastCommit = &c
152+ }
153+ }
154+ return t
155+}
156+
157+func clonePage(p ports.CommitPage) ports.CommitPage {
158+ p.Commits = slices.Clone(p.Commits)
159+ for i := range p.Commits {
160+ p.Commits[i] = cloneCommit(p.Commits[i])
161+ }
162+ return p
163+}
@@ -0,0 +1,163 @@
1+// SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+package app
4+
5+import (
6+ "container/list"
7+ "slices"
8+ "sync"
9+
10+ "bnema.dev/ermite/internal/domain"
11+ "bnema.dev/ermite/internal/ports"
12+)
13+
14+// Default bounds of the history cache of Browse.
15+const (
16+ // DefaultHistoryCacheEntries bounds the cached answers.
17+ DefaultHistoryCacheEntries = 256
18+ // DefaultHistoryCacheBytes bounds their estimated size.
19+ DefaultHistoryCacheBytes = 8 << 20
20+)
21+
22+// historyKey names an answer that depends only on immutable data: a commit
23+// designated by its full id, the path inside it, and for a page of history
24+// where it starts and how long it is. The repository is part of the key so
25+// that an answer is never served across a name.
26+type historyKey struct {
27+ repo domain.RepoName
28+ rev string // full commit id
29+ path string
30+ kind historyKind
31+ limit int
32+}
33+
34+type historyKind uint8
35+
36+const (
37+ kindTreeLast historyKind = iota // a directory with the last commit of its entries
38+ kindPage // the first page of the history of a path
39+)
40+
41+// historyCache is a small LRU of answers about commits, bounded by entry count
42+// and by estimated bytes. The answers are immutable (a commit id fixes its
43+// whole history), so there is no invalidation: entries leave by eviction only.
44+// It is safe for concurrent use. Values are stored and returned as copies, so
45+// a caller cannot alter what another one gets.
46+type historyCache struct {
47+ maxEntries int
48+ maxBytes int64
49+
50+ mu sync.Mutex
51+ order *list.List // front is the most recently used
52+ items map[historyKey]*list.Element
53+ bytes int64
54+}
55+
56+type historyEntry struct {
57+ key historyKey
58+ tree domain.Tree
59+ page ports.CommitPage
60+ size int64
61+}
62+
63+// newHistoryCache returns nil (no cache) unless both bounds are positive.
64+func newHistoryCache(maxEntries int, maxBytes int64) *historyCache {
65+ if maxEntries <= 0 || maxBytes <= 0 {
66+ return nil
67+ }
68+ return &historyCache{
69+ maxEntries: maxEntries, maxBytes: maxBytes,
70+ order: list.New(), items: map[historyKey]*list.Element{},
71+ }
72+}
73+
74+// get returns the entry of key and marks it recently used.
75+func (c *historyCache) get(key historyKey) (*historyEntry, bool) {
76+ if c == nil {
77+ return nil, false
78+ }
79+ c.mu.Lock()
80+ defer c.mu.Unlock()
81+ el, ok := c.items[key]
82+ if !ok {
83+ return nil, false
84+ }
85+ c.order.MoveToFront(el)
86+ return el.Value.(*historyEntry), true
87+}
88+
89+// put stores e, then evicts the least recently used entries until both bounds
90+// hold. An entry larger than the byte bound is not stored.
91+func (c *historyCache) put(e *historyEntry) {
92+ if c == nil || e.size > c.maxBytes {
93+ return
94+ }
95+ c.mu.Lock()
96+ defer c.mu.Unlock()
97+ if el, ok := c.items[e.key]; ok {
98+ c.bytes -= el.Value.(*historyEntry).size
99+ c.order.Remove(el)
100+ }
101+ c.items[e.key] = c.order.PushFront(e)
102+ c.bytes += e.size
103+ for c.order.Len() > c.maxEntries || c.bytes > c.maxBytes {
104+ oldest := c.order.Back()
105+ old := oldest.Value.(*historyEntry)
106+ c.order.Remove(oldest)
107+ delete(c.items, old.key)
108+ c.bytes -= old.size
109+ }
110+}
111+
112+// treeEntry builds the entry of a tree; the tree is copied.
113+func treeEntry(key historyKey, t domain.Tree) *historyEntry {
114+ t = cloneTree(t)
115+ size := int64(256 + len(t.Entries)*160)
116+ for _, e := range t.Entries {
117+ size += int64(len(e.Name) + len(e.Path))
118+ if e.LastCommit != nil {
119+ size += commitSize(*e.LastCommit)
120+ }
121+ }
122+ return &historyEntry{key: key, tree: t, size: size}
123+}
124+
125+// pageEntry builds the entry of a page of history; the page is copied.
126+func pageEntry(key historyKey, p ports.CommitPage) *historyEntry {
127+ p = clonePage(p)
128+ size := int64(128)
129+ for _, c := range p.Commits {
130+ size += commitSize(c)
131+ }
132+ return &historyEntry{key: key, page: p, size: size}
133+}
134+
135+func commitSize(c domain.Commit) int64 {
136+ n := 384 + len(c.Parents)*48 + len(c.Summary) + len(c.Message)
137+ n += len(c.Author.Name) + len(c.Author.Email) + len(c.Committer.Name) + len(c.Committer.Email)
138+ return int64(n)
139+}
140+
141+func cloneCommit(c domain.Commit) domain.Commit {
142+ c.Parents = slices.Clone(c.Parents)
143+ return c
144+}
145+
146+func cloneTree(t domain.Tree) domain.Tree {
147+ t.Entries = slices.Clone(t.Entries)
148+ for i := range t.Entries {
149+ if lc := t.Entries[i].LastCommit; lc != nil {
150+ c := cloneCommit(*lc)
151+ t.Entries[i].LastCommit = &c
152+ }
153+ }
154+ return t
155+}
156+
157+func clonePage(p ports.CommitPage) ports.CommitPage {
158+ p.Commits = slices.Clone(p.Commits)
159+ for i := range p.Commits {
160+ p.Commits[i] = cloneCommit(p.Commits[i])
161+ }
162+ return p
163+}
A internal/app/browsecache_test.go +296 -0
@@ -0,0 +1,296 @@
1+// SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+package app_test
4+
5+import (
6+ "context"
7+ "strings"
8+ "sync"
9+ "testing"
10+
11+ "github.com/stretchr/testify/assert"
12+ "github.com/stretchr/testify/mock"
13+ "github.com/stretchr/testify/require"
14+
15+ "bnema.dev/ermite/internal/app"
16+ "bnema.dev/ermite/internal/domain"
17+ "bnema.dev/ermite/internal/ports"
18+)
19+
20+func treePath(t *testing.T, s string) domain.TreePath {
21+ t.Helper()
22+ p, err := domain.ParseTreePath(s)
23+ require.NoError(t, err)
24+ return p
25+}
26+
27+func treeWithLast(c string) domain.Tree {
28+ commit := domain.Commit{ID: oid(c), Parents: []domain.ObjectID{oid("0")}, Summary: "s"}
29+ return domain.Tree{
30+ CommitID: oid("a"),
31+ Entries: []domain.TreeEntry{{Name: "f", Path: "d/f", LastCommit: &commit}},
32+ }
33+}
34+
35+// TestGetTreeLastCommitCache covers which GetTree calls are served from the
36+// cache: only the ones with last commits at a full commit id.
37+func TestGetTreeLastCommitCache(t *testing.T) {
38+ ctx := context.Background()
39+ pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic}
40+ full := domain.RevisionOf(oid("a"))
41+ branch, err := domain.ParseRevision("main")
42+ require.NoError(t, err)
43+ dir := treePath(t, "d")
44+
45+ tests := []struct {
46+ name string
47+ rev domain.Revision
48+ includeLast bool
49+ wantReaderOn int // reader calls for two identical requests
50+ }{
51+ {"full id with last commits is cached", full, true, 1},
52+ {"full id without last commits is not", full, false, 2},
53+ {"branch name is not", branch, true, 2},
54+ {"HEAD is not", domain.Revision{}, true, 2},
55+ }
56+ for _, tc := range tests {
57+ t.Run(tc.name, func(t *testing.T) {
58+ f := newBrowseFixture(t)
59+ f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil)
60+ f.reader.EXPECT().GetTree(mock.Anything, pub.Name, tc.rev, dir, tc.includeLast).
61+ Return(treeWithLast("b"), nil).Times(tc.wantReaderOn)
62+ for range 2 {
63+ tree, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, tc.rev, dir, tc.includeLast)
64+ require.NoError(t, err)
65+ assert.Equal(t, "d/f", tree.Entries[0].Path)
66+ }
67+ })
68+ }
69+}
70+
71+func TestGetTreeLastCommitCacheKey(t *testing.T) {
72+ ctx := context.Background()
73+ pubA := domain.Repo{Name: name(t, "a"), Visibility: domain.VisibilityPublic}
74+ pubB := domain.Repo{Name: name(t, "b"), Visibility: domain.VisibilityPublic}
75+ revA, revB := domain.RevisionOf(oid("a")), domain.RevisionOf(oid("b"))
76+ d1, d2 := treePath(t, "d1"), treePath(t, "d2")
77+
78+ // Each distinct (repo, commit, path) is one reader call, then a hit.
79+ keys := []struct {
80+ repo domain.Repo
81+ rev domain.Revision
82+ path domain.TreePath
83+ }{
84+ {pubA, revA, d1}, {pubB, revA, d1}, {pubA, revB, d1}, {pubA, revA, d2}, {pubA, revA, domain.TreePath{}},
85+ }
86+ f := newBrowseFixture(t)
87+ f.meta.EXPECT().Get(mock.Anything, mock.Anything).RunAndReturn(
88+ func(_ context.Context, n domain.RepoName) (domain.Repo, error) {
89+ return domain.Repo{Name: n, Visibility: domain.VisibilityPublic}, nil
90+ })
91+ for _, k := range keys {
92+ f.reader.EXPECT().GetTree(mock.Anything, k.repo.Name, k.rev, k.path, true).Return(treeWithLast("b"), nil).Once()
93+ }
94+ for range 2 {
95+ for _, k := range keys {
96+ _, err := f.browse.GetTree(ctx, domain.Anonymous(), k.repo.Name, k.rev, k.path, true)
97+ require.NoError(t, err)
98+ }
99+ }
100+}
101+
102+// TestGetTreeCacheKeepsAccessRule: a hit never skips the access check, and an
103+// error is not cached.
104+func TestGetTreeCacheKeepsAccessRule(t *testing.T) {
105+ ctx := context.Background()
106+ priv := domain.Repo{Name: name(t, "priv"), Visibility: domain.VisibilityPrivate}
107+ rev, dir := domain.RevisionOf(oid("a")), treePath(t, "d")
108+
109+ f := newBrowseFixture(t)
110+ f.meta.EXPECT().Get(mock.Anything, priv.Name).Return(priv, nil)
111+ f.reader.EXPECT().GetTree(mock.Anything, priv.Name, rev, dir, true).Return(domain.Tree{}, domain.ErrNotDirectory).Once()
112+ f.reader.EXPECT().GetTree(mock.Anything, priv.Name, rev, dir, true).Return(treeWithLast("b"), nil).Once()
113+
114+ _, err := f.browse.GetTree(ctx, domain.Root(), priv.Name, rev, dir, true)
115+ require.ErrorIs(t, err, domain.ErrNotDirectory) // not cached
116+ _, err = f.browse.GetTree(ctx, domain.Root(), priv.Name, rev, dir, true)
117+ require.NoError(t, err) // now cached
118+ _, err = f.browse.GetTree(ctx, domain.Anonymous(), priv.Name, rev, dir, true)
119+ assert.ErrorIs(t, err, domain.ErrNotFound)
120+}
121+
122+// TestGetTreeCacheReturnsCopies: a caller that edits its tree does not alter
123+// what the next one gets.
124+func TestGetTreeCacheReturnsCopies(t *testing.T) {
125+ ctx := context.Background()
126+ pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic}
127+ rev, dir := domain.RevisionOf(oid("a")), treePath(t, "d")
128+ f := newBrowseFixture(t)
129+ f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil)
130+ f.reader.EXPECT().GetTree(mock.Anything, pub.Name, rev, dir, true).Return(treeWithLast("b"), nil).Once()
131+
132+ for range 3 {
133+ tree, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, rev, dir, true)
134+ require.NoError(t, err)
135+ require.Len(t, tree.Entries, 1)
136+ assert.Equal(t, "s", tree.Entries[0].LastCommit.Summary)
137+ tree.Entries[0].LastCommit.Summary = "changed"
138+ tree.Entries[0].LastCommit.Parents[0] = oid("f")
139+ tree.Entries[0].Name = "changed"
140+ }
141+}
142+
143+// TestHistoryCacheBounds: the entry bound and the byte bound evict the least
144+// recently used answer.
145+func TestHistoryCacheBounds(t *testing.T) {
146+ ctx := context.Background()
147+ pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic}
148+ paths := []string{"p0", "p1", "p2"}
149+
150+ bigTree := treeWithLast("b")
151+ bigTree.Entries[0].Name = strings.Repeat("x", 4096)
152+
153+ tests := []struct {
154+ name string
155+ entries int
156+ bytes int64
157+ tree domain.Tree
158+ wantCached []bool // after requesting p0, p1, p2 (and touching p0 before p2 when touch)
159+ touch bool
160+ }{
161+ {"entry bound", 2, 1 << 20, treeWithLast("b"), []bool{false, true, true}, false},
162+ {"entry bound is LRU", 2, 1 << 20, treeWithLast("b"), []bool{true, false, true}, true},
163+ {"byte bound", 100, 6000, bigTree, []bool{false, false, true}, false},
164+ {"entry larger than the byte bound", 100, 1000, bigTree, []bool{false, false, false}, false},
165+ {"disabled by entries", 0, 1 << 20, treeWithLast("b"), []bool{false, false, false}, false},
166+ {"disabled by bytes", 10, 0, treeWithLast("b"), []bool{false, false, false}, false},
167+ }
168+ for _, tc := range tests {
169+ t.Run(tc.name, func(t *testing.T) {
170+ f := newBrowseFixture(t)
171+ f.browse.WithHistoryCache(tc.entries, tc.bytes)
172+ f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil)
173+ rev := domain.RevisionOf(oid("a"))
174+ calls := map[string]int{}
175+ f.reader.EXPECT().GetTree(mock.Anything, pub.Name, rev, mock.Anything, true).
176+ RunAndReturn(func(_ context.Context, _ domain.RepoName, _ domain.Revision, p domain.TreePath, _ bool) (domain.Tree, error) {
177+ calls[p.String()]++
178+ return tc.tree, nil
179+ })
180+ get := func(p string) {
181+ _, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, rev, treePath(t, p), true)
182+ require.NoError(t, err)
183+ }
184+ get("p0")
185+ get("p1")
186+ if tc.touch {
187+ get("p0")
188+ }
189+ get("p2")
190+ before := map[string]int{"p0": calls["p0"], "p1": calls["p1"], "p2": calls["p2"]}
191+ // Probe from the newest to the oldest so probing one does not evict
192+ // the next one asked: asking a miss stores it.
193+ for i := len(paths) - 1; i >= 0; i-- {
194+ if !tc.wantCached[i] {
195+ continue
196+ }
197+ get(paths[i])
198+ assert.Equal(t, before[paths[i]], calls[paths[i]], "%s should be a hit", paths[i])
199+ }
200+ for i, p := range paths {
201+ if tc.wantCached[i] {
202+ continue
203+ }
204+ get(p)
205+ assert.Equal(t, before[p]+1, calls[p], "%s should be a miss", p)
206+ break // a miss stores the entry and may evict the others
207+ }
208+ })
209+ }
210+}
211+
212+// TestListCommitsLastChangeCache: the latest commit of a path at a full
213+// commit id is cached; every other page shape is not.
214+func TestListCommitsLastChangeCache(t *testing.T) {
215+ ctx := context.Background()
216+ pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic}
217+ full := domain.RevisionOf(oid("a"))
218+ main, err := domain.ParseRevision("main")
219+ require.NoError(t, err)
220+ file := treePath(t, "d/file.go")
221+
222+ tests := []struct {
223+ name string
224+ rev domain.Revision
225+ path domain.TreePath
226+ size int
227+ calls int // reader calls for two identical requests
228+ }{
229+ {"last change at a full id", full, file, 1, 1},
230+ {"last change of the root at a full id", full, domain.TreePath{}, 1, 1},
231+ {"name", main, file, 1, 2},
232+ {"HEAD", domain.Revision{}, file, 1, 2},
233+ {"larger page", full, file, 2, 2},
234+ }
235+ for _, tc := range tests {
236+ t.Run(tc.name, func(t *testing.T) {
237+ f := newBrowseFixture(t)
238+ f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil)
239+ q := ports.CommitQuery{Revision: tc.rev, Path: tc.path, Limit: tc.size}
240+ f.reader.EXPECT().ListCommits(mock.Anything, pub.Name, q).Return(ports.CommitPage{
241+ Head: oid("a"),
242+ Commits: []domain.Commit{{ID: oid("c"), Parents: []domain.ObjectID{oid("d")}}},
243+ }, nil).Times(tc.calls)
244+ for range 2 {
245+ page, err := f.browse.ListCommits(ctx, domain.Anonymous(), pub.Name,
246+ app.ListCommitsParams{Revision: tc.rev, Path: tc.path, PageSize: tc.size})
247+ require.NoError(t, err)
248+ require.Len(t, page.Commits, 1)
249+ page.Commits[0].Parents[0] = oid("f") // must not leak into the cache
250+ }
251+ })
252+ }
253+
254+ t.Run("a following page is not cached", func(t *testing.T) {
255+ f := newBrowseFixture(t)
256+ f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil)
257+ f.reader.EXPECT().ListCommits(mock.Anything, pub.Name, ports.CommitQuery{Revision: full, Path: file, Limit: 1}).
258+ Return(ports.CommitPage{Head: oid("a"), Commits: []domain.Commit{{ID: oid("c")}}, HasMore: true}, nil).Once()
259+ first, err := f.browse.ListCommits(ctx, domain.Anonymous(), pub.Name, app.ListCommitsParams{Revision: full, Path: file, PageSize: 1})
260+ require.NoError(t, err)
261+ require.NotEmpty(t, first.NextPageToken)
262+ f.reader.EXPECT().ListCommits(mock.Anything, pub.Name, ports.CommitQuery{Revision: domain.RevisionOf(oid("a")), Path: file, Skip: 1, Limit: 1}).
263+ Return(ports.CommitPage{Head: oid("a")}, nil).Twice()
264+ for range 2 {
265+ _, err = f.browse.ListCommits(ctx, domain.Anonymous(), pub.Name, app.ListCommitsParams{Path: file, PageSize: 1, PageToken: first.NextPageToken})
266+ require.NoError(t, err)
267+ }
268+ })
269+}
270+
271+func TestHistoryCacheConcurrent(t *testing.T) {
272+ ctx := context.Background()
273+ pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic}
274+ f := newBrowseFixture(t)
275+ f.browse.WithHistoryCache(4, 1<<20)
276+ f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil)
277+ f.reader.EXPECT().GetTree(mock.Anything, pub.Name, mock.Anything, mock.Anything, true).Return(treeWithLast("b"), nil)
278+ paths := []string{"a", "b", "c", "d", "e", "f", "g"}
279+ var wg sync.WaitGroup
280+ for w := range 8 {
281+ wg.Go(func() {
282+ for i := range 200 {
283+ p, err := domain.ParseTreePath(paths[(i+w)%len(paths)])
284+ if err != nil {
285+ t.Error(err)
286+ return
287+ }
288+ if _, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, domain.RevisionOf(oid("a")), p, true); err != nil {
289+ t.Error(err)
290+ return
291+ }
292+ }
293+ })
294+ }
295+ wg.Wait()
296+}
@@ -0,0 +1,296 @@
1+// SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+package app_test
4+
5+import (
6+ "context"
7+ "strings"
8+ "sync"
9+ "testing"
10+
11+ "github.com/stretchr/testify/assert"
12+ "github.com/stretchr/testify/mock"
13+ "github.com/stretchr/testify/require"
14+
15+ "bnema.dev/ermite/internal/app"
16+ "bnema.dev/ermite/internal/domain"
17+ "bnema.dev/ermite/internal/ports"
18+)
19+
20+func treePath(t *testing.T, s string) domain.TreePath {
21+ t.Helper()
22+ p, err := domain.ParseTreePath(s)
23+ require.NoError(t, err)
24+ return p
25+}
26+
27+func treeWithLast(c string) domain.Tree {
28+ commit := domain.Commit{ID: oid(c), Parents: []domain.ObjectID{oid("0")}, Summary: "s"}
29+ return domain.Tree{
30+ CommitID: oid("a"),
31+ Entries: []domain.TreeEntry{{Name: "f", Path: "d/f", LastCommit: &commit}},
32+ }
33+}
34+
35+// TestGetTreeLastCommitCache covers which GetTree calls are served from the
36+// cache: only the ones with last commits at a full commit id.
37+func TestGetTreeLastCommitCache(t *testing.T) {
38+ ctx := context.Background()
39+ pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic}
40+ full := domain.RevisionOf(oid("a"))
41+ branch, err := domain.ParseRevision("main")
42+ require.NoError(t, err)
43+ dir := treePath(t, "d")
44+
45+ tests := []struct {
46+ name string
47+ rev domain.Revision
48+ includeLast bool
49+ wantReaderOn int // reader calls for two identical requests
50+ }{
51+ {"full id with last commits is cached", full, true, 1},
52+ {"full id without last commits is not", full, false, 2},
53+ {"branch name is not", branch, true, 2},
54+ {"HEAD is not", domain.Revision{}, true, 2},
55+ }
56+ for _, tc := range tests {
57+ t.Run(tc.name, func(t *testing.T) {
58+ f := newBrowseFixture(t)
59+ f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil)
60+ f.reader.EXPECT().GetTree(mock.Anything, pub.Name, tc.rev, dir, tc.includeLast).
61+ Return(treeWithLast("b"), nil).Times(tc.wantReaderOn)
62+ for range 2 {
63+ tree, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, tc.rev, dir, tc.includeLast)
64+ require.NoError(t, err)
65+ assert.Equal(t, "d/f", tree.Entries[0].Path)
66+ }
67+ })
68+ }
69+}
70+
71+func TestGetTreeLastCommitCacheKey(t *testing.T) {
72+ ctx := context.Background()
73+ pubA := domain.Repo{Name: name(t, "a"), Visibility: domain.VisibilityPublic}
74+ pubB := domain.Repo{Name: name(t, "b"), Visibility: domain.VisibilityPublic}
75+ revA, revB := domain.RevisionOf(oid("a")), domain.RevisionOf(oid("b"))
76+ d1, d2 := treePath(t, "d1"), treePath(t, "d2")
77+
78+ // Each distinct (repo, commit, path) is one reader call, then a hit.
79+ keys := []struct {
80+ repo domain.Repo
81+ rev domain.Revision
82+ path domain.TreePath
83+ }{
84+ {pubA, revA, d1}, {pubB, revA, d1}, {pubA, revB, d1}, {pubA, revA, d2}, {pubA, revA, domain.TreePath{}},
85+ }
86+ f := newBrowseFixture(t)
87+ f.meta.EXPECT().Get(mock.Anything, mock.Anything).RunAndReturn(
88+ func(_ context.Context, n domain.RepoName) (domain.Repo, error) {
89+ return domain.Repo{Name: n, Visibility: domain.VisibilityPublic}, nil
90+ })
91+ for _, k := range keys {
92+ f.reader.EXPECT().GetTree(mock.Anything, k.repo.Name, k.rev, k.path, true).Return(treeWithLast("b"), nil).Once()
93+ }
94+ for range 2 {
95+ for _, k := range keys {
96+ _, err := f.browse.GetTree(ctx, domain.Anonymous(), k.repo.Name, k.rev, k.path, true)
97+ require.NoError(t, err)
98+ }
99+ }
100+}
101+
102+// TestGetTreeCacheKeepsAccessRule: a hit never skips the access check, and an
103+// error is not cached.
104+func TestGetTreeCacheKeepsAccessRule(t *testing.T) {
105+ ctx := context.Background()
106+ priv := domain.Repo{Name: name(t, "priv"), Visibility: domain.VisibilityPrivate}
107+ rev, dir := domain.RevisionOf(oid("a")), treePath(t, "d")
108+
109+ f := newBrowseFixture(t)
110+ f.meta.EXPECT().Get(mock.Anything, priv.Name).Return(priv, nil)
111+ f.reader.EXPECT().GetTree(mock.Anything, priv.Name, rev, dir, true).Return(domain.Tree{}, domain.ErrNotDirectory).Once()
112+ f.reader.EXPECT().GetTree(mock.Anything, priv.Name, rev, dir, true).Return(treeWithLast("b"), nil).Once()
113+
114+ _, err := f.browse.GetTree(ctx, domain.Root(), priv.Name, rev, dir, true)
115+ require.ErrorIs(t, err, domain.ErrNotDirectory) // not cached
116+ _, err = f.browse.GetTree(ctx, domain.Root(), priv.Name, rev, dir, true)
117+ require.NoError(t, err) // now cached
118+ _, err = f.browse.GetTree(ctx, domain.Anonymous(), priv.Name, rev, dir, true)
119+ assert.ErrorIs(t, err, domain.ErrNotFound)
120+}
121+
122+// TestGetTreeCacheReturnsCopies: a caller that edits its tree does not alter
123+// what the next one gets.
124+func TestGetTreeCacheReturnsCopies(t *testing.T) {
125+ ctx := context.Background()
126+ pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic}
127+ rev, dir := domain.RevisionOf(oid("a")), treePath(t, "d")
128+ f := newBrowseFixture(t)
129+ f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil)
130+ f.reader.EXPECT().GetTree(mock.Anything, pub.Name, rev, dir, true).Return(treeWithLast("b"), nil).Once()
131+
132+ for range 3 {
133+ tree, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, rev, dir, true)
134+ require.NoError(t, err)
135+ require.Len(t, tree.Entries, 1)
136+ assert.Equal(t, "s", tree.Entries[0].LastCommit.Summary)
137+ tree.Entries[0].LastCommit.Summary = "changed"
138+ tree.Entries[0].LastCommit.Parents[0] = oid("f")
139+ tree.Entries[0].Name = "changed"
140+ }
141+}
142+
143+// TestHistoryCacheBounds: the entry bound and the byte bound evict the least
144+// recently used answer.
145+func TestHistoryCacheBounds(t *testing.T) {
146+ ctx := context.Background()
147+ pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic}
148+ paths := []string{"p0", "p1", "p2"}
149+
150+ bigTree := treeWithLast("b")
151+ bigTree.Entries[0].Name = strings.Repeat("x", 4096)
152+
153+ tests := []struct {
154+ name string
155+ entries int
156+ bytes int64
157+ tree domain.Tree
158+ wantCached []bool // after requesting p0, p1, p2 (and touching p0 before p2 when touch)
159+ touch bool
160+ }{
161+ {"entry bound", 2, 1 << 20, treeWithLast("b"), []bool{false, true, true}, false},
162+ {"entry bound is LRU", 2, 1 << 20, treeWithLast("b"), []bool{true, false, true}, true},
163+ {"byte bound", 100, 6000, bigTree, []bool{false, false, true}, false},
164+ {"entry larger than the byte bound", 100, 1000, bigTree, []bool{false, false, false}, false},
165+ {"disabled by entries", 0, 1 << 20, treeWithLast("b"), []bool{false, false, false}, false},
166+ {"disabled by bytes", 10, 0, treeWithLast("b"), []bool{false, false, false}, false},
167+ }
168+ for _, tc := range tests {
169+ t.Run(tc.name, func(t *testing.T) {
170+ f := newBrowseFixture(t)
171+ f.browse.WithHistoryCache(tc.entries, tc.bytes)
172+ f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil)
173+ rev := domain.RevisionOf(oid("a"))
174+ calls := map[string]int{}
175+ f.reader.EXPECT().GetTree(mock.Anything, pub.Name, rev, mock.Anything, true).
176+ RunAndReturn(func(_ context.Context, _ domain.RepoName, _ domain.Revision, p domain.TreePath, _ bool) (domain.Tree, error) {
177+ calls[p.String()]++
178+ return tc.tree, nil
179+ })
180+ get := func(p string) {
181+ _, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, rev, treePath(t, p), true)
182+ require.NoError(t, err)
183+ }
184+ get("p0")
185+ get("p1")
186+ if tc.touch {
187+ get("p0")
188+ }
189+ get("p2")
190+ before := map[string]int{"p0": calls["p0"], "p1": calls["p1"], "p2": calls["p2"]}
191+ // Probe from the newest to the oldest so probing one does not evict
192+ // the next one asked: asking a miss stores it.
193+ for i := len(paths) - 1; i >= 0; i-- {
194+ if !tc.wantCached[i] {
195+ continue
196+ }
197+ get(paths[i])
198+ assert.Equal(t, before[paths[i]], calls[paths[i]], "%s should be a hit", paths[i])
199+ }
200+ for i, p := range paths {
201+ if tc.wantCached[i] {
202+ continue
203+ }
204+ get(p)
205+ assert.Equal(t, before[p]+1, calls[p], "%s should be a miss", p)
206+ break // a miss stores the entry and may evict the others
207+ }
208+ })
209+ }
210+}
211+
212+// TestListCommitsLastChangeCache: the latest commit of a path at a full
213+// commit id is cached; every other page shape is not.
214+func TestListCommitsLastChangeCache(t *testing.T) {
215+ ctx := context.Background()
216+ pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic}
217+ full := domain.RevisionOf(oid("a"))
218+ main, err := domain.ParseRevision("main")
219+ require.NoError(t, err)
220+ file := treePath(t, "d/file.go")
221+
222+ tests := []struct {
223+ name string
224+ rev domain.Revision
225+ path domain.TreePath
226+ size int
227+ calls int // reader calls for two identical requests
228+ }{
229+ {"last change at a full id", full, file, 1, 1},
230+ {"last change of the root at a full id", full, domain.TreePath{}, 1, 1},
231+ {"name", main, file, 1, 2},
232+ {"HEAD", domain.Revision{}, file, 1, 2},
233+ {"larger page", full, file, 2, 2},
234+ }
235+ for _, tc := range tests {
236+ t.Run(tc.name, func(t *testing.T) {
237+ f := newBrowseFixture(t)
238+ f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil)
239+ q := ports.CommitQuery{Revision: tc.rev, Path: tc.path, Limit: tc.size}
240+ f.reader.EXPECT().ListCommits(mock.Anything, pub.Name, q).Return(ports.CommitPage{
241+ Head: oid("a"),
242+ Commits: []domain.Commit{{ID: oid("c"), Parents: []domain.ObjectID{oid("d")}}},
243+ }, nil).Times(tc.calls)
244+ for range 2 {
245+ page, err := f.browse.ListCommits(ctx, domain.Anonymous(), pub.Name,
246+ app.ListCommitsParams{Revision: tc.rev, Path: tc.path, PageSize: tc.size})
247+ require.NoError(t, err)
248+ require.Len(t, page.Commits, 1)
249+ page.Commits[0].Parents[0] = oid("f") // must not leak into the cache
250+ }
251+ })
252+ }
253+
254+ t.Run("a following page is not cached", func(t *testing.T) {
255+ f := newBrowseFixture(t)
256+ f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil)
257+ f.reader.EXPECT().ListCommits(mock.Anything, pub.Name, ports.CommitQuery{Revision: full, Path: file, Limit: 1}).
258+ Return(ports.CommitPage{Head: oid("a"), Commits: []domain.Commit{{ID: oid("c")}}, HasMore: true}, nil).Once()
259+ first, err := f.browse.ListCommits(ctx, domain.Anonymous(), pub.Name, app.ListCommitsParams{Revision: full, Path: file, PageSize: 1})
260+ require.NoError(t, err)
261+ require.NotEmpty(t, first.NextPageToken)
262+ f.reader.EXPECT().ListCommits(mock.Anything, pub.Name, ports.CommitQuery{Revision: domain.RevisionOf(oid("a")), Path: file, Skip: 1, Limit: 1}).
263+ Return(ports.CommitPage{Head: oid("a")}, nil).Twice()
264+ for range 2 {
265+ _, err = f.browse.ListCommits(ctx, domain.Anonymous(), pub.Name, app.ListCommitsParams{Path: file, PageSize: 1, PageToken: first.NextPageToken})
266+ require.NoError(t, err)
267+ }
268+ })
269+}
270+
271+func TestHistoryCacheConcurrent(t *testing.T) {
272+ ctx := context.Background()
273+ pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic}
274+ f := newBrowseFixture(t)
275+ f.browse.WithHistoryCache(4, 1<<20)
276+ f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil)
277+ f.reader.EXPECT().GetTree(mock.Anything, pub.Name, mock.Anything, mock.Anything, true).Return(treeWithLast("b"), nil)
278+ paths := []string{"a", "b", "c", "d", "e", "f", "g"}
279+ var wg sync.WaitGroup
280+ for w := range 8 {
281+ wg.Go(func() {
282+ for i := range 200 {
283+ p, err := domain.ParseTreePath(paths[(i+w)%len(paths)])
284+ if err != nil {
285+ t.Error(err)
286+ return
287+ }
288+ if _, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, domain.RevisionOf(oid("a")), p, true); err != nil {
289+ t.Error(err)
290+ return
291+ }
292+ }
293+ })
294+ }
295+ wg.Wait()
296+}

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 2

Tags 45

↑↓ moveenter openesc close