perf(browse): route warm reads to ermite-git and cache immutable last-commit results
- ListRefs, ResolveRevision, ResolveRefPath and tree last-commit listings (root and subdirectory) now go to ermite-git; routing weighs warm time and bytes instead of cold allocation counts - app.Browse caches last-commit tree listings and single-commit path lookups keyed by repo, full commit id and path, in a bounded LRU (256 entries, 8 MiB)
- commit
- c33e43c9e86c0234ed243b028e7bd62c10935077
- author
- brice · · 2026-10-11 18:40 UTC
- parent
- e42fec5
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
5 files · +577 -47
M internal/adapters/ermitegit/ermitegit.go +11 -4
| @@ -87,9 +87,13 @@ type Routes struct { | |||
| 87 | 87 | ResolveRefPath Backend | |
| 88 | 88 | // GetTree is a directory listing without last commits. | |
| 89 | 89 | GetTree Backend | |
| 90 | - | // GetTreeLastCommits is a listing with the last commit of each entry. | |
| 90 | + | // GetTreeLastCommits is the listing of the root with the last commit of | |
| 91 | + | // each entry. | |
| 91 | 92 | GetTreeLastCommits Backend | |
| 92 | - | OpenBlob Backend | |
| 93 | + | // GetTreeSubdirLastCommits is the same for a subdirectory, which costs a | |
| 94 | + | // much longer history walk (195 ms with git on lazygit). | |
| 95 | + | GetTreeSubdirLastCommits Backend | |
| 96 | + | OpenBlob Backend | |
| 93 | 97 | // ListCommits is a page of history of a revision. | |
| 94 | 98 | ListCommits Backend | |
| 95 | 99 | // ListCommitsPath is a page of the history of a file or a directory. | |
| @@ -99,7 +103,7 @@ type Routes struct { | |||
| 99 | 103 | // AllRoutes returns routes that send every method to b. The tests and the | |
| 100 | 104 | // benchmarks use it to exercise a backend whatever the production routes are. | |
| 101 | 105 | func AllRoutes(b Backend) Routes { | |
| 102 | - | return Routes{b, b, b, b, b, b, b, b, b} | |
| 106 | + | return Routes{b, b, b, b, b, b, b, b, b, b} | |
| 103 | 107 | } | |
| 104 | 108 | ||
| 105 | 109 | // Config configures an Adapter. | |
| @@ -320,7 +324,10 @@ func (a *Adapter) GetTree(ctx context.Context, repo domain.RepoName, rev domain. | |||
| 320 | 324 | cli := func() (domain.Tree, error) { return a.cli.GetTree(ctx, repo, rev, path, includeLastCommit) } | |
| 321 | 325 | route := a.routes.GetTree | |
| 322 | 326 | if includeLastCommit { | |
| 323 | - | route = a.routes.GetTreeLastCommits | |
| 327 | + | route = a.routes.GetTreeSubdirLastCommits | |
| 328 | + | if path.IsRoot() { | |
| 329 | + | route = a.routes.GetTreeLastCommits | |
| 330 | + | } | |
| 324 | 331 | } | |
| 325 | 332 | if route == CLI { | |
| 326 | 333 | return cli() | |
| @@ -87,9 +87,13 @@ type Routes struct { | |||||
| 87 | ResolveRefPath Backend | 87 | ResolveRefPath Backend | ||
| 88 | // GetTree is a directory listing without last commits. | 88 | // GetTree is a directory listing without last commits. | ||
| 89 | GetTree Backend | 89 | GetTree Backend | ||
| 90 | - | // GetTreeLastCommits is a listing with the last commit of each entry. | 90 | + | // GetTreeLastCommits is the listing of the root with the last commit of |
| 91 | + | // each entry. | |||
| 91 | GetTreeLastCommits Backend | 92 | GetTreeLastCommits Backend | ||
| 92 | - | OpenBlob Backend | 93 | + | // GetTreeSubdirLastCommits is the same for a subdirectory, which costs a |
| 94 | + | // much longer history walk (195 ms with git on lazygit). | |||
| 95 | + | GetTreeSubdirLastCommits Backend | |||
| 96 | + | OpenBlob Backend | |||
| 93 | // ListCommits is a page of history of a revision. | 97 | // ListCommits is a page of history of a revision. | ||
| 94 | ListCommits Backend | 98 | ListCommits Backend | ||
| 95 | // ListCommitsPath is a page of the history of a file or a directory. | 99 | // ListCommitsPath is a page of the history of a file or a directory. | ||
| @@ -99,7 +103,7 @@ type Routes struct { | |||||
| 99 | // AllRoutes returns routes that send every method to b. The tests and the | 103 | // AllRoutes returns routes that send every method to b. The tests and the | ||
| 100 | // benchmarks use it to exercise a backend whatever the production routes are. | 104 | // benchmarks use it to exercise a backend whatever the production routes are. | ||
| 101 | func AllRoutes(b Backend) Routes { | 105 | func AllRoutes(b Backend) Routes { | ||
| 102 | - | return Routes{b, b, b, b, b, b, b, b, b} | 106 | + | return Routes{b, b, b, b, b, b, b, b, b, b} |
| 103 | } | 107 | } | ||
| 104 | 108 | ||||
| 105 | // Config configures an Adapter. | 109 | // Config configures an Adapter. | ||
| @@ -320,7 +324,10 @@ func (a *Adapter) GetTree(ctx context.Context, repo domain.RepoName, rev domain. | |||||
| 320 | cli := func() (domain.Tree, error) { return a.cli.GetTree(ctx, repo, rev, path, includeLastCommit) } | 324 | cli := func() (domain.Tree, error) { return a.cli.GetTree(ctx, repo, rev, path, includeLastCommit) } | ||
| 321 | route := a.routes.GetTree | 325 | route := a.routes.GetTree | ||
| 322 | if includeLastCommit { | 326 | if includeLastCommit { | ||
| 323 | - | route = a.routes.GetTreeLastCommits | 327 | + | route = a.routes.GetTreeSubdirLastCommits |
| 328 | + | if path.IsRoot() { | |||
| 329 | + | route = a.routes.GetTreeLastCommits | |||
| 330 | + | } | |||
| 324 | } | 331 | } | ||
| 325 | if route == CLI { | 332 | if route == CLI { | ||
| 326 | return cli() | 333 | return cli() | ||
M internal/adapters/ermitegit/routes.go +51 -39
| @@ -4,15 +4,19 @@ package ermitegit | |||
| 4 | 4 | ||
| 5 | 5 | // DefaultRoutes returns the routes of production. | |
| 6 | 6 | // | |
| 7 | - | // The rule: a method is routed to ermite-git only when it passes the shared | |
| 7 | + | // The rule: a method is routed to ermite-git when it passes the shared | |
| 8 | 8 | // conformance table (TestConformance, TestCompareWithCLI) and its benchmark | |
| 9 | - | // beats the CLI adapter in time, in bytes AND in allocations per call, warm | |
| 10 | - | // AND cold, on the generated fixture AND on lazygit (BenchmarkRepoReader, | |
| 11 | - | // ERMITE_GIT_BENCH_REPO). Where a method has several shapes, every shape must | |
| 12 | - | // win: the constants GetTree, GetTreeLastCommits, ResolveRevisionAbbrev and | |
| 13 | - | // ListCommitsPath cover directories of any size and files of any history. A | |
| 14 | - | // method that does not win stays on the CLI. GetCommit is not routable: the | |
| 15 | - | // diff stays on git. | |
| 9 | + | // beats the CLI adapter WARM in time AND in bytes per call, on the generated | |
| 10 | + | // fixture AND on lazygit (BenchmarkRepoReader, ERMITE_GIT_BENCH_REPO), and its | |
| 11 | + | // COLD run does not lose in time by more than the open itself (about 65 µs, | |
| 12 | + | // paid once per open, see below). Allocation counts are reported but do not | |
| 13 | + | // decide: the repository cache (cache.go) keeps up to 64 repositories open for | |
| 14 | + | // ten minutes, so nearly every request is warm, and the CLI's own cost (a | |
| 15 | + | // process per call, absent from the Go counters) is not in its numbers. Where | |
| 16 | + | // a method has several shapes that behave differently, each has its own | |
| 17 | + | // constant: GetTree, GetTreeLastCommits (root), GetTreeSubdirLastCommits, | |
| 18 | + | // ResolveRevisionAbbrev and ListCommitsPath. A method that does not win stays | |
| 19 | + | // on the CLI. GetCommit is not routable: the diff stays on git. | |
| 16 | 20 | // | |
| 17 | 21 | // Warm is a repository that is open; cold closes it before every call, the | |
| 18 | 22 | // price of the first read after the idle TTL, an eviction or a restart (a | |
| @@ -92,39 +96,47 @@ package ermitegit | |||
| 92 | 96 | // ListCommits/dirpath lazygit warm 6.7ms 127.8K 403 2.0ms 50.2K 181 win | |
| 93 | 97 | // ListCommits/dirpath lazygit cold 6.7ms 127.8K 403 3.3ms 887.1K 2019 loses ba | |
| 94 | 98 | // | |
| 95 | - | // What decides the routes, with ermite-git 5228465 (zlib reader with a reused | |
| 96 | - | // adler32, header-size cache, Refresh and LookupRefID without allocations): | |
| 99 | + | // The "verdict" column is the strict reading, every line of the table won; | |
| 100 | + | // the routes follow the rule above. What decides them, with ermite-git 5228465 | |
| 101 | + | // (zlib reader with a reused adler32, header-size cache, Refresh and | |
| 102 | + | // LookupRefID without allocations): | |
| 97 | 103 | // | |
| 98 | - | // - The warm runs all win but GetTree of a large directory on lazygit (time | |
| 99 | - | // is won since the zlib change, bytes are not) and OpenBlob, whose | |
| 100 | - | // allocations come from the inflate of a small blob and whose time is | |
| 101 | - | // lost for a 10 MiB blob (inflate speed: 52 ms against 21 ms). | |
| 102 | - | // - The cold runs lose in allocations for every method that does little | |
| 103 | - | // work after the open (ListRefs, GetTree, ResolveRefPath: 150 to 200 | |
| 104 | - | // against 60 to 140), and in time too for the ones that cost a few | |
| 105 | - | // microseconds warm (ResolveRevision: 65 µs against 9 to 63 µs). | |
| 106 | - | // - Only the history of a revision (ListCommits page 1 and deep) wins every | |
| 107 | - | // line, so it is the only method routed. GetTree dir+last also wins every | |
| 108 | - | // line, but GetTreeLastCommits covers GetTree root+last too, which loses | |
| 109 | - | // cold allocations on lazygit (4204 against 3073): it is the nearest | |
| 110 | - | // miss, and the first to flip once opening a repository allocates less. | |
| 111 | - | // - OpenBlob by size: below 1 MiB the small blob loses cold and, on lazygit, | |
| 112 | - | // warm bytes and allocations; above, it loses time. No size makes it win, | |
| 113 | - | // so there is no size-based route. | |
| 114 | - | // | |
| 115 | - | // Once opening a repository allocates less, re-run the benchmark and flip the | |
| 116 | - | // constants it wins: ListRefs, ResolveRefPath and ResolveRevision win warm by 4 | |
| 117 | - | // to 60 times in time. | |
| 104 | + | // - ListRefs (every repository page, through loadRepo): 905 µs against 24 µs | |
| 105 | + | // warm, 120 KiB against 19 KiB (fixture), 960 against 16 µs and 108 KiB | |
| 106 | + | // against 2 KiB (lazygit). Cold it is still 4 to 10 times faster | |
| 107 | + | // (93 µs); only its allocation count loses. Routed. | |
| 108 | + | // - ResolveRevision (name, full id, abbreviation) and ResolveRefPath: 4 to 8 | |
| 109 | + | // times faster warm, with fewer bytes. Cold they lose 45 µs of time to the | |
| 110 | + | // open, but the ListRefs of the same page has opened the repository | |
| 111 | + | // already, so the open is paid by a call that wins it back. Routed. | |
| 112 | + | // - GetTree with last commits: root listing 4.1 ms against 68 µs on the | |
| 113 | + | // fixture, 20.5 ms against 6.2 ms on lazygit (cold 8.7 ms; its 4,206 | |
| 114 | + | // allocations against 3,082 are the only line lost); a subdirectory of | |
| 115 | + | // lazygit 195-199 ms against 8-9 ms warm, 15-17 ms cold, 20 MiB against | |
| 116 | + | // 1.5 MiB. They are two constants so that either can be flipped back on | |
| 117 | + | // its own. Both routed. | |
| 118 | + | // - GetTree without last commits wins warm in time everywhere (13 µs against | |
| 119 | + | // 0.7 ms at the root), but a large directory of lazygit loses warm bytes | |
| 120 | + | // (325 KiB against 275 KiB) and, cold, 4.0 ms against 1.4 ms in time. It | |
| 121 | + | // stays on the CLI. | |
| 122 | + | // - ListCommits of a revision wins every line and stays routed. | |
| 123 | + | // ListCommitsPath wins warm and cold in time (lazygit file: 12.9 ms against | |
| 124 | + | // 6.0 ms warm), only its cold bytes lose (2.4 MiB): it is a candidate, not | |
| 125 | + | // flipped here, to be decided with its own conformance review. | |
| 126 | + | // - OpenBlob by size: below 1 MiB the small blob loses warm bytes and | |
| 127 | + | // allocations on lazygit; above, it loses time (inflate speed: 52 ms | |
| 128 | + | // against 21 ms). No size makes it win, so there is no size-based route. | |
| 118 | 129 | func DefaultRoutes() Routes { | |
| 119 | 130 | return Routes{ | |
| 120 | - | ListRefs: CLI, | |
| 121 | - | ResolveRevision: CLI, | |
| 122 | - | ResolveRevisionAbbrev: CLI, | |
| 123 | - | ResolveRefPath: CLI, | |
| 124 | - | GetTree: CLI, | |
| 125 | - | GetTreeLastCommits: CLI, | |
| 126 | - | OpenBlob: CLI, | |
| 127 | - | ListCommits: ErmiteGit, | |
| 128 | - | ListCommitsPath: CLI, | |
| 131 | + | ListRefs: ErmiteGit, | |
| 132 | + | ResolveRevision: ErmiteGit, | |
| 133 | + | ResolveRevisionAbbrev: ErmiteGit, | |
| 134 | + | ResolveRefPath: ErmiteGit, | |
| 135 | + | GetTree: CLI, | |
| 136 | + | GetTreeLastCommits: ErmiteGit, | |
| 137 | + | GetTreeSubdirLastCommits: ErmiteGit, | |
| 138 | + | OpenBlob: CLI, | |
| 139 | + | ListCommits: ErmiteGit, | |
| 140 | + | ListCommitsPath: CLI, | |
| 129 | 141 | } | |
| 130 | 142 | } | |
| @@ -4,15 +4,19 @@ package ermitegit | |||||
| 4 | 4 | ||||
| 5 | // DefaultRoutes returns the routes of production. | 5 | // DefaultRoutes returns the routes of production. | ||
| 6 | // | 6 | // | ||
| 7 | - | // The rule: a method is routed to ermite-git only when it passes the shared | 7 | + | // The rule: a method is routed to ermite-git when it passes the shared |
| 8 | // conformance table (TestConformance, TestCompareWithCLI) and its benchmark | 8 | // conformance table (TestConformance, TestCompareWithCLI) and its benchmark | ||
| 9 | - | // beats the CLI adapter in time, in bytes AND in allocations per call, warm | 9 | + | // beats the CLI adapter WARM in time AND in bytes per call, on the generated |
| 10 | - | // AND cold, on the generated fixture AND on lazygit (BenchmarkRepoReader, | 10 | + | // fixture AND on lazygit (BenchmarkRepoReader, ERMITE_GIT_BENCH_REPO), and its |
| 11 | - | // ERMITE_GIT_BENCH_REPO). Where a method has several shapes, every shape must | 11 | + | // COLD run does not lose in time by more than the open itself (about 65 µs, |
| 12 | - | // win: the constants GetTree, GetTreeLastCommits, ResolveRevisionAbbrev and | 12 | + | // paid once per open, see below). Allocation counts are reported but do not |
| 13 | - | // ListCommitsPath cover directories of any size and files of any history. A | 13 | + | // decide: the repository cache (cache.go) keeps up to 64 repositories open for |
| 14 | - | // method that does not win stays on the CLI. GetCommit is not routable: the | 14 | + | // ten minutes, so nearly every request is warm, and the CLI's own cost (a |
| 15 | - | // diff stays on git. | 15 | + | // process per call, absent from the Go counters) is not in its numbers. Where |
| 16 | + | // a method has several shapes that behave differently, each has its own | |||
| 17 | + | // constant: GetTree, GetTreeLastCommits (root), GetTreeSubdirLastCommits, | |||
| 18 | + | // ResolveRevisionAbbrev and ListCommitsPath. A method that does not win stays | |||
| 19 | + | // on the CLI. GetCommit is not routable: the diff stays on git. | |||
| 16 | // | 20 | // | ||
| 17 | // Warm is a repository that is open; cold closes it before every call, the | 21 | // Warm is a repository that is open; cold closes it before every call, the | ||
| 18 | // price of the first read after the idle TTL, an eviction or a restart (a | 22 | // price of the first read after the idle TTL, an eviction or a restart (a | ||
| @@ -92,39 +96,47 @@ package ermitegit | |||||
| 92 | // ListCommits/dirpath lazygit warm 6.7ms 127.8K 403 2.0ms 50.2K 181 win | 96 | // ListCommits/dirpath lazygit warm 6.7ms 127.8K 403 2.0ms 50.2K 181 win | ||
| 93 | // ListCommits/dirpath lazygit cold 6.7ms 127.8K 403 3.3ms 887.1K 2019 loses ba | 97 | // ListCommits/dirpath lazygit cold 6.7ms 127.8K 403 3.3ms 887.1K 2019 loses ba | ||
| 94 | // | 98 | // | ||
| 95 | - | // What decides the routes, with ermite-git 5228465 (zlib reader with a reused | 99 | + | // The "verdict" column is the strict reading, every line of the table won; |
| 96 | - | // adler32, header-size cache, Refresh and LookupRefID without allocations): | 100 | + | // the routes follow the rule above. What decides them, with ermite-git 5228465 |
| 101 | + | // (zlib reader with a reused adler32, header-size cache, Refresh and | |||
| 102 | + | // LookupRefID without allocations): | |||
| 97 | // | 103 | // | ||
| 98 | - | // - The warm runs all win but GetTree of a large directory on lazygit (time | 104 | + | // - ListRefs (every repository page, through loadRepo): 905 µs against 24 µs |
| 99 | - | // is won since the zlib change, bytes are not) and OpenBlob, whose | 105 | + | // warm, 120 KiB against 19 KiB (fixture), 960 against 16 µs and 108 KiB |
| 100 | - | // allocations come from the inflate of a small blob and whose time is | 106 | + | // against 2 KiB (lazygit). Cold it is still 4 to 10 times faster |
| 101 | - | // lost for a 10 MiB blob (inflate speed: 52 ms against 21 ms). | 107 | + | // (93 µs); only its allocation count loses. Routed. |
| 102 | - | // - The cold runs lose in allocations for every method that does little | 108 | + | // - ResolveRevision (name, full id, abbreviation) and ResolveRefPath: 4 to 8 |
| 103 | - | // work after the open (ListRefs, GetTree, ResolveRefPath: 150 to 200 | 109 | + | // times faster warm, with fewer bytes. Cold they lose 45 µs of time to the |
| 104 | - | // against 60 to 140), and in time too for the ones that cost a few | 110 | + | // open, but the ListRefs of the same page has opened the repository |
| 105 | - | // microseconds warm (ResolveRevision: 65 µs against 9 to 63 µs). | 111 | + | // already, so the open is paid by a call that wins it back. Routed. |
| 106 | - | // - Only the history of a revision (ListCommits page 1 and deep) wins every | 112 | + | // - GetTree with last commits: root listing 4.1 ms against 68 µs on the |
| 107 | - | // line, so it is the only method routed. GetTree dir+last also wins every | 113 | + | // fixture, 20.5 ms against 6.2 ms on lazygit (cold 8.7 ms; its 4,206 |
| 108 | - | // line, but GetTreeLastCommits covers GetTree root+last too, which loses | 114 | + | // allocations against 3,082 are the only line lost); a subdirectory of |
| 109 | - | // cold allocations on lazygit (4204 against 3073): it is the nearest | 115 | + | // lazygit 195-199 ms against 8-9 ms warm, 15-17 ms cold, 20 MiB against |
| 110 | - | // miss, and the first to flip once opening a repository allocates less. | 116 | + | // 1.5 MiB. They are two constants so that either can be flipped back on |
| 111 | - | // - OpenBlob by size: below 1 MiB the small blob loses cold and, on lazygit, | 117 | + | // its own. Both routed. |
| 112 | - | // warm bytes and allocations; above, it loses time. No size makes it win, | 118 | + | // - GetTree without last commits wins warm in time everywhere (13 µs against |
| 113 | - | // so there is no size-based route. | 119 | + | // 0.7 ms at the root), but a large directory of lazygit loses warm bytes |
| 114 | - | // | 120 | + | // (325 KiB against 275 KiB) and, cold, 4.0 ms against 1.4 ms in time. It |
| 115 | - | // Once opening a repository allocates less, re-run the benchmark and flip the | 121 | + | // stays on the CLI. |
| 116 | - | // constants it wins: ListRefs, ResolveRefPath and ResolveRevision win warm by 4 | 122 | + | // - ListCommits of a revision wins every line and stays routed. |
| 117 | - | // to 60 times in time. | 123 | + | // ListCommitsPath wins warm and cold in time (lazygit file: 12.9 ms against |
| 124 | + | // 6.0 ms warm), only its cold bytes lose (2.4 MiB): it is a candidate, not | |||
| 125 | + | // flipped here, to be decided with its own conformance review. | |||
| 126 | + | // - OpenBlob by size: below 1 MiB the small blob loses warm bytes and | |||
| 127 | + | // allocations on lazygit; above, it loses time (inflate speed: 52 ms | |||
| 128 | + | // against 21 ms). No size makes it win, so there is no size-based route. | |||
| 118 | func DefaultRoutes() Routes { | 129 | func DefaultRoutes() Routes { | ||
| 119 | return Routes{ | 130 | return Routes{ | ||
| 120 | - | ListRefs: CLI, | 131 | + | ListRefs: ErmiteGit, |
| 121 | - | ResolveRevision: CLI, | 132 | + | ResolveRevision: ErmiteGit, |
| 122 | - | ResolveRevisionAbbrev: CLI, | 133 | + | ResolveRevisionAbbrev: ErmiteGit, |
| 123 | - | ResolveRefPath: CLI, | 134 | + | ResolveRefPath: ErmiteGit, |
| 124 | - | GetTree: CLI, | 135 | + | GetTree: CLI, |
| 125 | - | GetTreeLastCommits: CLI, | 136 | + | GetTreeLastCommits: ErmiteGit, |
| 126 | - | OpenBlob: CLI, | 137 | + | GetTreeSubdirLastCommits: ErmiteGit, |
| 127 | - | ListCommits: ErmiteGit, | 138 | + | OpenBlob: CLI, |
| 128 | - | ListCommitsPath: CLI, | 139 | + | ListCommits: ErmiteGit, |
| 140 | + | ListCommitsPath: CLI, | |||
| 129 | } | 141 | } | ||
| 130 | } | 142 | } | ||
M internal/app/browse.go +56 -4
| @@ -24,11 +24,29 @@ import ( | |||
| 24 | 24 | type Browse struct { | |
| 25 | 25 | meta ports.RepoMetaStore | |
| 26 | 26 | reader ports.RepoReader | |
| 27 | + | // history keeps the answers about a commit designated by its full id (a | |
| 28 | + | // tree with the last commit of its entries, the latest commit of a path): | |
| 29 | + | // they never change, and computing them costs a history walk. Nil | |
| 30 | + | // disables it. | |
| 31 | + | history *historyCache | |
| 27 | 32 | } | |
| 28 | 33 | ||
| 29 | - | // NewBrowse builds the read use cases over their ports. | |
| 34 | + | // NewBrowse builds the read use cases over their ports, with a history cache | |
| 35 | + | // of the default bounds. | |
| 30 | 36 | func NewBrowse(meta ports.RepoMetaStore, reader ports.RepoReader) *Browse { | |
| 31 | - | return &Browse{meta: meta, reader: reader} | |
| 37 | + | return &Browse{ | |
| 38 | + | meta: meta, reader: reader, | |
| 39 | + | history: newHistoryCache(DefaultHistoryCacheEntries, DefaultHistoryCacheBytes), | |
| 40 | + | } | |
| 41 | + | } | |
| 42 | + | ||
| 43 | + | // WithHistoryCache replaces the bounds of the history cache: at most | |
| 44 | + | // maxEntries answers and about maxBytes of memory, least recently used first | |
| 45 | + | // out. A bound that is not positive disables the cache. Call it before the | |
| 46 | + | // first request. | |
| 47 | + | func (b *Browse) WithHistoryCache(maxEntries int, maxBytes int64) *Browse { | |
| 48 | + | b.history = newHistoryCache(maxEntries, maxBytes) | |
| 49 | + | return b | |
| 32 | 50 | } | |
| 33 | 51 | ||
| 34 | 52 | // authorize returns nil if p may read the repository. | |
| @@ -171,7 +189,22 @@ func (b *Browse) GetTree(ctx context.Context, p domain.Principal, repo domain.Re | |||
| 171 | 189 | if err := b.authorize(ctx, p, repo); err != nil { | |
| 172 | 190 | return domain.Tree{}, err | |
| 173 | 191 | } | |
| 174 | - | return b.reader.GetTree(ctx, repo, rev, path, includeLastCommit) | |
| 192 | + | // The last commits are the costly part (a history walk per entry) and | |
| 193 | + | // depend only on the commit and the path: cache them once the revision is | |
| 194 | + | // a full commit id. A name or HEAD can move, so it is never a key. | |
| 195 | + | if !includeLastCommit || !rev.IsObjectID() { | |
| 196 | + | return b.reader.GetTree(ctx, repo, rev, path, includeLastCommit) | |
| 197 | + | } | |
| 198 | + | key := historyKey{repo: repo, rev: rev.String(), path: path.String(), kind: kindTreeLast} | |
| 199 | + | if e, ok := b.history.get(key); ok { | |
| 200 | + | return cloneTree(e.tree), nil | |
| 201 | + | } | |
| 202 | + | tree, err := b.reader.GetTree(ctx, repo, rev, path, true) | |
| 203 | + | if err != nil { | |
| 204 | + | return domain.Tree{}, err | |
| 205 | + | } | |
| 206 | + | b.history.put(treeEntry(key, tree)) | |
| 207 | + | return tree, nil | |
| 175 | 208 | } | |
| 176 | 209 | ||
| 177 | 210 | // OpenBlob returns the description of a file and a reader of its content. The | |
| @@ -224,7 +257,7 @@ func (b *Browse) ListCommits(ctx context.Context, p domain.Principal, repo domai | |||
| 224 | 257 | if err := b.authorize(ctx, p, repo); err != nil { | |
| 225 | 258 | return CommitPage{}, err | |
| 226 | 259 | } | |
| 227 | - | res, err := b.reader.ListCommits(ctx, repo, q) | |
| 260 | + | res, err := b.listCommits(ctx, repo, q) | |
| 228 | 261 | if err != nil { | |
| 229 | 262 | return CommitPage{}, err | |
| 230 | 263 | } | |
| @@ -235,6 +268,25 @@ func (b *Browse) ListCommits(ctx context.Context, p domain.Principal, repo domai | |||
| 235 | 268 | return out, nil | |
| 236 | 269 | } | |
| 237 | 270 | ||
| 271 | + | // listCommits asks the reader, through the history cache for the first | |
| 272 | + | // commit of a path at a full commit id (the "last change" of a file): that | |
| 273 | + | // answer is immutable. | |
| 274 | + | func (b *Browse) listCommits(ctx context.Context, repo domain.RepoName, q ports.CommitQuery) (ports.CommitPage, error) { | |
| 275 | + | if q.Limit != 1 || q.Skip != 0 || !q.Revision.IsObjectID() { | |
| 276 | + | return b.reader.ListCommits(ctx, repo, q) | |
| 277 | + | } | |
| 278 | + | key := historyKey{repo: repo, rev: q.Revision.String(), path: q.Path.String(), kind: kindPage, limit: q.Limit} | |
| 279 | + | if e, ok := b.history.get(key); ok { | |
| 280 | + | return clonePage(e.page), nil | |
| 281 | + | } | |
| 282 | + | res, err := b.reader.ListCommits(ctx, repo, q) | |
| 283 | + | if err != nil { | |
| 284 | + | return ports.CommitPage{}, err | |
| 285 | + | } | |
| 286 | + | b.history.put(pageEntry(key, res)) | |
| 287 | + | return res, nil | |
| 288 | + | } | |
| 289 | + | ||
| 238 | 290 | func pathDigest(path domain.TreePath) string { | |
| 239 | 291 | sum := sha256.Sum256([]byte(path.String())) | |
| 240 | 292 | return hex.EncodeToString(sum[:6]) | |
| @@ -24,11 +24,29 @@ import ( | |||||
| 24 | type Browse struct { | 24 | type Browse struct { | ||
| 25 | meta ports.RepoMetaStore | 25 | meta ports.RepoMetaStore | ||
| 26 | reader ports.RepoReader | 26 | reader ports.RepoReader | ||
| 27 | + | // history keeps the answers about a commit designated by its full id (a | |||
| 28 | + | // tree with the last commit of its entries, the latest commit of a path): | |||
| 29 | + | // they never change, and computing them costs a history walk. Nil | |||
| 30 | + | // disables it. | |||
| 31 | + | history *historyCache | |||
| 27 | } | 32 | } | ||
| 28 | 33 | ||||
| 29 | - | // NewBrowse builds the read use cases over their ports. | 34 | + | // NewBrowse builds the read use cases over their ports, with a history cache |
| 35 | + | // of the default bounds. | |||
| 30 | func NewBrowse(meta ports.RepoMetaStore, reader ports.RepoReader) *Browse { | 36 | func NewBrowse(meta ports.RepoMetaStore, reader ports.RepoReader) *Browse { | ||
| 31 | - | return &Browse{meta: meta, reader: reader} | 37 | + | return &Browse{ |
| 38 | + | meta: meta, reader: reader, | |||
| 39 | + | history: newHistoryCache(DefaultHistoryCacheEntries, DefaultHistoryCacheBytes), | |||
| 40 | + | } | |||
| 41 | + | } | |||
| 42 | + | ||||
| 43 | + | // WithHistoryCache replaces the bounds of the history cache: at most | |||
| 44 | + | // maxEntries answers and about maxBytes of memory, least recently used first | |||
| 45 | + | // out. A bound that is not positive disables the cache. Call it before the | |||
| 46 | + | // first request. | |||
| 47 | + | func (b *Browse) WithHistoryCache(maxEntries int, maxBytes int64) *Browse { | |||
| 48 | + | b.history = newHistoryCache(maxEntries, maxBytes) | |||
| 49 | + | return b | |||
| 32 | } | 50 | } | ||
| 33 | 51 | ||||
| 34 | // authorize returns nil if p may read the repository. | 52 | // authorize returns nil if p may read the repository. | ||
| @@ -171,7 +189,22 @@ func (b *Browse) GetTree(ctx context.Context, p domain.Principal, repo domain.Re | |||||
| 171 | if err := b.authorize(ctx, p, repo); err != nil { | 189 | if err := b.authorize(ctx, p, repo); err != nil { | ||
| 172 | return domain.Tree{}, err | 190 | return domain.Tree{}, err | ||
| 173 | } | 191 | } | ||
| 174 | - | return b.reader.GetTree(ctx, repo, rev, path, includeLastCommit) | 192 | + | // The last commits are the costly part (a history walk per entry) and |
| 193 | + | // depend only on the commit and the path: cache them once the revision is | |||
| 194 | + | // a full commit id. A name or HEAD can move, so it is never a key. | |||
| 195 | + | if !includeLastCommit || !rev.IsObjectID() { | |||
| 196 | + | return b.reader.GetTree(ctx, repo, rev, path, includeLastCommit) | |||
| 197 | + | } | |||
| 198 | + | key := historyKey{repo: repo, rev: rev.String(), path: path.String(), kind: kindTreeLast} | |||
| 199 | + | if e, ok := b.history.get(key); ok { | |||
| 200 | + | return cloneTree(e.tree), nil | |||
| 201 | + | } | |||
| 202 | + | tree, err := b.reader.GetTree(ctx, repo, rev, path, true) | |||
| 203 | + | if err != nil { | |||
| 204 | + | return domain.Tree{}, err | |||
| 205 | + | } | |||
| 206 | + | b.history.put(treeEntry(key, tree)) | |||
| 207 | + | return tree, nil | |||
| 175 | } | 208 | } | ||
| 176 | 209 | ||||
| 177 | // OpenBlob returns the description of a file and a reader of its content. The | 210 | // OpenBlob returns the description of a file and a reader of its content. The | ||
| @@ -224,7 +257,7 @@ func (b *Browse) ListCommits(ctx context.Context, p domain.Principal, repo domai | |||||
| 224 | if err := b.authorize(ctx, p, repo); err != nil { | 257 | if err := b.authorize(ctx, p, repo); err != nil { | ||
| 225 | return CommitPage{}, err | 258 | return CommitPage{}, err | ||
| 226 | } | 259 | } | ||
| 227 | - | res, err := b.reader.ListCommits(ctx, repo, q) | 260 | + | res, err := b.listCommits(ctx, repo, q) |
| 228 | if err != nil { | 261 | if err != nil { | ||
| 229 | return CommitPage{}, err | 262 | return CommitPage{}, err | ||
| 230 | } | 263 | } | ||
| @@ -235,6 +268,25 @@ func (b *Browse) ListCommits(ctx context.Context, p domain.Principal, repo domai | |||||
| 235 | return out, nil | 268 | return out, nil | ||
| 236 | } | 269 | } | ||
| 237 | 270 | ||||
| 271 | + | // listCommits asks the reader, through the history cache for the first | |||
| 272 | + | // commit of a path at a full commit id (the "last change" of a file): that | |||
| 273 | + | // answer is immutable. | |||
| 274 | + | func (b *Browse) listCommits(ctx context.Context, repo domain.RepoName, q ports.CommitQuery) (ports.CommitPage, error) { | |||
| 275 | + | if q.Limit != 1 || q.Skip != 0 || !q.Revision.IsObjectID() { | |||
| 276 | + | return b.reader.ListCommits(ctx, repo, q) | |||
| 277 | + | } | |||
| 278 | + | key := historyKey{repo: repo, rev: q.Revision.String(), path: q.Path.String(), kind: kindPage, limit: q.Limit} | |||
| 279 | + | if e, ok := b.history.get(key); ok { | |||
| 280 | + | return clonePage(e.page), nil | |||
| 281 | + | } | |||
| 282 | + | res, err := b.reader.ListCommits(ctx, repo, q) | |||
| 283 | + | if err != nil { | |||
| 284 | + | return ports.CommitPage{}, err | |||
| 285 | + | } | |||
| 286 | + | b.history.put(pageEntry(key, res)) | |||
| 287 | + | return res, nil | |||
| 288 | + | } | |||
| 289 | + | ||||
| 238 | func pathDigest(path domain.TreePath) string { | 290 | func pathDigest(path domain.TreePath) string { | ||
| 239 | sum := sha256.Sum256([]byte(path.String())) | 291 | sum := sha256.Sum256([]byte(path.String())) | ||
| 240 | return hex.EncodeToString(sum[:6]) | 292 | return hex.EncodeToString(sum[:6]) | ||
A internal/app/browsecache.go +163 -0
| @@ -0,0 +1,163 @@ | |||
| 1 | + | // SPDX-License-Identifier: AGPL-3.0-or-later | |
| 2 | + | ||
| 3 | + | package app | |
| 4 | + | ||
| 5 | + | import ( | |
| 6 | + | "container/list" | |
| 7 | + | "slices" | |
| 8 | + | "sync" | |
| 9 | + | ||
| 10 | + | "bnema.dev/ermite/internal/domain" | |
| 11 | + | "bnema.dev/ermite/internal/ports" | |
| 12 | + | ) | |
| 13 | + | ||
| 14 | + | // Default bounds of the history cache of Browse. | |
| 15 | + | const ( | |
| 16 | + | // DefaultHistoryCacheEntries bounds the cached answers. | |
| 17 | + | DefaultHistoryCacheEntries = 256 | |
| 18 | + | // DefaultHistoryCacheBytes bounds their estimated size. | |
| 19 | + | DefaultHistoryCacheBytes = 8 << 20 | |
| 20 | + | ) | |
| 21 | + | ||
| 22 | + | // historyKey names an answer that depends only on immutable data: a commit | |
| 23 | + | // designated by its full id, the path inside it, and for a page of history | |
| 24 | + | // where it starts and how long it is. The repository is part of the key so | |
| 25 | + | // that an answer is never served across a name. | |
| 26 | + | type historyKey struct { | |
| 27 | + | repo domain.RepoName | |
| 28 | + | rev string // full commit id | |
| 29 | + | path string | |
| 30 | + | kind historyKind | |
| 31 | + | limit int | |
| 32 | + | } | |
| 33 | + | ||
| 34 | + | type historyKind uint8 | |
| 35 | + | ||
| 36 | + | const ( | |
| 37 | + | kindTreeLast historyKind = iota // a directory with the last commit of its entries | |
| 38 | + | kindPage // the first page of the history of a path | |
| 39 | + | ) | |
| 40 | + | ||
| 41 | + | // historyCache is a small LRU of answers about commits, bounded by entry count | |
| 42 | + | // and by estimated bytes. The answers are immutable (a commit id fixes its | |
| 43 | + | // whole history), so there is no invalidation: entries leave by eviction only. | |
| 44 | + | // It is safe for concurrent use. Values are stored and returned as copies, so | |
| 45 | + | // a caller cannot alter what another one gets. | |
| 46 | + | type historyCache struct { | |
| 47 | + | maxEntries int | |
| 48 | + | maxBytes int64 | |
| 49 | + | ||
| 50 | + | mu sync.Mutex | |
| 51 | + | order *list.List // front is the most recently used | |
| 52 | + | items map[historyKey]*list.Element | |
| 53 | + | bytes int64 | |
| 54 | + | } | |
| 55 | + | ||
| 56 | + | type historyEntry struct { | |
| 57 | + | key historyKey | |
| 58 | + | tree domain.Tree | |
| 59 | + | page ports.CommitPage | |
| 60 | + | size int64 | |
| 61 | + | } | |
| 62 | + | ||
| 63 | + | // newHistoryCache returns nil (no cache) unless both bounds are positive. | |
| 64 | + | func newHistoryCache(maxEntries int, maxBytes int64) *historyCache { | |
| 65 | + | if maxEntries <= 0 || maxBytes <= 0 { | |
| 66 | + | return nil | |
| 67 | + | } | |
| 68 | + | return &historyCache{ | |
| 69 | + | maxEntries: maxEntries, maxBytes: maxBytes, | |
| 70 | + | order: list.New(), items: map[historyKey]*list.Element{}, | |
| 71 | + | } | |
| 72 | + | } | |
| 73 | + | ||
| 74 | + | // get returns the entry of key and marks it recently used. | |
| 75 | + | func (c *historyCache) get(key historyKey) (*historyEntry, bool) { | |
| 76 | + | if c == nil { | |
| 77 | + | return nil, false | |
| 78 | + | } | |
| 79 | + | c.mu.Lock() | |
| 80 | + | defer c.mu.Unlock() | |
| 81 | + | el, ok := c.items[key] | |
| 82 | + | if !ok { | |
| 83 | + | return nil, false | |
| 84 | + | } | |
| 85 | + | c.order.MoveToFront(el) | |
| 86 | + | return el.Value.(*historyEntry), true | |
| 87 | + | } | |
| 88 | + | ||
| 89 | + | // put stores e, then evicts the least recently used entries until both bounds | |
| 90 | + | // hold. An entry larger than the byte bound is not stored. | |
| 91 | + | func (c *historyCache) put(e *historyEntry) { | |
| 92 | + | if c == nil || e.size > c.maxBytes { | |
| 93 | + | return | |
| 94 | + | } | |
| 95 | + | c.mu.Lock() | |
| 96 | + | defer c.mu.Unlock() | |
| 97 | + | if el, ok := c.items[e.key]; ok { | |
| 98 | + | c.bytes -= el.Value.(*historyEntry).size | |
| 99 | + | c.order.Remove(el) | |
| 100 | + | } | |
| 101 | + | c.items[e.key] = c.order.PushFront(e) | |
| 102 | + | c.bytes += e.size | |
| 103 | + | for c.order.Len() > c.maxEntries || c.bytes > c.maxBytes { | |
| 104 | + | oldest := c.order.Back() | |
| 105 | + | old := oldest.Value.(*historyEntry) | |
| 106 | + | c.order.Remove(oldest) | |
| 107 | + | delete(c.items, old.key) | |
| 108 | + | c.bytes -= old.size | |
| 109 | + | } | |
| 110 | + | } | |
| 111 | + | ||
| 112 | + | // treeEntry builds the entry of a tree; the tree is copied. | |
| 113 | + | func treeEntry(key historyKey, t domain.Tree) *historyEntry { | |
| 114 | + | t = cloneTree(t) | |
| 115 | + | size := int64(256 + len(t.Entries)*160) | |
| 116 | + | for _, e := range t.Entries { | |
| 117 | + | size += int64(len(e.Name) + len(e.Path)) | |
| 118 | + | if e.LastCommit != nil { | |
| 119 | + | size += commitSize(*e.LastCommit) | |
| 120 | + | } | |
| 121 | + | } | |
| 122 | + | return &historyEntry{key: key, tree: t, size: size} | |
| 123 | + | } | |
| 124 | + | ||
| 125 | + | // pageEntry builds the entry of a page of history; the page is copied. | |
| 126 | + | func pageEntry(key historyKey, p ports.CommitPage) *historyEntry { | |
| 127 | + | p = clonePage(p) | |
| 128 | + | size := int64(128) | |
| 129 | + | for _, c := range p.Commits { | |
| 130 | + | size += commitSize(c) | |
| 131 | + | } | |
| 132 | + | return &historyEntry{key: key, page: p, size: size} | |
| 133 | + | } | |
| 134 | + | ||
| 135 | + | func commitSize(c domain.Commit) int64 { | |
| 136 | + | n := 384 + len(c.Parents)*48 + len(c.Summary) + len(c.Message) | |
| 137 | + | n += len(c.Author.Name) + len(c.Author.Email) + len(c.Committer.Name) + len(c.Committer.Email) | |
| 138 | + | return int64(n) | |
| 139 | + | } | |
| 140 | + | ||
| 141 | + | func cloneCommit(c domain.Commit) domain.Commit { | |
| 142 | + | c.Parents = slices.Clone(c.Parents) | |
| 143 | + | return c | |
| 144 | + | } | |
| 145 | + | ||
| 146 | + | func cloneTree(t domain.Tree) domain.Tree { | |
| 147 | + | t.Entries = slices.Clone(t.Entries) | |
| 148 | + | for i := range t.Entries { | |
| 149 | + | if lc := t.Entries[i].LastCommit; lc != nil { | |
| 150 | + | c := cloneCommit(*lc) | |
| 151 | + | t.Entries[i].LastCommit = &c | |
| 152 | + | } | |
| 153 | + | } | |
| 154 | + | return t | |
| 155 | + | } | |
| 156 | + | ||
| 157 | + | func clonePage(p ports.CommitPage) ports.CommitPage { | |
| 158 | + | p.Commits = slices.Clone(p.Commits) | |
| 159 | + | for i := range p.Commits { | |
| 160 | + | p.Commits[i] = cloneCommit(p.Commits[i]) | |
| 161 | + | } | |
| 162 | + | return p | |
| 163 | + | } | |
| @@ -0,0 +1,163 @@ | |||||
| 1 | + | // SPDX-License-Identifier: AGPL-3.0-or-later | |||
| 2 | + | ||||
| 3 | + | package app | |||
| 4 | + | ||||
| 5 | + | import ( | |||
| 6 | + | "container/list" | |||
| 7 | + | "slices" | |||
| 8 | + | "sync" | |||
| 9 | + | ||||
| 10 | + | "bnema.dev/ermite/internal/domain" | |||
| 11 | + | "bnema.dev/ermite/internal/ports" | |||
| 12 | + | ) | |||
| 13 | + | ||||
| 14 | + | // Default bounds of the history cache of Browse. | |||
| 15 | + | const ( | |||
| 16 | + | // DefaultHistoryCacheEntries bounds the cached answers. | |||
| 17 | + | DefaultHistoryCacheEntries = 256 | |||
| 18 | + | // DefaultHistoryCacheBytes bounds their estimated size. | |||
| 19 | + | DefaultHistoryCacheBytes = 8 << 20 | |||
| 20 | + | ) | |||
| 21 | + | ||||
| 22 | + | // historyKey names an answer that depends only on immutable data: a commit | |||
| 23 | + | // designated by its full id, the path inside it, and for a page of history | |||
| 24 | + | // where it starts and how long it is. The repository is part of the key so | |||
| 25 | + | // that an answer is never served across a name. | |||
| 26 | + | type historyKey struct { | |||
| 27 | + | repo domain.RepoName | |||
| 28 | + | rev string // full commit id | |||
| 29 | + | path string | |||
| 30 | + | kind historyKind | |||
| 31 | + | limit int | |||
| 32 | + | } | |||
| 33 | + | ||||
| 34 | + | type historyKind uint8 | |||
| 35 | + | ||||
| 36 | + | const ( | |||
| 37 | + | kindTreeLast historyKind = iota // a directory with the last commit of its entries | |||
| 38 | + | kindPage // the first page of the history of a path | |||
| 39 | + | ) | |||
| 40 | + | ||||
| 41 | + | // historyCache is a small LRU of answers about commits, bounded by entry count | |||
| 42 | + | // and by estimated bytes. The answers are immutable (a commit id fixes its | |||
| 43 | + | // whole history), so there is no invalidation: entries leave by eviction only. | |||
| 44 | + | // It is safe for concurrent use. Values are stored and returned as copies, so | |||
| 45 | + | // a caller cannot alter what another one gets. | |||
| 46 | + | type historyCache struct { | |||
| 47 | + | maxEntries int | |||
| 48 | + | maxBytes int64 | |||
| 49 | + | ||||
| 50 | + | mu sync.Mutex | |||
| 51 | + | order *list.List // front is the most recently used | |||
| 52 | + | items map[historyKey]*list.Element | |||
| 53 | + | bytes int64 | |||
| 54 | + | } | |||
| 55 | + | ||||
| 56 | + | type historyEntry struct { | |||
| 57 | + | key historyKey | |||
| 58 | + | tree domain.Tree | |||
| 59 | + | page ports.CommitPage | |||
| 60 | + | size int64 | |||
| 61 | + | } | |||
| 62 | + | ||||
| 63 | + | // newHistoryCache returns nil (no cache) unless both bounds are positive. | |||
| 64 | + | func newHistoryCache(maxEntries int, maxBytes int64) *historyCache { | |||
| 65 | + | if maxEntries <= 0 || maxBytes <= 0 { | |||
| 66 | + | return nil | |||
| 67 | + | } | |||
| 68 | + | return &historyCache{ | |||
| 69 | + | maxEntries: maxEntries, maxBytes: maxBytes, | |||
| 70 | + | order: list.New(), items: map[historyKey]*list.Element{}, | |||
| 71 | + | } | |||
| 72 | + | } | |||
| 73 | + | ||||
| 74 | + | // get returns the entry of key and marks it recently used. | |||
| 75 | + | func (c *historyCache) get(key historyKey) (*historyEntry, bool) { | |||
| 76 | + | if c == nil { | |||
| 77 | + | return nil, false | |||
| 78 | + | } | |||
| 79 | + | c.mu.Lock() | |||
| 80 | + | defer c.mu.Unlock() | |||
| 81 | + | el, ok := c.items[key] | |||
| 82 | + | if !ok { | |||
| 83 | + | return nil, false | |||
| 84 | + | } | |||
| 85 | + | c.order.MoveToFront(el) | |||
| 86 | + | return el.Value.(*historyEntry), true | |||
| 87 | + | } | |||
| 88 | + | ||||
| 89 | + | // put stores e, then evicts the least recently used entries until both bounds | |||
| 90 | + | // hold. An entry larger than the byte bound is not stored. | |||
| 91 | + | func (c *historyCache) put(e *historyEntry) { | |||
| 92 | + | if c == nil || e.size > c.maxBytes { | |||
| 93 | + | return | |||
| 94 | + | } | |||
| 95 | + | c.mu.Lock() | |||
| 96 | + | defer c.mu.Unlock() | |||
| 97 | + | if el, ok := c.items[e.key]; ok { | |||
| 98 | + | c.bytes -= el.Value.(*historyEntry).size | |||
| 99 | + | c.order.Remove(el) | |||
| 100 | + | } | |||
| 101 | + | c.items[e.key] = c.order.PushFront(e) | |||
| 102 | + | c.bytes += e.size | |||
| 103 | + | for c.order.Len() > c.maxEntries || c.bytes > c.maxBytes { | |||
| 104 | + | oldest := c.order.Back() | |||
| 105 | + | old := oldest.Value.(*historyEntry) | |||
| 106 | + | c.order.Remove(oldest) | |||
| 107 | + | delete(c.items, old.key) | |||
| 108 | + | c.bytes -= old.size | |||
| 109 | + | } | |||
| 110 | + | } | |||
| 111 | + | ||||
| 112 | + | // treeEntry builds the entry of a tree; the tree is copied. | |||
| 113 | + | func treeEntry(key historyKey, t domain.Tree) *historyEntry { | |||
| 114 | + | t = cloneTree(t) | |||
| 115 | + | size := int64(256 + len(t.Entries)*160) | |||
| 116 | + | for _, e := range t.Entries { | |||
| 117 | + | size += int64(len(e.Name) + len(e.Path)) | |||
| 118 | + | if e.LastCommit != nil { | |||
| 119 | + | size += commitSize(*e.LastCommit) | |||
| 120 | + | } | |||
| 121 | + | } | |||
| 122 | + | return &historyEntry{key: key, tree: t, size: size} | |||
| 123 | + | } | |||
| 124 | + | ||||
| 125 | + | // pageEntry builds the entry of a page of history; the page is copied. | |||
| 126 | + | func pageEntry(key historyKey, p ports.CommitPage) *historyEntry { | |||
| 127 | + | p = clonePage(p) | |||
| 128 | + | size := int64(128) | |||
| 129 | + | for _, c := range p.Commits { | |||
| 130 | + | size += commitSize(c) | |||
| 131 | + | } | |||
| 132 | + | return &historyEntry{key: key, page: p, size: size} | |||
| 133 | + | } | |||
| 134 | + | ||||
| 135 | + | func commitSize(c domain.Commit) int64 { | |||
| 136 | + | n := 384 + len(c.Parents)*48 + len(c.Summary) + len(c.Message) | |||
| 137 | + | n += len(c.Author.Name) + len(c.Author.Email) + len(c.Committer.Name) + len(c.Committer.Email) | |||
| 138 | + | return int64(n) | |||
| 139 | + | } | |||
| 140 | + | ||||
| 141 | + | func cloneCommit(c domain.Commit) domain.Commit { | |||
| 142 | + | c.Parents = slices.Clone(c.Parents) | |||
| 143 | + | return c | |||
| 144 | + | } | |||
| 145 | + | ||||
| 146 | + | func cloneTree(t domain.Tree) domain.Tree { | |||
| 147 | + | t.Entries = slices.Clone(t.Entries) | |||
| 148 | + | for i := range t.Entries { | |||
| 149 | + | if lc := t.Entries[i].LastCommit; lc != nil { | |||
| 150 | + | c := cloneCommit(*lc) | |||
| 151 | + | t.Entries[i].LastCommit = &c | |||
| 152 | + | } | |||
| 153 | + | } | |||
| 154 | + | return t | |||
| 155 | + | } | |||
| 156 | + | ||||
| 157 | + | func clonePage(p ports.CommitPage) ports.CommitPage { | |||
| 158 | + | p.Commits = slices.Clone(p.Commits) | |||
| 159 | + | for i := range p.Commits { | |||
| 160 | + | p.Commits[i] = cloneCommit(p.Commits[i]) | |||
| 161 | + | } | |||
| 162 | + | return p | |||
| 163 | + | } | |||
A internal/app/browsecache_test.go +296 -0
| @@ -0,0 +1,296 @@ | |||
| 1 | + | // SPDX-License-Identifier: AGPL-3.0-or-later | |
| 2 | + | ||
| 3 | + | package app_test | |
| 4 | + | ||
| 5 | + | import ( | |
| 6 | + | "context" | |
| 7 | + | "strings" | |
| 8 | + | "sync" | |
| 9 | + | "testing" | |
| 10 | + | ||
| 11 | + | "github.com/stretchr/testify/assert" | |
| 12 | + | "github.com/stretchr/testify/mock" | |
| 13 | + | "github.com/stretchr/testify/require" | |
| 14 | + | ||
| 15 | + | "bnema.dev/ermite/internal/app" | |
| 16 | + | "bnema.dev/ermite/internal/domain" | |
| 17 | + | "bnema.dev/ermite/internal/ports" | |
| 18 | + | ) | |
| 19 | + | ||
| 20 | + | func treePath(t *testing.T, s string) domain.TreePath { | |
| 21 | + | t.Helper() | |
| 22 | + | p, err := domain.ParseTreePath(s) | |
| 23 | + | require.NoError(t, err) | |
| 24 | + | return p | |
| 25 | + | } | |
| 26 | + | ||
| 27 | + | func treeWithLast(c string) domain.Tree { | |
| 28 | + | commit := domain.Commit{ID: oid(c), Parents: []domain.ObjectID{oid("0")}, Summary: "s"} | |
| 29 | + | return domain.Tree{ | |
| 30 | + | CommitID: oid("a"), | |
| 31 | + | Entries: []domain.TreeEntry{{Name: "f", Path: "d/f", LastCommit: &commit}}, | |
| 32 | + | } | |
| 33 | + | } | |
| 34 | + | ||
| 35 | + | // TestGetTreeLastCommitCache covers which GetTree calls are served from the | |
| 36 | + | // cache: only the ones with last commits at a full commit id. | |
| 37 | + | func TestGetTreeLastCommitCache(t *testing.T) { | |
| 38 | + | ctx := context.Background() | |
| 39 | + | pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic} | |
| 40 | + | full := domain.RevisionOf(oid("a")) | |
| 41 | + | branch, err := domain.ParseRevision("main") | |
| 42 | + | require.NoError(t, err) | |
| 43 | + | dir := treePath(t, "d") | |
| 44 | + | ||
| 45 | + | tests := []struct { | |
| 46 | + | name string | |
| 47 | + | rev domain.Revision | |
| 48 | + | includeLast bool | |
| 49 | + | wantReaderOn int // reader calls for two identical requests | |
| 50 | + | }{ | |
| 51 | + | {"full id with last commits is cached", full, true, 1}, | |
| 52 | + | {"full id without last commits is not", full, false, 2}, | |
| 53 | + | {"branch name is not", branch, true, 2}, | |
| 54 | + | {"HEAD is not", domain.Revision{}, true, 2}, | |
| 55 | + | } | |
| 56 | + | for _, tc := range tests { | |
| 57 | + | t.Run(tc.name, func(t *testing.T) { | |
| 58 | + | f := newBrowseFixture(t) | |
| 59 | + | f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil) | |
| 60 | + | f.reader.EXPECT().GetTree(mock.Anything, pub.Name, tc.rev, dir, tc.includeLast). | |
| 61 | + | Return(treeWithLast("b"), nil).Times(tc.wantReaderOn) | |
| 62 | + | for range 2 { | |
| 63 | + | tree, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, tc.rev, dir, tc.includeLast) | |
| 64 | + | require.NoError(t, err) | |
| 65 | + | assert.Equal(t, "d/f", tree.Entries[0].Path) | |
| 66 | + | } | |
| 67 | + | }) | |
| 68 | + | } | |
| 69 | + | } | |
| 70 | + | ||
| 71 | + | func TestGetTreeLastCommitCacheKey(t *testing.T) { | |
| 72 | + | ctx := context.Background() | |
| 73 | + | pubA := domain.Repo{Name: name(t, "a"), Visibility: domain.VisibilityPublic} | |
| 74 | + | pubB := domain.Repo{Name: name(t, "b"), Visibility: domain.VisibilityPublic} | |
| 75 | + | revA, revB := domain.RevisionOf(oid("a")), domain.RevisionOf(oid("b")) | |
| 76 | + | d1, d2 := treePath(t, "d1"), treePath(t, "d2") | |
| 77 | + | ||
| 78 | + | // Each distinct (repo, commit, path) is one reader call, then a hit. | |
| 79 | + | keys := []struct { | |
| 80 | + | repo domain.Repo | |
| 81 | + | rev domain.Revision | |
| 82 | + | path domain.TreePath | |
| 83 | + | }{ | |
| 84 | + | {pubA, revA, d1}, {pubB, revA, d1}, {pubA, revB, d1}, {pubA, revA, d2}, {pubA, revA, domain.TreePath{}}, | |
| 85 | + | } | |
| 86 | + | f := newBrowseFixture(t) | |
| 87 | + | f.meta.EXPECT().Get(mock.Anything, mock.Anything).RunAndReturn( | |
| 88 | + | func(_ context.Context, n domain.RepoName) (domain.Repo, error) { | |
| 89 | + | return domain.Repo{Name: n, Visibility: domain.VisibilityPublic}, nil | |
| 90 | + | }) | |
| 91 | + | for _, k := range keys { | |
| 92 | + | f.reader.EXPECT().GetTree(mock.Anything, k.repo.Name, k.rev, k.path, true).Return(treeWithLast("b"), nil).Once() | |
| 93 | + | } | |
| 94 | + | for range 2 { | |
| 95 | + | for _, k := range keys { | |
| 96 | + | _, err := f.browse.GetTree(ctx, domain.Anonymous(), k.repo.Name, k.rev, k.path, true) | |
| 97 | + | require.NoError(t, err) | |
| 98 | + | } | |
| 99 | + | } | |
| 100 | + | } | |
| 101 | + | ||
| 102 | + | // TestGetTreeCacheKeepsAccessRule: a hit never skips the access check, and an | |
| 103 | + | // error is not cached. | |
| 104 | + | func TestGetTreeCacheKeepsAccessRule(t *testing.T) { | |
| 105 | + | ctx := context.Background() | |
| 106 | + | priv := domain.Repo{Name: name(t, "priv"), Visibility: domain.VisibilityPrivate} | |
| 107 | + | rev, dir := domain.RevisionOf(oid("a")), treePath(t, "d") | |
| 108 | + | ||
| 109 | + | f := newBrowseFixture(t) | |
| 110 | + | f.meta.EXPECT().Get(mock.Anything, priv.Name).Return(priv, nil) | |
| 111 | + | f.reader.EXPECT().GetTree(mock.Anything, priv.Name, rev, dir, true).Return(domain.Tree{}, domain.ErrNotDirectory).Once() | |
| 112 | + | f.reader.EXPECT().GetTree(mock.Anything, priv.Name, rev, dir, true).Return(treeWithLast("b"), nil).Once() | |
| 113 | + | ||
| 114 | + | _, err := f.browse.GetTree(ctx, domain.Root(), priv.Name, rev, dir, true) | |
| 115 | + | require.ErrorIs(t, err, domain.ErrNotDirectory) // not cached | |
| 116 | + | _, err = f.browse.GetTree(ctx, domain.Root(), priv.Name, rev, dir, true) | |
| 117 | + | require.NoError(t, err) // now cached | |
| 118 | + | _, err = f.browse.GetTree(ctx, domain.Anonymous(), priv.Name, rev, dir, true) | |
| 119 | + | assert.ErrorIs(t, err, domain.ErrNotFound) | |
| 120 | + | } | |
| 121 | + | ||
| 122 | + | // TestGetTreeCacheReturnsCopies: a caller that edits its tree does not alter | |
| 123 | + | // what the next one gets. | |
| 124 | + | func TestGetTreeCacheReturnsCopies(t *testing.T) { | |
| 125 | + | ctx := context.Background() | |
| 126 | + | pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic} | |
| 127 | + | rev, dir := domain.RevisionOf(oid("a")), treePath(t, "d") | |
| 128 | + | f := newBrowseFixture(t) | |
| 129 | + | f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil) | |
| 130 | + | f.reader.EXPECT().GetTree(mock.Anything, pub.Name, rev, dir, true).Return(treeWithLast("b"), nil).Once() | |
| 131 | + | ||
| 132 | + | for range 3 { | |
| 133 | + | tree, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, rev, dir, true) | |
| 134 | + | require.NoError(t, err) | |
| 135 | + | require.Len(t, tree.Entries, 1) | |
| 136 | + | assert.Equal(t, "s", tree.Entries[0].LastCommit.Summary) | |
| 137 | + | tree.Entries[0].LastCommit.Summary = "changed" | |
| 138 | + | tree.Entries[0].LastCommit.Parents[0] = oid("f") | |
| 139 | + | tree.Entries[0].Name = "changed" | |
| 140 | + | } | |
| 141 | + | } | |
| 142 | + | ||
| 143 | + | // TestHistoryCacheBounds: the entry bound and the byte bound evict the least | |
| 144 | + | // recently used answer. | |
| 145 | + | func TestHistoryCacheBounds(t *testing.T) { | |
| 146 | + | ctx := context.Background() | |
| 147 | + | pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic} | |
| 148 | + | paths := []string{"p0", "p1", "p2"} | |
| 149 | + | ||
| 150 | + | bigTree := treeWithLast("b") | |
| 151 | + | bigTree.Entries[0].Name = strings.Repeat("x", 4096) | |
| 152 | + | ||
| 153 | + | tests := []struct { | |
| 154 | + | name string | |
| 155 | + | entries int | |
| 156 | + | bytes int64 | |
| 157 | + | tree domain.Tree | |
| 158 | + | wantCached []bool // after requesting p0, p1, p2 (and touching p0 before p2 when touch) | |
| 159 | + | touch bool | |
| 160 | + | }{ | |
| 161 | + | {"entry bound", 2, 1 << 20, treeWithLast("b"), []bool{false, true, true}, false}, | |
| 162 | + | {"entry bound is LRU", 2, 1 << 20, treeWithLast("b"), []bool{true, false, true}, true}, | |
| 163 | + | {"byte bound", 100, 6000, bigTree, []bool{false, false, true}, false}, | |
| 164 | + | {"entry larger than the byte bound", 100, 1000, bigTree, []bool{false, false, false}, false}, | |
| 165 | + | {"disabled by entries", 0, 1 << 20, treeWithLast("b"), []bool{false, false, false}, false}, | |
| 166 | + | {"disabled by bytes", 10, 0, treeWithLast("b"), []bool{false, false, false}, false}, | |
| 167 | + | } | |
| 168 | + | for _, tc := range tests { | |
| 169 | + | t.Run(tc.name, func(t *testing.T) { | |
| 170 | + | f := newBrowseFixture(t) | |
| 171 | + | f.browse.WithHistoryCache(tc.entries, tc.bytes) | |
| 172 | + | f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil) | |
| 173 | + | rev := domain.RevisionOf(oid("a")) | |
| 174 | + | calls := map[string]int{} | |
| 175 | + | f.reader.EXPECT().GetTree(mock.Anything, pub.Name, rev, mock.Anything, true). | |
| 176 | + | RunAndReturn(func(_ context.Context, _ domain.RepoName, _ domain.Revision, p domain.TreePath, _ bool) (domain.Tree, error) { | |
| 177 | + | calls[p.String()]++ | |
| 178 | + | return tc.tree, nil | |
| 179 | + | }) | |
| 180 | + | get := func(p string) { | |
| 181 | + | _, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, rev, treePath(t, p), true) | |
| 182 | + | require.NoError(t, err) | |
| 183 | + | } | |
| 184 | + | get("p0") | |
| 185 | + | get("p1") | |
| 186 | + | if tc.touch { | |
| 187 | + | get("p0") | |
| 188 | + | } | |
| 189 | + | get("p2") | |
| 190 | + | before := map[string]int{"p0": calls["p0"], "p1": calls["p1"], "p2": calls["p2"]} | |
| 191 | + | // Probe from the newest to the oldest so probing one does not evict | |
| 192 | + | // the next one asked: asking a miss stores it. | |
| 193 | + | for i := len(paths) - 1; i >= 0; i-- { | |
| 194 | + | if !tc.wantCached[i] { | |
| 195 | + | continue | |
| 196 | + | } | |
| 197 | + | get(paths[i]) | |
| 198 | + | assert.Equal(t, before[paths[i]], calls[paths[i]], "%s should be a hit", paths[i]) | |
| 199 | + | } | |
| 200 | + | for i, p := range paths { | |
| 201 | + | if tc.wantCached[i] { | |
| 202 | + | continue | |
| 203 | + | } | |
| 204 | + | get(p) | |
| 205 | + | assert.Equal(t, before[p]+1, calls[p], "%s should be a miss", p) | |
| 206 | + | break // a miss stores the entry and may evict the others | |
| 207 | + | } | |
| 208 | + | }) | |
| 209 | + | } | |
| 210 | + | } | |
| 211 | + | ||
| 212 | + | // TestListCommitsLastChangeCache: the latest commit of a path at a full | |
| 213 | + | // commit id is cached; every other page shape is not. | |
| 214 | + | func TestListCommitsLastChangeCache(t *testing.T) { | |
| 215 | + | ctx := context.Background() | |
| 216 | + | pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic} | |
| 217 | + | full := domain.RevisionOf(oid("a")) | |
| 218 | + | main, err := domain.ParseRevision("main") | |
| 219 | + | require.NoError(t, err) | |
| 220 | + | file := treePath(t, "d/file.go") | |
| 221 | + | ||
| 222 | + | tests := []struct { | |
| 223 | + | name string | |
| 224 | + | rev domain.Revision | |
| 225 | + | path domain.TreePath | |
| 226 | + | size int | |
| 227 | + | calls int // reader calls for two identical requests | |
| 228 | + | }{ | |
| 229 | + | {"last change at a full id", full, file, 1, 1}, | |
| 230 | + | {"last change of the root at a full id", full, domain.TreePath{}, 1, 1}, | |
| 231 | + | {"name", main, file, 1, 2}, | |
| 232 | + | {"HEAD", domain.Revision{}, file, 1, 2}, | |
| 233 | + | {"larger page", full, file, 2, 2}, | |
| 234 | + | } | |
| 235 | + | for _, tc := range tests { | |
| 236 | + | t.Run(tc.name, func(t *testing.T) { | |
| 237 | + | f := newBrowseFixture(t) | |
| 238 | + | f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil) | |
| 239 | + | q := ports.CommitQuery{Revision: tc.rev, Path: tc.path, Limit: tc.size} | |
| 240 | + | f.reader.EXPECT().ListCommits(mock.Anything, pub.Name, q).Return(ports.CommitPage{ | |
| 241 | + | Head: oid("a"), | |
| 242 | + | Commits: []domain.Commit{{ID: oid("c"), Parents: []domain.ObjectID{oid("d")}}}, | |
| 243 | + | }, nil).Times(tc.calls) | |
| 244 | + | for range 2 { | |
| 245 | + | page, err := f.browse.ListCommits(ctx, domain.Anonymous(), pub.Name, | |
| 246 | + | app.ListCommitsParams{Revision: tc.rev, Path: tc.path, PageSize: tc.size}) | |
| 247 | + | require.NoError(t, err) | |
| 248 | + | require.Len(t, page.Commits, 1) | |
| 249 | + | page.Commits[0].Parents[0] = oid("f") // must not leak into the cache | |
| 250 | + | } | |
| 251 | + | }) | |
| 252 | + | } | |
| 253 | + | ||
| 254 | + | t.Run("a following page is not cached", func(t *testing.T) { | |
| 255 | + | f := newBrowseFixture(t) | |
| 256 | + | f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil) | |
| 257 | + | f.reader.EXPECT().ListCommits(mock.Anything, pub.Name, ports.CommitQuery{Revision: full, Path: file, Limit: 1}). | |
| 258 | + | Return(ports.CommitPage{Head: oid("a"), Commits: []domain.Commit{{ID: oid("c")}}, HasMore: true}, nil).Once() | |
| 259 | + | first, err := f.browse.ListCommits(ctx, domain.Anonymous(), pub.Name, app.ListCommitsParams{Revision: full, Path: file, PageSize: 1}) | |
| 260 | + | require.NoError(t, err) | |
| 261 | + | require.NotEmpty(t, first.NextPageToken) | |
| 262 | + | f.reader.EXPECT().ListCommits(mock.Anything, pub.Name, ports.CommitQuery{Revision: domain.RevisionOf(oid("a")), Path: file, Skip: 1, Limit: 1}). | |
| 263 | + | Return(ports.CommitPage{Head: oid("a")}, nil).Twice() | |
| 264 | + | for range 2 { | |
| 265 | + | _, err = f.browse.ListCommits(ctx, domain.Anonymous(), pub.Name, app.ListCommitsParams{Path: file, PageSize: 1, PageToken: first.NextPageToken}) | |
| 266 | + | require.NoError(t, err) | |
| 267 | + | } | |
| 268 | + | }) | |
| 269 | + | } | |
| 270 | + | ||
| 271 | + | func TestHistoryCacheConcurrent(t *testing.T) { | |
| 272 | + | ctx := context.Background() | |
| 273 | + | pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic} | |
| 274 | + | f := newBrowseFixture(t) | |
| 275 | + | f.browse.WithHistoryCache(4, 1<<20) | |
| 276 | + | f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil) | |
| 277 | + | f.reader.EXPECT().GetTree(mock.Anything, pub.Name, mock.Anything, mock.Anything, true).Return(treeWithLast("b"), nil) | |
| 278 | + | paths := []string{"a", "b", "c", "d", "e", "f", "g"} | |
| 279 | + | var wg sync.WaitGroup | |
| 280 | + | for w := range 8 { | |
| 281 | + | wg.Go(func() { | |
| 282 | + | for i := range 200 { | |
| 283 | + | p, err := domain.ParseTreePath(paths[(i+w)%len(paths)]) | |
| 284 | + | if err != nil { | |
| 285 | + | t.Error(err) | |
| 286 | + | return | |
| 287 | + | } | |
| 288 | + | if _, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, domain.RevisionOf(oid("a")), p, true); err != nil { | |
| 289 | + | t.Error(err) | |
| 290 | + | return | |
| 291 | + | } | |
| 292 | + | } | |
| 293 | + | }) | |
| 294 | + | } | |
| 295 | + | wg.Wait() | |
| 296 | + | } | |
| @@ -0,0 +1,296 @@ | |||||
| 1 | + | // SPDX-License-Identifier: AGPL-3.0-or-later | |||
| 2 | + | ||||
| 3 | + | package app_test | |||
| 4 | + | ||||
| 5 | + | import ( | |||
| 6 | + | "context" | |||
| 7 | + | "strings" | |||
| 8 | + | "sync" | |||
| 9 | + | "testing" | |||
| 10 | + | ||||
| 11 | + | "github.com/stretchr/testify/assert" | |||
| 12 | + | "github.com/stretchr/testify/mock" | |||
| 13 | + | "github.com/stretchr/testify/require" | |||
| 14 | + | ||||
| 15 | + | "bnema.dev/ermite/internal/app" | |||
| 16 | + | "bnema.dev/ermite/internal/domain" | |||
| 17 | + | "bnema.dev/ermite/internal/ports" | |||
| 18 | + | ) | |||
| 19 | + | ||||
| 20 | + | func treePath(t *testing.T, s string) domain.TreePath { | |||
| 21 | + | t.Helper() | |||
| 22 | + | p, err := domain.ParseTreePath(s) | |||
| 23 | + | require.NoError(t, err) | |||
| 24 | + | return p | |||
| 25 | + | } | |||
| 26 | + | ||||
| 27 | + | func treeWithLast(c string) domain.Tree { | |||
| 28 | + | commit := domain.Commit{ID: oid(c), Parents: []domain.ObjectID{oid("0")}, Summary: "s"} | |||
| 29 | + | return domain.Tree{ | |||
| 30 | + | CommitID: oid("a"), | |||
| 31 | + | Entries: []domain.TreeEntry{{Name: "f", Path: "d/f", LastCommit: &commit}}, | |||
| 32 | + | } | |||
| 33 | + | } | |||
| 34 | + | ||||
| 35 | + | // TestGetTreeLastCommitCache covers which GetTree calls are served from the | |||
| 36 | + | // cache: only the ones with last commits at a full commit id. | |||
| 37 | + | func TestGetTreeLastCommitCache(t *testing.T) { | |||
| 38 | + | ctx := context.Background() | |||
| 39 | + | pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic} | |||
| 40 | + | full := domain.RevisionOf(oid("a")) | |||
| 41 | + | branch, err := domain.ParseRevision("main") | |||
| 42 | + | require.NoError(t, err) | |||
| 43 | + | dir := treePath(t, "d") | |||
| 44 | + | ||||
| 45 | + | tests := []struct { | |||
| 46 | + | name string | |||
| 47 | + | rev domain.Revision | |||
| 48 | + | includeLast bool | |||
| 49 | + | wantReaderOn int // reader calls for two identical requests | |||
| 50 | + | }{ | |||
| 51 | + | {"full id with last commits is cached", full, true, 1}, | |||
| 52 | + | {"full id without last commits is not", full, false, 2}, | |||
| 53 | + | {"branch name is not", branch, true, 2}, | |||
| 54 | + | {"HEAD is not", domain.Revision{}, true, 2}, | |||
| 55 | + | } | |||
| 56 | + | for _, tc := range tests { | |||
| 57 | + | t.Run(tc.name, func(t *testing.T) { | |||
| 58 | + | f := newBrowseFixture(t) | |||
| 59 | + | f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil) | |||
| 60 | + | f.reader.EXPECT().GetTree(mock.Anything, pub.Name, tc.rev, dir, tc.includeLast). | |||
| 61 | + | Return(treeWithLast("b"), nil).Times(tc.wantReaderOn) | |||
| 62 | + | for range 2 { | |||
| 63 | + | tree, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, tc.rev, dir, tc.includeLast) | |||
| 64 | + | require.NoError(t, err) | |||
| 65 | + | assert.Equal(t, "d/f", tree.Entries[0].Path) | |||
| 66 | + | } | |||
| 67 | + | }) | |||
| 68 | + | } | |||
| 69 | + | } | |||
| 70 | + | ||||
| 71 | + | func TestGetTreeLastCommitCacheKey(t *testing.T) { | |||
| 72 | + | ctx := context.Background() | |||
| 73 | + | pubA := domain.Repo{Name: name(t, "a"), Visibility: domain.VisibilityPublic} | |||
| 74 | + | pubB := domain.Repo{Name: name(t, "b"), Visibility: domain.VisibilityPublic} | |||
| 75 | + | revA, revB := domain.RevisionOf(oid("a")), domain.RevisionOf(oid("b")) | |||
| 76 | + | d1, d2 := treePath(t, "d1"), treePath(t, "d2") | |||
| 77 | + | ||||
| 78 | + | // Each distinct (repo, commit, path) is one reader call, then a hit. | |||
| 79 | + | keys := []struct { | |||
| 80 | + | repo domain.Repo | |||
| 81 | + | rev domain.Revision | |||
| 82 | + | path domain.TreePath | |||
| 83 | + | }{ | |||
| 84 | + | {pubA, revA, d1}, {pubB, revA, d1}, {pubA, revB, d1}, {pubA, revA, d2}, {pubA, revA, domain.TreePath{}}, | |||
| 85 | + | } | |||
| 86 | + | f := newBrowseFixture(t) | |||
| 87 | + | f.meta.EXPECT().Get(mock.Anything, mock.Anything).RunAndReturn( | |||
| 88 | + | func(_ context.Context, n domain.RepoName) (domain.Repo, error) { | |||
| 89 | + | return domain.Repo{Name: n, Visibility: domain.VisibilityPublic}, nil | |||
| 90 | + | }) | |||
| 91 | + | for _, k := range keys { | |||
| 92 | + | f.reader.EXPECT().GetTree(mock.Anything, k.repo.Name, k.rev, k.path, true).Return(treeWithLast("b"), nil).Once() | |||
| 93 | + | } | |||
| 94 | + | for range 2 { | |||
| 95 | + | for _, k := range keys { | |||
| 96 | + | _, err := f.browse.GetTree(ctx, domain.Anonymous(), k.repo.Name, k.rev, k.path, true) | |||
| 97 | + | require.NoError(t, err) | |||
| 98 | + | } | |||
| 99 | + | } | |||
| 100 | + | } | |||
| 101 | + | ||||
| 102 | + | // TestGetTreeCacheKeepsAccessRule: a hit never skips the access check, and an | |||
| 103 | + | // error is not cached. | |||
| 104 | + | func TestGetTreeCacheKeepsAccessRule(t *testing.T) { | |||
| 105 | + | ctx := context.Background() | |||
| 106 | + | priv := domain.Repo{Name: name(t, "priv"), Visibility: domain.VisibilityPrivate} | |||
| 107 | + | rev, dir := domain.RevisionOf(oid("a")), treePath(t, "d") | |||
| 108 | + | ||||
| 109 | + | f := newBrowseFixture(t) | |||
| 110 | + | f.meta.EXPECT().Get(mock.Anything, priv.Name).Return(priv, nil) | |||
| 111 | + | f.reader.EXPECT().GetTree(mock.Anything, priv.Name, rev, dir, true).Return(domain.Tree{}, domain.ErrNotDirectory).Once() | |||
| 112 | + | f.reader.EXPECT().GetTree(mock.Anything, priv.Name, rev, dir, true).Return(treeWithLast("b"), nil).Once() | |||
| 113 | + | ||||
| 114 | + | _, err := f.browse.GetTree(ctx, domain.Root(), priv.Name, rev, dir, true) | |||
| 115 | + | require.ErrorIs(t, err, domain.ErrNotDirectory) // not cached | |||
| 116 | + | _, err = f.browse.GetTree(ctx, domain.Root(), priv.Name, rev, dir, true) | |||
| 117 | + | require.NoError(t, err) // now cached | |||
| 118 | + | _, err = f.browse.GetTree(ctx, domain.Anonymous(), priv.Name, rev, dir, true) | |||
| 119 | + | assert.ErrorIs(t, err, domain.ErrNotFound) | |||
| 120 | + | } | |||
| 121 | + | ||||
| 122 | + | // TestGetTreeCacheReturnsCopies: a caller that edits its tree does not alter | |||
| 123 | + | // what the next one gets. | |||
| 124 | + | func TestGetTreeCacheReturnsCopies(t *testing.T) { | |||
| 125 | + | ctx := context.Background() | |||
| 126 | + | pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic} | |||
| 127 | + | rev, dir := domain.RevisionOf(oid("a")), treePath(t, "d") | |||
| 128 | + | f := newBrowseFixture(t) | |||
| 129 | + | f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil) | |||
| 130 | + | f.reader.EXPECT().GetTree(mock.Anything, pub.Name, rev, dir, true).Return(treeWithLast("b"), nil).Once() | |||
| 131 | + | ||||
| 132 | + | for range 3 { | |||
| 133 | + | tree, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, rev, dir, true) | |||
| 134 | + | require.NoError(t, err) | |||
| 135 | + | require.Len(t, tree.Entries, 1) | |||
| 136 | + | assert.Equal(t, "s", tree.Entries[0].LastCommit.Summary) | |||
| 137 | + | tree.Entries[0].LastCommit.Summary = "changed" | |||
| 138 | + | tree.Entries[0].LastCommit.Parents[0] = oid("f") | |||
| 139 | + | tree.Entries[0].Name = "changed" | |||
| 140 | + | } | |||
| 141 | + | } | |||
| 142 | + | ||||
| 143 | + | // TestHistoryCacheBounds: the entry bound and the byte bound evict the least | |||
| 144 | + | // recently used answer. | |||
| 145 | + | func TestHistoryCacheBounds(t *testing.T) { | |||
| 146 | + | ctx := context.Background() | |||
| 147 | + | pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic} | |||
| 148 | + | paths := []string{"p0", "p1", "p2"} | |||
| 149 | + | ||||
| 150 | + | bigTree := treeWithLast("b") | |||
| 151 | + | bigTree.Entries[0].Name = strings.Repeat("x", 4096) | |||
| 152 | + | ||||
| 153 | + | tests := []struct { | |||
| 154 | + | name string | |||
| 155 | + | entries int | |||
| 156 | + | bytes int64 | |||
| 157 | + | tree domain.Tree | |||
| 158 | + | wantCached []bool // after requesting p0, p1, p2 (and touching p0 before p2 when touch) | |||
| 159 | + | touch bool | |||
| 160 | + | }{ | |||
| 161 | + | {"entry bound", 2, 1 << 20, treeWithLast("b"), []bool{false, true, true}, false}, | |||
| 162 | + | {"entry bound is LRU", 2, 1 << 20, treeWithLast("b"), []bool{true, false, true}, true}, | |||
| 163 | + | {"byte bound", 100, 6000, bigTree, []bool{false, false, true}, false}, | |||
| 164 | + | {"entry larger than the byte bound", 100, 1000, bigTree, []bool{false, false, false}, false}, | |||
| 165 | + | {"disabled by entries", 0, 1 << 20, treeWithLast("b"), []bool{false, false, false}, false}, | |||
| 166 | + | {"disabled by bytes", 10, 0, treeWithLast("b"), []bool{false, false, false}, false}, | |||
| 167 | + | } | |||
| 168 | + | for _, tc := range tests { | |||
| 169 | + | t.Run(tc.name, func(t *testing.T) { | |||
| 170 | + | f := newBrowseFixture(t) | |||
| 171 | + | f.browse.WithHistoryCache(tc.entries, tc.bytes) | |||
| 172 | + | f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil) | |||
| 173 | + | rev := domain.RevisionOf(oid("a")) | |||
| 174 | + | calls := map[string]int{} | |||
| 175 | + | f.reader.EXPECT().GetTree(mock.Anything, pub.Name, rev, mock.Anything, true). | |||
| 176 | + | RunAndReturn(func(_ context.Context, _ domain.RepoName, _ domain.Revision, p domain.TreePath, _ bool) (domain.Tree, error) { | |||
| 177 | + | calls[p.String()]++ | |||
| 178 | + | return tc.tree, nil | |||
| 179 | + | }) | |||
| 180 | + | get := func(p string) { | |||
| 181 | + | _, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, rev, treePath(t, p), true) | |||
| 182 | + | require.NoError(t, err) | |||
| 183 | + | } | |||
| 184 | + | get("p0") | |||
| 185 | + | get("p1") | |||
| 186 | + | if tc.touch { | |||
| 187 | + | get("p0") | |||
| 188 | + | } | |||
| 189 | + | get("p2") | |||
| 190 | + | before := map[string]int{"p0": calls["p0"], "p1": calls["p1"], "p2": calls["p2"]} | |||
| 191 | + | // Probe from the newest to the oldest so probing one does not evict | |||
| 192 | + | // the next one asked: asking a miss stores it. | |||
| 193 | + | for i := len(paths) - 1; i >= 0; i-- { | |||
| 194 | + | if !tc.wantCached[i] { | |||
| 195 | + | continue | |||
| 196 | + | } | |||
| 197 | + | get(paths[i]) | |||
| 198 | + | assert.Equal(t, before[paths[i]], calls[paths[i]], "%s should be a hit", paths[i]) | |||
| 199 | + | } | |||
| 200 | + | for i, p := range paths { | |||
| 201 | + | if tc.wantCached[i] { | |||
| 202 | + | continue | |||
| 203 | + | } | |||
| 204 | + | get(p) | |||
| 205 | + | assert.Equal(t, before[p]+1, calls[p], "%s should be a miss", p) | |||
| 206 | + | break // a miss stores the entry and may evict the others | |||
| 207 | + | } | |||
| 208 | + | }) | |||
| 209 | + | } | |||
| 210 | + | } | |||
| 211 | + | ||||
| 212 | + | // TestListCommitsLastChangeCache: the latest commit of a path at a full | |||
| 213 | + | // commit id is cached; every other page shape is not. | |||
| 214 | + | func TestListCommitsLastChangeCache(t *testing.T) { | |||
| 215 | + | ctx := context.Background() | |||
| 216 | + | pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic} | |||
| 217 | + | full := domain.RevisionOf(oid("a")) | |||
| 218 | + | main, err := domain.ParseRevision("main") | |||
| 219 | + | require.NoError(t, err) | |||
| 220 | + | file := treePath(t, "d/file.go") | |||
| 221 | + | ||||
| 222 | + | tests := []struct { | |||
| 223 | + | name string | |||
| 224 | + | rev domain.Revision | |||
| 225 | + | path domain.TreePath | |||
| 226 | + | size int | |||
| 227 | + | calls int // reader calls for two identical requests | |||
| 228 | + | }{ | |||
| 229 | + | {"last change at a full id", full, file, 1, 1}, | |||
| 230 | + | {"last change of the root at a full id", full, domain.TreePath{}, 1, 1}, | |||
| 231 | + | {"name", main, file, 1, 2}, | |||
| 232 | + | {"HEAD", domain.Revision{}, file, 1, 2}, | |||
| 233 | + | {"larger page", full, file, 2, 2}, | |||
| 234 | + | } | |||
| 235 | + | for _, tc := range tests { | |||
| 236 | + | t.Run(tc.name, func(t *testing.T) { | |||
| 237 | + | f := newBrowseFixture(t) | |||
| 238 | + | f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil) | |||
| 239 | + | q := ports.CommitQuery{Revision: tc.rev, Path: tc.path, Limit: tc.size} | |||
| 240 | + | f.reader.EXPECT().ListCommits(mock.Anything, pub.Name, q).Return(ports.CommitPage{ | |||
| 241 | + | Head: oid("a"), | |||
| 242 | + | Commits: []domain.Commit{{ID: oid("c"), Parents: []domain.ObjectID{oid("d")}}}, | |||
| 243 | + | }, nil).Times(tc.calls) | |||
| 244 | + | for range 2 { | |||
| 245 | + | page, err := f.browse.ListCommits(ctx, domain.Anonymous(), pub.Name, | |||
| 246 | + | app.ListCommitsParams{Revision: tc.rev, Path: tc.path, PageSize: tc.size}) | |||
| 247 | + | require.NoError(t, err) | |||
| 248 | + | require.Len(t, page.Commits, 1) | |||
| 249 | + | page.Commits[0].Parents[0] = oid("f") // must not leak into the cache | |||
| 250 | + | } | |||
| 251 | + | }) | |||
| 252 | + | } | |||
| 253 | + | ||||
| 254 | + | t.Run("a following page is not cached", func(t *testing.T) { | |||
| 255 | + | f := newBrowseFixture(t) | |||
| 256 | + | f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil) | |||
| 257 | + | f.reader.EXPECT().ListCommits(mock.Anything, pub.Name, ports.CommitQuery{Revision: full, Path: file, Limit: 1}). | |||
| 258 | + | Return(ports.CommitPage{Head: oid("a"), Commits: []domain.Commit{{ID: oid("c")}}, HasMore: true}, nil).Once() | |||
| 259 | + | first, err := f.browse.ListCommits(ctx, domain.Anonymous(), pub.Name, app.ListCommitsParams{Revision: full, Path: file, PageSize: 1}) | |||
| 260 | + | require.NoError(t, err) | |||
| 261 | + | require.NotEmpty(t, first.NextPageToken) | |||
| 262 | + | f.reader.EXPECT().ListCommits(mock.Anything, pub.Name, ports.CommitQuery{Revision: domain.RevisionOf(oid("a")), Path: file, Skip: 1, Limit: 1}). | |||
| 263 | + | Return(ports.CommitPage{Head: oid("a")}, nil).Twice() | |||
| 264 | + | for range 2 { | |||
| 265 | + | _, err = f.browse.ListCommits(ctx, domain.Anonymous(), pub.Name, app.ListCommitsParams{Path: file, PageSize: 1, PageToken: first.NextPageToken}) | |||
| 266 | + | require.NoError(t, err) | |||
| 267 | + | } | |||
| 268 | + | }) | |||
| 269 | + | } | |||
| 270 | + | ||||
| 271 | + | func TestHistoryCacheConcurrent(t *testing.T) { | |||
| 272 | + | ctx := context.Background() | |||
| 273 | + | pub := domain.Repo{Name: name(t, "pub"), Visibility: domain.VisibilityPublic} | |||
| 274 | + | f := newBrowseFixture(t) | |||
| 275 | + | f.browse.WithHistoryCache(4, 1<<20) | |||
| 276 | + | f.meta.EXPECT().Get(mock.Anything, pub.Name).Return(pub, nil) | |||
| 277 | + | f.reader.EXPECT().GetTree(mock.Anything, pub.Name, mock.Anything, mock.Anything, true).Return(treeWithLast("b"), nil) | |||
| 278 | + | paths := []string{"a", "b", "c", "d", "e", "f", "g"} | |||
| 279 | + | var wg sync.WaitGroup | |||
| 280 | + | for w := range 8 { | |||
| 281 | + | wg.Go(func() { | |||
| 282 | + | for i := range 200 { | |||
| 283 | + | p, err := domain.ParseTreePath(paths[(i+w)%len(paths)]) | |||
| 284 | + | if err != nil { | |||
| 285 | + | t.Error(err) | |||
| 286 | + | return | |||
| 287 | + | } | |||
| 288 | + | if _, err := f.browse.GetTree(ctx, domain.Anonymous(), pub.Name, domain.RevisionOf(oid("a")), p, true); err != nil { | |||
| 289 | + | t.Error(err) | |||
| 290 | + | return | |||
| 291 | + | } | |||
| 292 | + | } | |||
| 293 | + | }) | |||
| 294 | + | } | |||
| 295 | + | wg.Wait() | |||
| 296 | + | } | |||