perf(gitcli): fill tree last commits with a small worker pool

commit: 2a9866e74b1c5043b5a42628f3b62f86e27bbe16
author: brice · 12d ago · 2026-10-10 11:41 UTC
parent: 13c62e2
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

1 file · +51 -10

M internal/adapters/gitcli/read.go +51 -10


@@ -14,6 +14,7 @@ import (
14	14		"slices"
15	15		"strconv"
16	16		"strings"
	17	+	"sync"
17	18		"time"
18	19
19	20		"bnema.dev/ermite/internal/domain"

@@ -409,25 +410,65 @@ func parseTreeEntry(rec string, parent domain.TreePath) (domain.TreeEntry, error
409	410		}
410	411
411	412		// fillLastCommits sets the last commit of the first MaxLastCommitEntries
412		-	// entries with `git log -1 <commit> -- <path>`, one after the other. Git
413		-	// answers each from the commit-graph and its changed-path filters when they
414		-	// exist. Every command takes a slot of the exec semaphore, so a large
415		-	// directory cannot use more than its share of the processes the server may
416		-	// run. Entries beyond the bound are left without one and the tree says so.
	413	+	// entries with `git log -1 <commit> -- <path>`, run by a small pool of
	414	+	// workers: min(4, max(1, MaxExec/2)) at most. Git answers each from the
	415	+	// commit-graph and its changed-path filters when they exist. Every command
	416	+	// takes a slot of the exec semaphore on its own (the workers hold none), so a
	417	+	// large directory cannot use more than its share of the processes the server
	418	+	// may run. Each worker writes its own entry, so the result does not depend on
	419	+	// the scheduling; the first error stops the others and is returned. Entries
	420	+	// beyond the bound are left without one and the tree says so.
	421	+	//
	422	+	//hexcheck:ignore suspicious-business-logic-in-adapter fills the tree with git output; the bound comes from the port
417	423		func (g Git) fillLastCommits(ctx context.Context, dir string, tree domain.Tree) error {
418	424		n := len(tree.Entries)
419	425		if n > domain.MaxLastCommitEntries {
420	426		n = domain.MaxLastCommitEntries
421	427		tree.LastCommitsOmitted = true
422	428		}
	429	+	workers := min(4, max(1, cap(g.exec)/2), n)
	430	+	if workers == 0 {
	431	+	return nil
	432	+	}
	433	+	ctx, cancel := context.WithCancel(ctx)
	434	+	defer cancel()
	435	+	var (
	436	+	wg sync.WaitGroup
	437	+	mu sync.Mutex
	438	+	firstErr error
	439	+	)
	440	+	next := make(chan int)
	441	+	for range workers {
	442	+	wg.Go(func() {
	443	+	for i := range next {
	444	+	c, err := g.lastCommit(ctx, dir, tree.CommitID, tree.Entries[i].Path)
	445	+	if err != nil {
	446	+	mu.Lock()
	447	+	if firstErr == nil {
	448	+	firstErr = err
	449	+	cancel()
	450	+	}
	451	+	mu.Unlock()
	452	+	continue
	453	+	}
	454	+	tree.Entries[i].LastCommit = &c
	455	+	}
	456	+	})
	457	+	}
	458	+	feed:
423	459		for i := range n {
424		-	c, err := g.lastCommit(ctx, dir, tree.CommitID, tree.Entries[i].Path)
425		-	if err != nil {
426		-	return err
	460	+	select {
	461	+	case next <- i:
	462	+	case <-ctx.Done():
	463	+	break feed
427	464		}
428		-	tree.Entries[i].LastCommit = &c
429	465		}
430		-	return nil
	466	+	close(next)
	467	+	wg.Wait()
	468	+	if firstErr != nil {
	469	+	return firstErr
	470	+	}
	471	+	return ctx.Err()
431	472		}
432	473
433	474		// lastCommit returns the newest commit reachable from head that changed path;


@@ -14,6 +14,7 @@ import (
14		"slices"	14		"slices"
15		"strconv"	15		"strconv"
16		"strings"	16		"strings"
			17	+	"sync"
17		"time"	18		"time"
18			19
19		"bnema.dev/ermite/internal/domain"	20		"bnema.dev/ermite/internal/domain"

@@ -409,25 +410,65 @@ func parseTreeEntry(rec string, parent domain.TreePath) (domain.TreeEntry, error
409		}	410		}
410			411
411		// fillLastCommits sets the last commit of the first MaxLastCommitEntries	412		// fillLastCommits sets the last commit of the first MaxLastCommitEntries
412	-	// entries with `git log -1 <commit> -- <path>`, one after the other. Git	413	+	// entries with `git log -1 <commit> -- <path>`, run by a small pool of
413	-	// answers each from the commit-graph and its changed-path filters when they	414	+	// workers: min(4, max(1, MaxExec/2)) at most. Git answers each from the
414	-	// exist. Every command takes a slot of the exec semaphore, so a large	415	+	// commit-graph and its changed-path filters when they exist. Every command
415	-	// directory cannot use more than its share of the processes the server may	416	+	// takes a slot of the exec semaphore on its own (the workers hold none), so a
416	-	// run. Entries beyond the bound are left without one and the tree says so.	417	+	// large directory cannot use more than its share of the processes the server
			418	+	// may run. Each worker writes its own entry, so the result does not depend on
			419	+	// the scheduling; the first error stops the others and is returned. Entries
			420	+	// beyond the bound are left without one and the tree says so.
			421	+	//
			422	+	//hexcheck:ignore suspicious-business-logic-in-adapter fills the tree with git output; the bound comes from the port
417		func (g Git) fillLastCommits(ctx context.Context, dir string, tree domain.Tree) error {	423		func (g Git) fillLastCommits(ctx context.Context, dir string, tree domain.Tree) error {
418		n := len(tree.Entries)	424		n := len(tree.Entries)
419		if n > domain.MaxLastCommitEntries {	425		if n > domain.MaxLastCommitEntries {
420		n = domain.MaxLastCommitEntries	426		n = domain.MaxLastCommitEntries
421		tree.LastCommitsOmitted = true	427		tree.LastCommitsOmitted = true
422		}	428		}
			429	+	workers := min(4, max(1, cap(g.exec)/2), n)
			430	+	if workers == 0 {
			431	+	return nil
			432	+	}
			433	+	ctx, cancel := context.WithCancel(ctx)
			434	+	defer cancel()
			435	+	var (
			436	+	wg sync.WaitGroup
			437	+	mu sync.Mutex
			438	+	firstErr error
			439	+	)
			440	+	next := make(chan int)
			441	+	for range workers {
			442	+	wg.Go(func() {
			443	+	for i := range next {
			444	+	c, err := g.lastCommit(ctx, dir, tree.CommitID, tree.Entries[i].Path)
			445	+	if err != nil {
			446	+	mu.Lock()
			447	+	if firstErr == nil {
			448	+	firstErr = err
			449	+	cancel()
			450	+	}
			451	+	mu.Unlock()
			452	+	continue
			453	+	}
			454	+	tree.Entries[i].LastCommit = &c
			455	+	}
			456	+	})
			457	+	}
			458	+	feed:
423		for i := range n {	459		for i := range n {
424	-	c, err := g.lastCommit(ctx, dir, tree.CommitID, tree.Entries[i].Path)	460	+	select {
425	-	if err != nil {	461	+	case next <- i:
426	-	return err	462	+	case <-ctx.Done():
			463	+	break feed
427		}	464		}
428	-	tree.Entries[i].LastCommit = &c
429		}	465		}
430	-	return nil	466	+	close(next)
			467	+	wg.Wait()
			468	+	if firstErr != nil {
			469	+	return firstErr
			470	+	}
			471	+	return ctx.Err()
431		}	472		}
432			473
433		// lastCommit returns the newest commit reachable from head that changed path;	474		// lastCommit returns the newest commit reachable from head that changed path;