detached 36d1371

feat(search): replace fuzzy search with SQLite FTS5 for accurate history matching

commit
36d1371bc6f1be1f950100522b69900c61032eb9
author
brice · · 2025-12-17 19:23 UTC
committer
brice · · 2025-12-17 22:02 UTC
parent
77a7a6d
signature
unsigned
5 files · +75 -50
Unified
UnifiedSplit
M go.mod +1 -1
@@ -12,7 +12,6 @@ require (
1212 github.com/ncruces/go-sqlite3 v0.30.3
1313 github.com/pressly/goose/v3 v3.26.0
1414 github.com/rs/zerolog v1.33.0
15- github.com/sahilm/fuzzy v0.1.1
1615 github.com/spf13/cobra v1.10.2
1716 github.com/spf13/viper v1.21.0
1817 github.com/stretchr/testify v1.11.1
@@ -44,6 +43,7 @@ require (
4443 github.com/pmezard/go-difflib v1.0.0 // indirect
4544 github.com/rivo/uniseg v0.4.7 // indirect
4645 github.com/sagikazarmark/locafero v0.11.0 // indirect
46+ github.com/sahilm/fuzzy v0.1.1 // indirect
4747 github.com/sethvargo/go-retry v0.3.0 // indirect
4848 github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
4949 github.com/spf13/afero v1.15.0 // indirect
@@ -12,7 +12,6 @@ require (
12 github.com/ncruces/go-sqlite3 v0.30.312 github.com/ncruces/go-sqlite3 v0.30.3
13 github.com/pressly/goose/v3 v3.26.013 github.com/pressly/goose/v3 v3.26.0
14 github.com/rs/zerolog v1.33.014 github.com/rs/zerolog v1.33.0
15- github.com/sahilm/fuzzy v0.1.1
16 github.com/spf13/cobra v1.10.215 github.com/spf13/cobra v1.10.2
17 github.com/spf13/viper v1.21.016 github.com/spf13/viper v1.21.0
18 github.com/stretchr/testify v1.11.117 github.com/stretchr/testify v1.11.1
@@ -44,6 +43,7 @@ require (
44 github.com/pmezard/go-difflib v1.0.0 // indirect43 github.com/pmezard/go-difflib v1.0.0 // indirect
45 github.com/rivo/uniseg v0.4.7 // indirect44 github.com/rivo/uniseg v0.4.7 // indirect
46 github.com/sagikazarmark/locafero v0.11.0 // indirect45 github.com/sagikazarmark/locafero v0.11.0 // indirect
46+ github.com/sahilm/fuzzy v0.1.1 // indirect
47 github.com/sethvargo/go-retry v0.3.0 // indirect47 github.com/sethvargo/go-retry v0.3.0 // indirect
48 github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect48 github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
49 github.com/spf13/afero v1.15.0 // indirect49 github.com/spf13/afero v1.15.0 // indirect
M internal/application/usecase/search_history.go +8 -43
@@ -8,7 +8,6 @@ import (
88 "github.com/bnema/dumber/internal/domain/entity"
99 "github.com/bnema/dumber/internal/domain/repository"
1010 "github.com/bnema/dumber/internal/logging"
11- "github.com/sahilm/fuzzy"
1211 )
1312
1413 // SearchHistoryUseCase handles history search and retrieval operations.
@@ -34,8 +33,8 @@ type SearchOutput struct {
3433 Matches []entity.HistoryMatch
3534 }
3635
37-// Search performs a fuzzy search on history entries.
38-// Uses in-memory fuzzy matching on title + URL for token-based matching in any order.
36+// Search performs a full-text search on history entries using SQLite FTS5.
37+// Returns only entries that actually match the query terms.
3938 func (uc *SearchHistoryUseCase) Search(ctx context.Context, input SearchInput) (*SearchOutput, error) {
4039 log := logging.FromContext(ctx)
4140
@@ -48,52 +47,18 @@ func (uc *SearchHistoryUseCase) Search(ctx context.Context, input SearchInput) (
4847 limit = 20 // Default limit
4948 }
5049
51- // Fetch more candidates for fuzzy filtering (10x the limit)
52- candidateLimit := limit * 10
53- if candidateLimit < 100 {
54- candidateLimit = 100
55- }
56-
57- candidates, err := uc.historyRepo.GetRecent(ctx, candidateLimit, 0)
50+ // Use repository's FTS5 search
51+ matches, err := uc.historyRepo.Search(ctx, input.Query, limit)
5852 if err != nil {
59- return nil, fmt.Errorf("failed to get history candidates: %w", err)
60- }
61-
62- if len(candidates) == 0 {
63- return &SearchOutput{Matches: []entity.HistoryMatch{}}, nil
64- }
65-
66- // Build searchable strings: "title url" for each entry
67- searchStrings := make([]string, len(candidates))
68- for i, e := range candidates {
69- searchStrings[i] = e.Title + " " + e.URL
53+ return nil, fmt.Errorf("failed to search history: %w", err)
7054 }
7155
72- // Fuzzy match
73- fuzzyMatches := fuzzy.Find(input.Query, searchStrings)
74-
7556 log.Debug().
7657 Str("query", input.Query).
77- Int("candidates", len(candidates)).
78- Int("matches", len(fuzzyMatches)).
79- Msg("fuzzy search completed")
80-
81- // Convert to HistoryMatch with scores, limited to requested limit
82- resultLimit := limit
83- if len(fuzzyMatches) < resultLimit {
84- resultLimit = len(fuzzyMatches)
85- }
86-
87- results := make([]entity.HistoryMatch, 0, resultLimit)
88- for i := 0; i < resultLimit; i++ {
89- m := fuzzyMatches[i]
90- results = append(results, entity.HistoryMatch{
91- Entry: candidates[m.Index],
92- Score: float64(m.Score),
93- })
94- }
58+ Int("matches", len(matches)).
59+ Msg("FTS5 search completed")
9560
96- return &SearchOutput{Matches: results}, nil
61+ return &SearchOutput{Matches: matches}, nil
9762 }
9863
9964 // GetRecent retrieves recent history entries with pagination.
@@ -8,7 +8,6 @@ import (
8 "github.com/bnema/dumber/internal/domain/entity"8 "github.com/bnema/dumber/internal/domain/entity"
9 "github.com/bnema/dumber/internal/domain/repository"9 "github.com/bnema/dumber/internal/domain/repository"
10 "github.com/bnema/dumber/internal/logging"10 "github.com/bnema/dumber/internal/logging"
11- "github.com/sahilm/fuzzy"
12 )11 )
13 12
14 // SearchHistoryUseCase handles history search and retrieval operations.13 // SearchHistoryUseCase handles history search and retrieval operations.
@@ -34,8 +33,8 @@ type SearchOutput struct {
34 Matches []entity.HistoryMatch33 Matches []entity.HistoryMatch
35 }34 }
36 35
37-// Search performs a fuzzy search on history entries.36+// Search performs a full-text search on history entries using SQLite FTS5.
38-// Uses in-memory fuzzy matching on title + URL for token-based matching in any order.37+// Returns only entries that actually match the query terms.
39 func (uc *SearchHistoryUseCase) Search(ctx context.Context, input SearchInput) (*SearchOutput, error) {38 func (uc *SearchHistoryUseCase) Search(ctx context.Context, input SearchInput) (*SearchOutput, error) {
40 log := logging.FromContext(ctx)39 log := logging.FromContext(ctx)
41 40
@@ -48,52 +47,18 @@ func (uc *SearchHistoryUseCase) Search(ctx context.Context, input SearchInput) (
48 limit = 20 // Default limit47 limit = 20 // Default limit
49 }48 }
50 49
51- // Fetch more candidates for fuzzy filtering (10x the limit)50+ // Use repository's FTS5 search
52- candidateLimit := limit * 1051+ matches, err := uc.historyRepo.Search(ctx, input.Query, limit)
53- if candidateLimit < 100 {
54- candidateLimit = 100
55- }
56-
57- candidates, err := uc.historyRepo.GetRecent(ctx, candidateLimit, 0)
58 if err != nil {52 if err != nil {
59- return nil, fmt.Errorf("failed to get history candidates: %w", err)53+ return nil, fmt.Errorf("failed to search history: %w", err)
60- }
61-
62- if len(candidates) == 0 {
63- return &SearchOutput{Matches: []entity.HistoryMatch{}}, nil
64- }
65-
66- // Build searchable strings: "title url" for each entry
67- searchStrings := make([]string, len(candidates))
68- for i, e := range candidates {
69- searchStrings[i] = e.Title + " " + e.URL
70 }54 }
71 55
72- // Fuzzy match
73- fuzzyMatches := fuzzy.Find(input.Query, searchStrings)
74-
75 log.Debug().56 log.Debug().
76 Str("query", input.Query).57 Str("query", input.Query).
77- Int("candidates", len(candidates)).58+ Int("matches", len(matches)).
78- Int("matches", len(fuzzyMatches)).59+ Msg("FTS5 search completed")
79- Msg("fuzzy search completed")
80-
81- // Convert to HistoryMatch with scores, limited to requested limit
82- resultLimit := limit
83- if len(fuzzyMatches) < resultLimit {
84- resultLimit = len(fuzzyMatches)
85- }
86-
87- results := make([]entity.HistoryMatch, 0, resultLimit)
88- for i := 0; i < resultLimit; i++ {
89- m := fuzzyMatches[i]
90- results = append(results, entity.HistoryMatch{
91- Entry: candidates[m.Index],
92- Score: float64(m.Score),
93- })
94- }
95 60
96- return &SearchOutput{Matches: results}, nil61+ return &SearchOutput{Matches: matches}, nil
97 }62 }
98 63
99 // GetRecent retrieves recent history entries with pagination.64 // GetRecent retrieves recent history entries with pagination.
M internal/infrastructure/persistence/sqlite/history_repo.go +13 -6
@@ -43,20 +43,27 @@ func (r *historyRepo) FindByURL(ctx context.Context, url string) (*entity.Histor
4343 }
4444
4545 func (r *historyRepo) Search(ctx context.Context, query string, limit int) ([]entity.HistoryMatch, error) {
46- rows, err := r.queries.SearchHistory(ctx, sqlc.SearchHistoryParams{
47- Column1: sql.NullString{String: query, Valid: true},
48- Column2: sql.NullString{String: query, Valid: true},
49- Limit: int64(limit),
46+ log := logging.FromContext(ctx)
47+
48+ // Use FTS5 full-text search for better accuracy
49+ // FTS5 query syntax: use * for prefix matching
50+ ftsQuery := query + "*"
51+
52+ rows, err := r.queries.SearchHistoryFTS(ctx, sqlc.SearchHistoryFTSParams{
53+ Url: ftsQuery,
54+ Limit: int64(limit),
5055 })
5156 if err != nil {
52- return nil, err
57+ log.Debug().Err(err).Str("query", query).Msg("FTS search failed, no results")
58+ // FTS5 returns error on no match or invalid query, return empty results
59+ return []entity.HistoryMatch{}, nil
5360 }
5461
5562 matches := make([]entity.HistoryMatch, len(rows))
5663 for i, row := range rows {
5764 matches[i] = entity.HistoryMatch{
5865 Entry: historyFromRow(row),
59- Score: 1.0, // Basic score, could be enhanced with FTS ranking
66+ Score: 1.0, // FTS5 already ranked by bm25
6067 }
6168 }
6269 return matches, nil
@@ -43,20 +43,27 @@ func (r *historyRepo) FindByURL(ctx context.Context, url string) (*entity.Histor
43 }43 }
44 44
45 func (r *historyRepo) Search(ctx context.Context, query string, limit int) ([]entity.HistoryMatch, error) {45 func (r *historyRepo) Search(ctx context.Context, query string, limit int) ([]entity.HistoryMatch, error) {
46- rows, err := r.queries.SearchHistory(ctx, sqlc.SearchHistoryParams{46+ log := logging.FromContext(ctx)
47- Column1: sql.NullString{String: query, Valid: true},47+
48- Column2: sql.NullString{String: query, Valid: true},48+ // Use FTS5 full-text search for better accuracy
49- Limit: int64(limit),49+ // FTS5 query syntax: use * for prefix matching
50+ ftsQuery := query + "*"
51+
52+ rows, err := r.queries.SearchHistoryFTS(ctx, sqlc.SearchHistoryFTSParams{
53+ Url: ftsQuery,
54+ Limit: int64(limit),
50 })55 })
51 if err != nil {56 if err != nil {
52- return nil, err57+ log.Debug().Err(err).Str("query", query).Msg("FTS search failed, no results")
58+ // FTS5 returns error on no match or invalid query, return empty results
59+ return []entity.HistoryMatch{}, nil
53 }60 }
54 61
55 matches := make([]entity.HistoryMatch, len(rows))62 matches := make([]entity.HistoryMatch, len(rows))
56 for i, row := range rows {63 for i, row := range rows {
57 matches[i] = entity.HistoryMatch{64 matches[i] = entity.HistoryMatch{
58 Entry: historyFromRow(row),65 Entry: historyFromRow(row),
59- Score: 1.0, // Basic score, could be enhanced with FTS ranking66+ Score: 1.0, // FTS5 already ranked by bm25
60 }67 }
61 }68 }
62 return matches, nil69 return matches, nil
M internal/infrastructure/persistence/sqlite/queries/history.sql +8 -0
@@ -48,3 +48,11 @@ SELECT date(last_visited) as day, COUNT(*) as entries, SUM(visit_count) as visit
4848
4949 -- name: DeleteHistoryByDomain :exec
5050 DELETE FROM history WHERE url LIKE '%://' || ? || '/%' OR url LIKE '%://' || ? || '?%' OR url LIKE '%://' || ? || '#%' OR url LIKE '%://' || ?;
51+
52+-- name: SearchHistoryFTS :many
53+SELECT h.id, h.url, h.title, h.favicon_url, h.visit_count, h.last_visited, h.created_at
54+FROM history_fts fts
55+JOIN history h ON fts.rowid = h.id
56+WHERE fts.url MATCH ?
57+ORDER BY bm25(history_fts)
58+LIMIT ?;
@@ -48,3 +48,11 @@ SELECT date(last_visited) as day, COUNT(*) as entries, SUM(visit_count) as visit
48 48
49 -- name: DeleteHistoryByDomain :exec49 -- name: DeleteHistoryByDomain :exec
50 DELETE FROM history WHERE url LIKE '%://' || ? || '/%' OR url LIKE '%://' || ? || '?%' OR url LIKE '%://' || ? || '#%' OR url LIKE '%://' || ?;50 DELETE FROM history WHERE url LIKE '%://' || ? || '/%' OR url LIKE '%://' || ? || '?%' OR url LIKE '%://' || ? || '#%' OR url LIKE '%://' || ?;
51+
52+-- name: SearchHistoryFTS :many
53+SELECT h.id, h.url, h.title, h.favicon_url, h.visit_count, h.last_visited, h.created_at
54+FROM history_fts fts
55+JOIN history h ON fts.rowid = h.id
56+WHERE fts.url MATCH ?
57+ORDER BY bm25(history_fts)
58+LIMIT ?;
M internal/infrastructure/persistence/sqlite/sqlc/history.sql.go +45 -0
@@ -293,6 +293,51 @@ func (q *Queries) SearchHistory(ctx context.Context, arg SearchHistoryParams) ([
293293 return items, nil
294294 }
295295
296+const SearchHistoryFTS = `-- name: SearchHistoryFTS :many
297+SELECT h.id, h.url, h.title, h.favicon_url, h.visit_count, h.last_visited, h.created_at
298+FROM history_fts fts
299+JOIN history h ON fts.rowid = h.id
300+WHERE fts.url MATCH ?
301+ORDER BY bm25(history_fts)
302+LIMIT ?
303+`
304+
305+type SearchHistoryFTSParams struct {
306+ Url string `json:"url"`
307+ Limit int64 `json:"limit"`
308+}
309+
310+func (q *Queries) SearchHistoryFTS(ctx context.Context, arg SearchHistoryFTSParams) ([]History, error) {
311+ rows, err := q.db.QueryContext(ctx, SearchHistoryFTS, arg.Url, arg.Limit)
312+ if err != nil {
313+ return nil, err
314+ }
315+ defer rows.Close()
316+ items := []History{}
317+ for rows.Next() {
318+ var i History
319+ if err := rows.Scan(
320+ &i.ID,
321+ &i.Url,
322+ &i.Title,
323+ &i.FaviconUrl,
324+ &i.VisitCount,
325+ &i.LastVisited,
326+ &i.CreatedAt,
327+ ); err != nil {
328+ return nil, err
329+ }
330+ items = append(items, i)
331+ }
332+ if err := rows.Close(); err != nil {
333+ return nil, err
334+ }
335+ if err := rows.Err(); err != nil {
336+ return nil, err
337+ }
338+ return items, nil
339+}
340+
296341 const UpsertHistory = `-- name: UpsertHistory :exec
297342 INSERT INTO history (url, title, favicon_url)
298343 VALUES (?, ?, ?)
@@ -293,6 +293,51 @@ func (q *Queries) SearchHistory(ctx context.Context, arg SearchHistoryParams) ([
293 return items, nil293 return items, nil
294 }294 }
295 295
296+const SearchHistoryFTS = `-- name: SearchHistoryFTS :many
297+SELECT h.id, h.url, h.title, h.favicon_url, h.visit_count, h.last_visited, h.created_at
298+FROM history_fts fts
299+JOIN history h ON fts.rowid = h.id
300+WHERE fts.url MATCH ?
301+ORDER BY bm25(history_fts)
302+LIMIT ?
303+`
304+
305+type SearchHistoryFTSParams struct {
306+ Url string `json:"url"`
307+ Limit int64 `json:"limit"`
308+}
309+
310+func (q *Queries) SearchHistoryFTS(ctx context.Context, arg SearchHistoryFTSParams) ([]History, error) {
311+ rows, err := q.db.QueryContext(ctx, SearchHistoryFTS, arg.Url, arg.Limit)
312+ if err != nil {
313+ return nil, err
314+ }
315+ defer rows.Close()
316+ items := []History{}
317+ for rows.Next() {
318+ var i History
319+ if err := rows.Scan(
320+ &i.ID,
321+ &i.Url,
322+ &i.Title,
323+ &i.FaviconUrl,
324+ &i.VisitCount,
325+ &i.LastVisited,
326+ &i.CreatedAt,
327+ ); err != nil {
328+ return nil, err
329+ }
330+ items = append(items, i)
331+ }
332+ if err := rows.Close(); err != nil {
333+ return nil, err
334+ }
335+ if err := rows.Err(); err != nil {
336+ return nil, err
337+ }
338+ return items, nil
339+}
340+
296 const UpsertHistory = `-- name: UpsertHistory :exec341 const UpsertHistory = `-- name: UpsertHistory :exec
297 INSERT INTO history (url, title, favicon_url)342 INSERT INTO history (url, title, favicon_url)
298 VALUES (?, ?, ?)343 VALUES (?, ?, ?)

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 8

Tags 42

↑↓ moveenter openesc close