detached 61f600c

feat(favicon): add DuckDuckGo favicon API fallback with HTTP client

commit
61f600cbdd12837a85d0b04e545c67ea52c0e381
author
bnema · · 2025-12-16 21:41 UTC
parent
1c2f59f
signature
unsigned
1 file · +97 -96
Unified
UnifiedSplit
M internal/ui/cache/favicon_cache.go +97 -96
@@ -3,9 +3,13 @@ package cache
33
44 import (
55 "context"
6+ "fmt"
7+ "io"
8+ "net/http"
69 "net/url"
710 "strings"
811 "sync"
12+ "time"
913
1014 "github.com/bnema/dumber/internal/logging"
1115 "github.com/bnema/puregotk-webkit/webkit"
@@ -14,13 +18,22 @@ import (
1418 "github.com/jwijenbergh/puregotk/v4/glib"
1519 )
1620
21+const (
22+ // DuckDuckGo favicon API URL template (better transparency support)
23+ duckduckgoFaviconURL = "https://icons.duckduckgo.com/ip3/%s.ico"
24+ // HTTP client timeout for favicon fetch
25+ faviconFetchTimeout = 5 * time.Second
26+)
27+
1728 // FaviconCache provides a two-tier favicon caching system:
1829 // 1. In-memory cache keyed by domain for fast lookups
1930 // 2. WebKit FaviconDatabase as persistent backing store
31+// 3. Google favicon API as fallback
2032 type FaviconCache struct {
2133 mu sync.RWMutex
2234 cache map[string]*gdk.Texture // key: domain (e.g., "github.com")
2335 faviconDB *webkit.FaviconDatabase
36+ client *http.Client
2437 }
2538
2639 // NewFaviconCache creates a new FaviconCache with the given FaviconDatabase.
@@ -29,6 +42,9 @@ func NewFaviconCache(faviconDB *webkit.FaviconDatabase) *FaviconCache {
2942 return &FaviconCache{
3043 cache: make(map[string]*gdk.Texture),
3144 faviconDB: faviconDB,
45+ client: &http.Client{
46+ Timeout: faviconFetchTimeout,
47+ },
3248 }
3349 }
3450
@@ -84,54 +100,9 @@ func (fc *FaviconCache) GetByURL(pageURL string) *gdk.Texture {
84100 return fc.Get(domain)
85101 }
86102
87-// getURLVariants returns URL variants to try for favicon lookup.
88-// WebKit stores favicons under exact URLs (e.g., https://www.youtube.com/)
89-// so we need to try multiple variants: with/without www, with/without trailing slash.
90-func getURLVariants(pageURL string) []string {
91- parsed, err := url.Parse(pageURL)
92- if err != nil {
93- return nil
94- }
95-
96- variants := make([]string, 0, 4)
97-
98- // Generate host variants (with and without www.)
99- hosts := []string{parsed.Host}
100- if strings.HasPrefix(parsed.Host, "www.") {
101- hosts = append(hosts, strings.TrimPrefix(parsed.Host, "www."))
102- } else {
103- hosts = append(hosts, "www."+parsed.Host)
104- }
105-
106- // Generate path variants (with and without trailing slash)
107- paths := []string{parsed.Path}
108- if parsed.Path == "" || parsed.Path == "/" {
109- paths = []string{"", "/"}
110- } else if strings.HasSuffix(parsed.Path, "/") {
111- paths = append(paths, strings.TrimSuffix(parsed.Path, "/"))
112- } else {
113- paths = append(paths, parsed.Path+"/")
114- }
115-
116- // Generate all combinations (skip the original URL)
117- original := pageURL
118- for _, host := range hosts {
119- for _, path := range paths {
120- p := *parsed
121- p.Host = host
122- p.Path = path
123- variant := p.String()
124- if variant != original {
125- variants = append(variants, variant)
126- }
127- }
128- }
129-
130- return variants
131-}
132-
133-// GetOrFetch checks the in-memory cache first, then queries the FaviconDatabase
134-// asynchronously on cache miss. The callback is invoked with the texture (or nil).
103+// GetOrFetch checks the in-memory cache first, then queries the FaviconDatabase,
104+// and falls back to Google's favicon API if not found.
105+// The callback is invoked with the texture (or nil).
135106 // This is safe to call from any goroutine; the callback runs on the GLib main loop.
136107 func (fc *FaviconCache) GetOrFetch(ctx context.Context, pageURL string, callback func(*gdk.Texture)) {
137108 if callback == nil {
@@ -147,12 +118,6 @@ func (fc *FaviconCache) GetOrFetch(ctx context.Context, pageURL string, callback
147118 return
148119 }
149120
150- // No FaviconDatabase, return nil
151- if fc.faviconDB == nil {
152- callback(nil)
153- return
154- }
155-
156121 // Helper to invoke callback on main thread
157122 invokeCallback := func(texture *gdk.Texture) {
158123 cb := glib.SourceFunc(func(uintptr) bool {
@@ -162,70 +127,106 @@ func (fc *FaviconCache) GetOrFetch(ctx context.Context, pageURL string, callback
162127 glib.IdleAdd(&cb, 0)
163128 }
164129
165- // Get all URL variants to try (www/non-www, with/without trailing slash)
166- variants := getURLVariants(pageURL)
167-
168- // Recursive function to try variants one by one
169- var tryVariant func(index int)
170- tryVariant = func(index int) {
171- if index >= len(variants) {
172- // All variants exhausted
173- invokeCallback(nil)
174- return
175- }
176-
177- variantURL := variants[index]
178- variantCb := gio.AsyncReadyCallback(func(_ uintptr, resultPtr uintptr, _ uintptr) {
179- if resultPtr == 0 {
180- tryVariant(index + 1)
181- return
182- }
183-
184- result := &gio.AsyncResultBase{Ptr: resultPtr}
185- texture, err := fc.faviconDB.GetFaviconFinish(result)
186- if err != nil {
187- log.Debug().Err(err).Str("url", variantURL).Msg("favicon variant fetch failed")
188- tryVariant(index + 1)
189- return
190- }
191-
192- if texture != nil && domain != "" {
130+ // Helper to fetch from external API and invoke callback
131+ fetchExternal := func() {
132+ go func() {
133+ texture := fc.fetchFromExternal(ctx, domain)
134+ if texture != nil {
193135 fc.Set(domain, texture)
194136 }
195137 invokeCallback(texture)
196- })
138+ }()
139+ }
197140
198- fc.faviconDB.GetFavicon(variantURL, nil, &variantCb, 0)
141+ // No FaviconDatabase, fetch from external API directly
142+ if fc.faviconDB == nil {
143+ fetchExternal()
144+ return
199145 }
200146
201- // Query FaviconDatabase with original URL first
147+ // Try FaviconDatabase first (just once, no variant attempts)
202148 asyncCb := gio.AsyncReadyCallback(func(_ uintptr, resultPtr uintptr, _ uintptr) {
203149 if resultPtr == 0 {
204- log.Debug().Str("url", pageURL).Msg("favicon fetch: nil result, trying variants")
205- tryVariant(0)
150+ log.Debug().Str("domain", domain).Msg("favicon not in database, fetching from DuckDuckGo")
151+ fetchExternal()
206152 return
207153 }
208154
209155 result := &gio.AsyncResultBase{Ptr: resultPtr}
210156 texture, err := fc.faviconDB.GetFaviconFinish(result)
211- if err != nil {
212- log.Debug().Err(err).Str("url", pageURL).Msg("favicon fetch failed, trying variants")
213- tryVariant(0)
157+ if err != nil || texture == nil {
158+ log.Debug().Str("domain", domain).Msg("favicon not in database, fetching from DuckDuckGo")
159+ fetchExternal()
214160 return
215161 }
216162
217- // Cache the result
218- if texture != nil && domain != "" {
219- fc.Set(domain, texture)
220- }
221-
163+ // Found in database, cache and return
164+ fc.Set(domain, texture)
222165 invokeCallback(texture)
223166 })
224167
225- // Start async fetch (nil cancellable, 0 userData)
168+ // Try with the original URL
226169 fc.faviconDB.GetFavicon(pageURL, nil, &asyncCb, 0)
227170 }
228171
172+// fetchFromExternal fetches a favicon from DuckDuckGo's favicon API.
173+// Returns nil if fetch fails.
174+func (fc *FaviconCache) fetchFromExternal(ctx context.Context, domain string) *gdk.Texture {
175+ log := logging.FromContext(ctx)
176+
177+ if domain == "" {
178+ return nil
179+ }
180+
181+ faviconURL := fmt.Sprintf(duckduckgoFaviconURL, url.QueryEscape(domain))
182+ log.Debug().Str("url", faviconURL).Msg("fetching favicon from DuckDuckGo")
183+
184+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, faviconURL, nil)
185+ if err != nil {
186+ log.Debug().Err(err).Msg("failed to create favicon request")
187+ return nil
188+ }
189+
190+ resp, err := fc.client.Do(req)
191+ if err != nil {
192+ log.Debug().Err(err).Str("domain", domain).Msg("failed to fetch favicon from DuckDuckGo")
193+ return nil
194+ }
195+ defer resp.Body.Close()
196+
197+ if resp.StatusCode != http.StatusOK {
198+ log.Debug().Int("status", resp.StatusCode).Str("domain", domain).Msg("DuckDuckGo favicon API returned non-OK status")
199+ return nil
200+ }
201+
202+ data, err := io.ReadAll(resp.Body)
203+ if err != nil {
204+ log.Debug().Err(err).Str("domain", domain).Msg("failed to read favicon response")
205+ return nil
206+ }
207+
208+ if len(data) == 0 {
209+ log.Debug().Str("domain", domain).Msg("empty favicon response from DuckDuckGo")
210+ return nil
211+ }
212+
213+ // Convert bytes to GdkTexture
214+ bytes := glib.NewBytes(data, uint(len(data)))
215+ if bytes == nil {
216+ log.Debug().Str("domain", domain).Msg("failed to create GBytes from favicon data")
217+ return nil
218+ }
219+
220+ texture, err := gdk.NewTextureFromBytes(bytes)
221+ if err != nil {
222+ log.Debug().Err(err).Str("domain", domain).Msg("failed to create texture from favicon bytes")
223+ return nil
224+ }
225+
226+ log.Debug().Str("domain", domain).Msg("favicon fetched from DuckDuckGo and cached")
227+ return texture
228+}
229+
229230 // Clear removes all entries from the in-memory cache.
230231 func (fc *FaviconCache) Clear() {
231232 fc.mu.Lock()
@@ -3,9 +3,13 @@ package cache
3 3
4 import (4 import (
5 "context"5 "context"
6+ "fmt"
7+ "io"
8+ "net/http"
6 "net/url"9 "net/url"
7 "strings"10 "strings"
8 "sync"11 "sync"
12+ "time"
9 13
10 "github.com/bnema/dumber/internal/logging"14 "github.com/bnema/dumber/internal/logging"
11 "github.com/bnema/puregotk-webkit/webkit"15 "github.com/bnema/puregotk-webkit/webkit"
@@ -14,13 +18,22 @@ import (
14 "github.com/jwijenbergh/puregotk/v4/glib"18 "github.com/jwijenbergh/puregotk/v4/glib"
15 )19 )
16 20
21+const (
22+ // DuckDuckGo favicon API URL template (better transparency support)
23+ duckduckgoFaviconURL = "https://icons.duckduckgo.com/ip3/%s.ico"
24+ // HTTP client timeout for favicon fetch
25+ faviconFetchTimeout = 5 * time.Second
26+)
27+
17 // FaviconCache provides a two-tier favicon caching system:28 // FaviconCache provides a two-tier favicon caching system:
18 // 1. In-memory cache keyed by domain for fast lookups29 // 1. In-memory cache keyed by domain for fast lookups
19 // 2. WebKit FaviconDatabase as persistent backing store30 // 2. WebKit FaviconDatabase as persistent backing store
31+// 3. Google favicon API as fallback
20 type FaviconCache struct {32 type FaviconCache struct {
21 mu sync.RWMutex33 mu sync.RWMutex
22 cache map[string]*gdk.Texture // key: domain (e.g., "github.com")34 cache map[string]*gdk.Texture // key: domain (e.g., "github.com")
23 faviconDB *webkit.FaviconDatabase35 faviconDB *webkit.FaviconDatabase
36+ client *http.Client
24 }37 }
25 38
26 // NewFaviconCache creates a new FaviconCache with the given FaviconDatabase.39 // NewFaviconCache creates a new FaviconCache with the given FaviconDatabase.
@@ -29,6 +42,9 @@ func NewFaviconCache(faviconDB *webkit.FaviconDatabase) *FaviconCache {
29 return &FaviconCache{42 return &FaviconCache{
30 cache: make(map[string]*gdk.Texture),43 cache: make(map[string]*gdk.Texture),
31 faviconDB: faviconDB,44 faviconDB: faviconDB,
45+ client: &http.Client{
46+ Timeout: faviconFetchTimeout,
47+ },
32 }48 }
33 }49 }
34 50
@@ -84,54 +100,9 @@ func (fc *FaviconCache) GetByURL(pageURL string) *gdk.Texture {
84 return fc.Get(domain)100 return fc.Get(domain)
85 }101 }
86 102
87-// getURLVariants returns URL variants to try for favicon lookup.103+// GetOrFetch checks the in-memory cache first, then queries the FaviconDatabase,
88-// WebKit stores favicons under exact URLs (e.g., https://www.youtube.com/)104+// and falls back to Google's favicon API if not found.
89-// so we need to try multiple variants: with/without www, with/without trailing slash.105+// The callback is invoked with the texture (or nil).
90-func getURLVariants(pageURL string) []string {
91- parsed, err := url.Parse(pageURL)
92- if err != nil {
93- return nil
94- }
95-
96- variants := make([]string, 0, 4)
97-
98- // Generate host variants (with and without www.)
99- hosts := []string{parsed.Host}
100- if strings.HasPrefix(parsed.Host, "www.") {
101- hosts = append(hosts, strings.TrimPrefix(parsed.Host, "www."))
102- } else {
103- hosts = append(hosts, "www."+parsed.Host)
104- }
105-
106- // Generate path variants (with and without trailing slash)
107- paths := []string{parsed.Path}
108- if parsed.Path == "" || parsed.Path == "/" {
109- paths = []string{"", "/"}
110- } else if strings.HasSuffix(parsed.Path, "/") {
111- paths = append(paths, strings.TrimSuffix(parsed.Path, "/"))
112- } else {
113- paths = append(paths, parsed.Path+"/")
114- }
115-
116- // Generate all combinations (skip the original URL)
117- original := pageURL
118- for _, host := range hosts {
119- for _, path := range paths {
120- p := *parsed
121- p.Host = host
122- p.Path = path
123- variant := p.String()
124- if variant != original {
125- variants = append(variants, variant)
126- }
127- }
128- }
129-
130- return variants
131-}
132-
133-// GetOrFetch checks the in-memory cache first, then queries the FaviconDatabase
134-// asynchronously on cache miss. The callback is invoked with the texture (or nil).
135 // This is safe to call from any goroutine; the callback runs on the GLib main loop.106 // This is safe to call from any goroutine; the callback runs on the GLib main loop.
136 func (fc *FaviconCache) GetOrFetch(ctx context.Context, pageURL string, callback func(*gdk.Texture)) {107 func (fc *FaviconCache) GetOrFetch(ctx context.Context, pageURL string, callback func(*gdk.Texture)) {
137 if callback == nil {108 if callback == nil {
@@ -147,12 +118,6 @@ func (fc *FaviconCache) GetOrFetch(ctx context.Context, pageURL string, callback
147 return118 return
148 }119 }
149 120
150- // No FaviconDatabase, return nil
151- if fc.faviconDB == nil {
152- callback(nil)
153- return
154- }
155-
156 // Helper to invoke callback on main thread121 // Helper to invoke callback on main thread
157 invokeCallback := func(texture *gdk.Texture) {122 invokeCallback := func(texture *gdk.Texture) {
158 cb := glib.SourceFunc(func(uintptr) bool {123 cb := glib.SourceFunc(func(uintptr) bool {
@@ -162,70 +127,106 @@ func (fc *FaviconCache) GetOrFetch(ctx context.Context, pageURL string, callback
162 glib.IdleAdd(&cb, 0)127 glib.IdleAdd(&cb, 0)
163 }128 }
164 129
165- // Get all URL variants to try (www/non-www, with/without trailing slash)130+ // Helper to fetch from external API and invoke callback
166- variants := getURLVariants(pageURL)131+ fetchExternal := func() {
167-132+ go func() {
168- // Recursive function to try variants one by one133+ texture := fc.fetchFromExternal(ctx, domain)
169- var tryVariant func(index int)134+ if texture != nil {
170- tryVariant = func(index int) {
171- if index >= len(variants) {
172- // All variants exhausted
173- invokeCallback(nil)
174- return
175- }
176-
177- variantURL := variants[index]
178- variantCb := gio.AsyncReadyCallback(func(_ uintptr, resultPtr uintptr, _ uintptr) {
179- if resultPtr == 0 {
180- tryVariant(index + 1)
181- return
182- }
183-
184- result := &gio.AsyncResultBase{Ptr: resultPtr}
185- texture, err := fc.faviconDB.GetFaviconFinish(result)
186- if err != nil {
187- log.Debug().Err(err).Str("url", variantURL).Msg("favicon variant fetch failed")
188- tryVariant(index + 1)
189- return
190- }
191-
192- if texture != nil && domain != "" {
193 fc.Set(domain, texture)135 fc.Set(domain, texture)
194 }136 }
195 invokeCallback(texture)137 invokeCallback(texture)
196- })138+ }()
139+ }
197 140
198- fc.faviconDB.GetFavicon(variantURL, nil, &variantCb, 0)141+ // No FaviconDatabase, fetch from external API directly
142+ if fc.faviconDB == nil {
143+ fetchExternal()
144+ return
199 }145 }
200 146
201- // Query FaviconDatabase with original URL first147+ // Try FaviconDatabase first (just once, no variant attempts)
202 asyncCb := gio.AsyncReadyCallback(func(_ uintptr, resultPtr uintptr, _ uintptr) {148 asyncCb := gio.AsyncReadyCallback(func(_ uintptr, resultPtr uintptr, _ uintptr) {
203 if resultPtr == 0 {149 if resultPtr == 0 {
204- log.Debug().Str("url", pageURL).Msg("favicon fetch: nil result, trying variants")150+ log.Debug().Str("domain", domain).Msg("favicon not in database, fetching from DuckDuckGo")
205- tryVariant(0)151+ fetchExternal()
206 return152 return
207 }153 }
208 154
209 result := &gio.AsyncResultBase{Ptr: resultPtr}155 result := &gio.AsyncResultBase{Ptr: resultPtr}
210 texture, err := fc.faviconDB.GetFaviconFinish(result)156 texture, err := fc.faviconDB.GetFaviconFinish(result)
211- if err != nil {157+ if err != nil || texture == nil {
212- log.Debug().Err(err).Str("url", pageURL).Msg("favicon fetch failed, trying variants")158+ log.Debug().Str("domain", domain).Msg("favicon not in database, fetching from DuckDuckGo")
213- tryVariant(0)159+ fetchExternal()
214 return160 return
215 }161 }
216 162
217- // Cache the result163+ // Found in database, cache and return
218- if texture != nil && domain != "" {164+ fc.Set(domain, texture)
219- fc.Set(domain, texture)
220- }
221-
222 invokeCallback(texture)165 invokeCallback(texture)
223 })166 })
224 167
225- // Start async fetch (nil cancellable, 0 userData)168+ // Try with the original URL
226 fc.faviconDB.GetFavicon(pageURL, nil, &asyncCb, 0)169 fc.faviconDB.GetFavicon(pageURL, nil, &asyncCb, 0)
227 }170 }
228 171
172+// fetchFromExternal fetches a favicon from DuckDuckGo's favicon API.
173+// Returns nil if fetch fails.
174+func (fc *FaviconCache) fetchFromExternal(ctx context.Context, domain string) *gdk.Texture {
175+ log := logging.FromContext(ctx)
176+
177+ if domain == "" {
178+ return nil
179+ }
180+
181+ faviconURL := fmt.Sprintf(duckduckgoFaviconURL, url.QueryEscape(domain))
182+ log.Debug().Str("url", faviconURL).Msg("fetching favicon from DuckDuckGo")
183+
184+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, faviconURL, nil)
185+ if err != nil {
186+ log.Debug().Err(err).Msg("failed to create favicon request")
187+ return nil
188+ }
189+
190+ resp, err := fc.client.Do(req)
191+ if err != nil {
192+ log.Debug().Err(err).Str("domain", domain).Msg("failed to fetch favicon from DuckDuckGo")
193+ return nil
194+ }
195+ defer resp.Body.Close()
196+
197+ if resp.StatusCode != http.StatusOK {
198+ log.Debug().Int("status", resp.StatusCode).Str("domain", domain).Msg("DuckDuckGo favicon API returned non-OK status")
199+ return nil
200+ }
201+
202+ data, err := io.ReadAll(resp.Body)
203+ if err != nil {
204+ log.Debug().Err(err).Str("domain", domain).Msg("failed to read favicon response")
205+ return nil
206+ }
207+
208+ if len(data) == 0 {
209+ log.Debug().Str("domain", domain).Msg("empty favicon response from DuckDuckGo")
210+ return nil
211+ }
212+
213+ // Convert bytes to GdkTexture
214+ bytes := glib.NewBytes(data, uint(len(data)))
215+ if bytes == nil {
216+ log.Debug().Str("domain", domain).Msg("failed to create GBytes from favicon data")
217+ return nil
218+ }
219+
220+ texture, err := gdk.NewTextureFromBytes(bytes)
221+ if err != nil {
222+ log.Debug().Err(err).Str("domain", domain).Msg("failed to create texture from favicon bytes")
223+ return nil
224+ }
225+
226+ log.Debug().Str("domain", domain).Msg("favicon fetched from DuckDuckGo and cached")
227+ return texture
228+}
229+
229 // Clear removes all entries from the in-memory cache.230 // Clear removes all entries from the in-memory cache.
230 func (fc *FaviconCache) Clear() {231 func (fc *FaviconCache) Clear() {
231 fc.mu.Lock()232 fc.mu.Lock()

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 8

Tags 42

↑↓ moveenter openesc close