feat(hints): label underlined words in OCR windows

commit: 38b30bbf04b3a3f3cf3d9db05751b04e0c603c70
author: brice · 3d ago · 2026-10-19 15:45 UTC
parent: cf52ba6
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

5 files · +450 -5

M docs/config.md +6 -4


@@ -324,12 +324,14 @@ a link; accessibility targets get a plain click.
324	324		\| Source \| Apps \| Accuracy \| Speed \|
325	325		\|---\|---\|---\|---\|
326	326		\| Accessibility tree (AT-SPI) \| Firefox, Chromium, GTK, Qt, Electron \| Exact: the app reports its own links and buttons \| A few milliseconds \|
327		-	\| Text recognition (OCR) \| Everything else: terminals, apps without accessibility \| Best effort: only URLs and domains written on screen \| 150 ms or more, more on large screens \|
	327	+	\| Text recognition (OCR) \| Everything else: terminals, apps without accessibility \| Best effort: URLs, domains and underlined text written on screen \| 150 ms or more, more on large screens \|
328	328
329	329		NeferWL asks the accessibility tree first. Windows it does not cover are read
330		-	with OCR, which finds visible URLs, domains and email addresses. OCR cannot
331		-	see a link whose address is hidden behind its text, and it can misread small
332		-	or low-contrast text.
	330	+	with OCR, which finds visible URLs, domains, email addresses and underlined
	331	+	text. Underlined words get a plain click, and the words of one continuous
	332	+	underline form one target; URLs keep the `hint.text-click` modifiers. OCR
	333	+	cannot see a link whose address is hidden behind plain text, and it can misread
	334	+	small or low-contrast text.
333	335
334	336		### Requirements
335	337

 | Source | Apps | Accuracy | Speed |
 |---|---|---|---|
 | Accessibility tree (AT-SPI) | Firefox, Chromium, GTK, Qt, Electron | Exact: the app reports its own links and buttons | A few milliseconds |
-| Text recognition (OCR) | Everything else: terminals, apps without accessibility | Best effort: only URLs and domains written on screen | 150 ms or more, more on large screens |
+| Text recognition (OCR) | Everything else: terminals, apps without accessibility | Best effort: URLs, domains and underlined text written on screen | 150 ms or more, more on large screens |
 NeferWL asks the accessibility tree first. Windows it does not cover are read
-with OCR, which finds visible URLs, domains and email addresses. OCR cannot
+with OCR, which finds visible URLs, domains, email addresses and underlined
-see a link whose address is hidden behind its text, and it can misread small
+text. Underlined words get a plain click, and the words of one continuous
-or low-contrast text.
+underline form one target; URLs keep the `hint.text-click` modifiers. OCR
+cannot see a link whose address is hidden behind plain text, and it can misread
+small or low-contrast text.
 ### Requirements

M internal/adapters/hints/hints.go +3 -0


@@ -378,6 +378,9 @@ func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame,
378	378		nextCache[j.key] = bandCache{hash: j.hash, ext: j.ext, words: j.words}
379	379		for _, word := range j.words {
380	380		word.Rect = word.Rect.Add(r.Min)
	381	+	if word.Underlined {
	382	+	word.UnderL, word.UnderR = word.UnderL+r.Min.X, word.UnderR+r.Min.X
	383	+	}
381	384		if !duplicateWord(words, word) {
382	385		words = append(words, word)
383	386		}


@@ -378,6 +378,9 @@ func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame,
378	nextCache[j.key] = bandCache{hash: j.hash, ext: j.ext, words: j.words}	378		nextCache[j.key] = bandCache{hash: j.hash, ext: j.ext, words: j.words}
379	for _, word := range j.words {	379		for _, word := range j.words {
380	word.Rect = word.Rect.Add(r.Min)	380		word.Rect = word.Rect.Add(r.Min)
		381	+	if word.Underlined {
		382	+	word.UnderL, word.UnderR = word.UnderL+r.Min.X, word.UnderR+r.Min.X
		383	+	}
381	if !duplicateWord(words, word) {	384		if !duplicateWord(words, word) {
382	words = append(words, word)	385		words = append(words, word)
383	}	386		}

M internal/adapters/hints/ocr.go +10 -1


@@ -30,6 +30,12 @@ type OCRWord struct {
30	30		Text string `json:"text"`
31	31		Rect image.Rectangle `json:"rect"`
32	32		Confidence float64 `json:"confidence"`
	33	+	// Underlined is set when a thin line runs under the word. UnderL and
	34	+	// UnderR are the x extent of that line (crop-local), which can reach the
	35	+	// neighbouring words of one underlined link.
	36	+	Underlined bool `json:"underlined,omitempty"`
	37	+	UnderL int `json:"under_l,omitempty"`
	38	+	UnderR int `json:"under_r,omitempty"`
33	39		}
34	40
35	41		var languageWarning sync.Once

@@ -199,6 +205,7 @@ func readBand(ctx context.Context, st ocrState, j bandJob) ([]OCRWord, error)
199	205		words[i].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
200	206		}
201	207		}
	208	+	markUnderlined(j.gray, words)
202	209		owned := words[:0]
203	210		for _, word := range words {
204	211		// Overlap supplies context, but each word belongs to exactly one band.

@@ -285,6 +292,7 @@ func textLinks(words []OCRWord, scale float64) []ports.HintTarget {
285	292		return words[i].Rect.Min.X < words[j].Rect.Min.X
286	293		})
287	294		var out []ports.HintTarget
	295	+	var urls []image.Rectangle // boxes of the URL targets, in pixels
288	296		used := make([]bool, len(words))
289	297		for i, word := range words {
290	298		if used[i] {

@@ -322,8 +330,9 @@ func textLinks(words []OCRWord, scale float64) []ports.HintTarget {
322	330		x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
323	331		right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
324	332		out = append(out, ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
	333	+	urls = append(urls, rect)
325	334		}
326		-	return out
	335	+	return append(out, underlinedLinks(words, urls, scale)...)
327	336		}
328	337
329	338		func abs(x int) int {

 	Text       string          `json:"text"`
 	Rect       image.Rectangle `json:"rect"`
 	Confidence float64         `json:"confidence"`
+	// Underlined is set when a thin line runs under the word. UnderL and
+	// UnderR are the x extent of that line (crop-local), which can reach the
+	// neighbouring words of one underlined link.
+	Underlined bool `json:"underlined,omitempty"`
+	UnderL     int  `json:"under_l,omitempty"`
+	UnderR     int  `json:"under_r,omitempty"`
 }
 var languageWarning sync.Once
 			words[i].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
 		}
 	}
+	markUnderlined(j.gray, words)
 	owned := words[:0]
 	for _, word := range words {
 		// Overlap supplies context, but each word belongs to exactly one band.
 		return words[i].Rect.Min.X < words[j].Rect.Min.X
 	})
 	var out []ports.HintTarget
+	var urls []image.Rectangle // boxes of the URL targets, in pixels
 	used := make([]bool, len(words))
 	for i, word := range words {
 		if used[i] {
 		x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
 		right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
 		out = append(out, ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
+		urls = append(urls, rect)
 	}
-	return out
+	return append(out, underlinedLinks(words, urls, scale)...)
 }
 func abs(x int) int {

A internal/adapters/hints/underline.go +246 -0

@@ -0,0 +1,246 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"image"
	5	+	"math"
	6	+	"sort"
	7	+
	8	+	"github.com/bnema/neferwl/internal/ports"
	9	+	)
	10	+
	11	+	// underlineCandidate is a word with a thin line under it.
	12	+	type underlineCandidate struct {
	13	+	word int
	14	+	row int // the line's row
	15	+	from, to int // x extent of the continuous line through the word
	16	+	}
	17	+
	18	+	// markUnderlined sets Underlined, UnderL and UnderR on the words of a band
	19	+	// that have a thin line under them. words are in the coordinates of g.
	20	+	// Lines that run on far beyond the word (rules, table borders, input boxes)
	21	+	// do not count, unless the extension is made of underlined words too.
	22	+	func markUnderlined(src *image.Gray, words []OCRWord) {
	23	+	// Words are relative to the band's own origin, whatever the image is a
	24	+	// view of.
	25	+	b := src.Bounds()
	26	+	g := &image.Gray{Pix: src.Pix[src.PixOffset(b.Min.X, b.Min.Y):], Stride: src.Stride, Rect: image.Rect(0, 0, b.Dx(), b.Dy())}
	27	+	var cands []underlineCandidate
	28	+	for i, w := range words {
	29	+	// A single glyph, or a box narrower than tall, has a flat bottom
	30	+	// stroke (8, L, 2) that is not an underline.
	31	+	if len([]rune(w.Text)) < 2 \|\| w.Rect.Dx() < w.Rect.Dy() {
	32	+	continue
	33	+	}
	34	+	if c, ok := underlineOf(g, w.Rect); ok {
	35	+	c.word = i
	36	+	cands = append(cands, c)
	37	+	}
	38	+	}
	39	+	for _, c := range cands {
	40	+	w := words[c.word]
	41	+	h := w.Rect.Dy()
	42	+	if (w.Rect.Min.X-c.from > h && !covered(words, cands, c, -1)) \|\|
	43	+	(c.to-w.Rect.Max.X > h && !covered(words, cands, c, 1)) {
	44	+	continue
	45	+	}
	46	+	words[c.word].Underlined, words[c.word].UnderL, words[c.word].UnderR = true, c.from, c.to
	47	+	}
	48	+	}
	49	+
	50	+	// covered reports whether the line of c, on the given side of its word, is
	51	+	// explained by underlined words on the same line up to within a glyph of its
	52	+	// end.
	53	+	func covered(words []OCRWord, cands []underlineCandidate, c underlineCandidate, dir int) bool {
	54	+	w := words[c.word].Rect
	55	+	h := w.Dy()
	56	+	cur := w.Min.X
	57	+	if dir > 0 {
	58	+	cur = w.Max.X
	59	+	}
	60	+	for {
	61	+	if (dir < 0 && cur-c.from <= h) \|\| (dir > 0 && c.to-cur <= h) {
	62	+	return true
	63	+	}
	64	+	next := -1
	65	+	for _, o := range cands {
	66	+	r := words[o.word].Rect
	67	+	if abs(r.Max.Y-w.Max.Y) > max(3, h/3) {
	68	+	continue
	69	+	}
	70	+	var gap, reach int
	71	+	if dir < 0 {
	72	+	gap, reach = cur-r.Max.X, r.Min.X
	73	+	} else {
	74	+	gap, reach = r.Min.X-cur, r.Max.X
	75	+	}
	76	+	if gap < -2 \|\| gap > h {
	77	+	continue
	78	+	}
	79	+	if next < 0 \|\| (dir < 0 && reach < next) \|\| (dir > 0 && reach > next) {
	80	+	next = reach
	81	+	}
	82	+	}
	83	+	if next < 0 \|\| (dir < 0 && next >= cur) \|\| (dir > 0 && next <= cur) {
	84	+	return false
	85	+	}
	86	+	cur = next
	87	+	}
	88	+	}
	89	+
	90	+	// underlineOf looks for an underline under the word box r in g.
	91	+	func underlineOf(g *image.Gray, r image.Rectangle) (underlineCandidate, bool) {
	92	+	b := g.Bounds()
	93	+	box := r.Intersect(b)
	94	+	h := r.Dy()
	95	+	if box.Dx() < 4 \|\| h < 4 {
	96	+	return underlineCandidate{}, false
	97	+	}
	98	+	bg, ink := contrastOf(g, box)
	99	+	thr := min(40, max(20, ink/2))
	100	+	if ink < thr+10 {
	101	+	return underlineCandidate{}, false
	102	+	}
	103	+	top := max(b.Min.Y, r.Max.Y-(h*15+99)/100)
	104	+	bottom := min(b.Max.Y, r.Max.Y+(h*35+99)/100+1)
	105	+	maxThick := max(2, h/6)
	106	+	rowOK := func(y int) bool {
	107	+	row := g.Pix[(y-g.Rect.Min.Y)*g.Stride:]
	108	+	n := 0
	109	+	for x := box.Min.X; x < box.Max.X; x++ {
	110	+	if abs(int(row[x-g.Rect.Min.X])-bg) >= thr {
	111	+	n++
	112	+	}
	113	+	}
	114	+	return n100 >= box.Dx()85
	115	+	}
	116	+	thick, first := 0, 0
	117	+	for y := top; y <= bottom; y++ {
	118	+	if y < bottom && rowOK(y) {
	119	+	if thick == 0 {
	120	+	first = y
	121	+	}
	122	+	thick++
	123	+	continue
	124	+	}
	125	+	if thick > 0 {
	126	+	if thick > maxThick {
	127	+	return underlineCandidate{}, false // a block, not a line
	128	+	}
	129	+	row := first + thick/2
	130	+	from, to := box.Min.X, box.Max.X
	131	+	line := g.Pix[(row-g.Rect.Min.Y)*g.Stride:]
	132	+	on := func(x int) bool { return abs(int(line[x-g.Rect.Min.X])-bg) >= thr }
	133	+	for from > b.Min.X && on(from-1) {
	134	+	from--
	135	+	}
	136	+	for to < b.Max.X && on(to) {
	137	+	to++
	138	+	}
	139	+	return underlineCandidate{row: row, from: from, to: to}, true
	140	+	}
	141	+	}
	142	+	return underlineCandidate{}, false
	143	+	}
	144	+
	145	+	// contrastOf returns the background level of box (the most common level,
	146	+	// found in bins of 16 so antialiasing does not split it, then refined to the
	147	+	// mean of that bin) and the contrast of its ink (the deviation reached by
	148	+	// the strongest 3% of pixels).
	149	+	func contrastOf(g *image.Gray, box image.Rectangle) (bg, ink int) {
	150	+	var hist [256]int
	151	+	for y := box.Min.Y; y < box.Max.Y; y++ {
	152	+	for _, v := range g.Pix[(y-g.Rect.Min.Y)*g.Stride+box.Min.X-g.Rect.Min.X:][:box.Dx()] {
	153	+	hist[v]++
	154	+	}
	155	+	}
	156	+	total := box.Dx() * box.Dy()
	157	+	var bins [16]int
	158	+	for v, c := range hist {
	159	+	bins[v>>4] += c
	160	+	}
	161	+	best := 0
	162	+	for i, c := range bins {
	163	+	if c > bins[best] {
	164	+	best = i
	165	+	}
	166	+	}
	167	+	sum, n := 0, 0
	168	+	for v := best << 4; v < (best+1)<<4; v++ {
	169	+	sum += v * hist[v]
	170	+	n += hist[v]
	171	+	}
	172	+	bg = sum / max(n, 1)
	173	+	var dev [256]int
	174	+	for v, c := range hist {
	175	+	dev[abs(v-bg)] += c
	176	+	}
	177	+	n = 0
	178	+	for d := 255; d >= 0; d-- {
	179	+	n += dev[d]
	180	+	if n100 >= total3 {
	181	+	return bg, d
	182	+	}
	183	+	}
	184	+	return bg, 0
	185	+	}
	186	+
	187	+	// underlinedLinks returns the targets for underlined words, joining the
	188	+	// words of one continuous underline. A group that overlaps a URL target (in
	189	+	// pixels) is left to it. Coordinates are scaled to logical.
	190	+	func underlinedLinks(words []OCRWord, urls []image.Rectangle, scale float64) []ports.HintTarget {
	191	+	var us []OCRWord
	192	+	for _, w := range words {
	193	+	if w.Underlined {
	194	+	us = append(us, w)
	195	+	}
	196	+	}
	197	+	sort.Slice(us, func(i, j int) bool { return us[i].Rect.Min.X < us[j].Rect.Min.X })
	198	+	used := make([]bool, len(us))
	199	+	var out []ports.HintTarget
	200	+	for i := range us {
	201	+	if used[i] {
	202	+	continue
	203	+	}
	204	+	used[i] = true
	205	+	rect, last := us[i].Rect, us[i]
	206	+	chars, width := len(last.Text), last.Rect.Dx()
	207	+	for {
	208	+	next := -1
	209	+	for j := range us {
	210	+	c := us[j]
	211	+	if used[j] \|\| c.Rect.Min.X < rect.Max.X-2 \|\| abs(c.Rect.Max.Y-last.Rect.Max.Y) > max(3, rect.Dy()/3) {
	212	+	continue
	213	+	}
	214	+	avg := float64(width+c.Rect.Dx()) / float64(max(1, chars+len(c.Text)))
	215	+	gap := c.Rect.Min.X - rect.Max.X
	216	+	if float64(gap) > 1.2*avg \|\| c.Rect.Min.X > last.UnderR+2 {
	217	+	continue
	218	+	}
	219	+	if next < 0 \|\| c.Rect.Min.X < us[next].Rect.Min.X {
	220	+	next = j
	221	+	}
	222	+	}
	223	+	if next < 0 {
	224	+	break
	225	+	}
	226	+	used[next] = true
	227	+	last = us[next]
	228	+	rect = rect.Union(last.Rect)
	229	+	chars += len(last.Text)
	230	+	width += last.Rect.Dx()
	231	+	}
	232	+	dup := false
	233	+	for _, u := range urls {
	234	+	if overlap(rect, u) > .5 {
	235	+	dup = true
	236	+	}
	237	+	}
	238	+	if dup {
	239	+	continue
	240	+	}
	241	+	x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
	242	+	right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
	243	+	out = append(out, ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
	244	+	}
	245	+	return out
	246	+	}

@@ -0,0 +1,246 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"image"
	5	+	"math"
	6	+	"sort"
	7	+
	8	+	"github.com/bnema/neferwl/internal/ports"
	9	+	)
	10	+
	11	+	// underlineCandidate is a word with a thin line under it.
	12	+	type underlineCandidate struct {
	13	+	word int
	14	+	row int // the line's row
	15	+	from, to int // x extent of the continuous line through the word
	16	+	}
	17	+
	18	+	// markUnderlined sets Underlined, UnderL and UnderR on the words of a band
	19	+	// that have a thin line under them. words are in the coordinates of g.
	20	+	// Lines that run on far beyond the word (rules, table borders, input boxes)
	21	+	// do not count, unless the extension is made of underlined words too.
	22	+	func markUnderlined(src *image.Gray, words []OCRWord) {
	23	+	// Words are relative to the band's own origin, whatever the image is a
	24	+	// view of.
	25	+	b := src.Bounds()
	26	+	g := &image.Gray{Pix: src.Pix[src.PixOffset(b.Min.X, b.Min.Y):], Stride: src.Stride, Rect: image.Rect(0, 0, b.Dx(), b.Dy())}
	27	+	var cands []underlineCandidate
	28	+	for i, w := range words {
	29	+	// A single glyph, or a box narrower than tall, has a flat bottom
	30	+	// stroke (8, L, 2) that is not an underline.
	31	+	if len([]rune(w.Text)) < 2 \|\| w.Rect.Dx() < w.Rect.Dy() {
	32	+	continue
	33	+	}
	34	+	if c, ok := underlineOf(g, w.Rect); ok {
	35	+	c.word = i
	36	+	cands = append(cands, c)
	37	+	}
	38	+	}
	39	+	for _, c := range cands {
	40	+	w := words[c.word]
	41	+	h := w.Rect.Dy()
	42	+	if (w.Rect.Min.X-c.from > h && !covered(words, cands, c, -1)) \|\|
	43	+	(c.to-w.Rect.Max.X > h && !covered(words, cands, c, 1)) {
	44	+	continue
	45	+	}
	46	+	words[c.word].Underlined, words[c.word].UnderL, words[c.word].UnderR = true, c.from, c.to
	47	+	}
	48	+	}
	49	+
	50	+	// covered reports whether the line of c, on the given side of its word, is
	51	+	// explained by underlined words on the same line up to within a glyph of its
	52	+	// end.
	53	+	func covered(words []OCRWord, cands []underlineCandidate, c underlineCandidate, dir int) bool {
	54	+	w := words[c.word].Rect
	55	+	h := w.Dy()
	56	+	cur := w.Min.X
	57	+	if dir > 0 {
	58	+	cur = w.Max.X
	59	+	}
	60	+	for {
	61	+	if (dir < 0 && cur-c.from <= h) \|\| (dir > 0 && c.to-cur <= h) {
	62	+	return true
	63	+	}
	64	+	next := -1
	65	+	for _, o := range cands {
	66	+	r := words[o.word].Rect
	67	+	if abs(r.Max.Y-w.Max.Y) > max(3, h/3) {
	68	+	continue
	69	+	}
	70	+	var gap, reach int
	71	+	if dir < 0 {
	72	+	gap, reach = cur-r.Max.X, r.Min.X
	73	+	} else {
	74	+	gap, reach = r.Min.X-cur, r.Max.X
	75	+	}
	76	+	if gap < -2 \|\| gap > h {
	77	+	continue
	78	+	}
	79	+	if next < 0 \|\| (dir < 0 && reach < next) \|\| (dir > 0 && reach > next) {
	80	+	next = reach
	81	+	}
	82	+	}
	83	+	if next < 0 \|\| (dir < 0 && next >= cur) \|\| (dir > 0 && next <= cur) {
	84	+	return false
	85	+	}
	86	+	cur = next
	87	+	}
	88	+	}
	89	+
	90	+	// underlineOf looks for an underline under the word box r in g.
	91	+	func underlineOf(g *image.Gray, r image.Rectangle) (underlineCandidate, bool) {
	92	+	b := g.Bounds()
	93	+	box := r.Intersect(b)
	94	+	h := r.Dy()
	95	+	if box.Dx() < 4 \|\| h < 4 {
	96	+	return underlineCandidate{}, false
	97	+	}
	98	+	bg, ink := contrastOf(g, box)
	99	+	thr := min(40, max(20, ink/2))
	100	+	if ink < thr+10 {
	101	+	return underlineCandidate{}, false
	102	+	}
	103	+	top := max(b.Min.Y, r.Max.Y-(h*15+99)/100)
	104	+	bottom := min(b.Max.Y, r.Max.Y+(h*35+99)/100+1)
	105	+	maxThick := max(2, h/6)
	106	+	rowOK := func(y int) bool {
	107	+	row := g.Pix[(y-g.Rect.Min.Y)*g.Stride:]
	108	+	n := 0
	109	+	for x := box.Min.X; x < box.Max.X; x++ {
	110	+	if abs(int(row[x-g.Rect.Min.X])-bg) >= thr {
	111	+	n++
	112	+	}
	113	+	}
	114	+	return n100 >= box.Dx()85
	115	+	}
	116	+	thick, first := 0, 0
	117	+	for y := top; y <= bottom; y++ {
	118	+	if y < bottom && rowOK(y) {
	119	+	if thick == 0 {
	120	+	first = y
	121	+	}
	122	+	thick++
	123	+	continue
	124	+	}
	125	+	if thick > 0 {
	126	+	if thick > maxThick {
	127	+	return underlineCandidate{}, false // a block, not a line
	128	+	}
	129	+	row := first + thick/2
	130	+	from, to := box.Min.X, box.Max.X
	131	+	line := g.Pix[(row-g.Rect.Min.Y)*g.Stride:]
	132	+	on := func(x int) bool { return abs(int(line[x-g.Rect.Min.X])-bg) >= thr }
	133	+	for from > b.Min.X && on(from-1) {
	134	+	from--
	135	+	}
	136	+	for to < b.Max.X && on(to) {
	137	+	to++
	138	+	}
	139	+	return underlineCandidate{row: row, from: from, to: to}, true
	140	+	}
	141	+	}
	142	+	return underlineCandidate{}, false
	143	+	}
	144	+
	145	+	// contrastOf returns the background level of box (the most common level,
	146	+	// found in bins of 16 so antialiasing does not split it, then refined to the
	147	+	// mean of that bin) and the contrast of its ink (the deviation reached by
	148	+	// the strongest 3% of pixels).
	149	+	func contrastOf(g *image.Gray, box image.Rectangle) (bg, ink int) {
	150	+	var hist [256]int
	151	+	for y := box.Min.Y; y < box.Max.Y; y++ {
	152	+	for _, v := range g.Pix[(y-g.Rect.Min.Y)*g.Stride+box.Min.X-g.Rect.Min.X:][:box.Dx()] {
	153	+	hist[v]++
	154	+	}
	155	+	}
	156	+	total := box.Dx() * box.Dy()
	157	+	var bins [16]int
	158	+	for v, c := range hist {
	159	+	bins[v>>4] += c
	160	+	}
	161	+	best := 0
	162	+	for i, c := range bins {
	163	+	if c > bins[best] {
	164	+	best = i
	165	+	}
	166	+	}
	167	+	sum, n := 0, 0
	168	+	for v := best << 4; v < (best+1)<<4; v++ {
	169	+	sum += v * hist[v]
	170	+	n += hist[v]
	171	+	}
	172	+	bg = sum / max(n, 1)
	173	+	var dev [256]int
	174	+	for v, c := range hist {
	175	+	dev[abs(v-bg)] += c
	176	+	}
	177	+	n = 0
	178	+	for d := 255; d >= 0; d-- {
	179	+	n += dev[d]
	180	+	if n100 >= total3 {
	181	+	return bg, d
	182	+	}
	183	+	}
	184	+	return bg, 0
	185	+	}
	186	+
	187	+	// underlinedLinks returns the targets for underlined words, joining the
	188	+	// words of one continuous underline. A group that overlaps a URL target (in
	189	+	// pixels) is left to it. Coordinates are scaled to logical.
	190	+	func underlinedLinks(words []OCRWord, urls []image.Rectangle, scale float64) []ports.HintTarget {
	191	+	var us []OCRWord
	192	+	for _, w := range words {
	193	+	if w.Underlined {
	194	+	us = append(us, w)
	195	+	}
	196	+	}
	197	+	sort.Slice(us, func(i, j int) bool { return us[i].Rect.Min.X < us[j].Rect.Min.X })
	198	+	used := make([]bool, len(us))
	199	+	var out []ports.HintTarget
	200	+	for i := range us {
	201	+	if used[i] {
	202	+	continue
	203	+	}
	204	+	used[i] = true
	205	+	rect, last := us[i].Rect, us[i]
	206	+	chars, width := len(last.Text), last.Rect.Dx()
	207	+	for {
	208	+	next := -1
	209	+	for j := range us {
	210	+	c := us[j]
	211	+	if used[j] \|\| c.Rect.Min.X < rect.Max.X-2 \|\| abs(c.Rect.Max.Y-last.Rect.Max.Y) > max(3, rect.Dy()/3) {
	212	+	continue
	213	+	}
	214	+	avg := float64(width+c.Rect.Dx()) / float64(max(1, chars+len(c.Text)))
	215	+	gap := c.Rect.Min.X - rect.Max.X
	216	+	if float64(gap) > 1.2*avg \|\| c.Rect.Min.X > last.UnderR+2 {
	217	+	continue
	218	+	}
	219	+	if next < 0 \|\| c.Rect.Min.X < us[next].Rect.Min.X {
	220	+	next = j
	221	+	}
	222	+	}
	223	+	if next < 0 {
	224	+	break
	225	+	}
	226	+	used[next] = true
	227	+	last = us[next]
	228	+	rect = rect.Union(last.Rect)
	229	+	chars += len(last.Text)
	230	+	width += last.Rect.Dx()
	231	+	}
	232	+	dup := false
	233	+	for _, u := range urls {
	234	+	if overlap(rect, u) > .5 {
	235	+	dup = true
	236	+	}
	237	+	}
	238	+	if dup {
	239	+	continue
	240	+	}
	241	+	x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
	242	+	right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
	243	+	out = append(out, ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
	244	+	}
	245	+	return out
	246	+	}

A internal/adapters/hints/underline_test.go +185 -0

@@ -0,0 +1,185 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"image"
	5	+	"image/color"
	6	+	"os"
	7	+	"os/exec"
	8	+	"testing"
	9	+
	10	+	"github.com/stretchr/testify/require"
	11	+
	12	+	"github.com/bnema/neferwl/internal/ports"
	13	+	)
	14	+
	15	+	// canvas is a synthetic grayscale band: bg everywhere, ink for text.
	16	+	type canvas struct {
	17	+	g *image.Gray
	18	+	bg, ink uint8
	19	+	s int // scale
	20	+	}
	21	+
	22	+	func newCanvas(w, h int, bg, ink uint8, s int) *canvas {
	23	+	g := image.NewGray(image.Rect(0, 0, ws, hs))
	24	+	for i := range g.Pix {
	25	+	g.Pix[i] = bg
	26	+	}
	27	+	return &canvas{g, bg, ink, s}
	28	+	}
	29	+
	30	+	func (c *canvas) fill(x0, y0, x1, y1 int, v uint8) {
	31	+	for y := y0 * c.s; y < y1*c.s; y++ {
	32	+	for x := x0 * c.s; x < x1*c.s; x++ {
	33	+	c.g.SetGray(x, y, color.Gray{v})
	34	+	}
	35	+	}
	36	+	}
	37	+
	38	+	// word draws a 10 px high word of x-height body at (x,y) with the given
	39	+	// width: text-like noisy ink (a stem every 3 px) and optional descenders.
	40	+	// It returns the OCR box (descenders excluded, as a box around the body).
	41	+	func (c *canvas) word(x, y, w int, descender bool) OCRWord {
	42	+	for i := 0; i < w; i += 3 {
	43	+	c.fill(x+i, y, x+i+1, y+10, c.ink)
	44	+	}
	45	+	box := image.Rect(xc.s, yc.s, (x+w)c.s, (y+10)c.s)
	46	+	if descender {
	47	+	for i := 0; i < w; i += 6 {
	48	+	c.fill(x+i, y+10, x+i+1, y+13, c.ink)
	49	+	}
	50	+	box.Max.Y = (y + 13) * c.s
	51	+	}
	52	+	return OCRWord{Text: "word", Rect: box}
	53	+	}
	54	+
	55	+	func (c *canvas) underline(x0, x1, y int) { c.fill(x0, y, x1, y+1, c.ink) }
	56	+
	57	+	func scanUnder(c *canvas, words ...OCRWord) []ports.HintTarget {
	58	+	markUnderlined(c.g, words)
	59	+	return textLinks(words, float64(c.s))
	60	+	}
	61	+
	62	+	func TestUnderline(t *testing.T) {
	63	+	for _, tc := range []struct {
	64	+	name string
	65	+	bg, ink uint8
	66	+	}{{"dark-bg", 10, 204}, {"light-bg", 245, 30}} {
	67	+	for _, s := range []int{1, 2} {
	68	+	t.Run(tc.name, func(t *testing.T) {
	69	+	mk := func() *canvas { return newCanvas(200, 60, tc.bg, tc.ink, s) }
	70	+
	71	+	c := mk()
	72	+	w := c.word(10, 20, 40, false)
	73	+	c.underline(10, 50, 32)
	74	+	got := scanUnder(c, w)
	75	+	require.Len(t, got, 1)
	76	+	require.Equal(t, ports.HintLink, got[0].Kind)
	77	+	require.False(t, got[0].Text)
	78	+	require.Equal(t, ports.WindowID(0), got[0].Window)
	79	+	require.Equal(t, 10, got[0].Rect.X)
	80	+
	81	+	c = mk()
	82	+	require.Empty(t, scanUnder(c, c.word(10, 20, 40, false)), "plain word")
	83	+
	84	+	c = mk()
	85	+	require.Empty(t, scanUnder(c, c.word(10, 20, 40, true)), "descenders")
	86	+
	87	+	c = mk()
	88	+	w = c.word(10, 20, 40, false)
	89	+	c.underline(0, 200, 32)
	90	+	require.Empty(t, scanUnder(c, w), "rule across")
	91	+
	92	+	c = mk()
	93	+	w = c.word(10, 20, 40, false)
	94	+	c.fill(10, 24, 50, 25, c.ink)
	95	+	require.Empty(t, scanUnder(c, w), "strikethrough")
	96	+
	97	+	// Two words with a continuous underline: one target.
	98	+	c = mk()
	99	+	a, b := c.word(10, 20, 40, false), c.word(56, 20, 40, false)
	100	+	c.underline(10, 96, 32)
	101	+	got = scanUnder(c, a, b)
	102	+	require.Len(t, got, 1)
	103	+	require.Equal(t, 10, got[0].Rect.X)
	104	+	require.Equal(t, 86, got[0].Rect.W)
	105	+
	106	+	// A gap in the underline: two targets.
	107	+	c = mk()
	108	+	a, b = c.word(10, 20, 40, false), c.word(56, 20, 40, false)
	109	+	c.underline(10, 50, 32)
	110	+	c.underline(56, 96, 32)
	111	+	require.Len(t, scanUnder(c, a, b), 2)
	112	+	})
	113	+	}
	114	+	}
	115	+	}
	116	+
	117	+	// A box border under a word, with the word a part of a table row, is not a
	118	+	// link; neither is an underline that continues beyond into unrelated text.
	119	+	func TestUnderlineRejectsBox(t *testing.T) {
	120	+	c := newCanvas(300, 60, 10, 204, 1)
	121	+	w := c.word(100, 20, 40, false)
	122	+	c.underline(20, 280, 32)
	123	+	c.fill(20, 10, 21, 33, 204)
	124	+	c.fill(279, 10, 280, 33, 204)
	125	+	require.Empty(t, scanUnder(c, w))
	126	+	}
	127	+
	128	+	// The line must be thin.
	129	+	func TestUnderlineRejectsBlock(t *testing.T) {
	130	+	c := newCanvas(100, 60, 10, 204, 1)
	131	+	w := c.word(10, 20, 40, false)
	132	+	c.fill(10, 31, 50, 36, 204)
	133	+	require.Empty(t, scanUnder(c, w))
	134	+	}
	135	+
	136	+	func TestUnderlinedURLIsOneTextTarget(t *testing.T) {
	137	+	c := newCanvas(300, 60, 10, 204, 1)
	138	+	w := c.word(10, 20, 100, false)
	139	+	w.Text = "example.org"
	140	+	c.underline(10, 110, 32)
	141	+	got := scanUnder(c, w)
	142	+	require.Len(t, got, 1)
	143	+	require.True(t, got[0].Text)
	144	+	}
	145	+
	146	+	func TestUnderlineOnBrowserTestdata(t *testing.T) {
	147	+	if _, err := exec.LookPath("tesseract"); err != nil {
	148	+	t.Skip("tesseract not on PATH")
	149	+	}
	150	+	f, err := os.Open("testdata/browser.png")
	151	+	require.NoError(t, err)
	152	+	defer f.Close()
	153	+	img, _, err := image.Decode(f)
	154	+	require.NoError(t, err)
	155	+	g := image.NewGray(img.Bounds())
	156	+	for y := 0; y < g.Rect.Dy(); y++ {
	157	+	for x := 0; x < g.Rect.Dx(); x++ {
	158	+	g.Set(x, y, img.At(x, y))
	159	+	}
	160	+	}
	161	+	words, _, err := readWords(t.Context(), g)
	162	+	require.NoError(t, err)
	163	+	var under, urls int
	164	+	for _, w := range words {
	165	+	if w.Underlined {
	166	+	under++
	167	+	t.Logf("underlined: %q %v", w.Text, w.Rect)
	168	+	}
	169	+	}
	170	+	for _, tg := range textLinks(words, 1) {
	171	+	if tg.Text {
	172	+	urls++
	173	+	}
	174	+	}
	175	+	t.Logf("words=%d underlined=%d url targets=%d total targets=%d", len(words), under, urls, len(textLinks(words, 1)))
	176	+	}
	177	+
	178	+	// A single glyph with a flat bottom stroke (8, L) is not underlined text.
	179	+	func TestUnderlineIgnoresSingleGlyph(t *testing.T) {
	180	+	c := newCanvas(100, 60, 10, 204, 1)
	181	+	w := c.word(10, 20, 10, false)
	182	+	w.Text = "8"
	183	+	c.underline(10, 20, 32)
	184	+	require.Empty(t, scanUnder(c, w))
	185	+	}

@@ -0,0 +1,185 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"image"
	5	+	"image/color"
	6	+	"os"
	7	+	"os/exec"
	8	+	"testing"
	9	+
	10	+	"github.com/stretchr/testify/require"
	11	+
	12	+	"github.com/bnema/neferwl/internal/ports"
	13	+	)
	14	+
	15	+	// canvas is a synthetic grayscale band: bg everywhere, ink for text.
	16	+	type canvas struct {
	17	+	g *image.Gray
	18	+	bg, ink uint8
	19	+	s int // scale
	20	+	}
	21	+
	22	+	func newCanvas(w, h int, bg, ink uint8, s int) *canvas {
	23	+	g := image.NewGray(image.Rect(0, 0, ws, hs))
	24	+	for i := range g.Pix {
	25	+	g.Pix[i] = bg
	26	+	}
	27	+	return &canvas{g, bg, ink, s}
	28	+	}
	29	+
	30	+	func (c *canvas) fill(x0, y0, x1, y1 int, v uint8) {
	31	+	for y := y0 * c.s; y < y1*c.s; y++ {
	32	+	for x := x0 * c.s; x < x1*c.s; x++ {
	33	+	c.g.SetGray(x, y, color.Gray{v})
	34	+	}
	35	+	}
	36	+	}
	37	+
	38	+	// word draws a 10 px high word of x-height body at (x,y) with the given
	39	+	// width: text-like noisy ink (a stem every 3 px) and optional descenders.
	40	+	// It returns the OCR box (descenders excluded, as a box around the body).
	41	+	func (c *canvas) word(x, y, w int, descender bool) OCRWord {
	42	+	for i := 0; i < w; i += 3 {
	43	+	c.fill(x+i, y, x+i+1, y+10, c.ink)
	44	+	}
	45	+	box := image.Rect(xc.s, yc.s, (x+w)c.s, (y+10)c.s)
	46	+	if descender {
	47	+	for i := 0; i < w; i += 6 {
	48	+	c.fill(x+i, y+10, x+i+1, y+13, c.ink)
	49	+	}
	50	+	box.Max.Y = (y + 13) * c.s
	51	+	}
	52	+	return OCRWord{Text: "word", Rect: box}
	53	+	}
	54	+
	55	+	func (c *canvas) underline(x0, x1, y int) { c.fill(x0, y, x1, y+1, c.ink) }
	56	+
	57	+	func scanUnder(c *canvas, words ...OCRWord) []ports.HintTarget {
	58	+	markUnderlined(c.g, words)
	59	+	return textLinks(words, float64(c.s))
	60	+	}
	61	+
	62	+	func TestUnderline(t *testing.T) {
	63	+	for _, tc := range []struct {
	64	+	name string
	65	+	bg, ink uint8
	66	+	}{{"dark-bg", 10, 204}, {"light-bg", 245, 30}} {
	67	+	for _, s := range []int{1, 2} {
	68	+	t.Run(tc.name, func(t *testing.T) {
	69	+	mk := func() *canvas { return newCanvas(200, 60, tc.bg, tc.ink, s) }
	70	+
	71	+	c := mk()
	72	+	w := c.word(10, 20, 40, false)
	73	+	c.underline(10, 50, 32)
	74	+	got := scanUnder(c, w)
	75	+	require.Len(t, got, 1)
	76	+	require.Equal(t, ports.HintLink, got[0].Kind)
	77	+	require.False(t, got[0].Text)
	78	+	require.Equal(t, ports.WindowID(0), got[0].Window)
	79	+	require.Equal(t, 10, got[0].Rect.X)
	80	+
	81	+	c = mk()
	82	+	require.Empty(t, scanUnder(c, c.word(10, 20, 40, false)), "plain word")
	83	+
	84	+	c = mk()
	85	+	require.Empty(t, scanUnder(c, c.word(10, 20, 40, true)), "descenders")
	86	+
	87	+	c = mk()
	88	+	w = c.word(10, 20, 40, false)
	89	+	c.underline(0, 200, 32)
	90	+	require.Empty(t, scanUnder(c, w), "rule across")
	91	+
	92	+	c = mk()
	93	+	w = c.word(10, 20, 40, false)
	94	+	c.fill(10, 24, 50, 25, c.ink)
	95	+	require.Empty(t, scanUnder(c, w), "strikethrough")
	96	+
	97	+	// Two words with a continuous underline: one target.
	98	+	c = mk()
	99	+	a, b := c.word(10, 20, 40, false), c.word(56, 20, 40, false)
	100	+	c.underline(10, 96, 32)
	101	+	got = scanUnder(c, a, b)
	102	+	require.Len(t, got, 1)
	103	+	require.Equal(t, 10, got[0].Rect.X)
	104	+	require.Equal(t, 86, got[0].Rect.W)
	105	+
	106	+	// A gap in the underline: two targets.
	107	+	c = mk()
	108	+	a, b = c.word(10, 20, 40, false), c.word(56, 20, 40, false)
	109	+	c.underline(10, 50, 32)
	110	+	c.underline(56, 96, 32)
	111	+	require.Len(t, scanUnder(c, a, b), 2)
	112	+	})
	113	+	}
	114	+	}
	115	+	}
	116	+
	117	+	// A box border under a word, with the word a part of a table row, is not a
	118	+	// link; neither is an underline that continues beyond into unrelated text.
	119	+	func TestUnderlineRejectsBox(t *testing.T) {
	120	+	c := newCanvas(300, 60, 10, 204, 1)
	121	+	w := c.word(100, 20, 40, false)
	122	+	c.underline(20, 280, 32)
	123	+	c.fill(20, 10, 21, 33, 204)
	124	+	c.fill(279, 10, 280, 33, 204)
	125	+	require.Empty(t, scanUnder(c, w))
	126	+	}
	127	+
	128	+	// The line must be thin.
	129	+	func TestUnderlineRejectsBlock(t *testing.T) {
	130	+	c := newCanvas(100, 60, 10, 204, 1)
	131	+	w := c.word(10, 20, 40, false)
	132	+	c.fill(10, 31, 50, 36, 204)
	133	+	require.Empty(t, scanUnder(c, w))
	134	+	}
	135	+
	136	+	func TestUnderlinedURLIsOneTextTarget(t *testing.T) {
	137	+	c := newCanvas(300, 60, 10, 204, 1)
	138	+	w := c.word(10, 20, 100, false)
	139	+	w.Text = "example.org"
	140	+	c.underline(10, 110, 32)
	141	+	got := scanUnder(c, w)
	142	+	require.Len(t, got, 1)
	143	+	require.True(t, got[0].Text)
	144	+	}
	145	+
	146	+	func TestUnderlineOnBrowserTestdata(t *testing.T) {
	147	+	if _, err := exec.LookPath("tesseract"); err != nil {
	148	+	t.Skip("tesseract not on PATH")
	149	+	}
	150	+	f, err := os.Open("testdata/browser.png")
	151	+	require.NoError(t, err)
	152	+	defer f.Close()
	153	+	img, _, err := image.Decode(f)
	154	+	require.NoError(t, err)
	155	+	g := image.NewGray(img.Bounds())
	156	+	for y := 0; y < g.Rect.Dy(); y++ {
	157	+	for x := 0; x < g.Rect.Dx(); x++ {
	158	+	g.Set(x, y, img.At(x, y))
	159	+	}
	160	+	}
	161	+	words, _, err := readWords(t.Context(), g)
	162	+	require.NoError(t, err)
	163	+	var under, urls int
	164	+	for _, w := range words {
	165	+	if w.Underlined {
	166	+	under++
	167	+	t.Logf("underlined: %q %v", w.Text, w.Rect)
	168	+	}
	169	+	}
	170	+	for _, tg := range textLinks(words, 1) {
	171	+	if tg.Text {
	172	+	urls++
	173	+	}
	174	+	}
	175	+	t.Logf("words=%d underlined=%d url targets=%d total targets=%d", len(words), under, urls, len(textLinks(words, 1)))
	176	+	}
	177	+
	178	+	// A single glyph with a flat bottom stroke (8, L) is not underlined text.
	179	+	func TestUnderlineIgnoresSingleGlyph(t *testing.T) {
	180	+	c := newCanvas(100, 60, 10, 204, 1)
	181	+	w := c.word(10, 20, 10, false)
	182	+	w.Text = "8"
	183	+	c.underline(10, 20, 32)
	184	+	require.Empty(t, scanUnder(c, w))
	185	+	}