feat(hints): read uncovered windows with tesseract

commit: e9ffd048a21ccc45d3cd726df982400046a03463
author: brice · 3d ago · 2026-10-19 13:26 UTC
parent: 78dc04f
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

15 files · +903 -148

M go.mod +0 -3


@@ -21,7 +21,6 @@ require (
21	21		)
22	22
23	23		require (
24		-	bnema.dev/hintscan v0.0.0-00010101000000-000000000000
25	24		github.com/mattn/go-colorable v0.1.15 // indirect
26	25		github.com/mattn/go-isatty v0.0.22 // indirect
27	26		github.com/stretchr/objx v0.5.3 // indirect

@@ -29,5 +28,3 @@ require (
29	28		golang.org/x/text v0.42.0 // indirect
30	29		gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
31	30		)
32		-
33		-	replace bnema.dev/hintscan => ../../../hintscan/.worktrees/scanner


@@ -21,7 +21,6 @@ require (
21		)	21	)
22			22
23		require (	23	require (
24	-	bnema.dev/hintscan v0.0.0-00010101000000-000000000000
25		github.com/mattn/go-colorable v0.1.15 // indirect	24	github.com/mattn/go-colorable v0.1.15 // indirect
26		github.com/mattn/go-isatty v0.0.22 // indirect	25	github.com/mattn/go-isatty v0.0.22 // indirect
27		github.com/stretchr/objx v0.5.3 // indirect	26	github.com/stretchr/objx v0.5.3 // indirect

@@ -29,5 +28,3 @@ require (
29		golang.org/x/text v0.42.0 // indirect	28	golang.org/x/text v0.42.0 // indirect
30		gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect	29	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
31		)	30	)
32	-
33	-	replace bnema.dev/hintscan => ../../../hintscan/.worktrees/scanner

M internal/adapters/atspi/bus_test.go +1 -1


@@ -75,7 +75,7 @@ func TestDoDecodesMatchesPipelined(t *testing.T) {
75	75		for r.More(end) {
76	76		roles = append(roles, r.Int32())
77	77		}
78		-	ok := r.Err() == nil && len(states) == 2 && states[0] == 1<<25\|1<<30 && len(roles) == 5 && roles[2] == 1<<24
	78	+	ok := r.Err() == nil && len(states) == 2 && states[0] == 1<<25\|1<<30 && len(roles) == 5 && uint32(roles[2]) == 1<<24\|1<<18\|1<<31
79	79		if !ok {
80	80		app.NewError(req, "org.test.Error.BadRule", "s").Str("bad rule")
81	81		_, _ = app.Send()


@@ -75,7 +75,7 @@ func TestDoDecodesMatchesPipelined(t *testing.T) {
75		for r.More(end) {	75		for r.More(end) {
76		roles = append(roles, r.Int32())	76		roles = append(roles, r.Int32())
77		}	77		}
78	-	ok := r.Err() == nil && len(states) == 2 && states[0] == 1<<25\|1<<30 && len(roles) == 5 && roles[2] == 1<<24	78	+	ok := r.Err() == nil && len(states) == 2 && states[0] == 1<<25\|1<<30 && len(roles) == 5 && uint32(roles[2]) == 1<<24\|1<<18\|1<<31
79		if !ok {	79		if !ok {
80		app.NewError(req, "org.test.Error.BadRule", "s").Str("bad rule")	80		app.NewError(req, "org.test.Error.BadRule", "s").Str("bad rule")
81		_, _ = app.Send()	81		_, _ = app.Send()

M internal/adapters/atspi/model.go +18 -14


@@ -5,17 +5,19 @@ import "github.com/bnema/neferwl/internal/ports"
5	5		// AT-SPI constants (atspi-constants.h). The role ids are the AtspiRole enum
6	6		// values; the state ids are bit indices of the state set.
7	7		const (
8		-	roleCheckBox = 7
9		-	roleComboBox = 11
10		-	roleDialog = 16
11		-	roleFrame = 23
12		-	roleMenuItem = 35
13		-	rolePageTab = 37
14		-	rolePushButton = 43
15		-	roleRadioButton = 44
16		-	roleToggleButton = 62
17		-	roleWindow = 69
18		-	roleLink = 88
	8	+	roleCheckBox = 7
	9	+	roleComboBox = 11
	10	+	roleDialog = 16
	11	+	roleFrame = 23
	12	+	roleMenuItem = 35
	13	+	rolePageTab = 37
	14	+	rolePushButton = 43
	15	+	roleRadioButton = 44
	16	+	roleToggleButton = 62
	17	+	roleWindow = 69
	18	+	roleLink = 88
	19	+	roleDocumentFrame = 82
	20	+	roleDocumentWeb = 95
19	21
20	22		stateActive = 1
21	23		stateShowing = 25

@@ -43,14 +45,14 @@ var clickableRoles = []uint32{
43	45		}
44	46
45	47		// matchRule is the Collection query: showing and visible objects of the
46		-	// clickable roles (only links when linksOnly).
	48	+	// clickable roles (only links when linksOnly), plus web documents for coverage.
47	49		type matchRule struct{ linksOnly bool }
48	50
49	51		func (r matchRule) roles() []uint32 {
50	52		if r.linksOnly {
51		-	return []uint32{roleLink}
	53	+	return []uint32{roleLink, roleDocumentFrame, roleDocumentWeb}
52	54		}
53		-	return clickableRoles
	55	+	return append(append([]uint32(nil), clickableRoles...), roleDocumentFrame, roleDocumentWeb)
54	56		}
55	57
56	58		// roleBits packs role ids into the bit set of a MatchRule: bit id%32 of word

@@ -186,3 +188,5 @@ func toTarget(win ports.Rect, fx, fy int32, e element) (ports.HintTarget, bool)
186	188		}
187	189		return ports.HintTarget{Rect: ports.Rect{X: x0, Y: y0, W: x1 - x0, H: y1 - y0}, Kind: kindOf(e.role)}, true
188	190		}
	191	+
	192	+	func isDocument(role uint32) bool { return role == roleDocumentFrame \|\| role == roleDocumentWeb }


@@ -5,17 +5,19 @@ import "github.com/bnema/neferwl/internal/ports"
5		// AT-SPI constants (atspi-constants.h). The role ids are the AtspiRole enum	5		// AT-SPI constants (atspi-constants.h). The role ids are the AtspiRole enum
6		// values; the state ids are bit indices of the state set.	6		// values; the state ids are bit indices of the state set.
7		const (	7		const (
8	-	roleCheckBox = 7	8	+	roleCheckBox = 7
9	-	roleComboBox = 11	9	+	roleComboBox = 11
10	-	roleDialog = 16	10	+	roleDialog = 16
11	-	roleFrame = 23	11	+	roleFrame = 23
12	-	roleMenuItem = 35	12	+	roleMenuItem = 35
13	-	rolePageTab = 37	13	+	rolePageTab = 37
14	-	rolePushButton = 43	14	+	rolePushButton = 43
15	-	roleRadioButton = 44	15	+	roleRadioButton = 44
16	-	roleToggleButton = 62	16	+	roleToggleButton = 62
17	-	roleWindow = 69	17	+	roleWindow = 69
18	-	roleLink = 88	18	+	roleLink = 88
			19	+	roleDocumentFrame = 82
			20	+	roleDocumentWeb = 95
19			21
20		stateActive = 1	22		stateActive = 1
21		stateShowing = 25	23		stateShowing = 25

@@ -43,14 +45,14 @@ var clickableRoles = []uint32{
43		}	45		}
44			46
45		// matchRule is the Collection query: showing and visible objects of the	47		// matchRule is the Collection query: showing and visible objects of the
46	-	// clickable roles (only links when linksOnly).	48	+	// clickable roles (only links when linksOnly), plus web documents for coverage.
47		type matchRule struct{ linksOnly bool }	49		type matchRule struct{ linksOnly bool }
48			50
49		func (r matchRule) roles() []uint32 {	51		func (r matchRule) roles() []uint32 {
50		if r.linksOnly {	52		if r.linksOnly {
51	-	return []uint32{roleLink}	53	+	return []uint32{roleLink, roleDocumentFrame, roleDocumentWeb}
52		}	54		}
53	-	return clickableRoles	55	+	return append(append([]uint32(nil), clickableRoles...), roleDocumentFrame, roleDocumentWeb)
54		}	56		}
55			57
56		// roleBits packs role ids into the bit set of a MatchRule: bit id%32 of word	58		// roleBits packs role ids into the bit set of a MatchRule: bit id%32 of word

@@ -186,3 +188,5 @@ func toTarget(win ports.Rect, fx, fy int32, e element) (ports.HintTarget, bool)
186		}	188		}
187		return ports.HintTarget{Rect: ports.Rect{X: x0, Y: y0, W: x1 - x0, H: y1 - y0}, Kind: kindOf(e.role)}, true	189		return ports.HintTarget{Rect: ports.Rect{X: x0, Y: y0, W: x1 - x0, H: y1 - y0}, Kind: kindOf(e.role)}, true
188		}	190		}
			191	+
			192	+	func isDocument(role uint32) bool { return role == roleDocumentFrame \|\| role == roleDocumentWeb }

M internal/adapters/atspi/source.go +9 -3


@@ -93,7 +93,7 @@ type winMatch struct {
93	93		// Targets returns the clickable targets of the accessibility tree for
94	94		// windows (logical output-local, clipped to each window) and the set of
95	95		// windows those targets cover. A window is covered when its tree answered
96		-	// with at least one target; the pixels of an uncovered window are kept. It
	96	+	// with a link target or web document; uncovered windows also get OCR. It
97	97		// returns what is ready within the budget.
98	98		func (s *Source) Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error) {
99	99		covered := map[ports.WindowID]bool{}

@@ -243,6 +243,10 @@ func (s *Source) elements(do func([]call) []reply, matched []winMatch, linksOnly
243	243		var out []ports.HintTarget
244	244		for k, i := range owner {
245	245		role, ext := info[2k], info[2k+1]
	246	+	if role.err == nil && isDocument(role.u) {
	247	+	covered[matched[i].win.ID] = true
	248	+	continue
	249	+	}
246	250		if role.err != nil \|\| ext.err != nil \|\| !isClickable(role.u, linksOnly) {
247	251		continue
248	252		}

@@ -252,7 +256,9 @@ func (s *Source) elements(do func([]call) []reply, matched []winMatch, linksOnly
252	256		continue
253	257		}
254	258		out = append(out, t)
255		-	covered[m.win.ID] = true
	259	+	if t.Kind == ports.HintLink {
	260	+	covered[m.win.ID] = true
	261	+	}
256	262		}
257	263		return out
258	264		}

@@ -279,7 +285,7 @@ func (s *Source) walk(do func([]call) []reply, root ref, linksOnly bool) []ref {
279	285		if state.err == nil && !hasState(state.state, stateShowing) {
280	286		continue
281	287		}
282		-	if role.err == nil && isClickable(role.u, linksOnly) && state.err == nil && hasState(state.state, stateVisible) {
	288	+	if role.err == nil && (isClickable(role.u, linksOnly) \|\| isDocument(role.u)) && state.err == nil && hasState(state.state, stateVisible) {
283	289		out = append(out, n)
284	290		}
285	291		if ch.err == nil {

 // Targets returns the clickable targets of the accessibility tree for
 // windows (logical output-local, clipped to each window) and the set of
 // windows those targets cover. A window is covered when its tree answered
-// with at least one target; the pixels of an uncovered window are kept. It
+// with a link target or web document; uncovered windows also get OCR. It
 // returns what is ready within the budget.
 func (s *Source) Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error) {
 	covered := map[ports.WindowID]bool{}
 	var out []ports.HintTarget
 	for k, i := range owner {
 		role, ext := info[2*k], info[2*k+1]
+		if role.err == nil && isDocument(role.u) {
+			covered[matched[i].win.ID] = true
+			continue
+		}
 		if role.err != nil || ext.err != nil || !isClickable(role.u, linksOnly) {
 			continue
 		}
 			continue
 		}
 		out = append(out, t)
-		covered[m.win.ID] = true
+		if t.Kind == ports.HintLink {
+			covered[m.win.ID] = true
+		}
 	}
 	return out
 }
 			if state.err == nil && !hasState(state.state, stateShowing) {
 				continue
 			}
-			if role.err == nil && isClickable(role.u, linksOnly) && state.err == nil && hasState(state.state, stateVisible) {
+			if role.err == nil && (isClickable(role.u, linksOnly) || isDocument(role.u)) && state.err == nil && hasState(state.state, stateVisible) {
 				out = append(out, n)
 			}
 			if ch.err == nil {

M internal/adapters/atspi/source_test.go +22 -0


@@ -2,6 +2,7 @@ package atspi
2	2
3	3		import (
4	4		"context"
	5	+	"fmt"
5	6		"testing"
6	7		"time"
7	8

@@ -122,3 +123,24 @@ func TestTargetsWithoutPIDsAsksNothing(t *testing.T) {
122	123		require.NoError(t, err)
123	124		require.Empty(t, covered)
124	125		}
	126	+
	127	+	func TestCoverageRequiresLinkOrDocument(t *testing.T) {
	128	+	for _, collection := range []bool{true, false} {
	129	+	for _, role := range []uint32{rolePushButton, roleDocumentFrame, roleDocumentWeb, roleLink} {
	130	+	t.Run(fmt.Sprintf("collection=%t/role=%d", collection, role), func(t *testing.T) {
	131	+	tr := newMocktransport(t)
	132	+	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
	133	+	[4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {10, 20, 30, 40}}, map[string]uint32{"/a": role}, !collection))
	134	+	s := &Source{tr: tr}
	135	+	targets, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}, false)
	136	+	require.NoError(t, err)
	137	+	require.Equal(t, role != rolePushButton, covered[1])
	138	+	if isDocument(role) {
	139	+	require.Empty(t, targets)
	140	+	} else {
	141	+	require.Len(t, targets, 1)
	142	+	}
	143	+	})
	144	+	}
	145	+	}
	146	+	}


@@ -2,6 +2,7 @@ package atspi
2		2
3	import (	3		import (
4	"context"	4		"context"
		5	+	"fmt"
5	"testing"	6		"testing"
6	"time"	7		"time"
7		8

@@ -122,3 +123,24 @@ func TestTargetsWithoutPIDsAsksNothing(t *testing.T) {
122	require.NoError(t, err)	123		require.NoError(t, err)
123	require.Empty(t, covered)	124		require.Empty(t, covered)
124	}	125		}
		126	+
		127	+	func TestCoverageRequiresLinkOrDocument(t *testing.T) {
		128	+	for _, collection := range []bool{true, false} {
		129	+	for _, role := range []uint32{rolePushButton, roleDocumentFrame, roleDocumentWeb, roleLink} {
		130	+	t.Run(fmt.Sprintf("collection=%t/role=%d", collection, role), func(t *testing.T) {
		131	+	tr := newMocktransport(t)
		132	+	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
		133	+	[4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {10, 20, 30, 40}}, map[string]uint32{"/a": role}, !collection))
		134	+	s := &Source{tr: tr}
		135	+	targets, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}, false)
		136	+	require.NoError(t, err)
		137	+	require.Equal(t, role != rolePushButton, covered[1])
		138	+	if isDocument(role) {
		139	+	require.Empty(t, targets)
		140	+	} else {
		141	+	require.Len(t, targets, 1)
		142	+	}
		143	+	})
		144	+	}
		145	+	}
		146	+	}

A internal/adapters/hints/benchmark_test.go +61 -0

@@ -0,0 +1,61 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/draw"
	7	+	"image/png"
	8	+	"os"
	9	+	"os/exec"
	10	+	"path/filepath"
	11	+	"testing"
	12	+	"time"
	13	+	)
	14	+
	15	+	// TestFrameBenchmark logs counts and timings, never filenames or OCR text.
	16	+	func TestFrameBenchmark(t *testing.T) {
	17	+	dir := os.Getenv("NEFERWL_HINT_BENCH_DIR")
	18	+	if dir == "" {
	19	+	t.Skip("set NEFERWL_HINT_BENCH_DIR to a local PNG directory")
	20	+	}
	21	+	if _, err := exec.LookPath("tesseract"); err != nil {
	22	+	t.Skip("tesseract not on PATH")
	23	+	}
	24	+	files, err := filepath.Glob(filepath.Join(dir, "*.png"))
	25	+	if err != nil \|\| len(files) == 0 {
	26	+	t.Fatal("no PNG frames")
	27	+	}
	28	+	for i, file := range files {
	29	+	f, err := os.Open(file)
	30	+	if err != nil {
	31	+	t.Fatal("open frame failed")
	32	+	}
	33	+	src, err := png.Decode(f)
	34	+	_ = f.Close()
	35	+	if err != nil {
	36	+	t.Fatal("decode frame failed")
	37	+	}
	38	+	img := image.NewRGBA(image.Rect(0, 0, src.Bounds().Dx(), src.Bounds().Dy()))
	39	+	draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
	40	+	start := time.Now()
	41	+	words, bands, err := readWords(context.Background(), img)
	42	+	if err != nil {
	43	+	t.Fatal("recognition failed")
	44	+	}
	45	+	t.Logf("frame=%d px=%d ocr_ms=%.1f words=%d links=%d bands=%d", i, img.Bounds().Dx()*img.Bounds().Dy(), ms(time.Since(start)), len(words), len(textLinks(words, 1)), bands)
	46	+	// Experimental strips use known OCR word regions, not pixel targets.
	47	+	// This is an oracle comparison, not a production optimization.
	48	+	lines := make([]image.Rectangle, 0, len(words))
	49	+	for _, word := range words {
	50	+	lines = append(lines, word.Rect)
	51	+	}
	52	+	for _, normalized := range []bool{false, true} {
	53	+	begin := time.Now()
	54	+	compact, _, err := readStrips(context.Background(), img, lines, 8, normalized)
	55	+	if err != nil {
	56	+	t.Fatal("strip recognition failed")
	57	+	}
	58	+	t.Logf("frame=%d strips normalized=%t ocr_ms=%.1f words=%d links=%d", i, normalized, ms(time.Since(begin)), len(compact), len(textLinks(compact, 1)))
	59	+	}
	60	+	}
	61	+	}

@@ -0,0 +1,61 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/draw"
	7	+	"image/png"
	8	+	"os"
	9	+	"os/exec"
	10	+	"path/filepath"
	11	+	"testing"
	12	+	"time"
	13	+	)
	14	+
	15	+	// TestFrameBenchmark logs counts and timings, never filenames or OCR text.
	16	+	func TestFrameBenchmark(t *testing.T) {
	17	+	dir := os.Getenv("NEFERWL_HINT_BENCH_DIR")
	18	+	if dir == "" {
	19	+	t.Skip("set NEFERWL_HINT_BENCH_DIR to a local PNG directory")
	20	+	}
	21	+	if _, err := exec.LookPath("tesseract"); err != nil {
	22	+	t.Skip("tesseract not on PATH")
	23	+	}
	24	+	files, err := filepath.Glob(filepath.Join(dir, "*.png"))
	25	+	if err != nil \|\| len(files) == 0 {
	26	+	t.Fatal("no PNG frames")
	27	+	}
	28	+	for i, file := range files {
	29	+	f, err := os.Open(file)
	30	+	if err != nil {
	31	+	t.Fatal("open frame failed")
	32	+	}
	33	+	src, err := png.Decode(f)
	34	+	_ = f.Close()
	35	+	if err != nil {
	36	+	t.Fatal("decode frame failed")
	37	+	}
	38	+	img := image.NewRGBA(image.Rect(0, 0, src.Bounds().Dx(), src.Bounds().Dy()))
	39	+	draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
	40	+	start := time.Now()
	41	+	words, bands, err := readWords(context.Background(), img)
	42	+	if err != nil {
	43	+	t.Fatal("recognition failed")
	44	+	}
	45	+	t.Logf("frame=%d px=%d ocr_ms=%.1f words=%d links=%d bands=%d", i, img.Bounds().Dx()*img.Bounds().Dy(), ms(time.Since(start)), len(words), len(textLinks(words, 1)), bands)
	46	+	// Experimental strips use known OCR word regions, not pixel targets.
	47	+	// This is an oracle comparison, not a production optimization.
	48	+	lines := make([]image.Rectangle, 0, len(words))
	49	+	for _, word := range words {
	50	+	lines = append(lines, word.Rect)
	51	+	}
	52	+	for _, normalized := range []bool{false, true} {
	53	+	begin := time.Now()
	54	+	compact, _, err := readStrips(context.Background(), img, lines, 8, normalized)
	55	+	if err != nil {
	56	+	t.Fatal("strip recognition failed")
	57	+	}
	58	+	t.Logf("frame=%d strips normalized=%t ocr_ms=%.1f words=%d links=%d", i, normalized, ms(time.Since(begin)), len(compact), len(textLinks(compact, 1)))
	59	+	}
	60	+	}
	61	+	}

M internal/adapters/hints/dump.go +21 -13


@@ -27,7 +27,7 @@ type Options struct {
27	27		// PNG and JSON). They contain the screen in clear.
28	28		DumpDir string
29	29		// Source, when not nil, finds targets in the accessibility tree of the
30		-	// windows on screen; the pixel scan covers the rest.
	30	+	// windows on screen; OCR reads the rest.
31	31		Source targetSource
32	32		}
33	33

@@ -39,8 +39,10 @@ type dumpJob struct {
39	39		pix []byte
40	40		stride int
41	41		targets []ports.HintTarget
	42	+	crops []image.Rectangle
	43	+	words []OCRWord
42	44		// atspi is the number of leading targets that come from the
43		-	// accessibility tree; the others come from pixels.
	45	+	// accessibility tree; the others come from OCR.
44	46		atspi int
45	47		captureD time.Duration
46	48		scanD time.Duration

@@ -54,16 +56,18 @@ type dumpTarget struct {
54	56		}
55	57
56	58		type dumpMeta struct {
57		-	Output string `json:"output"`
58		-	Width int `json:"width"`
59		-	Height int `json:"height"`
60		-	Stride int `json:"stride"`
61		-	Scale float64 `json:"scale"`
62		-	LinksOnly bool `json:"links_only"`
63		-	CaptureMS float64 `json:"capture_ms"`
64		-	ScanMS float64 `json:"scan_ms"`
65		-	AtspiMS float64 `json:"atspi_ms"`
66		-	Targets []dumpTarget `json:"targets"`
	59	+	Output string `json:"output"`
	60	+	Width int `json:"width"`
	61	+	Height int `json:"height"`
	62	+	Stride int `json:"stride"`
	63	+	Scale float64 `json:"scale"`
	64	+	LinksOnly bool `json:"links_only"`
	65	+	CaptureMS float64 `json:"capture_ms"`
	66	+	ScanMS float64 `json:"scan_ms"`
	67	+	AtspiMS float64 `json:"atspi_ms"`
	68	+	Crops []image.Rectangle `json:"ocr_crops"`
	69	+	Words []OCRWord `json:"words"`
	70	+	Targets []dumpTarget `json:"targets"`
67	71		}
68	72
69	73		func kindName(k ports.HintKind) string {

@@ -116,6 +120,9 @@ func writeDump(dir string, j *dumpJob) error {
116	120		if scale <= 0 {
117	121		scale = 1
118	122		}
	123	+	for _, r := range j.crops {
	124	+	outline(img, r, color.RGBA{0x80, 0x80, 0x80, 0xff})
	125	+	}
119	126		for i, t := range j.targets {
120	127		r := image.Rect(int(float64(t.Rect.X)scale), int(float64(t.Rect.Y)scale),
121	128		int(float64(t.Rect.X+t.Rect.W)scale), int(float64(t.Rect.Y+t.Rect.H)scale))

@@ -133,10 +140,11 @@ func writeDump(dir string, j *dumpJob) error {
133	140		Output: j.req.Output, Width: j.req.Width, Height: j.req.Height, Stride: j.stride,
134	141		Scale: j.req.Scale, LinksOnly: j.req.LinksOnly,
135	142		CaptureMS: ms(j.captureD), ScanMS: ms(j.scanD), AtspiMS: ms(j.atspiD),
	143	+	Crops: j.crops, Words: j.words,
136	144		Targets: make([]dumpTarget, 0, len(j.targets)),
137	145		}
138	146		for i, t := range j.targets {
139		-	src := "pixels"
	147	+	src := "ocr"
140	148		if i < j.atspi {
141	149		src = "atspi"
142	150		}

 	// PNG and JSON). They contain the screen in clear.
 	DumpDir string
 	// Source, when not nil, finds targets in the accessibility tree of the
-	// windows on screen; the pixel scan covers the rest.
+	// windows on screen; OCR reads the rest.
 	Source targetSource
 }
 	pix     []byte
 	stride  int
 	targets []ports.HintTarget
+	crops   []image.Rectangle
+	words   []OCRWord
 	// atspi is the number of leading targets that come from the
-	// accessibility tree; the others come from pixels.
+	// accessibility tree; the others come from OCR.
 	atspi    int
 	captureD time.Duration
 	scanD    time.Duration
 }
 type dumpMeta struct {
-	Output    string       `json:"output"`
+	Output    string            `json:"output"`
-	Width     int          `json:"width"`
+	Width     int               `json:"width"`
-	Height    int          `json:"height"`
+	Height    int               `json:"height"`
-	Stride    int          `json:"stride"`
+	Stride    int               `json:"stride"`
-	Scale     float64      `json:"scale"`
+	Scale     float64           `json:"scale"`
-	LinksOnly bool         `json:"links_only"`
+	LinksOnly bool              `json:"links_only"`
-	CaptureMS float64      `json:"capture_ms"`
+	CaptureMS float64           `json:"capture_ms"`
-	ScanMS    float64      `json:"scan_ms"`
+	ScanMS    float64           `json:"scan_ms"`
-	AtspiMS   float64      `json:"atspi_ms"`
+	AtspiMS   float64           `json:"atspi_ms"`
-	Targets   []dumpTarget `json:"targets"`
+	Crops     []image.Rectangle `json:"ocr_crops"`
+	Words     []OCRWord         `json:"words"`
+	Targets   []dumpTarget      `json:"targets"`
 }
 func kindName(k ports.HintKind) string {
 	if scale <= 0 {
 		scale = 1
 	}
+	for _, r := range j.crops {
+		outline(img, r, color.RGBA{0x80, 0x80, 0x80, 0xff})
+	}
 	for i, t := range j.targets {
 		r := image.Rect(int(float64(t.Rect.X)*scale), int(float64(t.Rect.Y)*scale),
 			int(float64(t.Rect.X+t.Rect.W)*scale), int(float64(t.Rect.Y+t.Rect.H)*scale))
 		Output: j.req.Output, Width: j.req.Width, Height: j.req.Height, Stride: j.stride,
 		Scale: j.req.Scale, LinksOnly: j.req.LinksOnly,
 		CaptureMS: ms(j.captureD), ScanMS: ms(j.scanD), AtspiMS: ms(j.atspiD),
+		Crops: j.crops, Words: j.words,
 		Targets: make([]dumpTarget, 0, len(j.targets)),
 	}
 	for i, t := range j.targets {
-		src := "pixels"
+		src := "ocr"
 		if i < j.atspi {
 			src = "atspi"
 		}

M internal/adapters/hints/hints.go +105 -88

@@ -1,16 +1,17 @@
1	1		// Package hints finds clickable targets for hint mode: it captures the last
2		-	// frame of an output through the capture pipeline and scans it with hintscan.
	2	+	// frame of an output through the capture pipeline and reads uncovered windows with OCR.
3	3		package hints
4	4
5	5		import (
6	6		"context"
7	7		"errors"
8	8		"fmt"
	9	+	"hash/maphash"
9	10		"image"
	11	+	"image/draw"
	12	+	"math"
10	13		"time"
11	14
12		-	"bnema.dev/hintscan"
13		-
14	15		"github.com/bnema/neferwl/internal/adapters/logging"
15	16		"github.com/bnema/neferwl/internal/ports"
16	17		)

@@ -53,17 +54,18 @@ type atspiResult struct {
53	54		}
54	55
55	56		type worker struct {
56		-	source targetSource
57		-	scanner *hintscan.Scanner
58		-	buf []byte
59		-	dst []hintscan.Target
60		-	n uint64
61		-	dumpDir string
62		-	pending *dumpJob
	57	+	source targetSource
	58	+	buf []byte
	59	+	cache map[cropKey]cropCache
	60	+	cacheSeed maphash.Seed
	61	+	warnedSource bool
	62	+	n uint64
	63	+	dumpDir string
	64	+	pending *dumpJob
63	65		}
64	66
65	67		func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {
66		-	w := &worker{scanner: hintscan.NewScanner(), dumpDir: opts.DumpDir, source: opts.Source}
	68	+	w := &worker{dumpDir: opts.DumpDir, source: opts.Source}
67	69		for {
68	70		select {
69	71		case <-ctx.Done():

@@ -112,7 +114,7 @@ func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests
112	114		if req.Width <= 0 \|\| req.Height <= 0 \|\| req.Width*req.Height > maxPixels {
113	115		return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))
114	116		}
115		-	// The accessibility query runs while the frame is captured and scanned.
	117	+	// The accessibility query runs while the frame is captured; OCR waits for it.
116	118		// It is always waited for: the source is not shared between requests.
117	119		atspi := w.startSource(ctx, req)
118	120		defer func() { <-atspi.done }()

@@ -178,56 +180,108 @@ func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests
178	180
179	181		func (w worker) scan(ctx context.Context, req ports.HintScanRequest, buf []byte, stride int, captureDur time.Duration, res ports.HintScanResult, atspi pendingSource) ports.HintScanResult {
180	182		log := logging.For(ctx, "hints")
181		-	kinds := hintscan.Clickable
182		-	if req.LinksOnly {
183		-	kinds = hintscan.KindSet(1 << hintscan.KindLink)
	183	+	ar := atspi.wait()
	184	+	if ar.err != nil && !w.warnedSource {
	185	+	log.Warn().Err(ar.err).Msg("accessibility targets unavailable; using OCR only")
	186	+	w.warnedSource = true
184	187		}
185		-	frame := hintscan.Frame{Pix: buf, Stride: stride, Width: req.Width, Height: req.Height, Order: hintscan.OrderBGRA, Scale: req.Scale}
186		-
	188	+	out := make([]ports.HintTarget, 0, len(ar.targets))
	189	+	for _, t := range ar.targets {
	190	+	if !req.LinksOnly \|\| t.Kind == ports.HintLink {
	191	+	out = append(out, t)
	192	+	}
	193	+	}
	194	+	nAtspi := len(out)
187	195		t0 := time.Now()
188		-	targets, err := w.scanner.Scan(ctx, frame, hintscan.Options{Kinds: kinds}, w.dst)
189		-	scanDur := time.Since(t0)
190		-	if err != nil {
191		-	log.Info().Err(err).Str("output", req.Output).Msg("hint scan failed")
192		-	res.Err = err
193		-	return res
	196	+	if w.cacheSeed == (maphash.Seed{}) {
	197	+	w.cacheSeed = maphash.MakeSeed()
194	198		}
195		-	w.dst = targets
196		-
197		-	pix := make([]ports.HintTarget, 0, len(targets))
198		-	for _, t := range targets {
199		-	kind := ports.HintLink
200		-	if t.Kind == hintscan.KindButton {
201		-	kind = ports.HintButton
	199	+	nextCache := make(map[cropKey]cropCache)
	200	+	var crops []image.Rectangle
	201	+	var allWords []OCRWord
	202	+	pixels, links := 0, 0
	203	+	cacheHit := true
	204	+	for _, win := range req.Windows {
	205	+	if ar.covered[win.ID] {
	206	+	continue
202	207		}
203		-	pix = append(pix, ports.HintTarget{
204		-	Rect: ports.Rect{X: t.Rect.Min.X, Y: t.Rect.Min.Y, W: t.Rect.Dx(), H: t.Rect.Dy()},
205		-	Kind: kind,
206		-	})
207		-	}
208		-	ar := atspi.wait()
209		-	if ar.err != nil {
210		-	log.Info().Err(ar.err).Msg("accessibility targets unavailable")
	208	+	r := windowCrop(win.Rect, req.Scale, req.Width, req.Height)
	209	+	if r.Empty() {
	210	+	continue
	211	+	}
	212	+	crops = append(crops, r)
	213	+	pixels += r.Dx() * r.Dy()
	214	+	img := cropImage(buf, stride, r)
	215	+	key := cropKey{output: req.Output, window: win.ID}
	216	+	hash := maphash.Bytes(w.cacheSeed, img.Pix)
	217	+	entry, ok := w.cache[key]
	218	+	hit := ok && entry.hash == hash && entry.width == r.Dx() && entry.height == r.Dy()
	219	+	cacheHit = cacheHit && hit
	220	+	if !hit {
	221	+	words, _, err := readWords(ctx, img)
	222	+	if err != nil {
	223	+	failureWarning.Do(func() { log.Warn().Err(err).Msg("OCR unavailable; install tesseract and tesseract-data-eng") })
	224	+	continue
	225	+	}
	226	+	entry = cropCache{hash: hash, width: r.Dx(), height: r.Dy(), words: words}
	227	+	}
	228	+	nextCache[key] = entry
	229	+	words := append([]OCRWord(nil), entry.words...)
	230	+	for i := range words {
	231	+	words[i].Rect = words[i].Rect.Add(r.Min)
	232	+	}
	233	+	allWords = append(allWords, words...)
	234	+	targets := textLinks(words, req.Scale)
	235	+	links += len(targets)
	236	+	out = append(out, targets...)
211	237		}
212		-	out, nAtspi := merge(req.Windows, ar.targets, ar.covered, pix)
	238	+	w.cache = nextCache
	239	+	ocrDur := time.Since(t0)
213	240		res.Targets = out
214	241		if w.dumpDir != "" {
215		-	w.pending = &dumpJob{when: time.Now(), req: req, pix: buf, stride: stride, targets: out, atspi: nAtspi, captureD: captureDur, scanD: scanDur, atspiD: ar.dur}
	242	+	w.pending = &dumpJob{when: time.Now(), req: req, pix: buf, stride: stride, targets: out, atspi: nAtspi, crops: crops, words: allWords, captureD: captureDur, scanD: ocrDur, atspiD: ar.dur}
216	243		}
217		-	log.Info().
218		-	Str("output", req.Output).
219		-	Int("targets", len(out)).
220		-	Dur("atspi_ms", ar.dur).
221		-	Int("atspi_apps", coveredApps(req.Windows, ar.covered)).
222		-	Int("atspi_windows", len(ar.covered)).
223		-	Int("atspi_targets", nAtspi).
224		-	Int("pixel_targets", len(out)-nAtspi).
225		-	Dur("scan", scanDur).
226		-	Dur("capture", captureDur).
227		-	Msg("hint scan")
	244	+	log.Info().Str("output", req.Output).
	245	+	Float64("atspi_ms", ms(ar.dur)).Int("atspi_windows", len(ar.covered)).Int("atspi_targets", nAtspi).
	246	+	Int("ocr_windows", len(crops)).Float64("ocr_ms", ms(ocrDur)).Int("ocr_px", pixels).
	247	+	Int("words", len(allWords)).Int("ocr_links", links).Bool("cache_hit", len(crops) > 0 && cacheHit).
	248	+	Int("targets", len(out)).Msg("hint scan")
228	249		return res
229	250		}
230	251
	252	+	type cropKey struct {
	253	+	output string
	254	+	window ports.WindowID
	255	+	}
	256	+	type cropCache struct {
	257	+	hash uint64
	258	+	width, height int
	259	+	words []OCRWord
	260	+	}
	261	+
	262	+	// windowCrop rounds outwards in physical pixels, then intersects the output.
	263	+	func windowCrop(r ports.Rect, scale float64, width, height int) image.Rectangle {
	264	+	if scale <= 0 {
	265	+	scale = 1
	266	+	}
	267	+	if r.W <= 0 \|\| r.H <= 0 {
	268	+	return image.Rectangle{}
	269	+	}
	270	+	return image.Rect(int(math.Floor(float64(r.X)scale)), int(math.Floor(float64(r.Y)scale)),
	271	+	int(math.Ceil(float64(r.X+r.W)scale)), int(math.Ceil(float64(r.Y+r.H)scale))).Intersect(image.Rect(0, 0, width, height))
	272	+	}
	273	+
	274	+	func cropImage(buf []byte, stride int, r image.Rectangle) *image.RGBA {
	275	+	img := image.NewRGBA(image.Rect(0, 0, r.Dx(), r.Dy()))
	276	+	// Convert only this window's BGRA pixels, not the full output.
	277	+	src := &image.RGBA{Pix: buf, Stride: stride, Rect: image.Rect(0, 0, stride/4, len(buf)/stride)}
	278	+	draw.Draw(img, img.Bounds(), src, r.Min, draw.Src)
	279	+	for i := 0; i < len(img.Pix); i += 4 {
	280	+	img.Pix[i], img.Pix[i+2], img.Pix[i+3] = img.Pix[i+2], img.Pix[i], 255
	281	+	}
	282	+	return img
	283	+	}
	284	+
231	285		// pendingSource is an accessibility query in flight.
232	286		type pendingSource struct {
233	287		done chan struct{}

@@ -256,40 +310,3 @@ func (p *pendingSource) wait() atspiResult {
256	310		<-p.done
257	311		return p.res
258	312		}
259		-
260		-	// merge keeps the accessibility targets of the covered windows and the pixel
261		-	// targets whose centre is in no covered window. The accessibility targets
262		-	// come first; it returns how many there are.
263		-	func merge(windows []ports.HintWindow, atspi []ports.HintTarget, covered map[ports.WindowID]bool, pixels []ports.HintTarget) ([]ports.HintTarget, int) {
264		-	if len(covered) == 0 {
265		-	return pixels, 0
266		-	}
267		-	out := make([]ports.HintTarget, 0, len(atspi)+len(pixels))
268		-	out = append(out, atspi...)
269		-	for _, t := range pixels {
270		-	cx, cy := t.Rect.X+t.Rect.W/2, t.Rect.Y+t.Rect.H/2
271		-	in := false
272		-	for _, win := range windows {
273		-	r := win.Rect
274		-	if covered[win.ID] && cx >= r.X && cx < r.X+r.W && cy >= r.Y && cy < r.Y+r.H {
275		-	in = true
276		-	break
277		-	}
278		-	}
279		-	if !in {
280		-	out = append(out, t)
281		-	}
282		-	}
283		-	return out, len(atspi)
284		-	}
285		-
286		-	// coveredApps counts the processes of the covered windows.
287		-	func coveredApps(windows []ports.HintWindow, covered map[ports.WindowID]bool) int {
288		-	pids := map[int]struct{}{}
289		-	for _, w := range windows {
290		-	if covered[w.ID] {
291		-	pids[w.PID] = struct{}{}
292		-	}
293		-	}
294		-	return len(pids)
295		-	}

@@ -1,16 +1,17 @@
1		// Package hints finds clickable targets for hint mode: it captures the last	1		// Package hints finds clickable targets for hint mode: it captures the last
2	-	// frame of an output through the capture pipeline and scans it with hintscan.	2	+	// frame of an output through the capture pipeline and reads uncovered windows with OCR.
3		package hints	3		package hints
4			4
5		import (	5		import (
6		"context"	6		"context"
7		"errors"	7		"errors"
8		"fmt"	8		"fmt"
			9	+	"hash/maphash"
9		"image"	10		"image"
			11	+	"image/draw"
			12	+	"math"
10		"time"	13		"time"
11			14
12	-	"bnema.dev/hintscan"
13	-
14		"github.com/bnema/neferwl/internal/adapters/logging"	15		"github.com/bnema/neferwl/internal/adapters/logging"
15		"github.com/bnema/neferwl/internal/ports"	16		"github.com/bnema/neferwl/internal/ports"
16		)	17		)

@@ -53,17 +54,18 @@ type atspiResult struct {
53		}	54		}
54			55
55		type worker struct {	56		type worker struct {
56	-	source targetSource	57	+	source targetSource
57	-	scanner *hintscan.Scanner	58	+	buf []byte
58	-	buf []byte	59	+	cache map[cropKey]cropCache
59	-	dst []hintscan.Target	60	+	cacheSeed maphash.Seed
60	-	n uint64	61	+	warnedSource bool
61	-	dumpDir string	62	+	n uint64
62	-	pending *dumpJob	63	+	dumpDir string
			64	+	pending *dumpJob
63		}	65		}
64			66
65		func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {	67		func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {
66	-	w := &worker{scanner: hintscan.NewScanner(), dumpDir: opts.DumpDir, source: opts.Source}	68	+	w := &worker{dumpDir: opts.DumpDir, source: opts.Source}
67		for {	69		for {
68		select {	70		select {
69		case <-ctx.Done():	71		case <-ctx.Done():

@@ -112,7 +114,7 @@ func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests
112		if req.Width <= 0 \|\| req.Height <= 0 \|\| req.Width*req.Height > maxPixels {	114		if req.Width <= 0 \|\| req.Height <= 0 \|\| req.Width*req.Height > maxPixels {
113		return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))	115		return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))
114		}	116		}
115	-	// The accessibility query runs while the frame is captured and scanned.	117	+	// The accessibility query runs while the frame is captured; OCR waits for it.
116		// It is always waited for: the source is not shared between requests.	118		// It is always waited for: the source is not shared between requests.
117		atspi := w.startSource(ctx, req)	119		atspi := w.startSource(ctx, req)
118		defer func() { <-atspi.done }()	120		defer func() { <-atspi.done }()

@@ -178,56 +180,108 @@ func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests
178			180
179		func (w worker) scan(ctx context.Context, req ports.HintScanRequest, buf []byte, stride int, captureDur time.Duration, res ports.HintScanResult, atspi pendingSource) ports.HintScanResult {	181		func (w worker) scan(ctx context.Context, req ports.HintScanRequest, buf []byte, stride int, captureDur time.Duration, res ports.HintScanResult, atspi pendingSource) ports.HintScanResult {
180		log := logging.For(ctx, "hints")	182		log := logging.For(ctx, "hints")
181	-	kinds := hintscan.Clickable	183	+	ar := atspi.wait()
182	-	if req.LinksOnly {	184	+	if ar.err != nil && !w.warnedSource {
183	-	kinds = hintscan.KindSet(1 << hintscan.KindLink)	185	+	log.Warn().Err(ar.err).Msg("accessibility targets unavailable; using OCR only")
			186	+	w.warnedSource = true
184		}	187		}
185	-	frame := hintscan.Frame{Pix: buf, Stride: stride, Width: req.Width, Height: req.Height, Order: hintscan.OrderBGRA, Scale: req.Scale}	188	+	out := make([]ports.HintTarget, 0, len(ar.targets))
186	-		189	+	for _, t := range ar.targets {
			190	+	if !req.LinksOnly \|\| t.Kind == ports.HintLink {
			191	+	out = append(out, t)
			192	+	}
			193	+	}
			194	+	nAtspi := len(out)
187		t0 := time.Now()	195		t0 := time.Now()
188	-	targets, err := w.scanner.Scan(ctx, frame, hintscan.Options{Kinds: kinds}, w.dst)	196	+	if w.cacheSeed == (maphash.Seed{}) {
189	-	scanDur := time.Since(t0)	197	+	w.cacheSeed = maphash.MakeSeed()
190	-	if err != nil {
191	-	log.Info().Err(err).Str("output", req.Output).Msg("hint scan failed")
192	-	res.Err = err
193	-	return res
194		}	198		}
195	-	w.dst = targets	199	+	nextCache := make(map[cropKey]cropCache)
196	-		200	+	var crops []image.Rectangle
197	-	pix := make([]ports.HintTarget, 0, len(targets))	201	+	var allWords []OCRWord
198	-	for _, t := range targets {	202	+	pixels, links := 0, 0
199	-	kind := ports.HintLink	203	+	cacheHit := true
200	-	if t.Kind == hintscan.KindButton {	204	+	for _, win := range req.Windows {
201	-	kind = ports.HintButton	205	+	if ar.covered[win.ID] {
			206	+	continue
202		}	207		}
203	-	pix = append(pix, ports.HintTarget{	208	+	r := windowCrop(win.Rect, req.Scale, req.Width, req.Height)
204	-	Rect: ports.Rect{X: t.Rect.Min.X, Y: t.Rect.Min.Y, W: t.Rect.Dx(), H: t.Rect.Dy()},	209	+	if r.Empty() {
205	-	Kind: kind,	210	+	continue
206	-	})	211	+	}
207	-	}	212	+	crops = append(crops, r)
208	-	ar := atspi.wait()	213	+	pixels += r.Dx() * r.Dy()
209	-	if ar.err != nil {	214	+	img := cropImage(buf, stride, r)
210	-	log.Info().Err(ar.err).Msg("accessibility targets unavailable")	215	+	key := cropKey{output: req.Output, window: win.ID}
			216	+	hash := maphash.Bytes(w.cacheSeed, img.Pix)
			217	+	entry, ok := w.cache[key]
			218	+	hit := ok && entry.hash == hash && entry.width == r.Dx() && entry.height == r.Dy()
			219	+	cacheHit = cacheHit && hit
			220	+	if !hit {
			221	+	words, _, err := readWords(ctx, img)
			222	+	if err != nil {
			223	+	failureWarning.Do(func() { log.Warn().Err(err).Msg("OCR unavailable; install tesseract and tesseract-data-eng") })
			224	+	continue
			225	+	}
			226	+	entry = cropCache{hash: hash, width: r.Dx(), height: r.Dy(), words: words}
			227	+	}
			228	+	nextCache[key] = entry
			229	+	words := append([]OCRWord(nil), entry.words...)
			230	+	for i := range words {
			231	+	words[i].Rect = words[i].Rect.Add(r.Min)
			232	+	}
			233	+	allWords = append(allWords, words...)
			234	+	targets := textLinks(words, req.Scale)
			235	+	links += len(targets)
			236	+	out = append(out, targets...)
211		}	237		}
212	-	out, nAtspi := merge(req.Windows, ar.targets, ar.covered, pix)	238	+	w.cache = nextCache
			239	+	ocrDur := time.Since(t0)
213		res.Targets = out	240		res.Targets = out
214		if w.dumpDir != "" {	241		if w.dumpDir != "" {
215	-	w.pending = &dumpJob{when: time.Now(), req: req, pix: buf, stride: stride, targets: out, atspi: nAtspi, captureD: captureDur, scanD: scanDur, atspiD: ar.dur}	242	+	w.pending = &dumpJob{when: time.Now(), req: req, pix: buf, stride: stride, targets: out, atspi: nAtspi, crops: crops, words: allWords, captureD: captureDur, scanD: ocrDur, atspiD: ar.dur}
216		}	243		}
217	-	log.Info().	244	+	log.Info().Str("output", req.Output).
218	-	Str("output", req.Output).	245	+	Float64("atspi_ms", ms(ar.dur)).Int("atspi_windows", len(ar.covered)).Int("atspi_targets", nAtspi).
219	-	Int("targets", len(out)).	246	+	Int("ocr_windows", len(crops)).Float64("ocr_ms", ms(ocrDur)).Int("ocr_px", pixels).
220	-	Dur("atspi_ms", ar.dur).	247	+	Int("words", len(allWords)).Int("ocr_links", links).Bool("cache_hit", len(crops) > 0 && cacheHit).
221	-	Int("atspi_apps", coveredApps(req.Windows, ar.covered)).	248	+	Int("targets", len(out)).Msg("hint scan")
222	-	Int("atspi_windows", len(ar.covered)).
223	-	Int("atspi_targets", nAtspi).
224	-	Int("pixel_targets", len(out)-nAtspi).
225	-	Dur("scan", scanDur).
226	-	Dur("capture", captureDur).
227	-	Msg("hint scan")
228		return res	249		return res
229		}	250		}
230			251
			252	+	type cropKey struct {
			253	+	output string
			254	+	window ports.WindowID
			255	+	}
			256	+	type cropCache struct {
			257	+	hash uint64
			258	+	width, height int
			259	+	words []OCRWord
			260	+	}
			261	+
			262	+	// windowCrop rounds outwards in physical pixels, then intersects the output.
			263	+	func windowCrop(r ports.Rect, scale float64, width, height int) image.Rectangle {
			264	+	if scale <= 0 {
			265	+	scale = 1
			266	+	}
			267	+	if r.W <= 0 \|\| r.H <= 0 {
			268	+	return image.Rectangle{}
			269	+	}
			270	+	return image.Rect(int(math.Floor(float64(r.X)scale)), int(math.Floor(float64(r.Y)scale)),
			271	+	int(math.Ceil(float64(r.X+r.W)scale)), int(math.Ceil(float64(r.Y+r.H)scale))).Intersect(image.Rect(0, 0, width, height))
			272	+	}
			273	+
			274	+	func cropImage(buf []byte, stride int, r image.Rectangle) *image.RGBA {
			275	+	img := image.NewRGBA(image.Rect(0, 0, r.Dx(), r.Dy()))
			276	+	// Convert only this window's BGRA pixels, not the full output.
			277	+	src := &image.RGBA{Pix: buf, Stride: stride, Rect: image.Rect(0, 0, stride/4, len(buf)/stride)}
			278	+	draw.Draw(img, img.Bounds(), src, r.Min, draw.Src)
			279	+	for i := 0; i < len(img.Pix); i += 4 {
			280	+	img.Pix[i], img.Pix[i+2], img.Pix[i+3] = img.Pix[i+2], img.Pix[i], 255
			281	+	}
			282	+	return img
			283	+	}
			284	+
231		// pendingSource is an accessibility query in flight.	285		// pendingSource is an accessibility query in flight.
232		type pendingSource struct {	286		type pendingSource struct {
233		done chan struct{}	287		done chan struct{}

@@ -256,40 +310,3 @@ func (p *pendingSource) wait() atspiResult {
256		<-p.done	310		<-p.done
257		return p.res	311		return p.res
258		}	312		}
259	-
260	-	// merge keeps the accessibility targets of the covered windows and the pixel
261	-	// targets whose centre is in no covered window. The accessibility targets
262	-	// come first; it returns how many there are.
263	-	func merge(windows []ports.HintWindow, atspi []ports.HintTarget, covered map[ports.WindowID]bool, pixels []ports.HintTarget) ([]ports.HintTarget, int) {
264	-	if len(covered) == 0 {
265	-	return pixels, 0
266	-	}
267	-	out := make([]ports.HintTarget, 0, len(atspi)+len(pixels))
268	-	out = append(out, atspi...)
269	-	for _, t := range pixels {
270	-	cx, cy := t.Rect.X+t.Rect.W/2, t.Rect.Y+t.Rect.H/2
271	-	in := false
272	-	for _, win := range windows {
273	-	r := win.Rect
274	-	if covered[win.ID] && cx >= r.X && cx < r.X+r.W && cy >= r.Y && cy < r.Y+r.H {
275	-	in = true
276	-	break
277	-	}
278	-	}
279	-	if !in {
280	-	out = append(out, t)
281	-	}
282	-	}
283	-	return out, len(atspi)
284	-	}
285	-
286	-	// coveredApps counts the processes of the covered windows.
287	-	func coveredApps(windows []ports.HintWindow, covered map[ports.WindowID]bool) int {
288	-	pids := map[int]struct{}{}
289	-	for _, w := range windows {
290	-	if covered[w.ID] {
291	-	pids[w.PID] = struct{}{}
292	-	}
293	-	}
294	-	return len(pids)
295	-	}

M internal/adapters/hints/hints_test.go +13 -24


@@ -9,6 +9,7 @@ import (
9	9		"image/draw"
10	10		_ "image/png"
11	11		"os"
	12	+	"os/exec"
12	13		"path/filepath"
13	14		"strings"
14	15		"testing"

@@ -67,6 +68,9 @@ func loadBGRA(t *testing.T) (pix []byte, w, hgt int) {
67	68		}
68	69
69	70		func TestHintScan(t *testing.T) {
	71	+	if _, err := exec.LookPath("tesseract"); err != nil {
	72	+	t.Skip("tesseract not on PATH")
	73	+	}
70	74		pix, w, hgt := loadBGRA(t)
71	75		h := start(t, 5*time.Second)
72	76

@@ -77,7 +81,7 @@ func TestHintScan(t *testing.T) {
77	81		h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
78	82		}()
79	83
80		-	h.requests <- ports.HintScanRequest{Seq: 7, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1}
	84	+	h.requests <- ports.HintScanRequest{Seq: 7, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
81	85		res := <-h.results
82	86		require.NoError(t, res.Err)
83	87		require.Equal(t, uint64(7), res.Seq)

@@ -164,7 +168,7 @@ func scanOnce(t testing.T, h harness, pix []byte, w, hgt int, seq uint64) port
164	168		copy(creq.Mapped, pix)
165	169		h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
166	170		}()
167		-	h.requests <- ports.HintScanRequest{Seq: seq, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1}
	171	+	h.requests <- ports.HintScanRequest{Seq: seq, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
168	172		res := <-h.results
169	173		require.NoError(t, res.Err)
170	174		return res

@@ -182,6 +186,9 @@ func waitFiles(t *testing.T, dir string, n int) []os.DirEntry {
182	186		}
183	187
184	188		func TestHintDump(t *testing.T) {
	189	+	if _, err := exec.LookPath("tesseract"); err != nil {
	190	+	t.Skip("tesseract not on PATH")
	191	+	}
185	192		pix, w, hgt := loadBGRA(t)
186	193		dir := filepath.Join(t.TempDir(), "hints")
187	194		h := startWith(t, 5*time.Second, Options{DumpDir: dir})

@@ -287,27 +294,6 @@ func TestHintDumpRetention(t *testing.T) {
287	294		require.Equal(t, "20260101-000000.010-o.annotated.png", entries[0].Name())
288	295		}
289	296
290		-	func TestMergePolicy(t *testing.T) {
291		-	windows := []ports.HintWindow{
292		-	{ID: 1, Rect: ports.Rect{X: 0, Y: 0, W: 100, H: 100}},
293		-	{ID: 2, Rect: ports.Rect{X: 100, Y: 0, W: 100, H: 100}},
294		-	}
295		-	at := []ports.HintTarget{{Rect: ports.Rect{X: 10, Y: 10, W: 20, H: 10}, Kind: ports.HintLink}}
296		-	pixels := []ports.HintTarget{
297		-	{Rect: ports.Rect{X: 12, Y: 12, W: 10, H: 4}}, // centre in covered window 1: dropped
298		-	{Rect: ports.Rect{X: 120, Y: 10, W: 20, H: 10}}, // window 2 is not covered: kept
299		-	{Rect: ports.Rect{X: 500, Y: 10, W: 20, H: 10}}, // no window: kept
300		-	}
301		-	out, n := merge(windows, at, map[ports.WindowID]bool{1: true}, pixels)
302		-	require.Equal(t, 1, n)
303		-	require.Equal(t, []ports.HintTarget{at[0], pixels[1], pixels[2]}, out)
304		-
305		-	// Nothing covered: pixels only.
306		-	out, n = merge(windows, nil, nil, pixels)
307		-	require.Zero(t, n)
308		-	require.Equal(t, pixels, out)
309		-	}
310		-
311	297		func TestScanMergesSource(t *testing.T) {
312	298		pix, w, hgt := loadBGRA(t)
313	299		src := newMocktargetSource(t)

@@ -327,7 +313,10 @@ func TestScanMergesSource(t *testing.T) {
327	313		require.Equal(t, at, res.Targets)
328	314		}
329	315
330		-	func TestScanSourceErrorKeepsPixels(t *testing.T) {
	316	+	func TestScanSourceErrorUsesOCR(t *testing.T) {
	317	+	if _, err := exec.LookPath("tesseract"); err != nil {
	318	+	t.Skip("tesseract not on PATH")
	319	+	}
331	320		pix, w, hgt := loadBGRA(t)
332	321		src := newMocktargetSource(t)
333	322		wins := []ports.HintWindow{{ID: 4, PID: 9, Rect: ports.Rect{W: w, H: hgt}}}

 	"image/draw"
 	_ "image/png"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
 }
 func TestHintScan(t *testing.T) {
+	if _, err := exec.LookPath("tesseract"); err != nil {
+		t.Skip("tesseract not on PATH")
+	}
 	pix, w, hgt := loadBGRA(t)
 	h := start(t, 5*time.Second)
 		h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
 	}()
-	h.requests <- ports.HintScanRequest{Seq: 7, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1}
+	h.requests <- ports.HintScanRequest{Seq: 7, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
 	res := <-h.results
 	require.NoError(t, res.Err)
 	require.Equal(t, uint64(7), res.Seq)
 		copy(creq.Mapped, pix)
 		h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
 	}()
-	h.requests <- ports.HintScanRequest{Seq: seq, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1}
+	h.requests <- ports.HintScanRequest{Seq: seq, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
 	res := <-h.results
 	require.NoError(t, res.Err)
 	return res
 }
 func TestHintDump(t *testing.T) {
+	if _, err := exec.LookPath("tesseract"); err != nil {
+		t.Skip("tesseract not on PATH")
+	}
 	pix, w, hgt := loadBGRA(t)
 	dir := filepath.Join(t.TempDir(), "hints")
 	h := startWith(t, 5*time.Second, Options{DumpDir: dir})
 	require.Equal(t, "20260101-000000.010-o.annotated.png", entries[0].Name())
 }
-func TestMergePolicy(t *testing.T) {
-	windows := []ports.HintWindow{
-		{ID: 1, Rect: ports.Rect{X: 0, Y: 0, W: 100, H: 100}},
-		{ID: 2, Rect: ports.Rect{X: 100, Y: 0, W: 100, H: 100}},
-	}
-	at := []ports.HintTarget{{Rect: ports.Rect{X: 10, Y: 10, W: 20, H: 10}, Kind: ports.HintLink}}
-	pixels := []ports.HintTarget{
-		{Rect: ports.Rect{X: 12, Y: 12, W: 10, H: 4}},   // centre in covered window 1: dropped
-		{Rect: ports.Rect{X: 120, Y: 10, W: 20, H: 10}}, // window 2 is not covered: kept
-		{Rect: ports.Rect{X: 500, Y: 10, W: 20, H: 10}}, // no window: kept
-	}
-	out, n := merge(windows, at, map[ports.WindowID]bool{1: true}, pixels)
-	require.Equal(t, 1, n)
-	require.Equal(t, []ports.HintTarget{at[0], pixels[1], pixels[2]}, out)
-	// Nothing covered: pixels only.
-	out, n = merge(windows, nil, nil, pixels)
-	require.Zero(t, n)
-	require.Equal(t, pixels, out)
-}
 func TestScanMergesSource(t *testing.T) {
 	pix, w, hgt := loadBGRA(t)
 	src := newMocktargetSource(t)
 	require.Equal(t, at, res.Targets)
 }
-func TestScanSourceErrorKeepsPixels(t *testing.T) {
+func TestScanSourceErrorUsesOCR(t *testing.T) {
+	if _, err := exec.LookPath("tesseract"); err != nil {
+		t.Skip("tesseract not on PATH")
+	}
 	pix, w, hgt := loadBGRA(t)
 	src := newMocktargetSource(t)
 	wins := []ports.HintWindow{{ID: 4, PID: 9, Rect: ports.Rect{W: w, H: hgt}}}

A internal/adapters/hints/hybrid_test.go +88 -0

@@ -0,0 +1,88 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"hash/maphash"
	6	+	"image"
	7	+	"testing"
	8	+
	9	+	"github.com/bnema/neferwl/internal/ports"
	10	+	"github.com/stretchr/testify/require"
	11	+	)
	12	+
	13	+	func TestWindowCrop(t *testing.T) {
	14	+	for _, tc := range []struct {
	15	+	name string
	16	+	r ports.Rect
	17	+	scale float64
	18	+	want image.Rectangle
	19	+	}{
	20	+	{"unit", ports.Rect{X: 10, Y: 20, W: 100, H: 50}, 1, image.Rect(10, 20, 110, 70)},
	21	+	{"fractional", ports.Rect{X: 11, Y: 21, W: 100, H: 50}, 1.2, image.Rect(13, 25, 134, 86)},
	22	+	{"off output", ports.Rect{X: -10, Y: 20, W: 100, H: 100}, 1.2, image.Rect(0, 24, 108, 100)},
	23	+	{"outside", ports.Rect{X: 1000, Y: 20, W: 100, H: 100}, 1, image.Rectangle{}},
	24	+	} {
	25	+	t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.want, windowCrop(tc.r, tc.scale, 200, 100)) })
	26	+	}
	27	+	}
	28	+
	29	+	func sourceResult(ar atspiResult) *pendingSource {
	30	+	p := &pendingSource{done: make(chan struct{}), res: ar}
	31	+	close(p.done)
	32	+	return p
	33	+	}
	34	+
	35	+	func TestHybridCacheAndMerge(t *testing.T) {
	36	+	t.Setenv("PATH", t.TempDir()) // Cached OCR must not need tesseract.
	37	+	buf := make([]byte, 2001004)
	38	+	wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 100, H: 100}}, {ID: 2, Rect: ports.Rect{X: 100, W: 100, H: 100}}}
	39	+	req := ports.HintScanRequest{Output: "o", Width: 200, Height: 100, Scale: 1, Windows: wins}
	40	+	seed := maphash.MakeSeed()
	41	+	r := windowCrop(wins[1].Rect, 1, 200, 100)
	42	+	entry := cropCache{hash: maphash.Bytes(seed, cropImage(buf, 800, r).Pix), width: 100, height: 100, words: []OCRWord{{Text: "example.org", Rect: image.Rect(10, 20, 90, 40)}}}
	43	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 1, Y: 1, W: 10, H: 10}}
	44	+	link := ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: 110, Y: 20, W: 80, H: 20}}
	45	+	uncoveredButton := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 101, Y: 1, W: 10, H: 10}}
	46	+	ar := atspiResult{targets: []ports.HintTarget{button, uncoveredButton}, covered: map[ports.WindowID]bool{1: true}}
	47	+	w := worker{cacheSeed: seed, cache: map[cropKey]cropCache{{output: "o", window: 2}: entry}}
	48	+	got := w.scan(context.Background(), req, buf, 800, 0, ports.HintScanResult{}, sourceResult(ar))
	49	+	require.Equal(t, []ports.HintTarget{button, uncoveredButton, link}, got.Targets)
	50	+	req.LinksOnly = true
	51	+	got = w.scan(context.Background(), req, buf, 800, 0, ports.HintScanResult{}, sourceResult(ar))
	52	+	require.Equal(t, []ports.HintTarget{link}, got.Targets)
	53	+	// Changing one pixel invalidates only the crop, never leaving stale links.
	54	+	buf[100*4]++
	55	+	got = w.scan(context.Background(), req, buf, 800, 0, ports.HintScanResult{}, sourceResult(ar))
	56	+	require.Empty(t, got.Targets)
	57	+	}
	58	+
	59	+	func TestMissingTesseractKeepsAccessibilityButtons(t *testing.T) {
	60	+	t.Setenv("PATH", t.TempDir())
	61	+	w := worker{}
	62	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
	63	+	req := ports.HintScanRequest{Width: 20, Height: 20, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}}}
	64	+	got := w.scan(context.Background(), req, make([]byte, 1600), 80, 0, ports.HintScanResult{}, sourceResult(atspiResult{targets: []ports.HintTarget{button}}))
	65	+	require.NoError(t, got.Err)
	66	+	require.Equal(t, []ports.HintTarget{button}, got.Targets)
	67	+	}
	68	+
	69	+	func TestNoWindowsProducesNoPixelTargets(t *testing.T) {
	70	+	pix, width, height := loadBGRA(t)
	71	+	w := worker{}
	72	+	got := w.scan(context.Background(), ports.HintScanRequest{Width: width, Height: height}, pix, width*4, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	73	+	require.Empty(t, got.Targets)
	74	+	}
	75	+
	76	+	func TestStripMappingAndPolarity(t *testing.T) {
	77	+	img := image.NewRGBA(image.Rect(0, 0, 200, 200))
	78	+	strips := packLines(img, []image.Rectangle{image.Rect(20, 20, 160, 40), image.Rect(20, 80, 160, 112)}, 2, true)
	79	+	require.Len(t, strips, 2)
	80	+	for _, s := range strips {
	81	+	for _, off := range s.offsets {
	82	+	require.Equal(t, 1, off.factor)
	83	+	require.Equal(t, uint8(255), s.img.GrayAt(off.packed.Min.X, off.packed.Min.Y).Y)
	84	+	}
	85	+	}
	86	+	small := packLines(img, []image.Rectangle{image.Rect(20, 20, 100, 30)}, 1, true)
	87	+	require.Equal(t, 2, small[0].offsets[0].factor)
	88	+	}

@@ -0,0 +1,88 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"hash/maphash"
	6	+	"image"
	7	+	"testing"
	8	+
	9	+	"github.com/bnema/neferwl/internal/ports"
	10	+	"github.com/stretchr/testify/require"
	11	+	)
	12	+
	13	+	func TestWindowCrop(t *testing.T) {
	14	+	for _, tc := range []struct {
	15	+	name string
	16	+	r ports.Rect
	17	+	scale float64
	18	+	want image.Rectangle
	19	+	}{
	20	+	{"unit", ports.Rect{X: 10, Y: 20, W: 100, H: 50}, 1, image.Rect(10, 20, 110, 70)},
	21	+	{"fractional", ports.Rect{X: 11, Y: 21, W: 100, H: 50}, 1.2, image.Rect(13, 25, 134, 86)},
	22	+	{"off output", ports.Rect{X: -10, Y: 20, W: 100, H: 100}, 1.2, image.Rect(0, 24, 108, 100)},
	23	+	{"outside", ports.Rect{X: 1000, Y: 20, W: 100, H: 100}, 1, image.Rectangle{}},
	24	+	} {
	25	+	t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.want, windowCrop(tc.r, tc.scale, 200, 100)) })
	26	+	}
	27	+	}
	28	+
	29	+	func sourceResult(ar atspiResult) *pendingSource {
	30	+	p := &pendingSource{done: make(chan struct{}), res: ar}
	31	+	close(p.done)
	32	+	return p
	33	+	}
	34	+
	35	+	func TestHybridCacheAndMerge(t *testing.T) {
	36	+	t.Setenv("PATH", t.TempDir()) // Cached OCR must not need tesseract.
	37	+	buf := make([]byte, 2001004)
	38	+	wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 100, H: 100}}, {ID: 2, Rect: ports.Rect{X: 100, W: 100, H: 100}}}
	39	+	req := ports.HintScanRequest{Output: "o", Width: 200, Height: 100, Scale: 1, Windows: wins}
	40	+	seed := maphash.MakeSeed()
	41	+	r := windowCrop(wins[1].Rect, 1, 200, 100)
	42	+	entry := cropCache{hash: maphash.Bytes(seed, cropImage(buf, 800, r).Pix), width: 100, height: 100, words: []OCRWord{{Text: "example.org", Rect: image.Rect(10, 20, 90, 40)}}}
	43	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 1, Y: 1, W: 10, H: 10}}
	44	+	link := ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: 110, Y: 20, W: 80, H: 20}}
	45	+	uncoveredButton := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 101, Y: 1, W: 10, H: 10}}
	46	+	ar := atspiResult{targets: []ports.HintTarget{button, uncoveredButton}, covered: map[ports.WindowID]bool{1: true}}
	47	+	w := worker{cacheSeed: seed, cache: map[cropKey]cropCache{{output: "o", window: 2}: entry}}
	48	+	got := w.scan(context.Background(), req, buf, 800, 0, ports.HintScanResult{}, sourceResult(ar))
	49	+	require.Equal(t, []ports.HintTarget{button, uncoveredButton, link}, got.Targets)
	50	+	req.LinksOnly = true
	51	+	got = w.scan(context.Background(), req, buf, 800, 0, ports.HintScanResult{}, sourceResult(ar))
	52	+	require.Equal(t, []ports.HintTarget{link}, got.Targets)
	53	+	// Changing one pixel invalidates only the crop, never leaving stale links.
	54	+	buf[100*4]++
	55	+	got = w.scan(context.Background(), req, buf, 800, 0, ports.HintScanResult{}, sourceResult(ar))
	56	+	require.Empty(t, got.Targets)
	57	+	}
	58	+
	59	+	func TestMissingTesseractKeepsAccessibilityButtons(t *testing.T) {
	60	+	t.Setenv("PATH", t.TempDir())
	61	+	w := worker{}
	62	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
	63	+	req := ports.HintScanRequest{Width: 20, Height: 20, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}}}
	64	+	got := w.scan(context.Background(), req, make([]byte, 1600), 80, 0, ports.HintScanResult{}, sourceResult(atspiResult{targets: []ports.HintTarget{button}}))
	65	+	require.NoError(t, got.Err)
	66	+	require.Equal(t, []ports.HintTarget{button}, got.Targets)
	67	+	}
	68	+
	69	+	func TestNoWindowsProducesNoPixelTargets(t *testing.T) {
	70	+	pix, width, height := loadBGRA(t)
	71	+	w := worker{}
	72	+	got := w.scan(context.Background(), ports.HintScanRequest{Width: width, Height: height}, pix, width*4, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	73	+	require.Empty(t, got.Targets)
	74	+	}
	75	+
	76	+	func TestStripMappingAndPolarity(t *testing.T) {
	77	+	img := image.NewRGBA(image.Rect(0, 0, 200, 200))
	78	+	strips := packLines(img, []image.Rectangle{image.Rect(20, 20, 160, 40), image.Rect(20, 80, 160, 112)}, 2, true)
	79	+	require.Len(t, strips, 2)
	80	+	for _, s := range strips {
	81	+	for _, off := range s.offsets {
	82	+	require.Equal(t, 1, off.factor)
	83	+	require.Equal(t, uint8(255), s.img.GrayAt(off.packed.Min.X, off.packed.Min.Y).Y)
	84	+	}
	85	+	}
	86	+	small := packLines(img, []image.Rectangle{image.Rect(20, 20, 100, 30)}, 1, true)
	87	+	require.Equal(t, 2, small[0].offsets[0].factor)
	88	+	}

A internal/adapters/hints/ocr.go +298 -0

@@ -0,0 +1,298 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"bytes"
	6	+	"context"
	7	+	"fmt"
	8	+	"image"
	9	+	"image/png"
	10	+	"math"
	11	+	"net"
	12	+	"os"
	13	+	"os/exec"
	14	+	"path/filepath"
	15	+	"regexp"
	16	+	"runtime"
	17	+	"sort"
	18	+	"strconv"
	19	+	"strings"
	20	+	"sync"
	21	+	"time"
	22	+
	23	+	"github.com/bnema/neferwl/internal/adapters/logging"
	24	+	"github.com/bnema/neferwl/internal/ports"
	25	+	)
	26	+
	27	+	// OCRWord is a recognized word in physical output coordinates.
	28	+	type OCRWord struct {
	29	+	Text string `json:"text"`
	30	+	Rect image.Rectangle `json:"rect"`
	31	+	Confidence float64 `json:"confidence"`
	32	+	}
	33	+
	34	+	var languageWarning sync.Once
	35	+	var failureWarning sync.Once
	36	+
	37	+	func ocrLanguage(ctx context.Context, binary string) (string, error) {
	38	+	dir := os.Getenv("TESSDATA_PREFIX")
	39	+	if dir == "" {
	40	+	dir = "/usr/share/tessdata"
	41	+	}
	42	+	if _, err := os.Stat(filepath.Join(dir, "eng.traineddata")); err == nil {
	43	+	return "eng", nil
	44	+	}
	45	+	out, err := exec.CommandContext(ctx, binary, "--list-langs").Output()
	46	+	if err != nil {
	47	+	return "", err
	48	+	}
	49	+	for _, line := range strings.Split(string(out), "\n")[1:] {
	50	+	lang := strings.TrimSpace(line)
	51	+	if lang != "" && lang != "osd" {
	52	+	languageWarning.Do(func() {
	53	+	log := logging.For(ctx, "hints")
	54	+	log.Warn().Str("language", lang).Msg("English OCR data missing; install tesseract-data-eng")
	55	+	})
	56	+	return lang, nil
	57	+	}
	58	+	}
	59	+	return "", fmt.Errorf("no OCR language available")
	60	+	}
	61	+
	62	+	func parseTSV(data []byte, offset image.Point) ([]OCRWord, error) {
	63	+	var words []OCRWord
	64	+	scanner := bufio.NewScanner(bytes.NewReader(data))
	65	+	scanner.Buffer(make([]byte, 4096), 1024*1024)
	66	+	for scanner.Scan() {
	67	+	fields := strings.SplitN(scanner.Text(), "\t", 12)
	68	+	if len(fields) != 12 \|\| fields[0] != "5" {
	69	+	continue
	70	+	}
	71	+	var values [4]int
	72	+	for i := range values {
	73	+	v, err := strconv.Atoi(fields[6+i])
	74	+	if err != nil {
	75	+	return nil, fmt.Errorf("OCR TSV coordinate: %w", err)
	76	+	}
	77	+	values[i] = v
	78	+	}
	79	+	conf, err := strconv.ParseFloat(fields[10], 64)
	80	+	if err != nil {
	81	+	return nil, fmt.Errorf("OCR TSV confidence: %w", err)
	82	+	}
	83	+	if values[2] <= 0 \|\| values[3] <= 0 \|\| strings.TrimSpace(fields[11]) == "" {
	84	+	continue
	85	+	}
	86	+	r := image.Rect(values[0], values[1], values[0]+values[2], values[1]+values[3]).Add(offset)
	87	+	words = append(words, OCRWord{Text: fields[11], Rect: r, Confidence: conf})
	88	+	}
	89	+	return words, scanner.Err()
	90	+	}
	91	+
	92	+	func readWords(ctx context.Context, img *image.RGBA) ([]OCRWord, int, error) {
	93	+	binary, err := exec.LookPath("tesseract")
	94	+	if err != nil {
	95	+	return nil, 0, err
	96	+	}
	97	+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
	98	+	defer cancel()
	99	+	lang, err := ocrLanguage(ctx, binary)
	100	+	if err != nil {
	101	+	return nil, 0, err
	102	+	}
	103	+	n := min(runtime.GOMAXPROCS(0), 8, img.Bounds().Dy())
	104	+	results := make([][]OCRWord, n)
	105	+	errs := make([]error, n)
	106	+	var wg sync.WaitGroup
	107	+	for i := range n {
	108	+	wg.Go(func() {
	109	+	top := max(0, i*img.Bounds().Dy()/n-40)
	110	+	bottom := min(img.Bounds().Dy(), (i+1)*img.Bounds().Dy()/n+40)
	111	+	crop := img.SubImage(image.Rect(0, top, img.Bounds().Dx(), bottom))
	112	+	results[i], errs[i] = recognizeBand(ctx, binary, lang, crop)
	113	+	if errs[i] != nil {
	114	+	return
	115	+	}
	116	+	heights := make([]int, 0, len(results[i]))
	117	+	for _, word := range results[i] {
	118	+	if len(word.Text) >= 3 {
	119	+	heights = append(heights, word.Rect.Dy())
	120	+	}
	121	+	}
	122	+	sort.Ints(heights)
	123	+	if len(heights) > 0 && heights[len(heights)/2] < 12 {
	124	+	larger := upscaleBand(crop)
	125	+	results[i], errs[i] = recognizeBand(ctx, binary, lang, larger)
	126	+	for j := range results[i] {
	127	+	r := results[i][j].Rect
	128	+	results[i][j].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
	129	+	}
	130	+	}
	131	+	for j := range results[i] {
	132	+	results[i][j].Rect = results[i][j].Rect.Add(image.Pt(0, top))
	133	+	}
	134	+	})
	135	+	}
	136	+	wg.Wait()
	137	+	var words []OCRWord
	138	+	for i := range n {
	139	+	if errs[i] != nil {
	140	+	return nil, n, errs[i]
	141	+	}
	142	+	for _, word := range results[i] {
	143	+	// Overlap supplies context, but each word belongs to exactly one band.
	144	+	center := (word.Rect.Min.Y + word.Rect.Max.Y) / 2
	145	+	if center < iimg.Bounds().Dy()/n \|\| center >= (i+1)img.Bounds().Dy()/n {
	146	+	continue
	147	+	}
	148	+	duplicate := false
	149	+	for _, old := range words {
	150	+	if strings.EqualFold(word.Text, old.Text) && overlap(word.Rect, old.Rect) > .5 {
	151	+	duplicate = true
	152	+	break
	153	+	}
	154	+	}
	155	+	if !duplicate {
	156	+	words = append(words, word)
	157	+	}
	158	+	}
	159	+	}
	160	+	return words, n, nil
	161	+	}
	162	+
	163	+	var domainPattern = regexp.MustCompile(`(?i)^(?:[a-z0-9](?:[a-z0-9-][a-z0-9])?\.)+([a-z]{2,24})(?::[0-9]+)?(?:[/?#]\S)?$`)
	164	+	var schemePattern = regexp.MustCompile(`(?i)^(?:https?\|htps\|file\|ftp):/{1,2}\S+$`)
	165	+	var emailPattern = regexp.MustCompile(`(?i)^[a-z0-9.!#$%&'*+/=?^_` + "`" + `{\|}~-]+@(?:[a-z0-9-]+\.)+[a-z]{2,24}$`)
	166	+	var fileExtensions = strings.Fields("go md txt toml json yaml yml rs py js ts sh c h png jpg log conf lock mod sum html css")
	167	+
	168	+	func isTextLink(text string) bool {
	169	+	text = strings.Trim(text, "()[]<>.,;:'\"")
	170	+	if schemePattern.MatchString(text) \|\| (strings.HasPrefix(strings.ToLower(text), "www.") && len(text) > 4) \|\| emailPattern.MatchString(text) {
	171	+	return true
	172	+	}
	173	+	host := text
	174	+	if h, _, err := net.SplitHostPort(text); err == nil {
	175	+	host = h
	176	+	}
	177	+	if ip := net.ParseIP(host); ip != nil && ip.To4() != nil {
	178	+	return true
	179	+	}
	180	+	m := domainPattern.FindStringSubmatch(text)
	181	+	if m == nil {
	182	+	return false
	183	+	}
	184	+	for _, ext := range fileExtensions {
	185	+	if strings.EqualFold(m[1], ext) {
	186	+	return false
	187	+	}
	188	+	}
	189	+	return true
	190	+	}
	191	+
	192	+	func textLinks(words []OCRWord, scale float64) []ports.HintTarget {
	193	+	if scale <= 0 {
	194	+	scale = 1
	195	+	}
	196	+	words = append([]OCRWord(nil), words...)
	197	+	sort.Slice(words, func(i, j int) bool {
	198	+	if words[i].Rect.Min.Y != words[j].Rect.Min.Y {
	199	+	return words[i].Rect.Min.Y < words[j].Rect.Min.Y
	200	+	}
	201	+	return words[i].Rect.Min.X < words[j].Rect.Min.X
	202	+	})
	203	+	var out []ports.HintTarget
	204	+	used := make([]bool, len(words))
	205	+	for i, word := range words {
	206	+	if used[i] {
	207	+	continue
	208	+	}
	209	+	text, rect := word.Text, word.Rect
	210	+	// Grow only touching words on the same baseline; ordinary prose is not joined.
	211	+	for {
	212	+	next := -1
	213	+	for j, candidate := range words {
	214	+	if j == i \|\| used[j] \|\| candidate.Rect.Min.X < rect.Max.X-2 {
	215	+	continue
	216	+	}
	217	+	gap := candidate.Rect.Min.X - rect.Max.X
	218	+	if gap > max(3, min(rect.Dy(), candidate.Rect.Dy())/5) {
	219	+	continue
	220	+	}
	221	+	if abs(candidate.Rect.Max.Y-rect.Max.Y) > max(3, rect.Dy()/3) {
	222	+	continue
	223	+	}
	224	+	if next < 0 \|\| candidate.Rect.Min.X < words[next].Rect.Min.X {
	225	+	next = j
	226	+	}
	227	+	}
	228	+	if next < 0 {
	229	+	break
	230	+	}
	231	+	text += words[next].Text
	232	+	rect = rect.Union(words[next].Rect)
	233	+	used[next] = true
	234	+	}
	235	+	if !isTextLink(text) {
	236	+	continue
	237	+	}
	238	+	x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
	239	+	right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
	240	+	out = append(out, ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
	241	+	}
	242	+	return out
	243	+	}
	244	+
	245	+	func abs(x int) int {
	246	+	if x < 0 {
	247	+	return -x
	248	+	}
	249	+	return x
	250	+	}
	251	+	func overlap(a, b image.Rectangle) float64 {
	252	+	intersection := a.Intersect(b)
	253	+	area := min(a.Dx()a.Dy(), b.Dx()b.Dy())
	254	+	if intersection.Empty() \|\| area <= 0 {
	255	+	return 0
	256	+	}
	257	+	return float64(intersection.Dx()*intersection.Dy()) / float64(area)
	258	+	}
	259	+	func targetRect(t ports.HintTarget) image.Rectangle {
	260	+	return image.Rect(t.Rect.X, t.Rect.Y, t.Rect.X+t.Rect.W, t.Rect.Y+t.Rect.H)
	261	+	}
	262	+	func recognizeBand(ctx context.Context, binary, lang string, img image.Image) ([]OCRWord, error) {
	263	+	return recognizeImage(ctx, binary, lang, img, "11", false)
	264	+	}
	265	+
	266	+	func recognizeImage(ctx context.Context, binary, lang string, img image.Image, psm string, normalized bool) ([]OCRWord, error) {
	267	+	var input bytes.Buffer
	268	+	enc := png.Encoder{CompressionLevel: png.BestSpeed}
	269	+	if err := enc.Encode(&input, img); err != nil {
	270	+	return nil, err
	271	+	}
	272	+	args := []string{"stdin", "stdout", "--psm", psm, "-l", lang}
	273	+	if normalized {
	274	+	args = append(args, "-c", "tessedit_do_invert=0")
	275	+	}
	276	+	args = append(args, "tsv")
	277	+	cmd := exec.CommandContext(ctx, binary, args...)
	278	+	cmd.Env = append(os.Environ(), "OMP_THREAD_LIMIT=1")
	279	+	cmd.Stdin = &input
	280	+	var stderr bytes.Buffer
	281	+	cmd.Stderr = &stderr
	282	+	output, err := cmd.Output()
	283	+	if err != nil {
	284	+	return nil, fmt.Errorf("tesseract: %w: %s", err, stderr.String())
	285	+	}
	286	+	return parseTSV(output, image.Point{})
	287	+	}
	288	+
	289	+	func upscaleBand(src image.Image) *image.RGBA {
	290	+	b := src.Bounds()
	291	+	dst := image.NewRGBA(image.Rect(0, 0, b.Dx()2, b.Dy()2))
	292	+	for y := 0; y < dst.Bounds().Dy(); y++ {
	293	+	for x := 0; x < dst.Bounds().Dx(); x++ {
	294	+	dst.Set(x, y, src.At(b.Min.X+x/2, b.Min.Y+y/2))
	295	+	}
	296	+	}
	297	+	return dst
	298	+	}

@@ -0,0 +1,298 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"bytes"
	6	+	"context"
	7	+	"fmt"
	8	+	"image"
	9	+	"image/png"
	10	+	"math"
	11	+	"net"
	12	+	"os"
	13	+	"os/exec"
	14	+	"path/filepath"
	15	+	"regexp"
	16	+	"runtime"
	17	+	"sort"
	18	+	"strconv"
	19	+	"strings"
	20	+	"sync"
	21	+	"time"
	22	+
	23	+	"github.com/bnema/neferwl/internal/adapters/logging"
	24	+	"github.com/bnema/neferwl/internal/ports"
	25	+	)
	26	+
	27	+	// OCRWord is a recognized word in physical output coordinates.
	28	+	type OCRWord struct {
	29	+	Text string `json:"text"`
	30	+	Rect image.Rectangle `json:"rect"`
	31	+	Confidence float64 `json:"confidence"`
	32	+	}
	33	+
	34	+	var languageWarning sync.Once
	35	+	var failureWarning sync.Once
	36	+
	37	+	func ocrLanguage(ctx context.Context, binary string) (string, error) {
	38	+	dir := os.Getenv("TESSDATA_PREFIX")
	39	+	if dir == "" {
	40	+	dir = "/usr/share/tessdata"
	41	+	}
	42	+	if _, err := os.Stat(filepath.Join(dir, "eng.traineddata")); err == nil {
	43	+	return "eng", nil
	44	+	}
	45	+	out, err := exec.CommandContext(ctx, binary, "--list-langs").Output()
	46	+	if err != nil {
	47	+	return "", err
	48	+	}
	49	+	for _, line := range strings.Split(string(out), "\n")[1:] {
	50	+	lang := strings.TrimSpace(line)
	51	+	if lang != "" && lang != "osd" {
	52	+	languageWarning.Do(func() {
	53	+	log := logging.For(ctx, "hints")
	54	+	log.Warn().Str("language", lang).Msg("English OCR data missing; install tesseract-data-eng")
	55	+	})
	56	+	return lang, nil
	57	+	}
	58	+	}
	59	+	return "", fmt.Errorf("no OCR language available")
	60	+	}
	61	+
	62	+	func parseTSV(data []byte, offset image.Point) ([]OCRWord, error) {
	63	+	var words []OCRWord
	64	+	scanner := bufio.NewScanner(bytes.NewReader(data))
	65	+	scanner.Buffer(make([]byte, 4096), 1024*1024)
	66	+	for scanner.Scan() {
	67	+	fields := strings.SplitN(scanner.Text(), "\t", 12)
	68	+	if len(fields) != 12 \|\| fields[0] != "5" {
	69	+	continue
	70	+	}
	71	+	var values [4]int
	72	+	for i := range values {
	73	+	v, err := strconv.Atoi(fields[6+i])
	74	+	if err != nil {
	75	+	return nil, fmt.Errorf("OCR TSV coordinate: %w", err)
	76	+	}
	77	+	values[i] = v
	78	+	}
	79	+	conf, err := strconv.ParseFloat(fields[10], 64)
	80	+	if err != nil {
	81	+	return nil, fmt.Errorf("OCR TSV confidence: %w", err)
	82	+	}
	83	+	if values[2] <= 0 \|\| values[3] <= 0 \|\| strings.TrimSpace(fields[11]) == "" {
	84	+	continue
	85	+	}
	86	+	r := image.Rect(values[0], values[1], values[0]+values[2], values[1]+values[3]).Add(offset)
	87	+	words = append(words, OCRWord{Text: fields[11], Rect: r, Confidence: conf})
	88	+	}
	89	+	return words, scanner.Err()
	90	+	}
	91	+
	92	+	func readWords(ctx context.Context, img *image.RGBA) ([]OCRWord, int, error) {
	93	+	binary, err := exec.LookPath("tesseract")
	94	+	if err != nil {
	95	+	return nil, 0, err
	96	+	}
	97	+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
	98	+	defer cancel()
	99	+	lang, err := ocrLanguage(ctx, binary)
	100	+	if err != nil {
	101	+	return nil, 0, err
	102	+	}
	103	+	n := min(runtime.GOMAXPROCS(0), 8, img.Bounds().Dy())
	104	+	results := make([][]OCRWord, n)
	105	+	errs := make([]error, n)
	106	+	var wg sync.WaitGroup
	107	+	for i := range n {
	108	+	wg.Go(func() {
	109	+	top := max(0, i*img.Bounds().Dy()/n-40)
	110	+	bottom := min(img.Bounds().Dy(), (i+1)*img.Bounds().Dy()/n+40)
	111	+	crop := img.SubImage(image.Rect(0, top, img.Bounds().Dx(), bottom))
	112	+	results[i], errs[i] = recognizeBand(ctx, binary, lang, crop)
	113	+	if errs[i] != nil {
	114	+	return
	115	+	}
	116	+	heights := make([]int, 0, len(results[i]))
	117	+	for _, word := range results[i] {
	118	+	if len(word.Text) >= 3 {
	119	+	heights = append(heights, word.Rect.Dy())
	120	+	}
	121	+	}
	122	+	sort.Ints(heights)
	123	+	if len(heights) > 0 && heights[len(heights)/2] < 12 {
	124	+	larger := upscaleBand(crop)
	125	+	results[i], errs[i] = recognizeBand(ctx, binary, lang, larger)
	126	+	for j := range results[i] {
	127	+	r := results[i][j].Rect
	128	+	results[i][j].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
	129	+	}
	130	+	}
	131	+	for j := range results[i] {
	132	+	results[i][j].Rect = results[i][j].Rect.Add(image.Pt(0, top))
	133	+	}
	134	+	})
	135	+	}
	136	+	wg.Wait()
	137	+	var words []OCRWord
	138	+	for i := range n {
	139	+	if errs[i] != nil {
	140	+	return nil, n, errs[i]
	141	+	}
	142	+	for _, word := range results[i] {
	143	+	// Overlap supplies context, but each word belongs to exactly one band.
	144	+	center := (word.Rect.Min.Y + word.Rect.Max.Y) / 2
	145	+	if center < iimg.Bounds().Dy()/n \|\| center >= (i+1)img.Bounds().Dy()/n {
	146	+	continue
	147	+	}
	148	+	duplicate := false
	149	+	for _, old := range words {
	150	+	if strings.EqualFold(word.Text, old.Text) && overlap(word.Rect, old.Rect) > .5 {
	151	+	duplicate = true
	152	+	break
	153	+	}
	154	+	}
	155	+	if !duplicate {
	156	+	words = append(words, word)
	157	+	}
	158	+	}
	159	+	}
	160	+	return words, n, nil
	161	+	}
	162	+
	163	+	var domainPattern = regexp.MustCompile(`(?i)^(?:[a-z0-9](?:[a-z0-9-][a-z0-9])?\.)+([a-z]{2,24})(?::[0-9]+)?(?:[/?#]\S)?$`)
	164	+	var schemePattern = regexp.MustCompile(`(?i)^(?:https?\|htps\|file\|ftp):/{1,2}\S+$`)
	165	+	var emailPattern = regexp.MustCompile(`(?i)^[a-z0-9.!#$%&'*+/=?^_` + "`" + `{\|}~-]+@(?:[a-z0-9-]+\.)+[a-z]{2,24}$`)
	166	+	var fileExtensions = strings.Fields("go md txt toml json yaml yml rs py js ts sh c h png jpg log conf lock mod sum html css")
	167	+
	168	+	func isTextLink(text string) bool {
	169	+	text = strings.Trim(text, "()[]<>.,;:'\"")
	170	+	if schemePattern.MatchString(text) \|\| (strings.HasPrefix(strings.ToLower(text), "www.") && len(text) > 4) \|\| emailPattern.MatchString(text) {
	171	+	return true
	172	+	}
	173	+	host := text
	174	+	if h, _, err := net.SplitHostPort(text); err == nil {
	175	+	host = h
	176	+	}
	177	+	if ip := net.ParseIP(host); ip != nil && ip.To4() != nil {
	178	+	return true
	179	+	}
	180	+	m := domainPattern.FindStringSubmatch(text)
	181	+	if m == nil {
	182	+	return false
	183	+	}
	184	+	for _, ext := range fileExtensions {
	185	+	if strings.EqualFold(m[1], ext) {
	186	+	return false
	187	+	}
	188	+	}
	189	+	return true
	190	+	}
	191	+
	192	+	func textLinks(words []OCRWord, scale float64) []ports.HintTarget {
	193	+	if scale <= 0 {
	194	+	scale = 1
	195	+	}
	196	+	words = append([]OCRWord(nil), words...)
	197	+	sort.Slice(words, func(i, j int) bool {
	198	+	if words[i].Rect.Min.Y != words[j].Rect.Min.Y {
	199	+	return words[i].Rect.Min.Y < words[j].Rect.Min.Y
	200	+	}
	201	+	return words[i].Rect.Min.X < words[j].Rect.Min.X
	202	+	})
	203	+	var out []ports.HintTarget
	204	+	used := make([]bool, len(words))
	205	+	for i, word := range words {
	206	+	if used[i] {
	207	+	continue
	208	+	}
	209	+	text, rect := word.Text, word.Rect
	210	+	// Grow only touching words on the same baseline; ordinary prose is not joined.
	211	+	for {
	212	+	next := -1
	213	+	for j, candidate := range words {
	214	+	if j == i \|\| used[j] \|\| candidate.Rect.Min.X < rect.Max.X-2 {
	215	+	continue
	216	+	}
	217	+	gap := candidate.Rect.Min.X - rect.Max.X
	218	+	if gap > max(3, min(rect.Dy(), candidate.Rect.Dy())/5) {
	219	+	continue
	220	+	}
	221	+	if abs(candidate.Rect.Max.Y-rect.Max.Y) > max(3, rect.Dy()/3) {
	222	+	continue
	223	+	}
	224	+	if next < 0 \|\| candidate.Rect.Min.X < words[next].Rect.Min.X {
	225	+	next = j
	226	+	}
	227	+	}
	228	+	if next < 0 {
	229	+	break
	230	+	}
	231	+	text += words[next].Text
	232	+	rect = rect.Union(words[next].Rect)
	233	+	used[next] = true
	234	+	}
	235	+	if !isTextLink(text) {
	236	+	continue
	237	+	}
	238	+	x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
	239	+	right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
	240	+	out = append(out, ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
	241	+	}
	242	+	return out
	243	+	}
	244	+
	245	+	func abs(x int) int {
	246	+	if x < 0 {
	247	+	return -x
	248	+	}
	249	+	return x
	250	+	}
	251	+	func overlap(a, b image.Rectangle) float64 {
	252	+	intersection := a.Intersect(b)
	253	+	area := min(a.Dx()a.Dy(), b.Dx()b.Dy())
	254	+	if intersection.Empty() \|\| area <= 0 {
	255	+	return 0
	256	+	}
	257	+	return float64(intersection.Dx()*intersection.Dy()) / float64(area)
	258	+	}
	259	+	func targetRect(t ports.HintTarget) image.Rectangle {
	260	+	return image.Rect(t.Rect.X, t.Rect.Y, t.Rect.X+t.Rect.W, t.Rect.Y+t.Rect.H)
	261	+	}
	262	+	func recognizeBand(ctx context.Context, binary, lang string, img image.Image) ([]OCRWord, error) {
	263	+	return recognizeImage(ctx, binary, lang, img, "11", false)
	264	+	}
	265	+
	266	+	func recognizeImage(ctx context.Context, binary, lang string, img image.Image, psm string, normalized bool) ([]OCRWord, error) {
	267	+	var input bytes.Buffer
	268	+	enc := png.Encoder{CompressionLevel: png.BestSpeed}
	269	+	if err := enc.Encode(&input, img); err != nil {
	270	+	return nil, err
	271	+	}
	272	+	args := []string{"stdin", "stdout", "--psm", psm, "-l", lang}
	273	+	if normalized {
	274	+	args = append(args, "-c", "tessedit_do_invert=0")
	275	+	}
	276	+	args = append(args, "tsv")
	277	+	cmd := exec.CommandContext(ctx, binary, args...)
	278	+	cmd.Env = append(os.Environ(), "OMP_THREAD_LIMIT=1")
	279	+	cmd.Stdin = &input
	280	+	var stderr bytes.Buffer
	281	+	cmd.Stderr = &stderr
	282	+	output, err := cmd.Output()
	283	+	if err != nil {
	284	+	return nil, fmt.Errorf("tesseract: %w: %s", err, stderr.String())
	285	+	}
	286	+	return parseTSV(output, image.Point{})
	287	+	}
	288	+
	289	+	func upscaleBand(src image.Image) *image.RGBA {
	290	+	b := src.Bounds()
	291	+	dst := image.NewRGBA(image.Rect(0, 0, b.Dx()2, b.Dy()2))
	292	+	for y := 0; y < dst.Bounds().Dy(); y++ {
	293	+	for x := 0; x < dst.Bounds().Dx(); x++ {
	294	+	dst.Set(x, y, src.At(b.Min.X+x/2, b.Min.Y+y/2))
	295	+	}
	296	+	}
	297	+	return dst
	298	+	}

A internal/adapters/hints/ocr_test.go +141 -0

@@ -0,0 +1,141 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/draw"
	7	+	"image/png"
	8	+	"os"
	9	+	"os/exec"
	10	+	"strings"
	11	+	"testing"
	12	+	"time"
	13	+
	14	+	"github.com/bnema/neferwl/internal/ports"
	15	+	)
	16	+
	17	+	func TestTextLink(t *testing.T) {
	18	+	yes := []string{"ermite.bnema.dev", "google.fr", "example.org/path?q=1", "https.google.fr", "https://example.org", "file:///tmp/main.go", "ftp://host/file", "www.main.go", "user@example.org", "127.0.0.1", "192.168.1.1:8080", "(https://example.org),", "htps://google.fr", "https:/example.org"}
	19	+	no := []string{"ermite-runner", "main.go", "README.md", "v1.2.3", "1.5", "e.g.", "report.txt", "source.rs", "alpha", "999.999.999.999"}
	20	+	for _, s := range yes {
	21	+	t.Run(s, func(t *testing.T) {
	22	+	if !isTextLink(s) {
	23	+	t.Errorf("not accepted: %s", s)
	24	+	}
	25	+	})
	26	+	}
	27	+	for _, s := range no {
	28	+	t.Run(s, func(t *testing.T) {
	29	+	if isTextLink(s) {
	30	+	t.Errorf("incorrectly accepted: %s", s)
	31	+	}
	32	+	})
	33	+	}
	34	+	}
	35	+	func TestParseTSV(t *testing.T) {
	36	+	data := "level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext\n5\t1\t1\t1\t1\t1\t10\t20\t30\t12\t91.5\texample.org\n"
	37	+	words, err := parseTSV([]byte(data), image.Pt(0, 40))
	38	+	if err != nil \|\| len(words) != 1 {
	39	+	t.Fatalf("%v %v", words, err)
	40	+	}
	41	+	if words[0].Rect != image.Rect(10, 60, 40, 72) \|\| words[0].Confidence != 91.5 {
	42	+	t.Fatal(words)
	43	+	}
	44	+	_, err = parseTSV([]byte(strings.ReplaceAll(data, "\t10\t", "\tbad\t")), image.Point{})
	45	+	if err == nil {
	46	+	t.Fatal("expected malformed coordinate error")
	47	+	}
	48	+	}
	49	+	func TestJoinedOCRLink(t *testing.T) {
	50	+	words := []OCRWord{{Text: "https:/", Rect: image.Rect(20, 20, 90, 40)}, {Text: "/example.org", Rect: image.Rect(91, 20, 200, 40)}}
	51	+	got := textLinks(words, 2)
	52	+	if len(got) != 1 \|\| got[0].Rect != (ports.Rect{X: 10, Y: 10, W: 90, H: 10}) {
	53	+	t.Fatal(got)
	54	+	}
	55	+	}
	56	+	func TestRealOCR(t *testing.T) {
	57	+	if _, err := exec.LookPath("tesseract"); err != nil {
	58	+	t.Skip("tesseract not on PATH")
	59	+	}
	60	+	for _, name := range []string{"browser", "terminal"} {
	61	+	t.Run(name, func(t *testing.T) {
	62	+	f, err := os.Open("testdata/" + name + ".png")
	63	+	if err != nil {
	64	+	t.Fatal(err)
	65	+	}
	66	+	defer f.Close()
	67	+	src, err := png.Decode(f)
	68	+	if err != nil {
	69	+	t.Fatal(err)
	70	+	}
	71	+	img := image.NewRGBA(src.Bounds())
	72	+	draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
	73	+	start := time.Now()
	74	+	words, bands, err := readWords(context.Background(), img)
	75	+	if err != nil {
	76	+	t.Fatal(err)
	77	+	}
	78	+	links := textLinks(words, 1)
	79	+	t.Logf("%s %dx%d ocr_ms=%.1f words=%d links=%d bands=%d", name, img.Bounds().Dx(), img.Bounds().Dy(), ms(time.Since(start)), len(words), len(links), bands)
	80	+	for _, w := range words {
	81	+	t.Logf("%s %v", w.Text, w.Rect)
	82	+	}
	83	+	if name == "terminal" {
	84	+	for _, r := range []image.Rectangle{image.Rect(160, 65, 380, 85), image.Rect(140, 90, 490, 110), image.Rect(95, 115, 630, 138), image.Rect(120, 140, 330, 163), image.Rect(385, 140, 740, 163)} {
	85	+	found := false
	86	+	for _, l := range links {
	87	+	if overlap(targetRect(l), r) > .5 {
	88	+	found = true
	89	+	}
	90	+	}
	91	+	if !found {
	92	+	t.Errorf("missing terminal URL near %v", r)
	93	+	}
	94	+	}
	95	+	if len(links) != 5 {
	96	+	t.Errorf("want 5 terminal links, got %d: %v", len(links), links)
	97	+	}
	98	+	} else if len(links) == 0 {
	99	+	t.Error("no browser links")
	100	+	}
	101	+	})
	102	+	}
	103	+	}
	104	+
	105	+	func TestSmallGlyphOCR(t *testing.T) {
	106	+	binary, err := exec.LookPath("tesseract")
	107	+	if err != nil {
	108	+	t.Skip("tesseract not on PATH")
	109	+	}
	110	+	f, err := os.Open("testdata/terminal.png")
	111	+	if err != nil {
	112	+	t.Fatal(err)
	113	+	}
	114	+	defer f.Close()
	115	+	src, err := png.Decode(f)
	116	+	if err != nil {
	117	+	t.Fatal(err)
	118	+	}
	119	+	small := image.NewRGBA(image.Rect(0, 0, 640, 200))
	120	+	for y := 0; y < 200; y++ {
	121	+	for x := 0; x < 640; x++ {
	122	+	small.Set(x, y, src.At(x2, y2))
	123	+	}
	124	+	}
	125	+	lang, err := ocrLanguage(context.Background(), binary)
	126	+	if err != nil {
	127	+	t.Fatal(err)
	128	+	}
	129	+	for _, factor := range []int{1, 2} {
	130	+	var img image.Image = small
	131	+	if factor == 2 {
	132	+	img = upscaleBand(small)
	133	+	}
	134	+	start := time.Now()
	135	+	words, err := recognizeBand(context.Background(), binary, lang, img)
	136	+	if err != nil {
	137	+	t.Fatal(err)
	138	+	}
	139	+	t.Logf("glyph experiment %dx: %.1f ms, words=%d links=%d", factor, ms(time.Since(start)), len(words), len(textLinks(words, 1)))
	140	+	}
	141	+	}

@@ -0,0 +1,141 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/draw"
	7	+	"image/png"
	8	+	"os"
	9	+	"os/exec"
	10	+	"strings"
	11	+	"testing"
	12	+	"time"
	13	+
	14	+	"github.com/bnema/neferwl/internal/ports"
	15	+	)
	16	+
	17	+	func TestTextLink(t *testing.T) {
	18	+	yes := []string{"ermite.bnema.dev", "google.fr", "example.org/path?q=1", "https.google.fr", "https://example.org", "file:///tmp/main.go", "ftp://host/file", "www.main.go", "user@example.org", "127.0.0.1", "192.168.1.1:8080", "(https://example.org),", "htps://google.fr", "https:/example.org"}
	19	+	no := []string{"ermite-runner", "main.go", "README.md", "v1.2.3", "1.5", "e.g.", "report.txt", "source.rs", "alpha", "999.999.999.999"}
	20	+	for _, s := range yes {
	21	+	t.Run(s, func(t *testing.T) {
	22	+	if !isTextLink(s) {
	23	+	t.Errorf("not accepted: %s", s)
	24	+	}
	25	+	})
	26	+	}
	27	+	for _, s := range no {
	28	+	t.Run(s, func(t *testing.T) {
	29	+	if isTextLink(s) {
	30	+	t.Errorf("incorrectly accepted: %s", s)
	31	+	}
	32	+	})
	33	+	}
	34	+	}
	35	+	func TestParseTSV(t *testing.T) {
	36	+	data := "level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext\n5\t1\t1\t1\t1\t1\t10\t20\t30\t12\t91.5\texample.org\n"
	37	+	words, err := parseTSV([]byte(data), image.Pt(0, 40))
	38	+	if err != nil \|\| len(words) != 1 {
	39	+	t.Fatalf("%v %v", words, err)
	40	+	}
	41	+	if words[0].Rect != image.Rect(10, 60, 40, 72) \|\| words[0].Confidence != 91.5 {
	42	+	t.Fatal(words)
	43	+	}
	44	+	_, err = parseTSV([]byte(strings.ReplaceAll(data, "\t10\t", "\tbad\t")), image.Point{})
	45	+	if err == nil {
	46	+	t.Fatal("expected malformed coordinate error")
	47	+	}
	48	+	}
	49	+	func TestJoinedOCRLink(t *testing.T) {
	50	+	words := []OCRWord{{Text: "https:/", Rect: image.Rect(20, 20, 90, 40)}, {Text: "/example.org", Rect: image.Rect(91, 20, 200, 40)}}
	51	+	got := textLinks(words, 2)
	52	+	if len(got) != 1 \|\| got[0].Rect != (ports.Rect{X: 10, Y: 10, W: 90, H: 10}) {
	53	+	t.Fatal(got)
	54	+	}
	55	+	}
	56	+	func TestRealOCR(t *testing.T) {
	57	+	if _, err := exec.LookPath("tesseract"); err != nil {
	58	+	t.Skip("tesseract not on PATH")
	59	+	}
	60	+	for _, name := range []string{"browser", "terminal"} {
	61	+	t.Run(name, func(t *testing.T) {
	62	+	f, err := os.Open("testdata/" + name + ".png")
	63	+	if err != nil {
	64	+	t.Fatal(err)
	65	+	}
	66	+	defer f.Close()
	67	+	src, err := png.Decode(f)
	68	+	if err != nil {
	69	+	t.Fatal(err)
	70	+	}
	71	+	img := image.NewRGBA(src.Bounds())
	72	+	draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
	73	+	start := time.Now()
	74	+	words, bands, err := readWords(context.Background(), img)
	75	+	if err != nil {
	76	+	t.Fatal(err)
	77	+	}
	78	+	links := textLinks(words, 1)
	79	+	t.Logf("%s %dx%d ocr_ms=%.1f words=%d links=%d bands=%d", name, img.Bounds().Dx(), img.Bounds().Dy(), ms(time.Since(start)), len(words), len(links), bands)
	80	+	for _, w := range words {
	81	+	t.Logf("%s %v", w.Text, w.Rect)
	82	+	}
	83	+	if name == "terminal" {
	84	+	for _, r := range []image.Rectangle{image.Rect(160, 65, 380, 85), image.Rect(140, 90, 490, 110), image.Rect(95, 115, 630, 138), image.Rect(120, 140, 330, 163), image.Rect(385, 140, 740, 163)} {
	85	+	found := false
	86	+	for _, l := range links {
	87	+	if overlap(targetRect(l), r) > .5 {
	88	+	found = true
	89	+	}
	90	+	}
	91	+	if !found {
	92	+	t.Errorf("missing terminal URL near %v", r)
	93	+	}
	94	+	}
	95	+	if len(links) != 5 {
	96	+	t.Errorf("want 5 terminal links, got %d: %v", len(links), links)
	97	+	}
	98	+	} else if len(links) == 0 {
	99	+	t.Error("no browser links")
	100	+	}
	101	+	})
	102	+	}
	103	+	}
	104	+
	105	+	func TestSmallGlyphOCR(t *testing.T) {
	106	+	binary, err := exec.LookPath("tesseract")
	107	+	if err != nil {
	108	+	t.Skip("tesseract not on PATH")
	109	+	}
	110	+	f, err := os.Open("testdata/terminal.png")
	111	+	if err != nil {
	112	+	t.Fatal(err)
	113	+	}
	114	+	defer f.Close()
	115	+	src, err := png.Decode(f)
	116	+	if err != nil {
	117	+	t.Fatal(err)
	118	+	}
	119	+	small := image.NewRGBA(image.Rect(0, 0, 640, 200))
	120	+	for y := 0; y < 200; y++ {
	121	+	for x := 0; x < 640; x++ {
	122	+	small.Set(x, y, src.At(x2, y2))
	123	+	}
	124	+	}
	125	+	lang, err := ocrLanguage(context.Background(), binary)
	126	+	if err != nil {
	127	+	t.Fatal(err)
	128	+	}
	129	+	for _, factor := range []int{1, 2} {
	130	+	var img image.Image = small
	131	+	if factor == 2 {
	132	+	img = upscaleBand(small)
	133	+	}
	134	+	start := time.Now()
	135	+	words, err := recognizeBand(context.Background(), binary, lang, img)
	136	+	if err != nil {
	137	+	t.Fatal(err)
	138	+	}
	139	+	t.Logf("glyph experiment %dx: %.1f ms, words=%d links=%d", factor, ms(time.Since(start)), len(words), len(textLinks(words, 1)))
	140	+	}
	141	+	}

A internal/adapters/hints/strips_test.go +124 -0

@@ -0,0 +1,124 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/color"
	7	+	"image/draw"
	8	+	"os/exec"
	9	+	"runtime"
	10	+	"sort"
	11	+	"sync"
	12	+	"time"
	13	+	)
	14	+
	15	+	type stripOffset struct {
	16	+	source, packed image.Rectangle
	17	+	factor int
	18	+	}
	19	+	type ocrStrip struct {
	20	+	img *image.Gray
	21	+	offsets []stripOffset
	22	+	}
	23	+
	24	+	func packLines(src *image.RGBA, lines []image.Rectangle, workers int, normalize bool) []ocrStrip {
	25	+	n := min(workers, len(lines))
	26	+	if n == 0 {
	27	+	return nil
	28	+	}
	29	+	groups := make([][]image.Rectangle, n)
	30	+	areas := make([]int, n)
	31	+	lines = append([]image.Rectangle(nil), lines...)
	32	+	sort.Slice(lines, func(i, j int) bool { return lines[i].Dx()lines[i].Dy() > lines[j].Dx()lines[j].Dy() })
	33	+	for _, r := range lines {
	34	+	best := 0
	35	+	for i := range n {
	36	+	if areas[i] < areas[best] {
	37	+	best = i
	38	+	}
	39	+	}
	40	+	groups[best] = append(groups[best], r)
	41	+	areas[best] += r.Dx() * r.Dy()
	42	+	}
	43	+	strips := make([]ocrStrip, n)
	44	+	for i, group := range groups {
	45	+	width, height := 0, 8
	46	+	for _, r := range group {
	47	+	factor := 1
	48	+	if r.Dy() < 12 {
	49	+	factor = 2
	50	+	}
	51	+	crop := r.Inset(-4).Intersect(src.Bounds())
	52	+	packed := image.Rect(4, height, 4+crop.Dx()factor, height+crop.Dy()factor)
	53	+	strips[i].offsets = append(strips[i].offsets, stripOffset{crop, packed, factor})
	54	+	width = max(width, packed.Max.X+4)
	55	+	height = packed.Max.Y + 8
	56	+	}
	57	+	dst := image.NewGray(image.Rect(0, 0, width, height))
	58	+	draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
	59	+	for _, off := range strips[i].offsets {
	60	+	// The padded perimeter estimates the background without depending on text colour.
	61	+	var sum, count int
	62	+	r := off.source
	63	+	for x := r.Min.X; x < r.Max.X; x++ {
	64	+	sum += int(color.GrayModel.Convert(src.At(x, r.Min.Y)).(color.Gray).Y)
	65	+	count++
	66	+	}
	67	+	invert := normalize && count > 0 && sum/count < 128
	68	+	for y := off.packed.Min.Y; y < off.packed.Max.Y; y++ {
	69	+	for x := off.packed.Min.X; x < off.packed.Max.X; x++ {
	70	+	v := color.GrayModel.Convert(src.At(r.Min.X+(x-off.packed.Min.X)/off.factor, r.Min.Y+(y-off.packed.Min.Y)/off.factor)).(color.Gray).Y
	71	+	if invert {
	72	+	v = 255 - v
	73	+	}
	74	+	dst.SetGray(x, y, color.Gray{Y: v})
	75	+	}
	76	+	}
	77	+	}
	78	+	strips[i].img = dst
	79	+	}
	80	+	return strips
	81	+	}
	82	+
	83	+	func readStrips(ctx context.Context, img *image.RGBA, lines []image.Rectangle, workers int, normalize bool) ([]OCRWord, int, error) {
	84	+	binary, err := exec.LookPath("tesseract")
	85	+	if err != nil {
	86	+	return nil, 0, err
	87	+	}
	88	+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
	89	+	defer cancel()
	90	+	lang, err := ocrLanguage(ctx, binary)
	91	+	if err != nil {
	92	+	return nil, 0, err
	93	+	}
	94	+	strips := packLines(img, lines, min(workers, runtime.GOMAXPROCS(0), 16), normalize)
	95	+	results := make([][]OCRWord, len(strips))
	96	+	errs := make([]error, len(strips))
	97	+	var wg sync.WaitGroup
	98	+	for i, s := range strips {
	99	+	wg.Go(func() {
	100	+	words, e := recognizeImage(ctx, binary, lang, s.img, "6", normalize)
	101	+	errs[i] = e
	102	+	for _, w := range words {
	103	+	center := image.Pt((w.Rect.Min.X+w.Rect.Max.X)/2, (w.Rect.Min.Y+w.Rect.Max.Y)/2)
	104	+	for _, off := range s.offsets {
	105	+	if center.In(off.packed) {
	106	+	r := w.Rect.Sub(off.packed.Min)
	107	+	w.Rect = image.Rect(r.Min.X/off.factor, r.Min.Y/off.factor, (r.Max.X+off.factor-1)/off.factor, (r.Max.Y+off.factor-1)/off.factor).Add(off.source.Min)
	108	+	results[i] = append(results[i], w)
	109	+	break
	110	+	}
	111	+	}
	112	+	}
	113	+	})
	114	+	}
	115	+	wg.Wait()
	116	+	var words []OCRWord
	117	+	for i := range strips {
	118	+	if errs[i] != nil {
	119	+	return nil, len(strips), errs[i]
	120	+	}
	121	+	words = append(words, results[i]...)
	122	+	}
	123	+	return words, len(strips), nil
	124	+	}

@@ -0,0 +1,124 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/color"
	7	+	"image/draw"
	8	+	"os/exec"
	9	+	"runtime"
	10	+	"sort"
	11	+	"sync"
	12	+	"time"
	13	+	)
	14	+
	15	+	type stripOffset struct {
	16	+	source, packed image.Rectangle
	17	+	factor int
	18	+	}
	19	+	type ocrStrip struct {
	20	+	img *image.Gray
	21	+	offsets []stripOffset
	22	+	}
	23	+
	24	+	func packLines(src *image.RGBA, lines []image.Rectangle, workers int, normalize bool) []ocrStrip {
	25	+	n := min(workers, len(lines))
	26	+	if n == 0 {
	27	+	return nil
	28	+	}
	29	+	groups := make([][]image.Rectangle, n)
	30	+	areas := make([]int, n)
	31	+	lines = append([]image.Rectangle(nil), lines...)
	32	+	sort.Slice(lines, func(i, j int) bool { return lines[i].Dx()lines[i].Dy() > lines[j].Dx()lines[j].Dy() })
	33	+	for _, r := range lines {
	34	+	best := 0
	35	+	for i := range n {
	36	+	if areas[i] < areas[best] {
	37	+	best = i
	38	+	}
	39	+	}
	40	+	groups[best] = append(groups[best], r)
	41	+	areas[best] += r.Dx() * r.Dy()
	42	+	}
	43	+	strips := make([]ocrStrip, n)
	44	+	for i, group := range groups {
	45	+	width, height := 0, 8
	46	+	for _, r := range group {
	47	+	factor := 1
	48	+	if r.Dy() < 12 {
	49	+	factor = 2
	50	+	}
	51	+	crop := r.Inset(-4).Intersect(src.Bounds())
	52	+	packed := image.Rect(4, height, 4+crop.Dx()factor, height+crop.Dy()factor)
	53	+	strips[i].offsets = append(strips[i].offsets, stripOffset{crop, packed, factor})
	54	+	width = max(width, packed.Max.X+4)
	55	+	height = packed.Max.Y + 8
	56	+	}
	57	+	dst := image.NewGray(image.Rect(0, 0, width, height))
	58	+	draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
	59	+	for _, off := range strips[i].offsets {
	60	+	// The padded perimeter estimates the background without depending on text colour.
	61	+	var sum, count int
	62	+	r := off.source
	63	+	for x := r.Min.X; x < r.Max.X; x++ {
	64	+	sum += int(color.GrayModel.Convert(src.At(x, r.Min.Y)).(color.Gray).Y)
	65	+	count++
	66	+	}
	67	+	invert := normalize && count > 0 && sum/count < 128
	68	+	for y := off.packed.Min.Y; y < off.packed.Max.Y; y++ {
	69	+	for x := off.packed.Min.X; x < off.packed.Max.X; x++ {
	70	+	v := color.GrayModel.Convert(src.At(r.Min.X+(x-off.packed.Min.X)/off.factor, r.Min.Y+(y-off.packed.Min.Y)/off.factor)).(color.Gray).Y
	71	+	if invert {
	72	+	v = 255 - v
	73	+	}
	74	+	dst.SetGray(x, y, color.Gray{Y: v})
	75	+	}
	76	+	}
	77	+	}
	78	+	strips[i].img = dst
	79	+	}
	80	+	return strips
	81	+	}
	82	+
	83	+	func readStrips(ctx context.Context, img *image.RGBA, lines []image.Rectangle, workers int, normalize bool) ([]OCRWord, int, error) {
	84	+	binary, err := exec.LookPath("tesseract")
	85	+	if err != nil {
	86	+	return nil, 0, err
	87	+	}
	88	+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
	89	+	defer cancel()
	90	+	lang, err := ocrLanguage(ctx, binary)
	91	+	if err != nil {
	92	+	return nil, 0, err
	93	+	}
	94	+	strips := packLines(img, lines, min(workers, runtime.GOMAXPROCS(0), 16), normalize)
	95	+	results := make([][]OCRWord, len(strips))
	96	+	errs := make([]error, len(strips))
	97	+	var wg sync.WaitGroup
	98	+	for i, s := range strips {
	99	+	wg.Go(func() {
	100	+	words, e := recognizeImage(ctx, binary, lang, s.img, "6", normalize)
	101	+	errs[i] = e
	102	+	for _, w := range words {
	103	+	center := image.Pt((w.Rect.Min.X+w.Rect.Max.X)/2, (w.Rect.Min.Y+w.Rect.Max.Y)/2)
	104	+	for _, off := range s.offsets {
	105	+	if center.In(off.packed) {
	106	+	r := w.Rect.Sub(off.packed.Min)
	107	+	w.Rect = image.Rect(r.Min.X/off.factor, r.Min.Y/off.factor, (r.Max.X+off.factor-1)/off.factor, (r.Max.Y+off.factor-1)/off.factor).Add(off.source.Min)
	108	+	results[i] = append(results[i], w)
	109	+	break
	110	+	}
	111	+	}
	112	+	}
	113	+	})
	114	+	}
	115	+	wg.Wait()
	116	+	var words []OCRWord
	117	+	for i := range strips {
	118	+	if errs[i] != nil {
	119	+	return nil, len(strips), errs[i]
	120	+	}
	121	+	words = append(words, results[i]...)
	122	+	}
	123	+	return words, len(strips), nil
	124	+	}

A internal/adapters/hints/testdata/terminal.png +0 -0 binary file: no text diff

binary file: no text diff

M internal/app/hintdebug.go +2 -2


@@ -13,13 +13,13 @@ import (
13	13		// hintOptions builds the hint worker options: the accessibility source, and,
14	14		// when NEFERWL_HINT_DEBUG is "1", the dump directory of scanned frames,
15	15		// $XDG_STATE_HOME/neferwl/hints. Without the accessibility bus the worker
16		-	// scans pixels only.
	16	+	// reads uncovered windows with OCR only.
17	17		func hintOptions(ctx context.Context) hints.Options {
18	18		opts := hintDump(ctx)
19	19		src, err := atspi.New(ctx)
20	20		if err != nil {
21	21		log := logging.For(ctx, "atspi")
22		-	log.Warn().Err(err).Msg("accessibility hint targets unavailable, using pixels only")
	22	+	log.Warn().Err(err).Msg("accessibility hint targets unavailable, using OCR only")
23	23		return opts
24	24		}
25	25		context.AfterFunc(ctx, func() { _ = src.Close() })


@@ -13,13 +13,13 @@ import (
13		// hintOptions builds the hint worker options: the accessibility source, and,	13		// hintOptions builds the hint worker options: the accessibility source, and,
14		// when NEFERWL_HINT_DEBUG is "1", the dump directory of scanned frames,	14		// when NEFERWL_HINT_DEBUG is "1", the dump directory of scanned frames,
15		// $XDG_STATE_HOME/neferwl/hints. Without the accessibility bus the worker	15		// $XDG_STATE_HOME/neferwl/hints. Without the accessibility bus the worker
16	-	// scans pixels only.	16	+	// reads uncovered windows with OCR only.
17		func hintOptions(ctx context.Context) hints.Options {	17		func hintOptions(ctx context.Context) hints.Options {
18		opts := hintDump(ctx)	18		opts := hintDump(ctx)
19		src, err := atspi.New(ctx)	19		src, err := atspi.New(ctx)
20		if err != nil {	20		if err != nil {
21		log := logging.For(ctx, "atspi")	21		log := logging.For(ctx, "atspi")
22	-	log.Warn().Err(err).Msg("accessibility hint targets unavailable, using pixels only")	22	+	log.Warn().Err(err).Msg("accessibility hint targets unavailable, using OCR only")
23		return opts	23		return opts
24		}	24		}
25		context.AfterFunc(ctx, func() { _ = src.Close() })	25		context.AfterFunc(ctx, func() { _ = src.Close() })