feat(hints): find hint targets with the accessibility tree and OCR

commit: 463ec0a837d15732d6561b8b2a1edb214e8cef56
author: brice · 3d ago · 2026-10-19 16:03 UTC
committer: brice · 3d ago · 2026-10-19 17:36 UTC
parent: d9f6a44
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

17 files · +3089 -0

A internal/adapters/hints/benchmark_test.go +81 -0

@@ -0,0 +1,81 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/draw"
	7	+	"image/png"
	8	+	"os"
	9	+	"os/exec"
	10	+	"path/filepath"
	11	+	"strconv"
	12	+	"testing"
	13	+	"time"
	14	+	)
	15	+
	16	+	// TestFrameBenchmark logs counts and timings, never filenames or OCR text.
	17	+	func TestFrameBenchmark(t *testing.T) {
	18	+	dir := os.Getenv("NEFERWL_HINT_BENCH_DIR")
	19	+	if dir == "" {
	20	+	t.Skip("set NEFERWL_HINT_BENCH_DIR to a local PNG directory")
	21	+	}
	22	+	if _, err := exec.LookPath("tesseract"); err != nil {
	23	+	t.Skip("tesseract not on PATH")
	24	+	}
	25	+	files, err := filepath.Glob(filepath.Join(dir, "*.png"))
	26	+	if err != nil \|\| len(files) == 0 {
	27	+	t.Fatal("no PNG frames")
	28	+	}
	29	+	// Settings under test: NEFERWL_HINT_BENCH_WORKERS=<n>, NEFERWL_HINT_BENCH_PNG=speed\|none.
	30	+	if v, _ := strconv.Atoi(os.Getenv("NEFERWL_HINT_BENCH_WORKERS")); v > 0 {
	31	+	ocrWorkers = v
	32	+	}
	33	+	if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
	34	+	rawPGM = false
	35	+	}
	36	+	if os.Getenv("NEFERWL_HINT_BENCH_PNG") == "none" {
	37	+	pngLevel = png.NoCompression
	38	+	}
	39	+	for i, file := range files {
	40	+	f, err := os.Open(file)
	41	+	if err != nil {
	42	+	t.Fatal("open frame failed")
	43	+	}
	44	+	src, err := png.Decode(f)
	45	+	_ = f.Close()
	46	+	if err != nil {
	47	+	t.Fatal("decode frame failed")
	48	+	}
	49	+	img := image.NewRGBA(image.Rect(0, 0, src.Bounds().Dx(), src.Bounds().Dy()))
	50	+	draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
	51	+	start := time.Now()
	52	+	words, bands, err := readWords(context.Background(), toGray(img))
	53	+	if err != nil {
	54	+	t.Fatal("recognition failed")
	55	+	}
	56	+	t.Logf("frame=%d px=%d ocr_ms=%.1f words=%d links=%d bands=%d", i, img.Bounds().Dx()*img.Bounds().Dy(), ms(time.Since(start)), len(words), len(textLinks(words, 1)), bands)
	57	+	if os.Getenv("NEFERWL_HINT_BENCH_STRIPS") == "" {
	58	+	continue
	59	+	}
	60	+	// Experimental strips use known OCR word regions, not pixel targets.
	61	+	// This is an oracle comparison, not a production optimization.
	62	+	lines := make([]image.Rectangle, 0, len(words))
	63	+	for _, word := range words {
	64	+	lines = append(lines, word.Rect)
	65	+	}
	66	+	for _, normalized := range []bool{false, true} {
	67	+	begin := time.Now()
	68	+	compact, _, err := readStrips(context.Background(), img, lines, 8, normalized)
	69	+	if err != nil {
	70	+	t.Fatal("strip recognition failed")
	71	+	}
	72	+	t.Logf("frame=%d strips normalized=%t ocr_ms=%.1f words=%d links=%d", i, normalized, ms(time.Since(begin)), len(compact), len(textLinks(compact, 1)))
	73	+	}
	74	+	}
	75	+	}
	76	+
	77	+	func toGray(img image.RGBA) image.Gray {
	78	+	g := image.NewGray(img.Bounds())
	79	+	draw.Draw(g, g.Bounds(), img, img.Bounds().Min, draw.Src)
	80	+	return g
	81	+	}

@@ -0,0 +1,81 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/draw"
	7	+	"image/png"
	8	+	"os"
	9	+	"os/exec"
	10	+	"path/filepath"
	11	+	"strconv"
	12	+	"testing"
	13	+	"time"
	14	+	)
	15	+
	16	+	// TestFrameBenchmark logs counts and timings, never filenames or OCR text.
	17	+	func TestFrameBenchmark(t *testing.T) {
	18	+	dir := os.Getenv("NEFERWL_HINT_BENCH_DIR")
	19	+	if dir == "" {
	20	+	t.Skip("set NEFERWL_HINT_BENCH_DIR to a local PNG directory")
	21	+	}
	22	+	if _, err := exec.LookPath("tesseract"); err != nil {
	23	+	t.Skip("tesseract not on PATH")
	24	+	}
	25	+	files, err := filepath.Glob(filepath.Join(dir, "*.png"))
	26	+	if err != nil \|\| len(files) == 0 {
	27	+	t.Fatal("no PNG frames")
	28	+	}
	29	+	// Settings under test: NEFERWL_HINT_BENCH_WORKERS=<n>, NEFERWL_HINT_BENCH_PNG=speed\|none.
	30	+	if v, _ := strconv.Atoi(os.Getenv("NEFERWL_HINT_BENCH_WORKERS")); v > 0 {
	31	+	ocrWorkers = v
	32	+	}
	33	+	if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
	34	+	rawPGM = false
	35	+	}
	36	+	if os.Getenv("NEFERWL_HINT_BENCH_PNG") == "none" {
	37	+	pngLevel = png.NoCompression
	38	+	}
	39	+	for i, file := range files {
	40	+	f, err := os.Open(file)
	41	+	if err != nil {
	42	+	t.Fatal("open frame failed")
	43	+	}
	44	+	src, err := png.Decode(f)
	45	+	_ = f.Close()
	46	+	if err != nil {
	47	+	t.Fatal("decode frame failed")
	48	+	}
	49	+	img := image.NewRGBA(image.Rect(0, 0, src.Bounds().Dx(), src.Bounds().Dy()))
	50	+	draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
	51	+	start := time.Now()
	52	+	words, bands, err := readWords(context.Background(), toGray(img))
	53	+	if err != nil {
	54	+	t.Fatal("recognition failed")
	55	+	}
	56	+	t.Logf("frame=%d px=%d ocr_ms=%.1f words=%d links=%d bands=%d", i, img.Bounds().Dx()*img.Bounds().Dy(), ms(time.Since(start)), len(words), len(textLinks(words, 1)), bands)
	57	+	if os.Getenv("NEFERWL_HINT_BENCH_STRIPS") == "" {
	58	+	continue
	59	+	}
	60	+	// Experimental strips use known OCR word regions, not pixel targets.
	61	+	// This is an oracle comparison, not a production optimization.
	62	+	lines := make([]image.Rectangle, 0, len(words))
	63	+	for _, word := range words {
	64	+	lines = append(lines, word.Rect)
	65	+	}
	66	+	for _, normalized := range []bool{false, true} {
	67	+	begin := time.Now()
	68	+	compact, _, err := readStrips(context.Background(), img, lines, 8, normalized)
	69	+	if err != nil {
	70	+	t.Fatal("strip recognition failed")
	71	+	}
	72	+	t.Logf("frame=%d strips normalized=%t ocr_ms=%.1f words=%d links=%d", i, normalized, ms(time.Since(begin)), len(compact), len(textLinks(compact, 1)))
	73	+	}
	74	+	}
	75	+	}
	76	+
	77	+	func toGray(img image.RGBA) image.Gray {
	78	+	g := image.NewGray(img.Bounds())
	79	+	draw.Draw(g, g.Bounds(), img, img.Bounds().Min, draw.Src)
	80	+	return g
	81	+	}

A internal/adapters/hints/dump.go +285 -0

@@ -0,0 +1,285 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"encoding/json"
	6	+	"fmt"
	7	+	"image"
	8	+	"image/color"
	9	+	"image/png"
	10	+	"os"
	11	+	"path/filepath"
	12	+	"sort"
	13	+	"strings"
	14	+	"syscall"
	15	+	"time"
	16	+
	17	+	"github.com/bnema/neferwl/internal/ports"
	18	+	)
	19	+
	20	+	const (
	21	+	// maxDumps is the number of dump groups kept in the directory.
	22	+	maxDumps = 50
	23	+	outlineSize = 2
	24	+	)
	25	+
	26	+	// Options tunes Run.
	27	+	type Options struct {
	28	+	// DumpDir, when not empty, receives the scanned frames (PNG, annotated
	29	+	// PNG and JSON). They contain the screen in clear.
	30	+	DumpDir string
	31	+	// Source, when not nil, finds targets in the accessibility tree of the
	32	+	// windows on screen; OCR reads the rest.
	33	+	Source targetSource
	34	+	}
	35	+
	36	+	// dumpJob is one scan to write. pix is the capture buffer of the scan, owned
	37	+	// by the job: writeDump converts it in place.
	38	+	type dumpJob struct {
	39	+	when time.Time
	40	+	req ports.HintScanRequest
	41	+	pix []byte
	42	+	m *mapping // owns pix; released after the dump
	43	+	stride int
	44	+	targets []ports.HintTarget
	45	+	crops []image.Rectangle
	46	+	words []OCRWord
	47	+	// atspi is the number of leading targets that come from the
	48	+	// accessibility tree; the others come from OCR.
	49	+	atspi int
	50	+	captureD time.Duration
	51	+	scanD time.Duration
	52	+	atspiD time.Duration
	53	+	}
	54	+
	55	+	type dumpTarget struct {
	56	+	Kind string `json:"kind"`
	57	+	Source string `json:"source"`
	58	+	Rect [4]int `json:"rect"`
	59	+	}
	60	+
	61	+	type dumpMeta struct {
	62	+	Output string `json:"output"`
	63	+	Width int `json:"width"`
	64	+	Height int `json:"height"`
	65	+	Stride int `json:"stride"`
	66	+	Scale float64 `json:"scale"`
	67	+	LinksOnly bool `json:"links_only"`
	68	+	CaptureMS float64 `json:"capture_ms"`
	69	+	ScanMS float64 `json:"scan_ms"`
	70	+	AtspiMS float64 `json:"atspi_ms"`
	71	+	Crops []image.Rectangle `json:"ocr_crops"`
	72	+	Words []OCRWord `json:"words"`
	73	+	Targets []dumpTarget `json:"targets"`
	74	+	}
	75	+
	76	+	func kindName(k ports.HintKind) string {
	77	+	switch k {
	78	+	case ports.HintLink:
	79	+	return "link"
	80	+	case ports.HintButton:
	81	+	return "button"
	82	+	default:
	83	+	return "text"
	84	+	}
	85	+	}
	86	+
	87	+	func kindColor(k ports.HintKind) color.RGBA {
	88	+	switch k {
	89	+	case ports.HintLink:
	90	+	return color.RGBA{0x00, 0xc8, 0x53, 0xff}
	91	+	case ports.HintButton:
	92	+	return color.RGBA{0x29, 0x62, 0xff, 0xff}
	93	+	default:
	94	+	return color.RGBA{0x80, 0x80, 0x80, 0xff}
	95	+	}
	96	+	}
	97	+
	98	+	// atspiColor is the outline of targets found in the accessibility tree.
	99	+	var atspiColor = color.RGBA{0xff, 0x00, 0xff, 0xff}
	100	+
	101	+	func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 }
	102	+
	103	+	// writeDump writes the three files of j into dir, then prunes old groups.
	104	+	func writeDump(dir string, j *dumpJob) error {
	105	+	if err := secureDir(dir); err != nil {
	106	+	return err
	107	+	}
	108	+	prefix := filepath.Join(dir, j.when.Format("20060102-150405.000")+"-"+safeName(j.req.Output))
	109	+
	110	+	// The frame is converted in place to RGBA and encoded straight from the
	111	+	// capture buffer: no copy. The job owns the buffer, which is released
	112	+	// with it.
	113	+	for y := 0; y < j.req.Height; y++ {
	114	+	row := j.pix[yj.stride : yj.stride+j.req.Width*4]
	115	+	for x := 0; x < len(row); x += 4 {
	116	+	row[x], row[x+2], row[x+3] = row[x+2], row[x], 0xff
	117	+	}
	118	+	}
	119	+	img := &image.RGBA{Pix: j.pix, Stride: j.stride, Rect: image.Rect(0, 0, j.req.Width, j.req.Height)}
	120	+	if err := writePNG(prefix+".png", img); err != nil {
	121	+	return err
	122	+	}
	123	+
	124	+	scale := j.req.Scale
	125	+	if scale <= 0 {
	126	+	scale = 1
	127	+	}
	128	+	for _, r := range j.crops {
	129	+	outline(img, r, color.RGBA{0x80, 0x80, 0x80, 0xff})
	130	+	}
	131	+	for i, t := range j.targets {
	132	+	r := image.Rect(int(float64(t.Rect.X)scale), int(float64(t.Rect.Y)scale),
	133	+	int(float64(t.Rect.X+t.Rect.W)scale), int(float64(t.Rect.Y+t.Rect.H)scale))
	134	+	c := kindColor(t.Kind)
	135	+	if i < j.atspi {
	136	+	c = atspiColor
	137	+	}
	138	+	outline(img, r, c)
	139	+	}
	140	+	if err := writePNG(prefix+".annotated.png", img); err != nil {
	141	+	return err
	142	+	}
	143	+
	144	+	meta := dumpMeta{
	145	+	Output: j.req.Output, Width: j.req.Width, Height: j.req.Height, Stride: j.stride,
	146	+	Scale: j.req.Scale, LinksOnly: j.req.LinksOnly,
	147	+	CaptureMS: ms(j.captureD), ScanMS: ms(j.scanD), AtspiMS: ms(j.atspiD),
	148	+	Crops: j.crops, Words: j.words,
	149	+	Targets: make([]dumpTarget, 0, len(j.targets)),
	150	+	}
	151	+	for i, t := range j.targets {
	152	+	src := "ocr"
	153	+	if i < j.atspi {
	154	+	src = "atspi"
	155	+	}
	156	+	meta.Targets = append(meta.Targets, dumpTarget{Kind: kindName(t.Kind), Source: src, Rect: [4]int{t.Rect.X, t.Rect.Y, t.Rect.X + t.Rect.W, t.Rect.Y + t.Rect.H}})
	157	+	}
	158	+	data, err := json.Marshal(meta)
	159	+	if err != nil {
	160	+	return err
	161	+	}
	162	+	if err := writeFile(prefix+".json", func(f *os.File) error { _, err := f.Write(append(data, '\n')); return err }); err != nil {
	163	+	return err
	164	+	}
	165	+	return prune(dir, maxDumps)
	166	+	}
	167	+
	168	+	func safeName(s string) string {
	169	+	return strings.Map(func(r rune) rune {
	170	+	if r == '/' \|\| r == 0 {
	171	+	return '_'
	172	+	}
	173	+	return r
	174	+	}, s)
	175	+	}
	176	+
	177	+	func writePNG(path string, img image.Image) error {
	178	+	enc := png.Encoder{CompressionLevel: png.BestSpeed}
	179	+	return writeFile(path, func(f *os.File) error {
	180	+	bw := bufio.NewWriterSize(f, 1<<16)
	181	+	if err := enc.Encode(bw, img); err != nil {
	182	+	return err
	183	+	}
	184	+	return bw.Flush()
	185	+	})
	186	+	}
	187	+
	188	+	// secureDir makes sure dir is a real directory owned by the current user and
	189	+	// private to it, whatever it was before.
	190	+	func secureDir(dir string) error {
	191	+	if err := os.MkdirAll(dir, 0o700); err != nil {
	192	+	return err
	193	+	}
	194	+	fi, err := os.Lstat(dir)
	195	+	if err != nil {
	196	+	return err
	197	+	}
	198	+	if !fi.IsDir() {
	199	+	return fmt.Errorf("dump dir %s is not a directory", dir)
	200	+	}
	201	+	if st, ok := fi.Sys().(*syscall.Stat_t); ok && int(st.Uid) != os.Getuid() {
	202	+	return fmt.Errorf("dump dir %s is not owned by the current user", dir)
	203	+	}
	204	+	return os.Chmod(dir, 0o700)
	205	+	}
	206	+
	207	+	// writeFile fills a fresh private temporary file in the directory of path and
	208	+	// renames it over path: an existing file or symlink there is replaced, never
	209	+	// followed.
	210	+	func writeFile(path string, fill func(*os.File) error) error {
	211	+	f, err := os.CreateTemp(filepath.Dir(path), ".dump-*")
	212	+	if err != nil {
	213	+	return err
	214	+	}
	215	+	tmp := f.Name()
	216	+	err = f.Chmod(0o600)
	217	+	if err == nil {
	218	+	err = fill(f)
	219	+	}
	220	+	if cerr := f.Close(); err == nil {
	221	+	err = cerr
	222	+	}
	223	+	if err == nil {
	224	+	err = os.Rename(tmp, path)
	225	+	}
	226	+	if err != nil {
	227	+	_ = os.Remove(tmp)
	228	+	}
	229	+	return err
	230	+	}
	231	+
	232	+	// outline draws a 2 px frame inside r, clipped to img.
	233	+	func outline(img *image.RGBA, r image.Rectangle, c color.RGBA) {
	234	+	r = r.Intersect(img.Rect)
	235	+	if r.Empty() {
	236	+	return
	237	+	}
	238	+	for y := r.Min.Y; y < r.Max.Y; y++ {
	239	+	for x := r.Min.X; x < r.Max.X; x++ {
	240	+	if x-r.Min.X < outlineSize \|\| r.Max.X-1-x < outlineSize \|\| y-r.Min.Y < outlineSize \|\| r.Max.Y-1-y < outlineSize {
	241	+	img.SetRGBA(x, y, c)
	242	+	}
	243	+	}
	244	+	}
	245	+	}
	246	+
	247	+	// groupOf returns the "<timestamp>-<output>" prefix of a dump file name.
	248	+	func groupOf(name string) string {
	249	+	for _, suf := range []string{".annotated.png", ".png", ".json"} {
	250	+	if strings.HasSuffix(name, suf) {
	251	+	return strings.TrimSuffix(name, suf)
	252	+	}
	253	+	}
	254	+	return ""
	255	+	}
	256	+
	257	+	// prune keeps the keep most recent groups of dir; timestamps sort by name.
	258	+	func prune(dir string, keep int) error {
	259	+	entries, err := os.ReadDir(dir)
	260	+	if err != nil {
	261	+	return err
	262	+	}
	263	+	groups := map[string][]string{}
	264	+	for _, e := range entries {
	265	+	if g := groupOf(e.Name()); g != "" && !e.IsDir() {
	266	+	groups[g] = append(groups[g], e.Name())
	267	+	}
	268	+	}
	269	+	if len(groups) <= keep {
	270	+	return nil
	271	+	}
	272	+	names := make([]string, 0, len(groups))
	273	+	for g := range groups {
	274	+	names = append(names, g)
	275	+	}
	276	+	sort.Strings(names)
	277	+	for _, g := range names[:len(names)-keep] {
	278	+	for _, f := range groups[g] {
	279	+	if err := os.Remove(filepath.Join(dir, f)); err != nil {
	280	+	return fmt.Errorf("prune: %w", err)
	281	+	}
	282	+	}
	283	+	}
	284	+	return nil
	285	+	}

@@ -0,0 +1,285 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"encoding/json"
	6	+	"fmt"
	7	+	"image"
	8	+	"image/color"
	9	+	"image/png"
	10	+	"os"
	11	+	"path/filepath"
	12	+	"sort"
	13	+	"strings"
	14	+	"syscall"
	15	+	"time"
	16	+
	17	+	"github.com/bnema/neferwl/internal/ports"
	18	+	)
	19	+
	20	+	const (
	21	+	// maxDumps is the number of dump groups kept in the directory.
	22	+	maxDumps = 50
	23	+	outlineSize = 2
	24	+	)
	25	+
	26	+	// Options tunes Run.
	27	+	type Options struct {
	28	+	// DumpDir, when not empty, receives the scanned frames (PNG, annotated
	29	+	// PNG and JSON). They contain the screen in clear.
	30	+	DumpDir string
	31	+	// Source, when not nil, finds targets in the accessibility tree of the
	32	+	// windows on screen; OCR reads the rest.
	33	+	Source targetSource
	34	+	}
	35	+
	36	+	// dumpJob is one scan to write. pix is the capture buffer of the scan, owned
	37	+	// by the job: writeDump converts it in place.
	38	+	type dumpJob struct {
	39	+	when time.Time
	40	+	req ports.HintScanRequest
	41	+	pix []byte
	42	+	m *mapping // owns pix; released after the dump
	43	+	stride int
	44	+	targets []ports.HintTarget
	45	+	crops []image.Rectangle
	46	+	words []OCRWord
	47	+	// atspi is the number of leading targets that come from the
	48	+	// accessibility tree; the others come from OCR.
	49	+	atspi int
	50	+	captureD time.Duration
	51	+	scanD time.Duration
	52	+	atspiD time.Duration
	53	+	}
	54	+
	55	+	type dumpTarget struct {
	56	+	Kind string `json:"kind"`
	57	+	Source string `json:"source"`
	58	+	Rect [4]int `json:"rect"`
	59	+	}
	60	+
	61	+	type dumpMeta struct {
	62	+	Output string `json:"output"`
	63	+	Width int `json:"width"`
	64	+	Height int `json:"height"`
	65	+	Stride int `json:"stride"`
	66	+	Scale float64 `json:"scale"`
	67	+	LinksOnly bool `json:"links_only"`
	68	+	CaptureMS float64 `json:"capture_ms"`
	69	+	ScanMS float64 `json:"scan_ms"`
	70	+	AtspiMS float64 `json:"atspi_ms"`
	71	+	Crops []image.Rectangle `json:"ocr_crops"`
	72	+	Words []OCRWord `json:"words"`
	73	+	Targets []dumpTarget `json:"targets"`
	74	+	}
	75	+
	76	+	func kindName(k ports.HintKind) string {
	77	+	switch k {
	78	+	case ports.HintLink:
	79	+	return "link"
	80	+	case ports.HintButton:
	81	+	return "button"
	82	+	default:
	83	+	return "text"
	84	+	}
	85	+	}
	86	+
	87	+	func kindColor(k ports.HintKind) color.RGBA {
	88	+	switch k {
	89	+	case ports.HintLink:
	90	+	return color.RGBA{0x00, 0xc8, 0x53, 0xff}
	91	+	case ports.HintButton:
	92	+	return color.RGBA{0x29, 0x62, 0xff, 0xff}
	93	+	default:
	94	+	return color.RGBA{0x80, 0x80, 0x80, 0xff}
	95	+	}
	96	+	}
	97	+
	98	+	// atspiColor is the outline of targets found in the accessibility tree.
	99	+	var atspiColor = color.RGBA{0xff, 0x00, 0xff, 0xff}
	100	+
	101	+	func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 }
	102	+
	103	+	// writeDump writes the three files of j into dir, then prunes old groups.
	104	+	func writeDump(dir string, j *dumpJob) error {
	105	+	if err := secureDir(dir); err != nil {
	106	+	return err
	107	+	}
	108	+	prefix := filepath.Join(dir, j.when.Format("20060102-150405.000")+"-"+safeName(j.req.Output))
	109	+
	110	+	// The frame is converted in place to RGBA and encoded straight from the
	111	+	// capture buffer: no copy. The job owns the buffer, which is released
	112	+	// with it.
	113	+	for y := 0; y < j.req.Height; y++ {
	114	+	row := j.pix[yj.stride : yj.stride+j.req.Width*4]
	115	+	for x := 0; x < len(row); x += 4 {
	116	+	row[x], row[x+2], row[x+3] = row[x+2], row[x], 0xff
	117	+	}
	118	+	}
	119	+	img := &image.RGBA{Pix: j.pix, Stride: j.stride, Rect: image.Rect(0, 0, j.req.Width, j.req.Height)}
	120	+	if err := writePNG(prefix+".png", img); err != nil {
	121	+	return err
	122	+	}
	123	+
	124	+	scale := j.req.Scale
	125	+	if scale <= 0 {
	126	+	scale = 1
	127	+	}
	128	+	for _, r := range j.crops {
	129	+	outline(img, r, color.RGBA{0x80, 0x80, 0x80, 0xff})
	130	+	}
	131	+	for i, t := range j.targets {
	132	+	r := image.Rect(int(float64(t.Rect.X)scale), int(float64(t.Rect.Y)scale),
	133	+	int(float64(t.Rect.X+t.Rect.W)scale), int(float64(t.Rect.Y+t.Rect.H)scale))
	134	+	c := kindColor(t.Kind)
	135	+	if i < j.atspi {
	136	+	c = atspiColor
	137	+	}
	138	+	outline(img, r, c)
	139	+	}
	140	+	if err := writePNG(prefix+".annotated.png", img); err != nil {
	141	+	return err
	142	+	}
	143	+
	144	+	meta := dumpMeta{
	145	+	Output: j.req.Output, Width: j.req.Width, Height: j.req.Height, Stride: j.stride,
	146	+	Scale: j.req.Scale, LinksOnly: j.req.LinksOnly,
	147	+	CaptureMS: ms(j.captureD), ScanMS: ms(j.scanD), AtspiMS: ms(j.atspiD),
	148	+	Crops: j.crops, Words: j.words,
	149	+	Targets: make([]dumpTarget, 0, len(j.targets)),
	150	+	}
	151	+	for i, t := range j.targets {
	152	+	src := "ocr"
	153	+	if i < j.atspi {
	154	+	src = "atspi"
	155	+	}
	156	+	meta.Targets = append(meta.Targets, dumpTarget{Kind: kindName(t.Kind), Source: src, Rect: [4]int{t.Rect.X, t.Rect.Y, t.Rect.X + t.Rect.W, t.Rect.Y + t.Rect.H}})
	157	+	}
	158	+	data, err := json.Marshal(meta)
	159	+	if err != nil {
	160	+	return err
	161	+	}
	162	+	if err := writeFile(prefix+".json", func(f *os.File) error { _, err := f.Write(append(data, '\n')); return err }); err != nil {
	163	+	return err
	164	+	}
	165	+	return prune(dir, maxDumps)
	166	+	}
	167	+
	168	+	func safeName(s string) string {
	169	+	return strings.Map(func(r rune) rune {
	170	+	if r == '/' \|\| r == 0 {
	171	+	return '_'
	172	+	}
	173	+	return r
	174	+	}, s)
	175	+	}
	176	+
	177	+	func writePNG(path string, img image.Image) error {
	178	+	enc := png.Encoder{CompressionLevel: png.BestSpeed}
	179	+	return writeFile(path, func(f *os.File) error {
	180	+	bw := bufio.NewWriterSize(f, 1<<16)
	181	+	if err := enc.Encode(bw, img); err != nil {
	182	+	return err
	183	+	}
	184	+	return bw.Flush()
	185	+	})
	186	+	}
	187	+
	188	+	// secureDir makes sure dir is a real directory owned by the current user and
	189	+	// private to it, whatever it was before.
	190	+	func secureDir(dir string) error {
	191	+	if err := os.MkdirAll(dir, 0o700); err != nil {
	192	+	return err
	193	+	}
	194	+	fi, err := os.Lstat(dir)
	195	+	if err != nil {
	196	+	return err
	197	+	}
	198	+	if !fi.IsDir() {
	199	+	return fmt.Errorf("dump dir %s is not a directory", dir)
	200	+	}
	201	+	if st, ok := fi.Sys().(*syscall.Stat_t); ok && int(st.Uid) != os.Getuid() {
	202	+	return fmt.Errorf("dump dir %s is not owned by the current user", dir)
	203	+	}
	204	+	return os.Chmod(dir, 0o700)
	205	+	}
	206	+
	207	+	// writeFile fills a fresh private temporary file in the directory of path and
	208	+	// renames it over path: an existing file or symlink there is replaced, never
	209	+	// followed.
	210	+	func writeFile(path string, fill func(*os.File) error) error {
	211	+	f, err := os.CreateTemp(filepath.Dir(path), ".dump-*")
	212	+	if err != nil {
	213	+	return err
	214	+	}
	215	+	tmp := f.Name()
	216	+	err = f.Chmod(0o600)
	217	+	if err == nil {
	218	+	err = fill(f)
	219	+	}
	220	+	if cerr := f.Close(); err == nil {
	221	+	err = cerr
	222	+	}
	223	+	if err == nil {
	224	+	err = os.Rename(tmp, path)
	225	+	}
	226	+	if err != nil {
	227	+	_ = os.Remove(tmp)
	228	+	}
	229	+	return err
	230	+	}
	231	+
	232	+	// outline draws a 2 px frame inside r, clipped to img.
	233	+	func outline(img *image.RGBA, r image.Rectangle, c color.RGBA) {
	234	+	r = r.Intersect(img.Rect)
	235	+	if r.Empty() {
	236	+	return
	237	+	}
	238	+	for y := r.Min.Y; y < r.Max.Y; y++ {
	239	+	for x := r.Min.X; x < r.Max.X; x++ {
	240	+	if x-r.Min.X < outlineSize \|\| r.Max.X-1-x < outlineSize \|\| y-r.Min.Y < outlineSize \|\| r.Max.Y-1-y < outlineSize {
	241	+	img.SetRGBA(x, y, c)
	242	+	}
	243	+	}
	244	+	}
	245	+	}
	246	+
	247	+	// groupOf returns the "<timestamp>-<output>" prefix of a dump file name.
	248	+	func groupOf(name string) string {
	249	+	for _, suf := range []string{".annotated.png", ".png", ".json"} {
	250	+	if strings.HasSuffix(name, suf) {
	251	+	return strings.TrimSuffix(name, suf)
	252	+	}
	253	+	}
	254	+	return ""
	255	+	}
	256	+
	257	+	// prune keeps the keep most recent groups of dir; timestamps sort by name.
	258	+	func prune(dir string, keep int) error {
	259	+	entries, err := os.ReadDir(dir)
	260	+	if err != nil {
	261	+	return err
	262	+	}
	263	+	groups := map[string][]string{}
	264	+	for _, e := range entries {
	265	+	if g := groupOf(e.Name()); g != "" && !e.IsDir() {
	266	+	groups[g] = append(groups[g], e.Name())
	267	+	}
	268	+	}
	269	+	if len(groups) <= keep {
	270	+	return nil
	271	+	}
	272	+	names := make([]string, 0, len(groups))
	273	+	for g := range groups {
	274	+	names = append(names, g)
	275	+	}
	276	+	sort.Strings(names)
	277	+	for _, g := range names[:len(names)-keep] {
	278	+	for _, f := range groups[g] {
	279	+	if err := os.Remove(filepath.Join(dir, f)); err != nil {
	280	+	return fmt.Errorf("prune: %w", err)
	281	+	}
	282	+	}
	283	+	}
	284	+	return nil
	285	+	}

A internal/adapters/hints/dump_test.go +40 -0

@@ -0,0 +1,40 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"os"
	5	+	"path/filepath"
	6	+	"testing"
	7	+	"time"
	8	+
	9	+	"github.com/stretchr/testify/require"
	10	+
	11	+	"github.com/bnema/neferwl/internal/ports"
	12	+	)
	13	+
	14	+	// An existing dump directory is made private, and a symlink at a file name is
	15	+	// replaced instead of followed.
	16	+	func TestWriteDumpPrivate(t *testing.T) {
	17	+	base := t.TempDir()
	18	+	dir := filepath.Join(base, "hints")
	19	+	require.NoError(t, os.Mkdir(dir, 0o755))
	20	+	require.NoError(t, os.Chmod(dir, 0o755))
	21	+	victim := filepath.Join(base, "victim")
	22	+	require.NoError(t, os.WriteFile(victim, []byte("keep"), 0o644))
	23	+	when := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
	24	+	prefix := filepath.Join(dir, when.Format("20060102-150405.000")+"-o")
	25	+	require.NoError(t, os.Symlink(victim, prefix+".json"))
	26	+	pix := make([]byte, 444)
	27	+	j := &dumpJob{when: when, req: ports.HintScanRequest{Output: "o", Width: 4, Height: 4, Scale: 1}, pix: pix, stride: 16, m: &mapping{}}
	28	+	require.NoError(t, writeDump(dir, j))
	29	+
	30	+	fi, err := os.Stat(dir)
	31	+	require.NoError(t, err)
	32	+	require.Equal(t, os.FileMode(0o700), fi.Mode().Perm())
	33	+	got, err := os.ReadFile(victim)
	34	+	require.NoError(t, err)
	35	+	require.Equal(t, "keep", string(got))
	36	+	li, err := os.Lstat(prefix + ".json")
	37	+	require.NoError(t, err)
	38	+	require.True(t, li.Mode().IsRegular())
	39	+	require.Equal(t, os.FileMode(0o600), li.Mode().Perm())
	40	+	}

@@ -0,0 +1,40 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"os"
	5	+	"path/filepath"
	6	+	"testing"
	7	+	"time"
	8	+
	9	+	"github.com/stretchr/testify/require"
	10	+
	11	+	"github.com/bnema/neferwl/internal/ports"
	12	+	)
	13	+
	14	+	// An existing dump directory is made private, and a symlink at a file name is
	15	+	// replaced instead of followed.
	16	+	func TestWriteDumpPrivate(t *testing.T) {
	17	+	base := t.TempDir()
	18	+	dir := filepath.Join(base, "hints")
	19	+	require.NoError(t, os.Mkdir(dir, 0o755))
	20	+	require.NoError(t, os.Chmod(dir, 0o755))
	21	+	victim := filepath.Join(base, "victim")
	22	+	require.NoError(t, os.WriteFile(victim, []byte("keep"), 0o644))
	23	+	when := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
	24	+	prefix := filepath.Join(dir, when.Format("20060102-150405.000")+"-o")
	25	+	require.NoError(t, os.Symlink(victim, prefix+".json"))
	26	+	pix := make([]byte, 444)
	27	+	j := &dumpJob{when: when, req: ports.HintScanRequest{Output: "o", Width: 4, Height: 4, Scale: 1}, pix: pix, stride: 16, m: &mapping{}}
	28	+	require.NoError(t, writeDump(dir, j))
	29	+
	30	+	fi, err := os.Stat(dir)
	31	+	require.NoError(t, err)
	32	+	require.Equal(t, os.FileMode(0o700), fi.Mode().Perm())
	33	+	got, err := os.ReadFile(victim)
	34	+	require.NoError(t, err)
	35	+	require.Equal(t, "keep", string(got))
	36	+	li, err := os.Lstat(prefix + ".json")
	37	+	require.NoError(t, err)
	38	+	require.True(t, li.Mode().IsRegular())
	39	+	require.Equal(t, os.FileMode(0o600), li.Mode().Perm())
	40	+	}

A internal/adapters/hints/hints.go +507 -0

@@ -0,0 +1,507 @@
	1	+	// Package hints finds clickable targets for hint mode: it captures the last
	2	+	// frame of an output through the capture pipeline and reads uncovered windows with OCR.
	3	+	package hints
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"fmt"
	9	+	"hash/maphash"
	10	+	"image"
	11	+	"math"
	12	+	"strings"
	13	+	"time"
	14	+
	15	+	"github.com/bnema/neferwl/internal/adapters/logging"
	16	+	"github.com/bnema/neferwl/internal/ports"
	17	+	)
	18	+
	19	+	// ErrBusy answers a request that arrives while another one is in flight.
	20	+	var ErrBusy = errors.New("hints: scan in progress")
	21	+
	22	+	// ErrTimeout is the error of a capture that is not answered in time.
	23	+	var ErrTimeout = errors.New("hints: capture timed out")
	24	+
	25	+	// ErrBadRequest is the error of a request without a usable buffer size.
	26	+	var ErrBadRequest = errors.New("hints: invalid output size")
	27	+
	28	+	const (
	29	+	captureTimeout = time.Second
	30	+	// formatXRGB8888 is the capture format: opaque BGRA in memory.
	31	+	formatXRGB8888 = 1
	32	+	maxPixels = 1 << 28
	33	+	)
	34	+
	35	+	// Run serves requests until ctx is done. It handles one request at a time;
	36	+	// a request that arrives meanwhile is answered with ErrBusy. Every request
	37	+	// gets exactly one result on results.
	38	+	func Run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, opts Options) error {
	39	+	return run(ctx, requests, captures, captured, results, captureTimeout, opts)
	40	+	}
	41	+
	42	+	// targetSource finds hint targets in the accessibility tree of windows. It
	43	+	// returns the targets (logical output-local) and the windows they cover.
	44	+	type targetSource interface {
	45	+	Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)
	46	+	// Close releases the source. It is called once, by the worker goroutine,
	47	+	// when the worker returns; the source is not safe for concurrent use.
	48	+	Close() error
	49	+	}
	50	+
	51	+	// atspiResult is what the source found for one request.
	52	+	type atspiResult struct {
	53	+	targets []ports.HintTarget
	54	+	covered map[ports.WindowID]bool
	55	+	dur time.Duration
	56	+	err error
	57	+	}
	58	+
	59	+	type worker struct {
	60	+	source targetSource
	61	+	cache map[bandKey]bandCache
	62	+	cacheSeed maphash.Seed
	63	+	ocr ocrState
	64	+	warnedSource bool
	65	+	n uint64
	66	+	dumpDir string
	67	+	pending *dumpJob
	68	+	// orphans are capture buffers the output may still write: they are
	69	+	// released when its CaptureDone finally arrives.
	70	+	orphans map[uint64]*mapping
	71	+	live int // mappings not released yet
	72	+	}
	73	+
	74	+	// frame is the captured output: BGRA pixels, nil when no capture was needed.
	75	+	type frame struct {
	76	+	pix []byte
	77	+	stride int
	78	+	m *mapping
	79	+	}
	80	+
	81	+	func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {
	82	+	w := &worker{dumpDir: opts.DumpDir, source: opts.Source}
	83	+	if opts.Source != nil {
	84	+	// The worker goroutine is the only one to touch the source.
	85	+	defer func() { _ = opts.Source.Close() }()
	86	+	}
	87	+	for {
	88	+	select {
	89	+	case <-ctx.Done():
	90	+	return nil
	91	+	case done := <-captured:
	92	+	// A capture abandoned by a timeout is finally done: release it.
	93	+	w.reap(done.ID)
	94	+	case req := <-requests:
	95	+	res := w.serve(ctx, req, requests, captures, captured, results, timeout)
	96	+	if !send(ctx, results, res) {
	97	+	// The dump job will never run: release its frame.
	98	+	if j := w.pending; j != nil {
	99	+	w.pending = nil
	100	+	j.m.unmap()
	101	+	}
	102	+	return nil
	103	+	}
	104	+	w.dump(ctx)
	105	+	}
	106	+	}
	107	+	}
	108	+
	109	+	// dump writes the last successful scan, after its result was sent, then
	110	+	// releases its frame.
	111	+	func (w *worker) dump(ctx context.Context) {
	112	+	j := w.pending
	113	+	w.pending = nil
	114	+	if j == nil {
	115	+	return
	116	+	}
	117	+	defer j.m.unmap()
	118	+	if err := writeDump(w.dumpDir, j); err != nil {
	119	+	log := logging.For(ctx, "hints")
	120	+	log.Warn().Err(err).Str("dir", w.dumpDir).Msg("hint debug dump failed")
	121	+	}
	122	+	}
	123	+
	124	+	func send(ctx context.Context, results chan<- ports.HintScanResult, res ports.HintScanResult) bool {
	125	+	select {
	126	+	case results <- res:
	127	+	return true
	128	+	case <-ctx.Done():
	129	+	return false
	130	+	}
	131	+	}
	132	+
	133	+	// needFrame reports whether the scan must capture the output: OCR has a
	134	+	// window to read, or the debug dump wants the frame. With OCR unavailable
	135	+	// (tesseract missing) and no dump, the accessibility targets are the result.
	136	+	func (w *worker) needFrame(ctx context.Context, req ports.HintScanRequest, ar atspiResult) bool {
	137	+	ocr := w.ocrReady(ctx)
	138	+	if w.dumpDir != "" {
	139	+	return true
	140	+	}
	141	+	if !ocr {
	142	+	return false
	143	+	}
	144	+	for _, win := range req.Windows {
	145	+	if !ar.covered[win.ID] && !windowCrop(win.Rect, req.Scale, req.Width, req.Height).Empty() {
	146	+	return true
	147	+	}
	148	+	}
	149	+	return false
	150	+	}
	151	+
	152	+	func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration) ports.HintScanResult {
	153	+	log := logging.For(ctx, "hints")
	154	+	res := ports.HintScanResult{Seq: req.Seq, Output: req.Output}
	155	+	fail := func(err error) ports.HintScanResult {
	156	+	log.Info().Err(err).Str("output", req.Output).Msg("hint scan failed")
	157	+	res.Err = err
	158	+	return res
	159	+	}
	160	+
	161	+	if req.Width <= 0 \|\| req.Height <= 0 \|\| req.Width*req.Height > maxPixels {
	162	+	return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))
	163	+	}
	164	+	// The accessibility query is always waited for: the source is not shared
	165	+	// between requests. The capture only starts once it is known whether any
	166	+	// window needs OCR.
	167	+	atspi := w.startSource(ctx, req)
	168	+	defer func() { <-atspi.done }()
	169	+	if !w.needFrame(ctx, req, atspi.wait()) {
	170	+	return w.scan(ctx, req, frame{}, 0, res, atspi)
	171	+	}
	172	+	// The capture buffer is anonymous mapped memory, outside the Go heap. It
	173	+	// is released once the crops are extracted, or by the dump job.
	174	+	m := w.alloc(req.Width * req.Height * 4)
	175	+	keep := false // the mapping outlives this call: dump job or pending capture
	176	+	defer func() {
	177	+	if !keep {
	178	+	m.unmap()
	179	+	}
	180	+	}()
	181	+	buf := m.b
	182	+
	183	+	w.n++
	184	+	id := ports.CaptureIDInternal \| w.n
	185	+	stride := req.Width * 4
	186	+	start := time.Now()
	187	+	creq := ports.CaptureRequest{
	188	+	ID: id,
	189	+	Output: req.Output,
	190	+	Region: image.Rect(0, 0, req.Width, req.Height),
	191	+	Mapped: buf,
	192	+	Width: req.Width,
	193	+	Height: req.Height,
	194	+	Stride: stride,
	195	+	Format: formatXRGB8888,
	196	+	Indicate: false,
	197	+	}
	198	+
	199	+	timer := time.NewTimer(timeout)
	200	+	defer timer.Stop()
	201	+
	202	+	select {
	203	+	case captures <- creq:
	204	+	case <-timer.C:
	205	+	return fail(ErrTimeout)
	206	+	case <-ctx.Done():
	207	+	return fail(ctx.Err())
	208	+	}
	209	+	// From here the output owns buf until CaptureDone.
	210	+	abandon := func() {
	211	+	if w.orphans == nil {
	212	+	w.orphans = map[uint64]*mapping{}
	213	+	}
	214	+	w.orphans[id] = m
	215	+	keep = true
	216	+	}
	217	+
	218	+	for {
	219	+	select {
	220	+	case done := <-captured:
	221	+	if done.ID != id {
	222	+	w.reap(done.ID)
	223	+	continue
	224	+	}
	225	+	if done.Err != nil {
	226	+	return fail(fmt.Errorf("hints: capture: %w", done.Err))
	227	+	}
	228	+	res = w.scan(ctx, req, frame{pix: buf, stride: stride, m: m}, time.Since(start), res, atspi)
	229	+	keep = w.pending != nil && w.pending.m == m
	230	+	return res
	231	+	case other := <-requests:
	232	+	// The buffer is in use until the capture is done: answer now.
	233	+	busy := ports.HintScanResult{Seq: other.Seq, Output: other.Output, Err: ErrBusy}
	234	+	if !send(ctx, results, busy) {
	235	+	abandon()
	236	+	return fail(ctx.Err())
	237	+	}
	238	+	case <-timer.C:
	239	+	// The output may still write into buf: it is released when its
	240	+	// CaptureDone arrives.
	241	+	abandon()
	242	+	return fail(ErrTimeout)
	243	+	case <-ctx.Done():
	244	+	abandon()
	245	+	return fail(ctx.Err())
	246	+	}
	247	+	}
	248	+	}
	249	+
	250	+	// reap releases the buffer of an abandoned capture that is finally done.
	251	+	func (w *worker) reap(id uint64) {
	252	+	if m, ok := w.orphans[id]; ok {
	253	+	delete(w.orphans, id)
	254	+	m.unmap()
	255	+	}
	256	+	}
	257	+
	258	+	// bandJob is one OCR band of one window crop.
	259	+	type bandJob struct {
	260	+	win int // index into the scan's windows
	261	+	key bandKey //
	262	+	ext image.Rectangle // crop-local rows read, with context
	263	+	own [2]int // crop-local rows owned: words centred there are kept
	264	+	hash uint64
	265	+	hit bool
	266	+	gray *image.Gray
	267	+	words []OCRWord
	268	+	err error
	269	+	}
	270	+
	271	+	func (w worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame, captureDur time.Duration, res ports.HintScanResult, atspi pendingSource) ports.HintScanResult {
	272	+	log := logging.For(ctx, "hints")
	273	+	ar := atspi.wait()
	274	+	if ar.err != nil && !w.warnedSource {
	275	+	log.Warn().Err(ar.err).Msg("accessibility targets unavailable; using OCR only")
	276	+	w.warnedSource = true
	277	+	}
	278	+	out := make([]ports.HintTarget, 0, len(ar.targets))
	279	+	for _, t := range ar.targets {
	280	+	if !req.LinksOnly \|\| t.Kind == ports.HintLink {
	281	+	out = append(out, t)
	282	+	}
	283	+	}
	284	+	nAtspi := len(out)
	285	+	t0 := time.Now()
	286	+	if w.cacheSeed == (maphash.Seed{}) {
	287	+	w.cacheSeed = maphash.MakeSeed()
	288	+	}
	289	+	nextCache := make(map[bandKey]bandCache)
	290	+	var crops []image.Rectangle
	291	+	var allWords []OCRWord
	292	+	pixels, links := 0, 0
	293	+	ocrOn := fr.pix != nil && w.ocrReady(ctx)
	294	+	// Phase 1 reads the frame: the bands that miss the cache are copied, as
	295	+	// grayscale, into one arena. Phase 2 (OCR) does not need the frame.
	296	+	var jobs []*bandJob
	297	+	var rects []image.Rectangle // crop of each window that has bands
	298	+	var arena *mapping
	299	+	if ocrOn {
	300	+	type pending struct {
	301	+	r image.Rectangle
	302	+	jobs []*bandJob
	303	+	}
	304	+	var wins []pending
	305	+	need := 0
	306	+	for _, win := range req.Windows {
	307	+	if ar.covered[win.ID] {
	308	+	continue
	309	+	}
	310	+	r := windowCrop(win.Rect, req.Scale, req.Width, req.Height)
	311	+	if r.Empty() {
	312	+	continue
	313	+	}
	314	+	crops = append(crops, r)
	315	+	pixels += r.Dx() * r.Dy()
	316	+	p := pending{r: r}
	317	+	for _, b := range bandsOf(r.Dy()) {
	318	+	ext := image.Rect(0, b.top, r.Dx(), b.bottom)
	319	+	j := &bandJob{win: len(wins), ext: ext, own: [2]int{b.ownTop, b.ownBottom},
	320	+	key: bandKey{output: req.Output, window: win.ID, band: b.idx, bands: b.n}}
	321	+	j.hash = w.cropHash(fr.pix, fr.stride, ext.Add(r.Min))
	322	+	entry, ok := w.cache[j.key]
	323	+	if ok && entry.hash == j.hash && entry.ext == ext {
	324	+	j.hit, j.words = true, entry.words
	325	+	} else {
	326	+	need += ext.Dx() * ext.Dy()
	327	+	}
	328	+	p.jobs = append(p.jobs, j)
	329	+	}
	330	+	wins = append(wins, p)
	331	+	}
	332	+	if need > 0 {
	333	+	arena = w.alloc(need)
	334	+	}
	335	+	off := 0
	336	+	for _, p := range wins {
	337	+	rects = append(rects, p.r)
	338	+	for _, j := range p.jobs {
	339	+	if !j.hit {
	340	+	n := j.ext.Dx() * j.ext.Dy()
	341	+	j.gray = cropGray(fr.pix, fr.stride, j.ext.Add(p.r.Min), arena.b[off:off+n])
	342	+	off += n
	343	+	}
	344	+	jobs = append(jobs, j)
	345	+	}
	346	+	}
	347	+	} else if w.dumpDir != "" {
	348	+	for _, win := range req.Windows {
	349	+	if !ar.covered[win.ID] {
	350	+	if r := windowCrop(win.Rect, req.Scale, req.Width, req.Height); !r.Empty() {
	351	+	crops = append(crops, r)
	352	+	}
	353	+	}
	354	+	}
	355	+	}
	356	+	if w.dumpDir != "" && fr.pix != nil {
	357	+	// The dump reads the frame after the result is sent and releases it.
	358	+	w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, crops: crops, captureD: captureDur, atspiD: ar.dur}
	359	+	} else {
	360	+	fr.m.unmap() // the crops are extracted: give the frame back
	361	+	}
	362	+	fr.pix = nil
	363	+
	364	+	hits := 0
	365	+	if len(jobs) > 0 {
	366	+	w.recognize(ctx, jobs)
	367	+	for _, j := range jobs {
	368	+	if j.hit {
	369	+	hits++
	370	+	}
	371	+	}
	372	+	}
	373	+	arena.unmap()
	374	+
	375	+	// Merge the bands of each window, in order.
	376	+	k := 0
	377	+	for wi, r := range rects {
	378	+	var words []OCRWord
	379	+	for ; k < len(jobs) && jobs[k].win == wi; k++ {
	380	+	j := jobs[k]
	381	+	if j.err != nil {
	382	+	failureWarning.Do(func() { log.Warn().Err(j.err).Msg("OCR unavailable; install tesseract and tesseract-data-eng") })
	383	+	continue
	384	+	}
	385	+	nextCache[j.key] = bandCache{hash: j.hash, ext: j.ext, words: j.words}
	386	+	for _, word := range j.words {
	387	+	word.Rect = word.Rect.Add(r.Min)
	388	+	if word.Underlined {
	389	+	word.UnderL, word.UnderR = word.UnderL+r.Min.X, word.UnderR+r.Min.X
	390	+	}
	391	+	if !duplicateWord(words, word) {
	392	+	words = append(words, word)
	393	+	}
	394	+	}
	395	+	}
	396	+	allWords = append(allWords, words...)
	397	+	targets := textLinks(words, req.Scale)
	398	+	links += len(targets)
	399	+	out = append(out, targets...)
	400	+	}
	401	+	w.cache = nextCache
	402	+	ocrDur := time.Since(t0)
	403	+	res.Targets = out
	404	+	if p := w.pending; p != nil {
	405	+	p.targets, p.atspi, p.words, p.scanD = out, nAtspi, allWords, ocrDur
	406	+	}
	407	+	log.Info().Str("output", req.Output).
	408	+	Float64("atspi_ms", ms(ar.dur)).Int("atspi_windows", len(ar.covered)).Int("atspi_targets", nAtspi).
	409	+	Int("ocr_windows", len(rects)).Float64("ocr_ms", ms(ocrDur)).Int("ocr_px", pixels).
	410	+	Int("ocr_bands", len(jobs)).Int("band_hits", hits).
	411	+	Int("words", len(allWords)).Int("ocr_links", links).Bool("cache_hit", len(jobs) > 0 && hits == len(jobs)).
	412	+	Int("targets", len(out)).Msg("hint scan")
	413	+	return res
	414	+	}
	415	+
	416	+	// duplicateWord reports whether words already holds word (the same text at
	417	+	// the same place), as happens where two bands overlap.
	418	+	func duplicateWord(words []OCRWord, word OCRWord) bool {
	419	+	for _, old := range words {
	420	+	if strings.EqualFold(word.Text, old.Text) && overlap(word.Rect, old.Rect) > .5 {
	421	+	return true
	422	+	}
	423	+	}
	424	+	return false
	425	+	}
	426	+
	427	+	// bandKey identifies one OCR band of a window on an output.
	428	+	type bandKey struct {
	429	+	output string
	430	+	window ports.WindowID
	431	+	band int
	432	+	bands int
	433	+	}
	434	+
	435	+	// bandCache is the OCR result of a band whose pixels hash to hash. words are
	436	+	// crop-local and only the ones the band owns.
	437	+	type bandCache struct {
	438	+	hash uint64
	439	+	ext image.Rectangle
	440	+	words []OCRWord
	441	+	}
	442	+
	443	+	// windowCrop rounds outwards in physical pixels, then intersects the output.
	444	+	func windowCrop(r ports.Rect, scale float64, width, height int) image.Rectangle {
	445	+	if scale <= 0 {
	446	+	scale = 1
	447	+	}
	448	+	if r.W <= 0 \|\| r.H <= 0 {
	449	+	return image.Rectangle{}
	450	+	}
	451	+	return image.Rect(int(math.Floor(float64(r.X)scale)), int(math.Floor(float64(r.Y)scale)),
	452	+	int(math.Ceil(float64(r.X+r.W)scale)), int(math.Ceil(float64(r.Y+r.H)scale))).Intersect(image.Rect(0, 0, width, height))
	453	+	}
	454	+
	455	+	// cropHash hashes the raw pixels of r without copying them.
	456	+	func (w *worker) cropHash(buf []byte, stride int, r image.Rectangle) uint64 {
	457	+	var h maphash.Hash
	458	+	h.SetSeed(w.cacheSeed)
	459	+	for y := r.Min.Y; y < r.Max.Y; y++ {
	460	+	_, _ = h.Write(buf[ystride+r.Min.X4 : ystride+r.Max.X4])
	461	+	}
	462	+	return h.Sum64()
	463	+	}
	464	+
	465	+	// cropGray converts the BGRA pixels of r to 8-bit luma into dst, which must
	466	+	// hold r.Dx()*r.Dy() bytes; only this band is converted, not the output.
	467	+	func cropGray(buf []byte, stride int, r image.Rectangle, dst []byte) *image.Gray {
	468	+	img := &image.Gray{Pix: dst[:r.Dx()*r.Dy()], Stride: r.Dx(), Rect: image.Rect(0, 0, r.Dx(), r.Dy())}
	469	+	for y := 0; y < r.Dy(); y++ {
	470	+	src := buf[(r.Min.Y+y)stride+r.Min.X4:][:r.Dx()*4]
	471	+	row := img.Pix[y*img.Stride:][:r.Dx()]
	472	+	for x := range row {
	473	+	p := src[x4 : x4+4 : x*4+4]
	474	+	row[x] = uint8((uint32(p[2])299 + uint32(p[1])587 + uint32(p[0])*114 + 500) / 1000)
	475	+	}
	476	+	}
	477	+	return img
	478	+	}
	479	+
	480	+	// pendingSource is an accessibility query in flight.
	481	+	type pendingSource struct {
	482	+	done chan struct{}
	483	+	res atspiResult
	484	+	}
	485	+
	486	+	// startSource starts the accessibility query of req, if there is a source and
	487	+	// a window to ask about. done is closed when res is set.
	488	+	func (w worker) startSource(ctx context.Context, req ports.HintScanRequest) pendingSource {
	489	+	p := &pendingSource{done: make(chan struct{})}
	490	+	if w.source == nil \|\| len(req.Windows) == 0 {
	491	+	close(p.done)
	492	+	return p
	493	+	}
	494	+	go func() {
	495	+	defer close(p.done)
	496	+	t0 := time.Now()
	497	+	targets, covered, err := w.source.Targets(ctx, req.Windows, req.LinksOnly)
	498	+	p.res = atspiResult{targets: targets, covered: covered, dur: time.Since(t0), err: err}
	499	+	}()
	500	+	return p
	501	+	}
	502	+
	503	+	// wait returns the result of the query.
	504	+	func (p *pendingSource) wait() atspiResult {
	505	+	<-p.done
	506	+	return p.res
	507	+	}

@@ -0,0 +1,507 @@
	1	+	// Package hints finds clickable targets for hint mode: it captures the last
	2	+	// frame of an output through the capture pipeline and reads uncovered windows with OCR.
	3	+	package hints
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"fmt"
	9	+	"hash/maphash"
	10	+	"image"
	11	+	"math"
	12	+	"strings"
	13	+	"time"
	14	+
	15	+	"github.com/bnema/neferwl/internal/adapters/logging"
	16	+	"github.com/bnema/neferwl/internal/ports"
	17	+	)
	18	+
	19	+	// ErrBusy answers a request that arrives while another one is in flight.
	20	+	var ErrBusy = errors.New("hints: scan in progress")
	21	+
	22	+	// ErrTimeout is the error of a capture that is not answered in time.
	23	+	var ErrTimeout = errors.New("hints: capture timed out")
	24	+
	25	+	// ErrBadRequest is the error of a request without a usable buffer size.
	26	+	var ErrBadRequest = errors.New("hints: invalid output size")
	27	+
	28	+	const (
	29	+	captureTimeout = time.Second
	30	+	// formatXRGB8888 is the capture format: opaque BGRA in memory.
	31	+	formatXRGB8888 = 1
	32	+	maxPixels = 1 << 28
	33	+	)
	34	+
	35	+	// Run serves requests until ctx is done. It handles one request at a time;
	36	+	// a request that arrives meanwhile is answered with ErrBusy. Every request
	37	+	// gets exactly one result on results.
	38	+	func Run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, opts Options) error {
	39	+	return run(ctx, requests, captures, captured, results, captureTimeout, opts)
	40	+	}
	41	+
	42	+	// targetSource finds hint targets in the accessibility tree of windows. It
	43	+	// returns the targets (logical output-local) and the windows they cover.
	44	+	type targetSource interface {
	45	+	Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)
	46	+	// Close releases the source. It is called once, by the worker goroutine,
	47	+	// when the worker returns; the source is not safe for concurrent use.
	48	+	Close() error
	49	+	}
	50	+
	51	+	// atspiResult is what the source found for one request.
	52	+	type atspiResult struct {
	53	+	targets []ports.HintTarget
	54	+	covered map[ports.WindowID]bool
	55	+	dur time.Duration
	56	+	err error
	57	+	}
	58	+
	59	+	type worker struct {
	60	+	source targetSource
	61	+	cache map[bandKey]bandCache
	62	+	cacheSeed maphash.Seed
	63	+	ocr ocrState
	64	+	warnedSource bool
	65	+	n uint64
	66	+	dumpDir string
	67	+	pending *dumpJob
	68	+	// orphans are capture buffers the output may still write: they are
	69	+	// released when its CaptureDone finally arrives.
	70	+	orphans map[uint64]*mapping
	71	+	live int // mappings not released yet
	72	+	}
	73	+
	74	+	// frame is the captured output: BGRA pixels, nil when no capture was needed.
	75	+	type frame struct {
	76	+	pix []byte
	77	+	stride int
	78	+	m *mapping
	79	+	}
	80	+
	81	+	func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {
	82	+	w := &worker{dumpDir: opts.DumpDir, source: opts.Source}
	83	+	if opts.Source != nil {
	84	+	// The worker goroutine is the only one to touch the source.
	85	+	defer func() { _ = opts.Source.Close() }()
	86	+	}
	87	+	for {
	88	+	select {
	89	+	case <-ctx.Done():
	90	+	return nil
	91	+	case done := <-captured:
	92	+	// A capture abandoned by a timeout is finally done: release it.
	93	+	w.reap(done.ID)
	94	+	case req := <-requests:
	95	+	res := w.serve(ctx, req, requests, captures, captured, results, timeout)
	96	+	if !send(ctx, results, res) {
	97	+	// The dump job will never run: release its frame.
	98	+	if j := w.pending; j != nil {
	99	+	w.pending = nil
	100	+	j.m.unmap()
	101	+	}
	102	+	return nil
	103	+	}
	104	+	w.dump(ctx)
	105	+	}
	106	+	}
	107	+	}
	108	+
	109	+	// dump writes the last successful scan, after its result was sent, then
	110	+	// releases its frame.
	111	+	func (w *worker) dump(ctx context.Context) {
	112	+	j := w.pending
	113	+	w.pending = nil
	114	+	if j == nil {
	115	+	return
	116	+	}
	117	+	defer j.m.unmap()
	118	+	if err := writeDump(w.dumpDir, j); err != nil {
	119	+	log := logging.For(ctx, "hints")
	120	+	log.Warn().Err(err).Str("dir", w.dumpDir).Msg("hint debug dump failed")
	121	+	}
	122	+	}
	123	+
	124	+	func send(ctx context.Context, results chan<- ports.HintScanResult, res ports.HintScanResult) bool {
	125	+	select {
	126	+	case results <- res:
	127	+	return true
	128	+	case <-ctx.Done():
	129	+	return false
	130	+	}
	131	+	}
	132	+
	133	+	// needFrame reports whether the scan must capture the output: OCR has a
	134	+	// window to read, or the debug dump wants the frame. With OCR unavailable
	135	+	// (tesseract missing) and no dump, the accessibility targets are the result.
	136	+	func (w *worker) needFrame(ctx context.Context, req ports.HintScanRequest, ar atspiResult) bool {
	137	+	ocr := w.ocrReady(ctx)
	138	+	if w.dumpDir != "" {
	139	+	return true
	140	+	}
	141	+	if !ocr {
	142	+	return false
	143	+	}
	144	+	for _, win := range req.Windows {
	145	+	if !ar.covered[win.ID] && !windowCrop(win.Rect, req.Scale, req.Width, req.Height).Empty() {
	146	+	return true
	147	+	}
	148	+	}
	149	+	return false
	150	+	}
	151	+
	152	+	func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration) ports.HintScanResult {
	153	+	log := logging.For(ctx, "hints")
	154	+	res := ports.HintScanResult{Seq: req.Seq, Output: req.Output}
	155	+	fail := func(err error) ports.HintScanResult {
	156	+	log.Info().Err(err).Str("output", req.Output).Msg("hint scan failed")
	157	+	res.Err = err
	158	+	return res
	159	+	}
	160	+
	161	+	if req.Width <= 0 \|\| req.Height <= 0 \|\| req.Width*req.Height > maxPixels {
	162	+	return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))
	163	+	}
	164	+	// The accessibility query is always waited for: the source is not shared
	165	+	// between requests. The capture only starts once it is known whether any
	166	+	// window needs OCR.
	167	+	atspi := w.startSource(ctx, req)
	168	+	defer func() { <-atspi.done }()
	169	+	if !w.needFrame(ctx, req, atspi.wait()) {
	170	+	return w.scan(ctx, req, frame{}, 0, res, atspi)
	171	+	}
	172	+	// The capture buffer is anonymous mapped memory, outside the Go heap. It
	173	+	// is released once the crops are extracted, or by the dump job.
	174	+	m := w.alloc(req.Width * req.Height * 4)
	175	+	keep := false // the mapping outlives this call: dump job or pending capture
	176	+	defer func() {
	177	+	if !keep {
	178	+	m.unmap()
	179	+	}
	180	+	}()
	181	+	buf := m.b
	182	+
	183	+	w.n++
	184	+	id := ports.CaptureIDInternal \| w.n
	185	+	stride := req.Width * 4
	186	+	start := time.Now()
	187	+	creq := ports.CaptureRequest{
	188	+	ID: id,
	189	+	Output: req.Output,
	190	+	Region: image.Rect(0, 0, req.Width, req.Height),
	191	+	Mapped: buf,
	192	+	Width: req.Width,
	193	+	Height: req.Height,
	194	+	Stride: stride,
	195	+	Format: formatXRGB8888,
	196	+	Indicate: false,
	197	+	}
	198	+
	199	+	timer := time.NewTimer(timeout)
	200	+	defer timer.Stop()
	201	+
	202	+	select {
	203	+	case captures <- creq:
	204	+	case <-timer.C:
	205	+	return fail(ErrTimeout)
	206	+	case <-ctx.Done():
	207	+	return fail(ctx.Err())
	208	+	}
	209	+	// From here the output owns buf until CaptureDone.
	210	+	abandon := func() {
	211	+	if w.orphans == nil {
	212	+	w.orphans = map[uint64]*mapping{}
	213	+	}
	214	+	w.orphans[id] = m
	215	+	keep = true
	216	+	}
	217	+
	218	+	for {
	219	+	select {
	220	+	case done := <-captured:
	221	+	if done.ID != id {
	222	+	w.reap(done.ID)
	223	+	continue
	224	+	}
	225	+	if done.Err != nil {
	226	+	return fail(fmt.Errorf("hints: capture: %w", done.Err))
	227	+	}
	228	+	res = w.scan(ctx, req, frame{pix: buf, stride: stride, m: m}, time.Since(start), res, atspi)
	229	+	keep = w.pending != nil && w.pending.m == m
	230	+	return res
	231	+	case other := <-requests:
	232	+	// The buffer is in use until the capture is done: answer now.
	233	+	busy := ports.HintScanResult{Seq: other.Seq, Output: other.Output, Err: ErrBusy}
	234	+	if !send(ctx, results, busy) {
	235	+	abandon()
	236	+	return fail(ctx.Err())
	237	+	}
	238	+	case <-timer.C:
	239	+	// The output may still write into buf: it is released when its
	240	+	// CaptureDone arrives.
	241	+	abandon()
	242	+	return fail(ErrTimeout)
	243	+	case <-ctx.Done():
	244	+	abandon()
	245	+	return fail(ctx.Err())
	246	+	}
	247	+	}
	248	+	}
	249	+
	250	+	// reap releases the buffer of an abandoned capture that is finally done.
	251	+	func (w *worker) reap(id uint64) {
	252	+	if m, ok := w.orphans[id]; ok {
	253	+	delete(w.orphans, id)
	254	+	m.unmap()
	255	+	}
	256	+	}
	257	+
	258	+	// bandJob is one OCR band of one window crop.
	259	+	type bandJob struct {
	260	+	win int // index into the scan's windows
	261	+	key bandKey //
	262	+	ext image.Rectangle // crop-local rows read, with context
	263	+	own [2]int // crop-local rows owned: words centred there are kept
	264	+	hash uint64
	265	+	hit bool
	266	+	gray *image.Gray
	267	+	words []OCRWord
	268	+	err error
	269	+	}
	270	+
	271	+	func (w worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame, captureDur time.Duration, res ports.HintScanResult, atspi pendingSource) ports.HintScanResult {
	272	+	log := logging.For(ctx, "hints")
	273	+	ar := atspi.wait()
	274	+	if ar.err != nil && !w.warnedSource {
	275	+	log.Warn().Err(ar.err).Msg("accessibility targets unavailable; using OCR only")
	276	+	w.warnedSource = true
	277	+	}
	278	+	out := make([]ports.HintTarget, 0, len(ar.targets))
	279	+	for _, t := range ar.targets {
	280	+	if !req.LinksOnly \|\| t.Kind == ports.HintLink {
	281	+	out = append(out, t)
	282	+	}
	283	+	}
	284	+	nAtspi := len(out)
	285	+	t0 := time.Now()
	286	+	if w.cacheSeed == (maphash.Seed{}) {
	287	+	w.cacheSeed = maphash.MakeSeed()
	288	+	}
	289	+	nextCache := make(map[bandKey]bandCache)
	290	+	var crops []image.Rectangle
	291	+	var allWords []OCRWord
	292	+	pixels, links := 0, 0
	293	+	ocrOn := fr.pix != nil && w.ocrReady(ctx)
	294	+	// Phase 1 reads the frame: the bands that miss the cache are copied, as
	295	+	// grayscale, into one arena. Phase 2 (OCR) does not need the frame.
	296	+	var jobs []*bandJob
	297	+	var rects []image.Rectangle // crop of each window that has bands
	298	+	var arena *mapping
	299	+	if ocrOn {
	300	+	type pending struct {
	301	+	r image.Rectangle
	302	+	jobs []*bandJob
	303	+	}
	304	+	var wins []pending
	305	+	need := 0
	306	+	for _, win := range req.Windows {
	307	+	if ar.covered[win.ID] {
	308	+	continue
	309	+	}
	310	+	r := windowCrop(win.Rect, req.Scale, req.Width, req.Height)
	311	+	if r.Empty() {
	312	+	continue
	313	+	}
	314	+	crops = append(crops, r)
	315	+	pixels += r.Dx() * r.Dy()
	316	+	p := pending{r: r}
	317	+	for _, b := range bandsOf(r.Dy()) {
	318	+	ext := image.Rect(0, b.top, r.Dx(), b.bottom)
	319	+	j := &bandJob{win: len(wins), ext: ext, own: [2]int{b.ownTop, b.ownBottom},
	320	+	key: bandKey{output: req.Output, window: win.ID, band: b.idx, bands: b.n}}
	321	+	j.hash = w.cropHash(fr.pix, fr.stride, ext.Add(r.Min))
	322	+	entry, ok := w.cache[j.key]
	323	+	if ok && entry.hash == j.hash && entry.ext == ext {
	324	+	j.hit, j.words = true, entry.words
	325	+	} else {
	326	+	need += ext.Dx() * ext.Dy()
	327	+	}
	328	+	p.jobs = append(p.jobs, j)
	329	+	}
	330	+	wins = append(wins, p)
	331	+	}
	332	+	if need > 0 {
	333	+	arena = w.alloc(need)
	334	+	}
	335	+	off := 0
	336	+	for _, p := range wins {
	337	+	rects = append(rects, p.r)
	338	+	for _, j := range p.jobs {
	339	+	if !j.hit {
	340	+	n := j.ext.Dx() * j.ext.Dy()
	341	+	j.gray = cropGray(fr.pix, fr.stride, j.ext.Add(p.r.Min), arena.b[off:off+n])
	342	+	off += n
	343	+	}
	344	+	jobs = append(jobs, j)
	345	+	}
	346	+	}
	347	+	} else if w.dumpDir != "" {
	348	+	for _, win := range req.Windows {
	349	+	if !ar.covered[win.ID] {
	350	+	if r := windowCrop(win.Rect, req.Scale, req.Width, req.Height); !r.Empty() {
	351	+	crops = append(crops, r)
	352	+	}
	353	+	}
	354	+	}
	355	+	}
	356	+	if w.dumpDir != "" && fr.pix != nil {
	357	+	// The dump reads the frame after the result is sent and releases it.
	358	+	w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, crops: crops, captureD: captureDur, atspiD: ar.dur}
	359	+	} else {
	360	+	fr.m.unmap() // the crops are extracted: give the frame back
	361	+	}
	362	+	fr.pix = nil
	363	+
	364	+	hits := 0
	365	+	if len(jobs) > 0 {
	366	+	w.recognize(ctx, jobs)
	367	+	for _, j := range jobs {
	368	+	if j.hit {
	369	+	hits++
	370	+	}
	371	+	}
	372	+	}
	373	+	arena.unmap()
	374	+
	375	+	// Merge the bands of each window, in order.
	376	+	k := 0
	377	+	for wi, r := range rects {
	378	+	var words []OCRWord
	379	+	for ; k < len(jobs) && jobs[k].win == wi; k++ {
	380	+	j := jobs[k]
	381	+	if j.err != nil {
	382	+	failureWarning.Do(func() { log.Warn().Err(j.err).Msg("OCR unavailable; install tesseract and tesseract-data-eng") })
	383	+	continue
	384	+	}
	385	+	nextCache[j.key] = bandCache{hash: j.hash, ext: j.ext, words: j.words}
	386	+	for _, word := range j.words {
	387	+	word.Rect = word.Rect.Add(r.Min)
	388	+	if word.Underlined {
	389	+	word.UnderL, word.UnderR = word.UnderL+r.Min.X, word.UnderR+r.Min.X
	390	+	}
	391	+	if !duplicateWord(words, word) {
	392	+	words = append(words, word)
	393	+	}
	394	+	}
	395	+	}
	396	+	allWords = append(allWords, words...)
	397	+	targets := textLinks(words, req.Scale)
	398	+	links += len(targets)
	399	+	out = append(out, targets...)
	400	+	}
	401	+	w.cache = nextCache
	402	+	ocrDur := time.Since(t0)
	403	+	res.Targets = out
	404	+	if p := w.pending; p != nil {
	405	+	p.targets, p.atspi, p.words, p.scanD = out, nAtspi, allWords, ocrDur
	406	+	}
	407	+	log.Info().Str("output", req.Output).
	408	+	Float64("atspi_ms", ms(ar.dur)).Int("atspi_windows", len(ar.covered)).Int("atspi_targets", nAtspi).
	409	+	Int("ocr_windows", len(rects)).Float64("ocr_ms", ms(ocrDur)).Int("ocr_px", pixels).
	410	+	Int("ocr_bands", len(jobs)).Int("band_hits", hits).
	411	+	Int("words", len(allWords)).Int("ocr_links", links).Bool("cache_hit", len(jobs) > 0 && hits == len(jobs)).
	412	+	Int("targets", len(out)).Msg("hint scan")
	413	+	return res
	414	+	}
	415	+
	416	+	// duplicateWord reports whether words already holds word (the same text at
	417	+	// the same place), as happens where two bands overlap.
	418	+	func duplicateWord(words []OCRWord, word OCRWord) bool {
	419	+	for _, old := range words {
	420	+	if strings.EqualFold(word.Text, old.Text) && overlap(word.Rect, old.Rect) > .5 {
	421	+	return true
	422	+	}
	423	+	}
	424	+	return false
	425	+	}
	426	+
	427	+	// bandKey identifies one OCR band of a window on an output.
	428	+	type bandKey struct {
	429	+	output string
	430	+	window ports.WindowID
	431	+	band int
	432	+	bands int
	433	+	}
	434	+
	435	+	// bandCache is the OCR result of a band whose pixels hash to hash. words are
	436	+	// crop-local and only the ones the band owns.
	437	+	type bandCache struct {
	438	+	hash uint64
	439	+	ext image.Rectangle
	440	+	words []OCRWord
	441	+	}
	442	+
	443	+	// windowCrop rounds outwards in physical pixels, then intersects the output.
	444	+	func windowCrop(r ports.Rect, scale float64, width, height int) image.Rectangle {
	445	+	if scale <= 0 {
	446	+	scale = 1
	447	+	}
	448	+	if r.W <= 0 \|\| r.H <= 0 {
	449	+	return image.Rectangle{}
	450	+	}
	451	+	return image.Rect(int(math.Floor(float64(r.X)scale)), int(math.Floor(float64(r.Y)scale)),
	452	+	int(math.Ceil(float64(r.X+r.W)scale)), int(math.Ceil(float64(r.Y+r.H)scale))).Intersect(image.Rect(0, 0, width, height))
	453	+	}
	454	+
	455	+	// cropHash hashes the raw pixels of r without copying them.
	456	+	func (w *worker) cropHash(buf []byte, stride int, r image.Rectangle) uint64 {
	457	+	var h maphash.Hash
	458	+	h.SetSeed(w.cacheSeed)
	459	+	for y := r.Min.Y; y < r.Max.Y; y++ {
	460	+	_, _ = h.Write(buf[ystride+r.Min.X4 : ystride+r.Max.X4])
	461	+	}
	462	+	return h.Sum64()
	463	+	}
	464	+
	465	+	// cropGray converts the BGRA pixels of r to 8-bit luma into dst, which must
	466	+	// hold r.Dx()*r.Dy() bytes; only this band is converted, not the output.
	467	+	func cropGray(buf []byte, stride int, r image.Rectangle, dst []byte) *image.Gray {
	468	+	img := &image.Gray{Pix: dst[:r.Dx()*r.Dy()], Stride: r.Dx(), Rect: image.Rect(0, 0, r.Dx(), r.Dy())}
	469	+	for y := 0; y < r.Dy(); y++ {
	470	+	src := buf[(r.Min.Y+y)stride+r.Min.X4:][:r.Dx()*4]
	471	+	row := img.Pix[y*img.Stride:][:r.Dx()]
	472	+	for x := range row {
	473	+	p := src[x4 : x4+4 : x*4+4]
	474	+	row[x] = uint8((uint32(p[2])299 + uint32(p[1])587 + uint32(p[0])*114 + 500) / 1000)
	475	+	}
	476	+	}
	477	+	return img
	478	+	}
	479	+
	480	+	// pendingSource is an accessibility query in flight.
	481	+	type pendingSource struct {
	482	+	done chan struct{}
	483	+	res atspiResult
	484	+	}
	485	+
	486	+	// startSource starts the accessibility query of req, if there is a source and
	487	+	// a window to ask about. done is closed when res is set.
	488	+	func (w worker) startSource(ctx context.Context, req ports.HintScanRequest) pendingSource {
	489	+	p := &pendingSource{done: make(chan struct{})}
	490	+	if w.source == nil \|\| len(req.Windows) == 0 {
	491	+	close(p.done)
	492	+	return p
	493	+	}
	494	+	go func() {
	495	+	defer close(p.done)
	496	+	t0 := time.Now()
	497	+	targets, covered, err := w.source.Targets(ctx, req.Windows, req.LinksOnly)
	498	+	p.res = atspiResult{targets: targets, covered: covered, dur: time.Since(t0), err: err}
	499	+	}()
	500	+	return p
	501	+	}
	502	+
	503	+	// wait returns the result of the query.
	504	+	func (p *pendingSource) wait() atspiResult {
	505	+	<-p.done
	506	+	return p.res
	507	+	}

A internal/adapters/hints/hints_test.go +436 -0

@@ -0,0 +1,436 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"fmt"
	8	+	"image"
	9	+	"image/draw"
	10	+	_ "image/png"
	11	+	"os"
	12	+	"os/exec"
	13	+	"path/filepath"
	14	+	"strings"
	15	+	"testing"
	16	+	"time"
	17	+
	18	+	"github.com/stretchr/testify/mock"
	19	+	"github.com/stretchr/testify/require"
	20	+
	21	+	"github.com/bnema/neferwl/internal/ports"
	22	+	)
	23	+
	24	+	type harness struct {
	25	+	requests chan ports.HintScanRequest
	26	+	captures chan ports.CaptureRequest
	27	+	captured chan ports.CaptureDone
	28	+	results chan ports.HintScanResult
	29	+	}
	30	+
	31	+	func start(t testing.T, timeout time.Duration) harness {
	32	+	return startWith(t, timeout, Options{})
	33	+	}
	34	+
	35	+	// startCapturing starts a worker that always captures: debug dumps are on.
	36	+	func startCapturing(t testing.T, timeout time.Duration) harness {
	37	+	return startWith(t, timeout, Options{DumpDir: filepath.Join(t.TempDir(), "hints")})
	38	+	}
	39	+
	40	+	func startWith(t testing.T, timeout time.Duration, opts Options) harness {
	41	+	t.Helper()
	42	+	h := &harness{
	43	+	requests: make(chan ports.HintScanRequest, 1),
	44	+	captures: make(chan ports.CaptureRequest, 4),
	45	+	captured: make(chan ports.CaptureDone, 1),
	46	+	results: make(chan ports.HintScanResult, 4),
	47	+	}
	48	+	ctx, cancel := context.WithCancel(context.Background())
	49	+	exited := make(chan struct{})
	50	+	go func() {
	51	+	defer close(exited)
	52	+	_ = run(ctx, h.requests, h.captures, h.captured, h.results, timeout, opts)
	53	+	}()
	54	+	t.Cleanup(func() { cancel(); <-exited })
	55	+	return h
	56	+	}
	57	+
	58	+	func loadBGRA(t *testing.T) (pix []byte, w, hgt int) {
	59	+	t.Helper()
	60	+	f, err := os.Open("testdata/browser.png")
	61	+	require.NoError(t, err)
	62	+	defer f.Close()
	63	+	img, _, err := image.Decode(f)
	64	+	require.NoError(t, err)
	65	+	b := img.Bounds()
	66	+	rgba := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
	67	+	draw.Draw(rgba, rgba.Bounds(), img, b.Min, draw.Src)
	68	+	for i := 0; i < len(rgba.Pix); i += 4 {
	69	+	rgba.Pix[i], rgba.Pix[i+2] = rgba.Pix[i+2], rgba.Pix[i]
	70	+	rgba.Pix[i+3] = 0xff
	71	+	}
	72	+	return rgba.Pix, b.Dx(), b.Dy()
	73	+	}
	74	+
	75	+	func TestHintScan(t *testing.T) {
	76	+	if _, err := exec.LookPath("tesseract"); err != nil {
	77	+	t.Skip("tesseract not on PATH")
	78	+	}
	79	+	pix, w, hgt := loadBGRA(t)
	80	+	h := start(t, 5*time.Second)
	81	+
	82	+	// Plays the output owner.
	83	+	go func() {
	84	+	creq := <-h.captures
	85	+	copy(creq.Mapped, pix)
	86	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	87	+	}()
	88	+
	89	+	h.requests <- ports.HintScanRequest{Seq: 7, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
	90	+	res := <-h.results
	91	+	require.NoError(t, res.Err)
	92	+	require.Equal(t, uint64(7), res.Seq)
	93	+	require.Equal(t, "HEADLESS-1", res.Output)
	94	+	var links int
	95	+	for _, tg := range res.Targets {
	96	+	require.Positive(t, tg.Rect.W)
	97	+	require.Positive(t, tg.Rect.H)
	98	+	if tg.Kind == ports.HintLink {
	99	+	links++
	100	+	}
	101	+	}
	102	+	require.GreaterOrEqual(t, links, 1)
	103	+	}
	104	+
	105	+	func TestHintRequestShape(t *testing.T) {
	106	+	h := startCapturing(t, 5*time.Second)
	107	+	got := make(chan ports.CaptureRequest, 1)
	108	+	go func() {
	109	+	creq := <-h.captures
	110	+	got <- creq
	111	+	h.captured <- ports.CaptureDone{ID: creq.ID + 1} // another ID: ignored
	112	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output} // blank frame
	113	+	}()
	114	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 64, Height: 32, Scale: 1}
	115	+	res := <-h.results
	116	+	require.NoError(t, res.Err)
	117	+	creq := <-got
	118	+	require.NotZero(t, creq.ID&ports.CaptureIDInternal)
	119	+	require.False(t, creq.Indicate)
	120	+	require.Equal(t, image.Rect(0, 0, 64, 32), creq.Region)
	121	+	require.Len(t, creq.Mapped, 64324)
	122	+	require.Equal(t, 64*4, creq.Stride)
	123	+	require.Equal(t, uint32(1), creq.Format)
	124	+	}
	125	+
	126	+	func TestHintCaptureError(t *testing.T) {
	127	+	h := startCapturing(t, 5*time.Second)
	128	+	boom := errors.New("boom")
	129	+	go func() {
	130	+	creq := <-h.captures
	131	+	h.captured <- ports.CaptureDone{ID: creq.ID, Err: boom}
	132	+	}()
	133	+	h.requests <- ports.HintScanRequest{Seq: 3, Output: "o", Width: 64, Height: 32, Scale: 1}
	134	+	res := <-h.results
	135	+	require.ErrorIs(t, res.Err, boom)
	136	+	require.Equal(t, uint64(3), res.Seq)
	137	+	require.Empty(t, res.Targets)
	138	+	}
	139	+
	140	+	func TestHintTimeout(t *testing.T) {
	141	+	h := startCapturing(t, 50*time.Millisecond)
	142	+	h.requests <- ports.HintScanRequest{Seq: 4, Output: "o", Width: 64, Height: 32, Scale: 1}
	143	+	res := <-h.results
	144	+	require.ErrorIs(t, res.Err, ErrTimeout)
	145	+	require.Equal(t, uint64(4), res.Seq)
	146	+	}
	147	+
	148	+	func TestHintBusy(t *testing.T) {
	149	+	h := startCapturing(t, 5*time.Second)
	150	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 64, Height: 32, Scale: 1}
	151	+	creq := <-h.captures
	152	+	h.requests <- ports.HintScanRequest{Seq: 2, Output: "o", Width: 64, Height: 32, Scale: 1}
	153	+	res := <-h.results
	154	+	require.ErrorIs(t, res.Err, ErrBusy)
	155	+	require.Equal(t, uint64(2), res.Seq)
	156	+	h.captured <- ports.CaptureDone{ID: creq.ID}
	157	+	res = <-h.results
	158	+	require.NoError(t, res.Err)
	159	+	require.Equal(t, uint64(1), res.Seq)
	160	+	}
	161	+
	162	+	func TestHintBadSize(t *testing.T) {
	163	+	h := start(t, time.Second)
	164	+	h.requests <- ports.HintScanRequest{Seq: 5, Output: "o"}
	165	+	res := <-h.results
	166	+	require.ErrorIs(t, res.Err, ErrBadRequest)
	167	+	}
	168	+
	169	+	func scanOnce(t testing.T, h harness, pix []byte, w, hgt int, seq uint64) ports.HintScanResult {
	170	+	t.Helper()
	171	+	go func() {
	172	+	creq := <-h.captures
	173	+	copy(creq.Mapped, pix)
	174	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	175	+	}()
	176	+	h.requests <- ports.HintScanRequest{Seq: seq, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
	177	+	res := <-h.results
	178	+	require.NoError(t, res.Err)
	179	+	return res
	180	+	}
	181	+
	182	+	// waitFiles waits for the dump the worker writes after sending the result.
	183	+	func waitFiles(t *testing.T, dir string, n int) []os.DirEntry {
	184	+	t.Helper()
	185	+	var entries []os.DirEntry
	186	+	require.Eventually(t, func() bool {
	187	+	entries, _ = os.ReadDir(dir)
	188	+	return len(entries) >= n
	189	+	}, 5time.Second, 10time.Millisecond)
	190	+	return entries
	191	+	}
	192	+
	193	+	func TestHintDump(t *testing.T) {
	194	+	if _, err := exec.LookPath("tesseract"); err != nil {
	195	+	t.Skip("tesseract not on PATH")
	196	+	}
	197	+	pix, w, hgt := loadBGRA(t)
	198	+	dir := filepath.Join(t.TempDir(), "hints")
	199	+	h := startWith(t, 5*time.Second, Options{DumpDir: dir})
	200	+	res := scanOnce(t, h, pix, w, hgt, 1)
	201	+	require.NotEmpty(t, res.Targets)
	202	+
	203	+	entries := waitFiles(t, dir, 3)
	204	+	// A second request is only served once the dump is written.
	205	+	_ = scanOnceDone(t, h)
	206	+	var png, ann, js string
	207	+	for _, e := range entries {
	208	+	info, err := e.Info()
	209	+	require.NoError(t, err)
	210	+	require.Equal(t, os.FileMode(0o600), info.Mode().Perm())
	211	+	switch {
	212	+	case strings.HasSuffix(e.Name(), ".annotated.png"):
	213	+	ann = e.Name()
	214	+	case strings.HasSuffix(e.Name(), ".png"):
	215	+	png = e.Name()
	216	+	case strings.HasSuffix(e.Name(), ".json"):
	217	+	js = e.Name()
	218	+	}
	219	+	}
	220	+	require.NotEmpty(t, png)
	221	+	require.NotEmpty(t, ann)
	222	+	require.Regexp(t, `^\d{8}-\d{6}\.\d{3}-HEADLESS-1\.json$`, js)
	223	+
	224	+	st, err := os.Stat(dir)
	225	+	require.NoError(t, err)
	226	+	require.Equal(t, os.FileMode(0o700), st.Mode().Perm())
	227	+
	228	+	data, err := os.ReadFile(filepath.Join(dir, js))
	229	+	require.NoError(t, err)
	230	+	var meta struct {
	231	+	Output string `json:"output"`
	232	+	Width int `json:"width"`
	233	+	Height int `json:"height"`
	234	+	Stride int `json:"stride"`
	235	+	Scale float64 `json:"scale"`
	236	+	LinksOnly bool `json:"links_only"`
	237	+	Targets []struct {
	238	+	Kind string `json:"kind"`
	239	+	Rect [4]int `json:"rect"`
	240	+	} `json:"targets"`
	241	+	}
	242	+	require.NoError(t, json.Unmarshal(data, &meta))
	243	+	require.Equal(t, "HEADLESS-1", meta.Output)
	244	+	require.Equal(t, w, meta.Width)
	245	+	require.Equal(t, hgt, meta.Height)
	246	+	require.Equal(t, w*4, meta.Stride)
	247	+	require.Equal(t, 1.0, meta.Scale)
	248	+	require.Len(t, meta.Targets, len(res.Targets))
	249	+	for i, tg := range res.Targets {
	250	+	require.Equal(t, [4]int{tg.Rect.X, tg.Rect.Y, tg.Rect.X + tg.Rect.W, tg.Rect.Y + tg.Rect.H}, meta.Targets[i].Rect)
	251	+	}
	252	+
	253	+	f, err := os.Open(filepath.Join(dir, png))
	254	+	require.NoError(t, err)
	255	+	defer f.Close()
	256	+	img, _, err := image.Decode(f)
	257	+	require.NoError(t, err)
	258	+	require.Equal(t, image.Rect(0, 0, w, hgt), img.Bounds())
	259	+	r, g, b, _ := img.At(0, 0).RGBA()
	260	+	require.Equal(t, uint32(pix[2])*0x101, r)
	261	+	require.Equal(t, uint32(pix[1])*0x101, g)
	262	+	require.Equal(t, uint32(pix[0])*0x101, b)
	263	+	}
	264	+
	265	+	// scanOnceDone sends a request that fails fast on a blank capture and waits
	266	+	// for its result, which proves the previous dump finished.
	267	+	func scanOnceDone(t testing.T, h harness) ports.HintScanResult {
	268	+	t.Helper()
	269	+	go func() {
	270	+	creq := <-h.captures
	271	+	h.captured <- ports.CaptureDone{ID: creq.ID, Err: errors.New("stop")}
	272	+	}()
	273	+	h.requests <- ports.HintScanRequest{Seq: 99, Output: "x", Width: 8, Height: 8, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 8, H: 8}}}}
	274	+	return <-h.results
	275	+	}
	276	+
	277	+	func TestHintDumpDisabled(t *testing.T) {
	278	+	if _, err := exec.LookPath("tesseract"); err != nil {
	279	+	t.Skip("tesseract not on PATH")
	280	+	}
	281	+	pix, w, hgt := loadBGRA(t)
	282	+	dir := filepath.Join(t.TempDir(), "hints")
	283	+	h := start(t, 5*time.Second)
	284	+	scanOnce(t, h, pix, w, hgt, 1)
	285	+	scanOnceDone(t, h)
	286	+	_, err := os.Stat(dir)
	287	+	require.ErrorIs(t, err, os.ErrNotExist)
	288	+	}
	289	+
	290	+	func TestHintDumpRetention(t *testing.T) {
	291	+	dir := t.TempDir()
	292	+	for i := 0; i < 60; i++ {
	293	+	for _, ext := range []string{".png", ".annotated.png", ".json"} {
	294	+	name := fmt.Sprintf("20260101-000000.%03d-o%s", i, ext)
	295	+	require.NoError(t, os.WriteFile(filepath.Join(dir, name), nil, 0o600))
	296	+	}
	297	+	}
	298	+	require.NoError(t, prune(dir, maxDumps))
	299	+	entries, err := os.ReadDir(dir)
	300	+	require.NoError(t, err)
	301	+	require.Len(t, entries, maxDumps*3)
	302	+	require.Equal(t, "20260101-000000.010-o.annotated.png", entries[0].Name())
	303	+	}
	304	+
	305	+	func TestScanMergesSource(t *testing.T) {
	306	+	pix, w, hgt := loadBGRA(t)
	307	+	src := newMocktargetSource(t)
	308	+	src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
	309	+	wins := []ports.HintWindow{{ID: 4, PID: 9, Rect: ports.Rect{X: 0, Y: 0, W: w, H: hgt}}}
	310	+	at := []ports.HintTarget{{Rect: ports.Rect{X: 5, Y: 5, W: 30, H: 10}, Kind: ports.HintLink}}
	311	+	src.EXPECT().Targets(mock.Anything, wins, false).Return(at, map[ports.WindowID]bool{4: true}, nil).Once()
	312	+	h := startWith(t, 5*time.Second, Options{Source: src})
	313	+	go func() {
	314	+	creq := <-h.captures
	315	+	copy(creq.Mapped, pix)
	316	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	317	+	}()
	318	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: wins}
	319	+	res := <-h.results
	320	+	require.NoError(t, res.Err)
	321	+	// The whole frame is covered: only the accessibility target remains.
	322	+	require.Equal(t, at, res.Targets)
	323	+	}
	324	+
	325	+	func TestScanSourceErrorUsesOCR(t *testing.T) {
	326	+	if _, err := exec.LookPath("tesseract"); err != nil {
	327	+	t.Skip("tesseract not on PATH")
	328	+	}
	329	+	pix, w, hgt := loadBGRA(t)
	330	+	src := newMocktargetSource(t)
	331	+	src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
	332	+	wins := []ports.HintWindow{{ID: 4, PID: 9, Rect: ports.Rect{W: w, H: hgt}}}
	333	+	src.EXPECT().Targets(mock.Anything, wins, false).Return(nil, nil, errors.New("bus gone")).Once()
	334	+	h := startWith(t, 5*time.Second, Options{Source: src})
	335	+	go func() {
	336	+	creq := <-h.captures
	337	+	copy(creq.Mapped, pix)
	338	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	339	+	}()
	340	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: wins}
	341	+	res := <-h.results
	342	+	require.NoError(t, res.Err)
	343	+	require.NotEmpty(t, res.Targets)
	344	+	}
	345	+
	346	+	// The worker keeps no frame: the mapped buffer belongs to the scan, and to the
	347	+	// dump job until it is written; every mapping is released afterwards.
	348	+	func TestWorkerHoldsNoFrameAfterScan(t *testing.T) {
	349	+	for _, dumpDir := range []string{"", filepath.Join(t.TempDir(), "hints")} {
	350	+	// OCR is "installed" but its binary cannot run: only the memory matters.
	351	+	w := &worker{dumpDir: dumpDir, ocr: ocrState{done: true, binary: "/nonexistent/tesseract", lang: "eng"}}
	352	+	requests := make(chan ports.HintScanRequest)
	353	+	captures := make(chan ports.CaptureRequest, 1)
	354	+	captured := make(chan ports.CaptureDone, 1)
	355	+	results := make(chan ports.HintScanResult, 1)
	356	+	go func() {
	357	+	creq := <-captures
	358	+	captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	359	+	}()
	360	+	req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 64, H: 32}}}}
	361	+	res := w.serve(context.Background(), req, requests, captures, captured, results, 5*time.Second)
	362	+	require.NoError(t, res.Err)
	363	+	if dumpDir == "" {
	364	+	require.Nil(t, w.pending)
	365	+	require.Zero(t, w.live, "frame and crops are released before the result")
	366	+	} else {
	367	+	require.NotNil(t, w.pending)
	368	+	require.Equal(t, 1, w.live, "only the frame, kept for the dump")
	369	+	}
	370	+	w.dump(context.Background())
	371	+	require.Nil(t, w.pending)
	372	+	require.Zero(t, w.live)
	373	+	require.Empty(t, w.cache) // OCR failed: nothing cached
	374	+	}
	375	+	}
	376	+
	377	+	// A capture that timed out stays mapped until the output answers it.
	378	+	func TestTimedOutCaptureIsReleasedWhenDone(t *testing.T) {
	379	+	w := &worker{dumpDir: t.TempDir()}
	380	+	captures := make(chan ports.CaptureRequest, 1)
	381	+	captured := make(chan ports.CaptureDone, 1)
	382	+	req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1}
	383	+	res := w.serve(context.Background(), req, make(chan ports.HintScanRequest), captures, captured, make(chan ports.HintScanResult, 1), 20*time.Millisecond)
	384	+	require.ErrorIs(t, res.Err, ErrTimeout)
	385	+	creq := <-captures
	386	+	require.Equal(t, 1, w.live)
	387	+	creq.Mapped[0] = 1 // still valid: the output may write until CaptureDone
	388	+	w.reap(creq.ID)
	389	+	require.Zero(t, w.live)
	390	+	}
	391	+
	392	+	// Without tesseract OCR is off for good: no capture, only accessibility targets.
	393	+	func TestNoTesseractSkipsCapture(t *testing.T) {
	394	+	t.Setenv("PATH", t.TempDir())
	395	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
	396	+	covered := ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: 30, W: 5, H: 5}}
	397	+	wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}, {ID: 2, Rect: ports.Rect{X: 20, W: 20, H: 20}}}
	398	+	src := newMocktargetSource(t)
	399	+	src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
	400	+	src.EXPECT().Targets(mock.Anything, wins, false).Return([]ports.HintTarget{button, covered}, map[ports.WindowID]bool{2: true}, nil).Times(2)
	401	+	h := startWith(t, time.Second, Options{Source: src})
	402	+	for seq := uint64(1); seq <= 2; seq++ {
	403	+	h.requests <- ports.HintScanRequest{Seq: seq, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: wins}
	404	+	res := <-h.results
	405	+	require.NoError(t, res.Err)
	406	+	require.Equal(t, []ports.HintTarget{button, covered}, res.Targets)
	407	+	}
	408	+	require.Empty(t, h.captures)
	409	+	}
	410	+
	411	+	// With debug dumps on, the frame is still captured (for the dump only).
	412	+	func TestNoTesseractDumpStillCaptures(t *testing.T) {
	413	+	t.Setenv("PATH", t.TempDir())
	414	+	dir := filepath.Join(t.TempDir(), "hints")
	415	+	h := startWith(t, 5*time.Second, Options{DumpDir: dir})
	416	+	go func() {
	417	+	creq := <-h.captures
	418	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	419	+	}()
	420	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 40, H: 20}}}}
	421	+	require.NoError(t, (<-h.results).Err)
	422	+	waitFiles(t, dir, 3)
	423	+	}
	424	+
	425	+	// A capture that completes after its scan timed out is released while the
	426	+	// worker is idle, without another scan, and its reply does not linger.
	427	+	func TestHintLateCaptureReaped(t *testing.T) {
	428	+	h := startCapturing(t, 50*time.Millisecond)
	429	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 64, Height: 32, Scale: 1}
	430	+	creq := <-h.captures
	431	+	res := <-h.results
	432	+	require.ErrorIs(t, res.Err, ErrTimeout)
	433	+	// The worker is idle; the output finally answers.
	434	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	435	+	require.Eventually(t, func() bool { return len(h.captured) == 0 }, time.Second, time.Millisecond)
	436	+	}

@@ -0,0 +1,436 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"fmt"
	8	+	"image"
	9	+	"image/draw"
	10	+	_ "image/png"
	11	+	"os"
	12	+	"os/exec"
	13	+	"path/filepath"
	14	+	"strings"
	15	+	"testing"
	16	+	"time"
	17	+
	18	+	"github.com/stretchr/testify/mock"
	19	+	"github.com/stretchr/testify/require"
	20	+
	21	+	"github.com/bnema/neferwl/internal/ports"
	22	+	)
	23	+
	24	+	type harness struct {
	25	+	requests chan ports.HintScanRequest
	26	+	captures chan ports.CaptureRequest
	27	+	captured chan ports.CaptureDone
	28	+	results chan ports.HintScanResult
	29	+	}
	30	+
	31	+	func start(t testing.T, timeout time.Duration) harness {
	32	+	return startWith(t, timeout, Options{})
	33	+	}
	34	+
	35	+	// startCapturing starts a worker that always captures: debug dumps are on.
	36	+	func startCapturing(t testing.T, timeout time.Duration) harness {
	37	+	return startWith(t, timeout, Options{DumpDir: filepath.Join(t.TempDir(), "hints")})
	38	+	}
	39	+
	40	+	func startWith(t testing.T, timeout time.Duration, opts Options) harness {
	41	+	t.Helper()
	42	+	h := &harness{
	43	+	requests: make(chan ports.HintScanRequest, 1),
	44	+	captures: make(chan ports.CaptureRequest, 4),
	45	+	captured: make(chan ports.CaptureDone, 1),
	46	+	results: make(chan ports.HintScanResult, 4),
	47	+	}
	48	+	ctx, cancel := context.WithCancel(context.Background())
	49	+	exited := make(chan struct{})
	50	+	go func() {
	51	+	defer close(exited)
	52	+	_ = run(ctx, h.requests, h.captures, h.captured, h.results, timeout, opts)
	53	+	}()
	54	+	t.Cleanup(func() { cancel(); <-exited })
	55	+	return h
	56	+	}
	57	+
	58	+	func loadBGRA(t *testing.T) (pix []byte, w, hgt int) {
	59	+	t.Helper()
	60	+	f, err := os.Open("testdata/browser.png")
	61	+	require.NoError(t, err)
	62	+	defer f.Close()
	63	+	img, _, err := image.Decode(f)
	64	+	require.NoError(t, err)
	65	+	b := img.Bounds()
	66	+	rgba := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
	67	+	draw.Draw(rgba, rgba.Bounds(), img, b.Min, draw.Src)
	68	+	for i := 0; i < len(rgba.Pix); i += 4 {
	69	+	rgba.Pix[i], rgba.Pix[i+2] = rgba.Pix[i+2], rgba.Pix[i]
	70	+	rgba.Pix[i+3] = 0xff
	71	+	}
	72	+	return rgba.Pix, b.Dx(), b.Dy()
	73	+	}
	74	+
	75	+	func TestHintScan(t *testing.T) {
	76	+	if _, err := exec.LookPath("tesseract"); err != nil {
	77	+	t.Skip("tesseract not on PATH")
	78	+	}
	79	+	pix, w, hgt := loadBGRA(t)
	80	+	h := start(t, 5*time.Second)
	81	+
	82	+	// Plays the output owner.
	83	+	go func() {
	84	+	creq := <-h.captures
	85	+	copy(creq.Mapped, pix)
	86	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	87	+	}()
	88	+
	89	+	h.requests <- ports.HintScanRequest{Seq: 7, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
	90	+	res := <-h.results
	91	+	require.NoError(t, res.Err)
	92	+	require.Equal(t, uint64(7), res.Seq)
	93	+	require.Equal(t, "HEADLESS-1", res.Output)
	94	+	var links int
	95	+	for _, tg := range res.Targets {
	96	+	require.Positive(t, tg.Rect.W)
	97	+	require.Positive(t, tg.Rect.H)
	98	+	if tg.Kind == ports.HintLink {
	99	+	links++
	100	+	}
	101	+	}
	102	+	require.GreaterOrEqual(t, links, 1)
	103	+	}
	104	+
	105	+	func TestHintRequestShape(t *testing.T) {
	106	+	h := startCapturing(t, 5*time.Second)
	107	+	got := make(chan ports.CaptureRequest, 1)
	108	+	go func() {
	109	+	creq := <-h.captures
	110	+	got <- creq
	111	+	h.captured <- ports.CaptureDone{ID: creq.ID + 1} // another ID: ignored
	112	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output} // blank frame
	113	+	}()
	114	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 64, Height: 32, Scale: 1}
	115	+	res := <-h.results
	116	+	require.NoError(t, res.Err)
	117	+	creq := <-got
	118	+	require.NotZero(t, creq.ID&ports.CaptureIDInternal)
	119	+	require.False(t, creq.Indicate)
	120	+	require.Equal(t, image.Rect(0, 0, 64, 32), creq.Region)
	121	+	require.Len(t, creq.Mapped, 64324)
	122	+	require.Equal(t, 64*4, creq.Stride)
	123	+	require.Equal(t, uint32(1), creq.Format)
	124	+	}
	125	+
	126	+	func TestHintCaptureError(t *testing.T) {
	127	+	h := startCapturing(t, 5*time.Second)
	128	+	boom := errors.New("boom")
	129	+	go func() {
	130	+	creq := <-h.captures
	131	+	h.captured <- ports.CaptureDone{ID: creq.ID, Err: boom}
	132	+	}()
	133	+	h.requests <- ports.HintScanRequest{Seq: 3, Output: "o", Width: 64, Height: 32, Scale: 1}
	134	+	res := <-h.results
	135	+	require.ErrorIs(t, res.Err, boom)
	136	+	require.Equal(t, uint64(3), res.Seq)
	137	+	require.Empty(t, res.Targets)
	138	+	}
	139	+
	140	+	func TestHintTimeout(t *testing.T) {
	141	+	h := startCapturing(t, 50*time.Millisecond)
	142	+	h.requests <- ports.HintScanRequest{Seq: 4, Output: "o", Width: 64, Height: 32, Scale: 1}
	143	+	res := <-h.results
	144	+	require.ErrorIs(t, res.Err, ErrTimeout)
	145	+	require.Equal(t, uint64(4), res.Seq)
	146	+	}
	147	+
	148	+	func TestHintBusy(t *testing.T) {
	149	+	h := startCapturing(t, 5*time.Second)
	150	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 64, Height: 32, Scale: 1}
	151	+	creq := <-h.captures
	152	+	h.requests <- ports.HintScanRequest{Seq: 2, Output: "o", Width: 64, Height: 32, Scale: 1}
	153	+	res := <-h.results
	154	+	require.ErrorIs(t, res.Err, ErrBusy)
	155	+	require.Equal(t, uint64(2), res.Seq)
	156	+	h.captured <- ports.CaptureDone{ID: creq.ID}
	157	+	res = <-h.results
	158	+	require.NoError(t, res.Err)
	159	+	require.Equal(t, uint64(1), res.Seq)
	160	+	}
	161	+
	162	+	func TestHintBadSize(t *testing.T) {
	163	+	h := start(t, time.Second)
	164	+	h.requests <- ports.HintScanRequest{Seq: 5, Output: "o"}
	165	+	res := <-h.results
	166	+	require.ErrorIs(t, res.Err, ErrBadRequest)
	167	+	}
	168	+
	169	+	func scanOnce(t testing.T, h harness, pix []byte, w, hgt int, seq uint64) ports.HintScanResult {
	170	+	t.Helper()
	171	+	go func() {
	172	+	creq := <-h.captures
	173	+	copy(creq.Mapped, pix)
	174	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	175	+	}()
	176	+	h.requests <- ports.HintScanRequest{Seq: seq, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
	177	+	res := <-h.results
	178	+	require.NoError(t, res.Err)
	179	+	return res
	180	+	}
	181	+
	182	+	// waitFiles waits for the dump the worker writes after sending the result.
	183	+	func waitFiles(t *testing.T, dir string, n int) []os.DirEntry {
	184	+	t.Helper()
	185	+	var entries []os.DirEntry
	186	+	require.Eventually(t, func() bool {
	187	+	entries, _ = os.ReadDir(dir)
	188	+	return len(entries) >= n
	189	+	}, 5time.Second, 10time.Millisecond)
	190	+	return entries
	191	+	}
	192	+
	193	+	func TestHintDump(t *testing.T) {
	194	+	if _, err := exec.LookPath("tesseract"); err != nil {
	195	+	t.Skip("tesseract not on PATH")
	196	+	}
	197	+	pix, w, hgt := loadBGRA(t)
	198	+	dir := filepath.Join(t.TempDir(), "hints")
	199	+	h := startWith(t, 5*time.Second, Options{DumpDir: dir})
	200	+	res := scanOnce(t, h, pix, w, hgt, 1)
	201	+	require.NotEmpty(t, res.Targets)
	202	+
	203	+	entries := waitFiles(t, dir, 3)
	204	+	// A second request is only served once the dump is written.
	205	+	_ = scanOnceDone(t, h)
	206	+	var png, ann, js string
	207	+	for _, e := range entries {
	208	+	info, err := e.Info()
	209	+	require.NoError(t, err)
	210	+	require.Equal(t, os.FileMode(0o600), info.Mode().Perm())
	211	+	switch {
	212	+	case strings.HasSuffix(e.Name(), ".annotated.png"):
	213	+	ann = e.Name()
	214	+	case strings.HasSuffix(e.Name(), ".png"):
	215	+	png = e.Name()
	216	+	case strings.HasSuffix(e.Name(), ".json"):
	217	+	js = e.Name()
	218	+	}
	219	+	}
	220	+	require.NotEmpty(t, png)
	221	+	require.NotEmpty(t, ann)
	222	+	require.Regexp(t, `^\d{8}-\d{6}\.\d{3}-HEADLESS-1\.json$`, js)
	223	+
	224	+	st, err := os.Stat(dir)
	225	+	require.NoError(t, err)
	226	+	require.Equal(t, os.FileMode(0o700), st.Mode().Perm())
	227	+
	228	+	data, err := os.ReadFile(filepath.Join(dir, js))
	229	+	require.NoError(t, err)
	230	+	var meta struct {
	231	+	Output string `json:"output"`
	232	+	Width int `json:"width"`
	233	+	Height int `json:"height"`
	234	+	Stride int `json:"stride"`
	235	+	Scale float64 `json:"scale"`
	236	+	LinksOnly bool `json:"links_only"`
	237	+	Targets []struct {
	238	+	Kind string `json:"kind"`
	239	+	Rect [4]int `json:"rect"`
	240	+	} `json:"targets"`
	241	+	}
	242	+	require.NoError(t, json.Unmarshal(data, &meta))
	243	+	require.Equal(t, "HEADLESS-1", meta.Output)
	244	+	require.Equal(t, w, meta.Width)
	245	+	require.Equal(t, hgt, meta.Height)
	246	+	require.Equal(t, w*4, meta.Stride)
	247	+	require.Equal(t, 1.0, meta.Scale)
	248	+	require.Len(t, meta.Targets, len(res.Targets))
	249	+	for i, tg := range res.Targets {
	250	+	require.Equal(t, [4]int{tg.Rect.X, tg.Rect.Y, tg.Rect.X + tg.Rect.W, tg.Rect.Y + tg.Rect.H}, meta.Targets[i].Rect)
	251	+	}
	252	+
	253	+	f, err := os.Open(filepath.Join(dir, png))
	254	+	require.NoError(t, err)
	255	+	defer f.Close()
	256	+	img, _, err := image.Decode(f)
	257	+	require.NoError(t, err)
	258	+	require.Equal(t, image.Rect(0, 0, w, hgt), img.Bounds())
	259	+	r, g, b, _ := img.At(0, 0).RGBA()
	260	+	require.Equal(t, uint32(pix[2])*0x101, r)
	261	+	require.Equal(t, uint32(pix[1])*0x101, g)
	262	+	require.Equal(t, uint32(pix[0])*0x101, b)
	263	+	}
	264	+
	265	+	// scanOnceDone sends a request that fails fast on a blank capture and waits
	266	+	// for its result, which proves the previous dump finished.
	267	+	func scanOnceDone(t testing.T, h harness) ports.HintScanResult {
	268	+	t.Helper()
	269	+	go func() {
	270	+	creq := <-h.captures
	271	+	h.captured <- ports.CaptureDone{ID: creq.ID, Err: errors.New("stop")}
	272	+	}()
	273	+	h.requests <- ports.HintScanRequest{Seq: 99, Output: "x", Width: 8, Height: 8, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 8, H: 8}}}}
	274	+	return <-h.results
	275	+	}
	276	+
	277	+	func TestHintDumpDisabled(t *testing.T) {
	278	+	if _, err := exec.LookPath("tesseract"); err != nil {
	279	+	t.Skip("tesseract not on PATH")
	280	+	}
	281	+	pix, w, hgt := loadBGRA(t)
	282	+	dir := filepath.Join(t.TempDir(), "hints")
	283	+	h := start(t, 5*time.Second)
	284	+	scanOnce(t, h, pix, w, hgt, 1)
	285	+	scanOnceDone(t, h)
	286	+	_, err := os.Stat(dir)
	287	+	require.ErrorIs(t, err, os.ErrNotExist)
	288	+	}
	289	+
	290	+	func TestHintDumpRetention(t *testing.T) {
	291	+	dir := t.TempDir()
	292	+	for i := 0; i < 60; i++ {
	293	+	for _, ext := range []string{".png", ".annotated.png", ".json"} {
	294	+	name := fmt.Sprintf("20260101-000000.%03d-o%s", i, ext)
	295	+	require.NoError(t, os.WriteFile(filepath.Join(dir, name), nil, 0o600))
	296	+	}
	297	+	}
	298	+	require.NoError(t, prune(dir, maxDumps))
	299	+	entries, err := os.ReadDir(dir)
	300	+	require.NoError(t, err)
	301	+	require.Len(t, entries, maxDumps*3)
	302	+	require.Equal(t, "20260101-000000.010-o.annotated.png", entries[0].Name())
	303	+	}
	304	+
	305	+	func TestScanMergesSource(t *testing.T) {
	306	+	pix, w, hgt := loadBGRA(t)
	307	+	src := newMocktargetSource(t)
	308	+	src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
	309	+	wins := []ports.HintWindow{{ID: 4, PID: 9, Rect: ports.Rect{X: 0, Y: 0, W: w, H: hgt}}}
	310	+	at := []ports.HintTarget{{Rect: ports.Rect{X: 5, Y: 5, W: 30, H: 10}, Kind: ports.HintLink}}
	311	+	src.EXPECT().Targets(mock.Anything, wins, false).Return(at, map[ports.WindowID]bool{4: true}, nil).Once()
	312	+	h := startWith(t, 5*time.Second, Options{Source: src})
	313	+	go func() {
	314	+	creq := <-h.captures
	315	+	copy(creq.Mapped, pix)
	316	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	317	+	}()
	318	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: wins}
	319	+	res := <-h.results
	320	+	require.NoError(t, res.Err)
	321	+	// The whole frame is covered: only the accessibility target remains.
	322	+	require.Equal(t, at, res.Targets)
	323	+	}
	324	+
	325	+	func TestScanSourceErrorUsesOCR(t *testing.T) {
	326	+	if _, err := exec.LookPath("tesseract"); err != nil {
	327	+	t.Skip("tesseract not on PATH")
	328	+	}
	329	+	pix, w, hgt := loadBGRA(t)
	330	+	src := newMocktargetSource(t)
	331	+	src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
	332	+	wins := []ports.HintWindow{{ID: 4, PID: 9, Rect: ports.Rect{W: w, H: hgt}}}
	333	+	src.EXPECT().Targets(mock.Anything, wins, false).Return(nil, nil, errors.New("bus gone")).Once()
	334	+	h := startWith(t, 5*time.Second, Options{Source: src})
	335	+	go func() {
	336	+	creq := <-h.captures
	337	+	copy(creq.Mapped, pix)
	338	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	339	+	}()
	340	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: wins}
	341	+	res := <-h.results
	342	+	require.NoError(t, res.Err)
	343	+	require.NotEmpty(t, res.Targets)
	344	+	}
	345	+
	346	+	// The worker keeps no frame: the mapped buffer belongs to the scan, and to the
	347	+	// dump job until it is written; every mapping is released afterwards.
	348	+	func TestWorkerHoldsNoFrameAfterScan(t *testing.T) {
	349	+	for _, dumpDir := range []string{"", filepath.Join(t.TempDir(), "hints")} {
	350	+	// OCR is "installed" but its binary cannot run: only the memory matters.
	351	+	w := &worker{dumpDir: dumpDir, ocr: ocrState{done: true, binary: "/nonexistent/tesseract", lang: "eng"}}
	352	+	requests := make(chan ports.HintScanRequest)
	353	+	captures := make(chan ports.CaptureRequest, 1)
	354	+	captured := make(chan ports.CaptureDone, 1)
	355	+	results := make(chan ports.HintScanResult, 1)
	356	+	go func() {
	357	+	creq := <-captures
	358	+	captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	359	+	}()
	360	+	req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 64, H: 32}}}}
	361	+	res := w.serve(context.Background(), req, requests, captures, captured, results, 5*time.Second)
	362	+	require.NoError(t, res.Err)
	363	+	if dumpDir == "" {
	364	+	require.Nil(t, w.pending)
	365	+	require.Zero(t, w.live, "frame and crops are released before the result")
	366	+	} else {
	367	+	require.NotNil(t, w.pending)
	368	+	require.Equal(t, 1, w.live, "only the frame, kept for the dump")
	369	+	}
	370	+	w.dump(context.Background())
	371	+	require.Nil(t, w.pending)
	372	+	require.Zero(t, w.live)
	373	+	require.Empty(t, w.cache) // OCR failed: nothing cached
	374	+	}
	375	+	}
	376	+
	377	+	// A capture that timed out stays mapped until the output answers it.
	378	+	func TestTimedOutCaptureIsReleasedWhenDone(t *testing.T) {
	379	+	w := &worker{dumpDir: t.TempDir()}
	380	+	captures := make(chan ports.CaptureRequest, 1)
	381	+	captured := make(chan ports.CaptureDone, 1)
	382	+	req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1}
	383	+	res := w.serve(context.Background(), req, make(chan ports.HintScanRequest), captures, captured, make(chan ports.HintScanResult, 1), 20*time.Millisecond)
	384	+	require.ErrorIs(t, res.Err, ErrTimeout)
	385	+	creq := <-captures
	386	+	require.Equal(t, 1, w.live)
	387	+	creq.Mapped[0] = 1 // still valid: the output may write until CaptureDone
	388	+	w.reap(creq.ID)
	389	+	require.Zero(t, w.live)
	390	+	}
	391	+
	392	+	// Without tesseract OCR is off for good: no capture, only accessibility targets.
	393	+	func TestNoTesseractSkipsCapture(t *testing.T) {
	394	+	t.Setenv("PATH", t.TempDir())
	395	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
	396	+	covered := ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: 30, W: 5, H: 5}}
	397	+	wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}, {ID: 2, Rect: ports.Rect{X: 20, W: 20, H: 20}}}
	398	+	src := newMocktargetSource(t)
	399	+	src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
	400	+	src.EXPECT().Targets(mock.Anything, wins, false).Return([]ports.HintTarget{button, covered}, map[ports.WindowID]bool{2: true}, nil).Times(2)
	401	+	h := startWith(t, time.Second, Options{Source: src})
	402	+	for seq := uint64(1); seq <= 2; seq++ {
	403	+	h.requests <- ports.HintScanRequest{Seq: seq, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: wins}
	404	+	res := <-h.results
	405	+	require.NoError(t, res.Err)
	406	+	require.Equal(t, []ports.HintTarget{button, covered}, res.Targets)
	407	+	}
	408	+	require.Empty(t, h.captures)
	409	+	}
	410	+
	411	+	// With debug dumps on, the frame is still captured (for the dump only).
	412	+	func TestNoTesseractDumpStillCaptures(t *testing.T) {
	413	+	t.Setenv("PATH", t.TempDir())
	414	+	dir := filepath.Join(t.TempDir(), "hints")
	415	+	h := startWith(t, 5*time.Second, Options{DumpDir: dir})
	416	+	go func() {
	417	+	creq := <-h.captures
	418	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	419	+	}()
	420	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 40, H: 20}}}}
	421	+	require.NoError(t, (<-h.results).Err)
	422	+	waitFiles(t, dir, 3)
	423	+	}
	424	+
	425	+	// A capture that completes after its scan timed out is released while the
	426	+	// worker is idle, without another scan, and its reply does not linger.
	427	+	func TestHintLateCaptureReaped(t *testing.T) {
	428	+	h := startCapturing(t, 50*time.Millisecond)
	429	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 64, Height: 32, Scale: 1}
	430	+	creq := <-h.captures
	431	+	res := <-h.results
	432	+	require.ErrorIs(t, res.Err, ErrTimeout)
	433	+	// The worker is idle; the output finally answers.
	434	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	435	+	require.Eventually(t, func() bool { return len(h.captured) == 0 }, time.Second, time.Millisecond)
	436	+	}

A internal/adapters/hints/hybrid_test.go +177 -0

@@ -0,0 +1,177 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"hash/maphash"
	6	+	"image"
	7	+	"image/draw"
	8	+	"image/png"
	9	+	"os"
	10	+	"os/exec"
	11	+	"runtime"
	12	+	"testing"
	13	+
	14	+	"github.com/bnema/neferwl/internal/ports"
	15	+	"github.com/stretchr/testify/require"
	16	+	)
	17	+
	18	+	func TestWindowCrop(t *testing.T) {
	19	+	for _, tc := range []struct {
	20	+	name string
	21	+	r ports.Rect
	22	+	scale float64
	23	+	want image.Rectangle
	24	+	}{
	25	+	{"unit", ports.Rect{X: 10, Y: 20, W: 100, H: 50}, 1, image.Rect(10, 20, 110, 70)},
	26	+	{"fractional", ports.Rect{X: 11, Y: 21, W: 100, H: 50}, 1.2, image.Rect(13, 25, 134, 86)},
	27	+	{"off output", ports.Rect{X: -10, Y: 20, W: 100, H: 100}, 1.2, image.Rect(0, 24, 108, 100)},
	28	+	{"outside", ports.Rect{X: 1000, Y: 20, W: 100, H: 100}, 1, image.Rectangle{}},
	29	+	} {
	30	+	t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.want, windowCrop(tc.r, tc.scale, 200, 100)) })
	31	+	}
	32	+	}
	33	+
	34	+	func sourceResult(ar atspiResult) *pendingSource {
	35	+	p := &pendingSource{done: make(chan struct{}), res: ar}
	36	+	close(p.done)
	37	+	return p
	38	+	}
	39	+
	40	+	func TestHybridCacheAndMerge(t *testing.T) {
	41	+	t.Setenv("PATH", t.TempDir()) // Cached OCR must not need tesseract.
	42	+	buf := make([]byte, 2001004)
	43	+	wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 100, H: 100}}, {ID: 2, Rect: ports.Rect{X: 100, W: 100, H: 100}}}
	44	+	req := ports.HintScanRequest{Output: "o", Width: 200, Height: 100, Scale: 1, Windows: wins}
	45	+	seed := maphash.MakeSeed()
	46	+	r := windowCrop(wins[1].Rect, 1, 200, 100)
	47	+	oldWorkers := ocrWorkers
	48	+	ocrWorkers = 1 // one band covers the crop
	49	+	t.Cleanup(func() { ocrWorkers = oldWorkers })
	50	+	ext := image.Rect(0, 0, 100, 100)
	51	+	entry := bandCache{hash: (&worker{cacheSeed: seed}).cropHash(buf, 800, r), ext: ext, words: []OCRWord{{Text: "example.org", Rect: image.Rect(10, 20, 90, 40)}}}
	52	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 1, Y: 1, W: 10, H: 10}}
	53	+	link := ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: 110, Y: 20, W: 80, H: 20}}
	54	+	uncoveredButton := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 101, Y: 1, W: 10, H: 10}}
	55	+	ar := atspiResult{targets: []ports.HintTarget{button, uncoveredButton}, covered: map[ports.WindowID]bool{1: true}}
	56	+	w := worker{ocr: ocrState{done: true, binary: "tesseract", lang: "eng"}, cacheSeed: seed, cache: map[bandKey]bandCache{{output: "o", window: 2, band: 0, bands: 1}: entry}}
	57	+	got := w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
	58	+	require.Equal(t, []ports.HintTarget{button, uncoveredButton, link}, got.Targets)
	59	+	req.LinksOnly = true
	60	+	got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
	61	+	require.Equal(t, []ports.HintTarget{link}, got.Targets)
	62	+	// Changing one pixel invalidates only the crop, never leaving stale links.
	63	+	buf[100*4]++
	64	+	got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
	65	+	require.Empty(t, got.Targets)
	66	+	}
	67	+
	68	+	func TestMissingTesseractKeepsAccessibilityButtons(t *testing.T) {
	69	+	t.Setenv("PATH", t.TempDir())
	70	+	w := worker{}
	71	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
	72	+	req := ports.HintScanRequest{Width: 20, Height: 20, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}}}
	73	+	got := w.scan(context.Background(), req, frame{pix: make([]byte, 1600), stride: 80}, 0, ports.HintScanResult{}, sourceResult(atspiResult{targets: []ports.HintTarget{button}}))
	74	+	require.NoError(t, got.Err)
	75	+	require.Equal(t, []ports.HintTarget{button}, got.Targets)
	76	+	}
	77	+
	78	+	func TestNoWindowsProducesNoPixelTargets(t *testing.T) {
	79	+	pix, width, height := loadBGRA(t)
	80	+	w := worker{}
	81	+	got := w.scan(context.Background(), ports.HintScanRequest{Width: width, Height: height}, frame{pix: pix, stride: width * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	82	+	require.Empty(t, got.Targets)
	83	+	}
	84	+
	85	+	func TestStripMappingAndPolarity(t *testing.T) {
	86	+	img := image.NewRGBA(image.Rect(0, 0, 200, 200))
	87	+	strips := packLines(img, []image.Rectangle{image.Rect(20, 20, 160, 40), image.Rect(20, 80, 160, 112)}, 2, true)
	88	+	require.Len(t, strips, 2)
	89	+	for _, s := range strips {
	90	+	for _, off := range s.offsets {
	91	+	require.Equal(t, 1, off.factor)
	92	+	require.Equal(t, uint8(255), s.img.GrayAt(off.packed.Min.X, off.packed.Min.Y).Y)
	93	+	}
	94	+	}
	95	+	small := packLines(img, []image.Rectangle{image.Rect(20, 20, 100, 30)}, 1, true)
	96	+	require.Equal(t, 2, small[0].offsets[0].factor)
	97	+	}
	98	+
	99	+	func TestCropGrayAndHash(t *testing.T) {
	100	+	buf := make([]byte, 444)
	101	+	for y := 0; y < 4; y++ {
	102	+	for x := 0; x < 4; x++ {
	103	+	copy(buf[y16+x4:], []byte{0, 0, 255, 255}) // BGRA red
	104	+	}
	105	+	}
	106	+	g := cropGray(buf, 16, image.Rect(1, 1, 3, 3), make([]byte, 4))
	107	+	require.Equal(t, image.Rect(0, 0, 2, 2), g.Bounds())
	108	+	require.Equal(t, uint8(76), g.GrayAt(1, 1).Y)
	109	+	w := worker{cacheSeed: maphash.MakeSeed()}
	110	+	a := w.cropHash(buf, 16, image.Rect(1, 1, 3, 3))
	111	+	buf[0]++ // outside the crop
	112	+	require.Equal(t, a, w.cropHash(buf, 16, image.Rect(1, 1, 3, 3)))
	113	+	buf[16+4]++
	114	+	require.NotEqual(t, a, w.cropHash(buf, 16, image.Rect(1, 1, 3, 3)))
	115	+	}
	116	+
	117	+	func TestUpscaleBandGray(t *testing.T) {
	118	+	g := image.NewGray(image.Rect(0, 0, 3, 2))
	119	+	for i := range g.Pix {
	120	+	g.Pix[i] = uint8(i + 1)
	121	+	}
	122	+	sub := g.SubImage(image.Rect(1, 1, 3, 2)).(*image.Gray)
	123	+	up := upscaleBand(sub)
	124	+	require.Equal(t, image.Rect(0, 0, 4, 2), up.Bounds())
	125	+	require.Equal(t, []uint8{5, 5, 6, 6, 5, 5, 6, 6}, up.Pix)
	126	+	}
	127	+
	128	+	// A blinking cursor changes one band: the other bands keep their cached
	129	+	// words, and only the changed one is recognized again.
	130	+	func TestBandCacheSurvivesCursorBlink(t *testing.T) {
	131	+	binary, err := exec.LookPath("tesseract")
	132	+	if err != nil {
	133	+	t.Skip("tesseract not on PATH")
	134	+	}
	135	+	old := ocrWorkers
	136	+	ocrWorkers = 4
	137	+	t.Cleanup(func() { ocrWorkers = old })
	138	+	f, err := os.Open("testdata/terminal.png")
	139	+	require.NoError(t, err)
	140	+	defer f.Close()
	141	+	src, err := png.Decode(f)
	142	+	require.NoError(t, err)
	143	+	w, h := src.Bounds().Dx(), src.Bounds().Dy()
	144	+	rgba := image.NewRGBA(image.Rect(0, 0, w, h))
	145	+	draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
	146	+	buf := make([]byte, wh4)
	147	+	for i := 0; i < len(buf); i += 4 {
	148	+	buf[i], buf[i+1], buf[i+2], buf[i+3] = rgba.Pix[i+2], rgba.Pix[i+1], rgba.Pix[i], 0xff
	149	+	}
	150	+	_ = binary
	151	+	wk := &worker{}
	152	+	req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
	153	+	first := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	154	+	require.NotEmpty(t, first.Targets)
	155	+	require.Len(t, wk.cache, min(runtime.GOMAXPROCS(0), 4))
	156	+
	157	+	// The same frame: every band hits.
	158	+	again := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	159	+	require.Equal(t, first.Targets, again.Targets)
	160	+
	161	+	// One pixel at the bottom blinks: only the last band (and the context of
	162	+	// none other) is invalidated.
	163	+	before := map[bandKey]uint64{}
	164	+	for k, v := range wk.cache {
	165	+	before[k] = v.hash
	166	+	}
	167	+	buf[((h-1)w+3)4] ^= 0xff
	168	+	blink := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	169	+	require.Equal(t, first.Targets, blink.Targets)
	170	+	changed := 0
	171	+	for k, v := range wk.cache {
	172	+	if before[k] != v.hash {
	173	+	changed++
	174	+	}
	175	+	}
	176	+	require.Equal(t, 1, changed)
	177	+	}

@@ -0,0 +1,177 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"hash/maphash"
	6	+	"image"
	7	+	"image/draw"
	8	+	"image/png"
	9	+	"os"
	10	+	"os/exec"
	11	+	"runtime"
	12	+	"testing"
	13	+
	14	+	"github.com/bnema/neferwl/internal/ports"
	15	+	"github.com/stretchr/testify/require"
	16	+	)
	17	+
	18	+	func TestWindowCrop(t *testing.T) {
	19	+	for _, tc := range []struct {
	20	+	name string
	21	+	r ports.Rect
	22	+	scale float64
	23	+	want image.Rectangle
	24	+	}{
	25	+	{"unit", ports.Rect{X: 10, Y: 20, W: 100, H: 50}, 1, image.Rect(10, 20, 110, 70)},
	26	+	{"fractional", ports.Rect{X: 11, Y: 21, W: 100, H: 50}, 1.2, image.Rect(13, 25, 134, 86)},
	27	+	{"off output", ports.Rect{X: -10, Y: 20, W: 100, H: 100}, 1.2, image.Rect(0, 24, 108, 100)},
	28	+	{"outside", ports.Rect{X: 1000, Y: 20, W: 100, H: 100}, 1, image.Rectangle{}},
	29	+	} {
	30	+	t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.want, windowCrop(tc.r, tc.scale, 200, 100)) })
	31	+	}
	32	+	}
	33	+
	34	+	func sourceResult(ar atspiResult) *pendingSource {
	35	+	p := &pendingSource{done: make(chan struct{}), res: ar}
	36	+	close(p.done)
	37	+	return p
	38	+	}
	39	+
	40	+	func TestHybridCacheAndMerge(t *testing.T) {
	41	+	t.Setenv("PATH", t.TempDir()) // Cached OCR must not need tesseract.
	42	+	buf := make([]byte, 2001004)
	43	+	wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 100, H: 100}}, {ID: 2, Rect: ports.Rect{X: 100, W: 100, H: 100}}}
	44	+	req := ports.HintScanRequest{Output: "o", Width: 200, Height: 100, Scale: 1, Windows: wins}
	45	+	seed := maphash.MakeSeed()
	46	+	r := windowCrop(wins[1].Rect, 1, 200, 100)
	47	+	oldWorkers := ocrWorkers
	48	+	ocrWorkers = 1 // one band covers the crop
	49	+	t.Cleanup(func() { ocrWorkers = oldWorkers })
	50	+	ext := image.Rect(0, 0, 100, 100)
	51	+	entry := bandCache{hash: (&worker{cacheSeed: seed}).cropHash(buf, 800, r), ext: ext, words: []OCRWord{{Text: "example.org", Rect: image.Rect(10, 20, 90, 40)}}}
	52	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 1, Y: 1, W: 10, H: 10}}
	53	+	link := ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: 110, Y: 20, W: 80, H: 20}}
	54	+	uncoveredButton := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 101, Y: 1, W: 10, H: 10}}
	55	+	ar := atspiResult{targets: []ports.HintTarget{button, uncoveredButton}, covered: map[ports.WindowID]bool{1: true}}
	56	+	w := worker{ocr: ocrState{done: true, binary: "tesseract", lang: "eng"}, cacheSeed: seed, cache: map[bandKey]bandCache{{output: "o", window: 2, band: 0, bands: 1}: entry}}
	57	+	got := w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
	58	+	require.Equal(t, []ports.HintTarget{button, uncoveredButton, link}, got.Targets)
	59	+	req.LinksOnly = true
	60	+	got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
	61	+	require.Equal(t, []ports.HintTarget{link}, got.Targets)
	62	+	// Changing one pixel invalidates only the crop, never leaving stale links.
	63	+	buf[100*4]++
	64	+	got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
	65	+	require.Empty(t, got.Targets)
	66	+	}
	67	+
	68	+	func TestMissingTesseractKeepsAccessibilityButtons(t *testing.T) {
	69	+	t.Setenv("PATH", t.TempDir())
	70	+	w := worker{}
	71	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
	72	+	req := ports.HintScanRequest{Width: 20, Height: 20, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}}}
	73	+	got := w.scan(context.Background(), req, frame{pix: make([]byte, 1600), stride: 80}, 0, ports.HintScanResult{}, sourceResult(atspiResult{targets: []ports.HintTarget{button}}))
	74	+	require.NoError(t, got.Err)
	75	+	require.Equal(t, []ports.HintTarget{button}, got.Targets)
	76	+	}
	77	+
	78	+	func TestNoWindowsProducesNoPixelTargets(t *testing.T) {
	79	+	pix, width, height := loadBGRA(t)
	80	+	w := worker{}
	81	+	got := w.scan(context.Background(), ports.HintScanRequest{Width: width, Height: height}, frame{pix: pix, stride: width * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	82	+	require.Empty(t, got.Targets)
	83	+	}
	84	+
	85	+	func TestStripMappingAndPolarity(t *testing.T) {
	86	+	img := image.NewRGBA(image.Rect(0, 0, 200, 200))
	87	+	strips := packLines(img, []image.Rectangle{image.Rect(20, 20, 160, 40), image.Rect(20, 80, 160, 112)}, 2, true)
	88	+	require.Len(t, strips, 2)
	89	+	for _, s := range strips {
	90	+	for _, off := range s.offsets {
	91	+	require.Equal(t, 1, off.factor)
	92	+	require.Equal(t, uint8(255), s.img.GrayAt(off.packed.Min.X, off.packed.Min.Y).Y)
	93	+	}
	94	+	}
	95	+	small := packLines(img, []image.Rectangle{image.Rect(20, 20, 100, 30)}, 1, true)
	96	+	require.Equal(t, 2, small[0].offsets[0].factor)
	97	+	}
	98	+
	99	+	func TestCropGrayAndHash(t *testing.T) {
	100	+	buf := make([]byte, 444)
	101	+	for y := 0; y < 4; y++ {
	102	+	for x := 0; x < 4; x++ {
	103	+	copy(buf[y16+x4:], []byte{0, 0, 255, 255}) // BGRA red
	104	+	}
	105	+	}
	106	+	g := cropGray(buf, 16, image.Rect(1, 1, 3, 3), make([]byte, 4))
	107	+	require.Equal(t, image.Rect(0, 0, 2, 2), g.Bounds())
	108	+	require.Equal(t, uint8(76), g.GrayAt(1, 1).Y)
	109	+	w := worker{cacheSeed: maphash.MakeSeed()}
	110	+	a := w.cropHash(buf, 16, image.Rect(1, 1, 3, 3))
	111	+	buf[0]++ // outside the crop
	112	+	require.Equal(t, a, w.cropHash(buf, 16, image.Rect(1, 1, 3, 3)))
	113	+	buf[16+4]++
	114	+	require.NotEqual(t, a, w.cropHash(buf, 16, image.Rect(1, 1, 3, 3)))
	115	+	}
	116	+
	117	+	func TestUpscaleBandGray(t *testing.T) {
	118	+	g := image.NewGray(image.Rect(0, 0, 3, 2))
	119	+	for i := range g.Pix {
	120	+	g.Pix[i] = uint8(i + 1)
	121	+	}
	122	+	sub := g.SubImage(image.Rect(1, 1, 3, 2)).(*image.Gray)
	123	+	up := upscaleBand(sub)
	124	+	require.Equal(t, image.Rect(0, 0, 4, 2), up.Bounds())
	125	+	require.Equal(t, []uint8{5, 5, 6, 6, 5, 5, 6, 6}, up.Pix)
	126	+	}
	127	+
	128	+	// A blinking cursor changes one band: the other bands keep their cached
	129	+	// words, and only the changed one is recognized again.
	130	+	func TestBandCacheSurvivesCursorBlink(t *testing.T) {
	131	+	binary, err := exec.LookPath("tesseract")
	132	+	if err != nil {
	133	+	t.Skip("tesseract not on PATH")
	134	+	}
	135	+	old := ocrWorkers
	136	+	ocrWorkers = 4
	137	+	t.Cleanup(func() { ocrWorkers = old })
	138	+	f, err := os.Open("testdata/terminal.png")
	139	+	require.NoError(t, err)
	140	+	defer f.Close()
	141	+	src, err := png.Decode(f)
	142	+	require.NoError(t, err)
	143	+	w, h := src.Bounds().Dx(), src.Bounds().Dy()
	144	+	rgba := image.NewRGBA(image.Rect(0, 0, w, h))
	145	+	draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
	146	+	buf := make([]byte, wh4)
	147	+	for i := 0; i < len(buf); i += 4 {
	148	+	buf[i], buf[i+1], buf[i+2], buf[i+3] = rgba.Pix[i+2], rgba.Pix[i+1], rgba.Pix[i], 0xff
	149	+	}
	150	+	_ = binary
	151	+	wk := &worker{}
	152	+	req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
	153	+	first := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	154	+	require.NotEmpty(t, first.Targets)
	155	+	require.Len(t, wk.cache, min(runtime.GOMAXPROCS(0), 4))
	156	+
	157	+	// The same frame: every band hits.
	158	+	again := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	159	+	require.Equal(t, first.Targets, again.Targets)
	160	+
	161	+	// One pixel at the bottom blinks: only the last band (and the context of
	162	+	// none other) is invalidated.
	163	+	before := map[bandKey]uint64{}
	164	+	for k, v := range wk.cache {
	165	+	before[k] = v.hash
	166	+	}
	167	+	buf[((h-1)w+3)4] ^= 0xff
	168	+	blink := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	169	+	require.Equal(t, first.Targets, blink.Targets)
	170	+	changed := 0
	171	+	for k, v := range wk.cache {
	172	+	if before[k] != v.hash {
	173	+	changed++
	174	+	}
	175	+	}
	176	+	require.Equal(t, 1, changed)
	177	+	}

A internal/adapters/hints/mmap.go +40 -0

@@ -0,0 +1,40 @@
	1	+	package hints
	2	+
	3	+	import "syscall"
	4	+
	5	+	// mapping is a byte region that lives outside the Go heap: anonymous memory
	6	+	// from mmap, returned to the system by unmap. When mmap fails it falls back
	7	+	// to heap memory, which unmap then just drops.
	8	+	type mapping struct {
	9	+	b []byte
	10	+	mapped bool
	11	+	live *int
	12	+	}
	13	+
	14	+	// alloc returns a zeroed region of n bytes; w.live counts the regions not
	15	+	// yet released.
	16	+	func (w worker) alloc(n int) mapping {
	17	+	m := &mapping{live: &w.live}
	18	+	b, err := syscall.Mmap(-1, 0, n, syscall.PROT_READ\|syscall.PROT_WRITE, syscall.MAP_PRIVATE\|syscall.MAP_ANON)
	19	+	if err != nil {
	20	+	b = make([]byte, n)
	21	+	} else {
	22	+	m.mapped = true
	23	+	}
	24	+	m.b = b
	25	+	w.live++
	26	+	return m
	27	+	}
	28	+
	29	+	// unmap releases the region; it is safe to call twice and on nil. The
	30	+	// region must not be used afterwards, nor still be written by the output.
	31	+	func (m *mapping) unmap() {
	32	+	if m == nil \|\| m.b == nil {
	33	+	return
	34	+	}
	35	+	if m.mapped {
	36	+	_ = syscall.Munmap(m.b)
	37	+	}
	38	+	m.b = nil
	39	+	*m.live--
	40	+	}

@@ -0,0 +1,40 @@
	1	+	package hints
	2	+
	3	+	import "syscall"
	4	+
	5	+	// mapping is a byte region that lives outside the Go heap: anonymous memory
	6	+	// from mmap, returned to the system by unmap. When mmap fails it falls back
	7	+	// to heap memory, which unmap then just drops.
	8	+	type mapping struct {
	9	+	b []byte
	10	+	mapped bool
	11	+	live *int
	12	+	}
	13	+
	14	+	// alloc returns a zeroed region of n bytes; w.live counts the regions not
	15	+	// yet released.
	16	+	func (w worker) alloc(n int) mapping {
	17	+	m := &mapping{live: &w.live}
	18	+	b, err := syscall.Mmap(-1, 0, n, syscall.PROT_READ\|syscall.PROT_WRITE, syscall.MAP_PRIVATE\|syscall.MAP_ANON)
	19	+	if err != nil {
	20	+	b = make([]byte, n)
	21	+	} else {
	22	+	m.mapped = true
	23	+	}
	24	+	m.b = b
	25	+	w.live++
	26	+	return m
	27	+	}
	28	+
	29	+	// unmap releases the region; it is safe to call twice and on nil. The
	30	+	// region must not be used afterwards, nor still be written by the output.
	31	+	func (m *mapping) unmap() {
	32	+	if m == nil \|\| m.b == nil {
	33	+	return
	34	+	}
	35	+	if m.mapped {
	36	+	_ = syscall.Munmap(m.b)
	37	+	}
	38	+	m.b = nil
	39	+	*m.live--
	40	+	}

A internal/adapters/hints/ocr.go +440 -0

@@ -0,0 +1,440 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"bytes"
	6	+	"context"
	7	+	"fmt"
	8	+	"image"
	9	+	"image/png"
	10	+	"io"
	11	+	"math"
	12	+	"net"
	13	+	"os"
	14	+	"os/exec"
	15	+	"path/filepath"
	16	+	"regexp"
	17	+	"runtime"
	18	+	"sort"
	19	+	"strconv"
	20	+	"strings"
	21	+	"sync"
	22	+	"time"
	23	+
	24	+	"github.com/bnema/neferwl/internal/adapters/logging"
	25	+	"github.com/bnema/neferwl/internal/ports"
	26	+	)
	27	+
	28	+	// OCRWord is a recognized word in physical output coordinates.
	29	+	type OCRWord struct {
	30	+	Text string `json:"text"`
	31	+	Rect image.Rectangle `json:"rect"`
	32	+	Confidence float64 `json:"confidence"`
	33	+	// Underlined is set when a thin line runs under the word. UnderL and
	34	+	// UnderR are the x extent of that line (crop-local), which can reach the
	35	+	// neighbouring words of one underlined link.
	36	+	Underlined bool `json:"underlined,omitempty"`
	37	+	UnderL int `json:"under_l,omitempty"`
	38	+	UnderR int `json:"under_r,omitempty"`
	39	+	}
	40	+
	41	+	var languageWarning sync.Once
	42	+	var failureWarning sync.Once
	43	+
	44	+	func ocrLanguage(ctx context.Context, binary string) (string, error) {
	45	+	dir := os.Getenv("TESSDATA_PREFIX")
	46	+	if dir == "" {
	47	+	dir = "/usr/share/tessdata"
	48	+	}
	49	+	if _, err := os.Stat(filepath.Join(dir, "eng.traineddata")); err == nil {
	50	+	return "eng", nil
	51	+	}
	52	+	out, err := exec.CommandContext(ctx, binary, "--list-langs").Output()
	53	+	if err != nil {
	54	+	return "", err
	55	+	}
	56	+	for _, line := range strings.Split(string(out), "\n")[1:] {
	57	+	lang := strings.TrimSpace(line)
	58	+	if lang != "" && lang != "osd" {
	59	+	languageWarning.Do(func() {
	60	+	log := logging.For(ctx, "hints")
	61	+	log.Warn().Str("language", lang).Msg("English OCR data missing; install tesseract-data-eng")
	62	+	})
	63	+	return lang, nil
	64	+	}
	65	+	}
	66	+	return "", fmt.Errorf("no OCR language available")
	67	+	}
	68	+
	69	+	func parseTSV(data []byte, offset image.Point) ([]OCRWord, error) {
	70	+	var words []OCRWord
	71	+	scanner := bufio.NewScanner(bytes.NewReader(data))
	72	+	scanner.Buffer(make([]byte, 4096), 1024*1024)
	73	+	for scanner.Scan() {
	74	+	fields := strings.SplitN(scanner.Text(), "\t", 12)
	75	+	if len(fields) != 12 \|\| fields[0] != "5" {
	76	+	continue
	77	+	}
	78	+	var values [4]int
	79	+	for i := range values {
	80	+	v, err := strconv.Atoi(fields[6+i])
	81	+	if err != nil {
	82	+	return nil, fmt.Errorf("OCR TSV coordinate: %w", err)
	83	+	}
	84	+	values[i] = v
	85	+	}
	86	+	conf, err := strconv.ParseFloat(fields[10], 64)
	87	+	if err != nil {
	88	+	return nil, fmt.Errorf("OCR TSV confidence: %w", err)
	89	+	}
	90	+	if values[2] <= 0 \|\| values[3] <= 0 \|\| strings.TrimSpace(fields[11]) == "" {
	91	+	continue
	92	+	}
	93	+	r := image.Rect(values[0], values[1], values[0]+values[2], values[1]+values[3]).Add(offset)
	94	+	words = append(words, OCRWord{Text: fields[11], Rect: r, Confidence: conf})
	95	+	}
	96	+	return words, scanner.Err()
	97	+	}
	98	+
	99	+	// ocrWorkers is the maximum number of bands recognized in parallel. 4 keeps
	100	+	// the tesseract processes at about 60% of the memory of 8 for the same time.
	101	+	var ocrWorkers = 4
	102	+
	103	+	// pngLevel is the compression of non-grayscale images piped to tesseract.
	104	+	var pngLevel = png.BestSpeed
	105	+
	106	+	// rawPGM feeds grayscale images to tesseract as binary PGM instead of PNG.
	107	+	var rawPGM = true
	108	+
	109	+	// ocrState is the tesseract installation, resolved once.
	110	+	type ocrState struct {
	111	+	done bool
	112	+	binary string
	113	+	lang string
	114	+	err error
	115	+	}
	116	+
	117	+	func (s *ocrState) resolve(ctx context.Context) error {
	118	+	if !s.done {
	119	+	s.done = true
	120	+	if s.binary, s.err = exec.LookPath("tesseract"); s.err == nil {
	121	+	s.lang, s.err = ocrLanguage(ctx, s.binary)
	122	+	}
	123	+	}
	124	+	return s.err
	125	+	}
	126	+
	127	+	// ocrReady resolves tesseract on the first call. When it is missing, OCR is
	128	+	// disabled for the life of the process and the warning is logged once.
	129	+	func (w *worker) ocrReady(ctx context.Context) bool {
	130	+	if err := w.ocr.resolve(ctx); err != nil {
	131	+	failureWarning.Do(func() {
	132	+	log := logging.For(ctx, "hints")
	133	+	log.Warn().Err(err).Msg("OCR unavailable; install tesseract and tesseract-data-eng")
	134	+	})
	135	+	return false
	136	+	}
	137	+	return true
	138	+	}
	139	+
	140	+	// band is a horizontal slice of a window crop recognized by one tesseract
	141	+	// process. It reads rows [top,bottom) and owns the words centred in
	142	+	// [ownTop,ownBottom); the margin only supplies context.
	143	+	type band struct {
	144	+	idx, n int
	145	+	top, bottom int
	146	+	ownTop, ownBottom int
	147	+	}
	148	+
	149	+	func bandsOf(height int) []band {
	150	+	n := min(runtime.GOMAXPROCS(0), ocrWorkers, height)
	151	+	bands := make([]band, 0, max(n, 0))
	152	+	for i := range n {
	153	+	ownTop, ownBottom := iheight/n, (i+1)height/n
	154	+	bands = append(bands, band{idx: i, n: n, top: max(0, ownTop-40), bottom: min(height, ownBottom+40), ownTop: ownTop, ownBottom: ownBottom})
	155	+	}
	156	+	return bands
	157	+	}
	158	+
	159	+	// recognize reads the bands of jobs that missed the cache, in parallel.
	160	+	func (w worker) recognize(ctx context.Context, jobs []bandJob) {
	161	+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
	162	+	defer cancel()
	163	+	// One scan-wide pool: at most ocrWorkers tesseract processes run at once,
	164	+	// whatever the number of windows.
	165	+	sem := make(chan struct{}, max(ocrWorkers, 1))
	166	+	var wg sync.WaitGroup
	167	+	for _, j := range jobs {
	168	+	if j.hit {
	169	+	continue
	170	+	}
	171	+	select {
	172	+	case sem <- struct{}{}:
	173	+	case <-ctx.Done():
	174	+	j.err = ctx.Err()
	175	+	continue
	176	+	}
	177	+	wg.Go(func() {
	178	+	defer func() { <-sem }()
	179	+	j.words, j.err = readBand(ctx, &w.ocr, j)
	180	+	})
	181	+	}
	182	+	wg.Wait()
	183	+	}
	184	+
	185	+	// readBand recognizes one band; the words are crop-local and owned by it.
	186	+	func readBand(ctx context.Context, st ocrState, j bandJob) ([]OCRWord, error) {
	187	+	words, err := recognizeBand(ctx, st.binary, st.lang, j.gray)
	188	+	if err != nil {
	189	+	return nil, err
	190	+	}
	191	+	heights := make([]int, 0, len(words))
	192	+	for _, word := range words {
	193	+	if len(word.Text) >= 3 {
	194	+	heights = append(heights, word.Rect.Dy())
	195	+	}
	196	+	}
	197	+	sort.Ints(heights)
	198	+	if len(heights) > 0 && heights[len(heights)/2] < 12 {
	199	+	words, err = recognizeBand(ctx, st.binary, st.lang, upscaleBand(j.gray))
	200	+	if err != nil {
	201	+	return nil, err
	202	+	}
	203	+	for i := range words {
	204	+	r := words[i].Rect
	205	+	words[i].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
	206	+	}
	207	+	}
	208	+	markUnderlined(j.gray, words)
	209	+	owned := words[:0]
	210	+	for _, word := range words {
	211	+	// Overlap supplies context, but each word belongs to exactly one band.
	212	+	word.Rect = word.Rect.Add(image.Pt(0, j.ext.Min.Y))
	213	+	if center := (word.Rect.Min.Y + word.Rect.Max.Y) / 2; center >= j.own[0] && center < j.own[1] {
	214	+	owned = append(owned, word)
	215	+	}
	216	+	}
	217	+	return owned, nil
	218	+	}
	219	+
	220	+	// readWords recognizes the words of img, a window crop, in parallel bands.
	221	+	func readWords(ctx context.Context, img *image.Gray) ([]OCRWord, int, error) {
	222	+	w := &worker{}
	223	+	if err := w.ocr.resolve(ctx); err != nil {
	224	+	return nil, 0, err
	225	+	}
	226	+	var jobs []*bandJob
	227	+	for _, b := range bandsOf(img.Bounds().Dy()) {
	228	+	ext := image.Rect(0, b.top, img.Bounds().Dx(), b.bottom)
	229	+	jobs = append(jobs, &bandJob{ext: ext, own: [2]int{b.ownTop, b.ownBottom}, gray: img.SubImage(ext).(*image.Gray)})
	230	+	}
	231	+	w.recognize(ctx, jobs)
	232	+	var words []OCRWord
	233	+	for _, j := range jobs {
	234	+	if j.err != nil {
	235	+	return nil, len(jobs), j.err
	236	+	}
	237	+	for _, word := range j.words {
	238	+	if !duplicateWord(words, word) {
	239	+	words = append(words, word)
	240	+	}
	241	+	}
	242	+	}
	243	+	return words, len(jobs), nil
	244	+	}
	245	+
	246	+	// The link patterns compile on the first OCR link check, not at process
	247	+	// start: with hint mode off nothing here costs memory or CPU.
	248	+	var domainPattern = sync.OnceValue(func() *regexp.Regexp {
	249	+	return regexp.MustCompile(`(?i)^(?:[a-z0-9](?:[a-z0-9-][a-z0-9])?\.)+([a-z]{2,24})(?::[0-9]+)?(?:[/?#]\S)?$`)
	250	+	})
	251	+	var schemePattern = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`(?i)^(?:https?\|htps\|file\|ftp):/{1,2}\S+$`) })
	252	+	var emailPattern = sync.OnceValue(func() *regexp.Regexp {
	253	+	return regexp.MustCompile(`(?i)^[a-z0-9.!#$%&'*+/=?^_` + "`" + `{\|}~-]+@(?:[a-z0-9-]+\.)+[a-z]{2,24}$`)
	254	+	})
	255	+	var fileExtensions = sync.OnceValue(func() []string {
	256	+	return strings.Fields("go md txt toml json yaml yml rs py js ts sh c h png jpg log conf lock mod sum html css")
	257	+	})
	258	+
	259	+	func isTextLink(text string) bool {
	260	+	text = strings.Trim(text, "()[]<>.,;:'\"")
	261	+	if schemePattern().MatchString(text) \|\| (strings.HasPrefix(strings.ToLower(text), "www.") && len(text) > 4) \|\| emailPattern().MatchString(text) {
	262	+	return true
	263	+	}
	264	+	host := text
	265	+	if h, _, err := net.SplitHostPort(text); err == nil {
	266	+	host = h
	267	+	}
	268	+	if ip := net.ParseIP(host); ip != nil && ip.To4() != nil {
	269	+	return true
	270	+	}
	271	+	m := domainPattern().FindStringSubmatch(text)
	272	+	if m == nil {
	273	+	return false
	274	+	}
	275	+	for _, ext := range fileExtensions() {
	276	+	if strings.EqualFold(m[1], ext) {
	277	+	return false
	278	+	}
	279	+	}
	280	+	return true
	281	+	}
	282	+
	283	+	func textLinks(words []OCRWord, scale float64) []ports.HintTarget {
	284	+	if scale <= 0 {
	285	+	scale = 1
	286	+	}
	287	+	words = append([]OCRWord(nil), words...)
	288	+	sort.Slice(words, func(i, j int) bool {
	289	+	if words[i].Rect.Min.Y != words[j].Rect.Min.Y {
	290	+	return words[i].Rect.Min.Y < words[j].Rect.Min.Y
	291	+	}
	292	+	return words[i].Rect.Min.X < words[j].Rect.Min.X
	293	+	})
	294	+	var out []ports.HintTarget
	295	+	var urls []image.Rectangle // boxes of the URL targets, in pixels
	296	+	used := make([]bool, len(words))
	297	+	for i, word := range words {
	298	+	if used[i] {
	299	+	continue
	300	+	}
	301	+	text, rect := word.Text, word.Rect
	302	+	// Grow only touching words on the same baseline; ordinary prose is not joined.
	303	+	for {
	304	+	next := -1
	305	+	for j, candidate := range words {
	306	+	if j == i \|\| used[j] \|\| candidate.Rect.Min.X < rect.Max.X-2 {
	307	+	continue
	308	+	}
	309	+	gap := candidate.Rect.Min.X - rect.Max.X
	310	+	if gap > max(3, min(rect.Dy(), candidate.Rect.Dy())/5) {
	311	+	continue
	312	+	}
	313	+	if abs(candidate.Rect.Max.Y-rect.Max.Y) > max(3, rect.Dy()/3) {
	314	+	continue
	315	+	}
	316	+	if next < 0 \|\| candidate.Rect.Min.X < words[next].Rect.Min.X {
	317	+	next = j
	318	+	}
	319	+	}
	320	+	if next < 0 {
	321	+	break
	322	+	}
	323	+	text += words[next].Text
	324	+	rect = rect.Union(words[next].Rect)
	325	+	used[next] = true
	326	+	}
	327	+	if !isTextLink(text) {
	328	+	continue
	329	+	}
	330	+	x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
	331	+	right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
	332	+	out = append(out, ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
	333	+	urls = append(urls, rect)
	334	+	}
	335	+	return append(out, underlinedLinks(words, urls, scale)...)
	336	+	}
	337	+
	338	+	func abs(x int) int {
	339	+	if x < 0 {
	340	+	return -x
	341	+	}
	342	+	return x
	343	+	}
	344	+	func overlap(a, b image.Rectangle) float64 {
	345	+	intersection := a.Intersect(b)
	346	+	area := min(a.Dx()a.Dy(), b.Dx()b.Dy())
	347	+	if intersection.Empty() \|\| area <= 0 {
	348	+	return 0
	349	+	}
	350	+	return float64(intersection.Dx()*intersection.Dy()) / float64(area)
	351	+	}
	352	+	func targetRect(t ports.HintTarget) image.Rectangle {
	353	+	return image.Rect(t.Rect.X, t.Rect.Y, t.Rect.X+t.Rect.W, t.Rect.Y+t.Rect.H)
	354	+	}
	355	+	func recognizeBand(ctx context.Context, binary, lang string, img image.Image) ([]OCRWord, error) {
	356	+	return recognizeImage(ctx, binary, lang, img, "11", false)
	357	+	}
	358	+
	359	+	// pgmReader streams a grayscale image as binary PGM (P5), row by row.
	360	+	type pgmReader struct {
	361	+	head []byte
	362	+	img *image.Gray
	363	+	y int
	364	+	row []byte
	365	+	}
	366	+
	367	+	func newPGMReader(img image.Gray) pgmReader {
	368	+	b := img.Bounds()
	369	+	return &pgmReader{head: fmt.Appendf(nil, "P5\n%d %d\n255\n", b.Dx(), b.Dy()), img: img}
	370	+	}
	371	+
	372	+	func (r *pgmReader) Read(p []byte) (int, error) {
	373	+	n := 0
	374	+	for n < len(p) {
	375	+	if len(r.head) > 0 {
	376	+	c := copy(p[n:], r.head)
	377	+	r.head, n = r.head[c:], n+c
	378	+	continue
	379	+	}
	380	+	if len(r.row) == 0 {
	381	+	b := r.img.Bounds()
	382	+	if r.y >= b.Dy() {
	383	+	break
	384	+	}
	385	+	r.row = r.img.Pix[(r.y+b.Min.Y-r.img.Rect.Min.Y)*r.img.Stride+(b.Min.X-r.img.Rect.Min.X):][:b.Dx()]
	386	+	r.y++
	387	+	}
	388	+	c := copy(p[n:], r.row)
	389	+	r.row, n = r.row[c:], n+c
	390	+	}
	391	+	if n == 0 {
	392	+	return 0, io.EOF
	393	+	}
	394	+	return n, nil
	395	+	}
	396	+
	397	+	func recognizeImage(ctx context.Context, binary, lang string, img image.Image, psm string, normalized bool) ([]OCRWord, error) {
	398	+	args := []string{"stdin", "stdout", "--psm", psm, "-l", lang}
	399	+	if normalized {
	400	+	args = append(args, "-c", "tessedit_do_invert=0")
	401	+	}
	402	+	args = append(args, "tsv")
	403	+	cmd := exec.CommandContext(ctx, binary, args...)
	404	+	cmd.Env = append(os.Environ(), "OMP_THREAD_LIMIT=1")
	405	+	if g, ok := img.(*image.Gray); ok && rawPGM {
	406	+	// Raw grayscale: no deflate, no encoder buffers.
	407	+	cmd.Stdin = newPGMReader(g)
	408	+	} else {
	409	+	// The PNG streams into tesseract; no encoded copy is held.
	410	+	pr, pw := io.Pipe()
	411	+	defer func() { _ = pr.Close() }()
	412	+	go func() {
	413	+	enc := png.Encoder{CompressionLevel: pngLevel}
	414	+	_ = pw.CloseWithError(enc.Encode(pw, img))
	415	+	}()
	416	+	cmd.Stdin = pr
	417	+	}
	418	+	var stderr bytes.Buffer
	419	+	cmd.Stderr = &stderr
	420	+	output, err := cmd.Output()
	421	+	if err != nil {
	422	+	return nil, fmt.Errorf("tesseract: %w: %s", err, stderr.String())
	423	+	}
	424	+	return parseTSV(output, image.Point{})
	425	+	}
	426	+
	427	+	// upscaleBand doubles a grayscale band with nearest-neighbour sampling.
	428	+	func upscaleBand(src image.Gray) image.Gray {
	429	+	b := src.Bounds()
	430	+	dst := image.NewGray(image.Rect(0, 0, b.Dx()2, b.Dy()2))
	431	+	for y := 0; y < b.Dy(); y++ {
	432	+	row := src.Pix[(y+b.Min.Y-src.Rect.Min.Y)*src.Stride+(b.Min.X-src.Rect.Min.X):][:b.Dx()]
	433	+	top := dst.Pix[2ydst.Stride:][:dst.Stride]
	434	+	for x, v := range row {
	435	+	top[2x], top[2x+1] = v, v
	436	+	}
	437	+	copy(dst.Pix[(2y+1)dst.Stride:], top)
	438	+	}
	439	+	return dst
	440	+	}

@@ -0,0 +1,440 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"bytes"
	6	+	"context"
	7	+	"fmt"
	8	+	"image"
	9	+	"image/png"
	10	+	"io"
	11	+	"math"
	12	+	"net"
	13	+	"os"
	14	+	"os/exec"
	15	+	"path/filepath"
	16	+	"regexp"
	17	+	"runtime"
	18	+	"sort"
	19	+	"strconv"
	20	+	"strings"
	21	+	"sync"
	22	+	"time"
	23	+
	24	+	"github.com/bnema/neferwl/internal/adapters/logging"
	25	+	"github.com/bnema/neferwl/internal/ports"
	26	+	)
	27	+
	28	+	// OCRWord is a recognized word in physical output coordinates.
	29	+	type OCRWord struct {
	30	+	Text string `json:"text"`
	31	+	Rect image.Rectangle `json:"rect"`
	32	+	Confidence float64 `json:"confidence"`
	33	+	// Underlined is set when a thin line runs under the word. UnderL and
	34	+	// UnderR are the x extent of that line (crop-local), which can reach the
	35	+	// neighbouring words of one underlined link.
	36	+	Underlined bool `json:"underlined,omitempty"`
	37	+	UnderL int `json:"under_l,omitempty"`
	38	+	UnderR int `json:"under_r,omitempty"`
	39	+	}
	40	+
	41	+	var languageWarning sync.Once
	42	+	var failureWarning sync.Once
	43	+
	44	+	func ocrLanguage(ctx context.Context, binary string) (string, error) {
	45	+	dir := os.Getenv("TESSDATA_PREFIX")
	46	+	if dir == "" {
	47	+	dir = "/usr/share/tessdata"
	48	+	}
	49	+	if _, err := os.Stat(filepath.Join(dir, "eng.traineddata")); err == nil {
	50	+	return "eng", nil
	51	+	}
	52	+	out, err := exec.CommandContext(ctx, binary, "--list-langs").Output()
	53	+	if err != nil {
	54	+	return "", err
	55	+	}
	56	+	for _, line := range strings.Split(string(out), "\n")[1:] {
	57	+	lang := strings.TrimSpace(line)
	58	+	if lang != "" && lang != "osd" {
	59	+	languageWarning.Do(func() {
	60	+	log := logging.For(ctx, "hints")
	61	+	log.Warn().Str("language", lang).Msg("English OCR data missing; install tesseract-data-eng")
	62	+	})
	63	+	return lang, nil
	64	+	}
	65	+	}
	66	+	return "", fmt.Errorf("no OCR language available")
	67	+	}
	68	+
	69	+	func parseTSV(data []byte, offset image.Point) ([]OCRWord, error) {
	70	+	var words []OCRWord
	71	+	scanner := bufio.NewScanner(bytes.NewReader(data))
	72	+	scanner.Buffer(make([]byte, 4096), 1024*1024)
	73	+	for scanner.Scan() {
	74	+	fields := strings.SplitN(scanner.Text(), "\t", 12)
	75	+	if len(fields) != 12 \|\| fields[0] != "5" {
	76	+	continue
	77	+	}
	78	+	var values [4]int
	79	+	for i := range values {
	80	+	v, err := strconv.Atoi(fields[6+i])
	81	+	if err != nil {
	82	+	return nil, fmt.Errorf("OCR TSV coordinate: %w", err)
	83	+	}
	84	+	values[i] = v
	85	+	}
	86	+	conf, err := strconv.ParseFloat(fields[10], 64)
	87	+	if err != nil {
	88	+	return nil, fmt.Errorf("OCR TSV confidence: %w", err)
	89	+	}
	90	+	if values[2] <= 0 \|\| values[3] <= 0 \|\| strings.TrimSpace(fields[11]) == "" {
	91	+	continue
	92	+	}
	93	+	r := image.Rect(values[0], values[1], values[0]+values[2], values[1]+values[3]).Add(offset)
	94	+	words = append(words, OCRWord{Text: fields[11], Rect: r, Confidence: conf})
	95	+	}
	96	+	return words, scanner.Err()
	97	+	}
	98	+
	99	+	// ocrWorkers is the maximum number of bands recognized in parallel. 4 keeps
	100	+	// the tesseract processes at about 60% of the memory of 8 for the same time.
	101	+	var ocrWorkers = 4
	102	+
	103	+	// pngLevel is the compression of non-grayscale images piped to tesseract.
	104	+	var pngLevel = png.BestSpeed
	105	+
	106	+	// rawPGM feeds grayscale images to tesseract as binary PGM instead of PNG.
	107	+	var rawPGM = true
	108	+
	109	+	// ocrState is the tesseract installation, resolved once.
	110	+	type ocrState struct {
	111	+	done bool
	112	+	binary string
	113	+	lang string
	114	+	err error
	115	+	}
	116	+
	117	+	func (s *ocrState) resolve(ctx context.Context) error {
	118	+	if !s.done {
	119	+	s.done = true
	120	+	if s.binary, s.err = exec.LookPath("tesseract"); s.err == nil {
	121	+	s.lang, s.err = ocrLanguage(ctx, s.binary)
	122	+	}
	123	+	}
	124	+	return s.err
	125	+	}
	126	+
	127	+	// ocrReady resolves tesseract on the first call. When it is missing, OCR is
	128	+	// disabled for the life of the process and the warning is logged once.
	129	+	func (w *worker) ocrReady(ctx context.Context) bool {
	130	+	if err := w.ocr.resolve(ctx); err != nil {
	131	+	failureWarning.Do(func() {
	132	+	log := logging.For(ctx, "hints")
	133	+	log.Warn().Err(err).Msg("OCR unavailable; install tesseract and tesseract-data-eng")
	134	+	})
	135	+	return false
	136	+	}
	137	+	return true
	138	+	}
	139	+
	140	+	// band is a horizontal slice of a window crop recognized by one tesseract
	141	+	// process. It reads rows [top,bottom) and owns the words centred in
	142	+	// [ownTop,ownBottom); the margin only supplies context.
	143	+	type band struct {
	144	+	idx, n int
	145	+	top, bottom int
	146	+	ownTop, ownBottom int
	147	+	}
	148	+
	149	+	func bandsOf(height int) []band {
	150	+	n := min(runtime.GOMAXPROCS(0), ocrWorkers, height)
	151	+	bands := make([]band, 0, max(n, 0))
	152	+	for i := range n {
	153	+	ownTop, ownBottom := iheight/n, (i+1)height/n
	154	+	bands = append(bands, band{idx: i, n: n, top: max(0, ownTop-40), bottom: min(height, ownBottom+40), ownTop: ownTop, ownBottom: ownBottom})
	155	+	}
	156	+	return bands
	157	+	}
	158	+
	159	+	// recognize reads the bands of jobs that missed the cache, in parallel.
	160	+	func (w worker) recognize(ctx context.Context, jobs []bandJob) {
	161	+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
	162	+	defer cancel()
	163	+	// One scan-wide pool: at most ocrWorkers tesseract processes run at once,
	164	+	// whatever the number of windows.
	165	+	sem := make(chan struct{}, max(ocrWorkers, 1))
	166	+	var wg sync.WaitGroup
	167	+	for _, j := range jobs {
	168	+	if j.hit {
	169	+	continue
	170	+	}
	171	+	select {
	172	+	case sem <- struct{}{}:
	173	+	case <-ctx.Done():
	174	+	j.err = ctx.Err()
	175	+	continue
	176	+	}
	177	+	wg.Go(func() {
	178	+	defer func() { <-sem }()
	179	+	j.words, j.err = readBand(ctx, &w.ocr, j)
	180	+	})
	181	+	}
	182	+	wg.Wait()
	183	+	}
	184	+
	185	+	// readBand recognizes one band; the words are crop-local and owned by it.
	186	+	func readBand(ctx context.Context, st ocrState, j bandJob) ([]OCRWord, error) {
	187	+	words, err := recognizeBand(ctx, st.binary, st.lang, j.gray)
	188	+	if err != nil {
	189	+	return nil, err
	190	+	}
	191	+	heights := make([]int, 0, len(words))
	192	+	for _, word := range words {
	193	+	if len(word.Text) >= 3 {
	194	+	heights = append(heights, word.Rect.Dy())
	195	+	}
	196	+	}
	197	+	sort.Ints(heights)
	198	+	if len(heights) > 0 && heights[len(heights)/2] < 12 {
	199	+	words, err = recognizeBand(ctx, st.binary, st.lang, upscaleBand(j.gray))
	200	+	if err != nil {
	201	+	return nil, err
	202	+	}
	203	+	for i := range words {
	204	+	r := words[i].Rect
	205	+	words[i].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
	206	+	}
	207	+	}
	208	+	markUnderlined(j.gray, words)
	209	+	owned := words[:0]
	210	+	for _, word := range words {
	211	+	// Overlap supplies context, but each word belongs to exactly one band.
	212	+	word.Rect = word.Rect.Add(image.Pt(0, j.ext.Min.Y))
	213	+	if center := (word.Rect.Min.Y + word.Rect.Max.Y) / 2; center >= j.own[0] && center < j.own[1] {
	214	+	owned = append(owned, word)
	215	+	}
	216	+	}
	217	+	return owned, nil
	218	+	}
	219	+
	220	+	// readWords recognizes the words of img, a window crop, in parallel bands.
	221	+	func readWords(ctx context.Context, img *image.Gray) ([]OCRWord, int, error) {
	222	+	w := &worker{}
	223	+	if err := w.ocr.resolve(ctx); err != nil {
	224	+	return nil, 0, err
	225	+	}
	226	+	var jobs []*bandJob
	227	+	for _, b := range bandsOf(img.Bounds().Dy()) {
	228	+	ext := image.Rect(0, b.top, img.Bounds().Dx(), b.bottom)
	229	+	jobs = append(jobs, &bandJob{ext: ext, own: [2]int{b.ownTop, b.ownBottom}, gray: img.SubImage(ext).(*image.Gray)})
	230	+	}
	231	+	w.recognize(ctx, jobs)
	232	+	var words []OCRWord
	233	+	for _, j := range jobs {
	234	+	if j.err != nil {
	235	+	return nil, len(jobs), j.err
	236	+	}
	237	+	for _, word := range j.words {
	238	+	if !duplicateWord(words, word) {
	239	+	words = append(words, word)
	240	+	}
	241	+	}
	242	+	}
	243	+	return words, len(jobs), nil
	244	+	}
	245	+
	246	+	// The link patterns compile on the first OCR link check, not at process
	247	+	// start: with hint mode off nothing here costs memory or CPU.
	248	+	var domainPattern = sync.OnceValue(func() *regexp.Regexp {
	249	+	return regexp.MustCompile(`(?i)^(?:[a-z0-9](?:[a-z0-9-][a-z0-9])?\.)+([a-z]{2,24})(?::[0-9]+)?(?:[/?#]\S)?$`)
	250	+	})
	251	+	var schemePattern = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`(?i)^(?:https?\|htps\|file\|ftp):/{1,2}\S+$`) })
	252	+	var emailPattern = sync.OnceValue(func() *regexp.Regexp {
	253	+	return regexp.MustCompile(`(?i)^[a-z0-9.!#$%&'*+/=?^_` + "`" + `{\|}~-]+@(?:[a-z0-9-]+\.)+[a-z]{2,24}$`)
	254	+	})
	255	+	var fileExtensions = sync.OnceValue(func() []string {
	256	+	return strings.Fields("go md txt toml json yaml yml rs py js ts sh c h png jpg log conf lock mod sum html css")
	257	+	})
	258	+
	259	+	func isTextLink(text string) bool {
	260	+	text = strings.Trim(text, "()[]<>.,;:'\"")
	261	+	if schemePattern().MatchString(text) \|\| (strings.HasPrefix(strings.ToLower(text), "www.") && len(text) > 4) \|\| emailPattern().MatchString(text) {
	262	+	return true
	263	+	}
	264	+	host := text
	265	+	if h, _, err := net.SplitHostPort(text); err == nil {
	266	+	host = h
	267	+	}
	268	+	if ip := net.ParseIP(host); ip != nil && ip.To4() != nil {
	269	+	return true
	270	+	}
	271	+	m := domainPattern().FindStringSubmatch(text)
	272	+	if m == nil {
	273	+	return false
	274	+	}
	275	+	for _, ext := range fileExtensions() {
	276	+	if strings.EqualFold(m[1], ext) {
	277	+	return false
	278	+	}
	279	+	}
	280	+	return true
	281	+	}
	282	+
	283	+	func textLinks(words []OCRWord, scale float64) []ports.HintTarget {
	284	+	if scale <= 0 {
	285	+	scale = 1
	286	+	}
	287	+	words = append([]OCRWord(nil), words...)
	288	+	sort.Slice(words, func(i, j int) bool {
	289	+	if words[i].Rect.Min.Y != words[j].Rect.Min.Y {
	290	+	return words[i].Rect.Min.Y < words[j].Rect.Min.Y
	291	+	}
	292	+	return words[i].Rect.Min.X < words[j].Rect.Min.X
	293	+	})
	294	+	var out []ports.HintTarget
	295	+	var urls []image.Rectangle // boxes of the URL targets, in pixels
	296	+	used := make([]bool, len(words))
	297	+	for i, word := range words {
	298	+	if used[i] {
	299	+	continue
	300	+	}
	301	+	text, rect := word.Text, word.Rect
	302	+	// Grow only touching words on the same baseline; ordinary prose is not joined.
	303	+	for {
	304	+	next := -1
	305	+	for j, candidate := range words {
	306	+	if j == i \|\| used[j] \|\| candidate.Rect.Min.X < rect.Max.X-2 {
	307	+	continue
	308	+	}
	309	+	gap := candidate.Rect.Min.X - rect.Max.X
	310	+	if gap > max(3, min(rect.Dy(), candidate.Rect.Dy())/5) {
	311	+	continue
	312	+	}
	313	+	if abs(candidate.Rect.Max.Y-rect.Max.Y) > max(3, rect.Dy()/3) {
	314	+	continue
	315	+	}
	316	+	if next < 0 \|\| candidate.Rect.Min.X < words[next].Rect.Min.X {
	317	+	next = j
	318	+	}
	319	+	}
	320	+	if next < 0 {
	321	+	break
	322	+	}
	323	+	text += words[next].Text
	324	+	rect = rect.Union(words[next].Rect)
	325	+	used[next] = true
	326	+	}
	327	+	if !isTextLink(text) {
	328	+	continue
	329	+	}
	330	+	x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
	331	+	right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
	332	+	out = append(out, ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
	333	+	urls = append(urls, rect)
	334	+	}
	335	+	return append(out, underlinedLinks(words, urls, scale)...)
	336	+	}
	337	+
	338	+	func abs(x int) int {
	339	+	if x < 0 {
	340	+	return -x
	341	+	}
	342	+	return x
	343	+	}
	344	+	func overlap(a, b image.Rectangle) float64 {
	345	+	intersection := a.Intersect(b)
	346	+	area := min(a.Dx()a.Dy(), b.Dx()b.Dy())
	347	+	if intersection.Empty() \|\| area <= 0 {
	348	+	return 0
	349	+	}
	350	+	return float64(intersection.Dx()*intersection.Dy()) / float64(area)
	351	+	}
	352	+	func targetRect(t ports.HintTarget) image.Rectangle {
	353	+	return image.Rect(t.Rect.X, t.Rect.Y, t.Rect.X+t.Rect.W, t.Rect.Y+t.Rect.H)
	354	+	}
	355	+	func recognizeBand(ctx context.Context, binary, lang string, img image.Image) ([]OCRWord, error) {
	356	+	return recognizeImage(ctx, binary, lang, img, "11", false)
	357	+	}
	358	+
	359	+	// pgmReader streams a grayscale image as binary PGM (P5), row by row.
	360	+	type pgmReader struct {
	361	+	head []byte
	362	+	img *image.Gray
	363	+	y int
	364	+	row []byte
	365	+	}
	366	+
	367	+	func newPGMReader(img image.Gray) pgmReader {
	368	+	b := img.Bounds()
	369	+	return &pgmReader{head: fmt.Appendf(nil, "P5\n%d %d\n255\n", b.Dx(), b.Dy()), img: img}
	370	+	}
	371	+
	372	+	func (r *pgmReader) Read(p []byte) (int, error) {
	373	+	n := 0
	374	+	for n < len(p) {
	375	+	if len(r.head) > 0 {
	376	+	c := copy(p[n:], r.head)
	377	+	r.head, n = r.head[c:], n+c
	378	+	continue
	379	+	}
	380	+	if len(r.row) == 0 {
	381	+	b := r.img.Bounds()
	382	+	if r.y >= b.Dy() {
	383	+	break
	384	+	}
	385	+	r.row = r.img.Pix[(r.y+b.Min.Y-r.img.Rect.Min.Y)*r.img.Stride+(b.Min.X-r.img.Rect.Min.X):][:b.Dx()]
	386	+	r.y++
	387	+	}
	388	+	c := copy(p[n:], r.row)
	389	+	r.row, n = r.row[c:], n+c
	390	+	}
	391	+	if n == 0 {
	392	+	return 0, io.EOF
	393	+	}
	394	+	return n, nil
	395	+	}
	396	+
	397	+	func recognizeImage(ctx context.Context, binary, lang string, img image.Image, psm string, normalized bool) ([]OCRWord, error) {
	398	+	args := []string{"stdin", "stdout", "--psm", psm, "-l", lang}
	399	+	if normalized {
	400	+	args = append(args, "-c", "tessedit_do_invert=0")
	401	+	}
	402	+	args = append(args, "tsv")
	403	+	cmd := exec.CommandContext(ctx, binary, args...)
	404	+	cmd.Env = append(os.Environ(), "OMP_THREAD_LIMIT=1")
	405	+	if g, ok := img.(*image.Gray); ok && rawPGM {
	406	+	// Raw grayscale: no deflate, no encoder buffers.
	407	+	cmd.Stdin = newPGMReader(g)
	408	+	} else {
	409	+	// The PNG streams into tesseract; no encoded copy is held.
	410	+	pr, pw := io.Pipe()
	411	+	defer func() { _ = pr.Close() }()
	412	+	go func() {
	413	+	enc := png.Encoder{CompressionLevel: pngLevel}
	414	+	_ = pw.CloseWithError(enc.Encode(pw, img))
	415	+	}()
	416	+	cmd.Stdin = pr
	417	+	}
	418	+	var stderr bytes.Buffer
	419	+	cmd.Stderr = &stderr
	420	+	output, err := cmd.Output()
	421	+	if err != nil {
	422	+	return nil, fmt.Errorf("tesseract: %w: %s", err, stderr.String())
	423	+	}
	424	+	return parseTSV(output, image.Point{})
	425	+	}
	426	+
	427	+	// upscaleBand doubles a grayscale band with nearest-neighbour sampling.
	428	+	func upscaleBand(src image.Gray) image.Gray {
	429	+	b := src.Bounds()
	430	+	dst := image.NewGray(image.Rect(0, 0, b.Dx()2, b.Dy()2))
	431	+	for y := 0; y < b.Dy(); y++ {
	432	+	row := src.Pix[(y+b.Min.Y-src.Rect.Min.Y)*src.Stride+(b.Min.X-src.Rect.Min.X):][:b.Dx()]
	433	+	top := dst.Pix[2ydst.Stride:][:dst.Stride]
	434	+	for x, v := range row {
	435	+	top[2x], top[2x+1] = v, v
	436	+	}
	437	+	copy(dst.Pix[(2y+1)dst.Stride:], top)
	438	+	}
	439	+	return dst
	440	+	}

A internal/adapters/hints/ocr_test.go +238 -0

@@ -0,0 +1,238 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/draw"
	7	+	"image/png"
	8	+	"io"
	9	+	"os"
	10	+	"os/exec"
	11	+	"path/filepath"
	12	+	"reflect"
	13	+	"strings"
	14	+	"testing"
	15	+	"time"
	16	+
	17	+	"github.com/bnema/neferwl/internal/ports"
	18	+	)
	19	+
	20	+	func TestTextLink(t *testing.T) {
	21	+	yes := []string{"ermite.bnema.dev", "google.fr", "example.org/path?q=1", "https.google.fr", "https://example.org", "file:///tmp/main.go", "ftp://host/file", "www.main.go", "user@example.org", "127.0.0.1", "192.168.1.1:8080", "(https://example.org),", "htps://google.fr", "https:/example.org"}
	22	+	no := []string{"ermite-runner", "main.go", "README.md", "v1.2.3", "1.5", "e.g.", "report.txt", "source.rs", "alpha", "999.999.999.999"}
	23	+	for _, s := range yes {
	24	+	t.Run(s, func(t *testing.T) {
	25	+	if !isTextLink(s) {
	26	+	t.Errorf("not accepted: %s", s)
	27	+	}
	28	+	})
	29	+	}
	30	+	for _, s := range no {
	31	+	t.Run(s, func(t *testing.T) {
	32	+	if isTextLink(s) {
	33	+	t.Errorf("incorrectly accepted: %s", s)
	34	+	}
	35	+	})
	36	+	}
	37	+	}
	38	+	func TestParseTSV(t *testing.T) {
	39	+	data := "level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext\n5\t1\t1\t1\t1\t1\t10\t20\t30\t12\t91.5\texample.org\n"
	40	+	words, err := parseTSV([]byte(data), image.Pt(0, 40))
	41	+	if err != nil \|\| len(words) != 1 {
	42	+	t.Fatalf("%v %v", words, err)
	43	+	}
	44	+	if words[0].Rect != image.Rect(10, 60, 40, 72) \|\| words[0].Confidence != 91.5 {
	45	+	t.Fatal(words)
	46	+	}
	47	+	_, err = parseTSV([]byte(strings.ReplaceAll(data, "\t10\t", "\tbad\t")), image.Point{})
	48	+	if err == nil {
	49	+	t.Fatal("expected malformed coordinate error")
	50	+	}
	51	+	}
	52	+	func TestJoinedOCRLink(t *testing.T) {
	53	+	words := []OCRWord{{Text: "https:/", Rect: image.Rect(20, 20, 90, 40)}, {Text: "/example.org", Rect: image.Rect(91, 20, 200, 40)}}
	54	+	got := textLinks(words, 2)
	55	+	if len(got) != 1 \|\| got[0].Rect != (ports.Rect{X: 10, Y: 10, W: 90, H: 10}) {
	56	+	t.Fatal(got)
	57	+	}
	58	+	}
	59	+	func TestRealOCR(t *testing.T) {
	60	+	if _, err := exec.LookPath("tesseract"); err != nil {
	61	+	t.Skip("tesseract not on PATH")
	62	+	}
	63	+	for _, name := range []string{"browser", "terminal"} {
	64	+	t.Run(name, func(t *testing.T) {
	65	+	f, err := os.Open("testdata/" + name + ".png")
	66	+	if err != nil {
	67	+	t.Fatal(err)
	68	+	}
	69	+	defer f.Close()
	70	+	src, err := png.Decode(f)
	71	+	if err != nil {
	72	+	t.Fatal(err)
	73	+	}
	74	+	img := image.NewRGBA(src.Bounds())
	75	+	draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
	76	+	start := time.Now()
	77	+	words, bands, err := readWords(context.Background(), toGray(img))
	78	+	if err != nil {
	79	+	t.Fatal(err)
	80	+	}
	81	+	links := textLinks(words, 1)
	82	+	t.Logf("%s %dx%d ocr_ms=%.1f words=%d links=%d bands=%d", name, img.Bounds().Dx(), img.Bounds().Dy(), ms(time.Since(start)), len(words), len(links), bands)
	83	+	for _, w := range words {
	84	+	t.Logf("%s %v", w.Text, w.Rect)
	85	+	}
	86	+	if name == "terminal" {
	87	+	for _, r := range []image.Rectangle{image.Rect(160, 65, 380, 85), image.Rect(140, 90, 490, 110), image.Rect(95, 115, 630, 138), image.Rect(120, 140, 330, 163), image.Rect(385, 140, 740, 163)} {
	88	+	found := false
	89	+	for _, l := range links {
	90	+	if overlap(targetRect(l), r) > .5 {
	91	+	found = true
	92	+	}
	93	+	}
	94	+	if !found {
	95	+	t.Errorf("missing terminal URL near %v", r)
	96	+	}
	97	+	}
	98	+	if len(links) != 5 {
	99	+	t.Errorf("want 5 terminal links, got %d: %v", len(links), links)
	100	+	}
	101	+	} else if len(links) == 0 {
	102	+	t.Error("no browser links")
	103	+	}
	104	+	})
	105	+	}
	106	+	}
	107	+
	108	+	func TestSmallGlyphOCR(t *testing.T) {
	109	+	binary, err := exec.LookPath("tesseract")
	110	+	if err != nil {
	111	+	t.Skip("tesseract not on PATH")
	112	+	}
	113	+	f, err := os.Open("testdata/terminal.png")
	114	+	if err != nil {
	115	+	t.Fatal(err)
	116	+	}
	117	+	defer f.Close()
	118	+	src, err := png.Decode(f)
	119	+	if err != nil {
	120	+	t.Fatal(err)
	121	+	}
	122	+	small := image.NewGray(image.Rect(0, 0, 640, 200))
	123	+	for y := 0; y < 200; y++ {
	124	+	for x := 0; x < 640; x++ {
	125	+	small.Set(x, y, src.At(x2, y2))
	126	+	}
	127	+	}
	128	+	lang, err := ocrLanguage(context.Background(), binary)
	129	+	if err != nil {
	130	+	t.Fatal(err)
	131	+	}
	132	+	for _, factor := range []int{1, 2} {
	133	+	var img image.Image = small
	134	+	if factor == 2 {
	135	+	img = upscaleBand(small)
	136	+	}
	137	+	start := time.Now()
	138	+	words, err := recognizeBand(context.Background(), binary, lang, img)
	139	+	if err != nil {
	140	+	t.Fatal(err)
	141	+	}
	142	+	t.Logf("glyph experiment %dx: %.1f ms, words=%d links=%d", factor, ms(time.Since(start)), len(words), len(textLinks(words, 1)))
	143	+	}
	144	+	}
	145	+
	146	+	func TestPGMReader(t *testing.T) {
	147	+	g := image.NewGray(image.Rect(0, 0, 3, 3))
	148	+	for i := range g.Pix {
	149	+	g.Pix[i] = uint8(i + 1)
	150	+	}
	151	+	sub := g.SubImage(image.Rect(1, 1, 3, 3)).(*image.Gray)
	152	+	got, err := io.ReadAll(newPGMReader(sub))
	153	+	if err != nil {
	154	+	t.Fatal(err)
	155	+	}
	156	+	if want := "P5\n2 2\n255\n" + string([]byte{5, 6, 8, 9}); string(got) != want {
	157	+	t.Fatalf("%q", got)
	158	+	}
	159	+	}
	160	+
	161	+	// Tesseract must read the raw PGM exactly as it reads the PNG.
	162	+	func TestPGMMatchesPNG(t *testing.T) {
	163	+	binary, err := exec.LookPath("tesseract")
	164	+	if err != nil {
	165	+	t.Skip("tesseract not on PATH")
	166	+	}
	167	+	f, err := os.Open("testdata/terminal.png")
	168	+	if err != nil {
	169	+	t.Fatal(err)
	170	+	}
	171	+	defer f.Close()
	172	+	src, err := png.Decode(f)
	173	+	if err != nil {
	174	+	t.Fatal(err)
	175	+	}
	176	+	rgba := image.NewRGBA(src.Bounds())
	177	+	draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
	178	+	gray := toGray(rgba)
	179	+	lang, err := ocrLanguage(context.Background(), binary)
	180	+	if err != nil {
	181	+	t.Fatal(err)
	182	+	}
	183	+	defer func(v bool) { rawPGM = v }(rawPGM)
	184	+	var res [2][]OCRWord
	185	+	for i, raw := range []bool{false, true} {
	186	+	rawPGM = raw
	187	+	if res[i], err = recognizeBand(context.Background(), binary, lang, gray); err != nil {
	188	+	t.Fatal(err)
	189	+	}
	190	+	}
	191	+	if len(res[0]) == 0 \|\| !reflect.DeepEqual(res[0], res[1]) {
	192	+	t.Fatalf("PNG %d words, PGM %d words", len(res[0]), len(res[1]))
	193	+	}
	194	+	}
	195	+
	196	+	// TestRecognizeCapsProcesses runs the bands of several windows through a
	197	+	// stand-in tesseract that logs when each process starts and ends.
	198	+	func TestRecognizeCapsProcesses(t *testing.T) {
	199	+	dir := t.TempDir()
	200	+	logPath := filepath.Join(dir, "log")
	201	+	script := filepath.Join(dir, "tesseract")
	202	+	body := "#!/bin/sh\ncat >/dev/null\necho s >> " + logPath + "\nsleep 0.1\necho e >> " + logPath + "\nprintf 'level\\tpage_num\\tblock_num\\tpar_num\\tline_num\\tword_num\\tleft\\ttop\\twidth\\theight\\tconf\\ttext\\n'\n"
	203	+	if err := os.WriteFile(script, []byte(body), 0o700); err != nil {
	204	+	t.Fatal(err)
	205	+	}
	206	+	old := ocrWorkers
	207	+	ocrWorkers = 2
	208	+	t.Cleanup(func() { ocrWorkers = old })
	209	+	w := &worker{ocr: ocrState{done: true, binary: script, lang: "eng"}}
	210	+	var jobs []*bandJob
	211	+	for range 8 { // several windows' bands
	212	+	g := image.NewGray(image.Rect(0, 0, 8, 8))
	213	+	jobs = append(jobs, &bandJob{ext: g.Rect, own: [2]int{0, 8}, gray: g})
	214	+	}
	215	+	w.recognize(context.Background(), jobs)
	216	+	data, err := os.ReadFile(logPath)
	217	+	if err != nil {
	218	+	t.Fatal(err)
	219	+	}
	220	+	cur, peak, starts := 0, 0, 0
	221	+	for _, l := range strings.Fields(string(data)) {
	222	+	if l == "s" {
	223	+	cur++
	224	+	starts++
	225	+	peak = max(peak, cur)
	226	+	} else {
	227	+	cur--
	228	+	}
	229	+	}
	230	+	if starts != 8 \|\| peak > 2 {
	231	+	t.Fatalf("starts=%d peak=%d", starts, peak)
	232	+	}
	233	+	for _, j := range jobs {
	234	+	if j.err != nil {
	235	+	t.Fatal(j.err)
	236	+	}
	237	+	}
	238	+	}

@@ -0,0 +1,238 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/draw"
	7	+	"image/png"
	8	+	"io"
	9	+	"os"
	10	+	"os/exec"
	11	+	"path/filepath"
	12	+	"reflect"
	13	+	"strings"
	14	+	"testing"
	15	+	"time"
	16	+
	17	+	"github.com/bnema/neferwl/internal/ports"
	18	+	)
	19	+
	20	+	func TestTextLink(t *testing.T) {
	21	+	yes := []string{"ermite.bnema.dev", "google.fr", "example.org/path?q=1", "https.google.fr", "https://example.org", "file:///tmp/main.go", "ftp://host/file", "www.main.go", "user@example.org", "127.0.0.1", "192.168.1.1:8080", "(https://example.org),", "htps://google.fr", "https:/example.org"}
	22	+	no := []string{"ermite-runner", "main.go", "README.md", "v1.2.3", "1.5", "e.g.", "report.txt", "source.rs", "alpha", "999.999.999.999"}
	23	+	for _, s := range yes {
	24	+	t.Run(s, func(t *testing.T) {
	25	+	if !isTextLink(s) {
	26	+	t.Errorf("not accepted: %s", s)
	27	+	}
	28	+	})
	29	+	}
	30	+	for _, s := range no {
	31	+	t.Run(s, func(t *testing.T) {
	32	+	if isTextLink(s) {
	33	+	t.Errorf("incorrectly accepted: %s", s)
	34	+	}
	35	+	})
	36	+	}
	37	+	}
	38	+	func TestParseTSV(t *testing.T) {
	39	+	data := "level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext\n5\t1\t1\t1\t1\t1\t10\t20\t30\t12\t91.5\texample.org\n"
	40	+	words, err := parseTSV([]byte(data), image.Pt(0, 40))
	41	+	if err != nil \|\| len(words) != 1 {
	42	+	t.Fatalf("%v %v", words, err)
	43	+	}
	44	+	if words[0].Rect != image.Rect(10, 60, 40, 72) \|\| words[0].Confidence != 91.5 {
	45	+	t.Fatal(words)
	46	+	}
	47	+	_, err = parseTSV([]byte(strings.ReplaceAll(data, "\t10\t", "\tbad\t")), image.Point{})
	48	+	if err == nil {
	49	+	t.Fatal("expected malformed coordinate error")
	50	+	}
	51	+	}
	52	+	func TestJoinedOCRLink(t *testing.T) {
	53	+	words := []OCRWord{{Text: "https:/", Rect: image.Rect(20, 20, 90, 40)}, {Text: "/example.org", Rect: image.Rect(91, 20, 200, 40)}}
	54	+	got := textLinks(words, 2)
	55	+	if len(got) != 1 \|\| got[0].Rect != (ports.Rect{X: 10, Y: 10, W: 90, H: 10}) {
	56	+	t.Fatal(got)
	57	+	}
	58	+	}
	59	+	func TestRealOCR(t *testing.T) {
	60	+	if _, err := exec.LookPath("tesseract"); err != nil {
	61	+	t.Skip("tesseract not on PATH")
	62	+	}
	63	+	for _, name := range []string{"browser", "terminal"} {
	64	+	t.Run(name, func(t *testing.T) {
	65	+	f, err := os.Open("testdata/" + name + ".png")
	66	+	if err != nil {
	67	+	t.Fatal(err)
	68	+	}
	69	+	defer f.Close()
	70	+	src, err := png.Decode(f)
	71	+	if err != nil {
	72	+	t.Fatal(err)
	73	+	}
	74	+	img := image.NewRGBA(src.Bounds())
	75	+	draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
	76	+	start := time.Now()
	77	+	words, bands, err := readWords(context.Background(), toGray(img))
	78	+	if err != nil {
	79	+	t.Fatal(err)
	80	+	}
	81	+	links := textLinks(words, 1)
	82	+	t.Logf("%s %dx%d ocr_ms=%.1f words=%d links=%d bands=%d", name, img.Bounds().Dx(), img.Bounds().Dy(), ms(time.Since(start)), len(words), len(links), bands)
	83	+	for _, w := range words {
	84	+	t.Logf("%s %v", w.Text, w.Rect)
	85	+	}
	86	+	if name == "terminal" {
	87	+	for _, r := range []image.Rectangle{image.Rect(160, 65, 380, 85), image.Rect(140, 90, 490, 110), image.Rect(95, 115, 630, 138), image.Rect(120, 140, 330, 163), image.Rect(385, 140, 740, 163)} {
	88	+	found := false
	89	+	for _, l := range links {
	90	+	if overlap(targetRect(l), r) > .5 {
	91	+	found = true
	92	+	}
	93	+	}
	94	+	if !found {
	95	+	t.Errorf("missing terminal URL near %v", r)
	96	+	}
	97	+	}
	98	+	if len(links) != 5 {
	99	+	t.Errorf("want 5 terminal links, got %d: %v", len(links), links)
	100	+	}
	101	+	} else if len(links) == 0 {
	102	+	t.Error("no browser links")
	103	+	}
	104	+	})
	105	+	}
	106	+	}
	107	+
	108	+	func TestSmallGlyphOCR(t *testing.T) {
	109	+	binary, err := exec.LookPath("tesseract")
	110	+	if err != nil {
	111	+	t.Skip("tesseract not on PATH")
	112	+	}
	113	+	f, err := os.Open("testdata/terminal.png")
	114	+	if err != nil {
	115	+	t.Fatal(err)
	116	+	}
	117	+	defer f.Close()
	118	+	src, err := png.Decode(f)
	119	+	if err != nil {
	120	+	t.Fatal(err)
	121	+	}
	122	+	small := image.NewGray(image.Rect(0, 0, 640, 200))
	123	+	for y := 0; y < 200; y++ {
	124	+	for x := 0; x < 640; x++ {
	125	+	small.Set(x, y, src.At(x2, y2))
	126	+	}
	127	+	}
	128	+	lang, err := ocrLanguage(context.Background(), binary)
	129	+	if err != nil {
	130	+	t.Fatal(err)
	131	+	}
	132	+	for _, factor := range []int{1, 2} {
	133	+	var img image.Image = small
	134	+	if factor == 2 {
	135	+	img = upscaleBand(small)
	136	+	}
	137	+	start := time.Now()
	138	+	words, err := recognizeBand(context.Background(), binary, lang, img)
	139	+	if err != nil {
	140	+	t.Fatal(err)
	141	+	}
	142	+	t.Logf("glyph experiment %dx: %.1f ms, words=%d links=%d", factor, ms(time.Since(start)), len(words), len(textLinks(words, 1)))
	143	+	}
	144	+	}
	145	+
	146	+	func TestPGMReader(t *testing.T) {
	147	+	g := image.NewGray(image.Rect(0, 0, 3, 3))
	148	+	for i := range g.Pix {
	149	+	g.Pix[i] = uint8(i + 1)
	150	+	}
	151	+	sub := g.SubImage(image.Rect(1, 1, 3, 3)).(*image.Gray)
	152	+	got, err := io.ReadAll(newPGMReader(sub))
	153	+	if err != nil {
	154	+	t.Fatal(err)
	155	+	}
	156	+	if want := "P5\n2 2\n255\n" + string([]byte{5, 6, 8, 9}); string(got) != want {
	157	+	t.Fatalf("%q", got)
	158	+	}
	159	+	}
	160	+
	161	+	// Tesseract must read the raw PGM exactly as it reads the PNG.
	162	+	func TestPGMMatchesPNG(t *testing.T) {
	163	+	binary, err := exec.LookPath("tesseract")
	164	+	if err != nil {
	165	+	t.Skip("tesseract not on PATH")
	166	+	}
	167	+	f, err := os.Open("testdata/terminal.png")
	168	+	if err != nil {
	169	+	t.Fatal(err)
	170	+	}
	171	+	defer f.Close()
	172	+	src, err := png.Decode(f)
	173	+	if err != nil {
	174	+	t.Fatal(err)
	175	+	}
	176	+	rgba := image.NewRGBA(src.Bounds())
	177	+	draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
	178	+	gray := toGray(rgba)
	179	+	lang, err := ocrLanguage(context.Background(), binary)
	180	+	if err != nil {
	181	+	t.Fatal(err)
	182	+	}
	183	+	defer func(v bool) { rawPGM = v }(rawPGM)
	184	+	var res [2][]OCRWord
	185	+	for i, raw := range []bool{false, true} {
	186	+	rawPGM = raw
	187	+	if res[i], err = recognizeBand(context.Background(), binary, lang, gray); err != nil {
	188	+	t.Fatal(err)
	189	+	}
	190	+	}
	191	+	if len(res[0]) == 0 \|\| !reflect.DeepEqual(res[0], res[1]) {
	192	+	t.Fatalf("PNG %d words, PGM %d words", len(res[0]), len(res[1]))
	193	+	}
	194	+	}
	195	+
	196	+	// TestRecognizeCapsProcesses runs the bands of several windows through a
	197	+	// stand-in tesseract that logs when each process starts and ends.
	198	+	func TestRecognizeCapsProcesses(t *testing.T) {
	199	+	dir := t.TempDir()
	200	+	logPath := filepath.Join(dir, "log")
	201	+	script := filepath.Join(dir, "tesseract")
	202	+	body := "#!/bin/sh\ncat >/dev/null\necho s >> " + logPath + "\nsleep 0.1\necho e >> " + logPath + "\nprintf 'level\\tpage_num\\tblock_num\\tpar_num\\tline_num\\tword_num\\tleft\\ttop\\twidth\\theight\\tconf\\ttext\\n'\n"
	203	+	if err := os.WriteFile(script, []byte(body), 0o700); err != nil {
	204	+	t.Fatal(err)
	205	+	}
	206	+	old := ocrWorkers
	207	+	ocrWorkers = 2
	208	+	t.Cleanup(func() { ocrWorkers = old })
	209	+	w := &worker{ocr: ocrState{done: true, binary: script, lang: "eng"}}
	210	+	var jobs []*bandJob
	211	+	for range 8 { // several windows' bands
	212	+	g := image.NewGray(image.Rect(0, 0, 8, 8))
	213	+	jobs = append(jobs, &bandJob{ext: g.Rect, own: [2]int{0, 8}, gray: g})
	214	+	}
	215	+	w.recognize(context.Background(), jobs)
	216	+	data, err := os.ReadFile(logPath)
	217	+	if err != nil {
	218	+	t.Fatal(err)
	219	+	}
	220	+	cur, peak, starts := 0, 0, 0
	221	+	for _, l := range strings.Fields(string(data)) {
	222	+	if l == "s" {
	223	+	cur++
	224	+	starts++
	225	+	peak = max(peak, cur)
	226	+	} else {
	227	+	cur--
	228	+	}
	229	+	}
	230	+	if starts != 8 \|\| peak > 2 {
	231	+	t.Fatalf("starts=%d peak=%d", starts, peak)
	232	+	}
	233	+	for _, j := range jobs {
	234	+	if j.err != nil {
	235	+	t.Fatal(j.err)
	236	+	}
	237	+	}
	238	+	}

A internal/adapters/hints/scan_bench_test.go +49 -0

@@ -0,0 +1,49 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"os"
	6	+	"os/exec"
	7	+	"runtime"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/neferwl/internal/ports"
	12	+	)
	13	+
	14	+	// BenchmarkScan runs whole scans of a 5120x2160 frame with a noisy window
	15	+	// (the cache never hits) and reports the Go heap bytes allocated per scan.
	16	+	func BenchmarkScan(b *testing.B) {
	17	+	if _, err := exec.LookPath("tesseract"); err != nil {
	18	+	b.Skip("tesseract not on PATH")
	19	+	}
	20	+	const w, h = 5120, 2160
	21	+	ctx := context.Background()
	22	+	w0 := &worker{}
	23	+	requests := make(chan ports.HintScanRequest)
	24	+	captures := make(chan ports.CaptureRequest, 1)
	25	+	captured := make(chan ports.CaptureDone, 1)
	26	+	results := make(chan ports.HintScanResult, 1)
	27	+	req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
	28	+	if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
	29	+	rawPGM = false
	30	+	}
	31	+	b.ReportAllocs()
	32	+	var peak uint64
	33	+	for i := range b.N {
	34	+	go func() {
	35	+	creq := <-captures
	36	+	creq.Mapped[0] = byte(i) // each scan differs: no cache hit
	37	+	creq.Mapped[1] = byte(i >> 8)
	38	+	captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	39	+	}()
	40	+	res := w0.serve(ctx, req, requests, captures, captured, results, 5*time.Second)
	41	+	if res.Err != nil {
	42	+	b.Fatal(res.Err)
	43	+	}
	44	+	var m runtime.MemStats
	45	+	runtime.ReadMemStats(&m)
	46	+	peak = max(peak, m.HeapSys-m.HeapReleased)
	47	+	}
	48	+	b.ReportMetric(float64(peak)/1e6, "heapMB")
	49	+	}

@@ -0,0 +1,49 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"os"
	6	+	"os/exec"
	7	+	"runtime"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/neferwl/internal/ports"
	12	+	)
	13	+
	14	+	// BenchmarkScan runs whole scans of a 5120x2160 frame with a noisy window
	15	+	// (the cache never hits) and reports the Go heap bytes allocated per scan.
	16	+	func BenchmarkScan(b *testing.B) {
	17	+	if _, err := exec.LookPath("tesseract"); err != nil {
	18	+	b.Skip("tesseract not on PATH")
	19	+	}
	20	+	const w, h = 5120, 2160
	21	+	ctx := context.Background()
	22	+	w0 := &worker{}
	23	+	requests := make(chan ports.HintScanRequest)
	24	+	captures := make(chan ports.CaptureRequest, 1)
	25	+	captured := make(chan ports.CaptureDone, 1)
	26	+	results := make(chan ports.HintScanResult, 1)
	27	+	req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
	28	+	if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
	29	+	rawPGM = false
	30	+	}
	31	+	b.ReportAllocs()
	32	+	var peak uint64
	33	+	for i := range b.N {
	34	+	go func() {
	35	+	creq := <-captures
	36	+	creq.Mapped[0] = byte(i) // each scan differs: no cache hit
	37	+	creq.Mapped[1] = byte(i >> 8)
	38	+	captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	39	+	}()
	40	+	res := w0.serve(ctx, req, requests, captures, captured, results, 5*time.Second)
	41	+	if res.Err != nil {
	42	+	b.Fatal(res.Err)
	43	+	}
	44	+	var m runtime.MemStats
	45	+	runtime.ReadMemStats(&m)
	46	+	peak = max(peak, m.HeapSys-m.HeapReleased)
	47	+	}
	48	+	b.ReportMetric(float64(peak)/1e6, "heapMB")
	49	+	}

A internal/adapters/hints/strips_test.go +124 -0

@@ -0,0 +1,124 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/color"
	7	+	"image/draw"
	8	+	"os/exec"
	9	+	"runtime"
	10	+	"sort"
	11	+	"sync"
	12	+	"time"
	13	+	)
	14	+
	15	+	type stripOffset struct {
	16	+	source, packed image.Rectangle
	17	+	factor int
	18	+	}
	19	+	type ocrStrip struct {
	20	+	img *image.Gray
	21	+	offsets []stripOffset
	22	+	}
	23	+
	24	+	func packLines(src *image.RGBA, lines []image.Rectangle, workers int, normalize bool) []ocrStrip {
	25	+	n := min(workers, len(lines))
	26	+	if n == 0 {
	27	+	return nil
	28	+	}
	29	+	groups := make([][]image.Rectangle, n)
	30	+	areas := make([]int, n)
	31	+	lines = append([]image.Rectangle(nil), lines...)
	32	+	sort.Slice(lines, func(i, j int) bool { return lines[i].Dx()lines[i].Dy() > lines[j].Dx()lines[j].Dy() })
	33	+	for _, r := range lines {
	34	+	best := 0
	35	+	for i := range n {
	36	+	if areas[i] < areas[best] {
	37	+	best = i
	38	+	}
	39	+	}
	40	+	groups[best] = append(groups[best], r)
	41	+	areas[best] += r.Dx() * r.Dy()
	42	+	}
	43	+	strips := make([]ocrStrip, n)
	44	+	for i, group := range groups {
	45	+	width, height := 0, 8
	46	+	for _, r := range group {
	47	+	factor := 1
	48	+	if r.Dy() < 12 {
	49	+	factor = 2
	50	+	}
	51	+	crop := r.Inset(-4).Intersect(src.Bounds())
	52	+	packed := image.Rect(4, height, 4+crop.Dx()factor, height+crop.Dy()factor)
	53	+	strips[i].offsets = append(strips[i].offsets, stripOffset{crop, packed, factor})
	54	+	width = max(width, packed.Max.X+4)
	55	+	height = packed.Max.Y + 8
	56	+	}
	57	+	dst := image.NewGray(image.Rect(0, 0, width, height))
	58	+	draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
	59	+	for _, off := range strips[i].offsets {
	60	+	// The padded perimeter estimates the background without depending on text colour.
	61	+	var sum, count int
	62	+	r := off.source
	63	+	for x := r.Min.X; x < r.Max.X; x++ {
	64	+	sum += int(color.GrayModel.Convert(src.At(x, r.Min.Y)).(color.Gray).Y)
	65	+	count++
	66	+	}
	67	+	invert := normalize && count > 0 && sum/count < 128
	68	+	for y := off.packed.Min.Y; y < off.packed.Max.Y; y++ {
	69	+	for x := off.packed.Min.X; x < off.packed.Max.X; x++ {
	70	+	v := color.GrayModel.Convert(src.At(r.Min.X+(x-off.packed.Min.X)/off.factor, r.Min.Y+(y-off.packed.Min.Y)/off.factor)).(color.Gray).Y
	71	+	if invert {
	72	+	v = 255 - v
	73	+	}
	74	+	dst.SetGray(x, y, color.Gray{Y: v})
	75	+	}
	76	+	}
	77	+	}
	78	+	strips[i].img = dst
	79	+	}
	80	+	return strips
	81	+	}
	82	+
	83	+	func readStrips(ctx context.Context, img *image.RGBA, lines []image.Rectangle, workers int, normalize bool) ([]OCRWord, int, error) {
	84	+	binary, err := exec.LookPath("tesseract")
	85	+	if err != nil {
	86	+	return nil, 0, err
	87	+	}
	88	+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
	89	+	defer cancel()
	90	+	lang, err := ocrLanguage(ctx, binary)
	91	+	if err != nil {
	92	+	return nil, 0, err
	93	+	}
	94	+	strips := packLines(img, lines, min(workers, runtime.GOMAXPROCS(0), 16), normalize)
	95	+	results := make([][]OCRWord, len(strips))
	96	+	errs := make([]error, len(strips))
	97	+	var wg sync.WaitGroup
	98	+	for i, s := range strips {
	99	+	wg.Go(func() {
	100	+	words, e := recognizeImage(ctx, binary, lang, s.img, "6", normalize)
	101	+	errs[i] = e
	102	+	for _, w := range words {
	103	+	center := image.Pt((w.Rect.Min.X+w.Rect.Max.X)/2, (w.Rect.Min.Y+w.Rect.Max.Y)/2)
	104	+	for _, off := range s.offsets {
	105	+	if center.In(off.packed) {
	106	+	r := w.Rect.Sub(off.packed.Min)
	107	+	w.Rect = image.Rect(r.Min.X/off.factor, r.Min.Y/off.factor, (r.Max.X+off.factor-1)/off.factor, (r.Max.Y+off.factor-1)/off.factor).Add(off.source.Min)
	108	+	results[i] = append(results[i], w)
	109	+	break
	110	+	}
	111	+	}
	112	+	}
	113	+	})
	114	+	}
	115	+	wg.Wait()
	116	+	var words []OCRWord
	117	+	for i := range strips {
	118	+	if errs[i] != nil {
	119	+	return nil, len(strips), errs[i]
	120	+	}
	121	+	words = append(words, results[i]...)
	122	+	}
	123	+	return words, len(strips), nil
	124	+	}

@@ -0,0 +1,124 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/color"
	7	+	"image/draw"
	8	+	"os/exec"
	9	+	"runtime"
	10	+	"sort"
	11	+	"sync"
	12	+	"time"
	13	+	)
	14	+
	15	+	type stripOffset struct {
	16	+	source, packed image.Rectangle
	17	+	factor int
	18	+	}
	19	+	type ocrStrip struct {
	20	+	img *image.Gray
	21	+	offsets []stripOffset
	22	+	}
	23	+
	24	+	func packLines(src *image.RGBA, lines []image.Rectangle, workers int, normalize bool) []ocrStrip {
	25	+	n := min(workers, len(lines))
	26	+	if n == 0 {
	27	+	return nil
	28	+	}
	29	+	groups := make([][]image.Rectangle, n)
	30	+	areas := make([]int, n)
	31	+	lines = append([]image.Rectangle(nil), lines...)
	32	+	sort.Slice(lines, func(i, j int) bool { return lines[i].Dx()lines[i].Dy() > lines[j].Dx()lines[j].Dy() })
	33	+	for _, r := range lines {
	34	+	best := 0
	35	+	for i := range n {
	36	+	if areas[i] < areas[best] {
	37	+	best = i
	38	+	}
	39	+	}
	40	+	groups[best] = append(groups[best], r)
	41	+	areas[best] += r.Dx() * r.Dy()
	42	+	}
	43	+	strips := make([]ocrStrip, n)
	44	+	for i, group := range groups {
	45	+	width, height := 0, 8
	46	+	for _, r := range group {
	47	+	factor := 1
	48	+	if r.Dy() < 12 {
	49	+	factor = 2
	50	+	}
	51	+	crop := r.Inset(-4).Intersect(src.Bounds())
	52	+	packed := image.Rect(4, height, 4+crop.Dx()factor, height+crop.Dy()factor)
	53	+	strips[i].offsets = append(strips[i].offsets, stripOffset{crop, packed, factor})
	54	+	width = max(width, packed.Max.X+4)
	55	+	height = packed.Max.Y + 8
	56	+	}
	57	+	dst := image.NewGray(image.Rect(0, 0, width, height))
	58	+	draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
	59	+	for _, off := range strips[i].offsets {
	60	+	// The padded perimeter estimates the background without depending on text colour.
	61	+	var sum, count int
	62	+	r := off.source
	63	+	for x := r.Min.X; x < r.Max.X; x++ {
	64	+	sum += int(color.GrayModel.Convert(src.At(x, r.Min.Y)).(color.Gray).Y)
	65	+	count++
	66	+	}
	67	+	invert := normalize && count > 0 && sum/count < 128
	68	+	for y := off.packed.Min.Y; y < off.packed.Max.Y; y++ {
	69	+	for x := off.packed.Min.X; x < off.packed.Max.X; x++ {
	70	+	v := color.GrayModel.Convert(src.At(r.Min.X+(x-off.packed.Min.X)/off.factor, r.Min.Y+(y-off.packed.Min.Y)/off.factor)).(color.Gray).Y
	71	+	if invert {
	72	+	v = 255 - v
	73	+	}
	74	+	dst.SetGray(x, y, color.Gray{Y: v})
	75	+	}
	76	+	}
	77	+	}
	78	+	strips[i].img = dst
	79	+	}
	80	+	return strips
	81	+	}
	82	+
	83	+	func readStrips(ctx context.Context, img *image.RGBA, lines []image.Rectangle, workers int, normalize bool) ([]OCRWord, int, error) {
	84	+	binary, err := exec.LookPath("tesseract")
	85	+	if err != nil {
	86	+	return nil, 0, err
	87	+	}
	88	+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
	89	+	defer cancel()
	90	+	lang, err := ocrLanguage(ctx, binary)
	91	+	if err != nil {
	92	+	return nil, 0, err
	93	+	}
	94	+	strips := packLines(img, lines, min(workers, runtime.GOMAXPROCS(0), 16), normalize)
	95	+	results := make([][]OCRWord, len(strips))
	96	+	errs := make([]error, len(strips))
	97	+	var wg sync.WaitGroup
	98	+	for i, s := range strips {
	99	+	wg.Go(func() {
	100	+	words, e := recognizeImage(ctx, binary, lang, s.img, "6", normalize)
	101	+	errs[i] = e
	102	+	for _, w := range words {
	103	+	center := image.Pt((w.Rect.Min.X+w.Rect.Max.X)/2, (w.Rect.Min.Y+w.Rect.Max.Y)/2)
	104	+	for _, off := range s.offsets {
	105	+	if center.In(off.packed) {
	106	+	r := w.Rect.Sub(off.packed.Min)
	107	+	w.Rect = image.Rect(r.Min.X/off.factor, r.Min.Y/off.factor, (r.Max.X+off.factor-1)/off.factor, (r.Max.Y+off.factor-1)/off.factor).Add(off.source.Min)
	108	+	results[i] = append(results[i], w)
	109	+	break
	110	+	}
	111	+	}
	112	+	}
	113	+	})
	114	+	}
	115	+	wg.Wait()
	116	+	var words []OCRWord
	117	+	for i := range strips {
	118	+	if errs[i] != nil {
	119	+	return nil, len(strips), errs[i]
	120	+	}
	121	+	words = append(words, results[i]...)
	122	+	}
	123	+	return words, len(strips), nil
	124	+	}

A internal/adapters/hints/target_source_mock_test.go +174 -0

@@ -0,0 +1,174 @@
	1	+	// Code generated by mockery; DO NOT EDIT.
	2	+	// github.com/vektra/mockery
	3	+	// template: testify
	4	+
	5	+	package hints
	6	+
	7	+	import (
	8	+	"context"
	9	+
	10	+	"github.com/bnema/neferwl/internal/ports"
	11	+	mock "github.com/stretchr/testify/mock"
	12	+	)
	13	+
	14	+	// newMocktargetSource creates a new instance of mocktargetSource. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
	15	+	// The first argument is typically a *testing.T value.
	16	+	func newMocktargetSource(t interface {
	17	+	mock.TestingT
	18	+	Cleanup(func())
	19	+	}) *mocktargetSource {
	20	+	if helper, ok := t.(interface{ Helper() }); ok {
	21	+	helper.Helper()
	22	+	}
	23	+
	24	+	mock := &mocktargetSource{}
	25	+	mock.Mock.Test(t)
	26	+
	27	+	t.Cleanup(func() {
	28	+	if helper, ok := t.(interface{ Helper() }); ok {
	29	+	helper.Helper()
	30	+	}
	31	+	mock.AssertExpectations(t)
	32	+	})
	33	+
	34	+	return mock
	35	+	}
	36	+
	37	+	// mocktargetSource is an autogenerated mock type for the targetSource type
	38	+	type mocktargetSource struct {
	39	+	mock.Mock
	40	+	}
	41	+
	42	+	type mocktargetSource_Expecter struct {
	43	+	mock *mock.Mock
	44	+	}
	45	+
	46	+	func (_m mocktargetSource) EXPECT() mocktargetSource_Expecter {
	47	+	return &mocktargetSource_Expecter{mock: &_m.Mock}
	48	+	}
	49	+
	50	+	// Close provides a mock function for the type mocktargetSource
	51	+	func (_mock *mocktargetSource) Close() error {
	52	+	ret := _mock.Called()
	53	+
	54	+	if len(ret) == 0 {
	55	+	panic("no return value specified for Close")
	56	+	}
	57	+
	58	+	var r0 error
	59	+	if returnFunc, ok := ret.Get(0).(func() error); ok {
	60	+	r0 = returnFunc()
	61	+	} else {
	62	+	r0 = ret.Error(0)
	63	+	}
	64	+	return r0
	65	+	}
	66	+
	67	+	// mocktargetSource_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
	68	+	type mocktargetSource_Close_Call struct {
	69	+	*mock.Call
	70	+	}
	71	+
	72	+	// Close is a helper method to define mock.On call
	73	+	func (_e mocktargetSource_Expecter) Close() mocktargetSource_Close_Call {
	74	+	return &mocktargetSource_Close_Call{Call: _e.mock.On("Close")}
	75	+	}
	76	+
	77	+	func (_c mocktargetSource_Close_Call) Run(run func()) mocktargetSource_Close_Call {
	78	+	_c.Call.Run(func(args mock.Arguments) {
	79	+	run()
	80	+	})
	81	+	return _c
	82	+	}
	83	+
	84	+	func (_c mocktargetSource_Close_Call) Return(err error) mocktargetSource_Close_Call {
	85	+	_c.Call.Return(err)
	86	+	return _c
	87	+	}
	88	+
	89	+	func (_c mocktargetSource_Close_Call) RunAndReturn(run func() error) mocktargetSource_Close_Call {
	90	+	_c.Call.Return(run)
	91	+	return _c
	92	+	}
	93	+
	94	+	// Targets provides a mock function for the type mocktargetSource
	95	+	func (_mock *mocktargetSource) Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error) {
	96	+	ret := _mock.Called(ctx, windows, linksOnly)
	97	+
	98	+	if len(ret) == 0 {
	99	+	panic("no return value specified for Targets")
	100	+	}
	101	+
	102	+	var r0 []ports.HintTarget
	103	+	var r1 map[ports.WindowID]bool
	104	+	var r2 error
	105	+	if returnFunc, ok := ret.Get(0).(func(context.Context, []ports.HintWindow, bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)); ok {
	106	+	return returnFunc(ctx, windows, linksOnly)
	107	+	}
	108	+	if returnFunc, ok := ret.Get(0).(func(context.Context, []ports.HintWindow, bool) []ports.HintTarget); ok {
	109	+	r0 = returnFunc(ctx, windows, linksOnly)
	110	+	} else {
	111	+	if ret.Get(0) != nil {
	112	+	r0 = ret.Get(0).([]ports.HintTarget)
	113	+	}
	114	+	}
	115	+	if returnFunc, ok := ret.Get(1).(func(context.Context, []ports.HintWindow, bool) map[ports.WindowID]bool); ok {
	116	+	r1 = returnFunc(ctx, windows, linksOnly)
	117	+	} else {
	118	+	if ret.Get(1) != nil {
	119	+	r1 = ret.Get(1).(map[ports.WindowID]bool)
	120	+	}
	121	+	}
	122	+	if returnFunc, ok := ret.Get(2).(func(context.Context, []ports.HintWindow, bool) error); ok {
	123	+	r2 = returnFunc(ctx, windows, linksOnly)
	124	+	} else {
	125	+	r2 = ret.Error(2)
	126	+	}
	127	+	return r0, r1, r2
	128	+	}
	129	+
	130	+	// mocktargetSource_Targets_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Targets'
	131	+	type mocktargetSource_Targets_Call struct {
	132	+	*mock.Call
	133	+	}
	134	+
	135	+	// Targets is a helper method to define mock.On call
	136	+	// - ctx context.Context
	137	+	// - windows []ports.HintWindow
	138	+	// - linksOnly bool
	139	+	func (_e mocktargetSource_Expecter) Targets(ctx any, windows any, linksOnly any) mocktargetSource_Targets_Call {
	140	+	return &mocktargetSource_Targets_Call{Call: _e.mock.On("Targets", ctx, windows, linksOnly)}
	141	+	}
	142	+
	143	+	func (_c mocktargetSource_Targets_Call) Run(run func(ctx context.Context, windows []ports.HintWindow, linksOnly bool)) mocktargetSource_Targets_Call {
	144	+	_c.Call.Run(func(args mock.Arguments) {
	145	+	var arg0 context.Context
	146	+	if args[0] != nil {
	147	+	arg0 = args[0].(context.Context)
	148	+	}
	149	+	var arg1 []ports.HintWindow
	150	+	if args[1] != nil {
	151	+	arg1 = args[1].([]ports.HintWindow)
	152	+	}
	153	+	var arg2 bool
	154	+	if args[2] != nil {
	155	+	arg2 = args[2].(bool)
	156	+	}
	157	+	run(
	158	+	arg0,
	159	+	arg1,
	160	+	arg2,
	161	+	)
	162	+	})
	163	+	return _c
	164	+	}
	165	+
	166	+	func (_c mocktargetSource_Targets_Call) Return(hintTargets []ports.HintTarget, windowIDToBool map[ports.WindowID]bool, err error) mocktargetSource_Targets_Call {
	167	+	_c.Call.Return(hintTargets, windowIDToBool, err)
	168	+	return _c
	169	+	}
	170	+
	171	+	func (_c mocktargetSource_Targets_Call) RunAndReturn(run func(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)) mocktargetSource_Targets_Call {
	172	+	_c.Call.Return(run)
	173	+	return _c
	174	+	}

@@ -0,0 +1,174 @@
	1	+	// Code generated by mockery; DO NOT EDIT.
	2	+	// github.com/vektra/mockery
	3	+	// template: testify
	4	+
	5	+	package hints
	6	+
	7	+	import (
	8	+	"context"
	9	+
	10	+	"github.com/bnema/neferwl/internal/ports"
	11	+	mock "github.com/stretchr/testify/mock"
	12	+	)
	13	+
	14	+	// newMocktargetSource creates a new instance of mocktargetSource. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
	15	+	// The first argument is typically a *testing.T value.
	16	+	func newMocktargetSource(t interface {
	17	+	mock.TestingT
	18	+	Cleanup(func())
	19	+	}) *mocktargetSource {
	20	+	if helper, ok := t.(interface{ Helper() }); ok {
	21	+	helper.Helper()
	22	+	}
	23	+
	24	+	mock := &mocktargetSource{}
	25	+	mock.Mock.Test(t)
	26	+
	27	+	t.Cleanup(func() {
	28	+	if helper, ok := t.(interface{ Helper() }); ok {
	29	+	helper.Helper()
	30	+	}
	31	+	mock.AssertExpectations(t)
	32	+	})
	33	+
	34	+	return mock
	35	+	}
	36	+
	37	+	// mocktargetSource is an autogenerated mock type for the targetSource type
	38	+	type mocktargetSource struct {
	39	+	mock.Mock
	40	+	}
	41	+
	42	+	type mocktargetSource_Expecter struct {
	43	+	mock *mock.Mock
	44	+	}
	45	+
	46	+	func (_m mocktargetSource) EXPECT() mocktargetSource_Expecter {
	47	+	return &mocktargetSource_Expecter{mock: &_m.Mock}
	48	+	}
	49	+
	50	+	// Close provides a mock function for the type mocktargetSource
	51	+	func (_mock *mocktargetSource) Close() error {
	52	+	ret := _mock.Called()
	53	+
	54	+	if len(ret) == 0 {
	55	+	panic("no return value specified for Close")
	56	+	}
	57	+
	58	+	var r0 error
	59	+	if returnFunc, ok := ret.Get(0).(func() error); ok {
	60	+	r0 = returnFunc()
	61	+	} else {
	62	+	r0 = ret.Error(0)
	63	+	}
	64	+	return r0
	65	+	}
	66	+
	67	+	// mocktargetSource_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
	68	+	type mocktargetSource_Close_Call struct {
	69	+	*mock.Call
	70	+	}
	71	+
	72	+	// Close is a helper method to define mock.On call
	73	+	func (_e mocktargetSource_Expecter) Close() mocktargetSource_Close_Call {
	74	+	return &mocktargetSource_Close_Call{Call: _e.mock.On("Close")}
	75	+	}
	76	+
	77	+	func (_c mocktargetSource_Close_Call) Run(run func()) mocktargetSource_Close_Call {
	78	+	_c.Call.Run(func(args mock.Arguments) {
	79	+	run()
	80	+	})
	81	+	return _c
	82	+	}
	83	+
	84	+	func (_c mocktargetSource_Close_Call) Return(err error) mocktargetSource_Close_Call {
	85	+	_c.Call.Return(err)
	86	+	return _c
	87	+	}
	88	+
	89	+	func (_c mocktargetSource_Close_Call) RunAndReturn(run func() error) mocktargetSource_Close_Call {
	90	+	_c.Call.Return(run)
	91	+	return _c
	92	+	}
	93	+
	94	+	// Targets provides a mock function for the type mocktargetSource
	95	+	func (_mock *mocktargetSource) Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error) {
	96	+	ret := _mock.Called(ctx, windows, linksOnly)
	97	+
	98	+	if len(ret) == 0 {
	99	+	panic("no return value specified for Targets")
	100	+	}
	101	+
	102	+	var r0 []ports.HintTarget
	103	+	var r1 map[ports.WindowID]bool
	104	+	var r2 error
	105	+	if returnFunc, ok := ret.Get(0).(func(context.Context, []ports.HintWindow, bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)); ok {
	106	+	return returnFunc(ctx, windows, linksOnly)
	107	+	}
	108	+	if returnFunc, ok := ret.Get(0).(func(context.Context, []ports.HintWindow, bool) []ports.HintTarget); ok {
	109	+	r0 = returnFunc(ctx, windows, linksOnly)
	110	+	} else {
	111	+	if ret.Get(0) != nil {
	112	+	r0 = ret.Get(0).([]ports.HintTarget)
	113	+	}
	114	+	}
	115	+	if returnFunc, ok := ret.Get(1).(func(context.Context, []ports.HintWindow, bool) map[ports.WindowID]bool); ok {
	116	+	r1 = returnFunc(ctx, windows, linksOnly)
	117	+	} else {
	118	+	if ret.Get(1) != nil {
	119	+	r1 = ret.Get(1).(map[ports.WindowID]bool)
	120	+	}
	121	+	}
	122	+	if returnFunc, ok := ret.Get(2).(func(context.Context, []ports.HintWindow, bool) error); ok {
	123	+	r2 = returnFunc(ctx, windows, linksOnly)
	124	+	} else {
	125	+	r2 = ret.Error(2)
	126	+	}
	127	+	return r0, r1, r2
	128	+	}
	129	+
	130	+	// mocktargetSource_Targets_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Targets'
	131	+	type mocktargetSource_Targets_Call struct {
	132	+	*mock.Call
	133	+	}
	134	+
	135	+	// Targets is a helper method to define mock.On call
	136	+	// - ctx context.Context
	137	+	// - windows []ports.HintWindow
	138	+	// - linksOnly bool
	139	+	func (_e mocktargetSource_Expecter) Targets(ctx any, windows any, linksOnly any) mocktargetSource_Targets_Call {
	140	+	return &mocktargetSource_Targets_Call{Call: _e.mock.On("Targets", ctx, windows, linksOnly)}
	141	+	}
	142	+
	143	+	func (_c mocktargetSource_Targets_Call) Run(run func(ctx context.Context, windows []ports.HintWindow, linksOnly bool)) mocktargetSource_Targets_Call {
	144	+	_c.Call.Run(func(args mock.Arguments) {
	145	+	var arg0 context.Context
	146	+	if args[0] != nil {
	147	+	arg0 = args[0].(context.Context)
	148	+	}
	149	+	var arg1 []ports.HintWindow
	150	+	if args[1] != nil {
	151	+	arg1 = args[1].([]ports.HintWindow)
	152	+	}
	153	+	var arg2 bool
	154	+	if args[2] != nil {
	155	+	arg2 = args[2].(bool)
	156	+	}
	157	+	run(
	158	+	arg0,
	159	+	arg1,
	160	+	arg2,
	161	+	)
	162	+	})
	163	+	return _c
	164	+	}
	165	+
	166	+	func (_c mocktargetSource_Targets_Call) Return(hintTargets []ports.HintTarget, windowIDToBool map[ports.WindowID]bool, err error) mocktargetSource_Targets_Call {
	167	+	_c.Call.Return(hintTargets, windowIDToBool, err)
	168	+	return _c
	169	+	}
	170	+
	171	+	func (_c mocktargetSource_Targets_Call) RunAndReturn(run func(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)) mocktargetSource_Targets_Call {
	172	+	_c.Call.Return(run)
	173	+	return _c
	174	+	}

A internal/adapters/hints/testdata/browser.png +0 -0 binary file: no text diff

binary file: no text diff

A internal/adapters/hints/testdata/terminal.png +0 -0 binary file: no text diff

binary file: no text diff

A internal/adapters/hints/testdata/underline.png +0 -0 binary file: no text diff

binary file: no text diff

A internal/adapters/hints/underline.go +246 -0

@@ -0,0 +1,246 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"image"
	5	+	"math"
	6	+	"sort"
	7	+
	8	+	"github.com/bnema/neferwl/internal/ports"
	9	+	)
	10	+
	11	+	// underlineCandidate is a word with a thin line under it.
	12	+	type underlineCandidate struct {
	13	+	word int
	14	+	row int // the line's row
	15	+	from, to int // x extent of the continuous line through the word
	16	+	}
	17	+
	18	+	// markUnderlined sets Underlined, UnderL and UnderR on the words of a band
	19	+	// that have a thin line under them. words are in the coordinates of g.
	20	+	// Lines that run on far beyond the word (rules, table borders, input boxes)
	21	+	// do not count, unless the extension is made of underlined words too.
	22	+	func markUnderlined(src *image.Gray, words []OCRWord) {
	23	+	// Words are relative to the band's own origin, whatever the image is a
	24	+	// view of.
	25	+	b := src.Bounds()
	26	+	g := &image.Gray{Pix: src.Pix[src.PixOffset(b.Min.X, b.Min.Y):], Stride: src.Stride, Rect: image.Rect(0, 0, b.Dx(), b.Dy())}
	27	+	var cands []underlineCandidate
	28	+	for i, w := range words {
	29	+	// A single glyph, or a box narrower than tall, has a flat bottom
	30	+	// stroke (8, L, 2) that is not an underline.
	31	+	if len([]rune(w.Text)) < 2 \|\| w.Rect.Dx() < w.Rect.Dy() {
	32	+	continue
	33	+	}
	34	+	if c, ok := underlineOf(g, w.Rect); ok {
	35	+	c.word = i
	36	+	cands = append(cands, c)
	37	+	}
	38	+	}
	39	+	for _, c := range cands {
	40	+	w := words[c.word]
	41	+	h := w.Rect.Dy()
	42	+	if (w.Rect.Min.X-c.from > h && !covered(words, cands, c, -1)) \|\|
	43	+	(c.to-w.Rect.Max.X > h && !covered(words, cands, c, 1)) {
	44	+	continue
	45	+	}
	46	+	words[c.word].Underlined, words[c.word].UnderL, words[c.word].UnderR = true, c.from, c.to
	47	+	}
	48	+	}
	49	+
	50	+	// covered reports whether the line of c, on the given side of its word, is
	51	+	// explained by underlined words on the same line up to within a glyph of its
	52	+	// end.
	53	+	func covered(words []OCRWord, cands []underlineCandidate, c underlineCandidate, dir int) bool {
	54	+	w := words[c.word].Rect
	55	+	h := w.Dy()
	56	+	cur := w.Min.X
	57	+	if dir > 0 {
	58	+	cur = w.Max.X
	59	+	}
	60	+	for {
	61	+	if (dir < 0 && cur-c.from <= h) \|\| (dir > 0 && c.to-cur <= h) {
	62	+	return true
	63	+	}
	64	+	next := -1
	65	+	for _, o := range cands {
	66	+	r := words[o.word].Rect
	67	+	if abs(r.Max.Y-w.Max.Y) > max(3, h/3) {
	68	+	continue
	69	+	}
	70	+	var gap, reach int
	71	+	if dir < 0 {
	72	+	gap, reach = cur-r.Max.X, r.Min.X
	73	+	} else {
	74	+	gap, reach = r.Min.X-cur, r.Max.X
	75	+	}
	76	+	if gap < -2 \|\| gap > h {
	77	+	continue
	78	+	}
	79	+	if next < 0 \|\| (dir < 0 && reach < next) \|\| (dir > 0 && reach > next) {
	80	+	next = reach
	81	+	}
	82	+	}
	83	+	if next < 0 \|\| (dir < 0 && next >= cur) \|\| (dir > 0 && next <= cur) {
	84	+	return false
	85	+	}
	86	+	cur = next
	87	+	}
	88	+	}
	89	+
	90	+	// underlineOf looks for an underline under the word box r in g.
	91	+	func underlineOf(g *image.Gray, r image.Rectangle) (underlineCandidate, bool) {
	92	+	b := g.Bounds()
	93	+	box := r.Intersect(b)
	94	+	h := r.Dy()
	95	+	if box.Dx() < 4 \|\| h < 4 {
	96	+	return underlineCandidate{}, false
	97	+	}
	98	+	bg, ink := contrastOf(g, box)
	99	+	thr := min(40, max(20, ink/2))
	100	+	if ink < thr+10 {
	101	+	return underlineCandidate{}, false
	102	+	}
	103	+	top := max(b.Min.Y, r.Max.Y-(h*15+99)/100)
	104	+	bottom := min(b.Max.Y, r.Max.Y+(h*35+99)/100+1)
	105	+	maxThick := max(2, h/6)
	106	+	rowOK := func(y int) bool {
	107	+	row := g.Pix[(y-g.Rect.Min.Y)*g.Stride:]
	108	+	n := 0
	109	+	for x := box.Min.X; x < box.Max.X; x++ {
	110	+	if abs(int(row[x-g.Rect.Min.X])-bg) >= thr {
	111	+	n++
	112	+	}
	113	+	}
	114	+	return n100 >= box.Dx()85
	115	+	}
	116	+	thick, first := 0, 0
	117	+	for y := top; y <= bottom; y++ {
	118	+	if y < bottom && rowOK(y) {
	119	+	if thick == 0 {
	120	+	first = y
	121	+	}
	122	+	thick++
	123	+	continue
	124	+	}
	125	+	if thick > 0 {
	126	+	if thick > maxThick {
	127	+	return underlineCandidate{}, false // a block, not a line
	128	+	}
	129	+	row := first + thick/2
	130	+	from, to := box.Min.X, box.Max.X
	131	+	line := g.Pix[(row-g.Rect.Min.Y)*g.Stride:]
	132	+	on := func(x int) bool { return abs(int(line[x-g.Rect.Min.X])-bg) >= thr }
	133	+	for from > b.Min.X && on(from-1) {
	134	+	from--
	135	+	}
	136	+	for to < b.Max.X && on(to) {
	137	+	to++
	138	+	}
	139	+	return underlineCandidate{row: row, from: from, to: to}, true
	140	+	}
	141	+	}
	142	+	return underlineCandidate{}, false
	143	+	}
	144	+
	145	+	// contrastOf returns the background level of box (the most common level,
	146	+	// found in bins of 16 so antialiasing does not split it, then refined to the
	147	+	// mean of that bin) and the contrast of its ink (the deviation reached by
	148	+	// the strongest 3% of pixels).
	149	+	func contrastOf(g *image.Gray, box image.Rectangle) (bg, ink int) {
	150	+	var hist [256]int
	151	+	for y := box.Min.Y; y < box.Max.Y; y++ {
	152	+	for _, v := range g.Pix[(y-g.Rect.Min.Y)*g.Stride+box.Min.X-g.Rect.Min.X:][:box.Dx()] {
	153	+	hist[v]++
	154	+	}
	155	+	}
	156	+	total := box.Dx() * box.Dy()
	157	+	var bins [16]int
	158	+	for v, c := range hist {
	159	+	bins[v>>4] += c
	160	+	}
	161	+	best := 0
	162	+	for i, c := range bins {
	163	+	if c > bins[best] {
	164	+	best = i
	165	+	}
	166	+	}
	167	+	sum, n := 0, 0
	168	+	for v := best << 4; v < (best+1)<<4; v++ {
	169	+	sum += v * hist[v]
	170	+	n += hist[v]
	171	+	}
	172	+	bg = sum / max(n, 1)
	173	+	var dev [256]int
	174	+	for v, c := range hist {
	175	+	dev[abs(v-bg)] += c
	176	+	}
	177	+	n = 0
	178	+	for d := 255; d >= 0; d-- {
	179	+	n += dev[d]
	180	+	if n100 >= total3 {
	181	+	return bg, d
	182	+	}
	183	+	}
	184	+	return bg, 0
	185	+	}
	186	+
	187	+	// underlinedLinks returns the targets for underlined words, joining the
	188	+	// words of one continuous underline. A group that overlaps a URL target (in
	189	+	// pixels) is left to it. Coordinates are scaled to logical.
	190	+	func underlinedLinks(words []OCRWord, urls []image.Rectangle, scale float64) []ports.HintTarget {
	191	+	var us []OCRWord
	192	+	for _, w := range words {
	193	+	if w.Underlined {
	194	+	us = append(us, w)
	195	+	}
	196	+	}
	197	+	sort.Slice(us, func(i, j int) bool { return us[i].Rect.Min.X < us[j].Rect.Min.X })
	198	+	used := make([]bool, len(us))
	199	+	var out []ports.HintTarget
	200	+	for i := range us {
	201	+	if used[i] {
	202	+	continue
	203	+	}
	204	+	used[i] = true
	205	+	rect, last := us[i].Rect, us[i]
	206	+	chars, width := len(last.Text), last.Rect.Dx()
	207	+	for {
	208	+	next := -1
	209	+	for j := range us {
	210	+	c := us[j]
	211	+	if used[j] \|\| c.Rect.Min.X < rect.Max.X-2 \|\| abs(c.Rect.Max.Y-last.Rect.Max.Y) > max(3, rect.Dy()/3) {
	212	+	continue
	213	+	}
	214	+	avg := float64(width+c.Rect.Dx()) / float64(max(1, chars+len(c.Text)))
	215	+	gap := c.Rect.Min.X - rect.Max.X
	216	+	if float64(gap) > 1.2*avg \|\| c.Rect.Min.X > last.UnderR+2 {
	217	+	continue
	218	+	}
	219	+	if next < 0 \|\| c.Rect.Min.X < us[next].Rect.Min.X {
	220	+	next = j
	221	+	}
	222	+	}
	223	+	if next < 0 {
	224	+	break
	225	+	}
	226	+	used[next] = true
	227	+	last = us[next]
	228	+	rect = rect.Union(last.Rect)
	229	+	chars += len(last.Text)
	230	+	width += last.Rect.Dx()
	231	+	}
	232	+	dup := false
	233	+	for _, u := range urls {
	234	+	if overlap(rect, u) > .5 {
	235	+	dup = true
	236	+	}
	237	+	}
	238	+	if dup {
	239	+	continue
	240	+	}
	241	+	x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
	242	+	right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
	243	+	out = append(out, ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
	244	+	}
	245	+	return out
	246	+	}

@@ -0,0 +1,246 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"image"
	5	+	"math"
	6	+	"sort"
	7	+
	8	+	"github.com/bnema/neferwl/internal/ports"
	9	+	)
	10	+
	11	+	// underlineCandidate is a word with a thin line under it.
	12	+	type underlineCandidate struct {
	13	+	word int
	14	+	row int // the line's row
	15	+	from, to int // x extent of the continuous line through the word
	16	+	}
	17	+
	18	+	// markUnderlined sets Underlined, UnderL and UnderR on the words of a band
	19	+	// that have a thin line under them. words are in the coordinates of g.
	20	+	// Lines that run on far beyond the word (rules, table borders, input boxes)
	21	+	// do not count, unless the extension is made of underlined words too.
	22	+	func markUnderlined(src *image.Gray, words []OCRWord) {
	23	+	// Words are relative to the band's own origin, whatever the image is a
	24	+	// view of.
	25	+	b := src.Bounds()
	26	+	g := &image.Gray{Pix: src.Pix[src.PixOffset(b.Min.X, b.Min.Y):], Stride: src.Stride, Rect: image.Rect(0, 0, b.Dx(), b.Dy())}
	27	+	var cands []underlineCandidate
	28	+	for i, w := range words {
	29	+	// A single glyph, or a box narrower than tall, has a flat bottom
	30	+	// stroke (8, L, 2) that is not an underline.
	31	+	if len([]rune(w.Text)) < 2 \|\| w.Rect.Dx() < w.Rect.Dy() {
	32	+	continue
	33	+	}
	34	+	if c, ok := underlineOf(g, w.Rect); ok {
	35	+	c.word = i
	36	+	cands = append(cands, c)
	37	+	}
	38	+	}
	39	+	for _, c := range cands {
	40	+	w := words[c.word]
	41	+	h := w.Rect.Dy()
	42	+	if (w.Rect.Min.X-c.from > h && !covered(words, cands, c, -1)) \|\|
	43	+	(c.to-w.Rect.Max.X > h && !covered(words, cands, c, 1)) {
	44	+	continue
	45	+	}
	46	+	words[c.word].Underlined, words[c.word].UnderL, words[c.word].UnderR = true, c.from, c.to
	47	+	}
	48	+	}
	49	+
	50	+	// covered reports whether the line of c, on the given side of its word, is
	51	+	// explained by underlined words on the same line up to within a glyph of its
	52	+	// end.
	53	+	func covered(words []OCRWord, cands []underlineCandidate, c underlineCandidate, dir int) bool {
	54	+	w := words[c.word].Rect
	55	+	h := w.Dy()
	56	+	cur := w.Min.X
	57	+	if dir > 0 {
	58	+	cur = w.Max.X
	59	+	}
	60	+	for {
	61	+	if (dir < 0 && cur-c.from <= h) \|\| (dir > 0 && c.to-cur <= h) {
	62	+	return true
	63	+	}
	64	+	next := -1
	65	+	for _, o := range cands {
	66	+	r := words[o.word].Rect
	67	+	if abs(r.Max.Y-w.Max.Y) > max(3, h/3) {
	68	+	continue
	69	+	}
	70	+	var gap, reach int
	71	+	if dir < 0 {
	72	+	gap, reach = cur-r.Max.X, r.Min.X
	73	+	} else {
	74	+	gap, reach = r.Min.X-cur, r.Max.X
	75	+	}
	76	+	if gap < -2 \|\| gap > h {
	77	+	continue
	78	+	}
	79	+	if next < 0 \|\| (dir < 0 && reach < next) \|\| (dir > 0 && reach > next) {
	80	+	next = reach
	81	+	}
	82	+	}
	83	+	if next < 0 \|\| (dir < 0 && next >= cur) \|\| (dir > 0 && next <= cur) {
	84	+	return false
	85	+	}
	86	+	cur = next
	87	+	}
	88	+	}
	89	+
	90	+	// underlineOf looks for an underline under the word box r in g.
	91	+	func underlineOf(g *image.Gray, r image.Rectangle) (underlineCandidate, bool) {
	92	+	b := g.Bounds()
	93	+	box := r.Intersect(b)
	94	+	h := r.Dy()
	95	+	if box.Dx() < 4 \|\| h < 4 {
	96	+	return underlineCandidate{}, false
	97	+	}
	98	+	bg, ink := contrastOf(g, box)
	99	+	thr := min(40, max(20, ink/2))
	100	+	if ink < thr+10 {
	101	+	return underlineCandidate{}, false
	102	+	}
	103	+	top := max(b.Min.Y, r.Max.Y-(h*15+99)/100)
	104	+	bottom := min(b.Max.Y, r.Max.Y+(h*35+99)/100+1)
	105	+	maxThick := max(2, h/6)
	106	+	rowOK := func(y int) bool {
	107	+	row := g.Pix[(y-g.Rect.Min.Y)*g.Stride:]
	108	+	n := 0
	109	+	for x := box.Min.X; x < box.Max.X; x++ {
	110	+	if abs(int(row[x-g.Rect.Min.X])-bg) >= thr {
	111	+	n++
	112	+	}
	113	+	}
	114	+	return n100 >= box.Dx()85
	115	+	}
	116	+	thick, first := 0, 0
	117	+	for y := top; y <= bottom; y++ {
	118	+	if y < bottom && rowOK(y) {
	119	+	if thick == 0 {
	120	+	first = y
	121	+	}
	122	+	thick++
	123	+	continue
	124	+	}
	125	+	if thick > 0 {
	126	+	if thick > maxThick {
	127	+	return underlineCandidate{}, false // a block, not a line
	128	+	}
	129	+	row := first + thick/2
	130	+	from, to := box.Min.X, box.Max.X
	131	+	line := g.Pix[(row-g.Rect.Min.Y)*g.Stride:]
	132	+	on := func(x int) bool { return abs(int(line[x-g.Rect.Min.X])-bg) >= thr }
	133	+	for from > b.Min.X && on(from-1) {
	134	+	from--
	135	+	}
	136	+	for to < b.Max.X && on(to) {
	137	+	to++
	138	+	}
	139	+	return underlineCandidate{row: row, from: from, to: to}, true
	140	+	}
	141	+	}
	142	+	return underlineCandidate{}, false
	143	+	}
	144	+
	145	+	// contrastOf returns the background level of box (the most common level,
	146	+	// found in bins of 16 so antialiasing does not split it, then refined to the
	147	+	// mean of that bin) and the contrast of its ink (the deviation reached by
	148	+	// the strongest 3% of pixels).
	149	+	func contrastOf(g *image.Gray, box image.Rectangle) (bg, ink int) {
	150	+	var hist [256]int
	151	+	for y := box.Min.Y; y < box.Max.Y; y++ {
	152	+	for _, v := range g.Pix[(y-g.Rect.Min.Y)*g.Stride+box.Min.X-g.Rect.Min.X:][:box.Dx()] {
	153	+	hist[v]++
	154	+	}
	155	+	}
	156	+	total := box.Dx() * box.Dy()
	157	+	var bins [16]int
	158	+	for v, c := range hist {
	159	+	bins[v>>4] += c
	160	+	}
	161	+	best := 0
	162	+	for i, c := range bins {
	163	+	if c > bins[best] {
	164	+	best = i
	165	+	}
	166	+	}
	167	+	sum, n := 0, 0
	168	+	for v := best << 4; v < (best+1)<<4; v++ {
	169	+	sum += v * hist[v]
	170	+	n += hist[v]
	171	+	}
	172	+	bg = sum / max(n, 1)
	173	+	var dev [256]int
	174	+	for v, c := range hist {
	175	+	dev[abs(v-bg)] += c
	176	+	}
	177	+	n = 0
	178	+	for d := 255; d >= 0; d-- {
	179	+	n += dev[d]
	180	+	if n100 >= total3 {
	181	+	return bg, d
	182	+	}
	183	+	}
	184	+	return bg, 0
	185	+	}
	186	+
	187	+	// underlinedLinks returns the targets for underlined words, joining the
	188	+	// words of one continuous underline. A group that overlaps a URL target (in
	189	+	// pixels) is left to it. Coordinates are scaled to logical.
	190	+	func underlinedLinks(words []OCRWord, urls []image.Rectangle, scale float64) []ports.HintTarget {
	191	+	var us []OCRWord
	192	+	for _, w := range words {
	193	+	if w.Underlined {
	194	+	us = append(us, w)
	195	+	}
	196	+	}
	197	+	sort.Slice(us, func(i, j int) bool { return us[i].Rect.Min.X < us[j].Rect.Min.X })
	198	+	used := make([]bool, len(us))
	199	+	var out []ports.HintTarget
	200	+	for i := range us {
	201	+	if used[i] {
	202	+	continue
	203	+	}
	204	+	used[i] = true
	205	+	rect, last := us[i].Rect, us[i]
	206	+	chars, width := len(last.Text), last.Rect.Dx()
	207	+	for {
	208	+	next := -1
	209	+	for j := range us {
	210	+	c := us[j]
	211	+	if used[j] \|\| c.Rect.Min.X < rect.Max.X-2 \|\| abs(c.Rect.Max.Y-last.Rect.Max.Y) > max(3, rect.Dy()/3) {
	212	+	continue
	213	+	}
	214	+	avg := float64(width+c.Rect.Dx()) / float64(max(1, chars+len(c.Text)))
	215	+	gap := c.Rect.Min.X - rect.Max.X
	216	+	if float64(gap) > 1.2*avg \|\| c.Rect.Min.X > last.UnderR+2 {
	217	+	continue
	218	+	}
	219	+	if next < 0 \|\| c.Rect.Min.X < us[next].Rect.Min.X {
	220	+	next = j
	221	+	}
	222	+	}
	223	+	if next < 0 {
	224	+	break
	225	+	}
	226	+	used[next] = true
	227	+	last = us[next]
	228	+	rect = rect.Union(last.Rect)
	229	+	chars += len(last.Text)
	230	+	width += last.Rect.Dx()
	231	+	}
	232	+	dup := false
	233	+	for _, u := range urls {
	234	+	if overlap(rect, u) > .5 {
	235	+	dup = true
	236	+	}
	237	+	}
	238	+	if dup {
	239	+	continue
	240	+	}
	241	+	x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
	242	+	right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
	243	+	out = append(out, ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
	244	+	}
	245	+	return out
	246	+	}

A internal/adapters/hints/underline_test.go +252 -0

@@ -0,0 +1,252 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"image"
	5	+	"image/color"
	6	+	"os"
	7	+	"os/exec"
	8	+	"testing"
	9	+
	10	+	"github.com/stretchr/testify/require"
	11	+
	12	+	"github.com/bnema/neferwl/internal/ports"
	13	+	)
	14	+
	15	+	// canvas is a synthetic grayscale band: bg everywhere, ink for text.
	16	+	type canvas struct {
	17	+	g *image.Gray
	18	+	bg, ink uint8
	19	+	s int // scale
	20	+	}
	21	+
	22	+	func newCanvas(w, h int, bg, ink uint8, s int) *canvas {
	23	+	g := image.NewGray(image.Rect(0, 0, ws, hs))
	24	+	for i := range g.Pix {
	25	+	g.Pix[i] = bg
	26	+	}
	27	+	return &canvas{g, bg, ink, s}
	28	+	}
	29	+
	30	+	func (c *canvas) fill(x0, y0, x1, y1 int, v uint8) {
	31	+	for y := y0 * c.s; y < y1*c.s; y++ {
	32	+	for x := x0 * c.s; x < x1*c.s; x++ {
	33	+	c.g.SetGray(x, y, color.Gray{v})
	34	+	}
	35	+	}
	36	+	}
	37	+
	38	+	// word draws a 10 px high word of x-height body at (x,y) with the given
	39	+	// width: text-like noisy ink (a stem every 3 px) and optional descenders.
	40	+	// It returns the OCR box (descenders excluded, as a box around the body).
	41	+	func (c *canvas) word(x, y, w int, descender bool) OCRWord {
	42	+	for i := 0; i < w; i += 3 {
	43	+	c.fill(x+i, y, x+i+1, y+10, c.ink)
	44	+	}
	45	+	box := image.Rect(xc.s, yc.s, (x+w)c.s, (y+10)c.s)
	46	+	if descender {
	47	+	for i := 0; i < w; i += 6 {
	48	+	c.fill(x+i, y+10, x+i+1, y+13, c.ink)
	49	+	}
	50	+	box.Max.Y = (y + 13) * c.s
	51	+	}
	52	+	return OCRWord{Text: "word", Rect: box}
	53	+	}
	54	+
	55	+	func (c *canvas) underline(x0, x1, y int) { c.fill(x0, y, x1, y+1, c.ink) }
	56	+
	57	+	func scanUnder(c *canvas, words ...OCRWord) []ports.HintTarget {
	58	+	markUnderlined(c.g, words)
	59	+	return textLinks(words, float64(c.s))
	60	+	}
	61	+
	62	+	func TestUnderline(t *testing.T) {
	63	+	for _, tc := range []struct {
	64	+	name string
	65	+	bg, ink uint8
	66	+	}{{"dark-bg", 10, 204}, {"light-bg", 245, 30}} {
	67	+	for _, s := range []int{1, 2} {
	68	+	t.Run(tc.name, func(t *testing.T) {
	69	+	mk := func() *canvas { return newCanvas(200, 60, tc.bg, tc.ink, s) }
	70	+
	71	+	c := mk()
	72	+	w := c.word(10, 20, 40, false)
	73	+	c.underline(10, 50, 32)
	74	+	got := scanUnder(c, w)
	75	+	require.Len(t, got, 1)
	76	+	require.Equal(t, ports.HintLink, got[0].Kind)
	77	+	require.False(t, got[0].Text)
	78	+	require.Equal(t, ports.WindowID(0), got[0].Window)
	79	+	require.Equal(t, 10, got[0].Rect.X)
	80	+
	81	+	c = mk()
	82	+	require.Empty(t, scanUnder(c, c.word(10, 20, 40, false)), "plain word")
	83	+
	84	+	c = mk()
	85	+	require.Empty(t, scanUnder(c, c.word(10, 20, 40, true)), "descenders")
	86	+
	87	+	c = mk()
	88	+	w = c.word(10, 20, 40, false)
	89	+	c.underline(0, 200, 32)
	90	+	require.Empty(t, scanUnder(c, w), "rule across")
	91	+
	92	+	c = mk()
	93	+	w = c.word(10, 20, 40, false)
	94	+	c.fill(10, 24, 50, 25, c.ink)
	95	+	require.Empty(t, scanUnder(c, w), "strikethrough")
	96	+
	97	+	// Two words with a continuous underline: one target.
	98	+	c = mk()
	99	+	a, b := c.word(10, 20, 40, false), c.word(56, 20, 40, false)
	100	+	c.underline(10, 96, 32)
	101	+	got = scanUnder(c, a, b)
	102	+	require.Len(t, got, 1)
	103	+	require.Equal(t, 10, got[0].Rect.X)
	104	+	require.Equal(t, 86, got[0].Rect.W)
	105	+
	106	+	// A gap in the underline: two targets.
	107	+	c = mk()
	108	+	a, b = c.word(10, 20, 40, false), c.word(56, 20, 40, false)
	109	+	c.underline(10, 50, 32)
	110	+	c.underline(56, 96, 32)
	111	+	require.Len(t, scanUnder(c, a, b), 2)
	112	+	})
	113	+	}
	114	+	}
	115	+	}
	116	+
	117	+	// A box border under a word, with the word a part of a table row, is not a
	118	+	// link; neither is an underline that continues beyond into unrelated text.
	119	+	func TestUnderlineRejectsBox(t *testing.T) {
	120	+	c := newCanvas(300, 60, 10, 204, 1)
	121	+	w := c.word(100, 20, 40, false)
	122	+	c.underline(20, 280, 32)
	123	+	c.fill(20, 10, 21, 33, 204)
	124	+	c.fill(279, 10, 280, 33, 204)
	125	+	require.Empty(t, scanUnder(c, w))
	126	+	}
	127	+
	128	+	// The line must be thin.
	129	+	func TestUnderlineRejectsBlock(t *testing.T) {
	130	+	c := newCanvas(100, 60, 10, 204, 1)
	131	+	w := c.word(10, 20, 40, false)
	132	+	c.fill(10, 31, 50, 36, 204)
	133	+	require.Empty(t, scanUnder(c, w))
	134	+	}
	135	+
	136	+	func TestUnderlinedURLIsOneTextTarget(t *testing.T) {
	137	+	c := newCanvas(300, 60, 10, 204, 1)
	138	+	w := c.word(10, 20, 100, false)
	139	+	w.Text = "example.org"
	140	+	c.underline(10, 110, 32)
	141	+	got := scanUnder(c, w)
	142	+	require.Len(t, got, 1)
	143	+	require.True(t, got[0].Text)
	144	+	}
	145	+
	146	+	// ocrTestdata reads the words of a testdata image with the real tesseract.
	147	+	func ocrTestdata(t testing.T, name string) (image.Gray, []OCRWord) {
	148	+	t.Helper()
	149	+	if _, err := exec.LookPath("tesseract"); err != nil {
	150	+	t.Skip("tesseract not on PATH")
	151	+	}
	152	+	f, err := os.Open("testdata/" + name)
	153	+	require.NoError(t, err)
	154	+	defer f.Close()
	155	+	img, _, err := image.Decode(f)
	156	+	require.NoError(t, err)
	157	+	g := image.NewGray(img.Bounds())
	158	+	for y := 0; y < g.Rect.Dy(); y++ {
	159	+	for x := 0; x < g.Rect.Dx(); x++ {
	160	+	g.Set(x, y, img.At(x, y))
	161	+	}
	162	+	}
	163	+	words, _, err := readWords(t.Context(), g)
	164	+	require.NoError(t, err)
	165	+	return g, words
	166	+	}
	167	+
	168	+	// headingRule finds the thin horizontal rule under the heading of
	169	+	// browser.png: the first row, from y=150, with a 900 px run of non-white
	170	+	// pixels. It returns the rule as a rect.
	171	+	func headingRule(t testing.T, g image.Gray) image.Rectangle {
	172	+	t.Helper()
	173	+	for y := 150; y < 220; y++ {
	174	+	run, start := 0, 0
	175	+	for x := 0; x < g.Rect.Dx(); x++ {
	176	+	if g.GrayAt(x, y).Y < 240 {
	177	+	if run == 0 {
	178	+	start = x
	179	+	}
	180	+	run++
	181	+	if run >= 900 {
	182	+	return image.Rect(start, y, start+run, y+1)
	183	+	}
	184	+	} else {
	185	+	run = 0
	186	+	}
	187	+	}
	188	+	}
	189	+	t.Fatal("heading rule not found")
	190	+	return image.Rectangle{}
	191	+	}
	192	+
	193	+	// browser.png has blue links and no underlined link: the only underline-only
	194	+	// target is a known false positive, and the heading rule is not taken for an
	195	+	// underline.
	196	+	func TestUnderlineOnBrowserTestdata(t *testing.T) {
	197	+	g, words := ocrTestdata(t, "browser.png")
	198	+	rule := headingRule(t, g)
	199	+	// Known false positive: the word "as" (flat glyph bottoms, ~(878,353)) is
	200	+	// flagged underlined by the detector. Anything else is a regression.
	201	+	var under []ports.HintTarget
	202	+	for _, tg := range textLinks(words, 1) {
	203	+	if !tg.Text {
	204	+	under = append(under, tg)
	205	+	}
	206	+	}
	207	+	require.Len(t, under, 1, "underline-only targets: %v", under)
	208	+	require.True(t, under[0].Rect.X > 860 && under[0].Rect.X < 900 && under[0].Rect.Y > 340 && under[0].Rect.Y < 370, "unexpected target %v", under[0].Rect)
	209	+	for _, tg := range textLinks(words, 1) {
	210	+	r := image.Rect(tg.Rect.X, tg.Rect.Y, tg.Rect.X+tg.Rect.W, tg.Rect.Y+tg.Rect.H)
	211	+	require.False(t, r.Overlaps(rule), "target %v overlaps the heading rule %v", tg.Rect, rule)
	212	+	}
	213	+	}
	214	+
	215	+	// underline.png is a real render (Chromium) of a generic page: underlined
	216	+	// links `demo`, `source` and `ermite-runner hosts`, a table with row rules
	217	+	// and an input box, which must yield nothing.
	218	+	func TestUnderlineOnRealRender(t *testing.T) {
	219	+	_, words := ocrTestdata(t, "underline.png")
	220	+	near := func(r ports.Rect, x0, y0, x1, y1 int) bool {
	221	+	const tol = 8
	222	+	abs := func(v int) int { return max(v, -v) }
	223	+	return abs(r.X-x0) <= tol && abs(r.Y-y0) <= tol && abs(r.X+r.W-x1) <= tol && abs(r.Y+r.H-y1) <= tol
	224	+	}
	225	+	var got []ports.HintTarget
	226	+	for _, tg := range textLinks(words, 1) {
	227	+	if !tg.Text {
	228	+	got = append(got, tg)
	229	+	}
	230	+	}
	231	+	require.Len(t, got, 3, "targets: %v", got)
	232	+	for _, want := range [][4]int{{148, 122, 196, 137}, {306, 126, 364, 137}, {122, 165, 294, 180}} {
	233	+	found := false
	234	+	for _, tg := range got {
	235	+	found = found \|\| near(tg.Rect, want[0], want[1], want[2], want[3])
	236	+	}
	237	+	require.True(t, found, "no target near %v in %v", want, got)
	238	+	}
	239	+	// The table (y 215..335) and the input box (y 360..410) give nothing.
	240	+	for _, tg := range textLinks(words, 1) {
	241	+	require.False(t, tg.Rect.Y > 200 && tg.Rect.Y < 420, "unexpected target %v", tg.Rect)
	242	+	}
	243	+	}
	244	+
	245	+	// A single glyph with a flat bottom stroke (8, L) is not underlined text.
	246	+	func TestUnderlineIgnoresSingleGlyph(t *testing.T) {
	247	+	c := newCanvas(100, 60, 10, 204, 1)
	248	+	w := c.word(10, 20, 10, false)
	249	+	w.Text = "8"
	250	+	c.underline(10, 20, 32)
	251	+	require.Empty(t, scanUnder(c, w))
	252	+	}

@@ -0,0 +1,252 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"image"
	5	+	"image/color"
	6	+	"os"
	7	+	"os/exec"
	8	+	"testing"
	9	+
	10	+	"github.com/stretchr/testify/require"
	11	+
	12	+	"github.com/bnema/neferwl/internal/ports"
	13	+	)
	14	+
	15	+	// canvas is a synthetic grayscale band: bg everywhere, ink for text.
	16	+	type canvas struct {
	17	+	g *image.Gray
	18	+	bg, ink uint8
	19	+	s int // scale
	20	+	}
	21	+
	22	+	func newCanvas(w, h int, bg, ink uint8, s int) *canvas {
	23	+	g := image.NewGray(image.Rect(0, 0, ws, hs))
	24	+	for i := range g.Pix {
	25	+	g.Pix[i] = bg
	26	+	}
	27	+	return &canvas{g, bg, ink, s}
	28	+	}
	29	+
	30	+	func (c *canvas) fill(x0, y0, x1, y1 int, v uint8) {
	31	+	for y := y0 * c.s; y < y1*c.s; y++ {
	32	+	for x := x0 * c.s; x < x1*c.s; x++ {
	33	+	c.g.SetGray(x, y, color.Gray{v})
	34	+	}
	35	+	}
	36	+	}
	37	+
	38	+	// word draws a 10 px high word of x-height body at (x,y) with the given
	39	+	// width: text-like noisy ink (a stem every 3 px) and optional descenders.
	40	+	// It returns the OCR box (descenders excluded, as a box around the body).
	41	+	func (c *canvas) word(x, y, w int, descender bool) OCRWord {
	42	+	for i := 0; i < w; i += 3 {
	43	+	c.fill(x+i, y, x+i+1, y+10, c.ink)
	44	+	}
	45	+	box := image.Rect(xc.s, yc.s, (x+w)c.s, (y+10)c.s)
	46	+	if descender {
	47	+	for i := 0; i < w; i += 6 {
	48	+	c.fill(x+i, y+10, x+i+1, y+13, c.ink)
	49	+	}
	50	+	box.Max.Y = (y + 13) * c.s
	51	+	}
	52	+	return OCRWord{Text: "word", Rect: box}
	53	+	}
	54	+
	55	+	func (c *canvas) underline(x0, x1, y int) { c.fill(x0, y, x1, y+1, c.ink) }
	56	+
	57	+	func scanUnder(c *canvas, words ...OCRWord) []ports.HintTarget {
	58	+	markUnderlined(c.g, words)
	59	+	return textLinks(words, float64(c.s))
	60	+	}
	61	+
	62	+	func TestUnderline(t *testing.T) {
	63	+	for _, tc := range []struct {
	64	+	name string
	65	+	bg, ink uint8
	66	+	}{{"dark-bg", 10, 204}, {"light-bg", 245, 30}} {
	67	+	for _, s := range []int{1, 2} {
	68	+	t.Run(tc.name, func(t *testing.T) {
	69	+	mk := func() *canvas { return newCanvas(200, 60, tc.bg, tc.ink, s) }
	70	+
	71	+	c := mk()
	72	+	w := c.word(10, 20, 40, false)
	73	+	c.underline(10, 50, 32)
	74	+	got := scanUnder(c, w)
	75	+	require.Len(t, got, 1)
	76	+	require.Equal(t, ports.HintLink, got[0].Kind)
	77	+	require.False(t, got[0].Text)
	78	+	require.Equal(t, ports.WindowID(0), got[0].Window)
	79	+	require.Equal(t, 10, got[0].Rect.X)
	80	+
	81	+	c = mk()
	82	+	require.Empty(t, scanUnder(c, c.word(10, 20, 40, false)), "plain word")
	83	+
	84	+	c = mk()
	85	+	require.Empty(t, scanUnder(c, c.word(10, 20, 40, true)), "descenders")
	86	+
	87	+	c = mk()
	88	+	w = c.word(10, 20, 40, false)
	89	+	c.underline(0, 200, 32)
	90	+	require.Empty(t, scanUnder(c, w), "rule across")
	91	+
	92	+	c = mk()
	93	+	w = c.word(10, 20, 40, false)
	94	+	c.fill(10, 24, 50, 25, c.ink)
	95	+	require.Empty(t, scanUnder(c, w), "strikethrough")
	96	+
	97	+	// Two words with a continuous underline: one target.
	98	+	c = mk()
	99	+	a, b := c.word(10, 20, 40, false), c.word(56, 20, 40, false)
	100	+	c.underline(10, 96, 32)
	101	+	got = scanUnder(c, a, b)
	102	+	require.Len(t, got, 1)
	103	+	require.Equal(t, 10, got[0].Rect.X)
	104	+	require.Equal(t, 86, got[0].Rect.W)
	105	+
	106	+	// A gap in the underline: two targets.
	107	+	c = mk()
	108	+	a, b = c.word(10, 20, 40, false), c.word(56, 20, 40, false)
	109	+	c.underline(10, 50, 32)
	110	+	c.underline(56, 96, 32)
	111	+	require.Len(t, scanUnder(c, a, b), 2)
	112	+	})
	113	+	}
	114	+	}
	115	+	}
	116	+
	117	+	// A box border under a word, with the word a part of a table row, is not a
	118	+	// link; neither is an underline that continues beyond into unrelated text.
	119	+	func TestUnderlineRejectsBox(t *testing.T) {
	120	+	c := newCanvas(300, 60, 10, 204, 1)
	121	+	w := c.word(100, 20, 40, false)
	122	+	c.underline(20, 280, 32)
	123	+	c.fill(20, 10, 21, 33, 204)
	124	+	c.fill(279, 10, 280, 33, 204)
	125	+	require.Empty(t, scanUnder(c, w))
	126	+	}
	127	+
	128	+	// The line must be thin.
	129	+	func TestUnderlineRejectsBlock(t *testing.T) {
	130	+	c := newCanvas(100, 60, 10, 204, 1)
	131	+	w := c.word(10, 20, 40, false)
	132	+	c.fill(10, 31, 50, 36, 204)
	133	+	require.Empty(t, scanUnder(c, w))
	134	+	}
	135	+
	136	+	func TestUnderlinedURLIsOneTextTarget(t *testing.T) {
	137	+	c := newCanvas(300, 60, 10, 204, 1)
	138	+	w := c.word(10, 20, 100, false)
	139	+	w.Text = "example.org"
	140	+	c.underline(10, 110, 32)
	141	+	got := scanUnder(c, w)
	142	+	require.Len(t, got, 1)
	143	+	require.True(t, got[0].Text)
	144	+	}
	145	+
	146	+	// ocrTestdata reads the words of a testdata image with the real tesseract.
	147	+	func ocrTestdata(t testing.T, name string) (image.Gray, []OCRWord) {
	148	+	t.Helper()
	149	+	if _, err := exec.LookPath("tesseract"); err != nil {
	150	+	t.Skip("tesseract not on PATH")
	151	+	}
	152	+	f, err := os.Open("testdata/" + name)
	153	+	require.NoError(t, err)
	154	+	defer f.Close()
	155	+	img, _, err := image.Decode(f)
	156	+	require.NoError(t, err)
	157	+	g := image.NewGray(img.Bounds())
	158	+	for y := 0; y < g.Rect.Dy(); y++ {
	159	+	for x := 0; x < g.Rect.Dx(); x++ {
	160	+	g.Set(x, y, img.At(x, y))
	161	+	}
	162	+	}
	163	+	words, _, err := readWords(t.Context(), g)
	164	+	require.NoError(t, err)
	165	+	return g, words
	166	+	}
	167	+
	168	+	// headingRule finds the thin horizontal rule under the heading of
	169	+	// browser.png: the first row, from y=150, with a 900 px run of non-white
	170	+	// pixels. It returns the rule as a rect.
	171	+	func headingRule(t testing.T, g image.Gray) image.Rectangle {
	172	+	t.Helper()
	173	+	for y := 150; y < 220; y++ {
	174	+	run, start := 0, 0
	175	+	for x := 0; x < g.Rect.Dx(); x++ {
	176	+	if g.GrayAt(x, y).Y < 240 {
	177	+	if run == 0 {
	178	+	start = x
	179	+	}
	180	+	run++
	181	+	if run >= 900 {
	182	+	return image.Rect(start, y, start+run, y+1)
	183	+	}
	184	+	} else {
	185	+	run = 0
	186	+	}
	187	+	}
	188	+	}
	189	+	t.Fatal("heading rule not found")
	190	+	return image.Rectangle{}
	191	+	}
	192	+
	193	+	// browser.png has blue links and no underlined link: the only underline-only
	194	+	// target is a known false positive, and the heading rule is not taken for an
	195	+	// underline.
	196	+	func TestUnderlineOnBrowserTestdata(t *testing.T) {
	197	+	g, words := ocrTestdata(t, "browser.png")
	198	+	rule := headingRule(t, g)
	199	+	// Known false positive: the word "as" (flat glyph bottoms, ~(878,353)) is
	200	+	// flagged underlined by the detector. Anything else is a regression.
	201	+	var under []ports.HintTarget
	202	+	for _, tg := range textLinks(words, 1) {
	203	+	if !tg.Text {
	204	+	under = append(under, tg)
	205	+	}
	206	+	}
	207	+	require.Len(t, under, 1, "underline-only targets: %v", under)
	208	+	require.True(t, under[0].Rect.X > 860 && under[0].Rect.X < 900 && under[0].Rect.Y > 340 && under[0].Rect.Y < 370, "unexpected target %v", under[0].Rect)
	209	+	for _, tg := range textLinks(words, 1) {
	210	+	r := image.Rect(tg.Rect.X, tg.Rect.Y, tg.Rect.X+tg.Rect.W, tg.Rect.Y+tg.Rect.H)
	211	+	require.False(t, r.Overlaps(rule), "target %v overlaps the heading rule %v", tg.Rect, rule)
	212	+	}
	213	+	}
	214	+
	215	+	// underline.png is a real render (Chromium) of a generic page: underlined
	216	+	// links `demo`, `source` and `ermite-runner hosts`, a table with row rules
	217	+	// and an input box, which must yield nothing.
	218	+	func TestUnderlineOnRealRender(t *testing.T) {
	219	+	_, words := ocrTestdata(t, "underline.png")
	220	+	near := func(r ports.Rect, x0, y0, x1, y1 int) bool {
	221	+	const tol = 8
	222	+	abs := func(v int) int { return max(v, -v) }
	223	+	return abs(r.X-x0) <= tol && abs(r.Y-y0) <= tol && abs(r.X+r.W-x1) <= tol && abs(r.Y+r.H-y1) <= tol
	224	+	}
	225	+	var got []ports.HintTarget
	226	+	for _, tg := range textLinks(words, 1) {
	227	+	if !tg.Text {
	228	+	got = append(got, tg)
	229	+	}
	230	+	}
	231	+	require.Len(t, got, 3, "targets: %v", got)
	232	+	for _, want := range [][4]int{{148, 122, 196, 137}, {306, 126, 364, 137}, {122, 165, 294, 180}} {
	233	+	found := false
	234	+	for _, tg := range got {
	235	+	found = found \|\| near(tg.Rect, want[0], want[1], want[2], want[3])
	236	+	}
	237	+	require.True(t, found, "no target near %v in %v", want, got)
	238	+	}
	239	+	// The table (y 215..335) and the input box (y 360..410) give nothing.
	240	+	for _, tg := range textLinks(words, 1) {
	241	+	require.False(t, tg.Rect.Y > 200 && tg.Rect.Y < 420, "unexpected target %v", tg.Rect)
	242	+	}
	243	+	}
	244	+
	245	+	// A single glyph with a flat bottom stroke (8, L) is not underlined text.
	246	+	func TestUnderlineIgnoresSingleGlyph(t *testing.T) {
	247	+	c := newCanvas(100, 60, 10, 204, 1)
	248	+	w := c.word(10, 20, 10, false)
	249	+	w.Text = "8"
	250	+	c.underline(10, 20, 32)
	251	+	require.Empty(t, scanUnder(c, w))
	252	+	}