Merge branch 'feat/hint-alloc' into feat/hint-hybrid

commit: 61fd74991947869011fd51f5c81c4b448fc016d7
author: brice · 3d ago · 2026-10-19 15:20 UTC
merge of: c17152b + b0cc57c · diff against the first parent
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

9 files · +662 -154

M internal/adapters/hints/benchmark_test.go +3 -0


@@ -30,6 +30,9 @@ func TestFrameBenchmark(t *testing.T) {
30	30		if v, _ := strconv.Atoi(os.Getenv("NEFERWL_HINT_BENCH_WORKERS")); v > 0 {
31	31		ocrWorkers = v
32	32		}
	33	+	if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
	34	+	rawPGM = false
	35	+	}
33	36		if os.Getenv("NEFERWL_HINT_BENCH_PNG") == "none" {
34	37		pngLevel = png.NoCompression
35	38		}


@@ -30,6 +30,9 @@ func TestFrameBenchmark(t *testing.T) {
30	if v, _ := strconv.Atoi(os.Getenv("NEFERWL_HINT_BENCH_WORKERS")); v > 0 {	30		if v, _ := strconv.Atoi(os.Getenv("NEFERWL_HINT_BENCH_WORKERS")); v > 0 {
31	ocrWorkers = v	31		ocrWorkers = v
32	}	32		}
		33	+	if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
		34	+	rawPGM = false
		35	+	}
33	if os.Getenv("NEFERWL_HINT_BENCH_PNG") == "none" {	36		if os.Getenv("NEFERWL_HINT_BENCH_PNG") == "none" {
34	pngLevel = png.NoCompression	37		pngLevel = png.NoCompression
35	}	38		}

M internal/adapters/hints/dump.go +1 -0


@@ -38,6 +38,7 @@ type dumpJob struct {
38	38		when time.Time
39	39		req ports.HintScanRequest
40	40		pix []byte
	41	+	m *mapping // owns pix; released after the dump
41	42		stride int
42	43		targets []ports.HintTarget
43	44		crops []image.Rectangle


@@ -38,6 +38,7 @@ type dumpJob struct {
38	when time.Time	38		when time.Time
39	req ports.HintScanRequest	39		req ports.HintScanRequest
40	pix []byte	40		pix []byte
		41	+	m *mapping // owns pix; released after the dump
41	stride int	42		stride int
42	targets []ports.HintTarget	43		targets []ports.HintTarget
43	crops []image.Rectangle	44		crops []image.Rectangle

M internal/adapters/hints/hints.go +216 -71


@@ -9,7 +9,7 @@ import (
9	9		"hash/maphash"
10	10		"image"
11	11		"math"
12		-	"runtime/debug"
	12	+	"strings"
13	13		"time"
14	14
15	15		"github.com/bnema/neferwl/internal/adapters/logging"

@@ -55,12 +55,24 @@ type atspiResult struct {
55	55
56	56		type worker struct {
57	57		source targetSource
58		-	cache map[cropKey]cropCache
	58	+	cache map[bandKey]bandCache
59	59		cacheSeed maphash.Seed
	60	+	ocr ocrState
60	61		warnedSource bool
61	62		n uint64
62	63		dumpDir string
63	64		pending *dumpJob
	65	+	// orphans are capture buffers the output may still write: they are
	66	+	// released when its CaptureDone finally arrives.
	67	+	orphans map[uint64]*mapping
	68	+	live int // mappings not released yet
	69	+	}
	70	+
	71	+	// frame is the captured output: BGRA pixels, nil when no capture was needed.
	72	+	type frame struct {
	73	+	pix []byte
	74	+	stride int
	75	+	m *mapping
64	76		}
65	77
66	78		func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {

@@ -75,19 +87,19 @@ func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures ch
75	87		return nil
76	88		}
77	89		w.dump(ctx)
78		-	// Nothing of the scan is kept: give the frame back to the system.
79		-	debug.FreeOSMemory()
80	90		}
81	91		}
82	92		}
83	93
84		-	// dump writes the last successful scan, after its result was sent.
	94	+	// dump writes the last successful scan, after its result was sent, then
	95	+	// releases its frame.
85	96		func (w *worker) dump(ctx context.Context) {
86	97		j := w.pending
87	98		w.pending = nil
88	99		if j == nil {
89	100		return
90	101		}
	102	+	defer j.m.unmap()
91	103		if err := writeDump(w.dumpDir, j); err != nil {
92	104		log := logging.For(ctx, "hints")
93	105		log.Warn().Err(err).Str("dir", w.dumpDir).Msg("hint debug dump failed")

@@ -103,6 +115,25 @@ func send(ctx context.Context, results chan<- ports.HintScanResult, res ports.Hi
103	115		}
104	116		}
105	117
	118	+	// needFrame reports whether the scan must capture the output: OCR has a
	119	+	// window to read, or the debug dump wants the frame. With OCR unavailable
	120	+	// (tesseract missing) and no dump, the accessibility targets are the result.
	121	+	func (w *worker) needFrame(ctx context.Context, req ports.HintScanRequest, ar atspiResult) bool {
	122	+	ocr := w.ocrReady(ctx)
	123	+	if w.dumpDir != "" {
	124	+	return true
	125	+	}
	126	+	if !ocr {
	127	+	return false
	128	+	}
	129	+	for _, win := range req.Windows {
	130	+	if !ar.covered[win.ID] && !windowCrop(win.Rect, req.Scale, req.Width, req.Height).Empty() {
	131	+	return true
	132	+	}
	133	+	}
	134	+	return false
	135	+	}
	136	+
106	137		func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration) ports.HintScanResult {
107	138		log := logging.For(ctx, "hints")
108	139		res := ports.HintScanResult{Seq: req.Seq, Output: req.Output}

@@ -115,13 +146,24 @@ func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests
115	146		if req.Width <= 0 \|\| req.Height <= 0 \|\| req.Width*req.Height > maxPixels {
116	147		return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))
117	148		}
118		-	// The accessibility query runs while the frame is captured; OCR waits for it.
119		-	// It is always waited for: the source is not shared between requests.
	149	+	// The accessibility query is always waited for: the source is not shared
	150	+	// between requests. The capture only starts once it is known whether any
	151	+	// window needs OCR.
120	152		atspi := w.startSource(ctx, req)
121	153		defer func() { <-atspi.done }()
122		-	// The capture buffer lives for this scan only; a dump job may keep it
123		-	// until the dump is written.
124		-	buf := make([]byte, req.Widthreq.Height4)
	154	+	if !w.needFrame(ctx, req, atspi.wait()) {
	155	+	return w.scan(ctx, req, frame{}, 0, res, atspi)
	156	+	}
	157	+	// The capture buffer is anonymous mapped memory, outside the Go heap. It
	158	+	// is released once the crops are extracted, or by the dump job.
	159	+	m := w.alloc(req.Width * req.Height * 4)
	160	+	keep := false // the mapping outlives this call: dump job or pending capture
	161	+	defer func() {
	162	+	if !keep {
	163	+	m.unmap()
	164	+	}
	165	+	}()
	166	+	buf := m.b
125	167
126	168		w.n++
127	169		id := ports.CaptureIDInternal \| w.n

@@ -149,33 +191,69 @@ func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests
149	191		case <-ctx.Done():
150	192		return fail(ctx.Err())
151	193		}
	194	+	// From here the output owns buf until CaptureDone.
	195	+	abandon := func() {
	196	+	if w.orphans == nil {
	197	+	w.orphans = map[uint64]*mapping{}
	198	+	}
	199	+	w.orphans[id] = m
	200	+	keep = true
	201	+	}
152	202
153	203		for {
154	204		select {
155	205		case done := <-captured:
156	206		if done.ID != id {
	207	+	w.reap(done.ID)
157	208		continue
158	209		}
159	210		if done.Err != nil {
160	211		return fail(fmt.Errorf("hints: capture: %w", done.Err))
161	212		}
162		-	return w.scan(ctx, req, buf, stride, time.Since(start), res, atspi)
	213	+	res = w.scan(ctx, req, frame{pix: buf, stride: stride, m: m}, time.Since(start), res, atspi)
	214	+	keep = w.pending != nil && w.pending.m == m
	215	+	return res
163	216		case other := <-requests:
164	217		// The buffer is in use until the capture is done: answer now.
165	218		busy := ports.HintScanResult{Seq: other.Seq, Output: other.Output, Err: ErrBusy}
166	219		if !send(ctx, results, busy) {
	220	+	abandon()
167	221		return fail(ctx.Err())
168	222		}
169	223		case <-timer.C:
170		-	// The output may still write into buf: it is never reused.
	224	+	// The output may still write into buf: it is released when its
	225	+	// CaptureDone arrives.
	226	+	abandon()
171	227		return fail(ErrTimeout)
172	228		case <-ctx.Done():
	229	+	abandon()
173	230		return fail(ctx.Err())
174	231		}
175	232		}
176	233		}
177	234
178		-	func (w worker) scan(ctx context.Context, req ports.HintScanRequest, buf []byte, stride int, captureDur time.Duration, res ports.HintScanResult, atspi pendingSource) ports.HintScanResult {
	235	+	// reap releases the buffer of an abandoned capture that is finally done.
	236	+	func (w *worker) reap(id uint64) {
	237	+	if m, ok := w.orphans[id]; ok {
	238	+	delete(w.orphans, id)
	239	+	m.unmap()
	240	+	}
	241	+	}
	242	+
	243	+	// bandJob is one OCR band of one window crop.
	244	+	type bandJob struct {
	245	+	win int // index into the scan's windows
	246	+	key bandKey //
	247	+	ext image.Rectangle // crop-local rows read, with context
	248	+	own [2]int // crop-local rows owned: words centred there are kept
	249	+	hash uint64
	250	+	hit bool
	251	+	gray *image.Gray
	252	+	words []OCRWord
	253	+	err error
	254	+	}
	255	+
	256	+	func (w worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame, captureDur time.Duration, res ports.HintScanResult, atspi pendingSource) ports.HintScanResult {
179	257		log := logging.For(ctx, "hints")
180	258		ar := atspi.wait()
181	259		if ar.err != nil && !w.warnedSource {

@@ -193,60 +271,109 @@ func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, buf []byte
193	271		if w.cacheSeed == (maphash.Seed{}) {
194	272		w.cacheSeed = maphash.MakeSeed()
195	273		}
196		-	nextCache := make(map[cropKey]cropCache)
	274	+	nextCache := make(map[bandKey]bandCache)
197	275		var crops []image.Rectangle
198	276		var allWords []OCRWord
199	277		pixels, links := 0, 0
200		-	cacheHit := true
201		-	// Phase 1 reads the frame: only the crops that miss the cache are
202		-	// copied, as grayscale. Phase 2 (OCR) does not need the frame.
203		-	type job struct {
204		-	key cropKey
205		-	r image.Rectangle
206		-	hash uint64
207		-	gray *image.Gray
208		-	hit bool
209		-	}
210		-	var jobs []job
211		-	for _, win := range req.Windows {
212		-	if ar.covered[win.ID] {
213		-	continue
	278	+	ocrOn := fr.pix != nil && w.ocrReady(ctx)
	279	+	// Phase 1 reads the frame: the bands that miss the cache are copied, as
	280	+	// grayscale, into one arena. Phase 2 (OCR) does not need the frame.
	281	+	var jobs []*bandJob
	282	+	var rects []image.Rectangle // crop of each window that has bands
	283	+	var arena *mapping
	284	+	if ocrOn {
	285	+	type pending struct {
	286	+	r image.Rectangle
	287	+	jobs []*bandJob
	288	+	}
	289	+	var wins []pending
	290	+	need := 0
	291	+	for _, win := range req.Windows {
	292	+	if ar.covered[win.ID] {
	293	+	continue
	294	+	}
	295	+	r := windowCrop(win.Rect, req.Scale, req.Width, req.Height)
	296	+	if r.Empty() {
	297	+	continue
	298	+	}
	299	+	crops = append(crops, r)
	300	+	pixels += r.Dx() * r.Dy()
	301	+	p := pending{r: r}
	302	+	for _, b := range bandsOf(r.Dy()) {
	303	+	ext := image.Rect(0, b.top, r.Dx(), b.bottom)
	304	+	j := &bandJob{win: len(wins), ext: ext, own: [2]int{b.ownTop, b.ownBottom},
	305	+	key: bandKey{output: req.Output, window: win.ID, band: b.idx, bands: b.n}}
	306	+	j.hash = w.cropHash(fr.pix, fr.stride, ext.Add(r.Min))
	307	+	entry, ok := w.cache[j.key]
	308	+	if ok && entry.hash == j.hash && entry.ext == ext {
	309	+	j.hit, j.words = true, entry.words
	310	+	} else {
	311	+	need += ext.Dx() * ext.Dy()
	312	+	}
	313	+	p.jobs = append(p.jobs, j)
	314	+	}
	315	+	wins = append(wins, p)
	316	+	}
	317	+	if need > 0 {
	318	+	arena = w.alloc(need)
214	319		}
215		-	r := windowCrop(win.Rect, req.Scale, req.Width, req.Height)
216		-	if r.Empty() {
217		-	continue
	320	+	off := 0
	321	+	for _, p := range wins {
	322	+	rects = append(rects, p.r)
	323	+	for _, j := range p.jobs {
	324	+	if !j.hit {
	325	+	n := j.ext.Dx() * j.ext.Dy()
	326	+	j.gray = cropGray(fr.pix, fr.stride, j.ext.Add(p.r.Min), arena.b[off:off+n])
	327	+	off += n
	328	+	}
	329	+	jobs = append(jobs, j)
	330	+	}
218	331		}
219		-	crops = append(crops, r)
220		-	pixels += r.Dx() * r.Dy()
221		-	key := cropKey{output: req.Output, window: win.ID}
222		-	hash := w.cropHash(buf, stride, r)
223		-	entry, ok := w.cache[key]
224		-	j := job{key: key, r: r, hash: hash, hit: ok && entry.hash == hash && entry.width == r.Dx() && entry.height == r.Dy()}
225		-	cacheHit = cacheHit && j.hit
226		-	if !j.hit {
227		-	j.gray = cropGray(buf, stride, r)
	332	+	} else if w.dumpDir != "" {
	333	+	for _, win := range req.Windows {
	334	+	if !ar.covered[win.ID] {
	335	+	if r := windowCrop(win.Rect, req.Scale, req.Width, req.Height); !r.Empty() {
	336	+	crops = append(crops, r)
	337	+	}
	338	+	}
228	339		}
229		-	jobs = append(jobs, j)
230	340		}
231		-	if w.dumpDir != "" {
232		-	// The dump reads the frame after the result is sent.
233		-	w.pending = &dumpJob{when: time.Now(), req: req, pix: buf, stride: stride, crops: crops, captureD: captureDur, atspiD: ar.dur}
	341	+	if w.dumpDir != "" && fr.pix != nil {
	342	+	// The dump reads the frame after the result is sent and releases it.
	343	+	w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, crops: crops, captureD: captureDur, atspiD: ar.dur}
	344	+	} else {
	345	+	fr.m.unmap() // the crops are extracted: give the frame back
234	346		}
235		-	buf = nil
236		-	for _, j := range jobs {
237		-	entry := w.cache[j.key]
238		-	if !j.hit {
239		-	words, _, err := readWords(ctx, j.gray)
240		-	if err != nil {
241		-	failureWarning.Do(func() { log.Warn().Err(err).Msg("OCR unavailable; install tesseract and tesseract-data-eng") })
242		-	continue
	347	+	fr.pix = nil
	348	+
	349	+	hits := 0
	350	+	if len(jobs) > 0 {
	351	+	w.recognize(ctx, jobs)
	352	+	for _, j := range jobs {
	353	+	if j.hit {
	354	+	hits++
243	355		}
244		-	entry = cropCache{hash: j.hash, width: j.r.Dx(), height: j.r.Dy(), words: words}
245	356		}
246		-	nextCache[j.key] = entry
247		-	words := append([]OCRWord(nil), entry.words...)
248		-	for i := range words {
249		-	words[i].Rect = words[i].Rect.Add(j.r.Min)
	357	+	}
	358	+	arena.unmap()
	359	+
	360	+	// Merge the bands of each window, in order.
	361	+	k := 0
	362	+	for wi, r := range rects {
	363	+	var words []OCRWord
	364	+	for ; k < len(jobs) && jobs[k].win == wi; k++ {
	365	+	j := jobs[k]
	366	+	if j.err != nil {
	367	+	failureWarning.Do(func() { log.Warn().Err(j.err).Msg("OCR unavailable; install tesseract and tesseract-data-eng") })
	368	+	continue
	369	+	}
	370	+	nextCache[j.key] = bandCache{hash: j.hash, ext: j.ext, words: j.words}
	371	+	for _, word := range j.words {
	372	+	word.Rect = word.Rect.Add(r.Min)
	373	+	if !duplicateWord(words, word) {
	374	+	words = append(words, word)
	375	+	}
	376	+	}
250	377		}
251	378		allWords = append(allWords, words...)
252	379		targets := textLinks(words, req.Scale)

@@ -261,20 +388,38 @@ func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, buf []byte
261	388		}
262	389		log.Info().Str("output", req.Output).
263	390		Float64("atspi_ms", ms(ar.dur)).Int("atspi_windows", len(ar.covered)).Int("atspi_targets", nAtspi).
264		-	Int("ocr_windows", len(crops)).Float64("ocr_ms", ms(ocrDur)).Int("ocr_px", pixels).
265		-	Int("words", len(allWords)).Int("ocr_links", links).Bool("cache_hit", len(crops) > 0 && cacheHit).
	391	+	Int("ocr_windows", len(rects)).Float64("ocr_ms", ms(ocrDur)).Int("ocr_px", pixels).
	392	+	Int("ocr_bands", len(jobs)).Int("band_hits", hits).
	393	+	Int("words", len(allWords)).Int("ocr_links", links).Bool("cache_hit", len(jobs) > 0 && hits == len(jobs)).
266	394		Int("targets", len(out)).Msg("hint scan")
267	395		return res
268	396		}
269	397
270		-	type cropKey struct {
	398	+	// duplicateWord reports whether words already holds word (the same text at
	399	+	// the same place), as happens where two bands overlap.
	400	+	func duplicateWord(words []OCRWord, word OCRWord) bool {
	401	+	for _, old := range words {
	402	+	if strings.EqualFold(word.Text, old.Text) && overlap(word.Rect, old.Rect) > .5 {
	403	+	return true
	404	+	}
	405	+	}
	406	+	return false
	407	+	}
	408	+
	409	+	// bandKey identifies one OCR band of a window on an output.
	410	+	type bandKey struct {
271	411		output string
272	412		window ports.WindowID
	413	+	band int
	414	+	bands int
273	415		}
274		-	type cropCache struct {
275		-	hash uint64
276		-	width, height int
277		-	words []OCRWord
	416	+
	417	+	// bandCache is the OCR result of a band whose pixels hash to hash. words are
	418	+	// crop-local and only the ones the band owns.
	419	+	type bandCache struct {
	420	+	hash uint64
	421	+	ext image.Rectangle
	422	+	words []OCRWord
278	423		}
279	424
280	425		// windowCrop rounds outwards in physical pixels, then intersects the output.

@@ -299,16 +444,16 @@ func (w *worker) cropHash(buf []byte, stride int, r image.Rectangle) uint64 {
299	444		return h.Sum64()
300	445		}
301	446
302		-	// cropGray converts the BGRA pixels of r to 8-bit luma; only this window is
303		-	// converted, not the full output.
304		-	func cropGray(buf []byte, stride int, r image.Rectangle) *image.Gray {
305		-	img := image.NewGray(image.Rect(0, 0, r.Dx(), r.Dy()))
	447	+	// cropGray converts the BGRA pixels of r to 8-bit luma into dst, which must
	448	+	// hold r.Dx()*r.Dy() bytes; only this band is converted, not the output.
	449	+	func cropGray(buf []byte, stride int, r image.Rectangle, dst []byte) *image.Gray {
	450	+	img := &image.Gray{Pix: dst[:r.Dx()*r.Dy()], Stride: r.Dx(), Rect: image.Rect(0, 0, r.Dx(), r.Dy())}
306	451		for y := 0; y < r.Dy(); y++ {
307	452		src := buf[(r.Min.Y+y)stride+r.Min.X4:][:r.Dx()*4]
308		-	dst := img.Pix[y*img.Stride:][:r.Dx()]
309		-	for x := range dst {
	453	+	row := img.Pix[y*img.Stride:][:r.Dx()]
	454	+	for x := range row {
310	455		p := src[x4 : x4+4 : x*4+4]
311		-	dst[x] = uint8((uint32(p[2])299 + uint32(p[1])587 + uint32(p[0])*114 + 500) / 1000)
	456	+	row[x] = uint8((uint32(p[2])299 + uint32(p[1])587 + uint32(p[0])*114 + 500) / 1000)
312	457		}
313	458		}
314	459		return img


@@ -9,7 +9,7 @@ import (
9		"hash/maphash"	9		"hash/maphash"
10		"image"	10		"image"
11		"math"	11		"math"
12	-	"runtime/debug"	12	+	"strings"
13		"time"	13		"time"
14			14
15		"github.com/bnema/neferwl/internal/adapters/logging"	15		"github.com/bnema/neferwl/internal/adapters/logging"

@@ -55,12 +55,24 @@ type atspiResult struct {
55			55
56		type worker struct {	56		type worker struct {
57		source targetSource	57		source targetSource
58	-	cache map[cropKey]cropCache	58	+	cache map[bandKey]bandCache
59		cacheSeed maphash.Seed	59		cacheSeed maphash.Seed
			60	+	ocr ocrState
60		warnedSource bool	61		warnedSource bool
61		n uint64	62		n uint64
62		dumpDir string	63		dumpDir string
63		pending *dumpJob	64		pending *dumpJob
			65	+	// orphans are capture buffers the output may still write: they are
			66	+	// released when its CaptureDone finally arrives.
			67	+	orphans map[uint64]*mapping
			68	+	live int // mappings not released yet
			69	+	}
			70	+
			71	+	// frame is the captured output: BGRA pixels, nil when no capture was needed.
			72	+	type frame struct {
			73	+	pix []byte
			74	+	stride int
			75	+	m *mapping
64		}	76		}
65			77
66		func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {	78		func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {

@@ -75,19 +87,19 @@ func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures ch
75		return nil	87		return nil
76		}	88		}
77		w.dump(ctx)	89		w.dump(ctx)
78	-	// Nothing of the scan is kept: give the frame back to the system.
79	-	debug.FreeOSMemory()
80		}	90		}
81		}	91		}
82		}	92		}
83			93
84	-	// dump writes the last successful scan, after its result was sent.	94	+	// dump writes the last successful scan, after its result was sent, then
			95	+	// releases its frame.
85		func (w *worker) dump(ctx context.Context) {	96		func (w *worker) dump(ctx context.Context) {
86		j := w.pending	97		j := w.pending
87		w.pending = nil	98		w.pending = nil
88		if j == nil {	99		if j == nil {
89		return	100		return
90		}	101		}
			102	+	defer j.m.unmap()
91		if err := writeDump(w.dumpDir, j); err != nil {	103		if err := writeDump(w.dumpDir, j); err != nil {
92		log := logging.For(ctx, "hints")	104		log := logging.For(ctx, "hints")
93		log.Warn().Err(err).Str("dir", w.dumpDir).Msg("hint debug dump failed")	105		log.Warn().Err(err).Str("dir", w.dumpDir).Msg("hint debug dump failed")

@@ -103,6 +115,25 @@ func send(ctx context.Context, results chan<- ports.HintScanResult, res ports.Hi
103		}	115		}
104		}	116		}
105			117
			118	+	// needFrame reports whether the scan must capture the output: OCR has a
			119	+	// window to read, or the debug dump wants the frame. With OCR unavailable
			120	+	// (tesseract missing) and no dump, the accessibility targets are the result.
			121	+	func (w *worker) needFrame(ctx context.Context, req ports.HintScanRequest, ar atspiResult) bool {
			122	+	ocr := w.ocrReady(ctx)
			123	+	if w.dumpDir != "" {
			124	+	return true
			125	+	}
			126	+	if !ocr {
			127	+	return false
			128	+	}
			129	+	for _, win := range req.Windows {
			130	+	if !ar.covered[win.ID] && !windowCrop(win.Rect, req.Scale, req.Width, req.Height).Empty() {
			131	+	return true
			132	+	}
			133	+	}
			134	+	return false
			135	+	}
			136	+
106		func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration) ports.HintScanResult {	137		func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration) ports.HintScanResult {
107		log := logging.For(ctx, "hints")	138		log := logging.For(ctx, "hints")
108		res := ports.HintScanResult{Seq: req.Seq, Output: req.Output}	139		res := ports.HintScanResult{Seq: req.Seq, Output: req.Output}

@@ -115,13 +146,24 @@ func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests
115		if req.Width <= 0 \|\| req.Height <= 0 \|\| req.Width*req.Height > maxPixels {	146		if req.Width <= 0 \|\| req.Height <= 0 \|\| req.Width*req.Height > maxPixels {
116		return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))	147		return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))
117		}	148		}
118	-	// The accessibility query runs while the frame is captured; OCR waits for it.	149	+	// The accessibility query is always waited for: the source is not shared
119	-	// It is always waited for: the source is not shared between requests.	150	+	// between requests. The capture only starts once it is known whether any
			151	+	// window needs OCR.
120		atspi := w.startSource(ctx, req)	152		atspi := w.startSource(ctx, req)
121		defer func() { <-atspi.done }()	153		defer func() { <-atspi.done }()
122	-	// The capture buffer lives for this scan only; a dump job may keep it	154	+	if !w.needFrame(ctx, req, atspi.wait()) {
123	-	// until the dump is written.	155	+	return w.scan(ctx, req, frame{}, 0, res, atspi)
124	-	buf := make([]byte, req.Widthreq.Height4)	156	+	}
			157	+	// The capture buffer is anonymous mapped memory, outside the Go heap. It
			158	+	// is released once the crops are extracted, or by the dump job.
			159	+	m := w.alloc(req.Width * req.Height * 4)
			160	+	keep := false // the mapping outlives this call: dump job or pending capture
			161	+	defer func() {
			162	+	if !keep {
			163	+	m.unmap()
			164	+	}
			165	+	}()
			166	+	buf := m.b
125			167
126		w.n++	168		w.n++
127		id := ports.CaptureIDInternal \| w.n	169		id := ports.CaptureIDInternal \| w.n

@@ -149,33 +191,69 @@ func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests
149		case <-ctx.Done():	191		case <-ctx.Done():
150		return fail(ctx.Err())	192		return fail(ctx.Err())
151		}	193		}
			194	+	// From here the output owns buf until CaptureDone.
			195	+	abandon := func() {
			196	+	if w.orphans == nil {
			197	+	w.orphans = map[uint64]*mapping{}
			198	+	}
			199	+	w.orphans[id] = m
			200	+	keep = true
			201	+	}
152			202
153		for {	203		for {
154		select {	204		select {
155		case done := <-captured:	205		case done := <-captured:
156		if done.ID != id {	206		if done.ID != id {
			207	+	w.reap(done.ID)
157		continue	208		continue
158		}	209		}
159		if done.Err != nil {	210		if done.Err != nil {
160		return fail(fmt.Errorf("hints: capture: %w", done.Err))	211		return fail(fmt.Errorf("hints: capture: %w", done.Err))
161		}	212		}
162	-	return w.scan(ctx, req, buf, stride, time.Since(start), res, atspi)	213	+	res = w.scan(ctx, req, frame{pix: buf, stride: stride, m: m}, time.Since(start), res, atspi)
			214	+	keep = w.pending != nil && w.pending.m == m
			215	+	return res
163		case other := <-requests:	216		case other := <-requests:
164		// The buffer is in use until the capture is done: answer now.	217		// The buffer is in use until the capture is done: answer now.
165		busy := ports.HintScanResult{Seq: other.Seq, Output: other.Output, Err: ErrBusy}	218		busy := ports.HintScanResult{Seq: other.Seq, Output: other.Output, Err: ErrBusy}
166		if !send(ctx, results, busy) {	219		if !send(ctx, results, busy) {
			220	+	abandon()
167		return fail(ctx.Err())	221		return fail(ctx.Err())
168		}	222		}
169		case <-timer.C:	223		case <-timer.C:
170	-	// The output may still write into buf: it is never reused.	224	+	// The output may still write into buf: it is released when its
			225	+	// CaptureDone arrives.
			226	+	abandon()
171		return fail(ErrTimeout)	227		return fail(ErrTimeout)
172		case <-ctx.Done():	228		case <-ctx.Done():
			229	+	abandon()
173		return fail(ctx.Err())	230		return fail(ctx.Err())
174		}	231		}
175		}	232		}
176		}	233		}
177			234
178	-	func (w worker) scan(ctx context.Context, req ports.HintScanRequest, buf []byte, stride int, captureDur time.Duration, res ports.HintScanResult, atspi pendingSource) ports.HintScanResult {	235	+	// reap releases the buffer of an abandoned capture that is finally done.
			236	+	func (w *worker) reap(id uint64) {
			237	+	if m, ok := w.orphans[id]; ok {
			238	+	delete(w.orphans, id)
			239	+	m.unmap()
			240	+	}
			241	+	}
			242	+
			243	+	// bandJob is one OCR band of one window crop.
			244	+	type bandJob struct {
			245	+	win int // index into the scan's windows
			246	+	key bandKey //
			247	+	ext image.Rectangle // crop-local rows read, with context
			248	+	own [2]int // crop-local rows owned: words centred there are kept
			249	+	hash uint64
			250	+	hit bool
			251	+	gray *image.Gray
			252	+	words []OCRWord
			253	+	err error
			254	+	}
			255	+
			256	+	func (w worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame, captureDur time.Duration, res ports.HintScanResult, atspi pendingSource) ports.HintScanResult {
179		log := logging.For(ctx, "hints")	257		log := logging.For(ctx, "hints")
180		ar := atspi.wait()	258		ar := atspi.wait()
181		if ar.err != nil && !w.warnedSource {	259		if ar.err != nil && !w.warnedSource {

@@ -193,60 +271,109 @@ func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, buf []byte
193		if w.cacheSeed == (maphash.Seed{}) {	271		if w.cacheSeed == (maphash.Seed{}) {
194		w.cacheSeed = maphash.MakeSeed()	272		w.cacheSeed = maphash.MakeSeed()
195		}	273		}
196	-	nextCache := make(map[cropKey]cropCache)	274	+	nextCache := make(map[bandKey]bandCache)
197		var crops []image.Rectangle	275		var crops []image.Rectangle
198		var allWords []OCRWord	276		var allWords []OCRWord
199		pixels, links := 0, 0	277		pixels, links := 0, 0
200	-	cacheHit := true	278	+	ocrOn := fr.pix != nil && w.ocrReady(ctx)
201	-	// Phase 1 reads the frame: only the crops that miss the cache are	279	+	// Phase 1 reads the frame: the bands that miss the cache are copied, as
202	-	// copied, as grayscale. Phase 2 (OCR) does not need the frame.	280	+	// grayscale, into one arena. Phase 2 (OCR) does not need the frame.
203	-	type job struct {	281	+	var jobs []*bandJob
204	-	key cropKey	282	+	var rects []image.Rectangle // crop of each window that has bands
205	-	r image.Rectangle	283	+	var arena *mapping
206	-	hash uint64	284	+	if ocrOn {
207	-	gray *image.Gray	285	+	type pending struct {
208	-	hit bool	286	+	r image.Rectangle
209	-	}	287	+	jobs []*bandJob
210	-	var jobs []job	288	+	}
211	-	for _, win := range req.Windows {	289	+	var wins []pending
212	-	if ar.covered[win.ID] {	290	+	need := 0
213	-	continue	291	+	for _, win := range req.Windows {
			292	+	if ar.covered[win.ID] {
			293	+	continue
			294	+	}
			295	+	r := windowCrop(win.Rect, req.Scale, req.Width, req.Height)
			296	+	if r.Empty() {
			297	+	continue
			298	+	}
			299	+	crops = append(crops, r)
			300	+	pixels += r.Dx() * r.Dy()
			301	+	p := pending{r: r}
			302	+	for _, b := range bandsOf(r.Dy()) {
			303	+	ext := image.Rect(0, b.top, r.Dx(), b.bottom)
			304	+	j := &bandJob{win: len(wins), ext: ext, own: [2]int{b.ownTop, b.ownBottom},
			305	+	key: bandKey{output: req.Output, window: win.ID, band: b.idx, bands: b.n}}
			306	+	j.hash = w.cropHash(fr.pix, fr.stride, ext.Add(r.Min))
			307	+	entry, ok := w.cache[j.key]
			308	+	if ok && entry.hash == j.hash && entry.ext == ext {
			309	+	j.hit, j.words = true, entry.words
			310	+	} else {
			311	+	need += ext.Dx() * ext.Dy()
			312	+	}
			313	+	p.jobs = append(p.jobs, j)
			314	+	}
			315	+	wins = append(wins, p)
			316	+	}
			317	+	if need > 0 {
			318	+	arena = w.alloc(need)
214		}	319		}
215	-	r := windowCrop(win.Rect, req.Scale, req.Width, req.Height)	320	+	off := 0
216	-	if r.Empty() {	321	+	for _, p := range wins {
217	-	continue	322	+	rects = append(rects, p.r)
			323	+	for _, j := range p.jobs {
			324	+	if !j.hit {
			325	+	n := j.ext.Dx() * j.ext.Dy()
			326	+	j.gray = cropGray(fr.pix, fr.stride, j.ext.Add(p.r.Min), arena.b[off:off+n])
			327	+	off += n
			328	+	}
			329	+	jobs = append(jobs, j)
			330	+	}
218		}	331		}
219	-	crops = append(crops, r)	332	+	} else if w.dumpDir != "" {
220	-	pixels += r.Dx() * r.Dy()	333	+	for _, win := range req.Windows {
221	-	key := cropKey{output: req.Output, window: win.ID}	334	+	if !ar.covered[win.ID] {
222	-	hash := w.cropHash(buf, stride, r)	335	+	if r := windowCrop(win.Rect, req.Scale, req.Width, req.Height); !r.Empty() {
223	-	entry, ok := w.cache[key]	336	+	crops = append(crops, r)
224	-	j := job{key: key, r: r, hash: hash, hit: ok && entry.hash == hash && entry.width == r.Dx() && entry.height == r.Dy()}	337	+	}
225	-	cacheHit = cacheHit && j.hit	338	+	}
226	-	if !j.hit {
227	-	j.gray = cropGray(buf, stride, r)
228		}	339		}
229	-	jobs = append(jobs, j)
230		}	340		}
231	-	if w.dumpDir != "" {	341	+	if w.dumpDir != "" && fr.pix != nil {
232	-	// The dump reads the frame after the result is sent.	342	+	// The dump reads the frame after the result is sent and releases it.
233	-	w.pending = &dumpJob{when: time.Now(), req: req, pix: buf, stride: stride, crops: crops, captureD: captureDur, atspiD: ar.dur}	343	+	w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, crops: crops, captureD: captureDur, atspiD: ar.dur}
			344	+	} else {
			345	+	fr.m.unmap() // the crops are extracted: give the frame back
234		}	346		}
235	-	buf = nil	347	+	fr.pix = nil
236	-	for _, j := range jobs {	348	+
237	-	entry := w.cache[j.key]	349	+	hits := 0
238	-	if !j.hit {	350	+	if len(jobs) > 0 {
239	-	words, _, err := readWords(ctx, j.gray)	351	+	w.recognize(ctx, jobs)
240	-	if err != nil {	352	+	for _, j := range jobs {
241	-	failureWarning.Do(func() { log.Warn().Err(err).Msg("OCR unavailable; install tesseract and tesseract-data-eng") })	353	+	if j.hit {
242	-	continue	354	+	hits++
243		}	355		}
244	-	entry = cropCache{hash: j.hash, width: j.r.Dx(), height: j.r.Dy(), words: words}
245		}	356		}
246	-	nextCache[j.key] = entry	357	+	}
247	-	words := append([]OCRWord(nil), entry.words...)	358	+	arena.unmap()
248	-	for i := range words {	359	+
249	-	words[i].Rect = words[i].Rect.Add(j.r.Min)	360	+	// Merge the bands of each window, in order.
			361	+	k := 0
			362	+	for wi, r := range rects {
			363	+	var words []OCRWord
			364	+	for ; k < len(jobs) && jobs[k].win == wi; k++ {
			365	+	j := jobs[k]
			366	+	if j.err != nil {
			367	+	failureWarning.Do(func() { log.Warn().Err(j.err).Msg("OCR unavailable; install tesseract and tesseract-data-eng") })
			368	+	continue
			369	+	}
			370	+	nextCache[j.key] = bandCache{hash: j.hash, ext: j.ext, words: j.words}
			371	+	for _, word := range j.words {
			372	+	word.Rect = word.Rect.Add(r.Min)
			373	+	if !duplicateWord(words, word) {
			374	+	words = append(words, word)
			375	+	}
			376	+	}
250		}	377		}
251		allWords = append(allWords, words...)	378		allWords = append(allWords, words...)
252		targets := textLinks(words, req.Scale)	379		targets := textLinks(words, req.Scale)

@@ -261,20 +388,38 @@ func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, buf []byte
261		}	388		}
262		log.Info().Str("output", req.Output).	389		log.Info().Str("output", req.Output).
263		Float64("atspi_ms", ms(ar.dur)).Int("atspi_windows", len(ar.covered)).Int("atspi_targets", nAtspi).	390		Float64("atspi_ms", ms(ar.dur)).Int("atspi_windows", len(ar.covered)).Int("atspi_targets", nAtspi).
264	-	Int("ocr_windows", len(crops)).Float64("ocr_ms", ms(ocrDur)).Int("ocr_px", pixels).	391	+	Int("ocr_windows", len(rects)).Float64("ocr_ms", ms(ocrDur)).Int("ocr_px", pixels).
265	-	Int("words", len(allWords)).Int("ocr_links", links).Bool("cache_hit", len(crops) > 0 && cacheHit).	392	+	Int("ocr_bands", len(jobs)).Int("band_hits", hits).
			393	+	Int("words", len(allWords)).Int("ocr_links", links).Bool("cache_hit", len(jobs) > 0 && hits == len(jobs)).
266		Int("targets", len(out)).Msg("hint scan")	394		Int("targets", len(out)).Msg("hint scan")
267		return res	395		return res
268		}	396		}
269			397
270	-	type cropKey struct {	398	+	// duplicateWord reports whether words already holds word (the same text at
			399	+	// the same place), as happens where two bands overlap.
			400	+	func duplicateWord(words []OCRWord, word OCRWord) bool {
			401	+	for _, old := range words {
			402	+	if strings.EqualFold(word.Text, old.Text) && overlap(word.Rect, old.Rect) > .5 {
			403	+	return true
			404	+	}
			405	+	}
			406	+	return false
			407	+	}
			408	+
			409	+	// bandKey identifies one OCR band of a window on an output.
			410	+	type bandKey struct {
271		output string	411		output string
272		window ports.WindowID	412		window ports.WindowID
			413	+	band int
			414	+	bands int
273		}	415		}
274	-	type cropCache struct {	416	+
275	-	hash uint64	417	+	// bandCache is the OCR result of a band whose pixels hash to hash. words are
276	-	width, height int	418	+	// crop-local and only the ones the band owns.
277	-	words []OCRWord	419	+	type bandCache struct {
			420	+	hash uint64
			421	+	ext image.Rectangle
			422	+	words []OCRWord
278		}	423		}
279			424
280		// windowCrop rounds outwards in physical pixels, then intersects the output.	425		// windowCrop rounds outwards in physical pixels, then intersects the output.

@@ -299,16 +444,16 @@ func (w *worker) cropHash(buf []byte, stride int, r image.Rectangle) uint64 {
299		return h.Sum64()	444		return h.Sum64()
300		}	445		}
301			446
302	-	// cropGray converts the BGRA pixels of r to 8-bit luma; only this window is	447	+	// cropGray converts the BGRA pixels of r to 8-bit luma into dst, which must
303	-	// converted, not the full output.	448	+	// hold r.Dx()*r.Dy() bytes; only this band is converted, not the output.
304	-	func cropGray(buf []byte, stride int, r image.Rectangle) *image.Gray {	449	+	func cropGray(buf []byte, stride int, r image.Rectangle, dst []byte) *image.Gray {
305	-	img := image.NewGray(image.Rect(0, 0, r.Dx(), r.Dy()))	450	+	img := &image.Gray{Pix: dst[:r.Dx()*r.Dy()], Stride: r.Dx(), Rect: image.Rect(0, 0, r.Dx(), r.Dy())}
306		for y := 0; y < r.Dy(); y++ {	451		for y := 0; y < r.Dy(); y++ {
307		src := buf[(r.Min.Y+y)stride+r.Min.X4:][:r.Dx()*4]	452		src := buf[(r.Min.Y+y)stride+r.Min.X4:][:r.Dx()*4]
308	-	dst := img.Pix[y*img.Stride:][:r.Dx()]	453	+	row := img.Pix[y*img.Stride:][:r.Dx()]
309	-	for x := range dst {	454	+	for x := range row {
310		p := src[x4 : x4+4 : x*4+4]	455		p := src[x4 : x4+4 : x*4+4]
311	-	dst[x] = uint8((uint32(p[2])299 + uint32(p[1])587 + uint32(p[0])*114 + 500) / 1000)	456	+	row[x] = uint8((uint32(p[2])299 + uint32(p[1])587 + uint32(p[0])*114 + 500) / 1000)
312		}	457		}
313		}	458		}
314		return img	459		return img

M internal/adapters/hints/hints_test.go +69 -10


@@ -32,6 +32,11 @@ func start(t testing.T, timeout time.Duration) harness {
32	32		return startWith(t, timeout, Options{})
33	33		}
34	34
	35	+	// startCapturing starts a worker that always captures: debug dumps are on.
	36	+	func startCapturing(t testing.T, timeout time.Duration) harness {
	37	+	return startWith(t, timeout, Options{DumpDir: filepath.Join(t.TempDir(), "hints")})
	38	+	}
	39	+
35	40		func startWith(t testing.T, timeout time.Duration, opts Options) harness {
36	41		t.Helper()
37	42		h := &harness{

@@ -98,7 +103,7 @@ func TestHintScan(t *testing.T) {
98	103		}
99	104
100	105		func TestHintRequestShape(t *testing.T) {
101		-	h := start(t, 5*time.Second)
	106	+	h := startCapturing(t, 5*time.Second)
102	107		got := make(chan ports.CaptureRequest, 1)
103	108		go func() {
104	109		creq := <-h.captures

@@ -119,7 +124,7 @@ func TestHintRequestShape(t *testing.T) {
119	124		}
120	125
121	126		func TestHintCaptureError(t *testing.T) {
122		-	h := start(t, 5*time.Second)
	127	+	h := startCapturing(t, 5*time.Second)
123	128		boom := errors.New("boom")
124	129		go func() {
125	130		creq := <-h.captures

@@ -133,7 +138,7 @@ func TestHintCaptureError(t *testing.T) {
133	138		}
134	139
135	140		func TestHintTimeout(t *testing.T) {
136		-	h := start(t, 50*time.Millisecond)
	141	+	h := startCapturing(t, 50*time.Millisecond)
137	142		h.requests <- ports.HintScanRequest{Seq: 4, Output: "o", Width: 64, Height: 32, Scale: 1}
138	143		res := <-h.results
139	144		require.ErrorIs(t, res.Err, ErrTimeout)

@@ -141,7 +146,7 @@ func TestHintTimeout(t *testing.T) {
141	146		}
142	147
143	148		func TestHintBusy(t *testing.T) {
144		-	h := start(t, 5*time.Second)
	149	+	h := startCapturing(t, 5*time.Second)
145	150		h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 64, Height: 32, Scale: 1}
146	151		creq := <-h.captures
147	152		h.requests <- ports.HintScanRequest{Seq: 2, Output: "o", Width: 64, Height: 32, Scale: 1}

@@ -265,11 +270,14 @@ func scanOnceDone(t testing.T, h harness) ports.HintScanResult {
265	270		creq := <-h.captures
266	271		h.captured <- ports.CaptureDone{ID: creq.ID, Err: errors.New("stop")}
267	272		}()
268		-	h.requests <- ports.HintScanRequest{Seq: 99, Output: "x", Width: 8, Height: 8, Scale: 1}
	273	+	h.requests <- ports.HintScanRequest{Seq: 99, Output: "x", Width: 8, Height: 8, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 8, H: 8}}}}
269	274		return <-h.results
270	275		}
271	276
272	277		func TestHintDumpDisabled(t *testing.T) {
	278	+	if _, err := exec.LookPath("tesseract"); err != nil {
	279	+	t.Skip("tesseract not on PATH")
	280	+	}
273	281		pix, w, hgt := loadBGRA(t)
274	282		dir := filepath.Join(t.TempDir(), "hints")
275	283		h := start(t, 5*time.Second)

@@ -333,11 +341,12 @@ func TestScanSourceErrorUsesOCR(t *testing.T) {
333	341		require.NotEmpty(t, res.Targets)
334	342		}
335	343
336		-	// The worker keeps no frame: the buffer belongs to the scan, and to the dump
337		-	// job until it is written.
	344	+	// The worker keeps no frame: the mapped buffer belongs to the scan, and to the
	345	+	// dump job until it is written; every mapping is released afterwards.
338	346		func TestWorkerHoldsNoFrameAfterScan(t *testing.T) {
339	347		for _, dumpDir := range []string{"", filepath.Join(t.TempDir(), "hints")} {
340		-	w := &worker{dumpDir: dumpDir}
	348	+	// OCR is "installed" but its binary cannot run: only the memory matters.
	349	+	w := &worker{dumpDir: dumpDir, ocr: ocrState{done: true, binary: "/nonexistent/tesseract", lang: "eng"}}
341	350		requests := make(chan ports.HintScanRequest)
342	351		captures := make(chan ports.CaptureRequest, 1)
343	352		captured := make(chan ports.CaptureDone, 1)

@@ -346,16 +355,66 @@ func TestWorkerHoldsNoFrameAfterScan(t *testing.T) {
346	355		creq := <-captures
347	356		captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
348	357		}()
349		-	req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1}
	358	+	req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 64, H: 32}}}}
350	359		res := w.serve(context.Background(), req, requests, captures, captured, results, 5*time.Second)
351	360		require.NoError(t, res.Err)
352	361		if dumpDir == "" {
353	362		require.Nil(t, w.pending)
	363	+	require.Zero(t, w.live, "frame and crops are released before the result")
354	364		} else {
355	365		require.NotNil(t, w.pending)
	366	+	require.Equal(t, 1, w.live, "only the frame, kept for the dump")
356	367		}
357	368		w.dump(context.Background())
358	369		require.Nil(t, w.pending)
359		-	require.Empty(t, w.cache) // no window, no cached crop
	370	+	require.Zero(t, w.live)
	371	+	require.Empty(t, w.cache) // OCR failed: nothing cached
	372	+	}
	373	+	}
	374	+
	375	+	// A capture that timed out stays mapped until the output answers it.
	376	+	func TestTimedOutCaptureIsReleasedWhenDone(t *testing.T) {
	377	+	w := &worker{dumpDir: t.TempDir()}
	378	+	captures := make(chan ports.CaptureRequest, 1)
	379	+	captured := make(chan ports.CaptureDone, 1)
	380	+	req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1}
	381	+	res := w.serve(context.Background(), req, make(chan ports.HintScanRequest), captures, captured, make(chan ports.HintScanResult, 1), 20*time.Millisecond)
	382	+	require.ErrorIs(t, res.Err, ErrTimeout)
	383	+	creq := <-captures
	384	+	require.Equal(t, 1, w.live)
	385	+	creq.Mapped[0] = 1 // still valid: the output may write until CaptureDone
	386	+	w.reap(creq.ID)
	387	+	require.Zero(t, w.live)
	388	+	}
	389	+
	390	+	// Without tesseract OCR is off for good: no capture, only accessibility targets.
	391	+	func TestNoTesseractSkipsCapture(t *testing.T) {
	392	+	t.Setenv("PATH", t.TempDir())
	393	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
	394	+	covered := ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: 30, W: 5, H: 5}}
	395	+	wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}, {ID: 2, Rect: ports.Rect{X: 20, W: 20, H: 20}}}
	396	+	src := newMocktargetSource(t)
	397	+	src.EXPECT().Targets(mock.Anything, wins, false).Return([]ports.HintTarget{button, covered}, map[ports.WindowID]bool{2: true}, nil).Times(2)
	398	+	h := startWith(t, time.Second, Options{Source: src})
	399	+	for seq := uint64(1); seq <= 2; seq++ {
	400	+	h.requests <- ports.HintScanRequest{Seq: seq, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: wins}
	401	+	res := <-h.results
	402	+	require.NoError(t, res.Err)
	403	+	require.Equal(t, []ports.HintTarget{button, covered}, res.Targets)
360	404		}
	405	+	require.Empty(t, h.captures)
	406	+	}
	407	+
	408	+	// With debug dumps on, the frame is still captured (for the dump only).
	409	+	func TestNoTesseractDumpStillCaptures(t *testing.T) {
	410	+	t.Setenv("PATH", t.TempDir())
	411	+	dir := filepath.Join(t.TempDir(), "hints")
	412	+	h := startWith(t, 5*time.Second, Options{DumpDir: dir})
	413	+	go func() {
	414	+	creq := <-h.captures
	415	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	416	+	}()
	417	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 40, H: 20}}}}
	418	+	require.NoError(t, (<-h.results).Err)
	419	+	waitFiles(t, dir, 3)
361	420		}

 	return startWith(t, timeout, Options{})
 }
+// startCapturing starts a worker that always captures: debug dumps are on.
+func startCapturing(t *testing.T, timeout time.Duration) *harness {
+	return startWith(t, timeout, Options{DumpDir: filepath.Join(t.TempDir(), "hints")})
+}
 func startWith(t *testing.T, timeout time.Duration, opts Options) *harness {
 	t.Helper()
 	h := &harness{
 }
 func TestHintRequestShape(t *testing.T) {
-	h := start(t, 5*time.Second)
+	h := startCapturing(t, 5*time.Second)
 	got := make(chan ports.CaptureRequest, 1)
 	go func() {
 		creq := <-h.captures
 }
 func TestHintCaptureError(t *testing.T) {
-	h := start(t, 5*time.Second)
+	h := startCapturing(t, 5*time.Second)
 	boom := errors.New("boom")
 	go func() {
 		creq := <-h.captures
 }
 func TestHintTimeout(t *testing.T) {
-	h := start(t, 50*time.Millisecond)
+	h := startCapturing(t, 50*time.Millisecond)
 	h.requests <- ports.HintScanRequest{Seq: 4, Output: "o", Width: 64, Height: 32, Scale: 1}
 	res := <-h.results
 	require.ErrorIs(t, res.Err, ErrTimeout)
 }
 func TestHintBusy(t *testing.T) {
-	h := start(t, 5*time.Second)
+	h := startCapturing(t, 5*time.Second)
 	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 64, Height: 32, Scale: 1}
 	creq := <-h.captures
 	h.requests <- ports.HintScanRequest{Seq: 2, Output: "o", Width: 64, Height: 32, Scale: 1}
 		creq := <-h.captures
 		h.captured <- ports.CaptureDone{ID: creq.ID, Err: errors.New("stop")}
 	}()
-	h.requests <- ports.HintScanRequest{Seq: 99, Output: "x", Width: 8, Height: 8, Scale: 1}
+	h.requests <- ports.HintScanRequest{Seq: 99, Output: "x", Width: 8, Height: 8, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 8, H: 8}}}}
 	return <-h.results
 }
 func TestHintDumpDisabled(t *testing.T) {
+	if _, err := exec.LookPath("tesseract"); err != nil {
+		t.Skip("tesseract not on PATH")
+	}
 	pix, w, hgt := loadBGRA(t)
 	dir := filepath.Join(t.TempDir(), "hints")
 	h := start(t, 5*time.Second)
 	require.NotEmpty(t, res.Targets)
 }
-// The worker keeps no frame: the buffer belongs to the scan, and to the dump
+// The worker keeps no frame: the mapped buffer belongs to the scan, and to the
-// job until it is written.
+// dump job until it is written; every mapping is released afterwards.
 func TestWorkerHoldsNoFrameAfterScan(t *testing.T) {
 	for _, dumpDir := range []string{"", filepath.Join(t.TempDir(), "hints")} {
-		w := &worker{dumpDir: dumpDir}
+		// OCR is "installed" but its binary cannot run: only the memory matters.
+		w := &worker{dumpDir: dumpDir, ocr: ocrState{done: true, binary: "/nonexistent/tesseract", lang: "eng"}}
 		requests := make(chan ports.HintScanRequest)
 		captures := make(chan ports.CaptureRequest, 1)
 		captured := make(chan ports.CaptureDone, 1)
 			creq := <-captures
 			captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
 		}()
-		req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1}
+		req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 64, H: 32}}}}
 		res := w.serve(context.Background(), req, requests, captures, captured, results, 5*time.Second)
 		require.NoError(t, res.Err)
 		if dumpDir == "" {
 			require.Nil(t, w.pending)
+			require.Zero(t, w.live, "frame and crops are released before the result")
 		} else {
 			require.NotNil(t, w.pending)
+			require.Equal(t, 1, w.live, "only the frame, kept for the dump")
 		}
 		w.dump(context.Background())
 		require.Nil(t, w.pending)
-		require.Empty(t, w.cache) // no window, no cached crop
+		require.Zero(t, w.live)
+		require.Empty(t, w.cache) // OCR failed: nothing cached
+	}
+}
+// A capture that timed out stays mapped until the output answers it.
+func TestTimedOutCaptureIsReleasedWhenDone(t *testing.T) {
+	w := &worker{dumpDir: t.TempDir()}
+	captures := make(chan ports.CaptureRequest, 1)
+	captured := make(chan ports.CaptureDone, 1)
+	req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1}
+	res := w.serve(context.Background(), req, make(chan ports.HintScanRequest), captures, captured, make(chan ports.HintScanResult, 1), 20*time.Millisecond)
+	require.ErrorIs(t, res.Err, ErrTimeout)
+	creq := <-captures
+	require.Equal(t, 1, w.live)
+	creq.Mapped[0] = 1 // still valid: the output may write until CaptureDone
+	w.reap(creq.ID)
+	require.Zero(t, w.live)
+}
+// Without tesseract OCR is off for good: no capture, only accessibility targets.
+func TestNoTesseractSkipsCapture(t *testing.T) {
+	t.Setenv("PATH", t.TempDir())
+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
+	covered := ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: 30, W: 5, H: 5}}
+	wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}, {ID: 2, Rect: ports.Rect{X: 20, W: 20, H: 20}}}
+	src := newMocktargetSource(t)
+	src.EXPECT().Targets(mock.Anything, wins, false).Return([]ports.HintTarget{button, covered}, map[ports.WindowID]bool{2: true}, nil).Times(2)
+	h := startWith(t, time.Second, Options{Source: src})
+	for seq := uint64(1); seq <= 2; seq++ {
+		h.requests <- ports.HintScanRequest{Seq: seq, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: wins}
+		res := <-h.results
+		require.NoError(t, res.Err)
+		require.Equal(t, []ports.HintTarget{button, covered}, res.Targets)
 	}
+	require.Empty(t, h.captures)
+}
+// With debug dumps on, the frame is still captured (for the dump only).
+func TestNoTesseractDumpStillCaptures(t *testing.T) {
+	t.Setenv("PATH", t.TempDir())
+	dir := filepath.Join(t.TempDir(), "hints")
+	h := startWith(t, 5*time.Second, Options{DumpDir: dir})
+	go func() {
+		creq := <-h.captures
+		h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
+	}()
+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 40, H: 20}}}}
+	require.NoError(t, (<-h.results).Err)
+	waitFiles(t, dir, 3)
 }

M internal/adapters/hints/hybrid_test.go +68 -8


@@ -4,6 +4,11 @@ import (
4	4		"context"
5	5		"hash/maphash"
6	6		"image"
	7	+	"image/draw"
	8	+	"image/png"
	9	+	"os"
	10	+	"os/exec"
	11	+	"runtime"
7	12		"testing"
8	13
9	14		"github.com/bnema/neferwl/internal/ports"

@@ -39,20 +44,24 @@ func TestHybridCacheAndMerge(t *testing.T) {
39	44		req := ports.HintScanRequest{Output: "o", Width: 200, Height: 100, Scale: 1, Windows: wins}
40	45		seed := maphash.MakeSeed()
41	46		r := windowCrop(wins[1].Rect, 1, 200, 100)
42		-	entry := cropCache{hash: (&worker{cacheSeed: seed}).cropHash(buf, 800, r), width: 100, height: 100, words: []OCRWord{{Text: "example.org", Rect: image.Rect(10, 20, 90, 40)}}}
	47	+	oldWorkers := ocrWorkers
	48	+	ocrWorkers = 1 // one band covers the crop
	49	+	t.Cleanup(func() { ocrWorkers = oldWorkers })
	50	+	ext := image.Rect(0, 0, 100, 100)
	51	+	entry := bandCache{hash: (&worker{cacheSeed: seed}).cropHash(buf, 800, r), ext: ext, words: []OCRWord{{Text: "example.org", Rect: image.Rect(10, 20, 90, 40)}}}
43	52		button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 1, Y: 1, W: 10, H: 10}}
44	53		link := ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: 110, Y: 20, W: 80, H: 20}}
45	54		uncoveredButton := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 101, Y: 1, W: 10, H: 10}}
46	55		ar := atspiResult{targets: []ports.HintTarget{button, uncoveredButton}, covered: map[ports.WindowID]bool{1: true}}
47		-	w := worker{cacheSeed: seed, cache: map[cropKey]cropCache{{output: "o", window: 2}: entry}}
48		-	got := w.scan(context.Background(), req, buf, 800, 0, ports.HintScanResult{}, sourceResult(ar))
	56	+	w := worker{ocr: ocrState{done: true, binary: "tesseract", lang: "eng"}, cacheSeed: seed, cache: map[bandKey]bandCache{{output: "o", window: 2, band: 0, bands: 1}: entry}}
	57	+	got := w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
49	58		require.Equal(t, []ports.HintTarget{button, uncoveredButton, link}, got.Targets)
50	59		req.LinksOnly = true
51		-	got = w.scan(context.Background(), req, buf, 800, 0, ports.HintScanResult{}, sourceResult(ar))
	60	+	got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
52	61		require.Equal(t, []ports.HintTarget{link}, got.Targets)
53	62		// Changing one pixel invalidates only the crop, never leaving stale links.
54	63		buf[100*4]++
55		-	got = w.scan(context.Background(), req, buf, 800, 0, ports.HintScanResult{}, sourceResult(ar))
	64	+	got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
56	65		require.Empty(t, got.Targets)
57	66		}
58	67

@@ -61,7 +70,7 @@ func TestMissingTesseractKeepsAccessibilityButtons(t *testing.T) {
61	70		w := worker{}
62	71		button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
63	72		req := ports.HintScanRequest{Width: 20, Height: 20, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}}}
64		-	got := w.scan(context.Background(), req, make([]byte, 1600), 80, 0, ports.HintScanResult{}, sourceResult(atspiResult{targets: []ports.HintTarget{button}}))
	73	+	got := w.scan(context.Background(), req, frame{pix: make([]byte, 1600), stride: 80}, 0, ports.HintScanResult{}, sourceResult(atspiResult{targets: []ports.HintTarget{button}}))
65	74		require.NoError(t, got.Err)
66	75		require.Equal(t, []ports.HintTarget{button}, got.Targets)
67	76		}

@@ -69,7 +78,7 @@ func TestMissingTesseractKeepsAccessibilityButtons(t *testing.T) {
69	78		func TestNoWindowsProducesNoPixelTargets(t *testing.T) {
70	79		pix, width, height := loadBGRA(t)
71	80		w := worker{}
72		-	got := w.scan(context.Background(), ports.HintScanRequest{Width: width, Height: height}, pix, width*4, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	81	+	got := w.scan(context.Background(), ports.HintScanRequest{Width: width, Height: height}, frame{pix: pix, stride: width * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
73	82		require.Empty(t, got.Targets)
74	83		}
75	84

@@ -94,7 +103,7 @@ func TestCropGrayAndHash(t *testing.T) {
94	103		copy(buf[y16+x4:], []byte{0, 0, 255, 255}) // BGRA red
95	104		}
96	105		}
97		-	g := cropGray(buf, 16, image.Rect(1, 1, 3, 3))
	106	+	g := cropGray(buf, 16, image.Rect(1, 1, 3, 3), make([]byte, 4))
98	107		require.Equal(t, image.Rect(0, 0, 2, 2), g.Bounds())
99	108		require.Equal(t, uint8(76), g.GrayAt(1, 1).Y)
100	109		w := worker{cacheSeed: maphash.MakeSeed()}

@@ -115,3 +124,54 @@ func TestUpscaleBandGray(t *testing.T) {
115	124		require.Equal(t, image.Rect(0, 0, 4, 2), up.Bounds())
116	125		require.Equal(t, []uint8{5, 5, 6, 6, 5, 5, 6, 6}, up.Pix)
117	126		}
	127	+
	128	+	// A blinking cursor changes one band: the other bands keep their cached
	129	+	// words, and only the changed one is recognized again.
	130	+	func TestBandCacheSurvivesCursorBlink(t *testing.T) {
	131	+	binary, err := exec.LookPath("tesseract")
	132	+	if err != nil {
	133	+	t.Skip("tesseract not on PATH")
	134	+	}
	135	+	old := ocrWorkers
	136	+	ocrWorkers = 4
	137	+	t.Cleanup(func() { ocrWorkers = old })
	138	+	f, err := os.Open("testdata/terminal.png")
	139	+	require.NoError(t, err)
	140	+	defer f.Close()
	141	+	src, err := png.Decode(f)
	142	+	require.NoError(t, err)
	143	+	w, h := src.Bounds().Dx(), src.Bounds().Dy()
	144	+	rgba := image.NewRGBA(image.Rect(0, 0, w, h))
	145	+	draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
	146	+	buf := make([]byte, wh4)
	147	+	for i := 0; i < len(buf); i += 4 {
	148	+	buf[i], buf[i+1], buf[i+2], buf[i+3] = rgba.Pix[i+2], rgba.Pix[i+1], rgba.Pix[i], 0xff
	149	+	}
	150	+	_ = binary
	151	+	wk := &worker{}
	152	+	req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
	153	+	first := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	154	+	require.NotEmpty(t, first.Targets)
	155	+	require.Len(t, wk.cache, min(runtime.GOMAXPROCS(0), 4))
	156	+
	157	+	// The same frame: every band hits.
	158	+	again := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	159	+	require.Equal(t, first.Targets, again.Targets)
	160	+
	161	+	// One pixel at the bottom blinks: only the last band (and the context of
	162	+	// none other) is invalidated.
	163	+	before := map[bandKey]uint64{}
	164	+	for k, v := range wk.cache {
	165	+	before[k] = v.hash
	166	+	}
	167	+	buf[((h-1)w+3)4] ^= 0xff
	168	+	blink := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	169	+	require.Equal(t, first.Targets, blink.Targets)
	170	+	changed := 0
	171	+	for k, v := range wk.cache {
	172	+	if before[k] != v.hash {
	173	+	changed++
	174	+	}
	175	+	}
	176	+	require.Equal(t, 1, changed)
	177	+	}


@@ -4,6 +4,11 @@ import (
4		"context"	4		"context"
5		"hash/maphash"	5		"hash/maphash"
6		"image"	6		"image"
			7	+	"image/draw"
			8	+	"image/png"
			9	+	"os"
			10	+	"os/exec"
			11	+	"runtime"
7		"testing"	12		"testing"
8			13
9		"github.com/bnema/neferwl/internal/ports"	14		"github.com/bnema/neferwl/internal/ports"

@@ -39,20 +44,24 @@ func TestHybridCacheAndMerge(t *testing.T) {
39		req := ports.HintScanRequest{Output: "o", Width: 200, Height: 100, Scale: 1, Windows: wins}	44		req := ports.HintScanRequest{Output: "o", Width: 200, Height: 100, Scale: 1, Windows: wins}
40		seed := maphash.MakeSeed()	45		seed := maphash.MakeSeed()
41		r := windowCrop(wins[1].Rect, 1, 200, 100)	46		r := windowCrop(wins[1].Rect, 1, 200, 100)
42	-	entry := cropCache{hash: (&worker{cacheSeed: seed}).cropHash(buf, 800, r), width: 100, height: 100, words: []OCRWord{{Text: "example.org", Rect: image.Rect(10, 20, 90, 40)}}}	47	+	oldWorkers := ocrWorkers
			48	+	ocrWorkers = 1 // one band covers the crop
			49	+	t.Cleanup(func() { ocrWorkers = oldWorkers })
			50	+	ext := image.Rect(0, 0, 100, 100)
			51	+	entry := bandCache{hash: (&worker{cacheSeed: seed}).cropHash(buf, 800, r), ext: ext, words: []OCRWord{{Text: "example.org", Rect: image.Rect(10, 20, 90, 40)}}}
43		button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 1, Y: 1, W: 10, H: 10}}	52		button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 1, Y: 1, W: 10, H: 10}}
44		link := ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: 110, Y: 20, W: 80, H: 20}}	53		link := ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: 110, Y: 20, W: 80, H: 20}}
45		uncoveredButton := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 101, Y: 1, W: 10, H: 10}}	54		uncoveredButton := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 101, Y: 1, W: 10, H: 10}}
46		ar := atspiResult{targets: []ports.HintTarget{button, uncoveredButton}, covered: map[ports.WindowID]bool{1: true}}	55		ar := atspiResult{targets: []ports.HintTarget{button, uncoveredButton}, covered: map[ports.WindowID]bool{1: true}}
47	-	w := worker{cacheSeed: seed, cache: map[cropKey]cropCache{{output: "o", window: 2}: entry}}	56	+	w := worker{ocr: ocrState{done: true, binary: "tesseract", lang: "eng"}, cacheSeed: seed, cache: map[bandKey]bandCache{{output: "o", window: 2, band: 0, bands: 1}: entry}}
48	-	got := w.scan(context.Background(), req, buf, 800, 0, ports.HintScanResult{}, sourceResult(ar))	57	+	got := w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
49		require.Equal(t, []ports.HintTarget{button, uncoveredButton, link}, got.Targets)	58		require.Equal(t, []ports.HintTarget{button, uncoveredButton, link}, got.Targets)
50		req.LinksOnly = true	59		req.LinksOnly = true
51	-	got = w.scan(context.Background(), req, buf, 800, 0, ports.HintScanResult{}, sourceResult(ar))	60	+	got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
52		require.Equal(t, []ports.HintTarget{link}, got.Targets)	61		require.Equal(t, []ports.HintTarget{link}, got.Targets)
53		// Changing one pixel invalidates only the crop, never leaving stale links.	62		// Changing one pixel invalidates only the crop, never leaving stale links.
54		buf[100*4]++	63		buf[100*4]++
55	-	got = w.scan(context.Background(), req, buf, 800, 0, ports.HintScanResult{}, sourceResult(ar))	64	+	got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
56		require.Empty(t, got.Targets)	65		require.Empty(t, got.Targets)
57		}	66		}
58			67

@@ -61,7 +70,7 @@ func TestMissingTesseractKeepsAccessibilityButtons(t *testing.T) {
61		w := worker{}	70		w := worker{}
62		button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}	71		button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
63		req := ports.HintScanRequest{Width: 20, Height: 20, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}}}	72		req := ports.HintScanRequest{Width: 20, Height: 20, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}}}
64	-	got := w.scan(context.Background(), req, make([]byte, 1600), 80, 0, ports.HintScanResult{}, sourceResult(atspiResult{targets: []ports.HintTarget{button}}))	73	+	got := w.scan(context.Background(), req, frame{pix: make([]byte, 1600), stride: 80}, 0, ports.HintScanResult{}, sourceResult(atspiResult{targets: []ports.HintTarget{button}}))
65		require.NoError(t, got.Err)	74		require.NoError(t, got.Err)
66		require.Equal(t, []ports.HintTarget{button}, got.Targets)	75		require.Equal(t, []ports.HintTarget{button}, got.Targets)
67		}	76		}

@@ -69,7 +78,7 @@ func TestMissingTesseractKeepsAccessibilityButtons(t *testing.T) {
69		func TestNoWindowsProducesNoPixelTargets(t *testing.T) {	78		func TestNoWindowsProducesNoPixelTargets(t *testing.T) {
70		pix, width, height := loadBGRA(t)	79		pix, width, height := loadBGRA(t)
71		w := worker{}	80		w := worker{}
72	-	got := w.scan(context.Background(), ports.HintScanRequest{Width: width, Height: height}, pix, width*4, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))	81	+	got := w.scan(context.Background(), ports.HintScanRequest{Width: width, Height: height}, frame{pix: pix, stride: width * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
73		require.Empty(t, got.Targets)	82		require.Empty(t, got.Targets)
74		}	83		}
75			84

@@ -94,7 +103,7 @@ func TestCropGrayAndHash(t *testing.T) {
94		copy(buf[y16+x4:], []byte{0, 0, 255, 255}) // BGRA red	103		copy(buf[y16+x4:], []byte{0, 0, 255, 255}) // BGRA red
95		}	104		}
96		}	105		}
97	-	g := cropGray(buf, 16, image.Rect(1, 1, 3, 3))	106	+	g := cropGray(buf, 16, image.Rect(1, 1, 3, 3), make([]byte, 4))
98		require.Equal(t, image.Rect(0, 0, 2, 2), g.Bounds())	107		require.Equal(t, image.Rect(0, 0, 2, 2), g.Bounds())
99		require.Equal(t, uint8(76), g.GrayAt(1, 1).Y)	108		require.Equal(t, uint8(76), g.GrayAt(1, 1).Y)
100		w := worker{cacheSeed: maphash.MakeSeed()}	109		w := worker{cacheSeed: maphash.MakeSeed()}

@@ -115,3 +124,54 @@ func TestUpscaleBandGray(t *testing.T) {
115		require.Equal(t, image.Rect(0, 0, 4, 2), up.Bounds())	124		require.Equal(t, image.Rect(0, 0, 4, 2), up.Bounds())
116		require.Equal(t, []uint8{5, 5, 6, 6, 5, 5, 6, 6}, up.Pix)	125		require.Equal(t, []uint8{5, 5, 6, 6, 5, 5, 6, 6}, up.Pix)
117		}	126		}
			127	+
			128	+	// A blinking cursor changes one band: the other bands keep their cached
			129	+	// words, and only the changed one is recognized again.
			130	+	func TestBandCacheSurvivesCursorBlink(t *testing.T) {
			131	+	binary, err := exec.LookPath("tesseract")
			132	+	if err != nil {
			133	+	t.Skip("tesseract not on PATH")
			134	+	}
			135	+	old := ocrWorkers
			136	+	ocrWorkers = 4
			137	+	t.Cleanup(func() { ocrWorkers = old })
			138	+	f, err := os.Open("testdata/terminal.png")
			139	+	require.NoError(t, err)
			140	+	defer f.Close()
			141	+	src, err := png.Decode(f)
			142	+	require.NoError(t, err)
			143	+	w, h := src.Bounds().Dx(), src.Bounds().Dy()
			144	+	rgba := image.NewRGBA(image.Rect(0, 0, w, h))
			145	+	draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
			146	+	buf := make([]byte, wh4)
			147	+	for i := 0; i < len(buf); i += 4 {
			148	+	buf[i], buf[i+1], buf[i+2], buf[i+3] = rgba.Pix[i+2], rgba.Pix[i+1], rgba.Pix[i], 0xff
			149	+	}
			150	+	_ = binary
			151	+	wk := &worker{}
			152	+	req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
			153	+	first := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
			154	+	require.NotEmpty(t, first.Targets)
			155	+	require.Len(t, wk.cache, min(runtime.GOMAXPROCS(0), 4))
			156	+
			157	+	// The same frame: every band hits.
			158	+	again := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
			159	+	require.Equal(t, first.Targets, again.Targets)
			160	+
			161	+	// One pixel at the bottom blinks: only the last band (and the context of
			162	+	// none other) is invalidated.
			163	+	before := map[bandKey]uint64{}
			164	+	for k, v := range wk.cache {
			165	+	before[k] = v.hash
			166	+	}
			167	+	buf[((h-1)w+3)4] ^= 0xff
			168	+	blink := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
			169	+	require.Equal(t, first.Targets, blink.Targets)
			170	+	changed := 0
			171	+	for k, v := range wk.cache {
			172	+	if before[k] != v.hash {
			173	+	changed++
			174	+	}
			175	+	}
			176	+	require.Equal(t, 1, changed)
			177	+	}

A internal/adapters/hints/mmap.go +40 -0

@@ -0,0 +1,40 @@
	1	+	package hints
	2	+
	3	+	import "syscall"
	4	+
	5	+	// mapping is a byte region that lives outside the Go heap: anonymous memory
	6	+	// from mmap, returned to the system by unmap. When mmap fails it falls back
	7	+	// to heap memory, which unmap then just drops.
	8	+	type mapping struct {
	9	+	b []byte
	10	+	mapped bool
	11	+	live *int
	12	+	}
	13	+
	14	+	// alloc returns a zeroed region of n bytes; w.live counts the regions not
	15	+	// yet released.
	16	+	func (w worker) alloc(n int) mapping {
	17	+	m := &mapping{live: &w.live}
	18	+	b, err := syscall.Mmap(-1, 0, n, syscall.PROT_READ\|syscall.PROT_WRITE, syscall.MAP_PRIVATE\|syscall.MAP_ANON)
	19	+	if err != nil {
	20	+	b = make([]byte, n)
	21	+	} else {
	22	+	m.mapped = true
	23	+	}
	24	+	m.b = b
	25	+	w.live++
	26	+	return m
	27	+	}
	28	+
	29	+	// unmap releases the region; it is safe to call twice and on nil. The
	30	+	// region must not be used afterwards, nor still be written by the output.
	31	+	func (m *mapping) unmap() {
	32	+	if m == nil \|\| m.b == nil {
	33	+	return
	34	+	}
	35	+	if m.mapped {
	36	+	_ = syscall.Munmap(m.b)
	37	+	}
	38	+	m.b = nil
	39	+	*m.live--
	40	+	}

@@ -0,0 +1,40 @@
	1	+	package hints
	2	+
	3	+	import "syscall"
	4	+
	5	+	// mapping is a byte region that lives outside the Go heap: anonymous memory
	6	+	// from mmap, returned to the system by unmap. When mmap fails it falls back
	7	+	// to heap memory, which unmap then just drops.
	8	+	type mapping struct {
	9	+	b []byte
	10	+	mapped bool
	11	+	live *int
	12	+	}
	13	+
	14	+	// alloc returns a zeroed region of n bytes; w.live counts the regions not
	15	+	// yet released.
	16	+	func (w worker) alloc(n int) mapping {
	17	+	m := &mapping{live: &w.live}
	18	+	b, err := syscall.Mmap(-1, 0, n, syscall.PROT_READ\|syscall.PROT_WRITE, syscall.MAP_PRIVATE\|syscall.MAP_ANON)
	19	+	if err != nil {
	20	+	b = make([]byte, n)
	21	+	} else {
	22	+	m.mapped = true
	23	+	}
	24	+	m.b = b
	25	+	w.live++
	26	+	return m
	27	+	}
	28	+
	29	+	// unmap releases the region; it is safe to call twice and on nil. The
	30	+	// region must not be used afterwards, nor still be written by the output.
	31	+	func (m *mapping) unmap() {
	32	+	if m == nil \|\| m.b == nil {
	33	+	return
	34	+	}
	35	+	if m.mapped {
	36	+	_ = syscall.Munmap(m.b)
	37	+	}
	38	+	m.b = nil
	39	+	*m.live--
	40	+	}

M internal/adapters/hints/ocr.go +164 -65


@@ -94,78 +94,134 @@ func parseTSV(data []byte, offset image.Point) ([]OCRWord, error) {
94	94		// the tesseract processes at about 60% of the memory of 8 for the same time.
95	95		var ocrWorkers = 4
96	96
97		-	// pngLevel is the compression of the images piped to tesseract.
	97	+	// pngLevel is the compression of non-grayscale images piped to tesseract.
98	98		var pngLevel = png.BestSpeed
99	99
100		-	// readWords recognizes the words of img, a window crop, in parallel bands.
101		-	func readWords(ctx context.Context, img *image.Gray) ([]OCRWord, int, error) {
102		-	binary, err := exec.LookPath("tesseract")
103		-	if err != nil {
104		-	return nil, 0, err
	100	+	// rawPGM feeds grayscale images to tesseract as binary PGM instead of PNG.
	101	+	var rawPGM = true
	102	+
	103	+	// ocrState is the tesseract installation, resolved once.
	104	+	type ocrState struct {
	105	+	done bool
	106	+	binary string
	107	+	lang string
	108	+	err error
	109	+	}
	110	+
	111	+	func (s *ocrState) resolve(ctx context.Context) error {
	112	+	if !s.done {
	113	+	s.done = true
	114	+	if s.binary, s.err = exec.LookPath("tesseract"); s.err == nil {
	115	+	s.lang, s.err = ocrLanguage(ctx, s.binary)
	116	+	}
	117	+	}
	118	+	return s.err
	119	+	}
	120	+
	121	+	// ocrReady resolves tesseract on the first call. When it is missing, OCR is
	122	+	// disabled for the life of the process and the warning is logged once.
	123	+	func (w *worker) ocrReady(ctx context.Context) bool {
	124	+	if err := w.ocr.resolve(ctx); err != nil {
	125	+	failureWarning.Do(func() {
	126	+	log := logging.For(ctx, "hints")
	127	+	log.Warn().Err(err).Msg("OCR unavailable; install tesseract and tesseract-data-eng")
	128	+	})
	129	+	return false
105	130		}
	131	+	return true
	132	+	}
	133	+
	134	+	// band is a horizontal slice of a window crop recognized by one tesseract
	135	+	// process. It reads rows [top,bottom) and owns the words centred in
	136	+	// [ownTop,ownBottom); the margin only supplies context.
	137	+	type band struct {
	138	+	idx, n int
	139	+	top, bottom int
	140	+	ownTop, ownBottom int
	141	+	}
	142	+
	143	+	func bandsOf(height int) []band {
	144	+	n := min(runtime.GOMAXPROCS(0), ocrWorkers, height)
	145	+	bands := make([]band, 0, max(n, 0))
	146	+	for i := range n {
	147	+	ownTop, ownBottom := iheight/n, (i+1)height/n
	148	+	bands = append(bands, band{idx: i, n: n, top: max(0, ownTop-40), bottom: min(height, ownBottom+40), ownTop: ownTop, ownBottom: ownBottom})
	149	+	}
	150	+	return bands
	151	+	}
	152	+
	153	+	// recognize reads the bands of jobs that missed the cache, in parallel.
	154	+	func (w worker) recognize(ctx context.Context, jobs []bandJob) {
106	155		ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
107	156		defer cancel()
108		-	lang, err := ocrLanguage(ctx, binary)
	157	+	var wg sync.WaitGroup
	158	+	for _, j := range jobs {
	159	+	if j.hit {
	160	+	continue
	161	+	}
	162	+	wg.Go(func() { j.words, j.err = readBand(ctx, &w.ocr, j) })
	163	+	}
	164	+	wg.Wait()
	165	+	}
	166	+
	167	+	// readBand recognizes one band; the words are crop-local and owned by it.
	168	+	func readBand(ctx context.Context, st ocrState, j bandJob) ([]OCRWord, error) {
	169	+	words, err := recognizeBand(ctx, st.binary, st.lang, j.gray)
109	170		if err != nil {
	171	+	return nil, err
	172	+	}
	173	+	heights := make([]int, 0, len(words))
	174	+	for _, word := range words {
	175	+	if len(word.Text) >= 3 {
	176	+	heights = append(heights, word.Rect.Dy())
	177	+	}
	178	+	}
	179	+	sort.Ints(heights)
	180	+	if len(heights) > 0 && heights[len(heights)/2] < 12 {
	181	+	words, err = recognizeBand(ctx, st.binary, st.lang, upscaleBand(j.gray))
	182	+	if err != nil {
	183	+	return nil, err
	184	+	}
	185	+	for i := range words {
	186	+	r := words[i].Rect
	187	+	words[i].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
	188	+	}
	189	+	}
	190	+	owned := words[:0]
	191	+	for _, word := range words {
	192	+	// Overlap supplies context, but each word belongs to exactly one band.
	193	+	word.Rect = word.Rect.Add(image.Pt(0, j.ext.Min.Y))
	194	+	if center := (word.Rect.Min.Y + word.Rect.Max.Y) / 2; center >= j.own[0] && center < j.own[1] {
	195	+	owned = append(owned, word)
	196	+	}
	197	+	}
	198	+	return owned, nil
	199	+	}
	200	+
	201	+	// readWords recognizes the words of img, a window crop, in parallel bands.
	202	+	func readWords(ctx context.Context, img *image.Gray) ([]OCRWord, int, error) {
	203	+	w := &worker{}
	204	+	if err := w.ocr.resolve(ctx); err != nil {
110	205		return nil, 0, err
111	206		}
112		-	n := min(runtime.GOMAXPROCS(0), ocrWorkers, img.Bounds().Dy())
113		-	results := make([][]OCRWord, n)
114		-	errs := make([]error, n)
115		-	var wg sync.WaitGroup
116		-	for i := range n {
117		-	wg.Go(func() {
118		-	top := max(0, i*img.Bounds().Dy()/n-40)
119		-	bottom := min(img.Bounds().Dy(), (i+1)*img.Bounds().Dy()/n+40)
120		-	crop := img.SubImage(image.Rect(0, top, img.Bounds().Dx(), bottom)).(*image.Gray)
121		-	results[i], errs[i] = recognizeBand(ctx, binary, lang, crop)
122		-	if errs[i] != nil {
123		-	return
124		-	}
125		-	heights := make([]int, 0, len(results[i]))
126		-	for _, word := range results[i] {
127		-	if len(word.Text) >= 3 {
128		-	heights = append(heights, word.Rect.Dy())
129		-	}
130		-	}
131		-	sort.Ints(heights)
132		-	if len(heights) > 0 && heights[len(heights)/2] < 12 {
133		-	results[i], errs[i] = recognizeBand(ctx, binary, lang, upscaleBand(crop))
134		-	for j := range results[i] {
135		-	r := results[i][j].Rect
136		-	results[i][j].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
137		-	}
138		-	}
139		-	for j := range results[i] {
140		-	results[i][j].Rect = results[i][j].Rect.Add(image.Pt(0, top))
141		-	}
142		-	})
	207	+	var jobs []*bandJob
	208	+	for _, b := range bandsOf(img.Bounds().Dy()) {
	209	+	ext := image.Rect(0, b.top, img.Bounds().Dx(), b.bottom)
	210	+	jobs = append(jobs, &bandJob{ext: ext, own: [2]int{b.ownTop, b.ownBottom}, gray: img.SubImage(ext).(*image.Gray)})
143	211		}
144		-	wg.Wait()
	212	+	w.recognize(ctx, jobs)
145	213		var words []OCRWord
146		-	for i := range n {
147		-	if errs[i] != nil {
148		-	return nil, n, errs[i]
	214	+	for _, j := range jobs {
	215	+	if j.err != nil {
	216	+	return nil, len(jobs), j.err
149	217		}
150		-	for _, word := range results[i] {
151		-	// Overlap supplies context, but each word belongs to exactly one band.
152		-	center := (word.Rect.Min.Y + word.Rect.Max.Y) / 2
153		-	if center < iimg.Bounds().Dy()/n \|\| center >= (i+1)img.Bounds().Dy()/n {
154		-	continue
155		-	}
156		-	duplicate := false
157		-	for _, old := range words {
158		-	if strings.EqualFold(word.Text, old.Text) && overlap(word.Rect, old.Rect) > .5 {
159		-	duplicate = true
160		-	break
161		-	}
162		-	}
163		-	if !duplicate {
	218	+	for _, word := range j.words {
	219	+	if !duplicateWord(words, word) {
164	220		words = append(words, word)
165	221		}
166	222		}
167	223		}
168		-	return words, n, nil
	224	+	return words, len(jobs), nil
169	225		}
170	226
171	227		// The link patterns compile on the first OCR link check, not at process

@@ -279,22 +335,65 @@ func recognizeBand(ctx context.Context, binary, lang string, img image.Image) ([
279	335		return recognizeImage(ctx, binary, lang, img, "11", false)
280	336		}
281	337
	338	+	// pgmReader streams a grayscale image as binary PGM (P5), row by row.
	339	+	type pgmReader struct {
	340	+	head []byte
	341	+	img *image.Gray
	342	+	y int
	343	+	row []byte
	344	+	}
	345	+
	346	+	func newPGMReader(img image.Gray) pgmReader {
	347	+	b := img.Bounds()
	348	+	return &pgmReader{head: fmt.Appendf(nil, "P5\n%d %d\n255\n", b.Dx(), b.Dy()), img: img}
	349	+	}
	350	+
	351	+	func (r *pgmReader) Read(p []byte) (int, error) {
	352	+	n := 0
	353	+	for n < len(p) {
	354	+	if len(r.head) > 0 {
	355	+	c := copy(p[n:], r.head)
	356	+	r.head, n = r.head[c:], n+c
	357	+	continue
	358	+	}
	359	+	if len(r.row) == 0 {
	360	+	b := r.img.Bounds()
	361	+	if r.y >= b.Dy() {
	362	+	break
	363	+	}
	364	+	r.row = r.img.Pix[(r.y+b.Min.Y-r.img.Rect.Min.Y)*r.img.Stride+(b.Min.X-r.img.Rect.Min.X):][:b.Dx()]
	365	+	r.y++
	366	+	}
	367	+	c := copy(p[n:], r.row)
	368	+	r.row, n = r.row[c:], n+c
	369	+	}
	370	+	if n == 0 {
	371	+	return 0, io.EOF
	372	+	}
	373	+	return n, nil
	374	+	}
	375	+
282	376		func recognizeImage(ctx context.Context, binary, lang string, img image.Image, psm string, normalized bool) ([]OCRWord, error) {
283	377		args := []string{"stdin", "stdout", "--psm", psm, "-l", lang}
284	378		if normalized {
285	379		args = append(args, "-c", "tessedit_do_invert=0")
286	380		}
287	381		args = append(args, "tsv")
288		-	// The PNG streams into tesseract; no encoded copy is held.
289		-	pr, pw := io.Pipe()
290		-	defer func() { _ = pr.Close() }()
291		-	go func() {
292		-	enc := png.Encoder{CompressionLevel: pngLevel}
293		-	_ = pw.CloseWithError(enc.Encode(pw, img))
294		-	}()
295	382		cmd := exec.CommandContext(ctx, binary, args...)
296	383		cmd.Env = append(os.Environ(), "OMP_THREAD_LIMIT=1")
297		-	cmd.Stdin = pr
	384	+	if g, ok := img.(*image.Gray); ok && rawPGM {
	385	+	// Raw grayscale: no deflate, no encoder buffers.
	386	+	cmd.Stdin = newPGMReader(g)
	387	+	} else {
	388	+	// The PNG streams into tesseract; no encoded copy is held.
	389	+	pr, pw := io.Pipe()
	390	+	defer func() { _ = pr.Close() }()
	391	+	go func() {
	392	+	enc := png.Encoder{CompressionLevel: pngLevel}
	393	+	_ = pw.CloseWithError(enc.Encode(pw, img))
	394	+	}()
	395	+	cmd.Stdin = pr
	396	+	}
298	397		var stderr bytes.Buffer
299	398		cmd.Stderr = &stderr
300	399		output, err := cmd.Output()


@@ -94,78 +94,134 @@ func parseTSV(data []byte, offset image.Point) ([]OCRWord, error) {
94		// the tesseract processes at about 60% of the memory of 8 for the same time.	94		// the tesseract processes at about 60% of the memory of 8 for the same time.
95		var ocrWorkers = 4	95		var ocrWorkers = 4
96			96
97	-	// pngLevel is the compression of the images piped to tesseract.	97	+	// pngLevel is the compression of non-grayscale images piped to tesseract.
98		var pngLevel = png.BestSpeed	98		var pngLevel = png.BestSpeed
99			99
100	-	// readWords recognizes the words of img, a window crop, in parallel bands.	100	+	// rawPGM feeds grayscale images to tesseract as binary PGM instead of PNG.
101	-	func readWords(ctx context.Context, img *image.Gray) ([]OCRWord, int, error) {	101	+	var rawPGM = true
102	-	binary, err := exec.LookPath("tesseract")	102	+
103	-	if err != nil {	103	+	// ocrState is the tesseract installation, resolved once.
104	-	return nil, 0, err	104	+	type ocrState struct {
			105	+	done bool
			106	+	binary string
			107	+	lang string
			108	+	err error
			109	+	}
			110	+
			111	+	func (s *ocrState) resolve(ctx context.Context) error {
			112	+	if !s.done {
			113	+	s.done = true
			114	+	if s.binary, s.err = exec.LookPath("tesseract"); s.err == nil {
			115	+	s.lang, s.err = ocrLanguage(ctx, s.binary)
			116	+	}
			117	+	}
			118	+	return s.err
			119	+	}
			120	+
			121	+	// ocrReady resolves tesseract on the first call. When it is missing, OCR is
			122	+	// disabled for the life of the process and the warning is logged once.
			123	+	func (w *worker) ocrReady(ctx context.Context) bool {
			124	+	if err := w.ocr.resolve(ctx); err != nil {
			125	+	failureWarning.Do(func() {
			126	+	log := logging.For(ctx, "hints")
			127	+	log.Warn().Err(err).Msg("OCR unavailable; install tesseract and tesseract-data-eng")
			128	+	})
			129	+	return false
105		}	130		}
			131	+	return true
			132	+	}
			133	+
			134	+	// band is a horizontal slice of a window crop recognized by one tesseract
			135	+	// process. It reads rows [top,bottom) and owns the words centred in
			136	+	// [ownTop,ownBottom); the margin only supplies context.
			137	+	type band struct {
			138	+	idx, n int
			139	+	top, bottom int
			140	+	ownTop, ownBottom int
			141	+	}
			142	+
			143	+	func bandsOf(height int) []band {
			144	+	n := min(runtime.GOMAXPROCS(0), ocrWorkers, height)
			145	+	bands := make([]band, 0, max(n, 0))
			146	+	for i := range n {
			147	+	ownTop, ownBottom := iheight/n, (i+1)height/n
			148	+	bands = append(bands, band{idx: i, n: n, top: max(0, ownTop-40), bottom: min(height, ownBottom+40), ownTop: ownTop, ownBottom: ownBottom})
			149	+	}
			150	+	return bands
			151	+	}
			152	+
			153	+	// recognize reads the bands of jobs that missed the cache, in parallel.
			154	+	func (w worker) recognize(ctx context.Context, jobs []bandJob) {
106		ctx, cancel := context.WithTimeout(ctx, 15*time.Second)	155		ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
107		defer cancel()	156		defer cancel()
108	-	lang, err := ocrLanguage(ctx, binary)	157	+	var wg sync.WaitGroup
			158	+	for _, j := range jobs {
			159	+	if j.hit {
			160	+	continue
			161	+	}
			162	+	wg.Go(func() { j.words, j.err = readBand(ctx, &w.ocr, j) })
			163	+	}
			164	+	wg.Wait()
			165	+	}
			166	+
			167	+	// readBand recognizes one band; the words are crop-local and owned by it.
			168	+	func readBand(ctx context.Context, st ocrState, j bandJob) ([]OCRWord, error) {
			169	+	words, err := recognizeBand(ctx, st.binary, st.lang, j.gray)
109		if err != nil {	170		if err != nil {
			171	+	return nil, err
			172	+	}
			173	+	heights := make([]int, 0, len(words))
			174	+	for _, word := range words {
			175	+	if len(word.Text) >= 3 {
			176	+	heights = append(heights, word.Rect.Dy())
			177	+	}
			178	+	}
			179	+	sort.Ints(heights)
			180	+	if len(heights) > 0 && heights[len(heights)/2] < 12 {
			181	+	words, err = recognizeBand(ctx, st.binary, st.lang, upscaleBand(j.gray))
			182	+	if err != nil {
			183	+	return nil, err
			184	+	}
			185	+	for i := range words {
			186	+	r := words[i].Rect
			187	+	words[i].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
			188	+	}
			189	+	}
			190	+	owned := words[:0]
			191	+	for _, word := range words {
			192	+	// Overlap supplies context, but each word belongs to exactly one band.
			193	+	word.Rect = word.Rect.Add(image.Pt(0, j.ext.Min.Y))
			194	+	if center := (word.Rect.Min.Y + word.Rect.Max.Y) / 2; center >= j.own[0] && center < j.own[1] {
			195	+	owned = append(owned, word)
			196	+	}
			197	+	}
			198	+	return owned, nil
			199	+	}
			200	+
			201	+	// readWords recognizes the words of img, a window crop, in parallel bands.
			202	+	func readWords(ctx context.Context, img *image.Gray) ([]OCRWord, int, error) {
			203	+	w := &worker{}
			204	+	if err := w.ocr.resolve(ctx); err != nil {
110		return nil, 0, err	205		return nil, 0, err
111		}	206		}
112	-	n := min(runtime.GOMAXPROCS(0), ocrWorkers, img.Bounds().Dy())	207	+	var jobs []*bandJob
113	-	results := make([][]OCRWord, n)	208	+	for _, b := range bandsOf(img.Bounds().Dy()) {
114	-	errs := make([]error, n)	209	+	ext := image.Rect(0, b.top, img.Bounds().Dx(), b.bottom)
115	-	var wg sync.WaitGroup	210	+	jobs = append(jobs, &bandJob{ext: ext, own: [2]int{b.ownTop, b.ownBottom}, gray: img.SubImage(ext).(*image.Gray)})
116	-	for i := range n {
117	-	wg.Go(func() {
118	-	top := max(0, i*img.Bounds().Dy()/n-40)
119	-	bottom := min(img.Bounds().Dy(), (i+1)*img.Bounds().Dy()/n+40)
120	-	crop := img.SubImage(image.Rect(0, top, img.Bounds().Dx(), bottom)).(*image.Gray)
121	-	results[i], errs[i] = recognizeBand(ctx, binary, lang, crop)
122	-	if errs[i] != nil {
123	-	return
124	-	}
125	-	heights := make([]int, 0, len(results[i]))
126	-	for _, word := range results[i] {
127	-	if len(word.Text) >= 3 {
128	-	heights = append(heights, word.Rect.Dy())
129	-	}
130	-	}
131	-	sort.Ints(heights)
132	-	if len(heights) > 0 && heights[len(heights)/2] < 12 {
133	-	results[i], errs[i] = recognizeBand(ctx, binary, lang, upscaleBand(crop))
134	-	for j := range results[i] {
135	-	r := results[i][j].Rect
136	-	results[i][j].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
137	-	}
138	-	}
139	-	for j := range results[i] {
140	-	results[i][j].Rect = results[i][j].Rect.Add(image.Pt(0, top))
141	-	}
142	-	})
143		}	211		}
144	-	wg.Wait()	212	+	w.recognize(ctx, jobs)
145		var words []OCRWord	213		var words []OCRWord
146	-	for i := range n {	214	+	for _, j := range jobs {
147	-	if errs[i] != nil {	215	+	if j.err != nil {
148	-	return nil, n, errs[i]	216	+	return nil, len(jobs), j.err
149		}	217		}
150	-	for _, word := range results[i] {	218	+	for _, word := range j.words {
151	-	// Overlap supplies context, but each word belongs to exactly one band.	219	+	if !duplicateWord(words, word) {
152	-	center := (word.Rect.Min.Y + word.Rect.Max.Y) / 2
153	-	if center < iimg.Bounds().Dy()/n \|\| center >= (i+1)img.Bounds().Dy()/n {
154	-	continue
155	-	}
156	-	duplicate := false
157	-	for _, old := range words {
158	-	if strings.EqualFold(word.Text, old.Text) && overlap(word.Rect, old.Rect) > .5 {
159	-	duplicate = true
160	-	break
161	-	}
162	-	}
163	-	if !duplicate {
164		words = append(words, word)	220		words = append(words, word)
165		}	221		}
166		}	222		}
167		}	223		}
168	-	return words, n, nil	224	+	return words, len(jobs), nil
169		}	225		}
170			226
171		// The link patterns compile on the first OCR link check, not at process	227		// The link patterns compile on the first OCR link check, not at process

@@ -279,22 +335,65 @@ func recognizeBand(ctx context.Context, binary, lang string, img image.Image) ([
279		return recognizeImage(ctx, binary, lang, img, "11", false)	335		return recognizeImage(ctx, binary, lang, img, "11", false)
280		}	336		}
281			337
			338	+	// pgmReader streams a grayscale image as binary PGM (P5), row by row.
			339	+	type pgmReader struct {
			340	+	head []byte
			341	+	img *image.Gray
			342	+	y int
			343	+	row []byte
			344	+	}
			345	+
			346	+	func newPGMReader(img image.Gray) pgmReader {
			347	+	b := img.Bounds()
			348	+	return &pgmReader{head: fmt.Appendf(nil, "P5\n%d %d\n255\n", b.Dx(), b.Dy()), img: img}
			349	+	}
			350	+
			351	+	func (r *pgmReader) Read(p []byte) (int, error) {
			352	+	n := 0
			353	+	for n < len(p) {
			354	+	if len(r.head) > 0 {
			355	+	c := copy(p[n:], r.head)
			356	+	r.head, n = r.head[c:], n+c
			357	+	continue
			358	+	}
			359	+	if len(r.row) == 0 {
			360	+	b := r.img.Bounds()
			361	+	if r.y >= b.Dy() {
			362	+	break
			363	+	}
			364	+	r.row = r.img.Pix[(r.y+b.Min.Y-r.img.Rect.Min.Y)*r.img.Stride+(b.Min.X-r.img.Rect.Min.X):][:b.Dx()]
			365	+	r.y++
			366	+	}
			367	+	c := copy(p[n:], r.row)
			368	+	r.row, n = r.row[c:], n+c
			369	+	}
			370	+	if n == 0 {
			371	+	return 0, io.EOF
			372	+	}
			373	+	return n, nil
			374	+	}
			375	+
282		func recognizeImage(ctx context.Context, binary, lang string, img image.Image, psm string, normalized bool) ([]OCRWord, error) {	376		func recognizeImage(ctx context.Context, binary, lang string, img image.Image, psm string, normalized bool) ([]OCRWord, error) {
283		args := []string{"stdin", "stdout", "--psm", psm, "-l", lang}	377		args := []string{"stdin", "stdout", "--psm", psm, "-l", lang}
284		if normalized {	378		if normalized {
285		args = append(args, "-c", "tessedit_do_invert=0")	379		args = append(args, "-c", "tessedit_do_invert=0")
286		}	380		}
287		args = append(args, "tsv")	381		args = append(args, "tsv")
288	-	// The PNG streams into tesseract; no encoded copy is held.
289	-	pr, pw := io.Pipe()
290	-	defer func() { _ = pr.Close() }()
291	-	go func() {
292	-	enc := png.Encoder{CompressionLevel: pngLevel}
293	-	_ = pw.CloseWithError(enc.Encode(pw, img))
294	-	}()
295		cmd := exec.CommandContext(ctx, binary, args...)	382		cmd := exec.CommandContext(ctx, binary, args...)
296		cmd.Env = append(os.Environ(), "OMP_THREAD_LIMIT=1")	383		cmd.Env = append(os.Environ(), "OMP_THREAD_LIMIT=1")
297	-	cmd.Stdin = pr	384	+	if g, ok := img.(*image.Gray); ok && rawPGM {
			385	+	// Raw grayscale: no deflate, no encoder buffers.
			386	+	cmd.Stdin = newPGMReader(g)
			387	+	} else {
			388	+	// The PNG streams into tesseract; no encoded copy is held.
			389	+	pr, pw := io.Pipe()
			390	+	defer func() { _ = pr.Close() }()
			391	+	go func() {
			392	+	enc := png.Encoder{CompressionLevel: pngLevel}
			393	+	_ = pw.CloseWithError(enc.Encode(pw, img))
			394	+	}()
			395	+	cmd.Stdin = pr
			396	+	}
298		var stderr bytes.Buffer	397		var stderr bytes.Buffer
299		cmd.Stderr = &stderr	398		cmd.Stderr = &stderr
300		output, err := cmd.Output()	399		output, err := cmd.Output()

M internal/adapters/hints/ocr_test.go +52 -0


@@ -5,8 +5,10 @@ import (
5	5		"image"
6	6		"image/draw"
7	7		"image/png"
	8	+	"io"
8	9		"os"
9	10		"os/exec"
	11	+	"reflect"
10	12		"strings"
11	13		"testing"
12	14		"time"

@@ -139,3 +141,53 @@ func TestSmallGlyphOCR(t *testing.T) {
139	141		t.Logf("glyph experiment %dx: %.1f ms, words=%d links=%d", factor, ms(time.Since(start)), len(words), len(textLinks(words, 1)))
140	142		}
141	143		}
	144	+
	145	+	func TestPGMReader(t *testing.T) {
	146	+	g := image.NewGray(image.Rect(0, 0, 3, 3))
	147	+	for i := range g.Pix {
	148	+	g.Pix[i] = uint8(i + 1)
	149	+	}
	150	+	sub := g.SubImage(image.Rect(1, 1, 3, 3)).(*image.Gray)
	151	+	got, err := io.ReadAll(newPGMReader(sub))
	152	+	if err != nil {
	153	+	t.Fatal(err)
	154	+	}
	155	+	if want := "P5\n2 2\n255\n" + string([]byte{5, 6, 8, 9}); string(got) != want {
	156	+	t.Fatalf("%q", got)
	157	+	}
	158	+	}
	159	+
	160	+	// Tesseract must read the raw PGM exactly as it reads the PNG.
	161	+	func TestPGMMatchesPNG(t *testing.T) {
	162	+	binary, err := exec.LookPath("tesseract")
	163	+	if err != nil {
	164	+	t.Skip("tesseract not on PATH")
	165	+	}
	166	+	f, err := os.Open("testdata/terminal.png")
	167	+	if err != nil {
	168	+	t.Fatal(err)
	169	+	}
	170	+	defer f.Close()
	171	+	src, err := png.Decode(f)
	172	+	if err != nil {
	173	+	t.Fatal(err)
	174	+	}
	175	+	rgba := image.NewRGBA(src.Bounds())
	176	+	draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
	177	+	gray := toGray(rgba)
	178	+	lang, err := ocrLanguage(context.Background(), binary)
	179	+	if err != nil {
	180	+	t.Fatal(err)
	181	+	}
	182	+	defer func(v bool) { rawPGM = v }(rawPGM)
	183	+	var res [2][]OCRWord
	184	+	for i, raw := range []bool{false, true} {
	185	+	rawPGM = raw
	186	+	if res[i], err = recognizeBand(context.Background(), binary, lang, gray); err != nil {
	187	+	t.Fatal(err)
	188	+	}
	189	+	}
	190	+	if len(res[0]) == 0 \|\| !reflect.DeepEqual(res[0], res[1]) {
	191	+	t.Fatalf("PNG %d words, PGM %d words", len(res[0]), len(res[1]))
	192	+	}
	193	+	}


@@ -5,8 +5,10 @@ import (
5	"image"	5		"image"
6	"image/draw"	6		"image/draw"
7	"image/png"	7		"image/png"
		8	+	"io"
8	"os"	9		"os"
9	"os/exec"	10		"os/exec"
		11	+	"reflect"
10	"strings"	12		"strings"
11	"testing"	13		"testing"
12	"time"	14		"time"

@@ -139,3 +141,53 @@ func TestSmallGlyphOCR(t *testing.T) {
139	t.Logf("glyph experiment %dx: %.1f ms, words=%d links=%d", factor, ms(time.Since(start)), len(words), len(textLinks(words, 1)))	141		t.Logf("glyph experiment %dx: %.1f ms, words=%d links=%d", factor, ms(time.Since(start)), len(words), len(textLinks(words, 1)))
140	}	142		}
141	}	143		}
		144	+
		145	+	func TestPGMReader(t *testing.T) {
		146	+	g := image.NewGray(image.Rect(0, 0, 3, 3))
		147	+	for i := range g.Pix {
		148	+	g.Pix[i] = uint8(i + 1)
		149	+	}
		150	+	sub := g.SubImage(image.Rect(1, 1, 3, 3)).(*image.Gray)
		151	+	got, err := io.ReadAll(newPGMReader(sub))
		152	+	if err != nil {
		153	+	t.Fatal(err)
		154	+	}
		155	+	if want := "P5\n2 2\n255\n" + string([]byte{5, 6, 8, 9}); string(got) != want {
		156	+	t.Fatalf("%q", got)
		157	+	}
		158	+	}
		159	+
		160	+	// Tesseract must read the raw PGM exactly as it reads the PNG.
		161	+	func TestPGMMatchesPNG(t *testing.T) {
		162	+	binary, err := exec.LookPath("tesseract")
		163	+	if err != nil {
		164	+	t.Skip("tesseract not on PATH")
		165	+	}
		166	+	f, err := os.Open("testdata/terminal.png")
		167	+	if err != nil {
		168	+	t.Fatal(err)
		169	+	}
		170	+	defer f.Close()
		171	+	src, err := png.Decode(f)
		172	+	if err != nil {
		173	+	t.Fatal(err)
		174	+	}
		175	+	rgba := image.NewRGBA(src.Bounds())
		176	+	draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
		177	+	gray := toGray(rgba)
		178	+	lang, err := ocrLanguage(context.Background(), binary)
		179	+	if err != nil {
		180	+	t.Fatal(err)
		181	+	}
		182	+	defer func(v bool) { rawPGM = v }(rawPGM)
		183	+	var res [2][]OCRWord
		184	+	for i, raw := range []bool{false, true} {
		185	+	rawPGM = raw
		186	+	if res[i], err = recognizeBand(context.Background(), binary, lang, gray); err != nil {
		187	+	t.Fatal(err)
		188	+	}
		189	+	}
		190	+	if len(res[0]) == 0 \|\| !reflect.DeepEqual(res[0], res[1]) {
		191	+	t.Fatalf("PNG %d words, PGM %d words", len(res[0]), len(res[1]))
		192	+	}
		193	+	}

A internal/adapters/hints/scan_bench_test.go +49 -0

@@ -0,0 +1,49 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"os"
	6	+	"os/exec"
	7	+	"runtime"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/neferwl/internal/ports"
	12	+	)
	13	+
	14	+	// BenchmarkScan runs whole scans of a 5120x2160 frame with a noisy window
	15	+	// (the cache never hits) and reports the Go heap bytes allocated per scan.
	16	+	func BenchmarkScan(b *testing.B) {
	17	+	if _, err := exec.LookPath("tesseract"); err != nil {
	18	+	b.Skip("tesseract not on PATH")
	19	+	}
	20	+	const w, h = 5120, 2160
	21	+	ctx := context.Background()
	22	+	w0 := &worker{}
	23	+	requests := make(chan ports.HintScanRequest)
	24	+	captures := make(chan ports.CaptureRequest, 1)
	25	+	captured := make(chan ports.CaptureDone, 1)
	26	+	results := make(chan ports.HintScanResult, 1)
	27	+	req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
	28	+	if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
	29	+	rawPGM = false
	30	+	}
	31	+	b.ReportAllocs()
	32	+	var peak uint64
	33	+	for i := range b.N {
	34	+	go func() {
	35	+	creq := <-captures
	36	+	creq.Mapped[0] = byte(i) // each scan differs: no cache hit
	37	+	creq.Mapped[1] = byte(i >> 8)
	38	+	captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	39	+	}()
	40	+	res := w0.serve(ctx, req, requests, captures, captured, results, 5*time.Second)
	41	+	if res.Err != nil {
	42	+	b.Fatal(res.Err)
	43	+	}
	44	+	var m runtime.MemStats
	45	+	runtime.ReadMemStats(&m)
	46	+	peak = max(peak, m.HeapSys-m.HeapReleased)
	47	+	}
	48	+	b.ReportMetric(float64(peak)/1e6, "heapMB")
	49	+	}

@@ -0,0 +1,49 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"os"
	6	+	"os/exec"
	7	+	"runtime"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/neferwl/internal/ports"
	12	+	)
	13	+
	14	+	// BenchmarkScan runs whole scans of a 5120x2160 frame with a noisy window
	15	+	// (the cache never hits) and reports the Go heap bytes allocated per scan.
	16	+	func BenchmarkScan(b *testing.B) {
	17	+	if _, err := exec.LookPath("tesseract"); err != nil {
	18	+	b.Skip("tesseract not on PATH")
	19	+	}
	20	+	const w, h = 5120, 2160
	21	+	ctx := context.Background()
	22	+	w0 := &worker{}
	23	+	requests := make(chan ports.HintScanRequest)
	24	+	captures := make(chan ports.CaptureRequest, 1)
	25	+	captured := make(chan ports.CaptureDone, 1)
	26	+	results := make(chan ports.HintScanResult, 1)
	27	+	req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
	28	+	if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
	29	+	rawPGM = false
	30	+	}
	31	+	b.ReportAllocs()
	32	+	var peak uint64
	33	+	for i := range b.N {
	34	+	go func() {
	35	+	creq := <-captures
	36	+	creq.Mapped[0] = byte(i) // each scan differs: no cache hit
	37	+	creq.Mapped[1] = byte(i >> 8)
	38	+	captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	39	+	}()
	40	+	res := w0.serve(ctx, req, requests, captures, captured, results, 5*time.Second)
	41	+	if res.Err != nil {
	42	+	b.Fatal(res.Err)
	43	+	}
	44	+	var m runtime.MemStats
	45	+	runtime.ReadMemStats(&m)
	46	+	peak = max(peak, m.HeapSys-m.HeapReleased)
	47	+	}
	48	+	b.ReportMetric(float64(peak)/1e6, "heapMB")
	49	+	}