Merge 68ee891ede655ecbcff92bd9c049b2795df153b3 into f2d166b7fd4fa830ec06f487fcdcc64f1eacb4c9

commit: 9de93ac1c0bcc822f01e06eb23b171b24252681a
author: ermite · 13h ago · 2026-10-22 20:29 UTC
merge of: f2d166b + 68ee891 · diff against the first parent
signature: unsigned

50 files · +2122 -4856

M .mockery.yml +2 -19


@@ -138,28 +138,11 @@ packages:
138		filename: proc_tree_mock_test.go	138		filename: proc_tree_mock_test.go
139		github.com/bnema/neferwl/internal/adapters/hints:	139		github.com/bnema/neferwl/internal/adapters/hints:
140		interfaces:	140		interfaces:
141	-	languageRunner:	141	+	detector:
142		config:	142		config:
143		dir: internal/adapters/hints	143		dir: internal/adapters/hints
144		pkgname: hints	144		pkgname: hints
145	-	filename: language_runner_mock_test.go	145	+	filename: detector_mock_test.go
146	-	targetSource:
147	-	config:
148	-	dir: internal/adapters/hints
149	-	pkgname: hints
150	-	filename: target_source_mock_test.go
151	-	github.com/bnema/neferwl/internal/adapters/atspi:
152	-	interfaces:
153	-	transport:
154	-	config:
155	-	dir: internal/adapters/atspi
156	-	pkgname: atspi
157	-	filename: transport_mock_test.go
158	-	connector:
159	-	config:
160	-	dir: internal/adapters/atspi
161	-	pkgname: atspi
162	-	filename: connector_mock_test.go
163		github.com/bnema/neferwl/internal/adapters/screencast:	146		github.com/bnema/neferwl/internal/adapters/screencast:
164		interfaces:	147		interfaces:
165		bus:	148		bus:

M README.md +1 -1


@@ -21,7 +21,7 @@ https://github.com/user-attachments/assets/1ff42ce5-47b3-4cf8-a3a7-5fc5abde4af1
21		- Games: direct scanout, VRR, tearing, explicit sync and [HDR](docs/config.md#hdr) give games the whole GPU instead of spending it on composition; see [Performance](docs/performance.md).	21		- Games: direct scanout, VRR, tearing, explicit sync and [HDR](docs/config.md#hdr) give games the whole GPU instead of spending it on composition; see [Performance](docs/performance.md).
22		- Overview: `cmd+o` or a four-finger swipe up shows live workspace previews. Select a window with the keyboard, touchpad or mouse; see [Overview](docs/config.md#overview).	22		- Overview: `cmd+o` or a four-finger swipe up shows live workspace previews. Select a window with the keyboard, touchpad or mouse; see [Overview](docs/config.md#overview).
23		- Stash: `cmd+s` toggles a floating strip of windows set aside for the current workspace; `cmd+shift+s` moves a window into or out of it. See [Stash](docs/config.md#stash).	23		- Stash: `cmd+s` toggles a floating strip of windows set aside for the current workspace; `cmd+shift+s` moves a window into or out of it. See [Stash](docs/config.md#stash).
24	-	- Hint mode: click links and buttons by typing their labels, without reaching for the mouse. Enable experimental `hint.enable` and add a bind; see [Hint mode](docs/config.md#hint-mode).	24	+	- Hint mode: click what is on screen by typing its label, without reaching for the mouse; targets come from the `nefer-hints` detector. Enable experimental `hint.enable` and add a bind; see [Hint mode](docs/config.md#hint-mode).
25		- Animations: short, cheap and off by default. Opt in with experimental `animations = on`; see [Configuration](docs/config.md#keys).	25		- Animations: short, cheap and off by default. Opt in with experimental `animations = on`; see [Configuration](docs/config.md#keys).
26			26
27		Written in Go, without cgo. Each piece of state has one owner goroutine, connected to the others by channels; hot paths reuse memory and client buffers stay off the Go heap. See [Architecture](docs/dev/architecture.md) for the concurrency model, the GC strategy and the reusable Go libraries.	27		Written in Go, without cgo. Each piece of state has one owner goroutine, connected to the others by channels; hot paths reuse memory and client buffers stay off the Go heap. See [Architecture](docs/dev/architecture.md) for the concurrency model, the GC strategy and the reusable Go libraries.

M docs/config.md +23 -28

 | `focus.flash-duration` | `200ms` | Fade duration, `50ms` to `1s`; not scaled by `animations.speed` |
 | `focus.flash-color` | `border.active` | Flash color `#rrggbb`; empty uses `border.active` |
 | `hint.enable` | `off` | Turn on the experimental [hint mode](#hint-mode) (restart required) |
-| `hint.targets` | `clickable` | What [hint mode](#hint-mode) labels: `clickable` (links, URLs and buttons) or `links` (links and URLs only) |
+| `hint.detector` | `nefer-hints` | The [hint mode](#hint-mode) detector binary: a path, or a name looked up in `PATH` (restart required) |
-| `hint.text-click` | `ctrl+shift` | Modifiers held for clicks on links found by OCR (terminals open links on a modified click); `none` sends a plain click |
+| `hint.model` | `$XDG_DATA_HOME/nefer-hints/models` | Directory of the detector's model files; `~/.local/share/nefer-hints/models` when `XDG_DATA_HOME` is unset; a leading `~/` is expanded (restart required) |
+| `hint.threshold` | model default | Optional minimum detection score override, 0.01 to 0.99; unset or empty uses the model’s own threshold (`model.json`), else the runner default. Lower finds more targets and more false ones (restart required) |
 | `hint.capture` | `off` | Show hint labels in captures requested by Wayland clients (screenshots, screencasts); the compositor's own hint scans never see them; applies on reload |
 | `hint.size` | `auto` | Label text size in logical pixels (6 to 72), or `auto` for screen pixel density |
 | `hint.background` | empty | Label background (`#rrggbb`); empty uses `border.active` |
 | `toggle-window-stash` | Move the focused window to the [stash](#stash), or back to its column |
 | `toggle-stash-visible` | Show or hide the stash |
 | `toggle-overview` | Open or close the [overview](#overview) |
-| `hint` | Label the links and buttons on the focused screen, then click the one you type ([hint mode](#hint-mode)) |
+| `hint` | Label the clickable elements on the focused screen, then click the one you type ([hint mode](#hint-mode)) |
 | `switch-column-next` / `-prev` | Switch to the previously used column. Hold `cmd` and press again to go further back |
 | `maximize-column` | Toggle full width for the focused column |
 | `cycle-column-width` | Grow the focused column by one [cell](#column-widths), then back to equal widths |
 While it is off, nothing of hint mode runs or uses memory, and the `hint` bind
 logs a notice once and does nothing. A reload does not change it.
-`hint` labels the links and buttons on the focused output. Type a label to
+`hint` labels the clickable elements (buttons, links, icons, fields) on the
-click its target; BackSpace erases a letter, Escape cancels. No bind is set
+focused output. Type a label to click its target; BackSpace erases a letter,
-by default; for example `bind.cmd+g = hint`.
+Escape cancels. No bind is set by default; for example `bind.cmd+g = hint`.
 Labels use the `border.active` colour with a contrasting text colour, and
 their size follows the screen's pixel density. Each label sits just outside
-the top-left corner of its target. Links found by OCR (terminal text) are
+the top-left corner of its target. The click is a plain left click.
-clicked with the `hint.text-click` modifiers held, which is how terminals open
-a link; accessibility targets get a plain click.
 Labels are not part of screenshots and screencasts by default. With
 `hint.capture = on` they appear in the captures that Wayland clients request
-(including Exclude captures); NeferWL's own text recognition never sees them.
+(including Exclude captures); the detector never sees them.
 The capture indicator and the position hint stay out of every capture.
 A reload applies `hint.capture` to the next frame.
 ### Where targets come from
-| Source | Apps | Accuracy | Speed |
+Targets come from `nefer-hints`, a separate program that looks at the pixels of
-|---|---|---|---|
+the whole output with a neural network and returns a box for each clickable
-| Accessibility tree (AT-SPI) | Firefox, Chromium, GTK, Qt, Electron | Exact: the app reports its own links and buttons | A few milliseconds |
+element. It works the same for every app, toolkit and terminal. It sees what is
-| Text recognition (OCR) | Everything else: terminals, apps without accessibility | Best effort: URLs, domains and underlined text written on screen | 150 ms or more, more on large screens |
+drawn: it does not know what an element does, and it can miss or invent boxes
+(`hint.threshold` trades one for the other). Boxes under 4×4 logical pixels
+are dropped.
-NeferWL asks the accessibility tree first. Windows it does not cover are read
+NeferWL starts `nefer-hints` at the first hint scan (loading takes about half a
-with OCR, which finds visible URLs, domains, email addresses and underlined
+second), keeps it running, and stops it on exit. If it dies, fails or takes
-text. Underlined words get a plain click, and the words of one continuous
+more than 2 seconds on a scan, that scan is cancelled and a new process starts
-underline form one target; URLs keep the `hint.text-click` modifiers. OCR
+at the next one. Its device (GPU or CPU) is logged at startup, at info level.
-cannot see a link whose address is hidden behind plain text, and it can misread
-small or low-contrast text.
 ### Requirements
-- Accessibility: NeferWL enables it lazily on the first hint scan that queries
+- The `nefer-hints` binary, found in `PATH` or named by `hint.detector`.
-  app windows. Apps started before then may need restarting. Some apps need a flag:
+- Its model files, in `hint.model`.
-  - Chromium: `--force-renderer-accessibility`
+- A Vulkan GPU for speed; without one the detector falls back to the CPU, which is slower.
-  - Electron: `ACCESSIBILITY_ENABLED=1`
-  - Qt: `QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1`
-- OCR: `tesseract` and `tesseract-data-eng`. Without tesseract, OCR is off
-  and hint mode labels only apps that expose the accessibility tree.
 ### Debugging hint mode
-With `hint.enable = on` and `NEFERWL_HINT_DEBUG=1` set when NeferWL starts (for example `NEFERWL_HINT_DEBUG=1 make tty`, or the variable in the launch environment), every successful scan writes three files to `$XDG_STATE_HOME/neferwl/hints/` (`~/.local/state/neferwl/hints/` when unset): `<time>-<output>.png`, the frame as scanned; `<time>-<output>.annotated.png`, the same with each target outlined (OCR links green, accessibility targets magenta, OCR window crops grey); and `<time>-<output>.json`, the targets (with their `source`, `atspi` or `ocr`) and timings. NeferWL keeps the 50 most recent scans and logs a warning at startup. The files contain the screen in clear: delete them after use.
+With `hint.enable = on` and `NEFERWL_HINT_DEBUG=1` set when NeferWL starts (for example `NEFERWL_HINT_DEBUG=1 make tty`, or the variable in the launch environment), every successful scan writes three files to `$XDG_STATE_HOME/neferwl/hints/` (`~/.local/state/neferwl/hints/` when unset): `<time>-<output>.png`, the frame as scanned; `<time>-<output>.annotated.png`, the same with the detector boxes outlined; and `<time>-<output>.json`, the targets (rectangle and score) and the capture, detection and total timings. NeferWL keeps the 50 most recent scans and logs a warning at startup. The detector's own messages are logged at debug level (`log.debug = hints`). The files contain the screen in clear: delete them after use.

M examples/config +6 -3

 focus.flash-color =
 # Hint mode is experimental and off: turn it on here (restart required).
 # hint.enable = off
-# Hint mode labels the links, URLs and buttons on the focused screen:
+# Hint mode finds its targets with the nefer-hints detector (restart required).
-# clickable (links, URLs and buttons) or links (links and URLs only).
+# hint.detector = nefer-hints
-# hint.targets = clickable
+# hint.model = ~/.local/share/nefer-hints/models
+# Unset or empty uses the model threshold (model.json), else the runner default.
+# Set 0.01..0.99 to override it.
+# hint.threshold =
 # Show the labels in screenshots and screencasts taken by clients.
 # hint.capture = off

D internal/adapters/atspi/bus_test.go +0 -197

@@ -1,197 +0,0 @@
1	-	package atspi
2	-
3	-	import (
4	-	"bufio"
5	-	"os"
6	-	"os/exec"
7	-	"strings"
8	-	"testing"
9	-	"time"
10	-
11	-	"github.com/bnema/zerobus"
12	-	"github.com/stretchr/testify/require"
13	-	)
14	-
15	-	// startBus starts a private dbus-daemon, so the test never touches the
16	-	// desktop's buses. It skips when the daemon is not installed.
17	-	func startBus(t *testing.T) string {
18	-	t.Helper()
19	-	bin, err := exec.LookPath("dbus-daemon")
20	-	if err != nil {
21	-	if os.Getenv("CI") != "" {
22	-	t.Fatal("dbus-daemon not installed")
23	-	}
24	-	t.Skip("dbus-daemon not installed")
25	-	}
26	-	cmd := exec.Command(bin, "--session", "--nofork", "--nopidfile", "--print-address=1")
27	-	out, err := cmd.StdoutPipe()
28	-	require.NoError(t, err)
29	-	require.NoError(t, cmd.Start())
30	-	t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() })
31	-	line, err := bufio.NewReader(out).ReadString('\n')
32	-	require.NoError(t, err)
33	-	return strings.TrimSpace(line)
34	-	}
35	-
36	-	// TestDoDecodesMatchesPipelined runs Do against an app on a real bus: the
37	-	// GetMatches reply is decoded, and replies to several calls sent before any
38	-	// is read are matched by serial.
39	-	func TestDoDecodesMatchesPipelined(t *testing.T) {
40	-	addr := startBus(t)
41	-	app, err := zerobus.Dial(addr)
42	-	require.NoError(t, err)
43	-	t.Cleanup(func() { _ = app.Close() })
44	-	_, err = app.RequestName("org.test.App", 0)
45	-	require.NoError(t, err)
46	-
47	-	done := make(chan struct{})
48	-	go func() {
49	-	defer close(done)
50	-	for {
51	-	m, err := app.ReadMessage()
52	-	if err != nil {
53	-	return
54	-	}
55	-	if m.Type != zerobus.TypeMethodCall {
56	-	continue
57	-	}
58	-	req := &zerobus.Message{Sender: strings.Clone(m.Sender), Serial: m.Serial}
59	-	switch m.Member {
60	-	case "GetMatches":
61	-	// Check the rule: struct, then sort, count, traverse.
62	-	r := m.Body()
63	-	r.Struct()
64	-	end := r.Array('i')
65	-	var states []int32
66	-	for r.More(end) {
67	-	states = append(states, r.Int32())
68	-	}
69	-	_ = r.Int32()
70	-	ad := r.Array('{')
71	-	_ = ad
72	-	_ = r.Int32()
73	-	end = r.Array('i')
74	-	var roles []int32
75	-	for r.More(end) {
76	-	roles = append(roles, r.Int32())
77	-	}
78	-	ok := r.Err() == nil && len(states) == 2 && states[0] == 1<<25\|1<<30 && len(roles) == 5 && uint32(roles[2]) == 1<<24\|1<<18\|1<<31
79	-	if !ok {
80	-	app.NewError(req, "org.test.Error.BadRule", "s").Str("bad rule")
81	-	_, _ = app.Send()
82	-	continue
83	-	}
84	-	e := app.NewReply(req, "a(so)")
85	-	a := e.BeginArray('(')
86	-	for _, p := range []string{"/one", "/two"} {
87	-	e.Struct()
88	-	e.Str(":1.99")
89	-	e.ObjectPath(p)
90	-	}
91	-	e.EndArray(a)
92	-	case "GetExtents":
93	-	e := app.NewReply(req, "(iiii)")
94	-	e.Struct()
95	-	e.Int32(1)
96	-	e.Int32(2)
97	-	e.Int32(30)
98	-	e.Int32(40)
99	-	case "GetRole":
100	-	app.NewReply(req, "u").Uint32(roleLink)
101	-	case "GetState":
102	-	e := app.NewReply(req, "au")
103	-	a := e.BeginArray('u')
104	-	e.Uint32(stShowing)
105	-	e.Uint32(0)
106	-	e.EndArray(a)
107	-	default:
108	-	app.NewError(req, "org.freedesktop.DBus.Error.UnknownMethod", "s").Str("no")
109	-	}
110	-	_, _ = app.Send()
111	-	}
112	-	}()
113	-
114	-	cc, err := zerobus.Dial(addr)
115	-	require.NoError(t, err)
116	-	b, err := newConn(cc)
117	-	require.NoError(t, err)
118	-	t.Cleanup(func() { _ = b.Close() })
119	-
120	-	calls := []call{
121	-	{op: opMatches, name: "org.test.App", path: "/frame", rule: matchRule{linksOnly: true}},
122	-	{op: opExtents, name: "org.test.App", path: "/one", arg: coordWindow},
123	-	{op: opRole, name: "org.test.App", path: "/one"},
124	-	{op: opState, name: "org.test.App", path: "/one"},
125	-	{op: opChildren, name: "org.test.App", path: "/one"},
126	-	}
127	-	rep := b.Do(time.Now().Add(2*time.Second), time.Second, calls)
128	-	require.Len(t, rep, 5)
129	-	require.NoError(t, rep[0].err)
130	-	require.Equal(t, []ref{{":1.99", "/one"}, {":1.99", "/two"}}, rep[0].refs)
131	-	require.NoError(t, rep[1].err)
132	-	require.Equal(t, [4]int32{1, 2, 30, 40}, rep[1].rect)
133	-	require.Equal(t, uint32(roleLink), rep[2].u)
134	-	require.Equal(t, uint64(stShowing), rep[3].state)
135	-	var be *busError
136	-	require.ErrorAs(t, rep[4].err, &be)
137	-	require.Equal(t, "org.freedesktop.DBus.Error.UnknownMethod", be.name)
138	-
139	-	// A call to a name nobody owns fails fast, others still succeed.
140	-	rep = b.Do(time.Now().Add(2*time.Second), time.Second, []call{
141	-	{op: opRole, name: "org.test.Nobody", path: "/x"},
142	-	{op: opRole, name: "org.test.App", path: "/x"},
143	-	})
144	-	require.Error(t, rep[0].err)
145	-	require.NoError(t, rep[1].err)
146	-	}
147	-
148	-	func TestDoTimesOutOnSilentApp(t *testing.T) {
149	-	addr := startBus(t)
150	-	app, err := zerobus.Dial(addr)
151	-	require.NoError(t, err)
152	-	t.Cleanup(func() { _ = app.Close() })
153	-	_, err = app.RequestName("org.test.Mute", 0)
154	-	require.NoError(t, err)
155	-
156	-	cc, err := zerobus.Dial(addr)
157	-	require.NoError(t, err)
158	-	b, err := newConn(cc)
159	-	require.NoError(t, err)
160	-	t.Cleanup(func() { _ = b.Close() })
161	-
162	-	t0 := time.Now()
163	-	rep := b.Do(time.Now().Add(time.Second), 50*time.Millisecond, []call{{op: opRole, name: "org.test.Mute", path: "/x"}})
164	-	require.ErrorIs(t, rep[0].err, ErrTimeout)
165	-	require.Less(t, time.Since(t0), 500*time.Millisecond)
166	-	}
167	-
168	-	// A call left unanswered for idle gets ErrTimeout without breaking the
169	-	// connection; a batch that hits its deadline breaks it, and Do returns near
170	-	// the deadline even though the idle timeout is longer.
171	-	func TestDoDeadlineBreaksConnection(t *testing.T) {
172	-	addr := startBus(t)
173	-	app, err := zerobus.Dial(addr)
174	-	require.NoError(t, err)
175	-	t.Cleanup(func() { _ = app.Close() })
176	-	_, err = app.RequestName("org.test.Mute", 0)
177	-	require.NoError(t, err)
178	-
179	-	cc, err := zerobus.Dial(addr)
180	-	require.NoError(t, err)
181	-	b, err := newConn(cc)
182	-	require.NoError(t, err)
183	-	t.Cleanup(func() { _ = b.Close() })
184	-
185	-	mute := []call{{op: opRole, name: "org.test.Mute", path: "/x"}}
186	-	rep := b.Do(time.Now().Add(time.Second), 40*time.Millisecond, mute)
187	-	require.ErrorIs(t, rep[0].err, ErrTimeout)
188	-	require.NotErrorIs(t, rep[0].err, ErrBroken)
189	-
190	-	t0 := time.Now()
191	-	rep = b.Do(time.Now().Add(60*time.Millisecond), time.Second, mute)
192	-	require.ErrorIs(t, rep[0].err, ErrBroken)
193	-	require.Less(t, time.Since(t0), 300*time.Millisecond)
194	-	// The socket is closed: the next batch fails at once.
195	-	rep = b.Do(time.Now().Add(time.Second), time.Second, mute)
196	-	require.ErrorIs(t, rep[0].err, ErrBroken)
197	-	}

D internal/adapters/atspi/connector_mock_test.go +0 -109

@@ -1,109 +0,0 @@
1	-	// Code generated by mockery; DO NOT EDIT.
2	-	// github.com/vektra/mockery
3	-	// template: testify
4	-
5	-	package atspi
6	-
7	-	import (
8	-	"context"
9	-
10	-	mock "github.com/stretchr/testify/mock"
11	-	)
12	-
13	-	// newMockconnector creates a new instance of mockconnector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
14	-	// The first argument is typically a *testing.T value.
15	-	func newMockconnector(t interface {
16	-	mock.TestingT
17	-	Cleanup(func())
18	-	}) *mockconnector {
19	-	if helper, ok := t.(interface{ Helper() }); ok {
20	-	helper.Helper()
21	-	}
22	-
23	-	mock := &mockconnector{}
24	-	mock.Mock.Test(t)
25	-
26	-	t.Cleanup(func() {
27	-	if helper, ok := t.(interface{ Helper() }); ok {
28	-	helper.Helper()
29	-	}
30	-	mock.AssertExpectations(t)
31	-	})
32	-
33	-	return mock
34	-	}
35	-
36	-	// mockconnector is an autogenerated mock type for the connector type
37	-	type mockconnector struct {
38	-	mock.Mock
39	-	}
40	-
41	-	type mockconnector_Expecter struct {
42	-	mock *mock.Mock
43	-	}
44	-
45	-	func (_m mockconnector) EXPECT() mockconnector_Expecter {
46	-	return &mockconnector_Expecter{mock: &_m.Mock}
47	-	}
48	-
49	-	// Connect provides a mock function for the type mockconnector
50	-	func (_mock *mockconnector) Connect(ctx context.Context) (transport, error) {
51	-	ret := _mock.Called(ctx)
52	-
53	-	if len(ret) == 0 {
54	-	panic("no return value specified for Connect")
55	-	}
56	-
57	-	var r0 transport
58	-	var r1 error
59	-	if returnFunc, ok := ret.Get(0).(func(context.Context) (transport, error)); ok {
60	-	return returnFunc(ctx)
61	-	}
62	-	if returnFunc, ok := ret.Get(0).(func(context.Context) transport); ok {
63	-	r0 = returnFunc(ctx)
64	-	} else {
65	-	if ret.Get(0) != nil {
66	-	r0 = ret.Get(0).(transport)
67	-	}
68	-	}
69	-	if returnFunc, ok := ret.Get(1).(func(context.Context) error); ok {
70	-	r1 = returnFunc(ctx)
71	-	} else {
72	-	r1 = ret.Error(1)
73	-	}
74	-	return r0, r1
75	-	}
76	-
77	-	// mockconnector_Connect_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Connect'
78	-	type mockconnector_Connect_Call struct {
79	-	*mock.Call
80	-	}
81	-
82	-	// Connect is a helper method to define mock.On call
83	-	// - ctx context.Context
84	-	func (_e mockconnector_Expecter) Connect(ctx any) mockconnector_Connect_Call {
85	-	return &mockconnector_Connect_Call{Call: _e.mock.On("Connect", ctx)}
86	-	}
87	-
88	-	func (_c mockconnector_Connect_Call) Run(run func(ctx context.Context)) mockconnector_Connect_Call {
89	-	_c.Call.Run(func(args mock.Arguments) {
90	-	var arg0 context.Context
91	-	if args[0] != nil {
92	-	arg0 = args[0].(context.Context)
93	-	}
94	-	run(
95	-	arg0,
96	-	)
97	-	})
98	-	return _c
99	-	}
100	-
101	-	func (_c mockconnector_Connect_Call) Return(transportMoqParam transport, err error) mockconnector_Connect_Call {
102	-	_c.Call.Return(transportMoqParam, err)
103	-	return _c
104	-	}
105	-
106	-	func (_c mockconnector_Connect_Call) RunAndReturn(run func(ctx context.Context) (transport, error)) mockconnector_Connect_Call {
107	-	_c.Call.Return(run)
108	-	return _c
109	-	}

D internal/adapters/atspi/doc.go +0 -10

@@ -1,10 +0,0 @@
1	-	// Package atspi finds hint targets in the accessibility tree of the apps on
2	-	// screen. It talks to the AT-SPI registry over the accessibility bus with
3	-	// zerobus, from one connection used by one goroutine at a time, and pipelines
4	-	// the calls it needs: several are sent before their replies are read, and the
5	-	// replies are matched by serial.
6	-	//
7	-	// The approach (Collection.GetMatches on the window frame, bounded walk when
8	-	// the toolkit has no Collection, window-relative extents) is adapted from
9	-	// neru's AT-SPI client, https://github.com/y3owk1n/neru (MIT licence).
10	-	package atspi

D internal/adapters/atspi/model.go +0 -206

@@ -1,206 +0,0 @@
1	-	package atspi
2	-
3	-	import "github.com/bnema/neferwl/internal/ports"
4	-
5	-	// AT-SPI constants (atspi-constants.h). The role ids are the AtspiRole enum
6	-	// values; the state ids are bit indices of the state set.
7	-	const (
8	-	roleCheckBox = 7
9	-	roleComboBox = 11
10	-	roleDialog = 16
11	-	roleFrame = 23
12	-	roleMenuItem = 35
13	-	rolePageTab = 37
14	-	rolePushButton = 43
15	-	roleRadioButton = 44
16	-	roleToggleButton = 62
17	-	roleWindow = 69
18	-	roleLink = 88
19	-	roleDocumentFrame = 82
20	-	roleDocumentWeb = 95
21	-
22	-	stateActive = 1
23	-	stateShowing = 25
24	-	stateVisible = 30
25	-
26	-	roleWords = 5 // int32 words of a role set in a MatchRule
27	-	stateWords = 2 // int32 words of a state set in a MatchRule
28	-
29	-	// AtspiCollectionMatchType and AtspiCollectionSortOrder values.
30	-	matchAll = 1
31	-	matchAny = 2
32	-	sortCanonical = 1
33	-	matchCount = 500
34	-	maxDepth = 30
35	-	maxWalkNodes = 3000
36	-	maxFrames = 32
37	-	sizeTolerance = 40
38	-	maxMatchesRead = matchCount
39	-	)
40	-
41	-	// clickableRoles are the roles that become hint targets.
42	-	var clickableRoles = []uint32{
43	-	roleLink, rolePushButton, roleToggleButton, roleCheckBox,
44	-	roleRadioButton, roleMenuItem, rolePageTab, roleComboBox,
45	-	}
46	-
47	-	// matchRule is the Collection query: showing and visible objects of the
48	-	// clickable roles (only links when linksOnly), plus web documents for coverage.
49	-	type matchRule struct{ linksOnly bool }
50	-
51	-	func (r matchRule) roles() []uint32 {
52	-	if r.linksOnly {
53	-	return []uint32{roleLink, roleDocumentFrame, roleDocumentWeb}
54	-	}
55	-	return append(append([]uint32(nil), clickableRoles...), roleDocumentFrame, roleDocumentWeb)
56	-	}
57	-
58	-	// roleBits packs role ids into the bit set of a MatchRule: bit id%32 of word
59	-	// id/32.
60	-	func roleBits(ids []uint32) []int32 {
61	-	words := make([]int32, roleWords)
62	-	for _, id := range ids {
63	-	if w := int(id / 32); w < len(words) {
64	-	words[w] \|= int32(uint32(1) << (id % 32))
65	-	}
66	-	}
67	-	return words
68	-	}
69	-
70	-	// stateBits packs state ids into the bit set of a MatchRule.
71	-	func stateBits(ids ...uint32) []int32 {
72	-	words := make([]int32, stateWords)
73	-	for _, id := range ids {
74	-	if w := int(id / 32); w < len(words) {
75	-	words[w] \|= int32(uint32(1) << (id % 32))
76	-	}
77	-	}
78	-	return words
79	-	}
80	-
81	-	func hasState(set uint64, state uint) bool { return set&(1<<state) != 0 }
82	-
83	-	// isClickable reports whether role makes a target.
84	-	func isClickable(role uint32, linksOnly bool) bool {
85	-	if linksOnly {
86	-	return role == roleLink
87	-	}
88	-	for _, r := range clickableRoles {
89	-	if r == role {
90	-	return true
91	-	}
92	-	}
93	-	return false
94	-	}
95	-
96	-	// isWindowRole reports whether role is a top-level window role.
97	-	func isWindowRole(role uint32) bool {
98	-	return role == roleFrame \|\| role == roleWindow \|\| role == roleDialog
99	-	}
100	-
101	-	// kindOf maps a role to the hint kind: links are links, the rest buttons.
102	-	func kindOf(role uint32) ports.HintKind {
103	-	if role == roleLink {
104	-	return ports.HintLink
105	-	}
106	-	return ports.HintButton
107	-	}
108	-
109	-	// frameInfo is a showing top-level window of an app.
110	-	type frameInfo struct {
111	-	ref ref
112	-	active bool
113	-	w, h int // size of the frame, from its window extents
114	-	}
115	-
116	-	// matchFrames pairs the windows of one app with its frames, one to one. A
117	-	// window is paired with a frame when the pairing is certain:
118	-	// - one window and one frame;
119	-	// - the focused window with the one ACTIVE frame; that frame is then
120	-	// reserved and no other window can take it;
121	-	// - otherwise the one unreserved frame whose size is within sizeTolerance
122	-	// of the window, provided no other unmatched window claims it as well.
123	-	//
124	-	// Ambiguous windows are left out. The result maps the index of a window in
125	-	// wins to the index of its frame in frames.
126	-	func matchFrames(wins []ports.HintWindow, frames []frameInfo) map[int]int {
127	-	out := map[int]int{}
128	-	if len(wins) == 1 && len(frames) == 1 {
129	-	out[0] = 0
130	-	return out
131	-	}
132	-	// Active matches first: they are reserved against the size matches.
133	-	active, n := -1, 0
134	-	for j, f := range frames {
135	-	if f.active {
136	-	active, n = j, n+1
137	-	}
138	-	}
139	-	reserved := make([]bool, len(frames))
140	-	done := make([]bool, len(wins))
141	-	if n == 1 {
142	-	focused := -1
143	-	count := 0
144	-	for i, w := range wins {
145	-	if w.Focused {
146	-	focused, count = i, count+1
147	-	}
148	-	}
149	-	if count == 1 {
150	-	out[focused] = active
151	-	reserved[active] = true
152	-	done[focused] = true
153	-	}
154	-	}
155	-	claims := make([][]int, len(wins)) // size candidates per window
156	-	owners := make([]int, len(frames)) // windows that size-match a frame
157	-	for i, w := range wins {
158	-	if done[i] {
159	-	continue
160	-	}
161	-	for j, f := range frames {
162	-	if !reserved[j] && absInt(f.w-w.Rect.W) <= sizeTolerance && absInt(f.h-w.Rect.H) <= sizeTolerance {
163	-	claims[i] = append(claims[i], j)
164	-	owners[j]++
165	-	}
166	-	}
167	-	}
168	-	for i := range wins {
169	-	if !done[i] && len(claims[i]) == 1 && owners[claims[i][0]] == 1 {
170	-	out[i] = claims[i][0]
171	-	}
172	-	}
173	-	return out
174	-	}
175	-
176	-	func absInt(v int) int {
177	-	if v < 0 {
178	-	return -v
179	-	}
180	-	return v
181	-	}
182	-
183	-	// element is a match with its extents.
184	-	type element struct {
185	-	role uint32
186	-	x, y, w, h int32 // window-relative
187	-	}
188	-
189	-	// toTarget converts an element of a window into a logical output-local
190	-	// target: the element extents are relative to the top-level frame, whose own
191	-	// window-relative origin is (fx, fy); the result is clipped to the window.
192	-	// It reports false for an empty result.
193	-	func toTarget(win ports.Rect, fx, fy int32, e element) (ports.HintTarget, bool) {
194	-	if e.w <= 0 \|\| e.h <= 0 {
195	-	return ports.HintTarget{}, false
196	-	}
197	-	r := ports.Rect{X: win.X + int(e.x-fx), Y: win.Y + int(e.y-fy), W: int(e.w), H: int(e.h)}
198	-	x0, y0 := max(r.X, win.X), max(r.Y, win.Y)
199	-	x1, y1 := min(r.X+r.W, win.X+win.W), min(r.Y+r.H, win.Y+win.H)
200	-	if x1 <= x0 \|\| y1 <= y0 {
201	-	return ports.HintTarget{}, false
202	-	}
203	-	return ports.HintTarget{Rect: ports.Rect{X: x0, Y: y0, W: x1 - x0, H: y1 - y0}, Kind: kindOf(e.role)}, true
204	-	}
205	-
206	-	func isDocument(role uint32) bool { return role == roleDocumentFrame \|\| role == roleDocumentWeb }

D internal/adapters/atspi/model_test.go +0 -117

@@ -1,117 +0,0 @@
1	-	package atspi
2	-
3	-	import (
4	-	"testing"
5	-
6	-	"github.com/stretchr/testify/require"
7	-
8	-	"github.com/bnema/neferwl/internal/ports"
9	-	)
10	-
11	-	func TestToTargetConvertsWindowCoordinates(t *testing.T) {
12	-	win := ports.Rect{X: 100, Y: 50, W: 800, H: 600}
13	-	// The frame origin (fx, fy) is subtracted: toolkit offsets cancel out.
14	-	got, ok := toTarget(win, 10, 20, element{role: roleLink, x: 110, y: 120, w: 40, h: 16})
15	-	require.True(t, ok)
16	-	require.Equal(t, ports.HintTarget{Rect: ports.Rect{X: 200, Y: 150, W: 40, H: 16}, Kind: ports.HintLink}, got)
17	-	}
18	-
19	-	func TestToTargetClipsToWindow(t *testing.T) {
20	-	win := ports.Rect{X: 100, Y: 50, W: 200, H: 100}
21	-	got, ok := toTarget(win, 0, 0, element{role: rolePushButton, x: 180, y: 90, w: 60, h: 30})
22	-	require.True(t, ok)
23	-	require.Equal(t, ports.Rect{X: 280, Y: 140, W: 20, H: 10}, got.Rect)
24	-	require.Equal(t, ports.HintButton, got.Kind)
25	-	}
26	-
27	-	func TestToTargetDropsEmptyAndOutside(t *testing.T) {
28	-	win := ports.Rect{X: 0, Y: 0, W: 100, H: 100}
29	-	for name, e := range map[string]element{
30	-	"zero width": {role: roleLink, x: 1, y: 1, w: 0, h: 10},
31	-	"neg height": {role: roleLink, x: 1, y: 1, w: 10, h: -1},
32	-	"outside": {role: roleLink, x: 500, y: 1, w: 10, h: 10},
33	-	"above": {role: roleLink, x: 1, y: -50, w: 10, h: 10},
34	-	"left of win": {role: roleLink, x: -30, y: 1, w: 10, h: 10},
35	-	} {
36	-	_, ok := toTarget(win, 0, 0, e)
37	-	require.False(t, ok, name)
38	-	}
39	-	}
40	-
41	-	func TestRoleKindMapping(t *testing.T) {
42	-	require.Equal(t, ports.HintLink, kindOf(roleLink))
43	-	for _, r := range []uint32{rolePushButton, roleToggleButton, roleCheckBox, roleRadioButton, roleMenuItem, rolePageTab, roleComboBox} {
44	-	require.Equal(t, ports.HintButton, kindOf(r), r)
45	-	require.True(t, isClickable(r, false), r)
46	-	require.False(t, isClickable(r, true), r)
47	-	}
48	-	require.True(t, isClickable(roleLink, true))
49	-	require.False(t, isClickable(roleFrame, false))
50	-	require.True(t, isWindowRole(roleFrame) && isWindowRole(roleWindow) && isWindowRole(roleDialog))
51	-	require.False(t, isWindowRole(roleLink))
52	-	}
53	-
54	-	func TestRoleAndStateBits(t *testing.T) {
55	-	require.Equal(t, []int32{0, 0, 0, 0, 0}, roleBits(nil))
56	-	// link is role 88: word 2, bit 24; push button is 43: word 1, bit 11.
57	-	require.Equal(t, []int32{0, 1 << 11, 1 << 24, 0, 0}, roleBits([]uint32{roleLink, rolePushButton}))
58	-	// showing is state 25, visible 30: both in word 0.
59	-	require.Equal(t, []int32{1<<25 \| 1<<30, 0}, stateBits(stateShowing, stateVisible))
60	-	}
61	-
62	-	func wins(ws ...ports.HintWindow) []ports.HintWindow { return ws }
63	-
64	-	func win(id ports.WindowID, w, h int, focused bool) ports.HintWindow {
65	-	return ports.HintWindow{ID: id, PID: 1, Rect: ports.Rect{W: w, H: h}, Focused: focused}
66	-	}
67	-
68	-	func TestMatchFramesSingle(t *testing.T) {
69	-	got := matchFrames(wins(win(1, 800, 600, false)), []frameInfo{{w: 100, h: 100}})
70	-	require.Equal(t, map[int]int{0: 0}, got)
71	-	}
72	-
73	-	func TestMatchFramesActive(t *testing.T) {
74	-	frames := []frameInfo{{w: 800, h: 600}, {w: 800, h: 600, active: true}}
75	-	got := matchFrames(wins(win(1, 800, 600, true), win(2, 800, 600, false)), frames)
76	-	// The focused window takes the ACTIVE frame and reserves it; the other
77	-	// window then only has the remaining frame to match.
78	-	require.Equal(t, map[int]int{0: 1, 1: 0}, got)
79	-	}
80	-
81	-	func TestMatchFramesSize(t *testing.T) {
82	-	frames := []frameInfo{{w: 1000, h: 700}, {w: 400, h: 300}}
83	-	got := matchFrames(wins(win(1, 1000, 700, false), win(2, 420, 290, false)), frames)
84	-	require.Equal(t, map[int]int{0: 0, 1: 1}, got)
85	-	}
86	-
87	-	func TestMatchFramesSizeTolerance(t *testing.T) {
88	-	frames := []frameInfo{{w: 840, h: 640}, {w: 100, h: 100}}
89	-	other := win(2, 90, 90, false)
90	-	// 40 px off still matches; 41 px does not.
91	-	require.Equal(t, map[int]int{0: 0, 1: 1}, matchFrames(wins(win(1, 800, 600, false), other), frames))
92	-	require.Equal(t, map[int]int{1: 1}, matchFrames(wins(win(1, 799, 600, false), other), frames))
93	-	}
94	-
95	-	func TestMatchFramesAmbiguous(t *testing.T) {
96	-	// Two windows and two frames of the same size, none active: skipped.
97	-	frames := []frameInfo{{w: 800, h: 600}, {w: 810, h: 600}}
98	-	got := matchFrames(wins(win(1, 800, 600, false), win(2, 800, 600, false)), frames)
99	-	require.Empty(t, got)
100	-	// One window, two same-size frames, not focused: skipped.
101	-	require.Empty(t, matchFrames(wins(win(1, 800, 600, false)), frames))
102	-	// No frame close enough.
103	-	require.Empty(t, matchFrames(wins(win(1, 200, 200, false), win(2, 300, 300, false)), frames))
104	-	}
105	-
106	-	// The active frame is reserved by the focused window: an unfocused window of
107	-	// the same size as that frame must not be paired with it as well.
108	-	func TestMatchFramesOneToOne(t *testing.T) {
109	-	frames := []frameInfo{{ref: ref{"a", "/active"}, active: true, w: 400, h: 300}, {ref: ref{"a", "/idle"}, w: 800, h: 600}}
110	-	got := matchFrames(wins(win(1, 800, 600, true), win(2, 400, 300, false)), frames)
111	-	seen := map[int]bool{}
112	-	for _, f := range got {
113	-	require.False(t, seen[f], "frame %d assigned twice: %v", f, got)
114	-	seen[f] = true
115	-	}
116	-	require.Equal(t, map[int]int{0: 0}, got)
117	-	}

D internal/adapters/atspi/source.go +0 -481

@@ -1,481 +0,0 @@
1	-	package atspi
2	-
3	-	import (
4	-	"context"
5	-	"errors"
6	-	"fmt"
7	-	"strings"
8	-	"time"
9	-
10	-	"github.com/bnema/zerobus"
11	-
12	-	"github.com/bnema/neferwl/internal/adapters/logging"
13	-	"github.com/bnema/neferwl/internal/ports"
14	-	)
15	-
16	-	const (
17	-	// callTimeout is how long a batch waits for the next reply.
18	-	callTimeout = 100 * time.Millisecond
19	-	// budget is the whole time Targets may take.
20	-	budget = 300 * time.Millisecond
21	-	)
22	-
23	-	// connectBudget bounds the whole connection setup (session bus,
24	-	// GetAddress, IsEnabled, dial of the accessibility bus) of one attempt.
25	-	const connectBudget = 300 * time.Millisecond
26	-
27	-	// retryEvery is the minimum time between two connection attempts.
28	-	const retryEvery = 5 * time.Second
29	-
30	-	// connector opens a transport on the accessibility bus.
31	-	type connector interface {
32	-	Connect(ctx context.Context) (transport, error)
33	-	}
34	-
35	-	// stallWarnAfter is how long an attempt may be in flight before it is
36	-	// reported as a stalled accessibility bus.
37	-	const stallWarnAfter = 5 * time.Second
38	-
39	-	// dialResult is the outcome of a connection attempt.
40	-	type dialResult struct {
41	-	tr transport
42	-	err error
43	-	at time.Time // when the attempt finished
44	-	}
45	-
46	-	// Source reads hint targets from the accessibility tree. It owns one
47	-	// connection to the accessibility bus. It is NOT safe for concurrent use:
48	-	// it is owned by the hints worker goroutine, which calls Targets and,
49	-	// when it returns, Close. The connection is made on the first Targets call
50	-	// that needs it, never at startup, and made again after a failure, at most
51	-	// once every retryEvery; meanwhile Targets finds nothing and OCR takes over.
52	-	//
53	-	// At most one connection attempt (one goroutine and one socket) exists at
54	-	// any time: an attempt that outlives the wait of ensure stays in flight, is
55	-	// polled by the later Targets calls and is never doubled.
56	-	type Source struct {
57	-	tr transport
58	-	dial connector
59	-	// attempt delivers the outcome of the attempt in flight (buffered, so
60	-	// the attempt goroutine never blocks); nil when none runs.
61	-	attempt chan dialResult
62	-	attemptStarted time.Time
63	-	warnedStall bool
64	-	retryAt time.Time
65	-	closed bool
66	-	// wait is how long ensure waits for a new attempt; connectBudget if 0.
67	-	wait time.Duration
68	-	}
69	-
70	-	// New returns a Source. It does not touch the bus: the connection is made by
71	-	// the first Targets call.
72	-	func New() *Source { return &Source{dial: busConnector{}} }
73	-
74	-	// busConnector connects to the session bus, enables accessibility and dials
75	-	// the accessibility bus.
76	-	type busConnector struct{}
77	-
78	-	func (busConnector) Connect(ctx context.Context) (transport, error) {
79	-	log := logging.For(ctx, "atspi")
80	-	// zerobus has no deadline or context API: SessionBus and Dial run the
81	-	// socket dial, SASL auth and Hello without a bound, and the watchdog
82	-	// below cannot exist before the connection does. A stall there is
83	-	// bounded by the single-in-flight rule of Source.ensure, not by a
84	-	// deadline. Follow-up: a deadline-capable zerobus API.
85	-	sess, err := zerobus.SessionBus()
86	-	if err != nil {
87	-	return nil, fmt.Errorf("atspi: session bus: %w", err)
88	-	}
89	-	defer func() { _ = sess.Close() }()
90	-	// Call has no deadline: closing the connection ends the wait for a
91	-	// silent service.
92	-	stop := time.AfterFunc(connectBudget, func() { _ = sess.Close() })
93	-	defer stop.Stop()
94	-
95	-	sess.NewCall(a11yBusName, a11yBusPath, a11yBusName, "GetAddress", "")
96	-	m, err := sess.Call()
97	-	if err != nil {
98	-	return nil, fmt.Errorf("atspi: org.a11y.Bus GetAddress: %w", err)
99	-	}
100	-	r := m.Body()
101	-	addr := r.Str()
102	-	if r.Err() != nil \|\| addr == "" {
103	-	return nil, errors.New("atspi: no accessibility bus address")
104	-	}
105	-	addr = strings.Clone(addr)
106	-
107	-	e := sess.NewCall(a11yBusName, a11yBusPath, ifaceProps, "Set", "ssv")
108	-	e.Str(a11yStatus)
109	-	e.Str("IsEnabled")
110	-	e.Variant("b")
111	-	e.Bool(true)
112	-	if _, err := sess.Call(); err != nil {
113	-	// Toolkits that already see the bus keep working.
114	-	log.Warn().Err(err).Msg("could not enable accessibility")
115	-	} else {
116	-	log.Info().Msg("accessibility enabled")
117	-	}
118	-
119	-	zc, err := zerobus.Dial(addr)
120	-	if err != nil {
121	-	return nil, fmt.Errorf("atspi: accessibility bus: %w", err)
122	-	}
123	-	tr, err := newConn(zc)
124	-	if err != nil {
125	-	return nil, err
126	-	}
127	-	log.Info().Msg("accessibility bus connected")
128	-	return tr, nil
129	-	}
130	-
131	-	// ensure returns the connection, making one if none exists and the last
132	-	// attempt is old enough. It waits at most the budget for a new attempt; an
133	-	// attempt that does not answer in time stays in flight (abandoned, never
134	-	// doubled) and is polled without blocking by the next calls.
135	-	func (s *Source) ensure(ctx context.Context) transport {
136	-	if s.closed {
137	-	return nil
138	-	}
139	-	if s.tr != nil {
140	-	return s.tr
141	-	}
142	-	if s.dial == nil {
143	-	return nil
144	-	}
145	-	now := time.Now()
146	-	if s.attempt != nil {
147	-	select {
148	-	case r := <-s.attempt:
149	-	s.finish(ctx, r)
150	-	default:
151	-	if !s.warnedStall && now.Sub(s.attemptStarted) > stallWarnAfter {
152	-	s.warnedStall = true
153	-	log := logging.For(ctx, "atspi")
154	-	log.Warn().Dur("for", now.Sub(s.attemptStarted)).Msg("accessibility bus not answering")
155	-	}
156	-	}
157	-	return s.tr
158	-	}
159	-	if now.Before(s.retryAt) {
160	-	return nil
161	-	}
162	-	ch := make(chan dialResult, 1)
163	-	s.attempt = ch
164	-	s.attemptStarted = now
165	-	s.warnedStall = false
166	-	dial := s.dial
167	-	go func() {
168	-	tr, err := dial.Connect(ctx)
169	-	ch <- dialResult{tr: tr, err: err, at: time.Now()}
170	-	}()
171	-	wait := s.wait
172	-	if wait <= 0 {
173	-	wait = connectBudget
174	-	}
175	-	timer := time.NewTimer(wait)
176	-	defer timer.Stop()
177	-	select {
178	-	case r := <-ch:
179	-	s.finish(ctx, r)
180	-	case <-timer.C:
181	-	log := logging.For(ctx, "atspi")
182	-	log.Debug().Dur("after", wait).Msg("accessibility connect attempt abandoned in flight")
183	-	case <-ctx.Done():
184	-	}
185	-	return s.tr
186	-	}
187	-
188	-	// finish takes the outcome of the attempt in flight: a transport is kept,
189	-	// an error starts the backoff from the time the attempt finished.
190	-	func (s *Source) finish(ctx context.Context, r dialResult) {
191	-	s.attempt = nil
192	-	if r.err != nil \|\| r.tr == nil {
193	-	s.retryAt = r.at.Add(retryEvery)
194	-	err := r.err
195	-	if err == nil {
196	-	err = errors.New("atspi: no transport")
197	-	}
198	-	log := logging.For(ctx, "atspi")
199	-	log.Warn().Err(err).Msg("accessibility hint targets unavailable, using OCR only")
200	-	return
201	-	}
202	-	s.tr = r.tr
203	-	}
204	-
205	-	// drop closes a broken connection; the next attempt reconnects.
206	-	func (s *Source) drop() {
207	-	s.retryAt = time.Now().Add(retryEvery)
208	-	if s.tr != nil {
209	-	_ = s.tr.Close()
210	-	s.tr = nil
211	-	}
212	-	}
213	-
214	-	// Close closes the accessibility bus connection, and the one an attempt
215	-	// still in flight may deliver late. After Close, Targets finds nothing and
216	-	// never connects. Like Targets it is called by the owner goroutine only.
217	-	func (s *Source) Close() error {
218	-	if s.closed {
219	-	return nil
220	-	}
221	-	s.closed = true
222	-	var err error
223	-	if s.tr != nil {
224	-	err = s.tr.Close()
225	-	s.tr = nil
226	-	}
227	-	if ch := s.attempt; ch != nil {
228	-	s.attempt = nil
229	-	// One reaper at most: Close runs once, and one attempt exists.
230	-	go func() {
231	-	if r := <-ch; r.tr != nil {
232	-	_ = r.tr.Close()
233	-	}
234	-	}()
235	-	}
236	-	return err
237	-	}
238	-
239	-	// appInfo is an application of the registry whose process shows a window.
240	-	type appInfo struct {
241	-	root ref
242	-	pid int
243	-	}
244	-
245	-	// winMatch is a window with the frame it was paired with.
246	-	type winMatch struct {
247	-	win ports.HintWindow
248	-	frame ref
249	-	fx int32
250	-	fy int32
251	-	}
252	-
253	-	// Targets returns the clickable targets of the accessibility tree for
254	-	// windows (logical output-local, clipped to each window) and the set of
255	-	// windows those targets cover. A window is covered when its tree answered
256	-	// with a link target or web document; uncovered windows also get OCR. It
257	-	// returns what is ready within the budget.
258	-	func (s *Source) Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error) {
259	-	covered := map[ports.WindowID]bool{}
260	-	deadline := time.Now().Add(budget)
261	-	if d, ok := ctx.Deadline(); ok && d.Before(deadline) {
262	-	deadline = d
263	-	}
264	-	wanted := map[int]bool{}
265	-	for _, w := range windows {
266	-	if w.PID > 0 {
267	-	wanted[w.PID] = true
268	-	}
269	-	}
270	-	if len(wanted) == 0 {
271	-	return nil, covered, nil
272	-	}
273	-	tr := s.ensure(ctx)
274	-	if tr == nil {
275	-	return nil, covered, nil
276	-	}
277	-	broken := false
278	-	do := func(calls []call) []reply {
279	-	if len(calls) == 0 \|\| ctx.Err() != nil {
280	-	return make([]reply, len(calls))
281	-	}
282	-	if broken {
283	-	out := make([]reply, len(calls))
284	-	for i := range out {
285	-	out[i].err = ErrBroken
286	-	}
287	-	return out
288	-	}
289	-	out := tr.Do(deadline, callTimeout, calls)
290	-	for _, r := range out {
291	-	if errors.Is(r.err, ErrBroken) {
292	-	broken = true
293	-	s.drop()
294	-	break
295	-	}
296	-	}
297	-	return out
298	-	}
299	-
300	-	// Applications of the registry, kept when their process shows a window.
301	-	rep := do([]call{{op: opChildren, name: registryName, path: registryRoot}})
302	-	if len(rep) == 0 \|\| rep[0].err != nil {
303	-	if len(rep) > 0 {
304	-	return nil, covered, fmt.Errorf("atspi: registry: %w", rep[0].err)
305	-	}
306	-	return nil, covered, nil
307	-	}
308	-	apps := rep[0].refs
309	-	calls := make([]call, len(apps))
310	-	for i, a := range apps {
311	-	calls[i] = call{op: opPID, name: a.name}
312	-	}
313	-	var kept []appInfo
314	-	for i, r := range do(calls) {
315	-	if r.err == nil && wanted[int(r.u)] {
316	-	kept = append(kept, appInfo{root: apps[i], pid: int(r.u)})
317	-	}
318	-	}
319	-
320	-	// The showing windows of each kept app.
321	-	calls = calls[:0]
322	-	for _, a := range kept {
323	-	calls = append(calls, call{op: opChildren, name: a.root.name, path: a.root.path})
324	-	}
325	-	var cands []ref
326	-	var candApp []int
327	-	for i, r := range do(calls) {
328	-	if r.err != nil {
329	-	continue
330	-	}
331	-	for _, c := range r.refs[:min(len(r.refs), maxFrames)] {
332	-	cands = append(cands, c)
333	-	candApp = append(candApp, i)
334	-	}
335	-	}
336	-	calls = calls[:0]
337	-	for _, c := range cands {
338	-	calls = append(calls,
339	-	call{op: opRole, name: c.name, path: c.path},
340	-	call{op: opState, name: c.name, path: c.path},
341	-	call{op: opExtents, name: c.name, path: c.path, arg: coordWindow})
342	-	}
343	-	info := do(calls)
344	-	type frame struct {
345	-	frameInfo
346	-	pid int
347	-	fx, fy int32
348	-	}
349	-	var frames []frame
350	-	for i, c := range cands {
351	-	role, state, ext := info[3i], info[3i+1], info[3*i+2]
352	-	if role.err != nil \|\| state.err != nil \|\| ext.err != nil \|\| !isWindowRole(role.u) \|\| !hasState(state.state, stateShowing) {
353	-	continue
354	-	}
355	-	frames = append(frames, frame{
356	-	frameInfo: frameInfo{ref: c, active: hasState(state.state, stateActive), w: int(ext.rect[2]), h: int(ext.rect[3])},
357	-	pid: kept[candApp[i]].pid, fx: ext.rect[0], fy: ext.rect[1],
358	-	})
359	-	}
360	-
361	-	// Pair the windows with the frames, process by process.
362	-	var matched []winMatch
363	-	for pid := range wanted {
364	-	var wins []ports.HintWindow
365	-	for _, w := range windows {
366	-	if w.PID == pid {
367	-	wins = append(wins, w)
368	-	}
369	-	}
370	-	var fs []frameInfo
371	-	var origin []frame
372	-	for _, f := range frames {
373	-	if f.pid == pid {
374	-	fs = append(fs, f.frameInfo)
375	-	origin = append(origin, f)
376	-	}
377	-	}
378	-	for wi, fi := range matchFrames(wins, fs) {
379	-	matched = append(matched, winMatch{win: wins[wi], frame: origin[fi].ref, fx: origin[fi].fx, fy: origin[fi].fy})
380	-	}
381	-	}
382	-
383	-	targets := s.elements(do, matched, linksOnly, covered)
384	-	log := logging.For(ctx, "atspi")
385	-	log.Debug().Int("apps", len(kept)).Int("frames", len(frames)).Int("windows", len(matched)).Int("targets", len(targets)).Msg("atspi targets")
386	-	return targets, covered, nil
387	-	}
388	-
389	-	// elements reads the clickable elements of the matched frames.
390	-	func (s *Source) elements(do func([]call) []reply, matched []winMatch, linksOnly bool, covered map[ports.WindowID]bool) []ports.HintTarget {
391	-	calls := make([]call, len(matched))
392	-	for i, m := range matched {
393	-	calls[i] = call{op: opMatches, name: m.frame.name, path: m.frame.path, rule: matchRule{linksOnly: linksOnly}}
394	-	}
395	-	found := make([][]ref, len(matched))
396	-	var walk []int // frames without Collection
397	-	for i, r := range do(calls) {
398	-	if r.err != nil {
399	-	var be *busError
400	-	if errors.As(r.err, &be) {
401	-	walk = append(walk, i)
402	-	}
403	-	continue
404	-	}
405	-	found[i] = r.refs
406	-	}
407	-	for _, i := range walk {
408	-	found[i] = s.walk(do, matched[i].frame, linksOnly)
409	-	}
410	-
411	-	calls = calls[:0]
412	-	var owner []int
413	-	for i, refs := range found {
414	-	for _, r := range refs {
415	-	calls = append(calls,
416	-	call{op: opRole, name: r.name, path: r.path},
417	-	call{op: opExtents, name: r.name, path: r.path, arg: coordWindow})
418	-	owner = append(owner, i)
419	-	}
420	-	}
421	-	info := do(calls)
422	-	var out []ports.HintTarget
423	-	for k, i := range owner {
424	-	role, ext := info[2k], info[2k+1]
425	-	if role.err == nil && isDocument(role.u) {
426	-	covered[matched[i].win.ID] = true
427	-	continue
428	-	}
429	-	if role.err != nil \|\| ext.err != nil \|\| !isClickable(role.u, linksOnly) {
430	-	continue
431	-	}
432	-	m := matched[i]
433	-	t, ok := toTarget(m.win.Rect, m.fx, m.fy, element{role: role.u, x: ext.rect[0], y: ext.rect[1], w: ext.rect[2], h: ext.rect[3]})
434	-	if !ok {
435	-	continue
436	-	}
437	-	t.Window = m.win.ID
438	-	out = append(out, t)
439	-	if t.Kind == ports.HintLink {
440	-	covered[m.win.ID] = true
441	-	}
442	-	}
443	-	return out
444	-	}
445	-
446	-	// walk is the fallback for toolkits without Collection: a breadth-first
447	-	// walk of at most maxDepth levels and maxWalkNodes nodes, returning the
448	-	// clickable nodes.
449	-	func (s *Source) walk(do func([]call) []reply, root ref, linksOnly bool) []ref {
450	-	var out []ref
451	-	level := []ref{root}
452	-	seen := 0
453	-	for depth := 0; depth < maxDepth && len(level) > 0 && seen < maxWalkNodes; depth++ {
454	-	calls := make([]call, 0, 3*len(level))
455	-	for _, n := range level {
456	-	calls = append(calls,
457	-	call{op: opChildren, name: n.name, path: n.path},
458	-	call{op: opRole, name: n.name, path: n.path},
459	-	call{op: opState, name: n.name, path: n.path})
460	-	}
461	-	rep := do(calls)
462	-	room := max(0, maxWalkNodes-seen-len(level))
463	-	next := make([]ref, 0, room)
464	-	for i, n := range level {
465	-	ch, role, state := rep[3i], rep[3i+1], rep[3*i+2]
466	-	if state.err == nil && !hasState(state.state, stateShowing) {
467	-	continue
468	-	}
469	-	if role.err == nil && (isClickable(role.u, linksOnly) \|\| isDocument(role.u)) && state.err == nil && hasState(state.state, stateVisible) {
470	-	out = append(out, n)
471	-	}
472	-	if ch.err == nil {
473	-	take := min(len(ch.refs), room-len(next))
474	-	next = append(next, ch.refs[:take]...)
475	-	}
476	-	}
477	-	seen += len(level)
478	-	level = next
479	-	}
480	-	return out
481	-	}

D internal/adapters/atspi/source_test.go +0 -355

@@ -1,355 +0,0 @@
1	-	package atspi
2	-
3	-	import (
4	-	"context"
5	-	"errors"
6	-	"fmt"
7	-	"runtime"
8	-	"testing"
9	-	"time"
10	-
11	-	"github.com/stretchr/testify/mock"
12	-	"github.com/stretchr/testify/require"
13	-
14	-	"github.com/bnema/neferwl/internal/ports"
15	-	)
16	-
17	-	const (
18	-	stActive = 1 << stateActive
19	-	stShowing = 1 << stateShowing
20	-	)
21	-
22	-	// tree answers calls like an app: registry with one app (:1.5, pid 77) with
23	-	// one showing frame whose links sit at window-relative extents.
24	-	func answerTree(frameExt [4]int32, matches []ref, ext map[string][4]int32, roles map[string]uint32, noCollection bool) func(time.Time, time.Duration, []call) []reply {
25	-	return func(_ time.Time, _ time.Duration, calls []call) []reply {
26	-	out := make([]reply, len(calls))
27	-	for i, c := range calls {
28	-	switch {
29	-	case c.op == opChildren && c.name == registryName:
30	-	out[i].refs = []ref{{":1.5", "/root"}, {":1.9", "/root"}}
31	-	case c.op == opPID:
32	-	out[i].u = map[string]uint32{":1.5": 77, ":1.9": 99}[c.name]
33	-	case c.op == opChildren && c.path == "/root":
34	-	out[i].refs = []ref{{":1.5", "/frame"}}
35	-	case c.op == opRole && c.path == "/frame":
36	-	out[i].u = roleFrame
37	-	case c.op == opState && c.path == "/frame":
38	-	out[i].state = stShowing \| stActive
39	-	case c.op == opExtents && c.path == "/frame":
40	-	out[i].rect = frameExt
41	-	case c.op == opMatches:
42	-	if noCollection {
43	-	out[i].err = &busError{name: "org.freedesktop.DBus.Error.UnknownMethod"}
44	-	} else {
45	-	out[i].refs = matches
46	-	}
47	-	case c.op == opChildren && c.path == "/frame":
48	-	out[i].refs = matches
49	-	case c.op == opChildren:
50	-	case c.op == opRole:
51	-	out[i].u = roles[c.path]
52	-	case c.op == opState:
53	-	out[i].state = stShowing \| 1<<stateVisible
54	-	case c.op == opExtents:
55	-	out[i].rect = ext[c.path]
56	-	}
57	-	}
58	-	return out
59	-	}
60	-	}
61	-
62	-	func TestTargetsCoversWindowAndConverts(t *testing.T) {
63	-	tr := newMocktransport(t)
64	-	matches := []ref{{":1.5", "/a"}, {":1.5", "/b"}, {":1.5", "/zero"}}
65	-	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
66	-	[4]int32{4, 30, 800, 600}, matches,
67	-	map[string][4]int32{"/a": {14, 130, 50, 16}, "/b": {104, 230, 20, 20}, "/zero": {5, 5, 0, 0}},
68	-	map[string]uint32{"/a": roleLink, "/b": rolePushButton, "/zero": roleLink}, false))
69	-	s := &Source{tr: tr}
70	-
71	-	windows := []ports.HintWindow{
72	-	{ID: 1, PID: 77, Rect: ports.Rect{X: 100, Y: 200, W: 800, H: 600}, Focused: true},
73	-	{ID: 2, PID: 5, Rect: ports.Rect{X: 0, Y: 0, W: 50, H: 50}},
74	-	}
75	-	targets, covered, err := s.Targets(context.Background(), windows, false)
76	-	require.NoError(t, err)
77	-	require.Equal(t, map[ports.WindowID]bool{1: true}, covered)
78	-	require.Equal(t, []ports.HintTarget{
79	-	{Rect: ports.Rect{X: 110, Y: 300, W: 50, H: 16}, Kind: ports.HintLink, Window: 1},
80	-	{Rect: ports.Rect{X: 200, Y: 400, W: 20, H: 20}, Kind: ports.HintButton, Window: 1},
81	-	}, targets)
82	-	}
83	-
84	-	func TestTargetsLinksOnlyDropsButtons(t *testing.T) {
85	-	tr := newMocktransport(t)
86	-	matches := []ref{{":1.5", "/a"}, {":1.5", "/b"}}
87	-	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
88	-	[4]int32{0, 0, 800, 600}, matches,
89	-	map[string][4]int32{"/a": {14, 130, 50, 16}, "/b": {104, 230, 20, 20}},
90	-	map[string]uint32{"/a": roleLink, "/b": rolePushButton}, false))
91	-	s := &Source{tr: tr}
92	-	targets, _, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}, true)
93	-	require.NoError(t, err)
94	-	require.Len(t, targets, 1)
95	-	require.Equal(t, ports.HintLink, targets[0].Kind)
96	-	}
97	-
98	-	func TestTargetsWalkWithoutCollection(t *testing.T) {
99	-	tr := newMocktransport(t)
100	-	matches := []ref{{":1.5", "/a"}}
101	-	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
102	-	[4]int32{0, 0, 800, 600}, matches,
103	-	map[string][4]int32{"/a": {14, 130, 50, 16}},
104	-	map[string]uint32{"/a": roleLink}, true))
105	-	s := &Source{tr: tr}
106	-	targets, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}, false)
107	-	require.NoError(t, err)
108	-	require.Len(t, targets, 1)
109	-	require.True(t, covered[1])
110	-	}
111	-
112	-	func TestTargetsNoPIDMatchCoversNothing(t *testing.T) {
113	-	tr := newMocktransport(t)
114	-	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree([4]int32{}, nil, nil, nil, false))
115	-	s := &Source{tr: tr}
116	-	targets, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 1234, Rect: ports.Rect{W: 8, H: 6}}}, false)
117	-	require.NoError(t, err)
118	-	require.Empty(t, targets)
119	-	require.Empty(t, covered)
120	-	}
121	-
122	-	func TestTargetsWithoutPIDsAsksNothing(t *testing.T) {
123	-	s := &Source{tr: newMocktransport(t)}
124	-	_, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 8, H: 6}}}, false)
125	-	require.NoError(t, err)
126	-	require.Empty(t, covered)
127	-	}
128	-
129	-	func TestCoverageRequiresLinkOrDocument(t *testing.T) {
130	-	for _, collection := range []bool{true, false} {
131	-	for _, role := range []uint32{rolePushButton, roleDocumentFrame, roleDocumentWeb, roleLink} {
132	-	t.Run(fmt.Sprintf("collection=%t/role=%d", collection, role), func(t *testing.T) {
133	-	tr := newMocktransport(t)
134	-	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
135	-	[4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {10, 20, 30, 40}}, map[string]uint32{"/a": role}, !collection))
136	-	s := &Source{tr: tr}
137	-	targets, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}, false)
138	-	require.NoError(t, err)
139	-	require.Equal(t, role != rolePushButton, covered[1])
140	-	if isDocument(role) {
141	-	require.Empty(t, targets)
142	-	} else {
143	-	require.Len(t, targets, 1)
144	-	}
145	-	})
146	-	}
147	-	}
148	-	}
149	-
150	-	var oneWindow = []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}
151	-
152	-	// New does not touch the bus; the first Targets call connects, once.
153	-	func TestConnectsLazilyOnFirstTargets(t *testing.T) {
154	-	tr := newMocktransport(t)
155	-	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
156	-	[4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {1, 1, 9, 9}}, map[string]uint32{"/a": roleLink}, false))
157	-	c := newMockconnector(t)
158	-	c.EXPECT().Connect(mock.Anything).Return(tr, nil).Once()
159	-	s := &Source{dial: c}
160	-	for range 2 {
161	-	targets, _, err := s.Targets(context.Background(), oneWindow, false)
162	-	require.NoError(t, err)
163	-	require.Len(t, targets, 1)
164	-	}
165	-	}
166	-
167	-	// Without PIDs nothing is asked and nothing is connected.
168	-	func TestNoConnectWithoutWindows(t *testing.T) {
169	-	s := &Source{dial: newMockconnector(t)}
170	-	_, _, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1}}, false)
171	-	require.NoError(t, err)
172	-	}
173	-
174	-	// A failed connection is retried no sooner than retryEvery; meanwhile no
175	-	// targets are returned.
176	-	func TestConnectRetriesWithBackoff(t *testing.T) {
177	-	c := newMockconnector(t)
178	-	c.EXPECT().Connect(mock.Anything).Return(nil, errors.New("down")).Once()
179	-	s := &Source{dial: c}
180	-	for range 3 {
181	-	targets, covered, err := s.Targets(context.Background(), oneWindow, false)
182	-	require.NoError(t, err)
183	-	require.Empty(t, targets)
184	-	require.Empty(t, covered)
185	-	}
186	-	tr := newMocktransport(t)
187	-	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
188	-	[4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {1, 1, 9, 9}}, map[string]uint32{"/a": roleLink}, false))
189	-	c.EXPECT().Connect(mock.Anything).Return(tr, nil).Once()
190	-	s.retryAt = time.Now().Add(-time.Second)
191	-	targets, _, err := s.Targets(context.Background(), oneWindow, false)
192	-	require.NoError(t, err)
193	-	require.Len(t, targets, 1)
194	-	}
195	-
196	-	// A broken connection is closed and dropped; the next attempt reconnects.
197	-	func TestBrokenConnectionIsDropped(t *testing.T) {
198	-	tr := newMocktransport(t)
199	-	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(
200	-	func(_ time.Time, _ time.Duration, calls []call) []reply {
201	-	out := make([]reply, len(calls))
202	-	for i := range out {
203	-	out[i].err = ErrBroken
204	-	}
205	-	return out
206	-	}).Once()
207	-	tr.EXPECT().Close().Return(nil).Once()
208	-	c := newMockconnector(t)
209	-	c.EXPECT().Connect(mock.Anything).Return(tr, nil).Once()
210	-	s := &Source{dial: c}
211	-	_, _, err := s.Targets(context.Background(), oneWindow, false)
212	-	require.Error(t, err)
213	-	require.Nil(t, s.tr)
214	-	targets, _, err := s.Targets(context.Background(), oneWindow, false)
215	-	require.NoError(t, err)
216	-	require.Empty(t, targets) // backoff: no new attempt yet
217	-	}
218	-
219	-	const testWait = 20 * time.Millisecond
220	-
221	-	// blockedConnector returns a connector whose Connect blocks until release is
222	-	func blockedConnector(t testing.T, release <-chan struct{}, tr transport, err error, times int) mockconnector {
223	-	t.Helper()
224	-	c := newMockconnector(t)
225	-	c.EXPECT().Connect(mock.Anything).RunAndReturn(func(context.Context) (transport, error) {
226	-	<-release
227	-	return tr, err
228	-	}).Times(times)
229	-	return c
230	-	}
231	-
232	-	// A silent service cannot hold Targets beyond the wait, is not doubled while
233	-	// in flight, and its late transport is used.
234	-	func TestConnectIsBoundedAndNotDoubled(t *testing.T) {
235	-	release := make(chan struct{})
236	-	tr := newMocktransport(t)
237	-	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
238	-	[4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {1, 1, 9, 9}}, map[string]uint32{"/a": roleLink}, false))
239	-	c := blockedConnector(t, release, tr, nil, 1)
240	-	s := &Source{dial: c, wait: testWait}
241	-	t0 := time.Now()
242	-	targets, _, err := s.Targets(context.Background(), oneWindow, false)
243	-	require.NoError(t, err)
244	-	require.Empty(t, targets)
245	-	require.Less(t, time.Since(t0), testWait+200*time.Millisecond)
246	-	require.NotNil(t, s.attempt)
247	-	// Still in flight: no second Connect (Times(1)), and no blocking.
248	-	targets, _, err = s.Targets(context.Background(), oneWindow, false)
249	-	require.NoError(t, err)
250	-	require.Empty(t, targets)
251	-	// It connects late: the next call uses it.
252	-	close(release)
253	-	require.Eventually(t, func() bool { return len(s.attempt) == 1 }, time.Second, time.Millisecond)
254	-	targets, _, err = s.Targets(context.Background(), oneWindow, false)
255	-	require.NoError(t, err)
256	-	require.Len(t, targets, 1)
257	-	require.Nil(t, s.attempt)
258	-	}
259	-
260	-	// An abandoned attempt that fails applies the backoff, then one new attempt.
261	-	func TestAbandonedAttemptErrorBacksOff(t *testing.T) {
262	-	release := make(chan struct{})
263	-	tr := newMocktransport(t)
264	-	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
265	-	[4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {1, 1, 9, 9}}, map[string]uint32{"/a": roleLink}, false))
266	-	c := newMockconnector(t)
267	-	c.EXPECT().Connect(mock.Anything).RunAndReturn(func(context.Context) (transport, error) {
268	-	<-release
269	-	return nil, errors.New("down")
270	-	}).Once()
271	-	c.EXPECT().Connect(mock.Anything).Return(tr, nil).Once()
272	-	s := &Source{dial: c, wait: testWait}
273	-	targets, _, err := s.Targets(context.Background(), oneWindow, false)
274	-	require.NoError(t, err)
275	-	require.Empty(t, targets)
276	-	close(release)
277	-	require.Eventually(t, func() bool { return len(s.attempt) == 1 }, time.Second, time.Millisecond)
278	-	// The error is collected; the backoff runs from its finish time.
279	-	targets, _, err = s.Targets(context.Background(), oneWindow, false)
280	-	require.NoError(t, err)
281	-	require.Empty(t, targets)
282	-	require.Nil(t, s.attempt)
283	-	require.WithinDuration(t, time.Now().Add(retryEvery), s.retryAt, time.Second)
284	-	targets, _, err = s.Targets(context.Background(), oneWindow, false) // backoff: no Connect
285	-	require.NoError(t, err)
286	-	require.Empty(t, targets)
287	-	s.retryAt = time.Now().Add(-time.Second)
288	-	targets, _, err = s.Targets(context.Background(), oneWindow, false)
289	-	require.NoError(t, err)
290	-	require.Len(t, targets, 1)
291	-	}
292	-
293	-	// Close with an attempt in flight closes the late transport; afterwards
294	-	// Targets finds nothing and never connects.
295	-	func TestCloseReapsLateTransport(t *testing.T) {
296	-	release := make(chan struct{})
297	-	closed := make(chan struct{})
298	-	tr := newMocktransport(t)
299	-	tr.EXPECT().Close().RunAndReturn(func() error { close(closed); return nil }).Once()
300	-	c := blockedConnector(t, release, tr, nil, 1)
301	-	s := &Source{dial: c, wait: testWait}
302	-	_, _, err := s.Targets(context.Background(), oneWindow, false)
303	-	require.NoError(t, err)
304	-	require.NoError(t, s.Close())
305	-	close(release)
306	-	select {
307	-	case <-closed:
308	-	case <-time.After(5 * time.Second):
309	-	t.Fatal("late transport not closed")
310	-	}
311	-	targets, covered, err := s.Targets(context.Background(), oneWindow, false)
312	-	require.NoError(t, err)
313	-	require.Empty(t, targets)
314	-	require.Empty(t, covered)
315	-	require.NoError(t, s.Close()) // idempotent
316	-	}
317	-
318	-	func TestWalkWideTreeStaysWithinBudget(t *testing.T) {
319	-	tr := newMocktransport(t)
320	-	children := make([]ref, 4096)
321	-	for i := range children {
322	-	children[i] = ref{":1.5", fmt.Sprintf("/node%d", i)}
323	-	}
324	-	visited := 0
325	-	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(
326	-	func(_ time.Time, _ time.Duration, calls []call) []reply {
327	-	visited += len(calls) / 3
328	-	require.LessOrEqual(t, visited, maxWalkNodes)
329	-	rep := make([]reply, len(calls))
330	-	for i, c := range calls {
331	-	switch c.op {
332	-	case opChildren:
333	-	rep[i].refs = children
334	-	case opRole:
335	-	rep[i].u = roleLink
336	-	case opState:
337	-	rep[i].state = stShowing \| 1<<stateVisible
338	-	}
339	-	}
340	-	return rep
341	-	}).Twice()
342	-	s := &Source{tr: tr}
343	-	// A full second level replies with another 4096 children each. There is
344	-	// no remaining budget, so none of those millions of refs may be copied.
345	-	var out []ref
346	-	var before, after runtime.MemStats
347	-	runtime.ReadMemStats(&before)
348	-	out = s.walk(func(calls []call) []reply { return tr.Do(time.Time{}, time.Second, calls) }, ref{":1.5", "/frame"}, false)
349	-	runtime.ReadMemStats(&after)
350	-	require.Len(t, out, maxWalkNodes)
351	-	require.Equal(t, maxWalkNodes, visited)
352	-	// Normal replies and call batches take only a few MB; the old append-
353	-	// then-truncate path copied hundreds of MB for this tree.
354	-	require.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(32<<20))
355	-	}

D internal/adapters/atspi/transport.go +0 -362

@@ -1,362 +0,0 @@
1	-	package atspi
2	-
3	-	import (
4	-	"errors"
5	-	"fmt"
6	-	"strings"
7	-	"sync/atomic"
8	-	"syscall"
9	-	"time"
10	-
11	-	"github.com/bnema/zerobus"
12	-	"golang.org/x/sys/unix"
13	-	)
14	-
15	-	// D-Bus names, paths and interfaces.
16	-	const (
17	-	busDaemon = "org.freedesktop.DBus"
18	-	busDaemonPath = "/org/freedesktop/DBus"
19	-	ifaceProps = "org.freedesktop.DBus.Properties"
20	-
21	-	a11yBusName = "org.a11y.Bus"
22	-	a11yBusPath = "/org/a11y/bus"
23	-	a11yStatus = "org.a11y.Status"
24	-	registryName = "org.a11y.atspi.Registry"
25	-	registryRoot = "/org/a11y/atspi/accessible/root"
26	-
27	-	ifaceAccessible = "org.a11y.atspi.Accessible"
28	-	ifaceComponent = "org.a11y.atspi.Component"
29	-	ifaceCollection = "org.a11y.atspi.Collection"
30	-
31	-	// coordWindow is ATSPI_COORD_TYPE_WINDOW: extents relative to the
32	-	// top-level window.
33	-	coordWindow = 1
34	-
35	-	// maxInflight bounds the calls sent before a reply is read, so the bus
36	-	// never waits on a full socket while this side waits on it.
37	-	maxInflight = 64
38	-	// maxRefs bounds the objects read from one reply.
39	-	maxRefs = 4096
40	-	)
41	-
42	-	// op is a call the source makes.
43	-	type op uint8
44	-
45	-	const (
46	-	opChildren op = iota // Accessible.GetChildren -> refs
47	-	opPID // GetConnectionUnixProcessID(name) -> u
48	-	opRole // Accessible.GetRole -> u
49	-	opState // Accessible.GetState -> state
50	-	opExtents // Component.GetExtents(arg) -> rect
51	-	opMatches // Collection.GetMatches(rule) -> refs
52	-	)
53	-
54	-	// ref is an accessible object: a bus name and a path.
55	-	type ref struct{ name, path string }
56	-
57	-	// call is one request of a batch. name and path address the object; for
58	-	// opPID, name is the bus name asked about.
59	-	type call struct {
60	-	op op
61	-	name, path string
62	-	arg uint32
63	-	rule matchRule
64	-	}
65	-
66	-	// reply answers the call at the same index of the batch.
67	-	type reply struct {
68	-	err error
69	-	refs []ref
70	-	u uint32
71	-	state uint64
72	-	rect [4]int32 // x, y, w, h
73	-	}
74	-
75	-	// transport sends batches of calls on one bus connection. It is used by one
76	-	// goroutine at a time.
77	-	type transport interface {
78	-	// Do sends calls pipelined and returns one reply per call, in order.
79	-	// The batch ends at deadline. A call not answered idle after it was
80	-	// sent gets ErrTimeout. When the connection fails or the deadline
81	-	// finds calls unanswered, those calls get an error wrapping
82	-	// ErrBroken: the connection must be dropped.
83	-	Do(deadline time.Time, idle time.Duration, calls []call) []reply
84	-	Close() error
85	-	}
86	-
87	-	// ErrTimeout is the error of a call not answered in time.
88	-	var ErrTimeout = errors.New("atspi: call timed out")
89	-
90	-	// busError is a D-Bus error reply.
91	-	type busError struct{ name, msg string }
92	-
93	-	func (e *busError) Error() string {
94	-	if e.msg == "" {
95	-	return e.name
96	-	}
97	-	return e.name + ": " + e.msg
98	-	}
99	-
100	-	// conn is a transport on a zerobus connection.
101	-	type conn struct {
102	-	c *zerobus.Conn
103	-	raw syscall.RawConn
104	-	}
105	-
106	-	func newConn(c zerobus.Conn) (conn, error) {
107	-	raw, err := c.SyscallConn()
108	-	if err != nil {
109	-	_ = c.Close()
110	-	return nil, err
111	-	}
112	-	return &conn{c: c, raw: raw}, nil
113	-	}
114	-
115	-	func (b *conn) Close() error { return b.c.Close() }
116	-
117	-	// readable waits until a message can be read or d passes.
118	-	func (b *conn) readable(d time.Duration) (bool, error) {
119	-	if b.c.Buffered() {
120	-	return true, nil
121	-	}
122	-	ms := int((d + time.Millisecond - 1) / time.Millisecond)
123	-	if ms < 1 {
124	-	ms = 1
125	-	}
126	-	var n int
127	-	var perr error
128	-	if err := b.raw.Control(func(fd uintptr) {
129	-	fds := []unix.PollFd{{Fd: int32(fd), Events: unix.POLLIN}}
130	-	for {
131	-	n, perr = unix.Poll(fds, ms)
132	-	if perr != unix.EINTR {
133	-	return
134	-	}
135	-	}
136	-	}); err != nil {
137	-	return false, err
138	-	}
139	-	return n > 0, perr
140	-	}
141	-
142	-	// ErrBroken is the error of calls on a connection that failed or was closed
143	-	// at the deadline; the owner must drop it and connect again.
144	-	var ErrBroken = errors.New("atspi: connection broken")
145	-
146	-	// errDeadline is the error of calls still unanswered at the batch deadline:
147	-	// late replies and half-read messages leave the connection unusable.
148	-	var errDeadline = fmt.Errorf("%w: %w", ErrBroken, ErrTimeout)
149	-
150	-	// watchdogGrace is how long after the deadline the watchdog closes a socket
151	-	// that the batch loop could not leave by itself (a stuck read or write).
152	-	const watchdogGrace = 20 * time.Millisecond
153	-
154	-	// Do sends the calls pipelined. A watchdog closes the socket shortly after
155	-	// deadline, so neither a blocked write nor a read stuck in the middle of a
156	-	// message can outlive the batch; the connection is then broken. Each call
157	-	// has its own idle timeout, counted from its send: traffic for other calls
158	-	// does not extend it.
159	-	func (b *conn) Do(deadline time.Time, idle time.Duration, calls []call) []reply {
160	-	out := make([]reply, len(calls))
161	-	answered := make([]bool, len(calls))
162	-	type inflight struct {
163	-	idx int
164	-	sent time.Time
165	-	}
166	-	pending := make(map[uint32]inflight, min(len(calls), maxInflight))
167	-	next, done := 0, 0
168	-	var fired atomic.Bool
169	-	watchdog := time.AfterFunc(time.Until(deadline)+watchdogGrace, func() {
170	-	fired.Store(true)
171	-	_ = b.c.Close()
172	-	})
173	-	defer watchdog.Stop()
174	-	// finish gives err to every call without an answer.
175	-	finish := func(err error) []reply {
176	-	if !errors.Is(err, ErrBroken) && (fired.Load() \|\| !errors.Is(err, ErrTimeout)) {
177	-	err = fmt.Errorf("%w: %w", ErrBroken, err)
178	-	}
179	-	if errors.Is(err, ErrBroken) {
180	-	_ = b.c.Close() // late replies would desynchronize the next batch
181	-	}
182	-	for i := range out {
183	-	if !answered[i] {
184	-	out[i].err = err
185	-	}
186	-	}
187	-	return out
188	-	}
189	-	for done < len(calls) {
190	-	for next < len(calls) && len(pending) < maxInflight {
191	-	if !time.Now().Before(deadline) {
192	-	return finish(errDeadline)
193	-	}
194	-	serial, err := b.send(calls[next])
195	-	if err != nil {
196	-	return finish(err)
197	-	}
198	-	pending[serial] = inflight{idx: next, sent: time.Now()}
199	-	next++
200	-	}
201	-	// Calls that waited longer than idle get no reply.
202	-	now := time.Now()
203	-	var oldest time.Time
204	-	for serial, p := range pending {
205	-	if now.Sub(p.sent) >= idle {
206	-	out[p.idx].err = ErrTimeout
207	-	answered[p.idx] = true
208	-	done++
209	-	delete(pending, serial)
210	-	} else if oldest.IsZero() \|\| p.sent.Before(oldest) {
211	-	oldest = p.sent
212	-	}
213	-	}
214	-	if len(pending) == 0 {
215	-	continue
216	-	}
217	-	wait := min(time.Until(oldest.Add(idle)), time.Until(deadline))
218	-	if wait <= 0 {
219	-	if !time.Now().Before(deadline) {
220	-	break
221	-	}
222	-	continue
223	-	}
224	-	ok, err := b.readable(wait)
225	-	if err != nil {
226	-	return finish(err)
227	-	}
228	-	if !ok {
229	-	continue
230	-	}
231	-	m, err := b.c.ReadMessage()
232	-	if err != nil {
233	-	return finish(err)
234	-	}
235	-	p, found := pending[m.ReplySerial]
236	-	if !found \|\| (m.Type != zerobus.TypeMethodReturn && m.Type != zerobus.TypeError) {
237	-	// A signal, or the late reply of an earlier batch.
238	-	continue
239	-	}
240	-	delete(pending, m.ReplySerial)
241	-	decode(m, calls[p.idx], &out[p.idx])
242	-	answered[p.idx] = true
243	-	done++
244	-	}
245	-	if done == len(calls) {
246	-	return out
247	-	}
248	-	return finish(errDeadline)
249	-	}
250	-
251	-	// send builds and sends one call and returns its serial.
252	-	func (b *conn) send(c call) (uint32, error) {
253	-	switch c.op {
254	-	case opChildren:
255	-	b.c.NewCall(c.name, c.path, ifaceAccessible, "GetChildren", "")
256	-	case opPID:
257	-	b.c.NewCall(busDaemon, busDaemonPath, busDaemon, "GetConnectionUnixProcessID", "s").Str(c.name)
258	-	case opRole:
259	-	b.c.NewCall(c.name, c.path, ifaceAccessible, "GetRole", "")
260	-	case opState:
261	-	b.c.NewCall(c.name, c.path, ifaceAccessible, "GetState", "")
262	-	case opExtents:
263	-	b.c.NewCall(c.name, c.path, ifaceComponent, "GetExtents", "u").Uint32(c.arg)
264	-	case opMatches:
265	-	e := b.c.NewCall(c.name, c.path, ifaceCollection, "GetMatches", "(aiia{ss}iaiiasib)uib")
266	-	encodeRule(e, c.rule)
267	-	e.Uint32(sortCanonical)
268	-	e.Int32(matchCount)
269	-	e.Bool(true)
270	-	}
271	-	return b.c.Send()
272	-	}
273	-
274	-	// encodeRule writes the MatchRule struct (aiia{ss}iaiiasib): states, state
275	-	// match, attributes, attribute match, roles, role match, interfaces,
276	-	// interface match, invert.
277	-	func encodeRule(e *zerobus.Encoder, r matchRule) {
278	-	e.Struct()
279	-	a := e.BeginArray('i')
280	-	for _, w := range stateBits(stateShowing, stateVisible) {
281	-	e.Int32(w)
282	-	}
283	-	e.EndArray(a)
284	-	e.Int32(matchAll)
285	-	m := e.BeginArray('{')
286	-	e.EndArray(m)
287	-	e.Int32(matchAll)
288	-	a = e.BeginArray('i')
289	-	for _, w := range roleBits(r.roles()) {
290	-	e.Int32(w)
291	-	}
292	-	e.EndArray(a)
293	-	e.Int32(matchAny)
294	-	s := e.BeginArray('s')
295	-	e.EndArray(s)
296	-	e.Int32(matchAll)
297	-	e.Bool(false)
298	-	}
299	-
300	-	// decode reads the reply m of c into out.
301	-	func decode(m zerobus.Message, c call, out reply) {
302	-	if m.Type == zerobus.TypeError {
303	-	be := &busError{name: strings.Clone(m.ErrorName)}
304	-	if strings.HasPrefix(m.Signature, "s") {
305	-	be.msg = strings.Clone(m.Body().Str())
306	-	}
307	-	out.err = be
308	-	return
309	-	}
310	-	r := m.Body()
311	-	want := func(sig string) bool {
312	-	if m.Signature != sig {
313	-	out.err = fmt.Errorf("atspi: reply signature %q, want %q", m.Signature, sig)
314	-	return false
315	-	}
316	-	return true
317	-	}
318	-	switch c.op {
319	-	case opChildren, opMatches:
320	-	if !want("a(so)") {
321	-	return
322	-	}
323	-	end := r.Array('(')
324	-	for r.More(end) {
325	-	if len(out.refs) >= maxRefs {
326	-	break
327	-	}
328	-	r.Struct()
329	-	name := r.Str()
330	-	path := r.ObjectPath()
331	-	if r.Err() != nil {
332	-	break
333	-	}
334	-	out.refs = append(out.refs, ref{strings.Clone(name), strings.Clone(path)})
335	-	}
336	-	case opPID, opRole:
337	-	if want("u") {
338	-	out.u = r.Uint32()
339	-	}
340	-	case opState:
341	-	if !want("au") {
342	-	return
343	-	}
344	-	end := r.Array('u')
345	-	var words [2]uint32
346	-	for i := 0; r.More(end); i++ {
347	-	w := r.Uint32()
348	-	if i < len(words) {
349	-	words[i] = w
350	-	}
351	-	}
352	-	out.state = uint64(words[0]) \| uint64(words[1])<<32
353	-	case opExtents:
354	-	if want("(iiii)") {
355	-	r.Struct()
356	-	out.rect = [4]int32{r.Int32(), r.Int32(), r.Int32(), r.Int32()}
357	-	}
358	-	}
359	-	if out.err == nil && r.Err() != nil {
360	-	out.err = r.Err()
361	-	}
362	-	}

D internal/adapters/atspi/transport_mock_test.go +0 -156

@@ -1,156 +0,0 @@
1	-	// Code generated by mockery; DO NOT EDIT.
2	-	// github.com/vektra/mockery
3	-	// template: testify
4	-
5	-	package atspi
6	-
7	-	import (
8	-	"time"
9	-
10	-	mock "github.com/stretchr/testify/mock"
11	-	)
12	-
13	-	// newMocktransport creates a new instance of mocktransport. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
14	-	// The first argument is typically a *testing.T value.
15	-	func newMocktransport(t interface {
16	-	mock.TestingT
17	-	Cleanup(func())
18	-	}) *mocktransport {
19	-	if helper, ok := t.(interface{ Helper() }); ok {
20	-	helper.Helper()
21	-	}
22	-
23	-	mock := &mocktransport{}
24	-	mock.Mock.Test(t)
25	-
26	-	t.Cleanup(func() {
27	-	if helper, ok := t.(interface{ Helper() }); ok {
28	-	helper.Helper()
29	-	}
30	-	mock.AssertExpectations(t)
31	-	})
32	-
33	-	return mock
34	-	}
35	-
36	-	// mocktransport is an autogenerated mock type for the transport type
37	-	type mocktransport struct {
38	-	mock.Mock
39	-	}
40	-
41	-	type mocktransport_Expecter struct {
42	-	mock *mock.Mock
43	-	}
44	-
45	-	func (_m mocktransport) EXPECT() mocktransport_Expecter {
46	-	return &mocktransport_Expecter{mock: &_m.Mock}
47	-	}
48	-
49	-	// Close provides a mock function for the type mocktransport
50	-	func (_mock *mocktransport) Close() error {
51	-	ret := _mock.Called()
52	-
53	-	if len(ret) == 0 {
54	-	panic("no return value specified for Close")
55	-	}
56	-
57	-	var r0 error
58	-	if returnFunc, ok := ret.Get(0).(func() error); ok {
59	-	r0 = returnFunc()
60	-	} else {
61	-	r0 = ret.Error(0)
62	-	}
63	-	return r0
64	-	}
65	-
66	-	// mocktransport_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
67	-	type mocktransport_Close_Call struct {
68	-	*mock.Call
69	-	}
70	-
71	-	// Close is a helper method to define mock.On call
72	-	func (_e mocktransport_Expecter) Close() mocktransport_Close_Call {
73	-	return &mocktransport_Close_Call{Call: _e.mock.On("Close")}
74	-	}
75	-
76	-	func (_c mocktransport_Close_Call) Run(run func()) mocktransport_Close_Call {
77	-	_c.Call.Run(func(args mock.Arguments) {
78	-	run()
79	-	})
80	-	return _c
81	-	}
82	-
83	-	func (_c mocktransport_Close_Call) Return(err error) mocktransport_Close_Call {
84	-	_c.Call.Return(err)
85	-	return _c
86	-	}
87	-
88	-	func (_c mocktransport_Close_Call) RunAndReturn(run func() error) mocktransport_Close_Call {
89	-	_c.Call.Return(run)
90	-	return _c
91	-	}
92	-
93	-	// Do provides a mock function for the type mocktransport
94	-	func (_mock *mocktransport) Do(deadline time.Time, idle time.Duration, calls []call) []reply {
95	-	ret := _mock.Called(deadline, idle, calls)
96	-
97	-	if len(ret) == 0 {
98	-	panic("no return value specified for Do")
99	-	}
100	-
101	-	var r0 []reply
102	-	if returnFunc, ok := ret.Get(0).(func(time.Time, time.Duration, []call) []reply); ok {
103	-	r0 = returnFunc(deadline, idle, calls)
104	-	} else {
105	-	if ret.Get(0) != nil {
106	-	r0 = ret.Get(0).([]reply)
107	-	}
108	-	}
109	-	return r0
110	-	}
111	-
112	-	// mocktransport_Do_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Do'
113	-	type mocktransport_Do_Call struct {
114	-	*mock.Call
115	-	}
116	-
117	-	// Do is a helper method to define mock.On call
118	-	// - deadline time.Time
119	-	// - idle time.Duration
120	-	// - calls []call
121	-	func (_e mocktransport_Expecter) Do(deadline any, idle any, calls any) mocktransport_Do_Call {
122	-	return &mocktransport_Do_Call{Call: _e.mock.On("Do", deadline, idle, calls)}
123	-	}
124	-
125	-	func (_c mocktransport_Do_Call) Run(run func(deadline time.Time, idle time.Duration, calls []call)) mocktransport_Do_Call {
126	-	_c.Call.Run(func(args mock.Arguments) {
127	-	var arg0 time.Time
128	-	if args[0] != nil {
129	-	arg0 = args[0].(time.Time)
130	-	}
131	-	var arg1 time.Duration
132	-	if args[1] != nil {
133	-	arg1 = args[1].(time.Duration)
134	-	}
135	-	var arg2 []call
136	-	if args[2] != nil {
137	-	arg2 = args[2].([]call)
138	-	}
139	-	run(
140	-	arg0,
141	-	arg1,
142	-	arg2,
143	-	)
144	-	})
145	-	return _c
146	-	}
147	-
148	-	func (_c mocktransport_Do_Call) Return(replyMoqParams []reply) mocktransport_Do_Call {
149	-	_c.Call.Return(replyMoqParams)
150	-	return _c
151	-	}
152	-
153	-	func (_c mocktransport_Do_Call) RunAndReturn(run func(deadline time.Time, idle time.Duration, calls []call) []reply) mocktransport_Do_Call {
154	-	_c.Call.Return(run)
155	-	return _c
156	-	}

M internal/adapters/config/config.go +43 -40


@@ -116,7 +116,7 @@ var actions = map[string]bool{
116		"toggle-floating": true,	116		"toggle-floating": true,
117		"hint": true,	117		"hint": true,
118		}	118		}
119	-	var components = map[string]bool{"core": true, "wayland": true, "input": true, "drm": true, "seat": true, "render": true, "sync": true, "config": true, "app": true}	119	+	var components = map[string]bool{"core": true, "wayland": true, "input": true, "drm": true, "seat": true, "render": true, "sync": true, "config": true, "app": true, "hints": true}
120		var color = regexp.MustCompile(`^#[0-9a-fA-F]{6}$`)	120		var color = regexp.MustCompile(`^#[0-9a-fA-F]{6}$`)
121			121
122		// columnStep is the +N or -N step of set-column-width, in cells, N from 1	122		// columnStep is the +N or -N step of set-column-width, in cells, N from 1

@@ -166,7 +166,8 @@ func Defaults() ports.Config {
166		c.Terminal.AutoOpen = "first"	166		c.Terminal.AutoOpen = "first"
167		// Hint mode is experimental: off unless the config turns it on.	167		// Hint mode is experimental: off unless the config turns it on.
168		c.Hint.Enable = false	168		c.Hint.Enable = false
169	-	c.Hint.TextClick = ports.ModCtrl \| ports.ModShift	169	+	c.Hint.Detector = defaultHintDetector
			170	+	c.Hint.Model = defaultHintModel()
170		c.Xwayland = "xwayland-satellite"	171		c.Xwayland = "xwayland-satellite"
171		c.Background.Color = "#111111"	172		c.Background.Color = "#111111"
172		c.Floating.Dim = 0.3	173		c.Floating.Dim = 0.3

@@ -895,21 +896,27 @@ func set(c *ports.Config, key, v string) error {
895		} else {	896		} else {
896		c.Hint.Foreground = v	897		c.Hint.Foreground = v
897		}	898		}
898	-	case "hint.targets":	899	+	case "hint.detector":
899	-	switch v {	900	+	if v == "" {
900	-	case "clickable":	901	+	v = defaultHintDetector
901	-	c.Hint.LinksOnly = false
902	-	case "links":
903	-	c.Hint.LinksOnly = true
904	-	default:
905	-	return fmt.Errorf("must be clickable or links")
906		}	902		}
907	-	case "hint.text-click":	903	+	c.Hint.Detector = expandHome(v)
908	-	m, err := parseTextClick(v)	904	+	case "hint.model":
909	-	if err != nil {	905	+	if v == "" {
910	-	return err	906	+	c.Hint.Model = defaultHintModel()
			907	+	break
911		}	908		}
912	-	c.Hint.TextClick = m	909	+	c.Hint.Model = expandHome(v)
			910	+	case "hint.threshold":
			911	+	if v == "" {
			912	+	c.Hint.Threshold = 0
			913	+	break
			914	+	}
			915	+	n, err := strconv.ParseFloat(v, 64)
			916	+	if err != nil \|\| math.IsNaN(n) \|\| n < 0.01 \|\| n > 0.99 {
			917	+	return fmt.Errorf("must be a number from 0.01 to 0.99")
			918	+	}
			919	+	c.Hint.Threshold = n
913		case "focus.flash-width":	920		case "focus.flash-width":
914		n, err := strconv.Atoi(v)	921		n, err := strconv.Atoi(v)
915		if err != nil \|\| n < 1 \|\| n > 8 {	922		if err != nil \|\| n < 1 \|\| n > 8 {

@@ -1333,34 +1340,30 @@ func checkAction(v string) error {
1333		return nil	1340		return nil
1334		}	1341		}
1335			1342
1336	-	// parseTextClick parses hint.text-click: `+`-separated modifier names as in	1343	+	const defaultHintDetector = "nefer-hints"
1337	-	// binds, or none. cmd is an alias of super here (it is not resolved through	1344	+
1338	-	// keyboard.cmd, so the result does not depend on key order).	1345	+	// defaultHintModel is $XDG_DATA_HOME/nefer-hints/models, or
1339	-	func parseTextClick(v string) (ports.Mods, error) {	1346	+	// ~/.local/share/nefer-hints/models when XDG_DATA_HOME is unset.
1340	-	if strings.ToLower(v) == "none" {	1347	+	func defaultHintModel() string {
1341	-	return 0, nil	1348	+	base := os.Getenv("XDG_DATA_HOME")
1342	-	}	1349	+	if base == "" {
1343	-	var mods ports.Mods	1350	+	home, err := os.UserHomeDir()
1344	-	for _, p := range strings.Split(v, "+") {	1351	+	if err != nil {
1345	-	var m ports.Mods	1352	+	return ""
1346	-	switch strings.ToLower(strings.TrimSpace(p)) {
1347	-	case "shift":
1348	-	m = ports.ModShift
1349	-	case "ctrl":
1350	-	m = ports.ModCtrl
1351	-	case "alt":
1352	-	m = ports.ModAlt
1353	-	case "super", "cmd":
1354	-	m = ports.ModSuper
1355	-	default:
1356	-	return 0, fmt.Errorf("unknown modifier %q (use shift, ctrl, alt, super or cmd joined by +, or none)", p)
1357		}	1353		}
1358	-	if mods&m != 0 {	1354	+	base = filepath.Join(home, ".local", "share")
1359	-	return 0, fmt.Errorf("duplicate modifier %q", p)	1355	+	}
			1356	+	return filepath.Join(base, "nefer-hints", "models")
			1357	+	}
			1358	+
			1359	+	// expandHome replaces a leading ~/ with the home directory.
			1360	+	func expandHome(p string) string {
			1361	+	if rest, ok := strings.CutPrefix(p, "~/"); ok {
			1362	+	if home, err := os.UserHomeDir(); err == nil {
			1363	+	return filepath.Join(home, rest)
1360		}	1364		}
1361	-	mods \|= m
1362		}	1365		}
1363	-	return mods, nil	1366	+	return p
1364		}	1367		}
1365			1368
1366		// parseCombo turns `ctrl+cmd+é` into the canonical `Cmd+Ctrl+eacute`:	1369		// parseCombo turns `ctrl+cmd+é` into the canonical `Cmd+Ctrl+eacute`:

M internal/adapters/config/config_test.go +51 -41


@@ -954,26 +954,61 @@ func TestLoadRejectsFIFO(t *testing.T) {
954		}	954		}
955		}	955		}
956			956
957	-	func TestHintTargets(t *testing.T) {	957	+	func TestHintBind(t *testing.T) {
958	-	if Defaults().Hint.LinksOnly {	958	+	c, w := parseString(t, "bind.cmd+g = hint\n")
959	-	t.Fatal("hint targets default must be clickable")	959	+	if len(w) != 0 \|\| c.Binds["Cmd+g"] != "hint" {
			960	+	t.Fatal(c.Binds["Cmd+g"], w)
960		}	961		}
961	-	c, w := parseString(t, "hint.targets = links\n")	962	+	}
962	-	if !c.Hint.LinksOnly \|\| len(w) != 0 {	963	+
963	-	t.Fatal(c.Hint, w)	964	+	func TestHintRemovedKeysAreUnknown(t *testing.T) {
			965	+	for _, line := range []string{"hint.targets = links", "hint.text-click = none"} {
			966	+	_, w := parseString(t, line+"\n")
			967	+	if len(w) != 1 {
			968	+	t.Errorf("%q: %v", line, w)
			969	+	}
964		}	970		}
965	-	c, w = parseString(t, "hint.targets = clickable\n")	971	+	}
966	-	if c.Hint.LinksOnly \|\| len(w) != 0 {	972	+
967	-	t.Fatal(c.Hint, w)	973	+	func TestHintDetectorConfig(t *testing.T) {
			974	+	t.Setenv("HOME", "/home/u")
			975	+	t.Setenv("XDG_DATA_HOME", "")
			976	+	d := Defaults().Hint
			977	+	if d.Detector != "nefer-hints" \|\| d.Model != "/home/u/.local/share/nefer-hints/models" \|\| d.Threshold != 0 {
			978	+	t.Fatalf("defaults: %+v", d)
968		}	979		}
969	-	// An invalid value warns and keeps the default.	980	+	t.Setenv("XDG_DATA_HOME", "/data")
970	-	c, w = parseString(t, "hint.targets = buttons\n")	981	+	if m := Defaults().Hint.Model; m != "/data/nefer-hints/models" {
971	-	if c.Hint.LinksOnly \|\| len(w) != 1 {	982	+	t.Fatal(m)
972	-	t.Fatal(c.Hint, w)
973		}	983		}
974	-	c, w = parseString(t, "bind.cmd+g = hint\n")	984	+	c, w := parseString(t, "hint.detector = ~/bin/nh\nhint.model = ~/m\nhint.threshold = 0.3\n")
975	-	if len(w) != 0 \|\| c.Binds["Cmd+g"] != "hint" {	985	+	if len(w) != 0 \|\| c.Hint.Detector != "/home/u/bin/nh" \|\| c.Hint.Model != "/home/u/m" \|\| c.Hint.Threshold != 0.3 {
976	-	t.Fatal(c.Binds["Cmd+g"], w)	986	+	t.Fatalf("%+v %v", c.Hint, w)
			987	+	}
			988	+	c, w = parseString(t, "hint.detector = /opt/nh\nhint.model = /opt/models\n")
			989	+	if len(w) != 0 \|\| c.Hint.Detector != "/opt/nh" \|\| c.Hint.Model != "/opt/models" {
			990	+	t.Fatalf("%+v %v", c.Hint, w)
			991	+	}
			992	+	for _, input := range []string{"", "hint.threshold =\n"} {
			993	+	c, w := parseString(t, input)
			994	+	if len(w) != 0 \|\| c.Hint.Threshold != 0 {
			995	+	t.Errorf("unset threshold %q: %v %v", input, c.Hint.Threshold, w)
			996	+	}
			997	+	}
			998	+	c, w = parseString(t, "hint.threshold = 0.3\nhint.threshold =\n")
			999	+	if len(w) != 1 \|\| c.Hint.Threshold != 0 { // duplicate-key warning only
			1000	+	t.Fatalf("cleared threshold: %v %v", c.Hint.Threshold, w)
			1001	+	}
			1002	+	for _, v := range []string{"0", "0.009", "1", "0.995", "NaN", "Inf", "-Inf", "high", "-1"} {
			1003	+	c, w := parseString(t, "hint.threshold = "+v+"\n")
			1004	+	if len(w) != 1 \|\| c.Hint.Threshold != 0 {
			1005	+	t.Errorf("hint.threshold = %s: %v %v", v, c.Hint.Threshold, w)
			1006	+	}
			1007	+	}
			1008	+	for _, v := range []string{"0.01", "0.99"} {
			1009	+	if c, w := parseString(t, "hint.threshold = "+v+"\n"); len(w) != 0 \|\| c.Hint.Threshold == 0 {
			1010	+	t.Errorf("hint.threshold = %s: %v", v, w)
			1011	+	}
977		}	1012		}
978		}	1013		}
979			1014

@@ -1021,31 +1056,6 @@ func TestHintSizeBounds(t *testing.T) {
1021		}	1056		}
1022		}	1057		}
1023			1058
1024	-	func TestHintTextClick(t *testing.T) {
1025	-	if got := Defaults().Hint.TextClick; got != ports.ModCtrl\|ports.ModShift {
1026	-	t.Fatalf("default %d", got)
1027	-	}
1028	-	for in, want := range map[string]ports.Mods{
1029	-	"ctrl+shift": ports.ModCtrl \| ports.ModShift,
1030	-	"Alt": ports.ModAlt,
1031	-	"super": ports.ModSuper,
1032	-	"cmd+shift": ports.ModSuper \| ports.ModShift,
1033	-	"none": 0,
1034	-	"shift+ctrl+alt": ports.ModShift \| ports.ModCtrl \| ports.ModAlt,
1035	-	} {
1036	-	c, w := parseString(t, "hint.text-click = "+in+"\n")
1037	-	if c.Hint.TextClick != want \|\| len(w) != 0 {
1038	-	t.Errorf("%q: %d %v", in, c.Hint.TextClick, w)
1039	-	}
1040	-	}
1041	-	for _, bad := range []string{"hyper", "ctrl+", "ctrl+ctrl", "ctrl+none", ""} {
1042	-	c, w := parseString(t, "hint.text-click = "+bad+"\n")
1043	-	if len(w) != 1 \|\| c.Hint.TextClick != ports.ModCtrl\|ports.ModShift {
1044	-	t.Errorf("%q: %d %v", bad, c.Hint.TextClick, w)
1045	-	}
1046	-	}
1047	-	}
1048	-
1049		func TestHintEnable(t *testing.T) {	1059		func TestHintEnable(t *testing.T) {
1050		if Defaults().Hint.Enable {	1060		if Defaults().Hint.Enable {
1051		t.Fatal("hint mode must be off by default")	1061		t.Fatal("hint mode must be off by default")

D internal/adapters/hints/benchmark_test.go +0 -81

@@ -1,81 +0,0 @@
1	-	package hints
2	-
3	-	import (
4	-	"context"
5	-	"image"
6	-	"image/draw"
7	-	"image/png"
8	-	"os"
9	-	"os/exec"
10	-	"path/filepath"
11	-	"strconv"
12	-	"testing"
13	-	"time"
14	-	)
15	-
16	-	// TestFrameBenchmark logs counts and timings, never filenames or OCR text.
17	-	func TestFrameBenchmark(t *testing.T) {
18	-	dir := os.Getenv("NEFERWL_HINT_BENCH_DIR")
19	-	if dir == "" {
20	-	t.Skip("set NEFERWL_HINT_BENCH_DIR to a local PNG directory")
21	-	}
22	-	if _, err := exec.LookPath("tesseract"); err != nil {
23	-	t.Skip("tesseract not on PATH")
24	-	}
25	-	files, err := filepath.Glob(filepath.Join(dir, "*.png"))
26	-	if err != nil \|\| len(files) == 0 {
27	-	t.Fatal("no PNG frames")
28	-	}
29	-	// Settings under test: NEFERWL_HINT_BENCH_WORKERS=<n>, NEFERWL_HINT_BENCH_PNG=speed\|none.
30	-	if v, _ := strconv.Atoi(os.Getenv("NEFERWL_HINT_BENCH_WORKERS")); v > 0 {
31	-	ocrWorkers = v
32	-	}
33	-	if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
34	-	rawPGM = false
35	-	}
36	-	if os.Getenv("NEFERWL_HINT_BENCH_PNG") == "none" {
37	-	pngLevel = png.NoCompression
38	-	}
39	-	for i, file := range files {
40	-	f, err := os.Open(file)
41	-	if err != nil {
42	-	t.Fatal("open frame failed")
43	-	}
44	-	src, err := png.Decode(f)
45	-	_ = f.Close()
46	-	if err != nil {
47	-	t.Fatal("decode frame failed")
48	-	}
49	-	img := image.NewRGBA(image.Rect(0, 0, src.Bounds().Dx(), src.Bounds().Dy()))
50	-	draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
51	-	start := time.Now()
52	-	words, bands, err := readWords(context.Background(), toGray(img))
53	-	if err != nil {
54	-	t.Fatal("recognition failed")
55	-	}
56	-	t.Logf("frame=%d px=%d ocr_ms=%.1f words=%d links=%d bands=%d", i, img.Bounds().Dx()*img.Bounds().Dy(), ms(time.Since(start)), len(words), len(textLinks(words, 1)), bands)
57	-	if os.Getenv("NEFERWL_HINT_BENCH_STRIPS") == "" {
58	-	continue
59	-	}
60	-	// Experimental strips use known OCR word regions, not pixel targets.
61	-	// This is an oracle comparison, not a production optimization.
62	-	lines := make([]image.Rectangle, 0, len(words))
63	-	for _, word := range words {
64	-	lines = append(lines, word.Rect)
65	-	}
66	-	for _, normalized := range []bool{false, true} {
67	-	begin := time.Now()
68	-	compact, _, err := readStrips(context.Background(), img, lines, 8, normalized)
69	-	if err != nil {
70	-	t.Fatal("strip recognition failed")
71	-	}
72	-	t.Logf("frame=%d strips normalized=%t ocr_ms=%.1f words=%d links=%d", i, normalized, ms(time.Since(begin)), len(compact), len(textLinks(compact, 1)))
73	-	}
74	-	}
75	-	}
76	-
77	-	func toGray(img image.RGBA) image.Gray {
78	-	g := image.NewGray(img.Bounds())
79	-	draw.Draw(g, g.Bounds(), img, img.Bounds().Min, draw.Src)
80	-	return g
81	-	}

A internal/adapters/hints/detector.go +448 -0

@@ -0,0 +1,448 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"bytes"
	6	+	"context"
	7	+	"encoding/binary"
	8	+	"errors"
	9	+	"fmt"
	10	+	"io"
	11	+	"math"
	12	+	"os"
	13	+	"os/exec"
	14	+	"strconv"
	15	+	"sync"
	16	+	"syscall"
	17	+	"time"
	18	+
	19	+	"github.com/bnema/neferwl/internal/adapters/logging"
	20	+	)
	21	+
	22	+	// detFrame is a captured output as the detector receives it: BGRA pixels in
	23	+	// m, valid only during the Detect call. It holds the mapping, not the bytes,
	24	+	// so that recording a call (mocks print their arguments) never reads memory
	25	+	// that was unmapped since.
	26	+	type detFrame struct {
	27	+	width, height, stride int
	28	+	m *mapping
	29	+	// regions are the crops detected independently; none means the whole
	30	+	// frame.
	31	+	regions []region
	32	+	}
	33	+
	34	+	// detection is one box found by the detector, in frame (physical) pixels.
	35	+	type detection struct {
	36	+	x1, y1, x2, y2, score float32
	37	+	// region is the index of the request region the box comes from.
	38	+	region int
	39	+	}
	40	+
	41	+	// detector finds clickable elements in a frame. The worker goroutine is its
	42	+	// only caller.
	43	+	type detector interface {
	44	+	// Detect returns the boxes found in f. An error is for this call only:
	45	+	// the detector recovers by itself on the next one.
	46	+	Detect(ctx context.Context, f detFrame) ([]detection, error)
	47	+	// Close releases the detector. It is called once, by the worker
	48	+	// goroutine, when the worker returns.
	49	+	Close() error
	50	+	}
	51	+
	52	+	const (
	53	+	protocolVersion = 1
	54	+	helloTimeout = 10 * time.Second
	55	+	requestTimeout = 2 * time.Second
	56	+	imgsz = 1280
	57	+	// maxString and maxBoxes bound what a misbehaving child can make us
	58	+	// allocate.
	59	+	maxString = 1 << 16
	60	+	maxBoxes = 64 * 1000 // runner default max-det per region, at most 64 regions
	61	+	)
	62	+
	63	+	var (
	64	+	magicHello = [4]byte{'N', 'H', 'N', 'T'}
	65	+	magicRequest = [4]byte{'N', 'H', 'R', 'Q'}
	66	+	magicResponse = [4]byte{'N', 'H', 'R', 'S'}
	67	+
	68	+	// errProtocol is a framing error: the stream cannot be trusted any more.
	69	+	errProtocol = errors.New("hints: detector protocol error")
	70	+	// errDetector is an error the detector reported itself; it keeps serving.
	71	+	errDetector = errors.New("hints: detector error")
	72	+	)
	73	+
	74	+	var le = binary.LittleEndian
	75	+
	76	+	// writeRequest writes one request (the regions of f, default confidence)
	77	+	// followed by the pixels.
	78	+	func writeRequest(w io.Writer, seq uint64, f detFrame) error {
	79	+	var h [4 + 4 + 8 + 5*4]byte
	80	+	copy(h[0:], magicRequest[:])
	81	+	le.PutUint32(h[4:], protocolVersion)
	82	+	le.PutUint64(h[8:], seq)
	83	+	le.PutUint32(h[16:], uint32(f.width))
	84	+	le.PutUint32(h[20:], uint32(f.height))
	85	+	le.PutUint32(h[24:], uint32(f.stride))
	86	+	le.PutUint32(h[28:], formatXRGB8888)
	87	+	le.PutUint32(h[32:], uint32(len(f.regions)))
	88	+	if _, err := w.Write(h[:]); err != nil {
	89	+	return err
	90	+	}
	91	+	var rec [16]byte
	92	+	for _, r := range f.regions {
	93	+	le.PutUint32(rec[0:], uint32(int32(r.x)))
	94	+	le.PutUint32(rec[4:], uint32(int32(r.y)))
	95	+	le.PutUint32(rec[8:], uint32(r.w))
	96	+	le.PutUint32(rec[12:], uint32(r.h))
	97	+	if _, err := w.Write(rec[:]); err != nil {
	98	+	return err
	99	+	}
	100	+	}
	101	+	var conf [4]byte // confidence: startup default
	102	+	le.PutUint32(conf[:], math.Float32bits(0))
	103	+	if _, err := w.Write(conf[:]); err != nil {
	104	+	return err
	105	+	}
	106	+	_, err := w.Write(f.m.b[:f.height*f.stride])
	107	+	return err
	108	+	}
	109	+
	110	+	func readMagic(r io.Reader, want [4]byte) error {
	111	+	var m [4]byte
	112	+	if _, err := io.ReadFull(r, m[:]); err != nil {
	113	+	return err
	114	+	}
	115	+	if m != want {
	116	+	return fmt.Errorf("%w: bad magic %q", errProtocol, m[:])
	117	+	}
	118	+	return nil
	119	+	}
	120	+
	121	+	func readU32(r io.Reader) (uint32, error) {
	122	+	var b [4]byte
	123	+	_, err := io.ReadFull(r, b[:])
	124	+	return le.Uint32(b[:]), err
	125	+	}
	126	+
	127	+	func readString(r io.Reader) (string, error) {
	128	+	n, err := readU32(r)
	129	+	if err != nil {
	130	+	return "", err
	131	+	}
	132	+	if n > maxString {
	133	+	return "", fmt.Errorf("%w: string of %d bytes", errProtocol, n)
	134	+	}
	135	+	b := make([]byte, n)
	136	+	_, err = io.ReadFull(r, b)
	137	+	return string(b), err
	138	+	}
	139	+
	140	+	// readHello reads the hello: the device name, or the error of a failed start.
	141	+	func readHello(r io.Reader) (device string, err error) {
	142	+	if err := readMagic(r, magicHello); err != nil {
	143	+	return "", err
	144	+	}
	145	+	version, err := readU32(r)
	146	+	if err != nil {
	147	+	return "", err
	148	+	}
	149	+	if version != protocolVersion {
	150	+	return "", fmt.Errorf("%w: version %d", errProtocol, version)
	151	+	}
	152	+	status, err := readU32(r)
	153	+	if err != nil {
	154	+	return "", err
	155	+	}
	156	+	msg, err := readString(r)
	157	+	if err != nil {
	158	+	return "", err
	159	+	}
	160	+	if status != 0 {
	161	+	return "", fmt.Errorf("%w: start failed: %s", errDetector, msg)
	162	+	}
	163	+	if _, err := readU32(r); err != nil { // imgsz
	164	+	return "", err
	165	+	}
	166	+	return msg, nil
	167	+	}
	168	+
	169	+	// readResponse reads the response to request seq.
	170	+	func readResponse(r io.Reader, seq uint64) ([]detection, error) {
	171	+	if err := readMagic(r, magicResponse); err != nil {
	172	+	return nil, err
	173	+	}
	174	+	var h [16]byte
	175	+	if _, err := io.ReadFull(r, h[:]); err != nil {
	176	+	return nil, err
	177	+	}
	178	+	if v := le.Uint32(h[0:]); v != protocolVersion {
	179	+	return nil, fmt.Errorf("%w: version %d", errProtocol, v)
	180	+	}
	181	+	if got := le.Uint64(h[4:]); got != seq {
	182	+	return nil, fmt.Errorf("%w: response %d to request %d", errProtocol, got, seq)
	183	+	}
	184	+	if status := le.Uint32(h[12:]); status != 0 {
	185	+	msg, err := readString(r)
	186	+	if err != nil {
	187	+	return nil, err
	188	+	}
	189	+	return nil, fmt.Errorf("%w: status %d: %s", errDetector, status, msg)
	190	+	}
	191	+	var t [8]byte // inference ms, nboxes
	192	+	if _, err := io.ReadFull(r, t[:]); err != nil {
	193	+	return nil, err
	194	+	}
	195	+	n := le.Uint32(t[4:])
	196	+	if n > maxBoxes {
	197	+	return nil, fmt.Errorf("%w: %d boxes", errProtocol, n)
	198	+	}
	199	+	out := make([]detection, 0, min(n, 4096))
	200	+	var rec [24]byte
	201	+	for range n {
	202	+	if _, err := io.ReadFull(r, rec[:]); err != nil {
	203	+	return nil, err
	204	+	}
	205	+	f := func(i int) float32 { return math.Float32frombits(le.Uint32(rec[i*4:])) }
	206	+	out = append(out, detection{x1: f(0), y1: f(1), x2: f(2), y2: f(3), score: f(4), region: int(le.Uint32(rec[20:]))})
	207	+	}
	208	+	return out, nil
	209	+	}
	210	+
	211	+	// procDetector runs the nefer-hints binary as a persistent child. It starts
	212	+	// on the first Detect, and again after the child died or misbehaved.
	213	+	type procDetector struct {
	214	+	command, model string
	215	+	conf float64
	216	+	helloTimeout time.Duration
	217	+	reqTimeout time.Duration
	218	+
	219	+	proc *process
	220	+	seq uint64
	221	+	}
	222	+
	223	+	func newProcDetector(command, model string, threshold float64) *procDetector {
	224	+	return &procDetector{command: command, model: model, conf: threshold, helloTimeout: helloTimeout, reqTimeout: requestTimeout}
	225	+	}
	226	+
	227	+	// waitDelay bounds cmd.Wait once the child is dead: a descendant that kept
	228	+	// stderr open must not block the reaping.
	229	+	const (
	230	+	waitDelay = time.Second
	231	+	closeGrace = time.Second
	232	+	)
	233	+
	234	+	// process is one running child, alone in its process group. It keeps both
	235	+	// pipe ends so that closing them unblocks a pending read or write.
	236	+	type process struct {
	237	+	cmd *exec.Cmd
	238	+	in *os.File // write end of the child's stdin
	239	+	out *os.File // read end of the child's stdout
	240	+	rd *bufio.Reader
	241	+
	242	+	mu sync.Mutex
	243	+	reaped bool
	244	+	once sync.Once
	245	+	}
	246	+
	247	+	// interrupt kills the whole process group and closes our pipe ends, without
	248	+	// waiting: it unblocks a pending read or write from a timer or a context
	249	+	// callback. kill reaps the child afterwards.
	250	+	func (p *process) interrupt() {
	251	+	p.mu.Lock()
	252	+	if !p.reaped {
	253	+	_ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGKILL)
	254	+	}
	255	+	p.mu.Unlock()
	256	+	_ = p.out.Close()
	257	+	_ = p.in.Close()
	258	+	}
	259	+
	260	+	// kill stops the child and its descendants and reaps it; safe to call more
	261	+	// than once. Wait is bounded by cmd.WaitDelay.
	262	+	func (p *process) kill() {
	263	+	p.interrupt()
	264	+	p.wait()
	265	+	}
	266	+
	267	+	// wait reaps the child (once); bounded by cmd.WaitDelay after its death.
	268	+	func (p *process) wait() {
	269	+	p.once.Do(func() {
	270	+	_ = p.cmd.Wait()
	271	+	p.mu.Lock()
	272	+	p.reaped = true
	273	+	p.mu.Unlock()
	274	+	})
	275	+	}
	276	+
	277	+	// cmd leaves confidence selection to the runner unless explicitly configured.
	278	+	func (d procDetector) cmd() exec.Cmd {
	279	+	args := []string{"--model", d.model, "--imgsz", strconv.Itoa(imgsz), "--gpu", "auto"}
	280	+	if d.conf != 0 {
	281	+	args = append(args, "--conf", strconv.FormatFloat(d.conf, 'f', -1, 64))
	282	+	}
	283	+	return exec.Command(d.command, args...)
	284	+	}
	285	+
	286	+	// interruptOnDeadline binds both callbacks to this process only. Stopping joins
	287	+	// any callback that already started, so its result can never be accepted.
	288	+	func interruptOnDeadline(ctx context.Context, timeout time.Duration, p *process) func() (timedOut, canceled bool) {
	289	+	timerDone, contextDone := make(chan struct{}), make(chan struct{})
	290	+	timer := time.AfterFunc(timeout, func() { defer close(timerDone); p.interrupt() })
	291	+	stop := context.AfterFunc(ctx, func() { defer close(contextDone); p.interrupt() })
	292	+	return func() (bool, bool) {
	293	+	timedOut, canceled := !timer.Stop(), !stop()
	294	+	if timedOut {
	295	+	<-timerDone
	296	+	}
	297	+	if canceled {
	298	+	<-contextDone
	299	+	}
	300	+	return timedOut, canceled
	301	+	}
	302	+	}
	303	+
	304	+	func (d *procDetector) start(ctx context.Context) error {
	305	+	log := logging.For(ctx, "hints")
	306	+	cmd := d.cmd()
	307	+	cmd.Stderr = &lineLog{log: func(line string) { log.Debug().Str("detector", d.command).Msg(line) }}
	308	+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	309	+	cmd.WaitDelay = waitDelay
	310	+	inR, inW, err := os.Pipe()
	311	+	if err != nil {
	312	+	return err
	313	+	}
	314	+	outR, outW, err := os.Pipe()
	315	+	if err != nil {
	316	+	_ = inR.Close()
	317	+	_ = inW.Close()
	318	+	return err
	319	+	}
	320	+	cmd.Stdin, cmd.Stdout = inR, outW
	321	+	err = cmd.Start()
	322	+	// The child has its ends; ours are not needed any more.
	323	+	_ = inR.Close()
	324	+	_ = outW.Close()
	325	+	if err != nil {
	326	+	_ = inW.Close()
	327	+	_ = outR.Close()
	328	+	return fmt.Errorf("hints: start detector: %w", err)
	329	+	}
	330	+	p := &process{cmd: cmd, in: inW, out: outR, rd: bufio.NewReaderSize(outR, 1<<16)}
	331	+	t0 := time.Now()
	332	+	stop := interruptOnDeadline(ctx, d.helloTimeout, p)
	333	+	device, err := readHello(p.rd)
	334	+	timedOut, canceled := stop()
	335	+	if err != nil \|\| timedOut \|\| canceled {
	336	+	p.kill()
	337	+	if ctx.Err() != nil {
	338	+	err = ctx.Err()
	339	+	} else if timedOut {
	340	+	err = errors.New("hello timed out")
	341	+	}
	342	+	return fmt.Errorf("hints: detector hello: %w", err)
	343	+	}
	344	+	d.proc = p
	345	+	log.Info().Str("device", device).Float64("start_ms", ms(time.Since(t0))).Msg("hint detector ready")
	346	+	return nil
	347	+	}
	348	+
	349	+	func (d *procDetector) Detect(ctx context.Context, f detFrame) ([]detection, error) {
	350	+	if d.proc == nil {
	351	+	if err := d.start(ctx); err != nil {
	352	+	return nil, err
	353	+	}
	354	+	}
	355	+	p := d.proc
	356	+	d.seq++
	357	+	stop := interruptOnDeadline(ctx, d.reqTimeout, p)
	358	+	dets, err := d.roundTrip(p, f)
	359	+	timedOut, canceled := stop()
	360	+	if timedOut \|\| canceled {
	361	+	d.drop()
	362	+	if ctx.Err() != nil {
	363	+	return nil, ctx.Err()
	364	+	}
	365	+	return nil, fmt.Errorf("hints: detector timed out after %s", d.reqTimeout)
	366	+	}
	367	+	if err == nil {
	368	+	return dets, nil
	369	+	}
	370	+	if errors.Is(err, errDetector) {
	371	+	return nil, err // the child answered and keeps serving
	372	+	}
	373	+	d.drop()
	374	+	switch {
	375	+	case timedOut:
	376	+	return nil, fmt.Errorf("hints: detector timed out after %s", d.reqTimeout)
	377	+	case ctx.Err() != nil:
	378	+	return nil, ctx.Err()
	379	+	}
	380	+	return nil, fmt.Errorf("hints: detector failed: %w", err)
	381	+	}
	382	+
	383	+	func (d procDetector) roundTrip(p process, f detFrame) ([]detection, error) {
	384	+	bw := bufio.NewWriterSize(p.in, 1<<16)
	385	+	if err := writeRequest(bw, d.seq, f); err != nil {
	386	+	return nil, err
	387	+	}
	388	+	if err := bw.Flush(); err != nil {
	389	+	return nil, err
	390	+	}
	391	+	return readResponse(p.rd, d.seq)
	392	+	}
	393	+
	394	+	// drop kills the current child: the next Detect starts a new one.
	395	+	func (d *procDetector) drop() {
	396	+	if d.proc != nil {
	397	+	d.proc.kill()
	398	+	d.proc = nil
	399	+	}
	400	+	}
	401	+
	402	+	// Close asks the child to stop by closing its stdin (it exits 0 on EOF), then
	403	+	// kills it if it does not.
	404	+	func (d *procDetector) Close() error {
	405	+	p := d.proc
	406	+	d.proc = nil
	407	+	if p == nil {
	408	+	return nil
	409	+	}
	410	+	defer func() { _ = p.out.Close() }()
	411	+	_ = p.in.Close()
	412	+	done := make(chan struct{})
	413	+	go func() { p.wait(); close(done) }()
	414	+	select {
	415	+	case <-done:
	416	+	case <-time.After(closeGrace):
	417	+	p.interrupt()
	418	+	select {
	419	+	case <-done:
	420	+	case <-time.After(2 * waitDelay):
	421	+	}
	422	+	}
	423	+	return nil
	424	+	}
	425	+
	426	+	// lineLog is an io.Writer that hands each complete line to log.
	427	+	type lineLog struct {
	428	+	log func(string)
	429	+	buf []byte
	430	+	}
	431	+
	432	+	func (l *lineLog) Write(b []byte) (int, error) {
	433	+	l.buf = append(l.buf, b...)
	434	+	for {
	435	+	i := bytes.IndexByte(l.buf, '\n')
	436	+	if i < 0 {
	437	+	break
	438	+	}
	439	+	if line := string(l.buf[:i]); line != "" {
	440	+	l.log(line)
	441	+	}
	442	+	l.buf = l.buf[i+1:]
	443	+	}
	444	+	if len(l.buf) > maxString {
	445	+	l.buf = l.buf[:0]
	446	+	}
	447	+	return len(b), nil
	448	+	}

A internal/adapters/hints/detector_mock_test.go +159 -0

@@ -0,0 +1,159 @@
	1	+	// Code generated by mockery; DO NOT EDIT.
	2	+	// github.com/vektra/mockery
	3	+	// template: testify
	4	+
	5	+	package hints
	6	+
	7	+	import (
	8	+	"context"
	9	+
	10	+	mock "github.com/stretchr/testify/mock"
	11	+	)
	12	+
	13	+	// newMockdetector creates a new instance of mockdetector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
	14	+	// The first argument is typically a *testing.T value.
	15	+	func newMockdetector(t interface {
	16	+	mock.TestingT
	17	+	Cleanup(func())
	18	+	}) *mockdetector {
	19	+	if helper, ok := t.(interface{ Helper() }); ok {
	20	+	helper.Helper()
	21	+	}
	22	+
	23	+	mock := &mockdetector{}
	24	+	mock.Mock.Test(t)
	25	+
	26	+	t.Cleanup(func() {
	27	+	if helper, ok := t.(interface{ Helper() }); ok {
	28	+	helper.Helper()
	29	+	}
	30	+	mock.AssertExpectations(t)
	31	+	})
	32	+
	33	+	return mock
	34	+	}
	35	+
	36	+	// mockdetector is an autogenerated mock type for the detector type
	37	+	type mockdetector struct {
	38	+	mock.Mock
	39	+	}
	40	+
	41	+	type mockdetector_Expecter struct {
	42	+	mock *mock.Mock
	43	+	}
	44	+
	45	+	func (_m mockdetector) EXPECT() mockdetector_Expecter {
	46	+	return &mockdetector_Expecter{mock: &_m.Mock}
	47	+	}
	48	+
	49	+	// Close provides a mock function for the type mockdetector
	50	+	func (_mock *mockdetector) Close() error {
	51	+	ret := _mock.Called()
	52	+
	53	+	if len(ret) == 0 {
	54	+	panic("no return value specified for Close")
	55	+	}
	56	+
	57	+	var r0 error
	58	+	if returnFunc, ok := ret.Get(0).(func() error); ok {
	59	+	r0 = returnFunc()
	60	+	} else {
	61	+	r0 = ret.Error(0)
	62	+	}
	63	+	return r0
	64	+	}
	65	+
	66	+	// mockdetector_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
	67	+	type mockdetector_Close_Call struct {
	68	+	*mock.Call
	69	+	}
	70	+
	71	+	// Close is a helper method to define mock.On call
	72	+	func (_e mockdetector_Expecter) Close() mockdetector_Close_Call {
	73	+	return &mockdetector_Close_Call{Call: _e.mock.On("Close")}
	74	+	}
	75	+
	76	+	func (_c mockdetector_Close_Call) Run(run func()) mockdetector_Close_Call {
	77	+	_c.Call.Run(func(args mock.Arguments) {
	78	+	run()
	79	+	})
	80	+	return _c
	81	+	}
	82	+
	83	+	func (_c mockdetector_Close_Call) Return(err error) mockdetector_Close_Call {
	84	+	_c.Call.Return(err)
	85	+	return _c
	86	+	}
	87	+
	88	+	func (_c mockdetector_Close_Call) RunAndReturn(run func() error) mockdetector_Close_Call {
	89	+	_c.Call.Return(run)
	90	+	return _c
	91	+	}
	92	+
	93	+	// Detect provides a mock function for the type mockdetector
	94	+	func (_mock *mockdetector) Detect(ctx context.Context, f detFrame) ([]detection, error) {
	95	+	ret := _mock.Called(ctx, f)
	96	+
	97	+	if len(ret) == 0 {
	98	+	panic("no return value specified for Detect")
	99	+	}
	100	+
	101	+	var r0 []detection
	102	+	var r1 error
	103	+	if returnFunc, ok := ret.Get(0).(func(context.Context, detFrame) ([]detection, error)); ok {
	104	+	return returnFunc(ctx, f)
	105	+	}
	106	+	if returnFunc, ok := ret.Get(0).(func(context.Context, detFrame) []detection); ok {
	107	+	r0 = returnFunc(ctx, f)
	108	+	} else {
	109	+	if ret.Get(0) != nil {
	110	+	r0 = ret.Get(0).([]detection)
	111	+	}
	112	+	}
	113	+	if returnFunc, ok := ret.Get(1).(func(context.Context, detFrame) error); ok {
	114	+	r1 = returnFunc(ctx, f)
	115	+	} else {
	116	+	r1 = ret.Error(1)
	117	+	}
	118	+	return r0, r1
	119	+	}
	120	+
	121	+	// mockdetector_Detect_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Detect'
	122	+	type mockdetector_Detect_Call struct {
	123	+	*mock.Call
	124	+	}
	125	+
	126	+	// Detect is a helper method to define mock.On call
	127	+	// - ctx context.Context
	128	+	// - f detFrame
	129	+	func (_e mockdetector_Expecter) Detect(ctx any, f any) mockdetector_Detect_Call {
	130	+	return &mockdetector_Detect_Call{Call: _e.mock.On("Detect", ctx, f)}
	131	+	}
	132	+
	133	+	func (_c mockdetector_Detect_Call) Run(run func(ctx context.Context, f detFrame)) mockdetector_Detect_Call {
	134	+	_c.Call.Run(func(args mock.Arguments) {
	135	+	var arg0 context.Context
	136	+	if args[0] != nil {
	137	+	arg0 = args[0].(context.Context)
	138	+	}
	139	+	var arg1 detFrame
	140	+	if args[1] != nil {
	141	+	arg1 = args[1].(detFrame)
	142	+	}
	143	+	run(
	144	+	arg0,
	145	+	arg1,
	146	+	)
	147	+	})
	148	+	return _c
	149	+	}
	150	+
	151	+	func (_c mockdetector_Detect_Call) Return(detectionMoqParams []detection, err error) mockdetector_Detect_Call {
	152	+	_c.Call.Return(detectionMoqParams, err)
	153	+	return _c
	154	+	}
	155	+
	156	+	func (_c mockdetector_Detect_Call) RunAndReturn(run func(ctx context.Context, f detFrame) ([]detection, error)) mockdetector_Detect_Call {
	157	+	_c.Call.Return(run)
	158	+	return _c
	159	+	}

A internal/adapters/hints/detector_proc_test.go +172 -0

@@ -0,0 +1,172 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"io"
	6	+	"os"
	7	+	"os/exec"
	8	+	"strconv"
	9	+	"strings"
	10	+	"testing"
	11	+	"time"
	12	+
	13	+	"github.com/stretchr/testify/require"
	14	+	)
	15	+
	16	+	// The detector child of these tests is the test binary itself, re-executed:
	17	+	// a real process, started with the detector's own arguments. TestMain turns
	18	+	// it into helperChild when helperEnv is set.
	19	+	const (
	20	+	helperEnv = "NEFERWL_HINTS_HELPER"
	21	+	helperPIDFile = "NEFERWL_HINTS_HELPER_PIDFILE"
	22	+	helperChildPID = "NEFERWL_HINTS_HELPER_CHILDPID"
	23	+	)
	24	+
	25	+	func TestMain(m *testing.M) {
	26	+	if mode := os.Getenv(helperEnv); mode != "" {
	27	+	helperChild(mode)
	28	+	os.Exit(0)
	29	+	}
	30	+	os.Exit(m.Run())
	31	+	}
	32	+
	33	+	func helperHello() []byte {
	34	+	return cat(magicHello[:], u32(protocolVersion), u32(0), u32(4), []byte("test"), u32(imgsz))
	35	+	}
	36	+
	37	+	// helperChild plays the detector for mode:
	38	+	//
	39	+	// nohello never says hello
	40	+	// silent says hello, then never answers
	41	+	// grandchild like nohello, after spawning a grandchild that shares the
	42	+	// child's stdout, stderr and process group
	43	+	// ok says hello and answers every request with no box
	44	+	func helperChild(mode string) {
	45	+	if f := os.Getenv(helperChildPID); f != "" {
	46	+	_ = os.WriteFile(f, []byte(strconv.Itoa(os.Getpid())), 0o600)
	47	+	}
	48	+	if mode == "grandchild" {
	49	+	gc := exec.Command("sleep", "60")
	50	+	gc.Stdout, gc.Stderr = os.Stdout, os.Stderr
	51	+	if err := gc.Start(); err != nil {
	52	+	os.Exit(2)
	53	+	}
	54	+	_ = os.WriteFile(os.Getenv(helperPIDFile), []byte(strconv.Itoa(gc.Process.Pid)), 0o600)
	55	+	mode = "nohello"
	56	+	}
	57	+	if mode != "nohello" {
	58	+	_, _ = os.Stdout.Write(helperHello())
	59	+	}
	60	+	if mode != "ok" {
	61	+	time.Sleep(time.Minute)
	62	+	return
	63	+	}
	64	+	for {
	65	+	var h [40]byte
	66	+	if _, err := io.ReadFull(os.Stdin, h[:]); err != nil {
	67	+	return
	68	+	}
	69	+	_, err := io.CopyN(io.Discard, os.Stdin, int64(le.Uint32(h[20:])*le.Uint32(h[24:])))
	70	+	if err != nil {
	71	+	return
	72	+	}
	73	+	_, _ = os.Stdout.Write(cat(magicResponse[:], u32(protocolVersion), h[8:16], u32(0), u32(0), u32(0)))
	74	+	}
	75	+	}
	76	+
	77	+	func helperDetector(t testing.T, mode string) procDetector {
	78	+	t.Helper()
	79	+	t.Setenv(helperEnv, mode)
	80	+	d := newProcDetector(os.Args[0], t.TempDir(), 0.05)
	81	+	d.helloTimeout, d.reqTimeout = 300time.Millisecond, 300time.Millisecond
	82	+	t.Cleanup(func() { _ = d.Close() })
	83	+	return d
	84	+	}
	85	+
	86	+	func tinyFrame() detFrame {
	87	+	return detFrame{width: 1, height: 1, stride: 4, m: &mapping{b: make([]byte, 4), live: new(int)}}
	88	+	}
	89	+
	90	+	// processGone reports whether pid is dead (or a zombie waiting for its
	91	+	// reaper).
	92	+	func processGone(pid int) bool {
	93	+	b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
	94	+	if err != nil {
	95	+	return true
	96	+	}
	97	+	s := string(b)
	98	+	return strings.HasPrefix(s[strings.LastIndexByte(s, ')')+2:], "Z")
	99	+	}
	100	+
	101	+	func readPID(t *testing.T, file string) int {
	102	+	t.Helper()
	103	+	var pid int
	104	+	require.Eventually(t, func() bool {
	105	+	b, err := os.ReadFile(file)
	106	+	if err != nil {
	107	+	return false
	108	+	}
	109	+	pid, err = strconv.Atoi(string(b))
	110	+	return err == nil
	111	+	}, time.Second, 10*time.Millisecond)
	112	+	return pid
	113	+	}
	114	+
	115	+	func TestProcDetectorHungChildIsKilledAndRestarted(t *testing.T) {
	116	+	for _, mode := range []string{"nohello", "silent"} {
	117	+	t.Run(mode, func(t *testing.T) {
	118	+	pidFile := t.TempDir() + "/child"
	119	+	t.Setenv(helperChildPID, pidFile)
	120	+	d := helperDetector(t, mode)
	121	+	ctx := context.Background()
	122	+	start := time.Now()
	123	+	_, err := d.Detect(ctx, tinyFrame())
	124	+	require.ErrorContains(t, err, "timed out")
	125	+	if mode == "nohello" {
	126	+	require.ErrorContains(t, err, "hello")
	127	+	}
	128	+	require.Less(t, time.Since(start), 2*time.Second)
	129	+	require.Nil(t, d.proc)
	130	+	pid := readPID(t, pidFile)
	131	+	require.True(t, processGone(pid), "child %d survives", pid)
	132	+
	133	+	// The next Detect starts a new, healthy child.
	134	+	t.Setenv(helperEnv, "ok")
	135	+	t.Setenv(helperChildPID, "")
	136	+	_, err = d.Detect(ctx, tinyFrame())
	137	+	require.NoError(t, err)
	138	+	require.NotNil(t, d.proc)
	139	+	start = time.Now()
	140	+	require.NoError(t, d.Close())
	141	+	require.Less(t, time.Since(start), 3*time.Second)
	142	+	})
	143	+	}
	144	+	}
	145	+
	146	+	// A grandchild holding the child's stdout and stderr open must neither
	147	+	// survive the kill nor keep Wait blocked.
	148	+	func TestProcDetectorKillsGrandchild(t *testing.T) {
	149	+	pidFile := t.TempDir() + "/pid"
	150	+	t.Setenv(helperPIDFile, pidFile)
	151	+	d := helperDetector(t, "grandchild")
	152	+	start := time.Now()
	153	+	_, err := d.Detect(context.Background(), tinyFrame())
	154	+	require.ErrorContains(t, err, "hello timed out")
	155	+	require.Less(t, time.Since(start), 2500*time.Millisecond)
	156	+	pid := readPID(t, pidFile)
	157	+	require.Eventually(t, func() bool { return processGone(pid) }, 2time.Second, 20time.Millisecond, "grandchild %d survives", pid)
	158	+	require.Nil(t, d.proc)
	159	+	}
	160	+
	161	+	// Cancelling the context interrupts a pending read.
	162	+	func TestProcDetectorContextCancel(t *testing.T) {
	163	+	d := helperDetector(t, "silent")
	164	+	d.reqTimeout = time.Minute
	165	+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
	166	+	defer cancel()
	167	+	start := time.Now()
	168	+	_, err := d.Detect(ctx, tinyFrame())
	169	+	require.ErrorIs(t, err, context.DeadlineExceeded)
	170	+	require.Less(t, time.Since(start), 2*time.Second)
	171	+	require.Nil(t, d.proc)
	172	+	}

A internal/adapters/hints/detector_test.go +335 -0

@@ -0,0 +1,335 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"context"
	6	+	"encoding/binary"
	7	+	"fmt"
	8	+	"io"
	9	+	"math"
	10	+	"net"
	11	+	"os"
	12	+	"path/filepath"
	13	+	"testing"
	14	+	"testing/synctest"
	15	+	"time"
	16	+
	17	+	"github.com/stretchr/testify/require"
	18	+	)
	19	+
	20	+	// u32, u64 and f32 build the byte fixtures of the protocol.
	21	+	func u32(v uint32) []byte { return binary.LittleEndian.AppendUint32(nil, v) }
	22	+	func u64(v uint64) []byte { return binary.LittleEndian.AppendUint64(nil, v) }
	23	+	func f32(v float32) []byte { return u32(math.Float32bits(v)) }
	24	+	func cat(parts ...[]byte) []byte {
	25	+	return bytes.Join(parts, nil)
	26	+	}
	27	+
	28	+	func TestWriteRequest(t *testing.T) {
	29	+	var buf bytes.Buffer
	30	+	pix := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}
	31	+	require.NoError(t, writeRequest(&buf, 0x0102030405, detFrame{width: 2, height: 1, stride: 12, m: &mapping{b: append(pix, 99)}}))
	32	+	want := cat([]byte("NHRQ"), u32(1), u64(0x0102030405), u32(2), u32(1), u32(12), u32(1), u32(0), f32(0), pix)
	33	+	require.Equal(t, want, buf.Bytes())
	34	+	}
	35	+
	36	+	func TestWriteRequestRegions(t *testing.T) {
	37	+	var buf bytes.Buffer
	38	+	pix := []byte{1, 2, 3, 4}
	39	+	f := detFrame{width: 1, height: 1, stride: 4, m: &mapping{b: pix}, regions: []region{{0, 1, 2, 3}, {4, 5, 6, 7}}}
	40	+	require.NoError(t, writeRequest(&buf, 1, f))
	41	+	want := cat([]byte("NHRQ"), u32(1), u64(1), u32(1), u32(1), u32(4), u32(1), u32(2),
	42	+	u32(0), u32(1), u32(2), u32(3), u32(4), u32(5), u32(6), u32(7), f32(0), pix)
	43	+	require.Equal(t, want, buf.Bytes())
	44	+	}
	45	+
	46	+	func TestReadHello(t *testing.T) {
	47	+	dev, err := readHello(bytes.NewReader(cat([]byte("NHNT"), u32(1), u32(0), u32(6), []byte("Vulkan"), u32(1280))))
	48	+	require.NoError(t, err)
	49	+	require.Equal(t, "Vulkan", dev)
	50	+
	51	+	_, err = readHello(bytes.NewReader(cat([]byte("NHNT"), u32(1), u32(1), u32(4), []byte("boom"))))
	52	+	require.ErrorIs(t, err, errDetector)
	53	+	require.ErrorContains(t, err, "boom")
	54	+
	55	+	_, err = readHello(bytes.NewReader(cat([]byte("XXXX"), u32(1))))
	56	+	require.ErrorIs(t, err, errProtocol)
	57	+	_, err = readHello(bytes.NewReader(cat([]byte("NHNT"), u32(2), u32(0))))
	58	+	require.ErrorIs(t, err, errProtocol)
	59	+	_, err = readHello(bytes.NewReader(cat([]byte("NHNT"), u32(1), u32(0), u32(1<<20))))
	60	+	require.ErrorIs(t, err, errProtocol)
	61	+	_, err = readHello(bytes.NewReader([]byte("NHNT")))
	62	+	require.Error(t, err)
	63	+	}
	64	+
	65	+	func TestReadResponse(t *testing.T) {
	66	+	head := func(seq uint64, status uint32) []byte { return cat([]byte("NHRS"), u32(1), u64(seq), u32(status)) }
	67	+
	68	+	ok := cat(head(5, 0), f32(12.5), u32(2),
	69	+	f32(1), f32(2), f32(30), f32(40), f32(0.9), u32(0),
	70	+	f32(5), f32(6), f32(7), f32(8), f32(0.25), u32(3))
	71	+	got, err := readResponse(bytes.NewReader(ok), 5)
	72	+	require.NoError(t, err)
	73	+	require.Equal(t, []detection{{x1: 1, y1: 2, x2: 30, y2: 40, score: 0.9}, {x1: 5, y1: 6, x2: 7, y2: 8, score: 0.25, region: 3}}, got)
	74	+
	75	+	got, err = readResponse(bytes.NewReader(cat(head(5, 0), f32(1), u32(0))), 5)
	76	+	require.NoError(t, err)
	77	+	require.Empty(t, got)
	78	+
	79	+	_, err = readResponse(bytes.NewReader(cat(head(5, 1), u32(3), []byte("bad"))), 5)
	80	+	require.ErrorIs(t, err, errDetector)
	81	+	require.ErrorContains(t, err, "bad")
	82	+
	83	+	_, err = readResponse(bytes.NewReader(ok), 6) // wrong sequence
	84	+	require.ErrorIs(t, err, errProtocol)
	85	+	_, err = readResponse(bytes.NewReader(cat([]byte("NHRX"), u32(1))), 5)
	86	+	require.ErrorIs(t, err, errProtocol)
	87	+	_, err = readResponse(bytes.NewReader(cat([]byte("NHRS"), u32(9), u64(5), u32(0))), 5)
	88	+	require.ErrorIs(t, err, errProtocol)
	89	+	_, err = readResponse(bytes.NewReader(cat(head(5, 0), f32(1), u32(1<<30))), 5)
	90	+	require.ErrorIs(t, err, errProtocol)
	91	+	_, err = readResponse(bytes.NewReader(ok[:len(ok)-3]), 5) // truncated
	92	+	require.Error(t, err)
	93	+	}
	94	+
	95	+	func TestLineLog(t *testing.T) {
	96	+	var lines []string
	97	+	l := &lineLog{log: func(s string) { lines = append(lines, s) }}
	98	+	_, _ = l.Write([]byte("one\ntw"))
	99	+	_, _ = l.Write([]byte("o\n\nthree"))
	100	+	require.Equal(t, []string{"one", "two"}, lines)
	101	+	}
	102	+
	103	+	// The real detector, when the environment names it:
	104	+	// NEFERWL_TEST_DETECTOR (binary) and NEFERWL_TEST_DETECTOR_MODEL (models dir).
	105	+	func TestProcDetectorIntegration(t *testing.T) {
	106	+	bin, model := os.Getenv("NEFERWL_TEST_DETECTOR"), os.Getenv("NEFERWL_TEST_DETECTOR_MODEL")
	107	+	if bin == "" \|\| model == "" {
	108	+	t.Skip("NEFERWL_TEST_DETECTOR and NEFERWL_TEST_DETECTOR_MODEL not set")
	109	+	}
	110	+	d := newProcDetector(bin, model, 0.05)
	111	+	t.Cleanup(func() { _ = d.Close() })
	112	+	m := &mapping{b: make([]byte, 12807204), live: new(int)}
	113	+	for i := range m.b {
	114	+	m.b[i] = 0xf0 // a blank light frame
	115	+	}
	116	+	f := detFrame{width: 1280, height: 720, stride: 1280 * 4, m: m}
	117	+	ctx := context.Background()
	118	+
	119	+	dets, err := d.Detect(ctx, f)
	120	+	require.NoError(t, err)
	121	+	require.Empty(t, dets) // nothing clickable on a blank frame
	122	+	first := d.proc
	123	+	require.NotNil(t, first)
	124	+	_, err = d.Detect(ctx, f)
	125	+	require.NoError(t, err)
	126	+	require.Same(t, first, d.proc, "the child is kept alive between scans")
	127	+
	128	+	// A dead child fails one scan, and the next one starts a new child.
	129	+	require.NoError(t, first.cmd.Process.Kill())
	130	+	time.Sleep(50 * time.Millisecond)
	131	+	_, err = d.Detect(ctx, f)
	132	+	require.Error(t, err)
	133	+	require.Nil(t, d.proc)
	134	+	_, err = d.Detect(ctx, f)
	135	+	require.NoError(t, err)
	136	+	require.NotSame(t, first, d.proc)
	137	+
	138	+	// A wrong geometry is the detector's own error: the child stays.
	139	+	proc := d.proc
	140	+	_, err = d.Detect(ctx, detFrame{width: 1280, height: 720, stride: 4, m: m})
	141	+	require.ErrorIs(t, err, errDetector)
	142	+	require.Same(t, proc, d.proc)
	143	+	require.NoError(t, d.Close())
	144	+	}
	145	+
	146	+	func TestProcDetectorMissingBinary(t *testing.T) {
	147	+	d := newProcDetector("/nonexistent/nefer-hints", t.TempDir(), 0.05)
	148	+	_, err := d.Detect(context.Background(), detFrame{width: 1, height: 1, stride: 4, m: &mapping{b: make([]byte, 4)}})
	149	+	require.Error(t, err)
	150	+	require.Nil(t, d.proc)
	151	+	require.NoError(t, d.Close())
	152	+	}
	153	+
	154	+	func TestProcDetectorCommand(t *testing.T) {
	155	+	for _, threshold := range []float64{0, 0.3} {
	156	+	d := newProcDetector("nefer-hints", "/models", threshold)
	157	+	want := []string{"nefer-hints", "--model", "/models", "--imgsz", "1280", "--gpu", "auto"}
	158	+	if threshold != 0 {
	159	+	want = append(want, "--conf", "0.3")
	160	+	}
	161	+	require.Equal(t, want, d.cmd().Args)
	162	+	}
	163	+	}
	164	+
	165	+	func TestResponseBoxLimit(t *testing.T) {
	166	+	_, err := readResponse(bytes.NewReader(cat([]byte("NHRS"), u32(1), u64(1), u32(0), f32(0), u32(64001))), 1)
	167	+	require.ErrorIs(t, err, errProtocol)
	168	+	}
	169	+
	170	+	func TestPrepareDetections(t *testing.T) {
	171	+	regions := []region{{0, 0, 100, 100}, {50, 0, 100, 100}}
	172	+	valid := detection{x1: 90, y1: 10, x2: 100, y2: 30, score: .5}
	173	+	invalid := detection{x1: 80, y1: 0, x2: 120, y2: 40, score: float32(math.NaN()), region: 1}
	174	+	badRegion := valid
	175	+	badRegion.region = 2
	176	+	got := prepareDetections([]detection{valid, invalid, badRegion}, regions, 150, 100)
	177	+	require.Equal(t, []detection{valid}, dropCutBoxes(got, regions))
	178	+	many := make([]detection, maxWorkBoxes+5)
	179	+	for i := range many {
	180	+	many[i] = valid
	181	+	many[i].score = float32(i) / float32(len(many))
	182	+	}
	183	+	got = prepareDetections(many, nil, 150, 100)
	184	+	require.Len(t, got, maxWorkBoxes)
	185	+	require.Equal(t, many[len(many)-1].score, got[0].score)
	186	+	require.Equal(t, many[5].score, got[len(got)-1].score)
	187	+	}
	188	+
	189	+	// TestDetectorChild is a real subprocess speaking just enough protocol to
	190	+	// synchronize cancellation without timers or project-interface doubles.
	191	+	func TestDetectorChild(t *testing.T) {
	192	+	mode := os.Getenv("HINT_CHILD_MODE")
	193	+	if mode == "" {
	194	+	return
	195	+	}
	196	+	conn, err := net.Dial("unix", os.Getenv("HINT_CHILD_SOCKET"))
	197	+	if err != nil {
	198	+	os.Exit(2)
	199	+	}
	200	+	defer conn.Close()
	201	+	if mode != "hello" {
	202	+	_, _ = os.Stdout.Write(cat([]byte("NHNT"), u32(1), u32(0), u32(3), []byte("cpu"), u32(1280)))
	203	+	}
	204	+	if mode == "request" {
	205	+	b := make([]byte, 44) // request header, confidence and one BGRA pixel
	206	+	if _, err := io.ReadFull(os.Stdin, b); err != nil {
	207	+	os.Exit(3)
	208	+	}
	209	+	}
	210	+	_, _ = conn.Write([]byte{1})
	211	+	_, _ = io.Copy(io.Discard, os.Stdin)
	212	+	os.Exit(0)
	213	+	}
	214	+
	215	+	func childDetector(t testing.T, mode string) (procDetector, net.Listener) {
	216	+	t.Helper()
	217	+	dir := t.TempDir()
	218	+	socket := filepath.Join(dir, "ready")
	219	+	listener, err := net.Listen("unix", socket)
	220	+	require.NoError(t, err)
	221	+	t.Cleanup(func() { _ = listener.Close() })
	222	+	t.Setenv("HINT_CHILD_MODE", mode)
	223	+	t.Setenv("HINT_CHILD_SOCKET", socket)
	224	+	exe, err := os.Executable()
	225	+	require.NoError(t, err)
	226	+	script := filepath.Join(dir, "detector")
	227	+	require.NoError(t, os.WriteFile(script, fmt.Appendf(nil, "#!/bin/sh\nexec %q -test.run=^TestDetectorChild$\n", exe), 0700))
	228	+	d := newProcDetector(script, dir, 0)
	229	+	t.Cleanup(func() { _ = d.Close() })
	230	+	return d, listener
	231	+	}
	232	+
	233	+	func TestDetectorCancellation(t *testing.T) {
	234	+	for _, mode := range []string{"hello", "request"} {
	235	+	t.Run(mode, func(t *testing.T) {
	236	+	d, l := childDetector(t, mode)
	237	+	ctx, cancel := context.WithCancel(context.Background())
	238	+	defer cancel()
	239	+	done := make(chan error, 1)
	240	+	go func() {
	241	+	_, err := d.Detect(ctx, detFrame{width: 1, height: 1, stride: 4, m: &mapping{b: make([]byte, 4)}})
	242	+	done <- err
	243	+	}()
	244	+	require.NoError(t, l.(net.UnixListener).SetDeadline(time.Now().Add(5time.Second)))
	245	+	conn, err := l.Accept()
	246	+	require.NoError(t, err)
	247	+	require.NoError(t, conn.SetReadDeadline(time.Now().Add(5*time.Second)))
	248	+	var ready [1]byte
	249	+	_, err = io.ReadFull(conn, ready[:])
	250	+	_ = conn.Close()
	251	+	require.NoError(t, err)
	252	+	cancel()
	253	+	select {
	254	+	case err := <-done:
	255	+	require.ErrorIs(t, err, context.Canceled)
	256	+	case <-time.After(5 * time.Second):
	257	+	t.Fatal("cancellation did not finish")
	258	+	}
	259	+	})
	260	+	}
	261	+	}
	262	+
	263	+	func TestDetectorCloseClosesOutput(t *testing.T) {
	264	+	d, l := childDetector(t, "close")
	265	+	require.NoError(t, d.start(context.Background()))
	266	+	conn, err := l.Accept()
	267	+	require.NoError(t, err)
	268	+	_ = conn.Close()
	269	+	p := d.proc
	270	+	require.NoError(t, d.Close())
	271	+	_, err = p.out.Read(make([]byte, 1))
	272	+	require.ErrorIs(t, err, os.ErrClosed)
	273	+	require.NoError(t, d.Close())
	274	+	}
	275	+
	276	+	// A complete buffered response must not override an already launched context
	277	+	// callback, including a detector-reported error that normally keeps the child.
	278	+	func TestDetectorCanceledBufferedResponse(t *testing.T) {
	279	+	for _, status := range []uint32{0, 1} {
	280	+	t.Run(fmt.Sprint(status), func(t *testing.T) {
	281	+	d, l := childDetector(t, "close")
	282	+	require.NoError(t, d.start(context.Background()))
	283	+	conn, err := l.Accept()
	284	+	require.NoError(t, err)
	285	+	defer conn.Close()
	286	+	p := d.proc
	287	+	response := cat([]byte("NHRS"), u32(1), u64(1), u32(status))
	288	+	if status == 0 {
	289	+	response = cat(response, f32(0), u32(0))
	290	+	} else {
	291	+	response = cat(response, u32(3), []byte("bad"))
	292	+	}
	293	+	p.rd.Reset(bytes.NewReader(response))
	294	+	ctx, cancel := context.WithCancel(context.Background())
	295	+	cancel()
	296	+	_, err = d.Detect(ctx, detFrame{width: 1, height: 1, stride: 4, m: &mapping{b: make([]byte, 4)}})
	297	+	require.ErrorIs(t, err, context.Canceled)
	298	+	require.Nil(t, d.proc)
	299	+	require.True(t, p.reaped)
	300	+	})
	301	+	}
	302	+	}
	303	+
	304	+	func TestInterruptDeadlineBoundary(t *testing.T) {
	305	+	for _, cancelContext := range []bool{false, true} {
	306	+	t.Run(fmt.Sprint(cancelContext), func(t *testing.T) {
	307	+	synctest.Test(t, func(t *testing.T) {
	308	+	inR, inW, err := os.Pipe()
	309	+	require.NoError(t, err)
	310	+	defer inR.Close()
	311	+	outR, outW, err := os.Pipe()
	312	+	require.NoError(t, err)
	313	+	defer outW.Close()
	314	+	// Already reaped: only the real pipe endpoints remain to interrupt.
	315	+	p := &process{in: inW, out: outR, reaped: true}
	316	+	ctx, cancel := context.WithCancel(context.Background())
	317	+	defer cancel()
	318	+	stop := interruptOnDeadline(ctx, time.Second, p)
	319	+	if cancelContext {
	320	+	cancel()
	321	+	} else {
	322	+	time.Sleep(time.Second)
	323	+	}
	324	+	synctest.Wait()
	325	+	timedOut, canceled := stop()
	326	+	require.Equal(t, !cancelContext, timedOut)
	327	+	require.Equal(t, cancelContext, canceled)
	328	+	_, err = inW.Write([]byte{1})
	329	+	require.ErrorIs(t, err, os.ErrClosed)
	330	+	_, err = outR.Read(make([]byte, 1))
	331	+	require.ErrorIs(t, err, os.ErrClosed)
	332	+	})
	333	+	})
	334	+	}
	335	+	}

M internal/adapters/hints/dump.go +38 -79


@@ -7,6 +7,7 @@ import (
7		"image"	7		"image"
8		"image/color"	8		"image/color"
9		"image/png"	9		"image/png"
			10	+	"math"
10		"os"	11		"os"
11		"path/filepath"	12		"path/filepath"
12		"sort"	13		"sort"

@@ -25,78 +26,54 @@ const (
25			26
26		// Options tunes Run.	27		// Options tunes Run.
27		type Options struct {	28		type Options struct {
			29	+	// Command is the nefer-hints binary: a path, or a name looked up in PATH.
			30	+	Command string
			31	+	// ModelDir is the directory of the detector's model files.
			32	+	ModelDir string
			33	+	// Threshold overrides the minimum detection score, 0.01 to 0.99.
			34	+	// Zero uses model.json or, if absent, the runner default.
			35	+	Threshold float64
28		// DumpDir, when not empty, receives the scanned frames (PNG, annotated	36		// DumpDir, when not empty, receives the scanned frames (PNG, annotated
29		// PNG and JSON). They contain the screen in clear.	37		// PNG and JSON). They contain the screen in clear.
30		DumpDir string	38		DumpDir string
31	-	// Source, when not nil, finds targets in the accessibility tree of the	39	+
32	-	// windows on screen; OCR reads the rest.	40	+	// detector replaces the subprocess in tests.
33	-	Source targetSource	41	+	detector detector
34		}	42		}
35			43
36		// dumpJob is one scan to write. pix is the capture buffer of the scan, owned	44		// dumpJob is one scan to write. pix is the capture buffer of the scan, owned
37		// by the job: writeDump converts it in place.	45		// by the job: writeDump converts it in place.
38		type dumpJob struct {	46		type dumpJob struct {
39	-	when time.Time	47	+	when time.Time
40	-	req ports.HintScanRequest	48	+	req ports.HintScanRequest
41	-	pix []byte	49	+	pix []byte
42	-	m *mapping // owns pix; released after the dump	50	+	m *mapping // owns pix; released after the dump
43	-	stride int	51	+	stride int
44	-	targets []ports.HintTarget	52	+	targets []ports.HintTarget
45	-	crops []image.Rectangle	53	+	dets []detection // detector boxes, physical pixels
46	-	words []OCRWord
47	-	// atspi is the number of leading targets that come from the
48	-	// accessibility tree; the others come from OCR.
49	-	atspi int
50		captureD time.Duration	54		captureD time.Duration
51	-	scanD time.Duration	55	+	detectD time.Duration
52	-	atspiD time.Duration
53		}	56		}
54			57
55		type dumpTarget struct {	58		type dumpTarget struct {
56	-	Kind string `json:"kind"`	59	+	Rect [4]int `json:"rect"` // x1, y1, x2, y2, logical output-local
57	-	Source string `json:"source"`	60	+	Score float32 `json:"score"`
58	-	Rect [4]int `json:"rect"`
59		}	61		}
60			62
61		type dumpMeta struct {	63		type dumpMeta struct {
62	-	Output string `json:"output"`	64	+	Output string `json:"output"`
63	-	Width int `json:"width"`	65	+	Width int `json:"width"`
64	-	Height int `json:"height"`	66	+	Height int `json:"height"`
65	-	Stride int `json:"stride"`	67	+	Stride int `json:"stride"`
66	-	Scale float64 `json:"scale"`	68	+	Scale float64 `json:"scale"`
67	-	LinksOnly bool `json:"links_only"`	69	+	CaptureMS float64 `json:"capture_ms"`
68	-	CaptureMS float64 `json:"capture_ms"`	70	+	DetectMS float64 `json:"detect_ms"`
69	-	ScanMS float64 `json:"scan_ms"`	71	+	TotalMS float64 `json:"total_ms"`
70	-	AtspiMS float64 `json:"atspi_ms"`	72	+	Targets []dumpTarget `json:"targets"`
71	-	Crops []image.Rectangle `json:"ocr_crops"`
72	-	Words []OCRWord `json:"words"`
73	-	Targets []dumpTarget `json:"targets"`
74	-	}
75	-
76	-	func kindName(k ports.HintKind) string {
77	-	switch k {
78	-	case ports.HintLink:
79	-	return "link"
80	-	case ports.HintButton:
81	-	return "button"
82	-	default:
83	-	return "text"
84	-	}
85		}	73		}
86			74
87	-	func kindColor(k ports.HintKind) color.RGBA {	75	+	// boxColor is the outline of the detector boxes.
88	-	switch k {	76	+	var boxColor = color.RGBA{0x00, 0xc8, 0x53, 0xff}
89	-	case ports.HintLink:
90	-	return color.RGBA{0x00, 0xc8, 0x53, 0xff}
91	-	case ports.HintButton:
92	-	return color.RGBA{0x29, 0x62, 0xff, 0xff}
93	-	default:
94	-	return color.RGBA{0x80, 0x80, 0x80, 0xff}
95	-	}
96	-	}
97	-
98	-	// atspiColor is the outline of targets found in the accessibility tree.
99	-	var atspiColor = color.RGBA{0xff, 0x00, 0xff, 0xff}
100			77
101		func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 }	78		func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 }
102			79

@@ -121,21 +98,8 @@ func writeDump(dir string, j *dumpJob) error {
121		return err	98		return err
122		}	99		}
123			100
124	-	scale := j.req.Scale	101	+	for _, d := range j.dets {
125	-	if scale <= 0 {	102	+	outline(img, image.Rect(int(d.x1), int(d.y1), int(math.Ceil(float64(d.x2))), int(math.Ceil(float64(d.y2)))), boxColor)
126	-	scale = 1
127	-	}
128	-	for _, r := range j.crops {
129	-	outline(img, r, color.RGBA{0x80, 0x80, 0x80, 0xff})
130	-	}
131	-	for i, t := range j.targets {
132	-	r := image.Rect(int(float64(t.Rect.X)scale), int(float64(t.Rect.Y)scale),
133	-	int(float64(t.Rect.X+t.Rect.W)scale), int(float64(t.Rect.Y+t.Rect.H)scale))
134	-	c := kindColor(t.Kind)
135	-	if i < j.atspi {
136	-	c = atspiColor
137	-	}
138	-	outline(img, r, c)
139		}	103		}
140		if err := writePNG(prefix+".annotated.png", img); err != nil {	104		if err := writePNG(prefix+".annotated.png", img); err != nil {
141		return err	105		return err

@@ -143,17 +107,12 @@ func writeDump(dir string, j *dumpJob) error {
143			107
144		meta := dumpMeta{	108		meta := dumpMeta{
145		Output: j.req.Output, Width: j.req.Width, Height: j.req.Height, Stride: j.stride,	109		Output: j.req.Output, Width: j.req.Width, Height: j.req.Height, Stride: j.stride,
146	-	Scale: j.req.Scale, LinksOnly: j.req.LinksOnly,	110	+	Scale: j.req.Scale,
147	-	CaptureMS: ms(j.captureD), ScanMS: ms(j.scanD), AtspiMS: ms(j.atspiD),	111	+	CaptureMS: ms(j.captureD), DetectMS: ms(j.detectD), TotalMS: ms(j.captureD + j.detectD),
148	-	Crops: j.crops, Words: j.words,
149		Targets: make([]dumpTarget, 0, len(j.targets)),	112		Targets: make([]dumpTarget, 0, len(j.targets)),
150		}	113		}
151	-	for i, t := range j.targets {	114	+	for _, t := range j.targets {
152	-	src := "ocr"	115	+	meta.Targets = append(meta.Targets, dumpTarget{Rect: [4]int{t.Rect.X, t.Rect.Y, t.Rect.X + t.Rect.W, t.Rect.Y + t.Rect.H}, Score: t.Score})
153	-	if i < j.atspi {
154	-	src = "atspi"
155	-	}
156	-	meta.Targets = append(meta.Targets, dumpTarget{Kind: kindName(t.Kind), Source: src, Rect: [4]int{t.Rect.X, t.Rect.Y, t.Rect.X + t.Rect.W, t.Rect.Y + t.Rect.H}})
157		}	116		}
158		data, err := json.Marshal(meta)	117		data, err := json.Marshal(meta)
159		if err != nil {	118		if err != nil {

M internal/adapters/hints/hints.go +99 -292

@@ -1,15 +1,16 @@
1		// Package hints finds clickable targets for hint mode: it captures the last	1		// Package hints finds clickable targets for hint mode: it captures the last
2	-	// frame of an output through the capture pipeline and reads uncovered windows with OCR.	2	+	// frame of an output through the capture pipeline and sends it to the
			3	+	// nefer-hints detector, a persistent subprocess.
3		package hints	4		package hints
4			5
5		import (	6		import (
			7	+	"cmp"
6		"context"	8		"context"
7		"errors"	9		"errors"
8		"fmt"	10		"fmt"
9	-	"hash/maphash"
10		"image"	11		"image"
11		"math"	12		"math"
12	-	"strings"	13	+	"slices"
13		"time"	14		"time"
14			15
15		"github.com/bnema/neferwl/internal/adapters/logging"	16		"github.com/bnema/neferwl/internal/adapters/logging"

@@ -39,39 +40,18 @@ func Run(ctx context.Context, requests <-chan ports.HintScanRequest, captures ch
39		return run(ctx, requests, captures, captured, results, captureTimeout, opts)	40		return run(ctx, requests, captures, captured, results, captureTimeout, opts)
40		}	41		}
41			42
42	-	// targetSource finds hint targets in the accessibility tree of windows. It
43	-	// returns the targets (logical output-local) and the windows they cover.
44	-	type targetSource interface {
45	-	Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)
46	-	// Close releases the source. It is called once, by the worker goroutine,
47	-	// when the worker returns; the source is not safe for concurrent use.
48	-	Close() error
49	-	}
50	-
51	-	// atspiResult is what the source found for one request.
52	-	type atspiResult struct {
53	-	targets []ports.HintTarget
54	-	covered map[ports.WindowID]bool
55	-	dur time.Duration
56	-	err error
57	-	}
58	-
59		type worker struct {	43		type worker struct {
60	-	source targetSource	44	+	det detector
61	-	cache map[bandKey]bandCache	45	+	n uint64
62	-	cacheSeed maphash.Seed	46	+	dumpDir string
63	-	ocr ocrState	47	+	pending *dumpJob
64	-	warnedSource bool
65	-	n uint64
66	-	dumpDir string
67	-	pending *dumpJob
68		// orphans are capture buffers the output may still write: they are	48		// orphans are capture buffers the output may still write: they are
69		// released when its CaptureDone finally arrives.	49		// released when its CaptureDone finally arrives.
70		orphans map[uint64]*mapping	50		orphans map[uint64]*mapping
71		live int // mappings not released yet	51		live int // mappings not released yet
72		}	52		}
73			53
74	-	// frame is the captured output: BGRA pixels, nil when no capture was needed.	54	+	// frame is the captured output: BGRA pixels.
75		type frame struct {	55		type frame struct {
76		pix []byte	56		pix []byte
77		stride int	57		stride int

@@ -79,11 +59,12 @@ type frame struct {
79		}	59		}
80			60
81		func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {	61		func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {
82	-	w := &worker{dumpDir: opts.DumpDir, source: opts.Source}	62	+	w := &worker{dumpDir: opts.DumpDir, det: opts.detector}
83	-	if opts.Source != nil {	63	+	if w.det == nil {
84	-	// The worker goroutine is the only one to touch the source.	64	+	w.det = newProcDetector(opts.Command, opts.ModelDir, opts.Threshold)
85	-	defer func() { _ = opts.Source.Close() }()
86		}	65		}
			66	+	// The worker goroutine is the only one to touch the detector.
			67	+	defer func() { _ = w.det.Close() }()
87		for {	68		for {
88		select {	69		select {
89		case <-ctx.Done():	70		case <-ctx.Done():

@@ -130,25 +111,6 @@ func send(ctx context.Context, results chan<- ports.HintScanResult, res ports.Hi
130		}	111		}
131		}	112		}
132			113
133	-	// needFrame reports whether the scan must capture the output: OCR has a
134	-	// window to read, or the debug dump wants the frame. With OCR unavailable
135	-	// (tesseract missing) and no dump, the accessibility targets are the result.
136	-	func (w *worker) needFrame(ctx context.Context, req ports.HintScanRequest, ar atspiResult) bool {
137	-	ocr := w.ocrReady(ctx)
138	-	if w.dumpDir != "" {
139	-	return true
140	-	}
141	-	if !ocr {
142	-	return false
143	-	}
144	-	for _, win := range req.Windows {
145	-	if !ar.covered[win.ID] && !windowCrop(win.Rect, req.Scale, req.Width, req.Height).Empty() {
146	-	return true
147	-	}
148	-	}
149	-	return false
150	-	}
151	-
152		func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration) ports.HintScanResult {	114		func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration) ports.HintScanResult {
153		log := logging.For(ctx, "hints")	115		log := logging.For(ctx, "hints")
154		res := ports.HintScanResult{Seq: req.Seq, Output: req.Output}	116		res := ports.HintScanResult{Seq: req.Seq, Output: req.Output}

@@ -161,16 +123,8 @@ func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests
161		if req.Width <= 0 \|\| req.Height <= 0 \|\| req.Width*req.Height > maxPixels {	123		if req.Width <= 0 \|\| req.Height <= 0 \|\| req.Width*req.Height > maxPixels {
162		return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))	124		return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))
163		}	125		}
164	-	// The accessibility query is always waited for: the source is not shared
165	-	// between requests. The capture only starts once it is known whether any
166	-	// window needs OCR.
167	-	atspi := w.startSource(ctx, req)
168	-	defer func() { <-atspi.done }()
169	-	if !w.needFrame(ctx, req, atspi.wait()) {
170	-	return w.scan(ctx, req, frame{}, 0, res, atspi)
171	-	}
172		// The capture buffer is anonymous mapped memory, outside the Go heap. It	126		// The capture buffer is anonymous mapped memory, outside the Go heap. It
173	-	// is released once the crops are extracted, or by the dump job.	127	+	// is released once the detector answered, or by the dump job.
174		m := w.alloc(req.Width * req.Height * 4)	128		m := w.alloc(req.Width * req.Height * 4)
175		keep := false // the mapping outlives this call: dump job or pending capture	129		keep := false // the mapping outlives this call: dump job or pending capture
176		defer func() {	130		defer func() {

@@ -225,7 +179,7 @@ func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests
225		if done.Err != nil {	179		if done.Err != nil {
226		return fail(fmt.Errorf("hints: capture: %w", done.Err))	180		return fail(fmt.Errorf("hints: capture: %w", done.Err))
227		}	181		}
228	-	res = w.scan(ctx, req, frame{pix: buf, stride: stride, m: m}, time.Since(start), res, atspi)	182	+	res = w.scan(ctx, req, frame{pix: buf, stride: stride, m: m}, time.Since(start), res)
229		keep = w.pending != nil && w.pending.m == m	183		keep = w.pending != nil && w.pending.m == m
230		return res	184		return res
231		case other := <-requests:	185		case other := <-requests:

@@ -255,253 +209,106 @@ func (w *worker) reap(id uint64) {
255		}	209		}
256		}	210		}
257			211
258	-	// bandJob is one OCR band of one window crop.	212	+	// minTarget is the smallest target side kept, in logical pixels.
259	-	type bandJob struct {	213	+	const minTarget = 4
260	-	win int // index into the scan's windows
261	-	key bandKey //
262	-	ext image.Rectangle // crop-local rows read, with context
263	-	own [2]int // crop-local rows owned: words centred there are kept
264	-	hash uint64
265	-	hit bool
266	-	gray *image.Gray
267	-	words []OCRWord
268	-	err error
269	-	}
270			214
271	-	func (w worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame, captureDur time.Duration, res ports.HintScanResult, atspi pendingSource) ports.HintScanResult {	215	+	// maxWorkBoxes bounds the quadratic cut-box and merge passes.
			216	+	const maxWorkBoxes = 2000
			217	+
			218	+	// scan sends the frame to the detector and turns the boxes into targets. It
			219	+	// releases the frame, or hands it to the dump job.
			220	+	func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame, captureDur time.Duration, res ports.HintScanResult) ports.HintScanResult {
272		log := logging.For(ctx, "hints")	221		log := logging.For(ctx, "hints")
273	-	ar := atspi.wait()
274	-	if ar.err != nil && !w.warnedSource {
275	-	log.Warn().Err(ar.err).Msg("accessibility targets unavailable; using OCR only")
276	-	w.warnedSource = true
277	-	}
278	-	out := make([]ports.HintTarget, 0, len(ar.targets))
279	-	for _, t := range ar.targets {
280	-	if !req.LinksOnly \|\| t.Kind == ports.HintLink {
281	-	out = append(out, t)
282	-	}
283	-	}
284	-	nAtspi := len(out)
285		t0 := time.Now()	222		t0 := time.Now()
286	-	if w.cacheSeed == (maphash.Seed{}) {	223	+	regions := tileRegions(req.Width, req.Height)
287	-	w.cacheSeed = maphash.MakeSeed()	224	+	dets, err := w.det.Detect(ctx, detFrame{width: req.Width, height: req.Height, stride: fr.stride, m: fr.m, regions: regions})
288	-	}	225	+	rawN := len(dets)
289	-	nextCache := make(map[bandKey]bandCache)	226	+	detectDur := time.Since(t0)
290	-	var crops []image.Rectangle	227	+	if err != nil {
291	-	var allWords []OCRWord	228	+	fr.m.unmap()
292	-	pixels, links := 0, 0	229	+	log.Info().Err(err).Str("output", req.Output).Float64("detect_ms", ms(detectDur)).Msg("hint scan failed")
293	-	ocrOn := fr.pix != nil && w.ocrReady(ctx)	230	+	res.Err = err
294	-	// Phase 1 reads the frame: the bands that miss the cache are copied, as	231	+	return res
295	-	// grayscale, into one arena. Phase 2 (OCR) does not need the frame.
296	-	var jobs []*bandJob
297	-	var rects []image.Rectangle // crop of each window that has bands
298	-	var arena *mapping
299	-	if ocrOn {
300	-	type pending struct {
301	-	r image.Rectangle
302	-	jobs []*bandJob
303	-	}
304	-	var wins []pending
305	-	need := 0
306	-	for _, win := range req.Windows {
307	-	if ar.covered[win.ID] {
308	-	continue
309	-	}
310	-	r := windowCrop(win.Rect, req.Scale, req.Width, req.Height)
311	-	if r.Empty() {
312	-	continue
313	-	}
314	-	crops = append(crops, r)
315	-	pixels += r.Dx() * r.Dy()
316	-	p := pending{r: r}
317	-	for _, b := range bandsOf(r.Dy()) {
318	-	ext := image.Rect(0, b.top, r.Dx(), b.bottom)
319	-	j := &bandJob{win: len(wins), ext: ext, own: [2]int{b.ownTop, b.ownBottom},
320	-	key: bandKey{output: req.Output, window: win.ID, band: b.idx, bands: b.n}}
321	-	j.hash = w.cropHash(fr.pix, fr.stride, ext.Add(r.Min))
322	-	entry, ok := w.cache[j.key]
323	-	if ok && entry.hash == j.hash && entry.ext == ext {
324	-	j.hit, j.words = true, entry.words
325	-	} else {
326	-	need += ext.Dx() * ext.Dy()
327	-	}
328	-	p.jobs = append(p.jobs, j)
329	-	}
330	-	wins = append(wins, p)
331	-	}
332	-	if need > 0 {
333	-	arena = w.alloc(need)
334	-	}
335	-	off := 0
336	-	for _, p := range wins {
337	-	rects = append(rects, p.r)
338	-	for _, j := range p.jobs {
339	-	if !j.hit {
340	-	n := j.ext.Dx() * j.ext.Dy()
341	-	j.gray = cropGray(fr.pix, fr.stride, j.ext.Add(p.r.Min), arena.b[off:off+n])
342	-	off += n
343	-	}
344	-	jobs = append(jobs, j)
345	-	}
346	-	}
347	-	} else if w.dumpDir != "" {
348	-	for _, win := range req.Windows {
349	-	if !ar.covered[win.ID] {
350	-	if r := windowCrop(win.Rect, req.Scale, req.Width, req.Height); !r.Empty() {
351	-	crops = append(crops, r)
352	-	}
353	-	}
354	-	}
355		}	232		}
356	-	if w.dumpDir != "" && fr.pix != nil {	233	+	// Boxes cut by a tile's inner edge are dropped when a neighbour tile
			234	+	// saw the element whole; then, on the union of all tiles and after
			235	+	// rounding to logical pixels, duplicates and nested boxes are merged.
			236	+	dets = prepareDetections(dets, regions, req.Width, req.Height)
			237	+	res.Targets = mergeTargets(targetsOf(dropCutBoxes(dets, regions), req.Scale, req.Width, req.Height))
			238	+	if w.dumpDir != "" {
357		// The dump reads the frame after the result is sent and releases it.	239		// The dump reads the frame after the result is sent and releases it.
358	-	w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, crops: crops, captureD: captureDur, atspiD: ar.dur}	240	+	w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, targets: res.Targets, dets: dets, captureD: captureDur, detectD: detectDur}
359		} else {	241		} else {
360	-	fr.m.unmap() // the crops are extracted: give the frame back	242	+	fr.m.unmap()
361		}	243		}
362	-	fr.pix = nil	244	+	log.Info().Str("output", req.Output).Float64("capture_ms", ms(captureDur)).Float64("detect_ms", ms(detectDur)).
363	-		245	+	Int("regions", len(regions)).Int("raw_boxes", rawN).Int("prepared_boxes", len(dets)).Int("targets", len(res.Targets)).Msg("hint scan")
364	-	hits := 0
365	-	if len(jobs) > 0 {
366	-	w.recognize(ctx, jobs)
367	-	for _, j := range jobs {
368	-	if j.hit {
369	-	hits++
370	-	}
371	-	}
372	-	}
373	-	arena.unmap()
374	-
375	-	// Merge the bands of each window, in order.
376	-	k := 0
377	-	for wi, r := range rects {
378	-	var words []OCRWord
379	-	for ; k < len(jobs) && jobs[k].win == wi; k++ {
380	-	j := jobs[k]
381	-	if j.err != nil {
382	-	failureWarning.Do(func() { log.Warn().Err(j.err).Msg("OCR unavailable; install tesseract and tesseract-data-eng") })
383	-	continue
384	-	}
385	-	nextCache[j.key] = bandCache{hash: j.hash, ext: j.ext, words: j.words}
386	-	for _, word := range j.words {
387	-	word.Rect = word.Rect.Add(r.Min)
388	-	if word.Underlined {
389	-	word.UnderL, word.UnderR = word.UnderL+r.Min.X, word.UnderR+r.Min.X
390	-	}
391	-	if !duplicateWord(words, word) {
392	-	words = append(words, word)
393	-	}
394	-	}
395	-	}
396	-	allWords = append(allWords, words...)
397	-	targets := textLinks(words, req.Scale)
398	-	links += len(targets)
399	-	out = append(out, targets...)
400	-	}
401	-	w.cache = nextCache
402	-	ocrDur := time.Since(t0)
403	-	res.Targets = out
404	-	if p := w.pending; p != nil {
405	-	p.targets, p.atspi, p.words, p.scanD = out, nAtspi, allWords, ocrDur
406	-	}
407	-	log.Info().Str("output", req.Output).
408	-	Float64("atspi_ms", ms(ar.dur)).Int("atspi_windows", len(ar.covered)).Int("atspi_targets", nAtspi).
409	-	Int("ocr_windows", len(rects)).Float64("ocr_ms", ms(ocrDur)).Int("ocr_px", pixels).
410	-	Int("ocr_bands", len(jobs)).Int("band_hits", hits).
411	-	Int("words", len(allWords)).Int("ocr_links", links).Bool("cache_hit", len(jobs) > 0 && hits == len(jobs)).
412	-	Int("targets", len(out)).Msg("hint scan")
413		return res	246		return res
414		}	247		}
415			248
416	-	// duplicateWord reports whether words already holds word (the same text at	249	+	// targetsOf converts boxes in physical pixels to logical output-local
417	-	// the same place), as happens where two bands overlap.	250	+	// targets. A box with a non-finite coordinate or score, a score outside
418	-	func duplicateWord(words []OCRWord, word OCRWord) bool {	251	+	// [0,1] or no area is dropped; the others are clamped to the width x height
419	-	for _, old := range words {	252	+	// frame (physical), rounded outwards to logical pixels, and dropped when
420	-	if strings.EqualFold(word.Text, old.Text) && overlap(word.Rect, old.Rect) > .5 {	253	+	// under minTarget on a side.
421	-	return true	254	+	func targetsOf(dets []detection, scale float64, width, height int) []ports.HintTarget {
422	-	}
423	-	}
424	-	return false
425	-	}
426	-
427	-	// bandKey identifies one OCR band of a window on an output.
428	-	type bandKey struct {
429	-	output string
430	-	window ports.WindowID
431	-	band int
432	-	bands int
433	-	}
434	-
435	-	// bandCache is the OCR result of a band whose pixels hash to hash. words are
436	-	// crop-local and only the ones the band owns.
437	-	type bandCache struct {
438	-	hash uint64
439	-	ext image.Rectangle
440	-	words []OCRWord
441	-	}
442	-
443	-	// windowCrop rounds outwards in physical pixels, then intersects the output.
444	-	func windowCrop(r ports.Rect, scale float64, width, height int) image.Rectangle {
445		if scale <= 0 {	255		if scale <= 0 {
446		scale = 1	256		scale = 1
447		}	257		}
448	-	if r.W <= 0 \|\| r.H <= 0 {	258	+	fw, fh := float64(width), float64(height)
449	-	return image.Rectangle{}	259	+	out := make([]ports.HintTarget, 0, len(dets))
450	-	}	260	+	for _, d := range dets {
451	-	return image.Rect(int(math.Floor(float64(r.X)scale)), int(math.Floor(float64(r.Y)scale)),	261	+	v := [4]float64{float64(d.x1), float64(d.y1), float64(d.x2), float64(d.y2)}
452	-	int(math.Ceil(float64(r.X+r.W)scale)), int(math.Ceil(float64(r.Y+r.H)scale))).Intersect(image.Rect(0, 0, width, height))	262	+	ok := d.score >= 0 && d.score <= 1 // false for NaN too
453	-	}	263	+	for _, f := range v {
454	-		264	+	ok = ok && !math.IsNaN(f) && !math.IsInf(f, 0)
455	-	// cropHash hashes the raw pixels of r without copying them.	265	+	}
456	-	func (w *worker) cropHash(buf []byte, stride int, r image.Rectangle) uint64 {	266	+	if !ok \|\| v[2] <= v[0] \|\| v[3] <= v[1] {
457	-	var h maphash.Hash	267	+	continue
458	-	h.SetSeed(w.cacheSeed)	268	+	}
459	-	for y := r.Min.Y; y < r.Max.Y; y++ {	269	+	// Clamp before any int conversion: huge finite values stay safe.
460	-	_, _ = h.Write(buf[ystride+r.Min.X4 : ystride+r.Max.X4])	270	+	x1, y1 := math.Max(0, math.Min(v[0], fw)), math.Max(0, math.Min(v[1], fh))
			271	+	x2, y2 := math.Max(0, math.Min(v[2], fw)), math.Max(0, math.Min(v[3], fh))
			272	+	lx1, ly1 := int(math.Floor(x1/scale)), int(math.Floor(y1/scale))
			273	+	lx2, ly2 := int(math.Ceil(x2/scale)), int(math.Ceil(y2/scale))
			274	+	if lx2-lx1 < minTarget \|\| ly2-ly1 < minTarget {
			275	+	continue
			276	+	}
			277	+	out = append(out, ports.HintTarget{Rect: ports.Rect{X: lx1, Y: ly1, W: lx2 - lx1, H: ly2 - ly1}, Score: d.score})
461		}	278		}
462	-	return h.Sum64()	279	+	return out
463		}	280		}
464			281
465	-	// cropGray converts the BGRA pixels of r to 8-bit luma into dst, which must	282	+	// prepareDetections validates and clamps boxes before any pairwise work,
466	-	// hold r.Dx()*r.Dy() bytes; only this band is converted, not the output.	283	+	// preserving their region metadata for cut-edge suppression.
467	-	func cropGray(buf []byte, stride int, r image.Rectangle, dst []byte) *image.Gray {	284	+	func prepareDetections(in []detection, regions []region, width, height int) []detection {
468	-	img := &image.Gray{Pix: dst[:r.Dx()*r.Dy()], Stride: r.Dx(), Rect: image.Rect(0, 0, r.Dx(), r.Dy())}	285	+	out := make([]detection, 0, len(in))
469	-	for y := 0; y < r.Dy(); y++ {	286	+	for _, d := range in {
470	-	src := buf[(r.Min.Y+y)stride+r.Min.X4:][:r.Dx()*4]	287	+	if d.region < 0 \|\| d.region >= max(1, len(regions)) \|\| !(d.score >= 0 && d.score <= 1) {
471	-	row := img.Pix[y*img.Stride:][:r.Dx()]	288	+	continue
472	-	for x := range row {	289	+	}
473	-	p := src[x4 : x4+4 : x*4+4]	290	+	valid := true
474	-	row[x] = uint8((uint32(p[2])299 + uint32(p[1])587 + uint32(p[0])*114 + 500) / 1000)	291	+	for _, v := range []float32{d.x1, d.y1, d.x2, d.y2} {
			292	+	if math.IsNaN(float64(v)) \|\| math.IsInf(float64(v), 0) {
			293	+	valid = false
			294	+	break
			295	+	}
475		}	296		}
			297	+	if !valid {
			298	+	continue
			299	+	}
			300	+	d.x1 = max(0, min(d.x1, float32(width)))
			301	+	d.x2 = max(0, min(d.x2, float32(width)))
			302	+	d.y1 = max(0, min(d.y1, float32(height)))
			303	+	d.y2 = max(0, min(d.y2, float32(height)))
			304	+	if d.x2 <= d.x1 \|\| d.y2 <= d.y1 {
			305	+	continue
			306	+	}
			307	+	out = append(out, d)
476		}	308		}
477	-	return img	309	+	slices.SortStableFunc(out, func(a, b detection) int { return cmp.Compare(b.score, a.score) })
478	-	}	310	+	if len(out) > maxWorkBoxes {
479	-		311	+	out = out[:maxWorkBoxes]
480	-	// pendingSource is an accessibility query in flight.
481	-	type pendingSource struct {
482	-	done chan struct{}
483	-	res atspiResult
484	-	}
485	-
486	-	// startSource starts the accessibility query of req, if there is a source and
487	-	// a window to ask about. done is closed when res is set.
488	-	func (w worker) startSource(ctx context.Context, req ports.HintScanRequest) pendingSource {
489	-	p := &pendingSource{done: make(chan struct{})}
490	-	if w.source == nil \|\| len(req.Windows) == 0 {
491	-	close(p.done)
492	-	return p
493		}	312		}
494	-	go func() {	313	+	return out
495	-	defer close(p.done)
496	-	t0 := time.Now()
497	-	targets, covered, err := w.source.Targets(ctx, req.Windows, req.LinksOnly)
498	-	p.res = atspiResult{targets: targets, covered: covered, dur: time.Since(t0), err: err}
499	-	}()
500	-	return p
501	-	}
502	-
503	-	// wait returns the result of the query.
504	-	func (p *pendingSource) wait() atspiResult {
505	-	<-p.done
506	-	return p.res
507		}	314		}

M internal/adapters/hints/hints_test.go +160 -178

@@ -1,15 +1,16 @@
1		package hints	1		package hints
2			2
3		import (	3		import (
			4	+	"bytes"
4		"context"	5		"context"
5		"encoding/json"	6		"encoding/json"
6		"errors"	7		"errors"
7		"fmt"	8		"fmt"
			9	+	"github.com/bnema/zerowrap"
8		"image"	10		"image"
9	-	"image/draw"
10		_ "image/png"	11		_ "image/png"
			12	+	"math"
11		"os"	13		"os"
12	-	"os/exec"
13		"path/filepath"	14		"path/filepath"
14		"strings"	15		"strings"
15		"testing"	16		"testing"

@@ -39,6 +40,13 @@ func startCapturing(t testing.T, timeout time.Duration) harness {
39			40
40		func startWith(t testing.T, timeout time.Duration, opts Options) harness {	41		func startWith(t testing.T, timeout time.Duration, opts Options) harness {
41		t.Helper()	42		t.Helper()
			43	+	if opts.detector == nil {
			44	+	// Tests of the capture flow do not care what the detector finds.
			45	+	det := newMockdetector(t)
			46	+	det.EXPECT().Detect(mock.Anything, mock.Anything).Return(nil, nil).Maybe()
			47	+	det.EXPECT().Close().Return(nil).Maybe()
			48	+	opts.detector = det
			49	+	}
42		h := &harness{	50		h := &harness{
43		requests: make(chan ports.HintScanRequest, 1),	51		requests: make(chan ports.HintScanRequest, 1),
44		captures: make(chan ports.CaptureRequest, 4),	52		captures: make(chan ports.CaptureRequest, 4),

@@ -55,51 +63,99 @@ func startWith(t testing.T, timeout time.Duration, opts Options) harness {
55		return h	63		return h
56		}	64		}
57			65
58	-	func loadBGRA(t *testing.T) (pix []byte, w, hgt int) {
59	-	t.Helper()
60	-	f, err := os.Open("testdata/browser.png")
61	-	require.NoError(t, err)
62	-	defer f.Close()
63	-	img, _, err := image.Decode(f)
64	-	require.NoError(t, err)
65	-	b := img.Bounds()
66	-	rgba := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
67	-	draw.Draw(rgba, rgba.Bounds(), img, b.Min, draw.Src)
68	-	for i := 0; i < len(rgba.Pix); i += 4 {
69	-	rgba.Pix[i], rgba.Pix[i+2] = rgba.Pix[i+2], rgba.Pix[i]
70	-	rgba.Pix[i+3] = 0xff
71	-	}
72	-	return rgba.Pix, b.Dx(), b.Dy()
73	-	}
74	-
75		func TestHintScan(t *testing.T) {	66		func TestHintScan(t *testing.T) {
76	-	if _, err := exec.LookPath("tesseract"); err != nil {	67	+	det := newMockdetector(t)
77	-	t.Skip("tesseract not on PATH")	68	+	det.EXPECT().Close().Return(nil).Once()
78	-	}	69	+	var got detFrame
79	-	pix, w, hgt := loadBGRA(t)	70	+	det.EXPECT().Detect(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, f detFrame) ([]detection, error) {
80	-	h := start(t, 5*time.Second)	71	+	got = f
81	-		72	+	return []detection{{x1: 10, y1: 20, x2: 50, y2: 30, score: 0.8}}, nil
82	-	// Plays the output owner.	73	+	}).Once()
			74	+	h := startWith(t, 5*time.Second, Options{detector: det})
83		go func() {	75		go func() {
84		creq := <-h.captures	76		creq := <-h.captures
85	-	copy(creq.Mapped, pix)
86		h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}	77		h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
87		}()	78		}()
88	-		79	+	h.requests <- ports.HintScanRequest{Seq: 7, Output: "HEADLESS-1", Width: 64, Height: 32, Scale: 1}
89	-	h.requests <- ports.HintScanRequest{Seq: 7, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
90		res := <-h.results	80		res := <-h.results
91		require.NoError(t, res.Err)	81		require.NoError(t, res.Err)
92		require.Equal(t, uint64(7), res.Seq)	82		require.Equal(t, uint64(7), res.Seq)
93		require.Equal(t, "HEADLESS-1", res.Output)	83		require.Equal(t, "HEADLESS-1", res.Output)
94	-	var links int	84	+	require.Equal(t, []ports.HintTarget{{Rect: ports.Rect{X: 10, Y: 20, W: 40, H: 10}, Score: 0.8}}, res.Targets)
95	-	for _, tg := range res.Targets {	85	+	require.Equal(t, 64, got.width)
96	-	require.Positive(t, tg.Rect.W)	86	+	require.Equal(t, 32, got.height)
97	-	require.Positive(t, tg.Rect.H)	87	+	require.Equal(t, 64*4, got.stride)
98	-	if tg.Kind == ports.HintLink {	88	+	}
99	-	links++	89	+
			90	+	func TestTargetsOf(t *testing.T) {
			91	+	nan := float32(math.NaN())
			92	+	inf := float32(math.Inf(1))
			93	+	r := func(x, y, w, h int, score float32) []ports.HintTarget {
			94	+	return []ports.HintTarget{{Rect: ports.Rect{X: x, Y: y, W: w, H: h}, Score: score}}
			95	+	}
			96	+	for _, tc := range []struct {
			97	+	name string
			98	+	det detection
			99	+	scale float64
			100	+	want []ports.HintTarget
			101	+	}{
			102	+	{"inside", detection{10, 20, 50, 40, 0.5, 0}, 1, r(10, 20, 40, 20, 0.5)},
			103	+	{"scale 2 rounds outwards, clamped at 100", detection{11, 21, 100.2, 61, 0.5, 0}, 2, r(5, 10, 45, 21, 0.5)},
			104	+	{"zero scale reads as 1", detection{0, 0, 10, 10, 0.5, 0}, 0, r(0, 0, 10, 10, 0.5)},
			105	+	{"exactly 4x4 kept", detection{0, 0, 8, 8, 0.4, 0}, 2, r(0, 0, 4, 4, 0.4)},
			106	+	{"3 logical wide dropped", detection{0, 0, 5, 40, 0.9, 0}, 2, nil},
			107	+	{"3 logical high dropped", detection{0, 0, 40, 5, 0.9, 0}, 2, nil},
			108	+	{"fully outside", detection{200, 10, 300, 50, 0.5, 0}, 1, nil},
			109	+	{"fully before origin", detection{-50, -50, -10, -10, 0.5, 0}, 1, nil},
			110	+	{"partly outside is clamped", detection{-20, 70, 150, 120, 0.5, 0}, 1, r(0, 70, 100, 30, 0.5)},
			111	+	{"partly outside, scale 2", detection{-20, 70, 150, 120, 0.5, 0}, 2, r(0, 35, 50, 15, 0.5)},
			112	+	{"inverted x", detection{50, 10, 20, 40, 0.5, 0}, 1, nil},
			113	+	{"inverted y", detection{10, 40, 50, 10, 0.5, 0}, 1, nil},
			114	+	{"empty x", detection{10, 10, 10, 40, 0.5, 0}, 1, nil},
			115	+	{"empty y", detection{10, 10, 40, 10, 0.5, 0}, 1, nil},
			116	+	{"NaN coordinate", detection{nan, 0, 50, 50, 0.5, 0}, 1, nil},
			117	+	{"Inf coordinate", detection{0, 0, inf, 50, 0.5, 0}, 1, nil},
			118	+	{"NaN score", detection{0, 0, 50, 50, nan, 0}, 1, nil},
			119	+	{"Inf score", detection{0, 0, 50, 50, inf, 0}, 1, nil},
			120	+	{"score above 1", detection{0, 0, 50, 50, 1.5, 0}, 1, nil},
			121	+	{"negative score", detection{0, 0, 50, 50, -0.1, 0}, 1, nil},
			122	+	{"score 0 and 1 kept", detection{0, 0, 50, 50, 1, 0}, 1, r(0, 0, 50, 50, 1)},
			123	+	{"huge coordinates", detection{-1e30, -1e30, 1e30, 1e30, 0.5, 0}, 1, r(0, 0, 100, 100, 0.5)},
			124	+	{"huge positive only", detection{10, 10, 1e30, 1e30, 0.5, 0}, 2, r(5, 5, 45, 45, 0.5)},
			125	+	{"huge outside", detection{1e30, 1e30, 3e30, 3e30, 0.5, 0}, 1, nil},
			126	+	} {
			127	+	t.Run(tc.name, func(t *testing.T) {
			128	+	got := targetsOf([]detection{tc.det}, tc.scale, 100, 100)
			129	+	if tc.want == nil {
			130	+	require.Empty(t, got)
			131	+	return
			132	+	}
			133	+	require.Equal(t, tc.want, got)
			134	+	})
			135	+	}
			136	+	}
			137	+
			138	+	// A detector error fails the scan; the worker goes on and the buffer is released.
			139	+	func TestHintDetectorErrorFailsScan(t *testing.T) {
			140	+	boom := errors.New("detector gone")
			141	+	det := newMockdetector(t)
			142	+	det.EXPECT().Close().Return(nil).Once()
			143	+	det.EXPECT().Detect(mock.Anything, mock.Anything).Return(nil, boom).Once()
			144	+	det.EXPECT().Detect(mock.Anything, mock.Anything).Return(nil, nil).Once()
			145	+	h := startWith(t, 5*time.Second, Options{detector: det})
			146	+	for i, want := range []error{boom, nil} {
			147	+	go func() {
			148	+	creq := <-h.captures
			149	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
			150	+	}()
			151	+	h.requests <- ports.HintScanRequest{Seq: uint64(i + 1), Output: "o", Width: 16, Height: 16, Scale: 1}
			152	+	res := <-h.results
			153	+	if want != nil {
			154	+	require.ErrorIs(t, res.Err, want)
			155	+	} else {
			156	+	require.NoError(t, res.Err)
100		}	157		}
101		}	158		}
102	-	require.GreaterOrEqual(t, links, 1)
103		}	159		}
104			160
105		func TestHintRequestShape(t *testing.T) {	161		func TestHintRequestShape(t *testing.T) {

@@ -166,19 +222,6 @@ func TestHintBadSize(t *testing.T) {
166		require.ErrorIs(t, res.Err, ErrBadRequest)	222		require.ErrorIs(t, res.Err, ErrBadRequest)
167		}	223		}
168			224
169	-	func scanOnce(t testing.T, h harness, pix []byte, w, hgt int, seq uint64) ports.HintScanResult {
170	-	t.Helper()
171	-	go func() {
172	-	creq := <-h.captures
173	-	copy(creq.Mapped, pix)
174	-	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
175	-	}()
176	-	h.requests <- ports.HintScanRequest{Seq: seq, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
177	-	res := <-h.results
178	-	require.NoError(t, res.Err)
179	-	return res
180	-	}
181	-
182		// waitFiles waits for the dump the worker writes after sending the result.	225		// waitFiles waits for the dump the worker writes after sending the result.
183		func waitFiles(t *testing.T, dir string, n int) []os.DirEntry {	226		func waitFiles(t *testing.T, dir string, n int) []os.DirEntry {
184		t.Helper()	227		t.Helper()

@@ -190,19 +233,32 @@ func waitFiles(t *testing.T, dir string, n int) []os.DirEntry {
190		return entries	233		return entries
191		}	234		}
192			235
			236	+	// scanOnce runs one scan on a 64x32 frame and waits for its result.
			237	+	func scanOnce(t testing.T, h harness, seq uint64) ports.HintScanResult {
			238	+	t.Helper()
			239	+	go func() {
			240	+	creq := <-h.captures
			241	+	for i := 0; i < len(creq.Mapped); i += 4 {
			242	+	creq.Mapped[i], creq.Mapped[i+1], creq.Mapped[i+2] = 10, 20, 30
			243	+	}
			244	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
			245	+	}()
			246	+	h.requests <- ports.HintScanRequest{Seq: seq, Output: "HEADLESS-1", Width: 64, Height: 32, Scale: 1}
			247	+	res := <-h.results
			248	+	require.NoError(t, res.Err)
			249	+	return res
			250	+	}
			251	+
193		func TestHintDump(t *testing.T) {	252		func TestHintDump(t *testing.T) {
194	-	if _, err := exec.LookPath("tesseract"); err != nil {	253	+	det := newMockdetector(t)
195	-	t.Skip("tesseract not on PATH")	254	+	det.EXPECT().Close().Return(nil).Once()
196	-	}	255	+	det.EXPECT().Detect(mock.Anything, mock.Anything).Return([]detection{{x1: 8, y1: 4, x2: 40, y2: 20, score: 0.75}}, nil).Once()
197	-	pix, w, hgt := loadBGRA(t)
198		dir := filepath.Join(t.TempDir(), "hints")	256		dir := filepath.Join(t.TempDir(), "hints")
199	-	h := startWith(t, 5*time.Second, Options{DumpDir: dir})	257	+	h := startWith(t, 5*time.Second, Options{DumpDir: dir, detector: det})
200	-	res := scanOnce(t, h, pix, w, hgt, 1)	258	+	res := scanOnce(t, h, 1)
201	-	require.NotEmpty(t, res.Targets)	259	+	require.Len(t, res.Targets, 1)
202			260
203		entries := waitFiles(t, dir, 3)	261		entries := waitFiles(t, dir, 3)
204	-	// A second request is only served once the dump is written.
205	-	_ = scanOnceDone(t, h)
206		var png, ann, js string	262		var png, ann, js string
207		for _, e := range entries {	263		for _, e := range entries {
208		info, err := e.Info()	264		info, err := e.Info()

@@ -228,61 +284,45 @@ func TestHintDump(t *testing.T) {
228		data, err := os.ReadFile(filepath.Join(dir, js))	284		data, err := os.ReadFile(filepath.Join(dir, js))
229		require.NoError(t, err)	285		require.NoError(t, err)
230		var meta struct {	286		var meta struct {
231	-	Output string `json:"output"`	287	+	Output string `json:"output"`
232	-	Width int `json:"width"`	288	+	Width int `json:"width"`
233	-	Height int `json:"height"`	289	+	Height int `json:"height"`
234	-	Stride int `json:"stride"`	290	+	Stride int `json:"stride"`
235	-	Scale float64 `json:"scale"`	291	+	Scale float64 `json:"scale"`
236	-	LinksOnly bool `json:"links_only"`	292	+	DetectMS float64 `json:"detect_ms"`
237	-	Targets []struct {	293	+	TotalMS float64 `json:"total_ms"`
238	-	Kind string `json:"kind"`	294	+	Targets []struct {
239	-	Rect [4]int `json:"rect"`	295	+	Rect [4]int `json:"rect"`
			296	+	Score float32 `json:"score"`
240		} `json:"targets"`	297		} `json:"targets"`
241		}	298		}
242		require.NoError(t, json.Unmarshal(data, &meta))	299		require.NoError(t, json.Unmarshal(data, &meta))
243		require.Equal(t, "HEADLESS-1", meta.Output)	300		require.Equal(t, "HEADLESS-1", meta.Output)
244	-	require.Equal(t, w, meta.Width)	301	+	require.Equal(t, 64, meta.Width)
245	-	require.Equal(t, hgt, meta.Height)	302	+	require.Equal(t, 32, meta.Height)
246	-	require.Equal(t, w*4, meta.Stride)	303	+	require.Equal(t, 64*4, meta.Stride)
247		require.Equal(t, 1.0, meta.Scale)	304		require.Equal(t, 1.0, meta.Scale)
248	-	require.Len(t, meta.Targets, len(res.Targets))	305	+	require.GreaterOrEqual(t, meta.TotalMS, meta.DetectMS)
249	-	for i, tg := range res.Targets {	306	+	require.Len(t, meta.Targets, 1)
250	-	require.Equal(t, [4]int{tg.Rect.X, tg.Rect.Y, tg.Rect.X + tg.Rect.W, tg.Rect.Y + tg.Rect.H}, meta.Targets[i].Rect)	307	+	require.Equal(t, [4]int{8, 4, 40, 20}, meta.Targets[0].Rect)
251	-	}	308	+	require.Equal(t, float32(0.75), meta.Targets[0].Score)
252			309
253		f, err := os.Open(filepath.Join(dir, png))	310		f, err := os.Open(filepath.Join(dir, png))
254		require.NoError(t, err)	311		require.NoError(t, err)
255		defer f.Close()	312		defer f.Close()
256		img, _, err := image.Decode(f)	313		img, _, err := image.Decode(f)
257		require.NoError(t, err)	314		require.NoError(t, err)
258	-	require.Equal(t, image.Rect(0, 0, w, hgt), img.Bounds())	315	+	require.Equal(t, image.Rect(0, 0, 64, 32), img.Bounds())
259		r, g, b, _ := img.At(0, 0).RGBA()	316		r, g, b, _ := img.At(0, 0).RGBA()
260	-	require.Equal(t, uint32(pix[2])*0x101, r)	317	+	require.Equal(t, uint32(30)*0x101, r)
261	-	require.Equal(t, uint32(pix[1])*0x101, g)	318	+	require.Equal(t, uint32(20)*0x101, g)
262	-	require.Equal(t, uint32(pix[0])*0x101, b)	319	+	require.Equal(t, uint32(10)*0x101, b)
263	-	}
264	-
265	-	// scanOnceDone sends a request that fails fast on a blank capture and waits
266	-	// for its result, which proves the previous dump finished.
267	-	func scanOnceDone(t testing.T, h harness) ports.HintScanResult {
268	-	t.Helper()
269	-	go func() {
270	-	creq := <-h.captures
271	-	h.captured <- ports.CaptureDone{ID: creq.ID, Err: errors.New("stop")}
272	-	}()
273	-	h.requests <- ports.HintScanRequest{Seq: 99, Output: "x", Width: 8, Height: 8, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 8, H: 8}}}}
274	-	return <-h.results
275		}	320		}
276			321
277		func TestHintDumpDisabled(t *testing.T) {	322		func TestHintDumpDisabled(t *testing.T) {
278	-	if _, err := exec.LookPath("tesseract"); err != nil {
279	-	t.Skip("tesseract not on PATH")
280	-	}
281	-	pix, w, hgt := loadBGRA(t)
282		dir := filepath.Join(t.TempDir(), "hints")	323		dir := filepath.Join(t.TempDir(), "hints")
283		h := start(t, 5*time.Second)	324		h := start(t, 5*time.Second)
284	-	scanOnce(t, h, pix, w, hgt, 1)	325	+	scanOnce(t, h, 1)
285	-	scanOnceDone(t, h)
286		_, err := os.Stat(dir)	326		_, err := os.Stat(dir)
287		require.ErrorIs(t, err, os.ErrNotExist)	327		require.ErrorIs(t, err, os.ErrNotExist)
288		}	328		}

@@ -302,67 +342,25 @@ func TestHintDumpRetention(t *testing.T) {
302		require.Equal(t, "20260101-000000.010-o.annotated.png", entries[0].Name())	342		require.Equal(t, "20260101-000000.010-o.annotated.png", entries[0].Name())
303		}	343		}
304			344
305	-	func TestScanMergesSource(t *testing.T) {
306	-	pix, w, hgt := loadBGRA(t)
307	-	src := newMocktargetSource(t)
308	-	src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
309	-	wins := []ports.HintWindow{{ID: 4, PID: 9, Rect: ports.Rect{X: 0, Y: 0, W: w, H: hgt}}}
310	-	at := []ports.HintTarget{{Rect: ports.Rect{X: 5, Y: 5, W: 30, H: 10}, Kind: ports.HintLink}}
311	-	src.EXPECT().Targets(mock.Anything, wins, false).Return(at, map[ports.WindowID]bool{4: true}, nil).Once()
312	-	h := startWith(t, 5*time.Second, Options{Source: src})
313	-	go func() {
314	-	creq := <-h.captures
315	-	copy(creq.Mapped, pix)
316	-	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
317	-	}()
318	-	h.requests <- ports.HintScanRequest{Seq: 1, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: wins}
319	-	res := <-h.results
320	-	require.NoError(t, res.Err)
321	-	// The whole frame is covered: only the accessibility target remains.
322	-	require.Equal(t, at, res.Targets)
323	-	}
324	-
325	-	func TestScanSourceErrorUsesOCR(t *testing.T) {
326	-	if _, err := exec.LookPath("tesseract"); err != nil {
327	-	t.Skip("tesseract not on PATH")
328	-	}
329	-	pix, w, hgt := loadBGRA(t)
330	-	src := newMocktargetSource(t)
331	-	src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
332	-	wins := []ports.HintWindow{{ID: 4, PID: 9, Rect: ports.Rect{W: w, H: hgt}}}
333	-	src.EXPECT().Targets(mock.Anything, wins, false).Return(nil, nil, errors.New("bus gone")).Once()
334	-	h := startWith(t, 5*time.Second, Options{Source: src})
335	-	go func() {
336	-	creq := <-h.captures
337	-	copy(creq.Mapped, pix)
338	-	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
339	-	}()
340	-	h.requests <- ports.HintScanRequest{Seq: 1, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: wins}
341	-	res := <-h.results
342	-	require.NoError(t, res.Err)
343	-	require.NotEmpty(t, res.Targets)
344	-	}
345	-
346		// The worker keeps no frame: the mapped buffer belongs to the scan, and to the	345		// The worker keeps no frame: the mapped buffer belongs to the scan, and to the
347		// dump job until it is written; every mapping is released afterwards.	346		// dump job until it is written; every mapping is released afterwards.
348		func TestWorkerHoldsNoFrameAfterScan(t *testing.T) {	347		func TestWorkerHoldsNoFrameAfterScan(t *testing.T) {
349		for _, dumpDir := range []string{"", filepath.Join(t.TempDir(), "hints")} {	348		for _, dumpDir := range []string{"", filepath.Join(t.TempDir(), "hints")} {
350	-	// OCR is "installed" but its binary cannot run: only the memory matters.	349	+	det := newMockdetector(t)
351	-	w := &worker{dumpDir: dumpDir, ocr: ocrState{done: true, binary: "/nonexistent/tesseract", lang: "eng"}}	350	+	det.EXPECT().Detect(mock.Anything, mock.Anything).Return(nil, nil).Once()
352	-	requests := make(chan ports.HintScanRequest)	351	+	w := &worker{dumpDir: dumpDir, det: det}
353		captures := make(chan ports.CaptureRequest, 1)	352		captures := make(chan ports.CaptureRequest, 1)
354		captured := make(chan ports.CaptureDone, 1)	353		captured := make(chan ports.CaptureDone, 1)
355	-	results := make(chan ports.HintScanResult, 1)
356		go func() {	354		go func() {
357		creq := <-captures	355		creq := <-captures
358		captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}	356		captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
359		}()	357		}()
360	-	req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 64, H: 32}}}}	358	+	req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1}
361	-	res := w.serve(context.Background(), req, requests, captures, captured, results, 5*time.Second)	359	+	res := w.serve(context.Background(), req, make(chan ports.HintScanRequest), captures, captured, make(chan ports.HintScanResult, 1), 5*time.Second)
362		require.NoError(t, res.Err)	360		require.NoError(t, res.Err)
363		if dumpDir == "" {	361		if dumpDir == "" {
364		require.Nil(t, w.pending)	362		require.Nil(t, w.pending)
365	-	require.Zero(t, w.live, "frame and crops are released before the result")	363	+	require.Zero(t, w.live, "the frame is released before the result")
366		} else {	364		} else {
367		require.NotNil(t, w.pending)	365		require.NotNil(t, w.pending)
368		require.Equal(t, 1, w.live, "only the frame, kept for the dump")	366		require.Equal(t, 1, w.live, "only the frame, kept for the dump")

@@ -370,7 +368,6 @@ func TestWorkerHoldsNoFrameAfterScan(t *testing.T) {
370		w.dump(context.Background())	368		w.dump(context.Background())
371		require.Nil(t, w.pending)	369		require.Nil(t, w.pending)
372		require.Zero(t, w.live)	370		require.Zero(t, w.live)
373	-	require.Empty(t, w.cache) // OCR failed: nothing cached
374		}	371		}
375		}	372		}
376			373

@@ -389,39 +386,6 @@ func TestTimedOutCaptureIsReleasedWhenDone(t *testing.T) {
389		require.Zero(t, w.live)	386		require.Zero(t, w.live)
390		}	387		}
391			388
392	-	// Without tesseract OCR is off for good: no capture, only accessibility targets.
393	-	func TestNoTesseractSkipsCapture(t *testing.T) {
394	-	t.Setenv("PATH", t.TempDir())
395	-	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
396	-	covered := ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: 30, W: 5, H: 5}}
397	-	wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}, {ID: 2, Rect: ports.Rect{X: 20, W: 20, H: 20}}}
398	-	src := newMocktargetSource(t)
399	-	src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
400	-	src.EXPECT().Targets(mock.Anything, wins, false).Return([]ports.HintTarget{button, covered}, map[ports.WindowID]bool{2: true}, nil).Times(2)
401	-	h := startWith(t, time.Second, Options{Source: src})
402	-	for seq := uint64(1); seq <= 2; seq++ {
403	-	h.requests <- ports.HintScanRequest{Seq: seq, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: wins}
404	-	res := <-h.results
405	-	require.NoError(t, res.Err)
406	-	require.Equal(t, []ports.HintTarget{button, covered}, res.Targets)
407	-	}
408	-	require.Empty(t, h.captures)
409	-	}
410	-
411	-	// With debug dumps on, the frame is still captured (for the dump only).
412	-	func TestNoTesseractDumpStillCaptures(t *testing.T) {
413	-	t.Setenv("PATH", t.TempDir())
414	-	dir := filepath.Join(t.TempDir(), "hints")
415	-	h := startWith(t, 5*time.Second, Options{DumpDir: dir})
416	-	go func() {
417	-	creq := <-h.captures
418	-	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
419	-	}()
420	-	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 40, H: 20}}}}
421	-	require.NoError(t, (<-h.results).Err)
422	-	waitFiles(t, dir, 3)
423	-	}
424	-
425		// A capture that completes after its scan timed out is released while the	389		// A capture that completes after its scan timed out is released while the
426		// worker is idle, without another scan, and its reply does not linger.	390		// worker is idle, without another scan, and its reply does not linger.
427		func TestHintLateCaptureReaped(t *testing.T) {	391		func TestHintLateCaptureReaped(t *testing.T) {

@@ -434,3 +398,21 @@ func TestHintLateCaptureReaped(t *testing.T) {
434		h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}	398		h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
435		require.Eventually(t, func() bool { return len(h.captured) == 0 }, time.Second, time.Millisecond)	399		require.Eventually(t, func() bool { return len(h.captured) == 0 }, time.Second, time.Millisecond)
436		}	400		}
			401	+
			402	+	func TestScanLogsRawAndPreparedCounts(t *testing.T) {
			403	+	dets := make([]detection, maxWorkBoxes+5)
			404	+	for i := range dets {
			405	+	dets[i] = detection{x1: 10, y1: 10, x2: 30, y2: 30, score: .9}
			406	+	}
			407	+	det := newMockdetector(t)
			408	+	det.EXPECT().Detect(mock.Anything, mock.Anything).Return(dets, nil).Once()
			409	+	var out bytes.Buffer // scan logs synchronously on this goroutine
			410	+	ctx := zerowrap.WithCtx(context.Background(), zerowrap.New(zerowrap.Config{Level: "info", Format: "json", Output: &out}))
			411	+	w := &worker{det: det, dumpDir: "enabled"}
			412	+	w.scan(ctx, ports.HintScanRequest{Width: 100, Height: 100, Scale: 1}, frame{m: &mapping{}}, 0, ports.HintScanResult{})
			413	+	var fields map[string]any
			414	+	require.NoError(t, json.Unmarshal(out.Bytes(), &fields))
			415	+	require.Equal(t, float64(maxWorkBoxes+5), fields["raw_boxes"])
			416	+	require.Equal(t, float64(maxWorkBoxes), fields["prepared_boxes"])
			417	+	require.Equal(t, "hints", fields["component"])
			418	+	}

D internal/adapters/hints/hybrid_test.go +0 -177

@@ -1,177 +0,0 @@
1	-	package hints
2	-
3	-	import (
4	-	"context"
5	-	"hash/maphash"
6	-	"image"
7	-	"image/draw"
8	-	"image/png"
9	-	"os"
10	-	"os/exec"
11	-	"runtime"
12	-	"testing"
13	-
14	-	"github.com/bnema/neferwl/internal/ports"
15	-	"github.com/stretchr/testify/require"
16	-	)
17	-
18	-	func TestWindowCrop(t *testing.T) {
19	-	for _, tc := range []struct {
20	-	name string
21	-	r ports.Rect
22	-	scale float64
23	-	want image.Rectangle
24	-	}{
25	-	{"unit", ports.Rect{X: 10, Y: 20, W: 100, H: 50}, 1, image.Rect(10, 20, 110, 70)},
26	-	{"fractional", ports.Rect{X: 11, Y: 21, W: 100, H: 50}, 1.2, image.Rect(13, 25, 134, 86)},
27	-	{"off output", ports.Rect{X: -10, Y: 20, W: 100, H: 100}, 1.2, image.Rect(0, 24, 108, 100)},
28	-	{"outside", ports.Rect{X: 1000, Y: 20, W: 100, H: 100}, 1, image.Rectangle{}},
29	-	} {
30	-	t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.want, windowCrop(tc.r, tc.scale, 200, 100)) })
31	-	}
32	-	}
33	-
34	-	func sourceResult(ar atspiResult) *pendingSource {
35	-	p := &pendingSource{done: make(chan struct{}), res: ar}
36	-	close(p.done)
37	-	return p
38	-	}
39	-
40	-	func TestHybridCacheAndMerge(t *testing.T) {
41	-	t.Setenv("PATH", t.TempDir()) // Cached OCR must not need tesseract.
42	-	buf := make([]byte, 2001004)
43	-	wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 100, H: 100}}, {ID: 2, Rect: ports.Rect{X: 100, W: 100, H: 100}}}
44	-	req := ports.HintScanRequest{Output: "o", Width: 200, Height: 100, Scale: 1, Windows: wins}
45	-	seed := maphash.MakeSeed()
46	-	r := windowCrop(wins[1].Rect, 1, 200, 100)
47	-	oldWorkers := ocrWorkers
48	-	ocrWorkers = 1 // one band covers the crop
49	-	t.Cleanup(func() { ocrWorkers = oldWorkers })
50	-	ext := image.Rect(0, 0, 100, 100)
51	-	entry := bandCache{hash: (&worker{cacheSeed: seed}).cropHash(buf, 800, r), ext: ext, words: []OCRWord{{Text: "example.org", Rect: image.Rect(10, 20, 90, 40)}}}
52	-	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 1, Y: 1, W: 10, H: 10}}
53	-	link := ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: 110, Y: 20, W: 80, H: 20}}
54	-	uncoveredButton := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 101, Y: 1, W: 10, H: 10}}
55	-	ar := atspiResult{targets: []ports.HintTarget{button, uncoveredButton}, covered: map[ports.WindowID]bool{1: true}}
56	-	w := worker{ocr: ocrState{done: true, binary: "tesseract", lang: "eng"}, cacheSeed: seed, cache: map[bandKey]bandCache{{output: "o", window: 2, band: 0, bands: 1}: entry}}
57	-	got := w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
58	-	require.Equal(t, []ports.HintTarget{button, uncoveredButton, link}, got.Targets)
59	-	req.LinksOnly = true
60	-	got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
61	-	require.Equal(t, []ports.HintTarget{link}, got.Targets)
62	-	// Changing one pixel invalidates only the crop, never leaving stale links.
63	-	buf[100*4]++
64	-	got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
65	-	require.Empty(t, got.Targets)
66	-	}
67	-
68	-	func TestMissingTesseractKeepsAccessibilityButtons(t *testing.T) {
69	-	t.Setenv("PATH", t.TempDir())
70	-	w := worker{}
71	-	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
72	-	req := ports.HintScanRequest{Width: 20, Height: 20, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}}}
73	-	got := w.scan(context.Background(), req, frame{pix: make([]byte, 1600), stride: 80}, 0, ports.HintScanResult{}, sourceResult(atspiResult{targets: []ports.HintTarget{button}}))
74	-	require.NoError(t, got.Err)
75	-	require.Equal(t, []ports.HintTarget{button}, got.Targets)
76	-	}
77	-
78	-	func TestNoWindowsProducesNoPixelTargets(t *testing.T) {
79	-	pix, width, height := loadBGRA(t)
80	-	w := worker{}
81	-	got := w.scan(context.Background(), ports.HintScanRequest{Width: width, Height: height}, frame{pix: pix, stride: width * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
82	-	require.Empty(t, got.Targets)
83	-	}
84	-
85	-	func TestStripMappingAndPolarity(t *testing.T) {
86	-	img := image.NewRGBA(image.Rect(0, 0, 200, 200))
87	-	strips := packLines(img, []image.Rectangle{image.Rect(20, 20, 160, 40), image.Rect(20, 80, 160, 112)}, 2, true)
88	-	require.Len(t, strips, 2)
89	-	for _, s := range strips {
90	-	for _, off := range s.offsets {
91	-	require.Equal(t, 1, off.factor)
92	-	require.Equal(t, uint8(255), s.img.GrayAt(off.packed.Min.X, off.packed.Min.Y).Y)
93	-	}
94	-	}
95	-	small := packLines(img, []image.Rectangle{image.Rect(20, 20, 100, 30)}, 1, true)
96	-	require.Equal(t, 2, small[0].offsets[0].factor)
97	-	}
98	-
99	-	func TestCropGrayAndHash(t *testing.T) {
100	-	buf := make([]byte, 444)
101	-	for y := 0; y < 4; y++ {
102	-	for x := 0; x < 4; x++ {
103	-	copy(buf[y16+x4:], []byte{0, 0, 255, 255}) // BGRA red
104	-	}
105	-	}
106	-	g := cropGray(buf, 16, image.Rect(1, 1, 3, 3), make([]byte, 4))
107	-	require.Equal(t, image.Rect(0, 0, 2, 2), g.Bounds())
108	-	require.Equal(t, uint8(76), g.GrayAt(1, 1).Y)
109	-	w := worker{cacheSeed: maphash.MakeSeed()}
110	-	a := w.cropHash(buf, 16, image.Rect(1, 1, 3, 3))
111	-	buf[0]++ // outside the crop
112	-	require.Equal(t, a, w.cropHash(buf, 16, image.Rect(1, 1, 3, 3)))
113	-	buf[16+4]++
114	-	require.NotEqual(t, a, w.cropHash(buf, 16, image.Rect(1, 1, 3, 3)))
115	-	}
116	-
117	-	func TestUpscaleBandGray(t *testing.T) {
118	-	g := image.NewGray(image.Rect(0, 0, 3, 2))
119	-	for i := range g.Pix {
120	-	g.Pix[i] = uint8(i + 1)
121	-	}
122	-	sub := g.SubImage(image.Rect(1, 1, 3, 2)).(*image.Gray)
123	-	up := upscaleBand(sub)
124	-	require.Equal(t, image.Rect(0, 0, 4, 2), up.Bounds())
125	-	require.Equal(t, []uint8{5, 5, 6, 6, 5, 5, 6, 6}, up.Pix)
126	-	}
127	-
128	-	// A blinking cursor changes one band: the other bands keep their cached
129	-	// words, and only the changed one is recognized again.
130	-	func TestBandCacheSurvivesCursorBlink(t *testing.T) {
131	-	binary, err := exec.LookPath("tesseract")
132	-	if err != nil {
133	-	t.Skip("tesseract not on PATH")
134	-	}
135	-	old := ocrWorkers
136	-	ocrWorkers = 4
137	-	t.Cleanup(func() { ocrWorkers = old })
138	-	f, err := os.Open("testdata/terminal.png")
139	-	require.NoError(t, err)
140	-	defer f.Close()
141	-	src, err := png.Decode(f)
142	-	require.NoError(t, err)
143	-	w, h := src.Bounds().Dx(), src.Bounds().Dy()
144	-	rgba := image.NewRGBA(image.Rect(0, 0, w, h))
145	-	draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
146	-	buf := make([]byte, wh4)
147	-	for i := 0; i < len(buf); i += 4 {
148	-	buf[i], buf[i+1], buf[i+2], buf[i+3] = rgba.Pix[i+2], rgba.Pix[i+1], rgba.Pix[i], 0xff
149	-	}
150	-	_ = binary
151	-	wk := &worker{}
152	-	req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
153	-	first := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
154	-	require.NotEmpty(t, first.Targets)
155	-	require.Len(t, wk.cache, min(runtime.GOMAXPROCS(0), 4))
156	-
157	-	// The same frame: every band hits.
158	-	again := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
159	-	require.Equal(t, first.Targets, again.Targets)
160	-
161	-	// One pixel at the bottom blinks: only the last band (and the context of
162	-	// none other) is invalidated.
163	-	before := map[bandKey]uint64{}
164	-	for k, v := range wk.cache {
165	-	before[k] = v.hash
166	-	}
167	-	buf[((h-1)w+3)4] ^= 0xff
168	-	blink := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
169	-	require.Equal(t, first.Targets, blink.Targets)
170	-	changed := 0
171	-	for k, v := range wk.cache {
172	-	if before[k] != v.hash {
173	-	changed++
174	-	}
175	-	}
176	-	require.Equal(t, 1, changed)
177	-	}

D internal/adapters/hints/language_runner_mock_test.go +0 -130

@@ -1,130 +0,0 @@
1	-	// Code generated by mockery; DO NOT EDIT.
2	-	// github.com/vektra/mockery
3	-	// template: testify
4	-
5	-	package hints
6	-
7	-	import (
8	-	"context"
9	-
10	-	mock "github.com/stretchr/testify/mock"
11	-	)
12	-
13	-	// newMocklanguageRunner creates a new instance of mocklanguageRunner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
14	-	// The first argument is typically a *testing.T value.
15	-	func newMocklanguageRunner(t interface {
16	-	mock.TestingT
17	-	Cleanup(func())
18	-	}) *mocklanguageRunner {
19	-	if helper, ok := t.(interface{ Helper() }); ok {
20	-	helper.Helper()
21	-	}
22	-
23	-	mock := &mocklanguageRunner{}
24	-	mock.Mock.Test(t)
25	-
26	-	t.Cleanup(func() {
27	-	if helper, ok := t.(interface{ Helper() }); ok {
28	-	helper.Helper()
29	-	}
30	-	mock.AssertExpectations(t)
31	-	})
32	-
33	-	return mock
34	-	}
35	-
36	-	// mocklanguageRunner is an autogenerated mock type for the languageRunner type
37	-	type mocklanguageRunner struct {
38	-	mock.Mock
39	-	}
40	-
41	-	type mocklanguageRunner_Expecter struct {
42	-	mock *mock.Mock
43	-	}
44	-
45	-	func (_m mocklanguageRunner) EXPECT() mocklanguageRunner_Expecter {
46	-	return &mocklanguageRunner_Expecter{mock: &_m.Mock}
47	-	}
48	-
49	-	// Output provides a mock function for the type mocklanguageRunner
50	-	func (_mock *mocklanguageRunner) Output(context1 context.Context, s string, strings ...string) ([]byte, error) {
51	-	var tmpRet mock.Arguments
52	-	if len(strings) > 0 {
53	-	tmpRet = _mock.Called(context1, s, strings)
54	-	} else {
55	-	tmpRet = _mock.Called(context1, s)
56	-	}
57	-	ret := tmpRet
58	-
59	-	if len(ret) == 0 {
60	-	panic("no return value specified for Output")
61	-	}
62	-
63	-	var r0 []byte
64	-	var r1 error
65	-	if returnFunc, ok := ret.Get(0).(func(context.Context, string, ...string) ([]byte, error)); ok {
66	-	return returnFunc(context1, s, strings...)
67	-	}
68	-	if returnFunc, ok := ret.Get(0).(func(context.Context, string, ...string) []byte); ok {
69	-	r0 = returnFunc(context1, s, strings...)
70	-	} else {
71	-	if ret.Get(0) != nil {
72	-	r0 = ret.Get(0).([]byte)
73	-	}
74	-	}
75	-	if returnFunc, ok := ret.Get(1).(func(context.Context, string, ...string) error); ok {
76	-	r1 = returnFunc(context1, s, strings...)
77	-	} else {
78	-	r1 = ret.Error(1)
79	-	}
80	-	return r0, r1
81	-	}
82	-
83	-	// mocklanguageRunner_Output_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Output'
84	-	type mocklanguageRunner_Output_Call struct {
85	-	*mock.Call
86	-	}
87	-
88	-	// Output is a helper method to define mock.On call
89	-	// - context1 context.Context
90	-	// - s string
91	-	// - strings ...string
92	-	func (_e mocklanguageRunner_Expecter) Output(context1 any, s any, strings ...any) mocklanguageRunner_Output_Call {
93	-	return &mocklanguageRunner_Output_Call{Call: _e.mock.On("Output",
94	-	append([]any{context1, s}, strings...)...)}
95	-	}
96	-
97	-	func (_c mocklanguageRunner_Output_Call) Run(run func(context1 context.Context, s string, strings ...string)) mocklanguageRunner_Output_Call {
98	-	_c.Call.Run(func(args mock.Arguments) {
99	-	var arg0 context.Context
100	-	if args[0] != nil {
101	-	arg0 = args[0].(context.Context)
102	-	}
103	-	var arg1 string
104	-	if args[1] != nil {
105	-	arg1 = args[1].(string)
106	-	}
107	-	var arg2 []string
108	-	var variadicArgs []string
109	-	if len(args) > 2 {
110	-	variadicArgs = args[2].([]string)
111	-	}
112	-	arg2 = variadicArgs
113	-	run(
114	-	arg0,
115	-	arg1,
116	-	arg2...,
117	-	)
118	-	})
119	-	return _c
120	-	}
121	-
122	-	func (_c mocklanguageRunner_Output_Call) Return(bytes []byte, err error) mocklanguageRunner_Output_Call {
123	-	_c.Call.Return(bytes, err)
124	-	return _c
125	-	}
126	-
127	-	func (_c mocklanguageRunner_Output_Call) RunAndReturn(run func(context1 context.Context, s string, strings ...string) ([]byte, error)) mocklanguageRunner_Output_Call {
128	-	_c.Call.Return(run)
129	-	return _c
130	-	}

A internal/adapters/hints/merge.go +82 -0

@@ -0,0 +1,82 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"cmp"
	5	+	"slices"
	6	+
	7	+	"github.com/bnema/neferwl/internal/ports"
	8	+	)
	9	+
	10	+	const (
	11	+	// dupIoU is the IoU above which two boxes are the same element: the
	12	+	// higher score is kept.
	13	+	dupIoU = 0.5
	14	+	// nestRatio is the share of the smaller box that must lie inside the
	15	+	// other for the two to be nested.
	16	+	nestRatio = 0.85
	17	+	)
	18	+
	19	+	func area(r ports.Rect) int { return r.W * r.H }
	20	+
	21	+	// interArea is the area of the intersection of a and b.
	22	+	func interArea(a, b ports.Rect) int {
	23	+	w := min(a.X+a.W, b.X+b.W) - max(a.X, b.X)
	24	+	h := min(a.Y+a.H, b.Y+b.H) - max(a.Y, b.Y)
	25	+	if w <= 0 \|\| h <= 0 {
	26	+	return 0
	27	+	}
	28	+	return w * h
	29	+	}
	30	+
	31	+	// nested reports whether the smaller of a and b lies for at least nestRatio
	32	+	// inside the other.
	33	+	func nested(a, b ports.Rect) bool {
	34	+	return float64(interArea(a, b)) >= nestRatio*float64(min(area(a), area(b)))
	35	+	}
	36	+
	37	+	// mergeTargets removes the duplicate and nested boxes the detector returns
	38	+	// for one element. It works on logical targets (after rounding), in two
	39	+	// passes so that the result does not depend on the processing order:
	40	+	//
	41	+	// 1. duplicates: by descending score, a box whose IoU with an already kept
	42	+	// box is over dupIoU is dropped;
	43	+	// 2. nesting: a box that contains another surviving box (that box lies
	44	+	// nestRatio inside it) is dropped, whatever the scores: the smallest of
	45	+	// a nested chain is kept (a row loses against its icon and its text).
	46	+	//
	47	+	// The result is ordered by descending score, then position.
	48	+	func mergeTargets(in []ports.HintTarget) []ports.HintTarget {
	49	+	s := slices.Clone(in)
	50	+	slices.SortStableFunc(s, func(a, b ports.HintTarget) int {
	51	+	return cmp.Or(cmp.Compare(b.Score, a.Score), cmp.Compare(a.Rect.Y, b.Rect.Y), cmp.Compare(a.Rect.X, b.Rect.X),
	52	+	cmp.Compare(area(a.Rect), area(b.Rect)))
	53	+	})
	54	+	kept := s[:0]
	55	+	for _, t := range s {
	56	+	dup := false
	57	+	for _, k := range kept {
	58	+	inter := interArea(t.Rect, k.Rect)
	59	+	if inter > 0 && float64(inter) > dupIoU*float64(area(t.Rect)+area(k.Rect)-inter) {
	60	+	dup = true
	61	+	break
	62	+	}
	63	+	}
	64	+	if !dup {
	65	+	kept = append(kept, t)
	66	+	}
	67	+	}
	68	+	out := make([]ports.HintTarget, 0, len(kept))
	69	+	for i, t := range kept {
	70	+	contains := false
	71	+	for j, k := range kept {
	72	+	if i != j && area(k.Rect) < area(t.Rect) && nested(t.Rect, k.Rect) {
	73	+	contains = true
	74	+	break
	75	+	}
	76	+	}
	77	+	if !contains {
	78	+	out = append(out, t)
	79	+	}
	80	+	}
	81	+	return out
	82	+	}

A internal/adapters/hints/merge_test.go +38 -0

@@ -0,0 +1,38 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"testing"
	5	+
	6	+	"github.com/stretchr/testify/require"
	7	+
	8	+	"github.com/bnema/neferwl/internal/ports"
	9	+	)
	10	+
	11	+	func tg(x, y, w, h int, score float32) ports.HintTarget {
	12	+	return ports.HintTarget{Rect: ports.Rect{X: x, Y: y, W: w, H: h}, Score: score}
	13	+	}
	14	+
	15	+	func TestMergeTargets(t *testing.T) {
	16	+	tests := []struct {
	17	+	name string
	18	+	in []ports.HintTarget
	19	+	want []ports.HintTarget
	20	+	}{
	21	+	{"empty", nil, []ports.HintTarget{}},
	22	+	{"duplicate keeps the best score", []ports.HintTarget{tg(10, 10, 100, 40, 0.3), tg(12, 11, 100, 40, 0.9)}, []ports.HintTarget{tg(12, 11, 100, 40, 0.9)}},
	23	+	{"duplicate, input order irrelevant", []ports.HintTarget{tg(12, 11, 100, 40, 0.9), tg(10, 10, 100, 40, 0.3)}, []ports.HintTarget{tg(12, 11, 100, 40, 0.9)}},
	24	+	{"row with icon and text keeps both small ones", []ports.HintTarget{
	25	+	tg(0, 0, 400, 30, 0.95), tg(4, 4, 22, 22, 0.5), tg(40, 6, 200, 18, 0.4),
	26	+	}, []ports.HintTarget{tg(4, 4, 22, 22, 0.5), tg(40, 6, 200, 18, 0.4)}},
	27	+	{"nested keeps the smaller even with a lower score", []ports.HintTarget{tg(0, 0, 200, 100, 0.9), tg(20, 20, 50, 30, 0.1)}, []ports.HintTarget{tg(20, 20, 50, 30, 0.1)}},
	28	+	{"partly overlapping distinct buttons are kept", []ports.HintTarget{tg(0, 0, 100, 30, 0.8), tg(80, 0, 100, 30, 0.7)}, []ports.HintTarget{tg(0, 0, 100, 30, 0.8), tg(80, 0, 100, 30, 0.7)}},
	29	+	{"mostly (not 85%) inside is kept", []ports.HintTarget{tg(0, 0, 100, 100, 0.8), tg(60, 0, 100, 20, 0.7)}, []ports.HintTarget{tg(0, 0, 100, 100, 0.8), tg(60, 0, 100, 20, 0.7)}},
	30	+	{"chain of three nested keeps the innermost", []ports.HintTarget{tg(0, 0, 300, 200, 0.99), tg(10, 10, 200, 100, 0.5), tg(20, 20, 50, 30, 0.2)}, []ports.HintTarget{tg(20, 20, 50, 30, 0.2)}},
	31	+	{"disjoint", []ports.HintTarget{tg(0, 0, 10, 10, 0.2), tg(50, 50, 10, 10, 0.6)}, []ports.HintTarget{tg(50, 50, 10, 10, 0.6), tg(0, 0, 10, 10, 0.2)}},
	32	+	}
	33	+	for _, tc := range tests {
	34	+	t.Run(tc.name, func(t *testing.T) {
	35	+	require.Equal(t, tc.want, mergeTargets(tc.in))
	36	+	})
	37	+	}
	38	+	}

D internal/adapters/hints/ocr.go +0 -460

@@ -1,460 +0,0 @@
1	-	package hints
2	-
3	-	import (
4	-	"bufio"
5	-	"bytes"
6	-	"context"
7	-	"fmt"
8	-	"image"
9	-	"image/png"
10	-	"io"
11	-	"math"
12	-	"net"
13	-	"os"
14	-	"os/exec"
15	-	"path/filepath"
16	-	"regexp"
17	-	"runtime"
18	-	"sort"
19	-	"strconv"
20	-	"strings"
21	-	"sync"
22	-	"time"
23	-
24	-	"github.com/bnema/neferwl/internal/adapters/logging"
25	-	"github.com/bnema/neferwl/internal/ports"
26	-	)
27	-
28	-	// OCRWord is a recognized word in physical output coordinates.
29	-	type OCRWord struct {
30	-	Text string `json:"text"`
31	-	Rect image.Rectangle `json:"rect"`
32	-	Confidence float64 `json:"confidence"`
33	-	// Underlined is set when a thin line runs under the word. UnderL and
34	-	// UnderR are the x extent of that line (crop-local), which can reach the
35	-	// neighbouring words of one underlined link.
36	-	Underlined bool `json:"underlined,omitempty"`
37	-	UnderL int `json:"under_l,omitempty"`
38	-	UnderR int `json:"under_r,omitempty"`
39	-	}
40	-
41	-	var languageWarning sync.Once
42	-	var failureWarning sync.Once
43	-
44	-	// languageRunner bounds the external language-discovery process independently
45	-	// of recognition, which has not started yet when discovery runs.
46	-	type languageRunner interface {
47	-	Output(context.Context, string, ...string) ([]byte, error)
48	-	}
49	-
50	-	type execLanguageRunner struct{}
51	-
52	-	func (execLanguageRunner) Output(ctx context.Context, binary string, args ...string) ([]byte, error) {
53	-	return exec.CommandContext(ctx, binary, args...).Output()
54	-	}
55	-
56	-	const languageTimeout = 5 * time.Second
57	-
58	-	func ocrLanguage(ctx context.Context, binary string) (string, error) {
59	-	return discoverLanguage(ctx, binary, execLanguageRunner{}, languageTimeout)
60	-	}
61	-
62	-	func discoverLanguage(ctx context.Context, binary string, runner languageRunner, timeout time.Duration) (string, error) {
63	-	dir := os.Getenv("TESSDATA_PREFIX")
64	-	if dir == "" {
65	-	dir = "/usr/share/tessdata"
66	-	}
67	-	if _, err := os.Stat(filepath.Join(dir, "eng.traineddata")); err == nil {
68	-	return "eng", nil
69	-	}
70	-	ctx, cancel := context.WithTimeout(ctx, timeout)
71	-	defer cancel()
72	-	out, err := runner.Output(ctx, binary, "--list-langs")
73	-	if err != nil {
74	-	return "", err
75	-	}
76	-	for _, line := range strings.Split(string(out), "\n")[1:] {
77	-	lang := strings.TrimSpace(line)
78	-	if lang != "" && lang != "osd" {
79	-	languageWarning.Do(func() {
80	-	log := logging.For(ctx, "hints")
81	-	log.Warn().Str("language", lang).Msg("English OCR data missing; install tesseract-data-eng")
82	-	})
83	-	return lang, nil
84	-	}
85	-	}
86	-	return "", fmt.Errorf("no OCR language available")
87	-	}
88	-
89	-	func parseTSV(data []byte, offset image.Point) ([]OCRWord, error) {
90	-	var words []OCRWord
91	-	scanner := bufio.NewScanner(bytes.NewReader(data))
92	-	scanner.Buffer(make([]byte, 4096), 1024*1024)
93	-	for scanner.Scan() {
94	-	fields := strings.SplitN(scanner.Text(), "\t", 12)
95	-	if len(fields) != 12 \|\| fields[0] != "5" {
96	-	continue
97	-	}
98	-	var values [4]int
99	-	for i := range values {
100	-	v, err := strconv.Atoi(fields[6+i])
101	-	if err != nil {
102	-	return nil, fmt.Errorf("OCR TSV coordinate: %w", err)
103	-	}
104	-	values[i] = v
105	-	}
106	-	conf, err := strconv.ParseFloat(fields[10], 64)
107	-	if err != nil {
108	-	return nil, fmt.Errorf("OCR TSV confidence: %w", err)
109	-	}
110	-	if values[2] <= 0 \|\| values[3] <= 0 \|\| strings.TrimSpace(fields[11]) == "" {
111	-	continue
112	-	}
113	-	r := image.Rect(values[0], values[1], values[0]+values[2], values[1]+values[3]).Add(offset)
114	-	words = append(words, OCRWord{Text: fields[11], Rect: r, Confidence: conf})
115	-	}
116	-	return words, scanner.Err()
117	-	}
118	-
119	-	// ocrWorkers is the maximum number of bands recognized in parallel. 4 keeps
120	-	// the tesseract processes at about 60% of the memory of 8 for the same time.
121	-	var ocrWorkers = 4
122	-
123	-	// pngLevel is the compression of non-grayscale images piped to tesseract.
124	-	var pngLevel = png.BestSpeed
125	-
126	-	// rawPGM feeds grayscale images to tesseract as binary PGM instead of PNG.
127	-	var rawPGM = true
128	-
129	-	// ocrState is the tesseract installation, resolved once.
130	-	type ocrState struct {
131	-	done bool
132	-	binary string
133	-	lang string
134	-	err error
135	-	}
136	-
137	-	func (s *ocrState) resolve(ctx context.Context) error {
138	-	if !s.done {
139	-	s.done = true
140	-	if s.binary, s.err = exec.LookPath("tesseract"); s.err == nil {
141	-	s.lang, s.err = ocrLanguage(ctx, s.binary)
142	-	}
143	-	}
144	-	return s.err
145	-	}
146	-
147	-	// ocrReady resolves tesseract on the first call. When it is missing, OCR is
148	-	// disabled for the life of the process and the warning is logged once.
149	-	func (w *worker) ocrReady(ctx context.Context) bool {
150	-	if err := w.ocr.resolve(ctx); err != nil {
151	-	failureWarning.Do(func() {
152	-	log := logging.For(ctx, "hints")
153	-	log.Warn().Err(err).Msg("OCR unavailable; install tesseract and tesseract-data-eng")
154	-	})
155	-	return false
156	-	}
157	-	return true
158	-	}
159	-
160	-	// band is a horizontal slice of a window crop recognized by one tesseract
161	-	// process. It reads rows [top,bottom) and owns the words centred in
162	-	// [ownTop,ownBottom); the margin only supplies context.
163	-	type band struct {
164	-	idx, n int
165	-	top, bottom int
166	-	ownTop, ownBottom int
167	-	}
168	-
169	-	func bandsOf(height int) []band {
170	-	n := min(runtime.GOMAXPROCS(0), ocrWorkers, height)
171	-	bands := make([]band, 0, max(n, 0))
172	-	for i := range n {
173	-	ownTop, ownBottom := iheight/n, (i+1)height/n
174	-	bands = append(bands, band{idx: i, n: n, top: max(0, ownTop-40), bottom: min(height, ownBottom+40), ownTop: ownTop, ownBottom: ownBottom})
175	-	}
176	-	return bands
177	-	}
178	-
179	-	// recognize reads the bands of jobs that missed the cache, in parallel.
180	-	func (w worker) recognize(ctx context.Context, jobs []bandJob) {
181	-	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
182	-	defer cancel()
183	-	// One scan-wide pool: at most ocrWorkers tesseract processes run at once,
184	-	// whatever the number of windows.
185	-	sem := make(chan struct{}, max(ocrWorkers, 1))
186	-	var wg sync.WaitGroup
187	-	for _, j := range jobs {
188	-	if j.hit {
189	-	continue
190	-	}
191	-	select {
192	-	case sem <- struct{}{}:
193	-	case <-ctx.Done():
194	-	j.err = ctx.Err()
195	-	continue
196	-	}
197	-	wg.Go(func() {
198	-	defer func() { <-sem }()
199	-	j.words, j.err = readBand(ctx, &w.ocr, j)
200	-	})
201	-	}
202	-	wg.Wait()
203	-	}
204	-
205	-	// readBand recognizes one band; the words are crop-local and owned by it.
206	-	func readBand(ctx context.Context, st ocrState, j bandJob) ([]OCRWord, error) {
207	-	words, err := recognizeBand(ctx, st.binary, st.lang, j.gray)
208	-	if err != nil {
209	-	return nil, err
210	-	}
211	-	heights := make([]int, 0, len(words))
212	-	for _, word := range words {
213	-	if len(word.Text) >= 3 {
214	-	heights = append(heights, word.Rect.Dy())
215	-	}
216	-	}
217	-	sort.Ints(heights)
218	-	if len(heights) > 0 && heights[len(heights)/2] < 12 {
219	-	words, err = recognizeBand(ctx, st.binary, st.lang, upscaleBand(j.gray))
220	-	if err != nil {
221	-	return nil, err
222	-	}
223	-	for i := range words {
224	-	r := words[i].Rect
225	-	words[i].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
226	-	}
227	-	}
228	-	markUnderlined(j.gray, words)
229	-	owned := words[:0]
230	-	for _, word := range words {
231	-	// Overlap supplies context, but each word belongs to exactly one band.
232	-	word.Rect = word.Rect.Add(image.Pt(0, j.ext.Min.Y))
233	-	if center := (word.Rect.Min.Y + word.Rect.Max.Y) / 2; center >= j.own[0] && center < j.own[1] {
234	-	owned = append(owned, word)
235	-	}
236	-	}
237	-	return owned, nil
238	-	}
239	-
240	-	// readWords recognizes the words of img, a window crop, in parallel bands.
241	-	func readWords(ctx context.Context, img *image.Gray) ([]OCRWord, int, error) {
242	-	w := &worker{}
243	-	if err := w.ocr.resolve(ctx); err != nil {
244	-	return nil, 0, err
245	-	}
246	-	var jobs []*bandJob
247	-	for _, b := range bandsOf(img.Bounds().Dy()) {
248	-	ext := image.Rect(0, b.top, img.Bounds().Dx(), b.bottom)
249	-	jobs = append(jobs, &bandJob{ext: ext, own: [2]int{b.ownTop, b.ownBottom}, gray: img.SubImage(ext).(*image.Gray)})
250	-	}
251	-	w.recognize(ctx, jobs)
252	-	var words []OCRWord
253	-	for _, j := range jobs {
254	-	if j.err != nil {
255	-	return nil, len(jobs), j.err
256	-	}
257	-	for _, word := range j.words {
258	-	if !duplicateWord(words, word) {
259	-	words = append(words, word)
260	-	}
261	-	}
262	-	}
263	-	return words, len(jobs), nil
264	-	}
265	-
266	-	// The link patterns compile on the first OCR link check, not at process
267	-	// start: with hint mode off nothing here costs memory or CPU.
268	-	var domainPattern = sync.OnceValue(func() *regexp.Regexp {
269	-	return regexp.MustCompile(`(?i)^(?:[a-z0-9](?:[a-z0-9-][a-z0-9])?\.)+([a-z]{2,24})(?::[0-9]+)?(?:[/?#]\S)?$`)
270	-	})
271	-	var schemePattern = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`(?i)^(?:https?\|htps\|file\|ftp):/{1,2}\S+$`) })
272	-	var emailPattern = sync.OnceValue(func() *regexp.Regexp {
273	-	return regexp.MustCompile(`(?i)^[a-z0-9.!#$%&'*+/=?^_` + "`" + `{\|}~-]+@(?:[a-z0-9-]+\.)+[a-z]{2,24}$`)
274	-	})
275	-	var fileExtensions = sync.OnceValue(func() []string {
276	-	return strings.Fields("go md txt toml json yaml yml rs py js ts sh c h png jpg log conf lock mod sum html css")
277	-	})
278	-
279	-	func isTextLink(text string) bool {
280	-	text = strings.Trim(text, "()[]<>.,;:'\"")
281	-	if schemePattern().MatchString(text) \|\| (strings.HasPrefix(strings.ToLower(text), "www.") && len(text) > 4) \|\| emailPattern().MatchString(text) {
282	-	return true
283	-	}
284	-	host := text
285	-	if h, _, err := net.SplitHostPort(text); err == nil {
286	-	host = h
287	-	}
288	-	if ip := net.ParseIP(host); ip != nil && ip.To4() != nil {
289	-	return true
290	-	}
291	-	m := domainPattern().FindStringSubmatch(text)
292	-	if m == nil {
293	-	return false
294	-	}
295	-	for _, ext := range fileExtensions() {
296	-	if strings.EqualFold(m[1], ext) {
297	-	return false
298	-	}
299	-	}
300	-	return true
301	-	}
302	-
303	-	func textLinks(words []OCRWord, scale float64) []ports.HintTarget {
304	-	if scale <= 0 {
305	-	scale = 1
306	-	}
307	-	words = append([]OCRWord(nil), words...)
308	-	sort.Slice(words, func(i, j int) bool {
309	-	if words[i].Rect.Min.Y != words[j].Rect.Min.Y {
310	-	return words[i].Rect.Min.Y < words[j].Rect.Min.Y
311	-	}
312	-	return words[i].Rect.Min.X < words[j].Rect.Min.X
313	-	})
314	-	var out []ports.HintTarget
315	-	var urls []image.Rectangle // boxes of the URL targets, in pixels
316	-	used := make([]bool, len(words))
317	-	for i, word := range words {
318	-	if used[i] {
319	-	continue
320	-	}
321	-	text, rect := word.Text, word.Rect
322	-	// Grow only touching words on the same baseline; ordinary prose is not joined.
323	-	for {
324	-	next := -1
325	-	for j, candidate := range words {
326	-	if j == i \|\| used[j] \|\| candidate.Rect.Min.X < rect.Max.X-2 {
327	-	continue
328	-	}
329	-	gap := candidate.Rect.Min.X - rect.Max.X
330	-	if gap > max(3, min(rect.Dy(), candidate.Rect.Dy())/5) {
331	-	continue
332	-	}
333	-	if abs(candidate.Rect.Max.Y-rect.Max.Y) > max(3, rect.Dy()/3) {
334	-	continue
335	-	}
336	-	if next < 0 \|\| candidate.Rect.Min.X < words[next].Rect.Min.X {
337	-	next = j
338	-	}
339	-	}
340	-	if next < 0 {
341	-	break
342	-	}
343	-	text += words[next].Text
344	-	rect = rect.Union(words[next].Rect)
345	-	used[next] = true
346	-	}
347	-	if !isTextLink(text) {
348	-	continue
349	-	}
350	-	x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
351	-	right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
352	-	out = append(out, ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
353	-	urls = append(urls, rect)
354	-	}
355	-	return append(out, underlinedLinks(words, urls, scale)...)
356	-	}
357	-
358	-	func abs(x int) int {
359	-	if x < 0 {
360	-	return -x
361	-	}
362	-	return x
363	-	}
364	-	func overlap(a, b image.Rectangle) float64 {
365	-	intersection := a.Intersect(b)
366	-	area := min(a.Dx()a.Dy(), b.Dx()b.Dy())
367	-	if intersection.Empty() \|\| area <= 0 {
368	-	return 0
369	-	}
370	-	return float64(intersection.Dx()*intersection.Dy()) / float64(area)
371	-	}
372	-	func targetRect(t ports.HintTarget) image.Rectangle {
373	-	return image.Rect(t.Rect.X, t.Rect.Y, t.Rect.X+t.Rect.W, t.Rect.Y+t.Rect.H)
374	-	}
375	-	func recognizeBand(ctx context.Context, binary, lang string, img image.Image) ([]OCRWord, error) {
376	-	return recognizeImage(ctx, binary, lang, img, "11", false)
377	-	}
378	-
379	-	// pgmReader streams a grayscale image as binary PGM (P5), row by row.
380	-	type pgmReader struct {
381	-	head []byte
382	-	img *image.Gray
383	-	y int
384	-	row []byte
385	-	}
386	-
387	-	func newPGMReader(img image.Gray) pgmReader {
388	-	b := img.Bounds()
389	-	return &pgmReader{head: fmt.Appendf(nil, "P5\n%d %d\n255\n", b.Dx(), b.Dy()), img: img}
390	-	}
391	-
392	-	func (r *pgmReader) Read(p []byte) (int, error) {
393	-	n := 0
394	-	for n < len(p) {
395	-	if len(r.head) > 0 {
396	-	c := copy(p[n:], r.head)
397	-	r.head, n = r.head[c:], n+c
398	-	continue
399	-	}
400	-	if len(r.row) == 0 {
401	-	b := r.img.Bounds()
402	-	if r.y >= b.Dy() {
403	-	break
404	-	}
405	-	r.row = r.img.Pix[(r.y+b.Min.Y-r.img.Rect.Min.Y)*r.img.Stride+(b.Min.X-r.img.Rect.Min.X):][:b.Dx()]
406	-	r.y++
407	-	}
408	-	c := copy(p[n:], r.row)
409	-	r.row, n = r.row[c:], n+c
410	-	}
411	-	if n == 0 {
412	-	return 0, io.EOF
413	-	}
414	-	return n, nil
415	-	}
416	-
417	-	func recognizeImage(ctx context.Context, binary, lang string, img image.Image, psm string, normalized bool) ([]OCRWord, error) {
418	-	args := []string{"stdin", "stdout", "--psm", psm, "-l", lang}
419	-	if normalized {
420	-	args = append(args, "-c", "tessedit_do_invert=0")
421	-	}
422	-	args = append(args, "tsv")
423	-	cmd := exec.CommandContext(ctx, binary, args...)
424	-	cmd.Env = append(os.Environ(), "OMP_THREAD_LIMIT=1")
425	-	if g, ok := img.(*image.Gray); ok && rawPGM {
426	-	// Raw grayscale: no deflate, no encoder buffers.
427	-	cmd.Stdin = newPGMReader(g)
428	-	} else {
429	-	// The PNG streams into tesseract; no encoded copy is held.
430	-	pr, pw := io.Pipe()
431	-	defer func() { _ = pr.Close() }()
432	-	go func() {
433	-	enc := png.Encoder{CompressionLevel: pngLevel}
434	-	_ = pw.CloseWithError(enc.Encode(pw, img))
435	-	}()
436	-	cmd.Stdin = pr
437	-	}
438	-	var stderr bytes.Buffer
439	-	cmd.Stderr = &stderr
440	-	output, err := cmd.Output()
441	-	if err != nil {
442	-	return nil, fmt.Errorf("tesseract: %w: %s", err, stderr.String())
443	-	}
444	-	return parseTSV(output, image.Point{})
445	-	}
446	-
447	-	// upscaleBand doubles a grayscale band with nearest-neighbour sampling.
448	-	func upscaleBand(src image.Gray) image.Gray {
449	-	b := src.Bounds()
450	-	dst := image.NewGray(image.Rect(0, 0, b.Dx()2, b.Dy()2))
451	-	for y := 0; y < b.Dy(); y++ {
452	-	row := src.Pix[(y+b.Min.Y-src.Rect.Min.Y)*src.Stride+(b.Min.X-src.Rect.Min.X):][:b.Dx()]
453	-	top := dst.Pix[2ydst.Stride:][:dst.Stride]
454	-	for x, v := range row {
455	-	top[2x], top[2x+1] = v, v
456	-	}
457	-	copy(dst.Pix[(2y+1)dst.Stride:], top)
458	-	}
459	-	return dst
460	-	}

D internal/adapters/hints/ocr_test.go +0 -272

@@ -1,272 +0,0 @@
1	-	package hints
2	-
3	-	import (
4	-	"context"
5	-	"errors"
6	-	"image"
7	-	"image/draw"
8	-	"image/png"
9	-	"io"
10	-	"os"
11	-	"os/exec"
12	-	"path/filepath"
13	-	"reflect"
14	-	"strings"
15	-	"testing"
16	-	"time"
17	-
18	-	"github.com/stretchr/testify/mock"
19	-
20	-	"github.com/bnema/neferwl/internal/ports"
21	-	)
22	-
23	-	func TestTextLink(t *testing.T) {
24	-	yes := []string{"ermite.bnema.dev", "google.fr", "example.org/path?q=1", "https.google.fr", "https://example.org", "file:///tmp/main.go", "ftp://host/file", "www.main.go", "user@example.org", "127.0.0.1", "192.168.1.1:8080", "(https://example.org),", "htps://google.fr", "https:/example.org"}
25	-	no := []string{"ermite-runner", "main.go", "README.md", "v1.2.3", "1.5", "e.g.", "report.txt", "source.rs", "alpha", "999.999.999.999"}
26	-	for _, s := range yes {
27	-	t.Run(s, func(t *testing.T) {
28	-	if !isTextLink(s) {
29	-	t.Errorf("not accepted: %s", s)
30	-	}
31	-	})
32	-	}
33	-	for _, s := range no {
34	-	t.Run(s, func(t *testing.T) {
35	-	if isTextLink(s) {
36	-	t.Errorf("incorrectly accepted: %s", s)
37	-	}
38	-	})
39	-	}
40	-	}
41	-	func TestParseTSV(t *testing.T) {
42	-	data := "level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext\n5\t1\t1\t1\t1\t1\t10\t20\t30\t12\t91.5\texample.org\n"
43	-	words, err := parseTSV([]byte(data), image.Pt(0, 40))
44	-	if err != nil \|\| len(words) != 1 {
45	-	t.Fatalf("%v %v", words, err)
46	-	}
47	-	if words[0].Rect != image.Rect(10, 60, 40, 72) \|\| words[0].Confidence != 91.5 {
48	-	t.Fatal(words)
49	-	}
50	-	_, err = parseTSV([]byte(strings.ReplaceAll(data, "\t10\t", "\tbad\t")), image.Point{})
51	-	if err == nil {
52	-	t.Fatal("expected malformed coordinate error")
53	-	}
54	-	}
55	-	func TestJoinedOCRLink(t *testing.T) {
56	-	words := []OCRWord{{Text: "https:/", Rect: image.Rect(20, 20, 90, 40)}, {Text: "/example.org", Rect: image.Rect(91, 20, 200, 40)}}
57	-	got := textLinks(words, 2)
58	-	if len(got) != 1 \|\| got[0].Rect != (ports.Rect{X: 10, Y: 10, W: 90, H: 10}) {
59	-	t.Fatal(got)
60	-	}
61	-	}
62	-	func TestRealOCR(t *testing.T) {
63	-	if _, err := exec.LookPath("tesseract"); err != nil {
64	-	t.Skip("tesseract not on PATH")
65	-	}
66	-	for _, name := range []string{"browser", "terminal"} {
67	-	t.Run(name, func(t *testing.T) {
68	-	f, err := os.Open("testdata/" + name + ".png")
69	-	if err != nil {
70	-	t.Fatal(err)
71	-	}
72	-	defer f.Close()
73	-	src, err := png.Decode(f)
74	-	if err != nil {
75	-	t.Fatal(err)
76	-	}
77	-	img := image.NewRGBA(src.Bounds())
78	-	draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
79	-	start := time.Now()
80	-	words, bands, err := readWords(context.Background(), toGray(img))
81	-	if err != nil {
82	-	t.Fatal(err)
83	-	}
84	-	links := textLinks(words, 1)
85	-	t.Logf("%s %dx%d ocr_ms=%.1f words=%d links=%d bands=%d", name, img.Bounds().Dx(), img.Bounds().Dy(), ms(time.Since(start)), len(words), len(links), bands)
86	-	for _, w := range words {
87	-	t.Logf("%s %v", w.Text, w.Rect)
88	-	}
89	-	if name == "terminal" {
90	-	for _, r := range []image.Rectangle{image.Rect(160, 65, 380, 85), image.Rect(140, 90, 490, 110), image.Rect(95, 115, 630, 138), image.Rect(120, 140, 330, 163), image.Rect(385, 140, 740, 163)} {
91	-	found := false
92	-	for _, l := range links {
93	-	if overlap(targetRect(l), r) > .5 {
94	-	found = true
95	-	}
96	-	}
97	-	if !found {
98	-	t.Errorf("missing terminal URL near %v", r)
99	-	}
100	-	}
101	-	if len(links) != 5 {
102	-	t.Errorf("want 5 terminal links, got %d: %v", len(links), links)
103	-	}
104	-	} else if len(links) == 0 {
105	-	t.Error("no browser links")
106	-	}
107	-	})
108	-	}
109	-	}
110	-
111	-	func TestSmallGlyphOCR(t *testing.T) {
112	-	binary, err := exec.LookPath("tesseract")
113	-	if err != nil {
114	-	t.Skip("tesseract not on PATH")
115	-	}
116	-	f, err := os.Open("testdata/terminal.png")
117	-	if err != nil {
118	-	t.Fatal(err)
119	-	}
120	-	defer f.Close()
121	-	src, err := png.Decode(f)
122	-	if err != nil {
123	-	t.Fatal(err)
124	-	}
125	-	small := image.NewGray(image.Rect(0, 0, 640, 200))
126	-	for y := 0; y < 200; y++ {
127	-	for x := 0; x < 640; x++ {
128	-	small.Set(x, y, src.At(x2, y2))
129	-	}
130	-	}
131	-	lang, err := ocrLanguage(context.Background(), binary)
132	-	if err != nil {
133	-	t.Fatal(err)
134	-	}
135	-	for _, factor := range []int{1, 2} {
136	-	var img image.Image = small
137	-	if factor == 2 {
138	-	img = upscaleBand(small)
139	-	}
140	-	start := time.Now()
141	-	words, err := recognizeBand(context.Background(), binary, lang, img)
142	-	if err != nil {
143	-	t.Fatal(err)
144	-	}
145	-	t.Logf("glyph experiment %dx: %.1f ms, words=%d links=%d", factor, ms(time.Since(start)), len(words), len(textLinks(words, 1)))
146	-	}
147	-	}
148	-
149	-	func TestPGMReader(t *testing.T) {
150	-	g := image.NewGray(image.Rect(0, 0, 3, 3))
151	-	for i := range g.Pix {
152	-	g.Pix[i] = uint8(i + 1)
153	-	}
154	-	sub := g.SubImage(image.Rect(1, 1, 3, 3)).(*image.Gray)
155	-	got, err := io.ReadAll(newPGMReader(sub))
156	-	if err != nil {
157	-	t.Fatal(err)
158	-	}
159	-	if want := "P5\n2 2\n255\n" + string([]byte{5, 6, 8, 9}); string(got) != want {
160	-	t.Fatalf("%q", got)
161	-	}
162	-	}
163	-
164	-	// Tesseract must read the raw PGM exactly as it reads the PNG.
165	-	func TestPGMMatchesPNG(t *testing.T) {
166	-	binary, err := exec.LookPath("tesseract")
167	-	if err != nil {
168	-	t.Skip("tesseract not on PATH")
169	-	}
170	-	f, err := os.Open("testdata/terminal.png")
171	-	if err != nil {
172	-	t.Fatal(err)
173	-	}
174	-	defer f.Close()
175	-	src, err := png.Decode(f)
176	-	if err != nil {
177	-	t.Fatal(err)
178	-	}
179	-	rgba := image.NewRGBA(src.Bounds())
180	-	draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
181	-	gray := toGray(rgba)
182	-	lang, err := ocrLanguage(context.Background(), binary)
183	-	if err != nil {
184	-	t.Fatal(err)
185	-	}
186	-	defer func(v bool) { rawPGM = v }(rawPGM)
187	-	var res [2][]OCRWord
188	-	for i, raw := range []bool{false, true} {
189	-	rawPGM = raw
190	-	if res[i], err = recognizeBand(context.Background(), binary, lang, gray); err != nil {
191	-	t.Fatal(err)
192	-	}
193	-	}
194	-	if len(res[0]) == 0 \|\| !reflect.DeepEqual(res[0], res[1]) {
195	-	t.Fatalf("PNG %d words, PGM %d words", len(res[0]), len(res[1]))
196	-	}
197	-	}
198	-
199	-	// TestRecognizeCapsProcesses runs the bands of several windows through a
200	-	// stand-in tesseract that logs when each process starts and ends.
201	-	func TestRecognizeCapsProcesses(t *testing.T) {
202	-	dir := t.TempDir()
203	-	logPath := filepath.Join(dir, "log")
204	-	script := filepath.Join(dir, "tesseract")
205	-	body := "#!/bin/sh\ncat >/dev/null\necho s >> " + logPath + "\nsleep 0.1\necho e >> " + logPath + "\nprintf 'level\\tpage_num\\tblock_num\\tpar_num\\tline_num\\tword_num\\tleft\\ttop\\twidth\\theight\\tconf\\ttext\\n'\n"
206	-	if err := os.WriteFile(script, []byte(body), 0o700); err != nil {
207	-	t.Fatal(err)
208	-	}
209	-	old := ocrWorkers
210	-	ocrWorkers = 2
211	-	t.Cleanup(func() { ocrWorkers = old })
212	-	w := &worker{ocr: ocrState{done: true, binary: script, lang: "eng"}}
213	-	var jobs []*bandJob
214	-	for range 8 { // several windows' bands
215	-	g := image.NewGray(image.Rect(0, 0, 8, 8))
216	-	jobs = append(jobs, &bandJob{ext: g.Rect, own: [2]int{0, 8}, gray: g})
217	-	}
218	-	w.recognize(context.Background(), jobs)
219	-	data, err := os.ReadFile(logPath)
220	-	if err != nil {
221	-	t.Fatal(err)
222	-	}
223	-	cur, peak, starts := 0, 0, 0
224	-	for _, l := range strings.Fields(string(data)) {
225	-	if l == "s" {
226	-	cur++
227	-	starts++
228	-	peak = max(peak, cur)
229	-	} else {
230	-	cur--
231	-	}
232	-	}
233	-	if starts != 8 \|\| peak > 2 {
234	-	t.Fatalf("starts=%d peak=%d", starts, peak)
235	-	}
236	-	for _, j := range jobs {
237	-	if j.err != nil {
238	-	t.Fatal(j.err)
239	-	}
240	-	}
241	-	}
242	-
243	-	func TestLanguageDiscoveryTimeout(t *testing.T) {
244	-	t.Setenv("TESSDATA_PREFIX", t.TempDir())
245	-	runner := newMocklanguageRunner(t)
246	-	parent := context.Background()
247	-	runner.EXPECT().Output(mock.Anything, "tesseract", []string{"--list-langs"}).RunAndReturn(
248	-	func(ctx context.Context, _ string, _ ...string) ([]byte, error) {
249	-	deadline, ok := ctx.Deadline()
250	-	if !ok \|\| time.Until(deadline) > languageTimeout {
251	-	t.Fatal("language discovery has no bounded deadline")
252	-	}
253	-	<-ctx.Done()
254	-	return nil, ctx.Err()
255	-	}).Once()
256	-	lang, err := discoverLanguage(parent, "tesseract", runner, 0)
257	-	if lang != "" \|\| !errors.Is(err, context.DeadlineExceeded) \|\| parent.Err() != nil {
258	-	t.Fatalf("language=%q error=%v parent=%v", lang, err, parent.Err())
259	-	}
260	-	}
261	-
262	-	func TestLanguageDiscoveryFailureKeepsAccessibilityOnly(t *testing.T) {
263	-	w := &worker{ocr: ocrState{done: true, err: context.DeadlineExceeded}}
264	-	ar := atspiResult{targets: []ports.HintTarget{{Window: 1, Rect: ports.Rect{W: 20, H: 10}}}}
265	-	req := ports.HintScanRequest{Width: 100, Height: 100, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 100, H: 100}}}}
266	-	if w.needFrame(context.Background(), req, ar) {
267	-	t.Fatal("timed out discovery must not require an OCR capture")
268	-	}
269	-	if err := w.ocr.resolve(context.Background()); !errors.Is(err, context.DeadlineExceeded) {
270	-	t.Fatalf("discovery failure was not retained: %v", err)
271	-	}
272	-	}

D internal/adapters/hints/scan_bench_test.go +0 -49

@@ -1,49 +0,0 @@
1	-	package hints
2	-
3	-	import (
4	-	"context"
5	-	"os"
6	-	"os/exec"
7	-	"runtime"
8	-	"testing"
9	-	"time"
10	-
11	-	"github.com/bnema/neferwl/internal/ports"
12	-	)
13	-
14	-	// BenchmarkScan runs whole scans of a 5120x2160 frame with a noisy window
15	-	// (the cache never hits) and reports the Go heap bytes allocated per scan.
16	-	func BenchmarkScan(b *testing.B) {
17	-	if _, err := exec.LookPath("tesseract"); err != nil {
18	-	b.Skip("tesseract not on PATH")
19	-	}
20	-	const w, h = 5120, 2160
21	-	ctx := context.Background()
22	-	w0 := &worker{}
23	-	requests := make(chan ports.HintScanRequest)
24	-	captures := make(chan ports.CaptureRequest, 1)
25	-	captured := make(chan ports.CaptureDone, 1)
26	-	results := make(chan ports.HintScanResult, 1)
27	-	req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
28	-	if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
29	-	rawPGM = false
30	-	}
31	-	b.ReportAllocs()
32	-	var peak uint64
33	-	for i := range b.N {
34	-	go func() {
35	-	creq := <-captures
36	-	creq.Mapped[0] = byte(i) // each scan differs: no cache hit
37	-	creq.Mapped[1] = byte(i >> 8)
38	-	captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
39	-	}()
40	-	res := w0.serve(ctx, req, requests, captures, captured, results, 5*time.Second)
41	-	if res.Err != nil {
42	-	b.Fatal(res.Err)
43	-	}
44	-	var m runtime.MemStats
45	-	runtime.ReadMemStats(&m)
46	-	peak = max(peak, m.HeapSys-m.HeapReleased)
47	-	}
48	-	b.ReportMetric(float64(peak)/1e6, "heapMB")
49	-	}

D internal/adapters/hints/strips_test.go +0 -124

@@ -1,124 +0,0 @@
1	-	package hints
2	-
3	-	import (
4	-	"context"
5	-	"image"
6	-	"image/color"
7	-	"image/draw"
8	-	"os/exec"
9	-	"runtime"
10	-	"sort"
11	-	"sync"
12	-	"time"
13	-	)
14	-
15	-	type stripOffset struct {
16	-	source, packed image.Rectangle
17	-	factor int
18	-	}
19	-	type ocrStrip struct {
20	-	img *image.Gray
21	-	offsets []stripOffset
22	-	}
23	-
24	-	func packLines(src *image.RGBA, lines []image.Rectangle, workers int, normalize bool) []ocrStrip {
25	-	n := min(workers, len(lines))
26	-	if n == 0 {
27	-	return nil
28	-	}
29	-	groups := make([][]image.Rectangle, n)
30	-	areas := make([]int, n)
31	-	lines = append([]image.Rectangle(nil), lines...)
32	-	sort.Slice(lines, func(i, j int) bool { return lines[i].Dx()lines[i].Dy() > lines[j].Dx()lines[j].Dy() })
33	-	for _, r := range lines {
34	-	best := 0
35	-	for i := range n {
36	-	if areas[i] < areas[best] {
37	-	best = i
38	-	}
39	-	}
40	-	groups[best] = append(groups[best], r)
41	-	areas[best] += r.Dx() * r.Dy()
42	-	}
43	-	strips := make([]ocrStrip, n)
44	-	for i, group := range groups {
45	-	width, height := 0, 8
46	-	for _, r := range group {
47	-	factor := 1
48	-	if r.Dy() < 12 {
49	-	factor = 2
50	-	}
51	-	crop := r.Inset(-4).Intersect(src.Bounds())
52	-	packed := image.Rect(4, height, 4+crop.Dx()factor, height+crop.Dy()factor)
53	-	strips[i].offsets = append(strips[i].offsets, stripOffset{crop, packed, factor})
54	-	width = max(width, packed.Max.X+4)
55	-	height = packed.Max.Y + 8
56	-	}
57	-	dst := image.NewGray(image.Rect(0, 0, width, height))
58	-	draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
59	-	for _, off := range strips[i].offsets {
60	-	// The padded perimeter estimates the background without depending on text colour.
61	-	var sum, count int
62	-	r := off.source
63	-	for x := r.Min.X; x < r.Max.X; x++ {
64	-	sum += int(color.GrayModel.Convert(src.At(x, r.Min.Y)).(color.Gray).Y)
65	-	count++
66	-	}
67	-	invert := normalize && count > 0 && sum/count < 128
68	-	for y := off.packed.Min.Y; y < off.packed.Max.Y; y++ {
69	-	for x := off.packed.Min.X; x < off.packed.Max.X; x++ {
70	-	v := color.GrayModel.Convert(src.At(r.Min.X+(x-off.packed.Min.X)/off.factor, r.Min.Y+(y-off.packed.Min.Y)/off.factor)).(color.Gray).Y
71	-	if invert {
72	-	v = 255 - v
73	-	}
74	-	dst.SetGray(x, y, color.Gray{Y: v})
75	-	}
76	-	}
77	-	}
78	-	strips[i].img = dst
79	-	}
80	-	return strips
81	-	}
82	-
83	-	func readStrips(ctx context.Context, img *image.RGBA, lines []image.Rectangle, workers int, normalize bool) ([]OCRWord, int, error) {
84	-	binary, err := exec.LookPath("tesseract")
85	-	if err != nil {
86	-	return nil, 0, err
87	-	}
88	-	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
89	-	defer cancel()
90	-	lang, err := ocrLanguage(ctx, binary)
91	-	if err != nil {
92	-	return nil, 0, err
93	-	}
94	-	strips := packLines(img, lines, min(workers, runtime.GOMAXPROCS(0), 16), normalize)
95	-	results := make([][]OCRWord, len(strips))
96	-	errs := make([]error, len(strips))
97	-	var wg sync.WaitGroup
98	-	for i, s := range strips {
99	-	wg.Go(func() {
100	-	words, e := recognizeImage(ctx, binary, lang, s.img, "6", normalize)
101	-	errs[i] = e
102	-	for _, w := range words {
103	-	center := image.Pt((w.Rect.Min.X+w.Rect.Max.X)/2, (w.Rect.Min.Y+w.Rect.Max.Y)/2)
104	-	for _, off := range s.offsets {
105	-	if center.In(off.packed) {
106	-	r := w.Rect.Sub(off.packed.Min)
107	-	w.Rect = image.Rect(r.Min.X/off.factor, r.Min.Y/off.factor, (r.Max.X+off.factor-1)/off.factor, (r.Max.Y+off.factor-1)/off.factor).Add(off.source.Min)
108	-	results[i] = append(results[i], w)
109	-	break
110	-	}
111	-	}
112	-	}
113	-	})
114	-	}
115	-	wg.Wait()
116	-	var words []OCRWord
117	-	for i := range strips {
118	-	if errs[i] != nil {
119	-	return nil, len(strips), errs[i]
120	-	}
121	-	words = append(words, results[i]...)
122	-	}
123	-	return words, len(strips), nil
124	-	}

D internal/adapters/hints/target_source_mock_test.go +0 -174

@@ -1,174 +0,0 @@
1	-	// Code generated by mockery; DO NOT EDIT.
2	-	// github.com/vektra/mockery
3	-	// template: testify
4	-
5	-	package hints
6	-
7	-	import (
8	-	"context"
9	-
10	-	"github.com/bnema/neferwl/internal/ports"
11	-	mock "github.com/stretchr/testify/mock"
12	-	)
13	-
14	-	// newMocktargetSource creates a new instance of mocktargetSource. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
15	-	// The first argument is typically a *testing.T value.
16	-	func newMocktargetSource(t interface {
17	-	mock.TestingT
18	-	Cleanup(func())
19	-	}) *mocktargetSource {
20	-	if helper, ok := t.(interface{ Helper() }); ok {
21	-	helper.Helper()
22	-	}
23	-
24	-	mock := &mocktargetSource{}
25	-	mock.Mock.Test(t)
26	-
27	-	t.Cleanup(func() {
28	-	if helper, ok := t.(interface{ Helper() }); ok {
29	-	helper.Helper()
30	-	}
31	-	mock.AssertExpectations(t)
32	-	})
33	-
34	-	return mock
35	-	}
36	-
37	-	// mocktargetSource is an autogenerated mock type for the targetSource type
38	-	type mocktargetSource struct {
39	-	mock.Mock
40	-	}
41	-
42	-	type mocktargetSource_Expecter struct {
43	-	mock *mock.Mock
44	-	}
45	-
46	-	func (_m mocktargetSource) EXPECT() mocktargetSource_Expecter {
47	-	return &mocktargetSource_Expecter{mock: &_m.Mock}
48	-	}
49	-
50	-	// Close provides a mock function for the type mocktargetSource
51	-	func (_mock *mocktargetSource) Close() error {
52	-	ret := _mock.Called()
53	-
54	-	if len(ret) == 0 {
55	-	panic("no return value specified for Close")
56	-	}
57	-
58	-	var r0 error
59	-	if returnFunc, ok := ret.Get(0).(func() error); ok {
60	-	r0 = returnFunc()
61	-	} else {
62	-	r0 = ret.Error(0)
63	-	}
64	-	return r0
65	-	}
66	-
67	-	// mocktargetSource_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
68	-	type mocktargetSource_Close_Call struct {
69	-	*mock.Call
70	-	}
71	-
72	-	// Close is a helper method to define mock.On call
73	-	func (_e mocktargetSource_Expecter) Close() mocktargetSource_Close_Call {
74	-	return &mocktargetSource_Close_Call{Call: _e.mock.On("Close")}
75	-	}
76	-
77	-	func (_c mocktargetSource_Close_Call) Run(run func()) mocktargetSource_Close_Call {
78	-	_c.Call.Run(func(args mock.Arguments) {
79	-	run()
80	-	})
81	-	return _c
82	-	}
83	-
84	-	func (_c mocktargetSource_Close_Call) Return(err error) mocktargetSource_Close_Call {
85	-	_c.Call.Return(err)
86	-	return _c
87	-	}
88	-
89	-	func (_c mocktargetSource_Close_Call) RunAndReturn(run func() error) mocktargetSource_Close_Call {
90	-	_c.Call.Return(run)
91	-	return _c
92	-	}
93	-
94	-	// Targets provides a mock function for the type mocktargetSource
95	-	func (_mock *mocktargetSource) Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error) {
96	-	ret := _mock.Called(ctx, windows, linksOnly)
97	-
98	-	if len(ret) == 0 {
99	-	panic("no return value specified for Targets")
100	-	}
101	-
102	-	var r0 []ports.HintTarget
103	-	var r1 map[ports.WindowID]bool
104	-	var r2 error
105	-	if returnFunc, ok := ret.Get(0).(func(context.Context, []ports.HintWindow, bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)); ok {
106	-	return returnFunc(ctx, windows, linksOnly)
107	-	}
108	-	if returnFunc, ok := ret.Get(0).(func(context.Context, []ports.HintWindow, bool) []ports.HintTarget); ok {
109	-	r0 = returnFunc(ctx, windows, linksOnly)
110	-	} else {
111	-	if ret.Get(0) != nil {
112	-	r0 = ret.Get(0).([]ports.HintTarget)
113	-	}
114	-	}
115	-	if returnFunc, ok := ret.Get(1).(func(context.Context, []ports.HintWindow, bool) map[ports.WindowID]bool); ok {
116	-	r1 = returnFunc(ctx, windows, linksOnly)
117	-	} else {
118	-	if ret.Get(1) != nil {
119	-	r1 = ret.Get(1).(map[ports.WindowID]bool)
120	-	}
121	-	}
122	-	if returnFunc, ok := ret.Get(2).(func(context.Context, []ports.HintWindow, bool) error); ok {
123	-	r2 = returnFunc(ctx, windows, linksOnly)
124	-	} else {
125	-	r2 = ret.Error(2)
126	-	}
127	-	return r0, r1, r2
128	-	}
129	-
130	-	// mocktargetSource_Targets_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Targets'
131	-	type mocktargetSource_Targets_Call struct {
132	-	*mock.Call
133	-	}
134	-
135	-	// Targets is a helper method to define mock.On call
136	-	// - ctx context.Context
137	-	// - windows []ports.HintWindow
138	-	// - linksOnly bool
139	-	func (_e mocktargetSource_Expecter) Targets(ctx any, windows any, linksOnly any) mocktargetSource_Targets_Call {
140	-	return &mocktargetSource_Targets_Call{Call: _e.mock.On("Targets", ctx, windows, linksOnly)}
141	-	}
142	-
143	-	func (_c mocktargetSource_Targets_Call) Run(run func(ctx context.Context, windows []ports.HintWindow, linksOnly bool)) mocktargetSource_Targets_Call {
144	-	_c.Call.Run(func(args mock.Arguments) {
145	-	var arg0 context.Context
146	-	if args[0] != nil {
147	-	arg0 = args[0].(context.Context)
148	-	}
149	-	var arg1 []ports.HintWindow
150	-	if args[1] != nil {
151	-	arg1 = args[1].([]ports.HintWindow)
152	-	}
153	-	var arg2 bool
154	-	if args[2] != nil {
155	-	arg2 = args[2].(bool)
156	-	}
157	-	run(
158	-	arg0,
159	-	arg1,
160	-	arg2,
161	-	)
162	-	})
163	-	return _c
164	-	}
165	-
166	-	func (_c mocktargetSource_Targets_Call) Return(hintTargets []ports.HintTarget, windowIDToBool map[ports.WindowID]bool, err error) mocktargetSource_Targets_Call {
167	-	_c.Call.Return(hintTargets, windowIDToBool, err)
168	-	return _c
169	-	}
170	-
171	-	func (_c mocktargetSource_Targets_Call) RunAndReturn(run func(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)) mocktargetSource_Targets_Call {
172	-	_c.Call.Return(run)
173	-	return _c
174	-	}

D internal/adapters/hints/testdata/browser.png +0 -0 binary file: no text diff

binary file: no text diff

D internal/adapters/hints/testdata/terminal.png +0 -0 binary file: no text diff

binary file: no text diff

D internal/adapters/hints/testdata/underline.png +0 -0 binary file: no text diff

binary file: no text diff

A internal/adapters/hints/tile.go +122 -0

@@ -0,0 +1,122 @@
	1	+	package hints
	2	+
	3	+	const (
	4	+	// maxRegions is the protocol limit of regions per request.
	5	+	maxRegions = 64
	6	+	// wholeMaxSide is the longest frame side sent whole when the frame is not
	7	+	// 16:9: up to there the square letterbox of the detector loses little.
	8	+	wholeMaxSide = 2560
	9	+	// tileMaxH is the tallest tile; its width is 16/9 of it.
	10	+	tileMaxH = 1440
	11	+	// tileCutMargin is the distance, in physical pixels, to a tile's inner
	12	+	// edge under which a box counts as cut by it.
	13	+	tileCutMargin = 2
	14	+	)
	15	+
	16	+	// region is a crop of the frame, in physical pixels.
	17	+	type region struct{ x, y, w, h int }
	18	+
	19	+	// tileRegions returns the crops to detect separately on a width x height
	20	+	// frame, or nil for one whole-frame detection (a 16:9 frame, or one whose
	21	+	// long side is at most wholeMaxSide). Otherwise the frame is covered by exact
	22	+	// 16:9 tiles (w9 == h16; explicit crops always use square letterboxing) of
	23	+	// height min(height, 1440) rounded down to a multiple of 9, overlapping by at
	24	+	// least 10 % of the tile side on each axis; the last tile of a row or column
	25	+	// is aligned to the frame edge. Tiles are listed row by row. A frame that
	26	+	// would need more than maxRegions tiles is detected whole.
	27	+	func tileRegions(width, height int) []region {
	28	+	if width <= 0 \|\| height <= 0 \|\| width9 == height16 \|\| max(width, height) <= wholeMaxSide {
	29	+	return nil
	30	+	}
	31	+	th := min(height, tileMaxH) / 9 * 9
	32	+	tw := th / 9 * 16
	33	+	if tw > width { // a tall frame: the width bounds the tile
	34	+	tw = width / 16 * 16
	35	+	th = tw / 16 * 9
	36	+	}
	37	+	if tw == 0 \|\| th == 0 {
	38	+	return nil
	39	+	}
	40	+	xs, ys := tileStarts(width, tw), tileStarts(height, th)
	41	+	if len(xs)*len(ys) > maxRegions {
	42	+	return nil
	43	+	}
	44	+	out := make([]region, 0, len(xs)*len(ys))
	45	+	for _, y := range ys {
	46	+	for _, x := range xs {
	47	+	out = append(out, region{x, y, tw, th})
	48	+	}
	49	+	}
	50	+	return out
	51	+	}
	52	+
	53	+	// tileStarts returns the start offsets of tiles of size t covering length l,
	54	+	// evenly spread, the last one flush with the end, with consecutive tiles
	55	+	// overlapping by at least a tenth of t.
	56	+	func tileStarts(l, t int) []int {
	57	+	if l <= t {
	58	+	return []int{0}
	59	+	}
	60	+	d := l - t
	61	+	maxStep := t * 9 / 10
	62	+	n := (d+maxStep-1)/maxStep + 1
	63	+	out := make([]int, n)
	64	+	for i := range out {
	65	+	out[i] = (di + n - 2) / (n - 1) // ceil(di/(n-1)): steps never exceed maxStep
	66	+	}
	67	+	return out
	68	+	}
	69	+
	70	+	// dropCutBoxes drops the boxes that a tile's inner edge cut (an edge of the
	71	+	// tile that is not an edge of the frame) when another tile saw the same
	72	+	// element uncut: the cut box would otherwise survive as a smaller box nested
	73	+	// in the whole one. A box is "the same element" when the other box covers
	74	+	// nestRatio of it. Without regions the boxes are returned as they are.
	75	+	func dropCutBoxes(dets []detection, regions []region) []detection {
	76	+	if len(regions) == 0 {
	77	+	return dets
	78	+	}
	79	+	fw, fh := 0, 0
	80	+	for _, r := range regions {
	81	+	fw, fh = max(fw, r.x+r.w), max(fh, r.y+r.h)
	82	+	}
	83	+	cut := func(d detection) bool {
	84	+	if d.region < 0 \|\| d.region >= len(regions) {
	85	+	return false
	86	+	}
	87	+	r := regions[d.region]
	88	+	m := float32(tileCutMargin)
	89	+	return (r.x > 0 && d.x1 < float32(r.x)+m) \|\| (r.x+r.w < fw && d.x2 > float32(r.x+r.w)-m) \|\|
	90	+	(r.y > 0 && d.y1 < float32(r.y)+m) \|\| (r.y+r.h < fh && d.y2 > float32(r.y+r.h)-m)
	91	+	}
	92	+	isCut := make([]bool, len(dets))
	93	+	for i, d := range dets {
	94	+	isCut[i] = cut(d)
	95	+	}
	96	+	out := make([]detection, 0, len(dets))
	97	+	for i, d := range dets {
	98	+	drop := false
	99	+	if isCut[i] {
	100	+	for j, o := range dets {
	101	+	if j != i && !isCut[j] && o.region != d.region && covered(d, o) {
	102	+	drop = true
	103	+	break
	104	+	}
	105	+	}
	106	+	}
	107	+	if !drop {
	108	+	out = append(out, d)
	109	+	}
	110	+	}
	111	+	return out
	112	+	}
	113	+
	114	+	// covered reports whether o covers at least nestRatio of d's area.
	115	+	func covered(d, o detection) bool {
	116	+	w := min(d.x2, o.x2) - max(d.x1, o.x1)
	117	+	h := min(d.y2, o.y2) - max(d.y1, o.y1)
	118	+	if w <= 0 \|\| h <= 0 {
	119	+	return false
	120	+	}
	121	+	return float64(wh) >= nestRatiofloat64((d.x2-d.x1)*(d.y2-d.y1))
	122	+	}

A internal/adapters/hints/tile_test.go +75 -0

@@ -0,0 +1,75 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"testing"
	5	+
	6	+	"github.com/stretchr/testify/require"
	7	+	)
	8	+
	9	+	func TestTileRegions(t *testing.T) {
	10	+	whole := [][2]int{{3840, 2160}, {1920, 1080}, {2560, 1080}, {2560, 1440}, {1280, 720}}
	11	+	for _, s := range whole {
	12	+	require.Nil(t, tileRegions(s[0], s[1]), "%dx%d", s[0], s[1])
	13	+	}
	14	+	require.Equal(t, []region{{0, 0, 2560, 1440}, {1280, 0, 2560, 1440}, {2560, 0, 2560, 1440},
	15	+	{0, 720, 2560, 1440}, {1280, 720, 2560, 1440}, {2560, 720, 2560, 1440}}, tileRegions(5120, 2160))
	16	+	require.Equal(t, []region{{0, 0, 2560, 1440}, {880, 0, 2560, 1440}}, tileRegions(3440, 1440))
	17	+
	18	+	for _, s := range [][2]int{{5120, 2160}, {3440, 1440}, {5120, 1440}, {7680, 2160}, {3000, 2999}, {1000, 3000}, {5121, 2161}, {6000, 1001}} {
	19	+	w, h := s[0], s[1]
	20	+	rs := tileRegions(w, h)
	21	+	require.NotEmpty(t, rs, "%dx%d", w, h)
	22	+	require.LessOrEqual(t, len(rs), maxRegions)
	23	+	covered := make([]bool, w*h)
	24	+	for _, r := range rs {
	25	+	require.Equal(t, r.h16, r.w9, "%dx%d tile %+v is not 16:9", w, h, r)
	26	+	require.GreaterOrEqual(t, r.x, 0)
	27	+	require.GreaterOrEqual(t, r.y, 0)
	28	+	require.LessOrEqual(t, r.x+r.w, w)
	29	+	require.LessOrEqual(t, r.y+r.h, h)
	30	+	for y := r.y; y < r.y+r.h; y++ {
	31	+	for x := r.x; x < r.x+r.w; x++ {
	32	+	covered[y*w+x] = true
	33	+	}
	34	+	}
	35	+	}
	36	+	for i, c := range covered {
	37	+	if !c {
	38	+	t.Fatalf("%dx%d: pixel %d,%d not covered", w, h, i%w, i/w)
	39	+	}
	40	+	}
	41	+	}
	42	+	}
	43	+
	44	+	func TestTileStartsOverlap(t *testing.T) {
	45	+	for _, l := range []int{2561, 3440, 5120, 7680, 10000} {
	46	+	xs := tileStarts(l, 2560)
	47	+	require.Equal(t, 0, xs[0])
	48	+	require.Equal(t, l-2560, xs[len(xs)-1])
	49	+	for i := 1; i < len(xs); i++ {
	50	+	require.GreaterOrEqual(t, xs[i-1]+2560-xs[i], 256, "length %d", l) // >= 10 %
	51	+	}
	52	+	}
	53	+	}
	54	+
	55	+	func TestTileRegionsTooMany(t *testing.T) {
	56	+	require.Nil(t, tileRegions(60000, 20000))
	57	+	}
	58	+
	59	+	func TestDropCutBoxes(t *testing.T) {
	60	+	rs := tileRegions(5120, 2160) // tile 0: x 0..2560, tile 1: x 1280..3840
	61	+	// A button at x 2500..2700 is cut at 2560 by tile 0 and whole in tile 1.
	62	+	cutBox := detection{x1: 2500, y1: 100, x2: 2560, y2: 140, score: 0.6, region: 0}
	63	+	whole := detection{x1: 2500, y1: 100, x2: 2700, y2: 140, score: 0.8, region: 1}
	64	+	require.Equal(t, []detection{whole}, dropCutBoxes([]detection{cutBox, whole}, rs))
	65	+	// Nobody saw it whole: the cut box stays.
	66	+	require.Equal(t, []detection{cutBox}, dropCutBoxes([]detection{cutBox}, rs))
	67	+	// An edge of the frame is not an inner edge.
	68	+	edge := detection{x1: 0, y1: 0, x2: 60, y2: 30, score: 0.6, region: 0}
	69	+	// Region 3 starts at y 720: its box at the frame's left edge is not cut
	70	+	// either, so both stay though they sit in different regions.
	71	+	other := detection{x1: 0, y1: 800, x2: 60, y2: 830, score: 0.5, region: 3}
	72	+	require.Equal(t, []detection{edge, other}, dropCutBoxes([]detection{edge, other}, rs))
	73	+	// No regions: unchanged.
	74	+	require.Equal(t, []detection{cutBox}, dropCutBoxes([]detection{cutBox}, nil))
	75	+	}

D internal/adapters/hints/underline.go +0 -246

@@ -1,246 +0,0 @@
1	-	package hints
2	-
3	-	import (
4	-	"image"
5	-	"math"
6	-	"sort"
7	-
8	-	"github.com/bnema/neferwl/internal/ports"
9	-	)
10	-
11	-	// underlineCandidate is a word with a thin line under it.
12	-	type underlineCandidate struct {
13	-	word int
14	-	row int // the line's row
15	-	from, to int // x extent of the continuous line through the word
16	-	}
17	-
18	-	// markUnderlined sets Underlined, UnderL and UnderR on the words of a band
19	-	// that have a thin line under them. words are in the coordinates of g.
20	-	// Lines that run on far beyond the word (rules, table borders, input boxes)
21	-	// do not count, unless the extension is made of underlined words too.
22	-	func markUnderlined(src *image.Gray, words []OCRWord) {
23	-	// Words are relative to the band's own origin, whatever the image is a
24	-	// view of.
25	-	b := src.Bounds()
26	-	g := &image.Gray{Pix: src.Pix[src.PixOffset(b.Min.X, b.Min.Y):], Stride: src.Stride, Rect: image.Rect(0, 0, b.Dx(), b.Dy())}
27	-	var cands []underlineCandidate
28	-	for i, w := range words {
29	-	// A single glyph, or a box narrower than tall, has a flat bottom
30	-	// stroke (8, L, 2) that is not an underline.
31	-	if len([]rune(w.Text)) < 2 \|\| w.Rect.Dx() < w.Rect.Dy() {
32	-	continue
33	-	}
34	-	if c, ok := underlineOf(g, w.Rect); ok {
35	-	c.word = i
36	-	cands = append(cands, c)
37	-	}
38	-	}
39	-	for _, c := range cands {
40	-	w := words[c.word]
41	-	h := w.Rect.Dy()
42	-	if (w.Rect.Min.X-c.from > h && !covered(words, cands, c, -1)) \|\|
43	-	(c.to-w.Rect.Max.X > h && !covered(words, cands, c, 1)) {
44	-	continue
45	-	}
46	-	words[c.word].Underlined, words[c.word].UnderL, words[c.word].UnderR = true, c.from, c.to
47	-	}
48	-	}
49	-
50	-	// covered reports whether the line of c, on the given side of its word, is
51	-	// explained by underlined words on the same line up to within a glyph of its
52	-	// end.
53	-	func covered(words []OCRWord, cands []underlineCandidate, c underlineCandidate, dir int) bool {
54	-	w := words[c.word].Rect
55	-	h := w.Dy()
56	-	cur := w.Min.X
57	-	if dir > 0 {
58	-	cur = w.Max.X
59	-	}
60	-	for {
61	-	if (dir < 0 && cur-c.from <= h) \|\| (dir > 0 && c.to-cur <= h) {
62	-	return true
63	-	}
64	-	next := -1
65	-	for _, o := range cands {
66	-	r := words[o.word].Rect
67	-	if abs(r.Max.Y-w.Max.Y) > max(3, h/3) {
68	-	continue
69	-	}
70	-	var gap, reach int
71	-	if dir < 0 {
72	-	gap, reach = cur-r.Max.X, r.Min.X
73	-	} else {
74	-	gap, reach = r.Min.X-cur, r.Max.X
75	-	}
76	-	if gap < -2 \|\| gap > h {
77	-	continue
78	-	}
79	-	if next < 0 \|\| (dir < 0 && reach < next) \|\| (dir > 0 && reach > next) {
80	-	next = reach
81	-	}
82	-	}
83	-	if next < 0 \|\| (dir < 0 && next >= cur) \|\| (dir > 0 && next <= cur) {
84	-	return false
85	-	}
86	-	cur = next
87	-	}
88	-	}
89	-
90	-	// underlineOf looks for an underline under the word box r in g.
91	-	func underlineOf(g *image.Gray, r image.Rectangle) (underlineCandidate, bool) {
92	-	b := g.Bounds()
93	-	box := r.Intersect(b)
94	-	h := r.Dy()
95	-	if box.Dx() < 4 \|\| h < 4 {
96	-	return underlineCandidate{}, false
97	-	}
98	-	bg, ink := contrastOf(g, box)
99	-	thr := min(40, max(20, ink/2))
100	-	if ink < thr+10 {
101	-	return underlineCandidate{}, false
102	-	}
103	-	top := max(b.Min.Y, r.Max.Y-(h*15+99)/100)
104	-	bottom := min(b.Max.Y, r.Max.Y+(h*35+99)/100+1)
105	-	maxThick := max(2, h/6)
106	-	rowOK := func(y int) bool {
107	-	row := g.Pix[(y-g.Rect.Min.Y)*g.Stride:]
108	-	n := 0
109	-	for x := box.Min.X; x < box.Max.X; x++ {
110	-	if abs(int(row[x-g.Rect.Min.X])-bg) >= thr {
111	-	n++
112	-	}
113	-	}
114	-	return n100 >= box.Dx()85
115	-	}
116	-	thick, first := 0, 0
117	-	for y := top; y <= bottom; y++ {
118	-	if y < bottom && rowOK(y) {
119	-	if thick == 0 {
120	-	first = y
121	-	}
122	-	thick++
123	-	continue
124	-	}
125	-	if thick > 0 {
126	-	if thick > maxThick {
127	-	return underlineCandidate{}, false // a block, not a line
128	-	}
129	-	row := first + thick/2
130	-	from, to := box.Min.X, box.Max.X
131	-	line := g.Pix[(row-g.Rect.Min.Y)*g.Stride:]
132	-	on := func(x int) bool { return abs(int(line[x-g.Rect.Min.X])-bg) >= thr }
133	-	for from > b.Min.X && on(from-1) {
134	-	from--
135	-	}
136	-	for to < b.Max.X && on(to) {
137	-	to++
138	-	}
139	-	return underlineCandidate{row: row, from: from, to: to}, true
140	-	}
141	-	}
142	-	return underlineCandidate{}, false
143	-	}
144	-
145	-	// contrastOf returns the background level of box (the most common level,
146	-	// found in bins of 16 so antialiasing does not split it, then refined to the
147	-	// mean of that bin) and the contrast of its ink (the deviation reached by
148	-	// the strongest 3% of pixels).
149	-	func contrastOf(g *image.Gray, box image.Rectangle) (bg, ink int) {
150	-	var hist [256]int
151	-	for y := box.Min.Y; y < box.Max.Y; y++ {
152	-	for _, v := range g.Pix[(y-g.Rect.Min.Y)*g.Stride+box.Min.X-g.Rect.Min.X:][:box.Dx()] {
153	-	hist[v]++
154	-	}
155	-	}
156	-	total := box.Dx() * box.Dy()
157	-	var bins [16]int
158	-	for v, c := range hist {
159	-	bins[v>>4] += c
160	-	}
161	-	best := 0
162	-	for i, c := range bins {
163	-	if c > bins[best] {
164	-	best = i
165	-	}
166	-	}
167	-	sum, n := 0, 0
168	-	for v := best << 4; v < (best+1)<<4; v++ {
169	-	sum += v * hist[v]
170	-	n += hist[v]
171	-	}
172	-	bg = sum / max(n, 1)
173	-	var dev [256]int
174	-	for v, c := range hist {
175	-	dev[abs(v-bg)] += c
176	-	}
177	-	n = 0
178	-	for d := 255; d >= 0; d-- {
179	-	n += dev[d]
180	-	if n100 >= total3 {
181	-	return bg, d
182	-	}
183	-	}
184	-	return bg, 0
185	-	}
186	-
187	-	// underlinedLinks returns the targets for underlined words, joining the
188	-	// words of one continuous underline. A group that overlaps a URL target (in
189	-	// pixels) is left to it. Coordinates are scaled to logical.
190	-	func underlinedLinks(words []OCRWord, urls []image.Rectangle, scale float64) []ports.HintTarget {
191	-	var us []OCRWord
192	-	for _, w := range words {
193	-	if w.Underlined {
194	-	us = append(us, w)
195	-	}
196	-	}
197	-	sort.Slice(us, func(i, j int) bool { return us[i].Rect.Min.X < us[j].Rect.Min.X })
198	-	used := make([]bool, len(us))
199	-	var out []ports.HintTarget
200	-	for i := range us {
201	-	if used[i] {
202	-	continue
203	-	}
204	-	used[i] = true
205	-	rect, last := us[i].Rect, us[i]
206	-	chars, width := len(last.Text), last.Rect.Dx()
207	-	for {
208	-	next := -1
209	-	for j := range us {
210	-	c := us[j]
211	-	if used[j] \|\| c.Rect.Min.X < rect.Max.X-2 \|\| abs(c.Rect.Max.Y-last.Rect.Max.Y) > max(3, rect.Dy()/3) {
212	-	continue
213	-	}
214	-	avg := float64(width+c.Rect.Dx()) / float64(max(1, chars+len(c.Text)))
215	-	gap := c.Rect.Min.X - rect.Max.X
216	-	if float64(gap) > 1.2*avg \|\| c.Rect.Min.X > last.UnderR+2 {
217	-	continue
218	-	}
219	-	if next < 0 \|\| c.Rect.Min.X < us[next].Rect.Min.X {
220	-	next = j
221	-	}
222	-	}
223	-	if next < 0 {
224	-	break
225	-	}
226	-	used[next] = true
227	-	last = us[next]
228	-	rect = rect.Union(last.Rect)
229	-	chars += len(last.Text)
230	-	width += last.Rect.Dx()
231	-	}
232	-	dup := false
233	-	for _, u := range urls {
234	-	if overlap(rect, u) > .5 {
235	-	dup = true
236	-	}
237	-	}
238	-	if dup {
239	-	continue
240	-	}
241	-	x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
242	-	right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
243	-	out = append(out, ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
244	-	}
245	-	return out
246	-	}

D internal/adapters/hints/underline_test.go +0 -252

@@ -1,252 +0,0 @@
1	-	package hints
2	-
3	-	import (
4	-	"image"
5	-	"image/color"
6	-	"os"
7	-	"os/exec"
8	-	"testing"
9	-
10	-	"github.com/stretchr/testify/require"
11	-
12	-	"github.com/bnema/neferwl/internal/ports"
13	-	)
14	-
15	-	// canvas is a synthetic grayscale band: bg everywhere, ink for text.
16	-	type canvas struct {
17	-	g *image.Gray
18	-	bg, ink uint8
19	-	s int // scale
20	-	}
21	-
22	-	func newCanvas(w, h int, bg, ink uint8, s int) *canvas {
23	-	g := image.NewGray(image.Rect(0, 0, ws, hs))
24	-	for i := range g.Pix {
25	-	g.Pix[i] = bg
26	-	}
27	-	return &canvas{g, bg, ink, s}
28	-	}
29	-
30	-	func (c *canvas) fill(x0, y0, x1, y1 int, v uint8) {
31	-	for y := y0 * c.s; y < y1*c.s; y++ {
32	-	for x := x0 * c.s; x < x1*c.s; x++ {
33	-	c.g.SetGray(x, y, color.Gray{v})
34	-	}
35	-	}
36	-	}
37	-
38	-	// word draws a 10 px high word of x-height body at (x,y) with the given
39	-	// width: text-like noisy ink (a stem every 3 px) and optional descenders.
40	-	// It returns the OCR box (descenders excluded, as a box around the body).
41	-	func (c *canvas) word(x, y, w int, descender bool) OCRWord {
42	-	for i := 0; i < w; i += 3 {
43	-	c.fill(x+i, y, x+i+1, y+10, c.ink)
44	-	}
45	-	box := image.Rect(xc.s, yc.s, (x+w)c.s, (y+10)c.s)
46	-	if descender {
47	-	for i := 0; i < w; i += 6 {
48	-	c.fill(x+i, y+10, x+i+1, y+13, c.ink)
49	-	}
50	-	box.Max.Y = (y + 13) * c.s
51	-	}
52	-	return OCRWord{Text: "word", Rect: box}
53	-	}
54	-
55	-	func (c *canvas) underline(x0, x1, y int) { c.fill(x0, y, x1, y+1, c.ink) }
56	-
57	-	func scanUnder(c *canvas, words ...OCRWord) []ports.HintTarget {
58	-	markUnderlined(c.g, words)
59	-	return textLinks(words, float64(c.s))
60	-	}
61	-
62	-	func TestUnderline(t *testing.T) {
63	-	for _, tc := range []struct {
64	-	name string
65	-	bg, ink uint8
66	-	}{{"dark-bg", 10, 204}, {"light-bg", 245, 30}} {
67	-	for _, s := range []int{1, 2} {
68	-	t.Run(tc.name, func(t *testing.T) {
69	-	mk := func() *canvas { return newCanvas(200, 60, tc.bg, tc.ink, s) }
70	-
71	-	c := mk()
72	-	w := c.word(10, 20, 40, false)
73	-	c.underline(10, 50, 32)
74	-	got := scanUnder(c, w)
75	-	require.Len(t, got, 1)
76	-	require.Equal(t, ports.HintLink, got[0].Kind)
77	-	require.False(t, got[0].Text)
78	-	require.Equal(t, ports.WindowID(0), got[0].Window)
79	-	require.Equal(t, 10, got[0].Rect.X)
80	-
81	-	c = mk()
82	-	require.Empty(t, scanUnder(c, c.word(10, 20, 40, false)), "plain word")
83	-
84	-	c = mk()
85	-	require.Empty(t, scanUnder(c, c.word(10, 20, 40, true)), "descenders")
86	-
87	-	c = mk()
88	-	w = c.word(10, 20, 40, false)
89	-	c.underline(0, 200, 32)
90	-	require.Empty(t, scanUnder(c, w), "rule across")
91	-
92	-	c = mk()
93	-	w = c.word(10, 20, 40, false)
94	-	c.fill(10, 24, 50, 25, c.ink)
95	-	require.Empty(t, scanUnder(c, w), "strikethrough")
96	-
97	-	// Two words with a continuous underline: one target.
98	-	c = mk()
99	-	a, b := c.word(10, 20, 40, false), c.word(56, 20, 40, false)
100	-	c.underline(10, 96, 32)
101	-	got = scanUnder(c, a, b)
102	-	require.Len(t, got, 1)
103	-	require.Equal(t, 10, got[0].Rect.X)
104	-	require.Equal(t, 86, got[0].Rect.W)
105	-
106	-	// A gap in the underline: two targets.
107	-	c = mk()
108	-	a, b = c.word(10, 20, 40, false), c.word(56, 20, 40, false)
109	-	c.underline(10, 50, 32)
110	-	c.underline(56, 96, 32)
111	-	require.Len(t, scanUnder(c, a, b), 2)
112	-	})
113	-	}
114	-	}
115	-	}
116	-
117	-	// A box border under a word, with the word a part of a table row, is not a
118	-	// link; neither is an underline that continues beyond into unrelated text.
119	-	func TestUnderlineRejectsBox(t *testing.T) {
120	-	c := newCanvas(300, 60, 10, 204, 1)
121	-	w := c.word(100, 20, 40, false)
122	-	c.underline(20, 280, 32)
123	-	c.fill(20, 10, 21, 33, 204)
124	-	c.fill(279, 10, 280, 33, 204)
125	-	require.Empty(t, scanUnder(c, w))
126	-	}
127	-
128	-	// The line must be thin.
129	-	func TestUnderlineRejectsBlock(t *testing.T) {
130	-	c := newCanvas(100, 60, 10, 204, 1)
131	-	w := c.word(10, 20, 40, false)
132	-	c.fill(10, 31, 50, 36, 204)
133	-	require.Empty(t, scanUnder(c, w))
134	-	}
135	-
136	-	func TestUnderlinedURLIsOneTextTarget(t *testing.T) {
137	-	c := newCanvas(300, 60, 10, 204, 1)
138	-	w := c.word(10, 20, 100, false)
139	-	w.Text = "example.org"
140	-	c.underline(10, 110, 32)
141	-	got := scanUnder(c, w)
142	-	require.Len(t, got, 1)
143	-	require.True(t, got[0].Text)
144	-	}
145	-
146	-	// ocrTestdata reads the words of a testdata image with the real tesseract.
147	-	func ocrTestdata(t testing.T, name string) (image.Gray, []OCRWord) {
148	-	t.Helper()
149	-	if _, err := exec.LookPath("tesseract"); err != nil {
150	-	t.Skip("tesseract not on PATH")
151	-	}
152	-	f, err := os.Open("testdata/" + name)
153	-	require.NoError(t, err)
154	-	defer f.Close()
155	-	img, _, err := image.Decode(f)
156	-	require.NoError(t, err)
157	-	g := image.NewGray(img.Bounds())
158	-	for y := 0; y < g.Rect.Dy(); y++ {
159	-	for x := 0; x < g.Rect.Dx(); x++ {
160	-	g.Set(x, y, img.At(x, y))
161	-	}
162	-	}
163	-	words, _, err := readWords(t.Context(), g)
164	-	require.NoError(t, err)
165	-	return g, words
166	-	}
167	-
168	-	// headingRule finds the thin horizontal rule under the heading of
169	-	// browser.png: the first row, from y=150, with a 900 px run of non-white
170	-	// pixels. It returns the rule as a rect.
171	-	func headingRule(t testing.T, g image.Gray) image.Rectangle {
172	-	t.Helper()
173	-	for y := 150; y < 220; y++ {
174	-	run, start := 0, 0
175	-	for x := 0; x < g.Rect.Dx(); x++ {
176	-	if g.GrayAt(x, y).Y < 240 {
177	-	if run == 0 {
178	-	start = x
179	-	}
180	-	run++
181	-	if run >= 900 {
182	-	return image.Rect(start, y, start+run, y+1)
183	-	}
184	-	} else {
185	-	run = 0
186	-	}
187	-	}
188	-	}
189	-	t.Fatal("heading rule not found")
190	-	return image.Rectangle{}
191	-	}
192	-
193	-	// browser.png has blue links and no underlined link: the only underline-only
194	-	// target is a known false positive, and the heading rule is not taken for an
195	-	// underline.
196	-	func TestUnderlineOnBrowserTestdata(t *testing.T) {
197	-	g, words := ocrTestdata(t, "browser.png")
198	-	rule := headingRule(t, g)
199	-	// Known false positive: the word "as" (flat glyph bottoms, ~(878,353)) is
200	-	// flagged underlined by the detector. Anything else is a regression.
201	-	var under []ports.HintTarget
202	-	for _, tg := range textLinks(words, 1) {
203	-	if !tg.Text {
204	-	under = append(under, tg)
205	-	}
206	-	}
207	-	require.Len(t, under, 1, "underline-only targets: %v", under)
208	-	require.True(t, under[0].Rect.X > 860 && under[0].Rect.X < 900 && under[0].Rect.Y > 340 && under[0].Rect.Y < 370, "unexpected target %v", under[0].Rect)
209	-	for _, tg := range textLinks(words, 1) {
210	-	r := image.Rect(tg.Rect.X, tg.Rect.Y, tg.Rect.X+tg.Rect.W, tg.Rect.Y+tg.Rect.H)
211	-	require.False(t, r.Overlaps(rule), "target %v overlaps the heading rule %v", tg.Rect, rule)
212	-	}
213	-	}
214	-
215	-	// underline.png is a real render (Chromium) of a generic page: underlined
216	-	// links `demo`, `source` and `ermite-runner hosts`, a table with row rules
217	-	// and an input box, which must yield nothing.
218	-	func TestUnderlineOnRealRender(t *testing.T) {
219	-	_, words := ocrTestdata(t, "underline.png")
220	-	near := func(r ports.Rect, x0, y0, x1, y1 int) bool {
221	-	const tol = 8
222	-	abs := func(v int) int { return max(v, -v) }
223	-	return abs(r.X-x0) <= tol && abs(r.Y-y0) <= tol && abs(r.X+r.W-x1) <= tol && abs(r.Y+r.H-y1) <= tol
224	-	}
225	-	var got []ports.HintTarget
226	-	for _, tg := range textLinks(words, 1) {
227	-	if !tg.Text {
228	-	got = append(got, tg)
229	-	}
230	-	}
231	-	require.Len(t, got, 3, "targets: %v", got)
232	-	for _, want := range [][4]int{{148, 122, 196, 137}, {306, 126, 364, 137}, {122, 165, 294, 180}} {
233	-	found := false
234	-	for _, tg := range got {
235	-	found = found \|\| near(tg.Rect, want[0], want[1], want[2], want[3])
236	-	}
237	-	require.True(t, found, "no target near %v in %v", want, got)
238	-	}
239	-	// The table (y 215..335) and the input box (y 360..410) give nothing.
240	-	for _, tg := range textLinks(words, 1) {
241	-	require.False(t, tg.Rect.Y > 200 && tg.Rect.Y < 420, "unexpected target %v", tg.Rect)
242	-	}
243	-	}
244	-
245	-	// A single glyph with a flat bottom stroke (8, L) is not underlined text.
246	-	func TestUnderlineIgnoresSingleGlyph(t *testing.T) {
247	-	c := newCanvas(100, 60, 10, 204, 1)
248	-	w := c.word(10, 20, 10, false)
249	-	w.Text = "8"
250	-	c.underline(10, 20, 32)
251	-	require.Empty(t, scanUnder(c, w))
252	-	}

M internal/adapters/logging/logging.go +1 -1


@@ -22,7 +22,7 @@ import (
22		var components = map[string]bool{	22		var components = map[string]bool{
23		"core": true, "wayland": true, "input": true, "drm": true, "seat": true,	23		"core": true, "wayland": true, "input": true, "drm": true, "seat": true,
24		"render": true, "sync": true, "config": true, "app": true,	24		"render": true, "sync": true, "config": true, "app": true,
25	-	"screensaver": true, "powerkey": true, "sched": true, "screencast": true,	25	+	"screensaver": true, "powerkey": true, "sched": true, "screencast": true, "hints": true,
26		}	26		}
27			27
28		// categories are debug switches narrower than a component, only on when	28		// categories are debug switches narrower than a component, only on when

M internal/app/app.go +6 -6


@@ -84,7 +84,7 @@ const busRetry = 2 * time.Second
84		func relayConfig(ctx context.Context, cur ports.Config, in <-chan ports.ConfigChanged, out chan<- ports.ConfigChanged, keymaps chan xkb.Keymap, deviceConfigs chan ports.InputDevicesConfig, curs cursors, commands chan<- ports.ClientCommand, log zerowrap.Logger) {	84		func relayConfig(ctx context.Context, cur ports.Config, in <-chan ports.ConfigChanged, out chan<- ports.ConfigChanged, keymaps chan xkb.Keymap, deviceConfigs chan ports.InputDevicesConfig, curs cursors, commands chan<- ports.ClientCommand, log zerowrap.Logger) {
85		kb := cur.Keyboard	85		kb := cur.Keyboard
86		devCfg := cur.InputDevicesConfig	86		devCfg := cur.InputDevicesConfig
87	-	hintOn := cur.Hint.Enable	87	+	hint := cur.Hint
88		for {	88		for {
89		var ev ports.ConfigChanged	89		var ev ports.ConfigChanged
90		select {	90		select {

@@ -92,11 +92,11 @@ func relayConfig(ctx context.Context, cur ports.Config, in <-chan ports.ConfigCh
92		return	92		return
93		case ev = <-in:	93		case ev = <-in:
94		}	94		}
95	-	// hint.enable is read at startup: nothing of hint mode starts or	95	+	// hint.enable, hint.detector, hint.model and hint.threshold are read
96	-	// stops on a reload.	96	+	// at startup: nothing of hint mode starts or stops on a reload.
97	-	if ev.Config.Hint.Enable != hintOn {	97	+	if h := ev.Config.Hint; h.Enable != hint.Enable \|\| h.Detector != hint.Detector \|\| h.Model != hint.Model \|\| h.Threshold != hint.Threshold {
98	-	log.Info().Bool("running", hintOn).Msg("hint.enable changes apply after restart")	98	+	log.Info().Bool("running", hint.Enable).Msg("hint.enable, hint.detector, hint.model and hint.threshold apply after restart")
99	-	ev.Config.Hint.Enable = hintOn	99	+	ev.Config.Hint.Enable, ev.Config.Hint.Detector, ev.Config.Hint.Model, ev.Config.Hint.Threshold = hint.Enable, hint.Detector, hint.Model, hint.Threshold
100		}	100		}
101		next := ev.Config.Keyboard	101		next := ev.Config.Keyboard
102		layoutChanged := next.Layout != kb.Layout \|\| next.Variant != kb.Variant \|\| next.Options != kb.Options	102		layoutChanged := next.Layout != kb.Layout \|\| next.Variant != kb.Variant \|\| next.Options != kb.Options

R internal/app/hintdebug.go → internal/app/hint.go +7 -10 renamed, 66% similar

 	"os"
 	"path/filepath"
-	"github.com/bnema/neferwl/internal/adapters/atspi"
 	"github.com/bnema/neferwl/internal/adapters/hints"
 	"github.com/bnema/neferwl/internal/adapters/logging"
+	"github.com/bnema/neferwl/internal/ports"
 )
-// hintOptions builds the hint worker options: the accessibility source, and,
+// hintOptions builds the hint worker options from the configuration (read at
-// when NEFERWL_HINT_DEBUG is "1", the dump directory of scanned frames,
+// startup) and, when NEFERWL_HINT_DEBUG is "1", the dump directory of scanned
-// $XDG_STATE_HOME/neferwl/hints. The accessibility bus is only contacted
+// frames, $XDG_STATE_HOME/neferwl/hints. The detector process only starts at
-// by the first scan; without it the worker reads windows with OCR only.
+// the first scan.
-func hintOptions(ctx context.Context) hints.Options {
+func hintOptions(ctx context.Context, cfg ports.Config) hints.Options {
 	opts := hintDump(ctx)
-	// The source connects on its first scan, and again after a failure. The
+	opts.Command, opts.ModelDir, opts.Threshold = cfg.Hint.Detector, cfg.Hint.Model, cfg.Hint.Threshold
-	// hints worker owns it and closes it when it returns.
-	src := atspi.New()
-	opts.Source = src
 	return opts
 }

M internal/app/hintoptin_test.go +18 -2


@@ -45,7 +45,6 @@ func TestHintOffStartsNoHintGoroutine(t *testing.T) {
45		buf := make([]byte, 1<<20)	45		buf := make([]byte, 1<<20)
46		dump := string(buf[:runtime.Stack(buf, true)])	46		dump := string(buf[:runtime.Stack(buf, true)])
47		require.NotContains(t, dump, "adapters/hints.")	47		require.NotContains(t, dump, "adapters/hints.")
48	-	require.NotContains(t, dump, "adapters/atspi.")
49		require.NotContains(t, dump, "routeCaptured")	48		require.NotContains(t, dump, "routeCaptured")
50		select {	49		select {
51		case <-done:	50		case <-done:

@@ -64,5 +63,22 @@ func TestRelayConfigKeepsStartupHintEnable(t *testing.T) {
64		next := config.Defaults()	63		next := config.Defaults()
65		next.Hint.Enable = true	64		next.Hint.Enable = true
66		in <- ports.ConfigChanged{Config: next}	65		in <- ports.ConfigChanged{Config: next}
67	-	require.False(t, (<-out).Config.Hint.Enable)	66	+	got := (<-out).Config.Hint
			67	+	require.False(t, got.Enable)
			68	+	}
			69	+
			70	+	func TestRelayConfigKeepsStartupHintDetector(t *testing.T) {
			71	+	ctx, cancel := context.WithCancel(context.Background())
			72	+	defer cancel()
			73	+	cur := config.Defaults()
			74	+	in := make(chan ports.ConfigChanged, 1)
			75	+	out := make(chan ports.ConfigChanged, 1)
			76	+	go relayConfig(ctx, cur, in, out, make(chan *xkb.Keymap, 1), make(chan ports.InputDevicesConfig, 1), newCursors(clock.System{}, 0), make(chan ports.ClientCommand, 1), logging.For(ctx, "config"))
			77	+	next := config.Defaults()
			78	+	next.Hint.Detector, next.Hint.Model, next.Hint.Threshold = "/other/nh", "/other/models", 0.5
			79	+	in <- ports.ConfigChanged{Config: next}
			80	+	got := (<-out).Config.Hint
			81	+	require.Equal(t, cur.Hint.Detector, got.Detector)
			82	+	require.Equal(t, cur.Hint.Model, got.Model)
			83	+	require.Equal(t, cur.Hint.Threshold, got.Threshold)
68		}	84		}

M internal/app/session.go +2 -2


@@ -102,9 +102,9 @@ func assembleSession(ctx context.Context, opts Options) (s *session, err error)
102		return s, err	102		return s, err
103		}	103		}
104		runtimeDir := os.Getenv("XDG_RUNTIME_DIR")	104		runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
105	-	// Hint mode off: no accessibility connection, no debug env, nothing.	105	+	// Hint mode off: no detector, no debug env, nothing.
106		if opts.Config.Hint.Enable {	106		if opts.Config.Hint.Enable {
107	-	s.hintOpts = hintOptions(ctx)	107	+	s.hintOpts = hintOptions(ctx, opts.Config)
108		}	108		}
109		// Every output renders on the same GPU: its formats are the clients'.	109		// Every output renders on the same GPU: its formats are the clients'.
110		s.dmabuf = vulkan.Probe()	110		s.dmabuf = vulkan.Probe()

M internal/core/core.go +3 -0


@@ -353,6 +353,9 @@ func (c *Core) apply(cfg ports.Config) error {
353	for _, s := range c.screens {	353		for _, s := range c.screens {
354	axes[s.mon] = s.mon.Current().policy()	354		axes[s.mon] = s.mon.Current().policy()
355	}	355		}
		356	+	if c.cfg.Hint.Size != cfg.Hint.Size {
		357	+	c.hintCancel()
		358	+	}
356	c.cfg = cfg	359		c.cfg = cfg
357	c.binds = binds	360		c.binds = binds
358	c.cmdMod = map[string]ports.Mods{"super": ports.ModSuper, "alt": ports.ModAlt, "ctrl": ports.ModCtrl}[cfg.Keyboard.CmdKey]	361		c.cmdMod = map[string]ports.Mods{"super": ports.ModSuper, "alt": ports.ModAlt, "ctrl": ports.ModCtrl}[cfg.Keyboard.CmdKey]

M internal/core/hint.go +84 -35

 import (
 	"cmp"
 	"context"
+	"math"
 	"slices"
 	"github.com/bnema/neferwl/internal/ports"
 // coordinates of its centre.
 type hintTarget struct {
 	x, y   float64
+	score  float32
 	window WindowID
 	source hintSourceGeom // source window and its request-time client geometry
 	label  string
 	rect   ports.Rect
-	text   bool // found by OCR
 }
 type hintSourceGeom struct {
 	// one on screen ends it.
 	ws      *Workspace
 	geom    hintGeom
-	windows []ports.HintWindow // visible windows at request time
+	windows []hintSourceGeom // visible windows and their client geometry at request time
 	targets []hintTarget
 	typed   string
 }
 		return
 	}
 	wins := c.hintWindows(sc)
-	req := ports.HintScanRequest{Seq: c.hint.seq + 1, Output: sc.name(), Width: sc.info.Width, Height: sc.info.Height, Scale: sc.scale, LinksOnly: c.cfg.Hint.LinksOnly, Windows: wins}
+	req := ports.HintScanRequest{Seq: c.hint.seq + 1, Output: sc.name(), Width: sc.info.Width, Height: sc.info.Height, Scale: sc.scale}
 	select {
 	case c.ch.HintScans <- req:
 	default:
 }
 // hintWindows lists the windows drawn on sc, bottom to top, with their client
-// area (placements are output-local).
+// area (placements are output-local). A target is only clicked while the
-func (c *Core) hintWindows(sc *screen) []ports.HintWindow {
+// geometry of its window is the one seen here.
+func (c *Core) hintWindows(sc *screen) []hintSourceGeom {
 	view := sc.mon.viewport()
-	var out []ports.HintWindow
+	var out []hintSourceGeom
 	for _, p := range sc.shownLayout() {
 		if p.Hidden || p.Preview > 0 || !view.shows(p) {
 			continue
 		if r.W <= 0 || r.H <= 0 {
 			continue
 		}
-		w := ports.HintWindow{ID: p.ID, Rect: r, Focused: p.Focused}
+		out = append(out, hintSourceGeom{id: p.ID, rect: r})
-		rec := c.windows.lookup(p.ID)
-		w.PID, w.AppID = rec.client.PID, rec.client.AppID
-		out = append(out, w)
 	}
 	return out
 }
 			// A layer surface, or a window of another output.
 			continue
 		}
-		source, ok := c.hintSource(sc, t, root)
+		source, ok := c.hintSource(sc, root)
 		if !ok {
 			continue
 		}
-		kept = append(kept, hintTarget{x: x, y: y, window: id, source: source, rect: t.Rect, text: t.Text})
+		kept = append(kept, hintTarget{x: x, y: y, window: id, source: source, rect: t.Rect, score: t.Score})
 	}
 	if len(kept) == 0 {
 		c.hintCancel()
 		return true
 	}
+	kept = c.hintSpread(sc, kept)
 	slices.SortStableFunc(kept, func(a, b hintTarget) int {
 		return cmp.Or(cmp.Compare(a.rect.Y, b.rect.Y), cmp.Compare(a.rect.X, b.rect.X))
 	})
 	return true
 }
-// hintSource resolves the request-time source window only while its client
+// hintLabelSize estimates, in logical pixels, the box of a two-letter label on
-// geometry is unchanged. This applies equally to accessibility and OCR targets.
+// sc: a monospace face (advance 0.6 em) plus padding and outline, rounded up.
-func (c *Core) hintSource(sc *screen, t ports.HintTarget, root WindowID) (hintSourceGeom, bool) {
+// The font size follows the renderer: hint.size, or the output's density when
-	if t.Window != 0 && root != t.Window {
+// it is automatic (same rules as hintlabel.FontPixels, which core cannot
-		return hintSourceGeom{}, false
+// import).
+func (c *Core) hintLabelSize(sc *screen) (w, h int) {
+	scale := sc.scale
+	if scale <= 0 {
+		scale = 1
+	}
+	font := c.cfg.Hint.Size // logical
+	if font <= 0 {
+		px := 13 * scale * math.Sqrt(math.Max(1, float64(sc.info.Height)/(1080*scale)))
+		if sc.info.Width > 0 && sc.info.Height > 0 && sc.info.PhysicalW > 0 && sc.info.PhysicalH > 0 {
+			ratio := (float64(sc.info.Width) / float64(sc.info.Height)) / (float64(sc.info.PhysicalW) / float64(sc.info.PhysicalH))
+			if math.Abs(ratio-1) <= .2 {
+				px = math.Min(math.Round(40*scale), math.Max(math.Round(13*scale), math.Round(3.2*float64(sc.info.Width)/float64(sc.info.PhysicalW))))
+			}
+		}
+		font = px / scale
+	}
+	font = math.Min(font, 256/scale)
+	s := font / 13
+	return int(math.Ceil(2*0.6*font + 6*s + 2/scale + 1)), int(math.Ceil(1.25*font + 2*s + 2/scale + 1))
+}
+// hintLabelRect mirrors the renderer's outside top-left anchor and output
+// clamping, in output-local logical coordinates.
+func hintLabelRect(sc *screen, target ports.Rect, lw, lh int) ports.Rect {
+	bounds := sc.rect()
+	inset := int(math.Round(0.25 * float64(lh)))
+	return ports.Rect{X: min(max(target.X-lw+inset, 0), bounds.W-lw), Y: min(max(target.Y-lh+inset, 0), bounds.H-lh), W: lw, H: lh}
+}
+// hintSpread drops targets whose output-clamped label rectangles intersect.
+// Highest scores win; four times the label budget gives spreading headroom
+// while bounding the quadratic pass. hintlabel.FontPixels is the source of
+// truth for the font rule mirrored above.
+func (c *Core) hintSpread(sc *screen, ts []hintTarget) []hintTarget {
+	lw, lh := c.hintLabelSize(sc)
+	order := slices.Clone(ts)
+	slices.SortStableFunc(order, func(a, b hintTarget) int {
+		return cmp.Or(cmp.Compare(b.score, a.score), cmp.Compare(a.rect.Y, b.rect.Y), cmp.Compare(a.rect.X, b.rect.X))
+	})
+	if len(order) > 4*hintMaxTargets {
+		order = order[:4*hintMaxTargets]
+	}
+	kept := order[:0]
+	for _, t := range order {
+		clash := false
+		for _, k := range kept {
+			a, b := hintLabelRect(sc, t.rect, lw, lh), hintLabelRect(sc, k.rect, lw, lh)
+			if a.X < b.X+b.W && b.X < a.X+a.W && a.Y < b.Y+b.H && b.Y < a.Y+a.H {
+				clash = true
+				break
+			}
+		}
+		if !clash {
+			kept = append(kept, t)
+		}
 	}
+	return kept
+}
+// hintSource resolves the request-time source window only while its client
+// geometry is unchanged.
+func (c *Core) hintSource(sc *screen, root WindowID) (hintSourceGeom, bool) {
 	for _, w := range c.hint.windows {
-		if w.ID == root && c.hintSourceUnchanged(sc, hintSourceGeom{id: w.ID, rect: w.Rect}) {
+		if w.id == root && c.hintSourceUnchanged(sc, w) {
-			return hintSourceGeom{id: w.ID, rect: w.Rect}, true
+			return w, true
 		}
 	}
 	return hintSourceGeom{}, false
 // synthetic: c.synthetic keeps the physical modifiers (the bind's Cmd may
 // still be down) from turning it into a window drag, without touching c.mods.
 //
-// A target found by OCR is clicked with hint.text-click held (terminals open
+// The window gets the keyboard first.
-// a link on a modified click). Every click gives the window the keyboard
-// first, then sends exactly the modifiers to hold (none for other targets),
-// and the real modifier state is always sent back afterwards.
 func (c *Core) hintClick(ctx context.Context, t hintTarget) (err error) {
 	c.hintCancel()
 	if err := c.publish(ctx); err != nil {
 	if !c.hintClickable(t) {
 		return nil
 	}
-	// Every click states its modifiers (possibly none): the bind's Cmd may
-	// still be down and the client must not see it on a plain click.
 	if err := c.clickFocus(ctx, t.window); err != nil {
 		return err
 	}
-	if err := c.command(ctx, ports.SetModifiers{ID: t.window, Mods: c.hintMods(t)}); err != nil {
+	// The physical modifiers (bind's Super, Ctrl, Shift) must not reach the
+	// client: state none, and always send the real state back afterwards.
+	if err := c.command(ctx, ports.SetModifiers{ID: t.window, Mods: 0}); err != nil {
 		return err
 	}
 	defer func() {
 	return c.pointerButton(ctx, ports.PointerButton{Button: btnLeft, Time: c.motionTime})
 }
-// hintMods are the modifiers to hold for a click on t, read at click time so
-// a reloaded configuration applies.
-func (c *Core) hintMods(t hintTarget) ports.Mods {
-	if !t.text {
-		return 0
-	}
-	return c.cfg.Hint.TextClick
-}
 // hintClickable reports whether a click on t may go out now: unprotected,
 // pointer free, and the selected window is still under the target's centre.
 func (c *Core) hintClickable(t hintTarget) bool {

A internal/core/hint_internal_test.go +45 -0

@@ -0,0 +1,45 @@
	1	+	package core
	2	+
	3	+	import (
	4	+	"github.com/bnema/neferwl/internal/ports"
	5	+	"github.com/stretchr/testify/require"
	6	+	"math"
	7	+	"testing"
	8	+	)
	9	+
	10	+	func TestHintLabelGeometry(t *testing.T) {
	11	+	for _, scale := range []float64{1, 1.5} {
	12	+	sc := &screen{scale: scale, info: ports.OutputInfo{Width: 1500, Height: 1200}}
	13	+	sc.mon = newMonitor("", "")
	14	+	sc.mon.SetOutput(int(float64(sc.info.Width)/sc.scale), int(float64(sc.info.Height)/sc.scale))
	15	+	c := &Core{}
	16	+	c.cfg.Hint.Size = 200
	17	+	w, h := c.hintLabelSize(sc)
	18	+	font := math.Min(200, 256/scale)
	19	+	require.Equal(t, int(math.Ceil(1.2font+6font/13+2/scale+1)), w)
	20	+	require.Equal(t, int(math.Ceil(1.25font+2font/13+2/scale+1)), h)
	21	+	bounds := sc.rect()
	22	+	// Corner separation exceeds label width, but right-edge clamping folds
	23	+	// the second label back over the first.
	24	+	a := hintTarget{rect: ports.Rect{X: bounds.W - w - 1, Y: bounds.H - h - 1}, score: 1}
	25	+	b := hintTarget{rect: ports.Rect{X: bounds.W - 1, Y: bounds.H - 1}, score: .5}
	26	+	require.Len(t, c.hintSpread(sc, []hintTarget{a, b}), 1)
	27	+	b.rect = ports.Rect{X: 0, Y: 0}
	28	+	require.Len(t, c.hintSpread(sc, []hintTarget{a, b}), 2)
	29	+	}
	30	+	}
	31	+
	32	+	func TestHintSpreadBudget(t *testing.T) {
	33	+	sc := &screen{scale: 1, info: ports.OutputInfo{Width: 1000000, Height: 1000}}
	34	+	sc.mon = newMonitor("", "")
	35	+	sc.mon.SetOutput(int(float64(sc.info.Width)/sc.scale), int(float64(sc.info.Height)/sc.scale))
	36	+	c := &Core{}
	37	+	ts := make([]hintTarget, 4*hintMaxTargets+10)
	38	+	for i := range ts {
	39	+	ts[i] = hintTarget{rect: ports.Rect{X: 100 + i*100, Y: 100}, score: float32(i)}
	40	+	}
	41	+	got := c.hintSpread(sc, ts)
	42	+	require.Len(t, got, 4*hintMaxTargets)
	43	+	require.Equal(t, float32(len(ts)-1), got[0].score)
	44	+	require.Equal(t, float32(10), got[len(got)-1].score)
	45	+	}

M internal/core/hint_test.go +92 -124

 	hints    chan ports.HintScanResult
 	win      ports.Rect // window 1, output-local
 	// state is the security state input is stamped with.
-	state ports.SecurityState
+	state  ports.SecurityState
+	config chan ports.ConfigChanged
+	cfg    ports.Config
 }
 // newHint starts a core with one 1000x800 output "OUT-1" and window 1 mapped.
 func newHint(t *testing.T, opts core.Options) *hintHarness {
 	t.Helper()
-	return newHintMods(t, opts, 0)
+	return newHintConfig(t, opts, func(*ports.Config) {})
-}
-// newHintMods is newHint with hint.text-click set to textClick.
-func newHintMods(t *testing.T, opts core.Options, textClick ports.Mods) *hintHarness {
-	t.Helper()
-	return newHintConfig(t, opts, func(cfg *ports.Config) { cfg.Hint.TextClick = textClick })
 }
 // newHintConfig is newHint with the config edited by edit.
 	cfg.Layout.Gaps = 20
 	cfg.Binds["Cmd+g"] = "hint"
 	h := &hintHarness{
+		config:   make(chan ports.ConfigChanged),
+		cfg:      cfg,
 		t:        t,
 		input:    make(chan ports.InputEvent, 32),
 		client:   make(chan ports.ClientEvent, 32),
 		// Unbuffered: a send returns once core took the result.
 		hints: make(chan ports.HintScanResult),
 	}
-	ch := core.Channels{Client: h.client, Input: h.input, Output: h.output, Commands: h.commands, Scenes: h.scenes, HintScans: h.scans, Hints: h.hints}
+	ch := core.Channels{Config: h.config, Client: h.client, Input: h.input, Output: h.output, Commands: h.commands, Scenes: h.scenes, HintScans: h.scans, Hints: h.hints}
 	c, err := core.New(cfg, ch, opts)
 	if err != nil {
 		t.Fatal(err)
 // inside is a target of the given size inside window 1.
 func (h *hintHarness) inside(i int) ports.HintTarget {
-	return ports.HintTarget{Rect: ports.Rect{X: h.win.X + 10 + i, Y: h.win.Y + 10 + 12*i, W: 40, H: 10}}
+	return ports.HintTarget{Rect: ports.Rect{X: h.win.X + 10 + 100*(i%4), Y: h.win.Y + 10 + 40*(i/4), W: 40, H: 10}}
 }
 func TestHintRequestOnBind(t *testing.T) {
 	h := newHint(t, core.Options{})
 	req := h.start()
-	if req.Output != "OUT-1" || req.Width != 1000 || req.Height != 800 || req.Scale != 1 || req.LinksOnly || req.Seq == 0 {
+	if req.Output != "OUT-1" || req.Width != 1000 || req.Height != 800 || req.Scale != 1 || req.Seq == 0 {
 		t.Fatalf("request %+v", req)
 	}
-	want := []ports.HintWindow{{ID: 1, PID: 4242, AppID: "org.test.app", Rect: h.win, Focused: true}}
-	if !slices.Equal(req.Windows, want) {
-		t.Fatalf("windows %+v, want %+v", req.Windows, want)
-	}
 	// A second bind while pending does not ask again; the pending phase
 	// cancels on any key.
 	h.press("g", ports.ModSuper)
 	h := newHint(t, core.Options{})
 	var targets []ports.HintTarget
 	for i := 0; i < 27; i++ {
-		targets = append(targets, ports.HintTarget{Rect: ports.Rect{X: h.win.X + 5, Y: h.win.Y + 5 + 14*i, W: 30, H: 10}})
+		targets = append(targets, ports.HintTarget{Rect: ports.Rect{X: h.win.X + 5 + 100*(i%4), Y: h.win.Y + 5 + 40*(i/4), W: 30, H: 10}})
 	}
 	s := h.show(targets...)
 	if len(s.HintLabels) != 27 {
 	h := newHint(t, core.Options{})
 	var targets []ports.HintTarget
 	for i := 0; i < 27; i++ {
-		targets = append(targets, ports.HintTarget{Rect: ports.Rect{X: h.win.X + 5, Y: h.win.Y + 5 + 14*i, W: 30, H: 10}})
+		targets = append(targets, ports.HintTarget{Rect: ports.Rect{X: h.win.X + 5 + 100*(i%4), Y: h.win.Y + 5 + 40*(i/4), W: 30, H: 10}})
 	}
 	h.show(targets...)
 	h.press("f", 0)
 	h.drain()
 	h.press("f", ports.ModSuper) // Super is still down
 	h.labelled(0, 0)
-	presses, releases := h.buttonsSent()
+	cmds := h.barrier()
+	if got, want := commandKinds(cmds), []string{"mods:0", "motion", "press", "release", "mods:restore"}; !slices.Equal(got, want) {
+		t.Fatalf("commands %v, want %v", got, want)
+	}
+	presses, releases := 0, 0
+	for _, c := range cmds {
+		if b, ok := c.(ports.PointerButtonTo); ok {
+			if b.Pressed {
+				presses++
+			} else {
+				releases++
+			}
+		}
+	}
 	if presses != 1 || releases != 1 {
 		t.Fatalf("client got %d presses, %d releases", presses, releases)
 	}
 }
 // commandsSent drains the commands up to a barrier, keeping the ones that
-// matter for a click: modifiers, focus, motion and buttons.
+// matter for a click: focus, motion and buttons.
 func (h *hintHarness) commandsSent() []ports.ClientCommand {
 	var out []ports.ClientCommand
 	for _, cmd := range h.barrier() {
 		switch cmd.(type) {
-		case ports.SetModifiers, ports.FocusWindow, ports.PointerMotionTo, ports.PointerButtonTo:
+		case ports.FocusWindow, ports.SetModifiers, ports.PointerMotionTo, ports.PointerButtonTo:
 			out = append(out, cmd)
 		}
 	}
 	var out []string
 	for _, c := range cmds {
 		switch v := c.(type) {
+		case ports.FocusWindow:
+			out = append(out, fmt.Sprintf("focus:%d", v.ID))
 		case ports.SetModifiers:
 			if v.Restore {
-				out = append(out, "restore")
+				out = append(out, "mods:restore")
 			} else {
 				out = append(out, fmt.Sprintf("mods:%d", v.Mods))
 			}
-		case ports.FocusWindow:
-			out = append(out, fmt.Sprintf("focus:%d", v.ID))
 		case ports.PointerMotionTo:
 			out = append(out, "motion")
 		case ports.PointerButtonTo:
 	return out
 }
-// An OCR target is clicked with the configured modifiers held: they go out
+// A click on the focused window moves the pointer to the target and presses
-// before the motion and are restored after the release.
+// and releases the left button; it states no modifiers.
-func TestHintTextClickHoldsModifiers(t *testing.T) {
+func TestHintClickSequence(t *testing.T) {
-	mods := ports.ModCtrl | ports.ModShift
+	h := newHint(t, core.Options{})
-	h := newHintMods(t, core.Options{}, mods)
+	h.show(h.inside(0))
-	tg := h.inside(0)
-	tg.Text = true
-	h.show(tg)
 	h.drain()
 	h.press("f", 0)
 	got := commandKinds(h.commandsSent())
-	want := []string{fmt.Sprintf("mods:%d", mods), "motion", "press", "release", "restore"}
+	want := []string{"mods:0", "motion", "press", "release", "mods:restore"}
 	if !slices.Equal(got, want) {
 		t.Fatalf("commands %v, want %v", got, want)
 	}
 }
-// Without text targets or without modifiers the click states no modifiers
+// A window that moved while the scan ran: a target over it is dropped.
-// (depressed 0) rather than leave the held bind modifier to the client.
+func TestHintResultDroppedWhenWindowMoved(t *testing.T) {
-func TestHintClickWithoutModifiers(t *testing.T) {
-	for name, c := range map[string]struct {
-		mods ports.Mods
-		text bool
-	}{"accessibility target": {ports.ModCtrl, false}, "none configured": {0, true}} {
-		t.Run(name, func(t *testing.T) {
-			h := newHintMods(t, core.Options{}, c.mods)
-			tg := h.inside(0)
-			tg.Text = c.text
-			h.show(tg)
-			h.drain()
-			h.press("f", 0)
-			got := commandKinds(h.commandsSent())
-			if want := []string{"mods:0", "motion", "press", "release", "restore"}; !slices.Equal(got, want) {
-				t.Fatalf("commands %v, want %v", got, want)
-			}
-		})
-	}
-}
-// The bind's Super is still physically held: an accessibility click states
-// depressed 0, and the restore carries the real state of the latest key,
-// swallowed ones included (a latch a label key consumed).
-func TestHintClickHeldBindModifierAndLatchRestore(t *testing.T) {
-	h := newHint(t, core.Options{})
-	tg := h.inside(0)
-	h.press("g", ports.ModSuper)
-	req := receive(t, h.scans)
-	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{tg, h.inside(1)}}
-	h.labelled(2, 0)
-	h.drain()
-	real := ports.ModState{Depressed: 64, Latched: 0, Locked: 2}
-	h.send(ports.KeyEvent{Keysym: "f", Base: "f", Mods: ports.ModSuper, Pressed: true, State: real})
-	cmds := h.commandsSent()
-	var restore *ports.SetModifiers
-	for i, c := range cmds {
-		if v, ok := c.(ports.SetModifiers); ok {
-			if i == 0 && (v.Restore || v.Mods != 0) {
-				t.Fatalf("first modifiers %+v, want depressed 0", v)
-			}
-			if v.Restore {
-				restore = &v
-			}
-		}
-	}
-	if restore == nil || restore.State != real {
-		t.Fatalf("restore %+v, want state %+v (commands %v)", restore, real, commandKinds(cmds))
-	}
-}
-// A result is kept only for the window it was scanned on.
-func TestHintResultWindowChecks(t *testing.T) {
-	h := newHint(t, core.Options{})
-	req := h.start()
-	// Accessibility target naming another window: dropped; naming window 1: kept.
-	ok := h.inside(1)
-	ok.Window = 1
-	bad := h.inside(0)
-	bad.Window = 7
-	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{bad, ok}}
-	s := h.labelled(1, 0)
-	if len(s.HintLabels) != 1 || s.HintLabels[0].Rect.Y != ok.Rect.Y {
-		t.Fatalf("labels %+v", s.HintLabels)
-	}
-}
-// A window that moved while the scan ran: an OCR target over it is dropped.
-func TestHintOCRResultDroppedWhenWindowMoved(t *testing.T) {
 	h := newHint(t, core.Options{})
 	req := h.start()
 	tg := h.inside(0)
-	tg.Text = true
 	h.client <- ports.WindowMapped{ID: 2}
 	h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 })
 	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{tg}}
 }
 // A new window covering the target (not in the snapshot) drops it as well.
-func TestHintOCRResultDroppedForUnseenWindow(t *testing.T) {
+func TestHintResultDroppedForUnseenWindow(t *testing.T) {
 	h := newHint(t, core.Options{})
 	req := h.start()
 	h.client <- ports.WindowMapped{ID: 2}
 	s := h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 })
-	tg := ports.HintTarget{Text: true}
+	var tg ports.HintTarget
 	for _, w := range s.Windows {
 		if w.ID == 2 {
 			tg.Rect = ports.Rect{X: w.Rect.X + 5, Y: w.Rect.Y + 5, W: 20, H: 10}
 	}
 }
-// A window that does not have the keyboard gets it before the modifiers.
+// A window that does not have the keyboard gets it before the click.
-func TestHintTextClickFocusesWindowFirst(t *testing.T) {
+func TestHintClickFocusesWindowFirst(t *testing.T) {
-	h := newHintMods(t, core.Options{}, ports.ModCtrl)
+	h := newHint(t, core.Options{})
 	h.client <- ports.WindowMapped{ID: 2}
 	s := h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 })
 	var r ports.Rect
 		}
 	}
 	h.win = r
-	tg := h.inside(0)
+	h.show(h.inside(0))
-	tg.Text = true
-	h.show(tg)
 	h.drain()
 	h.press("f", 0)
 	got := commandKinds(h.commandsSent())
-	want := []string{"focus:1", fmt.Sprintf("mods:%d", ports.ModCtrl), "motion", "press", "release", "restore"}
+	want := []string{"focus:1", "mods:0", "motion", "press", "release", "mods:restore"}
 	if !slices.Equal(got, want) {
 		t.Fatalf("commands %v, want %v", got, want)
 	}
 }
-// A click that is no longer allowed sends no modifiers at all.
+// A click that is no longer allowed sends no command at all.
-func TestHintTextClickNoModifiersWhenNotClickable(t *testing.T) {
+func TestHintClickNothingWhenNotClickable(t *testing.T) {
-	h := newHintMods(t, core.Options{}, ports.ModCtrl)
+	h := newHint(t, core.Options{})
-	tg := h.inside(0)
+	h.show(h.inside(0))
-	tg.Text = true
-	h.show(tg)
 	h.constrain(ports.ConstraintLock, ports.Rect{})
 	h.drain()
 	h.press("f", 0)
 // Resizing window 1 leaves the old target centre inside it. Identity alone
 // therefore cannot protect either admission or the eventual synthetic click.
 func TestHintRejectsResizedSource(t *testing.T) {
-	for _, accessibility := range []bool{false, true} {
+	{
 		for _, beforeResult := range []bool{false, true} {
-			t.Run(fmt.Sprintf("accessibility=%v/beforeResult=%v", accessibility, beforeResult), func(t *testing.T) {
+			t.Run(fmt.Sprintf("beforeResult=%v", beforeResult), func(t *testing.T) {
 				h := newHint(t, core.Options{})
 				tg := h.inside(0)
-				tg.Text = !accessibility
-				if accessibility {
-					tg.Window = 1
-				}
 				req := h.start()
 				if !beforeResult {
 					h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{tg}}
 		}
 	}
 }
+// Labels hang off the targets' top-left corners: targets whose corners are
+// closer than a label drop, the lowest score first, whatever the input order.
+func TestHintOverlappingLabelsDropLowerScore(t *testing.T) {
+	at := func(h *hintHarness, dx, dy int, score float32) ports.HintTarget {
+		return ports.HintTarget{Rect: ports.Rect{X: h.win.X + 20 + dx, Y: h.win.Y + 20 + dy, W: 80, H: 30}, Score: score}
+	}
+	for _, rev := range []bool{false, true} {
+		h := newHint(t, core.Options{})
+		ts := []ports.HintTarget{
+			at(h, 0, 0, 0.4),    // clashes with the next one, loses
+			at(h, 3, 2, 0.9),    // wins
+			at(h, 300, 0, 0.1),  // far away: kept, whatever its score
+			at(h, 0, 200, 0.5),  // far below
+			at(h, 4, 200, 0.45), // clashes with the previous one
+		}
+		if rev {
+			slices.Reverse(ts)
+		}
+		req := h.start()
+		h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: ts}
+		s := h.labelled(3, 0)
+		var got [][2]int
+		for _, l := range s.HintLabels {
+			got = append(got, [2]int{l.Rect.X - h.win.X - 20, l.Rect.Y - h.win.Y - 20})
+		}
+		want := [][2]int{{300, 0}, {3, 2}, {0, 200}} // sorted by Y, then X
+		if !slices.Equal(got, want) {
+			t.Fatalf("rev=%v: label corners %v, want %v", rev, got, want)
+		}
+	}
+}
+func TestHintSizeReloadCancelsShownHints(t *testing.T) {
+	h := newHint(t, core.Options{})
+	h.show(h.inside(0))
+	cfg := h.cfg
+	cfg.Hint.Size = 40
+	h.config <- ports.ConfigChanged{Config: cfg}
+	h.until(func(s ports.Scene) bool { return s.HintStyle.Size == 40 && len(s.HintLabels) == 0 })
+}

M internal/ports/config.go +8 -6

 		// Enable turns the experimental hint mode on (hint.enable). It is
 		// read at startup: a reload keeps the startup value.
 		Enable bool
-		// LinksOnly limits the targets to links and URLs; false also
-		// includes buttons (hint.targets = clickable).
-		LinksOnly bool
-		// TextClick are the modifiers held for a click on a link found by
-		// OCR (hint.text-click); zero sends a plain click.
-		TextClick Mods
 		// Capture shows the labels in the captures requested by Wayland
 		// clients (hint.capture); the compositor's own scans never see them.
 		Capture                bool
 		Size                   float64 // logical pixels; zero selects automatic density sizing
 		Background, Foreground string  // empty derives from Border.Active
+		// Detector is the nefer-hints binary (hint.detector): a path, or a
+		// name looked up in PATH. Model is the directory of its model files
+		// (hint.model), with a leading ~/ already expanded. Threshold is the
+		// minimum detection score (hint.threshold, 0.01 to 0.99); zero leaves
+		// it unset, using model.json or the runner default. All three
+		// are read at startup.
+		Detector, Model string
+		Threshold       float64
 	}
 	// Workspaces are declared with workspace.<name>.* keys, in first-seen order.
 	Workspaces []WorkspaceConfig

M internal/ports/hint_messages.go +2 -31

@@ -1,15 +1,5 @@
1		package ports	1		package ports
2			2
3	-	// HintKind is the kind of a clickable hint target.
4	-	type HintKind uint8
5	-
6	-	const (
7	-	// HintLink is a link or URL.
8	-	HintLink HintKind = iota
9	-	// HintButton is a button or control.
10	-	HintButton
11	-	)
12	-
13		// HintScanRequest asks the hint scanner to find clickable targets in the last	3		// HintScanRequest asks the hint scanner to find clickable targets in the last
14		// frame of Output. Core sends it and expects one HintScanResult echoing Seq.	4		// frame of Output. Core sends it and expects one HintScanResult echoing Seq.
15		type HintScanRequest struct {	5		type HintScanRequest struct {

@@ -19,31 +9,12 @@ type HintScanRequest struct {
19		// (OutputInfo.Width/Height).	9		// (OutputInfo.Width/Height).
20		Width, Height int	10		Width, Height int
21		Scale float64 // screen scale	11		Scale float64 // screen scale
22	-	LinksOnly bool
23	-	// Windows are the windows visible on Output, for sources that read a
24	-	// toolkit's accessibility tree. Core owns the slice per request.
25	-	Windows []HintWindow
26	-	}
27	-
28	-	// HintWindow is a window visible on the hint output.
29	-	type HintWindow struct {
30	-	ID WindowID
31	-	PID int
32	-	AppID string
33	-	// Rect is the client area, logical output-local.
34	-	Rect Rect
35	-	Focused bool
36		}	12		}
37			13
38		// HintTarget is a clickable region found in a frame.	14		// HintTarget is a clickable region found in a frame.
39		type HintTarget struct {	15		type HintTarget struct {
40	-	Rect Rect // logical, output-local	16	+	Rect Rect // logical, output-local
41	-	Kind HintKind	17	+	Score float32 // detector confidence, 0 to 1
42	-	// Text is set for targets found by OCR (read from pixels, not from an
43	-	// accessibility tree).
44	-	Text bool
45	-	// Window is the source window of an accessibility target, 0 for OCR.
46	-	Window WindowID
47		}	18		}
48			19
49		// HintScanResult answers a HintScanRequest. Core owns Targets.	20		// HintScanResult answers a HintScanRequest. Core owns Targets.