feat(hints): add experimental keyboard hint mode (#149)

* feat(ports): add hint scan, modifier override and hint config messages

* feat(core): add keyboard hint mode

* feat(hintlabel): rasterize hint labels sized and coloured for the output

* feat(vulkan): draw hint labels and keep them out of captures

* feat(wayland): apply temporary modifier overrides for hint clicks

* feat(config): add hint mode keys

* feat(atspi): find hint targets in the accessibility tree

* feat(hints): find hint targets with the accessibility tree and OCR

* feat(app): wire hint mode when hint.enable is on

* docs(config): document the experimental hint mode

commit: 353b4e351f6beb5e2901ea31c47e2377239169b2
author: bnema · 3d ago · 2026-10-19 18:32 UTC
committer: GitHub · 3d ago · 2026-10-19 18:32 UTC
parent: 9793569
signature: GPG UnverifiedB5690EEEBB952194 The key is unknown
checks: 1/1

75 files · +7747 -37

M .hexcheck.yaml +1 -1


@@ -25,7 +25,7 @@ allow:
25		ruleSettings:	25		ruleSettings:
26		missing-generated-mock-for-port:	26		missing-generated-mock-for-port:
27		# Sealed message unions are channel data, not ports; tested by sending messages (ADR 005).	27		# Sealed message unions are channel data, not ports; tested by sending messages (ADR 005).
28	-	excludePaths: [internal/ports/client_events.go, internal/ports/client_commands.go, internal/ports/input_events.go, internal/ports/output_events.go, internal/ports/lease.go, internal/ports/session_lock_messages.go, internal/ports/screencast_messages.go, internal/ports/session_backend.go]	28	+	excludePaths: [internal/ports/client_events.go, internal/ports/client_commands.go, internal/ports/input_events.go, internal/ports/output_events.go, internal/ports/lease.go, internal/ports/session_lock_messages.go, internal/ports/screencast_messages.go, internal/ports/session_backend.go, internal/ports/hint_messages.go]
29		suspicious-business-logic-in-adapter:	29		suspicious-business-logic-in-adapter:
30		# app is the composition root; starting core and wiring channels is its job.	30		# app is the composition root; starting core and wiring channels is its job.
31		excludePaths: [internal/app/app.go, internal/app/session.go]	31		excludePaths: [internal/app/app.go, internal/app/session.go]

M .mockery.yml +19 -0


@@ -111,6 +111,25 @@ packages:
111	dir: internal/adapters/statefile	111		dir: internal/adapters/statefile
112	pkgname: statefile	112		pkgname: statefile
113	filename: proc_tree_mock_test.go	113		filename: proc_tree_mock_test.go
		114	+	github.com/bnema/neferwl/internal/adapters/hints:
		115	+	interfaces:
		116	+	targetSource:
		117	+	config:
		118	+	dir: internal/adapters/hints
		119	+	pkgname: hints
		120	+	filename: target_source_mock_test.go
		121	+	github.com/bnema/neferwl/internal/adapters/atspi:
		122	+	interfaces:
		123	+	transport:
		124	+	config:
		125	+	dir: internal/adapters/atspi
		126	+	pkgname: atspi
		127	+	filename: transport_mock_test.go
		128	+	connector:
		129	+	config:
		130	+	dir: internal/adapters/atspi
		131	+	pkgname: atspi
		132	+	filename: connector_mock_test.go
114	github.com/bnema/neferwl/internal/adapters/screencast:	133		github.com/bnema/neferwl/internal/adapters/screencast:
115	interfaces:	134		interfaces:
116	bus:	135		bus:

M Makefile +1 -1


@@ -61,7 +61,7 @@ arch: adapter-imports
61		# Adapters stay independent: production code may import only the shared	61		# Adapters stay independent: production code may import only the shared
62		# adapter libraries below. outputkit holds what both output backends (drm,	62		# adapter libraries below. outputkit holds what both output backends (drm,
63		# headless) run; the others are leaf helpers.	63		# headless) run; the others are leaf helpers.
64	-	ADAPTER_LIBS := fdwait\|busretry\|outputkit/capture\|outputkit/corewake\|outputkit/presented\|outputkit/surfaces\|captureallow\|clock\|logging\|peerexe\|sessionsecurity\|syncfile\|workspaceid\|xkb	64	+	ADAPTER_LIBS := fdwait\|busretry\|hintlabel\|outputkit/capture\|outputkit/corewake\|outputkit/presented\|outputkit/surfaces\|captureallow\|clock\|logging\|peerexe\|sessionsecurity\|syncfile\|workspaceid\|xkb
65		adapter-imports:	65		adapter-imports:
66		@edges=$$(CGO_ENABLED=0 go list -f '{{$$p := .ImportPath}}{{range .Imports}}{{$$p}} {{.}}{{"\n"}}{{end}}' ./internal/adapters/...) \|\| exit 1; \	66		@edges=$$(CGO_ENABLED=0 go list -f '{{$$p := .ImportPath}}{{range .Imports}}{{$$p}} {{.}}{{"\n"}}{{end}}' ./internal/adapters/...) \|\| exit 1; \
67		bad=$$(echo "$$edges" \| grep -E ' \S+/internal/adapters/' \| grep -vE '/internal/adapters/($(ADAPTER_LIBS))$$' \	67		bad=$$(echo "$$edges" \| grep -E ' \S+/internal/adapters/' \| grep -vE '/internal/adapters/($(ADAPTER_LIBS))$$' \

M README.md +1 -0


@@ -281,6 +281,7 @@ bind.cmd+shift+f = toggle-fullscreen
281	bind.cmd+s = toggle-stash-visible	281		bind.cmd+s = toggle-stash-visible
282	bind.cmd+shift+s = toggle-window-stash	282		bind.cmd+shift+s = toggle-window-stash
283	bind.cmd+o = toggle-overview	283		bind.cmd+o = toggle-overview
		284	+	# bind.cmd+g = hint
284	bind.cmd+tab = switch-column-next	285		bind.cmd+tab = switch-column-next
285	bind.cmd+shift+tab = switch-column-prev	286		bind.cmd+shift+tab = switch-column-prev
286		287

M docs/config.md +50 -0


@@ -55,6 +55,12 @@ See [examples/config](../examples/config) for every key with its default.
55	\| `focus.flash-opacity` \| `0.6` \| Starting opacity of the flash, `0.1` to `1` \|	55		\| `focus.flash-opacity` \| `0.6` \| Starting opacity of the flash, `0.1` to `1` \|
56	\| `focus.flash-duration` \| `200ms` \| Fade duration, `50ms` to `1s`; not scaled by `animations.speed` \|	56		\| `focus.flash-duration` \| `200ms` \| Fade duration, `50ms` to `1s`; not scaled by `animations.speed` \|
57	\| `focus.flash-color` \| `border.active` \| Flash color `#rrggbb`; empty uses `border.active` \|	57		\| `focus.flash-color` \| `border.active` \| Flash color `#rrggbb`; empty uses `border.active` \|
		58	+	\| `hint.enable` \| `off` \| Turn on the experimental [hint mode](#hint-mode) (restart required) \|
		59	+	\| `hint.targets` \| `clickable` \| What [hint mode](#hint-mode) labels: `clickable` (links, URLs and buttons) or `links` (links and URLs only) \|
		60	+	\| `hint.text-click` \| `ctrl+shift` \| Modifiers held for clicks on links found by OCR (terminals open links on a modified click); `none` sends a plain click \|
		61	+	\| `hint.size` \| `auto` \| Label text size in logical pixels (6 to 72), or `auto` for screen pixel density \|
		62	+	\| `hint.background` \| empty \| Label background (`#rrggbb`); empty uses `border.active` \|
		63	+	\| `hint.foreground` \| empty \| Label text (`#rrggbb`); empty chooses a contrasting colour \|
58	\| `workspace.<name>.monitor` \| focused output \| Home monitor of a named workspace \|	64		\| `workspace.<name>.monitor` \| focused output \| Home monitor of a named workspace \|
59	\| `workspace.<name>.size` \| `inherit` \| [Smaller viewport](#workspace-size), e.g. `1920x1080` \|	65		\| `workspace.<name>.size` \| `inherit` \| [Smaller viewport](#workspace-size), e.g. `1920x1080` \|
60	\| `workspace.<name>.max-columns`, `.overflow` \| output values \| Per-workspace layout \|	66		\| `workspace.<name>.max-columns`, `.overflow` \| output values \| Per-workspace layout \|

@@ -191,6 +197,7 @@ bind.XF86MonBrightnessDown = spawn brightnessctl set 5%-
191	\| `toggle-window-stash` \| Move the focused window to the [stash](#stash), or back to its column \|	197		\| `toggle-window-stash` \| Move the focused window to the [stash](#stash), or back to its column \|
192	\| `toggle-stash-visible` \| Show or hide the stash \|	198		\| `toggle-stash-visible` \| Show or hide the stash \|
193	\| `toggle-overview` \| Open or close the [overview](#overview) \|	199		\| `toggle-overview` \| Open or close the [overview](#overview) \|
		200	+	\| `hint` \| Label the links and buttons on the focused screen, then click the one you type ([hint mode](#hint-mode)) \|
194	\| `switch-column-next` / `-prev` \| Switch to the previously used column. Hold `cmd` and press again to go further back \|	201		\| `switch-column-next` / `-prev` \| Switch to the previously used column. Hold `cmd` and press again to go further back \|
195	\| `maximize-column` \| Toggle full width for the focused column \|	202		\| `maximize-column` \| Toggle full width for the focused column \|
196	\| `cycle-column-width` \| Grow the focused column by one [cell](#column-widths), then back to equal widths \|	203		\| `cycle-column-width` \| Grow the focused column by one [cell](#column-widths), then back to equal widths \|

@@ -295,3 +302,46 @@ A workspace's stash shows as a pile of cards. Selecting the pile unfolds it into
295	With `layout.overflow = cascade`, workspaces sit side by side, each band of columns is a lane, and the stash sits under the lanes. Down (or fingers down on the touchpad, with natural scroll off) moves to the lane below; past the last lane it unfolds the stash. Left/right walk the cards; where the carousel does not wrap round, they go to the neighbor workspace past either end. Up leaves the stash for the last lane, at the column you came from.	302		With `layout.overflow = cascade`, workspaces sit side by side, each band of columns is a lane, and the stash sits under the lanes. Down (or fingers down on the touchpad, with natural scroll off) moves to the lane below; past the last lane it unfolds the stash. Left/right walk the cards; where the carousel does not wrap round, they go to the neighbor workspace past either end. Up leaves the stash for the last lane, at the column you came from.
296		303
297	Otherwise, left from the first column unfolds the stash. Up/down walk the cards, the focus-workspace binds change workspace, and right goes back to the columns.	304		Otherwise, left from the first column unfolds the stash. Up/down walk the cards, the focus-workspace binds change workspace, and right goes back to the columns.
		305	+
		306	+	## Hint mode
		307	+
		308	+	Hint mode is experimental and off by default: set `hint.enable = on` and restart NeferWL.
		309	+	While it is off, nothing of hint mode runs or uses memory, and the `hint` bind
		310	+	logs a notice once and does nothing. A reload does not change it.
		311	+
		312	+	`hint` labels the links and buttons on the focused output. Type a label to
		313	+	click its target; BackSpace erases a letter, Escape cancels. No bind is set
		314	+	by default; for example `bind.cmd+g = hint`.
		315	+
		316	+	Labels use the `border.active` colour with a contrasting text colour, and
		317	+	their size follows the screen's pixel density. Each label sits just outside
		318	+	the top-left corner of its target. Links found by OCR (terminal text) are
		319	+	clicked with the `hint.text-click` modifiers held, which is how terminals open
		320	+	a link; accessibility targets get a plain click.
		321	+
		322	+	### Where targets come from
		323	+
		324	+	\| Source \| Apps \| Accuracy \| Speed \|
		325	+	\|---\|---\|---\|---\|
		326	+	\| Accessibility tree (AT-SPI) \| Firefox, Chromium, GTK, Qt, Electron \| Exact: the app reports its own links and buttons \| A few milliseconds \|
		327	+	\| Text recognition (OCR) \| Everything else: terminals, apps without accessibility \| Best effort: URLs, domains and underlined text written on screen \| 150 ms or more, more on large screens \|
		328	+
		329	+	NeferWL asks the accessibility tree first. Windows it does not cover are read
		330	+	with OCR, which finds visible URLs, domains, email addresses and underlined
		331	+	text. Underlined words get a plain click, and the words of one continuous
		332	+	underline form one target; URLs keep the `hint.text-click` modifiers. OCR
		333	+	cannot see a link whose address is hidden behind plain text, and it can misread
		334	+	small or low-contrast text.
		335	+
		336	+	### Requirements
		337	+
		338	+	- Accessibility: NeferWL turns it on at startup. Some apps need a flag:
		339	+	- Chromium: `--force-renderer-accessibility`
		340	+	- Electron: `ACCESSIBILITY_ENABLED=1`
		341	+	- Qt: `QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1`
		342	+	- OCR: `tesseract` and `tesseract-data-eng`. Without tesseract, OCR is off
		343	+	and hint mode labels only apps that expose the accessibility tree.
		344	+
		345	+	### Debugging hint mode
		346	+
		347	+	With `hint.enable = on` and `NEFERWL_HINT_DEBUG=1` set when NeferWL starts (for example `NEFERWL_HINT_DEBUG=1 make tty`, or the variable in the launch environment), every successful scan writes three files to `$XDG_STATE_HOME/neferwl/hints/` (`~/.local/state/neferwl/hints/` when unset): `<time>-<output>.png`, the frame as scanned; `<time>-<output>.annotated.png`, the same with each target outlined (OCR links green, accessibility targets magenta, OCR window crops grey); and `<time>-<output>.json`, the targets (with their `source`, `atspi` or `ocr`) and timings. NeferWL keeps the 50 most recent scans and logs a warning at startup. The files contain the screen in clear: delete them after use.

M examples/config +7 -0


@@ -94,6 +94,11 @@ focus.flash-width = 2
94	focus.flash-opacity = 0.6	94		focus.flash-opacity = 0.6
95	focus.flash-duration = 200ms	95		focus.flash-duration = 200ms
96	focus.flash-color =	96		focus.flash-color =
		97	+	# Hint mode is experimental and off: turn it on here (restart required).
		98	+	# hint.enable = off
		99	+	# Hint mode labels the links, URLs and buttons on the focused screen:
		100	+	# clickable (links, URLs and buttons) or links (links and URLs only).
		101	+	# hint.targets = clickable
97		102
98	# Named workspaces are outside the numbered list and require a bind. The bind	103		# Named workspaces are outside the numbered list and require a bind. The bind
99	# shows one; pressing it again returns to the previous workspace.	104		# shows one; pressing it again returns to the previous workspace.

@@ -195,6 +200,8 @@ bind.cmd+shift+f = toggle-fullscreen
195	bind.cmd+s = toggle-stash-visible	200		bind.cmd+s = toggle-stash-visible
196	bind.cmd+shift+s = toggle-window-stash	201		bind.cmd+shift+s = toggle-window-stash
197	bind.cmd+o = toggle-overview	202		bind.cmd+o = toggle-overview
		203	+	# Label the links and buttons on screen and click one by typing its label.
		204	+	# bind.cmd+g = hint
198	bind.cmd+q = close-window	205		bind.cmd+q = close-window
199	bind.ctrl+alt+backspace = quit	206		bind.ctrl+alt+backspace = quit
200	bind.ctrl+cmd+space = spawn fuzzel	207		bind.ctrl+cmd+space = spawn fuzzel

M go.mod +2 -0


@@ -15,6 +15,7 @@ require (
15	github.com/godbus/dbus/v5 v5.2.2	15		github.com/godbus/dbus/v5 v5.2.2
16	github.com/rs/zerolog v1.35.1	16		github.com/rs/zerolog v1.35.1
17	github.com/stretchr/testify v1.12.1	17		github.com/stretchr/testify v1.12.1
		18	+	golang.org/x/image v0.46.0
18	golang.org/x/sys v0.48.0	19		golang.org/x/sys v0.48.0
19	golang.org/x/term v0.46.0	20		golang.org/x/term v0.46.0
20	)	21		)

@@ -24,5 +25,6 @@ require (
24	github.com/mattn/go-isatty v0.0.22 // indirect	25		github.com/mattn/go-isatty v0.0.22 // indirect
25	github.com/stretchr/objx v0.5.3 // indirect	26		github.com/stretchr/objx v0.5.3 // indirect
26	go.yaml.in/yaml/v3 v3.0.5 // indirect	27		go.yaml.in/yaml/v3 v3.0.5 // indirect
		28	+	golang.org/x/text v0.42.0 // indirect
27	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect	29		gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
28	)	30		)

M go.sum +4 -0 generated file: collapsed

load diff

A internal/adapters/atspi/bus_test.go +197 -0

@@ -0,0 +1,197 @@
	1	+	package atspi
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"os"
	6	+	"os/exec"
	7	+	"strings"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/zerobus"
	12	+	"github.com/stretchr/testify/require"
	13	+	)
	14	+
	15	+	// startBus starts a private dbus-daemon, so the test never touches the
	16	+	// desktop's buses. It skips when the daemon is not installed.
	17	+	func startBus(t *testing.T) string {
	18	+	t.Helper()
	19	+	bin, err := exec.LookPath("dbus-daemon")
	20	+	if err != nil {
	21	+	if os.Getenv("CI") != "" {
	22	+	t.Fatal("dbus-daemon not installed")
	23	+	}
	24	+	t.Skip("dbus-daemon not installed")
	25	+	}
	26	+	cmd := exec.Command(bin, "--session", "--nofork", "--nopidfile", "--print-address=1")
	27	+	out, err := cmd.StdoutPipe()
	28	+	require.NoError(t, err)
	29	+	require.NoError(t, cmd.Start())
	30	+	t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() })
	31	+	line, err := bufio.NewReader(out).ReadString('\n')
	32	+	require.NoError(t, err)
	33	+	return strings.TrimSpace(line)
	34	+	}
	35	+
	36	+	// TestDoDecodesMatchesPipelined runs Do against an app on a real bus: the
	37	+	// GetMatches reply is decoded, and replies to several calls sent before any
	38	+	// is read are matched by serial.
	39	+	func TestDoDecodesMatchesPipelined(t *testing.T) {
	40	+	addr := startBus(t)
	41	+	app, err := zerobus.Dial(addr)
	42	+	require.NoError(t, err)
	43	+	t.Cleanup(func() { _ = app.Close() })
	44	+	_, err = app.RequestName("org.test.App", 0)
	45	+	require.NoError(t, err)
	46	+
	47	+	done := make(chan struct{})
	48	+	go func() {
	49	+	defer close(done)
	50	+	for {
	51	+	m, err := app.ReadMessage()
	52	+	if err != nil {
	53	+	return
	54	+	}
	55	+	if m.Type != zerobus.TypeMethodCall {
	56	+	continue
	57	+	}
	58	+	req := &zerobus.Message{Sender: strings.Clone(m.Sender), Serial: m.Serial}
	59	+	switch m.Member {
	60	+	case "GetMatches":
	61	+	// Check the rule: struct, then sort, count, traverse.
	62	+	r := m.Body()
	63	+	r.Struct()
	64	+	end := r.Array('i')
	65	+	var states []int32
	66	+	for r.More(end) {
	67	+	states = append(states, r.Int32())
	68	+	}
	69	+	_ = r.Int32()
	70	+	ad := r.Array('{')
	71	+	_ = ad
	72	+	_ = r.Int32()
	73	+	end = r.Array('i')
	74	+	var roles []int32
	75	+	for r.More(end) {
	76	+	roles = append(roles, r.Int32())
	77	+	}
	78	+	ok := r.Err() == nil && len(states) == 2 && states[0] == 1<<25\|1<<30 && len(roles) == 5 && uint32(roles[2]) == 1<<24\|1<<18\|1<<31
	79	+	if !ok {
	80	+	app.NewError(req, "org.test.Error.BadRule", "s").Str("bad rule")
	81	+	_, _ = app.Send()
	82	+	continue
	83	+	}
	84	+	e := app.NewReply(req, "a(so)")
	85	+	a := e.BeginArray('(')
	86	+	for _, p := range []string{"/one", "/two"} {
	87	+	e.Struct()
	88	+	e.Str(":1.99")
	89	+	e.ObjectPath(p)
	90	+	}
	91	+	e.EndArray(a)
	92	+	case "GetExtents":
	93	+	e := app.NewReply(req, "(iiii)")
	94	+	e.Struct()
	95	+	e.Int32(1)
	96	+	e.Int32(2)
	97	+	e.Int32(30)
	98	+	e.Int32(40)
	99	+	case "GetRole":
	100	+	app.NewReply(req, "u").Uint32(roleLink)
	101	+	case "GetState":
	102	+	e := app.NewReply(req, "au")
	103	+	a := e.BeginArray('u')
	104	+	e.Uint32(stShowing)
	105	+	e.Uint32(0)
	106	+	e.EndArray(a)
	107	+	default:
	108	+	app.NewError(req, "org.freedesktop.DBus.Error.UnknownMethod", "s").Str("no")
	109	+	}
	110	+	_, _ = app.Send()
	111	+	}
	112	+	}()
	113	+
	114	+	cc, err := zerobus.Dial(addr)
	115	+	require.NoError(t, err)
	116	+	b, err := newConn(cc)
	117	+	require.NoError(t, err)
	118	+	t.Cleanup(func() { _ = b.Close() })
	119	+
	120	+	calls := []call{
	121	+	{op: opMatches, name: "org.test.App", path: "/frame", rule: matchRule{linksOnly: true}},
	122	+	{op: opExtents, name: "org.test.App", path: "/one", arg: coordWindow},
	123	+	{op: opRole, name: "org.test.App", path: "/one"},
	124	+	{op: opState, name: "org.test.App", path: "/one"},
	125	+	{op: opChildren, name: "org.test.App", path: "/one"},
	126	+	}
	127	+	rep := b.Do(time.Now().Add(2*time.Second), time.Second, calls)
	128	+	require.Len(t, rep, 5)
	129	+	require.NoError(t, rep[0].err)
	130	+	require.Equal(t, []ref{{":1.99", "/one"}, {":1.99", "/two"}}, rep[0].refs)
	131	+	require.NoError(t, rep[1].err)
	132	+	require.Equal(t, [4]int32{1, 2, 30, 40}, rep[1].rect)
	133	+	require.Equal(t, uint32(roleLink), rep[2].u)
	134	+	require.Equal(t, uint64(stShowing), rep[3].state)
	135	+	var be *busError
	136	+	require.ErrorAs(t, rep[4].err, &be)
	137	+	require.Equal(t, "org.freedesktop.DBus.Error.UnknownMethod", be.name)
	138	+
	139	+	// A call to a name nobody owns fails fast, others still succeed.
	140	+	rep = b.Do(time.Now().Add(2*time.Second), time.Second, []call{
	141	+	{op: opRole, name: "org.test.Nobody", path: "/x"},
	142	+	{op: opRole, name: "org.test.App", path: "/x"},
	143	+	})
	144	+	require.Error(t, rep[0].err)
	145	+	require.NoError(t, rep[1].err)
	146	+	}
	147	+
	148	+	func TestDoTimesOutOnSilentApp(t *testing.T) {
	149	+	addr := startBus(t)
	150	+	app, err := zerobus.Dial(addr)
	151	+	require.NoError(t, err)
	152	+	t.Cleanup(func() { _ = app.Close() })
	153	+	_, err = app.RequestName("org.test.Mute", 0)
	154	+	require.NoError(t, err)
	155	+
	156	+	cc, err := zerobus.Dial(addr)
	157	+	require.NoError(t, err)
	158	+	b, err := newConn(cc)
	159	+	require.NoError(t, err)
	160	+	t.Cleanup(func() { _ = b.Close() })
	161	+
	162	+	t0 := time.Now()
	163	+	rep := b.Do(time.Now().Add(time.Second), 50*time.Millisecond, []call{{op: opRole, name: "org.test.Mute", path: "/x"}})
	164	+	require.ErrorIs(t, rep[0].err, ErrTimeout)
	165	+	require.Less(t, time.Since(t0), 500*time.Millisecond)
	166	+	}
	167	+
	168	+	// A call left unanswered for idle gets ErrTimeout without breaking the
	169	+	// connection; a batch that hits its deadline breaks it, and Do returns near
	170	+	// the deadline even though the idle timeout is longer.
	171	+	func TestDoDeadlineBreaksConnection(t *testing.T) {
	172	+	addr := startBus(t)
	173	+	app, err := zerobus.Dial(addr)
	174	+	require.NoError(t, err)
	175	+	t.Cleanup(func() { _ = app.Close() })
	176	+	_, err = app.RequestName("org.test.Mute", 0)
	177	+	require.NoError(t, err)
	178	+
	179	+	cc, err := zerobus.Dial(addr)
	180	+	require.NoError(t, err)
	181	+	b, err := newConn(cc)
	182	+	require.NoError(t, err)
	183	+	t.Cleanup(func() { _ = b.Close() })
	184	+
	185	+	mute := []call{{op: opRole, name: "org.test.Mute", path: "/x"}}
	186	+	rep := b.Do(time.Now().Add(time.Second), 40*time.Millisecond, mute)
	187	+	require.ErrorIs(t, rep[0].err, ErrTimeout)
	188	+	require.NotErrorIs(t, rep[0].err, ErrBroken)
	189	+
	190	+	t0 := time.Now()
	191	+	rep = b.Do(time.Now().Add(60*time.Millisecond), time.Second, mute)
	192	+	require.ErrorIs(t, rep[0].err, ErrBroken)
	193	+	require.Less(t, time.Since(t0), 300*time.Millisecond)
	194	+	// The socket is closed: the next batch fails at once.
	195	+	rep = b.Do(time.Now().Add(time.Second), time.Second, mute)
	196	+	require.ErrorIs(t, rep[0].err, ErrBroken)
	197	+	}

A internal/adapters/atspi/connector_mock_test.go +109 -0

@@ -0,0 +1,109 @@
	1	+	// Code generated by mockery; DO NOT EDIT.
	2	+	// github.com/vektra/mockery
	3	+	// template: testify
	4	+
	5	+	package atspi
	6	+
	7	+	import (
	8	+	"context"
	9	+
	10	+	mock "github.com/stretchr/testify/mock"
	11	+	)
	12	+
	13	+	// newMockconnector creates a new instance of mockconnector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
	14	+	// The first argument is typically a *testing.T value.
	15	+	func newMockconnector(t interface {
	16	+	mock.TestingT
	17	+	Cleanup(func())
	18	+	}) *mockconnector {
	19	+	if helper, ok := t.(interface{ Helper() }); ok {
	20	+	helper.Helper()
	21	+	}
	22	+
	23	+	mock := &mockconnector{}
	24	+	mock.Mock.Test(t)
	25	+
	26	+	t.Cleanup(func() {
	27	+	if helper, ok := t.(interface{ Helper() }); ok {
	28	+	helper.Helper()
	29	+	}
	30	+	mock.AssertExpectations(t)
	31	+	})
	32	+
	33	+	return mock
	34	+	}
	35	+
	36	+	// mockconnector is an autogenerated mock type for the connector type
	37	+	type mockconnector struct {
	38	+	mock.Mock
	39	+	}
	40	+
	41	+	type mockconnector_Expecter struct {
	42	+	mock *mock.Mock
	43	+	}
	44	+
	45	+	func (_m mockconnector) EXPECT() mockconnector_Expecter {
	46	+	return &mockconnector_Expecter{mock: &_m.Mock}
	47	+	}
	48	+
	49	+	// Connect provides a mock function for the type mockconnector
	50	+	func (_mock *mockconnector) Connect(ctx context.Context) (transport, error) {
	51	+	ret := _mock.Called(ctx)
	52	+
	53	+	if len(ret) == 0 {
	54	+	panic("no return value specified for Connect")
	55	+	}
	56	+
	57	+	var r0 transport
	58	+	var r1 error
	59	+	if returnFunc, ok := ret.Get(0).(func(context.Context) (transport, error)); ok {
	60	+	return returnFunc(ctx)
	61	+	}
	62	+	if returnFunc, ok := ret.Get(0).(func(context.Context) transport); ok {
	63	+	r0 = returnFunc(ctx)
	64	+	} else {
	65	+	if ret.Get(0) != nil {
	66	+	r0 = ret.Get(0).(transport)
	67	+	}
	68	+	}
	69	+	if returnFunc, ok := ret.Get(1).(func(context.Context) error); ok {
	70	+	r1 = returnFunc(ctx)
	71	+	} else {
	72	+	r1 = ret.Error(1)
	73	+	}
	74	+	return r0, r1
	75	+	}
	76	+
	77	+	// mockconnector_Connect_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Connect'
	78	+	type mockconnector_Connect_Call struct {
	79	+	*mock.Call
	80	+	}
	81	+
	82	+	// Connect is a helper method to define mock.On call
	83	+	// - ctx context.Context
	84	+	func (_e mockconnector_Expecter) Connect(ctx any) mockconnector_Connect_Call {
	85	+	return &mockconnector_Connect_Call{Call: _e.mock.On("Connect", ctx)}
	86	+	}
	87	+
	88	+	func (_c mockconnector_Connect_Call) Run(run func(ctx context.Context)) mockconnector_Connect_Call {
	89	+	_c.Call.Run(func(args mock.Arguments) {
	90	+	var arg0 context.Context
	91	+	if args[0] != nil {
	92	+	arg0 = args[0].(context.Context)
	93	+	}
	94	+	run(
	95	+	arg0,
	96	+	)
	97	+	})
	98	+	return _c
	99	+	}
	100	+
	101	+	func (_c mockconnector_Connect_Call) Return(transportMoqParam transport, err error) mockconnector_Connect_Call {
	102	+	_c.Call.Return(transportMoqParam, err)
	103	+	return _c
	104	+	}
	105	+
	106	+	func (_c mockconnector_Connect_Call) RunAndReturn(run func(ctx context.Context) (transport, error)) mockconnector_Connect_Call {
	107	+	_c.Call.Return(run)
	108	+	return _c
	109	+	}

A internal/adapters/atspi/doc.go +10 -0

@@ -0,0 +1,10 @@
	1	+	// Package atspi finds hint targets in the accessibility tree of the apps on
	2	+	// screen. It talks to the AT-SPI registry over the accessibility bus with
	3	+	// zerobus, from one connection used by one goroutine at a time, and pipelines
	4	+	// the calls it needs: several are sent before their replies are read, and the
	5	+	// replies are matched by serial.
	6	+	//
	7	+	// The approach (Collection.GetMatches on the window frame, bounded walk when
	8	+	// the toolkit has no Collection, window-relative extents) is adapted from
	9	+	// neru's AT-SPI client, https://github.com/y3owk1n/neru (MIT licence).
	10	+	package atspi

A internal/adapters/atspi/model.go +206 -0

@@ -0,0 +1,206 @@
	1	+	package atspi
	2	+
	3	+	import "github.com/bnema/neferwl/internal/ports"
	4	+
	5	+	// AT-SPI constants (atspi-constants.h). The role ids are the AtspiRole enum
	6	+	// values; the state ids are bit indices of the state set.
	7	+	const (
	8	+	roleCheckBox = 7
	9	+	roleComboBox = 11
	10	+	roleDialog = 16
	11	+	roleFrame = 23
	12	+	roleMenuItem = 35
	13	+	rolePageTab = 37
	14	+	rolePushButton = 43
	15	+	roleRadioButton = 44
	16	+	roleToggleButton = 62
	17	+	roleWindow = 69
	18	+	roleLink = 88
	19	+	roleDocumentFrame = 82
	20	+	roleDocumentWeb = 95
	21	+
	22	+	stateActive = 1
	23	+	stateShowing = 25
	24	+	stateVisible = 30
	25	+
	26	+	roleWords = 5 // int32 words of a role set in a MatchRule
	27	+	stateWords = 2 // int32 words of a state set in a MatchRule
	28	+
	29	+	// AtspiCollectionMatchType and AtspiCollectionSortOrder values.
	30	+	matchAll = 1
	31	+	matchAny = 2
	32	+	sortCanonical = 1
	33	+	matchCount = 500
	34	+	maxDepth = 30
	35	+	maxWalkNodes = 3000
	36	+	maxFrames = 32
	37	+	sizeTolerance = 40
	38	+	maxMatchesRead = matchCount
	39	+	)
	40	+
	41	+	// clickableRoles are the roles that become hint targets.
	42	+	var clickableRoles = []uint32{
	43	+	roleLink, rolePushButton, roleToggleButton, roleCheckBox,
	44	+	roleRadioButton, roleMenuItem, rolePageTab, roleComboBox,
	45	+	}
	46	+
	47	+	// matchRule is the Collection query: showing and visible objects of the
	48	+	// clickable roles (only links when linksOnly), plus web documents for coverage.
	49	+	type matchRule struct{ linksOnly bool }
	50	+
	51	+	func (r matchRule) roles() []uint32 {
	52	+	if r.linksOnly {
	53	+	return []uint32{roleLink, roleDocumentFrame, roleDocumentWeb}
	54	+	}
	55	+	return append(append([]uint32(nil), clickableRoles...), roleDocumentFrame, roleDocumentWeb)
	56	+	}
	57	+
	58	+	// roleBits packs role ids into the bit set of a MatchRule: bit id%32 of word
	59	+	// id/32.
	60	+	func roleBits(ids []uint32) []int32 {
	61	+	words := make([]int32, roleWords)
	62	+	for _, id := range ids {
	63	+	if w := int(id / 32); w < len(words) {
	64	+	words[w] \|= int32(uint32(1) << (id % 32))
	65	+	}
	66	+	}
	67	+	return words
	68	+	}
	69	+
	70	+	// stateBits packs state ids into the bit set of a MatchRule.
	71	+	func stateBits(ids ...uint32) []int32 {
	72	+	words := make([]int32, stateWords)
	73	+	for _, id := range ids {
	74	+	if w := int(id / 32); w < len(words) {
	75	+	words[w] \|= int32(uint32(1) << (id % 32))
	76	+	}
	77	+	}
	78	+	return words
	79	+	}
	80	+
	81	+	func hasState(set uint64, state uint) bool { return set&(1<<state) != 0 }
	82	+
	83	+	// isClickable reports whether role makes a target.
	84	+	func isClickable(role uint32, linksOnly bool) bool {
	85	+	if linksOnly {
	86	+	return role == roleLink
	87	+	}
	88	+	for _, r := range clickableRoles {
	89	+	if r == role {
	90	+	return true
	91	+	}
	92	+	}
	93	+	return false
	94	+	}
	95	+
	96	+	// isWindowRole reports whether role is a top-level window role.
	97	+	func isWindowRole(role uint32) bool {
	98	+	return role == roleFrame \|\| role == roleWindow \|\| role == roleDialog
	99	+	}
	100	+
	101	+	// kindOf maps a role to the hint kind: links are links, the rest buttons.
	102	+	func kindOf(role uint32) ports.HintKind {
	103	+	if role == roleLink {
	104	+	return ports.HintLink
	105	+	}
	106	+	return ports.HintButton
	107	+	}
	108	+
	109	+	// frameInfo is a showing top-level window of an app.
	110	+	type frameInfo struct {
	111	+	ref ref
	112	+	active bool
	113	+	w, h int // size of the frame, from its window extents
	114	+	}
	115	+
	116	+	// matchFrames pairs the windows of one app with its frames, one to one. A
	117	+	// window is paired with a frame when the pairing is certain:
	118	+	// - one window and one frame;
	119	+	// - the focused window with the one ACTIVE frame; that frame is then
	120	+	// reserved and no other window can take it;
	121	+	// - otherwise the one unreserved frame whose size is within sizeTolerance
	122	+	// of the window, provided no other unmatched window claims it as well.
	123	+	//
	124	+	// Ambiguous windows are left out. The result maps the index of a window in
	125	+	// wins to the index of its frame in frames.
	126	+	func matchFrames(wins []ports.HintWindow, frames []frameInfo) map[int]int {
	127	+	out := map[int]int{}
	128	+	if len(wins) == 1 && len(frames) == 1 {
	129	+	out[0] = 0
	130	+	return out
	131	+	}
	132	+	// Active matches first: they are reserved against the size matches.
	133	+	active, n := -1, 0
	134	+	for j, f := range frames {
	135	+	if f.active {
	136	+	active, n = j, n+1
	137	+	}
	138	+	}
	139	+	reserved := make([]bool, len(frames))
	140	+	done := make([]bool, len(wins))
	141	+	if n == 1 {
	142	+	focused := -1
	143	+	count := 0
	144	+	for i, w := range wins {
	145	+	if w.Focused {
	146	+	focused, count = i, count+1
	147	+	}
	148	+	}
	149	+	if count == 1 {
	150	+	out[focused] = active
	151	+	reserved[active] = true
	152	+	done[focused] = true
	153	+	}
	154	+	}
	155	+	claims := make([][]int, len(wins)) // size candidates per window
	156	+	owners := make([]int, len(frames)) // windows that size-match a frame
	157	+	for i, w := range wins {
	158	+	if done[i] {
	159	+	continue
	160	+	}
	161	+	for j, f := range frames {
	162	+	if !reserved[j] && absInt(f.w-w.Rect.W) <= sizeTolerance && absInt(f.h-w.Rect.H) <= sizeTolerance {
	163	+	claims[i] = append(claims[i], j)
	164	+	owners[j]++
	165	+	}
	166	+	}
	167	+	}
	168	+	for i := range wins {
	169	+	if !done[i] && len(claims[i]) == 1 && owners[claims[i][0]] == 1 {
	170	+	out[i] = claims[i][0]
	171	+	}
	172	+	}
	173	+	return out
	174	+	}
	175	+
	176	+	func absInt(v int) int {
	177	+	if v < 0 {
	178	+	return -v
	179	+	}
	180	+	return v
	181	+	}
	182	+
	183	+	// element is a match with its extents.
	184	+	type element struct {
	185	+	role uint32
	186	+	x, y, w, h int32 // window-relative
	187	+	}
	188	+
	189	+	// toTarget converts an element of a window into a logical output-local
	190	+	// target: the element extents are relative to the top-level frame, whose own
	191	+	// window-relative origin is (fx, fy); the result is clipped to the window.
	192	+	// It reports false for an empty result.
	193	+	func toTarget(win ports.Rect, fx, fy int32, e element) (ports.HintTarget, bool) {
	194	+	if e.w <= 0 \|\| e.h <= 0 {
	195	+	return ports.HintTarget{}, false
	196	+	}
	197	+	r := ports.Rect{X: win.X + int(e.x-fx), Y: win.Y + int(e.y-fy), W: int(e.w), H: int(e.h)}
	198	+	x0, y0 := max(r.X, win.X), max(r.Y, win.Y)
	199	+	x1, y1 := min(r.X+r.W, win.X+win.W), min(r.Y+r.H, win.Y+win.H)
	200	+	if x1 <= x0 \|\| y1 <= y0 {
	201	+	return ports.HintTarget{}, false
	202	+	}
	203	+	return ports.HintTarget{Rect: ports.Rect{X: x0, Y: y0, W: x1 - x0, H: y1 - y0}, Kind: kindOf(e.role)}, true
	204	+	}
	205	+
	206	+	func isDocument(role uint32) bool { return role == roleDocumentFrame \|\| role == roleDocumentWeb }

A internal/adapters/atspi/model_test.go +117 -0

@@ -0,0 +1,117 @@
	1	+	package atspi
	2	+
	3	+	import (
	4	+	"testing"
	5	+
	6	+	"github.com/stretchr/testify/require"
	7	+
	8	+	"github.com/bnema/neferwl/internal/ports"
	9	+	)
	10	+
	11	+	func TestToTargetConvertsWindowCoordinates(t *testing.T) {
	12	+	win := ports.Rect{X: 100, Y: 50, W: 800, H: 600}
	13	+	// The frame origin (fx, fy) is subtracted: toolkit offsets cancel out.
	14	+	got, ok := toTarget(win, 10, 20, element{role: roleLink, x: 110, y: 120, w: 40, h: 16})
	15	+	require.True(t, ok)
	16	+	require.Equal(t, ports.HintTarget{Rect: ports.Rect{X: 200, Y: 150, W: 40, H: 16}, Kind: ports.HintLink}, got)
	17	+	}
	18	+
	19	+	func TestToTargetClipsToWindow(t *testing.T) {
	20	+	win := ports.Rect{X: 100, Y: 50, W: 200, H: 100}
	21	+	got, ok := toTarget(win, 0, 0, element{role: rolePushButton, x: 180, y: 90, w: 60, h: 30})
	22	+	require.True(t, ok)
	23	+	require.Equal(t, ports.Rect{X: 280, Y: 140, W: 20, H: 10}, got.Rect)
	24	+	require.Equal(t, ports.HintButton, got.Kind)
	25	+	}
	26	+
	27	+	func TestToTargetDropsEmptyAndOutside(t *testing.T) {
	28	+	win := ports.Rect{X: 0, Y: 0, W: 100, H: 100}
	29	+	for name, e := range map[string]element{
	30	+	"zero width": {role: roleLink, x: 1, y: 1, w: 0, h: 10},
	31	+	"neg height": {role: roleLink, x: 1, y: 1, w: 10, h: -1},
	32	+	"outside": {role: roleLink, x: 500, y: 1, w: 10, h: 10},
	33	+	"above": {role: roleLink, x: 1, y: -50, w: 10, h: 10},
	34	+	"left of win": {role: roleLink, x: -30, y: 1, w: 10, h: 10},
	35	+	} {
	36	+	_, ok := toTarget(win, 0, 0, e)
	37	+	require.False(t, ok, name)
	38	+	}
	39	+	}
	40	+
	41	+	func TestRoleKindMapping(t *testing.T) {
	42	+	require.Equal(t, ports.HintLink, kindOf(roleLink))
	43	+	for _, r := range []uint32{rolePushButton, roleToggleButton, roleCheckBox, roleRadioButton, roleMenuItem, rolePageTab, roleComboBox} {
	44	+	require.Equal(t, ports.HintButton, kindOf(r), r)
	45	+	require.True(t, isClickable(r, false), r)
	46	+	require.False(t, isClickable(r, true), r)
	47	+	}
	48	+	require.True(t, isClickable(roleLink, true))
	49	+	require.False(t, isClickable(roleFrame, false))
	50	+	require.True(t, isWindowRole(roleFrame) && isWindowRole(roleWindow) && isWindowRole(roleDialog))
	51	+	require.False(t, isWindowRole(roleLink))
	52	+	}
	53	+
	54	+	func TestRoleAndStateBits(t *testing.T) {
	55	+	require.Equal(t, []int32{0, 0, 0, 0, 0}, roleBits(nil))
	56	+	// link is role 88: word 2, bit 24; push button is 43: word 1, bit 11.
	57	+	require.Equal(t, []int32{0, 1 << 11, 1 << 24, 0, 0}, roleBits([]uint32{roleLink, rolePushButton}))
	58	+	// showing is state 25, visible 30: both in word 0.
	59	+	require.Equal(t, []int32{1<<25 \| 1<<30, 0}, stateBits(stateShowing, stateVisible))
	60	+	}
	61	+
	62	+	func wins(ws ...ports.HintWindow) []ports.HintWindow { return ws }
	63	+
	64	+	func win(id ports.WindowID, w, h int, focused bool) ports.HintWindow {
	65	+	return ports.HintWindow{ID: id, PID: 1, Rect: ports.Rect{W: w, H: h}, Focused: focused}
	66	+	}
	67	+
	68	+	func TestMatchFramesSingle(t *testing.T) {
	69	+	got := matchFrames(wins(win(1, 800, 600, false)), []frameInfo{{w: 100, h: 100}})
	70	+	require.Equal(t, map[int]int{0: 0}, got)
	71	+	}
	72	+
	73	+	func TestMatchFramesActive(t *testing.T) {
	74	+	frames := []frameInfo{{w: 800, h: 600}, {w: 800, h: 600, active: true}}
	75	+	got := matchFrames(wins(win(1, 800, 600, true), win(2, 800, 600, false)), frames)
	76	+	// The focused window takes the ACTIVE frame and reserves it; the other
	77	+	// window then only has the remaining frame to match.
	78	+	require.Equal(t, map[int]int{0: 1, 1: 0}, got)
	79	+	}
	80	+
	81	+	func TestMatchFramesSize(t *testing.T) {
	82	+	frames := []frameInfo{{w: 1000, h: 700}, {w: 400, h: 300}}
	83	+	got := matchFrames(wins(win(1, 1000, 700, false), win(2, 420, 290, false)), frames)
	84	+	require.Equal(t, map[int]int{0: 0, 1: 1}, got)
	85	+	}
	86	+
	87	+	func TestMatchFramesSizeTolerance(t *testing.T) {
	88	+	frames := []frameInfo{{w: 840, h: 640}, {w: 100, h: 100}}
	89	+	other := win(2, 90, 90, false)
	90	+	// 40 px off still matches; 41 px does not.
	91	+	require.Equal(t, map[int]int{0: 0, 1: 1}, matchFrames(wins(win(1, 800, 600, false), other), frames))
	92	+	require.Equal(t, map[int]int{1: 1}, matchFrames(wins(win(1, 799, 600, false), other), frames))
	93	+	}
	94	+
	95	+	func TestMatchFramesAmbiguous(t *testing.T) {
	96	+	// Two windows and two frames of the same size, none active: skipped.
	97	+	frames := []frameInfo{{w: 800, h: 600}, {w: 810, h: 600}}
	98	+	got := matchFrames(wins(win(1, 800, 600, false), win(2, 800, 600, false)), frames)
	99	+	require.Empty(t, got)
	100	+	// One window, two same-size frames, not focused: skipped.
	101	+	require.Empty(t, matchFrames(wins(win(1, 800, 600, false)), frames))
	102	+	// No frame close enough.
	103	+	require.Empty(t, matchFrames(wins(win(1, 200, 200, false), win(2, 300, 300, false)), frames))
	104	+	}
	105	+
	106	+	// The active frame is reserved by the focused window: an unfocused window of
	107	+	// the same size as that frame must not be paired with it as well.
	108	+	func TestMatchFramesOneToOne(t *testing.T) {
	109	+	frames := []frameInfo{{ref: ref{"a", "/active"}, active: true, w: 400, h: 300}, {ref: ref{"a", "/idle"}, w: 800, h: 600}}
	110	+	got := matchFrames(wins(win(1, 800, 600, true), win(2, 400, 300, false)), frames)
	111	+	seen := map[int]bool{}
	112	+	for _, f := range got {
	113	+	require.False(t, seen[f], "frame %d assigned twice: %v", f, got)
	114	+	seen[f] = true
	115	+	}
	116	+	require.Equal(t, map[int]int{0: 0}, got)
	117	+	}

A internal/adapters/atspi/source.go +482 -0

@@ -0,0 +1,482 @@
	1	+	package atspi
	2	+
	3	+	import (
	4	+	"context"
	5	+	"errors"
	6	+	"fmt"
	7	+	"strings"
	8	+	"time"
	9	+
	10	+	"github.com/bnema/zerobus"
	11	+
	12	+	"github.com/bnema/neferwl/internal/adapters/logging"
	13	+	"github.com/bnema/neferwl/internal/ports"
	14	+	)
	15	+
	16	+	const (
	17	+	// callTimeout is how long a batch waits for the next reply.
	18	+	callTimeout = 100 * time.Millisecond
	19	+	// budget is the whole time Targets may take.
	20	+	budget = 300 * time.Millisecond
	21	+	)
	22	+
	23	+	// connectBudget bounds the whole connection setup (session bus,
	24	+	// GetAddress, IsEnabled, dial of the accessibility bus) of one attempt.
	25	+	const connectBudget = 300 * time.Millisecond
	26	+
	27	+	// retryEvery is the minimum time between two connection attempts.
	28	+	const retryEvery = 5 * time.Second
	29	+
	30	+	// connector opens a transport on the accessibility bus.
	31	+	type connector interface {
	32	+	Connect(ctx context.Context) (transport, error)
	33	+	}
	34	+
	35	+	// stallWarnAfter is how long an attempt may be in flight before it is
	36	+	// reported as a stalled accessibility bus.
	37	+	const stallWarnAfter = 5 * time.Second
	38	+
	39	+	// dialResult is the outcome of a connection attempt.
	40	+	type dialResult struct {
	41	+	tr transport
	42	+	err error
	43	+	at time.Time // when the attempt finished
	44	+	}
	45	+
	46	+	// Source reads hint targets from the accessibility tree. It owns one
	47	+	// connection to the accessibility bus. It is NOT safe for concurrent use:
	48	+	// it is owned by the hints worker goroutine, which calls Targets and,
	49	+	// when it returns, Close. The connection is made on the first Targets call
	50	+	// that needs it, never at startup, and made again after a failure, at most
	51	+	// once every retryEvery; meanwhile Targets finds nothing and OCR takes over.
	52	+	//
	53	+	// At most one connection attempt (one goroutine and one socket) exists at
	54	+	// any time: an attempt that outlives the wait of ensure stays in flight, is
	55	+	// polled by the later Targets calls and is never doubled.
	56	+	type Source struct {
	57	+	tr transport
	58	+	dial connector
	59	+	// attempt delivers the outcome of the attempt in flight (buffered, so
	60	+	// the attempt goroutine never blocks); nil when none runs.
	61	+	attempt chan dialResult
	62	+	attemptStarted time.Time
	63	+	warnedStall bool
	64	+	retryAt time.Time
	65	+	closed bool
	66	+	// wait is how long ensure waits for a new attempt; connectBudget if 0.
	67	+	wait time.Duration
	68	+	}
	69	+
	70	+	// New returns a Source. It does not touch the bus: the connection is made by
	71	+	// the first Targets call.
	72	+	func New() *Source { return &Source{dial: busConnector{}} }
	73	+
	74	+	// busConnector connects to the session bus, enables accessibility and dials
	75	+	// the accessibility bus.
	76	+	type busConnector struct{}
	77	+
	78	+	func (busConnector) Connect(ctx context.Context) (transport, error) {
	79	+	log := logging.For(ctx, "atspi")
	80	+	// zerobus has no deadline or context API: SessionBus and Dial run the
	81	+	// socket dial, SASL auth and Hello without a bound, and the watchdog
	82	+	// below cannot exist before the connection does. A stall there is
	83	+	// bounded by the single-in-flight rule of Source.ensure, not by a
	84	+	// deadline. Follow-up: a deadline-capable zerobus API.
	85	+	sess, err := zerobus.SessionBus()
	86	+	if err != nil {
	87	+	return nil, fmt.Errorf("atspi: session bus: %w", err)
	88	+	}
	89	+	defer func() { _ = sess.Close() }()
	90	+	// Call has no deadline: closing the connection ends the wait for a
	91	+	// silent service.
	92	+	stop := time.AfterFunc(connectBudget, func() { _ = sess.Close() })
	93	+	defer stop.Stop()
	94	+
	95	+	sess.NewCall(a11yBusName, a11yBusPath, a11yBusName, "GetAddress", "")
	96	+	m, err := sess.Call()
	97	+	if err != nil {
	98	+	return nil, fmt.Errorf("atspi: org.a11y.Bus GetAddress: %w", err)
	99	+	}
	100	+	r := m.Body()
	101	+	addr := r.Str()
	102	+	if r.Err() != nil \|\| addr == "" {
	103	+	return nil, errors.New("atspi: no accessibility bus address")
	104	+	}
	105	+	addr = strings.Clone(addr)
	106	+
	107	+	e := sess.NewCall(a11yBusName, a11yBusPath, ifaceProps, "Set", "ssv")
	108	+	e.Str(a11yStatus)
	109	+	e.Str("IsEnabled")
	110	+	e.Variant("b")
	111	+	e.Bool(true)
	112	+	if _, err := sess.Call(); err != nil {
	113	+	// Toolkits that already see the bus keep working.
	114	+	log.Warn().Err(err).Msg("could not enable accessibility")
	115	+	} else {
	116	+	log.Info().Msg("accessibility enabled")
	117	+	}
	118	+
	119	+	zc, err := zerobus.Dial(addr)
	120	+	if err != nil {
	121	+	return nil, fmt.Errorf("atspi: accessibility bus: %w", err)
	122	+	}
	123	+	tr, err := newConn(zc)
	124	+	if err != nil {
	125	+	return nil, err
	126	+	}
	127	+	log.Info().Msg("accessibility bus connected")
	128	+	return tr, nil
	129	+	}
	130	+
	131	+	// ensure returns the connection, making one if none exists and the last
	132	+	// attempt is old enough. It waits at most the budget for a new attempt; an
	133	+	// attempt that does not answer in time stays in flight (abandoned, never
	134	+	// doubled) and is polled without blocking by the next calls.
	135	+	func (s *Source) ensure(ctx context.Context) transport {
	136	+	if s.closed {
	137	+	return nil
	138	+	}
	139	+	if s.tr != nil {
	140	+	return s.tr
	141	+	}
	142	+	if s.dial == nil {
	143	+	return nil
	144	+	}
	145	+	now := time.Now()
	146	+	if s.attempt != nil {
	147	+	select {
	148	+	case r := <-s.attempt:
	149	+	s.finish(ctx, r)
	150	+	default:
	151	+	if !s.warnedStall && now.Sub(s.attemptStarted) > stallWarnAfter {
	152	+	s.warnedStall = true
	153	+	log := logging.For(ctx, "atspi")
	154	+	log.Warn().Dur("for", now.Sub(s.attemptStarted)).Msg("accessibility bus not answering")
	155	+	}
	156	+	}
	157	+	return s.tr
	158	+	}
	159	+	if now.Before(s.retryAt) {
	160	+	return nil
	161	+	}
	162	+	ch := make(chan dialResult, 1)
	163	+	s.attempt = ch
	164	+	s.attemptStarted = now
	165	+	s.warnedStall = false
	166	+	dial := s.dial
	167	+	go func() {
	168	+	tr, err := dial.Connect(ctx)
	169	+	ch <- dialResult{tr: tr, err: err, at: time.Now()}
	170	+	}()
	171	+	wait := s.wait
	172	+	if wait <= 0 {
	173	+	wait = connectBudget
	174	+	}
	175	+	timer := time.NewTimer(wait)
	176	+	defer timer.Stop()
	177	+	select {
	178	+	case r := <-ch:
	179	+	s.finish(ctx, r)
	180	+	case <-timer.C:
	181	+	log := logging.For(ctx, "atspi")
	182	+	log.Debug().Dur("after", wait).Msg("accessibility connect attempt abandoned in flight")
	183	+	case <-ctx.Done():
	184	+	}
	185	+	return s.tr
	186	+	}
	187	+
	188	+	// finish takes the outcome of the attempt in flight: a transport is kept,
	189	+	// an error starts the backoff from the time the attempt finished.
	190	+	func (s *Source) finish(ctx context.Context, r dialResult) {
	191	+	s.attempt = nil
	192	+	if r.err != nil \|\| r.tr == nil {
	193	+	s.retryAt = r.at.Add(retryEvery)
	194	+	err := r.err
	195	+	if err == nil {
	196	+	err = errors.New("atspi: no transport")
	197	+	}
	198	+	log := logging.For(ctx, "atspi")
	199	+	log.Warn().Err(err).Msg("accessibility hint targets unavailable, using OCR only")
	200	+	return
	201	+	}
	202	+	s.tr = r.tr
	203	+	}
	204	+
	205	+	// drop closes a broken connection; the next attempt reconnects.
	206	+	func (s *Source) drop() {
	207	+	s.retryAt = time.Now().Add(retryEvery)
	208	+	if s.tr != nil {
	209	+	_ = s.tr.Close()
	210	+	s.tr = nil
	211	+	}
	212	+	}
	213	+
	214	+	// Close closes the accessibility bus connection, and the one an attempt
	215	+	// still in flight may deliver late. After Close, Targets finds nothing and
	216	+	// never connects. Like Targets it is called by the owner goroutine only.
	217	+	func (s *Source) Close() error {
	218	+	if s.closed {
	219	+	return nil
	220	+	}
	221	+	s.closed = true
	222	+	var err error
	223	+	if s.tr != nil {
	224	+	err = s.tr.Close()
	225	+	s.tr = nil
	226	+	}
	227	+	if ch := s.attempt; ch != nil {
	228	+	s.attempt = nil
	229	+	// One reaper at most: Close runs once, and one attempt exists.
	230	+	go func() {
	231	+	if r := <-ch; r.tr != nil {
	232	+	_ = r.tr.Close()
	233	+	}
	234	+	}()
	235	+	}
	236	+	return err
	237	+	}
	238	+
	239	+	// appInfo is an application of the registry whose process shows a window.
	240	+	type appInfo struct {
	241	+	root ref
	242	+	pid int
	243	+	}
	244	+
	245	+	// winMatch is a window with the frame it was paired with.
	246	+	type winMatch struct {
	247	+	win ports.HintWindow
	248	+	frame ref
	249	+	fx int32
	250	+	fy int32
	251	+	}
	252	+
	253	+	// Targets returns the clickable targets of the accessibility tree for
	254	+	// windows (logical output-local, clipped to each window) and the set of
	255	+	// windows those targets cover. A window is covered when its tree answered
	256	+	// with a link target or web document; uncovered windows also get OCR. It
	257	+	// returns what is ready within the budget.
	258	+	func (s *Source) Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error) {
	259	+	covered := map[ports.WindowID]bool{}
	260	+	deadline := time.Now().Add(budget)
	261	+	if d, ok := ctx.Deadline(); ok && d.Before(deadline) {
	262	+	deadline = d
	263	+	}
	264	+	wanted := map[int]bool{}
	265	+	for _, w := range windows {
	266	+	if w.PID > 0 {
	267	+	wanted[w.PID] = true
	268	+	}
	269	+	}
	270	+	if len(wanted) == 0 {
	271	+	return nil, covered, nil
	272	+	}
	273	+	tr := s.ensure(ctx)
	274	+	if tr == nil {
	275	+	return nil, covered, nil
	276	+	}
	277	+	broken := false
	278	+	do := func(calls []call) []reply {
	279	+	if len(calls) == 0 \|\| ctx.Err() != nil {
	280	+	return make([]reply, len(calls))
	281	+	}
	282	+	if broken {
	283	+	out := make([]reply, len(calls))
	284	+	for i := range out {
	285	+	out[i].err = ErrBroken
	286	+	}
	287	+	return out
	288	+	}
	289	+	out := tr.Do(deadline, callTimeout, calls)
	290	+	for _, r := range out {
	291	+	if errors.Is(r.err, ErrBroken) {
	292	+	broken = true
	293	+	s.drop()
	294	+	break
	295	+	}
	296	+	}
	297	+	return out
	298	+	}
	299	+
	300	+	// Applications of the registry, kept when their process shows a window.
	301	+	rep := do([]call{{op: opChildren, name: registryName, path: registryRoot}})
	302	+	if len(rep) == 0 \|\| rep[0].err != nil {
	303	+	if len(rep) > 0 {
	304	+	return nil, covered, fmt.Errorf("atspi: registry: %w", rep[0].err)
	305	+	}
	306	+	return nil, covered, nil
	307	+	}
	308	+	apps := rep[0].refs
	309	+	calls := make([]call, len(apps))
	310	+	for i, a := range apps {
	311	+	calls[i] = call{op: opPID, name: a.name}
	312	+	}
	313	+	var kept []appInfo
	314	+	for i, r := range do(calls) {
	315	+	if r.err == nil && wanted[int(r.u)] {
	316	+	kept = append(kept, appInfo{root: apps[i], pid: int(r.u)})
	317	+	}
	318	+	}
	319	+
	320	+	// The showing windows of each kept app.
	321	+	calls = calls[:0]
	322	+	for _, a := range kept {
	323	+	calls = append(calls, call{op: opChildren, name: a.root.name, path: a.root.path})
	324	+	}
	325	+	var cands []ref
	326	+	var candApp []int
	327	+	for i, r := range do(calls) {
	328	+	if r.err != nil {
	329	+	continue
	330	+	}
	331	+	for _, c := range r.refs[:min(len(r.refs), maxFrames)] {
	332	+	cands = append(cands, c)
	333	+	candApp = append(candApp, i)
	334	+	}
	335	+	}
	336	+	calls = calls[:0]
	337	+	for _, c := range cands {
	338	+	calls = append(calls,
	339	+	call{op: opRole, name: c.name, path: c.path},
	340	+	call{op: opState, name: c.name, path: c.path},
	341	+	call{op: opExtents, name: c.name, path: c.path, arg: coordWindow})
	342	+	}
	343	+	info := do(calls)
	344	+	type frame struct {
	345	+	frameInfo
	346	+	pid int
	347	+	fx, fy int32
	348	+	}
	349	+	var frames []frame
	350	+	for i, c := range cands {
	351	+	role, state, ext := info[3i], info[3i+1], info[3*i+2]
	352	+	if role.err != nil \|\| state.err != nil \|\| ext.err != nil \|\| !isWindowRole(role.u) \|\| !hasState(state.state, stateShowing) {
	353	+	continue
	354	+	}
	355	+	frames = append(frames, frame{
	356	+	frameInfo: frameInfo{ref: c, active: hasState(state.state, stateActive), w: int(ext.rect[2]), h: int(ext.rect[3])},
	357	+	pid: kept[candApp[i]].pid, fx: ext.rect[0], fy: ext.rect[1],
	358	+	})
	359	+	}
	360	+
	361	+	// Pair the windows with the frames, process by process.
	362	+	var matched []winMatch
	363	+	for pid := range wanted {
	364	+	var wins []ports.HintWindow
	365	+	for _, w := range windows {
	366	+	if w.PID == pid {
	367	+	wins = append(wins, w)
	368	+	}
	369	+	}
	370	+	var fs []frameInfo
	371	+	var origin []frame
	372	+	for _, f := range frames {
	373	+	if f.pid == pid {
	374	+	fs = append(fs, f.frameInfo)
	375	+	origin = append(origin, f)
	376	+	}
	377	+	}
	378	+	for wi, fi := range matchFrames(wins, fs) {
	379	+	matched = append(matched, winMatch{win: wins[wi], frame: origin[fi].ref, fx: origin[fi].fx, fy: origin[fi].fy})
	380	+	}
	381	+	}
	382	+
	383	+	targets := s.elements(do, matched, linksOnly, covered)
	384	+	log := logging.For(ctx, "atspi")
	385	+	log.Debug().Int("apps", len(kept)).Int("frames", len(frames)).Int("windows", len(matched)).Int("targets", len(targets)).Msg("atspi targets")
	386	+	return targets, covered, nil
	387	+	}
	388	+
	389	+	// elements reads the clickable elements of the matched frames.
	390	+	func (s *Source) elements(do func([]call) []reply, matched []winMatch, linksOnly bool, covered map[ports.WindowID]bool) []ports.HintTarget {
	391	+	calls := make([]call, len(matched))
	392	+	for i, m := range matched {
	393	+	calls[i] = call{op: opMatches, name: m.frame.name, path: m.frame.path, rule: matchRule{linksOnly: linksOnly}}
	394	+	}
	395	+	found := make([][]ref, len(matched))
	396	+	var walk []int // frames without Collection
	397	+	for i, r := range do(calls) {
	398	+	if r.err != nil {
	399	+	var be *busError
	400	+	if errors.As(r.err, &be) {
	401	+	walk = append(walk, i)
	402	+	}
	403	+	continue
	404	+	}
	405	+	found[i] = r.refs
	406	+	}
	407	+	for _, i := range walk {
	408	+	found[i] = s.walk(do, matched[i].frame, linksOnly)
	409	+	}
	410	+
	411	+	calls = calls[:0]
	412	+	var owner []int
	413	+	for i, refs := range found {
	414	+	for _, r := range refs {
	415	+	calls = append(calls,
	416	+	call{op: opRole, name: r.name, path: r.path},
	417	+	call{op: opExtents, name: r.name, path: r.path, arg: coordWindow})
	418	+	owner = append(owner, i)
	419	+	}
	420	+	}
	421	+	info := do(calls)
	422	+	var out []ports.HintTarget
	423	+	for k, i := range owner {
	424	+	role, ext := info[2k], info[2k+1]
	425	+	if role.err == nil && isDocument(role.u) {
	426	+	covered[matched[i].win.ID] = true
	427	+	continue
	428	+	}
	429	+	if role.err != nil \|\| ext.err != nil \|\| !isClickable(role.u, linksOnly) {
	430	+	continue
	431	+	}
	432	+	m := matched[i]
	433	+	t, ok := toTarget(m.win.Rect, m.fx, m.fy, element{role: role.u, x: ext.rect[0], y: ext.rect[1], w: ext.rect[2], h: ext.rect[3]})
	434	+	if !ok {
	435	+	continue
	436	+	}
	437	+	t.Window = m.win.ID
	438	+	out = append(out, t)
	439	+	if t.Kind == ports.HintLink {
	440	+	covered[m.win.ID] = true
	441	+	}
	442	+	}
	443	+	return out
	444	+	}
	445	+
	446	+	// walk is the fallback for toolkits without Collection: a breadth-first
	447	+	// walk of at most maxDepth levels and maxWalkNodes nodes, returning the
	448	+	// clickable nodes.
	449	+	func (s *Source) walk(do func([]call) []reply, root ref, linksOnly bool) []ref {
	450	+	var out []ref
	451	+	level := []ref{root}
	452	+	seen := 0
	453	+	for depth := 0; depth < maxDepth && len(level) > 0 && seen < maxWalkNodes; depth++ {
	454	+	calls := make([]call, 0, 3*len(level))
	455	+	for _, n := range level {
	456	+	calls = append(calls,
	457	+	call{op: opChildren, name: n.name, path: n.path},
	458	+	call{op: opRole, name: n.name, path: n.path},
	459	+	call{op: opState, name: n.name, path: n.path})
	460	+	}
	461	+	rep := do(calls)
	462	+	var next []ref
	463	+	for i, n := range level {
	464	+	ch, role, state := rep[3i], rep[3i+1], rep[3*i+2]
	465	+	if state.err == nil && !hasState(state.state, stateShowing) {
	466	+	continue
	467	+	}
	468	+	if role.err == nil && (isClickable(role.u, linksOnly) \|\| isDocument(role.u)) && state.err == nil && hasState(state.state, stateVisible) {
	469	+	out = append(out, n)
	470	+	}
	471	+	if ch.err == nil {
	472	+	next = append(next, ch.refs...)
	473	+	}
	474	+	}
	475	+	seen += len(level)
	476	+	if room := maxWalkNodes - seen; len(next) > room {
	477	+	next = next[:max(room, 0)]
	478	+	}
	479	+	level = next
	480	+	}
	481	+	return out
	482	+	}

A internal/adapters/atspi/source_test.go +315 -0

@@ -0,0 +1,315 @@
	1	+	package atspi
	2	+
	3	+	import (
	4	+	"context"
	5	+	"errors"
	6	+	"fmt"
	7	+	"testing"
	8	+	"time"
	9	+
	10	+	"github.com/stretchr/testify/mock"
	11	+	"github.com/stretchr/testify/require"
	12	+
	13	+	"github.com/bnema/neferwl/internal/ports"
	14	+	)
	15	+
	16	+	const (
	17	+	stActive = 1 << stateActive
	18	+	stShowing = 1 << stateShowing
	19	+	)
	20	+
	21	+	// tree answers calls like an app: registry with one app (:1.5, pid 77) with
	22	+	// one showing frame whose links sit at window-relative extents.
	23	+	func answerTree(frameExt [4]int32, matches []ref, ext map[string][4]int32, roles map[string]uint32, noCollection bool) func(time.Time, time.Duration, []call) []reply {
	24	+	return func(_ time.Time, _ time.Duration, calls []call) []reply {
	25	+	out := make([]reply, len(calls))
	26	+	for i, c := range calls {
	27	+	switch {
	28	+	case c.op == opChildren && c.name == registryName:
	29	+	out[i].refs = []ref{{":1.5", "/root"}, {":1.9", "/root"}}
	30	+	case c.op == opPID:
	31	+	out[i].u = map[string]uint32{":1.5": 77, ":1.9": 99}[c.name]
	32	+	case c.op == opChildren && c.path == "/root":
	33	+	out[i].refs = []ref{{":1.5", "/frame"}}
	34	+	case c.op == opRole && c.path == "/frame":
	35	+	out[i].u = roleFrame
	36	+	case c.op == opState && c.path == "/frame":
	37	+	out[i].state = stShowing \| stActive
	38	+	case c.op == opExtents && c.path == "/frame":
	39	+	out[i].rect = frameExt
	40	+	case c.op == opMatches:
	41	+	if noCollection {
	42	+	out[i].err = &busError{name: "org.freedesktop.DBus.Error.UnknownMethod"}
	43	+	} else {
	44	+	out[i].refs = matches
	45	+	}
	46	+	case c.op == opChildren && c.path == "/frame":
	47	+	out[i].refs = matches
	48	+	case c.op == opChildren:
	49	+	case c.op == opRole:
	50	+	out[i].u = roles[c.path]
	51	+	case c.op == opState:
	52	+	out[i].state = stShowing \| 1<<stateVisible
	53	+	case c.op == opExtents:
	54	+	out[i].rect = ext[c.path]
	55	+	}
	56	+	}
	57	+	return out
	58	+	}
	59	+	}
	60	+
	61	+	func TestTargetsCoversWindowAndConverts(t *testing.T) {
	62	+	tr := newMocktransport(t)
	63	+	matches := []ref{{":1.5", "/a"}, {":1.5", "/b"}, {":1.5", "/zero"}}
	64	+	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
	65	+	[4]int32{4, 30, 800, 600}, matches,
	66	+	map[string][4]int32{"/a": {14, 130, 50, 16}, "/b": {104, 230, 20, 20}, "/zero": {5, 5, 0, 0}},
	67	+	map[string]uint32{"/a": roleLink, "/b": rolePushButton, "/zero": roleLink}, false))
	68	+	s := &Source{tr: tr}
	69	+
	70	+	windows := []ports.HintWindow{
	71	+	{ID: 1, PID: 77, Rect: ports.Rect{X: 100, Y: 200, W: 800, H: 600}, Focused: true},
	72	+	{ID: 2, PID: 5, Rect: ports.Rect{X: 0, Y: 0, W: 50, H: 50}},
	73	+	}
	74	+	targets, covered, err := s.Targets(context.Background(), windows, false)
	75	+	require.NoError(t, err)
	76	+	require.Equal(t, map[ports.WindowID]bool{1: true}, covered)
	77	+	require.Equal(t, []ports.HintTarget{
	78	+	{Rect: ports.Rect{X: 110, Y: 300, W: 50, H: 16}, Kind: ports.HintLink, Window: 1},
	79	+	{Rect: ports.Rect{X: 200, Y: 400, W: 20, H: 20}, Kind: ports.HintButton, Window: 1},
	80	+	}, targets)
	81	+	}
	82	+
	83	+	func TestTargetsLinksOnlyDropsButtons(t *testing.T) {
	84	+	tr := newMocktransport(t)
	85	+	matches := []ref{{":1.5", "/a"}, {":1.5", "/b"}}
	86	+	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
	87	+	[4]int32{0, 0, 800, 600}, matches,
	88	+	map[string][4]int32{"/a": {14, 130, 50, 16}, "/b": {104, 230, 20, 20}},
	89	+	map[string]uint32{"/a": roleLink, "/b": rolePushButton}, false))
	90	+	s := &Source{tr: tr}
	91	+	targets, _, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}, true)
	92	+	require.NoError(t, err)
	93	+	require.Len(t, targets, 1)
	94	+	require.Equal(t, ports.HintLink, targets[0].Kind)
	95	+	}
	96	+
	97	+	func TestTargetsWalkWithoutCollection(t *testing.T) {
	98	+	tr := newMocktransport(t)
	99	+	matches := []ref{{":1.5", "/a"}}
	100	+	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
	101	+	[4]int32{0, 0, 800, 600}, matches,
	102	+	map[string][4]int32{"/a": {14, 130, 50, 16}},
	103	+	map[string]uint32{"/a": roleLink}, true))
	104	+	s := &Source{tr: tr}
	105	+	targets, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}, false)
	106	+	require.NoError(t, err)
	107	+	require.Len(t, targets, 1)
	108	+	require.True(t, covered[1])
	109	+	}
	110	+
	111	+	func TestTargetsNoPIDMatchCoversNothing(t *testing.T) {
	112	+	tr := newMocktransport(t)
	113	+	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree([4]int32{}, nil, nil, nil, false))
	114	+	s := &Source{tr: tr}
	115	+	targets, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 1234, Rect: ports.Rect{W: 8, H: 6}}}, false)
	116	+	require.NoError(t, err)
	117	+	require.Empty(t, targets)
	118	+	require.Empty(t, covered)
	119	+	}
	120	+
	121	+	func TestTargetsWithoutPIDsAsksNothing(t *testing.T) {
	122	+	s := &Source{tr: newMocktransport(t)}
	123	+	_, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 8, H: 6}}}, false)
	124	+	require.NoError(t, err)
	125	+	require.Empty(t, covered)
	126	+	}
	127	+
	128	+	func TestCoverageRequiresLinkOrDocument(t *testing.T) {
	129	+	for _, collection := range []bool{true, false} {
	130	+	for _, role := range []uint32{rolePushButton, roleDocumentFrame, roleDocumentWeb, roleLink} {
	131	+	t.Run(fmt.Sprintf("collection=%t/role=%d", collection, role), func(t *testing.T) {
	132	+	tr := newMocktransport(t)
	133	+	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
	134	+	[4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {10, 20, 30, 40}}, map[string]uint32{"/a": role}, !collection))
	135	+	s := &Source{tr: tr}
	136	+	targets, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}, false)
	137	+	require.NoError(t, err)
	138	+	require.Equal(t, role != rolePushButton, covered[1])
	139	+	if isDocument(role) {
	140	+	require.Empty(t, targets)
	141	+	} else {
	142	+	require.Len(t, targets, 1)
	143	+	}
	144	+	})
	145	+	}
	146	+	}
	147	+	}
	148	+
	149	+	var oneWindow = []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}
	150	+
	151	+	// New does not touch the bus; the first Targets call connects, once.
	152	+	func TestConnectsLazilyOnFirstTargets(t *testing.T) {
	153	+	tr := newMocktransport(t)
	154	+	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
	155	+	[4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {1, 1, 9, 9}}, map[string]uint32{"/a": roleLink}, false))
	156	+	c := newMockconnector(t)
	157	+	c.EXPECT().Connect(mock.Anything).Return(tr, nil).Once()
	158	+	s := &Source{dial: c}
	159	+	for range 2 {
	160	+	targets, _, err := s.Targets(context.Background(), oneWindow, false)
	161	+	require.NoError(t, err)
	162	+	require.Len(t, targets, 1)
	163	+	}
	164	+	}
	165	+
	166	+	// Without PIDs nothing is asked and nothing is connected.
	167	+	func TestNoConnectWithoutWindows(t *testing.T) {
	168	+	s := &Source{dial: newMockconnector(t)}
	169	+	_, _, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1}}, false)
	170	+	require.NoError(t, err)
	171	+	}
	172	+
	173	+	// A failed connection is retried no sooner than retryEvery; meanwhile no
	174	+	// targets are returned.
	175	+	func TestConnectRetriesWithBackoff(t *testing.T) {
	176	+	c := newMockconnector(t)
	177	+	c.EXPECT().Connect(mock.Anything).Return(nil, errors.New("down")).Once()
	178	+	s := &Source{dial: c}
	179	+	for range 3 {
	180	+	targets, covered, err := s.Targets(context.Background(), oneWindow, false)
	181	+	require.NoError(t, err)
	182	+	require.Empty(t, targets)
	183	+	require.Empty(t, covered)
	184	+	}
	185	+	tr := newMocktransport(t)
	186	+	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
	187	+	[4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {1, 1, 9, 9}}, map[string]uint32{"/a": roleLink}, false))
	188	+	c.EXPECT().Connect(mock.Anything).Return(tr, nil).Once()
	189	+	s.retryAt = time.Now().Add(-time.Second)
	190	+	targets, _, err := s.Targets(context.Background(), oneWindow, false)
	191	+	require.NoError(t, err)
	192	+	require.Len(t, targets, 1)
	193	+	}
	194	+
	195	+	// A broken connection is closed and dropped; the next attempt reconnects.
	196	+	func TestBrokenConnectionIsDropped(t *testing.T) {
	197	+	tr := newMocktransport(t)
	198	+	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(
	199	+	func(_ time.Time, _ time.Duration, calls []call) []reply {
	200	+	out := make([]reply, len(calls))
	201	+	for i := range out {
	202	+	out[i].err = ErrBroken
	203	+	}
	204	+	return out
	205	+	}).Once()
	206	+	tr.EXPECT().Close().Return(nil).Once()
	207	+	c := newMockconnector(t)
	208	+	c.EXPECT().Connect(mock.Anything).Return(tr, nil).Once()
	209	+	s := &Source{dial: c}
	210	+	_, _, err := s.Targets(context.Background(), oneWindow, false)
	211	+	require.Error(t, err)
	212	+	require.Nil(t, s.tr)
	213	+	targets, _, err := s.Targets(context.Background(), oneWindow, false)
	214	+	require.NoError(t, err)
	215	+	require.Empty(t, targets) // backoff: no new attempt yet
	216	+	}
	217	+
	218	+	const testWait = 20 * time.Millisecond
	219	+
	220	+	// blockedConnector returns a connector whose Connect blocks until release is
	221	+	func blockedConnector(t testing.T, release <-chan struct{}, tr transport, err error, times int) mockconnector {
	222	+	t.Helper()
	223	+	c := newMockconnector(t)
	224	+	c.EXPECT().Connect(mock.Anything).RunAndReturn(func(context.Context) (transport, error) {
	225	+	<-release
	226	+	return tr, err
	227	+	}).Times(times)
	228	+	return c
	229	+	}
	230	+
	231	+	// A silent service cannot hold Targets beyond the wait, is not doubled while
	232	+	// in flight, and its late transport is used.
	233	+	func TestConnectIsBoundedAndNotDoubled(t *testing.T) {
	234	+	release := make(chan struct{})
	235	+	tr := newMocktransport(t)
	236	+	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
	237	+	[4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {1, 1, 9, 9}}, map[string]uint32{"/a": roleLink}, false))
	238	+	c := blockedConnector(t, release, tr, nil, 1)
	239	+	s := &Source{dial: c, wait: testWait}
	240	+	t0 := time.Now()
	241	+	targets, _, err := s.Targets(context.Background(), oneWindow, false)
	242	+	require.NoError(t, err)
	243	+	require.Empty(t, targets)
	244	+	require.Less(t, time.Since(t0), testWait+200*time.Millisecond)
	245	+	require.NotNil(t, s.attempt)
	246	+	// Still in flight: no second Connect (Times(1)), and no blocking.
	247	+	targets, _, err = s.Targets(context.Background(), oneWindow, false)
	248	+	require.NoError(t, err)
	249	+	require.Empty(t, targets)
	250	+	// It connects late: the next call uses it.
	251	+	close(release)
	252	+	require.Eventually(t, func() bool { return len(s.attempt) == 1 }, time.Second, time.Millisecond)
	253	+	targets, _, err = s.Targets(context.Background(), oneWindow, false)
	254	+	require.NoError(t, err)
	255	+	require.Len(t, targets, 1)
	256	+	require.Nil(t, s.attempt)
	257	+	}
	258	+
	259	+	// An abandoned attempt that fails applies the backoff, then one new attempt.
	260	+	func TestAbandonedAttemptErrorBacksOff(t *testing.T) {
	261	+	release := make(chan struct{})
	262	+	tr := newMocktransport(t)
	263	+	tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
	264	+	[4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {1, 1, 9, 9}}, map[string]uint32{"/a": roleLink}, false))
	265	+	c := newMockconnector(t)
	266	+	c.EXPECT().Connect(mock.Anything).RunAndReturn(func(context.Context) (transport, error) {
	267	+	<-release
	268	+	return nil, errors.New("down")
	269	+	}).Once()
	270	+	c.EXPECT().Connect(mock.Anything).Return(tr, nil).Once()
	271	+	s := &Source{dial: c, wait: testWait}
	272	+	targets, _, err := s.Targets(context.Background(), oneWindow, false)
	273	+	require.NoError(t, err)
	274	+	require.Empty(t, targets)
	275	+	close(release)
	276	+	require.Eventually(t, func() bool { return len(s.attempt) == 1 }, time.Second, time.Millisecond)
	277	+	// The error is collected; the backoff runs from its finish time.
	278	+	targets, _, err = s.Targets(context.Background(), oneWindow, false)
	279	+	require.NoError(t, err)
	280	+	require.Empty(t, targets)
	281	+	require.Nil(t, s.attempt)
	282	+	require.WithinDuration(t, time.Now().Add(retryEvery), s.retryAt, time.Second)
	283	+	targets, _, err = s.Targets(context.Background(), oneWindow, false) // backoff: no Connect
	284	+	require.NoError(t, err)
	285	+	require.Empty(t, targets)
	286	+	s.retryAt = time.Now().Add(-time.Second)
	287	+	targets, _, err = s.Targets(context.Background(), oneWindow, false)
	288	+	require.NoError(t, err)
	289	+	require.Len(t, targets, 1)
	290	+	}
	291	+
	292	+	// Close with an attempt in flight closes the late transport; afterwards
	293	+	// Targets finds nothing and never connects.
	294	+	func TestCloseReapsLateTransport(t *testing.T) {
	295	+	release := make(chan struct{})
	296	+	closed := make(chan struct{})
	297	+	tr := newMocktransport(t)
	298	+	tr.EXPECT().Close().RunAndReturn(func() error { close(closed); return nil }).Once()
	299	+	c := blockedConnector(t, release, tr, nil, 1)
	300	+	s := &Source{dial: c, wait: testWait}
	301	+	_, _, err := s.Targets(context.Background(), oneWindow, false)
	302	+	require.NoError(t, err)
	303	+	require.NoError(t, s.Close())
	304	+	close(release)
	305	+	select {
	306	+	case <-closed:
	307	+	case <-time.After(5 * time.Second):
	308	+	t.Fatal("late transport not closed")
	309	+	}
	310	+	targets, covered, err := s.Targets(context.Background(), oneWindow, false)
	311	+	require.NoError(t, err)
	312	+	require.Empty(t, targets)
	313	+	require.Empty(t, covered)
	314	+	require.NoError(t, s.Close()) // idempotent
	315	+	}

A internal/adapters/atspi/transport.go +362 -0

@@ -0,0 +1,362 @@
	1	+	package atspi
	2	+
	3	+	import (
	4	+	"errors"
	5	+	"fmt"
	6	+	"strings"
	7	+	"sync/atomic"
	8	+	"syscall"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/zerobus"
	12	+	"golang.org/x/sys/unix"
	13	+	)
	14	+
	15	+	// D-Bus names, paths and interfaces.
	16	+	const (
	17	+	busDaemon = "org.freedesktop.DBus"
	18	+	busDaemonPath = "/org/freedesktop/DBus"
	19	+	ifaceProps = "org.freedesktop.DBus.Properties"
	20	+
	21	+	a11yBusName = "org.a11y.Bus"
	22	+	a11yBusPath = "/org/a11y/bus"
	23	+	a11yStatus = "org.a11y.Status"
	24	+	registryName = "org.a11y.atspi.Registry"
	25	+	registryRoot = "/org/a11y/atspi/accessible/root"
	26	+
	27	+	ifaceAccessible = "org.a11y.atspi.Accessible"
	28	+	ifaceComponent = "org.a11y.atspi.Component"
	29	+	ifaceCollection = "org.a11y.atspi.Collection"
	30	+
	31	+	// coordWindow is ATSPI_COORD_TYPE_WINDOW: extents relative to the
	32	+	// top-level window.
	33	+	coordWindow = 1
	34	+
	35	+	// maxInflight bounds the calls sent before a reply is read, so the bus
	36	+	// never waits on a full socket while this side waits on it.
	37	+	maxInflight = 64
	38	+	// maxRefs bounds the objects read from one reply.
	39	+	maxRefs = 4096
	40	+	)
	41	+
	42	+	// op is a call the source makes.
	43	+	type op uint8
	44	+
	45	+	const (
	46	+	opChildren op = iota // Accessible.GetChildren -> refs
	47	+	opPID // GetConnectionUnixProcessID(name) -> u
	48	+	opRole // Accessible.GetRole -> u
	49	+	opState // Accessible.GetState -> state
	50	+	opExtents // Component.GetExtents(arg) -> rect
	51	+	opMatches // Collection.GetMatches(rule) -> refs
	52	+	)
	53	+
	54	+	// ref is an accessible object: a bus name and a path.
	55	+	type ref struct{ name, path string }
	56	+
	57	+	// call is one request of a batch. name and path address the object; for
	58	+	// opPID, name is the bus name asked about.
	59	+	type call struct {
	60	+	op op
	61	+	name, path string
	62	+	arg uint32
	63	+	rule matchRule
	64	+	}
	65	+
	66	+	// reply answers the call at the same index of the batch.
	67	+	type reply struct {
	68	+	err error
	69	+	refs []ref
	70	+	u uint32
	71	+	state uint64
	72	+	rect [4]int32 // x, y, w, h
	73	+	}
	74	+
	75	+	// transport sends batches of calls on one bus connection. It is used by one
	76	+	// goroutine at a time.
	77	+	type transport interface {
	78	+	// Do sends calls pipelined and returns one reply per call, in order.
	79	+	// The batch ends at deadline. A call not answered idle after it was
	80	+	// sent gets ErrTimeout. When the connection fails or the deadline
	81	+	// finds calls unanswered, those calls get an error wrapping
	82	+	// ErrBroken: the connection must be dropped.
	83	+	Do(deadline time.Time, idle time.Duration, calls []call) []reply
	84	+	Close() error
	85	+	}
	86	+
	87	+	// ErrTimeout is the error of a call not answered in time.
	88	+	var ErrTimeout = errors.New("atspi: call timed out")
	89	+
	90	+	// busError is a D-Bus error reply.
	91	+	type busError struct{ name, msg string }
	92	+
	93	+	func (e *busError) Error() string {
	94	+	if e.msg == "" {
	95	+	return e.name
	96	+	}
	97	+	return e.name + ": " + e.msg
	98	+	}
	99	+
	100	+	// conn is a transport on a zerobus connection.
	101	+	type conn struct {
	102	+	c *zerobus.Conn
	103	+	raw syscall.RawConn
	104	+	}
	105	+
	106	+	func newConn(c zerobus.Conn) (conn, error) {
	107	+	raw, err := c.SyscallConn()
	108	+	if err != nil {
	109	+	_ = c.Close()
	110	+	return nil, err
	111	+	}
	112	+	return &conn{c: c, raw: raw}, nil
	113	+	}
	114	+
	115	+	func (b *conn) Close() error { return b.c.Close() }
	116	+
	117	+	// readable waits until a message can be read or d passes.
	118	+	func (b *conn) readable(d time.Duration) (bool, error) {
	119	+	if b.c.Buffered() {
	120	+	return true, nil
	121	+	}
	122	+	ms := int((d + time.Millisecond - 1) / time.Millisecond)
	123	+	if ms < 1 {
	124	+	ms = 1
	125	+	}
	126	+	var n int
	127	+	var perr error
	128	+	if err := b.raw.Control(func(fd uintptr) {
	129	+	fds := []unix.PollFd{{Fd: int32(fd), Events: unix.POLLIN}}
	130	+	for {
	131	+	n, perr = unix.Poll(fds, ms)
	132	+	if perr != unix.EINTR {
	133	+	return
	134	+	}
	135	+	}
	136	+	}); err != nil {
	137	+	return false, err
	138	+	}
	139	+	return n > 0, perr
	140	+	}
	141	+
	142	+	// ErrBroken is the error of calls on a connection that failed or was closed
	143	+	// at the deadline; the owner must drop it and connect again.
	144	+	var ErrBroken = errors.New("atspi: connection broken")
	145	+
	146	+	// errDeadline is the error of calls still unanswered at the batch deadline:
	147	+	// late replies and half-read messages leave the connection unusable.
	148	+	var errDeadline = fmt.Errorf("%w: %w", ErrBroken, ErrTimeout)
	149	+
	150	+	// watchdogGrace is how long after the deadline the watchdog closes a socket
	151	+	// that the batch loop could not leave by itself (a stuck read or write).
	152	+	const watchdogGrace = 20 * time.Millisecond
	153	+
	154	+	// Do sends the calls pipelined. A watchdog closes the socket shortly after
	155	+	// deadline, so neither a blocked write nor a read stuck in the middle of a
	156	+	// message can outlive the batch; the connection is then broken. Each call
	157	+	// has its own idle timeout, counted from its send: traffic for other calls
	158	+	// does not extend it.
	159	+	func (b *conn) Do(deadline time.Time, idle time.Duration, calls []call) []reply {
	160	+	out := make([]reply, len(calls))
	161	+	answered := make([]bool, len(calls))
	162	+	type inflight struct {
	163	+	idx int
	164	+	sent time.Time
	165	+	}
	166	+	pending := make(map[uint32]inflight, min(len(calls), maxInflight))
	167	+	next, done := 0, 0
	168	+	var fired atomic.Bool
	169	+	watchdog := time.AfterFunc(time.Until(deadline)+watchdogGrace, func() {
	170	+	fired.Store(true)
	171	+	_ = b.c.Close()
	172	+	})
	173	+	defer watchdog.Stop()
	174	+	// finish gives err to every call without an answer.
	175	+	finish := func(err error) []reply {
	176	+	if !errors.Is(err, ErrBroken) && (fired.Load() \|\| !errors.Is(err, ErrTimeout)) {
	177	+	err = fmt.Errorf("%w: %w", ErrBroken, err)
	178	+	}
	179	+	if errors.Is(err, ErrBroken) {
	180	+	_ = b.c.Close() // late replies would desynchronize the next batch
	181	+	}
	182	+	for i := range out {
	183	+	if !answered[i] {
	184	+	out[i].err = err
	185	+	}
	186	+	}
	187	+	return out
	188	+	}
	189	+	for done < len(calls) {
	190	+	for next < len(calls) && len(pending) < maxInflight {
	191	+	if !time.Now().Before(deadline) {
	192	+	return finish(errDeadline)
	193	+	}
	194	+	serial, err := b.send(calls[next])
	195	+	if err != nil {
	196	+	return finish(err)
	197	+	}
	198	+	pending[serial] = inflight{idx: next, sent: time.Now()}
	199	+	next++
	200	+	}
	201	+	// Calls that waited longer than idle get no reply.
	202	+	now := time.Now()
	203	+	var oldest time.Time
	204	+	for serial, p := range pending {
	205	+	if now.Sub(p.sent) >= idle {
	206	+	out[p.idx].err = ErrTimeout
	207	+	answered[p.idx] = true
	208	+	done++
	209	+	delete(pending, serial)
	210	+	} else if oldest.IsZero() \|\| p.sent.Before(oldest) {
	211	+	oldest = p.sent
	212	+	}
	213	+	}
	214	+	if len(pending) == 0 {
	215	+	continue
	216	+	}
	217	+	wait := min(time.Until(oldest.Add(idle)), time.Until(deadline))
	218	+	if wait <= 0 {
	219	+	if !time.Now().Before(deadline) {
	220	+	break
	221	+	}
	222	+	continue
	223	+	}
	224	+	ok, err := b.readable(wait)
	225	+	if err != nil {
	226	+	return finish(err)
	227	+	}
	228	+	if !ok {
	229	+	continue
	230	+	}
	231	+	m, err := b.c.ReadMessage()
	232	+	if err != nil {
	233	+	return finish(err)
	234	+	}
	235	+	p, found := pending[m.ReplySerial]
	236	+	if !found \|\| (m.Type != zerobus.TypeMethodReturn && m.Type != zerobus.TypeError) {
	237	+	// A signal, or the late reply of an earlier batch.
	238	+	continue
	239	+	}
	240	+	delete(pending, m.ReplySerial)
	241	+	decode(m, calls[p.idx], &out[p.idx])
	242	+	answered[p.idx] = true
	243	+	done++
	244	+	}
	245	+	if done == len(calls) {
	246	+	return out
	247	+	}
	248	+	return finish(errDeadline)
	249	+	}
	250	+
	251	+	// send builds and sends one call and returns its serial.
	252	+	func (b *conn) send(c call) (uint32, error) {
	253	+	switch c.op {
	254	+	case opChildren:
	255	+	b.c.NewCall(c.name, c.path, ifaceAccessible, "GetChildren", "")
	256	+	case opPID:
	257	+	b.c.NewCall(busDaemon, busDaemonPath, busDaemon, "GetConnectionUnixProcessID", "s").Str(c.name)
	258	+	case opRole:
	259	+	b.c.NewCall(c.name, c.path, ifaceAccessible, "GetRole", "")
	260	+	case opState:
	261	+	b.c.NewCall(c.name, c.path, ifaceAccessible, "GetState", "")
	262	+	case opExtents:
	263	+	b.c.NewCall(c.name, c.path, ifaceComponent, "GetExtents", "u").Uint32(c.arg)
	264	+	case opMatches:
	265	+	e := b.c.NewCall(c.name, c.path, ifaceCollection, "GetMatches", "(aiia{ss}iaiiasib)uib")
	266	+	encodeRule(e, c.rule)
	267	+	e.Uint32(sortCanonical)
	268	+	e.Int32(matchCount)
	269	+	e.Bool(true)
	270	+	}
	271	+	return b.c.Send()
	272	+	}
	273	+
	274	+	// encodeRule writes the MatchRule struct (aiia{ss}iaiiasib): states, state
	275	+	// match, attributes, attribute match, roles, role match, interfaces,
	276	+	// interface match, invert.
	277	+	func encodeRule(e *zerobus.Encoder, r matchRule) {
	278	+	e.Struct()
	279	+	a := e.BeginArray('i')
	280	+	for _, w := range stateBits(stateShowing, stateVisible) {
	281	+	e.Int32(w)
	282	+	}
	283	+	e.EndArray(a)
	284	+	e.Int32(matchAll)
	285	+	m := e.BeginArray('{')
	286	+	e.EndArray(m)
	287	+	e.Int32(matchAll)
	288	+	a = e.BeginArray('i')
	289	+	for _, w := range roleBits(r.roles()) {
	290	+	e.Int32(w)
	291	+	}
	292	+	e.EndArray(a)
	293	+	e.Int32(matchAny)
	294	+	s := e.BeginArray('s')
	295	+	e.EndArray(s)
	296	+	e.Int32(matchAll)
	297	+	e.Bool(false)
	298	+	}
	299	+
	300	+	// decode reads the reply m of c into out.
	301	+	func decode(m zerobus.Message, c call, out reply) {
	302	+	if m.Type == zerobus.TypeError {
	303	+	be := &busError{name: strings.Clone(m.ErrorName)}
	304	+	if strings.HasPrefix(m.Signature, "s") {
	305	+	be.msg = strings.Clone(m.Body().Str())
	306	+	}
	307	+	out.err = be
	308	+	return
	309	+	}
	310	+	r := m.Body()
	311	+	want := func(sig string) bool {
	312	+	if m.Signature != sig {
	313	+	out.err = fmt.Errorf("atspi: reply signature %q, want %q", m.Signature, sig)
	314	+	return false
	315	+	}
	316	+	return true
	317	+	}
	318	+	switch c.op {
	319	+	case opChildren, opMatches:
	320	+	if !want("a(so)") {
	321	+	return
	322	+	}
	323	+	end := r.Array('(')
	324	+	for r.More(end) {
	325	+	if len(out.refs) >= maxRefs {
	326	+	break
	327	+	}
	328	+	r.Struct()
	329	+	name := r.Str()
	330	+	path := r.ObjectPath()
	331	+	if r.Err() != nil {
	332	+	break
	333	+	}
	334	+	out.refs = append(out.refs, ref{strings.Clone(name), strings.Clone(path)})
	335	+	}
	336	+	case opPID, opRole:
	337	+	if want("u") {
	338	+	out.u = r.Uint32()
	339	+	}
	340	+	case opState:
	341	+	if !want("au") {
	342	+	return
	343	+	}
	344	+	end := r.Array('u')
	345	+	var words [2]uint32
	346	+	for i := 0; r.More(end); i++ {
	347	+	w := r.Uint32()
	348	+	if i < len(words) {
	349	+	words[i] = w
	350	+	}
	351	+	}
	352	+	out.state = uint64(words[0]) \| uint64(words[1])<<32
	353	+	case opExtents:
	354	+	if want("(iiii)") {
	355	+	r.Struct()
	356	+	out.rect = [4]int32{r.Int32(), r.Int32(), r.Int32(), r.Int32()}
	357	+	}
	358	+	}
	359	+	if out.err == nil && r.Err() != nil {
	360	+	out.err = r.Err()
	361	+	}
	362	+	}

A internal/adapters/atspi/transport_mock_test.go +156 -0

@@ -0,0 +1,156 @@
	1	+	// Code generated by mockery; DO NOT EDIT.
	2	+	// github.com/vektra/mockery
	3	+	// template: testify
	4	+
	5	+	package atspi
	6	+
	7	+	import (
	8	+	"time"
	9	+
	10	+	mock "github.com/stretchr/testify/mock"
	11	+	)
	12	+
	13	+	// newMocktransport creates a new instance of mocktransport. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
	14	+	// The first argument is typically a *testing.T value.
	15	+	func newMocktransport(t interface {
	16	+	mock.TestingT
	17	+	Cleanup(func())
	18	+	}) *mocktransport {
	19	+	if helper, ok := t.(interface{ Helper() }); ok {
	20	+	helper.Helper()
	21	+	}
	22	+
	23	+	mock := &mocktransport{}
	24	+	mock.Mock.Test(t)
	25	+
	26	+	t.Cleanup(func() {
	27	+	if helper, ok := t.(interface{ Helper() }); ok {
	28	+	helper.Helper()
	29	+	}
	30	+	mock.AssertExpectations(t)
	31	+	})
	32	+
	33	+	return mock
	34	+	}
	35	+
	36	+	// mocktransport is an autogenerated mock type for the transport type
	37	+	type mocktransport struct {
	38	+	mock.Mock
	39	+	}
	40	+
	41	+	type mocktransport_Expecter struct {
	42	+	mock *mock.Mock
	43	+	}
	44	+
	45	+	func (_m mocktransport) EXPECT() mocktransport_Expecter {
	46	+	return &mocktransport_Expecter{mock: &_m.Mock}
	47	+	}
	48	+
	49	+	// Close provides a mock function for the type mocktransport
	50	+	func (_mock *mocktransport) Close() error {
	51	+	ret := _mock.Called()
	52	+
	53	+	if len(ret) == 0 {
	54	+	panic("no return value specified for Close")
	55	+	}
	56	+
	57	+	var r0 error
	58	+	if returnFunc, ok := ret.Get(0).(func() error); ok {
	59	+	r0 = returnFunc()
	60	+	} else {
	61	+	r0 = ret.Error(0)
	62	+	}
	63	+	return r0
	64	+	}
	65	+
	66	+	// mocktransport_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
	67	+	type mocktransport_Close_Call struct {
	68	+	*mock.Call
	69	+	}
	70	+
	71	+	// Close is a helper method to define mock.On call
	72	+	func (_e mocktransport_Expecter) Close() mocktransport_Close_Call {
	73	+	return &mocktransport_Close_Call{Call: _e.mock.On("Close")}
	74	+	}
	75	+
	76	+	func (_c mocktransport_Close_Call) Run(run func()) mocktransport_Close_Call {
	77	+	_c.Call.Run(func(args mock.Arguments) {
	78	+	run()
	79	+	})
	80	+	return _c
	81	+	}
	82	+
	83	+	func (_c mocktransport_Close_Call) Return(err error) mocktransport_Close_Call {
	84	+	_c.Call.Return(err)
	85	+	return _c
	86	+	}
	87	+
	88	+	func (_c mocktransport_Close_Call) RunAndReturn(run func() error) mocktransport_Close_Call {
	89	+	_c.Call.Return(run)
	90	+	return _c
	91	+	}
	92	+
	93	+	// Do provides a mock function for the type mocktransport
	94	+	func (_mock *mocktransport) Do(deadline time.Time, idle time.Duration, calls []call) []reply {
	95	+	ret := _mock.Called(deadline, idle, calls)
	96	+
	97	+	if len(ret) == 0 {
	98	+	panic("no return value specified for Do")
	99	+	}
	100	+
	101	+	var r0 []reply
	102	+	if returnFunc, ok := ret.Get(0).(func(time.Time, time.Duration, []call) []reply); ok {
	103	+	r0 = returnFunc(deadline, idle, calls)
	104	+	} else {
	105	+	if ret.Get(0) != nil {
	106	+	r0 = ret.Get(0).([]reply)
	107	+	}
	108	+	}
	109	+	return r0
	110	+	}
	111	+
	112	+	// mocktransport_Do_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Do'
	113	+	type mocktransport_Do_Call struct {
	114	+	*mock.Call
	115	+	}
	116	+
	117	+	// Do is a helper method to define mock.On call
	118	+	// - deadline time.Time
	119	+	// - idle time.Duration
	120	+	// - calls []call
	121	+	func (_e mocktransport_Expecter) Do(deadline any, idle any, calls any) mocktransport_Do_Call {
	122	+	return &mocktransport_Do_Call{Call: _e.mock.On("Do", deadline, idle, calls)}
	123	+	}
	124	+
	125	+	func (_c mocktransport_Do_Call) Run(run func(deadline time.Time, idle time.Duration, calls []call)) mocktransport_Do_Call {
	126	+	_c.Call.Run(func(args mock.Arguments) {
	127	+	var arg0 time.Time
	128	+	if args[0] != nil {
	129	+	arg0 = args[0].(time.Time)
	130	+	}
	131	+	var arg1 time.Duration
	132	+	if args[1] != nil {
	133	+	arg1 = args[1].(time.Duration)
	134	+	}
	135	+	var arg2 []call
	136	+	if args[2] != nil {
	137	+	arg2 = args[2].([]call)
	138	+	}
	139	+	run(
	140	+	arg0,
	141	+	arg1,
	142	+	arg2,
	143	+	)
	144	+	})
	145	+	return _c
	146	+	}
	147	+
	148	+	func (_c mocktransport_Do_Call) Return(replyMoqParams []reply) mocktransport_Do_Call {
	149	+	_c.Call.Return(replyMoqParams)
	150	+	return _c
	151	+	}
	152	+
	153	+	func (_c mocktransport_Do_Call) RunAndReturn(run func(deadline time.Time, idle time.Duration, calls []call) []reply) mocktransport_Do_Call {
	154	+	_c.Call.Return(run)
	155	+	return _c
	156	+	}

M internal/adapters/config/config.go +74 -0


@@ -114,6 +114,7 @@ var actions = map[string]bool{
114	"move-workspace-prev": true,	114		"move-workspace-prev": true,
115	"move-workspace-next": true,	115		"move-workspace-next": true,
116	"toggle-floating": true,	116		"toggle-floating": true,
		117	+	"hint": true,
117	}	118		}
118	var components = map[string]bool{"core": true, "wayland": true, "input": true, "drm": true, "seat": true, "render": true, "sync": true, "config": true, "app": true}	119		var components = map[string]bool{"core": true, "wayland": true, "input": true, "drm": true, "seat": true, "render": true, "sync": true, "config": true, "app": true}
119	var color = regexp.MustCompile(`^#[0-9a-fA-F]{6}$`)	120		var color = regexp.MustCompile(`^#[0-9a-fA-F]{6}$`)

@@ -163,6 +164,9 @@ func Defaults() ports.Config {
163	c.Terminal.Command = []string{"foot"}	164		c.Terminal.Command = []string{"foot"}
164	}	165		}
165	c.Terminal.AutoOpen = "first"	166		c.Terminal.AutoOpen = "first"
		167	+	// Hint mode is experimental: off unless the config turns it on.
		168	+	c.Hint.Enable = false
		169	+	c.Hint.TextClick = ports.ModCtrl \| ports.ModShift
166	c.Xwayland = "xwayland-satellite"	170		c.Xwayland = "xwayland-satellite"
167	c.Background.Color = "#111111"	171		c.Background.Color = "#111111"
168	c.Floating.Dim = 0.3	172		c.Floating.Dim = 0.3

@@ -847,6 +851,46 @@ func set(c *ports.Config, key, v string) error {
847	return err	851		return err
848	}	852		}
849	c.Focus.Flash = b	853		c.Focus.Flash = b
		854	+	case "hint.enable":
		855	+	b, err := onOff(v)
		856	+	if err != nil {
		857	+	return err
		858	+	}
		859	+	c.Hint.Enable = b
		860	+	case "hint.size":
		861	+	if v == "auto" {
		862	+	c.Hint.Size = 0
		863	+	break
		864	+	}
		865	+	n, err := strconv.ParseFloat(v, 64)
		866	+	if err != nil \|\| math.IsNaN(n) \|\| n < 6 \|\| n > 72 {
		867	+	return fmt.Errorf("must be auto or a number from 6 to 72 (logical pixels)")
		868	+	}
		869	+	c.Hint.Size = n
		870	+	case "hint.background", "hint.foreground":
		871	+	if v != "" && !color.MatchString(v) {
		872	+	return fmt.Errorf("must be #rrggbb or empty (derived)")
		873	+	}
		874	+	if key == "hint.background" {
		875	+	c.Hint.Background = v
		876	+	} else {
		877	+	c.Hint.Foreground = v
		878	+	}
		879	+	case "hint.targets":
		880	+	switch v {
		881	+	case "clickable":
		882	+	c.Hint.LinksOnly = false
		883	+	case "links":
		884	+	c.Hint.LinksOnly = true
		885	+	default:
		886	+	return fmt.Errorf("must be clickable or links")
		887	+	}
		888	+	case "hint.text-click":
		889	+	m, err := parseTextClick(v)
		890	+	if err != nil {
		891	+	return err
		892	+	}
		893	+	c.Hint.TextClick = m
850	case "focus.flash-width":	894		case "focus.flash-width":
851	n, err := strconv.Atoi(v)	895		n, err := strconv.Atoi(v)
852	if err != nil \|\| n < 1 \|\| n > 8 {	896		if err != nil \|\| n < 1 \|\| n > 8 {

@@ -1270,6 +1314,36 @@ func checkAction(v string) error {
1270	return nil	1314		return nil
1271	}	1315		}
1272		1316
		1317	+	// parseTextClick parses hint.text-click: `+`-separated modifier names as in
		1318	+	// binds, or none. cmd is an alias of super here (it is not resolved through
		1319	+	// keyboard.cmd, so the result does not depend on key order).
		1320	+	func parseTextClick(v string) (ports.Mods, error) {
		1321	+	if strings.ToLower(v) == "none" {
		1322	+	return 0, nil
		1323	+	}
		1324	+	var mods ports.Mods
		1325	+	for _, p := range strings.Split(v, "+") {
		1326	+	var m ports.Mods
		1327	+	switch strings.ToLower(strings.TrimSpace(p)) {
		1328	+	case "shift":
		1329	+	m = ports.ModShift
		1330	+	case "ctrl":
		1331	+	m = ports.ModCtrl
		1332	+	case "alt":
		1333	+	m = ports.ModAlt
		1334	+	case "super", "cmd":
		1335	+	m = ports.ModSuper
		1336	+	default:
		1337	+	return 0, fmt.Errorf("unknown modifier %q (use shift, ctrl, alt, super or cmd joined by +, or none)", p)
		1338	+	}
		1339	+	if mods&m != 0 {
		1340	+	return 0, fmt.Errorf("duplicate modifier %q", p)
		1341	+	}
		1342	+	mods \|= m
		1343	+	}
		1344	+	return mods, nil
		1345	+	}
		1346	+
1273	// parseCombo turns `ctrl+cmd+é` into the canonical `Cmd+Ctrl+eacute`:	1347		// parseCombo turns `ctrl+cmd+é` into the canonical `Cmd+Ctrl+eacute`:
1274	// modifiers sorted, the key as an xkb keysym name.	1348		// modifiers sorted, the key as an xkb keysym name.
1275	func parseCombo(s string) (string, error) {	1349		func parseCombo(s string) (string, error) {

M internal/adapters/config/config_test.go +92 -0


@@ -953,3 +953,95 @@ func TestLoadRejectsFIFO(t *testing.T) {
953	t.Fatal("Load blocked on a FIFO")	953		t.Fatal("Load blocked on a FIFO")
954	}	954		}
955	}	955		}
		956	+
		957	+	func TestHintTargets(t *testing.T) {
		958	+	if Defaults().Hint.LinksOnly {
		959	+	t.Fatal("hint targets default must be clickable")
		960	+	}
		961	+	c, w := parseString(t, "hint.targets = links\n")
		962	+	if !c.Hint.LinksOnly \|\| len(w) != 0 {
		963	+	t.Fatal(c.Hint, w)
		964	+	}
		965	+	c, w = parseString(t, "hint.targets = clickable\n")
		966	+	if c.Hint.LinksOnly \|\| len(w) != 0 {
		967	+	t.Fatal(c.Hint, w)
		968	+	}
		969	+	// An invalid value warns and keeps the default.
		970	+	c, w = parseString(t, "hint.targets = buttons\n")
		971	+	if c.Hint.LinksOnly \|\| len(w) != 1 {
		972	+	t.Fatal(c.Hint, w)
		973	+	}
		974	+	c, w = parseString(t, "bind.cmd+g = hint\n")
		975	+	if len(w) != 0 \|\| c.Binds["Cmd+g"] != "hint" {
		976	+	t.Fatal(c.Binds["Cmd+g"], w)
		977	+	}
		978	+	}
		979	+
		980	+	func TestHintStyleConfig(t *testing.T) {
		981	+	c, warnings := parseString(t, "hint.size = 17.5\nhint.background = #89b4fa\nhint.foreground = #111111\n")
		982	+	if len(warnings) != 0 \|\| c.Hint.Size != 17.5 \|\| c.Hint.Background != "#89b4fa" \|\| c.Hint.Foreground != "#111111" {
		983	+	t.Fatalf("style: %+v, warnings: %v", c.Hint, warnings)
		984	+	}
		985	+	c, warnings = parseString(t, "hint.size = auto\nhint.background =\nhint.foreground =\n")
		986	+	if len(warnings) != 0 \|\| c.Hint.Size != 0 \|\| c.Hint.Background != "" \|\| c.Hint.Foreground != "" {
		987	+	t.Fatalf("automatic: %+v %v", c.Hint, warnings)
		988	+	}
		989	+	for _, line := range []string{"hint.size = 0", "hint.size = -1", "hint.size = NaN", "hint.size = Inf", "hint.size = huge", "hint.size = 5.9", "hint.size = 72.1", "hint.size = 1e9", "hint.background = red", "hint.foreground = #fff"} {
		990	+	_, warnings := parseString(t, line+"\n")
		991	+	if len(warnings) == 0 {
		992	+	t.Errorf("expected validate-config warning for %s", line)
		993	+	}
		994	+	}
		995	+	}
		996	+
		997	+	func TestHintSizeBounds(t *testing.T) {
		998	+	for _, v := range []string{"6", "72", "13"} {
		999	+	c, w := parseString(t, "hint.size = "+v+"\n")
		1000	+	if len(w) != 0 \|\| c.Hint.Size == 0 {
		1001	+	t.Errorf("hint.size = %s: %+v %v", v, c.Hint, w)
		1002	+	}
		1003	+	}
		1004	+	}
		1005	+
		1006	+	func TestHintTextClick(t *testing.T) {
		1007	+	if got := Defaults().Hint.TextClick; got != ports.ModCtrl\|ports.ModShift {
		1008	+	t.Fatalf("default %d", got)
		1009	+	}
		1010	+	for in, want := range map[string]ports.Mods{
		1011	+	"ctrl+shift": ports.ModCtrl \| ports.ModShift,
		1012	+	"Alt": ports.ModAlt,
		1013	+	"super": ports.ModSuper,
		1014	+	"cmd+shift": ports.ModSuper \| ports.ModShift,
		1015	+	"none": 0,
		1016	+	"shift+ctrl+alt": ports.ModShift \| ports.ModCtrl \| ports.ModAlt,
		1017	+	} {
		1018	+	c, w := parseString(t, "hint.text-click = "+in+"\n")
		1019	+	if c.Hint.TextClick != want \|\| len(w) != 0 {
		1020	+	t.Errorf("%q: %d %v", in, c.Hint.TextClick, w)
		1021	+	}
		1022	+	}
		1023	+	for _, bad := range []string{"hyper", "ctrl+", "ctrl+ctrl", "ctrl+none", ""} {
		1024	+	c, w := parseString(t, "hint.text-click = "+bad+"\n")
		1025	+	if len(w) != 1 \|\| c.Hint.TextClick != ports.ModCtrl\|ports.ModShift {
		1026	+	t.Errorf("%q: %d %v", bad, c.Hint.TextClick, w)
		1027	+	}
		1028	+	}
		1029	+	}
		1030	+
		1031	+	func TestHintEnable(t *testing.T) {
		1032	+	if Defaults().Hint.Enable {
		1033	+	t.Fatal("hint mode must be off by default")
		1034	+	}
		1035	+	c, w := parseString(t, "hint.enable = on\n")
		1036	+	if !c.Hint.Enable \|\| len(w) != 0 {
		1037	+	t.Fatal(c.Hint, w)
		1038	+	}
		1039	+	c, w = parseString(t, "hint.enable = off\n")
		1040	+	if c.Hint.Enable \|\| len(w) != 0 {
		1041	+	t.Fatal(c.Hint, w)
		1042	+	}
		1043	+	c, w = parseString(t, "hint.enable = maybe\n")
		1044	+	if c.Hint.Enable \|\| len(w) != 1 {
		1045	+	t.Fatal(c.Hint, w)
		1046	+	}
		1047	+	}

M internal/adapters/drm/overlay.go +3 -0


@@ -70,6 +70,9 @@ func overlayCandidate(s ports.Scene, surfaces map[ports.WindowID]ports.SurfaceCo
70	if s.Flash != nil {	70		if s.Flash != nil {
71	return ports.SceneWindow{}, ports.SurfaceContent{}, colorBypass, "focus_flash"	71		return ports.SceneWindow{}, ports.SurfaceContent{}, colorBypass, "focus_flash"
72	}	72		}
		73	+	if len(s.HintLabels) > 0 {
		74	+	return ports.SceneWindow{}, ports.SurfaceContent{}, colorBypass, "hint_labels"
		75	+	}
73	if s.Transform != 0 {	76		if s.Transform != 0 {
74	return ports.SceneWindow{}, ports.SurfaceContent{}, colorBypass, "output_transform"	77		return ports.SceneWindow{}, ports.SurfaceContent{}, colorBypass, "output_transform"
75	}	78		}

M internal/adapters/drm/scanout.go +4 -0


@@ -94,6 +94,10 @@ func scanoutCandidate(s ports.Scene, surfaces map[ports.WindowID]ports.SurfaceCo
94	if s.Flash != nil {	94		if s.Flash != nil {
95	return ports.SurfaceContent{}, "focus_flash"	95		return ports.SurfaceContent{}, "focus_flash"
96	}	96		}
		97	+	// Hint labels are composed over the window.
		98	+	if len(s.HintLabels) > 0 {
		99	+	return ports.SurfaceContent{}, "hint_labels"
		100	+	}
97	if s.Transform != 0 {	101		if s.Transform != 0 {
98	// The scanout buffer would have to be pre-rotated: rotated outputs compose.	102		// The scanout buffer would have to be pre-rotated: rotated outputs compose.
99	return ports.SurfaceContent{}, "output_transform"	103		return ports.SurfaceContent{}, "output_transform"

A internal/adapters/hintlabel/hintlabel.go +137 -0

@@ -0,0 +1,137 @@
	1	+	// Package hintlabel rasterizes the keyboard hint labels on the CPU: a small
	2	+	// box with the label text in Go Mono, as premultiplied BGRA pixels for the
	3	+	// renderer to upload and draw.
	4	+	package hintlabel
	5	+
	6	+	import (
	7	+	"image"
	8	+	"image/color"
	9	+	"image/draw"
	10	+	"math"
	11	+	"sync"
	12	+
	13	+	"golang.org/x/image/font"
	14	+	"golang.org/x/image/font/gofont/gomono"
	15	+	"golang.org/x/image/font/opentype"
	16	+	"golang.org/x/image/math/fixed"
	17	+	)
	18	+
	19	+	const (
	20	+	// fontSize is the text size in logical pixels.
	21	+	fontSize = 13
	22	+	// padX and padY are the padding around the text in logical pixels.
	23	+	padX, padY = 3, 1
	24	+	// maxEntries bounds the cache: it is cleared when it grows past it.
	25	+	maxEntries = 1024
	26	+	)
	27	+
	28	+	var (
	29	+	parseOnce sync.Once
	30	+	parsed *opentype.Font
	31	+	parseErr error
	32	+	)
	33	+
	34	+	type key struct {
	35	+	text string
	36	+	typed int
	37	+	scale float64
	38	+	style Style
	39	+	}
	40	+
	41	+	type entry struct {
	42	+	pix []byte
	43	+	w, h int
	44	+	}
	45	+
	46	+	// Raster draws and caches labels. It is not safe for concurrent use: the
	47	+	// renderer owns it.
	48	+	type Raster struct {
	49	+	faces map[float64]font.Face
	50	+	cache map[key]entry
	51	+	}
	52	+
	53	+	// Label returns the label box for text, whose first typed characters are
	54	+	// drawn dimmed, as premultiplied BGRA at scale physical pixels per logical
	55	+	// pixel. The slice is owned by the cache and stays valid until the cache is
	56	+	// cleared by a later call: do not modify it. Empty text, or a font that does
	57	+	// not load, yields a nil slice.
	58	+	func (r *Raster) Label(text string, typed int, scale float64, styles ...Style) (pix []byte, w, h int) {
	59	+	if text == "" {
	60	+	return nil, 0, 0
	61	+	}
	62	+	if scale <= 0 \|\| math.IsNaN(scale) \|\| math.IsInf(scale, 0) {
	63	+	scale = 1
	64	+	}
	65	+	typed = max(0, min(typed, len(text)))
	66	+	bg, fg := Colors("", "", "")
	67	+	style := Style{fontSize * scale, bg, fg}
	68	+	if len(styles) > 0 {
	69	+	style = styles[0]
	70	+	}
	71	+	k := key{text, typed, scale, style}
	72	+	if e, ok := r.cache[k]; ok {
	73	+	return e.pix, e.w, e.h
	74	+	}
	75	+	face := r.face(style.FontPx)
	76	+	if face == nil {
	77	+	return nil, 0, 0
	78	+	}
	79	+	if r.cache == nil \|\| len(r.cache) >= maxEntries {
	80	+	r.cache = map[key]entry{}
	81	+	}
	82	+	e := render(face, text, typed, style)
	83	+	r.cache[k] = e
	84	+	return e.pix, e.w, e.h
	85	+	}
	86	+
	87	+	func (r *Raster) face(scale float64) font.Face {
	88	+	if f, ok := r.faces[scale]; ok {
	89	+	return f
	90	+	}
	91	+	parseOnce.Do(func() { parsed, parseErr = opentype.Parse(gomono.TTF) })
	92	+	if parseErr != nil {
	93	+	return nil
	94	+	}
	95	+	f, err := opentype.NewFace(parsed, &opentype.FaceOptions{Size: scale, DPI: 72, Hinting: font.HintingFull})
	96	+	if err != nil {
	97	+	return nil
	98	+	}
	99	+	if r.faces == nil \|\| len(r.faces) >= 16 {
	100	+	for _, old := range r.faces {
	101	+	_ = old.Close()
	102	+	}
	103	+	r.faces = map[float64]font.Face{}
	104	+	}
	105	+	r.faces[scale] = f
	106	+	return f
	107	+	}
	108	+
	109	+	// render draws an opaque box inside a one-physical-pixel translucent outline.
	110	+	// All colours are premultiplied.
	111	+	func render(face font.Face, text string, typed int, style Style) entry {
	112	+	scale := style.FontPx / fontSize
	113	+	background, ink := style.Background, style.Foreground
	114	+	typedInk := mix(ink, background)
	115	+	outline := color.RGBA{uint8(math.Round(float64(ink.R) * .35)), uint8(math.Round(float64(ink.G) * .35)), uint8(math.Round(float64(ink.B) * .35)), 89}
	116	+	m := face.Metrics()
	117	+	px := int(math.Round(padX * scale))
	118	+	py := int(math.Round(padY * scale))
	119	+	border := 1
	120	+	adv := font.MeasureString(face, text).Ceil()
	121	+	w := adv + 2px + 2border
	122	+	h := m.Height.Ceil() + 2py + 2border
	123	+	img := image.NewRGBA(image.Rect(0, 0, w, h))
	124	+	draw.Draw(img, img.Bounds(), image.NewUniform(outline), image.Point{}, draw.Src)
	125	+	draw.Draw(img, img.Bounds().Inset(border), image.NewUniform(background), image.Point{}, draw.Src)
	126	+	d := font.Drawer{Dst: img, Face: face, Dot: fixed.Point26_6{X: fixed.I(border + px), Y: fixed.I(border+py) + m.Ascent}}
	127	+	d.Src = image.NewUniform(typedInk)
	128	+	d.DrawString(text[:typed])
	129	+	d.Src = image.NewUniform(ink)
	130	+	d.DrawString(text[typed:])
	131	+	// Premultiplied RGBA to BGRA in place.
	132	+	pix := img.Pix
	133	+	for i := 0; i+3 < len(pix); i += 4 {
	134	+	pix[i], pix[i+2] = pix[i+2], pix[i]
	135	+	}
	136	+	return entry{pix: pix, w: w, h: h}
	137	+	}

A internal/adapters/hintlabel/hintlabel_test.go +60 -0

@@ -0,0 +1,60 @@
	1	+	package hintlabel
	2	+
	3	+	import (
	4	+	"testing"
	5	+
	6	+	"github.com/stretchr/testify/require"
	7	+	)
	8	+
	9	+	func TestHintLabelSize(t *testing.T) {
	10	+	var r Raster
	11	+	_, w1, h1 := r.Label("f", 0, 1)
	12	+	_, w2, h2 := r.Label("fj", 0, 1)
	13	+	require.Greater(t, w1, 0)
	14	+	require.Greater(t, w2, w1, "a longer label is wider")
	15	+	require.Equal(t, h1, h2)
	16	+	_, w3, h3 := r.Label("fj", 0, 2)
	17	+	// The 1 px border does not scale: compare the boxes without it.
	18	+	require.InDelta(t, 2*(w2-2), w3-2, 1)
	19	+	require.InDelta(t, 2*(h2-2), h3-2, 1)
	20	+	}
	21	+
	22	+	func TestHintLabelCache(t *testing.T) {
	23	+	var r Raster
	24	+	a, w, h := r.Label("fj", 1, 1)
	25	+	b, _, _ := r.Label("fj", 1, 1)
	26	+	require.Len(t, a, wh4)
	27	+	require.Same(t, &a[0], &b[0], "same key returns the cached slice")
	28	+	c, _, _ := r.Label("fj", 0, 1)
	29	+	require.NotSame(t, &a[0], &c[0], "typed count is part of the key")
	30	+	}
	31	+
	32	+	func TestHintLabelPremultiplied(t *testing.T) {
	33	+	var r Raster
	34	+	pix, w, h := r.Label("ab", 1, 1.5)
	35	+	require.Len(t, pix, wh4)
	36	+	for i := 0; i < len(pix); i += 4 {
	37	+	a := pix[i+3]
	38	+	require.GreaterOrEqual(t, a, pix[i])
	39	+	require.GreaterOrEqual(t, a, pix[i+1])
	40	+	require.GreaterOrEqual(t, a, pix[i+2])
	41	+	}
	42	+	}
	43	+
	44	+	func TestHintLabelTypedIsDimmed(t *testing.T) {
	45	+	var r Raster
	46	+	plain, _, _ := r.Label("fj", 0, 1)
	47	+	dim, _, _ := r.Label("fj", 1, 1)
	48	+	require.NotEqual(t, plain, dim)
	49	+	}
	50	+
	51	+	func TestHintLabelEmptyAndCacheBound(t *testing.T) {
	52	+	var r Raster
	53	+	pix, w, h := r.Label("", 0, 1)
	54	+	require.Nil(t, pix)
	55	+	require.Zero(t, w+h)
	56	+	for i := 0; i < maxEntries+10; i++ {
	57	+	r.Label("a", 0, 1+float64(i)/100000)
	58	+	}
	59	+	require.LessOrEqual(t, len(r.cache), maxEntries)
	60	+	}

A internal/adapters/hintlabel/style.go +102 -0

@@ -0,0 +1,102 @@
	1	+	package hintlabel
	2	+
	3	+	import (
	4	+	"image/color"
	5	+	"math"
	6	+	"strconv"
	7	+	)
	8	+
	9	+	// Style is the resolved, physical-pixel label style and is also a cache key.
	10	+	type Style struct {
	11	+	FontPx float64
	12	+	Background, Foreground color.RGBA
	13	+	}
	14	+
	15	+	// FontPixels resolves an explicit logical size or the output's pixel density.
	16	+	// Both millimetre dimensions must be plausible for the untransformed mode.
	17	+	func FontPixels(width, height, mmWidth, mmHeight int, scale, logical float64) float64 {
	18	+	return min(maxFontPx, fontPixels(width, height, mmWidth, mmHeight, scale, logical))
	19	+	}
	20	+
	21	+	// maxFontPx bounds the physical font size, whatever the configuration or the
	22	+	// scale ask for.
	23	+	const maxFontPx = 256
	24	+
	25	+	func fontPixels(width, height, mmWidth, mmHeight int, scale, logical float64) float64 {
	26	+	if scale <= 0 \|\| math.IsNaN(scale) \|\| math.IsInf(scale, 0) {
	27	+	scale = 1
	28	+	}
	29	+	if logical > 0 && !math.IsInf(logical, 0) {
	30	+	return logical * scale
	31	+	}
	32	+	if width > 0 && height > 0 && mmWidth > 0 && mmHeight > 0 {
	33	+	ratio := (float64(width) / float64(height)) / (float64(mmWidth) / float64(mmHeight))
	34	+	if math.Abs(ratio-1) <= .2 {
	35	+	return min(math.Round(40scale), max(math.Round(13scale), math.Round(3.2*float64(width)/float64(mmWidth))))
	36	+	}
	37	+	}
	38	+	return 13 * scale * math.Sqrt(max(1, float64(height)/(1080*scale)))
	39	+	}
	40	+
	41	+	func parseColor(s string, fallback color.RGBA) color.RGBA {
	42	+	if len(s) != 7 \|\| s[0] != '#' {
	43	+	return fallback
	44	+	}
	45	+	n, err := strconv.ParseUint(s[1:], 16, 24)
	46	+	if err != nil {
	47	+	return fallback
	48	+	}
	49	+	return color.RGBA{uint8(n >> 16), uint8(n >> 8), uint8(n), 255}
	50	+	}
	51	+
	52	+	func luminance(c color.RGBA) float64 {
	53	+	linear := func(v uint8) float64 {
	54	+	x := float64(v) / 255
	55	+	if x <= .04045 {
	56	+	return x / 12.92
	57	+	}
	58	+	return math.Pow((x+.055)/1.055, 2.4)
	59	+	}
	60	+	return .2126linear(c.R) + .7152linear(c.G) + .0722*linear(c.B)
	61	+	}
	62	+
	63	+	func contrast(a, b color.RGBA) float64 {
	64	+	x, y := luminance(a), luminance(b)
	65	+	return (max(x, y) + .05) / (min(x, y) + .05)
	66	+	}
	67	+
	68	+	// Colors derives ink from the background. Near-black and near-white have a
	69	+	// narrow luminance gap where neither meets AA; in that gap move HSL
	70	+	// lightness away from .5, preserving hue and saturation.
	71	+	// Explicit foreground overrides are honoured as supplied.
	72	+	func Colors(active, background, foreground string) (color.RGBA, color.RGBA) {
	73	+	bg := parseColor(background, parseColor(active, color.RGBA{128, 128, 128, 255}))
	74	+	dark, light := color.RGBA{17, 17, 17, 255}, color.RGBA{245, 245, 245, 255}
	75	+	ink := dark
	76	+	if contrast(bg, light) > contrast(bg, dark) {
	77	+	ink = light
	78	+	}
	79	+	for contrast(bg, ink) < 4.5 {
	80	+	// Move HSL lightness toward zero while preserving hue and saturation.
	81	+	hi, lo := max(bg.R, bg.G, bg.B), min(bg.R, bg.G, bg.B)
	82	+	l := (float64(hi) + float64(lo)) / 510
	83	+	if l > .5 {
	84	+	// Above .5, moving toward white increases contrast with dark ink.
	85	+	bg.R += uint8(math.Ceil(float64(255-bg.R) * .01))
	86	+	bg.G += uint8(math.Ceil(float64(255-bg.G) * .01))
	87	+	bg.B += uint8(math.Ceil(float64(255-bg.B) * .01))
	88	+	ink = dark
	89	+	} else {
	90	+	bg.R = uint8(float64(bg.R) * .99)
	91	+	bg.G = uint8(float64(bg.G) * .99)
	92	+	bg.B = uint8(float64(bg.B) * .99)
	93	+	ink = light
	94	+	}
	95	+	}
	96	+	return bg, parseColor(foreground, ink)
	97	+	}
	98	+
	99	+	func mix(ink, bg color.RGBA) color.RGBA {
	100	+	channel := func(a, b uint8) uint8 { return uint8(math.Round(.45float64(a) + .55float64(b))) }
	101	+	return color.RGBA{channel(ink.R, bg.R), channel(ink.G, bg.G), channel(ink.B, bg.B), 255}
	102	+	}

A internal/adapters/hintlabel/style_test.go +80 -0

@@ -0,0 +1,80 @@
	1	+	package hintlabel
	2	+
	3	+	import (
	4	+	"image/color"
	5	+	"testing"
	6	+
	7	+	"github.com/stretchr/testify/require"
	8	+	)
	9	+
	10	+	func TestDensitySize(t *testing.T) {
	11	+	for _, tt := range []struct {
	12	+	name string
	13	+	w, h, mw, mh int
	14	+	scale, logical, want float64
	15	+	}{
	16	+	{"ultrawide", 5120, 2160, 990, 420, 1, 0, 17},
	17	+	{"4k", 3840, 2160, 600, 340, 1.2, 0, 20},
	18	+	{"unknown", 1920, 1080, 0, 0, 1, 0, 13},
	19	+	{"bogus aspect", 1920, 1080, 100, 100, 1, 0, 13},
	20	+	{"fallback high", 5120, 2160, 0, 0, 1, 0, 18.38477631085024},
	21	+	{"minimum", 1920, 1080, 1000, 563, 1.2, 0, 16},
	22	+	{"maximum", 5120, 2160, 100, 42, 1, 0, 40},
	23	+	{"override", 5120, 2160, 990, 420, 1.2, 24, 28.8},
	24	+	} {
	25	+	t.Run(tt.name, func(t *testing.T) {
	26	+	require.InDelta(t, tt.want, FontPixels(tt.w, tt.h, tt.mw, tt.mh, tt.scale, tt.logical), .00001)
	27	+	})
	28	+	}
	29	+	}
	30	+
	31	+	func TestDerivedContrast(t *testing.T) {
	32	+	for _, s := range []string{"#f9e2af", "#89b4fa", "#808080", "#000000", "#ffffff", "#1e1e2e", "#747474", "#757575", "#ff0000", "#00ff00", "#0000ff"} {
	33	+	t.Run(s, func(t *testing.T) {
	34	+	bg, fg := Colors(s, "", "")
	35	+	require.GreaterOrEqual(t, contrast(bg, fg), 4.5)
	36	+	require.Contains(t, []color.RGBA{{17, 17, 17, 255}, {245, 245, 245, 255}}, fg)
	37	+	})
	38	+	}
	39	+	// Sweep the gap between near-black and near-white contrast thresholds.
	40	+	for i := 0; i < 256; i++ {
	41	+	c := color.RGBA{uint8(i), uint8(i), uint8(i), 255}
	42	+	s := "#" + hexByte(c.R) + hexByte(c.G) + hexByte(c.B)
	43	+	bg, fg := Colors(s, "", "")
	44	+	require.GreaterOrEqual(t, contrast(bg, fg), 4.5, s)
	45	+	}
	46	+	}
	47	+
	48	+	func hexByte(v uint8) string {
	49	+	const digits = "0123456789abcdef"
	50	+	return string([]byte{digits[v>>4], digits[v&15]})
	51	+	}
	52	+
	53	+	func TestOverridesAndTyped(t *testing.T) {
	54	+	bg, fg := Colors("#ffffff", "#000000", "#89b4fa")
	55	+	require.Equal(t, color.RGBA{0, 0, 0, 255}, bg)
	56	+	require.Equal(t, color.RGBA{137, 180, 250, 255}, fg)
	57	+	require.Equal(t, color.RGBA{62, 81, 113, 255}, mix(fg, bg))
	58	+	fallback, _ := Colors("", "", "")
	59	+	require.Equal(t, color.RGBA{128, 128, 128, 255}, fallback)
	60	+	}
	61	+
	62	+	func TestStyledRasterSizeAndCache(t *testing.T) {
	63	+	var r Raster
	64	+	bg, fg := Colors("#f9e2af", "", "")
	65	+	style := Style{13, bg, fg}
	66	+	a, _, small := r.Label("fj", 0, 1, style)
	67	+	style.FontPx = 26
	68	+	b, _, large := r.Label("fj", 0, 1, style)
	69	+	require.Greater(t, large, small)
	70	+	require.NotEqual(t, a, b)
	71	+	style.Background = color.RGBA{0, 0, 0, 255}
	72	+	c, _, _ := r.Label("fj", 0, 1, style)
	73	+	require.NotEqual(t, b, c)
	74	+	require.Equal(t, byte(89), c[3], "outline has 35 percent physical-pixel alpha")
	75	+	}
	76	+
	77	+	func TestFontPixelsClamped(t *testing.T) {
	78	+	require.InDelta(t, 256, FontPixels(1920, 1080, 0, 0, 3, 1e9), .00001)
	79	+	require.InDelta(t, 256, FontPixels(1920, 1080, 0, 0, 4, 72), .00001)
	80	+	}

A internal/adapters/hints/benchmark_test.go +81 -0

@@ -0,0 +1,81 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/draw"
	7	+	"image/png"
	8	+	"os"
	9	+	"os/exec"
	10	+	"path/filepath"
	11	+	"strconv"
	12	+	"testing"
	13	+	"time"
	14	+	)
	15	+
	16	+	// TestFrameBenchmark logs counts and timings, never filenames or OCR text.
	17	+	func TestFrameBenchmark(t *testing.T) {
	18	+	dir := os.Getenv("NEFERWL_HINT_BENCH_DIR")
	19	+	if dir == "" {
	20	+	t.Skip("set NEFERWL_HINT_BENCH_DIR to a local PNG directory")
	21	+	}
	22	+	if _, err := exec.LookPath("tesseract"); err != nil {
	23	+	t.Skip("tesseract not on PATH")
	24	+	}
	25	+	files, err := filepath.Glob(filepath.Join(dir, "*.png"))
	26	+	if err != nil \|\| len(files) == 0 {
	27	+	t.Fatal("no PNG frames")
	28	+	}
	29	+	// Settings under test: NEFERWL_HINT_BENCH_WORKERS=<n>, NEFERWL_HINT_BENCH_PNG=speed\|none.
	30	+	if v, _ := strconv.Atoi(os.Getenv("NEFERWL_HINT_BENCH_WORKERS")); v > 0 {
	31	+	ocrWorkers = v
	32	+	}
	33	+	if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
	34	+	rawPGM = false
	35	+	}
	36	+	if os.Getenv("NEFERWL_HINT_BENCH_PNG") == "none" {
	37	+	pngLevel = png.NoCompression
	38	+	}
	39	+	for i, file := range files {
	40	+	f, err := os.Open(file)
	41	+	if err != nil {
	42	+	t.Fatal("open frame failed")
	43	+	}
	44	+	src, err := png.Decode(f)
	45	+	_ = f.Close()
	46	+	if err != nil {
	47	+	t.Fatal("decode frame failed")
	48	+	}
	49	+	img := image.NewRGBA(image.Rect(0, 0, src.Bounds().Dx(), src.Bounds().Dy()))
	50	+	draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
	51	+	start := time.Now()
	52	+	words, bands, err := readWords(context.Background(), toGray(img))
	53	+	if err != nil {
	54	+	t.Fatal("recognition failed")
	55	+	}
	56	+	t.Logf("frame=%d px=%d ocr_ms=%.1f words=%d links=%d bands=%d", i, img.Bounds().Dx()*img.Bounds().Dy(), ms(time.Since(start)), len(words), len(textLinks(words, 1)), bands)
	57	+	if os.Getenv("NEFERWL_HINT_BENCH_STRIPS") == "" {
	58	+	continue
	59	+	}
	60	+	// Experimental strips use known OCR word regions, not pixel targets.
	61	+	// This is an oracle comparison, not a production optimization.
	62	+	lines := make([]image.Rectangle, 0, len(words))
	63	+	for _, word := range words {
	64	+	lines = append(lines, word.Rect)
	65	+	}
	66	+	for _, normalized := range []bool{false, true} {
	67	+	begin := time.Now()
	68	+	compact, _, err := readStrips(context.Background(), img, lines, 8, normalized)
	69	+	if err != nil {
	70	+	t.Fatal("strip recognition failed")
	71	+	}
	72	+	t.Logf("frame=%d strips normalized=%t ocr_ms=%.1f words=%d links=%d", i, normalized, ms(time.Since(begin)), len(compact), len(textLinks(compact, 1)))
	73	+	}
	74	+	}
	75	+	}
	76	+
	77	+	func toGray(img image.RGBA) image.Gray {
	78	+	g := image.NewGray(img.Bounds())
	79	+	draw.Draw(g, g.Bounds(), img, img.Bounds().Min, draw.Src)
	80	+	return g
	81	+	}

A internal/adapters/hints/dump.go +285 -0

@@ -0,0 +1,285 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"encoding/json"
	6	+	"fmt"
	7	+	"image"
	8	+	"image/color"
	9	+	"image/png"
	10	+	"os"
	11	+	"path/filepath"
	12	+	"sort"
	13	+	"strings"
	14	+	"syscall"
	15	+	"time"
	16	+
	17	+	"github.com/bnema/neferwl/internal/ports"
	18	+	)
	19	+
	20	+	const (
	21	+	// maxDumps is the number of dump groups kept in the directory.
	22	+	maxDumps = 50
	23	+	outlineSize = 2
	24	+	)
	25	+
	26	+	// Options tunes Run.
	27	+	type Options struct {
	28	+	// DumpDir, when not empty, receives the scanned frames (PNG, annotated
	29	+	// PNG and JSON). They contain the screen in clear.
	30	+	DumpDir string
	31	+	// Source, when not nil, finds targets in the accessibility tree of the
	32	+	// windows on screen; OCR reads the rest.
	33	+	Source targetSource
	34	+	}
	35	+
	36	+	// dumpJob is one scan to write. pix is the capture buffer of the scan, owned
	37	+	// by the job: writeDump converts it in place.
	38	+	type dumpJob struct {
	39	+	when time.Time
	40	+	req ports.HintScanRequest
	41	+	pix []byte
	42	+	m *mapping // owns pix; released after the dump
	43	+	stride int
	44	+	targets []ports.HintTarget
	45	+	crops []image.Rectangle
	46	+	words []OCRWord
	47	+	// atspi is the number of leading targets that come from the
	48	+	// accessibility tree; the others come from OCR.
	49	+	atspi int
	50	+	captureD time.Duration
	51	+	scanD time.Duration
	52	+	atspiD time.Duration
	53	+	}
	54	+
	55	+	type dumpTarget struct {
	56	+	Kind string `json:"kind"`
	57	+	Source string `json:"source"`
	58	+	Rect [4]int `json:"rect"`
	59	+	}
	60	+
	61	+	type dumpMeta struct {
	62	+	Output string `json:"output"`
	63	+	Width int `json:"width"`
	64	+	Height int `json:"height"`
	65	+	Stride int `json:"stride"`
	66	+	Scale float64 `json:"scale"`
	67	+	LinksOnly bool `json:"links_only"`
	68	+	CaptureMS float64 `json:"capture_ms"`
	69	+	ScanMS float64 `json:"scan_ms"`
	70	+	AtspiMS float64 `json:"atspi_ms"`
	71	+	Crops []image.Rectangle `json:"ocr_crops"`
	72	+	Words []OCRWord `json:"words"`
	73	+	Targets []dumpTarget `json:"targets"`
	74	+	}
	75	+
	76	+	func kindName(k ports.HintKind) string {
	77	+	switch k {
	78	+	case ports.HintLink:
	79	+	return "link"
	80	+	case ports.HintButton:
	81	+	return "button"
	82	+	default:
	83	+	return "text"
	84	+	}
	85	+	}
	86	+
	87	+	func kindColor(k ports.HintKind) color.RGBA {
	88	+	switch k {
	89	+	case ports.HintLink:
	90	+	return color.RGBA{0x00, 0xc8, 0x53, 0xff}
	91	+	case ports.HintButton:
	92	+	return color.RGBA{0x29, 0x62, 0xff, 0xff}
	93	+	default:
	94	+	return color.RGBA{0x80, 0x80, 0x80, 0xff}
	95	+	}
	96	+	}
	97	+
	98	+	// atspiColor is the outline of targets found in the accessibility tree.
	99	+	var atspiColor = color.RGBA{0xff, 0x00, 0xff, 0xff}
	100	+
	101	+	func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 }
	102	+
	103	+	// writeDump writes the three files of j into dir, then prunes old groups.
	104	+	func writeDump(dir string, j *dumpJob) error {
	105	+	if err := secureDir(dir); err != nil {
	106	+	return err
	107	+	}
	108	+	prefix := filepath.Join(dir, j.when.Format("20060102-150405.000")+"-"+safeName(j.req.Output))
	109	+
	110	+	// The frame is converted in place to RGBA and encoded straight from the
	111	+	// capture buffer: no copy. The job owns the buffer, which is released
	112	+	// with it.
	113	+	for y := 0; y < j.req.Height; y++ {
	114	+	row := j.pix[yj.stride : yj.stride+j.req.Width*4]
	115	+	for x := 0; x < len(row); x += 4 {
	116	+	row[x], row[x+2], row[x+3] = row[x+2], row[x], 0xff
	117	+	}
	118	+	}
	119	+	img := &image.RGBA{Pix: j.pix, Stride: j.stride, Rect: image.Rect(0, 0, j.req.Width, j.req.Height)}
	120	+	if err := writePNG(prefix+".png", img); err != nil {
	121	+	return err
	122	+	}
	123	+
	124	+	scale := j.req.Scale
	125	+	if scale <= 0 {
	126	+	scale = 1
	127	+	}
	128	+	for _, r := range j.crops {
	129	+	outline(img, r, color.RGBA{0x80, 0x80, 0x80, 0xff})
	130	+	}
	131	+	for i, t := range j.targets {
	132	+	r := image.Rect(int(float64(t.Rect.X)scale), int(float64(t.Rect.Y)scale),
	133	+	int(float64(t.Rect.X+t.Rect.W)scale), int(float64(t.Rect.Y+t.Rect.H)scale))
	134	+	c := kindColor(t.Kind)
	135	+	if i < j.atspi {
	136	+	c = atspiColor
	137	+	}
	138	+	outline(img, r, c)
	139	+	}
	140	+	if err := writePNG(prefix+".annotated.png", img); err != nil {
	141	+	return err
	142	+	}
	143	+
	144	+	meta := dumpMeta{
	145	+	Output: j.req.Output, Width: j.req.Width, Height: j.req.Height, Stride: j.stride,
	146	+	Scale: j.req.Scale, LinksOnly: j.req.LinksOnly,
	147	+	CaptureMS: ms(j.captureD), ScanMS: ms(j.scanD), AtspiMS: ms(j.atspiD),
	148	+	Crops: j.crops, Words: j.words,
	149	+	Targets: make([]dumpTarget, 0, len(j.targets)),
	150	+	}
	151	+	for i, t := range j.targets {
	152	+	src := "ocr"
	153	+	if i < j.atspi {
	154	+	src = "atspi"
	155	+	}
	156	+	meta.Targets = append(meta.Targets, dumpTarget{Kind: kindName(t.Kind), Source: src, Rect: [4]int{t.Rect.X, t.Rect.Y, t.Rect.X + t.Rect.W, t.Rect.Y + t.Rect.H}})
	157	+	}
	158	+	data, err := json.Marshal(meta)
	159	+	if err != nil {
	160	+	return err
	161	+	}
	162	+	if err := writeFile(prefix+".json", func(f *os.File) error { _, err := f.Write(append(data, '\n')); return err }); err != nil {
	163	+	return err
	164	+	}
	165	+	return prune(dir, maxDumps)
	166	+	}
	167	+
	168	+	func safeName(s string) string {
	169	+	return strings.Map(func(r rune) rune {
	170	+	if r == '/' \|\| r == 0 {
	171	+	return '_'
	172	+	}
	173	+	return r
	174	+	}, s)
	175	+	}
	176	+
	177	+	func writePNG(path string, img image.Image) error {
	178	+	enc := png.Encoder{CompressionLevel: png.BestSpeed}
	179	+	return writeFile(path, func(f *os.File) error {
	180	+	bw := bufio.NewWriterSize(f, 1<<16)
	181	+	if err := enc.Encode(bw, img); err != nil {
	182	+	return err
	183	+	}
	184	+	return bw.Flush()
	185	+	})
	186	+	}
	187	+
	188	+	// secureDir makes sure dir is a real directory owned by the current user and
	189	+	// private to it, whatever it was before.
	190	+	func secureDir(dir string) error {
	191	+	if err := os.MkdirAll(dir, 0o700); err != nil {
	192	+	return err
	193	+	}
	194	+	fi, err := os.Lstat(dir)
	195	+	if err != nil {
	196	+	return err
	197	+	}
	198	+	if !fi.IsDir() {
	199	+	return fmt.Errorf("dump dir %s is not a directory", dir)
	200	+	}
	201	+	if st, ok := fi.Sys().(*syscall.Stat_t); ok && int(st.Uid) != os.Getuid() {
	202	+	return fmt.Errorf("dump dir %s is not owned by the current user", dir)
	203	+	}
	204	+	return os.Chmod(dir, 0o700)
	205	+	}
	206	+
	207	+	// writeFile fills a fresh private temporary file in the directory of path and
	208	+	// renames it over path: an existing file or symlink there is replaced, never
	209	+	// followed.
	210	+	func writeFile(path string, fill func(*os.File) error) error {
	211	+	f, err := os.CreateTemp(filepath.Dir(path), ".dump-*")
	212	+	if err != nil {
	213	+	return err
	214	+	}
	215	+	tmp := f.Name()
	216	+	err = f.Chmod(0o600)
	217	+	if err == nil {
	218	+	err = fill(f)
	219	+	}
	220	+	if cerr := f.Close(); err == nil {
	221	+	err = cerr
	222	+	}
	223	+	if err == nil {
	224	+	err = os.Rename(tmp, path)
	225	+	}
	226	+	if err != nil {
	227	+	_ = os.Remove(tmp)
	228	+	}
	229	+	return err
	230	+	}
	231	+
	232	+	// outline draws a 2 px frame inside r, clipped to img.
	233	+	func outline(img *image.RGBA, r image.Rectangle, c color.RGBA) {
	234	+	r = r.Intersect(img.Rect)
	235	+	if r.Empty() {
	236	+	return
	237	+	}
	238	+	for y := r.Min.Y; y < r.Max.Y; y++ {
	239	+	for x := r.Min.X; x < r.Max.X; x++ {
	240	+	if x-r.Min.X < outlineSize \|\| r.Max.X-1-x < outlineSize \|\| y-r.Min.Y < outlineSize \|\| r.Max.Y-1-y < outlineSize {
	241	+	img.SetRGBA(x, y, c)
	242	+	}
	243	+	}
	244	+	}
	245	+	}
	246	+
	247	+	// groupOf returns the "<timestamp>-<output>" prefix of a dump file name.
	248	+	func groupOf(name string) string {
	249	+	for _, suf := range []string{".annotated.png", ".png", ".json"} {
	250	+	if strings.HasSuffix(name, suf) {
	251	+	return strings.TrimSuffix(name, suf)
	252	+	}
	253	+	}
	254	+	return ""
	255	+	}
	256	+
	257	+	// prune keeps the keep most recent groups of dir; timestamps sort by name.
	258	+	func prune(dir string, keep int) error {
	259	+	entries, err := os.ReadDir(dir)
	260	+	if err != nil {
	261	+	return err
	262	+	}
	263	+	groups := map[string][]string{}
	264	+	for _, e := range entries {
	265	+	if g := groupOf(e.Name()); g != "" && !e.IsDir() {
	266	+	groups[g] = append(groups[g], e.Name())
	267	+	}
	268	+	}
	269	+	if len(groups) <= keep {
	270	+	return nil
	271	+	}
	272	+	names := make([]string, 0, len(groups))
	273	+	for g := range groups {
	274	+	names = append(names, g)
	275	+	}
	276	+	sort.Strings(names)
	277	+	for _, g := range names[:len(names)-keep] {
	278	+	for _, f := range groups[g] {
	279	+	if err := os.Remove(filepath.Join(dir, f)); err != nil {
	280	+	return fmt.Errorf("prune: %w", err)
	281	+	}
	282	+	}
	283	+	}
	284	+	return nil
	285	+	}

A internal/adapters/hints/dump_test.go +40 -0

@@ -0,0 +1,40 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"os"
	5	+	"path/filepath"
	6	+	"testing"
	7	+	"time"
	8	+
	9	+	"github.com/stretchr/testify/require"
	10	+
	11	+	"github.com/bnema/neferwl/internal/ports"
	12	+	)
	13	+
	14	+	// An existing dump directory is made private, and a symlink at a file name is
	15	+	// replaced instead of followed.
	16	+	func TestWriteDumpPrivate(t *testing.T) {
	17	+	base := t.TempDir()
	18	+	dir := filepath.Join(base, "hints")
	19	+	require.NoError(t, os.Mkdir(dir, 0o755))
	20	+	require.NoError(t, os.Chmod(dir, 0o755))
	21	+	victim := filepath.Join(base, "victim")
	22	+	require.NoError(t, os.WriteFile(victim, []byte("keep"), 0o644))
	23	+	when := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
	24	+	prefix := filepath.Join(dir, when.Format("20060102-150405.000")+"-o")
	25	+	require.NoError(t, os.Symlink(victim, prefix+".json"))
	26	+	pix := make([]byte, 444)
	27	+	j := &dumpJob{when: when, req: ports.HintScanRequest{Output: "o", Width: 4, Height: 4, Scale: 1}, pix: pix, stride: 16, m: &mapping{}}
	28	+	require.NoError(t, writeDump(dir, j))
	29	+
	30	+	fi, err := os.Stat(dir)
	31	+	require.NoError(t, err)
	32	+	require.Equal(t, os.FileMode(0o700), fi.Mode().Perm())
	33	+	got, err := os.ReadFile(victim)
	34	+	require.NoError(t, err)
	35	+	require.Equal(t, "keep", string(got))
	36	+	li, err := os.Lstat(prefix + ".json")
	37	+	require.NoError(t, err)
	38	+	require.True(t, li.Mode().IsRegular())
	39	+	require.Equal(t, os.FileMode(0o600), li.Mode().Perm())
	40	+	}

A internal/adapters/hints/hints.go +507 -0

@@ -0,0 +1,507 @@
	1	+	// Package hints finds clickable targets for hint mode: it captures the last
	2	+	// frame of an output through the capture pipeline and reads uncovered windows with OCR.
	3	+	package hints
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"fmt"
	9	+	"hash/maphash"
	10	+	"image"
	11	+	"math"
	12	+	"strings"
	13	+	"time"
	14	+
	15	+	"github.com/bnema/neferwl/internal/adapters/logging"
	16	+	"github.com/bnema/neferwl/internal/ports"
	17	+	)
	18	+
	19	+	// ErrBusy answers a request that arrives while another one is in flight.
	20	+	var ErrBusy = errors.New("hints: scan in progress")
	21	+
	22	+	// ErrTimeout is the error of a capture that is not answered in time.
	23	+	var ErrTimeout = errors.New("hints: capture timed out")
	24	+
	25	+	// ErrBadRequest is the error of a request without a usable buffer size.
	26	+	var ErrBadRequest = errors.New("hints: invalid output size")
	27	+
	28	+	const (
	29	+	captureTimeout = time.Second
	30	+	// formatXRGB8888 is the capture format: opaque BGRA in memory.
	31	+	formatXRGB8888 = 1
	32	+	maxPixels = 1 << 28
	33	+	)
	34	+
	35	+	// Run serves requests until ctx is done. It handles one request at a time;
	36	+	// a request that arrives meanwhile is answered with ErrBusy. Every request
	37	+	// gets exactly one result on results.
	38	+	func Run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, opts Options) error {
	39	+	return run(ctx, requests, captures, captured, results, captureTimeout, opts)
	40	+	}
	41	+
	42	+	// targetSource finds hint targets in the accessibility tree of windows. It
	43	+	// returns the targets (logical output-local) and the windows they cover.
	44	+	type targetSource interface {
	45	+	Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)
	46	+	// Close releases the source. It is called once, by the worker goroutine,
	47	+	// when the worker returns; the source is not safe for concurrent use.
	48	+	Close() error
	49	+	}
	50	+
	51	+	// atspiResult is what the source found for one request.
	52	+	type atspiResult struct {
	53	+	targets []ports.HintTarget
	54	+	covered map[ports.WindowID]bool
	55	+	dur time.Duration
	56	+	err error
	57	+	}
	58	+
	59	+	type worker struct {
	60	+	source targetSource
	61	+	cache map[bandKey]bandCache
	62	+	cacheSeed maphash.Seed
	63	+	ocr ocrState
	64	+	warnedSource bool
	65	+	n uint64
	66	+	dumpDir string
	67	+	pending *dumpJob
	68	+	// orphans are capture buffers the output may still write: they are
	69	+	// released when its CaptureDone finally arrives.
	70	+	orphans map[uint64]*mapping
	71	+	live int // mappings not released yet
	72	+	}
	73	+
	74	+	// frame is the captured output: BGRA pixels, nil when no capture was needed.
	75	+	type frame struct {
	76	+	pix []byte
	77	+	stride int
	78	+	m *mapping
	79	+	}
	80	+
	81	+	func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {
	82	+	w := &worker{dumpDir: opts.DumpDir, source: opts.Source}
	83	+	if opts.Source != nil {
	84	+	// The worker goroutine is the only one to touch the source.
	85	+	defer func() { _ = opts.Source.Close() }()
	86	+	}
	87	+	for {
	88	+	select {
	89	+	case <-ctx.Done():
	90	+	return nil
	91	+	case done := <-captured:
	92	+	// A capture abandoned by a timeout is finally done: release it.
	93	+	w.reap(done.ID)
	94	+	case req := <-requests:
	95	+	res := w.serve(ctx, req, requests, captures, captured, results, timeout)
	96	+	if !send(ctx, results, res) {
	97	+	// The dump job will never run: release its frame.
	98	+	if j := w.pending; j != nil {
	99	+	w.pending = nil
	100	+	j.m.unmap()
	101	+	}
	102	+	return nil
	103	+	}
	104	+	w.dump(ctx)
	105	+	}
	106	+	}
	107	+	}
	108	+
	109	+	// dump writes the last successful scan, after its result was sent, then
	110	+	// releases its frame.
	111	+	func (w *worker) dump(ctx context.Context) {
	112	+	j := w.pending
	113	+	w.pending = nil
	114	+	if j == nil {
	115	+	return
	116	+	}
	117	+	defer j.m.unmap()
	118	+	if err := writeDump(w.dumpDir, j); err != nil {
	119	+	log := logging.For(ctx, "hints")
	120	+	log.Warn().Err(err).Str("dir", w.dumpDir).Msg("hint debug dump failed")
	121	+	}
	122	+	}
	123	+
	124	+	func send(ctx context.Context, results chan<- ports.HintScanResult, res ports.HintScanResult) bool {
	125	+	select {
	126	+	case results <- res:
	127	+	return true
	128	+	case <-ctx.Done():
	129	+	return false
	130	+	}
	131	+	}
	132	+
	133	+	// needFrame reports whether the scan must capture the output: OCR has a
	134	+	// window to read, or the debug dump wants the frame. With OCR unavailable
	135	+	// (tesseract missing) and no dump, the accessibility targets are the result.
	136	+	func (w *worker) needFrame(ctx context.Context, req ports.HintScanRequest, ar atspiResult) bool {
	137	+	ocr := w.ocrReady(ctx)
	138	+	if w.dumpDir != "" {
	139	+	return true
	140	+	}
	141	+	if !ocr {
	142	+	return false
	143	+	}
	144	+	for _, win := range req.Windows {
	145	+	if !ar.covered[win.ID] && !windowCrop(win.Rect, req.Scale, req.Width, req.Height).Empty() {
	146	+	return true
	147	+	}
	148	+	}
	149	+	return false
	150	+	}
	151	+
	152	+	func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration) ports.HintScanResult {
	153	+	log := logging.For(ctx, "hints")
	154	+	res := ports.HintScanResult{Seq: req.Seq, Output: req.Output}
	155	+	fail := func(err error) ports.HintScanResult {
	156	+	log.Info().Err(err).Str("output", req.Output).Msg("hint scan failed")
	157	+	res.Err = err
	158	+	return res
	159	+	}
	160	+
	161	+	if req.Width <= 0 \|\| req.Height <= 0 \|\| req.Width*req.Height > maxPixels {
	162	+	return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))
	163	+	}
	164	+	// The accessibility query is always waited for: the source is not shared
	165	+	// between requests. The capture only starts once it is known whether any
	166	+	// window needs OCR.
	167	+	atspi := w.startSource(ctx, req)
	168	+	defer func() { <-atspi.done }()
	169	+	if !w.needFrame(ctx, req, atspi.wait()) {
	170	+	return w.scan(ctx, req, frame{}, 0, res, atspi)
	171	+	}
	172	+	// The capture buffer is anonymous mapped memory, outside the Go heap. It
	173	+	// is released once the crops are extracted, or by the dump job.
	174	+	m := w.alloc(req.Width * req.Height * 4)
	175	+	keep := false // the mapping outlives this call: dump job or pending capture
	176	+	defer func() {
	177	+	if !keep {
	178	+	m.unmap()
	179	+	}
	180	+	}()
	181	+	buf := m.b
	182	+
	183	+	w.n++
	184	+	id := ports.CaptureIDInternal \| w.n
	185	+	stride := req.Width * 4
	186	+	start := time.Now()
	187	+	creq := ports.CaptureRequest{
	188	+	ID: id,
	189	+	Output: req.Output,
	190	+	Region: image.Rect(0, 0, req.Width, req.Height),
	191	+	Mapped: buf,
	192	+	Width: req.Width,
	193	+	Height: req.Height,
	194	+	Stride: stride,
	195	+	Format: formatXRGB8888,
	196	+	Indicate: false,
	197	+	}
	198	+
	199	+	timer := time.NewTimer(timeout)
	200	+	defer timer.Stop()
	201	+
	202	+	select {
	203	+	case captures <- creq:
	204	+	case <-timer.C:
	205	+	return fail(ErrTimeout)
	206	+	case <-ctx.Done():
	207	+	return fail(ctx.Err())
	208	+	}
	209	+	// From here the output owns buf until CaptureDone.
	210	+	abandon := func() {
	211	+	if w.orphans == nil {
	212	+	w.orphans = map[uint64]*mapping{}
	213	+	}
	214	+	w.orphans[id] = m
	215	+	keep = true
	216	+	}
	217	+
	218	+	for {
	219	+	select {
	220	+	case done := <-captured:
	221	+	if done.ID != id {
	222	+	w.reap(done.ID)
	223	+	continue
	224	+	}
	225	+	if done.Err != nil {
	226	+	return fail(fmt.Errorf("hints: capture: %w", done.Err))
	227	+	}
	228	+	res = w.scan(ctx, req, frame{pix: buf, stride: stride, m: m}, time.Since(start), res, atspi)
	229	+	keep = w.pending != nil && w.pending.m == m
	230	+	return res
	231	+	case other := <-requests:
	232	+	// The buffer is in use until the capture is done: answer now.
	233	+	busy := ports.HintScanResult{Seq: other.Seq, Output: other.Output, Err: ErrBusy}
	234	+	if !send(ctx, results, busy) {
	235	+	abandon()
	236	+	return fail(ctx.Err())
	237	+	}
	238	+	case <-timer.C:
	239	+	// The output may still write into buf: it is released when its
	240	+	// CaptureDone arrives.
	241	+	abandon()
	242	+	return fail(ErrTimeout)
	243	+	case <-ctx.Done():
	244	+	abandon()
	245	+	return fail(ctx.Err())
	246	+	}
	247	+	}
	248	+	}
	249	+
	250	+	// reap releases the buffer of an abandoned capture that is finally done.
	251	+	func (w *worker) reap(id uint64) {
	252	+	if m, ok := w.orphans[id]; ok {
	253	+	delete(w.orphans, id)
	254	+	m.unmap()
	255	+	}
	256	+	}
	257	+
	258	+	// bandJob is one OCR band of one window crop.
	259	+	type bandJob struct {
	260	+	win int // index into the scan's windows
	261	+	key bandKey //
	262	+	ext image.Rectangle // crop-local rows read, with context
	263	+	own [2]int // crop-local rows owned: words centred there are kept
	264	+	hash uint64
	265	+	hit bool
	266	+	gray *image.Gray
	267	+	words []OCRWord
	268	+	err error
	269	+	}
	270	+
	271	+	func (w worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame, captureDur time.Duration, res ports.HintScanResult, atspi pendingSource) ports.HintScanResult {
	272	+	log := logging.For(ctx, "hints")
	273	+	ar := atspi.wait()
	274	+	if ar.err != nil && !w.warnedSource {
	275	+	log.Warn().Err(ar.err).Msg("accessibility targets unavailable; using OCR only")
	276	+	w.warnedSource = true
	277	+	}
	278	+	out := make([]ports.HintTarget, 0, len(ar.targets))
	279	+	for _, t := range ar.targets {
	280	+	if !req.LinksOnly \|\| t.Kind == ports.HintLink {
	281	+	out = append(out, t)
	282	+	}
	283	+	}
	284	+	nAtspi := len(out)
	285	+	t0 := time.Now()
	286	+	if w.cacheSeed == (maphash.Seed{}) {
	287	+	w.cacheSeed = maphash.MakeSeed()
	288	+	}
	289	+	nextCache := make(map[bandKey]bandCache)
	290	+	var crops []image.Rectangle
	291	+	var allWords []OCRWord
	292	+	pixels, links := 0, 0
	293	+	ocrOn := fr.pix != nil && w.ocrReady(ctx)
	294	+	// Phase 1 reads the frame: the bands that miss the cache are copied, as
	295	+	// grayscale, into one arena. Phase 2 (OCR) does not need the frame.
	296	+	var jobs []*bandJob
	297	+	var rects []image.Rectangle // crop of each window that has bands
	298	+	var arena *mapping
	299	+	if ocrOn {
	300	+	type pending struct {
	301	+	r image.Rectangle
	302	+	jobs []*bandJob
	303	+	}
	304	+	var wins []pending
	305	+	need := 0
	306	+	for _, win := range req.Windows {
	307	+	if ar.covered[win.ID] {
	308	+	continue
	309	+	}
	310	+	r := windowCrop(win.Rect, req.Scale, req.Width, req.Height)
	311	+	if r.Empty() {
	312	+	continue
	313	+	}
	314	+	crops = append(crops, r)
	315	+	pixels += r.Dx() * r.Dy()
	316	+	p := pending{r: r}
	317	+	for _, b := range bandsOf(r.Dy()) {
	318	+	ext := image.Rect(0, b.top, r.Dx(), b.bottom)
	319	+	j := &bandJob{win: len(wins), ext: ext, own: [2]int{b.ownTop, b.ownBottom},
	320	+	key: bandKey{output: req.Output, window: win.ID, band: b.idx, bands: b.n}}
	321	+	j.hash = w.cropHash(fr.pix, fr.stride, ext.Add(r.Min))
	322	+	entry, ok := w.cache[j.key]
	323	+	if ok && entry.hash == j.hash && entry.ext == ext {
	324	+	j.hit, j.words = true, entry.words
	325	+	} else {
	326	+	need += ext.Dx() * ext.Dy()
	327	+	}
	328	+	p.jobs = append(p.jobs, j)
	329	+	}
	330	+	wins = append(wins, p)
	331	+	}
	332	+	if need > 0 {
	333	+	arena = w.alloc(need)
	334	+	}
	335	+	off := 0
	336	+	for _, p := range wins {
	337	+	rects = append(rects, p.r)
	338	+	for _, j := range p.jobs {
	339	+	if !j.hit {
	340	+	n := j.ext.Dx() * j.ext.Dy()
	341	+	j.gray = cropGray(fr.pix, fr.stride, j.ext.Add(p.r.Min), arena.b[off:off+n])
	342	+	off += n
	343	+	}
	344	+	jobs = append(jobs, j)
	345	+	}
	346	+	}
	347	+	} else if w.dumpDir != "" {
	348	+	for _, win := range req.Windows {
	349	+	if !ar.covered[win.ID] {
	350	+	if r := windowCrop(win.Rect, req.Scale, req.Width, req.Height); !r.Empty() {
	351	+	crops = append(crops, r)
	352	+	}
	353	+	}
	354	+	}
	355	+	}
	356	+	if w.dumpDir != "" && fr.pix != nil {
	357	+	// The dump reads the frame after the result is sent and releases it.
	358	+	w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, crops: crops, captureD: captureDur, atspiD: ar.dur}
	359	+	} else {
	360	+	fr.m.unmap() // the crops are extracted: give the frame back
	361	+	}
	362	+	fr.pix = nil
	363	+
	364	+	hits := 0
	365	+	if len(jobs) > 0 {
	366	+	w.recognize(ctx, jobs)
	367	+	for _, j := range jobs {
	368	+	if j.hit {
	369	+	hits++
	370	+	}
	371	+	}
	372	+	}
	373	+	arena.unmap()
	374	+
	375	+	// Merge the bands of each window, in order.
	376	+	k := 0
	377	+	for wi, r := range rects {
	378	+	var words []OCRWord
	379	+	for ; k < len(jobs) && jobs[k].win == wi; k++ {
	380	+	j := jobs[k]
	381	+	if j.err != nil {
	382	+	failureWarning.Do(func() { log.Warn().Err(j.err).Msg("OCR unavailable; install tesseract and tesseract-data-eng") })
	383	+	continue
	384	+	}
	385	+	nextCache[j.key] = bandCache{hash: j.hash, ext: j.ext, words: j.words}
	386	+	for _, word := range j.words {
	387	+	word.Rect = word.Rect.Add(r.Min)
	388	+	if word.Underlined {
	389	+	word.UnderL, word.UnderR = word.UnderL+r.Min.X, word.UnderR+r.Min.X
	390	+	}
	391	+	if !duplicateWord(words, word) {
	392	+	words = append(words, word)
	393	+	}
	394	+	}
	395	+	}
	396	+	allWords = append(allWords, words...)
	397	+	targets := textLinks(words, req.Scale)
	398	+	links += len(targets)
	399	+	out = append(out, targets...)
	400	+	}
	401	+	w.cache = nextCache
	402	+	ocrDur := time.Since(t0)
	403	+	res.Targets = out
	404	+	if p := w.pending; p != nil {
	405	+	p.targets, p.atspi, p.words, p.scanD = out, nAtspi, allWords, ocrDur
	406	+	}
	407	+	log.Info().Str("output", req.Output).
	408	+	Float64("atspi_ms", ms(ar.dur)).Int("atspi_windows", len(ar.covered)).Int("atspi_targets", nAtspi).
	409	+	Int("ocr_windows", len(rects)).Float64("ocr_ms", ms(ocrDur)).Int("ocr_px", pixels).
	410	+	Int("ocr_bands", len(jobs)).Int("band_hits", hits).
	411	+	Int("words", len(allWords)).Int("ocr_links", links).Bool("cache_hit", len(jobs) > 0 && hits == len(jobs)).
	412	+	Int("targets", len(out)).Msg("hint scan")
	413	+	return res
	414	+	}
	415	+
	416	+	// duplicateWord reports whether words already holds word (the same text at
	417	+	// the same place), as happens where two bands overlap.
	418	+	func duplicateWord(words []OCRWord, word OCRWord) bool {
	419	+	for _, old := range words {
	420	+	if strings.EqualFold(word.Text, old.Text) && overlap(word.Rect, old.Rect) > .5 {
	421	+	return true
	422	+	}
	423	+	}
	424	+	return false
	425	+	}
	426	+
	427	+	// bandKey identifies one OCR band of a window on an output.
	428	+	type bandKey struct {
	429	+	output string
	430	+	window ports.WindowID
	431	+	band int
	432	+	bands int
	433	+	}
	434	+
	435	+	// bandCache is the OCR result of a band whose pixels hash to hash. words are
	436	+	// crop-local and only the ones the band owns.
	437	+	type bandCache struct {
	438	+	hash uint64
	439	+	ext image.Rectangle
	440	+	words []OCRWord
	441	+	}
	442	+
	443	+	// windowCrop rounds outwards in physical pixels, then intersects the output.
	444	+	func windowCrop(r ports.Rect, scale float64, width, height int) image.Rectangle {
	445	+	if scale <= 0 {
	446	+	scale = 1
	447	+	}
	448	+	if r.W <= 0 \|\| r.H <= 0 {
	449	+	return image.Rectangle{}
	450	+	}
	451	+	return image.Rect(int(math.Floor(float64(r.X)scale)), int(math.Floor(float64(r.Y)scale)),
	452	+	int(math.Ceil(float64(r.X+r.W)scale)), int(math.Ceil(float64(r.Y+r.H)scale))).Intersect(image.Rect(0, 0, width, height))
	453	+	}
	454	+
	455	+	// cropHash hashes the raw pixels of r without copying them.
	456	+	func (w *worker) cropHash(buf []byte, stride int, r image.Rectangle) uint64 {
	457	+	var h maphash.Hash
	458	+	h.SetSeed(w.cacheSeed)
	459	+	for y := r.Min.Y; y < r.Max.Y; y++ {
	460	+	_, _ = h.Write(buf[ystride+r.Min.X4 : ystride+r.Max.X4])
	461	+	}
	462	+	return h.Sum64()
	463	+	}
	464	+
	465	+	// cropGray converts the BGRA pixels of r to 8-bit luma into dst, which must
	466	+	// hold r.Dx()*r.Dy() bytes; only this band is converted, not the output.
	467	+	func cropGray(buf []byte, stride int, r image.Rectangle, dst []byte) *image.Gray {
	468	+	img := &image.Gray{Pix: dst[:r.Dx()*r.Dy()], Stride: r.Dx(), Rect: image.Rect(0, 0, r.Dx(), r.Dy())}
	469	+	for y := 0; y < r.Dy(); y++ {
	470	+	src := buf[(r.Min.Y+y)stride+r.Min.X4:][:r.Dx()*4]
	471	+	row := img.Pix[y*img.Stride:][:r.Dx()]
	472	+	for x := range row {
	473	+	p := src[x4 : x4+4 : x*4+4]
	474	+	row[x] = uint8((uint32(p[2])299 + uint32(p[1])587 + uint32(p[0])*114 + 500) / 1000)
	475	+	}
	476	+	}
	477	+	return img
	478	+	}
	479	+
	480	+	// pendingSource is an accessibility query in flight.
	481	+	type pendingSource struct {
	482	+	done chan struct{}
	483	+	res atspiResult
	484	+	}
	485	+
	486	+	// startSource starts the accessibility query of req, if there is a source and
	487	+	// a window to ask about. done is closed when res is set.
	488	+	func (w worker) startSource(ctx context.Context, req ports.HintScanRequest) pendingSource {
	489	+	p := &pendingSource{done: make(chan struct{})}
	490	+	if w.source == nil \|\| len(req.Windows) == 0 {
	491	+	close(p.done)
	492	+	return p
	493	+	}
	494	+	go func() {
	495	+	defer close(p.done)
	496	+	t0 := time.Now()
	497	+	targets, covered, err := w.source.Targets(ctx, req.Windows, req.LinksOnly)
	498	+	p.res = atspiResult{targets: targets, covered: covered, dur: time.Since(t0), err: err}
	499	+	}()
	500	+	return p
	501	+	}
	502	+
	503	+	// wait returns the result of the query.
	504	+	func (p *pendingSource) wait() atspiResult {
	505	+	<-p.done
	506	+	return p.res
	507	+	}

A internal/adapters/hints/hints_test.go +436 -0

@@ -0,0 +1,436 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"fmt"
	8	+	"image"
	9	+	"image/draw"
	10	+	_ "image/png"
	11	+	"os"
	12	+	"os/exec"
	13	+	"path/filepath"
	14	+	"strings"
	15	+	"testing"
	16	+	"time"
	17	+
	18	+	"github.com/stretchr/testify/mock"
	19	+	"github.com/stretchr/testify/require"
	20	+
	21	+	"github.com/bnema/neferwl/internal/ports"
	22	+	)
	23	+
	24	+	type harness struct {
	25	+	requests chan ports.HintScanRequest
	26	+	captures chan ports.CaptureRequest
	27	+	captured chan ports.CaptureDone
	28	+	results chan ports.HintScanResult
	29	+	}
	30	+
	31	+	func start(t testing.T, timeout time.Duration) harness {
	32	+	return startWith(t, timeout, Options{})
	33	+	}
	34	+
	35	+	// startCapturing starts a worker that always captures: debug dumps are on.
	36	+	func startCapturing(t testing.T, timeout time.Duration) harness {
	37	+	return startWith(t, timeout, Options{DumpDir: filepath.Join(t.TempDir(), "hints")})
	38	+	}
	39	+
	40	+	func startWith(t testing.T, timeout time.Duration, opts Options) harness {
	41	+	t.Helper()
	42	+	h := &harness{
	43	+	requests: make(chan ports.HintScanRequest, 1),
	44	+	captures: make(chan ports.CaptureRequest, 4),
	45	+	captured: make(chan ports.CaptureDone, 1),
	46	+	results: make(chan ports.HintScanResult, 4),
	47	+	}
	48	+	ctx, cancel := context.WithCancel(context.Background())
	49	+	exited := make(chan struct{})
	50	+	go func() {
	51	+	defer close(exited)
	52	+	_ = run(ctx, h.requests, h.captures, h.captured, h.results, timeout, opts)
	53	+	}()
	54	+	t.Cleanup(func() { cancel(); <-exited })
	55	+	return h
	56	+	}
	57	+
	58	+	func loadBGRA(t *testing.T) (pix []byte, w, hgt int) {
	59	+	t.Helper()
	60	+	f, err := os.Open("testdata/browser.png")
	61	+	require.NoError(t, err)
	62	+	defer f.Close()
	63	+	img, _, err := image.Decode(f)
	64	+	require.NoError(t, err)
	65	+	b := img.Bounds()
	66	+	rgba := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
	67	+	draw.Draw(rgba, rgba.Bounds(), img, b.Min, draw.Src)
	68	+	for i := 0; i < len(rgba.Pix); i += 4 {
	69	+	rgba.Pix[i], rgba.Pix[i+2] = rgba.Pix[i+2], rgba.Pix[i]
	70	+	rgba.Pix[i+3] = 0xff
	71	+	}
	72	+	return rgba.Pix, b.Dx(), b.Dy()
	73	+	}
	74	+
	75	+	func TestHintScan(t *testing.T) {
	76	+	if _, err := exec.LookPath("tesseract"); err != nil {
	77	+	t.Skip("tesseract not on PATH")
	78	+	}
	79	+	pix, w, hgt := loadBGRA(t)
	80	+	h := start(t, 5*time.Second)
	81	+
	82	+	// Plays the output owner.
	83	+	go func() {
	84	+	creq := <-h.captures
	85	+	copy(creq.Mapped, pix)
	86	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	87	+	}()
	88	+
	89	+	h.requests <- ports.HintScanRequest{Seq: 7, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
	90	+	res := <-h.results
	91	+	require.NoError(t, res.Err)
	92	+	require.Equal(t, uint64(7), res.Seq)
	93	+	require.Equal(t, "HEADLESS-1", res.Output)
	94	+	var links int
	95	+	for _, tg := range res.Targets {
	96	+	require.Positive(t, tg.Rect.W)
	97	+	require.Positive(t, tg.Rect.H)
	98	+	if tg.Kind == ports.HintLink {
	99	+	links++
	100	+	}
	101	+	}
	102	+	require.GreaterOrEqual(t, links, 1)
	103	+	}
	104	+
	105	+	func TestHintRequestShape(t *testing.T) {
	106	+	h := startCapturing(t, 5*time.Second)
	107	+	got := make(chan ports.CaptureRequest, 1)
	108	+	go func() {
	109	+	creq := <-h.captures
	110	+	got <- creq
	111	+	h.captured <- ports.CaptureDone{ID: creq.ID + 1} // another ID: ignored
	112	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output} // blank frame
	113	+	}()
	114	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 64, Height: 32, Scale: 1}
	115	+	res := <-h.results
	116	+	require.NoError(t, res.Err)
	117	+	creq := <-got
	118	+	require.NotZero(t, creq.ID&ports.CaptureIDInternal)
	119	+	require.False(t, creq.Indicate)
	120	+	require.Equal(t, image.Rect(0, 0, 64, 32), creq.Region)
	121	+	require.Len(t, creq.Mapped, 64324)
	122	+	require.Equal(t, 64*4, creq.Stride)
	123	+	require.Equal(t, uint32(1), creq.Format)
	124	+	}
	125	+
	126	+	func TestHintCaptureError(t *testing.T) {
	127	+	h := startCapturing(t, 5*time.Second)
	128	+	boom := errors.New("boom")
	129	+	go func() {
	130	+	creq := <-h.captures
	131	+	h.captured <- ports.CaptureDone{ID: creq.ID, Err: boom}
	132	+	}()
	133	+	h.requests <- ports.HintScanRequest{Seq: 3, Output: "o", Width: 64, Height: 32, Scale: 1}
	134	+	res := <-h.results
	135	+	require.ErrorIs(t, res.Err, boom)
	136	+	require.Equal(t, uint64(3), res.Seq)
	137	+	require.Empty(t, res.Targets)
	138	+	}
	139	+
	140	+	func TestHintTimeout(t *testing.T) {
	141	+	h := startCapturing(t, 50*time.Millisecond)
	142	+	h.requests <- ports.HintScanRequest{Seq: 4, Output: "o", Width: 64, Height: 32, Scale: 1}
	143	+	res := <-h.results
	144	+	require.ErrorIs(t, res.Err, ErrTimeout)
	145	+	require.Equal(t, uint64(4), res.Seq)
	146	+	}
	147	+
	148	+	func TestHintBusy(t *testing.T) {
	149	+	h := startCapturing(t, 5*time.Second)
	150	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 64, Height: 32, Scale: 1}
	151	+	creq := <-h.captures
	152	+	h.requests <- ports.HintScanRequest{Seq: 2, Output: "o", Width: 64, Height: 32, Scale: 1}
	153	+	res := <-h.results
	154	+	require.ErrorIs(t, res.Err, ErrBusy)
	155	+	require.Equal(t, uint64(2), res.Seq)
	156	+	h.captured <- ports.CaptureDone{ID: creq.ID}
	157	+	res = <-h.results
	158	+	require.NoError(t, res.Err)
	159	+	require.Equal(t, uint64(1), res.Seq)
	160	+	}
	161	+
	162	+	func TestHintBadSize(t *testing.T) {
	163	+	h := start(t, time.Second)
	164	+	h.requests <- ports.HintScanRequest{Seq: 5, Output: "o"}
	165	+	res := <-h.results
	166	+	require.ErrorIs(t, res.Err, ErrBadRequest)
	167	+	}
	168	+
	169	+	func scanOnce(t testing.T, h harness, pix []byte, w, hgt int, seq uint64) ports.HintScanResult {
	170	+	t.Helper()
	171	+	go func() {
	172	+	creq := <-h.captures
	173	+	copy(creq.Mapped, pix)
	174	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	175	+	}()
	176	+	h.requests <- ports.HintScanRequest{Seq: seq, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
	177	+	res := <-h.results
	178	+	require.NoError(t, res.Err)
	179	+	return res
	180	+	}
	181	+
	182	+	// waitFiles waits for the dump the worker writes after sending the result.
	183	+	func waitFiles(t *testing.T, dir string, n int) []os.DirEntry {
	184	+	t.Helper()
	185	+	var entries []os.DirEntry
	186	+	require.Eventually(t, func() bool {
	187	+	entries, _ = os.ReadDir(dir)
	188	+	return len(entries) >= n
	189	+	}, 5time.Second, 10time.Millisecond)
	190	+	return entries
	191	+	}
	192	+
	193	+	func TestHintDump(t *testing.T) {
	194	+	if _, err := exec.LookPath("tesseract"); err != nil {
	195	+	t.Skip("tesseract not on PATH")
	196	+	}
	197	+	pix, w, hgt := loadBGRA(t)
	198	+	dir := filepath.Join(t.TempDir(), "hints")
	199	+	h := startWith(t, 5*time.Second, Options{DumpDir: dir})
	200	+	res := scanOnce(t, h, pix, w, hgt, 1)
	201	+	require.NotEmpty(t, res.Targets)
	202	+
	203	+	entries := waitFiles(t, dir, 3)
	204	+	// A second request is only served once the dump is written.
	205	+	_ = scanOnceDone(t, h)
	206	+	var png, ann, js string
	207	+	for _, e := range entries {
	208	+	info, err := e.Info()
	209	+	require.NoError(t, err)
	210	+	require.Equal(t, os.FileMode(0o600), info.Mode().Perm())
	211	+	switch {
	212	+	case strings.HasSuffix(e.Name(), ".annotated.png"):
	213	+	ann = e.Name()
	214	+	case strings.HasSuffix(e.Name(), ".png"):
	215	+	png = e.Name()
	216	+	case strings.HasSuffix(e.Name(), ".json"):
	217	+	js = e.Name()
	218	+	}
	219	+	}
	220	+	require.NotEmpty(t, png)
	221	+	require.NotEmpty(t, ann)
	222	+	require.Regexp(t, `^\d{8}-\d{6}\.\d{3}-HEADLESS-1\.json$`, js)
	223	+
	224	+	st, err := os.Stat(dir)
	225	+	require.NoError(t, err)
	226	+	require.Equal(t, os.FileMode(0o700), st.Mode().Perm())
	227	+
	228	+	data, err := os.ReadFile(filepath.Join(dir, js))
	229	+	require.NoError(t, err)
	230	+	var meta struct {
	231	+	Output string `json:"output"`
	232	+	Width int `json:"width"`
	233	+	Height int `json:"height"`
	234	+	Stride int `json:"stride"`
	235	+	Scale float64 `json:"scale"`
	236	+	LinksOnly bool `json:"links_only"`
	237	+	Targets []struct {
	238	+	Kind string `json:"kind"`
	239	+	Rect [4]int `json:"rect"`
	240	+	} `json:"targets"`
	241	+	}
	242	+	require.NoError(t, json.Unmarshal(data, &meta))
	243	+	require.Equal(t, "HEADLESS-1", meta.Output)
	244	+	require.Equal(t, w, meta.Width)
	245	+	require.Equal(t, hgt, meta.Height)
	246	+	require.Equal(t, w*4, meta.Stride)
	247	+	require.Equal(t, 1.0, meta.Scale)
	248	+	require.Len(t, meta.Targets, len(res.Targets))
	249	+	for i, tg := range res.Targets {
	250	+	require.Equal(t, [4]int{tg.Rect.X, tg.Rect.Y, tg.Rect.X + tg.Rect.W, tg.Rect.Y + tg.Rect.H}, meta.Targets[i].Rect)
	251	+	}
	252	+
	253	+	f, err := os.Open(filepath.Join(dir, png))
	254	+	require.NoError(t, err)
	255	+	defer f.Close()
	256	+	img, _, err := image.Decode(f)
	257	+	require.NoError(t, err)
	258	+	require.Equal(t, image.Rect(0, 0, w, hgt), img.Bounds())
	259	+	r, g, b, _ := img.At(0, 0).RGBA()
	260	+	require.Equal(t, uint32(pix[2])*0x101, r)
	261	+	require.Equal(t, uint32(pix[1])*0x101, g)
	262	+	require.Equal(t, uint32(pix[0])*0x101, b)
	263	+	}
	264	+
	265	+	// scanOnceDone sends a request that fails fast on a blank capture and waits
	266	+	// for its result, which proves the previous dump finished.
	267	+	func scanOnceDone(t testing.T, h harness) ports.HintScanResult {
	268	+	t.Helper()
	269	+	go func() {
	270	+	creq := <-h.captures
	271	+	h.captured <- ports.CaptureDone{ID: creq.ID, Err: errors.New("stop")}
	272	+	}()
	273	+	h.requests <- ports.HintScanRequest{Seq: 99, Output: "x", Width: 8, Height: 8, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 8, H: 8}}}}
	274	+	return <-h.results
	275	+	}
	276	+
	277	+	func TestHintDumpDisabled(t *testing.T) {
	278	+	if _, err := exec.LookPath("tesseract"); err != nil {
	279	+	t.Skip("tesseract not on PATH")
	280	+	}
	281	+	pix, w, hgt := loadBGRA(t)
	282	+	dir := filepath.Join(t.TempDir(), "hints")
	283	+	h := start(t, 5*time.Second)
	284	+	scanOnce(t, h, pix, w, hgt, 1)
	285	+	scanOnceDone(t, h)
	286	+	_, err := os.Stat(dir)
	287	+	require.ErrorIs(t, err, os.ErrNotExist)
	288	+	}
	289	+
	290	+	func TestHintDumpRetention(t *testing.T) {
	291	+	dir := t.TempDir()
	292	+	for i := 0; i < 60; i++ {
	293	+	for _, ext := range []string{".png", ".annotated.png", ".json"} {
	294	+	name := fmt.Sprintf("20260101-000000.%03d-o%s", i, ext)
	295	+	require.NoError(t, os.WriteFile(filepath.Join(dir, name), nil, 0o600))
	296	+	}
	297	+	}
	298	+	require.NoError(t, prune(dir, maxDumps))
	299	+	entries, err := os.ReadDir(dir)
	300	+	require.NoError(t, err)
	301	+	require.Len(t, entries, maxDumps*3)
	302	+	require.Equal(t, "20260101-000000.010-o.annotated.png", entries[0].Name())
	303	+	}
	304	+
	305	+	func TestScanMergesSource(t *testing.T) {
	306	+	pix, w, hgt := loadBGRA(t)
	307	+	src := newMocktargetSource(t)
	308	+	src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
	309	+	wins := []ports.HintWindow{{ID: 4, PID: 9, Rect: ports.Rect{X: 0, Y: 0, W: w, H: hgt}}}
	310	+	at := []ports.HintTarget{{Rect: ports.Rect{X: 5, Y: 5, W: 30, H: 10}, Kind: ports.HintLink}}
	311	+	src.EXPECT().Targets(mock.Anything, wins, false).Return(at, map[ports.WindowID]bool{4: true}, nil).Once()
	312	+	h := startWith(t, 5*time.Second, Options{Source: src})
	313	+	go func() {
	314	+	creq := <-h.captures
	315	+	copy(creq.Mapped, pix)
	316	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	317	+	}()
	318	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: wins}
	319	+	res := <-h.results
	320	+	require.NoError(t, res.Err)
	321	+	// The whole frame is covered: only the accessibility target remains.
	322	+	require.Equal(t, at, res.Targets)
	323	+	}
	324	+
	325	+	func TestScanSourceErrorUsesOCR(t *testing.T) {
	326	+	if _, err := exec.LookPath("tesseract"); err != nil {
	327	+	t.Skip("tesseract not on PATH")
	328	+	}
	329	+	pix, w, hgt := loadBGRA(t)
	330	+	src := newMocktargetSource(t)
	331	+	src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
	332	+	wins := []ports.HintWindow{{ID: 4, PID: 9, Rect: ports.Rect{W: w, H: hgt}}}
	333	+	src.EXPECT().Targets(mock.Anything, wins, false).Return(nil, nil, errors.New("bus gone")).Once()
	334	+	h := startWith(t, 5*time.Second, Options{Source: src})
	335	+	go func() {
	336	+	creq := <-h.captures
	337	+	copy(creq.Mapped, pix)
	338	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	339	+	}()
	340	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: wins}
	341	+	res := <-h.results
	342	+	require.NoError(t, res.Err)
	343	+	require.NotEmpty(t, res.Targets)
	344	+	}
	345	+
	346	+	// The worker keeps no frame: the mapped buffer belongs to the scan, and to the
	347	+	// dump job until it is written; every mapping is released afterwards.
	348	+	func TestWorkerHoldsNoFrameAfterScan(t *testing.T) {
	349	+	for _, dumpDir := range []string{"", filepath.Join(t.TempDir(), "hints")} {
	350	+	// OCR is "installed" but its binary cannot run: only the memory matters.
	351	+	w := &worker{dumpDir: dumpDir, ocr: ocrState{done: true, binary: "/nonexistent/tesseract", lang: "eng"}}
	352	+	requests := make(chan ports.HintScanRequest)
	353	+	captures := make(chan ports.CaptureRequest, 1)
	354	+	captured := make(chan ports.CaptureDone, 1)
	355	+	results := make(chan ports.HintScanResult, 1)
	356	+	go func() {
	357	+	creq := <-captures
	358	+	captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	359	+	}()
	360	+	req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 64, H: 32}}}}
	361	+	res := w.serve(context.Background(), req, requests, captures, captured, results, 5*time.Second)
	362	+	require.NoError(t, res.Err)
	363	+	if dumpDir == "" {
	364	+	require.Nil(t, w.pending)
	365	+	require.Zero(t, w.live, "frame and crops are released before the result")
	366	+	} else {
	367	+	require.NotNil(t, w.pending)
	368	+	require.Equal(t, 1, w.live, "only the frame, kept for the dump")
	369	+	}
	370	+	w.dump(context.Background())
	371	+	require.Nil(t, w.pending)
	372	+	require.Zero(t, w.live)
	373	+	require.Empty(t, w.cache) // OCR failed: nothing cached
	374	+	}
	375	+	}
	376	+
	377	+	// A capture that timed out stays mapped until the output answers it.
	378	+	func TestTimedOutCaptureIsReleasedWhenDone(t *testing.T) {
	379	+	w := &worker{dumpDir: t.TempDir()}
	380	+	captures := make(chan ports.CaptureRequest, 1)
	381	+	captured := make(chan ports.CaptureDone, 1)
	382	+	req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1}
	383	+	res := w.serve(context.Background(), req, make(chan ports.HintScanRequest), captures, captured, make(chan ports.HintScanResult, 1), 20*time.Millisecond)
	384	+	require.ErrorIs(t, res.Err, ErrTimeout)
	385	+	creq := <-captures
	386	+	require.Equal(t, 1, w.live)
	387	+	creq.Mapped[0] = 1 // still valid: the output may write until CaptureDone
	388	+	w.reap(creq.ID)
	389	+	require.Zero(t, w.live)
	390	+	}
	391	+
	392	+	// Without tesseract OCR is off for good: no capture, only accessibility targets.
	393	+	func TestNoTesseractSkipsCapture(t *testing.T) {
	394	+	t.Setenv("PATH", t.TempDir())
	395	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
	396	+	covered := ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: 30, W: 5, H: 5}}
	397	+	wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}, {ID: 2, Rect: ports.Rect{X: 20, W: 20, H: 20}}}
	398	+	src := newMocktargetSource(t)
	399	+	src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
	400	+	src.EXPECT().Targets(mock.Anything, wins, false).Return([]ports.HintTarget{button, covered}, map[ports.WindowID]bool{2: true}, nil).Times(2)
	401	+	h := startWith(t, time.Second, Options{Source: src})
	402	+	for seq := uint64(1); seq <= 2; seq++ {
	403	+	h.requests <- ports.HintScanRequest{Seq: seq, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: wins}
	404	+	res := <-h.results
	405	+	require.NoError(t, res.Err)
	406	+	require.Equal(t, []ports.HintTarget{button, covered}, res.Targets)
	407	+	}
	408	+	require.Empty(t, h.captures)
	409	+	}
	410	+
	411	+	// With debug dumps on, the frame is still captured (for the dump only).
	412	+	func TestNoTesseractDumpStillCaptures(t *testing.T) {
	413	+	t.Setenv("PATH", t.TempDir())
	414	+	dir := filepath.Join(t.TempDir(), "hints")
	415	+	h := startWith(t, 5*time.Second, Options{DumpDir: dir})
	416	+	go func() {
	417	+	creq := <-h.captures
	418	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	419	+	}()
	420	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 40, H: 20}}}}
	421	+	require.NoError(t, (<-h.results).Err)
	422	+	waitFiles(t, dir, 3)
	423	+	}
	424	+
	425	+	// A capture that completes after its scan timed out is released while the
	426	+	// worker is idle, without another scan, and its reply does not linger.
	427	+	func TestHintLateCaptureReaped(t *testing.T) {
	428	+	h := startCapturing(t, 50*time.Millisecond)
	429	+	h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 64, Height: 32, Scale: 1}
	430	+	creq := <-h.captures
	431	+	res := <-h.results
	432	+	require.ErrorIs(t, res.Err, ErrTimeout)
	433	+	// The worker is idle; the output finally answers.
	434	+	h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	435	+	require.Eventually(t, func() bool { return len(h.captured) == 0 }, time.Second, time.Millisecond)
	436	+	}

A internal/adapters/hints/hybrid_test.go +177 -0

@@ -0,0 +1,177 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"hash/maphash"
	6	+	"image"
	7	+	"image/draw"
	8	+	"image/png"
	9	+	"os"
	10	+	"os/exec"
	11	+	"runtime"
	12	+	"testing"
	13	+
	14	+	"github.com/bnema/neferwl/internal/ports"
	15	+	"github.com/stretchr/testify/require"
	16	+	)
	17	+
	18	+	func TestWindowCrop(t *testing.T) {
	19	+	for _, tc := range []struct {
	20	+	name string
	21	+	r ports.Rect
	22	+	scale float64
	23	+	want image.Rectangle
	24	+	}{
	25	+	{"unit", ports.Rect{X: 10, Y: 20, W: 100, H: 50}, 1, image.Rect(10, 20, 110, 70)},
	26	+	{"fractional", ports.Rect{X: 11, Y: 21, W: 100, H: 50}, 1.2, image.Rect(13, 25, 134, 86)},
	27	+	{"off output", ports.Rect{X: -10, Y: 20, W: 100, H: 100}, 1.2, image.Rect(0, 24, 108, 100)},
	28	+	{"outside", ports.Rect{X: 1000, Y: 20, W: 100, H: 100}, 1, image.Rectangle{}},
	29	+	} {
	30	+	t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.want, windowCrop(tc.r, tc.scale, 200, 100)) })
	31	+	}
	32	+	}
	33	+
	34	+	func sourceResult(ar atspiResult) *pendingSource {
	35	+	p := &pendingSource{done: make(chan struct{}), res: ar}
	36	+	close(p.done)
	37	+	return p
	38	+	}
	39	+
	40	+	func TestHybridCacheAndMerge(t *testing.T) {
	41	+	t.Setenv("PATH", t.TempDir()) // Cached OCR must not need tesseract.
	42	+	buf := make([]byte, 2001004)
	43	+	wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 100, H: 100}}, {ID: 2, Rect: ports.Rect{X: 100, W: 100, H: 100}}}
	44	+	req := ports.HintScanRequest{Output: "o", Width: 200, Height: 100, Scale: 1, Windows: wins}
	45	+	seed := maphash.MakeSeed()
	46	+	r := windowCrop(wins[1].Rect, 1, 200, 100)
	47	+	oldWorkers := ocrWorkers
	48	+	ocrWorkers = 1 // one band covers the crop
	49	+	t.Cleanup(func() { ocrWorkers = oldWorkers })
	50	+	ext := image.Rect(0, 0, 100, 100)
	51	+	entry := bandCache{hash: (&worker{cacheSeed: seed}).cropHash(buf, 800, r), ext: ext, words: []OCRWord{{Text: "example.org", Rect: image.Rect(10, 20, 90, 40)}}}
	52	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 1, Y: 1, W: 10, H: 10}}
	53	+	link := ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: 110, Y: 20, W: 80, H: 20}}
	54	+	uncoveredButton := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 101, Y: 1, W: 10, H: 10}}
	55	+	ar := atspiResult{targets: []ports.HintTarget{button, uncoveredButton}, covered: map[ports.WindowID]bool{1: true}}
	56	+	w := worker{ocr: ocrState{done: true, binary: "tesseract", lang: "eng"}, cacheSeed: seed, cache: map[bandKey]bandCache{{output: "o", window: 2, band: 0, bands: 1}: entry}}
	57	+	got := w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
	58	+	require.Equal(t, []ports.HintTarget{button, uncoveredButton, link}, got.Targets)
	59	+	req.LinksOnly = true
	60	+	got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
	61	+	require.Equal(t, []ports.HintTarget{link}, got.Targets)
	62	+	// Changing one pixel invalidates only the crop, never leaving stale links.
	63	+	buf[100*4]++
	64	+	got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
	65	+	require.Empty(t, got.Targets)
	66	+	}
	67	+
	68	+	func TestMissingTesseractKeepsAccessibilityButtons(t *testing.T) {
	69	+	t.Setenv("PATH", t.TempDir())
	70	+	w := worker{}
	71	+	button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
	72	+	req := ports.HintScanRequest{Width: 20, Height: 20, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}}}
	73	+	got := w.scan(context.Background(), req, frame{pix: make([]byte, 1600), stride: 80}, 0, ports.HintScanResult{}, sourceResult(atspiResult{targets: []ports.HintTarget{button}}))
	74	+	require.NoError(t, got.Err)
	75	+	require.Equal(t, []ports.HintTarget{button}, got.Targets)
	76	+	}
	77	+
	78	+	func TestNoWindowsProducesNoPixelTargets(t *testing.T) {
	79	+	pix, width, height := loadBGRA(t)
	80	+	w := worker{}
	81	+	got := w.scan(context.Background(), ports.HintScanRequest{Width: width, Height: height}, frame{pix: pix, stride: width * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	82	+	require.Empty(t, got.Targets)
	83	+	}
	84	+
	85	+	func TestStripMappingAndPolarity(t *testing.T) {
	86	+	img := image.NewRGBA(image.Rect(0, 0, 200, 200))
	87	+	strips := packLines(img, []image.Rectangle{image.Rect(20, 20, 160, 40), image.Rect(20, 80, 160, 112)}, 2, true)
	88	+	require.Len(t, strips, 2)
	89	+	for _, s := range strips {
	90	+	for _, off := range s.offsets {
	91	+	require.Equal(t, 1, off.factor)
	92	+	require.Equal(t, uint8(255), s.img.GrayAt(off.packed.Min.X, off.packed.Min.Y).Y)
	93	+	}
	94	+	}
	95	+	small := packLines(img, []image.Rectangle{image.Rect(20, 20, 100, 30)}, 1, true)
	96	+	require.Equal(t, 2, small[0].offsets[0].factor)
	97	+	}
	98	+
	99	+	func TestCropGrayAndHash(t *testing.T) {
	100	+	buf := make([]byte, 444)
	101	+	for y := 0; y < 4; y++ {
	102	+	for x := 0; x < 4; x++ {
	103	+	copy(buf[y16+x4:], []byte{0, 0, 255, 255}) // BGRA red
	104	+	}
	105	+	}
	106	+	g := cropGray(buf, 16, image.Rect(1, 1, 3, 3), make([]byte, 4))
	107	+	require.Equal(t, image.Rect(0, 0, 2, 2), g.Bounds())
	108	+	require.Equal(t, uint8(76), g.GrayAt(1, 1).Y)
	109	+	w := worker{cacheSeed: maphash.MakeSeed()}
	110	+	a := w.cropHash(buf, 16, image.Rect(1, 1, 3, 3))
	111	+	buf[0]++ // outside the crop
	112	+	require.Equal(t, a, w.cropHash(buf, 16, image.Rect(1, 1, 3, 3)))
	113	+	buf[16+4]++
	114	+	require.NotEqual(t, a, w.cropHash(buf, 16, image.Rect(1, 1, 3, 3)))
	115	+	}
	116	+
	117	+	func TestUpscaleBandGray(t *testing.T) {
	118	+	g := image.NewGray(image.Rect(0, 0, 3, 2))
	119	+	for i := range g.Pix {
	120	+	g.Pix[i] = uint8(i + 1)
	121	+	}
	122	+	sub := g.SubImage(image.Rect(1, 1, 3, 2)).(*image.Gray)
	123	+	up := upscaleBand(sub)
	124	+	require.Equal(t, image.Rect(0, 0, 4, 2), up.Bounds())
	125	+	require.Equal(t, []uint8{5, 5, 6, 6, 5, 5, 6, 6}, up.Pix)
	126	+	}
	127	+
	128	+	// A blinking cursor changes one band: the other bands keep their cached
	129	+	// words, and only the changed one is recognized again.
	130	+	func TestBandCacheSurvivesCursorBlink(t *testing.T) {
	131	+	binary, err := exec.LookPath("tesseract")
	132	+	if err != nil {
	133	+	t.Skip("tesseract not on PATH")
	134	+	}
	135	+	old := ocrWorkers
	136	+	ocrWorkers = 4
	137	+	t.Cleanup(func() { ocrWorkers = old })
	138	+	f, err := os.Open("testdata/terminal.png")
	139	+	require.NoError(t, err)
	140	+	defer f.Close()
	141	+	src, err := png.Decode(f)
	142	+	require.NoError(t, err)
	143	+	w, h := src.Bounds().Dx(), src.Bounds().Dy()
	144	+	rgba := image.NewRGBA(image.Rect(0, 0, w, h))
	145	+	draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
	146	+	buf := make([]byte, wh4)
	147	+	for i := 0; i < len(buf); i += 4 {
	148	+	buf[i], buf[i+1], buf[i+2], buf[i+3] = rgba.Pix[i+2], rgba.Pix[i+1], rgba.Pix[i], 0xff
	149	+	}
	150	+	_ = binary
	151	+	wk := &worker{}
	152	+	req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
	153	+	first := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	154	+	require.NotEmpty(t, first.Targets)
	155	+	require.Len(t, wk.cache, min(runtime.GOMAXPROCS(0), 4))
	156	+
	157	+	// The same frame: every band hits.
	158	+	again := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	159	+	require.Equal(t, first.Targets, again.Targets)
	160	+
	161	+	// One pixel at the bottom blinks: only the last band (and the context of
	162	+	// none other) is invalidated.
	163	+	before := map[bandKey]uint64{}
	164	+	for k, v := range wk.cache {
	165	+	before[k] = v.hash
	166	+	}
	167	+	buf[((h-1)w+3)4] ^= 0xff
	168	+	blink := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
	169	+	require.Equal(t, first.Targets, blink.Targets)
	170	+	changed := 0
	171	+	for k, v := range wk.cache {
	172	+	if before[k] != v.hash {
	173	+	changed++
	174	+	}
	175	+	}
	176	+	require.Equal(t, 1, changed)
	177	+	}

A internal/adapters/hints/mmap.go +40 -0

@@ -0,0 +1,40 @@
	1	+	package hints
	2	+
	3	+	import "syscall"
	4	+
	5	+	// mapping is a byte region that lives outside the Go heap: anonymous memory
	6	+	// from mmap, returned to the system by unmap. When mmap fails it falls back
	7	+	// to heap memory, which unmap then just drops.
	8	+	type mapping struct {
	9	+	b []byte
	10	+	mapped bool
	11	+	live *int
	12	+	}
	13	+
	14	+	// alloc returns a zeroed region of n bytes; w.live counts the regions not
	15	+	// yet released.
	16	+	func (w worker) alloc(n int) mapping {
	17	+	m := &mapping{live: &w.live}
	18	+	b, err := syscall.Mmap(-1, 0, n, syscall.PROT_READ\|syscall.PROT_WRITE, syscall.MAP_PRIVATE\|syscall.MAP_ANON)
	19	+	if err != nil {
	20	+	b = make([]byte, n)
	21	+	} else {
	22	+	m.mapped = true
	23	+	}
	24	+	m.b = b
	25	+	w.live++
	26	+	return m
	27	+	}
	28	+
	29	+	// unmap releases the region; it is safe to call twice and on nil. The
	30	+	// region must not be used afterwards, nor still be written by the output.
	31	+	func (m *mapping) unmap() {
	32	+	if m == nil \|\| m.b == nil {
	33	+	return
	34	+	}
	35	+	if m.mapped {
	36	+	_ = syscall.Munmap(m.b)
	37	+	}
	38	+	m.b = nil
	39	+	*m.live--
	40	+	}

A internal/adapters/hints/ocr.go +440 -0

@@ -0,0 +1,440 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"bufio"
	5	+	"bytes"
	6	+	"context"
	7	+	"fmt"
	8	+	"image"
	9	+	"image/png"
	10	+	"io"
	11	+	"math"
	12	+	"net"
	13	+	"os"
	14	+	"os/exec"
	15	+	"path/filepath"
	16	+	"regexp"
	17	+	"runtime"
	18	+	"sort"
	19	+	"strconv"
	20	+	"strings"
	21	+	"sync"
	22	+	"time"
	23	+
	24	+	"github.com/bnema/neferwl/internal/adapters/logging"
	25	+	"github.com/bnema/neferwl/internal/ports"
	26	+	)
	27	+
	28	+	// OCRWord is a recognized word in physical output coordinates.
	29	+	type OCRWord struct {
	30	+	Text string `json:"text"`
	31	+	Rect image.Rectangle `json:"rect"`
	32	+	Confidence float64 `json:"confidence"`
	33	+	// Underlined is set when a thin line runs under the word. UnderL and
	34	+	// UnderR are the x extent of that line (crop-local), which can reach the
	35	+	// neighbouring words of one underlined link.
	36	+	Underlined bool `json:"underlined,omitempty"`
	37	+	UnderL int `json:"under_l,omitempty"`
	38	+	UnderR int `json:"under_r,omitempty"`
	39	+	}
	40	+
	41	+	var languageWarning sync.Once
	42	+	var failureWarning sync.Once
	43	+
	44	+	func ocrLanguage(ctx context.Context, binary string) (string, error) {
	45	+	dir := os.Getenv("TESSDATA_PREFIX")
	46	+	if dir == "" {
	47	+	dir = "/usr/share/tessdata"
	48	+	}
	49	+	if _, err := os.Stat(filepath.Join(dir, "eng.traineddata")); err == nil {
	50	+	return "eng", nil
	51	+	}
	52	+	out, err := exec.CommandContext(ctx, binary, "--list-langs").Output()
	53	+	if err != nil {
	54	+	return "", err
	55	+	}
	56	+	for _, line := range strings.Split(string(out), "\n")[1:] {
	57	+	lang := strings.TrimSpace(line)
	58	+	if lang != "" && lang != "osd" {
	59	+	languageWarning.Do(func() {
	60	+	log := logging.For(ctx, "hints")
	61	+	log.Warn().Str("language", lang).Msg("English OCR data missing; install tesseract-data-eng")
	62	+	})
	63	+	return lang, nil
	64	+	}
	65	+	}
	66	+	return "", fmt.Errorf("no OCR language available")
	67	+	}
	68	+
	69	+	func parseTSV(data []byte, offset image.Point) ([]OCRWord, error) {
	70	+	var words []OCRWord
	71	+	scanner := bufio.NewScanner(bytes.NewReader(data))
	72	+	scanner.Buffer(make([]byte, 4096), 1024*1024)
	73	+	for scanner.Scan() {
	74	+	fields := strings.SplitN(scanner.Text(), "\t", 12)
	75	+	if len(fields) != 12 \|\| fields[0] != "5" {
	76	+	continue
	77	+	}
	78	+	var values [4]int
	79	+	for i := range values {
	80	+	v, err := strconv.Atoi(fields[6+i])
	81	+	if err != nil {
	82	+	return nil, fmt.Errorf("OCR TSV coordinate: %w", err)
	83	+	}
	84	+	values[i] = v
	85	+	}
	86	+	conf, err := strconv.ParseFloat(fields[10], 64)
	87	+	if err != nil {
	88	+	return nil, fmt.Errorf("OCR TSV confidence: %w", err)
	89	+	}
	90	+	if values[2] <= 0 \|\| values[3] <= 0 \|\| strings.TrimSpace(fields[11]) == "" {
	91	+	continue
	92	+	}
	93	+	r := image.Rect(values[0], values[1], values[0]+values[2], values[1]+values[3]).Add(offset)
	94	+	words = append(words, OCRWord{Text: fields[11], Rect: r, Confidence: conf})
	95	+	}
	96	+	return words, scanner.Err()
	97	+	}
	98	+
	99	+	// ocrWorkers is the maximum number of bands recognized in parallel. 4 keeps
	100	+	// the tesseract processes at about 60% of the memory of 8 for the same time.
	101	+	var ocrWorkers = 4
	102	+
	103	+	// pngLevel is the compression of non-grayscale images piped to tesseract.
	104	+	var pngLevel = png.BestSpeed
	105	+
	106	+	// rawPGM feeds grayscale images to tesseract as binary PGM instead of PNG.
	107	+	var rawPGM = true
	108	+
	109	+	// ocrState is the tesseract installation, resolved once.
	110	+	type ocrState struct {
	111	+	done bool
	112	+	binary string
	113	+	lang string
	114	+	err error
	115	+	}
	116	+
	117	+	func (s *ocrState) resolve(ctx context.Context) error {
	118	+	if !s.done {
	119	+	s.done = true
	120	+	if s.binary, s.err = exec.LookPath("tesseract"); s.err == nil {
	121	+	s.lang, s.err = ocrLanguage(ctx, s.binary)
	122	+	}
	123	+	}
	124	+	return s.err
	125	+	}
	126	+
	127	+	// ocrReady resolves tesseract on the first call. When it is missing, OCR is
	128	+	// disabled for the life of the process and the warning is logged once.
	129	+	func (w *worker) ocrReady(ctx context.Context) bool {
	130	+	if err := w.ocr.resolve(ctx); err != nil {
	131	+	failureWarning.Do(func() {
	132	+	log := logging.For(ctx, "hints")
	133	+	log.Warn().Err(err).Msg("OCR unavailable; install tesseract and tesseract-data-eng")
	134	+	})
	135	+	return false
	136	+	}
	137	+	return true
	138	+	}
	139	+
	140	+	// band is a horizontal slice of a window crop recognized by one tesseract
	141	+	// process. It reads rows [top,bottom) and owns the words centred in
	142	+	// [ownTop,ownBottom); the margin only supplies context.
	143	+	type band struct {
	144	+	idx, n int
	145	+	top, bottom int
	146	+	ownTop, ownBottom int
	147	+	}
	148	+
	149	+	func bandsOf(height int) []band {
	150	+	n := min(runtime.GOMAXPROCS(0), ocrWorkers, height)
	151	+	bands := make([]band, 0, max(n, 0))
	152	+	for i := range n {
	153	+	ownTop, ownBottom := iheight/n, (i+1)height/n
	154	+	bands = append(bands, band{idx: i, n: n, top: max(0, ownTop-40), bottom: min(height, ownBottom+40), ownTop: ownTop, ownBottom: ownBottom})
	155	+	}
	156	+	return bands
	157	+	}
	158	+
	159	+	// recognize reads the bands of jobs that missed the cache, in parallel.
	160	+	func (w worker) recognize(ctx context.Context, jobs []bandJob) {
	161	+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
	162	+	defer cancel()
	163	+	// One scan-wide pool: at most ocrWorkers tesseract processes run at once,
	164	+	// whatever the number of windows.
	165	+	sem := make(chan struct{}, max(ocrWorkers, 1))
	166	+	var wg sync.WaitGroup
	167	+	for _, j := range jobs {
	168	+	if j.hit {
	169	+	continue
	170	+	}
	171	+	select {
	172	+	case sem <- struct{}{}:
	173	+	case <-ctx.Done():
	174	+	j.err = ctx.Err()
	175	+	continue
	176	+	}
	177	+	wg.Go(func() {
	178	+	defer func() { <-sem }()
	179	+	j.words, j.err = readBand(ctx, &w.ocr, j)
	180	+	})
	181	+	}
	182	+	wg.Wait()
	183	+	}
	184	+
	185	+	// readBand recognizes one band; the words are crop-local and owned by it.
	186	+	func readBand(ctx context.Context, st ocrState, j bandJob) ([]OCRWord, error) {
	187	+	words, err := recognizeBand(ctx, st.binary, st.lang, j.gray)
	188	+	if err != nil {
	189	+	return nil, err
	190	+	}
	191	+	heights := make([]int, 0, len(words))
	192	+	for _, word := range words {
	193	+	if len(word.Text) >= 3 {
	194	+	heights = append(heights, word.Rect.Dy())
	195	+	}
	196	+	}
	197	+	sort.Ints(heights)
	198	+	if len(heights) > 0 && heights[len(heights)/2] < 12 {
	199	+	words, err = recognizeBand(ctx, st.binary, st.lang, upscaleBand(j.gray))
	200	+	if err != nil {
	201	+	return nil, err
	202	+	}
	203	+	for i := range words {
	204	+	r := words[i].Rect
	205	+	words[i].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
	206	+	}
	207	+	}
	208	+	markUnderlined(j.gray, words)
	209	+	owned := words[:0]
	210	+	for _, word := range words {
	211	+	// Overlap supplies context, but each word belongs to exactly one band.
	212	+	word.Rect = word.Rect.Add(image.Pt(0, j.ext.Min.Y))
	213	+	if center := (word.Rect.Min.Y + word.Rect.Max.Y) / 2; center >= j.own[0] && center < j.own[1] {
	214	+	owned = append(owned, word)
	215	+	}
	216	+	}
	217	+	return owned, nil
	218	+	}
	219	+
	220	+	// readWords recognizes the words of img, a window crop, in parallel bands.
	221	+	func readWords(ctx context.Context, img *image.Gray) ([]OCRWord, int, error) {
	222	+	w := &worker{}
	223	+	if err := w.ocr.resolve(ctx); err != nil {
	224	+	return nil, 0, err
	225	+	}
	226	+	var jobs []*bandJob
	227	+	for _, b := range bandsOf(img.Bounds().Dy()) {
	228	+	ext := image.Rect(0, b.top, img.Bounds().Dx(), b.bottom)
	229	+	jobs = append(jobs, &bandJob{ext: ext, own: [2]int{b.ownTop, b.ownBottom}, gray: img.SubImage(ext).(*image.Gray)})
	230	+	}
	231	+	w.recognize(ctx, jobs)
	232	+	var words []OCRWord
	233	+	for _, j := range jobs {
	234	+	if j.err != nil {
	235	+	return nil, len(jobs), j.err
	236	+	}
	237	+	for _, word := range j.words {
	238	+	if !duplicateWord(words, word) {
	239	+	words = append(words, word)
	240	+	}
	241	+	}
	242	+	}
	243	+	return words, len(jobs), nil
	244	+	}
	245	+
	246	+	// The link patterns compile on the first OCR link check, not at process
	247	+	// start: with hint mode off nothing here costs memory or CPU.
	248	+	var domainPattern = sync.OnceValue(func() *regexp.Regexp {
	249	+	return regexp.MustCompile(`(?i)^(?:[a-z0-9](?:[a-z0-9-][a-z0-9])?\.)+([a-z]{2,24})(?::[0-9]+)?(?:[/?#]\S)?$`)
	250	+	})
	251	+	var schemePattern = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`(?i)^(?:https?\|htps\|file\|ftp):/{1,2}\S+$`) })
	252	+	var emailPattern = sync.OnceValue(func() *regexp.Regexp {
	253	+	return regexp.MustCompile(`(?i)^[a-z0-9.!#$%&'*+/=?^_` + "`" + `{\|}~-]+@(?:[a-z0-9-]+\.)+[a-z]{2,24}$`)
	254	+	})
	255	+	var fileExtensions = sync.OnceValue(func() []string {
	256	+	return strings.Fields("go md txt toml json yaml yml rs py js ts sh c h png jpg log conf lock mod sum html css")
	257	+	})
	258	+
	259	+	func isTextLink(text string) bool {
	260	+	text = strings.Trim(text, "()[]<>.,;:'\"")
	261	+	if schemePattern().MatchString(text) \|\| (strings.HasPrefix(strings.ToLower(text), "www.") && len(text) > 4) \|\| emailPattern().MatchString(text) {
	262	+	return true
	263	+	}
	264	+	host := text
	265	+	if h, _, err := net.SplitHostPort(text); err == nil {
	266	+	host = h
	267	+	}
	268	+	if ip := net.ParseIP(host); ip != nil && ip.To4() != nil {
	269	+	return true
	270	+	}
	271	+	m := domainPattern().FindStringSubmatch(text)
	272	+	if m == nil {
	273	+	return false
	274	+	}
	275	+	for _, ext := range fileExtensions() {
	276	+	if strings.EqualFold(m[1], ext) {
	277	+	return false
	278	+	}
	279	+	}
	280	+	return true
	281	+	}
	282	+
	283	+	func textLinks(words []OCRWord, scale float64) []ports.HintTarget {
	284	+	if scale <= 0 {
	285	+	scale = 1
	286	+	}
	287	+	words = append([]OCRWord(nil), words...)
	288	+	sort.Slice(words, func(i, j int) bool {
	289	+	if words[i].Rect.Min.Y != words[j].Rect.Min.Y {
	290	+	return words[i].Rect.Min.Y < words[j].Rect.Min.Y
	291	+	}
	292	+	return words[i].Rect.Min.X < words[j].Rect.Min.X
	293	+	})
	294	+	var out []ports.HintTarget
	295	+	var urls []image.Rectangle // boxes of the URL targets, in pixels
	296	+	used := make([]bool, len(words))
	297	+	for i, word := range words {
	298	+	if used[i] {
	299	+	continue
	300	+	}
	301	+	text, rect := word.Text, word.Rect
	302	+	// Grow only touching words on the same baseline; ordinary prose is not joined.
	303	+	for {
	304	+	next := -1
	305	+	for j, candidate := range words {
	306	+	if j == i \|\| used[j] \|\| candidate.Rect.Min.X < rect.Max.X-2 {
	307	+	continue
	308	+	}
	309	+	gap := candidate.Rect.Min.X - rect.Max.X
	310	+	if gap > max(3, min(rect.Dy(), candidate.Rect.Dy())/5) {
	311	+	continue
	312	+	}
	313	+	if abs(candidate.Rect.Max.Y-rect.Max.Y) > max(3, rect.Dy()/3) {
	314	+	continue
	315	+	}
	316	+	if next < 0 \|\| candidate.Rect.Min.X < words[next].Rect.Min.X {
	317	+	next = j
	318	+	}
	319	+	}
	320	+	if next < 0 {
	321	+	break
	322	+	}
	323	+	text += words[next].Text
	324	+	rect = rect.Union(words[next].Rect)
	325	+	used[next] = true
	326	+	}
	327	+	if !isTextLink(text) {
	328	+	continue
	329	+	}
	330	+	x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
	331	+	right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
	332	+	out = append(out, ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
	333	+	urls = append(urls, rect)
	334	+	}
	335	+	return append(out, underlinedLinks(words, urls, scale)...)
	336	+	}
	337	+
	338	+	func abs(x int) int {
	339	+	if x < 0 {
	340	+	return -x
	341	+	}
	342	+	return x
	343	+	}
	344	+	func overlap(a, b image.Rectangle) float64 {
	345	+	intersection := a.Intersect(b)
	346	+	area := min(a.Dx()a.Dy(), b.Dx()b.Dy())
	347	+	if intersection.Empty() \|\| area <= 0 {
	348	+	return 0
	349	+	}
	350	+	return float64(intersection.Dx()*intersection.Dy()) / float64(area)
	351	+	}
	352	+	func targetRect(t ports.HintTarget) image.Rectangle {
	353	+	return image.Rect(t.Rect.X, t.Rect.Y, t.Rect.X+t.Rect.W, t.Rect.Y+t.Rect.H)
	354	+	}
	355	+	func recognizeBand(ctx context.Context, binary, lang string, img image.Image) ([]OCRWord, error) {
	356	+	return recognizeImage(ctx, binary, lang, img, "11", false)
	357	+	}
	358	+
	359	+	// pgmReader streams a grayscale image as binary PGM (P5), row by row.
	360	+	type pgmReader struct {
	361	+	head []byte
	362	+	img *image.Gray
	363	+	y int
	364	+	row []byte
	365	+	}
	366	+
	367	+	func newPGMReader(img image.Gray) pgmReader {
	368	+	b := img.Bounds()
	369	+	return &pgmReader{head: fmt.Appendf(nil, "P5\n%d %d\n255\n", b.Dx(), b.Dy()), img: img}
	370	+	}
	371	+
	372	+	func (r *pgmReader) Read(p []byte) (int, error) {
	373	+	n := 0
	374	+	for n < len(p) {
	375	+	if len(r.head) > 0 {
	376	+	c := copy(p[n:], r.head)
	377	+	r.head, n = r.head[c:], n+c
	378	+	continue
	379	+	}
	380	+	if len(r.row) == 0 {
	381	+	b := r.img.Bounds()
	382	+	if r.y >= b.Dy() {
	383	+	break
	384	+	}
	385	+	r.row = r.img.Pix[(r.y+b.Min.Y-r.img.Rect.Min.Y)*r.img.Stride+(b.Min.X-r.img.Rect.Min.X):][:b.Dx()]
	386	+	r.y++
	387	+	}
	388	+	c := copy(p[n:], r.row)
	389	+	r.row, n = r.row[c:], n+c
	390	+	}
	391	+	if n == 0 {
	392	+	return 0, io.EOF
	393	+	}
	394	+	return n, nil
	395	+	}
	396	+
	397	+	func recognizeImage(ctx context.Context, binary, lang string, img image.Image, psm string, normalized bool) ([]OCRWord, error) {
	398	+	args := []string{"stdin", "stdout", "--psm", psm, "-l", lang}
	399	+	if normalized {
	400	+	args = append(args, "-c", "tessedit_do_invert=0")
	401	+	}
	402	+	args = append(args, "tsv")
	403	+	cmd := exec.CommandContext(ctx, binary, args...)
	404	+	cmd.Env = append(os.Environ(), "OMP_THREAD_LIMIT=1")
	405	+	if g, ok := img.(*image.Gray); ok && rawPGM {
	406	+	// Raw grayscale: no deflate, no encoder buffers.
	407	+	cmd.Stdin = newPGMReader(g)
	408	+	} else {
	409	+	// The PNG streams into tesseract; no encoded copy is held.
	410	+	pr, pw := io.Pipe()
	411	+	defer func() { _ = pr.Close() }()
	412	+	go func() {
	413	+	enc := png.Encoder{CompressionLevel: pngLevel}
	414	+	_ = pw.CloseWithError(enc.Encode(pw, img))
	415	+	}()
	416	+	cmd.Stdin = pr
	417	+	}
	418	+	var stderr bytes.Buffer
	419	+	cmd.Stderr = &stderr
	420	+	output, err := cmd.Output()
	421	+	if err != nil {
	422	+	return nil, fmt.Errorf("tesseract: %w: %s", err, stderr.String())
	423	+	}
	424	+	return parseTSV(output, image.Point{})
	425	+	}
	426	+
	427	+	// upscaleBand doubles a grayscale band with nearest-neighbour sampling.
	428	+	func upscaleBand(src image.Gray) image.Gray {
	429	+	b := src.Bounds()
	430	+	dst := image.NewGray(image.Rect(0, 0, b.Dx()2, b.Dy()2))
	431	+	for y := 0; y < b.Dy(); y++ {
	432	+	row := src.Pix[(y+b.Min.Y-src.Rect.Min.Y)*src.Stride+(b.Min.X-src.Rect.Min.X):][:b.Dx()]
	433	+	top := dst.Pix[2ydst.Stride:][:dst.Stride]
	434	+	for x, v := range row {
	435	+	top[2x], top[2x+1] = v, v
	436	+	}
	437	+	copy(dst.Pix[(2y+1)dst.Stride:], top)
	438	+	}
	439	+	return dst
	440	+	}

A internal/adapters/hints/ocr_test.go +238 -0

@@ -0,0 +1,238 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/draw"
	7	+	"image/png"
	8	+	"io"
	9	+	"os"
	10	+	"os/exec"
	11	+	"path/filepath"
	12	+	"reflect"
	13	+	"strings"
	14	+	"testing"
	15	+	"time"
	16	+
	17	+	"github.com/bnema/neferwl/internal/ports"
	18	+	)
	19	+
	20	+	func TestTextLink(t *testing.T) {
	21	+	yes := []string{"ermite.bnema.dev", "google.fr", "example.org/path?q=1", "https.google.fr", "https://example.org", "file:///tmp/main.go", "ftp://host/file", "www.main.go", "user@example.org", "127.0.0.1", "192.168.1.1:8080", "(https://example.org),", "htps://google.fr", "https:/example.org"}
	22	+	no := []string{"ermite-runner", "main.go", "README.md", "v1.2.3", "1.5", "e.g.", "report.txt", "source.rs", "alpha", "999.999.999.999"}
	23	+	for _, s := range yes {
	24	+	t.Run(s, func(t *testing.T) {
	25	+	if !isTextLink(s) {
	26	+	t.Errorf("not accepted: %s", s)
	27	+	}
	28	+	})
	29	+	}
	30	+	for _, s := range no {
	31	+	t.Run(s, func(t *testing.T) {
	32	+	if isTextLink(s) {
	33	+	t.Errorf("incorrectly accepted: %s", s)
	34	+	}
	35	+	})
	36	+	}
	37	+	}
	38	+	func TestParseTSV(t *testing.T) {
	39	+	data := "level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext\n5\t1\t1\t1\t1\t1\t10\t20\t30\t12\t91.5\texample.org\n"
	40	+	words, err := parseTSV([]byte(data), image.Pt(0, 40))
	41	+	if err != nil \|\| len(words) != 1 {
	42	+	t.Fatalf("%v %v", words, err)
	43	+	}
	44	+	if words[0].Rect != image.Rect(10, 60, 40, 72) \|\| words[0].Confidence != 91.5 {
	45	+	t.Fatal(words)
	46	+	}
	47	+	_, err = parseTSV([]byte(strings.ReplaceAll(data, "\t10\t", "\tbad\t")), image.Point{})
	48	+	if err == nil {
	49	+	t.Fatal("expected malformed coordinate error")
	50	+	}
	51	+	}
	52	+	func TestJoinedOCRLink(t *testing.T) {
	53	+	words := []OCRWord{{Text: "https:/", Rect: image.Rect(20, 20, 90, 40)}, {Text: "/example.org", Rect: image.Rect(91, 20, 200, 40)}}
	54	+	got := textLinks(words, 2)
	55	+	if len(got) != 1 \|\| got[0].Rect != (ports.Rect{X: 10, Y: 10, W: 90, H: 10}) {
	56	+	t.Fatal(got)
	57	+	}
	58	+	}
	59	+	func TestRealOCR(t *testing.T) {
	60	+	if _, err := exec.LookPath("tesseract"); err != nil {
	61	+	t.Skip("tesseract not on PATH")
	62	+	}
	63	+	for _, name := range []string{"browser", "terminal"} {
	64	+	t.Run(name, func(t *testing.T) {
	65	+	f, err := os.Open("testdata/" + name + ".png")
	66	+	if err != nil {
	67	+	t.Fatal(err)
	68	+	}
	69	+	defer f.Close()
	70	+	src, err := png.Decode(f)
	71	+	if err != nil {
	72	+	t.Fatal(err)
	73	+	}
	74	+	img := image.NewRGBA(src.Bounds())
	75	+	draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
	76	+	start := time.Now()
	77	+	words, bands, err := readWords(context.Background(), toGray(img))
	78	+	if err != nil {
	79	+	t.Fatal(err)
	80	+	}
	81	+	links := textLinks(words, 1)
	82	+	t.Logf("%s %dx%d ocr_ms=%.1f words=%d links=%d bands=%d", name, img.Bounds().Dx(), img.Bounds().Dy(), ms(time.Since(start)), len(words), len(links), bands)
	83	+	for _, w := range words {
	84	+	t.Logf("%s %v", w.Text, w.Rect)
	85	+	}
	86	+	if name == "terminal" {
	87	+	for _, r := range []image.Rectangle{image.Rect(160, 65, 380, 85), image.Rect(140, 90, 490, 110), image.Rect(95, 115, 630, 138), image.Rect(120, 140, 330, 163), image.Rect(385, 140, 740, 163)} {
	88	+	found := false
	89	+	for _, l := range links {
	90	+	if overlap(targetRect(l), r) > .5 {
	91	+	found = true
	92	+	}
	93	+	}
	94	+	if !found {
	95	+	t.Errorf("missing terminal URL near %v", r)
	96	+	}
	97	+	}
	98	+	if len(links) != 5 {
	99	+	t.Errorf("want 5 terminal links, got %d: %v", len(links), links)
	100	+	}
	101	+	} else if len(links) == 0 {
	102	+	t.Error("no browser links")
	103	+	}
	104	+	})
	105	+	}
	106	+	}
	107	+
	108	+	func TestSmallGlyphOCR(t *testing.T) {
	109	+	binary, err := exec.LookPath("tesseract")
	110	+	if err != nil {
	111	+	t.Skip("tesseract not on PATH")
	112	+	}
	113	+	f, err := os.Open("testdata/terminal.png")
	114	+	if err != nil {
	115	+	t.Fatal(err)
	116	+	}
	117	+	defer f.Close()
	118	+	src, err := png.Decode(f)
	119	+	if err != nil {
	120	+	t.Fatal(err)
	121	+	}
	122	+	small := image.NewGray(image.Rect(0, 0, 640, 200))
	123	+	for y := 0; y < 200; y++ {
	124	+	for x := 0; x < 640; x++ {
	125	+	small.Set(x, y, src.At(x2, y2))
	126	+	}
	127	+	}
	128	+	lang, err := ocrLanguage(context.Background(), binary)
	129	+	if err != nil {
	130	+	t.Fatal(err)
	131	+	}
	132	+	for _, factor := range []int{1, 2} {
	133	+	var img image.Image = small
	134	+	if factor == 2 {
	135	+	img = upscaleBand(small)
	136	+	}
	137	+	start := time.Now()
	138	+	words, err := recognizeBand(context.Background(), binary, lang, img)
	139	+	if err != nil {
	140	+	t.Fatal(err)
	141	+	}
	142	+	t.Logf("glyph experiment %dx: %.1f ms, words=%d links=%d", factor, ms(time.Since(start)), len(words), len(textLinks(words, 1)))
	143	+	}
	144	+	}
	145	+
	146	+	func TestPGMReader(t *testing.T) {
	147	+	g := image.NewGray(image.Rect(0, 0, 3, 3))
	148	+	for i := range g.Pix {
	149	+	g.Pix[i] = uint8(i + 1)
	150	+	}
	151	+	sub := g.SubImage(image.Rect(1, 1, 3, 3)).(*image.Gray)
	152	+	got, err := io.ReadAll(newPGMReader(sub))
	153	+	if err != nil {
	154	+	t.Fatal(err)
	155	+	}
	156	+	if want := "P5\n2 2\n255\n" + string([]byte{5, 6, 8, 9}); string(got) != want {
	157	+	t.Fatalf("%q", got)
	158	+	}
	159	+	}
	160	+
	161	+	// Tesseract must read the raw PGM exactly as it reads the PNG.
	162	+	func TestPGMMatchesPNG(t *testing.T) {
	163	+	binary, err := exec.LookPath("tesseract")
	164	+	if err != nil {
	165	+	t.Skip("tesseract not on PATH")
	166	+	}
	167	+	f, err := os.Open("testdata/terminal.png")
	168	+	if err != nil {
	169	+	t.Fatal(err)
	170	+	}
	171	+	defer f.Close()
	172	+	src, err := png.Decode(f)
	173	+	if err != nil {
	174	+	t.Fatal(err)
	175	+	}
	176	+	rgba := image.NewRGBA(src.Bounds())
	177	+	draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
	178	+	gray := toGray(rgba)
	179	+	lang, err := ocrLanguage(context.Background(), binary)
	180	+	if err != nil {
	181	+	t.Fatal(err)
	182	+	}
	183	+	defer func(v bool) { rawPGM = v }(rawPGM)
	184	+	var res [2][]OCRWord
	185	+	for i, raw := range []bool{false, true} {
	186	+	rawPGM = raw
	187	+	if res[i], err = recognizeBand(context.Background(), binary, lang, gray); err != nil {
	188	+	t.Fatal(err)
	189	+	}
	190	+	}
	191	+	if len(res[0]) == 0 \|\| !reflect.DeepEqual(res[0], res[1]) {
	192	+	t.Fatalf("PNG %d words, PGM %d words", len(res[0]), len(res[1]))
	193	+	}
	194	+	}
	195	+
	196	+	// TestRecognizeCapsProcesses runs the bands of several windows through a
	197	+	// stand-in tesseract that logs when each process starts and ends.
	198	+	func TestRecognizeCapsProcesses(t *testing.T) {
	199	+	dir := t.TempDir()
	200	+	logPath := filepath.Join(dir, "log")
	201	+	script := filepath.Join(dir, "tesseract")
	202	+	body := "#!/bin/sh\ncat >/dev/null\necho s >> " + logPath + "\nsleep 0.1\necho e >> " + logPath + "\nprintf 'level\\tpage_num\\tblock_num\\tpar_num\\tline_num\\tword_num\\tleft\\ttop\\twidth\\theight\\tconf\\ttext\\n'\n"
	203	+	if err := os.WriteFile(script, []byte(body), 0o700); err != nil {
	204	+	t.Fatal(err)
	205	+	}
	206	+	old := ocrWorkers
	207	+	ocrWorkers = 2
	208	+	t.Cleanup(func() { ocrWorkers = old })
	209	+	w := &worker{ocr: ocrState{done: true, binary: script, lang: "eng"}}
	210	+	var jobs []*bandJob
	211	+	for range 8 { // several windows' bands
	212	+	g := image.NewGray(image.Rect(0, 0, 8, 8))
	213	+	jobs = append(jobs, &bandJob{ext: g.Rect, own: [2]int{0, 8}, gray: g})
	214	+	}
	215	+	w.recognize(context.Background(), jobs)
	216	+	data, err := os.ReadFile(logPath)
	217	+	if err != nil {
	218	+	t.Fatal(err)
	219	+	}
	220	+	cur, peak, starts := 0, 0, 0
	221	+	for _, l := range strings.Fields(string(data)) {
	222	+	if l == "s" {
	223	+	cur++
	224	+	starts++
	225	+	peak = max(peak, cur)
	226	+	} else {
	227	+	cur--
	228	+	}
	229	+	}
	230	+	if starts != 8 \|\| peak > 2 {
	231	+	t.Fatalf("starts=%d peak=%d", starts, peak)
	232	+	}
	233	+	for _, j := range jobs {
	234	+	if j.err != nil {
	235	+	t.Fatal(j.err)
	236	+	}
	237	+	}
	238	+	}

A internal/adapters/hints/scan_bench_test.go +49 -0

@@ -0,0 +1,49 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"os"
	6	+	"os/exec"
	7	+	"runtime"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/neferwl/internal/ports"
	12	+	)
	13	+
	14	+	// BenchmarkScan runs whole scans of a 5120x2160 frame with a noisy window
	15	+	// (the cache never hits) and reports the Go heap bytes allocated per scan.
	16	+	func BenchmarkScan(b *testing.B) {
	17	+	if _, err := exec.LookPath("tesseract"); err != nil {
	18	+	b.Skip("tesseract not on PATH")
	19	+	}
	20	+	const w, h = 5120, 2160
	21	+	ctx := context.Background()
	22	+	w0 := &worker{}
	23	+	requests := make(chan ports.HintScanRequest)
	24	+	captures := make(chan ports.CaptureRequest, 1)
	25	+	captured := make(chan ports.CaptureDone, 1)
	26	+	results := make(chan ports.HintScanResult, 1)
	27	+	req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
	28	+	if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
	29	+	rawPGM = false
	30	+	}
	31	+	b.ReportAllocs()
	32	+	var peak uint64
	33	+	for i := range b.N {
	34	+	go func() {
	35	+	creq := <-captures
	36	+	creq.Mapped[0] = byte(i) // each scan differs: no cache hit
	37	+	creq.Mapped[1] = byte(i >> 8)
	38	+	captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
	39	+	}()
	40	+	res := w0.serve(ctx, req, requests, captures, captured, results, 5*time.Second)
	41	+	if res.Err != nil {
	42	+	b.Fatal(res.Err)
	43	+	}
	44	+	var m runtime.MemStats
	45	+	runtime.ReadMemStats(&m)
	46	+	peak = max(peak, m.HeapSys-m.HeapReleased)
	47	+	}
	48	+	b.ReportMetric(float64(peak)/1e6, "heapMB")
	49	+	}

A internal/adapters/hints/strips_test.go +124 -0

@@ -0,0 +1,124 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"context"
	5	+	"image"
	6	+	"image/color"
	7	+	"image/draw"
	8	+	"os/exec"
	9	+	"runtime"
	10	+	"sort"
	11	+	"sync"
	12	+	"time"
	13	+	)
	14	+
	15	+	type stripOffset struct {
	16	+	source, packed image.Rectangle
	17	+	factor int
	18	+	}
	19	+	type ocrStrip struct {
	20	+	img *image.Gray
	21	+	offsets []stripOffset
	22	+	}
	23	+
	24	+	func packLines(src *image.RGBA, lines []image.Rectangle, workers int, normalize bool) []ocrStrip {
	25	+	n := min(workers, len(lines))
	26	+	if n == 0 {
	27	+	return nil
	28	+	}
	29	+	groups := make([][]image.Rectangle, n)
	30	+	areas := make([]int, n)
	31	+	lines = append([]image.Rectangle(nil), lines...)
	32	+	sort.Slice(lines, func(i, j int) bool { return lines[i].Dx()lines[i].Dy() > lines[j].Dx()lines[j].Dy() })
	33	+	for _, r := range lines {
	34	+	best := 0
	35	+	for i := range n {
	36	+	if areas[i] < areas[best] {
	37	+	best = i
	38	+	}
	39	+	}
	40	+	groups[best] = append(groups[best], r)
	41	+	areas[best] += r.Dx() * r.Dy()
	42	+	}
	43	+	strips := make([]ocrStrip, n)
	44	+	for i, group := range groups {
	45	+	width, height := 0, 8
	46	+	for _, r := range group {
	47	+	factor := 1
	48	+	if r.Dy() < 12 {
	49	+	factor = 2
	50	+	}
	51	+	crop := r.Inset(-4).Intersect(src.Bounds())
	52	+	packed := image.Rect(4, height, 4+crop.Dx()factor, height+crop.Dy()factor)
	53	+	strips[i].offsets = append(strips[i].offsets, stripOffset{crop, packed, factor})
	54	+	width = max(width, packed.Max.X+4)
	55	+	height = packed.Max.Y + 8
	56	+	}
	57	+	dst := image.NewGray(image.Rect(0, 0, width, height))
	58	+	draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
	59	+	for _, off := range strips[i].offsets {
	60	+	// The padded perimeter estimates the background without depending on text colour.
	61	+	var sum, count int
	62	+	r := off.source
	63	+	for x := r.Min.X; x < r.Max.X; x++ {
	64	+	sum += int(color.GrayModel.Convert(src.At(x, r.Min.Y)).(color.Gray).Y)
	65	+	count++
	66	+	}
	67	+	invert := normalize && count > 0 && sum/count < 128
	68	+	for y := off.packed.Min.Y; y < off.packed.Max.Y; y++ {
	69	+	for x := off.packed.Min.X; x < off.packed.Max.X; x++ {
	70	+	v := color.GrayModel.Convert(src.At(r.Min.X+(x-off.packed.Min.X)/off.factor, r.Min.Y+(y-off.packed.Min.Y)/off.factor)).(color.Gray).Y
	71	+	if invert {
	72	+	v = 255 - v
	73	+	}
	74	+	dst.SetGray(x, y, color.Gray{Y: v})
	75	+	}
	76	+	}
	77	+	}
	78	+	strips[i].img = dst
	79	+	}
	80	+	return strips
	81	+	}
	82	+
	83	+	func readStrips(ctx context.Context, img *image.RGBA, lines []image.Rectangle, workers int, normalize bool) ([]OCRWord, int, error) {
	84	+	binary, err := exec.LookPath("tesseract")
	85	+	if err != nil {
	86	+	return nil, 0, err
	87	+	}
	88	+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
	89	+	defer cancel()
	90	+	lang, err := ocrLanguage(ctx, binary)
	91	+	if err != nil {
	92	+	return nil, 0, err
	93	+	}
	94	+	strips := packLines(img, lines, min(workers, runtime.GOMAXPROCS(0), 16), normalize)
	95	+	results := make([][]OCRWord, len(strips))
	96	+	errs := make([]error, len(strips))
	97	+	var wg sync.WaitGroup
	98	+	for i, s := range strips {
	99	+	wg.Go(func() {
	100	+	words, e := recognizeImage(ctx, binary, lang, s.img, "6", normalize)
	101	+	errs[i] = e
	102	+	for _, w := range words {
	103	+	center := image.Pt((w.Rect.Min.X+w.Rect.Max.X)/2, (w.Rect.Min.Y+w.Rect.Max.Y)/2)
	104	+	for _, off := range s.offsets {
	105	+	if center.In(off.packed) {
	106	+	r := w.Rect.Sub(off.packed.Min)
	107	+	w.Rect = image.Rect(r.Min.X/off.factor, r.Min.Y/off.factor, (r.Max.X+off.factor-1)/off.factor, (r.Max.Y+off.factor-1)/off.factor).Add(off.source.Min)
	108	+	results[i] = append(results[i], w)
	109	+	break
	110	+	}
	111	+	}
	112	+	}
	113	+	})
	114	+	}
	115	+	wg.Wait()
	116	+	var words []OCRWord
	117	+	for i := range strips {
	118	+	if errs[i] != nil {
	119	+	return nil, len(strips), errs[i]
	120	+	}
	121	+	words = append(words, results[i]...)
	122	+	}
	123	+	return words, len(strips), nil
	124	+	}

A internal/adapters/hints/target_source_mock_test.go +174 -0

@@ -0,0 +1,174 @@
	1	+	// Code generated by mockery; DO NOT EDIT.
	2	+	// github.com/vektra/mockery
	3	+	// template: testify
	4	+
	5	+	package hints
	6	+
	7	+	import (
	8	+	"context"
	9	+
	10	+	"github.com/bnema/neferwl/internal/ports"
	11	+	mock "github.com/stretchr/testify/mock"
	12	+	)
	13	+
	14	+	// newMocktargetSource creates a new instance of mocktargetSource. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
	15	+	// The first argument is typically a *testing.T value.
	16	+	func newMocktargetSource(t interface {
	17	+	mock.TestingT
	18	+	Cleanup(func())
	19	+	}) *mocktargetSource {
	20	+	if helper, ok := t.(interface{ Helper() }); ok {
	21	+	helper.Helper()
	22	+	}
	23	+
	24	+	mock := &mocktargetSource{}
	25	+	mock.Mock.Test(t)
	26	+
	27	+	t.Cleanup(func() {
	28	+	if helper, ok := t.(interface{ Helper() }); ok {
	29	+	helper.Helper()
	30	+	}
	31	+	mock.AssertExpectations(t)
	32	+	})
	33	+
	34	+	return mock
	35	+	}
	36	+
	37	+	// mocktargetSource is an autogenerated mock type for the targetSource type
	38	+	type mocktargetSource struct {
	39	+	mock.Mock
	40	+	}
	41	+
	42	+	type mocktargetSource_Expecter struct {
	43	+	mock *mock.Mock
	44	+	}
	45	+
	46	+	func (_m mocktargetSource) EXPECT() mocktargetSource_Expecter {
	47	+	return &mocktargetSource_Expecter{mock: &_m.Mock}
	48	+	}
	49	+
	50	+	// Close provides a mock function for the type mocktargetSource
	51	+	func (_mock *mocktargetSource) Close() error {
	52	+	ret := _mock.Called()
	53	+
	54	+	if len(ret) == 0 {
	55	+	panic("no return value specified for Close")
	56	+	}
	57	+
	58	+	var r0 error
	59	+	if returnFunc, ok := ret.Get(0).(func() error); ok {
	60	+	r0 = returnFunc()
	61	+	} else {
	62	+	r0 = ret.Error(0)
	63	+	}
	64	+	return r0
	65	+	}
	66	+
	67	+	// mocktargetSource_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
	68	+	type mocktargetSource_Close_Call struct {
	69	+	*mock.Call
	70	+	}
	71	+
	72	+	// Close is a helper method to define mock.On call
	73	+	func (_e mocktargetSource_Expecter) Close() mocktargetSource_Close_Call {
	74	+	return &mocktargetSource_Close_Call{Call: _e.mock.On("Close")}
	75	+	}
	76	+
	77	+	func (_c mocktargetSource_Close_Call) Run(run func()) mocktargetSource_Close_Call {
	78	+	_c.Call.Run(func(args mock.Arguments) {
	79	+	run()
	80	+	})
	81	+	return _c
	82	+	}
	83	+
	84	+	func (_c mocktargetSource_Close_Call) Return(err error) mocktargetSource_Close_Call {
	85	+	_c.Call.Return(err)
	86	+	return _c
	87	+	}
	88	+
	89	+	func (_c mocktargetSource_Close_Call) RunAndReturn(run func() error) mocktargetSource_Close_Call {
	90	+	_c.Call.Return(run)
	91	+	return _c
	92	+	}
	93	+
	94	+	// Targets provides a mock function for the type mocktargetSource
	95	+	func (_mock *mocktargetSource) Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error) {
	96	+	ret := _mock.Called(ctx, windows, linksOnly)
	97	+
	98	+	if len(ret) == 0 {
	99	+	panic("no return value specified for Targets")
	100	+	}
	101	+
	102	+	var r0 []ports.HintTarget
	103	+	var r1 map[ports.WindowID]bool
	104	+	var r2 error
	105	+	if returnFunc, ok := ret.Get(0).(func(context.Context, []ports.HintWindow, bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)); ok {
	106	+	return returnFunc(ctx, windows, linksOnly)
	107	+	}
	108	+	if returnFunc, ok := ret.Get(0).(func(context.Context, []ports.HintWindow, bool) []ports.HintTarget); ok {
	109	+	r0 = returnFunc(ctx, windows, linksOnly)
	110	+	} else {
	111	+	if ret.Get(0) != nil {
	112	+	r0 = ret.Get(0).([]ports.HintTarget)
	113	+	}
	114	+	}
	115	+	if returnFunc, ok := ret.Get(1).(func(context.Context, []ports.HintWindow, bool) map[ports.WindowID]bool); ok {
	116	+	r1 = returnFunc(ctx, windows, linksOnly)
	117	+	} else {
	118	+	if ret.Get(1) != nil {
	119	+	r1 = ret.Get(1).(map[ports.WindowID]bool)
	120	+	}
	121	+	}
	122	+	if returnFunc, ok := ret.Get(2).(func(context.Context, []ports.HintWindow, bool) error); ok {
	123	+	r2 = returnFunc(ctx, windows, linksOnly)
	124	+	} else {
	125	+	r2 = ret.Error(2)
	126	+	}
	127	+	return r0, r1, r2
	128	+	}
	129	+
	130	+	// mocktargetSource_Targets_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Targets'
	131	+	type mocktargetSource_Targets_Call struct {
	132	+	*mock.Call
	133	+	}
	134	+
	135	+	// Targets is a helper method to define mock.On call
	136	+	// - ctx context.Context
	137	+	// - windows []ports.HintWindow
	138	+	// - linksOnly bool
	139	+	func (_e mocktargetSource_Expecter) Targets(ctx any, windows any, linksOnly any) mocktargetSource_Targets_Call {
	140	+	return &mocktargetSource_Targets_Call{Call: _e.mock.On("Targets", ctx, windows, linksOnly)}
	141	+	}
	142	+
	143	+	func (_c mocktargetSource_Targets_Call) Run(run func(ctx context.Context, windows []ports.HintWindow, linksOnly bool)) mocktargetSource_Targets_Call {
	144	+	_c.Call.Run(func(args mock.Arguments) {
	145	+	var arg0 context.Context
	146	+	if args[0] != nil {
	147	+	arg0 = args[0].(context.Context)
	148	+	}
	149	+	var arg1 []ports.HintWindow
	150	+	if args[1] != nil {
	151	+	arg1 = args[1].([]ports.HintWindow)
	152	+	}
	153	+	var arg2 bool
	154	+	if args[2] != nil {
	155	+	arg2 = args[2].(bool)
	156	+	}
	157	+	run(
	158	+	arg0,
	159	+	arg1,
	160	+	arg2,
	161	+	)
	162	+	})
	163	+	return _c
	164	+	}
	165	+
	166	+	func (_c mocktargetSource_Targets_Call) Return(hintTargets []ports.HintTarget, windowIDToBool map[ports.WindowID]bool, err error) mocktargetSource_Targets_Call {
	167	+	_c.Call.Return(hintTargets, windowIDToBool, err)
	168	+	return _c
	169	+	}
	170	+
	171	+	func (_c mocktargetSource_Targets_Call) RunAndReturn(run func(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)) mocktargetSource_Targets_Call {
	172	+	_c.Call.Return(run)
	173	+	return _c
	174	+	}

A internal/adapters/hints/testdata/browser.png +0 -0 binary file: no text diff

binary file: no text diff

A internal/adapters/hints/testdata/terminal.png +0 -0 binary file: no text diff

binary file: no text diff

A internal/adapters/hints/testdata/underline.png +0 -0 binary file: no text diff

binary file: no text diff

A internal/adapters/hints/underline.go +246 -0

@@ -0,0 +1,246 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"image"
	5	+	"math"
	6	+	"sort"
	7	+
	8	+	"github.com/bnema/neferwl/internal/ports"
	9	+	)
	10	+
	11	+	// underlineCandidate is a word with a thin line under it.
	12	+	type underlineCandidate struct {
	13	+	word int
	14	+	row int // the line's row
	15	+	from, to int // x extent of the continuous line through the word
	16	+	}
	17	+
	18	+	// markUnderlined sets Underlined, UnderL and UnderR on the words of a band
	19	+	// that have a thin line under them. words are in the coordinates of g.
	20	+	// Lines that run on far beyond the word (rules, table borders, input boxes)
	21	+	// do not count, unless the extension is made of underlined words too.
	22	+	func markUnderlined(src *image.Gray, words []OCRWord) {
	23	+	// Words are relative to the band's own origin, whatever the image is a
	24	+	// view of.
	25	+	b := src.Bounds()
	26	+	g := &image.Gray{Pix: src.Pix[src.PixOffset(b.Min.X, b.Min.Y):], Stride: src.Stride, Rect: image.Rect(0, 0, b.Dx(), b.Dy())}
	27	+	var cands []underlineCandidate
	28	+	for i, w := range words {
	29	+	// A single glyph, or a box narrower than tall, has a flat bottom
	30	+	// stroke (8, L, 2) that is not an underline.
	31	+	if len([]rune(w.Text)) < 2 \|\| w.Rect.Dx() < w.Rect.Dy() {
	32	+	continue
	33	+	}
	34	+	if c, ok := underlineOf(g, w.Rect); ok {
	35	+	c.word = i
	36	+	cands = append(cands, c)
	37	+	}
	38	+	}
	39	+	for _, c := range cands {
	40	+	w := words[c.word]
	41	+	h := w.Rect.Dy()
	42	+	if (w.Rect.Min.X-c.from > h && !covered(words, cands, c, -1)) \|\|
	43	+	(c.to-w.Rect.Max.X > h && !covered(words, cands, c, 1)) {
	44	+	continue
	45	+	}
	46	+	words[c.word].Underlined, words[c.word].UnderL, words[c.word].UnderR = true, c.from, c.to
	47	+	}
	48	+	}
	49	+
	50	+	// covered reports whether the line of c, on the given side of its word, is
	51	+	// explained by underlined words on the same line up to within a glyph of its
	52	+	// end.
	53	+	func covered(words []OCRWord, cands []underlineCandidate, c underlineCandidate, dir int) bool {
	54	+	w := words[c.word].Rect
	55	+	h := w.Dy()
	56	+	cur := w.Min.X
	57	+	if dir > 0 {
	58	+	cur = w.Max.X
	59	+	}
	60	+	for {
	61	+	if (dir < 0 && cur-c.from <= h) \|\| (dir > 0 && c.to-cur <= h) {
	62	+	return true
	63	+	}
	64	+	next := -1
	65	+	for _, o := range cands {
	66	+	r := words[o.word].Rect
	67	+	if abs(r.Max.Y-w.Max.Y) > max(3, h/3) {
	68	+	continue
	69	+	}
	70	+	var gap, reach int
	71	+	if dir < 0 {
	72	+	gap, reach = cur-r.Max.X, r.Min.X
	73	+	} else {
	74	+	gap, reach = r.Min.X-cur, r.Max.X
	75	+	}
	76	+	if gap < -2 \|\| gap > h {
	77	+	continue
	78	+	}
	79	+	if next < 0 \|\| (dir < 0 && reach < next) \|\| (dir > 0 && reach > next) {
	80	+	next = reach
	81	+	}
	82	+	}
	83	+	if next < 0 \|\| (dir < 0 && next >= cur) \|\| (dir > 0 && next <= cur) {
	84	+	return false
	85	+	}
	86	+	cur = next
	87	+	}
	88	+	}
	89	+
	90	+	// underlineOf looks for an underline under the word box r in g.
	91	+	func underlineOf(g *image.Gray, r image.Rectangle) (underlineCandidate, bool) {
	92	+	b := g.Bounds()
	93	+	box := r.Intersect(b)
	94	+	h := r.Dy()
	95	+	if box.Dx() < 4 \|\| h < 4 {
	96	+	return underlineCandidate{}, false
	97	+	}
	98	+	bg, ink := contrastOf(g, box)
	99	+	thr := min(40, max(20, ink/2))
	100	+	if ink < thr+10 {
	101	+	return underlineCandidate{}, false
	102	+	}
	103	+	top := max(b.Min.Y, r.Max.Y-(h*15+99)/100)
	104	+	bottom := min(b.Max.Y, r.Max.Y+(h*35+99)/100+1)
	105	+	maxThick := max(2, h/6)
	106	+	rowOK := func(y int) bool {
	107	+	row := g.Pix[(y-g.Rect.Min.Y)*g.Stride:]
	108	+	n := 0
	109	+	for x := box.Min.X; x < box.Max.X; x++ {
	110	+	if abs(int(row[x-g.Rect.Min.X])-bg) >= thr {
	111	+	n++
	112	+	}
	113	+	}
	114	+	return n100 >= box.Dx()85
	115	+	}
	116	+	thick, first := 0, 0
	117	+	for y := top; y <= bottom; y++ {
	118	+	if y < bottom && rowOK(y) {
	119	+	if thick == 0 {
	120	+	first = y
	121	+	}
	122	+	thick++
	123	+	continue
	124	+	}
	125	+	if thick > 0 {
	126	+	if thick > maxThick {
	127	+	return underlineCandidate{}, false // a block, not a line
	128	+	}
	129	+	row := first + thick/2
	130	+	from, to := box.Min.X, box.Max.X
	131	+	line := g.Pix[(row-g.Rect.Min.Y)*g.Stride:]
	132	+	on := func(x int) bool { return abs(int(line[x-g.Rect.Min.X])-bg) >= thr }
	133	+	for from > b.Min.X && on(from-1) {
	134	+	from--
	135	+	}
	136	+	for to < b.Max.X && on(to) {
	137	+	to++
	138	+	}
	139	+	return underlineCandidate{row: row, from: from, to: to}, true
	140	+	}
	141	+	}
	142	+	return underlineCandidate{}, false
	143	+	}
	144	+
	145	+	// contrastOf returns the background level of box (the most common level,
	146	+	// found in bins of 16 so antialiasing does not split it, then refined to the
	147	+	// mean of that bin) and the contrast of its ink (the deviation reached by
	148	+	// the strongest 3% of pixels).
	149	+	func contrastOf(g *image.Gray, box image.Rectangle) (bg, ink int) {
	150	+	var hist [256]int
	151	+	for y := box.Min.Y; y < box.Max.Y; y++ {
	152	+	for _, v := range g.Pix[(y-g.Rect.Min.Y)*g.Stride+box.Min.X-g.Rect.Min.X:][:box.Dx()] {
	153	+	hist[v]++
	154	+	}
	155	+	}
	156	+	total := box.Dx() * box.Dy()
	157	+	var bins [16]int
	158	+	for v, c := range hist {
	159	+	bins[v>>4] += c
	160	+	}
	161	+	best := 0
	162	+	for i, c := range bins {
	163	+	if c > bins[best] {
	164	+	best = i
	165	+	}
	166	+	}
	167	+	sum, n := 0, 0
	168	+	for v := best << 4; v < (best+1)<<4; v++ {
	169	+	sum += v * hist[v]
	170	+	n += hist[v]
	171	+	}
	172	+	bg = sum / max(n, 1)
	173	+	var dev [256]int
	174	+	for v, c := range hist {
	175	+	dev[abs(v-bg)] += c
	176	+	}
	177	+	n = 0
	178	+	for d := 255; d >= 0; d-- {
	179	+	n += dev[d]
	180	+	if n100 >= total3 {
	181	+	return bg, d
	182	+	}
	183	+	}
	184	+	return bg, 0
	185	+	}
	186	+
	187	+	// underlinedLinks returns the targets for underlined words, joining the
	188	+	// words of one continuous underline. A group that overlaps a URL target (in
	189	+	// pixels) is left to it. Coordinates are scaled to logical.
	190	+	func underlinedLinks(words []OCRWord, urls []image.Rectangle, scale float64) []ports.HintTarget {
	191	+	var us []OCRWord
	192	+	for _, w := range words {
	193	+	if w.Underlined {
	194	+	us = append(us, w)
	195	+	}
	196	+	}
	197	+	sort.Slice(us, func(i, j int) bool { return us[i].Rect.Min.X < us[j].Rect.Min.X })
	198	+	used := make([]bool, len(us))
	199	+	var out []ports.HintTarget
	200	+	for i := range us {
	201	+	if used[i] {
	202	+	continue
	203	+	}
	204	+	used[i] = true
	205	+	rect, last := us[i].Rect, us[i]
	206	+	chars, width := len(last.Text), last.Rect.Dx()
	207	+	for {
	208	+	next := -1
	209	+	for j := range us {
	210	+	c := us[j]
	211	+	if used[j] \|\| c.Rect.Min.X < rect.Max.X-2 \|\| abs(c.Rect.Max.Y-last.Rect.Max.Y) > max(3, rect.Dy()/3) {
	212	+	continue
	213	+	}
	214	+	avg := float64(width+c.Rect.Dx()) / float64(max(1, chars+len(c.Text)))
	215	+	gap := c.Rect.Min.X - rect.Max.X
	216	+	if float64(gap) > 1.2*avg \|\| c.Rect.Min.X > last.UnderR+2 {
	217	+	continue
	218	+	}
	219	+	if next < 0 \|\| c.Rect.Min.X < us[next].Rect.Min.X {
	220	+	next = j
	221	+	}
	222	+	}
	223	+	if next < 0 {
	224	+	break
	225	+	}
	226	+	used[next] = true
	227	+	last = us[next]
	228	+	rect = rect.Union(last.Rect)
	229	+	chars += len(last.Text)
	230	+	width += last.Rect.Dx()
	231	+	}
	232	+	dup := false
	233	+	for _, u := range urls {
	234	+	if overlap(rect, u) > .5 {
	235	+	dup = true
	236	+	}
	237	+	}
	238	+	if dup {
	239	+	continue
	240	+	}
	241	+	x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
	242	+	right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
	243	+	out = append(out, ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
	244	+	}
	245	+	return out
	246	+	}

A internal/adapters/hints/underline_test.go +252 -0

@@ -0,0 +1,252 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"image"
	5	+	"image/color"
	6	+	"os"
	7	+	"os/exec"
	8	+	"testing"
	9	+
	10	+	"github.com/stretchr/testify/require"
	11	+
	12	+	"github.com/bnema/neferwl/internal/ports"
	13	+	)
	14	+
	15	+	// canvas is a synthetic grayscale band: bg everywhere, ink for text.
	16	+	type canvas struct {
	17	+	g *image.Gray
	18	+	bg, ink uint8
	19	+	s int // scale
	20	+	}
	21	+
	22	+	func newCanvas(w, h int, bg, ink uint8, s int) *canvas {
	23	+	g := image.NewGray(image.Rect(0, 0, ws, hs))
	24	+	for i := range g.Pix {
	25	+	g.Pix[i] = bg
	26	+	}
	27	+	return &canvas{g, bg, ink, s}
	28	+	}
	29	+
	30	+	func (c *canvas) fill(x0, y0, x1, y1 int, v uint8) {
	31	+	for y := y0 * c.s; y < y1*c.s; y++ {
	32	+	for x := x0 * c.s; x < x1*c.s; x++ {
	33	+	c.g.SetGray(x, y, color.Gray{v})
	34	+	}
	35	+	}
	36	+	}
	37	+
	38	+	// word draws a 10 px high word of x-height body at (x,y) with the given
	39	+	// width: text-like noisy ink (a stem every 3 px) and optional descenders.
	40	+	// It returns the OCR box (descenders excluded, as a box around the body).
	41	+	func (c *canvas) word(x, y, w int, descender bool) OCRWord {
	42	+	for i := 0; i < w; i += 3 {
	43	+	c.fill(x+i, y, x+i+1, y+10, c.ink)
	44	+	}
	45	+	box := image.Rect(xc.s, yc.s, (x+w)c.s, (y+10)c.s)
	46	+	if descender {
	47	+	for i := 0; i < w; i += 6 {
	48	+	c.fill(x+i, y+10, x+i+1, y+13, c.ink)
	49	+	}
	50	+	box.Max.Y = (y + 13) * c.s
	51	+	}
	52	+	return OCRWord{Text: "word", Rect: box}
	53	+	}
	54	+
	55	+	func (c *canvas) underline(x0, x1, y int) { c.fill(x0, y, x1, y+1, c.ink) }
	56	+
	57	+	func scanUnder(c *canvas, words ...OCRWord) []ports.HintTarget {
	58	+	markUnderlined(c.g, words)
	59	+	return textLinks(words, float64(c.s))
	60	+	}
	61	+
	62	+	func TestUnderline(t *testing.T) {
	63	+	for _, tc := range []struct {
	64	+	name string
	65	+	bg, ink uint8
	66	+	}{{"dark-bg", 10, 204}, {"light-bg", 245, 30}} {
	67	+	for _, s := range []int{1, 2} {
	68	+	t.Run(tc.name, func(t *testing.T) {
	69	+	mk := func() *canvas { return newCanvas(200, 60, tc.bg, tc.ink, s) }
	70	+
	71	+	c := mk()
	72	+	w := c.word(10, 20, 40, false)
	73	+	c.underline(10, 50, 32)
	74	+	got := scanUnder(c, w)
	75	+	require.Len(t, got, 1)
	76	+	require.Equal(t, ports.HintLink, got[0].Kind)
	77	+	require.False(t, got[0].Text)
	78	+	require.Equal(t, ports.WindowID(0), got[0].Window)
	79	+	require.Equal(t, 10, got[0].Rect.X)
	80	+
	81	+	c = mk()
	82	+	require.Empty(t, scanUnder(c, c.word(10, 20, 40, false)), "plain word")
	83	+
	84	+	c = mk()
	85	+	require.Empty(t, scanUnder(c, c.word(10, 20, 40, true)), "descenders")
	86	+
	87	+	c = mk()
	88	+	w = c.word(10, 20, 40, false)
	89	+	c.underline(0, 200, 32)
	90	+	require.Empty(t, scanUnder(c, w), "rule across")
	91	+
	92	+	c = mk()
	93	+	w = c.word(10, 20, 40, false)
	94	+	c.fill(10, 24, 50, 25, c.ink)
	95	+	require.Empty(t, scanUnder(c, w), "strikethrough")
	96	+
	97	+	// Two words with a continuous underline: one target.
	98	+	c = mk()
	99	+	a, b := c.word(10, 20, 40, false), c.word(56, 20, 40, false)
	100	+	c.underline(10, 96, 32)
	101	+	got = scanUnder(c, a, b)
	102	+	require.Len(t, got, 1)
	103	+	require.Equal(t, 10, got[0].Rect.X)
	104	+	require.Equal(t, 86, got[0].Rect.W)
	105	+
	106	+	// A gap in the underline: two targets.
	107	+	c = mk()
	108	+	a, b = c.word(10, 20, 40, false), c.word(56, 20, 40, false)
	109	+	c.underline(10, 50, 32)
	110	+	c.underline(56, 96, 32)
	111	+	require.Len(t, scanUnder(c, a, b), 2)
	112	+	})
	113	+	}
	114	+	}
	115	+	}
	116	+
	117	+	// A box border under a word, with the word a part of a table row, is not a
	118	+	// link; neither is an underline that continues beyond into unrelated text.
	119	+	func TestUnderlineRejectsBox(t *testing.T) {
	120	+	c := newCanvas(300, 60, 10, 204, 1)
	121	+	w := c.word(100, 20, 40, false)
	122	+	c.underline(20, 280, 32)
	123	+	c.fill(20, 10, 21, 33, 204)
	124	+	c.fill(279, 10, 280, 33, 204)
	125	+	require.Empty(t, scanUnder(c, w))
	126	+	}
	127	+
	128	+	// The line must be thin.
	129	+	func TestUnderlineRejectsBlock(t *testing.T) {
	130	+	c := newCanvas(100, 60, 10, 204, 1)
	131	+	w := c.word(10, 20, 40, false)
	132	+	c.fill(10, 31, 50, 36, 204)
	133	+	require.Empty(t, scanUnder(c, w))
	134	+	}
	135	+
	136	+	func TestUnderlinedURLIsOneTextTarget(t *testing.T) {
	137	+	c := newCanvas(300, 60, 10, 204, 1)
	138	+	w := c.word(10, 20, 100, false)
	139	+	w.Text = "example.org"
	140	+	c.underline(10, 110, 32)
	141	+	got := scanUnder(c, w)
	142	+	require.Len(t, got, 1)
	143	+	require.True(t, got[0].Text)
	144	+	}
	145	+
	146	+	// ocrTestdata reads the words of a testdata image with the real tesseract.
	147	+	func ocrTestdata(t testing.T, name string) (image.Gray, []OCRWord) {
	148	+	t.Helper()
	149	+	if _, err := exec.LookPath("tesseract"); err != nil {
	150	+	t.Skip("tesseract not on PATH")
	151	+	}
	152	+	f, err := os.Open("testdata/" + name)
	153	+	require.NoError(t, err)
	154	+	defer f.Close()
	155	+	img, _, err := image.Decode(f)
	156	+	require.NoError(t, err)
	157	+	g := image.NewGray(img.Bounds())
	158	+	for y := 0; y < g.Rect.Dy(); y++ {
	159	+	for x := 0; x < g.Rect.Dx(); x++ {
	160	+	g.Set(x, y, img.At(x, y))
	161	+	}
	162	+	}
	163	+	words, _, err := readWords(t.Context(), g)
	164	+	require.NoError(t, err)
	165	+	return g, words
	166	+	}
	167	+
	168	+	// headingRule finds the thin horizontal rule under the heading of
	169	+	// browser.png: the first row, from y=150, with a 900 px run of non-white
	170	+	// pixels. It returns the rule as a rect.
	171	+	func headingRule(t testing.T, g image.Gray) image.Rectangle {
	172	+	t.Helper()
	173	+	for y := 150; y < 220; y++ {
	174	+	run, start := 0, 0
	175	+	for x := 0; x < g.Rect.Dx(); x++ {
	176	+	if g.GrayAt(x, y).Y < 240 {
	177	+	if run == 0 {
	178	+	start = x
	179	+	}
	180	+	run++
	181	+	if run >= 900 {
	182	+	return image.Rect(start, y, start+run, y+1)
	183	+	}
	184	+	} else {
	185	+	run = 0
	186	+	}
	187	+	}
	188	+	}
	189	+	t.Fatal("heading rule not found")
	190	+	return image.Rectangle{}
	191	+	}
	192	+
	193	+	// browser.png has blue links and no underlined link: the only underline-only
	194	+	// target is a known false positive, and the heading rule is not taken for an
	195	+	// underline.
	196	+	func TestUnderlineOnBrowserTestdata(t *testing.T) {
	197	+	g, words := ocrTestdata(t, "browser.png")
	198	+	rule := headingRule(t, g)
	199	+	// Known false positive: the word "as" (flat glyph bottoms, ~(878,353)) is
	200	+	// flagged underlined by the detector. Anything else is a regression.
	201	+	var under []ports.HintTarget
	202	+	for _, tg := range textLinks(words, 1) {
	203	+	if !tg.Text {
	204	+	under = append(under, tg)
	205	+	}
	206	+	}
	207	+	require.Len(t, under, 1, "underline-only targets: %v", under)
	208	+	require.True(t, under[0].Rect.X > 860 && under[0].Rect.X < 900 && under[0].Rect.Y > 340 && under[0].Rect.Y < 370, "unexpected target %v", under[0].Rect)
	209	+	for _, tg := range textLinks(words, 1) {
	210	+	r := image.Rect(tg.Rect.X, tg.Rect.Y, tg.Rect.X+tg.Rect.W, tg.Rect.Y+tg.Rect.H)
	211	+	require.False(t, r.Overlaps(rule), "target %v overlaps the heading rule %v", tg.Rect, rule)
	212	+	}
	213	+	}
	214	+
	215	+	// underline.png is a real render (Chromium) of a generic page: underlined
	216	+	// links `demo`, `source` and `ermite-runner hosts`, a table with row rules
	217	+	// and an input box, which must yield nothing.
	218	+	func TestUnderlineOnRealRender(t *testing.T) {
	219	+	_, words := ocrTestdata(t, "underline.png")
	220	+	near := func(r ports.Rect, x0, y0, x1, y1 int) bool {
	221	+	const tol = 8
	222	+	abs := func(v int) int { return max(v, -v) }
	223	+	return abs(r.X-x0) <= tol && abs(r.Y-y0) <= tol && abs(r.X+r.W-x1) <= tol && abs(r.Y+r.H-y1) <= tol
	224	+	}
	225	+	var got []ports.HintTarget
	226	+	for _, tg := range textLinks(words, 1) {
	227	+	if !tg.Text {
	228	+	got = append(got, tg)
	229	+	}
	230	+	}
	231	+	require.Len(t, got, 3, "targets: %v", got)
	232	+	for _, want := range [][4]int{{148, 122, 196, 137}, {306, 126, 364, 137}, {122, 165, 294, 180}} {
	233	+	found := false
	234	+	for _, tg := range got {
	235	+	found = found \|\| near(tg.Rect, want[0], want[1], want[2], want[3])
	236	+	}
	237	+	require.True(t, found, "no target near %v in %v", want, got)
	238	+	}
	239	+	// The table (y 215..335) and the input box (y 360..410) give nothing.
	240	+	for _, tg := range textLinks(words, 1) {
	241	+	require.False(t, tg.Rect.Y > 200 && tg.Rect.Y < 420, "unexpected target %v", tg.Rect)
	242	+	}
	243	+	}
	244	+
	245	+	// A single glyph with a flat bottom stroke (8, L) is not underlined text.
	246	+	func TestUnderlineIgnoresSingleGlyph(t *testing.T) {
	247	+	c := newCanvas(100, 60, 10, 204, 1)
	248	+	w := c.word(10, 20, 10, false)
	249	+	w.Text = "8"
	250	+	c.underline(10, 20, 32)
	251	+	require.Empty(t, scanUnder(c, w))
	252	+	}

M internal/adapters/outputkit/capture/excluded.go +5 -5


@@ -10,7 +10,7 @@ import (
10			10
11		// A request with Exclude leaves out the exclusion's HUD layers and popups;	11		// A request with Exclude leaves out the exclusion's HUD layers and popups;
12		// every other capture shows them. No capture shows the capture indicator	12		// every other capture shows them. No capture shows the capture indicator
13	-	// (Scene.CaptureIndicators): a scene that carries it is never the source of a	13	+	// (Scene.CaptureIndicators): a scene that carries it, or hint labels, is never the source of a
14		// capture image. Requests of the displayed frame are then served from the	14		// capture image. Requests of the displayed frame are then served from the
15		// frame rendered without it first (SubmitPlain, ExcludedScene).	15		// frame rendered without it first (SubmitPlain, ExcludedScene).
16			16

@@ -64,10 +64,10 @@ func excludedShown(s ports.Scene) bool {
64		return false	64		return false
65		}	65		}
66			66
67	-	// plain returns s without its capture state and indicators, Seq 0 so the	67	+	// plain returns s without its capture state, indicators and hint labels, Seq 0 so the
68		// renderer redraws the whole target instead of reusing a displayed frame.	68		// renderer redraws the whole target instead of reusing a displayed frame.
69		func plain(s ports.Scene) ports.Scene {	69		func plain(s ports.Scene) ports.Scene {
70	-	s.Capture, s.CaptureIndicators, s.Seq = nil, nil, 0	70	+	s.Capture, s.CaptureIndicators, s.HintLabels, s.Seq = nil, nil, nil, 0
71		return s	71		return s
72		}	72		}
73			73

@@ -85,7 +85,7 @@ func plain(s ports.Scene) ports.Scene {
85		// the plain render reads client buffers even when the requests are rejected,	85		// the plain render reads client buffers even when the requests are rejected,
86		// so the caller owns it (hold or wait it) whatever happened, error included.	86		// so the caller owns it (hold or wait it) whatever happened, error included.
87		func (p Pipeline) SubmitPlain(r ports.Renderer, s ports.Scene, surfaces map[ports.WindowID]ports.SurfaceContent, normal []ports.CaptureRequest) (took bool, done os.File, err error) {	87		func (p Pipeline) SubmitPlain(r ports.Renderer, s ports.Scene, surfaces map[ports.WindowID]ports.SurfaceContent, normal []ports.CaptureRequest) (took bool, done os.File, err error) {
88	-	if len(normal) == 0 \|\| len(s.CaptureIndicators) == 0 {	88	+	if len(normal) == 0 \|\| len(s.CaptureIndicators) == 0 && len(s.HintLabels) == 0 {
89		return false, nil, nil	89		return false, nil, nil
90		}	90		}
91		done, err = r.Render(plain(s), surfaces)	91		done, err = r.Render(plain(s), surfaces)

@@ -164,7 +164,7 @@ func (p *Pipeline) Split(s ports.Scene, reqs []ports.CaptureRequest) (normal, ex
164		}	164		}
165		clear(reqs[n:])	165		clear(reqs[n:])
166		reqs = reqs[:n]	166		reqs = reqs[:n]
167	-	excl := excludedShown(s) \|\| len(s.CaptureIndicators) > 0	167	+	excl := excludedShown(s) \|\| len(s.CaptureIndicators) > 0 \|\| len(s.HintLabels) > 0
168		group := func(q ports.CaptureRequest) int {	168		group := func(q ports.CaptureRequest) int {
169		switch {	169		switch {
170		case q.OffScreen:	170		case q.OffScreen:

M internal/adapters/outputkit/capture/indicator_test.go +40 -0


@@ -112,3 +112,43 @@ func TestSubmitPlainRejectsProtectedEpoch(t *testing.T) {
112	_, statErr := file.Stat()	112		_, statErr := file.Stat()
113	require.ErrorIs(t, statErr, os.ErrClosed)	113		require.ErrorIs(t, statErr, os.ErrClosed)
114	}	114		}
		115	+
		116	+	// Hint labels are never captured either: a scene with only labels is served
		117	+	// from the plain frame, and an Exclude request is ordered to the excluded scene.
		118	+	func TestSubmitPlainServesHintLabelScenesWithoutLabels(t *testing.T) {
		119	+	replies := make(chan ports.CaptureDone, 1)
		120	+	p := NewPipeline(context.Background(), replies)
		121	+	r := portsmocks.NewMockRenderer(t)
		122	+	frame := portsmocks.NewMockCaptureFrame(t)
		123	+	frame.EXPECT().Done().Return((*os.File)(nil))
		124	+	frame.EXPECT().Read(image.Rect(0, 0, 2, 2), mock.Anything, 8).Return(nil).Once()
		125	+	var rendered []ports.Scene
		126	+	r.EXPECT().Render(mock.Anything, mock.Anything).RunAndReturn(func(s ports.Scene, _ map[ports.WindowID]ports.SurfaceContent) (*os.File, error) {
		127	+	rendered = append(rendered, s)
		128	+	return nil, nil
		129	+	}).Once()
		130	+	r.EXPECT().BeginCapture().Return(frame, nil).Once()
		131	+	r.EXPECT().EndCapture(frame).Return().Once()
		132	+	s := excludedTestScene()
		133	+	s.Capture = nil
		134	+	s.HintLabels = []ports.HintLabel{{Text: "f"}}
		135	+	reqs := []ports.CaptureRequest{pipelineRequest(t, 1)}
		136	+	require.False(t, reqs[0].Indicate)
		137	+	took, done, err := p.SubmitPlain(r, s, nil, reqs)
		138	+	require.NoError(t, err)
		139	+	require.Nil(t, done)
		140	+	require.True(t, took)
		141	+	require.Len(t, rendered, 1)
		142	+	require.Nil(t, rendered[0].HintLabels)
		143	+	require.Zero(t, rendered[0].Seq)
		144	+	require.Equal(t, uint64(1), awaitCapture(t, replies).ID)
		145	+	p.Close(r)
		146	+	frame.Calls = nil
		147	+
		148	+	withSession := s
		149	+	withSession.Capture = &ports.SceneCapture{Session: 7}
		150	+	normal, excluded, _ := p.Split(withSession, []ports.CaptureRequest{{ID: 2, Exclude: true, Session: 7}})
		151	+	require.Empty(t, normal)
		152	+	require.Equal(t, []uint64{2}, ids(excluded), "labels force the Exclude request to the excluded scene")
		153	+	require.Nil(t, p.ExcludedScene(s).HintLabels)
		154	+	}

M internal/adapters/vulkan/damage.go +4 -2

 	tg.heldLayers = append(tg.heldLayers[:0], s.Layers...)
 	tg.heldHints = append(tg.heldHints[:0], s.DropHints...)
 	tg.heldIndicators = append(tg.heldIndicators[:0], s.CaptureIndicators...)
+	tg.heldLabels = append(tg.heldLabels[:0], s.HintLabels...)
 	tg.valid, tg.sceneSeq, tg.scene, tg.windows = true, s.Seq, s, d.drawn
 	tg.scene.Windows, tg.scene.Separators, tg.scene.Layers = tg.heldWindows, tg.heldSeparators, tg.heldLayers
 	tg.scene.DropHints, tg.scene.CaptureIndicators = tg.heldHints, tg.heldIndicators
+	tg.scene.HintLabels = tg.heldLabels
 	// Only whether they are set matters (any capture redraws everything):
 	// sentinels, so no pointer into the caller's capture state is kept.
 	tg.scene.Capture, tg.scene.CaptureScene = nil, nil
 	if old.Security != cur.Security || old.Output != cur.Output ||
 		old.OutputWidth != cur.OutputWidth || old.OutputHeight != cur.OutputHeight ||
 		old.Scale != cur.Scale || old.Transform != cur.Transform || old.Off != cur.Off ||
-		old.Background != cur.Background || old.Border != cur.Border ||
+		old.HintStyle != cur.HintStyle || old.Background != cur.Background || old.Border != cur.Border ||
 		old.WorkspaceClip != cur.WorkspaceClip || old.TileInset != cur.TileInset || old.Dim != cur.Dim || old.DimBehind != cur.DimBehind ||
 		old.Capture != nil || cur.Capture != nil || old.CaptureScene != nil || cur.CaptureScene != nil ||
 		!slices.Equal(old.Layers, cur.Layers) || !slices.Equal(old.DropHints, cur.DropHints) ||
-		!slices.Equal(old.CaptureIndicators, cur.CaptureIndicators) ||
+		!slices.Equal(old.CaptureIndicators, cur.CaptureIndicators) || !slices.Equal(old.HintLabels, cur.HintLabels) ||
 		len(old.Windows) != len(cur.Windows) {
 		return image.Rectangle{}, false
 	}

M internal/adapters/vulkan/damage_delta_test.go +2 -2


@@ -17,9 +17,9 @@ import (
17		// TestSceneFieldCount fails when Scene gains or loses a field: sceneDelta	17		// TestSceneFieldCount fails when Scene gains or loses a field: sceneDelta
18		// lists the fields that force a full redraw.	18		// lists the fields that force a full redraw.
19		func TestSceneFieldCount(t *testing.T) {	19		func TestSceneFieldCount(t *testing.T) {
20	-	// 23 with Animating, which sceneDelta intentionally ignores: it only	20	+	// 25 with Animating, which sceneDelta intentionally ignores: it only
21		// schedules the next frame, it draws nothing.	21		// schedules the next frame, it draws nothing.
22	-	const fields = 23	22	+	const fields = 25
23		assert.Equal(t, fields, len(reflect.VisibleFields(reflect.TypeOf(ports.Scene{}))),	23		assert.Equal(t, fields, len(reflect.VisibleFields(reflect.TypeOf(ports.Scene{}))),
24		"Scene fields changed: check sceneDelta (a field it does not compare must force a full redraw or be covered), then this count")	24		"Scene fields changed: check sceneDelta (a field it does not compare must force a full redraw or be covered), then this count")
25		}	25		}

A internal/adapters/vulkan/hint_labels.go +52 -0

@@ -0,0 +1,52 @@
	1	+	package vulkan
	2	+
	3	+	import (
	4	+	"image"
	5	+	"math"
	6	+
	7	+	"github.com/bnema/neferwl/internal/adapters/hintlabel"
	8	+	)
	9	+
	10	+	// hintLabels draws the keyboard hint labels (ports.Scene.HintLabels) over
	11	+	// everything, after the capture indicators. Each label is rasterized on the CPU
	12	+	// (hintlabel) at the scene's scale and drawn from the SHM copy path: the GPU
	13	+	// buffer is cached under a key naming the label, so a label that stays the
	14	+	// same uploads once. Captures are rendered from scenes without labels.
	15	+	func (w *sceneWalk) hintLabels() {
	16	+	if len(w.s.HintLabels) == 0 {
	17	+	return
	18	+	}
	19	+	r := w.r
	20	+	hs := w.s.HintStyle
	21	+	bg, fg := hintlabel.Colors(w.s.Border.Active, hs.Background, hs.Foreground)
	22	+	style := hintlabel.Style{FontPx: hintlabel.FontPixels(hs.Width, hs.Height, hs.PhysicalW, hs.PhysicalH, w.scale, hs.Size), Background: bg, Foreground: fg}
	23	+	for _, l := range w.s.HintLabels {
	24	+	pix, lw, lh := r.labels.Label(l.Text, l.Typed, w.scale, style)
	25	+	if lw <= 0 \|\| lh <= 0 {
	26	+	continue
	27	+	}
	28	+	// Anchored outside the target's top-left corner, overlapping it by a
	29	+	// quarter of the label height, then kept inside the output.
	30	+	inset := int(math.Round(0.25 * float64(lh)))
	31	+	x := min(max(w.phys(l.Rect.X)-lw+inset, w.bounds.Min.X), max(w.bounds.Max.X-lw, w.bounds.Min.X))
	32	+	y := min(max(w.phys(l.Rect.Y)-lh+inset, w.bounds.Min.Y), max(w.bounds.Max.Y-lh, w.bounds.Min.Y))
	33	+	rect := image.Rect(x, y, x+lw, y+lh)
	34	+	if rect.Intersect(w.bounds).Empty() {
	35	+	continue
	36	+	}
	37	+	key := shmKey{label: l.Text, typed: l.Typed, scale: w.scale, style: style}
	38	+	st := shmState{w: lw, h: lh, seq: 1}
	39	+	c := r.shmCached(key, st)
	40	+	if c == nil {
	41	+	var err error
	42	+	if c, err = r.shmCopyFor(key, st, pix, 0, lw*4, nil); err != nil {
	43	+	continue
	44	+	}
	45	+	}
	46	+	dr := r.contentDraw(rect, rect, lw, lh, [4]float32{}, 0, modeBuffer, true)
	47	+	dr.pc.color[3] = 1
	48	+	dr.shm = c
	49	+	dr.pc.buf = [4]uint32{0, uint32(lw), uint32(lh), 0}
	50	+	w.draws = append(w.draws, dr)
	51	+	}
	52	+	}

A internal/adapters/vulkan/hint_labels_test.go +137 -0

@@ -0,0 +1,137 @@
	1	+	package vulkan
	2	+
	3	+	import (
	4	+	"image"
	5	+	"math"
	6	+	"testing"
	7	+
	8	+	"github.com/stretchr/testify/require"
	9	+
	10	+	"github.com/bnema/neferwl/internal/ports"
	11	+	)
	12	+
	13	+	func TestHintLabelsDrawnAfterIndicators(t *testing.T) {
	14	+	r, err := New(256, 256)
	15	+	if err != nil {
	16	+	t.Skipf("Vulkan unavailable: %v", err)
	17	+	}
	18	+	defer r.Close()
	19	+	for _, scale := range []int{1, 2} {
	20	+	s := ports.Scene{Scale: float64(scale),
	21	+	CaptureIndicators: []ports.CaptureIndicator{{Rect: ports.Rect{W: 64, H: 64}}},
	22	+	HintLabels: []ports.HintLabel{
	23	+	{Rect: ports.Rect{X: 80, Y: 90}, Text: "f"},
	24	+	{Rect: ports.Rect{X: 100, Y: 110}, Text: "jk", Typed: 1},
	25	+	}}
	26	+	bare := s
	27	+	bare.HintLabels = nil
	28	+	before := len(r.draws(bare, nil, newDamage(&target{}, bare, image.Rect(0, 0, 256, 256))))
	29	+	ds := r.draws(s, nil, newDamage(&target{}, s, image.Rect(0, 0, 256, 256)))
	30	+	require.Len(t, ds, before+2, "one draw per label, after the indicator's")
	31	+	for i, tl := range []image.Point{{80 * scale, 90 * scale}, {100 * scale, 110 * scale}} {
	32	+	d := ds[before+i]
	33	+	require.NotNil(t, d.shm)
	34	+	lw, lh := int(d.pc.rect[2]-d.pc.rect[0]), int(d.pc.rect[3]-d.pc.rect[1])
	35	+	inset := int(math.Round(0.25 * float64(lh)))
	36	+	require.Equal(t, int32(tl.X-lw+inset), d.pc.rect[0])
	37	+	require.Equal(t, int32(tl.Y-lh+inset), d.pc.rect[1])
	38	+	require.Greater(t, d.pc.rect[2], d.pc.rect[0])
	39	+	require.Greater(t, d.pc.rect[3], d.pc.rect[1])
	40	+	require.Equal(t, float32(1), d.pc.color[3])
	41	+	}
	42	+	}
	43	+	}
	44	+
	45	+	func TestHintLabelsAnchoredAndClampedAtTopLeft(t *testing.T) {
	46	+	r, err := New(128, 128)
	47	+	if err != nil {
	48	+	t.Skipf("Vulkan unavailable: %v", err)
	49	+	}
	50	+	defer r.Close()
	51	+	// A target at the origin puts the anchor off-screen: clamped to 0,0.
	52	+	s := ports.Scene{Scale: 1, HintLabels: []ports.HintLabel{{Rect: ports.Rect{X: 0, Y: 0}, Text: "f"}, {Rect: ports.Rect{X: 60, Y: 60}, Text: "f"}}}
	53	+	ds := r.draws(s, nil, newDamage(&target{}, s, image.Rect(0, 0, 128, 128)))
	54	+	require.Len(t, ds, 2)
	55	+	require.Equal(t, int32(0), ds[0].pc.rect[0])
	56	+	require.Equal(t, int32(0), ds[0].pc.rect[1])
	57	+	lw, lh := int(ds[1].pc.rect[2]-ds[1].pc.rect[0]), int(ds[1].pc.rect[3]-ds[1].pc.rect[1])
	58	+	inset := int(math.Round(0.25 * float64(lh)))
	59	+	require.Equal(t, int32(60-lw+inset), ds[1].pc.rect[0])
	60	+	require.Equal(t, int32(60-lh+inset), ds[1].pc.rect[1])
	61	+	require.Less(t, int(ds[1].pc.rect[2]), 60+lw, "label sits left of the target, mostly outside it")
	62	+	}
	63	+
	64	+	func TestHintLabelsAbsentDrawNothing(t *testing.T) {
	65	+	r := &Renderer{width: 64, height: 64}
	66	+	s := ports.Scene{Scale: 1}
	67	+	require.Empty(t, r.draws(s, nil, newDamage(&target{}, s, image.Rect(0, 0, 64, 64))))
	68	+	}
	69	+
	70	+	func TestHintLabelsClampedInsideOutput(t *testing.T) {
	71	+	r, err := New(128, 128)
	72	+	if err != nil {
	73	+	t.Skipf("Vulkan unavailable: %v", err)
	74	+	}
	75	+	defer r.Close()
	76	+	s := ports.Scene{Scale: 1, HintStyle: ports.HintStyle{Size: 40}, HintLabels: []ports.HintLabel{{Rect: ports.Rect{X: 127, Y: 127}, Text: "ff"}}}
	77	+	ds := r.draws(s, nil, newDamage(&target{}, s, image.Rect(0, 0, 128, 128)))
	78	+	require.Len(t, ds, 1)
	79	+	require.LessOrEqual(t, ds[0].pc.rect[2], int32(128))
	80	+	require.LessOrEqual(t, ds[0].pc.rect[3], int32(128))
	81	+	}
	82	+
	83	+	func TestHintLabelsRenderAndUploadOnce(t *testing.T) {
	84	+	r, err := New(128, 128)
	85	+	if err != nil {
	86	+	t.Skipf("Vulkan unavailable: %v", err)
	87	+	}
	88	+	defer r.Close()
	89	+	s := ports.Scene{Scale: 1, Seq: 1, HintLabels: []ports.HintLabel{{Rect: ports.Rect{X: 4, Y: 4}, Text: "fj"}}}
	90	+	require.NoError(t, render(r, s, nil))
	91	+	first := r.copied
	92	+	require.Positive(t, first)
	93	+	s.Seq = 2
	94	+	require.NoError(t, render(r, s, nil))
	95	+	require.Equal(t, first, r.copied, "an unchanged label is not uploaded again")
	96	+	s.HintStyle.Size = 26
	97	+	s.Seq++
	98	+	require.NoError(t, render(r, s, nil))
	99	+	require.Greater(t, r.copied, first, "size reload uploads a new raster")
	100	+	sized := r.copied
	101	+	s.HintStyle.Background = "#89b4fa"
	102	+	s.Seq++
	103	+	require.NoError(t, render(r, s, nil))
	104	+	require.Greater(t, r.copied, sized, "colour reload uploads a new raster")
	105	+	}
	106	+
	107	+	func TestHintSceneDeltaOnlyHintLabelsChange(t *testing.T) {
	108	+	phys := func(r ports.Rect) image.Rectangle { return image.Rect(r.X, r.Y, r.X+r.W, r.Y+r.H) }
	109	+	base := ports.Scene{Scale: 1, OutputWidth: 100, OutputHeight: 100}
	110	+	with := base
	111	+	with.HintLabels = []ports.HintLabel{{Rect: ports.Rect{X: 1, Y: 1}, Text: "f"}}
	112	+	_, ok := sceneDelta(base, with, phys, 1<<30)
	113	+	require.False(t, ok, "labels appearing redraw the output")
	114	+	typed := with
	115	+	typed.HintLabels = []ports.HintLabel{{Rect: ports.Rect{X: 1, Y: 1}, Text: "f", Typed: 1}}
	116	+	_, ok = sceneDelta(with, typed, phys, 1<<30)
	117	+	require.False(t, ok)
	118	+	_, ok = sceneDelta(with, with, phys, 1<<30)
	119	+	require.True(t, ok, "same labels, no damage")
	120	+	}
	121	+
	122	+	func TestHintLabelsHeldByTarget(t *testing.T) {
	123	+	s := ports.Scene{Seq: 1, HintLabels: []ports.HintLabel{{Text: "f"}}}
	124	+	tg := &target{}
	125	+	tg.hold(s, &damageRegion{drawn: map[ports.WindowID]heldWindow{}})
	126	+	s.HintLabels[0].Text = "j"
	127	+	require.Equal(t, "f", tg.scene.HintLabels[0].Text, "the held scene owns its labels")
	128	+	}
	129	+
	130	+	func TestHintStyleChangesSceneDamage(t *testing.T) {
	131	+	old := ports.Scene{Scale: 1, HintLabels: []ports.HintLabel{{Text: "f"}}}
	132	+	changed := old
	133	+	changed.HintStyle.Size = 24
	134	+	require.False(t, old.SameAs(changed))
	135	+	_, partial := sceneDelta(old, changed, func(r ports.Rect) image.Rectangle { return image.Rect(r.X, r.Y, r.X+r.W, r.Y+r.H) }, 10)
	136	+	require.False(t, partial)
	137	+	}

M internal/adapters/vulkan/renderer.go +3 -0


@@ -9,6 +9,7 @@ import (
9	"strconv"	9		"strconv"
10	"unsafe"	10		"unsafe"
11		11
		12	+	"github.com/bnema/neferwl/internal/adapters/hintlabel"
12	"github.com/bnema/neferwl/internal/ports"	13		"github.com/bnema/neferwl/internal/ports"
13	"github.com/bnema/zerowrap"	14		"github.com/bnema/zerowrap"
14		15

@@ -85,6 +86,8 @@ type Renderer struct {
85	// the bytes copied into them, for tests.	86		// the bytes copied into them, for tests.
86	shm map[shmKey]*shmSurface	87		shm map[shmKey]*shmSurface
87	copied int	88		copied int
		89	+	// labels rasterizes the hint labels (hint_labels.go); the zero value works.
		90	+	labels hintlabel.Raster
88	// shmRects is scratch for shmCopyFor.	91		// shmRects is scratch for shmCopyFor.
89	shmRects []image.Rectangle	92		shmRects []image.Rectangle
90	// marks date the frames for Trim's wall-clock eviction (trim.go).	93		// marks date the frames for Trim's wall-clock eviction (trim.go).

M internal/adapters/vulkan/scene.go +4 -3

 	w.popups(true)
 	w.focusFlash()
 	w.captureIndicators()
+	w.hintLabels()
 	r.scratchDraws = w.draws
 	return w.draws
 }
 	for i := range content.Children {
 		if ch := &content.Children[i]; ch.Below {
 			cx, cy := at(ch)
-			w.surface(&ch.SurfaceContent, cx, cy, clip, shmKey{id, ch.Surface}, ch.Version, covers, false)
+			w.surface(&ch.SurfaceContent, cx, cy, clip, shmKey{win: id, surface: ch.Surface}, ch.Version, covers, false)
 		}
 	}
-	w.surface(content, ox, oy, clip, shmKey{id, content.Surface}, content.Version, covers, true)
+	w.surface(content, ox, oy, clip, shmKey{win: id, surface: content.Surface}, content.Version, covers, true)
 	for i := range content.Children {
 		if ch := &content.Children[i]; !ch.Below {
 			cx, cy := at(ch)
-			w.surface(&ch.SurfaceContent, cx, cy, clip, shmKey{id, ch.Surface}, ch.Version, nil, false)
+			w.surface(&ch.SurfaceContent, cx, cy, clip, shmKey{win: id, surface: ch.Surface}, ch.Version, nil, false)
 		}
 	}
 }

M internal/adapters/vulkan/shm.go +8 -0


@@ -6,6 +6,7 @@ import (
6	"runtime/debug"	6		"runtime/debug"
7	"unsafe"	7		"unsafe"
8		8
		9	+	"github.com/bnema/neferwl/internal/adapters/hintlabel"
9	"github.com/bnema/neferwl/internal/ports"	10		"github.com/bnema/neferwl/internal/ports"
10	vk "github.com/bnema/purego-vulkan/vulkan"	11		vk "github.com/bnema/purego-vulkan/vulkan"
11	"golang.org/x/sys/unix"	12		"golang.org/x/sys/unix"

@@ -154,6 +155,13 @@ func (r Renderer) freeGPUBuffer(b gpuBuffer) {
154	type shmKey struct {	155		type shmKey struct {
155	win ports.WindowID	156		win ports.WindowID
156	surface uint64	157		surface uint64
		158	+	// label, typed and scale name a compositor-drawn hint label (hint_labels.go)
		159	+	// instead of a client surface: win and surface are 0 and label is never
		160	+	// empty, so such a key cannot collide with a client's.
		161	+	label string
		162	+	typed int
		163	+	scale float64
		164	+	style hintlabel.Style
157	}	165		}
158		166
159	// shmState is what a GPU buffer holds: a w×h client buffer at a content	167		// shmState is what a GPU buffer holds: a w×h client buffer at a content

M internal/adapters/vulkan/targets.go +1 -0


@@ -43,6 +43,7 @@ type target struct {
43	heldLayers []ports.SceneLayer	43		heldLayers []ports.SceneLayer
44	heldHints []ports.Rect	44		heldHints []ports.Rect
45	heldIndicators []ports.CaptureIndicator	45		heldIndicators []ports.CaptureIndicator
		46	+	heldLabels []ports.HintLabel
46	windows map[ports.WindowID]heldWindow	47		windows map[ports.WindowID]heldWindow
47	}	48		}
48		49

M internal/adapters/wayland/keyboard_test.go +88 -1


@@ -130,13 +130,14 @@ type eventsProxy struct {
130		opcodes []uint16	130		opcodes []uint16
131		keymaps []string	131		keymaps []string
132		locked []uint32 // locked modifiers of each modifiers event	132		locked []uint32 // locked modifiers of each modifiers event
			133	+	depress []uint32 // depressed modifiers of each modifiers event
133		}	134		}
134			135
135		func (p eventsProxy) Dispatch(e wlturbo.Event) {	136		func (p eventsProxy) Dispatch(e wlturbo.Event) {
136		p.opcodes = append(p.opcodes, e.Opcode)	137		p.opcodes = append(p.opcodes, e.Opcode)
137		if e.Opcode == uint16(wayland.KeyboardEventModifiers) {	138		if e.Opcode == uint16(wayland.KeyboardEventModifiers) {
138		e.Uint32() // serial	139		e.Uint32() // serial
139	-	e.Uint32() // depressed	140	+	p.depress = append(p.depress, e.Uint32())
140		e.Uint32() // latched	141		e.Uint32() // latched
141		p.locked = append(p.locked, e.Uint32())	142		p.locked = append(p.locked, e.Uint32())
142		return	143		return

@@ -254,3 +255,89 @@ func TestFootKeyboard(t *testing.T) {
254		}	255		}
255		}	256		}
256		}	257		}
			258	+
			259	+	// SetModifiers sends the requested depressed mask with the real latched and
			260	+	// locked state, and Restore sends the real state back; the seat's own state
			261	+	// is untouched. A window that is not focused is ignored.
			262	+	func TestSetModifiersOverrideAndRestore(t *testing.T) {
			263	+	s, events, commands, dir := keyboardServer(t)
			264	+	target, proxy := focusedTarget(t, s, events, commands, dir)
			265	+	const capsLocked = 2
			266	+	var id ports.WindowID
			267	+	set := make(chan struct{})
			268	+	s.display.Do(func() { s.seat.modState = ports.ModState{Locked: capsLocked}; id = s.seat.focused; close(set) })
			269	+	<-set
			270	+	// A marker command applied on the display means every command sent before
			271	+	// it was applied (they apply in order); the roundtrip then delivers the
			272	+	// events they sent.
			273	+	var pending bool
			274	+	wait := func() {
			275	+	t.Helper()
			276	+	pending = !pending
			277	+	commands <- ports.SlotsPending{Pending: pending}
			278	+	deadline := time.Now().Add(2 * time.Second)
			279	+	for {
			280	+	var now bool
			281	+	if !s.display.Do(func() { now = s.slotsPending }) {
			282	+	t.Fatal("display stopped")
			283	+	}
			284	+	if now == pending {
			285	+	break
			286	+	}
			287	+	if time.Now().After(deadline) {
			288	+	t.Fatal("commands not applied")
			289	+	}
			290	+	time.Sleep(time.Millisecond)
			291	+	}
			292	+	roundtrip(t, target)
			293	+	}
			294	+	proxy.depress, proxy.locked = nil, nil
			295	+	commands <- ports.SetModifiers{ID: id + 100, Mods: ports.ModCtrl}
			296	+	wait()
			297	+	if len(proxy.depress) != 0 {
			298	+	t.Fatalf("unfocused window got modifiers %v", proxy.depress)
			299	+	}
			300	+	commands <- ports.SetModifiers{ID: id, Mods: ports.ModCtrl \| ports.ModShift}
			301	+	wait()
			302	+	if !slices.Equal(proxy.depress, []uint32{xkbControl \| xkbShift}) \|\| !slices.Equal(proxy.locked, []uint32{capsLocked}) {
			303	+	t.Fatalf("override depressed %v locked %v", proxy.depress, proxy.locked)
			304	+	}
			305	+	done := make(chan ports.ModState, 1)
			306	+	commands <- ports.SetModifiers{ID: id, Restore: true, State: ports.ModState{Locked: capsLocked}}
			307	+	wait()
			308	+	if !slices.Equal(proxy.depress, []uint32{xkbControl \| xkbShift, 0}) \|\| !slices.Equal(proxy.locked, []uint32{capsLocked, capsLocked}) {
			309	+	t.Fatalf("restore depressed %v locked %v", proxy.depress, proxy.locked)
			310	+	}
			311	+	s.display.Do(func() { done <- s.seat.modState })
			312	+	if m := <-done; m != (ports.ModState{Locked: capsLocked}) {
			313	+	t.Fatalf("seat state changed: %+v", m)
			314	+	}
			315	+	}
			316	+
			317	+	// Restore carries the real state core saw, which swallowed keys (a latch a
			318	+	// hint label key consumed) never gave the seat: it is sent and adopted.
			319	+	func TestSetModifiersRestoreAdoptsCoreState(t *testing.T) {
			320	+	s, events, commands, dir := keyboardServer(t)
			321	+	target, proxy := focusedTarget(t, s, events, commands, dir)
			322	+	const shiftLatched, capsLocked = 1, 2
			323	+	var id ports.WindowID
			324	+	set := make(chan struct{})
			325	+	s.display.Do(func() { s.seat.modState = ports.ModState{Latched: shiftLatched}; id = s.seat.focused; close(set) })
			326	+	<-set
			327	+	proxy.depress, proxy.locked = nil, nil
			328	+	want := ports.ModState{Locked: capsLocked} // the latch was consumed
			329	+	commands <- ports.SetModifiers{ID: id, Restore: true, State: want}
			330	+	var got ports.ModState
			331	+	deadline := time.Now().Add(2 * time.Second)
			332	+	for got != want {
			333	+	if time.Now().After(deadline) {
			334	+	t.Fatalf("seat state %+v, want %+v", got, want)
			335	+	}
			336	+	s.display.Do(func() { got = s.seat.modState })
			337	+	time.Sleep(time.Millisecond)
			338	+	}
			339	+	roundtrip(t, target)
			340	+	if !slices.Equal(proxy.locked, []uint32{capsLocked}) \|\| !slices.Equal(proxy.depress, []uint32{0}) {
			341	+	t.Fatalf("restore depressed %v locked %v", proxy.depress, proxy.locked)
			342	+	}
			343	+	}

M internal/adapters/wayland/seat.go +56 -0


@@ -83,6 +83,8 @@ func (s *Server) applyInput(cmd ports.ClientCommand) {
83	target = c.ID	83		target = c.ID
84	case ports.ForwardKey:	84		case ports.ForwardKey:
85	target = c.ID	85		target = c.ID
		86	+	case ports.SetModifiers:
		87	+	target = c.ID
86	}	88		}
87	if target != 0 && !s.lockInputTarget(target) {	89		if target != 0 && !s.lockInputTarget(target) {
88	return	90		return

@@ -177,6 +179,8 @@ func (s *Server) applyInput(cmd ports.ClientCommand) {
177	if s.seat.focused != c.ID {	179		if s.seat.focused != c.ID {
178	s.changeFocus(c.ID)	180		s.changeFocus(c.ID)
179	}	181		}
		182	+	case ports.SetModifiers:
		183	+	s.setModifiers(c)
180	case ports.ForwardKey:	184		case ports.ForwardKey:
181	if !(s.security != nil && s.security.Snapshot().Protected) && s.grabKey(c) {	185		if !(s.security != nil && s.security.Snapshot().Protected) && s.grabKey(c) {
182	return	186		return

@@ -219,6 +223,58 @@ func (s *Server) applyInput(cmd ports.ClientCommand) {
219	}	223		}
220	}	224		}
221		225
		226	+	// xkb modifier masks of the default keymaps: Shift, Control, Mod1 (Alt) and
		227	+	// Mod4 (Super). The server does not parse the keymap for its modifier indices.
		228	+	const (
		229	+	xkbShift = 1 << 0
		230	+	xkbControl = 1 << 2
		231	+	xkbMod1 = 1 << 3
		232	+	xkbMod4 = 1 << 6
		233	+	)
		234	+
		235	+	// depressedMask maps core modifier flags to xkb depressed masks.
		236	+	func depressedMask(m ports.Mods) uint32 {
		237	+	var d uint32
		238	+	for _, f := range []struct {
		239	+	flag ports.Mods
		240	+	mask uint32
		241	+	}{{ports.ModShift, xkbShift}, {ports.ModCtrl, xkbControl}, {ports.ModAlt, xkbMod1}, {ports.ModSuper, xkbMod4}} {
		242	+	if m&f.flag != 0 {
		243	+	d \|= f.mask
		244	+	}
		245	+	}
		246	+	return d
		247	+	}
		248	+
		249	+	// setModifiers overrides, then restores, the modifiers the focused window's
		250	+	// keyboards (or the input method grab) see. The override replaces the
		251	+	// depressed modifiers and keeps latched, locked and group; the seat's real
		252	+	// modState is only replaced by a Restore, with the state core carries.
		253	+	func (s *Server) setModifiers(c ports.SetModifiers) {
		254	+	_, keyboards := s.focusTarget(c.ID)
		255	+	if c.ID != s.seat.focused \|\| len(keyboards) == 0 {
		256	+	s.log.Debug().Uint64("id", uint64(c.ID)).Msg("ignored set modifiers")
		257	+	return
		258	+	}
		259	+	s.useKeymap(nil)
		260	+	s.serial++
		261	+	if c.Restore {
		262	+	// Swallowed keys never reached the seat: core's snapshot is the
		263	+	// real state.
		264	+	s.seat.modState = c.State
		265	+	}
		266	+	m := s.seat.modState
		267	+	if !c.Restore {
		268	+	m.Depressed = depressedMask(c.Mods)
		269	+	}
		270	+	for _, k := range keyboards {
		271	+	k.SendModifiers(s.serial, m.Depressed, m.Latched, m.Locked, m.Group)
		272	+	}
		273	+	if g := s.grab(); g != nil {
		274	+	g.res.SendModifiers(s.serial, m.Depressed, m.Latched, m.Locked, m.Group)
		275	+	}
		276	+	}
		277	+
222	func (s Server) windowKeyboards(w window) []*wayland.Keyboard {	278		func (s Server) windowKeyboards(w window) []*wayland.Keyboard {
223	if w == nil \|\| !w.mapped \|\| !w.xdg.resource.Resource.Alive() {	279		if w == nil \|\| !w.mapped \|\| !w.xdg.resource.Resource.Alive() {
224	return nil	280		return nil

M internal/adapters/wayland/server.go +2 -2


@@ -752,13 +752,13 @@ func (s *Server) apply(cmd ports.ClientCommand) {
752		// Only input/focus commands require an owner envelope. Config relay	752		// Only input/focus commands require an owner envelope. Config relay
753		// also produces SetKeymap and other epoch-independent commands.	753		// also produces SetKeymap and other epoch-independent commands.
754		switch cmd.(type) {	754		switch cmd.(type) {
755	-	case ports.PointerFocus, ports.PointerMotionTo, ports.PointerButtonTo, ports.PointerAxisTo, ports.GestureBeginTo, ports.GestureUpdateTo, ports.GestureEndTo, ports.FocusWindow, ports.ForwardKey,	755	+	case ports.PointerFocus, ports.PointerMotionTo, ports.PointerButtonTo, ports.PointerAxisTo, ports.GestureBeginTo, ports.GestureUpdateTo, ports.GestureEndTo, ports.FocusWindow, ports.ForwardKey, ports.SetModifiers,
756		ports.DataDragFocus, ports.DataDragMotion, ports.DataDragDrop, ports.DataDragCancel:	756		ports.DataDragFocus, ports.DataDragMotion, ports.DataDragDrop, ports.DataDragCancel:
757		return	757		return
758		}	758		}
759		}	759		}
760		switch c := cmd.(type) {	760		switch c := cmd.(type) {
761	-	case ports.PointerFocus, ports.PointerMotionTo, ports.PointerButtonTo, ports.PointerAxisTo, ports.GestureBeginTo, ports.GestureUpdateTo, ports.GestureEndTo, ports.SetKeymap, ports.FocusWindow, ports.ForwardKey:	761	+	case ports.PointerFocus, ports.PointerMotionTo, ports.PointerButtonTo, ports.PointerAxisTo, ports.GestureBeginTo, ports.GestureUpdateTo, ports.GestureEndTo, ports.SetKeymap, ports.FocusWindow, ports.ForwardKey, ports.SetModifiers:
762		s.applyInput(c)	762		s.applyInput(c)
763		case ports.DataDragFocus, ports.DataDragMotion, ports.DataDragDrop, ports.DataDragCancel:	763		case ports.DataDragFocus, ports.DataDragMotion, ports.DataDragDrop, ports.DataDragCancel:
764		s.applyDrag(c)	764		s.applyDrag(c)

M internal/app/app.go +7 -0


@@ -84,6 +84,7 @@ const busRetry = 2 * time.Second
84	func relayConfig(ctx context.Context, cur ports.Config, in <-chan ports.ConfigChanged, out chan<- ports.ConfigChanged, keymaps chan xkb.Keymap, deviceConfigs chan ports.InputDevicesConfig, curs cursors, commands chan<- ports.ClientCommand, log zerowrap.Logger) {	84		func relayConfig(ctx context.Context, cur ports.Config, in <-chan ports.ConfigChanged, out chan<- ports.ConfigChanged, keymaps chan xkb.Keymap, deviceConfigs chan ports.InputDevicesConfig, curs cursors, commands chan<- ports.ClientCommand, log zerowrap.Logger) {
85	kb := cur.Keyboard	85		kb := cur.Keyboard
86	devCfg := cur.InputDevicesConfig	86		devCfg := cur.InputDevicesConfig
		87	+	hintOn := cur.Hint.Enable
87	for {	88		for {
88	var ev ports.ConfigChanged	89		var ev ports.ConfigChanged
89	select {	90		select {

@@ -91,6 +92,12 @@ func relayConfig(ctx context.Context, cur ports.Config, in <-chan ports.ConfigCh
91	return	92		return
92	case ev = <-in:	93		case ev = <-in:
93	}	94		}
		95	+	// hint.enable is read at startup: nothing of hint mode starts or
		96	+	// stops on a reload.
		97	+	if ev.Config.Hint.Enable != hintOn {
		98	+	log.Info().Bool("running", hintOn).Msg("hint.enable changes apply after restart")
		99	+	ev.Config.Hint.Enable = hintOn
		100	+	}
94	next := ev.Config.Keyboard	101		next := ev.Config.Keyboard
95	layoutChanged := next.Layout != kb.Layout \|\| next.Variant != kb.Variant \|\| next.Options != kb.Options	102		layoutChanged := next.Layout != kb.Layout \|\| next.Variant != kb.Variant \|\| next.Options != kb.Options
96	repeatChanged := next.RepeatRate != kb.RepeatRate \|\| next.RepeatDelay != kb.RepeatDelay	103		repeatChanged := next.RepeatRate != kb.RepeatRate \|\| next.RepeatDelay != kb.RepeatDelay

A internal/app/captured.go +41 -0

@@ -0,0 +1,41 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+
	6	+	"github.com/bnema/neferwl/internal/adapters/logging"
	7	+	"github.com/bnema/neferwl/internal/ports"
	8	+	)
	9	+
	10	+	// routeCaptured demultiplexes capture replies from the outputs: replies whose
	11	+	// ID carries ports.CaptureIDInternal go to internal, all others to wayland.
	12	+	// Both sends give way to ctx. A nil internal means no internal consumer is
	13	+	// wired: internal replies are dropped (logged at debug) instead of blocking
	14	+	// forever, since nothing would ever receive them. It returns when ctx is done
	15	+	// or in is closed.
	16	+	func routeCaptured(ctx context.Context, in <-chan ports.CaptureDone, wayland, internal chan<- ports.CaptureDone) {
	17	+	log := logging.For(ctx, "app")
	18	+	for {
	19	+	select {
	20	+	case <-ctx.Done():
	21	+	return
	22	+	case done, ok := <-in:
	23	+	if !ok {
	24	+	return
	25	+	}
	26	+	dst := wayland
	27	+	if done.ID&ports.CaptureIDInternal != 0 {
	28	+	if internal == nil {
	29	+	log.Debug().Uint64("id", done.ID).Msg("internal capture reply dropped: no consumer")
	30	+	continue
	31	+	}
	32	+	dst = internal
	33	+	}
	34	+	select {
	35	+	case dst <- done:
	36	+	case <-ctx.Done():
	37	+	return
	38	+	}
	39	+	}
	40	+	}
	41	+	}

A internal/app/captured_test.go +72 -0

@@ -0,0 +1,72 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"testing"
	6	+	"time"
	7	+
	8	+	"github.com/stretchr/testify/require"
	9	+
	10	+	"github.com/bnema/neferwl/internal/ports"
	11	+	)
	12	+
	13	+	func TestCapturedDemux(t *testing.T) {
	14	+	ctx, cancel := context.WithCancel(context.Background())
	15	+	defer cancel()
	16	+	in := make(chan ports.CaptureDone, 4)
	17	+	wl := make(chan ports.CaptureDone, 4)
	18	+	internal := make(chan ports.CaptureDone, 4)
	19	+	exited := make(chan struct{})
	20	+	go func() { routeCaptured(ctx, in, wl, internal); close(exited) }()
	21	+
	22	+	in <- ports.CaptureDone{ID: 7}
	23	+	in <- ports.CaptureDone{ID: ports.CaptureIDInternal \| 3}
	24	+	require.Equal(t, uint64(7), recvDone(t, wl).ID)
	25	+	require.Equal(t, ports.CaptureIDInternal\|3, recvDone(t, internal).ID)
	26	+	require.Empty(t, wl)
	27	+	require.Empty(t, internal)
	28	+
	29	+	cancel()
	30	+	select {
	31	+	case <-exited:
	32	+	case <-time.After(2 * time.Second):
	33	+	t.Fatal("routeCaptured did not return on cancel")
	34	+	}
	35	+	}
	36	+
	37	+	func TestCapturedDemuxCancelWhileBlocked(t *testing.T) {
	38	+	ctx, cancel := context.WithCancel(context.Background())
	39	+	in := make(chan ports.CaptureDone, 1)
	40	+	wl := make(chan ports.CaptureDone) // never read
	41	+	exited := make(chan struct{})
	42	+	go func() { routeCaptured(ctx, in, wl, nil); close(exited) }()
	43	+	in <- ports.CaptureDone{ID: 1}
	44	+	cancel()
	45	+	select {
	46	+	case <-exited:
	47	+	case <-time.After(2 * time.Second):
	48	+	t.Fatal("routeCaptured blocked on a full destination after cancel")
	49	+	}
	50	+	}
	51	+
	52	+	func TestCapturedDemuxDropsInternalWithoutConsumer(t *testing.T) {
	53	+	ctx, cancel := context.WithCancel(context.Background())
	54	+	defer cancel()
	55	+	in := make(chan ports.CaptureDone, 2)
	56	+	wl := make(chan ports.CaptureDone, 2)
	57	+	go routeCaptured(ctx, in, wl, nil)
	58	+	in <- ports.CaptureDone{ID: ports.CaptureIDInternal \| 1}
	59	+	in <- ports.CaptureDone{ID: 2}
	60	+	require.Equal(t, uint64(2), recvDone(t, wl).ID)
	61	+	}
	62	+
	63	+	func recvDone(t *testing.T, ch <-chan ports.CaptureDone) ports.CaptureDone {
	64	+	t.Helper()
	65	+	select {
	66	+	case d := <-ch:
	67	+	return d
	68	+	case <-time.After(2 * time.Second):
	69	+	t.Fatal("no capture reply")
	70	+	return ports.CaptureDone{}
	71	+	}
	72	+	}

A internal/app/hintdebug.go +51 -0

@@ -0,0 +1,51 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"os"
	6	+	"path/filepath"
	7	+
	8	+	"github.com/bnema/neferwl/internal/adapters/atspi"
	9	+	"github.com/bnema/neferwl/internal/adapters/hints"
	10	+	"github.com/bnema/neferwl/internal/adapters/logging"
	11	+	)
	12	+
	13	+	// hintOptions builds the hint worker options: the accessibility source, and,
	14	+	// when NEFERWL_HINT_DEBUG is "1", the dump directory of scanned frames,
	15	+	// $XDG_STATE_HOME/neferwl/hints. The accessibility bus is only contacted
	16	+	// by the first scan; without it the worker reads windows with OCR only.
	17	+	func hintOptions(ctx context.Context) hints.Options {
	18	+	opts := hintDump(ctx)
	19	+	// The source connects on its first scan, and again after a failure. The
	20	+	// hints worker owns it and closes it when it returns.
	21	+	src := atspi.New()
	22	+	opts.Source = src
	23	+	return opts
	24	+	}
	25	+
	26	+	// hintDump reads NEFERWL_HINT_DEBUG once.
	27	+	func hintDump(ctx context.Context) hints.Options {
	28	+	if os.Getenv("NEFERWL_HINT_DEBUG") != "1" {
	29	+	return hints.Options{}
	30	+	}
	31	+	log := logging.For(ctx, "hints")
	32	+	state := os.Getenv("XDG_STATE_HOME")
	33	+	if state == "" {
	34	+	home, err := os.UserHomeDir()
	35	+	if err != nil {
	36	+	log.Warn().Err(err).Msg("hint debug dumps disabled")
	37	+	return hints.Options{}
	38	+	}
	39	+	state = filepath.Join(home, ".local", "state")
	40	+	}
	41	+	dir := filepath.Join(state, "neferwl", "hints")
	42	+	log.Warn().Str("dir", dir).Msg("hint debug dumps enabled")
	43	+	return hints.Options{DumpDir: dir}
	44	+	}
	45	+
	46	+	// hintDisabledNote returns the once-only notice for a hint bind pressed while
	47	+	// hint mode is off. The logger is made now; nothing else of hint mode exists.
	48	+	func hintDisabledNote(ctx context.Context) func() {
	49	+	log := logging.For(ctx, "core")
	50	+	return func() { log.Info().Msg("hint mode is disabled; set hint.enable = on and restart") }
	51	+	}

A internal/app/hintoptin_test.go +68 -0

@@ -0,0 +1,68 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"runtime"
	6	+	"testing"
	7	+	"time"
	8	+
	9	+	"github.com/stretchr/testify/require"
	10	+
	11	+	"github.com/bnema/neferwl/internal/adapters/clock"
	12	+	"github.com/bnema/neferwl/internal/adapters/config"
	13	+	"github.com/bnema/neferwl/internal/adapters/logging"
	14	+	"github.com/bnema/neferwl/internal/adapters/xkb"
	15	+	"github.com/bnema/neferwl/internal/ports"
	16	+	)
	17	+
	18	+	func TestHintPipesOffByDefault(t *testing.T) {
	19	+	require.False(t, config.Defaults().Hint.Enable)
	20	+	off := newPipes(false, false)
	21	+	require.Nil(t, off.hintScans)
	22	+	require.Nil(t, off.hintResults)
	23	+	require.Nil(t, off.hintCaptured)
	24	+	// No demultiplexing: the outputs reply on the wayland channel itself.
	25	+	require.Equal(t, off.captured, off.capturedAll)
	26	+	ch := off.core()
	27	+	require.Nil(t, ch.HintScans)
	28	+	require.Nil(t, ch.Hints)
	29	+	on := newPipes(false, true)
	30	+	require.NotNil(t, on.hintScans)
	31	+	require.NotNil(t, on.hintResults)
	32	+	require.NotNil(t, on.hintCaptured)
	33	+	require.NotEqual(t, on.captured, on.capturedAll)
	34	+	}
	35	+
	36	+	func TestHintOffStartsNoHintGoroutine(t *testing.T) {
	37	+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
	38	+	ctx, cancel := context.WithCancel(context.Background())
	39	+	defer cancel()
	40	+	done := make(chan error, 1)
	41	+	go func() {
	42	+	done <- Run(ctx, Options{Backend: "headless", NoXwayland: true, Config: config.Defaults(), NoTerminal: true, Timeout: 800 * time.Millisecond})
	43	+	}()
	44	+	time.Sleep(400 * time.Millisecond)
	45	+	buf := make([]byte, 1<<20)
	46	+	dump := string(buf[:runtime.Stack(buf, true)])
	47	+	require.NotContains(t, dump, "adapters/hints.")
	48	+	require.NotContains(t, dump, "adapters/atspi.")
	49	+	require.NotContains(t, dump, "routeCaptured")
	50	+	select {
	51	+	case <-done:
	52	+	case <-time.After(3 * time.Second):
	53	+	t.Fatal("session did not stop")
	54	+	}
	55	+	}
	56	+
	57	+	func TestRelayConfigKeepsStartupHintEnable(t *testing.T) {
	58	+	ctx, cancel := context.WithCancel(context.Background())
	59	+	defer cancel()
	60	+	cur := config.Defaults()
	61	+	in := make(chan ports.ConfigChanged, 1)
	62	+	out := make(chan ports.ConfigChanged, 1)
	63	+	go relayConfig(ctx, cur, in, out, make(chan *xkb.Keymap, 1), make(chan ports.InputDevicesConfig, 1), newCursors(clock.System{}, 0), make(chan ports.ClientCommand, 1), logging.For(ctx, "config"))
	64	+	next := config.Defaults()
	65	+	next.Hint.Enable = true
	66	+	in <- ports.ConfigChanged{Config: next}
	67	+	require.False(t, (<-out).Config.Hint.Enable)
	68	+	}

M internal/app/session.go +17 -2

 	"github.com/bnema/neferwl/internal/adapters/config"
 	"github.com/bnema/neferwl/internal/adapters/drm"
 	"github.com/bnema/neferwl/internal/adapters/headlessinput"
+	"github.com/bnema/neferwl/internal/adapters/hints"
 	"github.com/bnema/neferwl/internal/adapters/launcher"
 	"github.com/bnema/neferwl/internal/adapters/libinput"
 	"github.com/bnema/neferwl/internal/adapters/logging"
 	km           *xkb.Keymap
 	core         *core.Core
 	dmabuf       ports.DMABufSupport
+	hintOpts     hints.Options
 	wsIDs        *workspaceid.IDs
 	captureAllow *captureallow.Store
 	cast         *screencast.Service // nil without the D-Bus services
 			return s, err
 		}
 	}
-	s.p = newPipes(s.hw != nil)
+	s.p = newPipes(s.hw != nil, opts.Config.Hint.Enable)
 	// core() carries channels only; the gate, clock and terminal flag are
 	// explicit core.Options.
-	if s.core, err = core.New(opts.Config, s.p.core(), core.Options{Security: s.security, Clock: s.clock, Terminal: !opts.NoTerminal}); err != nil {
+	if s.core, err = core.New(opts.Config, s.p.core(), core.Options{Security: s.security, Clock: s.clock, Terminal: !opts.NoTerminal, HintDisabled: hintDisabledNote(ctx)}); err != nil {
 		return s, err
 	}
 	if s.km, err = xkb.New(xkb.RMLVO{Layout: opts.Config.Keyboard.Layout, Variant: opts.Config.Keyboard.Variant, Options: opts.Config.Keyboard.Options}); err != nil {
 		return s, err
 	}
 	runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
+	// Hint mode off: no accessibility connection, no debug env, nothing.
+	if opts.Config.Hint.Enable {
+		s.hintOpts = hintOptions(ctx)
+	}
 	// Every output renders on the same GPU: its formats are the clients'.
 	s.dmabuf = vulkan.Probe()
 	log.Info().Int("formats", len(s.dmabuf.Formats)).Msg("dmabuf")
 			log.Warn().Err(err).Msg("capture allowlist not watched, restart to apply changes")
 		}
 	}()
+	if p.hintScans != nil {
+		workers.Add(2)
+		go func() { defer workers.Done(); routeCaptured(ctx, p.capturedAll, p.captured, p.internalCaptured()) }()
+		go func() {
+			defer workers.Done()
+			// Run only returns when ctx is done: nothing to report.
+			_ = hints.Run(ctx, p.hintScans, p.captures, p.hintCaptured, p.hintResults, s.hintOpts)
+		}()
+	}
 	go func() { defer workers.Done(); done <- child.Run(ctx, p.spawn) }()
 	go func() { defer workers.Done(); done <- c.Run(ctx) }()
 	workers.Add(1)

M internal/app/wiring.go +34 -5

 	// move running slides.
 	frames chan ports.OutputFrame
+	// Hint mode: core asks the hints adapter for a scan and gets the result.
+	hintScans   chan ports.HintScanRequest
+	hintResults chan ports.HintScanResult
 	captures chan ports.CaptureRequest
-	// Every admitted capture owns one reply slot until wayland consumes it.
+	// Every admitted capture owns one reply slot until routeCaptured takes
-	// Keep this capacity tied to admission so output routing never waits.
+	// it. Keep this capacity tied to admission so output routing never waits.
-	captured chan ports.CaptureDone
+	// With hint mode on, the outputs reply on capturedAll and routeCaptured splits the replies by
+	// ports.CaptureIDInternal into captured (the wayland consumer) and
+	// hintCaptured (internal consumers). With hint mode off, capturedAll is
+	// captured itself and there is no routeCaptured.
+	capturedAll chan ports.CaptureDone
+	captured    chan ports.CaptureDone
+	// hintCaptured carries internal capture replies to the hints adapter.
+	hintCaptured chan ports.CaptureDone
 	outputFormats chan ports.OutputFormats
 	outputHeads   chan ports.OutputHeads
 	castCommands chan ports.CastCommand
 }
-func newPipes(realSession bool) *pipes {
+// newPipes makes the channels. The hint channels exist only with hint set
+// (hint.enable = on): without them core's hint action is a no-op and nothing
+// of hint mode is allocated.
+func newPipes(realSession, hint bool) *pipes {
 	p := &pipes{
 		securityChanges: make(chan ports.SecurityState, 8),
 		securityEvents:  make(chan ports.SecurityBackendEvent, 64),
 		castCommands: make(chan ports.CastCommand, 64),
 	}
+	// Without hint mode every reply is for the wayland consumer: the outputs
+	// reply on its channel directly and nothing demultiplexes.
+	p.capturedAll = p.captured
+	if hint {
+		p.capturedAll = make(chan ports.CaptureDone, ports.MaxCaptureInflight)
+		p.hintScans = make(chan ports.HintScanRequest, 1)
+		p.hintResults = make(chan ports.HintScanResult, 1)
+		p.hintCaptured = make(chan ports.CaptureDone, 1)
+	}
 	if realSession {
 		p.scales = make(chan ports.ScaleChanged, 8)
 	}
 		ConfigErrors: p.configErrors,
 		Scales:       p.scales,
 		Frames:       p.frames,
+		HintScans:    p.hintScans,
+		Hints:        p.hintResults,
 	}
 }
+// internalCaptured is the channel internal capture replies are routed to: the
+// hints adapter reads it.
+func (p *pipes) internalCaptured() chan<- ports.CaptureDone { return p.hintCaptured }
 // wayland is the wayland server's side.
 func (p *pipes) wayland() wayland.Channels {
 	return wayland.Channels{
 		content:         content,
 		frames:          p.frames,
 		captures:        p.captures,
-		captured:        p.captured,
+		captured:        p.capturedAll,
 		formats:         p.outputFormats,
 		heads:           p.outputHeads,
 		leaseRequests:   p.leaseRequests,

M internal/core/action.go +6 -0


@@ -63,6 +63,9 @@ const (
63	// command key is held (switcher.go).	63		// command key is held (switcher.go).
64	ActionSwitchColumnNext Action = "switch-column-next"	64		ActionSwitchColumnNext Action = "switch-column-next"
65	ActionSwitchColumnPrev Action = "switch-column-prev"	65		ActionSwitchColumnPrev Action = "switch-column-prev"
		66	+	// Show keyboard hint labels over the links and buttons of the focused
		67	+	// output (hint.go).
		68	+	ActionHint Action = "hint"
66	)	69		)
67		70
68	// monitorDirection returns the direction of the neighbor screen targeted by	71		// monitorDirection returns the direction of the neighbor screen targeted by

@@ -303,6 +306,9 @@ func (c *Core) dispatchAction(a Action) Effect {
303	return Effect{}	306		return Effect{}
304	}	307		}
305	switch a {	308		switch a {
		309	+	case ActionHint:
		310	+	c.hintRequest()
		311	+	return Effect{}
306	case ActionFocusMonitorLeft, ActionFocusMonitorRight, ActionFocusMonitorUp, ActionFocusMonitorDown:	312		case ActionFocusMonitorLeft, ActionFocusMonitorRight, ActionFocusMonitorUp, ActionFocusMonitorDown:
307	if i := c.neighbor(monitorDirection(a)); i >= 0 {	313		if i := c.neighbor(monitorDirection(a)); i >= 0 {
308	c.focusScreen = i	314		c.focusScreen = i

M internal/core/core.go +30 -1

 	// animating scene (Scene.Animating): a running slide moves one step per
 	// report. Without them it moves on a timer.
 	Frames <-chan ports.OutputFrame
+	// HintScans, when set, receives the scan request of each hint action
+	// (capacity 1; core never waits: a full channel cancels the hint).
+	// Hints carries the results back. With HintScans nil the hint action
+	// does nothing.
+	HintScans chan<- ports.HintScanRequest
+	Hints     <-chan ports.HintScanResult
 }
 // Options are the services and policy core is constructed with.
 	Clock ports.Clock
 	// Terminal enables automatic terminal opening according to terminal.auto-open.
 	Terminal bool
+	// HintDisabled, when set, is called the first time the hint action runs
+	// without a scanner (hint mode off). It must not block.
+	HintDisabled func()
 }
 // fullscreenGrace is how long after mapping a window's fullscreen request
 	// capt is the capture state (capture.go, capindicator.go).
 	capt  captureState
 	flash focusFlash
+	// hint is keyboard hint mode (hint.go).
+	hint hintState
+	// keyState is the real modifier state of the latest key event, swallowed
+	// keys included.
+	keyState ports.ModState
+	// synthetic is set while a hint click runs through the pointer handlers:
+	// compositor modifier+button actions (window drag) do not apply to it.
+	synthetic bool
 	// switcher is the column switcher in progress: the screen it is on and
 	// the timer that shows its cards (switcher.go).
 	switcher struct {
 		}
 		switch Action(a) {
 		case "none", ActionSpawnTerminal, ActionFocusColumnLeft, ActionFocusColumnRight, ActionFocusWindowUp, ActionFocusWindowDown, ActionMoveColumnLeft, ActionMoveColumnRight, ActionCycleColumnWidth, ActionMaximizeColumn, ActionToggleFullscreen, ActionToggleWindowStash, ActionToggleStashVisible, ActionToggleOverview, ActionCloseWindow, ActionQuit, ActionFocusWorkspacePrev, ActionFocusWorkspaceNext, ActionMoveColumnToWorkspacePrev, ActionMoveColumnToWorkspaceNext, ActionMoveWindowToWorkspacePrev, ActionMoveWindowToWorkspaceNext, ActionFocusMonitorLeft, ActionFocusMonitorRight, ActionFocusMonitorUp, ActionFocusMonitorDown, ActionMoveWorkspaceLeft, ActionMoveWorkspaceRight, ActionMoveWorkspaceToMonitorUp, ActionMoveWorkspaceToMonitorDown, ActionConsumeOrExpelLeft, ActionConsumeOrExpelRight,
-			ActionMoveWindowUp, ActionMoveWindowDown, ActionMoveWorkspacePrev, ActionMoveWorkspaceNext, ActionToggleFloating, ActionSwitchColumnNext, ActionSwitchColumnPrev:
+			ActionMoveWindowUp, ActionMoveWindowDown, ActionMoveWorkspacePrev, ActionMoveWorkspaceNext, ActionToggleFloating, ActionSwitchColumnNext, ActionSwitchColumnPrev, ActionHint:
 		default:
 			return fmt.Errorf("invalid action %q", a)
 		}
 		return errSecurityChanged
 	}
 	c.syncSecurity()
+	c.hintValidate()
 	if c.security.Protected {
 		return c.publishProtected(ctx)
 	}
 				return nil
 			}
 			continue
+		case res, ok := <-c.ch.Hints:
+			if !ok {
+				c.ch.Hints = nil
+				continue
+			}
+			if c.securityCheckpoint(ctx) != nil {
+				return nil
+			}
+			if c.hintResult(res) && c.publish(ctx) != nil {
+				return nil
+			}
 		case ev, ok := <-c.ch.Output:
 			if !ok {
 				c.ch.Output = nil

M internal/core/drag.go +1 -1


@@ -90,7 +90,7 @@ func (c Core) draggable(id WindowID) (screen, *Workspace, bool) {
90		// startDrag starts a cmd + button drag on the window under the pointer.	90		// startDrag starts a cmd + button drag on the window under the pointer.
91		// It reports whether it did: the press is then not forwarded.	91		// It reports whether it did: the press is then not forwarded.
92		func (c *Core) startDrag(ctx context.Context, v ports.PointerButton) (bool, error) {	92		func (c *Core) startDrag(ctx context.Context, v ports.PointerButton) (bool, error) {
93	-	if !v.Pressed \|\| c.drag != nil \|\| c.held() != 0 \|\| c.grab != 0 \|\| c.cmdMod == 0 \|\| c.mods&c.cmdMod == 0 {	93	+	if c.synthetic \|\| !v.Pressed \|\| c.drag != nil \|\| c.held() != 0 \|\| c.grab != 0 \|\| c.cmdMod == 0 \|\| c.mods&c.cmdMod == 0 {
94		return false, nil	94		return false, nil
95		}	95		}
96		if v.Button != btnLeft && v.Button != btnRight {	96		if v.Button != btnLeft && v.Button != btnRight {

A internal/core/hint.go +361 -0

@@ -0,0 +1,361 @@
	1	+	package core
	2	+
	3	+	import (
	4	+	"cmp"
	5	+	"context"
	6	+	"slices"
	7	+
	8	+	"github.com/bnema/neferwl/internal/ports"
	9	+	)
	10	+
	11	+	// Hint mode: the hint action asks the scanner (HintScans) for the clickable
	12	+	// targets in the last frame of the focused output, labels the ones that sit
	13	+	// over a managed window, grabs the keyboard for the label letters, and clicks
	14	+	// the centre of the target whose label was typed. Everything here runs on the
	15	+	// owner goroutine.
	16	+
	17	+	// hintAlphabet orders the label letters: home row first.
	18	+	const hintAlphabet = "fjdkslaghrueiwoqptyvbcnxmz"
	19	+
	20	+	// hintMaxTargets is the most targets labelled: every two-letter label.
	21	+	const hintMaxTargets = len(hintAlphabet) * len(hintAlphabet)
	22	+
	23	+	type hintPhase uint8
	24	+
	25	+	const (
	26	+	hintIdle hintPhase = iota
	27	+	hintPending
	28	+	hintShown
	29	+	)
	30	+
	31	+	// hintTarget is a labelled click target. x and y are global logical
	32	+	// coordinates of its centre.
	33	+	type hintTarget struct {
	34	+	x, y float64
	35	+	window WindowID
	36	+	label string
	37	+	rect ports.Rect
	38	+	text bool // found by OCR
	39	+	}
	40	+
	41	+	type hintState struct {
	42	+	phase hintPhase
	43	+	seq uint64 // last request sequence number
	44	+	output string
	45	+	// ws is the workspace on the hint output when hint mode began: another
	46	+	// one on screen ends it.
	47	+	ws *Workspace
	48	+	geom hintGeom
	49	+	windows []ports.HintWindow // visible windows at request time
	50	+	targets []hintTarget
	51	+	typed string
	52	+	}
	53	+
	54	+	// hintGeom is the output geometry a scan was requested on: a result or a
	55	+	// label set is only good for the same one.
	56	+	type hintGeom struct {
	57	+	x, y int
	58	+	w, h int // logical size
	59	+	width, height int // buffer size, physical pixels
	60	+	scale float64
	61	+	transform ports.BufferTransform
	62	+	}
	63	+
	64	+	func (sc *screen) hintGeom() hintGeom {
	65	+	r := sc.rect()
	66	+	return hintGeom{x: sc.x, y: sc.y, w: r.W, h: r.H, width: sc.info.Width, height: sc.info.Height, scale: sc.scale, transform: sc.transform}
	67	+	}
	68	+
	69	+	// hintRequest starts hint mode on the focused screen. It does nothing
	70	+	// (reporting once through Options.HintDisabled) without a scanner, while the session is protected, while hint mode is
	71	+	// active, or on an output with a non-normal transform.
	72	+	func (c *Core) hintRequest() {
	73	+	if c.ch.HintScans == nil {
	74	+	if c.opts.HintDisabled != nil {
	75	+	c.opts.HintDisabled()
	76	+	c.opts.HintDisabled = nil
	77	+	}
	78	+	return
	79	+	}
	80	+	if c.security.Protected \|\| c.hint.phase != hintIdle \|\| c.hintPointerBusy() {
	81	+	return
	82	+	}
	83	+	sc := c.cur()
	84	+	if sc.name() == "" \|\| sc.transform != 0 \|\| sc.mon.ov.open \|\| sc.info.Width <= 0 \|\| sc.info.Height <= 0 {
	85	+	return
	86	+	}
	87	+	wins := c.hintWindows(sc)
	88	+	req := ports.HintScanRequest{Seq: c.hint.seq + 1, Output: sc.name(), Width: sc.info.Width, Height: sc.info.Height, Scale: sc.scale, LinksOnly: c.cfg.Hint.LinksOnly, Windows: wins}
	89	+	select {
	90	+	case c.ch.HintScans <- req:
	91	+	default:
	92	+	return
	93	+	}
	94	+	c.hint = hintState{phase: hintPending, seq: req.Seq, output: sc.name(), ws: sc.mon.Current(), geom: sc.hintGeom(), windows: slices.Clone(wins)}
	95	+	}
	96	+
	97	+	// hintWindows lists the windows drawn on sc, bottom to top, with their client
	98	+	// area (placements are output-local).
	99	+	func (c Core) hintWindows(sc screen) []ports.HintWindow {
	100	+	view := sc.mon.viewport()
	101	+	var out []ports.HintWindow
	102	+	for _, p := range sc.shownLayout() {
	103	+	if p.Hidden \|\| p.Preview > 0 \|\| !view.shows(p) {
	104	+	continue
	105	+	}
	106	+	r := c.clientRect(p)
	107	+	if r.W <= 0 \|\| r.H <= 0 {
	108	+	continue
	109	+	}
	110	+	w := ports.HintWindow{ID: p.ID, Rect: r, Focused: p.Focused}
	111	+	rec := c.windows.lookup(p.ID)
	112	+	w.PID, w.AppID = rec.client.PID, rec.client.AppID
	113	+	out = append(out, w)
	114	+	}
	115	+	return out
	116	+	}
	117	+
	118	+	// hintPointerBusy reports whether the pointer is in use by something a
	119	+	// synthetic click would disturb: a pointer constraint, a grab, a held
	120	+	// button (swallowed ones included: they stay in c.buttons), a window or data
	121	+	// drag, or the column switcher.
	122	+	func (c *Core) hintPointerBusy() bool {
	123	+	return c.constraint.Mode != ports.ConstraintNone \|\| c.constrained.Mode != ports.ConstraintNone \|\|
	124	+	c.grab != 0 \|\| len(c.buttons) > 0 \|\| c.drag != nil \|\| c.dataDrag != nil \|\| c.switching()
	125	+	}
	126	+
	127	+	// hintResult handles a scanner result and reports whether to publish.
	128	+	func (c *Core) hintResult(res ports.HintScanResult) bool {
	129	+	h := &c.hint
	130	+	if h.phase != hintPending \|\| res.Seq != h.seq \|\| res.Output != h.output {
	131	+	return false
	132	+	}
	133	+	if res.Err != nil \|\| c.security.Protected {
	134	+	c.hintCancel()
	135	+	return true
	136	+	}
	137	+	i := c.screenIndex(h.output)
	138	+	if i < 0 {
	139	+	c.hintCancel()
	140	+	return true
	141	+	}
	142	+	sc := c.screens[i]
	143	+	if sc.hintGeom() != h.geom {
	144	+	c.hintCancel()
	145	+	return true
	146	+	}
	147	+	out := sc.rect()
	148	+	var kept []hintTarget
	149	+	for _, t := range res.Targets {
	150	+	x := float64(sc.x) + float64(t.Rect.X) + float64(t.Rect.W)/2
	151	+	y := float64(sc.y) + float64(t.Rect.Y) + float64(t.Rect.H)/2
	152	+	if x < float64(out.X) \|\| y < float64(out.Y) \|\| x >= float64(out.X+out.W) \|\| y >= float64(out.Y+out.H) {
	153	+	continue
	154	+	}
	155	+	id, _, _ := c.hit(x, y)
	156	+	if id == 0 {
	157	+	continue
	158	+	}
	159	+	root := c.popupRoot(id)
	160	+	if s, _ := c.screenOf(root); s != sc {
	161	+	// A layer surface, or a window of another output.
	162	+	continue
	163	+	}
	164	+	if !c.hintOwned(sc, t, root) {
	165	+	continue
	166	+	}
	167	+	kept = append(kept, hintTarget{x: x, y: y, window: id, rect: t.Rect, text: t.Text})
	168	+	}
	169	+	if len(kept) == 0 {
	170	+	c.hintCancel()
	171	+	return true
	172	+	}
	173	+	slices.SortStableFunc(kept, func(a, b hintTarget) int {
	174	+	return cmp.Or(cmp.Compare(a.rect.Y, b.rect.Y), cmp.Compare(a.rect.X, b.rect.X))
	175	+	})
	176	+	if len(kept) > hintMaxTargets {
	177	+	kept = kept[:hintMaxTargets]
	178	+	}
	179	+	n := len(hintAlphabet)
	180	+	for i := range kept {
	181	+	if len(kept) <= n {
	182	+	kept[i].label = string(hintAlphabet[i])
	183	+	} else {
	184	+	kept[i].label = string([]byte{hintAlphabet[i/n], hintAlphabet[i%n]})
	185	+	}
	186	+	}
	187	+	h.phase, h.targets, h.typed = hintShown, kept, ""
	188	+	return true
	189	+	}
	190	+
	191	+	// hintOwned reports whether the window under a target is the one it was
	192	+	// scanned on. An accessibility target names its window; an OCR target only
	193	+	// counts over a window that was visible when the scan was requested and has
	194	+	// not moved or resized since.
	195	+	func (c Core) hintOwned(sc screen, t ports.HintTarget, root WindowID) bool {
	196	+	if t.Window != 0 {
	197	+	return root == t.Window
	198	+	}
	199	+	for _, w := range c.hint.windows {
	200	+	if w.ID != root {
	201	+	continue
	202	+	}
	203	+	for _, p := range sc.shownLayout() {
	204	+	if p.ID == root {
	205	+	return c.clientRect(p) == w.Rect
	206	+	}
	207	+	}
	208	+	}
	209	+	return false
	210	+	}
	211	+
	212	+	// hintCancel ends hint mode.
	213	+	func (c *Core) hintCancel() { c.hint = hintState{seq: c.hint.seq} }
	214	+
	215	+	// hintValidate ends hint mode when what it was started on is gone: the
	216	+	// session became protected, the output went or changed geometry, another workspace is on screen
	217	+	// there, or the overview opened. It runs at the start of every publish.
	218	+	func (c *Core) hintValidate() {
	219	+	h := &c.hint
	220	+	if h.phase == hintIdle {
	221	+	return
	222	+	}
	223	+	i := c.screenIndex(h.output)
	224	+	if c.security.Protected \|\| i < 0 \|\| c.screens[i].mon.ov.open \|\| c.screens[i].mon.Current() != h.ws \|\| c.cur().mon.ov.open {
	225	+	c.hintCancel()
	226	+	return
	227	+	}
	228	+	if c.screens[i].hintGeom() != h.geom {
	229	+	c.hintCancel()
	230	+	}
	231	+	}
	232	+
	233	+	// hintKey handles a key while hint mode is active: the key never goes on.
	234	+	// A press is the whole job; releases take the usual swallow path.
	235	+	func (c *Core) hintKey(ctx context.Context, key ports.KeyEvent) (bool, error) {
	236	+	if c.hint.phase == hintIdle \|\| !key.Pressed {
	237	+	return false, nil
	238	+	}
	239	+	c.pressed[heldKey(key)] = true
	240	+	h := &c.hint
	241	+	if h.phase == hintPending \|\| keyName(key.Keysym) == "Escape" {
	242	+	c.hintCancel()
	243	+	return true, c.publish(ctx)
	244	+	}
	245	+	if keyName(key.Keysym) == "BackSpace" {
	246	+	if h.typed != "" {
	247	+	h.typed = h.typed[:len(h.typed)-1]
	248	+	}
	249	+	return true, c.publish(ctx)
	250	+	}
	251	+	name := key.Base
	252	+	if name == "" {
	253	+	name = key.Keysym
	254	+	}
	255	+	name = keyName(name)
	256	+	if len(name) != 1 \|\| name[0] < 'a' \|\| name[0] > 'z' {
	257	+	c.hintCancel()
	258	+	return true, c.publish(ctx)
	259	+	}
	260	+	typed := h.typed + name
	261	+	var match *hintTarget
	262	+	any := false
	263	+	for i := range h.targets {
	264	+	if len(h.targets[i].label) >= len(typed) && h.targets[i].label[:len(typed)] == typed {
	265	+	any = true
	266	+	if h.targets[i].label == typed {
	267	+	match = &h.targets[i]
	268	+	}
	269	+	}
	270	+	}
	271	+	if !any {
	272	+	return true, nil
	273	+	}
	274	+	if match == nil {
	275	+	h.typed = typed
	276	+	return true, c.publish(ctx)
	277	+	}
	278	+	return true, c.hintClick(ctx, *match)
	279	+	}
	280	+
	281	+	// hintClick ends hint mode and left-clicks the target's centre, provided the
	282	+	// pointer is free and the same window is still under it. The click is
	283	+	// synthetic: c.synthetic keeps the physical modifiers (the bind's Cmd may
	284	+	// still be down) from turning it into a window drag, without touching c.mods.
	285	+	//
	286	+	// A target found by OCR is clicked with hint.text-click held (terminals open
	287	+	// a link on a modified click). Every click gives the window the keyboard
	288	+	// first, then sends exactly the modifiers to hold (none for other targets),
	289	+	// and the real modifier state is always sent back afterwards.
	290	+	func (c *Core) hintClick(ctx context.Context, t hintTarget) (err error) {
	291	+	c.hintCancel()
	292	+	if err := c.publish(ctx); err != nil {
	293	+	return err
	294	+	}
	295	+	if !c.hintClickable(t) {
	296	+	return nil
	297	+	}
	298	+	// Every click states its modifiers (possibly none): the bind's Cmd may
	299	+	// still be down and the client must not see it on a plain click.
	300	+	if err := c.clickFocus(ctx, t.window); err != nil {
	301	+	return err
	302	+	}
	303	+	if err := c.command(ctx, ports.SetModifiers{ID: t.window, Mods: c.hintMods(t)}); err != nil {
	304	+	return err
	305	+	}
	306	+	defer func() {
	307	+	if e := c.command(ctx, ports.SetModifiers{ID: t.window, Restore: true, State: c.keyState}); err == nil {
	308	+	err = e
	309	+	}
	310	+	}()
	311	+	c.synthetic = true
	312	+	defer func() { c.synthetic = false }()
	313	+	if err := c.pointerMotion(ctx, ports.PointerMotion{X: t.x, Y: t.y, Time: c.motionTime}); err != nil {
	314	+	return err
	315	+	}
	316	+	// The motion may have been bent (constraint) or the layout changed:
	317	+	// press only where the target is.
	318	+	if c.cursorX != t.x \|\| c.cursorY != t.y \|\| !c.hintClickable(t) {
	319	+	return nil
	320	+	}
	321	+	if err := c.pointerButton(ctx, ports.PointerButton{Button: btnLeft, Pressed: true, Time: c.motionTime}); err != nil {
	322	+	return err
	323	+	}
	324	+	return c.pointerButton(ctx, ports.PointerButton{Button: btnLeft, Time: c.motionTime})
	325	+	}
	326	+
	327	+	// hintMods are the modifiers to hold for a click on t, read at click time so
	328	+	// a reloaded configuration applies.
	329	+	func (c *Core) hintMods(t hintTarget) ports.Mods {
	330	+	if !t.text {
	331	+	return 0
	332	+	}
	333	+	return c.cfg.Hint.TextClick
	334	+	}
	335	+
	336	+	// hintClickable reports whether a click on t may go out now: unprotected,
	337	+	// pointer free, and the selected window is still under the target's centre.
	338	+	func (c *Core) hintClickable(t hintTarget) bool {
	339	+	if c.security.Protected \|\| c.hintPointerBusy() {
	340	+	return false
	341	+	}
	342	+	id, _, _ := c.hit(t.x, t.y)
	343	+	return id == t.window
	344	+	}
	345	+
	346	+	// hintLabels are the labels to draw on sc: those still matching what was
	347	+	// typed. Nil unless sc is the hint output.
	348	+	func (c Core) hintLabels(sc screen) []ports.HintLabel {
	349	+	h := &c.hint
	350	+	if h.phase != hintShown \|\| sc.name() != h.output {
	351	+	return nil
	352	+	}
	353	+	var out []ports.HintLabel
	354	+	for _, t := range h.targets {
	355	+	if len(t.label) < len(h.typed) \|\| t.label[:len(h.typed)] != h.typed {
	356	+	continue
	357	+	}
	358	+	out = append(out, ports.HintLabel{Rect: ports.Rect{X: t.rect.X, Y: t.rect.Y}, Text: t.label, Typed: len(h.typed)})
	359	+	}
	360	+	return out
	361	+	}

A internal/core/hint_test.go +834 -0

@@ -0,0 +1,834 @@
	1	+	package core_test
	2	+
	3	+	import (
	4	+	"context"
	5	+	"fmt"
	6	+	"slices"
	7	+	"sync/atomic"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/neferwl/internal/core"
	12	+	portsmocks "github.com/bnema/neferwl/internal/mocks/ports"
	13	+	"github.com/bnema/neferwl/internal/ports"
	14	+	)
	15	+
	16	+	type hintHarness struct {
	17	+	t *testing.T
	18	+	input chan ports.InputEvent
	19	+	client chan ports.ClientEvent
	20	+	output chan ports.OutputEvent
	21	+	commands chan ports.ClientCommand
	22	+	scenes chan []ports.Scene
	23	+	scans chan ports.HintScanRequest
	24	+	hints chan ports.HintScanResult
	25	+	win ports.Rect // window 1, output-local
	26	+	// state is the security state input is stamped with.
	27	+	state ports.SecurityState
	28	+	}
	29	+
	30	+	// newHint starts a core with one 1000x800 output "OUT-1" and window 1 mapped.
	31	+	func newHint(t testing.T, opts core.Options) hintHarness {
	32	+	t.Helper()
	33	+	return newHintMods(t, opts, 0)
	34	+	}
	35	+
	36	+	// newHintMods is newHint with hint.text-click set to textClick.
	37	+	func newHintMods(t testing.T, opts core.Options, textClick ports.Mods) hintHarness {
	38	+	t.Helper()
	39	+	cfg := scrollDefaults()
	40	+	cfg.Hint.TextClick = textClick
	41	+	cfg.Border.Width = 0
	42	+	cfg.Layout.Gaps = 20
	43	+	cfg.Binds["Cmd+g"] = "hint"
	44	+	h := &hintHarness{
	45	+	t: t,
	46	+	input: make(chan ports.InputEvent, 32),
	47	+	client: make(chan ports.ClientEvent, 32),
	48	+	output: make(chan ports.OutputEvent, 8),
	49	+	commands: make(chan ports.ClientCommand, 256),
	50	+	scenes: make(chan []ports.Scene, 1),
	51	+	scans: make(chan ports.HintScanRequest, 1),
	52	+	// Unbuffered: a send returns once core took the result.
	53	+	hints: make(chan ports.HintScanResult),
	54	+	}
	55	+	ch := core.Channels{Client: h.client, Input: h.input, Output: h.output, Commands: h.commands, Scenes: h.scenes, HintScans: h.scans, Hints: h.hints}
	56	+	c, err := core.New(cfg, ch, opts)
	57	+	if err != nil {
	58	+	t.Fatal(err)
	59	+	}
	60	+	ctx, cancel := context.WithCancel(context.Background())
	61	+	done := make(chan error, 1)
	62	+	go func() { done <- c.Run(ctx) }()
	63	+	t.Cleanup(func() { cancel(); receive(t, done) })
	64	+	h.output <- ports.OutputAdded{Info: ports.OutputInfo{Name: "OUT-1", Width: 1000, Height: 800}}
	65	+	scene(t, h.scenes)
	66	+	h.client <- ports.WindowMapped{ID: 1, AppID: "org.test.app", PID: 4242}
	67	+	s := scene(t, h.scenes)
	68	+	h.win = s.Windows[0].Rect
	69	+	h.drain()
	70	+	return h
	71	+	}
	72	+
	73	+	func (h *hintHarness) drain() {
	74	+	for {
	75	+	select {
	76	+	case <-h.commands:
	77	+	default:
	78	+	return
	79	+	}
	80	+	}
	81	+	}
	82	+
	83	+	// press and release send one key event.
	84	+	func (h *hintHarness) press(sym string, mods ports.Mods) {
	85	+	h.send(ports.KeyEvent{Keysym: sym, Base: sym, Mods: mods, Pressed: true})
	86	+	}
	87	+
	88	+	func (h *hintHarness) release(sym string, mods ports.Mods) {
	89	+	h.send(ports.KeyEvent{Keysym: sym, Base: sym, Mods: mods})
	90	+	}
	91	+
	92	+	// send gives core an input event stamped with the current security state.
	93	+	func (h *hintHarness) send(ev ports.InputEvent) {
	94	+	h.input <- ports.SecurityInput{State: h.state, Event: ev}
	95	+	}
	96	+
	97	+	// start presses the hint bind and returns the request.
	98	+	func (h *hintHarness) start() ports.HintScanRequest {
	99	+	h.t.Helper()
	100	+	h.press("g", ports.ModSuper)
	101	+	req := receive(h.t, h.scans)
	102	+	h.release("g", 0) // the user lets go of the whole chord
	103	+	return req
	104	+	}
	105	+
	106	+	// show starts hint mode and answers with targets, returning the next scene.
	107	+	func (h *hintHarness) show(targets ...ports.HintTarget) ports.Scene {
	108	+	h.t.Helper()
	109	+	req := h.start()
	110	+	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: targets}
	111	+	return h.labelled(len(targets), 0)
	112	+	}
	113	+
	114	+	// until returns the first scene, from the next one on, that satisfies ok:
	115	+	// the bind press and other input publish scenes before the one waited for.
	116	+	func (h *hintHarness) until(ok func(ports.Scene) bool) ports.Scene {
	117	+	h.t.Helper()
	118	+	for {
	119	+	if s := scene(h.t, h.scenes); ok(s) {
	120	+	return s
	121	+	}
	122	+	}
	123	+	}
	124	+
	125	+	// labelled waits for a scene with n labels, all typed with typed letters.
	126	+	func (h *hintHarness) labelled(n, typed int) ports.Scene {
	127	+	h.t.Helper()
	128	+	return h.until(func(s ports.Scene) bool {
	129	+	return len(s.HintLabels) == n && (n == 0 \|\| s.HintLabels[0].Typed == typed)
	130	+	})
	131	+	}
	132	+
	133	+	// inside is a target of the given size inside window 1.
	134	+	func (h *hintHarness) inside(i int) ports.HintTarget {
	135	+	return ports.HintTarget{Rect: ports.Rect{X: h.win.X + 10 + i, Y: h.win.Y + 10 + 12*i, W: 40, H: 10}}
	136	+	}
	137	+
	138	+	func TestHintRequestOnBind(t *testing.T) {
	139	+	h := newHint(t, core.Options{})
	140	+	req := h.start()
	141	+	if req.Output != "OUT-1" \|\| req.Width != 1000 \|\| req.Height != 800 \|\| req.Scale != 1 \|\| req.LinksOnly \|\| req.Seq == 0 {
	142	+	t.Fatalf("request %+v", req)
	143	+	}
	144	+	want := []ports.HintWindow{{ID: 1, PID: 4242, AppID: "org.test.app", Rect: h.win, Focused: true}}
	145	+	if !slices.Equal(req.Windows, want) {
	146	+	t.Fatalf("windows %+v, want %+v", req.Windows, want)
	147	+	}
	148	+	// A second bind while pending does not ask again; the pending phase
	149	+	// cancels on any key.
	150	+	h.press("g", ports.ModSuper)
	151	+	select {
	152	+	case r := <-h.scans:
	153	+	t.Fatalf("second request %+v", r)
	154	+	case <-time.After(50 * time.Millisecond):
	155	+	}
	156	+	}
	157	+
	158	+	func TestHintRequestNilScansIsNoop(t *testing.T) {
	159	+	cfg := scrollDefaults()
	160	+	cfg.Binds["Cmd+g"] = "hint"
	161	+	input := make(chan ports.InputEvent, 4)
	162	+	client := make(chan ports.ClientEvent, 4)
	163	+	output := make(chan ports.OutputEvent, 4)
	164	+	commands := make(chan ports.ClientCommand, 32)
	165	+	scenes := make(chan []ports.Scene, 1)
	166	+	c, err := core.New(cfg, core.Channels{Client: client, Input: input, Output: output, Commands: commands, Scenes: scenes}, core.Options{})
	167	+	if err != nil {
	168	+	t.Fatal(err)
	169	+	}
	170	+	ctx, cancel := context.WithCancel(context.Background())
	171	+	done := make(chan error, 1)
	172	+	go func() { done <- c.Run(ctx) }()
	173	+	defer func() { cancel(); receive(t, done) }()
	174	+	output <- ports.OutputAdded{Info: ports.OutputInfo{Name: "OUT-1", Width: 1000, Height: 800}}
	175	+	scene(t, scenes)
	176	+	input <- ports.KeyEvent{Keysym: "g", Base: "g", Mods: ports.ModSuper, Pressed: true}
	177	+	if s := scene(t, scenes); len(s.HintLabels) != 0 {
	178	+	t.Fatal(s.HintLabels)
	179	+	}
	180	+	}
	181	+
	182	+	func TestHintRefusedWhenProtected(t *testing.T) {
	183	+	var state atomic.Pointer[ports.SecurityState]
	184	+	state.Store(&ports.SecurityState{})
	185	+	gate := portsmocks.NewMockSessionSecurity(t)
	186	+	gate.EXPECT().Snapshot().RunAndReturn(func() ports.SecurityState { return *state.Load() }).Maybe()
	187	+	h := newHint(t, core.Options{Security: gate})
	188	+	// Hint mode shown, then the session locks: the labels go away.
	189	+	h.show(h.inside(0))
	190	+	state.Store(&ports.SecurityState{Generation: 2, Protected: true})
	191	+	h.state = *state.Load()
	192	+	h.client <- ports.WindowMapped{ID: 2}
	193	+	s := h.until(func(s ports.Scene) bool { return s.Security.Protected })
	194	+	if len(s.HintLabels) != 0 {
	195	+	t.Fatalf("labels while protected %+v", s.HintLabels)
	196	+	}
	197	+	// And a new request is refused while it lasts.
	198	+	h.press("g", ports.ModSuper)
	199	+	select {
	200	+	case r := <-h.scans:
	201	+	t.Fatalf("request while protected %+v", r)
	202	+	case <-time.After(100 * time.Millisecond):
	203	+	}
	204	+	}
	205	+
	206	+	func TestHintResultShowsLabels(t *testing.T) {
	207	+	h := newHint(t, core.Options{})
	208	+	var targets []ports.HintTarget
	209	+	for i := 0; i < 26; i++ {
	210	+	targets = append(targets, h.inside(i))
	211	+	}
	212	+	s := h.show(targets...)
	213	+	if len(s.HintLabels) != 26 {
	214	+	t.Fatalf("%d labels", len(s.HintLabels))
	215	+	}
	216	+	seen := map[string]bool{}
	217	+	for _, l := range s.HintLabels {
	218	+	if len(l.Text) != 1 \|\| l.Typed != 0 \|\| seen[l.Text] {
	219	+	t.Fatalf("label %+v", l)
	220	+	}
	221	+	seen[l.Text] = true
	222	+	}
	223	+	// Sorted by position: the first target gets the first letter.
	224	+	if s.HintLabels[0].Text != "f" \|\| s.HintLabels[0].Rect.X != targets[0].Rect.X \|\| s.HintLabels[0].Rect.Y != targets[0].Rect.Y {
	225	+	t.Fatalf("first %+v", s.HintLabels[0])
	226	+	}
	227	+	}
	228	+
	229	+	func TestHintTwoLetterLabels(t *testing.T) {
	230	+	h := newHint(t, core.Options{})
	231	+	var targets []ports.HintTarget
	232	+	for i := 0; i < 27; i++ {
	233	+	targets = append(targets, ports.HintTarget{Rect: ports.Rect{X: h.win.X + 5, Y: h.win.Y + 5 + 14*i, W: 30, H: 10}})
	234	+	}
	235	+	s := h.show(targets...)
	236	+	if len(s.HintLabels) != 27 {
	237	+	t.Fatalf("%d labels", len(s.HintLabels))
	238	+	}
	239	+	seen := map[string]bool{}
	240	+	for _, l := range s.HintLabels {
	241	+	if len(l.Text) != 2 \|\| seen[l.Text] {
	242	+	t.Fatalf("label %+v", l)
	243	+	}
	244	+	seen[l.Text] = true
	245	+	}
	246	+	if s.HintLabels[0].Text != "ff" \|\| s.HintLabels[1].Text != "fj" {
	247	+	t.Fatalf("labels %v %v", s.HintLabels[0].Text, s.HintLabels[1].Text)
	248	+	}
	249	+	}
	250	+
	251	+	func TestHintDropsTargetsOutsideWindows(t *testing.T) {
	252	+	h := newHint(t, core.Options{})
	253	+	in := h.inside(0)
	254	+	// In the gap around the window, over the background.
	255	+	out := ports.HintTarget{Rect: ports.Rect{X: 2, Y: 2, W: 4, H: 4}}
	256	+	if h.win.X < 8 \|\| h.win.Y < 8 {
	257	+	t.Fatalf("window %+v leaves no gap", h.win)
	258	+	}
	259	+	req := h.start()
	260	+	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{out, in}}
	261	+	s := h.labelled(1, 0)
	262	+	if len(s.HintLabels) != 1 \|\| s.HintLabels[0].Rect.X != in.Rect.X {
	263	+	t.Fatalf("labels %+v (window %+v)", s.HintLabels, h.win)
	264	+	}
	265	+	}
	266	+
	267	+	func TestHintTypingFilters(t *testing.T) {
	268	+	h := newHint(t, core.Options{})
	269	+	var targets []ports.HintTarget
	270	+	for i := 0; i < 27; i++ {
	271	+	targets = append(targets, ports.HintTarget{Rect: ports.Rect{X: h.win.X + 5, Y: h.win.Y + 5 + 14*i, W: 30, H: 10}})
	272	+	}
	273	+	h.show(targets...)
	274	+	h.press("f", 0)
	275	+	s := h.labelled(26, 1)
	276	+	if len(s.HintLabels) != 26 {
	277	+	t.Fatalf("%d labels after f", len(s.HintLabels))
	278	+	}
	279	+	for _, l := range s.HintLabels {
	280	+	if l.Typed != 1 \|\| l.Text[0] != 'f' {
	281	+	t.Fatalf("label %+v", l)
	282	+	}
	283	+	}
	284	+	h.press("BackSpace", 0)
	285	+	h.labelled(27, 0)
	286	+	// Only the 27th target starts with j: its label is "jf".
	287	+	h.press("j", 0)
	288	+	s = h.labelled(1, 1)
	289	+	if len(s.HintLabels) != 1 \|\| s.HintLabels[0].Text != "jf" {
	290	+	t.Fatalf("after j %+v", s.HintLabels)
	291	+	}
	292	+	}
	293	+
	294	+	func TestHintClick(t *testing.T) {
	295	+	h := newHint(t, core.Options{})
	296	+	tg := h.inside(0)
	297	+	h.show(tg)
	298	+	h.drain()
	299	+	h.press("f", 0)
	300	+	s := h.labelled(0, 0)
	301	+	if len(s.HintLabels) != 0 {
	302	+	t.Fatalf("labels after click %+v", s.HintLabels)
	303	+	}
	304	+	cx := float64(tg.Rect.X) + float64(tg.Rect.W)/2 - float64(h.win.X)
	305	+	cy := float64(tg.Rect.Y) + float64(tg.Rect.H)/2 - float64(h.win.Y)
	306	+	var gotMotion, gotPress, gotRelease, gotFocus bool
	307	+	for len(h.commands) > 0 \|\| !gotRelease {
	308	+	var cmd ports.ClientCommand
	309	+	select {
	310	+	case cmd = <-h.commands:
	311	+	case <-time.After(time.Second):
	312	+	t.Fatalf("missing commands: focus %v motion %v press %v release %v", gotFocus, gotMotion, gotPress, gotRelease)
	313	+	}
	314	+	switch v := cmd.(type) {
	315	+	case ports.PointerFocus:
	316	+	if v.ID == 1 && v.X == cx && v.Y == cy {
	317	+	gotFocus = true
	318	+	}
	319	+	case ports.PointerMotionTo:
	320	+	if v.ID == 1 && v.X == cx && v.Y == cy {
	321	+	gotMotion = true
	322	+	}
	323	+	case ports.PointerButtonTo:
	324	+	if v.ID != 1 \|\| v.Button != 0x110 {
	325	+	t.Fatalf("button %+v", v)
	326	+	}
	327	+	if v.Pressed {
	328	+	gotPress = true
	329	+	} else {
	330	+	gotRelease = gotPress
	331	+	}
	332	+	}
	333	+	}
	334	+	if !gotFocus \|\| !gotMotion {
	335	+	t.Fatalf("focus %v motion %v", gotFocus, gotMotion)
	336	+	}
	337	+	}
	338	+
	339	+	func TestHintEscapeCancels(t *testing.T) {
	340	+	h := newHint(t, core.Options{})
	341	+	h.show(h.inside(0))
	342	+	h.drain()
	343	+	h.press("Escape", 0)
	344	+	s := h.labelled(0, 0)
	345	+	if len(s.HintLabels) != 0 {
	346	+	t.Fatalf("labels %+v", s.HintLabels)
	347	+	}
	348	+	h.release("Escape", 0)
	349	+	// A later key goes to the window again: the Escape release did not.
	350	+	h.press("x", 0)
	351	+	cmd := command(t, h.commands)
	352	+	if v, ok := cmd.(ports.ForwardKey); !ok \|\| v.Key.Keysym != "x" {
	353	+	t.Fatalf("command %#v", cmd)
	354	+	}
	355	+	for len(h.commands) > 0 {
	356	+	if _, ok := (<-h.commands).(ports.PointerButtonTo); ok {
	357	+	t.Fatal("button sent")
	358	+	}
	359	+	}
	360	+	}
	361	+
	362	+	func TestHintKeysNotForwarded(t *testing.T) {
	363	+	h := newHint(t, core.Options{})
	364	+	h.show(h.inside(0), h.inside(1))
	365	+	h.drain()
	366	+	// A letter that matches nothing is swallowed and does not cancel.
	367	+	h.press("q", 0)
	368	+	h.release("q", 0)
	369	+	// Any other key cancels and is swallowed.
	370	+	h.press("Return", 0)
	371	+	s := h.labelled(0, 0)
	372	+	if len(s.HintLabels) != 0 {
	373	+	t.Fatalf("labels %+v", s.HintLabels)
	374	+	}
	375	+	h.release("Return", 0)
	376	+	h.press("x", 0)
	377	+	cmd := command(t, h.commands)
	378	+	if v, ok := cmd.(ports.ForwardKey); !ok \|\| v.Key.Keysym != "x" {
	379	+	t.Fatalf("first forwarded command %#v", cmd)
	380	+	}
	381	+	}
	382	+
	383	+	func TestHintStaleSeqIgnored(t *testing.T) {
	384	+	h := newHint(t, core.Options{})
	385	+	req := h.start()
	386	+	h.hints <- ports.HintScanResult{Seq: req.Seq + 7, Output: req.Output, Targets: []ports.HintTarget{h.inside(0)}}
	387	+	// A matching result afterwards shows: the stale one changed nothing.
	388	+	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{h.inside(0), h.inside(1)}}
	389	+	s := h.labelled(2, 0)
	390	+	if len(s.HintLabels) != 2 {
	391	+	t.Fatalf("labels %+v", s.HintLabels)
	392	+	}
	393	+	}
	394	+
	395	+	func TestHintErrorCancels(t *testing.T) {
	396	+	h := newHint(t, core.Options{})
	397	+	req := h.start()
	398	+	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Err: fmt.Errorf("boom")}
	399	+	h.press("x", 0)
	400	+	cmd := command(t, h.commands)
	401	+	if v, ok := cmd.(ports.ForwardKey); !ok \|\| v.Key.Keysym != "x" {
	402	+	t.Fatalf("command %#v", cmd)
	403	+	}
	404	+	}
	405	+
	406	+	func TestHintWindowMovedAborts(t *testing.T) {
	407	+	h := newHint(t, core.Options{})
	408	+	tg := h.inside(0)
	409	+	h.show(tg)
	410	+	// The window closes before the label is typed: the click must not go
	411	+	// anywhere.
	412	+	h.client <- ports.WindowUnmapped{ID: 1}
	413	+	h.until(func(s ports.Scene) bool { return len(s.Windows) == 0 })
	414	+	h.drain()
	415	+	h.press("f", 0)
	416	+	// With no window left a key goes nowhere; the next bind press, answered
	417	+	// by a scan request, shows core handled the label key before it.
	418	+	h.press("g", ports.ModSuper)
	419	+	receive(t, h.scans)
	420	+	for len(h.commands) > 0 {
	421	+	if _, ok := (<-h.commands).(ports.PointerButtonTo); ok {
	422	+	t.Fatal("button sent to a moved window")
	423	+	}
	424	+	}
	425	+	}
	426	+
	427	+	// barrier returns the commands core sent for everything given to it so far:
	428	+	// a marker key goes through the same input channel and owner, and is
	429	+	// forwarded to the focused window after the commands of the earlier events.
	430	+	func (h *hintHarness) barrier() []ports.ClientCommand {
	431	+	h.t.Helper()
	432	+	h.press("marker", 0)
	433	+	var out []ports.ClientCommand
	434	+	for {
	435	+	cmd := command(h.t, h.commands)
	436	+	if k, ok := cmd.(ports.ForwardKey); ok && k.Key.Keysym == "marker" {
	437	+	return out
	438	+	}
	439	+	out = append(out, cmd)
	440	+	}
	441	+	}
	442	+
	443	+	// buttonsSent drains the commands and counts pointer button commands.
	444	+	func (h *hintHarness) buttonsSent() (presses, releases int) {
	445	+	for _, cmd := range h.barrier() {
	446	+	if b, ok := cmd.(ports.PointerButtonTo); ok {
	447	+	if b.Pressed {
	448	+	presses++
	449	+	} else {
	450	+	releases++
	451	+	}
	452	+	}
	453	+	}
	454	+	return presses, releases
	455	+	}
	456	+
	457	+	// The bind's modifier is still held while the label is typed: the synthetic
	458	+	// click must reach the client, not start a Cmd+left-button window drag.
	459	+	func TestHintClickWithBindModifierHeld(t *testing.T) {
	460	+	h := newHint(t, core.Options{})
	461	+	tg := h.inside(0)
	462	+	h.press("g", ports.ModSuper)
	463	+	req := receive(t, h.scans)
	464	+	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{tg}}
	465	+	h.labelled(1, 0)
	466	+	h.drain()
	467	+	h.press("f", ports.ModSuper) // Super is still down
	468	+	h.labelled(0, 0)
	469	+	presses, releases := h.buttonsSent()
	470	+	if presses != 1 \|\| releases != 1 {
	471	+	t.Fatalf("client got %d presses, %d releases", presses, releases)
	472	+	}
	473	+	// No drag is left behind: a later plain motion and a click work as usual.
	474	+	h.release("f", ports.ModSuper)
	475	+	h.release("Super_L", 0)
	476	+	h.send(ports.PointerMotion{X: float64(tg.Rect.X), Y: float64(tg.Rect.Y)})
	477	+	h.send(ports.PointerButton{Button: 0x110, Pressed: true})
	478	+	h.send(ports.PointerButton{Button: 0x110})
	479	+	if presses, releases = h.buttonsSent(); presses != 1 \|\| releases != 1 {
	480	+	t.Fatalf("after: %d presses, %d releases (drag left active?)", presses, releases)
	481	+	}
	482	+	}
	483	+
	484	+	// constrain gives window 1 a pointer constraint and waits until core applied it.
	485	+	func (h *hintHarness) constrain(mode ports.ConstraintMode, r ports.Rect) {
	486	+	h.client <- ports.PointerConstrained{ID: 1, PointerConstraint: ports.PointerConstraint{Mode: mode, Rect: r}}
	487	+	h.client <- ports.WindowMapped{ID: 1} // a publish after the event
	488	+	h.until(func(ports.Scene) bool { return true })
	489	+	}
	490	+
	491	+	func TestHintRefusedWithPointerConstraint(t *testing.T) {
	492	+	for name, c := range map[string]ports.PointerConstraint{
	493	+	"lock": {Mode: ports.ConstraintLock},
	494	+	"confine": {Mode: ports.ConstraintConfine, Rect: ports.Rect{X: 0, Y: 0, W: 5, H: 5}},
	495	+	} {
	496	+	t.Run(name, func(t *testing.T) {
	497	+	h := newHint(t, core.Options{})
	498	+	h.constrain(c.Mode, c.Rect)
	499	+	h.press("g", ports.ModSuper)
	500	+	select {
	501	+	case r := <-h.scans:
	502	+	t.Fatalf("request under a %s: %+v", name, r)
	503	+	case <-time.After(100 * time.Millisecond):
	504	+	}
	505	+	})
	506	+	}
	507	+	}
	508	+
	509	+	// A constraint that appears while the labels are shown (the pointer would be
	510	+	// held or bent away from the target) cancels the click.
	511	+	func TestHintNoClickWhenConstraintAppears(t *testing.T) {
	512	+	for name, c := range map[string]ports.PointerConstraint{
	513	+	"lock": {Mode: ports.ConstraintLock},
	514	+	"confine": {Mode: ports.ConstraintConfine, Rect: ports.Rect{X: 0, Y: 0, W: 5, H: 5}},
	515	+	} {
	516	+	t.Run(name, func(t *testing.T) {
	517	+	h := newHint(t, core.Options{})
	518	+	h.show(h.inside(0))
	519	+	h.constrain(c.Mode, c.Rect)
	520	+	h.drain()
	521	+	h.press("f", 0)
	522	+	if p, r := h.buttonsSent(); p+r != 0 {
	523	+	t.Fatalf("%d presses, %d releases under a %s", p, r, name)
	524	+	}
	525	+	})
	526	+	}
	527	+	}
	528	+
	529	+	// pressReal sends a real left press over window 1 and waits for the client to
	530	+	// see it: the window then holds the pointer grab.
	531	+	func (h *hintHarness) pressReal() {
	532	+	h.t.Helper()
	533	+	h.send(ports.PointerMotion{X: float64(h.win.X + 1), Y: float64(h.win.Y + 1)})
	534	+	h.send(ports.PointerButton{Button: 0x110, Pressed: true})
	535	+	for {
	536	+	cmd := command(h.t, h.commands)
	537	+	if b, ok := cmd.(ports.PointerButtonTo); ok && b.Pressed {
	538	+	return
	539	+	}
	540	+	}
	541	+	}
	542	+
	543	+	func TestHintRefusedWhileButtonHeld(t *testing.T) {
	544	+	h := newHint(t, core.Options{})
	545	+	h.pressReal()
	546	+	h.press("g", ports.ModSuper)
	547	+	select {
	548	+	case r := <-h.scans:
	549	+	t.Fatalf("request with a button held: %+v", r)
	550	+	case <-time.After(100 * time.Millisecond):
	551	+	}
	552	+	}
	553	+
	554	+	// A button pressed (and so a grab taken) after the labels appeared: the label
	555	+	// is typed, no synthetic button goes out and the real press stays held.
	556	+	func TestHintNoClickWhileGrabbed(t *testing.T) {
	557	+	h := newHint(t, core.Options{})
	558	+	h.show(h.inside(0))
	559	+	h.pressReal()
	560	+	h.press("f", 0)
	561	+	if p, r := h.buttonsSent(); p+r != 0 {
	562	+	t.Fatalf("synthetic click during a grab: %d presses, %d releases", p, r)
	563	+	}
	564	+	// The real release still reaches the client: c.buttons was not touched.
	565	+	h.send(ports.PointerButton{Button: 0x110})
	566	+	if _, r := h.buttonsSent(); r != 1 {
	567	+	t.Fatalf("real release delivered %d times", r)
	568	+	}
	569	+	}
	570	+
	571	+	// The window stays mapped but moves away from the target's centre.
	572	+	func TestHintMappedWindowMovedAborts(t *testing.T) {
	573	+	h := newHint(t, core.Options{})
	574	+	// A target at the right edge of window 1.
	575	+	tg := ports.HintTarget{Rect: ports.Rect{X: h.win.X + h.win.W - 50, Y: h.win.Y + 10, W: 40, H: 10}}
	576	+	h.show(tg)
	577	+	h.client <- ports.WindowMapped{ID: 2}
	578	+	s := h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 && len(s.HintLabels) <= 1 })
	579	+	if s.Windows[0].Rect == h.win {
	580	+	t.Fatalf("window 1 did not move: %+v", s.Windows)
	581	+	}
	582	+	h.drain()
	583	+	h.press("f", 0)
	584	+	if p, r := h.buttonsSent(); p+r != 0 {
	585	+	t.Fatalf("click went to %d presses %d releases after the window moved", p, r)
	586	+	}
	587	+	}
	588	+
	589	+	// commandsSent drains the commands up to a barrier, keeping the ones that
	590	+	// matter for a click: modifiers, focus, motion and buttons.
	591	+	func (h *hintHarness) commandsSent() []ports.ClientCommand {
	592	+	var out []ports.ClientCommand
	593	+	for _, cmd := range h.barrier() {
	594	+	switch cmd.(type) {
	595	+	case ports.SetModifiers, ports.FocusWindow, ports.PointerMotionTo, ports.PointerButtonTo:
	596	+	out = append(out, cmd)
	597	+	}
	598	+	}
	599	+	return out
	600	+	}
	601	+
	602	+	func commandKinds(cmds []ports.ClientCommand) []string {
	603	+	var out []string
	604	+	for _, c := range cmds {
	605	+	switch v := c.(type) {
	606	+	case ports.SetModifiers:
	607	+	if v.Restore {
	608	+	out = append(out, "restore")
	609	+	} else {
	610	+	out = append(out, fmt.Sprintf("mods:%d", v.Mods))
	611	+	}
	612	+	case ports.FocusWindow:
	613	+	out = append(out, fmt.Sprintf("focus:%d", v.ID))
	614	+	case ports.PointerMotionTo:
	615	+	out = append(out, "motion")
	616	+	case ports.PointerButtonTo:
	617	+	if v.Pressed {
	618	+	out = append(out, "press")
	619	+	} else {
	620	+	out = append(out, "release")
	621	+	}
	622	+	}
	623	+	}
	624	+	return out
	625	+	}
	626	+
	627	+	// An OCR target is clicked with the configured modifiers held: they go out
	628	+	// before the motion and are restored after the release.
	629	+	func TestHintTextClickHoldsModifiers(t *testing.T) {
	630	+	mods := ports.ModCtrl \| ports.ModShift
	631	+	h := newHintMods(t, core.Options{}, mods)
	632	+	tg := h.inside(0)
	633	+	tg.Text = true
	634	+	h.show(tg)
	635	+	h.drain()
	636	+	h.press("f", 0)
	637	+	got := commandKinds(h.commandsSent())
	638	+	want := []string{fmt.Sprintf("mods:%d", mods), "motion", "press", "release", "restore"}
	639	+	if !slices.Equal(got, want) {
	640	+	t.Fatalf("commands %v, want %v", got, want)
	641	+	}
	642	+	}
	643	+
	644	+	// Without text targets or without modifiers the click states no modifiers
	645	+	// (depressed 0) rather than leave the held bind modifier to the client.
	646	+	func TestHintClickWithoutModifiers(t *testing.T) {
	647	+	for name, c := range map[string]struct {
	648	+	mods ports.Mods
	649	+	text bool
	650	+	}{"accessibility target": {ports.ModCtrl, false}, "none configured": {0, true}} {
	651	+	t.Run(name, func(t *testing.T) {
	652	+	h := newHintMods(t, core.Options{}, c.mods)
	653	+	tg := h.inside(0)
	654	+	tg.Text = c.text
	655	+	h.show(tg)
	656	+	h.drain()
	657	+	h.press("f", 0)
	658	+	got := commandKinds(h.commandsSent())
	659	+	if want := []string{"mods:0", "motion", "press", "release", "restore"}; !slices.Equal(got, want) {
	660	+	t.Fatalf("commands %v, want %v", got, want)
	661	+	}
	662	+	})
	663	+	}
	664	+	}
	665	+
	666	+	// The bind's Super is still physically held: an accessibility click states
	667	+	// depressed 0, and the restore carries the real state of the latest key,
	668	+	// swallowed ones included (a latch a label key consumed).
	669	+	func TestHintClickHeldBindModifierAndLatchRestore(t *testing.T) {
	670	+	h := newHint(t, core.Options{})
	671	+	tg := h.inside(0)
	672	+	h.press("g", ports.ModSuper)
	673	+	req := receive(t, h.scans)
	674	+	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{tg, h.inside(1)}}
	675	+	h.labelled(2, 0)
	676	+	h.drain()
	677	+	real := ports.ModState{Depressed: 64, Latched: 0, Locked: 2}
	678	+	h.send(ports.KeyEvent{Keysym: "f", Base: "f", Mods: ports.ModSuper, Pressed: true, State: real})
	679	+	cmds := h.commandsSent()
	680	+	var restore *ports.SetModifiers
	681	+	for i, c := range cmds {
	682	+	if v, ok := c.(ports.SetModifiers); ok {
	683	+	if i == 0 && (v.Restore \|\| v.Mods != 0) {
	684	+	t.Fatalf("first modifiers %+v, want depressed 0", v)
	685	+	}
	686	+	if v.Restore {
	687	+	restore = &v
	688	+	}
	689	+	}
	690	+	}
	691	+	if restore == nil \|\| restore.State != real {
	692	+	t.Fatalf("restore %+v, want state %+v (commands %v)", restore, real, commandKinds(cmds))
	693	+	}
	694	+	}
	695	+
	696	+	// A result is kept only for the window it was scanned on.
	697	+	func TestHintResultWindowChecks(t *testing.T) {
	698	+	h := newHint(t, core.Options{})
	699	+	req := h.start()
	700	+	// Accessibility target naming another window: dropped; naming window 1: kept.
	701	+	ok := h.inside(1)
	702	+	ok.Window = 1
	703	+	bad := h.inside(0)
	704	+	bad.Window = 7
	705	+	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{bad, ok}}
	706	+	s := h.labelled(1, 0)
	707	+	if len(s.HintLabels) != 1 \|\| s.HintLabels[0].Rect.Y != ok.Rect.Y {
	708	+	t.Fatalf("labels %+v", s.HintLabels)
	709	+	}
	710	+	}
	711	+
	712	+	// A window that moved while the scan ran: an OCR target over it is dropped.
	713	+	func TestHintOCRResultDroppedWhenWindowMoved(t *testing.T) {
	714	+	h := newHint(t, core.Options{})
	715	+	req := h.start()
	716	+	tg := h.inside(0)
	717	+	tg.Text = true
	718	+	h.client <- ports.WindowMapped{ID: 2}
	719	+	h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 })
	720	+	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{tg}}
	721	+	// Whatever window now covers the target was not scanned: no labels.
	722	+	if s := h.labelled(0, 0); len(s.HintLabels) != 0 {
	723	+	t.Fatalf("labels %+v", s.HintLabels)
	724	+	}
	725	+	}
	726	+
	727	+	// A new window covering the target (not in the snapshot) drops it as well.
	728	+	func TestHintOCRResultDroppedForUnseenWindow(t *testing.T) {
	729	+	h := newHint(t, core.Options{})
	730	+	req := h.start()
	731	+	h.client <- ports.WindowMapped{ID: 2}
	732	+	s := h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 })
	733	+	tg := ports.HintTarget{Text: true}
	734	+	for _, w := range s.Windows {
	735	+	if w.ID == 2 {
	736	+	tg.Rect = ports.Rect{X: w.Rect.X + 5, Y: w.Rect.Y + 5, W: 20, H: 10}
	737	+	}
	738	+	}
	739	+	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{tg}}
	740	+	if s := h.labelled(0, 0); len(s.HintLabels) != 0 {
	741	+	t.Fatalf("labels %+v", s.HintLabels)
	742	+	}
	743	+	}
	744	+
	745	+	// Targets outside the output are dropped.
	746	+	func TestHintDropsTargetsOutsideOutput(t *testing.T) {
	747	+	h := newHint(t, core.Options{})
	748	+	req := h.start()
	749	+	out := ports.HintTarget{Rect: ports.Rect{X: 2000, Y: 10, W: 10, H: 10}}
	750	+	h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{out, h.inside(0)}}
	751	+	if s := h.labelled(1, 0); len(s.HintLabels) != 1 {
	752	+	t.Fatalf("labels %+v", s.HintLabels)
	753	+	}
	754	+	}
	755	+
	756	+	// The output changing size while labels are shown ends hint mode.
	757	+	func TestHintCancelsWhenOutputChanges(t *testing.T) {
	758	+	h := newHint(t, core.Options{})
	759	+	h.show(h.inside(0))
	760	+	h.output <- ports.OutputAdded{Info: ports.OutputInfo{Name: "OUT-1", Width: 1200, Height: 800}}
	761	+	if s := h.labelled(0, 0); len(s.HintLabels) != 0 {
	762	+	t.Fatalf("labels %+v", s.HintLabels)
	763	+	}
	764	+	}
	765	+
	766	+	// A window that does not have the keyboard gets it before the modifiers.
	767	+	func TestHintTextClickFocusesWindowFirst(t *testing.T) {
	768	+	h := newHintMods(t, core.Options{}, ports.ModCtrl)
	769	+	h.client <- ports.WindowMapped{ID: 2}
	770	+	s := h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 })
	771	+	var r ports.Rect
	772	+	for _, w := range s.Windows {
	773	+	if w.ID == 1 {
	774	+	r = w.Rect
	775	+	}
	776	+	}
	777	+	h.win = r
	778	+	tg := h.inside(0)
	779	+	tg.Text = true
	780	+	h.show(tg)
	781	+	h.drain()
	782	+	h.press("f", 0)
	783	+	got := commandKinds(h.commandsSent())
	784	+	want := []string{"focus:1", fmt.Sprintf("mods:%d", ports.ModCtrl), "motion", "press", "release", "restore"}
	785	+	if !slices.Equal(got, want) {
	786	+	t.Fatalf("commands %v, want %v", got, want)
	787	+	}
	788	+	}
	789	+
	790	+	// A click that is no longer allowed sends no modifiers at all.
	791	+	func TestHintTextClickNoModifiersWhenNotClickable(t *testing.T) {
	792	+	h := newHintMods(t, core.Options{}, ports.ModCtrl)
	793	+	tg := h.inside(0)
	794	+	tg.Text = true
	795	+	h.show(tg)
	796	+	h.constrain(ports.ConstraintLock, ports.Rect{})
	797	+	h.drain()
	798	+	h.press("f", 0)
	799	+	if got := commandKinds(h.commandsSent()); len(got) != 0 {
	800	+	t.Fatalf("commands %v under a constraint", got)
	801	+	}
	802	+	}
	803	+
	804	+	func TestHintDisabledNotesOnce(t *testing.T) {
	805	+	cfg := scrollDefaults()
	806	+	cfg.Binds["Cmd+g"] = "hint"
	807	+	var noted atomic.Int32
	808	+	h := &hintHarness{
	809	+	input: make(chan ports.InputEvent, 32),
	810	+	client: make(chan ports.ClientEvent, 32),
	811	+	output: make(chan ports.OutputEvent, 8),
	812	+	commands: make(chan ports.ClientCommand, 256),
	813	+	scenes: make(chan []ports.Scene, 1),
	814	+	}
	815	+	ch := core.Channels{Client: h.client, Input: h.input, Output: h.output, Commands: h.commands, Scenes: h.scenes}
	816	+	c, err := core.New(cfg, ch, core.Options{HintDisabled: func() { noted.Add(1) }})
	817	+	if err != nil {
	818	+	t.Fatal(err)
	819	+	}
	820	+	ctx, cancel := context.WithCancel(context.Background())
	821	+	done := make(chan error, 1)
	822	+	go func() { done <- c.Run(ctx) }()
	823	+	t.Cleanup(func() { cancel(); receive(t, done) })
	824	+	h.output <- ports.OutputAdded{Info: ports.OutputInfo{Name: "OUT-1", Width: 1000, Height: 800}}
	825	+	scene(t, h.scenes)
	826	+	for range 3 {
	827	+	h.press("g", ports.ModSuper)
	828	+	h.release("g", 0)
	829	+	}
	830	+	c.Ping(ctx)
	831	+	if got := noted.Load(); got != 1 {
	832	+	t.Fatalf("disabled note ran %d times, want 1", got)
	833	+	}
	834	+	}

M internal/core/input.go +22 -8

 	return nil
 }
+// clickFocus focuses the window id and its output the way a click on it
+// does, unless it already has the keyboard.
+func (c *Core) clickFocus(ctx context.Context, id WindowID) error {
+	s, w := c.screenOf(id)
+	if s == nil || w != s.mon.Current() || (c.keyboard.sent == id && s == c.cur()) {
+		return nil
+	}
+	now := c.now()
+	before := c.snapshot(now)
+	w.Click(id)
+	c.focusScreen = c.screenIndex(s.name())
+	c.transition(before, now)
+	return c.publish(ctx)
+}
 func (c *Core) pointerButton(ctx context.Context, v ports.PointerButton) error {
 	// Buttons a drag took stay in c.buttons until released, so the
 	// security gate admits their release; no client sees it.
 			}
 		}
 		// A click focuses the window and its output.
-		s, w := c.screenOf(id)
+		if v.Pressed {
-		if v.Pressed && s != nil && w == s.mon.Current() && (c.keyboard.sent != id || s != c.cur()) {
+			if err := c.clickFocus(ctx, id); err != nil {
-			now := c.now()
-			before := c.snapshot(now)
-			w.Click(id)
-			c.focusScreen = c.screenIndex(s.name())
-			c.transition(before, now)
-			if err := c.publish(ctx); err != nil {
 				return err
 			}
 		}
 func (c *Core) keyEvent(ctx context.Context, key ports.KeyEvent) error {
 	c.mods = key.Mods
+	c.keyState = key.State
 	c.repeatKey(key)
 	// Releasing the command key commits the switcher; the release still
 	// goes on to the bind lookup and the clients.
 		}
 		return c.rehit(ctx)
 	}
+	// Hint mode takes every key press before binds and windows.
+	if handled, err := c.hintKey(ctx, key); handled || err != nil {
+		return err
+	}
 	// The overview takes its keys before any window; others still
 	// run binds, and are not forwarded.
 	// A launcher or a menu holding the keyboard gets them first.

M internal/core/scene.go +2 -0


@@ -123,6 +123,8 @@ func (c Core) sceneFor(ctx context.Context, i int, sc screen, only *screen, ca
123	}	123		}
124	scene.Flash = c.flashFor(sc)	124		scene.Flash = c.flashFor(sc)
125	scene.CaptureIndicators = c.captureIndicators(sc)	125		scene.CaptureIndicators = c.captureIndicators(sc)
		126	+	scene.HintLabels = c.hintLabels(sc)
		127	+	scene.HintStyle = ports.HintStyle{Size: c.cfg.Hint.Size, Background: c.cfg.Hint.Background, Foreground: c.cfg.Hint.Foreground, Width: sc.info.Width, Height: sc.info.Height, PhysicalW: sc.info.PhysicalW, PhysicalH: sc.info.PhysicalH}
126	scene.Capture = c.captureSceneFor(sc, capture)	128		scene.Capture = c.captureSceneFor(sc, capture)
127	// The springs of this frame are advanced: a scene shown while one still	129		// The springs of this frame are advanced: a scene shown while one still
128	// runs wakes core when it flips; the one a spring settles on does not.	130		// runs wakes core when it flips; the one a spring settles on does not.

M internal/ports/capture_messages.go +6 -0


@@ -65,3 +65,9 @@ type CaptureDone struct {
65	Err error	65		Err error
66	Time time.Time	66		Time time.Time
67	}	67		}
		68	+
		69	+	// CaptureIDInternal is the bit set in the ID of captures requested by the
		70	+	// compositor itself (hint scanning) rather than by a Wayland client. Wayland
		71	+	// IDs start at 1 and increment, so they never set it. Replies carrying it are
		72	+	// demultiplexed away from the Wayland consumer.
		73	+	const CaptureIDInternal uint64 = 1 << 63

M internal/ports/client_commands.go +14 -0


@@ -289,3 +289,17 @@ type SpawnRequest struct {
289	// Env adds KEY=value entries to the child environment.	289		// Env adds KEY=value entries to the child environment.
290	Env []string	290		Env []string
291	}	291		}
		292	+
		293	+	// SetModifiers carries core → wayland a temporary modifier override for the
		294	+	// focused window's keyboards: Mods are sent as depressed modifiers (mapped
		295	+	// to the current keymap's masks), latched/locked/group kept from the real
		296	+	// state. Mods 0 with Restore true sends the real modifier state again: State
		297	+	// is that state as core last saw it, which the seat adopts.
		298	+	type SetModifiers struct {
		299	+	ID WindowID
		300	+	Mods Mods
		301	+	Restore bool
		302	+	State ModState
		303	+	}
		304	+
		305	+	func (SetModifiers) clientCommand() {}

M internal/ports/config.go +14 -0


@@ -161,6 +161,20 @@ type Config struct {
161	FlashDuration time.Duration	161		FlashDuration time.Duration
162	FlashColor string	162		FlashColor string
163	}	163		}
		164	+	// Hint configures keyboard hint mode (action hint).
		165	+	Hint struct {
		166	+	// Enable turns the experimental hint mode on (hint.enable). It is
		167	+	// read at startup: a reload keeps the startup value.
		168	+	Enable bool
		169	+	// LinksOnly limits the targets to links and URLs; false also
		170	+	// includes buttons (hint.targets = clickable).
		171	+	LinksOnly bool
		172	+	// TextClick are the modifiers held for a click on a link found by
		173	+	// OCR (hint.text-click); zero sends a plain click.
		174	+	TextClick Mods
		175	+	Size float64 // logical pixels; zero selects automatic density sizing
		176	+	Background, Foreground string // empty derives from Border.Active
		177	+	}
164	// Workspaces are declared with workspace.<name>.* keys, in first-seen order.	178		// Workspaces are declared with workspace.<name>.* keys, in first-seen order.
165	Workspaces []WorkspaceConfig	179		Workspaces []WorkspaceConfig
166	// Rules are the window rules (rule.<name>.* keys), in first-seen order.	180		// Rules are the window rules (rule.<name>.* keys), in first-seen order.

A internal/ports/hint_messages.go +67 -0

@@ -0,0 +1,67 @@
	1	+	package ports
	2	+
	3	+	// HintKind is the kind of a clickable hint target.
	4	+	type HintKind uint8
	5	+
	6	+	const (
	7	+	// HintLink is a link or URL.
	8	+	HintLink HintKind = iota
	9	+	// HintButton is a button or control.
	10	+	HintButton
	11	+	)
	12	+
	13	+	// HintScanRequest asks the hint scanner to find clickable targets in the last
	14	+	// frame of Output. Core sends it and expects one HintScanResult echoing Seq.
	15	+	type HintScanRequest struct {
	16	+	Seq uint64 // core-chosen, echoed back
	17	+	Output string
	18	+	// Width and Height are the output buffer size in physical pixels
	19	+	// (OutputInfo.Width/Height).
	20	+	Width, Height int
	21	+	Scale float64 // screen scale
	22	+	LinksOnly bool
	23	+	// Windows are the windows visible on Output, for sources that read a
	24	+	// toolkit's accessibility tree. Core owns the slice per request.
	25	+	Windows []HintWindow
	26	+	}
	27	+
	28	+	// HintWindow is a window visible on the hint output.
	29	+	type HintWindow struct {
	30	+	ID WindowID
	31	+	PID int
	32	+	AppID string
	33	+	// Rect is the client area, logical output-local.
	34	+	Rect Rect
	35	+	Focused bool
	36	+	}
	37	+
	38	+	// HintTarget is a clickable region found in a frame.
	39	+	type HintTarget struct {
	40	+	Rect Rect // logical, output-local
	41	+	Kind HintKind
	42	+	// Text is set for targets found by OCR (read from pixels, not from an
	43	+	// accessibility tree).
	44	+	Text bool
	45	+	// Window is the source window of an accessibility target, 0 for OCR.
	46	+	Window WindowID
	47	+	}
	48	+
	49	+	// HintScanResult answers a HintScanRequest. Core owns Targets.
	50	+	type HintScanResult struct {
	51	+	Seq uint64
	52	+	Output string
	53	+	Targets []HintTarget
	54	+	Err error
	55	+	}
	56	+
	57	+	// HintLabel is a label drawn over a hint target.
	58	+	type HintLabel struct {
	59	+	// Rect is the label box in logical output-local coordinates; X and Y are
	60	+	// the target's top-left corner, which the renderer anchors the label
	61	+	// outside of. W and H may be 0: the renderer sizes the box from Text.
	62	+	Rect Rect
	63	+	// Text is the full label, lowercase a–z.
	64	+	Text string
	65	+	// Typed is the number of leading characters already typed (drawn dimmed).
	66	+	Typed int
	67	+	}

M internal/ports/scene.go +14 -1

 	// drawn over everything; nil when none. Captures never hold them: the
 	// capture pipeline serves requests from a scene without them.
 	CaptureIndicators []CaptureIndicator
+	// HintLabels are the keyboard hint labels of this output, drawn over
+	// everything, after CaptureIndicators; nil when hint mode is not shown
+	// here. Captures never contain them.
+	HintLabels []HintLabel
+	HintStyle  HintStyle
 	// Flash is the focus flash drawn over everything of this output; nil
 	// draws nothing.
 	Flash *FocusFlash
 	}
 	return slices.Equal(s.Windows, o.Windows) && slices.Equal(s.Separators, o.Separators) &&
 		slices.Equal(s.DropHints, o.DropHints) && slices.Equal(s.Layers, o.Layers) &&
-		slices.Equal(s.CaptureIndicators, o.CaptureIndicators)
+		slices.Equal(s.CaptureIndicators, o.CaptureIndicators) &&
+		s.HintStyle == o.HintStyle && slices.Equal(s.HintLabels, o.HintLabels)
 }
 // Shows reports whether the scene draws the surface of id: only its
 func (w SceneWindow) Tile() bool {
 	return !w.Floating && !w.Fullscreen && !w.Popup && w.Preview == 0
 }
+// HintStyle carries reloadable label settings and untransformed output density.
+type HintStyle struct {
+	Size                                float64
+	Background, Foreground              string
+	Width, Height, PhysicalW, PhysicalH int
+}

M internal/ports/scene_same_test.go +4 -2

 		Flash:             &FocusFlash{Rect: Rect{W: 20, H: 10}, Width: 2, Color: "#ff0000", Alpha: 0.6},
 		CaptureIndicators: []CaptureIndicator{{Rect: Rect{W: 4, H: 4}}},
+		HintLabels:        []HintLabel{{Rect: Rect{X: 3, Y: 4}, Text: "fj", Typed: 1}},
 	}
 }
 		"drop hint":         func(s *Scene) { s.DropHints[0].W++ },
 		"layer":             func(s *Scene) { s.Layers[0].Rect.H++ },
 		"indicator":         func(s *Scene) { s.CaptureIndicators[0].Pill = true },
+		"hint style":        func(s *Scene) { s.HintStyle.Size = 24 },
 		"excluded":          func(s *Scene) { s.Capture.Excluded[0] = 4 },
 		"transform":         func(s *Scene) { s.Transform = 1 },
 		"capture shown":     func(s *Scene) { s.Capture.Shown++ },
 // TestSceneSameAsCoversEveryField fails when Scene gains or loses a field:
 // update Scene.SameAs, then this count.
 func TestSceneSameAsCoversEveryField(t *testing.T) {
-	// 23 with Animating, which SameAs intentionally does not compare: it
+	// 25 with Animating, which SameAs intentionally does not compare: it
 	// only schedules the next frame, it draws nothing.
-	const fields = 23
+	const fields = 25
 	assert.Equal(t, fields, len(reflect.VisibleFields(reflect.TypeOf(Scene{}))),
 		"Scene fields changed: update Scene.SameAs (and TestSceneSameAs), then this count")
 }