detached 596b553

feat(hints): detect targets with the nefer-hints subprocess

Hint mode now gets its targets only from nefer-hints, a persistent
detector subprocess (binary protocol over stdin/stdout, lazy start,
restart after a failure). The OCR and AT-SPI sources, the text-click
modifiers and hint.targets/hint.text-click are removed. New keys:
hint.detector, hint.model, hint.threshold.
commit
596b55394e36120807d399835617c4645cfd22a3
author
brice · · 2026-10-21 13:28 UTC
parent
e0a8611
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
39 files · +1011 -4831
Split
UnifiedSplit
M .mockery.yml +2 -19
@@ -113,28 +113,11 @@ packages:
113 filename: proc_tree_mock_test.go113 filename: proc_tree_mock_test.go
114 github.com/bnema/neferwl/internal/adapters/hints:114 github.com/bnema/neferwl/internal/adapters/hints:
115 interfaces:115 interfaces:
116- languageRunner:116+ detector:
117 config:117 config:
118 dir: internal/adapters/hints118 dir: internal/adapters/hints
119 pkgname: hints119 pkgname: hints
120- filename: language_runner_mock_test.go120+ filename: detector_mock_test.go
121- targetSource:
122- config:
123- dir: internal/adapters/hints
124- pkgname: hints
125- filename: target_source_mock_test.go
126- github.com/bnema/neferwl/internal/adapters/atspi:
127- interfaces:
128- transport:
129- config:
130- dir: internal/adapters/atspi
131- pkgname: atspi
132- filename: transport_mock_test.go
133- connector:
134- config:
135- dir: internal/adapters/atspi
136- pkgname: atspi
137- filename: connector_mock_test.go
138 github.com/bnema/neferwl/internal/adapters/screencast:121 github.com/bnema/neferwl/internal/adapters/screencast:
139 interfaces:122 interfaces:
140 bus:123 bus:
D internal/adapters/atspi/bus_test.go +0 -197
@@ -1,197 +0,0 @@
1-package atspi
2-
3-import (
4- "bufio"
5- "os"
6- "os/exec"
7- "strings"
8- "testing"
9- "time"
10-
11- "github.com/bnema/zerobus"
12- "github.com/stretchr/testify/require"
13-)
14-
15-// startBus starts a private dbus-daemon, so the test never touches the
16-// desktop's buses. It skips when the daemon is not installed.
17-func startBus(t *testing.T) string {
18- t.Helper()
19- bin, err := exec.LookPath("dbus-daemon")
20- if err != nil {
21- if os.Getenv("CI") != "" {
22- t.Fatal("dbus-daemon not installed")
23- }
24- t.Skip("dbus-daemon not installed")
25- }
26- cmd := exec.Command(bin, "--session", "--nofork", "--nopidfile", "--print-address=1")
27- out, err := cmd.StdoutPipe()
28- require.NoError(t, err)
29- require.NoError(t, cmd.Start())
30- t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() })
31- line, err := bufio.NewReader(out).ReadString('\n')
32- require.NoError(t, err)
33- return strings.TrimSpace(line)
34-}
35-
36-// TestDoDecodesMatchesPipelined runs Do against an app on a real bus: the
37-// GetMatches reply is decoded, and replies to several calls sent before any
38-// is read are matched by serial.
39-func TestDoDecodesMatchesPipelined(t *testing.T) {
40- addr := startBus(t)
41- app, err := zerobus.Dial(addr)
42- require.NoError(t, err)
43- t.Cleanup(func() { _ = app.Close() })
44- _, err = app.RequestName("org.test.App", 0)
45- require.NoError(t, err)
46-
47- done := make(chan struct{})
48- go func() {
49- defer close(done)
50- for {
51- m, err := app.ReadMessage()
52- if err != nil {
53- return
54- }
55- if m.Type != zerobus.TypeMethodCall {
56- continue
57- }
58- req := &zerobus.Message{Sender: strings.Clone(m.Sender), Serial: m.Serial}
59- switch m.Member {
60- case "GetMatches":
61- // Check the rule: struct, then sort, count, traverse.
62- r := m.Body()
63- r.Struct()
64- end := r.Array('i')
65- var states []int32
66- for r.More(end) {
67- states = append(states, r.Int32())
68- }
69- _ = r.Int32()
70- ad := r.Array('{')
71- _ = ad
72- _ = r.Int32()
73- end = r.Array('i')
74- var roles []int32
75- for r.More(end) {
76- roles = append(roles, r.Int32())
77- }
78- ok := r.Err() == nil && len(states) == 2 && states[0] == 1<<25|1<<30 && len(roles) == 5 && uint32(roles[2]) == 1<<24|1<<18|1<<31
79- if !ok {
80- app.NewError(req, "org.test.Error.BadRule", "s").Str("bad rule")
81- _, _ = app.Send()
82- continue
83- }
84- e := app.NewReply(req, "a(so)")
85- a := e.BeginArray('(')
86- for _, p := range []string{"/one", "/two"} {
87- e.Struct()
88- e.Str(":1.99")
89- e.ObjectPath(p)
90- }
91- e.EndArray(a)
92- case "GetExtents":
93- e := app.NewReply(req, "(iiii)")
94- e.Struct()
95- e.Int32(1)
96- e.Int32(2)
97- e.Int32(30)
98- e.Int32(40)
99- case "GetRole":
100- app.NewReply(req, "u").Uint32(roleLink)
101- case "GetState":
102- e := app.NewReply(req, "au")
103- a := e.BeginArray('u')
104- e.Uint32(stShowing)
105- e.Uint32(0)
106- e.EndArray(a)
107- default:
108- app.NewError(req, "org.freedesktop.DBus.Error.UnknownMethod", "s").Str("no")
109- }
110- _, _ = app.Send()
111- }
112- }()
113-
114- cc, err := zerobus.Dial(addr)
115- require.NoError(t, err)
116- b, err := newConn(cc)
117- require.NoError(t, err)
118- t.Cleanup(func() { _ = b.Close() })
119-
120- calls := []call{
121- {op: opMatches, name: "org.test.App", path: "/frame", rule: matchRule{linksOnly: true}},
122- {op: opExtents, name: "org.test.App", path: "/one", arg: coordWindow},
123- {op: opRole, name: "org.test.App", path: "/one"},
124- {op: opState, name: "org.test.App", path: "/one"},
125- {op: opChildren, name: "org.test.App", path: "/one"},
126- }
127- rep := b.Do(time.Now().Add(2*time.Second), time.Second, calls)
128- require.Len(t, rep, 5)
129- require.NoError(t, rep[0].err)
130- require.Equal(t, []ref{{":1.99", "/one"}, {":1.99", "/two"}}, rep[0].refs)
131- require.NoError(t, rep[1].err)
132- require.Equal(t, [4]int32{1, 2, 30, 40}, rep[1].rect)
133- require.Equal(t, uint32(roleLink), rep[2].u)
134- require.Equal(t, uint64(stShowing), rep[3].state)
135- var be *busError
136- require.ErrorAs(t, rep[4].err, &be)
137- require.Equal(t, "org.freedesktop.DBus.Error.UnknownMethod", be.name)
138-
139- // A call to a name nobody owns fails fast, others still succeed.
140- rep = b.Do(time.Now().Add(2*time.Second), time.Second, []call{
141- {op: opRole, name: "org.test.Nobody", path: "/x"},
142- {op: opRole, name: "org.test.App", path: "/x"},
143- })
144- require.Error(t, rep[0].err)
145- require.NoError(t, rep[1].err)
146-}
147-
148-func TestDoTimesOutOnSilentApp(t *testing.T) {
149- addr := startBus(t)
150- app, err := zerobus.Dial(addr)
151- require.NoError(t, err)
152- t.Cleanup(func() { _ = app.Close() })
153- _, err = app.RequestName("org.test.Mute", 0)
154- require.NoError(t, err)
155-
156- cc, err := zerobus.Dial(addr)
157- require.NoError(t, err)
158- b, err := newConn(cc)
159- require.NoError(t, err)
160- t.Cleanup(func() { _ = b.Close() })
161-
162- t0 := time.Now()
163- rep := b.Do(time.Now().Add(time.Second), 50*time.Millisecond, []call{{op: opRole, name: "org.test.Mute", path: "/x"}})
164- require.ErrorIs(t, rep[0].err, ErrTimeout)
165- require.Less(t, time.Since(t0), 500*time.Millisecond)
166-}
167-
168-// A call left unanswered for idle gets ErrTimeout without breaking the
169-// connection; a batch that hits its deadline breaks it, and Do returns near
170-// the deadline even though the idle timeout is longer.
171-func TestDoDeadlineBreaksConnection(t *testing.T) {
172- addr := startBus(t)
173- app, err := zerobus.Dial(addr)
174- require.NoError(t, err)
175- t.Cleanup(func() { _ = app.Close() })
176- _, err = app.RequestName("org.test.Mute", 0)
177- require.NoError(t, err)
178-
179- cc, err := zerobus.Dial(addr)
180- require.NoError(t, err)
181- b, err := newConn(cc)
182- require.NoError(t, err)
183- t.Cleanup(func() { _ = b.Close() })
184-
185- mute := []call{{op: opRole, name: "org.test.Mute", path: "/x"}}
186- rep := b.Do(time.Now().Add(time.Second), 40*time.Millisecond, mute)
187- require.ErrorIs(t, rep[0].err, ErrTimeout)
188- require.NotErrorIs(t, rep[0].err, ErrBroken)
189-
190- t0 := time.Now()
191- rep = b.Do(time.Now().Add(60*time.Millisecond), time.Second, mute)
192- require.ErrorIs(t, rep[0].err, ErrBroken)
193- require.Less(t, time.Since(t0), 300*time.Millisecond)
194- // The socket is closed: the next batch fails at once.
195- rep = b.Do(time.Now().Add(time.Second), time.Second, mute)
196- require.ErrorIs(t, rep[0].err, ErrBroken)
197-}
D internal/adapters/atspi/connector_mock_test.go +0 -109
@@ -1,109 +0,0 @@
1-// Code generated by mockery; DO NOT EDIT.
2-// github.com/vektra/mockery
3-// template: testify
4-
5-package atspi
6-
7-import (
8- "context"
9-
10- mock "github.com/stretchr/testify/mock"
11-)
12-
13-// newMockconnector creates a new instance of mockconnector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
14-// The first argument is typically a *testing.T value.
15-func newMockconnector(t interface {
16- mock.TestingT
17- Cleanup(func())
18-}) *mockconnector {
19- if helper, ok := t.(interface{ Helper() }); ok {
20- helper.Helper()
21- }
22-
23- mock := &mockconnector{}
24- mock.Mock.Test(t)
25-
26- t.Cleanup(func() {
27- if helper, ok := t.(interface{ Helper() }); ok {
28- helper.Helper()
29- }
30- mock.AssertExpectations(t)
31- })
32-
33- return mock
34-}
35-
36-// mockconnector is an autogenerated mock type for the connector type
37-type mockconnector struct {
38- mock.Mock
39-}
40-
41-type mockconnector_Expecter struct {
42- mock *mock.Mock
43-}
44-
45-func (_m *mockconnector) EXPECT() *mockconnector_Expecter {
46- return &mockconnector_Expecter{mock: &_m.Mock}
47-}
48-
49-// Connect provides a mock function for the type mockconnector
50-func (_mock *mockconnector) Connect(ctx context.Context) (transport, error) {
51- ret := _mock.Called(ctx)
52-
53- if len(ret) == 0 {
54- panic("no return value specified for Connect")
55- }
56-
57- var r0 transport
58- var r1 error
59- if returnFunc, ok := ret.Get(0).(func(context.Context) (transport, error)); ok {
60- return returnFunc(ctx)
61- }
62- if returnFunc, ok := ret.Get(0).(func(context.Context) transport); ok {
63- r0 = returnFunc(ctx)
64- } else {
65- if ret.Get(0) != nil {
66- r0 = ret.Get(0).(transport)
67- }
68- }
69- if returnFunc, ok := ret.Get(1).(func(context.Context) error); ok {
70- r1 = returnFunc(ctx)
71- } else {
72- r1 = ret.Error(1)
73- }
74- return r0, r1
75-}
76-
77-// mockconnector_Connect_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Connect'
78-type mockconnector_Connect_Call struct {
79- *mock.Call
80-}
81-
82-// Connect is a helper method to define mock.On call
83-// - ctx context.Context
84-func (_e *mockconnector_Expecter) Connect(ctx any) *mockconnector_Connect_Call {
85- return &mockconnector_Connect_Call{Call: _e.mock.On("Connect", ctx)}
86-}
87-
88-func (_c *mockconnector_Connect_Call) Run(run func(ctx context.Context)) *mockconnector_Connect_Call {
89- _c.Call.Run(func(args mock.Arguments) {
90- var arg0 context.Context
91- if args[0] != nil {
92- arg0 = args[0].(context.Context)
93- }
94- run(
95- arg0,
96- )
97- })
98- return _c
99-}
100-
101-func (_c *mockconnector_Connect_Call) Return(transportMoqParam transport, err error) *mockconnector_Connect_Call {
102- _c.Call.Return(transportMoqParam, err)
103- return _c
104-}
105-
106-func (_c *mockconnector_Connect_Call) RunAndReturn(run func(ctx context.Context) (transport, error)) *mockconnector_Connect_Call {
107- _c.Call.Return(run)
108- return _c
109-}
D internal/adapters/atspi/doc.go +0 -10
@@ -1,10 +0,0 @@
1-// Package atspi finds hint targets in the accessibility tree of the apps on
2-// screen. It talks to the AT-SPI registry over the accessibility bus with
3-// zerobus, from one connection used by one goroutine at a time, and pipelines
4-// the calls it needs: several are sent before their replies are read, and the
5-// replies are matched by serial.
6-//
7-// The approach (Collection.GetMatches on the window frame, bounded walk when
8-// the toolkit has no Collection, window-relative extents) is adapted from
9-// neru's AT-SPI client, https://github.com/y3owk1n/neru (MIT licence).
10-package atspi
D internal/adapters/atspi/model.go +0 -206
@@ -1,206 +0,0 @@
1-package atspi
2-
3-import "github.com/bnema/neferwl/internal/ports"
4-
5-// AT-SPI constants (atspi-constants.h). The role ids are the AtspiRole enum
6-// values; the state ids are bit indices of the state set.
7-const (
8- roleCheckBox = 7
9- roleComboBox = 11
10- roleDialog = 16
11- roleFrame = 23
12- roleMenuItem = 35
13- rolePageTab = 37
14- rolePushButton = 43
15- roleRadioButton = 44
16- roleToggleButton = 62
17- roleWindow = 69
18- roleLink = 88
19- roleDocumentFrame = 82
20- roleDocumentWeb = 95
21-
22- stateActive = 1
23- stateShowing = 25
24- stateVisible = 30
25-
26- roleWords = 5 // int32 words of a role set in a MatchRule
27- stateWords = 2 // int32 words of a state set in a MatchRule
28-
29- // AtspiCollectionMatchType and AtspiCollectionSortOrder values.
30- matchAll = 1
31- matchAny = 2
32- sortCanonical = 1
33- matchCount = 500
34- maxDepth = 30
35- maxWalkNodes = 3000
36- maxFrames = 32
37- sizeTolerance = 40
38- maxMatchesRead = matchCount
39-)
40-
41-// clickableRoles are the roles that become hint targets.
42-var clickableRoles = []uint32{
43- roleLink, rolePushButton, roleToggleButton, roleCheckBox,
44- roleRadioButton, roleMenuItem, rolePageTab, roleComboBox,
45-}
46-
47-// matchRule is the Collection query: showing and visible objects of the
48-// clickable roles (only links when linksOnly), plus web documents for coverage.
49-type matchRule struct{ linksOnly bool }
50-
51-func (r matchRule) roles() []uint32 {
52- if r.linksOnly {
53- return []uint32{roleLink, roleDocumentFrame, roleDocumentWeb}
54- }
55- return append(append([]uint32(nil), clickableRoles...), roleDocumentFrame, roleDocumentWeb)
56-}
57-
58-// roleBits packs role ids into the bit set of a MatchRule: bit id%32 of word
59-// id/32.
60-func roleBits(ids []uint32) []int32 {
61- words := make([]int32, roleWords)
62- for _, id := range ids {
63- if w := int(id / 32); w < len(words) {
64- words[w] |= int32(uint32(1) << (id % 32))
65- }
66- }
67- return words
68-}
69-
70-// stateBits packs state ids into the bit set of a MatchRule.
71-func stateBits(ids ...uint32) []int32 {
72- words := make([]int32, stateWords)
73- for _, id := range ids {
74- if w := int(id / 32); w < len(words) {
75- words[w] |= int32(uint32(1) << (id % 32))
76- }
77- }
78- return words
79-}
80-
81-func hasState(set uint64, state uint) bool { return set&(1<<state) != 0 }
82-
83-// isClickable reports whether role makes a target.
84-func isClickable(role uint32, linksOnly bool) bool {
85- if linksOnly {
86- return role == roleLink
87- }
88- for _, r := range clickableRoles {
89- if r == role {
90- return true
91- }
92- }
93- return false
94-}
95-
96-// isWindowRole reports whether role is a top-level window role.
97-func isWindowRole(role uint32) bool {
98- return role == roleFrame || role == roleWindow || role == roleDialog
99-}
100-
101-// kindOf maps a role to the hint kind: links are links, the rest buttons.
102-func kindOf(role uint32) ports.HintKind {
103- if role == roleLink {
104- return ports.HintLink
105- }
106- return ports.HintButton
107-}
108-
109-// frameInfo is a showing top-level window of an app.
110-type frameInfo struct {
111- ref ref
112- active bool
113- w, h int // size of the frame, from its window extents
114-}
115-
116-// matchFrames pairs the windows of one app with its frames, one to one. A
117-// window is paired with a frame when the pairing is certain:
118-// - one window and one frame;
119-// - the focused window with the one ACTIVE frame; that frame is then
120-// reserved and no other window can take it;
121-// - otherwise the one unreserved frame whose size is within sizeTolerance
122-// of the window, provided no other unmatched window claims it as well.
123-//
124-// Ambiguous windows are left out. The result maps the index of a window in
125-// wins to the index of its frame in frames.
126-func matchFrames(wins []ports.HintWindow, frames []frameInfo) map[int]int {
127- out := map[int]int{}
128- if len(wins) == 1 && len(frames) == 1 {
129- out[0] = 0
130- return out
131- }
132- // Active matches first: they are reserved against the size matches.
133- active, n := -1, 0
134- for j, f := range frames {
135- if f.active {
136- active, n = j, n+1
137- }
138- }
139- reserved := make([]bool, len(frames))
140- done := make([]bool, len(wins))
141- if n == 1 {
142- focused := -1
143- count := 0
144- for i, w := range wins {
145- if w.Focused {
146- focused, count = i, count+1
147- }
148- }
149- if count == 1 {
150- out[focused] = active
151- reserved[active] = true
152- done[focused] = true
153- }
154- }
155- claims := make([][]int, len(wins)) // size candidates per window
156- owners := make([]int, len(frames)) // windows that size-match a frame
157- for i, w := range wins {
158- if done[i] {
159- continue
160- }
161- for j, f := range frames {
162- if !reserved[j] && absInt(f.w-w.Rect.W) <= sizeTolerance && absInt(f.h-w.Rect.H) <= sizeTolerance {
163- claims[i] = append(claims[i], j)
164- owners[j]++
165- }
166- }
167- }
168- for i := range wins {
169- if !done[i] && len(claims[i]) == 1 && owners[claims[i][0]] == 1 {
170- out[i] = claims[i][0]
171- }
172- }
173- return out
174-}
175-
176-func absInt(v int) int {
177- if v < 0 {
178- return -v
179- }
180- return v
181-}
182-
183-// element is a match with its extents.
184-type element struct {
185- role uint32
186- x, y, w, h int32 // window-relative
187-}
188-
189-// toTarget converts an element of a window into a logical output-local
190-// target: the element extents are relative to the top-level frame, whose own
191-// window-relative origin is (fx, fy); the result is clipped to the window.
192-// It reports false for an empty result.
193-func toTarget(win ports.Rect, fx, fy int32, e element) (ports.HintTarget, bool) {
194- if e.w <= 0 || e.h <= 0 {
195- return ports.HintTarget{}, false
196- }
197- r := ports.Rect{X: win.X + int(e.x-fx), Y: win.Y + int(e.y-fy), W: int(e.w), H: int(e.h)}
198- x0, y0 := max(r.X, win.X), max(r.Y, win.Y)
199- x1, y1 := min(r.X+r.W, win.X+win.W), min(r.Y+r.H, win.Y+win.H)
200- if x1 <= x0 || y1 <= y0 {
201- return ports.HintTarget{}, false
202- }
203- return ports.HintTarget{Rect: ports.Rect{X: x0, Y: y0, W: x1 - x0, H: y1 - y0}, Kind: kindOf(e.role)}, true
204-}
205-
206-func isDocument(role uint32) bool { return role == roleDocumentFrame || role == roleDocumentWeb }
D internal/adapters/atspi/model_test.go +0 -117
@@ -1,117 +0,0 @@
1-package atspi
2-
3-import (
4- "testing"
5-
6- "github.com/stretchr/testify/require"
7-
8- "github.com/bnema/neferwl/internal/ports"
9-)
10-
11-func TestToTargetConvertsWindowCoordinates(t *testing.T) {
12- win := ports.Rect{X: 100, Y: 50, W: 800, H: 600}
13- // The frame origin (fx, fy) is subtracted: toolkit offsets cancel out.
14- got, ok := toTarget(win, 10, 20, element{role: roleLink, x: 110, y: 120, w: 40, h: 16})
15- require.True(t, ok)
16- require.Equal(t, ports.HintTarget{Rect: ports.Rect{X: 200, Y: 150, W: 40, H: 16}, Kind: ports.HintLink}, got)
17-}
18-
19-func TestToTargetClipsToWindow(t *testing.T) {
20- win := ports.Rect{X: 100, Y: 50, W: 200, H: 100}
21- got, ok := toTarget(win, 0, 0, element{role: rolePushButton, x: 180, y: 90, w: 60, h: 30})
22- require.True(t, ok)
23- require.Equal(t, ports.Rect{X: 280, Y: 140, W: 20, H: 10}, got.Rect)
24- require.Equal(t, ports.HintButton, got.Kind)
25-}
26-
27-func TestToTargetDropsEmptyAndOutside(t *testing.T) {
28- win := ports.Rect{X: 0, Y: 0, W: 100, H: 100}
29- for name, e := range map[string]element{
30- "zero width": {role: roleLink, x: 1, y: 1, w: 0, h: 10},
31- "neg height": {role: roleLink, x: 1, y: 1, w: 10, h: -1},
32- "outside": {role: roleLink, x: 500, y: 1, w: 10, h: 10},
33- "above": {role: roleLink, x: 1, y: -50, w: 10, h: 10},
34- "left of win": {role: roleLink, x: -30, y: 1, w: 10, h: 10},
35- } {
36- _, ok := toTarget(win, 0, 0, e)
37- require.False(t, ok, name)
38- }
39-}
40-
41-func TestRoleKindMapping(t *testing.T) {
42- require.Equal(t, ports.HintLink, kindOf(roleLink))
43- for _, r := range []uint32{rolePushButton, roleToggleButton, roleCheckBox, roleRadioButton, roleMenuItem, rolePageTab, roleComboBox} {
44- require.Equal(t, ports.HintButton, kindOf(r), r)
45- require.True(t, isClickable(r, false), r)
46- require.False(t, isClickable(r, true), r)
47- }
48- require.True(t, isClickable(roleLink, true))
49- require.False(t, isClickable(roleFrame, false))
50- require.True(t, isWindowRole(roleFrame) && isWindowRole(roleWindow) && isWindowRole(roleDialog))
51- require.False(t, isWindowRole(roleLink))
52-}
53-
54-func TestRoleAndStateBits(t *testing.T) {
55- require.Equal(t, []int32{0, 0, 0, 0, 0}, roleBits(nil))
56- // link is role 88: word 2, bit 24; push button is 43: word 1, bit 11.
57- require.Equal(t, []int32{0, 1 << 11, 1 << 24, 0, 0}, roleBits([]uint32{roleLink, rolePushButton}))
58- // showing is state 25, visible 30: both in word 0.
59- require.Equal(t, []int32{1<<25 | 1<<30, 0}, stateBits(stateShowing, stateVisible))
60-}
61-
62-func wins(ws ...ports.HintWindow) []ports.HintWindow { return ws }
63-
64-func win(id ports.WindowID, w, h int, focused bool) ports.HintWindow {
65- return ports.HintWindow{ID: id, PID: 1, Rect: ports.Rect{W: w, H: h}, Focused: focused}
66-}
67-
68-func TestMatchFramesSingle(t *testing.T) {
69- got := matchFrames(wins(win(1, 800, 600, false)), []frameInfo{{w: 100, h: 100}})
70- require.Equal(t, map[int]int{0: 0}, got)
71-}
72-
73-func TestMatchFramesActive(t *testing.T) {
74- frames := []frameInfo{{w: 800, h: 600}, {w: 800, h: 600, active: true}}
75- got := matchFrames(wins(win(1, 800, 600, true), win(2, 800, 600, false)), frames)
76- // The focused window takes the ACTIVE frame and reserves it; the other
77- // window then only has the remaining frame to match.
78- require.Equal(t, map[int]int{0: 1, 1: 0}, got)
79-}
80-
81-func TestMatchFramesSize(t *testing.T) {
82- frames := []frameInfo{{w: 1000, h: 700}, {w: 400, h: 300}}
83- got := matchFrames(wins(win(1, 1000, 700, false), win(2, 420, 290, false)), frames)
84- require.Equal(t, map[int]int{0: 0, 1: 1}, got)
85-}
86-
87-func TestMatchFramesSizeTolerance(t *testing.T) {
88- frames := []frameInfo{{w: 840, h: 640}, {w: 100, h: 100}}
89- other := win(2, 90, 90, false)
90- // 40 px off still matches; 41 px does not.
91- require.Equal(t, map[int]int{0: 0, 1: 1}, matchFrames(wins(win(1, 800, 600, false), other), frames))
92- require.Equal(t, map[int]int{1: 1}, matchFrames(wins(win(1, 799, 600, false), other), frames))
93-}
94-
95-func TestMatchFramesAmbiguous(t *testing.T) {
96- // Two windows and two frames of the same size, none active: skipped.
97- frames := []frameInfo{{w: 800, h: 600}, {w: 810, h: 600}}
98- got := matchFrames(wins(win(1, 800, 600, false), win(2, 800, 600, false)), frames)
99- require.Empty(t, got)
100- // One window, two same-size frames, not focused: skipped.
101- require.Empty(t, matchFrames(wins(win(1, 800, 600, false)), frames))
102- // No frame close enough.
103- require.Empty(t, matchFrames(wins(win(1, 200, 200, false), win(2, 300, 300, false)), frames))
104-}
105-
106-// The active frame is reserved by the focused window: an unfocused window of
107-// the same size as that frame must not be paired with it as well.
108-func TestMatchFramesOneToOne(t *testing.T) {
109- frames := []frameInfo{{ref: ref{"a", "/active"}, active: true, w: 400, h: 300}, {ref: ref{"a", "/idle"}, w: 800, h: 600}}
110- got := matchFrames(wins(win(1, 800, 600, true), win(2, 400, 300, false)), frames)
111- seen := map[int]bool{}
112- for _, f := range got {
113- require.False(t, seen[f], "frame %d assigned twice: %v", f, got)
114- seen[f] = true
115- }
116- require.Equal(t, map[int]int{0: 0}, got)
117-}
D internal/adapters/atspi/source.go +0 -481
@@ -1,481 +0,0 @@
1-package atspi
2-
3-import (
4- "context"
5- "errors"
6- "fmt"
7- "strings"
8- "time"
9-
10- "github.com/bnema/zerobus"
11-
12- "github.com/bnema/neferwl/internal/adapters/logging"
13- "github.com/bnema/neferwl/internal/ports"
14-)
15-
16-const (
17- // callTimeout is how long a batch waits for the next reply.
18- callTimeout = 100 * time.Millisecond
19- // budget is the whole time Targets may take.
20- budget = 300 * time.Millisecond
21-)
22-
23-// connectBudget bounds the whole connection setup (session bus,
24-// GetAddress, IsEnabled, dial of the accessibility bus) of one attempt.
25-const connectBudget = 300 * time.Millisecond
26-
27-// retryEvery is the minimum time between two connection attempts.
28-const retryEvery = 5 * time.Second
29-
30-// connector opens a transport on the accessibility bus.
31-type connector interface {
32- Connect(ctx context.Context) (transport, error)
33-}
34-
35-// stallWarnAfter is how long an attempt may be in flight before it is
36-// reported as a stalled accessibility bus.
37-const stallWarnAfter = 5 * time.Second
38-
39-// dialResult is the outcome of a connection attempt.
40-type dialResult struct {
41- tr transport
42- err error
43- at time.Time // when the attempt finished
44-}
45-
46-// Source reads hint targets from the accessibility tree. It owns one
47-// connection to the accessibility bus. It is NOT safe for concurrent use:
48-// it is owned by the hints worker goroutine, which calls Targets and,
49-// when it returns, Close. The connection is made on the first Targets call
50-// that needs it, never at startup, and made again after a failure, at most
51-// once every retryEvery; meanwhile Targets finds nothing and OCR takes over.
52-//
53-// At most one connection attempt (one goroutine and one socket) exists at
54-// any time: an attempt that outlives the wait of ensure stays in flight, is
55-// polled by the later Targets calls and is never doubled.
56-type Source struct {
57- tr transport
58- dial connector
59- // attempt delivers the outcome of the attempt in flight (buffered, so
60- // the attempt goroutine never blocks); nil when none runs.
61- attempt chan dialResult
62- attemptStarted time.Time
63- warnedStall bool
64- retryAt time.Time
65- closed bool
66- // wait is how long ensure waits for a new attempt; connectBudget if 0.
67- wait time.Duration
68-}
69-
70-// New returns a Source. It does not touch the bus: the connection is made by
71-// the first Targets call.
72-func New() *Source { return &Source{dial: busConnector{}} }
73-
74-// busConnector connects to the session bus, enables accessibility and dials
75-// the accessibility bus.
76-type busConnector struct{}
77-
78-func (busConnector) Connect(ctx context.Context) (transport, error) {
79- log := logging.For(ctx, "atspi")
80- // zerobus has no deadline or context API: SessionBus and Dial run the
81- // socket dial, SASL auth and Hello without a bound, and the watchdog
82- // below cannot exist before the connection does. A stall there is
83- // bounded by the single-in-flight rule of Source.ensure, not by a
84- // deadline. Follow-up: a deadline-capable zerobus API.
85- sess, err := zerobus.SessionBus()
86- if err != nil {
87- return nil, fmt.Errorf("atspi: session bus: %w", err)
88- }
89- defer func() { _ = sess.Close() }()
90- // Call has no deadline: closing the connection ends the wait for a
91- // silent service.
92- stop := time.AfterFunc(connectBudget, func() { _ = sess.Close() })
93- defer stop.Stop()
94-
95- sess.NewCall(a11yBusName, a11yBusPath, a11yBusName, "GetAddress", "")
96- m, err := sess.Call()
97- if err != nil {
98- return nil, fmt.Errorf("atspi: org.a11y.Bus GetAddress: %w", err)
99- }
100- r := m.Body()
101- addr := r.Str()
102- if r.Err() != nil || addr == "" {
103- return nil, errors.New("atspi: no accessibility bus address")
104- }
105- addr = strings.Clone(addr)
106-
107- e := sess.NewCall(a11yBusName, a11yBusPath, ifaceProps, "Set", "ssv")
108- e.Str(a11yStatus)
109- e.Str("IsEnabled")
110- e.Variant("b")
111- e.Bool(true)
112- if _, err := sess.Call(); err != nil {
113- // Toolkits that already see the bus keep working.
114- log.Warn().Err(err).Msg("could not enable accessibility")
115- } else {
116- log.Info().Msg("accessibility enabled")
117- }
118-
119- zc, err := zerobus.Dial(addr)
120- if err != nil {
121- return nil, fmt.Errorf("atspi: accessibility bus: %w", err)
122- }
123- tr, err := newConn(zc)
124- if err != nil {
125- return nil, err
126- }
127- log.Info().Msg("accessibility bus connected")
128- return tr, nil
129-}
130-
131-// ensure returns the connection, making one if none exists and the last
132-// attempt is old enough. It waits at most the budget for a new attempt; an
133-// attempt that does not answer in time stays in flight (abandoned, never
134-// doubled) and is polled without blocking by the next calls.
135-func (s *Source) ensure(ctx context.Context) transport {
136- if s.closed {
137- return nil
138- }
139- if s.tr != nil {
140- return s.tr
141- }
142- if s.dial == nil {
143- return nil
144- }
145- now := time.Now()
146- if s.attempt != nil {
147- select {
148- case r := <-s.attempt:
149- s.finish(ctx, r)
150- default:
151- if !s.warnedStall && now.Sub(s.attemptStarted) > stallWarnAfter {
152- s.warnedStall = true
153- log := logging.For(ctx, "atspi")
154- log.Warn().Dur("for", now.Sub(s.attemptStarted)).Msg("accessibility bus not answering")
155- }
156- }
157- return s.tr
158- }
159- if now.Before(s.retryAt) {
160- return nil
161- }
162- ch := make(chan dialResult, 1)
163- s.attempt = ch
164- s.attemptStarted = now
165- s.warnedStall = false
166- dial := s.dial
167- go func() {
168- tr, err := dial.Connect(ctx)
169- ch <- dialResult{tr: tr, err: err, at: time.Now()}
170- }()
171- wait := s.wait
172- if wait <= 0 {
173- wait = connectBudget
174- }
175- timer := time.NewTimer(wait)
176- defer timer.Stop()
177- select {
178- case r := <-ch:
179- s.finish(ctx, r)
180- case <-timer.C:
181- log := logging.For(ctx, "atspi")
182- log.Debug().Dur("after", wait).Msg("accessibility connect attempt abandoned in flight")
183- case <-ctx.Done():
184- }
185- return s.tr
186-}
187-
188-// finish takes the outcome of the attempt in flight: a transport is kept,
189-// an error starts the backoff from the time the attempt finished.
190-func (s *Source) finish(ctx context.Context, r dialResult) {
191- s.attempt = nil
192- if r.err != nil || r.tr == nil {
193- s.retryAt = r.at.Add(retryEvery)
194- err := r.err
195- if err == nil {
196- err = errors.New("atspi: no transport")
197- }
198- log := logging.For(ctx, "atspi")
199- log.Warn().Err(err).Msg("accessibility hint targets unavailable, using OCR only")
200- return
201- }
202- s.tr = r.tr
203-}
204-
205-// drop closes a broken connection; the next attempt reconnects.
206-func (s *Source) drop() {
207- s.retryAt = time.Now().Add(retryEvery)
208- if s.tr != nil {
209- _ = s.tr.Close()
210- s.tr = nil
211- }
212-}
213-
214-// Close closes the accessibility bus connection, and the one an attempt
215-// still in flight may deliver late. After Close, Targets finds nothing and
216-// never connects. Like Targets it is called by the owner goroutine only.
217-func (s *Source) Close() error {
218- if s.closed {
219- return nil
220- }
221- s.closed = true
222- var err error
223- if s.tr != nil {
224- err = s.tr.Close()
225- s.tr = nil
226- }
227- if ch := s.attempt; ch != nil {
228- s.attempt = nil
229- // One reaper at most: Close runs once, and one attempt exists.
230- go func() {
231- if r := <-ch; r.tr != nil {
232- _ = r.tr.Close()
233- }
234- }()
235- }
236- return err
237-}
238-
239-// appInfo is an application of the registry whose process shows a window.
240-type appInfo struct {
241- root ref
242- pid int
243-}
244-
245-// winMatch is a window with the frame it was paired with.
246-type winMatch struct {
247- win ports.HintWindow
248- frame ref
249- fx int32
250- fy int32
251-}
252-
253-// Targets returns the clickable targets of the accessibility tree for
254-// windows (logical output-local, clipped to each window) and the set of
255-// windows those targets cover. A window is covered when its tree answered
256-// with a link target or web document; uncovered windows also get OCR. It
257-// returns what is ready within the budget.
258-func (s *Source) Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error) {
259- covered := map[ports.WindowID]bool{}
260- deadline := time.Now().Add(budget)
261- if d, ok := ctx.Deadline(); ok && d.Before(deadline) {
262- deadline = d
263- }
264- wanted := map[int]bool{}
265- for _, w := range windows {
266- if w.PID > 0 {
267- wanted[w.PID] = true
268- }
269- }
270- if len(wanted) == 0 {
271- return nil, covered, nil
272- }
273- tr := s.ensure(ctx)
274- if tr == nil {
275- return nil, covered, nil
276- }
277- broken := false
278- do := func(calls []call) []reply {
279- if len(calls) == 0 || ctx.Err() != nil {
280- return make([]reply, len(calls))
281- }
282- if broken {
283- out := make([]reply, len(calls))
284- for i := range out {
285- out[i].err = ErrBroken
286- }
287- return out
288- }
289- out := tr.Do(deadline, callTimeout, calls)
290- for _, r := range out {
291- if errors.Is(r.err, ErrBroken) {
292- broken = true
293- s.drop()
294- break
295- }
296- }
297- return out
298- }
299-
300- // Applications of the registry, kept when their process shows a window.
301- rep := do([]call{{op: opChildren, name: registryName, path: registryRoot}})
302- if len(rep) == 0 || rep[0].err != nil {
303- if len(rep) > 0 {
304- return nil, covered, fmt.Errorf("atspi: registry: %w", rep[0].err)
305- }
306- return nil, covered, nil
307- }
308- apps := rep[0].refs
309- calls := make([]call, len(apps))
310- for i, a := range apps {
311- calls[i] = call{op: opPID, name: a.name}
312- }
313- var kept []appInfo
314- for i, r := range do(calls) {
315- if r.err == nil && wanted[int(r.u)] {
316- kept = append(kept, appInfo{root: apps[i], pid: int(r.u)})
317- }
318- }
319-
320- // The showing windows of each kept app.
321- calls = calls[:0]
322- for _, a := range kept {
323- calls = append(calls, call{op: opChildren, name: a.root.name, path: a.root.path})
324- }
325- var cands []ref
326- var candApp []int
327- for i, r := range do(calls) {
328- if r.err != nil {
329- continue
330- }
331- for _, c := range r.refs[:min(len(r.refs), maxFrames)] {
332- cands = append(cands, c)
333- candApp = append(candApp, i)
334- }
335- }
336- calls = calls[:0]
337- for _, c := range cands {
338- calls = append(calls,
339- call{op: opRole, name: c.name, path: c.path},
340- call{op: opState, name: c.name, path: c.path},
341- call{op: opExtents, name: c.name, path: c.path, arg: coordWindow})
342- }
343- info := do(calls)
344- type frame struct {
345- frameInfo
346- pid int
347- fx, fy int32
348- }
349- var frames []frame
350- for i, c := range cands {
351- role, state, ext := info[3*i], info[3*i+1], info[3*i+2]
352- if role.err != nil || state.err != nil || ext.err != nil || !isWindowRole(role.u) || !hasState(state.state, stateShowing) {
353- continue
354- }
355- frames = append(frames, frame{
356- frameInfo: frameInfo{ref: c, active: hasState(state.state, stateActive), w: int(ext.rect[2]), h: int(ext.rect[3])},
357- pid: kept[candApp[i]].pid, fx: ext.rect[0], fy: ext.rect[1],
358- })
359- }
360-
361- // Pair the windows with the frames, process by process.
362- var matched []winMatch
363- for pid := range wanted {
364- var wins []ports.HintWindow
365- for _, w := range windows {
366- if w.PID == pid {
367- wins = append(wins, w)
368- }
369- }
370- var fs []frameInfo
371- var origin []frame
372- for _, f := range frames {
373- if f.pid == pid {
374- fs = append(fs, f.frameInfo)
375- origin = append(origin, f)
376- }
377- }
378- for wi, fi := range matchFrames(wins, fs) {
379- matched = append(matched, winMatch{win: wins[wi], frame: origin[fi].ref, fx: origin[fi].fx, fy: origin[fi].fy})
380- }
381- }
382-
383- targets := s.elements(do, matched, linksOnly, covered)
384- log := logging.For(ctx, "atspi")
385- log.Debug().Int("apps", len(kept)).Int("frames", len(frames)).Int("windows", len(matched)).Int("targets", len(targets)).Msg("atspi targets")
386- return targets, covered, nil
387-}
388-
389-// elements reads the clickable elements of the matched frames.
390-func (s *Source) elements(do func([]call) []reply, matched []winMatch, linksOnly bool, covered map[ports.WindowID]bool) []ports.HintTarget {
391- calls := make([]call, len(matched))
392- for i, m := range matched {
393- calls[i] = call{op: opMatches, name: m.frame.name, path: m.frame.path, rule: matchRule{linksOnly: linksOnly}}
394- }
395- found := make([][]ref, len(matched))
396- var walk []int // frames without Collection
397- for i, r := range do(calls) {
398- if r.err != nil {
399- var be *busError
400- if errors.As(r.err, &be) {
401- walk = append(walk, i)
402- }
403- continue
404- }
405- found[i] = r.refs
406- }
407- for _, i := range walk {
408- found[i] = s.walk(do, matched[i].frame, linksOnly)
409- }
410-
411- calls = calls[:0]
412- var owner []int
413- for i, refs := range found {
414- for _, r := range refs {
415- calls = append(calls,
416- call{op: opRole, name: r.name, path: r.path},
417- call{op: opExtents, name: r.name, path: r.path, arg: coordWindow})
418- owner = append(owner, i)
419- }
420- }
421- info := do(calls)
422- var out []ports.HintTarget
423- for k, i := range owner {
424- role, ext := info[2*k], info[2*k+1]
425- if role.err == nil && isDocument(role.u) {
426- covered[matched[i].win.ID] = true
427- continue
428- }
429- if role.err != nil || ext.err != nil || !isClickable(role.u, linksOnly) {
430- continue
431- }
432- m := matched[i]
433- t, ok := toTarget(m.win.Rect, m.fx, m.fy, element{role: role.u, x: ext.rect[0], y: ext.rect[1], w: ext.rect[2], h: ext.rect[3]})
434- if !ok {
435- continue
436- }
437- t.Window = m.win.ID
438- out = append(out, t)
439- if t.Kind == ports.HintLink {
440- covered[m.win.ID] = true
441- }
442- }
443- return out
444-}
445-
446-// walk is the fallback for toolkits without Collection: a breadth-first
447-// walk of at most maxDepth levels and maxWalkNodes nodes, returning the
448-// clickable nodes.
449-func (s *Source) walk(do func([]call) []reply, root ref, linksOnly bool) []ref {
450- var out []ref
451- level := []ref{root}
452- seen := 0
453- for depth := 0; depth < maxDepth && len(level) > 0 && seen < maxWalkNodes; depth++ {
454- calls := make([]call, 0, 3*len(level))
455- for _, n := range level {
456- calls = append(calls,
457- call{op: opChildren, name: n.name, path: n.path},
458- call{op: opRole, name: n.name, path: n.path},
459- call{op: opState, name: n.name, path: n.path})
460- }
461- rep := do(calls)
462- room := max(0, maxWalkNodes-seen-len(level))
463- next := make([]ref, 0, room)
464- for i, n := range level {
465- ch, role, state := rep[3*i], rep[3*i+1], rep[3*i+2]
466- if state.err == nil && !hasState(state.state, stateShowing) {
467- continue
468- }
469- if role.err == nil && (isClickable(role.u, linksOnly) || isDocument(role.u)) && state.err == nil && hasState(state.state, stateVisible) {
470- out = append(out, n)
471- }
472- if ch.err == nil {
473- take := min(len(ch.refs), room-len(next))
474- next = append(next, ch.refs[:take]...)
475- }
476- }
477- seen += len(level)
478- level = next
479- }
480- return out
481-}
D internal/adapters/atspi/source_test.go +0 -355
@@ -1,355 +0,0 @@
1-package atspi
2-
3-import (
4- "context"
5- "errors"
6- "fmt"
7- "runtime"
8- "testing"
9- "time"
10-
11- "github.com/stretchr/testify/mock"
12- "github.com/stretchr/testify/require"
13-
14- "github.com/bnema/neferwl/internal/ports"
15-)
16-
17-const (
18- stActive = 1 << stateActive
19- stShowing = 1 << stateShowing
20-)
21-
22-// tree answers calls like an app: registry with one app (:1.5, pid 77) with
23-// one showing frame whose links sit at window-relative extents.
24-func answerTree(frameExt [4]int32, matches []ref, ext map[string][4]int32, roles map[string]uint32, noCollection bool) func(time.Time, time.Duration, []call) []reply {
25- return func(_ time.Time, _ time.Duration, calls []call) []reply {
26- out := make([]reply, len(calls))
27- for i, c := range calls {
28- switch {
29- case c.op == opChildren && c.name == registryName:
30- out[i].refs = []ref{{":1.5", "/root"}, {":1.9", "/root"}}
31- case c.op == opPID:
32- out[i].u = map[string]uint32{":1.5": 77, ":1.9": 99}[c.name]
33- case c.op == opChildren && c.path == "/root":
34- out[i].refs = []ref{{":1.5", "/frame"}}
35- case c.op == opRole && c.path == "/frame":
36- out[i].u = roleFrame
37- case c.op == opState && c.path == "/frame":
38- out[i].state = stShowing | stActive
39- case c.op == opExtents && c.path == "/frame":
40- out[i].rect = frameExt
41- case c.op == opMatches:
42- if noCollection {
43- out[i].err = &busError{name: "org.freedesktop.DBus.Error.UnknownMethod"}
44- } else {
45- out[i].refs = matches
46- }
47- case c.op == opChildren && c.path == "/frame":
48- out[i].refs = matches
49- case c.op == opChildren:
50- case c.op == opRole:
51- out[i].u = roles[c.path]
52- case c.op == opState:
53- out[i].state = stShowing | 1<<stateVisible
54- case c.op == opExtents:
55- out[i].rect = ext[c.path]
56- }
57- }
58- return out
59- }
60-}
61-
62-func TestTargetsCoversWindowAndConverts(t *testing.T) {
63- tr := newMocktransport(t)
64- matches := []ref{{":1.5", "/a"}, {":1.5", "/b"}, {":1.5", "/zero"}}
65- tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
66- [4]int32{4, 30, 800, 600}, matches,
67- map[string][4]int32{"/a": {14, 130, 50, 16}, "/b": {104, 230, 20, 20}, "/zero": {5, 5, 0, 0}},
68- map[string]uint32{"/a": roleLink, "/b": rolePushButton, "/zero": roleLink}, false))
69- s := &Source{tr: tr}
70-
71- windows := []ports.HintWindow{
72- {ID: 1, PID: 77, Rect: ports.Rect{X: 100, Y: 200, W: 800, H: 600}, Focused: true},
73- {ID: 2, PID: 5, Rect: ports.Rect{X: 0, Y: 0, W: 50, H: 50}},
74- }
75- targets, covered, err := s.Targets(context.Background(), windows, false)
76- require.NoError(t, err)
77- require.Equal(t, map[ports.WindowID]bool{1: true}, covered)
78- require.Equal(t, []ports.HintTarget{
79- {Rect: ports.Rect{X: 110, Y: 300, W: 50, H: 16}, Kind: ports.HintLink, Window: 1},
80- {Rect: ports.Rect{X: 200, Y: 400, W: 20, H: 20}, Kind: ports.HintButton, Window: 1},
81- }, targets)
82-}
83-
84-func TestTargetsLinksOnlyDropsButtons(t *testing.T) {
85- tr := newMocktransport(t)
86- matches := []ref{{":1.5", "/a"}, {":1.5", "/b"}}
87- tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
88- [4]int32{0, 0, 800, 600}, matches,
89- map[string][4]int32{"/a": {14, 130, 50, 16}, "/b": {104, 230, 20, 20}},
90- map[string]uint32{"/a": roleLink, "/b": rolePushButton}, false))
91- s := &Source{tr: tr}
92- targets, _, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}, true)
93- require.NoError(t, err)
94- require.Len(t, targets, 1)
95- require.Equal(t, ports.HintLink, targets[0].Kind)
96-}
97-
98-func TestTargetsWalkWithoutCollection(t *testing.T) {
99- tr := newMocktransport(t)
100- matches := []ref{{":1.5", "/a"}}
101- tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
102- [4]int32{0, 0, 800, 600}, matches,
103- map[string][4]int32{"/a": {14, 130, 50, 16}},
104- map[string]uint32{"/a": roleLink}, true))
105- s := &Source{tr: tr}
106- targets, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}, false)
107- require.NoError(t, err)
108- require.Len(t, targets, 1)
109- require.True(t, covered[1])
110-}
111-
112-func TestTargetsNoPIDMatchCoversNothing(t *testing.T) {
113- tr := newMocktransport(t)
114- tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree([4]int32{}, nil, nil, nil, false))
115- s := &Source{tr: tr}
116- targets, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 1234, Rect: ports.Rect{W: 8, H: 6}}}, false)
117- require.NoError(t, err)
118- require.Empty(t, targets)
119- require.Empty(t, covered)
120-}
121-
122-func TestTargetsWithoutPIDsAsksNothing(t *testing.T) {
123- s := &Source{tr: newMocktransport(t)}
124- _, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 8, H: 6}}}, false)
125- require.NoError(t, err)
126- require.Empty(t, covered)
127-}
128-
129-func TestCoverageRequiresLinkOrDocument(t *testing.T) {
130- for _, collection := range []bool{true, false} {
131- for _, role := range []uint32{rolePushButton, roleDocumentFrame, roleDocumentWeb, roleLink} {
132- t.Run(fmt.Sprintf("collection=%t/role=%d", collection, role), func(t *testing.T) {
133- tr := newMocktransport(t)
134- tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
135- [4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {10, 20, 30, 40}}, map[string]uint32{"/a": role}, !collection))
136- s := &Source{tr: tr}
137- targets, covered, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}, false)
138- require.NoError(t, err)
139- require.Equal(t, role != rolePushButton, covered[1])
140- if isDocument(role) {
141- require.Empty(t, targets)
142- } else {
143- require.Len(t, targets, 1)
144- }
145- })
146- }
147- }
148-}
149-
150-var oneWindow = []ports.HintWindow{{ID: 1, PID: 77, Rect: ports.Rect{W: 800, H: 600}}}
151-
152-// New does not touch the bus; the first Targets call connects, once.
153-func TestConnectsLazilyOnFirstTargets(t *testing.T) {
154- tr := newMocktransport(t)
155- tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
156- [4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {1, 1, 9, 9}}, map[string]uint32{"/a": roleLink}, false))
157- c := newMockconnector(t)
158- c.EXPECT().Connect(mock.Anything).Return(tr, nil).Once()
159- s := &Source{dial: c}
160- for range 2 {
161- targets, _, err := s.Targets(context.Background(), oneWindow, false)
162- require.NoError(t, err)
163- require.Len(t, targets, 1)
164- }
165-}
166-
167-// Without PIDs nothing is asked and nothing is connected.
168-func TestNoConnectWithoutWindows(t *testing.T) {
169- s := &Source{dial: newMockconnector(t)}
170- _, _, err := s.Targets(context.Background(), []ports.HintWindow{{ID: 1}}, false)
171- require.NoError(t, err)
172-}
173-
174-// A failed connection is retried no sooner than retryEvery; meanwhile no
175-// targets are returned.
176-func TestConnectRetriesWithBackoff(t *testing.T) {
177- c := newMockconnector(t)
178- c.EXPECT().Connect(mock.Anything).Return(nil, errors.New("down")).Once()
179- s := &Source{dial: c}
180- for range 3 {
181- targets, covered, err := s.Targets(context.Background(), oneWindow, false)
182- require.NoError(t, err)
183- require.Empty(t, targets)
184- require.Empty(t, covered)
185- }
186- tr := newMocktransport(t)
187- tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
188- [4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {1, 1, 9, 9}}, map[string]uint32{"/a": roleLink}, false))
189- c.EXPECT().Connect(mock.Anything).Return(tr, nil).Once()
190- s.retryAt = time.Now().Add(-time.Second)
191- targets, _, err := s.Targets(context.Background(), oneWindow, false)
192- require.NoError(t, err)
193- require.Len(t, targets, 1)
194-}
195-
196-// A broken connection is closed and dropped; the next attempt reconnects.
197-func TestBrokenConnectionIsDropped(t *testing.T) {
198- tr := newMocktransport(t)
199- tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(
200- func(_ time.Time, _ time.Duration, calls []call) []reply {
201- out := make([]reply, len(calls))
202- for i := range out {
203- out[i].err = ErrBroken
204- }
205- return out
206- }).Once()
207- tr.EXPECT().Close().Return(nil).Once()
208- c := newMockconnector(t)
209- c.EXPECT().Connect(mock.Anything).Return(tr, nil).Once()
210- s := &Source{dial: c}
211- _, _, err := s.Targets(context.Background(), oneWindow, false)
212- require.Error(t, err)
213- require.Nil(t, s.tr)
214- targets, _, err := s.Targets(context.Background(), oneWindow, false)
215- require.NoError(t, err)
216- require.Empty(t, targets) // backoff: no new attempt yet
217-}
218-
219-const testWait = 20 * time.Millisecond
220-
221-// blockedConnector returns a connector whose Connect blocks until release is
222-func blockedConnector(t *testing.T, release <-chan struct{}, tr transport, err error, times int) *mockconnector {
223- t.Helper()
224- c := newMockconnector(t)
225- c.EXPECT().Connect(mock.Anything).RunAndReturn(func(context.Context) (transport, error) {
226- <-release
227- return tr, err
228- }).Times(times)
229- return c
230-}
231-
232-// A silent service cannot hold Targets beyond the wait, is not doubled while
233-// in flight, and its late transport is used.
234-func TestConnectIsBoundedAndNotDoubled(t *testing.T) {
235- release := make(chan struct{})
236- tr := newMocktransport(t)
237- tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
238- [4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {1, 1, 9, 9}}, map[string]uint32{"/a": roleLink}, false))
239- c := blockedConnector(t, release, tr, nil, 1)
240- s := &Source{dial: c, wait: testWait}
241- t0 := time.Now()
242- targets, _, err := s.Targets(context.Background(), oneWindow, false)
243- require.NoError(t, err)
244- require.Empty(t, targets)
245- require.Less(t, time.Since(t0), testWait+200*time.Millisecond)
246- require.NotNil(t, s.attempt)
247- // Still in flight: no second Connect (Times(1)), and no blocking.
248- targets, _, err = s.Targets(context.Background(), oneWindow, false)
249- require.NoError(t, err)
250- require.Empty(t, targets)
251- // It connects late: the next call uses it.
252- close(release)
253- require.Eventually(t, func() bool { return len(s.attempt) == 1 }, time.Second, time.Millisecond)
254- targets, _, err = s.Targets(context.Background(), oneWindow, false)
255- require.NoError(t, err)
256- require.Len(t, targets, 1)
257- require.Nil(t, s.attempt)
258-}
259-
260-// An abandoned attempt that fails applies the backoff, then one new attempt.
261-func TestAbandonedAttemptErrorBacksOff(t *testing.T) {
262- release := make(chan struct{})
263- tr := newMocktransport(t)
264- tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(answerTree(
265- [4]int32{0, 0, 800, 600}, []ref{{":1.5", "/a"}}, map[string][4]int32{"/a": {1, 1, 9, 9}}, map[string]uint32{"/a": roleLink}, false))
266- c := newMockconnector(t)
267- c.EXPECT().Connect(mock.Anything).RunAndReturn(func(context.Context) (transport, error) {
268- <-release
269- return nil, errors.New("down")
270- }).Once()
271- c.EXPECT().Connect(mock.Anything).Return(tr, nil).Once()
272- s := &Source{dial: c, wait: testWait}
273- targets, _, err := s.Targets(context.Background(), oneWindow, false)
274- require.NoError(t, err)
275- require.Empty(t, targets)
276- close(release)
277- require.Eventually(t, func() bool { return len(s.attempt) == 1 }, time.Second, time.Millisecond)
278- // The error is collected; the backoff runs from its finish time.
279- targets, _, err = s.Targets(context.Background(), oneWindow, false)
280- require.NoError(t, err)
281- require.Empty(t, targets)
282- require.Nil(t, s.attempt)
283- require.WithinDuration(t, time.Now().Add(retryEvery), s.retryAt, time.Second)
284- targets, _, err = s.Targets(context.Background(), oneWindow, false) // backoff: no Connect
285- require.NoError(t, err)
286- require.Empty(t, targets)
287- s.retryAt = time.Now().Add(-time.Second)
288- targets, _, err = s.Targets(context.Background(), oneWindow, false)
289- require.NoError(t, err)
290- require.Len(t, targets, 1)
291-}
292-
293-// Close with an attempt in flight closes the late transport; afterwards
294-// Targets finds nothing and never connects.
295-func TestCloseReapsLateTransport(t *testing.T) {
296- release := make(chan struct{})
297- closed := make(chan struct{})
298- tr := newMocktransport(t)
299- tr.EXPECT().Close().RunAndReturn(func() error { close(closed); return nil }).Once()
300- c := blockedConnector(t, release, tr, nil, 1)
301- s := &Source{dial: c, wait: testWait}
302- _, _, err := s.Targets(context.Background(), oneWindow, false)
303- require.NoError(t, err)
304- require.NoError(t, s.Close())
305- close(release)
306- select {
307- case <-closed:
308- case <-time.After(5 * time.Second):
309- t.Fatal("late transport not closed")
310- }
311- targets, covered, err := s.Targets(context.Background(), oneWindow, false)
312- require.NoError(t, err)
313- require.Empty(t, targets)
314- require.Empty(t, covered)
315- require.NoError(t, s.Close()) // idempotent
316-}
317-
318-func TestWalkWideTreeStaysWithinBudget(t *testing.T) {
319- tr := newMocktransport(t)
320- children := make([]ref, 4096)
321- for i := range children {
322- children[i] = ref{":1.5", fmt.Sprintf("/node%d", i)}
323- }
324- visited := 0
325- tr.EXPECT().Do(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(
326- func(_ time.Time, _ time.Duration, calls []call) []reply {
327- visited += len(calls) / 3
328- require.LessOrEqual(t, visited, maxWalkNodes)
329- rep := make([]reply, len(calls))
330- for i, c := range calls {
331- switch c.op {
332- case opChildren:
333- rep[i].refs = children
334- case opRole:
335- rep[i].u = roleLink
336- case opState:
337- rep[i].state = stShowing | 1<<stateVisible
338- }
339- }
340- return rep
341- }).Twice()
342- s := &Source{tr: tr}
343- // A full second level replies with another 4096 children each. There is
344- // no remaining budget, so none of those millions of refs may be copied.
345- var out []ref
346- var before, after runtime.MemStats
347- runtime.ReadMemStats(&before)
348- out = s.walk(func(calls []call) []reply { return tr.Do(time.Time{}, time.Second, calls) }, ref{":1.5", "/frame"}, false)
349- runtime.ReadMemStats(&after)
350- require.Len(t, out, maxWalkNodes)
351- require.Equal(t, maxWalkNodes, visited)
352- // Normal replies and call batches take only a few MB; the old append-
353- // then-truncate path copied hundreds of MB for this tree.
354- require.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(32<<20))
355-}
D internal/adapters/atspi/transport.go +0 -362
@@ -1,362 +0,0 @@
1-package atspi
2-
3-import (
4- "errors"
5- "fmt"
6- "strings"
7- "sync/atomic"
8- "syscall"
9- "time"
10-
11- "github.com/bnema/zerobus"
12- "golang.org/x/sys/unix"
13-)
14-
15-// D-Bus names, paths and interfaces.
16-const (
17- busDaemon = "org.freedesktop.DBus"
18- busDaemonPath = "/org/freedesktop/DBus"
19- ifaceProps = "org.freedesktop.DBus.Properties"
20-
21- a11yBusName = "org.a11y.Bus"
22- a11yBusPath = "/org/a11y/bus"
23- a11yStatus = "org.a11y.Status"
24- registryName = "org.a11y.atspi.Registry"
25- registryRoot = "/org/a11y/atspi/accessible/root"
26-
27- ifaceAccessible = "org.a11y.atspi.Accessible"
28- ifaceComponent = "org.a11y.atspi.Component"
29- ifaceCollection = "org.a11y.atspi.Collection"
30-
31- // coordWindow is ATSPI_COORD_TYPE_WINDOW: extents relative to the
32- // top-level window.
33- coordWindow = 1
34-
35- // maxInflight bounds the calls sent before a reply is read, so the bus
36- // never waits on a full socket while this side waits on it.
37- maxInflight = 64
38- // maxRefs bounds the objects read from one reply.
39- maxRefs = 4096
40-)
41-
42-// op is a call the source makes.
43-type op uint8
44-
45-const (
46- opChildren op = iota // Accessible.GetChildren -> refs
47- opPID // GetConnectionUnixProcessID(name) -> u
48- opRole // Accessible.GetRole -> u
49- opState // Accessible.GetState -> state
50- opExtents // Component.GetExtents(arg) -> rect
51- opMatches // Collection.GetMatches(rule) -> refs
52-)
53-
54-// ref is an accessible object: a bus name and a path.
55-type ref struct{ name, path string }
56-
57-// call is one request of a batch. name and path address the object; for
58-// opPID, name is the bus name asked about.
59-type call struct {
60- op op
61- name, path string
62- arg uint32
63- rule matchRule
64-}
65-
66-// reply answers the call at the same index of the batch.
67-type reply struct {
68- err error
69- refs []ref
70- u uint32
71- state uint64
72- rect [4]int32 // x, y, w, h
73-}
74-
75-// transport sends batches of calls on one bus connection. It is used by one
76-// goroutine at a time.
77-type transport interface {
78- // Do sends calls pipelined and returns one reply per call, in order.
79- // The batch ends at deadline. A call not answered idle after it was
80- // sent gets ErrTimeout. When the connection fails or the deadline
81- // finds calls unanswered, those calls get an error wrapping
82- // ErrBroken: the connection must be dropped.
83- Do(deadline time.Time, idle time.Duration, calls []call) []reply
84- Close() error
85-}
86-
87-// ErrTimeout is the error of a call not answered in time.
88-var ErrTimeout = errors.New("atspi: call timed out")
89-
90-// busError is a D-Bus error reply.
91-type busError struct{ name, msg string }
92-
93-func (e *busError) Error() string {
94- if e.msg == "" {
95- return e.name
96- }
97- return e.name + ": " + e.msg
98-}
99-
100-// conn is a transport on a zerobus connection.
101-type conn struct {
102- c *zerobus.Conn
103- raw syscall.RawConn
104-}
105-
106-func newConn(c *zerobus.Conn) (*conn, error) {
107- raw, err := c.SyscallConn()
108- if err != nil {
109- _ = c.Close()
110- return nil, err
111- }
112- return &conn{c: c, raw: raw}, nil
113-}
114-
115-func (b *conn) Close() error { return b.c.Close() }
116-
117-// readable waits until a message can be read or d passes.
118-func (b *conn) readable(d time.Duration) (bool, error) {
119- if b.c.Buffered() {
120- return true, nil
121- }
122- ms := int((d + time.Millisecond - 1) / time.Millisecond)
123- if ms < 1 {
124- ms = 1
125- }
126- var n int
127- var perr error
128- if err := b.raw.Control(func(fd uintptr) {
129- fds := []unix.PollFd{{Fd: int32(fd), Events: unix.POLLIN}}
130- for {
131- n, perr = unix.Poll(fds, ms)
132- if perr != unix.EINTR {
133- return
134- }
135- }
136- }); err != nil {
137- return false, err
138- }
139- return n > 0, perr
140-}
141-
142-// ErrBroken is the error of calls on a connection that failed or was closed
143-// at the deadline; the owner must drop it and connect again.
144-var ErrBroken = errors.New("atspi: connection broken")
145-
146-// errDeadline is the error of calls still unanswered at the batch deadline:
147-// late replies and half-read messages leave the connection unusable.
148-var errDeadline = fmt.Errorf("%w: %w", ErrBroken, ErrTimeout)
149-
150-// watchdogGrace is how long after the deadline the watchdog closes a socket
151-// that the batch loop could not leave by itself (a stuck read or write).
152-const watchdogGrace = 20 * time.Millisecond
153-
154-// Do sends the calls pipelined. A watchdog closes the socket shortly after
155-// deadline, so neither a blocked write nor a read stuck in the middle of a
156-// message can outlive the batch; the connection is then broken. Each call
157-// has its own idle timeout, counted from its send: traffic for other calls
158-// does not extend it.
159-func (b *conn) Do(deadline time.Time, idle time.Duration, calls []call) []reply {
160- out := make([]reply, len(calls))
161- answered := make([]bool, len(calls))
162- type inflight struct {
163- idx int
164- sent time.Time
165- }
166- pending := make(map[uint32]inflight, min(len(calls), maxInflight))
167- next, done := 0, 0
168- var fired atomic.Bool
169- watchdog := time.AfterFunc(time.Until(deadline)+watchdogGrace, func() {
170- fired.Store(true)
171- _ = b.c.Close()
172- })
173- defer watchdog.Stop()
174- // finish gives err to every call without an answer.
175- finish := func(err error) []reply {
176- if !errors.Is(err, ErrBroken) && (fired.Load() || !errors.Is(err, ErrTimeout)) {
177- err = fmt.Errorf("%w: %w", ErrBroken, err)
178- }
179- if errors.Is(err, ErrBroken) {
180- _ = b.c.Close() // late replies would desynchronize the next batch
181- }
182- for i := range out {
183- if !answered[i] {
184- out[i].err = err
185- }
186- }
187- return out
188- }
189- for done < len(calls) {
190- for next < len(calls) && len(pending) < maxInflight {
191- if !time.Now().Before(deadline) {
192- return finish(errDeadline)
193- }
194- serial, err := b.send(calls[next])
195- if err != nil {
196- return finish(err)
197- }
198- pending[serial] = inflight{idx: next, sent: time.Now()}
199- next++
200- }
201- // Calls that waited longer than idle get no reply.
202- now := time.Now()
203- var oldest time.Time
204- for serial, p := range pending {
205- if now.Sub(p.sent) >= idle {
206- out[p.idx].err = ErrTimeout
207- answered[p.idx] = true
208- done++
209- delete(pending, serial)
210- } else if oldest.IsZero() || p.sent.Before(oldest) {
211- oldest = p.sent
212- }
213- }
214- if len(pending) == 0 {
215- continue
216- }
217- wait := min(time.Until(oldest.Add(idle)), time.Until(deadline))
218- if wait <= 0 {
219- if !time.Now().Before(deadline) {
220- break
221- }
222- continue
223- }
224- ok, err := b.readable(wait)
225- if err != nil {
226- return finish(err)
227- }
228- if !ok {
229- continue
230- }
231- m, err := b.c.ReadMessage()
232- if err != nil {
233- return finish(err)
234- }
235- p, found := pending[m.ReplySerial]
236- if !found || (m.Type != zerobus.TypeMethodReturn && m.Type != zerobus.TypeError) {
237- // A signal, or the late reply of an earlier batch.
238- continue
239- }
240- delete(pending, m.ReplySerial)
241- decode(m, calls[p.idx], &out[p.idx])
242- answered[p.idx] = true
243- done++
244- }
245- if done == len(calls) {
246- return out
247- }
248- return finish(errDeadline)
249-}
250-
251-// send builds and sends one call and returns its serial.
252-func (b *conn) send(c call) (uint32, error) {
253- switch c.op {
254- case opChildren:
255- b.c.NewCall(c.name, c.path, ifaceAccessible, "GetChildren", "")
256- case opPID:
257- b.c.NewCall(busDaemon, busDaemonPath, busDaemon, "GetConnectionUnixProcessID", "s").Str(c.name)
258- case opRole:
259- b.c.NewCall(c.name, c.path, ifaceAccessible, "GetRole", "")
260- case opState:
261- b.c.NewCall(c.name, c.path, ifaceAccessible, "GetState", "")
262- case opExtents:
263- b.c.NewCall(c.name, c.path, ifaceComponent, "GetExtents", "u").Uint32(c.arg)
264- case opMatches:
265- e := b.c.NewCall(c.name, c.path, ifaceCollection, "GetMatches", "(aiia{ss}iaiiasib)uib")
266- encodeRule(e, c.rule)
267- e.Uint32(sortCanonical)
268- e.Int32(matchCount)
269- e.Bool(true)
270- }
271- return b.c.Send()
272-}
273-
274-// encodeRule writes the MatchRule struct (aiia{ss}iaiiasib): states, state
275-// match, attributes, attribute match, roles, role match, interfaces,
276-// interface match, invert.
277-func encodeRule(e *zerobus.Encoder, r matchRule) {
278- e.Struct()
279- a := e.BeginArray('i')
280- for _, w := range stateBits(stateShowing, stateVisible) {
281- e.Int32(w)
282- }
283- e.EndArray(a)
284- e.Int32(matchAll)
285- m := e.BeginArray('{')
286- e.EndArray(m)
287- e.Int32(matchAll)
288- a = e.BeginArray('i')
289- for _, w := range roleBits(r.roles()) {
290- e.Int32(w)
291- }
292- e.EndArray(a)
293- e.Int32(matchAny)
294- s := e.BeginArray('s')
295- e.EndArray(s)
296- e.Int32(matchAll)
297- e.Bool(false)
298-}
299-
300-// decode reads the reply m of c into out.
301-func decode(m *zerobus.Message, c call, out *reply) {
302- if m.Type == zerobus.TypeError {
303- be := &busError{name: strings.Clone(m.ErrorName)}
304- if strings.HasPrefix(m.Signature, "s") {
305- be.msg = strings.Clone(m.Body().Str())
306- }
307- out.err = be
308- return
309- }
310- r := m.Body()
311- want := func(sig string) bool {
312- if m.Signature != sig {
313- out.err = fmt.Errorf("atspi: reply signature %q, want %q", m.Signature, sig)
314- return false
315- }
316- return true
317- }
318- switch c.op {
319- case opChildren, opMatches:
320- if !want("a(so)") {
321- return
322- }
323- end := r.Array('(')
324- for r.More(end) {
325- if len(out.refs) >= maxRefs {
326- break
327- }
328- r.Struct()
329- name := r.Str()
330- path := r.ObjectPath()
331- if r.Err() != nil {
332- break
333- }
334- out.refs = append(out.refs, ref{strings.Clone(name), strings.Clone(path)})
335- }
336- case opPID, opRole:
337- if want("u") {
338- out.u = r.Uint32()
339- }
340- case opState:
341- if !want("au") {
342- return
343- }
344- end := r.Array('u')
345- var words [2]uint32
346- for i := 0; r.More(end); i++ {
347- w := r.Uint32()
348- if i < len(words) {
349- words[i] = w
350- }
351- }
352- out.state = uint64(words[0]) | uint64(words[1])<<32
353- case opExtents:
354- if want("(iiii)") {
355- r.Struct()
356- out.rect = [4]int32{r.Int32(), r.Int32(), r.Int32(), r.Int32()}
357- }
358- }
359- if out.err == nil && r.Err() != nil {
360- out.err = r.Err()
361- }
362-}
D internal/adapters/atspi/transport_mock_test.go +0 -156
@@ -1,156 +0,0 @@
1-// Code generated by mockery; DO NOT EDIT.
2-// github.com/vektra/mockery
3-// template: testify
4-
5-package atspi
6-
7-import (
8- "time"
9-
10- mock "github.com/stretchr/testify/mock"
11-)
12-
13-// newMocktransport creates a new instance of mocktransport. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
14-// The first argument is typically a *testing.T value.
15-func newMocktransport(t interface {
16- mock.TestingT
17- Cleanup(func())
18-}) *mocktransport {
19- if helper, ok := t.(interface{ Helper() }); ok {
20- helper.Helper()
21- }
22-
23- mock := &mocktransport{}
24- mock.Mock.Test(t)
25-
26- t.Cleanup(func() {
27- if helper, ok := t.(interface{ Helper() }); ok {
28- helper.Helper()
29- }
30- mock.AssertExpectations(t)
31- })
32-
33- return mock
34-}
35-
36-// mocktransport is an autogenerated mock type for the transport type
37-type mocktransport struct {
38- mock.Mock
39-}
40-
41-type mocktransport_Expecter struct {
42- mock *mock.Mock
43-}
44-
45-func (_m *mocktransport) EXPECT() *mocktransport_Expecter {
46- return &mocktransport_Expecter{mock: &_m.Mock}
47-}
48-
49-// Close provides a mock function for the type mocktransport
50-func (_mock *mocktransport) Close() error {
51- ret := _mock.Called()
52-
53- if len(ret) == 0 {
54- panic("no return value specified for Close")
55- }
56-
57- var r0 error
58- if returnFunc, ok := ret.Get(0).(func() error); ok {
59- r0 = returnFunc()
60- } else {
61- r0 = ret.Error(0)
62- }
63- return r0
64-}
65-
66-// mocktransport_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
67-type mocktransport_Close_Call struct {
68- *mock.Call
69-}
70-
71-// Close is a helper method to define mock.On call
72-func (_e *mocktransport_Expecter) Close() *mocktransport_Close_Call {
73- return &mocktransport_Close_Call{Call: _e.mock.On("Close")}
74-}
75-
76-func (_c *mocktransport_Close_Call) Run(run func()) *mocktransport_Close_Call {
77- _c.Call.Run(func(args mock.Arguments) {
78- run()
79- })
80- return _c
81-}
82-
83-func (_c *mocktransport_Close_Call) Return(err error) *mocktransport_Close_Call {
84- _c.Call.Return(err)
85- return _c
86-}
87-
88-func (_c *mocktransport_Close_Call) RunAndReturn(run func() error) *mocktransport_Close_Call {
89- _c.Call.Return(run)
90- return _c
91-}
92-
93-// Do provides a mock function for the type mocktransport
94-func (_mock *mocktransport) Do(deadline time.Time, idle time.Duration, calls []call) []reply {
95- ret := _mock.Called(deadline, idle, calls)
96-
97- if len(ret) == 0 {
98- panic("no return value specified for Do")
99- }
100-
101- var r0 []reply
102- if returnFunc, ok := ret.Get(0).(func(time.Time, time.Duration, []call) []reply); ok {
103- r0 = returnFunc(deadline, idle, calls)
104- } else {
105- if ret.Get(0) != nil {
106- r0 = ret.Get(0).([]reply)
107- }
108- }
109- return r0
110-}
111-
112-// mocktransport_Do_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Do'
113-type mocktransport_Do_Call struct {
114- *mock.Call
115-}
116-
117-// Do is a helper method to define mock.On call
118-// - deadline time.Time
119-// - idle time.Duration
120-// - calls []call
121-func (_e *mocktransport_Expecter) Do(deadline any, idle any, calls any) *mocktransport_Do_Call {
122- return &mocktransport_Do_Call{Call: _e.mock.On("Do", deadline, idle, calls)}
123-}
124-
125-func (_c *mocktransport_Do_Call) Run(run func(deadline time.Time, idle time.Duration, calls []call)) *mocktransport_Do_Call {
126- _c.Call.Run(func(args mock.Arguments) {
127- var arg0 time.Time
128- if args[0] != nil {
129- arg0 = args[0].(time.Time)
130- }
131- var arg1 time.Duration
132- if args[1] != nil {
133- arg1 = args[1].(time.Duration)
134- }
135- var arg2 []call
136- if args[2] != nil {
137- arg2 = args[2].([]call)
138- }
139- run(
140- arg0,
141- arg1,
142- arg2,
143- )
144- })
145- return _c
146-}
147-
148-func (_c *mocktransport_Do_Call) Return(replyMoqParams []reply) *mocktransport_Do_Call {
149- _c.Call.Return(replyMoqParams)
150- return _c
151-}
152-
153-func (_c *mocktransport_Do_Call) RunAndReturn(run func(deadline time.Time, idle time.Duration, calls []call) []reply) *mocktransport_Do_Call {
154- _c.Call.Return(run)
155- return _c
156-}
M internal/adapters/config/config.go +42 -39
@@ -166,7 +166,9 @@ func Defaults() ports.Config {
166 c.Terminal.AutoOpen = "first"166 c.Terminal.AutoOpen = "first"
167 // Hint mode is experimental: off unless the config turns it on.167 // Hint mode is experimental: off unless the config turns it on.
168 c.Hint.Enable = false168 c.Hint.Enable = false
169- c.Hint.TextClick = ports.ModCtrl | ports.ModShift169+ c.Hint.Detector = defaultHintDetector
170+ c.Hint.Model = defaultHintModel()
171+ c.Hint.Threshold = defaultHintThreshold
170 c.Xwayland = "xwayland-satellite"172 c.Xwayland = "xwayland-satellite"
171 c.Background.Color = "#111111"173 c.Background.Color = "#111111"
172 c.Floating.Dim = 0.3174 c.Floating.Dim = 0.3
@@ -895,21 +897,23 @@ func set(c *ports.Config, key, v string) error {
895 } else {897 } else {
896 c.Hint.Foreground = v898 c.Hint.Foreground = v
897 }899 }
898- case "hint.targets":900+ case "hint.detector":
899- switch v {901+ if v == "" {
900- case "clickable":902+ v = defaultHintDetector
901- c.Hint.LinksOnly = false
902- case "links":
903- c.Hint.LinksOnly = true
904- default:
905- return fmt.Errorf("must be clickable or links")
906 }903 }
907- case "hint.text-click":904+ c.Hint.Detector = expandHome(v)
908- m, err := parseTextClick(v)905+ case "hint.model":
909- if err != nil {906+ if v == "" {
910- return err907+ c.Hint.Model = defaultHintModel()
908+ break
911 }909 }
912- c.Hint.TextClick = m910+ c.Hint.Model = expandHome(v)
911+ case "hint.threshold":
912+ n, err := strconv.ParseFloat(v, 64)
913+ if err != nil || math.IsNaN(n) || n < 0.01 || n > 0.99 {
914+ return fmt.Errorf("must be a number from 0.01 to 0.99")
915+ }
916+ c.Hint.Threshold = n
913 case "focus.flash-width":917 case "focus.flash-width":
914 n, err := strconv.Atoi(v)918 n, err := strconv.Atoi(v)
915 if err != nil || n < 1 || n > 8 {919 if err != nil || n < 1 || n > 8 {
@@ -1333,34 +1337,33 @@ func checkAction(v string) error {
1333 return nil1337 return nil
1334 }1338 }
1335 1339
1336-// parseTextClick parses hint.text-click: `+`-separated modifier names as in1340+const (
1337-// binds, or none. cmd is an alias of super here (it is not resolved through1341+ defaultHintDetector = "nefer-hints"
1338-// keyboard.cmd, so the result does not depend on key order).1342+ defaultHintThreshold = 0.05
1339-func parseTextClick(v string) (ports.Mods, error) {1343+)
1340- if strings.ToLower(v) == "none" {1344+
1341- return 0, nil1345+// defaultHintModel is $XDG_DATA_HOME/nefer-hints/models, or
1342- }1346+// ~/.local/share/nefer-hints/models when XDG_DATA_HOME is unset.
1343- var mods ports.Mods1347+func defaultHintModel() string {
1344- for _, p := range strings.Split(v, "+") {1348+ base := os.Getenv("XDG_DATA_HOME")
1345- var m ports.Mods1349+ if base == "" {
1346- switch strings.ToLower(strings.TrimSpace(p)) {1350+ home, err := os.UserHomeDir()
1347- case "shift":1351+ if err != nil {
1348- m = ports.ModShift1352+ return ""
1349- case "ctrl":
1350- m = ports.ModCtrl
1351- case "alt":
1352- m = ports.ModAlt
1353- case "super", "cmd":
1354- m = ports.ModSuper
1355- default:
1356- return 0, fmt.Errorf("unknown modifier %q (use shift, ctrl, alt, super or cmd joined by +, or none)", p)
1357 }1353 }
1358- if mods&m != 0 {1354+ base = filepath.Join(home, ".local", "share")
1359- return 0, fmt.Errorf("duplicate modifier %q", p)1355+ }
1356+ return filepath.Join(base, "nefer-hints", "models")
1357+}
1358+
1359+// expandHome replaces a leading ~/ with the home directory.
1360+func expandHome(p string) string {
1361+ if rest, ok := strings.CutPrefix(p, "~/"); ok {
1362+ if home, err := os.UserHomeDir(); err == nil {
1363+ return filepath.Join(home, rest)
1360 }1364 }
1361- mods |= m
1362 }1365 }
1363- return mods, nil1366+ return p
1364 }1367 }
1365 1368
1366 // parseCombo turns `ctrl+cmd+é` into the canonical `Cmd+Ctrl+eacute`:1369 // parseCombo turns `ctrl+cmd+é` into the canonical `Cmd+Ctrl+eacute`:
M internal/adapters/config/config_test.go +41 -41
@@ -954,26 +954,51 @@ func TestLoadRejectsFIFO(t *testing.T) {
954 }954 }
955 }955 }
956 956
957-func TestHintTargets(t *testing.T) {957+func TestHintBind(t *testing.T) {
958- if Defaults().Hint.LinksOnly {958+ c, w := parseString(t, "bind.cmd+g = hint\n")
959- t.Fatal("hint targets default must be clickable")959+ if len(w) != 0 || c.Binds["Cmd+g"] != "hint" {
960+ t.Fatal(c.Binds["Cmd+g"], w)
960 }961 }
961- c, w := parseString(t, "hint.targets = links\n")962+}
962- if !c.Hint.LinksOnly || len(w) != 0 {963+
963- t.Fatal(c.Hint, w)964+func TestHintRemovedKeysAreUnknown(t *testing.T) {
965+ for _, line := range []string{"hint.targets = links", "hint.text-click = none"} {
966+ _, w := parseString(t, line+"\n")
967+ if len(w) != 1 {
968+ t.Errorf("%q: %v", line, w)
969+ }
964 }970 }
965- c, w = parseString(t, "hint.targets = clickable\n")971+}
966- if c.Hint.LinksOnly || len(w) != 0 {972+
967- t.Fatal(c.Hint, w)973+func TestHintDetectorConfig(t *testing.T) {
974+ t.Setenv("HOME", "/home/u")
975+ t.Setenv("XDG_DATA_HOME", "")
976+ d := Defaults().Hint
977+ if d.Detector != "nefer-hints" || d.Model != "/home/u/.local/share/nefer-hints/models" || d.Threshold != 0.05 {
978+ t.Fatalf("defaults: %+v", d)
968 }979 }
969- // An invalid value warns and keeps the default.980+ t.Setenv("XDG_DATA_HOME", "/data")
970- c, w = parseString(t, "hint.targets = buttons\n")981+ if m := Defaults().Hint.Model; m != "/data/nefer-hints/models" {
971- if c.Hint.LinksOnly || len(w) != 1 {982+ t.Fatal(m)
972- t.Fatal(c.Hint, w)
973 }983 }
974- c, w = parseString(t, "bind.cmd+g = hint\n")984+ c, w := parseString(t, "hint.detector = ~/bin/nh\nhint.model = ~/m\nhint.threshold = 0.3\n")
975- if len(w) != 0 || c.Binds["Cmd+g"] != "hint" {985+ if len(w) != 0 || c.Hint.Detector != "/home/u/bin/nh" || c.Hint.Model != "/home/u/m" || c.Hint.Threshold != 0.3 {
976- t.Fatal(c.Binds["Cmd+g"], w)986+ t.Fatalf("%+v %v", c.Hint, w)
987+ }
988+ c, w = parseString(t, "hint.detector = /opt/nh\nhint.model = /opt/models\n")
989+ if len(w) != 0 || c.Hint.Detector != "/opt/nh" || c.Hint.Model != "/opt/models" {
990+ t.Fatalf("%+v %v", c.Hint, w)
991+ }
992+ for _, v := range []string{"0", "0.009", "1", "0.995", "NaN", "high", "-1"} {
993+ c, w := parseString(t, "hint.threshold = "+v+"\n")
994+ if len(w) != 1 || c.Hint.Threshold != 0.05 {
995+ t.Errorf("hint.threshold = %s: %v %v", v, c.Hint.Threshold, w)
996+ }
997+ }
998+ for _, v := range []string{"0.01", "0.99"} {
999+ if c, w := parseString(t, "hint.threshold = "+v+"\n"); len(w) != 0 || c.Hint.Threshold == 0.05 {
1000+ t.Errorf("hint.threshold = %s: %v", v, w)
1001+ }
977 }1002 }
978 }1003 }
979 1004
@@ -1021,31 +1046,6 @@ func TestHintSizeBounds(t *testing.T) {
1021 }1046 }
1022 }1047 }
1023 1048
1024-func TestHintTextClick(t *testing.T) {
1025- if got := Defaults().Hint.TextClick; got != ports.ModCtrl|ports.ModShift {
1026- t.Fatalf("default %d", got)
1027- }
1028- for in, want := range map[string]ports.Mods{
1029- "ctrl+shift": ports.ModCtrl | ports.ModShift,
1030- "Alt": ports.ModAlt,
1031- "super": ports.ModSuper,
1032- "cmd+shift": ports.ModSuper | ports.ModShift,
1033- "none": 0,
1034- "shift+ctrl+alt": ports.ModShift | ports.ModCtrl | ports.ModAlt,
1035- } {
1036- c, w := parseString(t, "hint.text-click = "+in+"\n")
1037- if c.Hint.TextClick != want || len(w) != 0 {
1038- t.Errorf("%q: %d %v", in, c.Hint.TextClick, w)
1039- }
1040- }
1041- for _, bad := range []string{"hyper", "ctrl+", "ctrl+ctrl", "ctrl+none", ""} {
1042- c, w := parseString(t, "hint.text-click = "+bad+"\n")
1043- if len(w) != 1 || c.Hint.TextClick != ports.ModCtrl|ports.ModShift {
1044- t.Errorf("%q: %d %v", bad, c.Hint.TextClick, w)
1045- }
1046- }
1047-}
1048-
1049 func TestHintEnable(t *testing.T) {1049 func TestHintEnable(t *testing.T) {
1050 if Defaults().Hint.Enable {1050 if Defaults().Hint.Enable {
1051 t.Fatal("hint mode must be off by default")1051 t.Fatal("hint mode must be off by default")
D internal/adapters/hints/benchmark_test.go +0 -81
@@ -1,81 +0,0 @@
1-package hints
2-
3-import (
4- "context"
5- "image"
6- "image/draw"
7- "image/png"
8- "os"
9- "os/exec"
10- "path/filepath"
11- "strconv"
12- "testing"
13- "time"
14-)
15-
16-// TestFrameBenchmark logs counts and timings, never filenames or OCR text.
17-func TestFrameBenchmark(t *testing.T) {
18- dir := os.Getenv("NEFERWL_HINT_BENCH_DIR")
19- if dir == "" {
20- t.Skip("set NEFERWL_HINT_BENCH_DIR to a local PNG directory")
21- }
22- if _, err := exec.LookPath("tesseract"); err != nil {
23- t.Skip("tesseract not on PATH")
24- }
25- files, err := filepath.Glob(filepath.Join(dir, "*.png"))
26- if err != nil || len(files) == 0 {
27- t.Fatal("no PNG frames")
28- }
29- // Settings under test: NEFERWL_HINT_BENCH_WORKERS=<n>, NEFERWL_HINT_BENCH_PNG=speed|none.
30- if v, _ := strconv.Atoi(os.Getenv("NEFERWL_HINT_BENCH_WORKERS")); v > 0 {
31- ocrWorkers = v
32- }
33- if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
34- rawPGM = false
35- }
36- if os.Getenv("NEFERWL_HINT_BENCH_PNG") == "none" {
37- pngLevel = png.NoCompression
38- }
39- for i, file := range files {
40- f, err := os.Open(file)
41- if err != nil {
42- t.Fatal("open frame failed")
43- }
44- src, err := png.Decode(f)
45- _ = f.Close()
46- if err != nil {
47- t.Fatal("decode frame failed")
48- }
49- img := image.NewRGBA(image.Rect(0, 0, src.Bounds().Dx(), src.Bounds().Dy()))
50- draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
51- start := time.Now()
52- words, bands, err := readWords(context.Background(), toGray(img))
53- if err != nil {
54- t.Fatal("recognition failed")
55- }
56- t.Logf("frame=%d px=%d ocr_ms=%.1f words=%d links=%d bands=%d", i, img.Bounds().Dx()*img.Bounds().Dy(), ms(time.Since(start)), len(words), len(textLinks(words, 1)), bands)
57- if os.Getenv("NEFERWL_HINT_BENCH_STRIPS") == "" {
58- continue
59- }
60- // Experimental strips use known OCR word regions, not pixel targets.
61- // This is an oracle comparison, not a production optimization.
62- lines := make([]image.Rectangle, 0, len(words))
63- for _, word := range words {
64- lines = append(lines, word.Rect)
65- }
66- for _, normalized := range []bool{false, true} {
67- begin := time.Now()
68- compact, _, err := readStrips(context.Background(), img, lines, 8, normalized)
69- if err != nil {
70- t.Fatal("strip recognition failed")
71- }
72- t.Logf("frame=%d strips normalized=%t ocr_ms=%.1f words=%d links=%d", i, normalized, ms(time.Since(begin)), len(compact), len(textLinks(compact, 1)))
73- }
74- }
75-}
76-
77-func toGray(img *image.RGBA) *image.Gray {
78- g := image.NewGray(img.Bounds())
79- draw.Draw(g, g.Bounds(), img, img.Bounds().Min, draw.Src)
80- return g
81-}
A internal/adapters/hints/detector.go +352 -0
@@ -0,0 +1,352 @@
1+package hints
2+
3+import (
4+ "bufio"
5+ "bytes"
6+ "context"
7+ "encoding/binary"
8+ "errors"
9+ "fmt"
10+ "io"
11+ "math"
12+ "os/exec"
13+ "strconv"
14+ "sync"
15+ "time"
16+
17+ "github.com/bnema/neferwl/internal/adapters/logging"
18+)
19+
20+// detFrame is a captured output as the detector receives it: BGRA pixels in
21+// m, valid only during the Detect call. It holds the mapping, not the bytes,
22+// so that recording a call (mocks print their arguments) never reads memory
23+// that was unmapped since.
24+type detFrame struct {
25+ width, height, stride int
26+ m *mapping
27+}
28+
29+// detection is one box found by the detector, in frame (physical) pixels.
30+type detection struct {
31+ x1, y1, x2, y2, score float32
32+}
33+
34+// detector finds clickable elements in a frame. The worker goroutine is its
35+// only caller.
36+type detector interface {
37+ // Detect returns the boxes found in f. An error is for this call only:
38+ // the detector recovers by itself on the next one.
39+ Detect(ctx context.Context, f detFrame) ([]detection, error)
40+ // Close releases the detector. It is called once, by the worker
41+ // goroutine, when the worker returns.
42+ Close() error
43+}
44+
45+const (
46+ protocolVersion = 1
47+ helloTimeout = 10 * time.Second
48+ requestTimeout = 2 * time.Second
49+ imgsz = 1280
50+ // maxString and maxBoxes bound what a misbehaving child can make us
51+ // allocate.
52+ maxString = 1 << 16
53+ maxBoxes = 1 << 20
54+)
55+
56+var (
57+ magicHello = [4]byte{'N', 'H', 'N', 'T'}
58+ magicRequest = [4]byte{'N', 'H', 'R', 'Q'}
59+ magicResponse = [4]byte{'N', 'H', 'R', 'S'}
60+
61+ // errProtocol is a framing error: the stream cannot be trusted any more.
62+ errProtocol = errors.New("hints: detector protocol error")
63+ // errDetector is an error the detector reported itself; it keeps serving.
64+ errDetector = errors.New("hints: detector error")
65+)
66+
67+var le = binary.LittleEndian
68+
69+// writeRequest writes one whole-frame request (no region, default
70+// confidence) followed by the pixels.
71+func writeRequest(w io.Writer, seq uint64, f detFrame) error {
72+ var h [4 + 4 + 8 + 5*4 + 4]byte
73+ copy(h[0:], magicRequest[:])
74+ le.PutUint32(h[4:], protocolVersion)
75+ le.PutUint64(h[8:], seq)
76+ le.PutUint32(h[16:], uint32(f.width))
77+ le.PutUint32(h[20:], uint32(f.height))
78+ le.PutUint32(h[24:], uint32(f.stride))
79+ le.PutUint32(h[28:], formatXRGB8888)
80+ le.PutUint32(h[32:], 0) // nregions
81+ le.PutUint32(h[36:], math.Float32bits(0)) // confidence: startup default
82+ if _, err := w.Write(h[:]); err != nil {
83+ return err
84+ }
85+ _, err := w.Write(f.m.b[:f.height*f.stride])
86+ return err
87+}
88+
89+func readMagic(r io.Reader, want [4]byte) error {
90+ var m [4]byte
91+ if _, err := io.ReadFull(r, m[:]); err != nil {
92+ return err
93+ }
94+ if m != want {
95+ return fmt.Errorf("%w: bad magic %q", errProtocol, m[:])
96+ }
97+ return nil
98+}
99+
100+func readU32(r io.Reader) (uint32, error) {
101+ var b [4]byte
102+ _, err := io.ReadFull(r, b[:])
103+ return le.Uint32(b[:]), err
104+}
105+
106+func readString(r io.Reader) (string, error) {
107+ n, err := readU32(r)
108+ if err != nil {
109+ return "", err
110+ }
111+ if n > maxString {
112+ return "", fmt.Errorf("%w: string of %d bytes", errProtocol, n)
113+ }
114+ b := make([]byte, n)
115+ _, err = io.ReadFull(r, b)
116+ return string(b), err
117+}
118+
119+// readHello reads the hello: the device name, or the error of a failed start.
120+func readHello(r io.Reader) (device string, err error) {
121+ if err := readMagic(r, magicHello); err != nil {
122+ return "", err
123+ }
124+ version, err := readU32(r)
125+ if err != nil {
126+ return "", err
127+ }
128+ if version != protocolVersion {
129+ return "", fmt.Errorf("%w: version %d", errProtocol, version)
130+ }
131+ status, err := readU32(r)
132+ if err != nil {
133+ return "", err
134+ }
135+ msg, err := readString(r)
136+ if err != nil {
137+ return "", err
138+ }
139+ if status != 0 {
140+ return "", fmt.Errorf("%w: start failed: %s", errDetector, msg)
141+ }
142+ if _, err := readU32(r); err != nil { // imgsz
143+ return "", err
144+ }
145+ return msg, nil
146+}
147+
148+// readResponse reads the response to request seq.
149+func readResponse(r io.Reader, seq uint64) ([]detection, error) {
150+ if err := readMagic(r, magicResponse); err != nil {
151+ return nil, err
152+ }
153+ var h [16]byte
154+ if _, err := io.ReadFull(r, h[:]); err != nil {
155+ return nil, err
156+ }
157+ if v := le.Uint32(h[0:]); v != protocolVersion {
158+ return nil, fmt.Errorf("%w: version %d", errProtocol, v)
159+ }
160+ if got := le.Uint64(h[4:]); got != seq {
161+ return nil, fmt.Errorf("%w: response %d to request %d", errProtocol, got, seq)
162+ }
163+ if status := le.Uint32(h[12:]); status != 0 {
164+ msg, err := readString(r)
165+ if err != nil {
166+ return nil, err
167+ }
168+ return nil, fmt.Errorf("%w: status %d: %s", errDetector, status, msg)
169+ }
170+ var t [8]byte // inference ms, nboxes
171+ if _, err := io.ReadFull(r, t[:]); err != nil {
172+ return nil, err
173+ }
174+ n := le.Uint32(t[4:])
175+ if n > maxBoxes {
176+ return nil, fmt.Errorf("%w: %d boxes", errProtocol, n)
177+ }
178+ out := make([]detection, 0, min(n, 4096))
179+ var rec [24]byte
180+ for range n {
181+ if _, err := io.ReadFull(r, rec[:]); err != nil {
182+ return nil, err
183+ }
184+ f := func(i int) float32 { return math.Float32frombits(le.Uint32(rec[i*4:])) }
185+ out = append(out, detection{x1: f(0), y1: f(1), x2: f(2), y2: f(3), score: f(4)})
186+ }
187+ return out, nil
188+}
189+
190+// procDetector runs the nefer-hints binary as a persistent child. It starts
191+// on the first Detect, and again after the child died or misbehaved.
192+type procDetector struct {
193+ command, model string
194+ conf float64
195+ helloTimeout time.Duration
196+ reqTimeout time.Duration
197+
198+ proc *process
199+ seq uint64
200+}
201+
202+func newProcDetector(command, model string, threshold float64) *procDetector {
203+ return &procDetector{command: command, model: model, conf: threshold, helloTimeout: helloTimeout, reqTimeout: requestTimeout}
204+}
205+
206+// process is one running child.
207+type process struct {
208+ cmd *exec.Cmd
209+ in io.WriteCloser
210+ out *bufio.Reader
211+ once sync.Once
212+}
213+
214+// kill stops the child and reaps it; safe to call more than once.
215+func (p *process) kill() {
216+ p.once.Do(func() {
217+ _ = p.cmd.Process.Kill()
218+ _ = p.in.Close()
219+ _ = p.cmd.Wait()
220+ })
221+}
222+
223+// interrupt kills the child without waiting: it unblocks a pending read or
224+// write from a timer or a context callback. kill reaps it afterwards.
225+func (p *process) interrupt() { _ = p.cmd.Process.Kill() }
226+
227+func (d *procDetector) start(ctx context.Context) error {
228+ log := logging.For(ctx, "hints")
229+ cmd := exec.Command(d.command, "--model", d.model, "--imgsz", strconv.Itoa(imgsz),
230+ "--conf", strconv.FormatFloat(d.conf, 'f', -1, 64), "--gpu", "auto")
231+ cmd.Stderr = &lineLog{log: func(line string) { log.Debug().Str("detector", d.command).Msg(line) }}
232+ in, err := cmd.StdinPipe()
233+ if err != nil {
234+ return err
235+ }
236+ out, err := cmd.StdoutPipe()
237+ if err != nil {
238+ return err
239+ }
240+ if err := cmd.Start(); err != nil {
241+ return fmt.Errorf("hints: start detector: %w", err)
242+ }
243+ p := &process{cmd: cmd, in: in, out: bufio.NewReaderSize(out, 1<<16)}
244+ t0 := time.Now()
245+ timer := time.AfterFunc(d.helloTimeout, p.interrupt)
246+ stop := context.AfterFunc(ctx, p.interrupt)
247+ device, err := readHello(p.out)
248+ stop()
249+ timedOut := !timer.Stop()
250+ if err != nil {
251+ p.kill()
252+ if timedOut {
253+ err = errors.New("hello timed out")
254+ }
255+ return fmt.Errorf("hints: detector hello: %w", err)
256+ }
257+ d.proc = p
258+ log.Info().Str("device", device).Float64("start_ms", ms(time.Since(t0))).Msg("hint detector ready")
259+ return nil
260+}
261+
262+func (d *procDetector) Detect(ctx context.Context, f detFrame) ([]detection, error) {
263+ if d.proc == nil {
264+ if err := d.start(ctx); err != nil {
265+ return nil, err
266+ }
267+ }
268+ p := d.proc
269+ d.seq++
270+ timer := time.AfterFunc(d.reqTimeout, p.interrupt)
271+ stop := context.AfterFunc(ctx, p.interrupt)
272+ dets, err := d.roundTrip(p, f)
273+ stop()
274+ timedOut := !timer.Stop()
275+ if err == nil {
276+ return dets, nil
277+ }
278+ if errors.Is(err, errDetector) {
279+ return nil, err // the child answered and keeps serving
280+ }
281+ d.drop()
282+ switch {
283+ case timedOut:
284+ return nil, fmt.Errorf("hints: detector timed out after %s", d.reqTimeout)
285+ case ctx.Err() != nil:
286+ return nil, ctx.Err()
287+ }
288+ return nil, fmt.Errorf("hints: detector failed: %w", err)
289+}
290+
291+func (d *procDetector) roundTrip(p *process, f detFrame) ([]detection, error) {
292+ bw := bufio.NewWriterSize(p.in, 1<<16)
293+ if err := writeRequest(bw, d.seq, f); err != nil {
294+ return nil, err
295+ }
296+ if err := bw.Flush(); err != nil {
297+ return nil, err
298+ }
299+ return readResponse(p.out, d.seq)
300+}
301+
302+// drop kills the current child: the next Detect starts a new one.
303+func (d *procDetector) drop() {
304+ if d.proc != nil {
305+ d.proc.kill()
306+ d.proc = nil
307+ }
308+}
309+
310+// Close asks the child to stop by closing its stdin (it exits 0 on EOF), then
311+// kills it if it does not.
312+func (d *procDetector) Close() error {
313+ p := d.proc
314+ d.proc = nil
315+ if p == nil {
316+ return nil
317+ }
318+ _ = p.in.Close()
319+ done := make(chan struct{})
320+ go func() { _ = p.cmd.Wait(); close(done) }()
321+ select {
322+ case <-done:
323+ case <-time.After(time.Second):
324+ _ = p.cmd.Process.Kill()
325+ <-done
326+ }
327+ return nil
328+}
329+
330+// lineLog is an io.Writer that hands each complete line to log.
331+type lineLog struct {
332+ log func(string)
333+ buf []byte
334+}
335+
336+func (l *lineLog) Write(b []byte) (int, error) {
337+ l.buf = append(l.buf, b...)
338+ for {
339+ i := bytes.IndexByte(l.buf, '\n')
340+ if i < 0 {
341+ break
342+ }
343+ if line := string(l.buf[:i]); line != "" {
344+ l.log(line)
345+ }
346+ l.buf = l.buf[i+1:]
347+ }
348+ if len(l.buf) > maxString {
349+ l.buf = l.buf[:0]
350+ }
351+ return len(b), nil
352+}
A internal/adapters/hints/detector_mock_test.go +159 -0
@@ -0,0 +1,159 @@
1+// Code generated by mockery; DO NOT EDIT.
2+// github.com/vektra/mockery
3+// template: testify
4+
5+package hints
6+
7+import (
8+ "context"
9+
10+ mock "github.com/stretchr/testify/mock"
11+)
12+
13+// newMockdetector creates a new instance of mockdetector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
14+// The first argument is typically a *testing.T value.
15+func newMockdetector(t interface {
16+ mock.TestingT
17+ Cleanup(func())
18+}) *mockdetector {
19+ if helper, ok := t.(interface{ Helper() }); ok {
20+ helper.Helper()
21+ }
22+
23+ mock := &mockdetector{}
24+ mock.Mock.Test(t)
25+
26+ t.Cleanup(func() {
27+ if helper, ok := t.(interface{ Helper() }); ok {
28+ helper.Helper()
29+ }
30+ mock.AssertExpectations(t)
31+ })
32+
33+ return mock
34+}
35+
36+// mockdetector is an autogenerated mock type for the detector type
37+type mockdetector struct {
38+ mock.Mock
39+}
40+
41+type mockdetector_Expecter struct {
42+ mock *mock.Mock
43+}
44+
45+func (_m *mockdetector) EXPECT() *mockdetector_Expecter {
46+ return &mockdetector_Expecter{mock: &_m.Mock}
47+}
48+
49+// Close provides a mock function for the type mockdetector
50+func (_mock *mockdetector) Close() error {
51+ ret := _mock.Called()
52+
53+ if len(ret) == 0 {
54+ panic("no return value specified for Close")
55+ }
56+
57+ var r0 error
58+ if returnFunc, ok := ret.Get(0).(func() error); ok {
59+ r0 = returnFunc()
60+ } else {
61+ r0 = ret.Error(0)
62+ }
63+ return r0
64+}
65+
66+// mockdetector_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
67+type mockdetector_Close_Call struct {
68+ *mock.Call
69+}
70+
71+// Close is a helper method to define mock.On call
72+func (_e *mockdetector_Expecter) Close() *mockdetector_Close_Call {
73+ return &mockdetector_Close_Call{Call: _e.mock.On("Close")}
74+}
75+
76+func (_c *mockdetector_Close_Call) Run(run func()) *mockdetector_Close_Call {
77+ _c.Call.Run(func(args mock.Arguments) {
78+ run()
79+ })
80+ return _c
81+}
82+
83+func (_c *mockdetector_Close_Call) Return(err error) *mockdetector_Close_Call {
84+ _c.Call.Return(err)
85+ return _c
86+}
87+
88+func (_c *mockdetector_Close_Call) RunAndReturn(run func() error) *mockdetector_Close_Call {
89+ _c.Call.Return(run)
90+ return _c
91+}
92+
93+// Detect provides a mock function for the type mockdetector
94+func (_mock *mockdetector) Detect(ctx context.Context, f detFrame) ([]detection, error) {
95+ ret := _mock.Called(ctx, f)
96+
97+ if len(ret) == 0 {
98+ panic("no return value specified for Detect")
99+ }
100+
101+ var r0 []detection
102+ var r1 error
103+ if returnFunc, ok := ret.Get(0).(func(context.Context, detFrame) ([]detection, error)); ok {
104+ return returnFunc(ctx, f)
105+ }
106+ if returnFunc, ok := ret.Get(0).(func(context.Context, detFrame) []detection); ok {
107+ r0 = returnFunc(ctx, f)
108+ } else {
109+ if ret.Get(0) != nil {
110+ r0 = ret.Get(0).([]detection)
111+ }
112+ }
113+ if returnFunc, ok := ret.Get(1).(func(context.Context, detFrame) error); ok {
114+ r1 = returnFunc(ctx, f)
115+ } else {
116+ r1 = ret.Error(1)
117+ }
118+ return r0, r1
119+}
120+
121+// mockdetector_Detect_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Detect'
122+type mockdetector_Detect_Call struct {
123+ *mock.Call
124+}
125+
126+// Detect is a helper method to define mock.On call
127+// - ctx context.Context
128+// - f detFrame
129+func (_e *mockdetector_Expecter) Detect(ctx any, f any) *mockdetector_Detect_Call {
130+ return &mockdetector_Detect_Call{Call: _e.mock.On("Detect", ctx, f)}
131+}
132+
133+func (_c *mockdetector_Detect_Call) Run(run func(ctx context.Context, f detFrame)) *mockdetector_Detect_Call {
134+ _c.Call.Run(func(args mock.Arguments) {
135+ var arg0 context.Context
136+ if args[0] != nil {
137+ arg0 = args[0].(context.Context)
138+ }
139+ var arg1 detFrame
140+ if args[1] != nil {
141+ arg1 = args[1].(detFrame)
142+ }
143+ run(
144+ arg0,
145+ arg1,
146+ )
147+ })
148+ return _c
149+}
150+
151+func (_c *mockdetector_Detect_Call) Return(detectionMoqParams []detection, err error) *mockdetector_Detect_Call {
152+ _c.Call.Return(detectionMoqParams, err)
153+ return _c
154+}
155+
156+func (_c *mockdetector_Detect_Call) RunAndReturn(run func(ctx context.Context, f detFrame) ([]detection, error)) *mockdetector_Detect_Call {
157+ _c.Call.Return(run)
158+ return _c
159+}
A internal/adapters/hints/detector_test.go +137 -0
@@ -0,0 +1,137 @@
1+package hints
2+
3+import (
4+ "bytes"
5+ "context"
6+ "encoding/binary"
7+ "math"
8+ "os"
9+ "testing"
10+ "time"
11+
12+ "github.com/stretchr/testify/require"
13+)
14+
15+// u32, u64 and f32 build the byte fixtures of the protocol.
16+func u32(v uint32) []byte { return binary.LittleEndian.AppendUint32(nil, v) }
17+func u64(v uint64) []byte { return binary.LittleEndian.AppendUint64(nil, v) }
18+func f32(v float32) []byte { return u32(math.Float32bits(v)) }
19+func cat(parts ...[]byte) []byte {
20+ return bytes.Join(parts, nil)
21+}
22+
23+func TestWriteRequest(t *testing.T) {
24+ var buf bytes.Buffer
25+ pix := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}
26+ require.NoError(t, writeRequest(&buf, 0x0102030405, detFrame{width: 2, height: 1, stride: 12, m: &mapping{b: append(pix, 99)}}))
27+ want := cat([]byte("NHRQ"), u32(1), u64(0x0102030405), u32(2), u32(1), u32(12), u32(1), u32(0), f32(0), pix)
28+ require.Equal(t, want, buf.Bytes())
29+}
30+
31+func TestReadHello(t *testing.T) {
32+ dev, err := readHello(bytes.NewReader(cat([]byte("NHNT"), u32(1), u32(0), u32(6), []byte("Vulkan"), u32(1280))))
33+ require.NoError(t, err)
34+ require.Equal(t, "Vulkan", dev)
35+
36+ _, err = readHello(bytes.NewReader(cat([]byte("NHNT"), u32(1), u32(1), u32(4), []byte("boom"))))
37+ require.ErrorIs(t, err, errDetector)
38+ require.ErrorContains(t, err, "boom")
39+
40+ _, err = readHello(bytes.NewReader(cat([]byte("XXXX"), u32(1))))
41+ require.ErrorIs(t, err, errProtocol)
42+ _, err = readHello(bytes.NewReader(cat([]byte("NHNT"), u32(2), u32(0))))
43+ require.ErrorIs(t, err, errProtocol)
44+ _, err = readHello(bytes.NewReader(cat([]byte("NHNT"), u32(1), u32(0), u32(1<<20))))
45+ require.ErrorIs(t, err, errProtocol)
46+ _, err = readHello(bytes.NewReader([]byte("NHNT")))
47+ require.Error(t, err)
48+}
49+
50+func TestReadResponse(t *testing.T) {
51+ head := func(seq uint64, status uint32) []byte { return cat([]byte("NHRS"), u32(1), u64(seq), u32(status)) }
52+
53+ ok := cat(head(5, 0), f32(12.5), u32(2),
54+ f32(1), f32(2), f32(30), f32(40), f32(0.9), u32(0),
55+ f32(5), f32(6), f32(7), f32(8), f32(0.25), u32(0))
56+ got, err := readResponse(bytes.NewReader(ok), 5)
57+ require.NoError(t, err)
58+ require.Equal(t, []detection{{1, 2, 30, 40, 0.9}, {5, 6, 7, 8, 0.25}}, got)
59+
60+ got, err = readResponse(bytes.NewReader(cat(head(5, 0), f32(1), u32(0))), 5)
61+ require.NoError(t, err)
62+ require.Empty(t, got)
63+
64+ _, err = readResponse(bytes.NewReader(cat(head(5, 1), u32(3), []byte("bad"))), 5)
65+ require.ErrorIs(t, err, errDetector)
66+ require.ErrorContains(t, err, "bad")
67+
68+ _, err = readResponse(bytes.NewReader(ok), 6) // wrong sequence
69+ require.ErrorIs(t, err, errProtocol)
70+ _, err = readResponse(bytes.NewReader(cat([]byte("NHRX"), u32(1))), 5)
71+ require.ErrorIs(t, err, errProtocol)
72+ _, err = readResponse(bytes.NewReader(cat([]byte("NHRS"), u32(9), u64(5), u32(0))), 5)
73+ require.ErrorIs(t, err, errProtocol)
74+ _, err = readResponse(bytes.NewReader(cat(head(5, 0), f32(1), u32(1<<30))), 5)
75+ require.ErrorIs(t, err, errProtocol)
76+ _, err = readResponse(bytes.NewReader(ok[:len(ok)-3]), 5) // truncated
77+ require.Error(t, err)
78+}
79+
80+func TestLineLog(t *testing.T) {
81+ var lines []string
82+ l := &lineLog{log: func(s string) { lines = append(lines, s) }}
83+ _, _ = l.Write([]byte("one\ntw"))
84+ _, _ = l.Write([]byte("o\n\nthree"))
85+ require.Equal(t, []string{"one", "two"}, lines)
86+}
87+
88+// The real detector, when the environment names it:
89+// NEFERWL_TEST_DETECTOR (binary) and NEFERWL_TEST_DETECTOR_MODEL (models dir).
90+func TestProcDetectorIntegration(t *testing.T) {
91+ bin, model := os.Getenv("NEFERWL_TEST_DETECTOR"), os.Getenv("NEFERWL_TEST_DETECTOR_MODEL")
92+ if bin == "" || model == "" {
93+ t.Skip("NEFERWL_TEST_DETECTOR and NEFERWL_TEST_DETECTOR_MODEL not set")
94+ }
95+ d := newProcDetector(bin, model, 0.05)
96+ t.Cleanup(func() { _ = d.Close() })
97+ m := &mapping{b: make([]byte, 1280*720*4), live: new(int)}
98+ for i := range m.b {
99+ m.b[i] = 0xf0 // a blank light frame
100+ }
101+ f := detFrame{width: 1280, height: 720, stride: 1280 * 4, m: m}
102+ ctx := context.Background()
103+
104+ dets, err := d.Detect(ctx, f)
105+ require.NoError(t, err)
106+ require.Empty(t, dets) // nothing clickable on a blank frame
107+ first := d.proc
108+ require.NotNil(t, first)
109+ _, err = d.Detect(ctx, f)
110+ require.NoError(t, err)
111+ require.Same(t, first, d.proc, "the child is kept alive between scans")
112+
113+ // A dead child fails one scan, and the next one starts a new child.
114+ require.NoError(t, first.cmd.Process.Kill())
115+ time.Sleep(50 * time.Millisecond)
116+ _, err = d.Detect(ctx, f)
117+ require.Error(t, err)
118+ require.Nil(t, d.proc)
119+ _, err = d.Detect(ctx, f)
120+ require.NoError(t, err)
121+ require.NotSame(t, first, d.proc)
122+
123+ // A wrong geometry is the detector's own error: the child stays.
124+ proc := d.proc
125+ _, err = d.Detect(ctx, detFrame{width: 1280, height: 720, stride: 4, m: m})
126+ require.ErrorIs(t, err, errDetector)
127+ require.Same(t, proc, d.proc)
128+ require.NoError(t, d.Close())
129+}
130+
131+func TestProcDetectorMissingBinary(t *testing.T) {
132+ d := newProcDetector("/nonexistent/nefer-hints", t.TempDir(), 0.05)
133+ _, err := d.Detect(context.Background(), detFrame{width: 1, height: 1, stride: 4, m: &mapping{b: make([]byte, 4)}})
134+ require.Error(t, err)
135+ require.Nil(t, d.proc)
136+ require.NoError(t, d.Close())
137+}
M internal/adapters/hints/dump.go +37 -79
@@ -7,6 +7,7 @@ import (
7 "image"7 "image"
8 "image/color"8 "image/color"
9 "image/png"9 "image/png"
10+ "math"
10 "os"11 "os"
11 "path/filepath"12 "path/filepath"
12 "sort"13 "sort"
@@ -25,78 +26,53 @@ const (
25 26
26 // Options tunes Run.27 // Options tunes Run.
27 type Options struct {28 type Options struct {
29+ // Command is the nefer-hints binary: a path, or a name looked up in PATH.
30+ Command string
31+ // ModelDir is the directory of the detector's model files.
32+ ModelDir string
33+ // Threshold is the minimum detection score, 0.01 to 0.99.
34+ Threshold float64
28 // DumpDir, when not empty, receives the scanned frames (PNG, annotated35 // DumpDir, when not empty, receives the scanned frames (PNG, annotated
29 // PNG and JSON). They contain the screen in clear.36 // PNG and JSON). They contain the screen in clear.
30 DumpDir string37 DumpDir string
31- // Source, when not nil, finds targets in the accessibility tree of the38+
32- // windows on screen; OCR reads the rest.39+ // detector replaces the subprocess in tests.
33- Source targetSource40+ detector detector
34 }41 }
35 42
36 // dumpJob is one scan to write. pix is the capture buffer of the scan, owned43 // dumpJob is one scan to write. pix is the capture buffer of the scan, owned
37 // by the job: writeDump converts it in place.44 // by the job: writeDump converts it in place.
38 type dumpJob struct {45 type dumpJob struct {
39- when time.Time46+ when time.Time
40- req ports.HintScanRequest47+ req ports.HintScanRequest
41- pix []byte48+ pix []byte
42- m *mapping // owns pix; released after the dump49+ m *mapping // owns pix; released after the dump
43- stride int50+ stride int
44- targets []ports.HintTarget51+ targets []ports.HintTarget
45- crops []image.Rectangle52+ dets []detection // detector boxes, physical pixels
46- words []OCRWord
47- // atspi is the number of leading targets that come from the
48- // accessibility tree; the others come from OCR.
49- atspi int
50 captureD time.Duration53 captureD time.Duration
51- scanD time.Duration54+ detectD time.Duration
52- atspiD time.Duration
53 }55 }
54 56
55 type dumpTarget struct {57 type dumpTarget struct {
56- Kind string `json:"kind"`58+ Rect [4]int `json:"rect"` // x1, y1, x2, y2, logical output-local
57- Source string `json:"source"`59+ Score float32 `json:"score"`
58- Rect [4]int `json:"rect"`
59 }60 }
60 61
61 type dumpMeta struct {62 type dumpMeta struct {
62- Output string `json:"output"`63+ Output string `json:"output"`
63- Width int `json:"width"`64+ Width int `json:"width"`
64- Height int `json:"height"`65+ Height int `json:"height"`
65- Stride int `json:"stride"`66+ Stride int `json:"stride"`
66- Scale float64 `json:"scale"`67+ Scale float64 `json:"scale"`
67- LinksOnly bool `json:"links_only"`68+ CaptureMS float64 `json:"capture_ms"`
68- CaptureMS float64 `json:"capture_ms"`69+ DetectMS float64 `json:"detect_ms"`
69- ScanMS float64 `json:"scan_ms"`70+ TotalMS float64 `json:"total_ms"`
70- AtspiMS float64 `json:"atspi_ms"`71+ Targets []dumpTarget `json:"targets"`
71- Crops []image.Rectangle `json:"ocr_crops"`
72- Words []OCRWord `json:"words"`
73- Targets []dumpTarget `json:"targets"`
74-}
75-
76-func kindName(k ports.HintKind) string {
77- switch k {
78- case ports.HintLink:
79- return "link"
80- case ports.HintButton:
81- return "button"
82- default:
83- return "text"
84- }
85 }72 }
86 73
87-func kindColor(k ports.HintKind) color.RGBA {74+// boxColor is the outline of the detector boxes.
88- switch k {75+var boxColor = color.RGBA{0x00, 0xc8, 0x53, 0xff}
89- case ports.HintLink:
90- return color.RGBA{0x00, 0xc8, 0x53, 0xff}
91- case ports.HintButton:
92- return color.RGBA{0x29, 0x62, 0xff, 0xff}
93- default:
94- return color.RGBA{0x80, 0x80, 0x80, 0xff}
95- }
96-}
97-
98-// atspiColor is the outline of targets found in the accessibility tree.
99-var atspiColor = color.RGBA{0xff, 0x00, 0xff, 0xff}
100 76
101 func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 }77 func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 }
102 78
@@ -121,21 +97,8 @@ func writeDump(dir string, j *dumpJob) error {
121 return err97 return err
122 }98 }
123 99
124- scale := j.req.Scale100+ for _, d := range j.dets {
125- if scale <= 0 {101+ outline(img, image.Rect(int(d.x1), int(d.y1), int(math.Ceil(float64(d.x2))), int(math.Ceil(float64(d.y2)))), boxColor)
126- scale = 1
127- }
128- for _, r := range j.crops {
129- outline(img, r, color.RGBA{0x80, 0x80, 0x80, 0xff})
130- }
131- for i, t := range j.targets {
132- r := image.Rect(int(float64(t.Rect.X)*scale), int(float64(t.Rect.Y)*scale),
133- int(float64(t.Rect.X+t.Rect.W)*scale), int(float64(t.Rect.Y+t.Rect.H)*scale))
134- c := kindColor(t.Kind)
135- if i < j.atspi {
136- c = atspiColor
137- }
138- outline(img, r, c)
139 }102 }
140 if err := writePNG(prefix+".annotated.png", img); err != nil {103 if err := writePNG(prefix+".annotated.png", img); err != nil {
141 return err104 return err
@@ -143,17 +106,12 @@ func writeDump(dir string, j *dumpJob) error {
143 106
144 meta := dumpMeta{107 meta := dumpMeta{
145 Output: j.req.Output, Width: j.req.Width, Height: j.req.Height, Stride: j.stride,108 Output: j.req.Output, Width: j.req.Width, Height: j.req.Height, Stride: j.stride,
146- Scale: j.req.Scale, LinksOnly: j.req.LinksOnly,109+ Scale: j.req.Scale,
147- CaptureMS: ms(j.captureD), ScanMS: ms(j.scanD), AtspiMS: ms(j.atspiD),110+ CaptureMS: ms(j.captureD), DetectMS: ms(j.detectD), TotalMS: ms(j.captureD + j.detectD),
148- Crops: j.crops, Words: j.words,
149 Targets: make([]dumpTarget, 0, len(j.targets)),111 Targets: make([]dumpTarget, 0, len(j.targets)),
150 }112 }
151- for i, t := range j.targets {113+ for _, t := range j.targets {
152- src := "ocr"114+ meta.Targets = append(meta.Targets, dumpTarget{Rect: [4]int{t.Rect.X, t.Rect.Y, t.Rect.X + t.Rect.W, t.Rect.Y + t.Rect.H}, Score: t.Score})
153- if i < j.atspi {
154- src = "atspi"
155- }
156- meta.Targets = append(meta.Targets, dumpTarget{Kind: kindName(t.Kind), Source: src, Rect: [4]int{t.Rect.X, t.Rect.Y, t.Rect.X + t.Rect.W, t.Rect.Y + t.Rect.H}})
157 }115 }
158 data, err := json.Marshal(meta)116 data, err := json.Marshal(meta)
159 if err != nil {117 if err != nil {
M internal/adapters/hints/hints.go +51 -296
@@ -1,15 +1,14 @@
1 // Package hints finds clickable targets for hint mode: it captures the last1 // Package hints finds clickable targets for hint mode: it captures the last
2-// frame of an output through the capture pipeline and reads uncovered windows with OCR.2+// frame of an output through the capture pipeline and sends it to the
3+// nefer-hints detector, a persistent subprocess.
3 package hints4 package hints
4 5
5 import (6 import (
6 "context"7 "context"
7 "errors"8 "errors"
8 "fmt"9 "fmt"
9- "hash/maphash"
10 "image"10 "image"
11 "math"11 "math"
12- "strings"
13 "time"12 "time"
14 13
15 "github.com/bnema/neferwl/internal/adapters/logging"14 "github.com/bnema/neferwl/internal/adapters/logging"
@@ -39,39 +38,18 @@ func Run(ctx context.Context, requests <-chan ports.HintScanRequest, captures ch
39 return run(ctx, requests, captures, captured, results, captureTimeout, opts)38 return run(ctx, requests, captures, captured, results, captureTimeout, opts)
40 }39 }
41 40
42-// targetSource finds hint targets in the accessibility tree of windows. It
43-// returns the targets (logical output-local) and the windows they cover.
44-type targetSource interface {
45- Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)
46- // Close releases the source. It is called once, by the worker goroutine,
47- // when the worker returns; the source is not safe for concurrent use.
48- Close() error
49-}
50-
51-// atspiResult is what the source found for one request.
52-type atspiResult struct {
53- targets []ports.HintTarget
54- covered map[ports.WindowID]bool
55- dur time.Duration
56- err error
57-}
58-
59 type worker struct {41 type worker struct {
60- source targetSource42+ det detector
61- cache map[bandKey]bandCache43+ n uint64
62- cacheSeed maphash.Seed44+ dumpDir string
63- ocr ocrState45+ pending *dumpJob
64- warnedSource bool
65- n uint64
66- dumpDir string
67- pending *dumpJob
68 // orphans are capture buffers the output may still write: they are46 // orphans are capture buffers the output may still write: they are
69 // released when its CaptureDone finally arrives.47 // released when its CaptureDone finally arrives.
70 orphans map[uint64]*mapping48 orphans map[uint64]*mapping
71 live int // mappings not released yet49 live int // mappings not released yet
72 }50 }
73 51
74-// frame is the captured output: BGRA pixels, nil when no capture was needed.52+// frame is the captured output: BGRA pixels.
75 type frame struct {53 type frame struct {
76 pix []byte54 pix []byte
77 stride int55 stride int
@@ -79,11 +57,12 @@ type frame struct {
79 }57 }
80 58
81 func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {59 func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {
82- w := &worker{dumpDir: opts.DumpDir, source: opts.Source}60+ w := &worker{dumpDir: opts.DumpDir, det: opts.detector}
83- if opts.Source != nil {61+ if w.det == nil {
84- // The worker goroutine is the only one to touch the source.62+ w.det = newProcDetector(opts.Command, opts.ModelDir, opts.Threshold)
85- defer func() { _ = opts.Source.Close() }()
86 }63 }
64+ // The worker goroutine is the only one to touch the detector.
65+ defer func() { _ = w.det.Close() }()
87 for {66 for {
88 select {67 select {
89 case <-ctx.Done():68 case <-ctx.Done():
@@ -130,25 +109,6 @@ func send(ctx context.Context, results chan<- ports.HintScanResult, res ports.Hi
130 }109 }
131 }110 }
132 111
133-// needFrame reports whether the scan must capture the output: OCR has a
134-// window to read, or the debug dump wants the frame. With OCR unavailable
135-// (tesseract missing) and no dump, the accessibility targets are the result.
136-func (w *worker) needFrame(ctx context.Context, req ports.HintScanRequest, ar atspiResult) bool {
137- ocr := w.ocrReady(ctx)
138- if w.dumpDir != "" {
139- return true
140- }
141- if !ocr {
142- return false
143- }
144- for _, win := range req.Windows {
145- if !ar.covered[win.ID] && !windowCrop(win.Rect, req.Scale, req.Width, req.Height).Empty() {
146- return true
147- }
148- }
149- return false
150-}
151-
152 func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration) ports.HintScanResult {112 func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration) ports.HintScanResult {
153 log := logging.For(ctx, "hints")113 log := logging.For(ctx, "hints")
154 res := ports.HintScanResult{Seq: req.Seq, Output: req.Output}114 res := ports.HintScanResult{Seq: req.Seq, Output: req.Output}
@@ -161,16 +121,8 @@ func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests
161 if req.Width <= 0 || req.Height <= 0 || req.Width*req.Height > maxPixels {121 if req.Width <= 0 || req.Height <= 0 || req.Width*req.Height > maxPixels {
162 return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))122 return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))
163 }123 }
164- // The accessibility query is always waited for: the source is not shared
165- // between requests. The capture only starts once it is known whether any
166- // window needs OCR.
167- atspi := w.startSource(ctx, req)
168- defer func() { <-atspi.done }()
169- if !w.needFrame(ctx, req, atspi.wait()) {
170- return w.scan(ctx, req, frame{}, 0, res, atspi)
171- }
172 // The capture buffer is anonymous mapped memory, outside the Go heap. It124 // The capture buffer is anonymous mapped memory, outside the Go heap. It
173- // is released once the crops are extracted, or by the dump job.125+ // is released once the detector answered, or by the dump job.
174 m := w.alloc(req.Width * req.Height * 4)126 m := w.alloc(req.Width * req.Height * 4)
175 keep := false // the mapping outlives this call: dump job or pending capture127 keep := false // the mapping outlives this call: dump job or pending capture
176 defer func() {128 defer func() {
@@ -225,7 +177,7 @@ func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests
225 if done.Err != nil {177 if done.Err != nil {
226 return fail(fmt.Errorf("hints: capture: %w", done.Err))178 return fail(fmt.Errorf("hints: capture: %w", done.Err))
227 }179 }
228- res = w.scan(ctx, req, frame{pix: buf, stride: stride, m: m}, time.Since(start), res, atspi)180+ res = w.scan(ctx, req, frame{pix: buf, stride: stride, m: m}, time.Since(start), res)
229 keep = w.pending != nil && w.pending.m == m181 keep = w.pending != nil && w.pending.m == m
230 return res182 return res
231 case other := <-requests:183 case other := <-requests:
@@ -255,253 +207,56 @@ func (w *worker) reap(id uint64) {
255 }207 }
256 }208 }
257 209
258-// bandJob is one OCR band of one window crop.210+// minTarget is the smallest target side kept, in logical pixels.
259-type bandJob struct {211+const minTarget = 4
260- win int // index into the scan's windows
261- key bandKey //
262- ext image.Rectangle // crop-local rows read, with context
263- own [2]int // crop-local rows owned: words centred there are kept
264- hash uint64
265- hit bool
266- gray *image.Gray
267- words []OCRWord
268- err error
269-}
270 212
271-func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame, captureDur time.Duration, res ports.HintScanResult, atspi *pendingSource) ports.HintScanResult {213+// scan sends the frame to the detector and turns the boxes into targets. It
214+// releases the frame, or hands it to the dump job.
215+func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame, captureDur time.Duration, res ports.HintScanResult) ports.HintScanResult {
272 log := logging.For(ctx, "hints")216 log := logging.For(ctx, "hints")
273- ar := atspi.wait()
274- if ar.err != nil && !w.warnedSource {
275- log.Warn().Err(ar.err).Msg("accessibility targets unavailable; using OCR only")
276- w.warnedSource = true
277- }
278- out := make([]ports.HintTarget, 0, len(ar.targets))
279- for _, t := range ar.targets {
280- if !req.LinksOnly || t.Kind == ports.HintLink {
281- out = append(out, t)
282- }
283- }
284- nAtspi := len(out)
285 t0 := time.Now()217 t0 := time.Now()
286- if w.cacheSeed == (maphash.Seed{}) {218+ dets, err := w.det.Detect(ctx, detFrame{width: req.Width, height: req.Height, stride: fr.stride, m: fr.m})
287- w.cacheSeed = maphash.MakeSeed()219+ detectDur := time.Since(t0)
288- }220+ if err != nil {
289- nextCache := make(map[bandKey]bandCache)221+ fr.m.unmap()
290- var crops []image.Rectangle222+ log.Info().Err(err).Str("output", req.Output).Float64("detect_ms", ms(detectDur)).Msg("hint scan failed")
291- var allWords []OCRWord223+ res.Err = err
292- pixels, links := 0, 0224+ return res
293- ocrOn := fr.pix != nil && w.ocrReady(ctx)
294- // Phase 1 reads the frame: the bands that miss the cache are copied, as
295- // grayscale, into one arena. Phase 2 (OCR) does not need the frame.
296- var jobs []*bandJob
297- var rects []image.Rectangle // crop of each window that has bands
298- var arena *mapping
299- if ocrOn {
300- type pending struct {
301- r image.Rectangle
302- jobs []*bandJob
303- }
304- var wins []pending
305- need := 0
306- for _, win := range req.Windows {
307- if ar.covered[win.ID] {
308- continue
309- }
310- r := windowCrop(win.Rect, req.Scale, req.Width, req.Height)
311- if r.Empty() {
312- continue
313- }
314- crops = append(crops, r)
315- pixels += r.Dx() * r.Dy()
316- p := pending{r: r}
317- for _, b := range bandsOf(r.Dy()) {
318- ext := image.Rect(0, b.top, r.Dx(), b.bottom)
319- j := &bandJob{win: len(wins), ext: ext, own: [2]int{b.ownTop, b.ownBottom},
320- key: bandKey{output: req.Output, window: win.ID, band: b.idx, bands: b.n}}
321- j.hash = w.cropHash(fr.pix, fr.stride, ext.Add(r.Min))
322- entry, ok := w.cache[j.key]
323- if ok && entry.hash == j.hash && entry.ext == ext {
324- j.hit, j.words = true, entry.words
325- } else {
326- need += ext.Dx() * ext.Dy()
327- }
328- p.jobs = append(p.jobs, j)
329- }
330- wins = append(wins, p)
331- }
332- if need > 0 {
333- arena = w.alloc(need)
334- }
335- off := 0
336- for _, p := range wins {
337- rects = append(rects, p.r)
338- for _, j := range p.jobs {
339- if !j.hit {
340- n := j.ext.Dx() * j.ext.Dy()
341- j.gray = cropGray(fr.pix, fr.stride, j.ext.Add(p.r.Min), arena.b[off:off+n])
342- off += n
343- }
344- jobs = append(jobs, j)
345- }
346- }
347- } else if w.dumpDir != "" {
348- for _, win := range req.Windows {
349- if !ar.covered[win.ID] {
350- if r := windowCrop(win.Rect, req.Scale, req.Width, req.Height); !r.Empty() {
351- crops = append(crops, r)
352- }
353- }
354- }
355 }225 }
356- if w.dumpDir != "" && fr.pix != nil {226+ res.Targets = targetsOf(dets, req.Scale)
227+ if w.dumpDir != "" {
357 // The dump reads the frame after the result is sent and releases it.228 // The dump reads the frame after the result is sent and releases it.
358- w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, crops: crops, captureD: captureDur, atspiD: ar.dur}229+ w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, targets: res.Targets, dets: dets, captureD: captureDur, detectD: detectDur}
359 } else {230 } else {
360- fr.m.unmap() // the crops are extracted: give the frame back231+ fr.m.unmap()
361 }232 }
362- fr.pix = nil233+ log.Info().Str("output", req.Output).Float64("capture_ms", ms(captureDur)).Float64("detect_ms", ms(detectDur)).
363-234+ Int("boxes", len(dets)).Int("targets", len(res.Targets)).Msg("hint scan")
364- hits := 0
365- if len(jobs) > 0 {
366- w.recognize(ctx, jobs)
367- for _, j := range jobs {
368- if j.hit {
369- hits++
370- }
371- }
372- }
373- arena.unmap()
374-
375- // Merge the bands of each window, in order.
376- k := 0
377- for wi, r := range rects {
378- var words []OCRWord
379- for ; k < len(jobs) && jobs[k].win == wi; k++ {
380- j := jobs[k]
381- if j.err != nil {
382- failureWarning.Do(func() { log.Warn().Err(j.err).Msg("OCR unavailable; install tesseract and tesseract-data-eng") })
383- continue
384- }
385- nextCache[j.key] = bandCache{hash: j.hash, ext: j.ext, words: j.words}
386- for _, word := range j.words {
387- word.Rect = word.Rect.Add(r.Min)
388- if word.Underlined {
389- word.UnderL, word.UnderR = word.UnderL+r.Min.X, word.UnderR+r.Min.X
390- }
391- if !duplicateWord(words, word) {
392- words = append(words, word)
393- }
394- }
395- }
396- allWords = append(allWords, words...)
397- targets := textLinks(words, req.Scale)
398- links += len(targets)
399- out = append(out, targets...)
400- }
401- w.cache = nextCache
402- ocrDur := time.Since(t0)
403- res.Targets = out
404- if p := w.pending; p != nil {
405- p.targets, p.atspi, p.words, p.scanD = out, nAtspi, allWords, ocrDur
406- }
407- log.Info().Str("output", req.Output).
408- Float64("atspi_ms", ms(ar.dur)).Int("atspi_windows", len(ar.covered)).Int("atspi_targets", nAtspi).
409- Int("ocr_windows", len(rects)).Float64("ocr_ms", ms(ocrDur)).Int("ocr_px", pixels).
410- Int("ocr_bands", len(jobs)).Int("band_hits", hits).
411- Int("words", len(allWords)).Int("ocr_links", links).Bool("cache_hit", len(jobs) > 0 && hits == len(jobs)).
412- Int("targets", len(out)).Msg("hint scan")
413 return res235 return res
414 }236 }
415 237
416-// duplicateWord reports whether words already holds word (the same text at238+// targetsOf converts boxes in physical pixels to logical output-local
417-// the same place), as happens where two bands overlap.239+// targets, rounding outwards, and drops the ones under minTarget on a side.
418-func duplicateWord(words []OCRWord, word OCRWord) bool {240+func targetsOf(dets []detection, scale float64) []ports.HintTarget {
419- for _, old := range words {
420- if strings.EqualFold(word.Text, old.Text) && overlap(word.Rect, old.Rect) > .5 {
421- return true
422- }
423- }
424- return false
425-}
426-
427-// bandKey identifies one OCR band of a window on an output.
428-type bandKey struct {
429- output string
430- window ports.WindowID
431- band int
432- bands int
433-}
434-
435-// bandCache is the OCR result of a band whose pixels hash to hash. words are
436-// crop-local and only the ones the band owns.
437-type bandCache struct {
438- hash uint64
439- ext image.Rectangle
440- words []OCRWord
441-}
442-
443-// windowCrop rounds outwards in physical pixels, then intersects the output.
444-func windowCrop(r ports.Rect, scale float64, width, height int) image.Rectangle {
445 if scale <= 0 {241 if scale <= 0 {
446 scale = 1242 scale = 1
447 }243 }
448- if r.W <= 0 || r.H <= 0 {244+ out := make([]ports.HintTarget, 0, len(dets))
449- return image.Rectangle{}245+ for _, d := range dets {
450- }246+ v := [4]float64{float64(d.x1), float64(d.y1), float64(d.x2), float64(d.y2)}
451- return image.Rect(int(math.Floor(float64(r.X)*scale)), int(math.Floor(float64(r.Y)*scale)),247+ ok := true
452- int(math.Ceil(float64(r.X+r.W)*scale)), int(math.Ceil(float64(r.Y+r.H)*scale))).Intersect(image.Rect(0, 0, width, height))248+ for _, f := range v {
453-}249+ ok = ok && !math.IsNaN(f) && !math.IsInf(f, 0)
454-
455-// cropHash hashes the raw pixels of r without copying them.
456-func (w *worker) cropHash(buf []byte, stride int, r image.Rectangle) uint64 {
457- var h maphash.Hash
458- h.SetSeed(w.cacheSeed)
459- for y := r.Min.Y; y < r.Max.Y; y++ {
460- _, _ = h.Write(buf[y*stride+r.Min.X*4 : y*stride+r.Max.X*4])
461- }
462- return h.Sum64()
463-}
464-
465-// cropGray converts the BGRA pixels of r to 8-bit luma into dst, which must
466-// hold r.Dx()*r.Dy() bytes; only this band is converted, not the output.
467-func cropGray(buf []byte, stride int, r image.Rectangle, dst []byte) *image.Gray {
468- img := &image.Gray{Pix: dst[:r.Dx()*r.Dy()], Stride: r.Dx(), Rect: image.Rect(0, 0, r.Dx(), r.Dy())}
469- for y := 0; y < r.Dy(); y++ {
470- src := buf[(r.Min.Y+y)*stride+r.Min.X*4:][:r.Dx()*4]
471- row := img.Pix[y*img.Stride:][:r.Dx()]
472- for x := range row {
473- p := src[x*4 : x*4+4 : x*4+4]
474- row[x] = uint8((uint32(p[2])*299 + uint32(p[1])*587 + uint32(p[0])*114 + 500) / 1000)
475 }250 }
251+ if !ok {
252+ continue
253+ }
254+ x1, y1 := int(math.Floor(v[0]/scale)), int(math.Floor(v[1]/scale))
255+ x2, y2 := int(math.Ceil(v[2]/scale)), int(math.Ceil(v[3]/scale))
256+ if x2-x1 < minTarget || y2-y1 < minTarget {
257+ continue
258+ }
259+ out = append(out, ports.HintTarget{Rect: ports.Rect{X: x1, Y: y1, W: x2 - x1, H: y2 - y1}, Score: d.score})
476 }260 }
477- return img261+ return out
478-}
479-
480-// pendingSource is an accessibility query in flight.
481-type pendingSource struct {
482- done chan struct{}
483- res atspiResult
484-}
485-
486-// startSource starts the accessibility query of req, if there is a source and
487-// a window to ask about. done is closed when res is set.
488-func (w *worker) startSource(ctx context.Context, req ports.HintScanRequest) *pendingSource {
489- p := &pendingSource{done: make(chan struct{})}
490- if w.source == nil || len(req.Windows) == 0 {
491- close(p.done)
492- return p
493- }
494- go func() {
495- defer close(p.done)
496- t0 := time.Now()
497- targets, covered, err := w.source.Targets(ctx, req.Windows, req.LinksOnly)
498- p.res = atspiResult{targets: targets, covered: covered, dur: time.Since(t0), err: err}
499- }()
500- return p
501-}
502-
503-// wait returns the result of the query.
504-func (p *pendingSource) wait() atspiResult {
505- <-p.done
506- return p.res
507 }262 }
M internal/adapters/hints/hints_test.go +108 -178
@@ -6,10 +6,9 @@ import (
6 "errors"6 "errors"
7 "fmt"7 "fmt"
8 "image"8 "image"
9- "image/draw"
10 _ "image/png"9 _ "image/png"
10+ "math"
11 "os"11 "os"
12- "os/exec"
13 "path/filepath"12 "path/filepath"
14 "strings"13 "strings"
15 "testing"14 "testing"
@@ -39,6 +38,13 @@ func startCapturing(t *testing.T, timeout time.Duration) *harness {
39 38
40 func startWith(t *testing.T, timeout time.Duration, opts Options) *harness {39 func startWith(t *testing.T, timeout time.Duration, opts Options) *harness {
41 t.Helper()40 t.Helper()
41+ if opts.detector == nil {
42+ // Tests of the capture flow do not care what the detector finds.
43+ det := newMockdetector(t)
44+ det.EXPECT().Detect(mock.Anything, mock.Anything).Return(nil, nil).Maybe()
45+ det.EXPECT().Close().Return(nil).Maybe()
46+ opts.detector = det
47+ }
42 h := &harness{48 h := &harness{
43 requests: make(chan ports.HintScanRequest, 1),49 requests: make(chan ports.HintScanRequest, 1),
44 captures: make(chan ports.CaptureRequest, 4),50 captures: make(chan ports.CaptureRequest, 4),
@@ -55,51 +61,67 @@ func startWith(t *testing.T, timeout time.Duration, opts Options) *harness {
55 return h61 return h
56 }62 }
57 63
58-func loadBGRA(t *testing.T) (pix []byte, w, hgt int) {
59- t.Helper()
60- f, err := os.Open("testdata/browser.png")
61- require.NoError(t, err)
62- defer f.Close()
63- img, _, err := image.Decode(f)
64- require.NoError(t, err)
65- b := img.Bounds()
66- rgba := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
67- draw.Draw(rgba, rgba.Bounds(), img, b.Min, draw.Src)
68- for i := 0; i < len(rgba.Pix); i += 4 {
69- rgba.Pix[i], rgba.Pix[i+2] = rgba.Pix[i+2], rgba.Pix[i]
70- rgba.Pix[i+3] = 0xff
71- }
72- return rgba.Pix, b.Dx(), b.Dy()
73-}
74-
75 func TestHintScan(t *testing.T) {64 func TestHintScan(t *testing.T) {
76- if _, err := exec.LookPath("tesseract"); err != nil {65+ det := newMockdetector(t)
77- t.Skip("tesseract not on PATH")66+ det.EXPECT().Close().Return(nil).Once()
78- }67+ var got detFrame
79- pix, w, hgt := loadBGRA(t)68+ det.EXPECT().Detect(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, f detFrame) ([]detection, error) {
80- h := start(t, 5*time.Second)69+ got = f
81-70+ return []detection{{x1: 10, y1: 20, x2: 110, y2: 60, score: 0.8}}, nil
82- // Plays the output owner.71+ }).Once()
72+ h := startWith(t, 5*time.Second, Options{detector: det})
83 go func() {73 go func() {
84 creq := <-h.captures74 creq := <-h.captures
85- copy(creq.Mapped, pix)
86 h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}75 h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
87 }()76 }()
88-77+ h.requests <- ports.HintScanRequest{Seq: 7, Output: "HEADLESS-1", Width: 64, Height: 32, Scale: 1}
89- h.requests <- ports.HintScanRequest{Seq: 7, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
90 res := <-h.results78 res := <-h.results
91 require.NoError(t, res.Err)79 require.NoError(t, res.Err)
92 require.Equal(t, uint64(7), res.Seq)80 require.Equal(t, uint64(7), res.Seq)
93 require.Equal(t, "HEADLESS-1", res.Output)81 require.Equal(t, "HEADLESS-1", res.Output)
94- var links int82+ require.Equal(t, []ports.HintTarget{{Rect: ports.Rect{X: 10, Y: 20, W: 100, H: 40}, Score: 0.8}}, res.Targets)
95- for _, tg := range res.Targets {83+ require.Equal(t, 64, got.width)
96- require.Positive(t, tg.Rect.W)84+ require.Equal(t, 32, got.height)
97- require.Positive(t, tg.Rect.H)85+ require.Equal(t, 64*4, got.stride)
98- if tg.Kind == ports.HintLink {86+}
99- links++87+
88+func TestTargetsOf(t *testing.T) {
89+ got := targetsOf([]detection{
90+ {x1: 11, y1: 21, x2: 100.2, y2: 61, score: 0.5}, // scale 2: x 5..51, y 10..31
91+ {x1: 0, y1: 0, x2: 5, y2: 40, score: 0.9}, // 2.5 -> 3 logical wide: dropped
92+ {x1: 0, y1: 0, x2: 40, y2: 5, score: 0.9}, // 3 logical high: dropped
93+ {x1: 0, y1: 0, x2: 8, y2: 8, score: 0.4}, // exactly 4x4: kept
94+ {x1: float32(math.NaN()), y1: 0, x2: 50, y2: 50},
95+ }, 2)
96+ require.Equal(t, []ports.HintTarget{
97+ {Rect: ports.Rect{X: 5, Y: 10, W: 46, H: 21}, Score: 0.5},
98+ {Rect: ports.Rect{X: 0, Y: 0, W: 4, H: 4}, Score: 0.4},
99+ }, got)
100+ // A zero scale reads as 1.
101+ require.Len(t, targetsOf([]detection{{x2: 10, y2: 10}}, 0), 1)
102+}
103+
104+// A detector error fails the scan; the worker goes on and the buffer is released.
105+func TestHintDetectorErrorFailsScan(t *testing.T) {
106+ boom := errors.New("detector gone")
107+ det := newMockdetector(t)
108+ det.EXPECT().Close().Return(nil).Once()
109+ det.EXPECT().Detect(mock.Anything, mock.Anything).Return(nil, boom).Once()
110+ det.EXPECT().Detect(mock.Anything, mock.Anything).Return(nil, nil).Once()
111+ h := startWith(t, 5*time.Second, Options{detector: det})
112+ for i, want := range []error{boom, nil} {
113+ go func() {
114+ creq := <-h.captures
115+ h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
116+ }()
117+ h.requests <- ports.HintScanRequest{Seq: uint64(i + 1), Output: "o", Width: 16, Height: 16, Scale: 1}
118+ res := <-h.results
119+ if want != nil {
120+ require.ErrorIs(t, res.Err, want)
121+ } else {
122+ require.NoError(t, res.Err)
100 }123 }
101 }124 }
102- require.GreaterOrEqual(t, links, 1)
103 }125 }
104 126
105 func TestHintRequestShape(t *testing.T) {127 func TestHintRequestShape(t *testing.T) {
@@ -166,19 +188,6 @@ func TestHintBadSize(t *testing.T) {
166 require.ErrorIs(t, res.Err, ErrBadRequest)188 require.ErrorIs(t, res.Err, ErrBadRequest)
167 }189 }
168 190
169-func scanOnce(t *testing.T, h *harness, pix []byte, w, hgt int, seq uint64) ports.HintScanResult {
170- t.Helper()
171- go func() {
172- creq := <-h.captures
173- copy(creq.Mapped, pix)
174- h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
175- }()
176- h.requests <- ports.HintScanRequest{Seq: seq, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: hgt}}}}
177- res := <-h.results
178- require.NoError(t, res.Err)
179- return res
180-}
181-
182 // waitFiles waits for the dump the worker writes after sending the result.191 // waitFiles waits for the dump the worker writes after sending the result.
183 func waitFiles(t *testing.T, dir string, n int) []os.DirEntry {192 func waitFiles(t *testing.T, dir string, n int) []os.DirEntry {
184 t.Helper()193 t.Helper()
@@ -190,19 +199,32 @@ func waitFiles(t *testing.T, dir string, n int) []os.DirEntry {
190 return entries199 return entries
191 }200 }
192 201
202+// scanOnce runs one scan on a 64x32 frame and waits for its result.
203+func scanOnce(t *testing.T, h *harness, seq uint64) ports.HintScanResult {
204+ t.Helper()
205+ go func() {
206+ creq := <-h.captures
207+ for i := 0; i < len(creq.Mapped); i += 4 {
208+ creq.Mapped[i], creq.Mapped[i+1], creq.Mapped[i+2] = 10, 20, 30
209+ }
210+ h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
211+ }()
212+ h.requests <- ports.HintScanRequest{Seq: seq, Output: "HEADLESS-1", Width: 64, Height: 32, Scale: 1}
213+ res := <-h.results
214+ require.NoError(t, res.Err)
215+ return res
216+}
217+
193 func TestHintDump(t *testing.T) {218 func TestHintDump(t *testing.T) {
194- if _, err := exec.LookPath("tesseract"); err != nil {219+ det := newMockdetector(t)
195- t.Skip("tesseract not on PATH")220+ det.EXPECT().Close().Return(nil).Once()
196- }221+ det.EXPECT().Detect(mock.Anything, mock.Anything).Return([]detection{{x1: 8, y1: 4, x2: 40, y2: 20, score: 0.75}}, nil).Once()
197- pix, w, hgt := loadBGRA(t)
198 dir := filepath.Join(t.TempDir(), "hints")222 dir := filepath.Join(t.TempDir(), "hints")
199- h := startWith(t, 5*time.Second, Options{DumpDir: dir})223+ h := startWith(t, 5*time.Second, Options{DumpDir: dir, detector: det})
200- res := scanOnce(t, h, pix, w, hgt, 1)224+ res := scanOnce(t, h, 1)
201- require.NotEmpty(t, res.Targets)225+ require.Len(t, res.Targets, 1)
202 226
203 entries := waitFiles(t, dir, 3)227 entries := waitFiles(t, dir, 3)
204- // A second request is only served once the dump is written.
205- _ = scanOnceDone(t, h)
206 var png, ann, js string228 var png, ann, js string
207 for _, e := range entries {229 for _, e := range entries {
208 info, err := e.Info()230 info, err := e.Info()
@@ -228,61 +250,45 @@ func TestHintDump(t *testing.T) {
228 data, err := os.ReadFile(filepath.Join(dir, js))250 data, err := os.ReadFile(filepath.Join(dir, js))
229 require.NoError(t, err)251 require.NoError(t, err)
230 var meta struct {252 var meta struct {
231- Output string `json:"output"`253+ Output string `json:"output"`
232- Width int `json:"width"`254+ Width int `json:"width"`
233- Height int `json:"height"`255+ Height int `json:"height"`
234- Stride int `json:"stride"`256+ Stride int `json:"stride"`
235- Scale float64 `json:"scale"`257+ Scale float64 `json:"scale"`
236- LinksOnly bool `json:"links_only"`258+ DetectMS float64 `json:"detect_ms"`
237- Targets []struct {259+ TotalMS float64 `json:"total_ms"`
238- Kind string `json:"kind"`260+ Targets []struct {
239- Rect [4]int `json:"rect"`261+ Rect [4]int `json:"rect"`
262+ Score float32 `json:"score"`
240 } `json:"targets"`263 } `json:"targets"`
241 }264 }
242 require.NoError(t, json.Unmarshal(data, &meta))265 require.NoError(t, json.Unmarshal(data, &meta))
243 require.Equal(t, "HEADLESS-1", meta.Output)266 require.Equal(t, "HEADLESS-1", meta.Output)
244- require.Equal(t, w, meta.Width)267+ require.Equal(t, 64, meta.Width)
245- require.Equal(t, hgt, meta.Height)268+ require.Equal(t, 32, meta.Height)
246- require.Equal(t, w*4, meta.Stride)269+ require.Equal(t, 64*4, meta.Stride)
247 require.Equal(t, 1.0, meta.Scale)270 require.Equal(t, 1.0, meta.Scale)
248- require.Len(t, meta.Targets, len(res.Targets))271+ require.GreaterOrEqual(t, meta.TotalMS, meta.DetectMS)
249- for i, tg := range res.Targets {272+ require.Len(t, meta.Targets, 1)
250- require.Equal(t, [4]int{tg.Rect.X, tg.Rect.Y, tg.Rect.X + tg.Rect.W, tg.Rect.Y + tg.Rect.H}, meta.Targets[i].Rect)273+ require.Equal(t, [4]int{8, 4, 40, 20}, meta.Targets[0].Rect)
251- }274+ require.Equal(t, float32(0.75), meta.Targets[0].Score)
252 275
253 f, err := os.Open(filepath.Join(dir, png))276 f, err := os.Open(filepath.Join(dir, png))
254 require.NoError(t, err)277 require.NoError(t, err)
255 defer f.Close()278 defer f.Close()
256 img, _, err := image.Decode(f)279 img, _, err := image.Decode(f)
257 require.NoError(t, err)280 require.NoError(t, err)
258- require.Equal(t, image.Rect(0, 0, w, hgt), img.Bounds())281+ require.Equal(t, image.Rect(0, 0, 64, 32), img.Bounds())
259 r, g, b, _ := img.At(0, 0).RGBA()282 r, g, b, _ := img.At(0, 0).RGBA()
260- require.Equal(t, uint32(pix[2])*0x101, r)283+ require.Equal(t, uint32(30)*0x101, r)
261- require.Equal(t, uint32(pix[1])*0x101, g)284+ require.Equal(t, uint32(20)*0x101, g)
262- require.Equal(t, uint32(pix[0])*0x101, b)285+ require.Equal(t, uint32(10)*0x101, b)
263-}
264-
265-// scanOnceDone sends a request that fails fast on a blank capture and waits
266-// for its result, which proves the previous dump finished.
267-func scanOnceDone(t *testing.T, h *harness) ports.HintScanResult {
268- t.Helper()
269- go func() {
270- creq := <-h.captures
271- h.captured <- ports.CaptureDone{ID: creq.ID, Err: errors.New("stop")}
272- }()
273- h.requests <- ports.HintScanRequest{Seq: 99, Output: "x", Width: 8, Height: 8, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 8, H: 8}}}}
274- return <-h.results
275 }286 }
276 287
277 func TestHintDumpDisabled(t *testing.T) {288 func TestHintDumpDisabled(t *testing.T) {
278- if _, err := exec.LookPath("tesseract"); err != nil {
279- t.Skip("tesseract not on PATH")
280- }
281- pix, w, hgt := loadBGRA(t)
282 dir := filepath.Join(t.TempDir(), "hints")289 dir := filepath.Join(t.TempDir(), "hints")
283 h := start(t, 5*time.Second)290 h := start(t, 5*time.Second)
284- scanOnce(t, h, pix, w, hgt, 1)291+ scanOnce(t, h, 1)
285- scanOnceDone(t, h)
286 _, err := os.Stat(dir)292 _, err := os.Stat(dir)
287 require.ErrorIs(t, err, os.ErrNotExist)293 require.ErrorIs(t, err, os.ErrNotExist)
288 }294 }
@@ -302,67 +308,25 @@ func TestHintDumpRetention(t *testing.T) {
302 require.Equal(t, "20260101-000000.010-o.annotated.png", entries[0].Name())308 require.Equal(t, "20260101-000000.010-o.annotated.png", entries[0].Name())
303 }309 }
304 310
305-func TestScanMergesSource(t *testing.T) {
306- pix, w, hgt := loadBGRA(t)
307- src := newMocktargetSource(t)
308- src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
309- wins := []ports.HintWindow{{ID: 4, PID: 9, Rect: ports.Rect{X: 0, Y: 0, W: w, H: hgt}}}
310- at := []ports.HintTarget{{Rect: ports.Rect{X: 5, Y: 5, W: 30, H: 10}, Kind: ports.HintLink}}
311- src.EXPECT().Targets(mock.Anything, wins, false).Return(at, map[ports.WindowID]bool{4: true}, nil).Once()
312- h := startWith(t, 5*time.Second, Options{Source: src})
313- go func() {
314- creq := <-h.captures
315- copy(creq.Mapped, pix)
316- h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
317- }()
318- h.requests <- ports.HintScanRequest{Seq: 1, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: wins}
319- res := <-h.results
320- require.NoError(t, res.Err)
321- // The whole frame is covered: only the accessibility target remains.
322- require.Equal(t, at, res.Targets)
323-}
324-
325-func TestScanSourceErrorUsesOCR(t *testing.T) {
326- if _, err := exec.LookPath("tesseract"); err != nil {
327- t.Skip("tesseract not on PATH")
328- }
329- pix, w, hgt := loadBGRA(t)
330- src := newMocktargetSource(t)
331- src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
332- wins := []ports.HintWindow{{ID: 4, PID: 9, Rect: ports.Rect{W: w, H: hgt}}}
333- src.EXPECT().Targets(mock.Anything, wins, false).Return(nil, nil, errors.New("bus gone")).Once()
334- h := startWith(t, 5*time.Second, Options{Source: src})
335- go func() {
336- creq := <-h.captures
337- copy(creq.Mapped, pix)
338- h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
339- }()
340- h.requests <- ports.HintScanRequest{Seq: 1, Output: "HEADLESS-1", Width: w, Height: hgt, Scale: 1, Windows: wins}
341- res := <-h.results
342- require.NoError(t, res.Err)
343- require.NotEmpty(t, res.Targets)
344-}
345-
346 // The worker keeps no frame: the mapped buffer belongs to the scan, and to the311 // The worker keeps no frame: the mapped buffer belongs to the scan, and to the
347 // dump job until it is written; every mapping is released afterwards.312 // dump job until it is written; every mapping is released afterwards.
348 func TestWorkerHoldsNoFrameAfterScan(t *testing.T) {313 func TestWorkerHoldsNoFrameAfterScan(t *testing.T) {
349 for _, dumpDir := range []string{"", filepath.Join(t.TempDir(), "hints")} {314 for _, dumpDir := range []string{"", filepath.Join(t.TempDir(), "hints")} {
350- // OCR is "installed" but its binary cannot run: only the memory matters.315+ det := newMockdetector(t)
351- w := &worker{dumpDir: dumpDir, ocr: ocrState{done: true, binary: "/nonexistent/tesseract", lang: "eng"}}316+ det.EXPECT().Detect(mock.Anything, mock.Anything).Return(nil, nil).Once()
352- requests := make(chan ports.HintScanRequest)317+ w := &worker{dumpDir: dumpDir, det: det}
353 captures := make(chan ports.CaptureRequest, 1)318 captures := make(chan ports.CaptureRequest, 1)
354 captured := make(chan ports.CaptureDone, 1)319 captured := make(chan ports.CaptureDone, 1)
355- results := make(chan ports.HintScanResult, 1)
356 go func() {320 go func() {
357 creq := <-captures321 creq := <-captures
358 captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}322 captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
359 }()323 }()
360- req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 64, H: 32}}}}324+ req := ports.HintScanRequest{Output: "o", Width: 64, Height: 32, Scale: 1}
361- res := w.serve(context.Background(), req, requests, captures, captured, results, 5*time.Second)325+ res := w.serve(context.Background(), req, make(chan ports.HintScanRequest), captures, captured, make(chan ports.HintScanResult, 1), 5*time.Second)
362 require.NoError(t, res.Err)326 require.NoError(t, res.Err)
363 if dumpDir == "" {327 if dumpDir == "" {
364 require.Nil(t, w.pending)328 require.Nil(t, w.pending)
365- require.Zero(t, w.live, "frame and crops are released before the result")329+ require.Zero(t, w.live, "the frame is released before the result")
366 } else {330 } else {
367 require.NotNil(t, w.pending)331 require.NotNil(t, w.pending)
368 require.Equal(t, 1, w.live, "only the frame, kept for the dump")332 require.Equal(t, 1, w.live, "only the frame, kept for the dump")
@@ -370,7 +334,6 @@ func TestWorkerHoldsNoFrameAfterScan(t *testing.T) {
370 w.dump(context.Background())334 w.dump(context.Background())
371 require.Nil(t, w.pending)335 require.Nil(t, w.pending)
372 require.Zero(t, w.live)336 require.Zero(t, w.live)
373- require.Empty(t, w.cache) // OCR failed: nothing cached
374 }337 }
375 }338 }
376 339
@@ -389,39 +352,6 @@ func TestTimedOutCaptureIsReleasedWhenDone(t *testing.T) {
389 require.Zero(t, w.live)352 require.Zero(t, w.live)
390 }353 }
391 354
392-// Without tesseract OCR is off for good: no capture, only accessibility targets.
393-func TestNoTesseractSkipsCapture(t *testing.T) {
394- t.Setenv("PATH", t.TempDir())
395- button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
396- covered := ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: 30, W: 5, H: 5}}
397- wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}, {ID: 2, Rect: ports.Rect{X: 20, W: 20, H: 20}}}
398- src := newMocktargetSource(t)
399- src.EXPECT().Close().Return(nil).Once() // the worker closes its source on return
400- src.EXPECT().Targets(mock.Anything, wins, false).Return([]ports.HintTarget{button, covered}, map[ports.WindowID]bool{2: true}, nil).Times(2)
401- h := startWith(t, time.Second, Options{Source: src})
402- for seq := uint64(1); seq <= 2; seq++ {
403- h.requests <- ports.HintScanRequest{Seq: seq, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: wins}
404- res := <-h.results
405- require.NoError(t, res.Err)
406- require.Equal(t, []ports.HintTarget{button, covered}, res.Targets)
407- }
408- require.Empty(t, h.captures)
409-}
410-
411-// With debug dumps on, the frame is still captured (for the dump only).
412-func TestNoTesseractDumpStillCaptures(t *testing.T) {
413- t.Setenv("PATH", t.TempDir())
414- dir := filepath.Join(t.TempDir(), "hints")
415- h := startWith(t, 5*time.Second, Options{DumpDir: dir})
416- go func() {
417- creq := <-h.captures
418- h.captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
419- }()
420- h.requests <- ports.HintScanRequest{Seq: 1, Output: "o", Width: 40, Height: 20, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 40, H: 20}}}}
421- require.NoError(t, (<-h.results).Err)
422- waitFiles(t, dir, 3)
423-}
424-
425 // A capture that completes after its scan timed out is released while the355 // A capture that completes after its scan timed out is released while the
426 // worker is idle, without another scan, and its reply does not linger.356 // worker is idle, without another scan, and its reply does not linger.
427 func TestHintLateCaptureReaped(t *testing.T) {357 func TestHintLateCaptureReaped(t *testing.T) {
D internal/adapters/hints/hybrid_test.go +0 -177
@@ -1,177 +0,0 @@
1-package hints
2-
3-import (
4- "context"
5- "hash/maphash"
6- "image"
7- "image/draw"
8- "image/png"
9- "os"
10- "os/exec"
11- "runtime"
12- "testing"
13-
14- "github.com/bnema/neferwl/internal/ports"
15- "github.com/stretchr/testify/require"
16-)
17-
18-func TestWindowCrop(t *testing.T) {
19- for _, tc := range []struct {
20- name string
21- r ports.Rect
22- scale float64
23- want image.Rectangle
24- }{
25- {"unit", ports.Rect{X: 10, Y: 20, W: 100, H: 50}, 1, image.Rect(10, 20, 110, 70)},
26- {"fractional", ports.Rect{X: 11, Y: 21, W: 100, H: 50}, 1.2, image.Rect(13, 25, 134, 86)},
27- {"off output", ports.Rect{X: -10, Y: 20, W: 100, H: 100}, 1.2, image.Rect(0, 24, 108, 100)},
28- {"outside", ports.Rect{X: 1000, Y: 20, W: 100, H: 100}, 1, image.Rectangle{}},
29- } {
30- t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.want, windowCrop(tc.r, tc.scale, 200, 100)) })
31- }
32-}
33-
34-func sourceResult(ar atspiResult) *pendingSource {
35- p := &pendingSource{done: make(chan struct{}), res: ar}
36- close(p.done)
37- return p
38-}
39-
40-func TestHybridCacheAndMerge(t *testing.T) {
41- t.Setenv("PATH", t.TempDir()) // Cached OCR must not need tesseract.
42- buf := make([]byte, 200*100*4)
43- wins := []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 100, H: 100}}, {ID: 2, Rect: ports.Rect{X: 100, W: 100, H: 100}}}
44- req := ports.HintScanRequest{Output: "o", Width: 200, Height: 100, Scale: 1, Windows: wins}
45- seed := maphash.MakeSeed()
46- r := windowCrop(wins[1].Rect, 1, 200, 100)
47- oldWorkers := ocrWorkers
48- ocrWorkers = 1 // one band covers the crop
49- t.Cleanup(func() { ocrWorkers = oldWorkers })
50- ext := image.Rect(0, 0, 100, 100)
51- entry := bandCache{hash: (&worker{cacheSeed: seed}).cropHash(buf, 800, r), ext: ext, words: []OCRWord{{Text: "example.org", Rect: image.Rect(10, 20, 90, 40)}}}
52- button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 1, Y: 1, W: 10, H: 10}}
53- link := ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: 110, Y: 20, W: 80, H: 20}}
54- uncoveredButton := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{X: 101, Y: 1, W: 10, H: 10}}
55- ar := atspiResult{targets: []ports.HintTarget{button, uncoveredButton}, covered: map[ports.WindowID]bool{1: true}}
56- w := worker{ocr: ocrState{done: true, binary: "tesseract", lang: "eng"}, cacheSeed: seed, cache: map[bandKey]bandCache{{output: "o", window: 2, band: 0, bands: 1}: entry}}
57- got := w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
58- require.Equal(t, []ports.HintTarget{button, uncoveredButton, link}, got.Targets)
59- req.LinksOnly = true
60- got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
61- require.Equal(t, []ports.HintTarget{link}, got.Targets)
62- // Changing one pixel invalidates only the crop, never leaving stale links.
63- buf[100*4]++
64- got = w.scan(context.Background(), req, frame{pix: buf, stride: 800}, 0, ports.HintScanResult{}, sourceResult(ar))
65- require.Empty(t, got.Targets)
66-}
67-
68-func TestMissingTesseractKeepsAccessibilityButtons(t *testing.T) {
69- t.Setenv("PATH", t.TempDir())
70- w := worker{}
71- button := ports.HintTarget{Kind: ports.HintButton, Rect: ports.Rect{W: 5, H: 5}}
72- req := ports.HintScanRequest{Width: 20, Height: 20, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 20, H: 20}}}}
73- got := w.scan(context.Background(), req, frame{pix: make([]byte, 1600), stride: 80}, 0, ports.HintScanResult{}, sourceResult(atspiResult{targets: []ports.HintTarget{button}}))
74- require.NoError(t, got.Err)
75- require.Equal(t, []ports.HintTarget{button}, got.Targets)
76-}
77-
78-func TestNoWindowsProducesNoPixelTargets(t *testing.T) {
79- pix, width, height := loadBGRA(t)
80- w := worker{}
81- got := w.scan(context.Background(), ports.HintScanRequest{Width: width, Height: height}, frame{pix: pix, stride: width * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
82- require.Empty(t, got.Targets)
83-}
84-
85-func TestStripMappingAndPolarity(t *testing.T) {
86- img := image.NewRGBA(image.Rect(0, 0, 200, 200))
87- strips := packLines(img, []image.Rectangle{image.Rect(20, 20, 160, 40), image.Rect(20, 80, 160, 112)}, 2, true)
88- require.Len(t, strips, 2)
89- for _, s := range strips {
90- for _, off := range s.offsets {
91- require.Equal(t, 1, off.factor)
92- require.Equal(t, uint8(255), s.img.GrayAt(off.packed.Min.X, off.packed.Min.Y).Y)
93- }
94- }
95- small := packLines(img, []image.Rectangle{image.Rect(20, 20, 100, 30)}, 1, true)
96- require.Equal(t, 2, small[0].offsets[0].factor)
97-}
98-
99-func TestCropGrayAndHash(t *testing.T) {
100- buf := make([]byte, 4*4*4)
101- for y := 0; y < 4; y++ {
102- for x := 0; x < 4; x++ {
103- copy(buf[y*16+x*4:], []byte{0, 0, 255, 255}) // BGRA red
104- }
105- }
106- g := cropGray(buf, 16, image.Rect(1, 1, 3, 3), make([]byte, 4))
107- require.Equal(t, image.Rect(0, 0, 2, 2), g.Bounds())
108- require.Equal(t, uint8(76), g.GrayAt(1, 1).Y)
109- w := worker{cacheSeed: maphash.MakeSeed()}
110- a := w.cropHash(buf, 16, image.Rect(1, 1, 3, 3))
111- buf[0]++ // outside the crop
112- require.Equal(t, a, w.cropHash(buf, 16, image.Rect(1, 1, 3, 3)))
113- buf[16+4]++
114- require.NotEqual(t, a, w.cropHash(buf, 16, image.Rect(1, 1, 3, 3)))
115-}
116-
117-func TestUpscaleBandGray(t *testing.T) {
118- g := image.NewGray(image.Rect(0, 0, 3, 2))
119- for i := range g.Pix {
120- g.Pix[i] = uint8(i + 1)
121- }
122- sub := g.SubImage(image.Rect(1, 1, 3, 2)).(*image.Gray)
123- up := upscaleBand(sub)
124- require.Equal(t, image.Rect(0, 0, 4, 2), up.Bounds())
125- require.Equal(t, []uint8{5, 5, 6, 6, 5, 5, 6, 6}, up.Pix)
126-}
127-
128-// A blinking cursor changes one band: the other bands keep their cached
129-// words, and only the changed one is recognized again.
130-func TestBandCacheSurvivesCursorBlink(t *testing.T) {
131- binary, err := exec.LookPath("tesseract")
132- if err != nil {
133- t.Skip("tesseract not on PATH")
134- }
135- old := ocrWorkers
136- ocrWorkers = 4
137- t.Cleanup(func() { ocrWorkers = old })
138- f, err := os.Open("testdata/terminal.png")
139- require.NoError(t, err)
140- defer f.Close()
141- src, err := png.Decode(f)
142- require.NoError(t, err)
143- w, h := src.Bounds().Dx(), src.Bounds().Dy()
144- rgba := image.NewRGBA(image.Rect(0, 0, w, h))
145- draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
146- buf := make([]byte, w*h*4)
147- for i := 0; i < len(buf); i += 4 {
148- buf[i], buf[i+1], buf[i+2], buf[i+3] = rgba.Pix[i+2], rgba.Pix[i+1], rgba.Pix[i], 0xff
149- }
150- _ = binary
151- wk := &worker{}
152- req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
153- first := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
154- require.NotEmpty(t, first.Targets)
155- require.Len(t, wk.cache, min(runtime.GOMAXPROCS(0), 4))
156-
157- // The same frame: every band hits.
158- again := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
159- require.Equal(t, first.Targets, again.Targets)
160-
161- // One pixel at the bottom blinks: only the last band (and the context of
162- // none other) is invalidated.
163- before := map[bandKey]uint64{}
164- for k, v := range wk.cache {
165- before[k] = v.hash
166- }
167- buf[((h-1)*w+3)*4] ^= 0xff
168- blink := wk.scan(context.Background(), req, frame{pix: buf, stride: w * 4}, 0, ports.HintScanResult{}, sourceResult(atspiResult{}))
169- require.Equal(t, first.Targets, blink.Targets)
170- changed := 0
171- for k, v := range wk.cache {
172- if before[k] != v.hash {
173- changed++
174- }
175- }
176- require.Equal(t, 1, changed)
177-}
D internal/adapters/hints/language_runner_mock_test.go +0 -130
@@ -1,130 +0,0 @@
1-// Code generated by mockery; DO NOT EDIT.
2-// github.com/vektra/mockery
3-// template: testify
4-
5-package hints
6-
7-import (
8- "context"
9-
10- mock "github.com/stretchr/testify/mock"
11-)
12-
13-// newMocklanguageRunner creates a new instance of mocklanguageRunner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
14-// The first argument is typically a *testing.T value.
15-func newMocklanguageRunner(t interface {
16- mock.TestingT
17- Cleanup(func())
18-}) *mocklanguageRunner {
19- if helper, ok := t.(interface{ Helper() }); ok {
20- helper.Helper()
21- }
22-
23- mock := &mocklanguageRunner{}
24- mock.Mock.Test(t)
25-
26- t.Cleanup(func() {
27- if helper, ok := t.(interface{ Helper() }); ok {
28- helper.Helper()
29- }
30- mock.AssertExpectations(t)
31- })
32-
33- return mock
34-}
35-
36-// mocklanguageRunner is an autogenerated mock type for the languageRunner type
37-type mocklanguageRunner struct {
38- mock.Mock
39-}
40-
41-type mocklanguageRunner_Expecter struct {
42- mock *mock.Mock
43-}
44-
45-func (_m *mocklanguageRunner) EXPECT() *mocklanguageRunner_Expecter {
46- return &mocklanguageRunner_Expecter{mock: &_m.Mock}
47-}
48-
49-// Output provides a mock function for the type mocklanguageRunner
50-func (_mock *mocklanguageRunner) Output(context1 context.Context, s string, strings ...string) ([]byte, error) {
51- var tmpRet mock.Arguments
52- if len(strings) > 0 {
53- tmpRet = _mock.Called(context1, s, strings)
54- } else {
55- tmpRet = _mock.Called(context1, s)
56- }
57- ret := tmpRet
58-
59- if len(ret) == 0 {
60- panic("no return value specified for Output")
61- }
62-
63- var r0 []byte
64- var r1 error
65- if returnFunc, ok := ret.Get(0).(func(context.Context, string, ...string) ([]byte, error)); ok {
66- return returnFunc(context1, s, strings...)
67- }
68- if returnFunc, ok := ret.Get(0).(func(context.Context, string, ...string) []byte); ok {
69- r0 = returnFunc(context1, s, strings...)
70- } else {
71- if ret.Get(0) != nil {
72- r0 = ret.Get(0).([]byte)
73- }
74- }
75- if returnFunc, ok := ret.Get(1).(func(context.Context, string, ...string) error); ok {
76- r1 = returnFunc(context1, s, strings...)
77- } else {
78- r1 = ret.Error(1)
79- }
80- return r0, r1
81-}
82-
83-// mocklanguageRunner_Output_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Output'
84-type mocklanguageRunner_Output_Call struct {
85- *mock.Call
86-}
87-
88-// Output is a helper method to define mock.On call
89-// - context1 context.Context
90-// - s string
91-// - strings ...string
92-func (_e *mocklanguageRunner_Expecter) Output(context1 any, s any, strings ...any) *mocklanguageRunner_Output_Call {
93- return &mocklanguageRunner_Output_Call{Call: _e.mock.On("Output",
94- append([]any{context1, s}, strings...)...)}
95-}
96-
97-func (_c *mocklanguageRunner_Output_Call) Run(run func(context1 context.Context, s string, strings ...string)) *mocklanguageRunner_Output_Call {
98- _c.Call.Run(func(args mock.Arguments) {
99- var arg0 context.Context
100- if args[0] != nil {
101- arg0 = args[0].(context.Context)
102- }
103- var arg1 string
104- if args[1] != nil {
105- arg1 = args[1].(string)
106- }
107- var arg2 []string
108- var variadicArgs []string
109- if len(args) > 2 {
110- variadicArgs = args[2].([]string)
111- }
112- arg2 = variadicArgs
113- run(
114- arg0,
115- arg1,
116- arg2...,
117- )
118- })
119- return _c
120-}
121-
122-func (_c *mocklanguageRunner_Output_Call) Return(bytes []byte, err error) *mocklanguageRunner_Output_Call {
123- _c.Call.Return(bytes, err)
124- return _c
125-}
126-
127-func (_c *mocklanguageRunner_Output_Call) RunAndReturn(run func(context1 context.Context, s string, strings ...string) ([]byte, error)) *mocklanguageRunner_Output_Call {
128- _c.Call.Return(run)
129- return _c
130-}
D internal/adapters/hints/ocr.go +0 -460
@@ -1,460 +0,0 @@
1-package hints
2-
3-import (
4- "bufio"
5- "bytes"
6- "context"
7- "fmt"
8- "image"
9- "image/png"
10- "io"
11- "math"
12- "net"
13- "os"
14- "os/exec"
15- "path/filepath"
16- "regexp"
17- "runtime"
18- "sort"
19- "strconv"
20- "strings"
21- "sync"
22- "time"
23-
24- "github.com/bnema/neferwl/internal/adapters/logging"
25- "github.com/bnema/neferwl/internal/ports"
26-)
27-
28-// OCRWord is a recognized word in physical output coordinates.
29-type OCRWord struct {
30- Text string `json:"text"`
31- Rect image.Rectangle `json:"rect"`
32- Confidence float64 `json:"confidence"`
33- // Underlined is set when a thin line runs under the word. UnderL and
34- // UnderR are the x extent of that line (crop-local), which can reach the
35- // neighbouring words of one underlined link.
36- Underlined bool `json:"underlined,omitempty"`
37- UnderL int `json:"under_l,omitempty"`
38- UnderR int `json:"under_r,omitempty"`
39-}
40-
41-var languageWarning sync.Once
42-var failureWarning sync.Once
43-
44-// languageRunner bounds the external language-discovery process independently
45-// of recognition, which has not started yet when discovery runs.
46-type languageRunner interface {
47- Output(context.Context, string, ...string) ([]byte, error)
48-}
49-
50-type execLanguageRunner struct{}
51-
52-func (execLanguageRunner) Output(ctx context.Context, binary string, args ...string) ([]byte, error) {
53- return exec.CommandContext(ctx, binary, args...).Output()
54-}
55-
56-const languageTimeout = 5 * time.Second
57-
58-func ocrLanguage(ctx context.Context, binary string) (string, error) {
59- return discoverLanguage(ctx, binary, execLanguageRunner{}, languageTimeout)
60-}
61-
62-func discoverLanguage(ctx context.Context, binary string, runner languageRunner, timeout time.Duration) (string, error) {
63- dir := os.Getenv("TESSDATA_PREFIX")
64- if dir == "" {
65- dir = "/usr/share/tessdata"
66- }
67- if _, err := os.Stat(filepath.Join(dir, "eng.traineddata")); err == nil {
68- return "eng", nil
69- }
70- ctx, cancel := context.WithTimeout(ctx, timeout)
71- defer cancel()
72- out, err := runner.Output(ctx, binary, "--list-langs")
73- if err != nil {
74- return "", err
75- }
76- for _, line := range strings.Split(string(out), "\n")[1:] {
77- lang := strings.TrimSpace(line)
78- if lang != "" && lang != "osd" {
79- languageWarning.Do(func() {
80- log := logging.For(ctx, "hints")
81- log.Warn().Str("language", lang).Msg("English OCR data missing; install tesseract-data-eng")
82- })
83- return lang, nil
84- }
85- }
86- return "", fmt.Errorf("no OCR language available")
87-}
88-
89-func parseTSV(data []byte, offset image.Point) ([]OCRWord, error) {
90- var words []OCRWord
91- scanner := bufio.NewScanner(bytes.NewReader(data))
92- scanner.Buffer(make([]byte, 4096), 1024*1024)
93- for scanner.Scan() {
94- fields := strings.SplitN(scanner.Text(), "\t", 12)
95- if len(fields) != 12 || fields[0] != "5" {
96- continue
97- }
98- var values [4]int
99- for i := range values {
100- v, err := strconv.Atoi(fields[6+i])
101- if err != nil {
102- return nil, fmt.Errorf("OCR TSV coordinate: %w", err)
103- }
104- values[i] = v
105- }
106- conf, err := strconv.ParseFloat(fields[10], 64)
107- if err != nil {
108- return nil, fmt.Errorf("OCR TSV confidence: %w", err)
109- }
110- if values[2] <= 0 || values[3] <= 0 || strings.TrimSpace(fields[11]) == "" {
111- continue
112- }
113- r := image.Rect(values[0], values[1], values[0]+values[2], values[1]+values[3]).Add(offset)
114- words = append(words, OCRWord{Text: fields[11], Rect: r, Confidence: conf})
115- }
116- return words, scanner.Err()
117-}
118-
119-// ocrWorkers is the maximum number of bands recognized in parallel. 4 keeps
120-// the tesseract processes at about 60% of the memory of 8 for the same time.
121-var ocrWorkers = 4
122-
123-// pngLevel is the compression of non-grayscale images piped to tesseract.
124-var pngLevel = png.BestSpeed
125-
126-// rawPGM feeds grayscale images to tesseract as binary PGM instead of PNG.
127-var rawPGM = true
128-
129-// ocrState is the tesseract installation, resolved once.
130-type ocrState struct {
131- done bool
132- binary string
133- lang string
134- err error
135-}
136-
137-func (s *ocrState) resolve(ctx context.Context) error {
138- if !s.done {
139- s.done = true
140- if s.binary, s.err = exec.LookPath("tesseract"); s.err == nil {
141- s.lang, s.err = ocrLanguage(ctx, s.binary)
142- }
143- }
144- return s.err
145-}
146-
147-// ocrReady resolves tesseract on the first call. When it is missing, OCR is
148-// disabled for the life of the process and the warning is logged once.
149-func (w *worker) ocrReady(ctx context.Context) bool {
150- if err := w.ocr.resolve(ctx); err != nil {
151- failureWarning.Do(func() {
152- log := logging.For(ctx, "hints")
153- log.Warn().Err(err).Msg("OCR unavailable; install tesseract and tesseract-data-eng")
154- })
155- return false
156- }
157- return true
158-}
159-
160-// band is a horizontal slice of a window crop recognized by one tesseract
161-// process. It reads rows [top,bottom) and owns the words centred in
162-// [ownTop,ownBottom); the margin only supplies context.
163-type band struct {
164- idx, n int
165- top, bottom int
166- ownTop, ownBottom int
167-}
168-
169-func bandsOf(height int) []band {
170- n := min(runtime.GOMAXPROCS(0), ocrWorkers, height)
171- bands := make([]band, 0, max(n, 0))
172- for i := range n {
173- ownTop, ownBottom := i*height/n, (i+1)*height/n
174- bands = append(bands, band{idx: i, n: n, top: max(0, ownTop-40), bottom: min(height, ownBottom+40), ownTop: ownTop, ownBottom: ownBottom})
175- }
176- return bands
177-}
178-
179-// recognize reads the bands of jobs that missed the cache, in parallel.
180-func (w *worker) recognize(ctx context.Context, jobs []*bandJob) {
181- ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
182- defer cancel()
183- // One scan-wide pool: at most ocrWorkers tesseract processes run at once,
184- // whatever the number of windows.
185- sem := make(chan struct{}, max(ocrWorkers, 1))
186- var wg sync.WaitGroup
187- for _, j := range jobs {
188- if j.hit {
189- continue
190- }
191- select {
192- case sem <- struct{}{}:
193- case <-ctx.Done():
194- j.err = ctx.Err()
195- continue
196- }
197- wg.Go(func() {
198- defer func() { <-sem }()
199- j.words, j.err = readBand(ctx, &w.ocr, j)
200- })
201- }
202- wg.Wait()
203-}
204-
205-// readBand recognizes one band; the words are crop-local and owned by it.
206-func readBand(ctx context.Context, st *ocrState, j *bandJob) ([]OCRWord, error) {
207- words, err := recognizeBand(ctx, st.binary, st.lang, j.gray)
208- if err != nil {
209- return nil, err
210- }
211- heights := make([]int, 0, len(words))
212- for _, word := range words {
213- if len(word.Text) >= 3 {
214- heights = append(heights, word.Rect.Dy())
215- }
216- }
217- sort.Ints(heights)
218- if len(heights) > 0 && heights[len(heights)/2] < 12 {
219- words, err = recognizeBand(ctx, st.binary, st.lang, upscaleBand(j.gray))
220- if err != nil {
221- return nil, err
222- }
223- for i := range words {
224- r := words[i].Rect
225- words[i].Rect = image.Rect(r.Min.X/2, r.Min.Y/2, (r.Max.X+1)/2, (r.Max.Y+1)/2)
226- }
227- }
228- markUnderlined(j.gray, words)
229- owned := words[:0]
230- for _, word := range words {
231- // Overlap supplies context, but each word belongs to exactly one band.
232- word.Rect = word.Rect.Add(image.Pt(0, j.ext.Min.Y))
233- if center := (word.Rect.Min.Y + word.Rect.Max.Y) / 2; center >= j.own[0] && center < j.own[1] {
234- owned = append(owned, word)
235- }
236- }
237- return owned, nil
238-}
239-
240-// readWords recognizes the words of img, a window crop, in parallel bands.
241-func readWords(ctx context.Context, img *image.Gray) ([]OCRWord, int, error) {
242- w := &worker{}
243- if err := w.ocr.resolve(ctx); err != nil {
244- return nil, 0, err
245- }
246- var jobs []*bandJob
247- for _, b := range bandsOf(img.Bounds().Dy()) {
248- ext := image.Rect(0, b.top, img.Bounds().Dx(), b.bottom)
249- jobs = append(jobs, &bandJob{ext: ext, own: [2]int{b.ownTop, b.ownBottom}, gray: img.SubImage(ext).(*image.Gray)})
250- }
251- w.recognize(ctx, jobs)
252- var words []OCRWord
253- for _, j := range jobs {
254- if j.err != nil {
255- return nil, len(jobs), j.err
256- }
257- for _, word := range j.words {
258- if !duplicateWord(words, word) {
259- words = append(words, word)
260- }
261- }
262- }
263- return words, len(jobs), nil
264-}
265-
266-// The link patterns compile on the first OCR link check, not at process
267-// start: with hint mode off nothing here costs memory or CPU.
268-var domainPattern = sync.OnceValue(func() *regexp.Regexp {
269- return regexp.MustCompile(`(?i)^(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+([a-z]{2,24})(?::[0-9]+)?(?:[/?#]\S*)?$`)
270-})
271-var schemePattern = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`(?i)^(?:https?|htps|file|ftp):/{1,2}\S+$`) })
272-var emailPattern = sync.OnceValue(func() *regexp.Regexp {
273- return regexp.MustCompile(`(?i)^[a-z0-9.!#$%&'*+/=?^_` + "`" + `{|}~-]+@(?:[a-z0-9-]+\.)+[a-z]{2,24}$`)
274-})
275-var fileExtensions = sync.OnceValue(func() []string {
276- return strings.Fields("go md txt toml json yaml yml rs py js ts sh c h png jpg log conf lock mod sum html css")
277-})
278-
279-func isTextLink(text string) bool {
280- text = strings.Trim(text, "()[]<>.,;:'\"")
281- if schemePattern().MatchString(text) || (strings.HasPrefix(strings.ToLower(text), "www.") && len(text) > 4) || emailPattern().MatchString(text) {
282- return true
283- }
284- host := text
285- if h, _, err := net.SplitHostPort(text); err == nil {
286- host = h
287- }
288- if ip := net.ParseIP(host); ip != nil && ip.To4() != nil {
289- return true
290- }
291- m := domainPattern().FindStringSubmatch(text)
292- if m == nil {
293- return false
294- }
295- for _, ext := range fileExtensions() {
296- if strings.EqualFold(m[1], ext) {
297- return false
298- }
299- }
300- return true
301-}
302-
303-func textLinks(words []OCRWord, scale float64) []ports.HintTarget {
304- if scale <= 0 {
305- scale = 1
306- }
307- words = append([]OCRWord(nil), words...)
308- sort.Slice(words, func(i, j int) bool {
309- if words[i].Rect.Min.Y != words[j].Rect.Min.Y {
310- return words[i].Rect.Min.Y < words[j].Rect.Min.Y
311- }
312- return words[i].Rect.Min.X < words[j].Rect.Min.X
313- })
314- var out []ports.HintTarget
315- var urls []image.Rectangle // boxes of the URL targets, in pixels
316- used := make([]bool, len(words))
317- for i, word := range words {
318- if used[i] {
319- continue
320- }
321- text, rect := word.Text, word.Rect
322- // Grow only touching words on the same baseline; ordinary prose is not joined.
323- for {
324- next := -1
325- for j, candidate := range words {
326- if j == i || used[j] || candidate.Rect.Min.X < rect.Max.X-2 {
327- continue
328- }
329- gap := candidate.Rect.Min.X - rect.Max.X
330- if gap > max(3, min(rect.Dy(), candidate.Rect.Dy())/5) {
331- continue
332- }
333- if abs(candidate.Rect.Max.Y-rect.Max.Y) > max(3, rect.Dy()/3) {
334- continue
335- }
336- if next < 0 || candidate.Rect.Min.X < words[next].Rect.Min.X {
337- next = j
338- }
339- }
340- if next < 0 {
341- break
342- }
343- text += words[next].Text
344- rect = rect.Union(words[next].Rect)
345- used[next] = true
346- }
347- if !isTextLink(text) {
348- continue
349- }
350- x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
351- right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
352- out = append(out, ports.HintTarget{Kind: ports.HintLink, Text: true, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
353- urls = append(urls, rect)
354- }
355- return append(out, underlinedLinks(words, urls, scale)...)
356-}
357-
358-func abs(x int) int {
359- if x < 0 {
360- return -x
361- }
362- return x
363-}
364-func overlap(a, b image.Rectangle) float64 {
365- intersection := a.Intersect(b)
366- area := min(a.Dx()*a.Dy(), b.Dx()*b.Dy())
367- if intersection.Empty() || area <= 0 {
368- return 0
369- }
370- return float64(intersection.Dx()*intersection.Dy()) / float64(area)
371-}
372-func targetRect(t ports.HintTarget) image.Rectangle {
373- return image.Rect(t.Rect.X, t.Rect.Y, t.Rect.X+t.Rect.W, t.Rect.Y+t.Rect.H)
374-}
375-func recognizeBand(ctx context.Context, binary, lang string, img image.Image) ([]OCRWord, error) {
376- return recognizeImage(ctx, binary, lang, img, "11", false)
377-}
378-
379-// pgmReader streams a grayscale image as binary PGM (P5), row by row.
380-type pgmReader struct {
381- head []byte
382- img *image.Gray
383- y int
384- row []byte
385-}
386-
387-func newPGMReader(img *image.Gray) *pgmReader {
388- b := img.Bounds()
389- return &pgmReader{head: fmt.Appendf(nil, "P5\n%d %d\n255\n", b.Dx(), b.Dy()), img: img}
390-}
391-
392-func (r *pgmReader) Read(p []byte) (int, error) {
393- n := 0
394- for n < len(p) {
395- if len(r.head) > 0 {
396- c := copy(p[n:], r.head)
397- r.head, n = r.head[c:], n+c
398- continue
399- }
400- if len(r.row) == 0 {
401- b := r.img.Bounds()
402- if r.y >= b.Dy() {
403- break
404- }
405- r.row = r.img.Pix[(r.y+b.Min.Y-r.img.Rect.Min.Y)*r.img.Stride+(b.Min.X-r.img.Rect.Min.X):][:b.Dx()]
406- r.y++
407- }
408- c := copy(p[n:], r.row)
409- r.row, n = r.row[c:], n+c
410- }
411- if n == 0 {
412- return 0, io.EOF
413- }
414- return n, nil
415-}
416-
417-func recognizeImage(ctx context.Context, binary, lang string, img image.Image, psm string, normalized bool) ([]OCRWord, error) {
418- args := []string{"stdin", "stdout", "--psm", psm, "-l", lang}
419- if normalized {
420- args = append(args, "-c", "tessedit_do_invert=0")
421- }
422- args = append(args, "tsv")
423- cmd := exec.CommandContext(ctx, binary, args...)
424- cmd.Env = append(os.Environ(), "OMP_THREAD_LIMIT=1")
425- if g, ok := img.(*image.Gray); ok && rawPGM {
426- // Raw grayscale: no deflate, no encoder buffers.
427- cmd.Stdin = newPGMReader(g)
428- } else {
429- // The PNG streams into tesseract; no encoded copy is held.
430- pr, pw := io.Pipe()
431- defer func() { _ = pr.Close() }()
432- go func() {
433- enc := png.Encoder{CompressionLevel: pngLevel}
434- _ = pw.CloseWithError(enc.Encode(pw, img))
435- }()
436- cmd.Stdin = pr
437- }
438- var stderr bytes.Buffer
439- cmd.Stderr = &stderr
440- output, err := cmd.Output()
441- if err != nil {
442- return nil, fmt.Errorf("tesseract: %w: %s", err, stderr.String())
443- }
444- return parseTSV(output, image.Point{})
445-}
446-
447-// upscaleBand doubles a grayscale band with nearest-neighbour sampling.
448-func upscaleBand(src *image.Gray) *image.Gray {
449- b := src.Bounds()
450- dst := image.NewGray(image.Rect(0, 0, b.Dx()*2, b.Dy()*2))
451- for y := 0; y < b.Dy(); y++ {
452- row := src.Pix[(y+b.Min.Y-src.Rect.Min.Y)*src.Stride+(b.Min.X-src.Rect.Min.X):][:b.Dx()]
453- top := dst.Pix[2*y*dst.Stride:][:dst.Stride]
454- for x, v := range row {
455- top[2*x], top[2*x+1] = v, v
456- }
457- copy(dst.Pix[(2*y+1)*dst.Stride:], top)
458- }
459- return dst
460-}
D internal/adapters/hints/ocr_test.go +0 -272
@@ -1,272 +0,0 @@
1-package hints
2-
3-import (
4- "context"
5- "errors"
6- "image"
7- "image/draw"
8- "image/png"
9- "io"
10- "os"
11- "os/exec"
12- "path/filepath"
13- "reflect"
14- "strings"
15- "testing"
16- "time"
17-
18- "github.com/stretchr/testify/mock"
19-
20- "github.com/bnema/neferwl/internal/ports"
21-)
22-
23-func TestTextLink(t *testing.T) {
24- yes := []string{"ermite.bnema.dev", "google.fr", "example.org/path?q=1", "https.google.fr", "https://example.org", "file:///tmp/main.go", "ftp://host/file", "www.main.go", "user@example.org", "127.0.0.1", "192.168.1.1:8080", "(https://example.org),", "htps://google.fr", "https:/example.org"}
25- no := []string{"ermite-runner", "main.go", "README.md", "v1.2.3", "1.5", "e.g.", "report.txt", "source.rs", "alpha", "999.999.999.999"}
26- for _, s := range yes {
27- t.Run(s, func(t *testing.T) {
28- if !isTextLink(s) {
29- t.Errorf("not accepted: %s", s)
30- }
31- })
32- }
33- for _, s := range no {
34- t.Run(s, func(t *testing.T) {
35- if isTextLink(s) {
36- t.Errorf("incorrectly accepted: %s", s)
37- }
38- })
39- }
40-}
41-func TestParseTSV(t *testing.T) {
42- data := "level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext\n5\t1\t1\t1\t1\t1\t10\t20\t30\t12\t91.5\texample.org\n"
43- words, err := parseTSV([]byte(data), image.Pt(0, 40))
44- if err != nil || len(words) != 1 {
45- t.Fatalf("%v %v", words, err)
46- }
47- if words[0].Rect != image.Rect(10, 60, 40, 72) || words[0].Confidence != 91.5 {
48- t.Fatal(words)
49- }
50- _, err = parseTSV([]byte(strings.ReplaceAll(data, "\t10\t", "\tbad\t")), image.Point{})
51- if err == nil {
52- t.Fatal("expected malformed coordinate error")
53- }
54-}
55-func TestJoinedOCRLink(t *testing.T) {
56- words := []OCRWord{{Text: "https:/", Rect: image.Rect(20, 20, 90, 40)}, {Text: "/example.org", Rect: image.Rect(91, 20, 200, 40)}}
57- got := textLinks(words, 2)
58- if len(got) != 1 || got[0].Rect != (ports.Rect{X: 10, Y: 10, W: 90, H: 10}) {
59- t.Fatal(got)
60- }
61-}
62-func TestRealOCR(t *testing.T) {
63- if _, err := exec.LookPath("tesseract"); err != nil {
64- t.Skip("tesseract not on PATH")
65- }
66- for _, name := range []string{"browser", "terminal"} {
67- t.Run(name, func(t *testing.T) {
68- f, err := os.Open("testdata/" + name + ".png")
69- if err != nil {
70- t.Fatal(err)
71- }
72- defer f.Close()
73- src, err := png.Decode(f)
74- if err != nil {
75- t.Fatal(err)
76- }
77- img := image.NewRGBA(src.Bounds())
78- draw.Draw(img, img.Bounds(), src, src.Bounds().Min, draw.Src)
79- start := time.Now()
80- words, bands, err := readWords(context.Background(), toGray(img))
81- if err != nil {
82- t.Fatal(err)
83- }
84- links := textLinks(words, 1)
85- t.Logf("%s %dx%d ocr_ms=%.1f words=%d links=%d bands=%d", name, img.Bounds().Dx(), img.Bounds().Dy(), ms(time.Since(start)), len(words), len(links), bands)
86- for _, w := range words {
87- t.Logf("%s %v", w.Text, w.Rect)
88- }
89- if name == "terminal" {
90- for _, r := range []image.Rectangle{image.Rect(160, 65, 380, 85), image.Rect(140, 90, 490, 110), image.Rect(95, 115, 630, 138), image.Rect(120, 140, 330, 163), image.Rect(385, 140, 740, 163)} {
91- found := false
92- for _, l := range links {
93- if overlap(targetRect(l), r) > .5 {
94- found = true
95- }
96- }
97- if !found {
98- t.Errorf("missing terminal URL near %v", r)
99- }
100- }
101- if len(links) != 5 {
102- t.Errorf("want 5 terminal links, got %d: %v", len(links), links)
103- }
104- } else if len(links) == 0 {
105- t.Error("no browser links")
106- }
107- })
108- }
109-}
110-
111-func TestSmallGlyphOCR(t *testing.T) {
112- binary, err := exec.LookPath("tesseract")
113- if err != nil {
114- t.Skip("tesseract not on PATH")
115- }
116- f, err := os.Open("testdata/terminal.png")
117- if err != nil {
118- t.Fatal(err)
119- }
120- defer f.Close()
121- src, err := png.Decode(f)
122- if err != nil {
123- t.Fatal(err)
124- }
125- small := image.NewGray(image.Rect(0, 0, 640, 200))
126- for y := 0; y < 200; y++ {
127- for x := 0; x < 640; x++ {
128- small.Set(x, y, src.At(x*2, y*2))
129- }
130- }
131- lang, err := ocrLanguage(context.Background(), binary)
132- if err != nil {
133- t.Fatal(err)
134- }
135- for _, factor := range []int{1, 2} {
136- var img image.Image = small
137- if factor == 2 {
138- img = upscaleBand(small)
139- }
140- start := time.Now()
141- words, err := recognizeBand(context.Background(), binary, lang, img)
142- if err != nil {
143- t.Fatal(err)
144- }
145- t.Logf("glyph experiment %dx: %.1f ms, words=%d links=%d", factor, ms(time.Since(start)), len(words), len(textLinks(words, 1)))
146- }
147-}
148-
149-func TestPGMReader(t *testing.T) {
150- g := image.NewGray(image.Rect(0, 0, 3, 3))
151- for i := range g.Pix {
152- g.Pix[i] = uint8(i + 1)
153- }
154- sub := g.SubImage(image.Rect(1, 1, 3, 3)).(*image.Gray)
155- got, err := io.ReadAll(newPGMReader(sub))
156- if err != nil {
157- t.Fatal(err)
158- }
159- if want := "P5\n2 2\n255\n" + string([]byte{5, 6, 8, 9}); string(got) != want {
160- t.Fatalf("%q", got)
161- }
162-}
163-
164-// Tesseract must read the raw PGM exactly as it reads the PNG.
165-func TestPGMMatchesPNG(t *testing.T) {
166- binary, err := exec.LookPath("tesseract")
167- if err != nil {
168- t.Skip("tesseract not on PATH")
169- }
170- f, err := os.Open("testdata/terminal.png")
171- if err != nil {
172- t.Fatal(err)
173- }
174- defer f.Close()
175- src, err := png.Decode(f)
176- if err != nil {
177- t.Fatal(err)
178- }
179- rgba := image.NewRGBA(src.Bounds())
180- draw.Draw(rgba, rgba.Bounds(), src, src.Bounds().Min, draw.Src)
181- gray := toGray(rgba)
182- lang, err := ocrLanguage(context.Background(), binary)
183- if err != nil {
184- t.Fatal(err)
185- }
186- defer func(v bool) { rawPGM = v }(rawPGM)
187- var res [2][]OCRWord
188- for i, raw := range []bool{false, true} {
189- rawPGM = raw
190- if res[i], err = recognizeBand(context.Background(), binary, lang, gray); err != nil {
191- t.Fatal(err)
192- }
193- }
194- if len(res[0]) == 0 || !reflect.DeepEqual(res[0], res[1]) {
195- t.Fatalf("PNG %d words, PGM %d words", len(res[0]), len(res[1]))
196- }
197-}
198-
199-// TestRecognizeCapsProcesses runs the bands of several windows through a
200-// stand-in tesseract that logs when each process starts and ends.
201-func TestRecognizeCapsProcesses(t *testing.T) {
202- dir := t.TempDir()
203- logPath := filepath.Join(dir, "log")
204- script := filepath.Join(dir, "tesseract")
205- body := "#!/bin/sh\ncat >/dev/null\necho s >> " + logPath + "\nsleep 0.1\necho e >> " + logPath + "\nprintf 'level\\tpage_num\\tblock_num\\tpar_num\\tline_num\\tword_num\\tleft\\ttop\\twidth\\theight\\tconf\\ttext\\n'\n"
206- if err := os.WriteFile(script, []byte(body), 0o700); err != nil {
207- t.Fatal(err)
208- }
209- old := ocrWorkers
210- ocrWorkers = 2
211- t.Cleanup(func() { ocrWorkers = old })
212- w := &worker{ocr: ocrState{done: true, binary: script, lang: "eng"}}
213- var jobs []*bandJob
214- for range 8 { // several windows' bands
215- g := image.NewGray(image.Rect(0, 0, 8, 8))
216- jobs = append(jobs, &bandJob{ext: g.Rect, own: [2]int{0, 8}, gray: g})
217- }
218- w.recognize(context.Background(), jobs)
219- data, err := os.ReadFile(logPath)
220- if err != nil {
221- t.Fatal(err)
222- }
223- cur, peak, starts := 0, 0, 0
224- for _, l := range strings.Fields(string(data)) {
225- if l == "s" {
226- cur++
227- starts++
228- peak = max(peak, cur)
229- } else {
230- cur--
231- }
232- }
233- if starts != 8 || peak > 2 {
234- t.Fatalf("starts=%d peak=%d", starts, peak)
235- }
236- for _, j := range jobs {
237- if j.err != nil {
238- t.Fatal(j.err)
239- }
240- }
241-}
242-
243-func TestLanguageDiscoveryTimeout(t *testing.T) {
244- t.Setenv("TESSDATA_PREFIX", t.TempDir())
245- runner := newMocklanguageRunner(t)
246- parent := context.Background()
247- runner.EXPECT().Output(mock.Anything, "tesseract", []string{"--list-langs"}).RunAndReturn(
248- func(ctx context.Context, _ string, _ ...string) ([]byte, error) {
249- deadline, ok := ctx.Deadline()
250- if !ok || time.Until(deadline) > languageTimeout {
251- t.Fatal("language discovery has no bounded deadline")
252- }
253- <-ctx.Done()
254- return nil, ctx.Err()
255- }).Once()
256- lang, err := discoverLanguage(parent, "tesseract", runner, 0)
257- if lang != "" || !errors.Is(err, context.DeadlineExceeded) || parent.Err() != nil {
258- t.Fatalf("language=%q error=%v parent=%v", lang, err, parent.Err())
259- }
260-}
261-
262-func TestLanguageDiscoveryFailureKeepsAccessibilityOnly(t *testing.T) {
263- w := &worker{ocr: ocrState{done: true, err: context.DeadlineExceeded}}
264- ar := atspiResult{targets: []ports.HintTarget{{Window: 1, Rect: ports.Rect{W: 20, H: 10}}}}
265- req := ports.HintScanRequest{Width: 100, Height: 100, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: 100, H: 100}}}}
266- if w.needFrame(context.Background(), req, ar) {
267- t.Fatal("timed out discovery must not require an OCR capture")
268- }
269- if err := w.ocr.resolve(context.Background()); !errors.Is(err, context.DeadlineExceeded) {
270- t.Fatalf("discovery failure was not retained: %v", err)
271- }
272-}
D internal/adapters/hints/scan_bench_test.go +0 -49
@@ -1,49 +0,0 @@
1-package hints
2-
3-import (
4- "context"
5- "os"
6- "os/exec"
7- "runtime"
8- "testing"
9- "time"
10-
11- "github.com/bnema/neferwl/internal/ports"
12-)
13-
14-// BenchmarkScan runs whole scans of a 5120x2160 frame with a noisy window
15-// (the cache never hits) and reports the Go heap bytes allocated per scan.
16-func BenchmarkScan(b *testing.B) {
17- if _, err := exec.LookPath("tesseract"); err != nil {
18- b.Skip("tesseract not on PATH")
19- }
20- const w, h = 5120, 2160
21- ctx := context.Background()
22- w0 := &worker{}
23- requests := make(chan ports.HintScanRequest)
24- captures := make(chan ports.CaptureRequest, 1)
25- captured := make(chan ports.CaptureDone, 1)
26- results := make(chan ports.HintScanResult, 1)
27- req := ports.HintScanRequest{Output: "o", Width: w, Height: h, Scale: 1, Windows: []ports.HintWindow{{ID: 1, Rect: ports.Rect{W: w, H: h}}}}
28- if os.Getenv("NEFERWL_HINT_BENCH_PGM") == "0" {
29- rawPGM = false
30- }
31- b.ReportAllocs()
32- var peak uint64
33- for i := range b.N {
34- go func() {
35- creq := <-captures
36- creq.Mapped[0] = byte(i) // each scan differs: no cache hit
37- creq.Mapped[1] = byte(i >> 8)
38- captured <- ports.CaptureDone{ID: creq.ID, Output: creq.Output}
39- }()
40- res := w0.serve(ctx, req, requests, captures, captured, results, 5*time.Second)
41- if res.Err != nil {
42- b.Fatal(res.Err)
43- }
44- var m runtime.MemStats
45- runtime.ReadMemStats(&m)
46- peak = max(peak, m.HeapSys-m.HeapReleased)
47- }
48- b.ReportMetric(float64(peak)/1e6, "heapMB")
49-}
D internal/adapters/hints/strips_test.go +0 -124
@@ -1,124 +0,0 @@
1-package hints
2-
3-import (
4- "context"
5- "image"
6- "image/color"
7- "image/draw"
8- "os/exec"
9- "runtime"
10- "sort"
11- "sync"
12- "time"
13-)
14-
15-type stripOffset struct {
16- source, packed image.Rectangle
17- factor int
18-}
19-type ocrStrip struct {
20- img *image.Gray
21- offsets []stripOffset
22-}
23-
24-func packLines(src *image.RGBA, lines []image.Rectangle, workers int, normalize bool) []ocrStrip {
25- n := min(workers, len(lines))
26- if n == 0 {
27- return nil
28- }
29- groups := make([][]image.Rectangle, n)
30- areas := make([]int, n)
31- lines = append([]image.Rectangle(nil), lines...)
32- sort.Slice(lines, func(i, j int) bool { return lines[i].Dx()*lines[i].Dy() > lines[j].Dx()*lines[j].Dy() })
33- for _, r := range lines {
34- best := 0
35- for i := range n {
36- if areas[i] < areas[best] {
37- best = i
38- }
39- }
40- groups[best] = append(groups[best], r)
41- areas[best] += r.Dx() * r.Dy()
42- }
43- strips := make([]ocrStrip, n)
44- for i, group := range groups {
45- width, height := 0, 8
46- for _, r := range group {
47- factor := 1
48- if r.Dy() < 12 {
49- factor = 2
50- }
51- crop := r.Inset(-4).Intersect(src.Bounds())
52- packed := image.Rect(4, height, 4+crop.Dx()*factor, height+crop.Dy()*factor)
53- strips[i].offsets = append(strips[i].offsets, stripOffset{crop, packed, factor})
54- width = max(width, packed.Max.X+4)
55- height = packed.Max.Y + 8
56- }
57- dst := image.NewGray(image.Rect(0, 0, width, height))
58- draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
59- for _, off := range strips[i].offsets {
60- // The padded perimeter estimates the background without depending on text colour.
61- var sum, count int
62- r := off.source
63- for x := r.Min.X; x < r.Max.X; x++ {
64- sum += int(color.GrayModel.Convert(src.At(x, r.Min.Y)).(color.Gray).Y)
65- count++
66- }
67- invert := normalize && count > 0 && sum/count < 128
68- for y := off.packed.Min.Y; y < off.packed.Max.Y; y++ {
69- for x := off.packed.Min.X; x < off.packed.Max.X; x++ {
70- v := color.GrayModel.Convert(src.At(r.Min.X+(x-off.packed.Min.X)/off.factor, r.Min.Y+(y-off.packed.Min.Y)/off.factor)).(color.Gray).Y
71- if invert {
72- v = 255 - v
73- }
74- dst.SetGray(x, y, color.Gray{Y: v})
75- }
76- }
77- }
78- strips[i].img = dst
79- }
80- return strips
81-}
82-
83-func readStrips(ctx context.Context, img *image.RGBA, lines []image.Rectangle, workers int, normalize bool) ([]OCRWord, int, error) {
84- binary, err := exec.LookPath("tesseract")
85- if err != nil {
86- return nil, 0, err
87- }
88- ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
89- defer cancel()
90- lang, err := ocrLanguage(ctx, binary)
91- if err != nil {
92- return nil, 0, err
93- }
94- strips := packLines(img, lines, min(workers, runtime.GOMAXPROCS(0), 16), normalize)
95- results := make([][]OCRWord, len(strips))
96- errs := make([]error, len(strips))
97- var wg sync.WaitGroup
98- for i, s := range strips {
99- wg.Go(func() {
100- words, e := recognizeImage(ctx, binary, lang, s.img, "6", normalize)
101- errs[i] = e
102- for _, w := range words {
103- center := image.Pt((w.Rect.Min.X+w.Rect.Max.X)/2, (w.Rect.Min.Y+w.Rect.Max.Y)/2)
104- for _, off := range s.offsets {
105- if center.In(off.packed) {
106- r := w.Rect.Sub(off.packed.Min)
107- w.Rect = image.Rect(r.Min.X/off.factor, r.Min.Y/off.factor, (r.Max.X+off.factor-1)/off.factor, (r.Max.Y+off.factor-1)/off.factor).Add(off.source.Min)
108- results[i] = append(results[i], w)
109- break
110- }
111- }
112- }
113- })
114- }
115- wg.Wait()
116- var words []OCRWord
117- for i := range strips {
118- if errs[i] != nil {
119- return nil, len(strips), errs[i]
120- }
121- words = append(words, results[i]...)
122- }
123- return words, len(strips), nil
124-}
D internal/adapters/hints/target_source_mock_test.go +0 -174
@@ -1,174 +0,0 @@
1-// Code generated by mockery; DO NOT EDIT.
2-// github.com/vektra/mockery
3-// template: testify
4-
5-package hints
6-
7-import (
8- "context"
9-
10- "github.com/bnema/neferwl/internal/ports"
11- mock "github.com/stretchr/testify/mock"
12-)
13-
14-// newMocktargetSource creates a new instance of mocktargetSource. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
15-// The first argument is typically a *testing.T value.
16-func newMocktargetSource(t interface {
17- mock.TestingT
18- Cleanup(func())
19-}) *mocktargetSource {
20- if helper, ok := t.(interface{ Helper() }); ok {
21- helper.Helper()
22- }
23-
24- mock := &mocktargetSource{}
25- mock.Mock.Test(t)
26-
27- t.Cleanup(func() {
28- if helper, ok := t.(interface{ Helper() }); ok {
29- helper.Helper()
30- }
31- mock.AssertExpectations(t)
32- })
33-
34- return mock
35-}
36-
37-// mocktargetSource is an autogenerated mock type for the targetSource type
38-type mocktargetSource struct {
39- mock.Mock
40-}
41-
42-type mocktargetSource_Expecter struct {
43- mock *mock.Mock
44-}
45-
46-func (_m *mocktargetSource) EXPECT() *mocktargetSource_Expecter {
47- return &mocktargetSource_Expecter{mock: &_m.Mock}
48-}
49-
50-// Close provides a mock function for the type mocktargetSource
51-func (_mock *mocktargetSource) Close() error {
52- ret := _mock.Called()
53-
54- if len(ret) == 0 {
55- panic("no return value specified for Close")
56- }
57-
58- var r0 error
59- if returnFunc, ok := ret.Get(0).(func() error); ok {
60- r0 = returnFunc()
61- } else {
62- r0 = ret.Error(0)
63- }
64- return r0
65-}
66-
67-// mocktargetSource_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
68-type mocktargetSource_Close_Call struct {
69- *mock.Call
70-}
71-
72-// Close is a helper method to define mock.On call
73-func (_e *mocktargetSource_Expecter) Close() *mocktargetSource_Close_Call {
74- return &mocktargetSource_Close_Call{Call: _e.mock.On("Close")}
75-}
76-
77-func (_c *mocktargetSource_Close_Call) Run(run func()) *mocktargetSource_Close_Call {
78- _c.Call.Run(func(args mock.Arguments) {
79- run()
80- })
81- return _c
82-}
83-
84-func (_c *mocktargetSource_Close_Call) Return(err error) *mocktargetSource_Close_Call {
85- _c.Call.Return(err)
86- return _c
87-}
88-
89-func (_c *mocktargetSource_Close_Call) RunAndReturn(run func() error) *mocktargetSource_Close_Call {
90- _c.Call.Return(run)
91- return _c
92-}
93-
94-// Targets provides a mock function for the type mocktargetSource
95-func (_mock *mocktargetSource) Targets(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error) {
96- ret := _mock.Called(ctx, windows, linksOnly)
97-
98- if len(ret) == 0 {
99- panic("no return value specified for Targets")
100- }
101-
102- var r0 []ports.HintTarget
103- var r1 map[ports.WindowID]bool
104- var r2 error
105- if returnFunc, ok := ret.Get(0).(func(context.Context, []ports.HintWindow, bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)); ok {
106- return returnFunc(ctx, windows, linksOnly)
107- }
108- if returnFunc, ok := ret.Get(0).(func(context.Context, []ports.HintWindow, bool) []ports.HintTarget); ok {
109- r0 = returnFunc(ctx, windows, linksOnly)
110- } else {
111- if ret.Get(0) != nil {
112- r0 = ret.Get(0).([]ports.HintTarget)
113- }
114- }
115- if returnFunc, ok := ret.Get(1).(func(context.Context, []ports.HintWindow, bool) map[ports.WindowID]bool); ok {
116- r1 = returnFunc(ctx, windows, linksOnly)
117- } else {
118- if ret.Get(1) != nil {
119- r1 = ret.Get(1).(map[ports.WindowID]bool)
120- }
121- }
122- if returnFunc, ok := ret.Get(2).(func(context.Context, []ports.HintWindow, bool) error); ok {
123- r2 = returnFunc(ctx, windows, linksOnly)
124- } else {
125- r2 = ret.Error(2)
126- }
127- return r0, r1, r2
128-}
129-
130-// mocktargetSource_Targets_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Targets'
131-type mocktargetSource_Targets_Call struct {
132- *mock.Call
133-}
134-
135-// Targets is a helper method to define mock.On call
136-// - ctx context.Context
137-// - windows []ports.HintWindow
138-// - linksOnly bool
139-func (_e *mocktargetSource_Expecter) Targets(ctx any, windows any, linksOnly any) *mocktargetSource_Targets_Call {
140- return &mocktargetSource_Targets_Call{Call: _e.mock.On("Targets", ctx, windows, linksOnly)}
141-}
142-
143-func (_c *mocktargetSource_Targets_Call) Run(run func(ctx context.Context, windows []ports.HintWindow, linksOnly bool)) *mocktargetSource_Targets_Call {
144- _c.Call.Run(func(args mock.Arguments) {
145- var arg0 context.Context
146- if args[0] != nil {
147- arg0 = args[0].(context.Context)
148- }
149- var arg1 []ports.HintWindow
150- if args[1] != nil {
151- arg1 = args[1].([]ports.HintWindow)
152- }
153- var arg2 bool
154- if args[2] != nil {
155- arg2 = args[2].(bool)
156- }
157- run(
158- arg0,
159- arg1,
160- arg2,
161- )
162- })
163- return _c
164-}
165-
166-func (_c *mocktargetSource_Targets_Call) Return(hintTargets []ports.HintTarget, windowIDToBool map[ports.WindowID]bool, err error) *mocktargetSource_Targets_Call {
167- _c.Call.Return(hintTargets, windowIDToBool, err)
168- return _c
169-}
170-
171-func (_c *mocktargetSource_Targets_Call) RunAndReturn(run func(ctx context.Context, windows []ports.HintWindow, linksOnly bool) ([]ports.HintTarget, map[ports.WindowID]bool, error)) *mocktargetSource_Targets_Call {
172- _c.Call.Return(run)
173- return _c
174-}
D internal/adapters/hints/testdata/browser.png +0 -0 binary file: no text diff

binary file: no text diff

D internal/adapters/hints/testdata/terminal.png +0 -0 binary file: no text diff

binary file: no text diff

D internal/adapters/hints/testdata/underline.png +0 -0 binary file: no text diff

binary file: no text diff

D internal/adapters/hints/underline.go +0 -246
@@ -1,246 +0,0 @@
1-package hints
2-
3-import (
4- "image"
5- "math"
6- "sort"
7-
8- "github.com/bnema/neferwl/internal/ports"
9-)
10-
11-// underlineCandidate is a word with a thin line under it.
12-type underlineCandidate struct {
13- word int
14- row int // the line's row
15- from, to int // x extent of the continuous line through the word
16-}
17-
18-// markUnderlined sets Underlined, UnderL and UnderR on the words of a band
19-// that have a thin line under them. words are in the coordinates of g.
20-// Lines that run on far beyond the word (rules, table borders, input boxes)
21-// do not count, unless the extension is made of underlined words too.
22-func markUnderlined(src *image.Gray, words []OCRWord) {
23- // Words are relative to the band's own origin, whatever the image is a
24- // view of.
25- b := src.Bounds()
26- g := &image.Gray{Pix: src.Pix[src.PixOffset(b.Min.X, b.Min.Y):], Stride: src.Stride, Rect: image.Rect(0, 0, b.Dx(), b.Dy())}
27- var cands []underlineCandidate
28- for i, w := range words {
29- // A single glyph, or a box narrower than tall, has a flat bottom
30- // stroke (8, L, 2) that is not an underline.
31- if len([]rune(w.Text)) < 2 || w.Rect.Dx() < w.Rect.Dy() {
32- continue
33- }
34- if c, ok := underlineOf(g, w.Rect); ok {
35- c.word = i
36- cands = append(cands, c)
37- }
38- }
39- for _, c := range cands {
40- w := words[c.word]
41- h := w.Rect.Dy()
42- if (w.Rect.Min.X-c.from > h && !covered(words, cands, c, -1)) ||
43- (c.to-w.Rect.Max.X > h && !covered(words, cands, c, 1)) {
44- continue
45- }
46- words[c.word].Underlined, words[c.word].UnderL, words[c.word].UnderR = true, c.from, c.to
47- }
48-}
49-
50-// covered reports whether the line of c, on the given side of its word, is
51-// explained by underlined words on the same line up to within a glyph of its
52-// end.
53-func covered(words []OCRWord, cands []underlineCandidate, c underlineCandidate, dir int) bool {
54- w := words[c.word].Rect
55- h := w.Dy()
56- cur := w.Min.X
57- if dir > 0 {
58- cur = w.Max.X
59- }
60- for {
61- if (dir < 0 && cur-c.from <= h) || (dir > 0 && c.to-cur <= h) {
62- return true
63- }
64- next := -1
65- for _, o := range cands {
66- r := words[o.word].Rect
67- if abs(r.Max.Y-w.Max.Y) > max(3, h/3) {
68- continue
69- }
70- var gap, reach int
71- if dir < 0 {
72- gap, reach = cur-r.Max.X, r.Min.X
73- } else {
74- gap, reach = r.Min.X-cur, r.Max.X
75- }
76- if gap < -2 || gap > h {
77- continue
78- }
79- if next < 0 || (dir < 0 && reach < next) || (dir > 0 && reach > next) {
80- next = reach
81- }
82- }
83- if next < 0 || (dir < 0 && next >= cur) || (dir > 0 && next <= cur) {
84- return false
85- }
86- cur = next
87- }
88-}
89-
90-// underlineOf looks for an underline under the word box r in g.
91-func underlineOf(g *image.Gray, r image.Rectangle) (underlineCandidate, bool) {
92- b := g.Bounds()
93- box := r.Intersect(b)
94- h := r.Dy()
95- if box.Dx() < 4 || h < 4 {
96- return underlineCandidate{}, false
97- }
98- bg, ink := contrastOf(g, box)
99- thr := min(40, max(20, ink/2))
100- if ink < thr+10 {
101- return underlineCandidate{}, false
102- }
103- top := max(b.Min.Y, r.Max.Y-(h*15+99)/100)
104- bottom := min(b.Max.Y, r.Max.Y+(h*35+99)/100+1)
105- maxThick := max(2, h/6)
106- rowOK := func(y int) bool {
107- row := g.Pix[(y-g.Rect.Min.Y)*g.Stride:]
108- n := 0
109- for x := box.Min.X; x < box.Max.X; x++ {
110- if abs(int(row[x-g.Rect.Min.X])-bg) >= thr {
111- n++
112- }
113- }
114- return n*100 >= box.Dx()*85
115- }
116- thick, first := 0, 0
117- for y := top; y <= bottom; y++ {
118- if y < bottom && rowOK(y) {
119- if thick == 0 {
120- first = y
121- }
122- thick++
123- continue
124- }
125- if thick > 0 {
126- if thick > maxThick {
127- return underlineCandidate{}, false // a block, not a line
128- }
129- row := first + thick/2
130- from, to := box.Min.X, box.Max.X
131- line := g.Pix[(row-g.Rect.Min.Y)*g.Stride:]
132- on := func(x int) bool { return abs(int(line[x-g.Rect.Min.X])-bg) >= thr }
133- for from > b.Min.X && on(from-1) {
134- from--
135- }
136- for to < b.Max.X && on(to) {
137- to++
138- }
139- return underlineCandidate{row: row, from: from, to: to}, true
140- }
141- }
142- return underlineCandidate{}, false
143-}
144-
145-// contrastOf returns the background level of box (the most common level,
146-// found in bins of 16 so antialiasing does not split it, then refined to the
147-// mean of that bin) and the contrast of its ink (the deviation reached by
148-// the strongest 3% of pixels).
149-func contrastOf(g *image.Gray, box image.Rectangle) (bg, ink int) {
150- var hist [256]int
151- for y := box.Min.Y; y < box.Max.Y; y++ {
152- for _, v := range g.Pix[(y-g.Rect.Min.Y)*g.Stride+box.Min.X-g.Rect.Min.X:][:box.Dx()] {
153- hist[v]++
154- }
155- }
156- total := box.Dx() * box.Dy()
157- var bins [16]int
158- for v, c := range hist {
159- bins[v>>4] += c
160- }
161- best := 0
162- for i, c := range bins {
163- if c > bins[best] {
164- best = i
165- }
166- }
167- sum, n := 0, 0
168- for v := best << 4; v < (best+1)<<4; v++ {
169- sum += v * hist[v]
170- n += hist[v]
171- }
172- bg = sum / max(n, 1)
173- var dev [256]int
174- for v, c := range hist {
175- dev[abs(v-bg)] += c
176- }
177- n = 0
178- for d := 255; d >= 0; d-- {
179- n += dev[d]
180- if n*100 >= total*3 {
181- return bg, d
182- }
183- }
184- return bg, 0
185-}
186-
187-// underlinedLinks returns the targets for underlined words, joining the
188-// words of one continuous underline. A group that overlaps a URL target (in
189-// pixels) is left to it. Coordinates are scaled to logical.
190-func underlinedLinks(words []OCRWord, urls []image.Rectangle, scale float64) []ports.HintTarget {
191- var us []OCRWord
192- for _, w := range words {
193- if w.Underlined {
194- us = append(us, w)
195- }
196- }
197- sort.Slice(us, func(i, j int) bool { return us[i].Rect.Min.X < us[j].Rect.Min.X })
198- used := make([]bool, len(us))
199- var out []ports.HintTarget
200- for i := range us {
201- if used[i] {
202- continue
203- }
204- used[i] = true
205- rect, last := us[i].Rect, us[i]
206- chars, width := len(last.Text), last.Rect.Dx()
207- for {
208- next := -1
209- for j := range us {
210- c := us[j]
211- if used[j] || c.Rect.Min.X < rect.Max.X-2 || abs(c.Rect.Max.Y-last.Rect.Max.Y) > max(3, rect.Dy()/3) {
212- continue
213- }
214- avg := float64(width+c.Rect.Dx()) / float64(max(1, chars+len(c.Text)))
215- gap := c.Rect.Min.X - rect.Max.X
216- if float64(gap) > 1.2*avg || c.Rect.Min.X > last.UnderR+2 {
217- continue
218- }
219- if next < 0 || c.Rect.Min.X < us[next].Rect.Min.X {
220- next = j
221- }
222- }
223- if next < 0 {
224- break
225- }
226- used[next] = true
227- last = us[next]
228- rect = rect.Union(last.Rect)
229- chars += len(last.Text)
230- width += last.Rect.Dx()
231- }
232- dup := false
233- for _, u := range urls {
234- if overlap(rect, u) > .5 {
235- dup = true
236- }
237- }
238- if dup {
239- continue
240- }
241- x, y := int(math.Floor(float64(rect.Min.X)/scale)), int(math.Floor(float64(rect.Min.Y)/scale))
242- right, bottom := int(math.Ceil(float64(rect.Max.X)/scale)), int(math.Ceil(float64(rect.Max.Y)/scale))
243- out = append(out, ports.HintTarget{Kind: ports.HintLink, Rect: ports.Rect{X: x, Y: y, W: right - x, H: bottom - y}})
244- }
245- return out
246-}
D internal/adapters/hints/underline_test.go +0 -252
@@ -1,252 +0,0 @@
1-package hints
2-
3-import (
4- "image"
5- "image/color"
6- "os"
7- "os/exec"
8- "testing"
9-
10- "github.com/stretchr/testify/require"
11-
12- "github.com/bnema/neferwl/internal/ports"
13-)
14-
15-// canvas is a synthetic grayscale band: bg everywhere, ink for text.
16-type canvas struct {
17- g *image.Gray
18- bg, ink uint8
19- s int // scale
20-}
21-
22-func newCanvas(w, h int, bg, ink uint8, s int) *canvas {
23- g := image.NewGray(image.Rect(0, 0, w*s, h*s))
24- for i := range g.Pix {
25- g.Pix[i] = bg
26- }
27- return &canvas{g, bg, ink, s}
28-}
29-
30-func (c *canvas) fill(x0, y0, x1, y1 int, v uint8) {
31- for y := y0 * c.s; y < y1*c.s; y++ {
32- for x := x0 * c.s; x < x1*c.s; x++ {
33- c.g.SetGray(x, y, color.Gray{v})
34- }
35- }
36-}
37-
38-// word draws a 10 px high word of x-height body at (x,y) with the given
39-// width: text-like noisy ink (a stem every 3 px) and optional descenders.
40-// It returns the OCR box (descenders excluded, as a box around the body).
41-func (c *canvas) word(x, y, w int, descender bool) OCRWord {
42- for i := 0; i < w; i += 3 {
43- c.fill(x+i, y, x+i+1, y+10, c.ink)
44- }
45- box := image.Rect(x*c.s, y*c.s, (x+w)*c.s, (y+10)*c.s)
46- if descender {
47- for i := 0; i < w; i += 6 {
48- c.fill(x+i, y+10, x+i+1, y+13, c.ink)
49- }
50- box.Max.Y = (y + 13) * c.s
51- }
52- return OCRWord{Text: "word", Rect: box}
53-}
54-
55-func (c *canvas) underline(x0, x1, y int) { c.fill(x0, y, x1, y+1, c.ink) }
56-
57-func scanUnder(c *canvas, words ...OCRWord) []ports.HintTarget {
58- markUnderlined(c.g, words)
59- return textLinks(words, float64(c.s))
60-}
61-
62-func TestUnderline(t *testing.T) {
63- for _, tc := range []struct {
64- name string
65- bg, ink uint8
66- }{{"dark-bg", 10, 204}, {"light-bg", 245, 30}} {
67- for _, s := range []int{1, 2} {
68- t.Run(tc.name, func(t *testing.T) {
69- mk := func() *canvas { return newCanvas(200, 60, tc.bg, tc.ink, s) }
70-
71- c := mk()
72- w := c.word(10, 20, 40, false)
73- c.underline(10, 50, 32)
74- got := scanUnder(c, w)
75- require.Len(t, got, 1)
76- require.Equal(t, ports.HintLink, got[0].Kind)
77- require.False(t, got[0].Text)
78- require.Equal(t, ports.WindowID(0), got[0].Window)
79- require.Equal(t, 10, got[0].Rect.X)
80-
81- c = mk()
82- require.Empty(t, scanUnder(c, c.word(10, 20, 40, false)), "plain word")
83-
84- c = mk()
85- require.Empty(t, scanUnder(c, c.word(10, 20, 40, true)), "descenders")
86-
87- c = mk()
88- w = c.word(10, 20, 40, false)
89- c.underline(0, 200, 32)
90- require.Empty(t, scanUnder(c, w), "rule across")
91-
92- c = mk()
93- w = c.word(10, 20, 40, false)
94- c.fill(10, 24, 50, 25, c.ink)
95- require.Empty(t, scanUnder(c, w), "strikethrough")
96-
97- // Two words with a continuous underline: one target.
98- c = mk()
99- a, b := c.word(10, 20, 40, false), c.word(56, 20, 40, false)
100- c.underline(10, 96, 32)
101- got = scanUnder(c, a, b)
102- require.Len(t, got, 1)
103- require.Equal(t, 10, got[0].Rect.X)
104- require.Equal(t, 86, got[0].Rect.W)
105-
106- // A gap in the underline: two targets.
107- c = mk()
108- a, b = c.word(10, 20, 40, false), c.word(56, 20, 40, false)
109- c.underline(10, 50, 32)
110- c.underline(56, 96, 32)
111- require.Len(t, scanUnder(c, a, b), 2)
112- })
113- }
114- }
115-}
116-
117-// A box border under a word, with the word a part of a table row, is not a
118-// link; neither is an underline that continues beyond into unrelated text.
119-func TestUnderlineRejectsBox(t *testing.T) {
120- c := newCanvas(300, 60, 10, 204, 1)
121- w := c.word(100, 20, 40, false)
122- c.underline(20, 280, 32)
123- c.fill(20, 10, 21, 33, 204)
124- c.fill(279, 10, 280, 33, 204)
125- require.Empty(t, scanUnder(c, w))
126-}
127-
128-// The line must be thin.
129-func TestUnderlineRejectsBlock(t *testing.T) {
130- c := newCanvas(100, 60, 10, 204, 1)
131- w := c.word(10, 20, 40, false)
132- c.fill(10, 31, 50, 36, 204)
133- require.Empty(t, scanUnder(c, w))
134-}
135-
136-func TestUnderlinedURLIsOneTextTarget(t *testing.T) {
137- c := newCanvas(300, 60, 10, 204, 1)
138- w := c.word(10, 20, 100, false)
139- w.Text = "example.org"
140- c.underline(10, 110, 32)
141- got := scanUnder(c, w)
142- require.Len(t, got, 1)
143- require.True(t, got[0].Text)
144-}
145-
146-// ocrTestdata reads the words of a testdata image with the real tesseract.
147-func ocrTestdata(t *testing.T, name string) (*image.Gray, []OCRWord) {
148- t.Helper()
149- if _, err := exec.LookPath("tesseract"); err != nil {
150- t.Skip("tesseract not on PATH")
151- }
152- f, err := os.Open("testdata/" + name)
153- require.NoError(t, err)
154- defer f.Close()
155- img, _, err := image.Decode(f)
156- require.NoError(t, err)
157- g := image.NewGray(img.Bounds())
158- for y := 0; y < g.Rect.Dy(); y++ {
159- for x := 0; x < g.Rect.Dx(); x++ {
160- g.Set(x, y, img.At(x, y))
161- }
162- }
163- words, _, err := readWords(t.Context(), g)
164- require.NoError(t, err)
165- return g, words
166-}
167-
168-// headingRule finds the thin horizontal rule under the heading of
169-// browser.png: the first row, from y=150, with a 900 px run of non-white
170-// pixels. It returns the rule as a rect.
171-func headingRule(t *testing.T, g *image.Gray) image.Rectangle {
172- t.Helper()
173- for y := 150; y < 220; y++ {
174- run, start := 0, 0
175- for x := 0; x < g.Rect.Dx(); x++ {
176- if g.GrayAt(x, y).Y < 240 {
177- if run == 0 {
178- start = x
179- }
180- run++
181- if run >= 900 {
182- return image.Rect(start, y, start+run, y+1)
183- }
184- } else {
185- run = 0
186- }
187- }
188- }
189- t.Fatal("heading rule not found")
190- return image.Rectangle{}
191-}
192-
193-// browser.png has blue links and no underlined link: the only underline-only
194-// target is a known false positive, and the heading rule is not taken for an
195-// underline.
196-func TestUnderlineOnBrowserTestdata(t *testing.T) {
197- g, words := ocrTestdata(t, "browser.png")
198- rule := headingRule(t, g)
199- // Known false positive: the word "as" (flat glyph bottoms, ~(878,353)) is
200- // flagged underlined by the detector. Anything else is a regression.
201- var under []ports.HintTarget
202- for _, tg := range textLinks(words, 1) {
203- if !tg.Text {
204- under = append(under, tg)
205- }
206- }
207- require.Len(t, under, 1, "underline-only targets: %v", under)
208- require.True(t, under[0].Rect.X > 860 && under[0].Rect.X < 900 && under[0].Rect.Y > 340 && under[0].Rect.Y < 370, "unexpected target %v", under[0].Rect)
209- for _, tg := range textLinks(words, 1) {
210- r := image.Rect(tg.Rect.X, tg.Rect.Y, tg.Rect.X+tg.Rect.W, tg.Rect.Y+tg.Rect.H)
211- require.False(t, r.Overlaps(rule), "target %v overlaps the heading rule %v", tg.Rect, rule)
212- }
213-}
214-
215-// underline.png is a real render (Chromium) of a generic page: underlined
216-// links `demo`, `source` and `ermite-runner hosts`, a table with row rules
217-// and an input box, which must yield nothing.
218-func TestUnderlineOnRealRender(t *testing.T) {
219- _, words := ocrTestdata(t, "underline.png")
220- near := func(r ports.Rect, x0, y0, x1, y1 int) bool {
221- const tol = 8
222- abs := func(v int) int { return max(v, -v) }
223- return abs(r.X-x0) <= tol && abs(r.Y-y0) <= tol && abs(r.X+r.W-x1) <= tol && abs(r.Y+r.H-y1) <= tol
224- }
225- var got []ports.HintTarget
226- for _, tg := range textLinks(words, 1) {
227- if !tg.Text {
228- got = append(got, tg)
229- }
230- }
231- require.Len(t, got, 3, "targets: %v", got)
232- for _, want := range [][4]int{{148, 122, 196, 137}, {306, 126, 364, 137}, {122, 165, 294, 180}} {
233- found := false
234- for _, tg := range got {
235- found = found || near(tg.Rect, want[0], want[1], want[2], want[3])
236- }
237- require.True(t, found, "no target near %v in %v", want, got)
238- }
239- // The table (y 215..335) and the input box (y 360..410) give nothing.
240- for _, tg := range textLinks(words, 1) {
241- require.False(t, tg.Rect.Y > 200 && tg.Rect.Y < 420, "unexpected target %v", tg.Rect)
242- }
243-}
244-
245-// A single glyph with a flat bottom stroke (8, L) is not underlined text.
246-func TestUnderlineIgnoresSingleGlyph(t *testing.T) {
247- c := newCanvas(100, 60, 10, 204, 1)
248- w := c.word(10, 20, 10, false)
249- w.Text = "8"
250- c.underline(10, 20, 32)
251- require.Empty(t, scanUnder(c, w))
252-}
M internal/app/app.go +6 -6
@@ -84,7 +84,7 @@ const busRetry = 2 * time.Second
84 func relayConfig(ctx context.Context, cur ports.Config, in <-chan ports.ConfigChanged, out chan<- ports.ConfigChanged, keymaps chan *xkb.Keymap, deviceConfigs chan ports.InputDevicesConfig, curs *cursors, commands chan<- ports.ClientCommand, log zerowrap.Logger) {84 func relayConfig(ctx context.Context, cur ports.Config, in <-chan ports.ConfigChanged, out chan<- ports.ConfigChanged, keymaps chan *xkb.Keymap, deviceConfigs chan ports.InputDevicesConfig, curs *cursors, commands chan<- ports.ClientCommand, log zerowrap.Logger) {
85 kb := cur.Keyboard85 kb := cur.Keyboard
86 devCfg := cur.InputDevicesConfig86 devCfg := cur.InputDevicesConfig
87- hintOn := cur.Hint.Enable87+ hint := cur.Hint
88 for {88 for {
89 var ev ports.ConfigChanged89 var ev ports.ConfigChanged
90 select {90 select {
@@ -92,11 +92,11 @@ func relayConfig(ctx context.Context, cur ports.Config, in <-chan ports.ConfigCh
92 return92 return
93 case ev = <-in:93 case ev = <-in:
94 }94 }
95- // hint.enable is read at startup: nothing of hint mode starts or95+ // hint.enable, hint.detector, hint.model and hint.threshold are read
96- // stops on a reload.96+ // at startup: nothing of hint mode starts or stops on a reload.
97- if ev.Config.Hint.Enable != hintOn {97+ if h := ev.Config.Hint; h.Enable != hint.Enable || h.Detector != hint.Detector || h.Model != hint.Model || h.Threshold != hint.Threshold {
98- log.Info().Bool("running", hintOn).Msg("hint.enable changes apply after restart")98+ log.Info().Bool("running", hint.Enable).Msg("hint.enable, hint.detector, hint.model and hint.threshold apply after restart")
99- ev.Config.Hint.Enable = hintOn99+ ev.Config.Hint.Enable, ev.Config.Hint.Detector, ev.Config.Hint.Model, ev.Config.Hint.Threshold = hint.Enable, hint.Detector, hint.Model, hint.Threshold
100 }100 }
101 next := ev.Config.Keyboard101 next := ev.Config.Keyboard
102 layoutChanged := next.Layout != kb.Layout || next.Variant != kb.Variant || next.Options != kb.Options102 layoutChanged := next.Layout != kb.Layout || next.Variant != kb.Variant || next.Options != kb.Options
R internal/app/hintdebug.go → internal/app/hint.go +7 -10 renamed, 66% similar
@@ -5,21 +5,18 @@ import (
5 "os"5 "os"
6 "path/filepath"6 "path/filepath"
7 7
8- "github.com/bnema/neferwl/internal/adapters/atspi"
9 "github.com/bnema/neferwl/internal/adapters/hints"8 "github.com/bnema/neferwl/internal/adapters/hints"
10 "github.com/bnema/neferwl/internal/adapters/logging"9 "github.com/bnema/neferwl/internal/adapters/logging"
10+ "github.com/bnema/neferwl/internal/ports"
11 )11 )
12 12
13-// hintOptions builds the hint worker options: the accessibility source, and,13+// hintOptions builds the hint worker options from the configuration (read at
14-// when NEFERWL_HINT_DEBUG is "1", the dump directory of scanned frames,14+// startup) and, when NEFERWL_HINT_DEBUG is "1", the dump directory of scanned
15-// $XDG_STATE_HOME/neferwl/hints. The accessibility bus is only contacted15+// frames, $XDG_STATE_HOME/neferwl/hints. The detector process only starts at
16-// by the first scan; without it the worker reads windows with OCR only.16+// the first scan.
17-func hintOptions(ctx context.Context) hints.Options {17+func hintOptions(ctx context.Context, cfg ports.Config) hints.Options {
18 opts := hintDump(ctx)18 opts := hintDump(ctx)
19- // The source connects on its first scan, and again after a failure. The19+ opts.Command, opts.ModelDir, opts.Threshold = cfg.Hint.Detector, cfg.Hint.Model, cfg.Hint.Threshold
20- // hints worker owns it and closes it when it returns.
21- src := atspi.New()
22- opts.Source = src
23 return opts20 return opts
24 }21 }
25 22
M internal/app/hintoptin_test.go +18 -2
@@ -45,7 +45,6 @@ func TestHintOffStartsNoHintGoroutine(t *testing.T) {
45 buf := make([]byte, 1<<20)45 buf := make([]byte, 1<<20)
46 dump := string(buf[:runtime.Stack(buf, true)])46 dump := string(buf[:runtime.Stack(buf, true)])
47 require.NotContains(t, dump, "adapters/hints.")47 require.NotContains(t, dump, "adapters/hints.")
48- require.NotContains(t, dump, "adapters/atspi.")
49 require.NotContains(t, dump, "routeCaptured")48 require.NotContains(t, dump, "routeCaptured")
50 select {49 select {
51 case <-done:50 case <-done:
@@ -64,5 +63,22 @@ func TestRelayConfigKeepsStartupHintEnable(t *testing.T) {
64 next := config.Defaults()63 next := config.Defaults()
65 next.Hint.Enable = true64 next.Hint.Enable = true
66 in <- ports.ConfigChanged{Config: next}65 in <- ports.ConfigChanged{Config: next}
67- require.False(t, (<-out).Config.Hint.Enable)66+ got := (<-out).Config.Hint
67+ require.False(t, got.Enable)
68+}
69+
70+func TestRelayConfigKeepsStartupHintDetector(t *testing.T) {
71+ ctx, cancel := context.WithCancel(context.Background())
72+ defer cancel()
73+ cur := config.Defaults()
74+ in := make(chan ports.ConfigChanged, 1)
75+ out := make(chan ports.ConfigChanged, 1)
76+ go relayConfig(ctx, cur, in, out, make(chan *xkb.Keymap, 1), make(chan ports.InputDevicesConfig, 1), newCursors(clock.System{}, 0), make(chan ports.ClientCommand, 1), logging.For(ctx, "config"))
77+ next := config.Defaults()
78+ next.Hint.Detector, next.Hint.Model, next.Hint.Threshold = "/other/nh", "/other/models", 0.5
79+ in <- ports.ConfigChanged{Config: next}
80+ got := (<-out).Config.Hint
81+ require.Equal(t, cur.Hint.Detector, got.Detector)
82+ require.Equal(t, cur.Hint.Model, got.Model)
83+ require.Equal(t, cur.Hint.Threshold, got.Threshold)
68 }84 }
M internal/app/session.go +2 -2
@@ -102,9 +102,9 @@ func assembleSession(ctx context.Context, opts Options) (s *session, err error)
102 return s, err102 return s, err
103 }103 }
104 runtimeDir := os.Getenv("XDG_RUNTIME_DIR")104 runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
105- // Hint mode off: no accessibility connection, no debug env, nothing.105+ // Hint mode off: no detector, no debug env, nothing.
106 if opts.Config.Hint.Enable {106 if opts.Config.Hint.Enable {
107- s.hintOpts = hintOptions(ctx)107+ s.hintOpts = hintOptions(ctx, opts.Config)
108 }108 }
109 // Every output renders on the same GPU: its formats are the clients'.109 // Every output renders on the same GPU: its formats are the clients'.
110 s.dmabuf = vulkan.Probe()110 s.dmabuf = vulkan.Probe()
M internal/core/hint.go +15 -43
@@ -36,7 +36,6 @@ type hintTarget struct {
36 source hintSourceGeom // source window and its request-time client geometry36 source hintSourceGeom // source window and its request-time client geometry
37 label string37 label string
38 rect ports.Rect38 rect ports.Rect
39- text bool // found by OCR
40 }39 }
41 40
42 type hintSourceGeom struct {41 type hintSourceGeom struct {
@@ -52,7 +51,7 @@ type hintState struct {
52 // one on screen ends it.51 // one on screen ends it.
53 ws *Workspace52 ws *Workspace
54 geom hintGeom53 geom hintGeom
55- windows []ports.HintWindow // visible windows at request time54+ windows []hintSourceGeom // visible windows and their client geometry at request time
56 targets []hintTarget55 targets []hintTarget
57 typed string56 typed string
58 }57 }
@@ -91,7 +90,7 @@ func (c *Core) hintRequest() {
91 return90 return
92 }91 }
93 wins := c.hintWindows(sc)92 wins := c.hintWindows(sc)
94- req := ports.HintScanRequest{Seq: c.hint.seq + 1, Output: sc.name(), Width: sc.info.Width, Height: sc.info.Height, Scale: sc.scale, LinksOnly: c.cfg.Hint.LinksOnly, Windows: wins}93+ req := ports.HintScanRequest{Seq: c.hint.seq + 1, Output: sc.name(), Width: sc.info.Width, Height: sc.info.Height, Scale: sc.scale}
95 select {94 select {
96 case c.ch.HintScans <- req:95 case c.ch.HintScans <- req:
97 default:96 default:
@@ -101,10 +100,11 @@ func (c *Core) hintRequest() {
101 }100 }
102 101
103 // hintWindows lists the windows drawn on sc, bottom to top, with their client102 // hintWindows lists the windows drawn on sc, bottom to top, with their client
104-// area (placements are output-local).103+// area (placements are output-local). A target is only clicked while the
105-func (c *Core) hintWindows(sc *screen) []ports.HintWindow {104+// geometry of its window is the one seen here.
105+func (c *Core) hintWindows(sc *screen) []hintSourceGeom {
106 view := sc.mon.viewport()106 view := sc.mon.viewport()
107- var out []ports.HintWindow107+ var out []hintSourceGeom
108 for _, p := range sc.shownLayout() {108 for _, p := range sc.shownLayout() {
109 if p.Hidden || p.Preview > 0 || !view.shows(p) {109 if p.Hidden || p.Preview > 0 || !view.shows(p) {
110 continue110 continue
@@ -113,10 +113,7 @@ func (c *Core) hintWindows(sc *screen) []ports.HintWindow {
113 if r.W <= 0 || r.H <= 0 {113 if r.W <= 0 || r.H <= 0 {
114 continue114 continue
115 }115 }
116- w := ports.HintWindow{ID: p.ID, Rect: r, Focused: p.Focused}116+ out = append(out, hintSourceGeom{id: p.ID, rect: r})
117- rec := c.windows.lookup(p.ID)
118- w.PID, w.AppID = rec.client.PID, rec.client.AppID
119- out = append(out, w)
120 }117 }
121 return out118 return out
122 }119 }
@@ -167,11 +164,11 @@ func (c *Core) hintResult(res ports.HintScanResult) bool {
167 // A layer surface, or a window of another output.164 // A layer surface, or a window of another output.
168 continue165 continue
169 }166 }
170- source, ok := c.hintSource(sc, t, root)167+ source, ok := c.hintSource(sc, root)
171 if !ok {168 if !ok {
172 continue169 continue
173 }170 }
174- kept = append(kept, hintTarget{x: x, y: y, window: id, source: source, rect: t.Rect, text: t.Text})171+ kept = append(kept, hintTarget{x: x, y: y, window: id, source: source, rect: t.Rect})
175 }172 }
176 if len(kept) == 0 {173 if len(kept) == 0 {
177 c.hintCancel()174 c.hintCancel()
@@ -196,14 +193,11 @@ func (c *Core) hintResult(res ports.HintScanResult) bool {
196 }193 }
197 194
198 // hintSource resolves the request-time source window only while its client195 // hintSource resolves the request-time source window only while its client
199-// geometry is unchanged. This applies equally to accessibility and OCR targets.196+// geometry is unchanged.
200-func (c *Core) hintSource(sc *screen, t ports.HintTarget, root WindowID) (hintSourceGeom, bool) {197+func (c *Core) hintSource(sc *screen, root WindowID) (hintSourceGeom, bool) {
201- if t.Window != 0 && root != t.Window {
202- return hintSourceGeom{}, false
203- }
204 for _, w := range c.hint.windows {198 for _, w := range c.hint.windows {
205- if w.ID == root && c.hintSourceUnchanged(sc, hintSourceGeom{id: w.ID, rect: w.Rect}) {199+ if w.id == root && c.hintSourceUnchanged(sc, w) {
206- return hintSourceGeom{id: w.ID, rect: w.Rect}, true200+ return w, true
207 }201 }
208 }202 }
209 return hintSourceGeom{}, false203 return hintSourceGeom{}, false
@@ -292,11 +286,8 @@ func (c *Core) hintKey(ctx context.Context, key ports.KeyEvent) (bool, error) {
292 // synthetic: c.synthetic keeps the physical modifiers (the bind's Cmd may286 // synthetic: c.synthetic keeps the physical modifiers (the bind's Cmd may
293 // still be down) from turning it into a window drag, without touching c.mods.287 // still be down) from turning it into a window drag, without touching c.mods.
294 //288 //
295-// A target found by OCR is clicked with hint.text-click held (terminals open289+// The window gets the keyboard first.
296-// a link on a modified click). Every click gives the window the keyboard290+func (c *Core) hintClick(ctx context.Context, t hintTarget) error {
297-// first, then sends exactly the modifiers to hold (none for other targets),
298-// and the real modifier state is always sent back afterwards.
299-func (c *Core) hintClick(ctx context.Context, t hintTarget) (err error) {
300 c.hintCancel()291 c.hintCancel()
301 if err := c.publish(ctx); err != nil {292 if err := c.publish(ctx); err != nil {
302 return err293 return err
@@ -304,19 +295,9 @@ func (c *Core) hintClick(ctx context.Context, t hintTarget) (err error) {
304 if !c.hintClickable(t) {295 if !c.hintClickable(t) {
305 return nil296 return nil
306 }297 }
307- // Every click states its modifiers (possibly none): the bind's Cmd may
308- // still be down and the client must not see it on a plain click.
309 if err := c.clickFocus(ctx, t.window); err != nil {298 if err := c.clickFocus(ctx, t.window); err != nil {
310 return err299 return err
311 }300 }
312- if err := c.command(ctx, ports.SetModifiers{ID: t.window, Mods: c.hintMods(t)}); err != nil {
313- return err
314- }
315- defer func() {
316- if e := c.command(ctx, ports.SetModifiers{ID: t.window, Restore: true, State: c.keyState}); err == nil {
317- err = e
318- }
319- }()
320 c.synthetic = true301 c.synthetic = true
321 defer func() { c.synthetic = false }()302 defer func() { c.synthetic = false }()
322 if err := c.pointerMotion(ctx, ports.PointerMotion{X: t.x, Y: t.y, Time: c.motionTime}); err != nil {303 if err := c.pointerMotion(ctx, ports.PointerMotion{X: t.x, Y: t.y, Time: c.motionTime}); err != nil {
@@ -333,15 +314,6 @@ func (c *Core) hintClick(ctx context.Context, t hintTarget) (err error) {
333 return c.pointerButton(ctx, ports.PointerButton{Button: btnLeft, Time: c.motionTime})314 return c.pointerButton(ctx, ports.PointerButton{Button: btnLeft, Time: c.motionTime})
334 }315 }
335 316
336-// hintMods are the modifiers to hold for a click on t, read at click time so
337-// a reloaded configuration applies.
338-func (c *Core) hintMods(t hintTarget) ports.Mods {
339- if !t.text {
340- return 0
341- }
342- return c.cfg.Hint.TextClick
343-}
344-
345 // hintClickable reports whether a click on t may go out now: unprotected,317 // hintClickable reports whether a click on t may go out now: unprotected,
346 // pointer free, and the selected window is still under the target's centre.318 // pointer free, and the selected window is still under the target's centre.
347 func (c *Core) hintClickable(t hintTarget) bool {319 func (c *Core) hintClickable(t hintTarget) bool {
M internal/core/hint_test.go +25 -121
@@ -30,13 +30,7 @@ type hintHarness struct {
30 // newHint starts a core with one 1000x800 output "OUT-1" and window 1 mapped.30 // newHint starts a core with one 1000x800 output "OUT-1" and window 1 mapped.
31 func newHint(t *testing.T, opts core.Options) *hintHarness {31 func newHint(t *testing.T, opts core.Options) *hintHarness {
32 t.Helper()32 t.Helper()
33- return newHintMods(t, opts, 0)33+ return newHintConfig(t, opts, func(*ports.Config) {})
34-}
35-
36-// newHintMods is newHint with hint.text-click set to textClick.
37-func newHintMods(t *testing.T, opts core.Options, textClick ports.Mods) *hintHarness {
38- t.Helper()
39- return newHintConfig(t, opts, func(cfg *ports.Config) { cfg.Hint.TextClick = textClick })
40 }34 }
41 35
42 // newHintConfig is newHint with the config edited by edit.36 // newHintConfig is newHint with the config edited by edit.
@@ -144,13 +138,9 @@ func (h *hintHarness) inside(i int) ports.HintTarget {
144 func TestHintRequestOnBind(t *testing.T) {138 func TestHintRequestOnBind(t *testing.T) {
145 h := newHint(t, core.Options{})139 h := newHint(t, core.Options{})
146 req := h.start()140 req := h.start()
147- if req.Output != "OUT-1" || req.Width != 1000 || req.Height != 800 || req.Scale != 1 || req.LinksOnly || req.Seq == 0 {141+ if req.Output != "OUT-1" || req.Width != 1000 || req.Height != 800 || req.Scale != 1 || req.Seq == 0 {
148 t.Fatalf("request %+v", req)142 t.Fatalf("request %+v", req)
149 }143 }
150- want := []ports.HintWindow{{ID: 1, PID: 4242, AppID: "org.test.app", Rect: h.win, Focused: true}}
151- if !slices.Equal(req.Windows, want) {
152- t.Fatalf("windows %+v, want %+v", req.Windows, want)
153- }
154 // A second bind while pending does not ask again; the pending phase144 // A second bind while pending does not ask again; the pending phase
155 // cancels on any key.145 // cancels on any key.
156 h.press("g", ports.ModSuper)146 h.press("g", ports.ModSuper)
@@ -593,12 +583,12 @@ func TestHintMappedWindowMovedAborts(t *testing.T) {
593 }583 }
594 584
595 // commandsSent drains the commands up to a barrier, keeping the ones that585 // commandsSent drains the commands up to a barrier, keeping the ones that
596-// matter for a click: modifiers, focus, motion and buttons.586+// matter for a click: focus, motion and buttons.
597 func (h *hintHarness) commandsSent() []ports.ClientCommand {587 func (h *hintHarness) commandsSent() []ports.ClientCommand {
598 var out []ports.ClientCommand588 var out []ports.ClientCommand
599 for _, cmd := range h.barrier() {589 for _, cmd := range h.barrier() {
600 switch cmd.(type) {590 switch cmd.(type) {
601- case ports.SetModifiers, ports.FocusWindow, ports.PointerMotionTo, ports.PointerButtonTo:591+ case ports.FocusWindow, ports.PointerMotionTo, ports.PointerButtonTo:
602 out = append(out, cmd)592 out = append(out, cmd)
603 }593 }
604 }594 }
@@ -609,12 +599,6 @@ func commandKinds(cmds []ports.ClientCommand) []string {
609 var out []string599 var out []string
610 for _, c := range cmds {600 for _, c := range cmds {
611 switch v := c.(type) {601 switch v := c.(type) {
612- case ports.SetModifiers:
613- if v.Restore {
614- out = append(out, "restore")
615- } else {
616- out = append(out, fmt.Sprintf("mods:%d", v.Mods))
617- }
618 case ports.FocusWindow:602 case ports.FocusWindow:
619 out = append(out, fmt.Sprintf("focus:%d", v.ID))603 out = append(out, fmt.Sprintf("focus:%d", v.ID))
620 case ports.PointerMotionTo:604 case ports.PointerMotionTo:
@@ -630,97 +614,25 @@ func commandKinds(cmds []ports.ClientCommand) []string {
630 return out614 return out
631 }615 }
632 616
633-// An OCR target is clicked with the configured modifiers held: they go out617+// A click on the focused window moves the pointer to the target and presses
634-// before the motion and are restored after the release.618+// and releases the left button; it states no modifiers.
635-func TestHintTextClickHoldsModifiers(t *testing.T) {619+func TestHintClickSequence(t *testing.T) {
636- mods := ports.ModCtrl | ports.ModShift620+ h := newHint(t, core.Options{})
637- h := newHintMods(t, core.Options{}, mods)621+ h.show(h.inside(0))
638- tg := h.inside(0)
639- tg.Text = true
640- h.show(tg)
641 h.drain()622 h.drain()
642 h.press("f", 0)623 h.press("f", 0)
643 got := commandKinds(h.commandsSent())624 got := commandKinds(h.commandsSent())
644- want := []string{fmt.Sprintf("mods:%d", mods), "motion", "press", "release", "restore"}625+ want := []string{"motion", "press", "release"}
645 if !slices.Equal(got, want) {626 if !slices.Equal(got, want) {
646 t.Fatalf("commands %v, want %v", got, want)627 t.Fatalf("commands %v, want %v", got, want)
647 }628 }
648 }629 }
649 630
650-// Without text targets or without modifiers the click states no modifiers631+// A window that moved while the scan ran: a target over it is dropped.
651-// (depressed 0) rather than leave the held bind modifier to the client.632+func TestHintResultDroppedWhenWindowMoved(t *testing.T) {
652-func TestHintClickWithoutModifiers(t *testing.T) {
653- for name, c := range map[string]struct {
654- mods ports.Mods
655- text bool
656- }{"accessibility target": {ports.ModCtrl, false}, "none configured": {0, true}} {
657- t.Run(name, func(t *testing.T) {
658- h := newHintMods(t, core.Options{}, c.mods)
659- tg := h.inside(0)
660- tg.Text = c.text
661- h.show(tg)
662- h.drain()
663- h.press("f", 0)
664- got := commandKinds(h.commandsSent())
665- if want := []string{"mods:0", "motion", "press", "release", "restore"}; !slices.Equal(got, want) {
666- t.Fatalf("commands %v, want %v", got, want)
667- }
668- })
669- }
670-}
671-
672-// The bind's Super is still physically held: an accessibility click states
673-// depressed 0, and the restore carries the real state of the latest key,
674-// swallowed ones included (a latch a label key consumed).
675-func TestHintClickHeldBindModifierAndLatchRestore(t *testing.T) {
676- h := newHint(t, core.Options{})
677- tg := h.inside(0)
678- h.press("g", ports.ModSuper)
679- req := receive(t, h.scans)
680- h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{tg, h.inside(1)}}
681- h.labelled(2, 0)
682- h.drain()
683- real := ports.ModState{Depressed: 64, Latched: 0, Locked: 2}
684- h.send(ports.KeyEvent{Keysym: "f", Base: "f", Mods: ports.ModSuper, Pressed: true, State: real})
685- cmds := h.commandsSent()
686- var restore *ports.SetModifiers
687- for i, c := range cmds {
688- if v, ok := c.(ports.SetModifiers); ok {
689- if i == 0 && (v.Restore || v.Mods != 0) {
690- t.Fatalf("first modifiers %+v, want depressed 0", v)
691- }
692- if v.Restore {
693- restore = &v
694- }
695- }
696- }
697- if restore == nil || restore.State != real {
698- t.Fatalf("restore %+v, want state %+v (commands %v)", restore, real, commandKinds(cmds))
699- }
700-}
701-
702-// A result is kept only for the window it was scanned on.
703-func TestHintResultWindowChecks(t *testing.T) {
704- h := newHint(t, core.Options{})
705- req := h.start()
706- // Accessibility target naming another window: dropped; naming window 1: kept.
707- ok := h.inside(1)
708- ok.Window = 1
709- bad := h.inside(0)
710- bad.Window = 7
711- h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{bad, ok}}
712- s := h.labelled(1, 0)
713- if len(s.HintLabels) != 1 || s.HintLabels[0].Rect.Y != ok.Rect.Y {
714- t.Fatalf("labels %+v", s.HintLabels)
715- }
716-}
717-
718-// A window that moved while the scan ran: an OCR target over it is dropped.
719-func TestHintOCRResultDroppedWhenWindowMoved(t *testing.T) {
720 h := newHint(t, core.Options{})633 h := newHint(t, core.Options{})
721 req := h.start()634 req := h.start()
722 tg := h.inside(0)635 tg := h.inside(0)
723- tg.Text = true
724 h.client <- ports.WindowMapped{ID: 2}636 h.client <- ports.WindowMapped{ID: 2}
725 h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 })637 h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 })
726 h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{tg}}638 h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{tg}}
@@ -731,12 +643,12 @@ func TestHintOCRResultDroppedWhenWindowMoved(t *testing.T) {
731 }643 }
732 644
733 // A new window covering the target (not in the snapshot) drops it as well.645 // A new window covering the target (not in the snapshot) drops it as well.
734-func TestHintOCRResultDroppedForUnseenWindow(t *testing.T) {646+func TestHintResultDroppedForUnseenWindow(t *testing.T) {
735 h := newHint(t, core.Options{})647 h := newHint(t, core.Options{})
736 req := h.start()648 req := h.start()
737 h.client <- ports.WindowMapped{ID: 2}649 h.client <- ports.WindowMapped{ID: 2}
738 s := h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 })650 s := h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 })
739- tg := ports.HintTarget{Text: true}651+ var tg ports.HintTarget
740 for _, w := range s.Windows {652 for _, w := range s.Windows {
741 if w.ID == 2 {653 if w.ID == 2 {
742 tg.Rect = ports.Rect{X: w.Rect.X + 5, Y: w.Rect.Y + 5, W: 20, H: 10}654 tg.Rect = ports.Rect{X: w.Rect.X + 5, Y: w.Rect.Y + 5, W: 20, H: 10}
@@ -769,9 +681,9 @@ func TestHintCancelsWhenOutputChanges(t *testing.T) {
769 }681 }
770 }682 }
771 683
772-// A window that does not have the keyboard gets it before the modifiers.684+// A window that does not have the keyboard gets it before the click.
773-func TestHintTextClickFocusesWindowFirst(t *testing.T) {685+func TestHintClickFocusesWindowFirst(t *testing.T) {
774- h := newHintMods(t, core.Options{}, ports.ModCtrl)686+ h := newHint(t, core.Options{})
775 h.client <- ports.WindowMapped{ID: 2}687 h.client <- ports.WindowMapped{ID: 2}
776 s := h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 })688 s := h.until(func(s ports.Scene) bool { return len(s.Windows) == 2 })
777 var r ports.Rect689 var r ports.Rect
@@ -781,24 +693,20 @@ func TestHintTextClickFocusesWindowFirst(t *testing.T) {
781 }693 }
782 }694 }
783 h.win = r695 h.win = r
784- tg := h.inside(0)696+ h.show(h.inside(0))
785- tg.Text = true
786- h.show(tg)
787 h.drain()697 h.drain()
788 h.press("f", 0)698 h.press("f", 0)
789 got := commandKinds(h.commandsSent())699 got := commandKinds(h.commandsSent())
790- want := []string{"focus:1", fmt.Sprintf("mods:%d", ports.ModCtrl), "motion", "press", "release", "restore"}700+ want := []string{"focus:1", "motion", "press", "release"}
791 if !slices.Equal(got, want) {701 if !slices.Equal(got, want) {
792 t.Fatalf("commands %v, want %v", got, want)702 t.Fatalf("commands %v, want %v", got, want)
793 }703 }
794 }704 }
795 705
796-// A click that is no longer allowed sends no modifiers at all.706+// A click that is no longer allowed sends no command at all.
797-func TestHintTextClickNoModifiersWhenNotClickable(t *testing.T) {707+func TestHintClickNothingWhenNotClickable(t *testing.T) {
798- h := newHintMods(t, core.Options{}, ports.ModCtrl)708+ h := newHint(t, core.Options{})
799- tg := h.inside(0)709+ h.show(h.inside(0))
800- tg.Text = true
801- h.show(tg)
802 h.constrain(ports.ConstraintLock, ports.Rect{})710 h.constrain(ports.ConstraintLock, ports.Rect{})
803 h.drain()711 h.drain()
804 h.press("f", 0)712 h.press("f", 0)
@@ -842,15 +750,11 @@ func TestHintDisabledNotesOnce(t *testing.T) {
842 // Resizing window 1 leaves the old target centre inside it. Identity alone750 // Resizing window 1 leaves the old target centre inside it. Identity alone
843 // therefore cannot protect either admission or the eventual synthetic click.751 // therefore cannot protect either admission or the eventual synthetic click.
844 func TestHintRejectsResizedSource(t *testing.T) {752 func TestHintRejectsResizedSource(t *testing.T) {
845- for _, accessibility := range []bool{false, true} {753+ {
846 for _, beforeResult := range []bool{false, true} {754 for _, beforeResult := range []bool{false, true} {
847- t.Run(fmt.Sprintf("accessibility=%v/beforeResult=%v", accessibility, beforeResult), func(t *testing.T) {755+ t.Run(fmt.Sprintf("beforeResult=%v", beforeResult), func(t *testing.T) {
848 h := newHint(t, core.Options{})756 h := newHint(t, core.Options{})
849 tg := h.inside(0)757 tg := h.inside(0)
850- tg.Text = !accessibility
851- if accessibility {
852- tg.Window = 1
853- }
854 req := h.start()758 req := h.start()
855 if !beforeResult {759 if !beforeResult {
856 h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{tg}}760 h.hints <- ports.HintScanResult{Seq: req.Seq, Output: req.Output, Targets: []ports.HintTarget{tg}}
M internal/ports/config.go +7 -6
@@ -170,17 +170,18 @@ type Config struct {
170 // Enable turns the experimental hint mode on (hint.enable). It is170 // Enable turns the experimental hint mode on (hint.enable). It is
171 // read at startup: a reload keeps the startup value.171 // read at startup: a reload keeps the startup value.
172 Enable bool172 Enable bool
173- // LinksOnly limits the targets to links and URLs; false also
174- // includes buttons (hint.targets = clickable).
175- LinksOnly bool
176- // TextClick are the modifiers held for a click on a link found by
177- // OCR (hint.text-click); zero sends a plain click.
178- TextClick Mods
179 // Capture shows the labels in the captures requested by Wayland173 // Capture shows the labels in the captures requested by Wayland
180 // clients (hint.capture); the compositor's own scans never see them.174 // clients (hint.capture); the compositor's own scans never see them.
181 Capture bool175 Capture bool
182 Size float64 // logical pixels; zero selects automatic density sizing176 Size float64 // logical pixels; zero selects automatic density sizing
183 Background, Foreground string // empty derives from Border.Active177 Background, Foreground string // empty derives from Border.Active
178+ // Detector is the nefer-hints binary (hint.detector): a path, or a
179+ // name looked up in PATH. Model is the directory of its model files
180+ // (hint.model), with a leading ~/ already expanded. Threshold is the
181+ // minimum detection score (hint.threshold, 0.01 to 0.99). All three
182+ // are read at startup.
183+ Detector, Model string
184+ Threshold float64
184 }185 }
185 // Workspaces are declared with workspace.<name>.* keys, in first-seen order.186 // Workspaces are declared with workspace.<name>.* keys, in first-seen order.
186 Workspaces []WorkspaceConfig187 Workspaces []WorkspaceConfig
M internal/ports/hint_messages.go +2 -31
@@ -1,15 +1,5 @@
1 package ports1 package ports
2 2
3-// HintKind is the kind of a clickable hint target.
4-type HintKind uint8
5-
6-const (
7- // HintLink is a link or URL.
8- HintLink HintKind = iota
9- // HintButton is a button or control.
10- HintButton
11-)
12-
13 // HintScanRequest asks the hint scanner to find clickable targets in the last3 // HintScanRequest asks the hint scanner to find clickable targets in the last
14 // frame of Output. Core sends it and expects one HintScanResult echoing Seq.4 // frame of Output. Core sends it and expects one HintScanResult echoing Seq.
15 type HintScanRequest struct {5 type HintScanRequest struct {
@@ -19,31 +9,12 @@ type HintScanRequest struct {
19 // (OutputInfo.Width/Height).9 // (OutputInfo.Width/Height).
20 Width, Height int10 Width, Height int
21 Scale float64 // screen scale11 Scale float64 // screen scale
22- LinksOnly bool
23- // Windows are the windows visible on Output, for sources that read a
24- // toolkit's accessibility tree. Core owns the slice per request.
25- Windows []HintWindow
26-}
27-
28-// HintWindow is a window visible on the hint output.
29-type HintWindow struct {
30- ID WindowID
31- PID int
32- AppID string
33- // Rect is the client area, logical output-local.
34- Rect Rect
35- Focused bool
36 }12 }
37 13
38 // HintTarget is a clickable region found in a frame.14 // HintTarget is a clickable region found in a frame.
39 type HintTarget struct {15 type HintTarget struct {
40- Rect Rect // logical, output-local16+ Rect Rect // logical, output-local
41- Kind HintKind17+ Score float32 // detector confidence, 0 to 1
42- // Text is set for targets found by OCR (read from pixels, not from an
43- // accessibility tree).
44- Text bool
45- // Window is the source window of an accessibility target, 0 for OCR.
46- Window WindowID
47 }18 }
48 19
49 // HintScanResult answers a HintScanRequest. Core owns Targets.20 // HintScanResult answers a HintScanRequest. Core owns Targets.

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 25

Tags 10

↑↓ moveenter openesc close