3e96bd1 fix(atspi): match frames to windows one to one and tag targets with their window · neferwl

M internal/adapters/atspi/model.go +35 -21


@@ -113,12 +113,13 @@ type frameInfo struct {
113	113		w, h int // size of the frame, from its window extents
114	114		}
115	115
116		-	// matchFrames pairs the windows of one app with its frames. A window is
117		-	// paired with a frame when the pairing is certain:
	116	+	// matchFrames pairs the windows of one app with its frames, one to one. A
	117	+	// window is paired with a frame when the pairing is certain:
118	118		// - one window and one frame;
119		-	// - the focused window with the one ACTIVE frame;
120		-	// - otherwise the one frame whose size is within sizeTolerance of the
121		-	// window, provided no other window claims it as well.
	119	+	// - the focused window with the one ACTIVE frame; that frame is then
	120	+	// reserved and no other window can take it;
	121	+	// - otherwise the one unreserved frame whose size is within sizeTolerance
	122	+	// of the window, provided no other unmatched window claims it as well.
122	123		//
123	124		// Ambiguous windows are left out. The result maps the index of a window in
124	125		// wins to the index of its frame in frames.

@@ -128,31 +129,44 @@ func matchFrames(wins []ports.HintWindow, frames []frameInfo) map[int]int {
128	129		out[0] = 0
129	130		return out
130	131		}
	132	+	// Active matches first: they are reserved against the size matches.
	133	+	active, n := -1, 0
	134	+	for j, f := range frames {
	135	+	if f.active {
	136	+	active, n = j, n+1
	137	+	}
	138	+	}
	139	+	reserved := make([]bool, len(frames))
	140	+	done := make([]bool, len(wins))
	141	+	if n == 1 {
	142	+	focused := -1
	143	+	count := 0
	144	+	for i, w := range wins {
	145	+	if w.Focused {
	146	+	focused, count = i, count+1
	147	+	}
	148	+	}
	149	+	if count == 1 {
	150	+	out[focused] = active
	151	+	reserved[active] = true
	152	+	done[focused] = true
	153	+	}
	154	+	}
131	155		claims := make([][]int, len(wins)) // size candidates per window
132	156		owners := make([]int, len(frames)) // windows that size-match a frame
133	157		for i, w := range wins {
	158	+	if done[i] {
	159	+	continue
	160	+	}
134	161		for j, f := range frames {
135		-	if absInt(f.w-w.Rect.W) <= sizeTolerance && absInt(f.h-w.Rect.H) <= sizeTolerance {
	162	+	if !reserved[j] && absInt(f.w-w.Rect.W) <= sizeTolerance && absInt(f.h-w.Rect.H) <= sizeTolerance {
136	163		claims[i] = append(claims[i], j)
137	164		owners[j]++
138	165		}
139	166		}
140	167		}
141		-	for i, w := range wins {
142		-	if w.Focused {
143		-	active := -1
144		-	n := 0
145		-	for j, f := range frames {
146		-	if f.active {
147		-	active, n = j, n+1
148		-	}
149		-	}
150		-	if n == 1 {
151		-	out[i] = active
152		-	continue
153		-	}
154		-	}
155		-	if len(claims[i]) == 1 && owners[claims[i][0]] == 1 {
	168	+	for i := range wins {
	169	+	if !done[i] && len(claims[i]) == 1 && owners[claims[i][0]] == 1 {
156	170		out[i] = claims[i][0]
157	171		}
158	172		}

 	w, h   int // size of the frame, from its window extents
 }
-// matchFrames pairs the windows of one app with its frames. A window is
+// matchFrames pairs the windows of one app with its frames, one to one. A
-// paired with a frame when the pairing is certain:
+// window is paired with a frame when the pairing is certain:
 //   - one window and one frame;
-//   - the focused window with the one ACTIVE frame;
+//   - the focused window with the one ACTIVE frame; that frame is then
-//   - otherwise the one frame whose size is within sizeTolerance of the
+//     reserved and no other window can take it;
-//     window, provided no other window claims it as well.
+//   - otherwise the one unreserved frame whose size is within sizeTolerance
+//     of the window, provided no other unmatched window claims it as well.
 //
 // Ambiguous windows are left out. The result maps the index of a window in
 // wins to the index of its frame in frames.
 		out[0] = 0
 		return out
 	}
+	// Active matches first: they are reserved against the size matches.
+	active, n := -1, 0
+	for j, f := range frames {
+		if f.active {
+			active, n = j, n+1
+		}
+	}
+	reserved := make([]bool, len(frames))
+	done := make([]bool, len(wins))
+	if n == 1 {
+		focused := -1
+		count := 0
+		for i, w := range wins {
+			if w.Focused {
+				focused, count = i, count+1
+			}
+		}
+		if count == 1 {
+			out[focused] = active
+			reserved[active] = true
+			done[focused] = true
+		}
+	}
 	claims := make([][]int, len(wins)) // size candidates per window
 	owners := make([]int, len(frames)) // windows that size-match a frame
 	for i, w := range wins {
+		if done[i] {
+			continue
+		}
 		for j, f := range frames {
-			if absInt(f.w-w.Rect.W) <= sizeTolerance && absInt(f.h-w.Rect.H) <= sizeTolerance {
+			if !reserved[j] && absInt(f.w-w.Rect.W) <= sizeTolerance && absInt(f.h-w.Rect.H) <= sizeTolerance {
 				claims[i] = append(claims[i], j)
 				owners[j]++
 			}
 		}
 	}
-	for i, w := range wins {
+	for i := range wins {
-		if w.Focused {
+		if !done[i] && len(claims[i]) == 1 && owners[claims[i][0]] == 1 {
-			active := -1
-			n := 0
-			for j, f := range frames {
-				if f.active {
-					active, n = j, n+1
-				}
-			}
-			if n == 1 {
-				out[i] = active
-				continue
-			}
-		}
-		if len(claims[i]) == 1 && owners[claims[i][0]] == 1 {
 			out[i] = claims[i][0]
 		}
 	}

M internal/adapters/atspi/model_test.go +16 -3


@@ -73,9 +73,9 @@ func TestMatchFramesSingle(t *testing.T) {
73	73		func TestMatchFramesActive(t *testing.T) {
74	74		frames := []frameInfo{{w: 800, h: 600}, {w: 800, h: 600, active: true}}
75	75		got := matchFrames(wins(win(1, 800, 600, true), win(2, 800, 600, false)), frames)
76		-	// The focused window takes the ACTIVE frame; the other cannot be told
77		-	// apart by size (both frames match it) and is left out.
78		-	require.Equal(t, map[int]int{0: 1}, got)
	76	+	// The focused window takes the ACTIVE frame and reserves it; the other
	77	+	// window then only has the remaining frame to match.
	78	+	require.Equal(t, map[int]int{0: 1, 1: 0}, got)
79	79		}
80	80
81	81		func TestMatchFramesSize(t *testing.T) {

@@ -102,3 +102,16 @@ func TestMatchFramesAmbiguous(t *testing.T) {
102	102		// No frame close enough.
103	103		require.Empty(t, matchFrames(wins(win(1, 200, 200, false), win(2, 300, 300, false)), frames))
104	104		}
	105	+
	106	+	// The active frame is reserved by the focused window: an unfocused window of
	107	+	// the same size as that frame must not be paired with it as well.
	108	+	func TestMatchFramesOneToOne(t *testing.T) {
	109	+	frames := []frameInfo{{ref: ref{"a", "/active"}, active: true, w: 400, h: 300}, {ref: ref{"a", "/idle"}, w: 800, h: 600}}
	110	+	got := matchFrames(wins(win(1, 800, 600, true), win(2, 400, 300, false)), frames)
	111	+	seen := map[int]bool{}
	112	+	for _, f := range got {
	113	+	require.False(t, seen[f], "frame %d assigned twice: %v", f, got)
	114	+	seen[f] = true
	115	+	}
	116	+	require.Equal(t, map[int]int{0: 0}, got)
	117	+	}


@@ -73,9 +73,9 @@ func TestMatchFramesSingle(t *testing.T) {
73		func TestMatchFramesActive(t *testing.T) {	73		func TestMatchFramesActive(t *testing.T) {
74		frames := []frameInfo{{w: 800, h: 600}, {w: 800, h: 600, active: true}}	74		frames := []frameInfo{{w: 800, h: 600}, {w: 800, h: 600, active: true}}
75		got := matchFrames(wins(win(1, 800, 600, true), win(2, 800, 600, false)), frames)	75		got := matchFrames(wins(win(1, 800, 600, true), win(2, 800, 600, false)), frames)
76	-	// The focused window takes the ACTIVE frame; the other cannot be told	76	+	// The focused window takes the ACTIVE frame and reserves it; the other
77	-	// apart by size (both frames match it) and is left out.	77	+	// window then only has the remaining frame to match.
78	-	require.Equal(t, map[int]int{0: 1}, got)	78	+	require.Equal(t, map[int]int{0: 1, 1: 0}, got)
79		}	79		}
80			80
81		func TestMatchFramesSize(t *testing.T) {	81		func TestMatchFramesSize(t *testing.T) {

@@ -102,3 +102,16 @@ func TestMatchFramesAmbiguous(t *testing.T) {
102		// No frame close enough.	102		// No frame close enough.
103		require.Empty(t, matchFrames(wins(win(1, 200, 200, false), win(2, 300, 300, false)), frames))	103		require.Empty(t, matchFrames(wins(win(1, 200, 200, false), win(2, 300, 300, false)), frames))
104		}	104		}
			105	+
			106	+	// The active frame is reserved by the focused window: an unfocused window of
			107	+	// the same size as that frame must not be paired with it as well.
			108	+	func TestMatchFramesOneToOne(t *testing.T) {
			109	+	frames := []frameInfo{{ref: ref{"a", "/active"}, active: true, w: 400, h: 300}, {ref: ref{"a", "/idle"}, w: 800, h: 600}}
			110	+	got := matchFrames(wins(win(1, 800, 600, true), win(2, 400, 300, false)), frames)
			111	+	seen := map[int]bool{}
			112	+	for _, f := range got {
			113	+	require.False(t, seen[f], "frame %d assigned twice: %v", f, got)
			114	+	seen[f] = true
			115	+	}
			116	+	require.Equal(t, map[int]int{0: 0}, got)
			117	+	}

M internal/adapters/atspi/source.go +1 -0


@@ -255,6 +255,7 @@ func (s *Source) elements(do func([]call) []reply, matched []winMatch, linksOnly
255	255		if !ok {
256	256		continue
257	257		}
	258	+	t.Window = m.win.ID
258	259		out = append(out, t)
259	260		if t.Kind == ports.HintLink {
260	261		covered[m.win.ID] = true


@@ -255,6 +255,7 @@ func (s *Source) elements(do func([]call) []reply, matched []winMatch, linksOnly
255	if !ok {	255		if !ok {
256	continue	256		continue
257	}	257		}
		258	+	t.Window = m.win.ID
258	out = append(out, t)	259		out = append(out, t)
259	if t.Kind == ports.HintLink {	260		if t.Kind == ports.HintLink {
260	covered[m.win.ID] = true	261		covered[m.win.ID] = true

M internal/adapters/atspi/source_test.go +2 -2


@@ -74,8 +74,8 @@ func TestTargetsCoversWindowAndConverts(t *testing.T) {
74	74		require.NoError(t, err)
75	75		require.Equal(t, map[ports.WindowID]bool{1: true}, covered)
76	76		require.Equal(t, []ports.HintTarget{
77		-	{Rect: ports.Rect{X: 110, Y: 300, W: 50, H: 16}, Kind: ports.HintLink},
78		-	{Rect: ports.Rect{X: 200, Y: 400, W: 20, H: 20}, Kind: ports.HintButton},
	77	+	{Rect: ports.Rect{X: 110, Y: 300, W: 50, H: 16}, Kind: ports.HintLink, Window: 1},
	78	+	{Rect: ports.Rect{X: 200, Y: 400, W: 20, H: 20}, Kind: ports.HintButton, Window: 1},
79	79		}, targets)
80	80		}
81	81


@@ -74,8 +74,8 @@ func TestTargetsCoversWindowAndConverts(t *testing.T) {
74		require.NoError(t, err)	74		require.NoError(t, err)
75		require.Equal(t, map[ports.WindowID]bool{1: true}, covered)	75		require.Equal(t, map[ports.WindowID]bool{1: true}, covered)
76		require.Equal(t, []ports.HintTarget{	76		require.Equal(t, []ports.HintTarget{
77	-	{Rect: ports.Rect{X: 110, Y: 300, W: 50, H: 16}, Kind: ports.HintLink},	77	+	{Rect: ports.Rect{X: 110, Y: 300, W: 50, H: 16}, Kind: ports.HintLink, Window: 1},
78	-	{Rect: ports.Rect{X: 200, Y: 400, W: 20, H: 20}, Kind: ports.HintButton},	78	+	{Rect: ports.Rect{X: 200, Y: 400, W: 20, H: 20}, Kind: ports.HintButton, Window: 1},
79		}, targets)	79		}, targets)
80		}	80		}
81			81

M internal/ports/hint_messages.go +2 -0


@@ -42,6 +42,8 @@ type HintTarget struct {
42	42		// Text is set for targets found by OCR (read from pixels, not from an
43	43		// accessibility tree).
44	44		Text bool
	45	+	// Window is the source window of an accessibility target, 0 for OCR.
	46	+	Window WindowID
45	47		}
46	48
47	49		// HintScanResult answers a HintScanRequest. Core owns Targets.


@@ -42,6 +42,8 @@ type HintTarget struct {
42	// Text is set for targets found by OCR (read from pixels, not from an	42		// Text is set for targets found by OCR (read from pixels, not from an
43	// accessibility tree).	43		// accessibility tree).
44	Text bool	44		Text bool
		45	+	// Window is the source window of an accessibility target, 0 for OCR.
		46	+	Window WindowID
45	}	47		}
46		48
47	// HintScanResult answers a HintScanRequest. Core owns Targets.	49		// HintScanResult answers a HintScanRequest. Core owns Targets.