fix(hints): merge duplicate and nested detector boxes

commit: 8a43081a7df1e576d5527130b022d47e5b2a9b3f
author: brice · 40h ago · 2026-10-21 15:07 UTC
committer: brice · 11h ago · 2026-10-22 20:27 UTC
parent: ea2f35e
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

3 files · +122 -1

M internal/adapters/hints/hints.go +2 -1


@@ -223,7 +223,8 @@ func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame,
223	223		res.Err = err
224	224		return res
225	225		}
226		-	res.Targets = targetsOf(dets, req.Scale, req.Width, req.Height)
	226	+	// Merge after rounding to logical pixels, on the union of all boxes.
	227	+	res.Targets = mergeTargets(targetsOf(dets, req.Scale, req.Width, req.Height))
227	228		if w.dumpDir != "" {
228	229		// The dump reads the frame after the result is sent and releases it.
229	230		w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, targets: res.Targets, dets: dets, captureD: captureDur, detectD: detectDur}

 		res.Err = err
 		return res
 	}
-	res.Targets = targetsOf(dets, req.Scale, req.Width, req.Height)
+	// Merge after rounding to logical pixels, on the union of all boxes.
+	res.Targets = mergeTargets(targetsOf(dets, req.Scale, req.Width, req.Height))
 	if w.dumpDir != "" {
 		// The dump reads the frame after the result is sent and releases it.
 		w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, targets: res.Targets, dets: dets, captureD: captureDur, detectD: detectDur}

A internal/adapters/hints/merge.go +82 -0

@@ -0,0 +1,82 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"cmp"
	5	+	"slices"
	6	+
	7	+	"github.com/bnema/neferwl/internal/ports"
	8	+	)
	9	+
	10	+	const (
	11	+	// dupIoU is the IoU above which two boxes are the same element: the
	12	+	// higher score is kept.
	13	+	dupIoU = 0.5
	14	+	// nestRatio is the share of the smaller box that must lie inside the
	15	+	// other for the two to be nested.
	16	+	nestRatio = 0.85
	17	+	)
	18	+
	19	+	func area(r ports.Rect) int { return r.W * r.H }
	20	+
	21	+	// interArea is the area of the intersection of a and b.
	22	+	func interArea(a, b ports.Rect) int {
	23	+	w := min(a.X+a.W, b.X+b.W) - max(a.X, b.X)
	24	+	h := min(a.Y+a.H, b.Y+b.H) - max(a.Y, b.Y)
	25	+	if w <= 0 \|\| h <= 0 {
	26	+	return 0
	27	+	}
	28	+	return w * h
	29	+	}
	30	+
	31	+	// nested reports whether the smaller of a and b lies for at least nestRatio
	32	+	// inside the other.
	33	+	func nested(a, b ports.Rect) bool {
	34	+	return float64(interArea(a, b)) >= nestRatio*float64(min(area(a), area(b)))
	35	+	}
	36	+
	37	+	// mergeTargets removes the duplicate and nested boxes the detector returns
	38	+	// for one element. It works on logical targets (after rounding), in two
	39	+	// passes so that the result does not depend on the processing order:
	40	+	//
	41	+	// 1. duplicates: by descending score, a box whose IoU with an already kept
	42	+	// box is over dupIoU is dropped;
	43	+	// 2. nesting: a box that contains another surviving box (that box lies
	44	+	// nestRatio inside it) is dropped, whatever the scores: the smallest of
	45	+	// a nested chain is kept (a row loses against its icon and its text).
	46	+	//
	47	+	// The result is ordered by descending score, then position.
	48	+	func mergeTargets(in []ports.HintTarget) []ports.HintTarget {
	49	+	s := slices.Clone(in)
	50	+	slices.SortStableFunc(s, func(a, b ports.HintTarget) int {
	51	+	return cmp.Or(cmp.Compare(b.Score, a.Score), cmp.Compare(a.Rect.Y, b.Rect.Y), cmp.Compare(a.Rect.X, b.Rect.X),
	52	+	cmp.Compare(area(a.Rect), area(b.Rect)))
	53	+	})
	54	+	kept := s[:0:0]
	55	+	for _, t := range s {
	56	+	dup := false
	57	+	for _, k := range kept {
	58	+	inter := interArea(t.Rect, k.Rect)
	59	+	if inter > 0 && float64(inter) > dupIoU*float64(area(t.Rect)+area(k.Rect)-inter) {
	60	+	dup = true
	61	+	break
	62	+	}
	63	+	}
	64	+	if !dup {
	65	+	kept = append(kept, t)
	66	+	}
	67	+	}
	68	+	out := make([]ports.HintTarget, 0, len(kept))
	69	+	for i, t := range kept {
	70	+	contains := false
	71	+	for j, k := range kept {
	72	+	if i != j && area(k.Rect) < area(t.Rect) && nested(t.Rect, k.Rect) {
	73	+	contains = true
	74	+	break
	75	+	}
	76	+	}
	77	+	if !contains {
	78	+	out = append(out, t)
	79	+	}
	80	+	}
	81	+	return out
	82	+	}

@@ -0,0 +1,82 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"cmp"
	5	+	"slices"
	6	+
	7	+	"github.com/bnema/neferwl/internal/ports"
	8	+	)
	9	+
	10	+	const (
	11	+	// dupIoU is the IoU above which two boxes are the same element: the
	12	+	// higher score is kept.
	13	+	dupIoU = 0.5
	14	+	// nestRatio is the share of the smaller box that must lie inside the
	15	+	// other for the two to be nested.
	16	+	nestRatio = 0.85
	17	+	)
	18	+
	19	+	func area(r ports.Rect) int { return r.W * r.H }
	20	+
	21	+	// interArea is the area of the intersection of a and b.
	22	+	func interArea(a, b ports.Rect) int {
	23	+	w := min(a.X+a.W, b.X+b.W) - max(a.X, b.X)
	24	+	h := min(a.Y+a.H, b.Y+b.H) - max(a.Y, b.Y)
	25	+	if w <= 0 \|\| h <= 0 {
	26	+	return 0
	27	+	}
	28	+	return w * h
	29	+	}
	30	+
	31	+	// nested reports whether the smaller of a and b lies for at least nestRatio
	32	+	// inside the other.
	33	+	func nested(a, b ports.Rect) bool {
	34	+	return float64(interArea(a, b)) >= nestRatio*float64(min(area(a), area(b)))
	35	+	}
	36	+
	37	+	// mergeTargets removes the duplicate and nested boxes the detector returns
	38	+	// for one element. It works on logical targets (after rounding), in two
	39	+	// passes so that the result does not depend on the processing order:
	40	+	//
	41	+	// 1. duplicates: by descending score, a box whose IoU with an already kept
	42	+	// box is over dupIoU is dropped;
	43	+	// 2. nesting: a box that contains another surviving box (that box lies
	44	+	// nestRatio inside it) is dropped, whatever the scores: the smallest of
	45	+	// a nested chain is kept (a row loses against its icon and its text).
	46	+	//
	47	+	// The result is ordered by descending score, then position.
	48	+	func mergeTargets(in []ports.HintTarget) []ports.HintTarget {
	49	+	s := slices.Clone(in)
	50	+	slices.SortStableFunc(s, func(a, b ports.HintTarget) int {
	51	+	return cmp.Or(cmp.Compare(b.Score, a.Score), cmp.Compare(a.Rect.Y, b.Rect.Y), cmp.Compare(a.Rect.X, b.Rect.X),
	52	+	cmp.Compare(area(a.Rect), area(b.Rect)))
	53	+	})
	54	+	kept := s[:0:0]
	55	+	for _, t := range s {
	56	+	dup := false
	57	+	for _, k := range kept {
	58	+	inter := interArea(t.Rect, k.Rect)
	59	+	if inter > 0 && float64(inter) > dupIoU*float64(area(t.Rect)+area(k.Rect)-inter) {
	60	+	dup = true
	61	+	break
	62	+	}
	63	+	}
	64	+	if !dup {
	65	+	kept = append(kept, t)
	66	+	}
	67	+	}
	68	+	out := make([]ports.HintTarget, 0, len(kept))
	69	+	for i, t := range kept {
	70	+	contains := false
	71	+	for j, k := range kept {
	72	+	if i != j && area(k.Rect) < area(t.Rect) && nested(t.Rect, k.Rect) {
	73	+	contains = true
	74	+	break
	75	+	}
	76	+	}
	77	+	if !contains {
	78	+	out = append(out, t)
	79	+	}
	80	+	}
	81	+	return out
	82	+	}

A internal/adapters/hints/merge_test.go +38 -0

@@ -0,0 +1,38 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"testing"
	5	+
	6	+	"github.com/stretchr/testify/require"
	7	+
	8	+	"github.com/bnema/neferwl/internal/ports"
	9	+	)
	10	+
	11	+	func tg(x, y, w, h int, score float32) ports.HintTarget {
	12	+	return ports.HintTarget{Rect: ports.Rect{X: x, Y: y, W: w, H: h}, Score: score}
	13	+	}
	14	+
	15	+	func TestMergeTargets(t *testing.T) {
	16	+	tests := []struct {
	17	+	name string
	18	+	in []ports.HintTarget
	19	+	want []ports.HintTarget
	20	+	}{
	21	+	{"empty", nil, []ports.HintTarget{}},
	22	+	{"duplicate keeps the best score", []ports.HintTarget{tg(10, 10, 100, 40, 0.3), tg(12, 11, 100, 40, 0.9)}, []ports.HintTarget{tg(12, 11, 100, 40, 0.9)}},
	23	+	{"duplicate, input order irrelevant", []ports.HintTarget{tg(12, 11, 100, 40, 0.9), tg(10, 10, 100, 40, 0.3)}, []ports.HintTarget{tg(12, 11, 100, 40, 0.9)}},
	24	+	{"row with icon and text keeps both small ones", []ports.HintTarget{
	25	+	tg(0, 0, 400, 30, 0.95), tg(4, 4, 22, 22, 0.5), tg(40, 6, 200, 18, 0.4),
	26	+	}, []ports.HintTarget{tg(4, 4, 22, 22, 0.5), tg(40, 6, 200, 18, 0.4)}},
	27	+	{"nested keeps the smaller even with a lower score", []ports.HintTarget{tg(0, 0, 200, 100, 0.9), tg(20, 20, 50, 30, 0.1)}, []ports.HintTarget{tg(20, 20, 50, 30, 0.1)}},
	28	+	{"partly overlapping distinct buttons are kept", []ports.HintTarget{tg(0, 0, 100, 30, 0.8), tg(80, 0, 100, 30, 0.7)}, []ports.HintTarget{tg(0, 0, 100, 30, 0.8), tg(80, 0, 100, 30, 0.7)}},
	29	+	{"mostly (not 85%) inside is kept", []ports.HintTarget{tg(0, 0, 100, 100, 0.8), tg(60, 0, 100, 20, 0.7)}, []ports.HintTarget{tg(0, 0, 100, 100, 0.8), tg(60, 0, 100, 20, 0.7)}},
	30	+	{"chain of three nested keeps the innermost", []ports.HintTarget{tg(0, 0, 300, 200, 0.99), tg(10, 10, 200, 100, 0.5), tg(20, 20, 50, 30, 0.2)}, []ports.HintTarget{tg(20, 20, 50, 30, 0.2)}},
	31	+	{"disjoint", []ports.HintTarget{tg(0, 0, 10, 10, 0.2), tg(50, 50, 10, 10, 0.6)}, []ports.HintTarget{tg(50, 50, 10, 10, 0.6), tg(0, 0, 10, 10, 0.2)}},
	32	+	}
	33	+	for _, tc := range tests {
	34	+	t.Run(tc.name, func(t *testing.T) {
	35	+	require.Equal(t, tc.want, mergeTargets(tc.in))
	36	+	})
	37	+	}
	38	+	}

@@ -0,0 +1,38 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"testing"
	5	+
	6	+	"github.com/stretchr/testify/require"
	7	+
	8	+	"github.com/bnema/neferwl/internal/ports"
	9	+	)
	10	+
	11	+	func tg(x, y, w, h int, score float32) ports.HintTarget {
	12	+	return ports.HintTarget{Rect: ports.Rect{X: x, Y: y, W: w, H: h}, Score: score}
	13	+	}
	14	+
	15	+	func TestMergeTargets(t *testing.T) {
	16	+	tests := []struct {
	17	+	name string
	18	+	in []ports.HintTarget
	19	+	want []ports.HintTarget
	20	+	}{
	21	+	{"empty", nil, []ports.HintTarget{}},
	22	+	{"duplicate keeps the best score", []ports.HintTarget{tg(10, 10, 100, 40, 0.3), tg(12, 11, 100, 40, 0.9)}, []ports.HintTarget{tg(12, 11, 100, 40, 0.9)}},
	23	+	{"duplicate, input order irrelevant", []ports.HintTarget{tg(12, 11, 100, 40, 0.9), tg(10, 10, 100, 40, 0.3)}, []ports.HintTarget{tg(12, 11, 100, 40, 0.9)}},
	24	+	{"row with icon and text keeps both small ones", []ports.HintTarget{
	25	+	tg(0, 0, 400, 30, 0.95), tg(4, 4, 22, 22, 0.5), tg(40, 6, 200, 18, 0.4),
	26	+	}, []ports.HintTarget{tg(4, 4, 22, 22, 0.5), tg(40, 6, 200, 18, 0.4)}},
	27	+	{"nested keeps the smaller even with a lower score", []ports.HintTarget{tg(0, 0, 200, 100, 0.9), tg(20, 20, 50, 30, 0.1)}, []ports.HintTarget{tg(20, 20, 50, 30, 0.1)}},
	28	+	{"partly overlapping distinct buttons are kept", []ports.HintTarget{tg(0, 0, 100, 30, 0.8), tg(80, 0, 100, 30, 0.7)}, []ports.HintTarget{tg(0, 0, 100, 30, 0.8), tg(80, 0, 100, 30, 0.7)}},
	29	+	{"mostly (not 85%) inside is kept", []ports.HintTarget{tg(0, 0, 100, 100, 0.8), tg(60, 0, 100, 20, 0.7)}, []ports.HintTarget{tg(0, 0, 100, 100, 0.8), tg(60, 0, 100, 20, 0.7)}},
	30	+	{"chain of three nested keeps the innermost", []ports.HintTarget{tg(0, 0, 300, 200, 0.99), tg(10, 10, 200, 100, 0.5), tg(20, 20, 50, 30, 0.2)}, []ports.HintTarget{tg(20, 20, 50, 30, 0.2)}},
	31	+	{"disjoint", []ports.HintTarget{tg(0, 0, 10, 10, 0.2), tg(50, 50, 10, 10, 0.6)}, []ports.HintTarget{tg(50, 50, 10, 10, 0.6), tg(0, 0, 10, 10, 0.2)}},
	32	+	}
	33	+	for _, tc := range tests {
	34	+	t.Run(tc.name, func(t *testing.T) {
	35	+	require.Equal(t, tc.want, mergeTargets(tc.in))
	36	+	})
	37	+	}
	38	+	}