feat(hints): tile wide outputs into 16:9 regions

commit: 400321da84d1238e688b8f190aa6ec12045b76cf
author: brice · 35h ago · 2026-10-21 15:10 UTC
committer: brice · 5h ago · 2026-10-22 20:27 UTC
parent: ab2b2af
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

6 files · +264 -35

M internal/adapters/hints/detector.go +25 -6


@@ -26,11 +26,16 @@ import (
26	26		type detFrame struct {
27	27		width, height, stride int
28	28		m *mapping
	29	+	// regions are the crops detected independently; none means the whole
	30	+	// frame.
	31	+	regions []region
29	32		}
30	33
31	34		// detection is one box found by the detector, in frame (physical) pixels.
32	35		type detection struct {
33	36		x1, y1, x2, y2, score float32
	37	+	// region is the index of the request region the box comes from.
	38	+	region int
34	39		}
35	40
36	41		// detector finds clickable elements in a frame. The worker goroutine is its

@@ -68,10 +73,10 @@ var (
68	73
69	74		var le = binary.LittleEndian
70	75
71		-	// writeRequest writes one whole-frame request (no region, default
72		-	// confidence) followed by the pixels.
	76	+	// writeRequest writes one request (the regions of f, default confidence)
	77	+	// followed by the pixels.
73	78		func writeRequest(w io.Writer, seq uint64, f detFrame) error {
74		-	var h [4 + 4 + 8 + 5*4 + 4]byte
	79	+	var h [4 + 4 + 8 + 5*4]byte
75	80		copy(h[0:], magicRequest[:])
76	81		le.PutUint32(h[4:], protocolVersion)
77	82		le.PutUint64(h[8:], seq)

@@ -79,11 +84,25 @@ func writeRequest(w io.Writer, seq uint64, f detFrame) error {
79	84		le.PutUint32(h[20:], uint32(f.height))
80	85		le.PutUint32(h[24:], uint32(f.stride))
81	86		le.PutUint32(h[28:], formatXRGB8888)
82		-	le.PutUint32(h[32:], 0) // nregions
83		-	le.PutUint32(h[36:], math.Float32bits(0)) // confidence: startup default
	87	+	le.PutUint32(h[32:], uint32(len(f.regions)))
84	88		if _, err := w.Write(h[:]); err != nil {
85	89		return err
86	90		}
	91	+	var rec [16]byte
	92	+	for _, r := range f.regions {
	93	+	le.PutUint32(rec[0:], uint32(int32(r.x)))
	94	+	le.PutUint32(rec[4:], uint32(int32(r.y)))
	95	+	le.PutUint32(rec[8:], uint32(r.w))
	96	+	le.PutUint32(rec[12:], uint32(r.h))
	97	+	if _, err := w.Write(rec[:]); err != nil {
	98	+	return err
	99	+	}
	100	+	}
	101	+	var conf [4]byte // confidence: startup default
	102	+	le.PutUint32(conf[:], math.Float32bits(0))
	103	+	if _, err := w.Write(conf[:]); err != nil {
	104	+	return err
	105	+	}
87	106		_, err := w.Write(f.m.b[:f.height*f.stride])
88	107		return err
89	108		}

@@ -184,7 +203,7 @@ func readResponse(r io.Reader, seq uint64) ([]detection, error) {
184	203		return nil, err
185	204		}
186	205		f := func(i int) float32 { return math.Float32frombits(le.Uint32(rec[i*4:])) }
187		-	out = append(out, detection{x1: f(0), y1: f(1), x2: f(2), y2: f(3), score: f(4)})
	206	+	out = append(out, detection{x1: f(0), y1: f(1), x2: f(2), y2: f(3), score: f(4), region: int(le.Uint32(rec[20:]))})
188	207		}
189	208		return out, nil
190	209		}

 type detFrame struct {
 	width, height, stride int
 	m                     *mapping
+	// regions are the crops detected independently; none means the whole
+	// frame.
+	regions []region
 }
 // detection is one box found by the detector, in frame (physical) pixels.
 type detection struct {
 	x1, y1, x2, y2, score float32
+	// region is the index of the request region the box comes from.
+	region int
 }
 // detector finds clickable elements in a frame. The worker goroutine is its
 var le = binary.LittleEndian
-// writeRequest writes one whole-frame request (no region, default
+// writeRequest writes one request (the regions of f, default confidence)
-// confidence) followed by the pixels.
+// followed by the pixels.
 func writeRequest(w io.Writer, seq uint64, f detFrame) error {
-	var h [4 + 4 + 8 + 5*4 + 4]byte
+	var h [4 + 4 + 8 + 5*4]byte
 	copy(h[0:], magicRequest[:])
 	le.PutUint32(h[4:], protocolVersion)
 	le.PutUint64(h[8:], seq)
 	le.PutUint32(h[20:], uint32(f.height))
 	le.PutUint32(h[24:], uint32(f.stride))
 	le.PutUint32(h[28:], formatXRGB8888)
-	le.PutUint32(h[32:], 0)                   // nregions
+	le.PutUint32(h[32:], uint32(len(f.regions)))
-	le.PutUint32(h[36:], math.Float32bits(0)) // confidence: startup default
 	if _, err := w.Write(h[:]); err != nil {
 		return err
 	}
+	var rec [16]byte
+	for _, r := range f.regions {
+		le.PutUint32(rec[0:], uint32(int32(r.x)))
+		le.PutUint32(rec[4:], uint32(int32(r.y)))
+		le.PutUint32(rec[8:], uint32(r.w))
+		le.PutUint32(rec[12:], uint32(r.h))
+		if _, err := w.Write(rec[:]); err != nil {
+			return err
+		}
+	}
+	var conf [4]byte // confidence: startup default
+	le.PutUint32(conf[:], math.Float32bits(0))
+	if _, err := w.Write(conf[:]); err != nil {
+		return err
+	}
 	_, err := w.Write(f.m.b[:f.height*f.stride])
 	return err
 }
 			return nil, err
 		}
 		f := func(i int) float32 { return math.Float32frombits(le.Uint32(rec[i*4:])) }
-		out = append(out, detection{x1: f(0), y1: f(1), x2: f(2), y2: f(3), score: f(4)})
+		out = append(out, detection{x1: f(0), y1: f(1), x2: f(2), y2: f(3), score: f(4), region: int(le.Uint32(rec[20:]))})
 	}
 	return out, nil
 }

M internal/adapters/hints/detector_test.go +12 -2


@@ -28,6 +28,16 @@ func TestWriteRequest(t *testing.T) {
28	28		require.Equal(t, want, buf.Bytes())
29	29		}
30	30
	31	+	func TestWriteRequestRegions(t *testing.T) {
	32	+	var buf bytes.Buffer
	33	+	pix := []byte{1, 2, 3, 4}
	34	+	f := detFrame{width: 1, height: 1, stride: 4, m: &mapping{b: pix}, regions: []region{{0, 1, 2, 3}, {4, 5, 6, 7}}}
	35	+	require.NoError(t, writeRequest(&buf, 1, f))
	36	+	want := cat([]byte("NHRQ"), u32(1), u64(1), u32(1), u32(1), u32(4), u32(1), u32(2),
	37	+	u32(0), u32(1), u32(2), u32(3), u32(4), u32(5), u32(6), u32(7), f32(0), pix)
	38	+	require.Equal(t, want, buf.Bytes())
	39	+	}
	40	+
31	41		func TestReadHello(t *testing.T) {
32	42		dev, err := readHello(bytes.NewReader(cat([]byte("NHNT"), u32(1), u32(0), u32(6), []byte("Vulkan"), u32(1280))))
33	43		require.NoError(t, err)

@@ -52,10 +62,10 @@ func TestReadResponse(t *testing.T) {
52	62
53	63		ok := cat(head(5, 0), f32(12.5), u32(2),
54	64		f32(1), f32(2), f32(30), f32(40), f32(0.9), u32(0),
55		-	f32(5), f32(6), f32(7), f32(8), f32(0.25), u32(0))
	65	+	f32(5), f32(6), f32(7), f32(8), f32(0.25), u32(3))
56	66		got, err := readResponse(bytes.NewReader(ok), 5)
57	67		require.NoError(t, err)
58		-	require.Equal(t, []detection{{1, 2, 30, 40, 0.9}, {5, 6, 7, 8, 0.25}}, got)
	68	+	require.Equal(t, []detection{{x1: 1, y1: 2, x2: 30, y2: 40, score: 0.9}, {x1: 5, y1: 6, x2: 7, y2: 8, score: 0.25, region: 3}}, got)
59	69
60	70		got, err = readResponse(bytes.NewReader(cat(head(5, 0), f32(1), u32(0))), 5)
61	71		require.NoError(t, err)

 	require.Equal(t, want, buf.Bytes())
 }
+func TestWriteRequestRegions(t *testing.T) {
+	var buf bytes.Buffer
+	pix := []byte{1, 2, 3, 4}
+	f := detFrame{width: 1, height: 1, stride: 4, m: &mapping{b: pix}, regions: []region{{0, 1, 2, 3}, {4, 5, 6, 7}}}
+	require.NoError(t, writeRequest(&buf, 1, f))
+	want := cat([]byte("NHRQ"), u32(1), u64(1), u32(1), u32(1), u32(4), u32(1), u32(2),
+		u32(0), u32(1), u32(2), u32(3), u32(4), u32(5), u32(6), u32(7), f32(0), pix)
+	require.Equal(t, want, buf.Bytes())
+}
 func TestReadHello(t *testing.T) {
 	dev, err := readHello(bytes.NewReader(cat([]byte("NHNT"), u32(1), u32(0), u32(6), []byte("Vulkan"), u32(1280))))
 	require.NoError(t, err)
 	ok := cat(head(5, 0), f32(12.5), u32(2),
 		f32(1), f32(2), f32(30), f32(40), f32(0.9), u32(0),
-		f32(5), f32(6), f32(7), f32(8), f32(0.25), u32(0))
+		f32(5), f32(6), f32(7), f32(8), f32(0.25), u32(3))
 	got, err := readResponse(bytes.NewReader(ok), 5)
 	require.NoError(t, err)
-	require.Equal(t, []detection{{1, 2, 30, 40, 0.9}, {5, 6, 7, 8, 0.25}}, got)
+	require.Equal(t, []detection{{x1: 1, y1: 2, x2: 30, y2: 40, score: 0.9}, {x1: 5, y1: 6, x2: 7, y2: 8, score: 0.25, region: 3}}, got)
 	got, err = readResponse(bytes.NewReader(cat(head(5, 0), f32(1), u32(0))), 5)
 	require.NoError(t, err)

M internal/adapters/hints/hints.go +6 -3


@@ -215,7 +215,8 @@ const minTarget = 4
215	215		func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame, captureDur time.Duration, res ports.HintScanResult) ports.HintScanResult {
216	216		log := logging.For(ctx, "hints")
217	217		t0 := time.Now()
218		-	dets, err := w.det.Detect(ctx, detFrame{width: req.Width, height: req.Height, stride: fr.stride, m: fr.m})
	218	+	regions := tileRegions(req.Width, req.Height)
	219	+	dets, err := w.det.Detect(ctx, detFrame{width: req.Width, height: req.Height, stride: fr.stride, m: fr.m, regions: regions})
219	220		detectDur := time.Since(t0)
220	221		if err != nil {
221	222		fr.m.unmap()

@@ -223,8 +224,10 @@ func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame,
223	224		res.Err = err
224	225		return res
225	226		}
226		-	// Merge after rounding to logical pixels, on the union of all boxes.
227		-	res.Targets = mergeTargets(targetsOf(dets, req.Scale, req.Width, req.Height))
	227	+	// Boxes cut by a tile's inner edge are dropped when a neighbour tile
	228	+	// saw the element whole; then, on the union of all tiles and after
	229	+	// rounding to logical pixels, duplicates and nested boxes are merged.
	230	+	res.Targets = mergeTargets(targetsOf(dropCutBoxes(dets, regions), req.Scale, req.Width, req.Height))
228	231		if w.dumpDir != "" {
229	232		// The dump reads the frame after the result is sent and releases it.
230	233		w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, targets: res.Targets, dets: dets, captureD: captureDur, detectD: detectDur}

 func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame, captureDur time.Duration, res ports.HintScanResult) ports.HintScanResult {
 	log := logging.For(ctx, "hints")
 	t0 := time.Now()
-	dets, err := w.det.Detect(ctx, detFrame{width: req.Width, height: req.Height, stride: fr.stride, m: fr.m})
+	regions := tileRegions(req.Width, req.Height)
+	dets, err := w.det.Detect(ctx, detFrame{width: req.Width, height: req.Height, stride: fr.stride, m: fr.m, regions: regions})
 	detectDur := time.Since(t0)
 	if err != nil {
 		fr.m.unmap()
 		res.Err = err
 		return res
 	}
-	// Merge after rounding to logical pixels, on the union of all boxes.
+	// Boxes cut by a tile's inner edge are dropped when a neighbour tile
-	res.Targets = mergeTargets(targetsOf(dets, req.Scale, req.Width, req.Height))
+	// saw the element whole; then, on the union of all tiles and after
+	// rounding to logical pixels, duplicates and nested boxes are merged.
+	res.Targets = mergeTargets(targetsOf(dropCutBoxes(dets, regions), req.Scale, req.Width, req.Height))
 	if w.dumpDir != "" {
 		// The dump reads the frame after the result is sent and releases it.
 		w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, targets: res.Targets, dets: dets, captureD: captureDur, detectD: detectDur}

M internal/adapters/hints/hints_test.go +24 -24


@@ -97,30 +97,30 @@ func TestTargetsOf(t *testing.T) {
97	97		scale float64
98	98		want []ports.HintTarget
99	99		}{
100		-	{"inside", detection{10, 20, 50, 40, 0.5}, 1, r(10, 20, 40, 20, 0.5)},
101		-	{"scale 2 rounds outwards, clamped at 100", detection{11, 21, 100.2, 61, 0.5}, 2, r(5, 10, 45, 21, 0.5)},
102		-	{"zero scale reads as 1", detection{0, 0, 10, 10, 0.5}, 0, r(0, 0, 10, 10, 0.5)},
103		-	{"exactly 4x4 kept", detection{0, 0, 8, 8, 0.4}, 2, r(0, 0, 4, 4, 0.4)},
104		-	{"3 logical wide dropped", detection{0, 0, 5, 40, 0.9}, 2, nil},
105		-	{"3 logical high dropped", detection{0, 0, 40, 5, 0.9}, 2, nil},
106		-	{"fully outside", detection{200, 10, 300, 50, 0.5}, 1, nil},
107		-	{"fully before origin", detection{-50, -50, -10, -10, 0.5}, 1, nil},
108		-	{"partly outside is clamped", detection{-20, 70, 150, 120, 0.5}, 1, r(0, 70, 100, 30, 0.5)},
109		-	{"partly outside, scale 2", detection{-20, 70, 150, 120, 0.5}, 2, r(0, 35, 50, 15, 0.5)},
110		-	{"inverted x", detection{50, 10, 20, 40, 0.5}, 1, nil},
111		-	{"inverted y", detection{10, 40, 50, 10, 0.5}, 1, nil},
112		-	{"empty x", detection{10, 10, 10, 40, 0.5}, 1, nil},
113		-	{"empty y", detection{10, 10, 40, 10, 0.5}, 1, nil},
114		-	{"NaN coordinate", detection{nan, 0, 50, 50, 0.5}, 1, nil},
115		-	{"Inf coordinate", detection{0, 0, inf, 50, 0.5}, 1, nil},
116		-	{"NaN score", detection{0, 0, 50, 50, nan}, 1, nil},
117		-	{"Inf score", detection{0, 0, 50, 50, inf}, 1, nil},
118		-	{"score above 1", detection{0, 0, 50, 50, 1.5}, 1, nil},
119		-	{"negative score", detection{0, 0, 50, 50, -0.1}, 1, nil},
120		-	{"score 0 and 1 kept", detection{0, 0, 50, 50, 1}, 1, r(0, 0, 50, 50, 1)},
121		-	{"huge coordinates", detection{-1e30, -1e30, 1e30, 1e30, 0.5}, 1, r(0, 0, 100, 100, 0.5)},
122		-	{"huge positive only", detection{10, 10, 1e30, 1e30, 0.5}, 2, r(5, 5, 45, 45, 0.5)},
123		-	{"huge outside", detection{1e30, 1e30, 3e30, 3e30, 0.5}, 1, nil},
	100	+	{"inside", detection{10, 20, 50, 40, 0.5, 0}, 1, r(10, 20, 40, 20, 0.5)},
	101	+	{"scale 2 rounds outwards, clamped at 100", detection{11, 21, 100.2, 61, 0.5, 0}, 2, r(5, 10, 45, 21, 0.5)},
	102	+	{"zero scale reads as 1", detection{0, 0, 10, 10, 0.5, 0}, 0, r(0, 0, 10, 10, 0.5)},
	103	+	{"exactly 4x4 kept", detection{0, 0, 8, 8, 0.4, 0}, 2, r(0, 0, 4, 4, 0.4)},
	104	+	{"3 logical wide dropped", detection{0, 0, 5, 40, 0.9, 0}, 2, nil},
	105	+	{"3 logical high dropped", detection{0, 0, 40, 5, 0.9, 0}, 2, nil},
	106	+	{"fully outside", detection{200, 10, 300, 50, 0.5, 0}, 1, nil},
	107	+	{"fully before origin", detection{-50, -50, -10, -10, 0.5, 0}, 1, nil},
	108	+	{"partly outside is clamped", detection{-20, 70, 150, 120, 0.5, 0}, 1, r(0, 70, 100, 30, 0.5)},
	109	+	{"partly outside, scale 2", detection{-20, 70, 150, 120, 0.5, 0}, 2, r(0, 35, 50, 15, 0.5)},
	110	+	{"inverted x", detection{50, 10, 20, 40, 0.5, 0}, 1, nil},
	111	+	{"inverted y", detection{10, 40, 50, 10, 0.5, 0}, 1, nil},
	112	+	{"empty x", detection{10, 10, 10, 40, 0.5, 0}, 1, nil},
	113	+	{"empty y", detection{10, 10, 40, 10, 0.5, 0}, 1, nil},
	114	+	{"NaN coordinate", detection{nan, 0, 50, 50, 0.5, 0}, 1, nil},
	115	+	{"Inf coordinate", detection{0, 0, inf, 50, 0.5, 0}, 1, nil},
	116	+	{"NaN score", detection{0, 0, 50, 50, nan, 0}, 1, nil},
	117	+	{"Inf score", detection{0, 0, 50, 50, inf, 0}, 1, nil},
	118	+	{"score above 1", detection{0, 0, 50, 50, 1.5, 0}, 1, nil},
	119	+	{"negative score", detection{0, 0, 50, 50, -0.1, 0}, 1, nil},
	120	+	{"score 0 and 1 kept", detection{0, 0, 50, 50, 1, 0}, 1, r(0, 0, 50, 50, 1)},
	121	+	{"huge coordinates", detection{-1e30, -1e30, 1e30, 1e30, 0.5, 0}, 1, r(0, 0, 100, 100, 0.5)},
	122	+	{"huge positive only", detection{10, 10, 1e30, 1e30, 0.5, 0}, 2, r(5, 5, 45, 45, 0.5)},
	123	+	{"huge outside", detection{1e30, 1e30, 3e30, 3e30, 0.5, 0}, 1, nil},
124	124		} {
125	125		t.Run(tc.name, func(t *testing.T) {
126	126		got := targetsOf([]detection{tc.det}, tc.scale, 100, 100)


@@ -97,30 +97,30 @@ func TestTargetsOf(t *testing.T) {
97		scale float64	97		scale float64
98		want []ports.HintTarget	98		want []ports.HintTarget
99		}{	99		}{
100	-	{"inside", detection{10, 20, 50, 40, 0.5}, 1, r(10, 20, 40, 20, 0.5)},	100	+	{"inside", detection{10, 20, 50, 40, 0.5, 0}, 1, r(10, 20, 40, 20, 0.5)},
101	-	{"scale 2 rounds outwards, clamped at 100", detection{11, 21, 100.2, 61, 0.5}, 2, r(5, 10, 45, 21, 0.5)},	101	+	{"scale 2 rounds outwards, clamped at 100", detection{11, 21, 100.2, 61, 0.5, 0}, 2, r(5, 10, 45, 21, 0.5)},
102	-	{"zero scale reads as 1", detection{0, 0, 10, 10, 0.5}, 0, r(0, 0, 10, 10, 0.5)},	102	+	{"zero scale reads as 1", detection{0, 0, 10, 10, 0.5, 0}, 0, r(0, 0, 10, 10, 0.5)},
103	-	{"exactly 4x4 kept", detection{0, 0, 8, 8, 0.4}, 2, r(0, 0, 4, 4, 0.4)},	103	+	{"exactly 4x4 kept", detection{0, 0, 8, 8, 0.4, 0}, 2, r(0, 0, 4, 4, 0.4)},
104	-	{"3 logical wide dropped", detection{0, 0, 5, 40, 0.9}, 2, nil},	104	+	{"3 logical wide dropped", detection{0, 0, 5, 40, 0.9, 0}, 2, nil},
105	-	{"3 logical high dropped", detection{0, 0, 40, 5, 0.9}, 2, nil},	105	+	{"3 logical high dropped", detection{0, 0, 40, 5, 0.9, 0}, 2, nil},
106	-	{"fully outside", detection{200, 10, 300, 50, 0.5}, 1, nil},	106	+	{"fully outside", detection{200, 10, 300, 50, 0.5, 0}, 1, nil},
107	-	{"fully before origin", detection{-50, -50, -10, -10, 0.5}, 1, nil},	107	+	{"fully before origin", detection{-50, -50, -10, -10, 0.5, 0}, 1, nil},
108	-	{"partly outside is clamped", detection{-20, 70, 150, 120, 0.5}, 1, r(0, 70, 100, 30, 0.5)},	108	+	{"partly outside is clamped", detection{-20, 70, 150, 120, 0.5, 0}, 1, r(0, 70, 100, 30, 0.5)},
109	-	{"partly outside, scale 2", detection{-20, 70, 150, 120, 0.5}, 2, r(0, 35, 50, 15, 0.5)},	109	+	{"partly outside, scale 2", detection{-20, 70, 150, 120, 0.5, 0}, 2, r(0, 35, 50, 15, 0.5)},
110	-	{"inverted x", detection{50, 10, 20, 40, 0.5}, 1, nil},	110	+	{"inverted x", detection{50, 10, 20, 40, 0.5, 0}, 1, nil},
111	-	{"inverted y", detection{10, 40, 50, 10, 0.5}, 1, nil},	111	+	{"inverted y", detection{10, 40, 50, 10, 0.5, 0}, 1, nil},
112	-	{"empty x", detection{10, 10, 10, 40, 0.5}, 1, nil},	112	+	{"empty x", detection{10, 10, 10, 40, 0.5, 0}, 1, nil},
113	-	{"empty y", detection{10, 10, 40, 10, 0.5}, 1, nil},	113	+	{"empty y", detection{10, 10, 40, 10, 0.5, 0}, 1, nil},
114	-	{"NaN coordinate", detection{nan, 0, 50, 50, 0.5}, 1, nil},	114	+	{"NaN coordinate", detection{nan, 0, 50, 50, 0.5, 0}, 1, nil},
115	-	{"Inf coordinate", detection{0, 0, inf, 50, 0.5}, 1, nil},	115	+	{"Inf coordinate", detection{0, 0, inf, 50, 0.5, 0}, 1, nil},
116	-	{"NaN score", detection{0, 0, 50, 50, nan}, 1, nil},	116	+	{"NaN score", detection{0, 0, 50, 50, nan, 0}, 1, nil},
117	-	{"Inf score", detection{0, 0, 50, 50, inf}, 1, nil},	117	+	{"Inf score", detection{0, 0, 50, 50, inf, 0}, 1, nil},
118	-	{"score above 1", detection{0, 0, 50, 50, 1.5}, 1, nil},	118	+	{"score above 1", detection{0, 0, 50, 50, 1.5, 0}, 1, nil},
119	-	{"negative score", detection{0, 0, 50, 50, -0.1}, 1, nil},	119	+	{"negative score", detection{0, 0, 50, 50, -0.1, 0}, 1, nil},
120	-	{"score 0 and 1 kept", detection{0, 0, 50, 50, 1}, 1, r(0, 0, 50, 50, 1)},	120	+	{"score 0 and 1 kept", detection{0, 0, 50, 50, 1, 0}, 1, r(0, 0, 50, 50, 1)},
121	-	{"huge coordinates", detection{-1e30, -1e30, 1e30, 1e30, 0.5}, 1, r(0, 0, 100, 100, 0.5)},	121	+	{"huge coordinates", detection{-1e30, -1e30, 1e30, 1e30, 0.5, 0}, 1, r(0, 0, 100, 100, 0.5)},
122	-	{"huge positive only", detection{10, 10, 1e30, 1e30, 0.5}, 2, r(5, 5, 45, 45, 0.5)},	122	+	{"huge positive only", detection{10, 10, 1e30, 1e30, 0.5, 0}, 2, r(5, 5, 45, 45, 0.5)},
123	-	{"huge outside", detection{1e30, 1e30, 3e30, 3e30, 0.5}, 1, nil},	123	+	{"huge outside", detection{1e30, 1e30, 3e30, 3e30, 0.5, 0}, 1, nil},
124		} {	124		} {
125		t.Run(tc.name, func(t *testing.T) {	125		t.Run(tc.name, func(t *testing.T) {
126		got := targetsOf([]detection{tc.det}, tc.scale, 100, 100)	126		got := targetsOf([]detection{tc.det}, tc.scale, 100, 100)

A internal/adapters/hints/tile.go +122 -0

@@ -0,0 +1,122 @@
	1	+	package hints
	2	+
	3	+	const (
	4	+	// maxRegions is the protocol limit of regions per request.
	5	+	maxRegions = 64
	6	+	// wholeMaxSide is the longest frame side sent whole when the frame is not
	7	+	// 16:9: up to there the square letterbox of the detector loses little.
	8	+	wholeMaxSide = 2560
	9	+	// tileMaxH is the tallest tile; its width is 16/9 of it.
	10	+	tileMaxH = 1440
	11	+	// tileCutMargin is the distance, in physical pixels, to a tile's inner
	12	+	// edge under which a box counts as cut by it.
	13	+	tileCutMargin = 2
	14	+	)
	15	+
	16	+	// region is a crop of the frame, in physical pixels.
	17	+	type region struct{ x, y, w, h int }
	18	+
	19	+	// tileRegions returns the crops to detect separately on a width x height
	20	+	// frame, or nil for one whole-frame detection (a 16:9 frame, or one whose
	21	+	// long side is at most wholeMaxSide). Otherwise the frame is covered by exact
	22	+	// 16:9 tiles (w9 == h16: the detector then uses its rectangular graph) of
	23	+	// height min(height, 1440) rounded down to a multiple of 9, overlapping by at
	24	+	// least 10 % of the tile side on each axis; the last tile of a row or column
	25	+	// is aligned to the frame edge. Tiles are listed row by row. A frame that
	26	+	// would need more than maxRegions tiles is detected whole.
	27	+	func tileRegions(width, height int) []region {
	28	+	if width <= 0 \|\| height <= 0 \|\| width9 == height16 \|\| max(width, height) <= wholeMaxSide {
	29	+	return nil
	30	+	}
	31	+	th := min(height, tileMaxH) / 9 * 9
	32	+	tw := th / 9 * 16
	33	+	if tw > width { // a tall frame: the width bounds the tile
	34	+	tw = width / 16 * 16
	35	+	th = tw / 16 * 9
	36	+	}
	37	+	if tw == 0 \|\| th == 0 {
	38	+	return nil
	39	+	}
	40	+	xs, ys := tileStarts(width, tw), tileStarts(height, th)
	41	+	if len(xs)*len(ys) > maxRegions {
	42	+	return nil
	43	+	}
	44	+	out := make([]region, 0, len(xs)*len(ys))
	45	+	for _, y := range ys {
	46	+	for _, x := range xs {
	47	+	out = append(out, region{x, y, tw, th})
	48	+	}
	49	+	}
	50	+	return out
	51	+	}
	52	+
	53	+	// tileStarts returns the start offsets of tiles of size t covering length l,
	54	+	// evenly spread, the last one flush with the end, with consecutive tiles
	55	+	// overlapping by at least a tenth of t.
	56	+	func tileStarts(l, t int) []int {
	57	+	if l <= t {
	58	+	return []int{0}
	59	+	}
	60	+	d := l - t
	61	+	maxStep := t * 9 / 10
	62	+	n := (d+maxStep-1)/maxStep + 1
	63	+	out := make([]int, n)
	64	+	for i := range out {
	65	+	out[i] = (di + n - 2) / (n - 1) // ceil(di/(n-1)): steps never exceed maxStep
	66	+	}
	67	+	return out
	68	+	}
	69	+
	70	+	// dropCutBoxes drops the boxes that a tile's inner edge cut (an edge of the
	71	+	// tile that is not an edge of the frame) when another tile saw the same
	72	+	// element uncut: the cut box would otherwise survive as a smaller box nested
	73	+	// in the whole one. A box is "the same element" when the other box covers
	74	+	// nestRatio of it. Without regions the boxes are returned as they are.
	75	+	func dropCutBoxes(dets []detection, regions []region) []detection {
	76	+	if len(regions) == 0 {
	77	+	return dets
	78	+	}
	79	+	fw, fh := 0, 0
	80	+	for _, r := range regions {
	81	+	fw, fh = max(fw, r.x+r.w), max(fh, r.y+r.h)
	82	+	}
	83	+	cut := func(d detection) bool {
	84	+	if d.region < 0 \|\| d.region >= len(regions) {
	85	+	return false
	86	+	}
	87	+	r := regions[d.region]
	88	+	m := float32(tileCutMargin)
	89	+	return (r.x > 0 && d.x1 < float32(r.x)+m) \|\| (r.x+r.w < fw && d.x2 > float32(r.x+r.w)-m) \|\|
	90	+	(r.y > 0 && d.y1 < float32(r.y)+m) \|\| (r.y+r.h < fh && d.y2 > float32(r.y+r.h)-m)
	91	+	}
	92	+	isCut := make([]bool, len(dets))
	93	+	for i, d := range dets {
	94	+	isCut[i] = cut(d)
	95	+	}
	96	+	out := make([]detection, 0, len(dets))
	97	+	for i, d := range dets {
	98	+	drop := false
	99	+	if isCut[i] {
	100	+	for j, o := range dets {
	101	+	if j != i && !isCut[j] && o.region != d.region && covered(d, o) {
	102	+	drop = true
	103	+	break
	104	+	}
	105	+	}
	106	+	}
	107	+	if !drop {
	108	+	out = append(out, d)
	109	+	}
	110	+	}
	111	+	return out
	112	+	}
	113	+
	114	+	// covered reports whether o covers at least nestRatio of d's area.
	115	+	func covered(d, o detection) bool {
	116	+	w := min(d.x2, o.x2) - max(d.x1, o.x1)
	117	+	h := min(d.y2, o.y2) - max(d.y1, o.y1)
	118	+	if w <= 0 \|\| h <= 0 {
	119	+	return false
	120	+	}
	121	+	return float64(wh) >= nestRatiofloat64((d.x2-d.x1)*(d.y2-d.y1))
	122	+	}

@@ -0,0 +1,122 @@
	1	+	package hints
	2	+
	3	+	const (
	4	+	// maxRegions is the protocol limit of regions per request.
	5	+	maxRegions = 64
	6	+	// wholeMaxSide is the longest frame side sent whole when the frame is not
	7	+	// 16:9: up to there the square letterbox of the detector loses little.
	8	+	wholeMaxSide = 2560
	9	+	// tileMaxH is the tallest tile; its width is 16/9 of it.
	10	+	tileMaxH = 1440
	11	+	// tileCutMargin is the distance, in physical pixels, to a tile's inner
	12	+	// edge under which a box counts as cut by it.
	13	+	tileCutMargin = 2
	14	+	)
	15	+
	16	+	// region is a crop of the frame, in physical pixels.
	17	+	type region struct{ x, y, w, h int }
	18	+
	19	+	// tileRegions returns the crops to detect separately on a width x height
	20	+	// frame, or nil for one whole-frame detection (a 16:9 frame, or one whose
	21	+	// long side is at most wholeMaxSide). Otherwise the frame is covered by exact
	22	+	// 16:9 tiles (w9 == h16: the detector then uses its rectangular graph) of
	23	+	// height min(height, 1440) rounded down to a multiple of 9, overlapping by at
	24	+	// least 10 % of the tile side on each axis; the last tile of a row or column
	25	+	// is aligned to the frame edge. Tiles are listed row by row. A frame that
	26	+	// would need more than maxRegions tiles is detected whole.
	27	+	func tileRegions(width, height int) []region {
	28	+	if width <= 0 \|\| height <= 0 \|\| width9 == height16 \|\| max(width, height) <= wholeMaxSide {
	29	+	return nil
	30	+	}
	31	+	th := min(height, tileMaxH) / 9 * 9
	32	+	tw := th / 9 * 16
	33	+	if tw > width { // a tall frame: the width bounds the tile
	34	+	tw = width / 16 * 16
	35	+	th = tw / 16 * 9
	36	+	}
	37	+	if tw == 0 \|\| th == 0 {
	38	+	return nil
	39	+	}
	40	+	xs, ys := tileStarts(width, tw), tileStarts(height, th)
	41	+	if len(xs)*len(ys) > maxRegions {
	42	+	return nil
	43	+	}
	44	+	out := make([]region, 0, len(xs)*len(ys))
	45	+	for _, y := range ys {
	46	+	for _, x := range xs {
	47	+	out = append(out, region{x, y, tw, th})
	48	+	}
	49	+	}
	50	+	return out
	51	+	}
	52	+
	53	+	// tileStarts returns the start offsets of tiles of size t covering length l,
	54	+	// evenly spread, the last one flush with the end, with consecutive tiles
	55	+	// overlapping by at least a tenth of t.
	56	+	func tileStarts(l, t int) []int {
	57	+	if l <= t {
	58	+	return []int{0}
	59	+	}
	60	+	d := l - t
	61	+	maxStep := t * 9 / 10
	62	+	n := (d+maxStep-1)/maxStep + 1
	63	+	out := make([]int, n)
	64	+	for i := range out {
	65	+	out[i] = (di + n - 2) / (n - 1) // ceil(di/(n-1)): steps never exceed maxStep
	66	+	}
	67	+	return out
	68	+	}
	69	+
	70	+	// dropCutBoxes drops the boxes that a tile's inner edge cut (an edge of the
	71	+	// tile that is not an edge of the frame) when another tile saw the same
	72	+	// element uncut: the cut box would otherwise survive as a smaller box nested
	73	+	// in the whole one. A box is "the same element" when the other box covers
	74	+	// nestRatio of it. Without regions the boxes are returned as they are.
	75	+	func dropCutBoxes(dets []detection, regions []region) []detection {
	76	+	if len(regions) == 0 {
	77	+	return dets
	78	+	}
	79	+	fw, fh := 0, 0
	80	+	for _, r := range regions {
	81	+	fw, fh = max(fw, r.x+r.w), max(fh, r.y+r.h)
	82	+	}
	83	+	cut := func(d detection) bool {
	84	+	if d.region < 0 \|\| d.region >= len(regions) {
	85	+	return false
	86	+	}
	87	+	r := regions[d.region]
	88	+	m := float32(tileCutMargin)
	89	+	return (r.x > 0 && d.x1 < float32(r.x)+m) \|\| (r.x+r.w < fw && d.x2 > float32(r.x+r.w)-m) \|\|
	90	+	(r.y > 0 && d.y1 < float32(r.y)+m) \|\| (r.y+r.h < fh && d.y2 > float32(r.y+r.h)-m)
	91	+	}
	92	+	isCut := make([]bool, len(dets))
	93	+	for i, d := range dets {
	94	+	isCut[i] = cut(d)
	95	+	}
	96	+	out := make([]detection, 0, len(dets))
	97	+	for i, d := range dets {
	98	+	drop := false
	99	+	if isCut[i] {
	100	+	for j, o := range dets {
	101	+	if j != i && !isCut[j] && o.region != d.region && covered(d, o) {
	102	+	drop = true
	103	+	break
	104	+	}
	105	+	}
	106	+	}
	107	+	if !drop {
	108	+	out = append(out, d)
	109	+	}
	110	+	}
	111	+	return out
	112	+	}
	113	+
	114	+	// covered reports whether o covers at least nestRatio of d's area.
	115	+	func covered(d, o detection) bool {
	116	+	w := min(d.x2, o.x2) - max(d.x1, o.x1)
	117	+	h := min(d.y2, o.y2) - max(d.y1, o.y1)
	118	+	if w <= 0 \|\| h <= 0 {
	119	+	return false
	120	+	}
	121	+	return float64(wh) >= nestRatiofloat64((d.x2-d.x1)*(d.y2-d.y1))
	122	+	}

A internal/adapters/hints/tile_test.go +75 -0

@@ -0,0 +1,75 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"testing"
	5	+
	6	+	"github.com/stretchr/testify/require"
	7	+	)
	8	+
	9	+	func TestTileRegions(t *testing.T) {
	10	+	whole := [][2]int{{3840, 2160}, {1920, 1080}, {2560, 1080}, {2560, 1440}, {1280, 720}}
	11	+	for _, s := range whole {
	12	+	require.Nil(t, tileRegions(s[0], s[1]), "%dx%d", s[0], s[1])
	13	+	}
	14	+	require.Equal(t, []region{{0, 0, 2560, 1440}, {1280, 0, 2560, 1440}, {2560, 0, 2560, 1440},
	15	+	{0, 720, 2560, 1440}, {1280, 720, 2560, 1440}, {2560, 720, 2560, 1440}}, tileRegions(5120, 2160))
	16	+	require.Equal(t, []region{{0, 0, 2560, 1440}, {880, 0, 2560, 1440}}, tileRegions(3440, 1440))
	17	+
	18	+	for _, s := range [][2]int{{5120, 2160}, {3440, 1440}, {5120, 1440}, {7680, 2160}, {3000, 2999}, {1000, 3000}, {5121, 2161}, {6000, 1001}} {
	19	+	w, h := s[0], s[1]
	20	+	rs := tileRegions(w, h)
	21	+	require.NotEmpty(t, rs, "%dx%d", w, h)
	22	+	require.LessOrEqual(t, len(rs), maxRegions)
	23	+	covered := make([]bool, w*h)
	24	+	for _, r := range rs {
	25	+	require.Equal(t, r.h16, r.w9, "%dx%d tile %+v is not 16:9", w, h, r)
	26	+	require.GreaterOrEqual(t, r.x, 0)
	27	+	require.GreaterOrEqual(t, r.y, 0)
	28	+	require.LessOrEqual(t, r.x+r.w, w)
	29	+	require.LessOrEqual(t, r.y+r.h, h)
	30	+	for y := r.y; y < r.y+r.h; y++ {
	31	+	for x := r.x; x < r.x+r.w; x++ {
	32	+	covered[y*w+x] = true
	33	+	}
	34	+	}
	35	+	}
	36	+	for i, c := range covered {
	37	+	if !c {
	38	+	t.Fatalf("%dx%d: pixel %d,%d not covered", w, h, i%w, i/w)
	39	+	}
	40	+	}
	41	+	}
	42	+	}
	43	+
	44	+	func TestTileStartsOverlap(t *testing.T) {
	45	+	for _, l := range []int{2561, 3440, 5120, 7680, 10000} {
	46	+	xs := tileStarts(l, 2560)
	47	+	require.Equal(t, 0, xs[0])
	48	+	require.Equal(t, l-2560, xs[len(xs)-1])
	49	+	for i := 1; i < len(xs); i++ {
	50	+	require.GreaterOrEqual(t, xs[i-1]+2560-xs[i], 256, "length %d", l) // >= 10 %
	51	+	}
	52	+	}
	53	+	}
	54	+
	55	+	func TestTileRegionsTooMany(t *testing.T) {
	56	+	require.Nil(t, tileRegions(60000, 20000))
	57	+	}
	58	+
	59	+	func TestDropCutBoxes(t *testing.T) {
	60	+	rs := tileRegions(5120, 2160) // tile 0: x 0..2560, tile 1: x 1280..3840
	61	+	// A button at x 2500..2700 is cut at 2560 by tile 0 and whole in tile 1.
	62	+	cutBox := detection{x1: 2500, y1: 100, x2: 2560, y2: 140, score: 0.6, region: 0}
	63	+	whole := detection{x1: 2500, y1: 100, x2: 2700, y2: 140, score: 0.8, region: 1}
	64	+	require.Equal(t, []detection{whole}, dropCutBoxes([]detection{cutBox, whole}, rs))
	65	+	// Nobody saw it whole: the cut box stays.
	66	+	require.Equal(t, []detection{cutBox}, dropCutBoxes([]detection{cutBox}, rs))
	67	+	// An edge of the frame is not an inner edge.
	68	+	edge := detection{x1: 0, y1: 0, x2: 60, y2: 30, score: 0.6, region: 0}
	69	+	// Region 3 starts at y 720: its box at the frame's left edge is not cut
	70	+	// either, so both stay though they sit in different regions.
	71	+	other := detection{x1: 0, y1: 800, x2: 60, y2: 830, score: 0.5, region: 3}
	72	+	require.Equal(t, []detection{edge, other}, dropCutBoxes([]detection{edge, other}, rs))
	73	+	// No regions: unchanged.
	74	+	require.Equal(t, []detection{cutBox}, dropCutBoxes([]detection{cutBox}, nil))
	75	+	}

@@ -0,0 +1,75 @@
	1	+	package hints
	2	+
	3	+	import (
	4	+	"testing"
	5	+
	6	+	"github.com/stretchr/testify/require"
	7	+	)
	8	+
	9	+	func TestTileRegions(t *testing.T) {
	10	+	whole := [][2]int{{3840, 2160}, {1920, 1080}, {2560, 1080}, {2560, 1440}, {1280, 720}}
	11	+	for _, s := range whole {
	12	+	require.Nil(t, tileRegions(s[0], s[1]), "%dx%d", s[0], s[1])
	13	+	}
	14	+	require.Equal(t, []region{{0, 0, 2560, 1440}, {1280, 0, 2560, 1440}, {2560, 0, 2560, 1440},
	15	+	{0, 720, 2560, 1440}, {1280, 720, 2560, 1440}, {2560, 720, 2560, 1440}}, tileRegions(5120, 2160))
	16	+	require.Equal(t, []region{{0, 0, 2560, 1440}, {880, 0, 2560, 1440}}, tileRegions(3440, 1440))
	17	+
	18	+	for _, s := range [][2]int{{5120, 2160}, {3440, 1440}, {5120, 1440}, {7680, 2160}, {3000, 2999}, {1000, 3000}, {5121, 2161}, {6000, 1001}} {
	19	+	w, h := s[0], s[1]
	20	+	rs := tileRegions(w, h)
	21	+	require.NotEmpty(t, rs, "%dx%d", w, h)
	22	+	require.LessOrEqual(t, len(rs), maxRegions)
	23	+	covered := make([]bool, w*h)
	24	+	for _, r := range rs {
	25	+	require.Equal(t, r.h16, r.w9, "%dx%d tile %+v is not 16:9", w, h, r)
	26	+	require.GreaterOrEqual(t, r.x, 0)
	27	+	require.GreaterOrEqual(t, r.y, 0)
	28	+	require.LessOrEqual(t, r.x+r.w, w)
	29	+	require.LessOrEqual(t, r.y+r.h, h)
	30	+	for y := r.y; y < r.y+r.h; y++ {
	31	+	for x := r.x; x < r.x+r.w; x++ {
	32	+	covered[y*w+x] = true
	33	+	}
	34	+	}
	35	+	}
	36	+	for i, c := range covered {
	37	+	if !c {
	38	+	t.Fatalf("%dx%d: pixel %d,%d not covered", w, h, i%w, i/w)
	39	+	}
	40	+	}
	41	+	}
	42	+	}
	43	+
	44	+	func TestTileStartsOverlap(t *testing.T) {
	45	+	for _, l := range []int{2561, 3440, 5120, 7680, 10000} {
	46	+	xs := tileStarts(l, 2560)
	47	+	require.Equal(t, 0, xs[0])
	48	+	require.Equal(t, l-2560, xs[len(xs)-1])
	49	+	for i := 1; i < len(xs); i++ {
	50	+	require.GreaterOrEqual(t, xs[i-1]+2560-xs[i], 256, "length %d", l) // >= 10 %
	51	+	}
	52	+	}
	53	+	}
	54	+
	55	+	func TestTileRegionsTooMany(t *testing.T) {
	56	+	require.Nil(t, tileRegions(60000, 20000))
	57	+	}
	58	+
	59	+	func TestDropCutBoxes(t *testing.T) {
	60	+	rs := tileRegions(5120, 2160) // tile 0: x 0..2560, tile 1: x 1280..3840
	61	+	// A button at x 2500..2700 is cut at 2560 by tile 0 and whole in tile 1.
	62	+	cutBox := detection{x1: 2500, y1: 100, x2: 2560, y2: 140, score: 0.6, region: 0}
	63	+	whole := detection{x1: 2500, y1: 100, x2: 2700, y2: 140, score: 0.8, region: 1}
	64	+	require.Equal(t, []detection{whole}, dropCutBoxes([]detection{cutBox, whole}, rs))
	65	+	// Nobody saw it whole: the cut box stays.
	66	+	require.Equal(t, []detection{cutBox}, dropCutBoxes([]detection{cutBox}, rs))
	67	+	// An edge of the frame is not an inner edge.
	68	+	edge := detection{x1: 0, y1: 0, x2: 60, y2: 30, score: 0.6, region: 0}
	69	+	// Region 3 starts at y 720: its box at the frame's left edge is not cut
	70	+	// either, so both stay though they sit in different regions.
	71	+	other := detection{x1: 0, y1: 800, x2: 60, y2: 830, score: 0.5, region: 3}
	72	+	require.Equal(t, []detection{edge, other}, dropCutBoxes([]detection{edge, other}, rs))
	73	+	// No regions: unchanged.
	74	+	require.Equal(t, []detection{cutBox}, dropCutBoxes([]detection{cutBox}, nil))
	75	+	}