1// Package hints finds clickable targets for hint mode: it captures the last
2// frame of an output through the capture pipeline and sends it to the
3// nefer-hints detector, a persistent subprocess.
4package hints
5
6import (
7 "cmp"
8 "context"
9 "errors"
10 "fmt"
11 "image"
12 "math"
13 "slices"
14 "time"
15
16 "github.com/bnema/neferwl/internal/adapters/logging"
17 "github.com/bnema/neferwl/internal/ports"
18)
19
20// ErrBusy answers a request that arrives while another one is in flight.
21var ErrBusy = errors.New("hints: scan in progress")
22
23// ErrTimeout is the error of a capture that is not answered in time.
24var ErrTimeout = errors.New("hints: capture timed out")
25
26// ErrBadRequest is the error of a request without a usable buffer size.
27var ErrBadRequest = errors.New("hints: invalid output size")
28
29const (
30 captureTimeout = time.Second
31 // formatXRGB8888 is the capture format: opaque BGRA in memory.
32 formatXRGB8888 = 1
33 maxPixels = 1 << 28
34)
35
36// Run serves requests until ctx is done. It handles one request at a time;
37// a request that arrives meanwhile is answered with ErrBusy. Every request
38// gets exactly one result on results.
39func Run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, opts Options) error {
40 return run(ctx, requests, captures, captured, results, captureTimeout, opts)
41}
42
43type worker struct {
44 det detector
45 n uint64
46 dumpDir string
47 pending *dumpJob
48 // orphans are capture buffers the output may still write: they are
49 // released when its CaptureDone finally arrives.
50 orphans map[uint64]*mapping
51 live int // mappings not released yet
52}
53
54// frame is the captured output: BGRA pixels.
55type frame struct {
56 pix []byte
57 stride int
58 m *mapping
59}
60
61func run(ctx context.Context, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration, opts Options) error {
62 w := &worker{dumpDir: opts.DumpDir, det: opts.detector}
63 if w.det == nil {
64 w.det = newProcDetector(opts.Command, opts.ModelDir, opts.Threshold)
65 }
66 // The worker goroutine is the only one to touch the detector.
67 defer func() { _ = w.det.Close() }()
68 for {
69 select {
70 case <-ctx.Done():
71 return nil
72 case done := <-captured:
73 // A capture abandoned by a timeout is finally done: release it.
74 w.reap(done.ID)
75 case req := <-requests:
76 res := w.serve(ctx, req, requests, captures, captured, results, timeout)
77 if !send(ctx, results, res) {
78 // The dump job will never run: release its frame.
79 if j := w.pending; j != nil {
80 w.pending = nil
81 j.m.unmap()
82 }
83 return nil
84 }
85 w.dump(ctx)
86 }
87 }
88}
89
90// dump writes the last successful scan, after its result was sent, then
91// releases its frame.
92func (w *worker) dump(ctx context.Context) {
93 j := w.pending
94 w.pending = nil
95 if j == nil {
96 return
97 }
98 defer j.m.unmap()
99 if err := writeDump(w.dumpDir, j); err != nil {
100 log := logging.For(ctx, "hints")
101 log.Warn().Err(err).Str("dir", w.dumpDir).Msg("hint debug dump failed")
102 }
103}
104
105func send(ctx context.Context, results chan<- ports.HintScanResult, res ports.HintScanResult) bool {
106 select {
107 case results <- res:
108 return true
109 case <-ctx.Done():
110 return false
111 }
112}
113
114func (w *worker) serve(ctx context.Context, req ports.HintScanRequest, requests <-chan ports.HintScanRequest, captures chan<- ports.CaptureRequest, captured <-chan ports.CaptureDone, results chan<- ports.HintScanResult, timeout time.Duration) ports.HintScanResult {
115 log := logging.For(ctx, "hints")
116 res := ports.HintScanResult{Seq: req.Seq, Output: req.Output}
117 fail := func(err error) ports.HintScanResult {
118 log.Info().Err(err).Str("output", req.Output).Msg("hint scan failed")
119 res.Err = err
120 return res
121 }
122
123 if req.Width <= 0 || req.Height <= 0 || req.Width*req.Height > maxPixels {
124 return fail(fmt.Errorf("%w: %dx%d", ErrBadRequest, req.Width, req.Height))
125 }
126 // The capture buffer is anonymous mapped memory, outside the Go heap. It
127 // is released once the detector answered, or by the dump job.
128 m := w.alloc(req.Width * req.Height * 4)
129 keep := false // the mapping outlives this call: dump job or pending capture
130 defer func() {
131 if !keep {
132 m.unmap()
133 }
134 }()
135 buf := m.b
136
137 w.n++
138 id := ports.CaptureIDInternal | w.n
139 stride := req.Width * 4
140 start := time.Now()
141 creq := ports.CaptureRequest{
142 ID: id,
143 Output: req.Output,
144 Region: image.Rect(0, 0, req.Width, req.Height),
145 Mapped: buf,
146 Width: req.Width,
147 Height: req.Height,
148 Stride: stride,
149 Format: formatXRGB8888,
150 Indicate: false,
151 }
152
153 timer := time.NewTimer(timeout)
154 defer timer.Stop()
155
156 select {
157 case captures <- creq:
158 case <-timer.C:
159 return fail(ErrTimeout)
160 case <-ctx.Done():
161 return fail(ctx.Err())
162 }
163 // From here the output owns buf until CaptureDone.
164 abandon := func() {
165 if w.orphans == nil {
166 w.orphans = map[uint64]*mapping{}
167 }
168 w.orphans[id] = m
169 keep = true
170 }
171
172 for {
173 select {
174 case done := <-captured:
175 if done.ID != id {
176 w.reap(done.ID)
177 continue
178 }
179 if done.Err != nil {
180 return fail(fmt.Errorf("hints: capture: %w", done.Err))
181 }
182 res = w.scan(ctx, req, frame{pix: buf, stride: stride, m: m}, time.Since(start), res)
183 keep = w.pending != nil && w.pending.m == m
184 return res
185 case other := <-requests:
186 // The buffer is in use until the capture is done: answer now.
187 busy := ports.HintScanResult{Seq: other.Seq, Output: other.Output, Err: ErrBusy}
188 if !send(ctx, results, busy) {
189 abandon()
190 return fail(ctx.Err())
191 }
192 case <-timer.C:
193 // The output may still write into buf: it is released when its
194 // CaptureDone arrives.
195 abandon()
196 return fail(ErrTimeout)
197 case <-ctx.Done():
198 abandon()
199 return fail(ctx.Err())
200 }
201 }
202}
203
204// reap releases the buffer of an abandoned capture that is finally done.
205func (w *worker) reap(id uint64) {
206 if m, ok := w.orphans[id]; ok {
207 delete(w.orphans, id)
208 m.unmap()
209 }
210}
211
212// minTarget is the smallest target side kept, in logical pixels.
213const minTarget = 4
214
215// maxWorkBoxes bounds the quadratic cut-box and merge passes.
216const maxWorkBoxes = 2000
217
218// scan sends the frame to the detector and turns the boxes into targets. It
219// releases the frame, or hands it to the dump job.
220func (w *worker) scan(ctx context.Context, req ports.HintScanRequest, fr frame, captureDur time.Duration, res ports.HintScanResult) ports.HintScanResult {
221 log := logging.For(ctx, "hints")
222 t0 := time.Now()
223 regions := tileRegions(req.Width, req.Height)
224 dets, err := w.det.Detect(ctx, detFrame{width: req.Width, height: req.Height, stride: fr.stride, m: fr.m, regions: regions})
225 rawN := len(dets)
226 detectDur := time.Since(t0)
227 if err != nil {
228 fr.m.unmap()
229 log.Info().Err(err).Str("output", req.Output).Float64("detect_ms", ms(detectDur)).Msg("hint scan failed")
230 res.Err = err
231 return res
232 }
233 // Boxes cut by a tile's inner edge are dropped when a neighbour tile
234 // saw the element whole; then, on the union of all tiles and after
235 // rounding to logical pixels, duplicates and nested boxes are merged.
236 dets = prepareDetections(dets, regions, req.Width, req.Height)
237 res.Targets = mergeTargets(targetsOf(dropCutBoxes(dets, regions), req.Scale, req.Width, req.Height))
238 if w.dumpDir != "" {
239 // The dump reads the frame after the result is sent and releases it.
240 w.pending = &dumpJob{when: time.Now(), req: req, pix: fr.pix, m: fr.m, stride: fr.stride, targets: res.Targets, dets: dets, captureD: captureDur, detectD: detectDur}
241 } else {
242 fr.m.unmap()
243 }
244 log.Info().Str("output", req.Output).Float64("capture_ms", ms(captureDur)).Float64("detect_ms", ms(detectDur)).
245 Int("regions", len(regions)).Int("raw_boxes", rawN).Int("prepared_boxes", len(dets)).Int("targets", len(res.Targets)).Msg("hint scan")
246 return res
247}
248
249// targetsOf converts boxes in physical pixels to logical output-local
250// targets. A box with a non-finite coordinate or score, a score outside
251// [0,1] or no area is dropped; the others are clamped to the width x height
252// frame (physical), rounded outwards to logical pixels, and dropped when
253// under minTarget on a side.
254func targetsOf(dets []detection, scale float64, width, height int) []ports.HintTarget {
255 if scale <= 0 {
256 scale = 1
257 }
258 fw, fh := float64(width), float64(height)
259 out := make([]ports.HintTarget, 0, len(dets))
260 for _, d := range dets {
261 v := [4]float64{float64(d.x1), float64(d.y1), float64(d.x2), float64(d.y2)}
262 ok := d.score >= 0 && d.score <= 1 // false for NaN too
263 for _, f := range v {
264 ok = ok && !math.IsNaN(f) && !math.IsInf(f, 0)
265 }
266 if !ok || v[2] <= v[0] || v[3] <= v[1] {
267 continue
268 }
269 // Clamp before any int conversion: huge finite values stay safe.
270 x1, y1 := math.Max(0, math.Min(v[0], fw)), math.Max(0, math.Min(v[1], fh))
271 x2, y2 := math.Max(0, math.Min(v[2], fw)), math.Max(0, math.Min(v[3], fh))
272 lx1, ly1 := int(math.Floor(x1/scale)), int(math.Floor(y1/scale))
273 lx2, ly2 := int(math.Ceil(x2/scale)), int(math.Ceil(y2/scale))
274 if lx2-lx1 < minTarget || ly2-ly1 < minTarget {
275 continue
276 }
277 out = append(out, ports.HintTarget{Rect: ports.Rect{X: lx1, Y: ly1, W: lx2 - lx1, H: ly2 - ly1}, Score: d.score})
278 }
279 return out
280}
281
282// prepareDetections validates and clamps boxes before any pairwise work,
283// preserving their region metadata for cut-edge suppression.
284func prepareDetections(in []detection, regions []region, width, height int) []detection {
285 out := make([]detection, 0, len(in))
286 for _, d := range in {
287 if d.region < 0 || d.region >= max(1, len(regions)) || !(d.score >= 0 && d.score <= 1) {
288 continue
289 }
290 valid := true
291 for _, v := range []float32{d.x1, d.y1, d.x2, d.y2} {
292 if math.IsNaN(float64(v)) || math.IsInf(float64(v), 0) {
293 valid = false
294 break
295 }
296 }
297 if !valid {
298 continue
299 }
300 d.x1 = max(0, min(d.x1, float32(width)))
301 d.x2 = max(0, min(d.x2, float32(width)))
302 d.y1 = max(0, min(d.y1, float32(height)))
303 d.y2 = max(0, min(d.y2, float32(height)))
304 if d.x2 <= d.x1 || d.y2 <= d.y1 {
305 continue
306 }
307 out = append(out, d)
308 }
309 slices.SortStableFunc(out, func(a, b detection) int { return cmp.Compare(b.score, a.score) })
310 if len(out) > maxWorkBoxes {
311 out = out[:maxWorkBoxes]
312 }
313 return out
314}