1package harness
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "io/fs"
9 "os"
10 "os/exec"
11 "path/filepath"
12 "strconv"
13 "strings"
14 "syscall"
15 "time"
16)
17
18const (
19 Pass = 0
20 Failed = 1
21 Usage = 2
22 Unavailable = 3
23 ProcessFailure = 4
24)
25
26type Options struct {
27 Size string
28 Scale string
29 Layout string
30 Background string
31 Input string
32 Out string
33 Keep bool
34 AllowUnpinned bool
35 Timeout time.Duration
36 ReadyTimeout time.Duration
37 Client []string
38 Expect Expectations
39 GoldenBase string
40}
41type Compositor struct {
42 Pinned bool `json:"pinned"`
43 SHA string `json:"sha,omitempty"`
44 Binary string `json:"binary,omitempty"`
45 PID int `json:"pid,omitempty"`
46}
47type Result struct {
48 Compositor Compositor `json:"compositor"`
49 Schema string `json:"schema"`
50 Status string `json:"status"`
51 Checks []Check `json:"checks"`
52 Artifacts map[string]string `json:"artifacts"`
53 Timings map[string]string `json:"timings"`
54 Error string `json:"error,omitempty"`
55}
56
57func validate(o Options) error {
58 if _, err := Config(o.Size, o.Scale, o.Layout, o.Background); err != nil {
59 return err
60 }
61 if o.Out == "" || o.Timeout <= 0 || o.ReadyTimeout <= 0 || o.ReadyTimeout >= o.Timeout {
62 return fmt.Errorf("out and positive timeouts (ready < total) required")
63 }
64 if o.Input != "" {
65 if fi, err := os.Stat(o.Input); err != nil || !fi.Mode().IsRegular() {
66 return fmt.Errorf("input script not readable: %s", o.Input)
67 }
68 }
69 for _, p := range o.Expect.Probes {
70 if _, err := ParseColor(p.Color); err != nil {
71 return err
72 }
73 }
74 for _, r := range o.Expect.Regions {
75 if r.Color != "" {
76 if _, err := ParseColor(r.Color); err != nil {
77 return err
78 }
79 }
80 }
81 for _, g := range o.Expect.Goldens {
82 if g.MaxDiffRatio < 0 || g.MaxDiffRatio > 1 || g.Path == "" {
83 return fmt.Errorf("invalid golden %q", g.Name)
84 }
85 }
86 return nil
87}
88
89// copyRunLog copies the compositor's structured run log into the artifacts.
90// A missing log (older NeferWL, early failure) is not an error; it reports
91// whether dst was written.
92func copyRunLog(src, dst string) (bool, error) {
93 data, err := os.ReadFile(src)
94 if errors.Is(err, fs.ErrNotExist) {
95 return false, nil
96 }
97 if err == nil {
98 err = os.WriteFile(dst, data, 0600)
99 }
100 return err == nil, err
101}
102func newLog(path string) (*os.File, error) {
103 return os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
104}
105func waitSocket(ctx context.Context, runtime string, exited <-chan error) (string, error) {
106 ticker := time.NewTicker(25 * time.Millisecond)
107 defer ticker.Stop()
108 for {
109 name, err := DiscoverSocket(ctx, runtime)
110 if err != nil || name != "" {
111 if ctx.Err() != nil {
112 return "", ctx.Err()
113 }
114 return name, err
115 }
116 select {
117 case err := <-exited:
118 return "", fmt.Errorf("compositor exited before ready: %v", err)
119 case <-ctx.Done():
120 return "", ctx.Err()
121 case <-ticker.C:
122 }
123 }
124}
125
126// Run writes result.json for every run that has acquired an artifact directory.
127// It never owns client-side debug files: the client receives NEFERGUI_DEBUG_DIR and writes them.
128func Run(ctx context.Context, o Options) (res Result, code int) {
129 res = Result{Schema: "nefergui-harness/v1", Status: "error", Checks: []Check{}, Artifacts: map[string]string{}, Timings: map[string]string{}}
130 code = ProcessFailure
131 if err := validate(o); err != nil {
132 res.Error = err.Error()
133 return res, Usage
134 }
135 start := time.Now()
136 runCtx, cancel := context.WithTimeout(ctx, o.Timeout)
137 defer cancel()
138 out, err := filepath.Abs(o.Out)
139 if err != nil {
140 res.Error = err.Error()
141 return res, Usage
142 }
143 if err = os.Mkdir(out, 0700); err != nil {
144 res.Error = fmt.Sprintf("out must be a new directory: %v", err)
145 return res, Usage
146 }
147 res.Artifacts["root"] = out
148 defer func() {
149 res.Timings["total"] = time.Since(start).String()
150 data, e := json.MarshalIndent(res, "", " ")
151 if e == nil {
152 e = os.WriteFile(filepath.Join(out, "result.json"), append(data, '\n'), 0600)
153 }
154 if e != nil {
155 writeErr := fmt.Errorf("write result.json: %w", e)
156 if res.Error != "" {
157 writeErr = errors.Join(errors.New(res.Error), writeErr)
158 }
159 res.Error = writeErr.Error()
160 res.Status = "error"
161 code = ProcessFailure
162 }
163 }()
164 fail := func(err error, c int) (Result, int) { res.Error = err.Error(); return res, c }
165 for key, dir := range map[string]string{"frames": "frames", "debug": "client-debug", "logs": "logs"} {
166 path := filepath.Join(out, dir)
167 if err = os.Mkdir(path, 0700); err != nil {
168 return fail(err, ProcessFailure)
169 }
170 res.Artifacts[key] = path
171 }
172 shots := FramesDir(filepath.Join(out, "frames"), o.Size) // first output's screenshots
173 res.Artifacts["frame"] = filepath.Join(shots, "latest.png")
174 res.Artifacts["compositor_log"] = filepath.Join(out, "logs", "compositor.log")
175 res.Artifacts["client_stdout"] = filepath.Join(out, "logs", "client.stdout")
176 res.Artifacts["client_stderr"] = filepath.Join(out, "logs", "client.stderr")
177 session, err := os.MkdirTemp("", "nefergui-harness-")
178 if err != nil {
179 return fail(err, ProcessFailure)
180 }
181 if !o.Keep {
182 defer os.RemoveAll(session)
183 } else {
184 res.Artifacts["session"] = session
185 }
186 runtime := filepath.Join(session, "runtime")
187 configHome := filepath.Join(session, "config")
188 for _, d := range []string{runtime, filepath.Join(configHome, "neferwl")} {
189 if err = os.MkdirAll(d, 0700); err != nil {
190 return fail(err, ProcessFailure)
191 }
192 }
193 cfg, err := Config(o.Size, o.Scale, o.Layout, o.Background)
194 if err != nil {
195 return fail(err, Usage)
196 }
197 if err = os.WriteFile(filepath.Join(configHome, "neferwl", "config"), []byte(cfg), 0600); err != nil {
198 return fail(err, ProcessFailure)
199 }
200 buildStart := time.Now()
201 if path := os.Getenv("NEFERGUI_NEFERWL"); path != "" {
202 res.Compositor = Compositor{Binary: path}
203 if !o.AllowUnpinned {
204 return fail(fmt.Errorf("NEFERGUI_NEFERWL requires --allow-unpinned"), Usage)
205 }
206 } else {
207 res.Compositor = Compositor{Pinned: true, SHA: NeferWLSHA}
208 }
209 buildCtx, buildCancel := context.WithTimeout(runCtx, 3*time.Minute)
210 binary, err := NeferWL(buildCtx)
211 buildCancel()
212 res.Timings["build"] = time.Since(buildStart).String()
213 if err != nil {
214 return fail(err, Unavailable)
215 }
216 if err = runCtx.Err(); err != nil {
217 return fail(err, Unavailable)
218 }
219 logFile, err := newLog(res.Artifacts["compositor_log"])
220 if err != nil {
221 return fail(err, ProcessFailure)
222 }
223 defer logFile.Close()
224 args := []string{"--backend=headless", "--no-terminal", "--no-xwayland", "--screenshot", res.Artifacts["frames"], "--size", o.Size, "--timeout", o.Timeout.String()}
225 if o.Input != "" {
226 args = append(args, "--input", o.Input)
227 }
228 cmd := exec.Command(binary, args...)
229 // NeferWL writes structured run logs under XDG_STATE_HOME; keep them in
230 // the session instead of the host state directory, and copy them out.
231 stateHome := filepath.Join(session, "state")
232 cmd.Env = withEnv(isolatedEnv(os.Environ()), map[string]string{"XDG_RUNTIME_DIR": runtime, "XDG_CONFIG_HOME": configHome, "XDG_STATE_HOME": stateHome, "WAYLAND_DISPLAY": ""})
233 // Registered before the reap defer, so it runs after the compositor exits.
234 defer func() {
235 dst := filepath.Join(out, "logs", "compositor-run.log")
236 ok, e := copyRunLog(filepath.Join(stateHome, "neferwl", "runs", "headless", "latest.log"), dst)
237 if ok {
238 res.Artifacts["compositor_run_log"] = dst
239 }
240 if e != nil {
241 copyErr := fmt.Errorf("copy compositor run log: %w", e)
242 if res.Error != "" {
243 copyErr = errors.Join(errors.New(res.Error), copyErr)
244 }
245 res.Error, res.Status, code = copyErr.Error(), "error", ProcessFailure
246 }
247 }()
248 cmd.Stdout = logFile
249 cmd.Stderr = logFile
250 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
251 if err = cmd.Start(); err != nil {
252 return fail(err, Unavailable)
253 }
254 res.Compositor.PID = cmd.Process.Pid
255 exited := make(chan error, 1)
256 go func() { exited <- cmd.Wait(); close(exited) }()
257 // Always reap the process; killing the group also stops any children.
258 defer func() {
259 _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
260 <-exited
261 }()
262 readyCtx, readyCancel := context.WithTimeout(runCtx, o.ReadyTimeout)
263 defer readyCancel()
264 readyStart := time.Now()
265 socket, err := waitSocket(readyCtx, runtime, exited)
266 res.Timings["ready"] = time.Since(readyStart).String()
267 if err != nil {
268 return fail(err, Unavailable)
269 }
270 // Create empty logs even for compositor-only sessions.
271 for _, path := range []string{res.Artifacts["client_stdout"], res.Artifacts["client_stderr"]} {
272 f, e := newLog(path)
273 if e != nil {
274 return fail(e, ProcessFailure)
275 }
276 _ = f.Close()
277 }
278 env := withEnv(isolatedEnv(os.Environ()), map[string]string{"XDG_RUNTIME_DIR": runtime, "XDG_CONFIG_HOME": configHome, "WAYLAND_DISPLAY": socket, "NEFERGUI_DEBUG_DIR": res.Artifacts["debug"]})
279 if len(o.Client) > 0 {
280 stdout, e := os.OpenFile(res.Artifacts["client_stdout"], os.O_WRONLY, 0600)
281 if e != nil {
282 return fail(e, ProcessFailure)
283 }
284 defer stdout.Close()
285 stderr, e := os.OpenFile(res.Artifacts["client_stderr"], os.O_WRONLY, 0600)
286 if e != nil {
287 return fail(e, ProcessFailure)
288 }
289 defer stderr.Close()
290 client := exec.Command(o.Client[0], o.Client[1:]...)
291 client.Env = env
292 client.Stdout = stdout
293 client.Stderr = stderr
294 client.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
295 if err = client.Start(); err != nil {
296 return fail(err, ProcessFailure)
297 }
298 clientExit := make(chan error, 1)
299 go func() { clientExit <- client.Wait(); close(clientExit) }()
300 defer func() {
301 _ = syscall.Kill(-client.Process.Pid, syscall.SIGKILL)
302 <-clientExit
303 }()
304 clientStart := time.Now()
305 capturePath := filepath.Join(res.Artifacts["frames"], "capture.png")
306 requestPath := filepath.Join(res.Artifacts["debug"], "capture-request")
307 requestSeen := false
308 var requestedAt time.Time
309 captureTick := time.NewTicker(10 * time.Millisecond)
310 defer captureTick.Stop()
311 clientLoop:
312 for {
313 select {
314 case err = <-clientExit:
315 break clientLoop
316 case err = <-exited:
317 err = fmt.Errorf("compositor exited during client: %v", err)
318 break clientLoop
319 case <-runCtx.Done():
320 err = runCtx.Err()
321 break clientLoop
322 case <-captureTick.C:
323 if !requestSeen {
324 requestedAt, err = captureRequestTime(requestPath)
325 if errors.Is(err, os.ErrNotExist) {
326 err = nil
327 } else if err == nil {
328 requestSeen = true
329 } else {
330 break clientLoop
331 }
332 }
333 if requestSeen {
334 err = copyNextFrame(shots, requestedAt, capturePath)
335 if err == nil {
336 res.Artifacts["capture"] = capturePath
337 requestSeen = false
338 break clientLoop
339 }
340 if !errors.Is(err, os.ErrNotExist) {
341 break clientLoop
342 }
343 err = nil
344 }
345 }
346 }
347 // After capture, still wait for the client to finish normally.
348 if res.Artifacts["capture"] != "" && err == nil {
349 select {
350 case err = <-clientExit:
351 case err = <-exited:
352 err = fmt.Errorf("compositor exited during client: %v", err)
353 case <-runCtx.Done():
354 err = runCtx.Err()
355 }
356 }
357 res.Timings["client"] = time.Since(clientStart).String()
358 // A request near client exit must not silently fall back to latest.png.
359 if err == nil && res.Artifacts["capture"] == "" {
360 if !requestSeen {
361 requestedAt, err = captureRequestTime(requestPath)
362 if errors.Is(err, os.ErrNotExist) {
363 err = nil
364 } else if err == nil {
365 requestSeen = true
366 }
367 }
368 if requestSeen && err == nil {
369 for runCtx.Err() == nil {
370 err = copyNextFrame(shots, requestedAt, capturePath)
371 if err == nil {
372 res.Artifacts["capture"] = capturePath
373 break
374 }
375 if !errors.Is(err, os.ErrNotExist) {
376 break
377 }
378 err = nil
379 select {
380 case <-runCtx.Done():
381 case <-captureTick.C:
382 }
383 }
384 if res.Artifacts["capture"] == "" && err == nil {
385 err = fmt.Errorf("capture-request: %w", runCtx.Err())
386 }
387 }
388 }
389 if err != nil {
390 return fail(fmt.Errorf("client/compositor: %w", err), ProcessFailure)
391 }
392 select {
393 case e := <-exited:
394 return fail(fmt.Errorf("compositor exited during client: %v", e), ProcessFailure)
395 default:
396 }
397 }
398 // Wait briefly for the asynchronous screenshot writer after the client exits.
399 frameCtx, frameCancel := context.WithTimeout(runCtx, 3*time.Second)
400 defer frameCancel()
401 // A compositor crash after readiness is not a successful capture, even if latest.png exists.
402 select {
403 case e := <-exited:
404 return fail(fmt.Errorf("compositor exited before capture: %v", e), ProcessFailure)
405 default:
406 }
407 tick := time.NewTicker(25 * time.Millisecond)
408 defer tick.Stop()
409 for {
410 _, e := LoadPNG(res.Artifacts["frame"])
411 if e == nil {
412 break
413 }
414 select {
415 case <-frameCtx.Done():
416 return fail(fmt.Errorf("no frame: %w", frameCtx.Err()), ProcessFailure)
417 case e := <-exited:
418 return fail(fmt.Errorf("compositor exited before frame: %v", e), ProcessFailure)
419 case <-tick.C:
420 }
421 }
422 checkFrame := res.Artifacts["frame"]
423 if res.Artifacts["capture"] != "" {
424 checkFrame = res.Artifacts["capture"]
425 }
426 checks, err := Evaluate(checkFrame, o.Expect, o.GoldenBase, out)
427 if err != nil {
428 return fail(err, ProcessFailure)
429 }
430 res.Checks = checks
431 for _, c := range checks {
432 if !c.Pass {
433 res.Status = "failed"
434 return res, Failed
435 }
436 }
437 res.Status = "pass"
438 return res, Pass
439}
440
441// frameIndex accepts only compositor screenshots, not latest.png or temporary files.
442func frameIndex(name string) (int, bool) {
443 if !strings.HasPrefix(name, "frame-") || !strings.HasSuffix(name, ".png") {
444 return 0, false
445 }
446 digits := strings.TrimSuffix(strings.TrimPrefix(name, "frame-"), ".png")
447 if len(digits) != 6 {
448 return 0, false
449 }
450 for _, c := range digits {
451 if c < '0' || c > '9' {
452 return 0, false
453 }
454 }
455 n, err := strconv.Atoi(digits)
456 return n, err == nil
457}
458func captureRequestTime(path string) (time.Time, error) {
459 data, err := os.ReadFile(path)
460 if err != nil {
461 return time.Time{}, err
462 }
463 ns, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
464 if err != nil || ns <= 0 {
465 return time.Time{}, fmt.Errorf("invalid capture-request timestamp %q", strings.TrimSpace(string(data)))
466 }
467 return time.Unix(0, ns), nil
468}
469
470// nextFrame picks the second screenshot written after the commit token.
471// NeferWL renders and writes screenshots one after another, so the first file
472// finished after the token may have been rendered before it; the second began
473// after the first finished and therefore shows the committed frame.
474// An incomplete candidate is retried instead of selecting a later frame.
475func nextFrame(dir string, after time.Time) (string, error) {
476 entries, err := os.ReadDir(dir)
477 if err != nil {
478 return "", err
479 }
480 first, second := int(^uint(0)>>1), int(^uint(0)>>1)
481 names := map[int]string{}
482 for _, entry := range entries {
483 if entry.IsDir() {
484 continue
485 }
486 n, ok := frameIndex(entry.Name())
487 if !ok {
488 continue
489 }
490 info, e := entry.Info()
491 if e != nil {
492 return "", e
493 }
494 if !info.ModTime().After(after) {
495 continue
496 }
497 names[n] = entry.Name()
498 if n < first {
499 first, second = n, first
500 } else if n < second {
501 second = n
502 }
503 }
504 return names[second], nil
505}
506func copyNextFrame(dir string, after time.Time, destination string) error {
507 name, err := nextFrame(dir, after)
508 if err != nil {
509 return err
510 }
511 if name == "" {
512 return os.ErrNotExist
513 }
514 // NeferWL writes frame-N directly. Retry an incomplete PNG on the next tick.
515 path := filepath.Join(dir, name)
516 if _, err := LoadPNG(path); err != nil {
517 return os.ErrNotExist
518 }
519 data, err := os.ReadFile(path)
520 if err != nil {
521 return err
522 }
523 return os.WriteFile(destination, data, 0600)
524}
525func Evaluate(frame string, expect Expectations, base, out string) ([]Check, error) {
526 img, err := LoadPNG(frame)
527 if err != nil {
528 return nil, err
529 }
530 checks := make([]Check, 0, len(expect.Probes)+len(expect.Regions)+len(expect.Goldens))
531 add := func(name string, err error) {
532 c := Check{Name: name, Pass: err == nil}
533 if err != nil {
534 c.Detail = err.Error()
535 }
536 checks = append(checks, c)
537 }
538 for _, p := range expect.Probes {
539 want, _ := ParseColor(p.Color)
540 add(p.Name, PixelProbe(img, p.X, p.Y, want, p.Tolerance))
541 }
542 for _, r := range expect.Regions {
543 err := BoundsCheck(img, r.Rect)
544 if err == nil && r.Color != "" {
545 want, _ := ParseColor(r.Color)
546 err = UniformRegion(img, r.Rect, want, r.Tolerance)
547 }
548 add(r.Name, err)
549 }
550 for i, g := range expect.Goldens {
551 path := g.Path
552 if !filepath.IsAbs(path) {
553 path = filepath.Join(base, path)
554 }
555 ref, e := LoadPNG(path)
556 if e != nil {
557 add(g.Name, e)
558 continue
559 }
560 pass, ratio, diff, e := CompareGolden(img, ref, g.Tolerance, g.MaxDiffRatio)
561 if diff != nil {
562 e = errors.Join(e, SavePNG(filepath.Join(out, fmt.Sprintf("diff-%d.png", i)), diff))
563 }
564 if e == nil && !pass {
565 e = fmt.Errorf("differing pixel ratio %.6f exceeds %.6f", ratio, g.MaxDiffRatio)
566 }
567 add(g.Name, e)
568 }
569 return checks, nil
570}