1package harness
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "io/fs"
9 "os"
10 "os/exec"
11 "path/filepath"
12 "strconv"
13 "strings"
14 "syscall"
15 "time"
16)
17
18const (
19 Pass = 0
20 Failed = 1
21 Usage = 2
22 Unavailable = 3
23 ProcessFailure = 4
24)
25
26type Options struct {
27 Size string
28 Scale string
29 Layout string
30 Background string
31 Input string
32 Out string
33 Keep bool
34 AllowUnpinned bool
35 Timeout time.Duration
36 ReadyTimeout time.Duration
37 Client []string
38 Expect Expectations
39 GoldenBase string
40}
41type Compositor struct {
42 Pinned bool `json:"pinned"`
43 SHA string `json:"sha,omitempty"`
44 Binary string `json:"binary,omitempty"`
45 PID int `json:"pid,omitempty"`
46}
47type Result struct {
48 Compositor Compositor `json:"compositor"`
49 Schema string `json:"schema"`
50 Status string `json:"status"`
51 Checks []Check `json:"checks"`
52 Artifacts map[string]string `json:"artifacts"`
53 Timings map[string]string `json:"timings"`
54 Error string `json:"error,omitempty"`
55}
56
57func validate(o Options) error {
58 if _, err := Config(o.Size, o.Scale, o.Layout, o.Background); err != nil {
59 return err
60 }
61 if o.Out == "" || o.Timeout <= 0 || o.ReadyTimeout <= 0 || o.ReadyTimeout >= o.Timeout {
62 return fmt.Errorf("out and positive timeouts (ready < total) required")
63 }
64 if o.Input != "" {
65 if fi, err := os.Stat(o.Input); err != nil || !fi.Mode().IsRegular() {
66 return fmt.Errorf("input script not readable: %s", o.Input)
67 }
68 }
69 for _, p := range o.Expect.Probes {
70 if _, err := ParseColor(p.Color); err != nil {
71 return err
72 }
73 }
74 for _, r := range o.Expect.Regions {
75 if r.Color != "" {
76 if _, err := ParseColor(r.Color); err != nil {
77 return err
78 }
79 }
80 }
81 for _, g := range o.Expect.Goldens {
82 if g.MaxDiffRatio < 0 || g.MaxDiffRatio > 1 || g.Path == "" {
83 return fmt.Errorf("invalid golden %q", g.Name)
84 }
85 }
86 return nil
87}
88
89// copyRunLog copies the compositor's structured run log into the artifacts.
90// A missing log (older NeferWL, early failure) is not an error; it reports
91// whether dst was written.
92func copyRunLog(src, dst string) (bool, error) {
93 data, err := os.ReadFile(src)
94 if errors.Is(err, fs.ErrNotExist) {
95 return false, nil
96 }
97 if err == nil {
98 err = os.WriteFile(dst, data, 0600)
99 }
100 return err == nil, err
101}
102func newLog(path string) (*os.File, error) {
103 return os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
104}
105func waitSocket(ctx context.Context, runtime string, exited <-chan error) (string, error) {
106 ticker := time.NewTicker(25 * time.Millisecond)
107 defer ticker.Stop()
108 for {
109 name, err := DiscoverSocket(ctx, runtime)
110 if err != nil || name != "" {
111 if ctx.Err() != nil {
112 return "", ctx.Err()
113 }
114 return name, err
115 }
116 select {
117 case err := <-exited:
118 return "", fmt.Errorf("compositor exited before ready: %v", err)
119 case <-ctx.Done():
120 return "", ctx.Err()
121 case <-ticker.C:
122 }
123 }
124}
125
126// Run writes result.json for every run that has acquired an artifact directory.
127// It never owns client-side debug files: the client receives NEFERGUI_DEBUG_DIR and writes them.
128func Run(ctx context.Context, o Options) (res Result, code int) {
129 res = Result{Schema: "nefergui-harness/v1", Status: "error", Checks: []Check{}, Artifacts: map[string]string{}, Timings: map[string]string{}}
130 code = ProcessFailure
131 if err := validate(o); err != nil {
132 res.Error = err.Error()
133 return res, Usage
134 }
135 start := time.Now()
136 runCtx, cancel := context.WithTimeout(ctx, o.Timeout)
137 defer cancel()
138 out, err := filepath.Abs(o.Out)
139 if err != nil {
140 res.Error = err.Error()
141 return res, Usage
142 }
143 if err = os.Mkdir(out, 0700); err != nil {
144 res.Error = fmt.Sprintf("out must be a new directory: %v", err)
145 return res, Usage
146 }
147 res.Artifacts["root"] = out
148 defer func() {
149 res.Timings["total"] = time.Since(start).String()
150 data, e := json.MarshalIndent(res, "", " ")
151 if e == nil {
152 e = os.WriteFile(filepath.Join(out, "result.json"), append(data, '\n'), 0600)
153 }
154 if e != nil {
155 writeErr := fmt.Errorf("write result.json: %w", e)
156 if res.Error != "" {
157 writeErr = errors.Join(errors.New(res.Error), writeErr)
158 }
159 res.Error = writeErr.Error()
160 res.Status = "error"
161 code = ProcessFailure
162 }
163 }()
164 fail := func(err error, c int) (Result, int) { res.Error = err.Error(); return res, c }
165 for key, dir := range map[string]string{"frames": "frames", "debug": "client-debug", "logs": "logs"} {
166 path := filepath.Join(out, dir)
167 if err = os.Mkdir(path, 0700); err != nil {
168 return fail(err, ProcessFailure)
169 }
170 res.Artifacts[key] = path
171 }
172 res.Artifacts["frame"] = filepath.Join(out, "frames", "latest.png")
173 res.Artifacts["compositor_log"] = filepath.Join(out, "logs", "compositor.log")
174 res.Artifacts["client_stdout"] = filepath.Join(out, "logs", "client.stdout")
175 res.Artifacts["client_stderr"] = filepath.Join(out, "logs", "client.stderr")
176 session, err := os.MkdirTemp("", "nefergui-harness-")
177 if err != nil {
178 return fail(err, ProcessFailure)
179 }
180 if !o.Keep {
181 defer os.RemoveAll(session)
182 } else {
183 res.Artifacts["session"] = session
184 }
185 runtime := filepath.Join(session, "runtime")
186 configHome := filepath.Join(session, "config")
187 for _, d := range []string{runtime, filepath.Join(configHome, "neferwl")} {
188 if err = os.MkdirAll(d, 0700); err != nil {
189 return fail(err, ProcessFailure)
190 }
191 }
192 cfg, err := Config(o.Size, o.Scale, o.Layout, o.Background)
193 if err != nil {
194 return fail(err, Usage)
195 }
196 if err = os.WriteFile(filepath.Join(configHome, "neferwl", "config"), []byte(cfg), 0600); err != nil {
197 return fail(err, ProcessFailure)
198 }
199 buildStart := time.Now()
200 if path := os.Getenv("NEFERGUI_NEFERWL"); path != "" {
201 res.Compositor = Compositor{Binary: path}
202 if !o.AllowUnpinned {
203 return fail(fmt.Errorf("NEFERGUI_NEFERWL requires --allow-unpinned"), Usage)
204 }
205 } else {
206 res.Compositor = Compositor{Pinned: true, SHA: NeferWLSHA}
207 }
208 buildCtx, buildCancel := context.WithTimeout(runCtx, 3*time.Minute)
209 binary, err := NeferWL(buildCtx)
210 buildCancel()
211 res.Timings["build"] = time.Since(buildStart).String()
212 if err != nil {
213 return fail(err, Unavailable)
214 }
215 if err = runCtx.Err(); err != nil {
216 return fail(err, Unavailable)
217 }
218 logFile, err := newLog(res.Artifacts["compositor_log"])
219 if err != nil {
220 return fail(err, ProcessFailure)
221 }
222 defer logFile.Close()
223 args := []string{"--backend=headless", "--no-terminal", "--no-xwayland", "--screenshot", res.Artifacts["frames"], "--size", o.Size, "--timeout", o.Timeout.String()}
224 if o.Input != "" {
225 args = append(args, "--input", o.Input)
226 }
227 cmd := exec.Command(binary, args...)
228 // NeferWL writes structured run logs under XDG_STATE_HOME; keep them in
229 // the session instead of the host state directory, and copy them out.
230 stateHome := filepath.Join(session, "state")
231 cmd.Env = withEnv(isolatedEnv(os.Environ()), map[string]string{"XDG_RUNTIME_DIR": runtime, "XDG_CONFIG_HOME": configHome, "XDG_STATE_HOME": stateHome, "WAYLAND_DISPLAY": ""})
232 // Registered before the reap defer, so it runs after the compositor exits.
233 defer func() {
234 dst := filepath.Join(out, "logs", "compositor-run.log")
235 ok, e := copyRunLog(filepath.Join(stateHome, "neferwl", "runs", "headless", "latest.log"), dst)
236 if ok {
237 res.Artifacts["compositor_run_log"] = dst
238 }
239 if e != nil {
240 copyErr := fmt.Errorf("copy compositor run log: %w", e)
241 if res.Error != "" {
242 copyErr = errors.Join(errors.New(res.Error), copyErr)
243 }
244 res.Error, res.Status, code = copyErr.Error(), "error", ProcessFailure
245 }
246 }()
247 cmd.Stdout = logFile
248 cmd.Stderr = logFile
249 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
250 if err = cmd.Start(); err != nil {
251 return fail(err, Unavailable)
252 }
253 res.Compositor.PID = cmd.Process.Pid
254 exited := make(chan error, 1)
255 go func() { exited <- cmd.Wait(); close(exited) }()
256 // Always reap the process; killing the group also stops any children.
257 defer func() {
258 _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
259 <-exited
260 }()
261 readyCtx, readyCancel := context.WithTimeout(runCtx, o.ReadyTimeout)
262 defer readyCancel()
263 readyStart := time.Now()
264 socket, err := waitSocket(readyCtx, runtime, exited)
265 res.Timings["ready"] = time.Since(readyStart).String()
266 if err != nil {
267 return fail(err, Unavailable)
268 }
269 // Create empty logs even for compositor-only sessions.
270 for _, path := range []string{res.Artifacts["client_stdout"], res.Artifacts["client_stderr"]} {
271 f, e := newLog(path)
272 if e != nil {
273 return fail(e, ProcessFailure)
274 }
275 _ = f.Close()
276 }
277 env := withEnv(isolatedEnv(os.Environ()), map[string]string{"XDG_RUNTIME_DIR": runtime, "XDG_CONFIG_HOME": configHome, "WAYLAND_DISPLAY": socket, "NEFERGUI_DEBUG_DIR": res.Artifacts["debug"]})
278 if len(o.Client) > 0 {
279 stdout, e := os.OpenFile(res.Artifacts["client_stdout"], os.O_WRONLY, 0600)
280 if e != nil {
281 return fail(e, ProcessFailure)
282 }
283 defer stdout.Close()
284 stderr, e := os.OpenFile(res.Artifacts["client_stderr"], os.O_WRONLY, 0600)
285 if e != nil {
286 return fail(e, ProcessFailure)
287 }
288 defer stderr.Close()
289 client := exec.Command(o.Client[0], o.Client[1:]...)
290 client.Env = env
291 client.Stdout = stdout
292 client.Stderr = stderr
293 client.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
294 if err = client.Start(); err != nil {
295 return fail(err, ProcessFailure)
296 }
297 clientExit := make(chan error, 1)
298 go func() { clientExit <- client.Wait(); close(clientExit) }()
299 defer func() {
300 _ = syscall.Kill(-client.Process.Pid, syscall.SIGKILL)
301 <-clientExit
302 }()
303 clientStart := time.Now()
304 capturePath := filepath.Join(res.Artifacts["frames"], "capture.png")
305 requestPath := filepath.Join(res.Artifacts["debug"], "capture-request")
306 requestSeen := false
307 var requestedAt time.Time
308 captureTick := time.NewTicker(10 * time.Millisecond)
309 defer captureTick.Stop()
310 clientLoop:
311 for {
312 select {
313 case err = <-clientExit:
314 break clientLoop
315 case err = <-exited:
316 err = fmt.Errorf("compositor exited during client: %v", err)
317 break clientLoop
318 case <-runCtx.Done():
319 err = runCtx.Err()
320 break clientLoop
321 case <-captureTick.C:
322 if !requestSeen {
323 requestedAt, err = captureRequestTime(requestPath)
324 if errors.Is(err, os.ErrNotExist) {
325 err = nil
326 } else if err == nil {
327 requestSeen = true
328 } else {
329 break clientLoop
330 }
331 }
332 if requestSeen {
333 err = copyNextFrame(res.Artifacts["frames"], requestedAt, capturePath)
334 if err == nil {
335 res.Artifacts["capture"] = capturePath
336 requestSeen = false
337 break clientLoop
338 }
339 if !errors.Is(err, os.ErrNotExist) {
340 break clientLoop
341 }
342 err = nil
343 }
344 }
345 }
346 // After capture, still wait for the client to finish normally.
347 if res.Artifacts["capture"] != "" && err == nil {
348 select {
349 case err = <-clientExit:
350 case err = <-exited:
351 err = fmt.Errorf("compositor exited during client: %v", err)
352 case <-runCtx.Done():
353 err = runCtx.Err()
354 }
355 }
356 res.Timings["client"] = time.Since(clientStart).String()
357 // A request near client exit must not silently fall back to latest.png.
358 if err == nil && res.Artifacts["capture"] == "" {
359 if !requestSeen {
360 requestedAt, err = captureRequestTime(requestPath)
361 if errors.Is(err, os.ErrNotExist) {
362 err = nil
363 } else if err == nil {
364 requestSeen = true
365 }
366 }
367 if requestSeen && err == nil {
368 for runCtx.Err() == nil {
369 err = copyNextFrame(res.Artifacts["frames"], requestedAt, capturePath)
370 if err == nil {
371 res.Artifacts["capture"] = capturePath
372 break
373 }
374 if !errors.Is(err, os.ErrNotExist) {
375 break
376 }
377 err = nil
378 select {
379 case <-runCtx.Done():
380 case <-captureTick.C:
381 }
382 }
383 if res.Artifacts["capture"] == "" && err == nil {
384 err = fmt.Errorf("capture-request: %w", runCtx.Err())
385 }
386 }
387 }
388 if err != nil {
389 return fail(fmt.Errorf("client/compositor: %w", err), ProcessFailure)
390 }
391 select {
392 case e := <-exited:
393 return fail(fmt.Errorf("compositor exited during client: %v", e), ProcessFailure)
394 default:
395 }
396 }
397 // Wait briefly for the asynchronous screenshot writer after the client exits.
398 frameCtx, frameCancel := context.WithTimeout(runCtx, 3*time.Second)
399 defer frameCancel()
400 // A compositor crash after readiness is not a successful capture, even if latest.png exists.
401 select {
402 case e := <-exited:
403 return fail(fmt.Errorf("compositor exited before capture: %v", e), ProcessFailure)
404 default:
405 }
406 tick := time.NewTicker(25 * time.Millisecond)
407 defer tick.Stop()
408 for {
409 _, e := LoadPNG(res.Artifacts["frame"])
410 if e == nil {
411 break
412 }
413 select {
414 case <-frameCtx.Done():
415 return fail(fmt.Errorf("no frame: %w", frameCtx.Err()), ProcessFailure)
416 case e := <-exited:
417 return fail(fmt.Errorf("compositor exited before frame: %v", e), ProcessFailure)
418 case <-tick.C:
419 }
420 }
421 checkFrame := res.Artifacts["frame"]
422 if res.Artifacts["capture"] != "" {
423 checkFrame = res.Artifacts["capture"]
424 }
425 checks, err := Evaluate(checkFrame, o.Expect, o.GoldenBase, out)
426 if err != nil {
427 return fail(err, ProcessFailure)
428 }
429 res.Checks = checks
430 for _, c := range checks {
431 if !c.Pass {
432 res.Status = "failed"
433 return res, Failed
434 }
435 }
436 res.Status = "pass"
437 return res, Pass
438}
439
440// frameIndex accepts only compositor screenshots, not latest.png or temporary files.
441func frameIndex(name string) (int, bool) {
442 if !strings.HasPrefix(name, "frame-") || !strings.HasSuffix(name, ".png") {
443 return 0, false
444 }
445 digits := strings.TrimSuffix(strings.TrimPrefix(name, "frame-"), ".png")
446 if len(digits) != 6 {
447 return 0, false
448 }
449 for _, c := range digits {
450 if c < '0' || c > '9' {
451 return 0, false
452 }
453 }
454 n, err := strconv.Atoi(digits)
455 return n, err == nil
456}
457func captureRequestTime(path string) (time.Time, error) {
458 data, err := os.ReadFile(path)
459 if err != nil {
460 return time.Time{}, err
461 }
462 ns, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
463 if err != nil || ns <= 0 {
464 return time.Time{}, fmt.Errorf("invalid capture-request timestamp %q", strings.TrimSpace(string(data)))
465 }
466 return time.Unix(0, ns), nil
467}
468
469// nextFrame picks the second screenshot written after the commit token.
470// NeferWL renders and writes screenshots one after another, so the first file
471// finished after the token may have been rendered before it; the second began
472// after the first finished and therefore shows the committed frame.
473// An incomplete candidate is retried instead of selecting a later frame.
474func nextFrame(dir string, after time.Time) (string, error) {
475 entries, err := os.ReadDir(dir)
476 if err != nil {
477 return "", err
478 }
479 first, second := int(^uint(0)>>1), int(^uint(0)>>1)
480 names := map[int]string{}
481 for _, entry := range entries {
482 if entry.IsDir() {
483 continue
484 }
485 n, ok := frameIndex(entry.Name())
486 if !ok {
487 continue
488 }
489 info, e := entry.Info()
490 if e != nil {
491 return "", e
492 }
493 if !info.ModTime().After(after) {
494 continue
495 }
496 names[n] = entry.Name()
497 if n < first {
498 first, second = n, first
499 } else if n < second {
500 second = n
501 }
502 }
503 return names[second], nil
504}
505func copyNextFrame(dir string, after time.Time, destination string) error {
506 name, err := nextFrame(dir, after)
507 if err != nil {
508 return err
509 }
510 if name == "" {
511 return os.ErrNotExist
512 }
513 // NeferWL writes frame-N directly. Retry an incomplete PNG on the next tick.
514 path := filepath.Join(dir, name)
515 if _, err := LoadPNG(path); err != nil {
516 return os.ErrNotExist
517 }
518 data, err := os.ReadFile(path)
519 if err != nil {
520 return err
521 }
522 return os.WriteFile(destination, data, 0600)
523}
524func Evaluate(frame string, expect Expectations, base, out string) ([]Check, error) {
525 img, err := LoadPNG(frame)
526 if err != nil {
527 return nil, err
528 }
529 checks := make([]Check, 0, len(expect.Probes)+len(expect.Regions)+len(expect.Goldens))
530 add := func(name string, err error) {
531 c := Check{Name: name, Pass: err == nil}
532 if err != nil {
533 c.Detail = err.Error()
534 }
535 checks = append(checks, c)
536 }
537 for _, p := range expect.Probes {
538 want, _ := ParseColor(p.Color)
539 add(p.Name, PixelProbe(img, p.X, p.Y, want, p.Tolerance))
540 }
541 for _, r := range expect.Regions {
542 err := BoundsCheck(img, r.Rect)
543 if err == nil && r.Color != "" {
544 want, _ := ParseColor(r.Color)
545 err = UniformRegion(img, r.Rect, want, r.Tolerance)
546 }
547 add(r.Name, err)
548 }
549 for i, g := range expect.Goldens {
550 path := g.Path
551 if !filepath.IsAbs(path) {
552 path = filepath.Join(base, path)
553 }
554 ref, e := LoadPNG(path)
555 if e != nil {
556 add(g.Name, e)
557 continue
558 }
559 pass, ratio, diff, e := CompareGolden(img, ref, g.Tolerance, g.MaxDiffRatio)
560 if diff != nil {
561 e = errors.Join(e, SavePNG(filepath.Join(out, fmt.Sprintf("diff-%d.png", i)), diff))
562 }
563 if e == nil && !pass {
564 e = fmt.Errorf("differing pixel ratio %.6f exceeds %.6f", ratio, g.MaxDiffRatio)
565 }
566 add(g.Name, e)
567 }
568 return checks, nil
569}