detached 7e47dd8

refactor(obs): decompose performance harness

commit
7e47dd8c142e216fedd7333db1973366104adfd0
author
brice · · 2026-07-13 03:20 UTC
parent
6d5b59f
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
7 files · +2606 -2560
Unified
UnifiedSplit
A cmd/vev-perf-harness/harness_1.go +755 -0
@@ -0,0 +1,755 @@
1+package main
2+
3+import (
4+ "bytes"
5+ "context"
6+ "errors"
7+ "fmt"
8+ "io"
9+ "os"
10+ "os/exec"
11+ "path/filepath"
12+ "strconv"
13+ "strings"
14+ "sync"
15+ "syscall"
16+ "time"
17+
18+ "github.com/bnema/vev/pkg/safedir"
19+)
20+
21+const (
22+ minimumDuration = 30 * time.Second
23+ minimumRepetitions = 10
24+ minimumInIntervalEventSamples = 10
25+ measuredEventCadence = time.Second
26+)
27+
28+type options struct {
29+ vevBin, manifest, out string
30+ scenario string
31+ warmup, duration time.Duration
32+ repetitions int
33+}
34+
35+type manifest struct {
36+ Schema uint16 `json:"schema"`
37+ Topologies []topology `json:"topologies"`
38+ Workloads []string `json:"workloads"`
39+ Transports []transport `json:"transports"`
40+ Scenarios []scenario `json:"scenarios"`
41+}
42+type topology struct {
43+ ID string `json:"id"`
44+ Geometry string `json:"geometry"`
45+ RowsPerPane int `json:"rows_per_pane"`
46+}
47+type transport struct {
48+ ID string `json:"id"`
49+ Kind string `json:"kind"`
50+ RTTMS int `json:"rtt_ms,omitempty"`
51+ LossPercent int `json:"loss_percent,omitempty"`
52+}
53+type scenario struct {
54+ ID string `json:"id"`
55+ Topology string `json:"topology"`
56+ Workload string `json:"workload"`
57+ Transport string `json:"transport"`
58+ Roles []string `json:"roles"`
59+ InapplicableReason string `json:"inapplicable_reason,omitempty"`
60+}
61+
62+type processMapping struct {
63+ ProcessID string `json:"process_id"`
64+ ClockDomain string `json:"clock_domain"`
65+ TracePath string `json:"trace_path"`
66+ Role string `json:"role"`
67+ Scenario string `json:"scenario"`
68+ Run int `json:"run"`
69+ Identity string `json:"identity"`
70+}
71+
72+type runManifest struct {
73+ Scenario string `json:"scenario"`
74+ Run int `json:"run"`
75+ Processes []processMapping `json:"processes"`
76+}
77+type harnessMark struct {
78+ Scenario string `json:"scenario"`
79+ Run int `json:"run"`
80+ Sequence uint64 `json:"sequence"`
81+ Kind string `json:"kind"`
82+ Tick int64 `json:"tick"`
83+ Valid bool `json:"valid"`
84+}
85+
86+// measuredInterval belongs solely to the harness clock domain. A paired event
87+// is eligible only when both owned boundaries fall inside this interval.
88+type measuredInterval struct {
89+ Start int64
90+ End int64
91+}
92+
93+type eventSample struct {
94+ Sequence uint64 `json:"sequence"`
95+ Injected int64 `json:"injected_tick"`
96+ Flushed int64 `json:"flushed_tick"`
97+ Latency int64 `json:"latency_nanos"`
98+}
99+type span struct {
100+ Component, Name string
101+ Samples []int64
102+}
103+type runResult struct {
104+ Spans []span `json:"process_local_spans"`
105+ Scenario string `json:"scenario"`
106+ Run int `json:"run"`
107+ Samples int `json:"samples"`
108+ EndToEnd []int64 `json:"end_to_end_nanos"`
109+ Event distribution `json:"event_end_to_end"`
110+ Cadence distribution `json:"event_cadence_nanos"`
111+ CadenceSamples []int64 `json:"-"`
112+ MaxGap int64 `json:"event_max_gap_nanos"`
113+ Processes []processMapping `json:"processes"`
114+}
115+type summary struct {
116+ Schema uint16 `json:"schema"`
117+ GitSHA string `json:"git_sha"`
118+ Warmup string `json:"warmup"`
119+ Duration string `json:"duration"`
120+ Repetitions int `json:"repetitions"`
121+ EndToEnd distribution `json:"harness_end_to_end"`
122+ Cadence distribution `json:"harness_event_cadence_nanos"`
123+ MaxGap int64 `json:"harness_event_max_gap_nanos"`
124+ RunP50Dispersion distribution `json:"run_p50_dispersion"`
125+ Spans []spanSummary `json:"process_local_spans"`
126+ Runs int `json:"runs"`
127+}
128+type distribution struct {
129+ Count int `json:"count"`
130+ P50 int64 `json:"p50"`
131+ P95 int64 `json:"p95"`
132+ P99 int64 `json:"p99"`
133+ Max int64 `json:"max"`
134+}
135+type spanSummary struct {
136+ Component string `json:"component"`
137+ Name string `json:"name"`
138+ Distribution distribution `json:"distribution"`
139+}
140+
141+type clock interface {
142+ Now() int64
143+ Sleep(time.Duration)
144+}
145+type systemClock struct{ start time.Time }
146+
147+func (c systemClock) Now() int64 { return time.Since(c.start).Nanoseconds() }
148+func (systemClock) Sleep(d time.Duration) { time.Sleep(d) }
149+
150+type launchedProcess interface {
151+ Warmup([]byte) error
152+ Measure([]byte, func() error, func() error) error
153+ Close() error
154+}
155+
156+// roleCommand contains only argv accepted by app.parseArgs. Transport routing
157+// metadata configures the harness's public-command seam; it is never passed as
158+// an unrecognised vev flag.
159+type roleCommand struct {
160+ Args []string
161+ Transport transport
162+}
163+
164+type launcher interface {
165+ Launch(processMapping, roleCommand) (launchedProcess, error)
166+}
167+
168+type launchedRole struct {
169+ mapping processMapping
170+ process launchedProcess
171+}
172+
173+// closeLaunchedRoles closes every role in reverse launch order. errors.Join
174+// retains every cleanup failure in that stable order.
175+func closeLaunchedRoles(processes []launchedRole) error {
176+ var failures []error
177+ for i := len(processes) - 1; i >= 0; i-- {
178+ if err := processes[i].process.Close(); err != nil {
179+ failures = append(failures, fmt.Errorf("close %s role: %w", processes[i].mapping.Role, err))
180+ }
181+ }
182+ return errors.Join(failures...)
183+}
184+
185+// cliLauncher is deliberately limited to executing the supplied public vev
186+// binary. Scenario orchestration never reaches into an in-process daemon.
187+type cliLauncher struct {
188+ bin string
189+ netemFactory udpNetemFactory
190+ mu sync.Mutex
191+ peers map[string]peerRoute
192+ runtimes map[string]string
193+}
194+
195+type peerRoute struct {
196+ mapping processMapping
197+ command roleCommand
198+ pidPath string
199+ netem udpNetem
200+}
201+
202+type preparedPeer struct {
203+ route peerRoute
204+ cleanupRuntime func() error
205+}
206+
207+func (p preparedPeer) Warmup([]byte) error { return nil }
208+func (p preparedPeer) Measure([]byte, func() error, func() error) error { return nil }
209+func (p preparedPeer) Close() error {
210+ var errs []error
211+ if p.route.pidPath != "" {
212+ b, err := os.ReadFile(p.route.pidPath)
213+ if err == nil {
214+ pid, parseErr := strconv.Atoi(strings.TrimSpace(string(b)))
215+ if parseErr != nil || pid <= 0 {
216+ errs = append(errs, fmt.Errorf("invalid owned peer pid: %q", b))
217+ } else if killErr := syscall.Kill(pid, syscall.SIGTERM); killErr != nil && !errors.Is(killErr, syscall.ESRCH) {
218+ errs = append(errs, killErr)
219+ }
220+ } else if !errors.Is(err, os.ErrNotExist) {
221+ errs = append(errs, err)
222+ }
223+ }
224+ if p.route.netem != nil {
225+ if err := p.route.netem.Close(); err != nil {
226+ errs = append(errs, err)
227+ }
228+ }
229+ if p.cleanupRuntime != nil {
230+ if err := p.cleanupRuntime(); err != nil {
231+ errs = append(errs, err)
232+ }
233+ }
234+ return errors.Join(errs...)
235+}
236+
237+type terminalOutput interface {
238+ io.Writer
239+ Sync() error
240+ Close() error
241+}
242+
243+type cliProcess struct {
244+ cmd *exec.Cmd
245+ pty io.ReadWriteCloser
246+ output terminalOutput
247+ chunks chan []byte
248+ done chan struct{}
249+ closed sync.Once
250+ waitErr chan error
251+ waitTimeout func() <-chan time.Time
252+ forceCleanup func()
253+ forceKill func()
254+ shutdown func() error
255+ cleanupRuntime func() error
256+ terminalReady bool
257+ readyPath string
258+}
259+
260+func (l *cliLauncher) Launch(m processMapping, role roleCommand) (launchedProcess, error) {
261+ if len(role.Args) == 0 {
262+ return nil, errors.New("public CLI role has no arguments")
263+ }
264+ if info, err := os.Stat(l.bin); err != nil {
265+ return nil, err
266+ } else if info.IsDir() {
267+ return nil, fmt.Errorf("public CLI binary %q is a directory", l.bin)
268+ }
269+ if m.Role == "ssh_stdio_peer" || m.Role == "udp_peer" {
270+ return l.preparePeer(m, role)
271+ }
272+ cmd := exec.Command(l.bin, role.Args...)
273+ runDir := filepath.Dir(m.TracePath)
274+ runtimeDir, err := l.runtimeDir(runDir)
275+ if err != nil {
276+ return nil, err
277+ }
278+ // All role inputs are absolute paths resolved at harness startup. Do not set
279+ // a role working directory: daemon-owned subprocess cleanup may otherwise
280+ // treat the evidence directory as its working tree and remove a preallocated
281+ // trace while a later repetition is being merged.
282+ cmd.Env = append(withoutEnv(os.Environ(), "VEV", "VEV_PERF_TRACE", "VEV_PERF_PROCESS_ID", "VEV_PERF_SCENARIO", "VEV_PERF_RUN", "VEV_REMOTE_TRANSPORT", "XDG_RUNTIME_DIR", "XDG_STATE_HOME"), traceEnvironment(m)...)
283+ cmd.Env = append(cmd.Env, "XDG_RUNTIME_DIR="+runtimeDir,
284+ "XDG_STATE_HOME="+filepath.Join(runDir, "state"), "TERM=xterm-256color")
285+ if m.Role == "client" {
286+ l.mu.Lock()
287+ peer, routed := l.peers[runDir]
288+ l.mu.Unlock()
289+ if routed {
290+ mode, err := remoteMode(peer.command.Transport)
291+ if err != nil {
292+ return nil, err
293+ }
294+ cmd.Env = append(cmd.Env, "PATH="+runDir+":"+os.Getenv("PATH"), "VEV_REMOTE_TRANSPORT="+mode)
295+ }
296+ }
297+ if err := safedir.EnsurePrivate(filepath.Join(runDir, "state")); err != nil {
298+ return nil, err
299+ }
300+ p := &cliProcess{cmd: cmd, waitErr: make(chan error, 1), readyPath: filepath.Join(runtimeDir, "vev", "daemon.sock")}
301+ if m.Role == "daemon" {
302+ p.cleanupRuntime = func() error { return l.releaseRuntime(runDir) }
303+ }
304+ if m.Role == "client" {
305+ master, slave, err := openPTY()
306+ if err != nil {
307+ return nil, err
308+ }
309+ output, err := os.OpenFile(filepath.Join(filepath.Dir(m.TracePath), "terminal-output"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o600)
310+ if err != nil {
311+ _ = master.Close()
312+ _ = slave.Close()
313+ return nil, err
314+ }
315+ cmd.Stdin, cmd.Stdout, cmd.Stderr = slave, slave, slave
316+ // Ctty is the child descriptor number after Start remaps slave to stdin.
317+ cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true, Setctty: true, Ctty: 0}
318+ if err := cmd.Start(); err != nil {
319+ _ = output.Close()
320+ _ = master.Close()
321+ _ = slave.Close()
322+ if p.cleanupRuntime != nil {
323+ _ = p.cleanupRuntime()
324+ }
325+ return nil, err
326+ }
327+ _ = slave.Close()
328+ p.pty, p.output, p.chunks, p.done = master, output, make(chan []byte, 32), make(chan struct{})
329+ go p.copyTerminal()
330+ } else {
331+ // Every launched role owns a dedicated process group, so the bounded
332+ // cleanup path never signals the harness/test runner's group.
333+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
334+ devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, 0)
335+ if err != nil {
336+ return nil, err
337+ }
338+ cmd.Stdin, cmd.Stdout, cmd.Stderr = devNull, devNull, devNull
339+ if err := cmd.Start(); err != nil {
340+ _ = devNull.Close()
341+ if p.cleanupRuntime != nil {
342+ _ = p.cleanupRuntime()
343+ }
344+ return nil, err
345+ }
346+ _ = devNull.Close()
347+ }
348+ if m.Role == "daemon" {
349+ // Stop the foreground daemon through its public CLI before its bounded
350+ // process-group fallback. This lets blocked carriage operations close and
351+ // serialize their failed end marks instead of being cut off by SIGKILL.
352+ p.shutdown = func() error {
353+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
354+ defer cancel()
355+ shutdown := exec.CommandContext(ctx, l.bin, "kill", "--daemon")
356+ shutdown.Env = withoutEnv(cmd.Env, "VEV_PERF_TRACE", "VEV_PERF_PROCESS_ID", "VEV_PERF_SCENARIO", "VEV_PERF_RUN")
357+ return shutdown.Run()
358+ }
359+ }
360+ go func() { p.waitErr <- cmd.Wait() }()
361+ return p, nil
362+}
363+
364+// runtimeDir returns one short, private runtime directory shared by the roles
365+// of one run. Keeping the Unix socket outside deeply nested evidence output
366+// avoids the platform socket-path limit without weakening run isolation.
367+func (l *cliLauncher) runtimeDir(runDir string) (string, error) {
368+ l.mu.Lock()
369+ defer l.mu.Unlock()
370+ if path := l.runtimes[runDir]; path != "" {
371+ return path, nil
372+ }
373+ path, err := os.MkdirTemp("", "vev-harness-runtime-")
374+ if err != nil {
375+ return "", err
376+ }
377+ if err := safedir.EnsurePrivate(path); err != nil {
378+ _ = os.RemoveAll(path)
379+ return "", err
380+ }
381+ if l.runtimes == nil {
382+ l.runtimes = make(map[string]string)
383+ }
384+ l.runtimes[runDir] = path
385+ return path, nil
386+}
387+
388+func (l *cliLauncher) releaseRuntime(runDir string) error {
389+ l.mu.Lock()
390+ path := l.runtimes[runDir]
391+ delete(l.runtimes, runDir)
392+ l.mu.Unlock()
393+ if path == "" {
394+ return nil
395+ }
396+ return os.RemoveAll(path)
397+}
398+
399+// preparePeer installs a per-run ssh command seam. The ordinary public client
400+// still executes its documented remote attach command; its ssh child invokes
401+// only the parsed public _stdio or _udp-proxy command. This makes the peer
402+// carrying client traffic the declared, exclusively traced role rather than a
403+// separately launched but unused process.
404+func (l *cliLauncher) preparePeer(m processMapping, role roleCommand) (launchedProcess, error) {
405+ runDir := filepath.Dir(m.TracePath)
406+ l.mu.Lock()
407+ runtimeDir := l.runtimes[runDir]
408+ l.mu.Unlock()
409+ peerOwnsRuntime := runtimeDir == ""
410+ if peerOwnsRuntime {
411+ var err error
412+ runtimeDir, err = l.runtimeDir(runDir)
413+ if err != nil {
414+ return nil, err
415+ }
416+ }
417+ if err := safedir.EnsurePrivate(runDir); err != nil {
418+ return nil, err
419+ }
420+ if err := safedir.EnsurePrivate(runtimeDir); err != nil {
421+ return nil, err
422+ }
423+ if err := safedir.EnsurePrivate(filepath.Join(runDir, "state")); err != nil {
424+ return nil, err
425+ }
426+ if len(role.Args) != 2 || (role.Args[0] != "_stdio" && role.Args[0] != "_udp-proxy") {
427+ return nil, fmt.Errorf("unsupported public peer command %q", role.Args)
428+ }
429+ route := peerRoute{mapping: m, command: role}
430+ if m.Role == "udp_peer" {
431+ route.pidPath = filepath.Join(runDir, "udp-peer.pid")
432+ factory := l.netemFactory
433+ if factory == nil {
434+ factory = newUDPNetem
435+ }
436+ netem, err := factory(udpNetemConfig{
437+ RTT: time.Duration(role.Transport.RTTMS) * time.Millisecond,
438+ LossPercent: role.Transport.LossPercent,
439+ TargetPath: filepath.Join(runDir, "udp-peer.target"),
440+ })
441+ if err != nil {
442+ return nil, fmt.Errorf("start harness UDP netem: %w", err)
443+ }
444+ route.netem = netem
445+ }
446+ shim := filepath.Join(runDir, "ssh")
447+ ready := filepath.Join(runDir, "udp-peer.ready")
448+ target := filepath.Join(runDir, "udp-peer.target")
449+ netemPort := 0
450+ if route.netem != nil {
451+ netemPort = route.netem.Port()
452+ }
453+ stderr := filepath.Join(runDir, "udp-peer.stderr")
454+ // The bootstrap protocol requires the SSH command to exit after forwarding
455+ // the readiness line. The long-lived proxy is owned by pidPath and receives
456+ // the role's unique trace identity, never the bootstrap/client identity.
457+ body := fmt.Sprintf(`#!/bin/sh
458+set -eu
459+case "$*" in
460+ *"_stdio"*)
461+ exec env VEV_PERF_TRACE=%[1]q VEV_PERF_PROCESS_ID=%[2]q VEV_PERF_SCENARIO=%[12]q VEV_PERF_RUN=%[13]d XDG_RUNTIME_DIR=%[3]q XDG_STATE_HOME=%[4]q TERM=xterm-256color %[5]q _stdio %[6]q
462+ ;;
463+ *"_udp-bootstrap"*)
464+ rm -f %[7]q %[8]q %[9]q
465+ env VEV_PERF_TRACE=%[1]q VEV_PERF_PROCESS_ID=%[2]q VEV_PERF_SCENARIO=%[12]q VEV_PERF_RUN=%[13]d XDG_RUNTIME_DIR=%[3]q XDG_STATE_HOME=%[4]q TERM=xterm-256color %[5]q _udp-proxy %[6]q >%[7]q 2>%[8]q &
466+ peer=$!
467+ printf '%%s\n' "$peer" > %[11]q
468+ i=0
469+ while [ ! -s %[7]q ] && [ "$i" -lt 1000 ]; do sleep 0.01; i=$((i+1)); done
470+ test -s %[7]q
471+ line=$(head -n 1 %[7]q)
472+ printf '%%s\n' "$line" > %[9]q
473+ set -- $line
474+ test "$1" = VEV-UDP
475+ test -n "$3"
476+ printf 'VEV-UDP %%s %%s\n' %[10]d "$3"
477+ exit 0
478+ ;;
479+ *) echo 'vev harness ssh seam rejected non-vev command' >&2; exit 64 ;;
480+esac
481+`, m.TracePath, m.ProcessID, runtimeDir, filepath.Join(runDir, "state"), l.bin, role.Args[1], ready, stderr, target, netemPort, route.pidPath, m.Scenario, m.Run)
482+ if err := os.WriteFile(shim, []byte(body), 0o700); err != nil {
483+ if route.netem != nil {
484+ _ = route.netem.Close()
485+ }
486+ return nil, err
487+ }
488+ l.mu.Lock()
489+ if l.peers == nil {
490+ l.peers = make(map[string]peerRoute)
491+ }
492+ if _, exists := l.peers[runDir]; exists {
493+ l.mu.Unlock()
494+ if route.netem != nil {
495+ _ = route.netem.Close()
496+ }
497+ return nil, fmt.Errorf("duplicate transport peer for %s", runDir)
498+ }
499+ l.peers[runDir] = route
500+ l.mu.Unlock()
501+ peer := preparedPeer{route: route}
502+ if peerOwnsRuntime {
503+ peer.cleanupRuntime = func() error { return l.releaseRuntime(runDir) }
504+ }
505+ return peer, nil
506+}
507+
508+func remoteMode(t transport) (string, error) {
509+ switch t.Kind {
510+ case "ssh_stdio":
511+ return "stdio", nil
512+ case "udp":
513+ return "udp", nil
514+ default:
515+ return "", fmt.Errorf("transport %q cannot route through a remote peer", t.ID)
516+ }
517+}
518+
519+// traceEnvironment is the complete process-local identity supplied by the
520+// persisted harness processMapping. App turns Scenario/Run into production
521+// mark correlation; sequence/request/epoch remain process-local operation IDs.
522+func traceEnvironment(m processMapping) []string {
523+ return []string{
524+ "VEV_PERF_TRACE=" + m.TracePath,
525+ "VEV_PERF_PROCESS_ID=" + m.ProcessID,
526+ "VEV_PERF_SCENARIO=" + m.Scenario,
527+ "VEV_PERF_RUN=" + strconv.Itoa(m.Run),
528+ }
529+}
530+
531+func withoutEnv(env []string, names ...string) []string {
532+ remove := make(map[string]bool, len(names))
533+ for _, name := range names {
534+ remove[name] = true
535+ }
536+ out := make([]string, 0, len(env))
537+ for _, entry := range env {
538+ name, _, _ := strings.Cut(entry, "=")
539+ if !remove[name] {
540+ out = append(out, entry)
541+ }
542+ }
543+ return out
544+}
545+
546+func (p *cliProcess) copyTerminal() {
547+ defer close(p.chunks)
548+ buf := make([]byte, 4096)
549+ for {
550+ n, err := p.pty.Read(buf)
551+ if n > 0 {
552+ chunk := append([]byte(nil), buf[:n]...)
553+ if _, writeErr := p.output.Write(chunk); writeErr != nil {
554+ return
555+ }
556+ select {
557+ case p.chunks <- chunk:
558+ case <-p.done:
559+ return
560+ }
561+ }
562+ if err != nil {
563+ return
564+ }
565+ }
566+}
567+
568+// WaitReady confirms the explicitly launched public daemon owns its unique
569+// socket before a client is launched. Without this barrier, the normal client
570+// auto-spawn behavior can create an unmanifested daemon process during startup.
571+func (p *cliProcess) WaitReady() error {
572+ if p.readyPath == "" {
573+ return errors.New("daemon readiness path is empty")
574+ }
575+ ticker := time.NewTicker(10 * time.Millisecond)
576+ defer ticker.Stop()
577+ timer := time.NewTimer(10 * time.Second)
578+ defer timer.Stop()
579+ for {
580+ if info, err := os.Stat(p.readyPath); err == nil {
581+ if info.Mode()&os.ModeSocket == 0 {
582+ return fmt.Errorf("daemon readiness path %s is not a socket", p.readyPath)
583+ }
584+ return nil
585+ }
586+ select {
587+ case err := <-p.waitErr:
588+ p.waitErr <- err
589+ if err == nil {
590+ return errors.New("daemon exited before readiness")
591+ }
592+ return fmt.Errorf("daemon exited before readiness: %w", err)
593+ case <-ticker.C:
594+ case <-timer.C:
595+ return fmt.Errorf("timed out waiting for daemon socket %s", p.readyPath)
596+ }
597+ }
598+}
599+
600+func (p *cliProcess) Warmup(input []byte) error {
601+ if p.pty == nil || len(input) == 0 {
602+ return nil
603+ }
604+ // A client cannot write application output until it has received Welcome and
605+ // entered raw mode. Require that observable output before submitting any
606+ // warmup input, so the PTY's line discipline cannot echo it locally.
607+ if !p.terminalReady {
608+ if err := p.waitForTerminalReady(); err != nil {
609+ return err
610+ }
611+ p.terminalReady = true
612+ }
613+ if _, err := p.pty.Write(input); err != nil {
614+ return err
615+ }
616+ // Do not begin measured input until the public client has rendered a real
617+ // shell response to warmup input. This is intentionally not a measurement.
618+ return p.waitForTerminalMarker(inputMarker(input), input)
619+}
620+
621+// Measure records a boundary only after the harness-owned terminal output file
622+// has accepted application output and Sync has reported a successful flush.
623+// The marker is injected through the real client PTY, never fabricated by the
624+// harness. PTY local echo is discarded and cannot satisfy this boundary.
625+func (p *cliProcess) Measure(input []byte, injected, flushed func() error) error {
626+ if p.pty == nil {
627+ return errors.New("measured process has no terminal PTY")
628+ }
629+ if _, err := p.pty.Write(input); err != nil {
630+ return err
631+ }
632+ // The injected callback runs only after the owned PTY accepts the bytes.
633+ if err := injected(); err != nil {
634+ return err
635+ }
636+ if err := p.waitForTerminalMarker(inputMarker(input), input); err != nil {
637+ return err
638+ }
639+ if err := p.output.Sync(); err != nil {
640+ return fmt.Errorf("flush terminal output: %w", err)
641+ }
642+ return flushed()
643+}
644+
645+// waitForTerminalReady waits for public client output produced before any
646+// harness input. The client enters raw mode before its output loop can render
647+// this initial prompt/state, so this is an observable readiness boundary.
648+func (p *cliProcess) waitForTerminalReady() error {
649+ timer := time.NewTimer(10 * time.Second)
650+ defer timer.Stop()
651+ for {
652+ select {
653+ case chunk, ok := <-p.chunks:
654+ if !ok {
655+ return p.terminalClosedError("readiness output")
656+ }
657+ if len(chunk) == 0 {
658+ continue
659+ }
660+ if err := p.output.Sync(); err != nil {
661+ return fmt.Errorf("flush terminal readiness output: %w", err)
662+ }
663+ return nil
664+ case err := <-p.waitErr:
665+ // Keep the exit result available to Close, which owns reaping.
666+ p.waitErr <- err
667+ if err == nil {
668+ return errors.New("client exited before readiness output")
669+ }
670+ return fmt.Errorf("client exited before readiness output: %w", err)
671+ case <-timer.C:
672+ return errors.New("timed out waiting for client readiness output")
673+ }
674+ }
675+}
676+
677+func (p *cliProcess) terminalClosedError(boundary string) error {
678+ if p.waitErr != nil {
679+ select {
680+ case err := <-p.waitErr:
681+ // Close owns reaping and may need the result after a failed boundary.
682+ p.waitErr <- err
683+ if err != nil {
684+ return fmt.Errorf("client terminal closed before %s: %w", boundary, err)
685+ }
686+ case <-time.After(10 * time.Millisecond):
687+ }
688+ }
689+ return fmt.Errorf("client terminal closed before %s", boundary)
690+}
691+
692+func (p *cliProcess) waitForTerminalMarker(marker, input []byte) error {
693+ var received []byte
694+ echo := newPTYLocalEcho(input)
695+ timer := time.NewTimer(10 * time.Second)
696+ defer timer.Stop()
697+ for !bytes.Contains(received, marker) {
698+ select {
699+ case chunk, ok := <-p.chunks:
700+ if !ok {
701+ return p.terminalClosedError("application output")
702+ }
703+ received = append(received, echo.applicationOutput(chunk)...)
704+ case err := <-p.waitErr:
705+ // Keep the exit result available to Close, which owns reaping.
706+ p.waitErr <- err
707+ if err == nil {
708+ return errors.New("client exited before application output")
709+ }
710+ return fmt.Errorf("client exited before application output: %w", err)
711+ case <-timer.C:
712+ return errors.New("timed out waiting for application terminal output")
713+ }
714+ }
715+ return nil
716+}
717+
718+// ptyLocalEcho removes the one ordered input echo that a not-yet-raw slave PTY
719+// can produce. It normalizes NL to the conventional ONLCR echo spelling; any
720+// divergence is retained as application output. The line discipline emits the
721+// echo before the child can read and respond to the same write.
722+type ptyLocalEcho struct {
723+ expected []byte
724+ matched int
725+ done bool
726+}
727+
728+func newPTYLocalEcho(input []byte) *ptyLocalEcho {
729+ expected := make([]byte, 0, len(input))
730+ for _, b := range input {
731+ if b == '\n' {
732+ expected = append(expected, '\r')
733+ }
734+ expected = append(expected, b)
735+ }
736+ return &ptyLocalEcho{expected: expected}
737+}
738+
739+func (e *ptyLocalEcho) applicationOutput(chunk []byte) []byte {
740+ if e.done || len(e.expected) == 0 {
741+ return chunk
742+ }
743+ for i, b := range chunk {
744+ if b != e.expected[e.matched] {
745+ e.done = true
746+ return append([]byte(nil), chunk[i:]...)
747+ }
748+ e.matched++
749+ if e.matched == len(e.expected) {
750+ e.done = true
751+ return append([]byte(nil), chunk[i+1:]...)
752+ }
753+ }
754+ return nil
755+}
A cmd/vev-perf-harness/harness_2.go +744 -0
@@ -0,0 +1,744 @@
1+package main
2+
3+import (
4+ "bytes"
5+ "encoding/json"
6+ "errors"
7+ "flag"
8+ "fmt"
9+ "io"
10+ "os"
11+ "os/exec"
12+ "path/filepath"
13+ "runtime/debug"
14+ "sort"
15+ "strings"
16+ "syscall"
17+ "time"
18+
19+ "github.com/bnema/vev/pkg/safedir"
20+ "golang.org/x/sys/unix"
21+)
22+
23+func (p *cliProcess) Close() error {
24+ var result error
25+ p.closed.Do(func() {
26+ if p.done != nil {
27+ close(p.done)
28+ }
29+ // A public terminal client owns a shell session. Ask that shell to exit
30+ // before closing the PTY so its transport can finish its receive span;
31+ // this avoids turning ordinary harness teardown into an aborted trace.
32+ exited := false
33+ if p.shutdown != nil && p.waitErr != nil {
34+ select {
35+ case <-p.waitErr:
36+ exited = true
37+ default:
38+ // The client may have just ended the last session and raced this
39+ // best-effort public shutdown. The normal bounded reaping path below
40+ // remains authoritative in that case.
41+ _ = p.shutdown()
42+ }
43+ }
44+ if p.pty != nil && p.waitErr != nil {
45+ _, _ = p.pty.Write([]byte("exit\n"))
46+ select {
47+ case <-p.waitErr:
48+ exited = true
49+ case <-p.closeDeadline():
50+ }
51+ }
52+ if p.pty != nil {
53+ _ = p.pty.Close()
54+ }
55+ if p.waitErr != nil && !exited {
56+ select {
57+ case <-p.waitErr:
58+ case <-p.closeDeadline():
59+ // Teardown is deliberately bounded: a stuck public CLI or child
60+ // must not hold a local harness run indefinitely.
61+ p.forceProcessGroupCleanup()
62+ select {
63+ case <-p.waitErr:
64+ case <-p.closeDeadline():
65+ p.forceProcessGroupKill()
66+ select {
67+ case <-p.waitErr:
68+ case <-p.closeDeadline():
69+ result = errors.New("timed out reaping public CLI process")
70+ }
71+ }
72+ }
73+ }
74+ if p.output != nil {
75+ if err := p.output.Close(); result == nil {
76+ result = err
77+ }
78+ }
79+ if p.cleanupRuntime != nil {
80+ if err := p.cleanupRuntime(); result == nil {
81+ result = err
82+ }
83+ }
84+ })
85+ return result
86+}
87+
88+func (p *cliProcess) closeDeadline() <-chan time.Time {
89+ if p.waitTimeout != nil {
90+ return p.waitTimeout()
91+ }
92+ if p.shutdown != nil {
93+ // A public kill asks the daemon to finish bounded detached notices and
94+ // close blocked carriage before it exits; do not turn that graceful
95+ // protocol into a SIGKILL at the two-second client deadline.
96+ return time.After(5 * time.Second)
97+ }
98+ return time.After(2 * time.Second)
99+}
100+
101+func (p *cliProcess) forceProcessGroupCleanup() {
102+ if p.forceCleanup != nil {
103+ p.forceCleanup()
104+ return
105+ }
106+ if p.cmd != nil && p.cmd.Process != nil {
107+ // The client is a session/process-group leader; forced cleanup must
108+ // include its ssh seam/_stdio descendants.
109+ _ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGTERM)
110+ }
111+}
112+
113+func (p *cliProcess) forceProcessGroupKill() {
114+ if p.forceKill != nil {
115+ p.forceKill()
116+ return
117+ }
118+ if p.cmd != nil && p.cmd.Process != nil {
119+ _ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGKILL)
120+ }
121+}
122+
123+// openPTY is the small Linux PTY boundary needed to drive the public terminal
124+// client. No daemon or adapter implementation is imported by this command.
125+func openPTY() (*os.File, *os.File, error) {
126+ masterFD, err := unix.Open("/dev/ptmx", unix.O_RDWR|unix.O_NOCTTY, 0)
127+ if err != nil {
128+ return nil, nil, err
129+ }
130+ closeMaster := func(err error) (*os.File, *os.File, error) { _ = unix.Close(masterFD); return nil, nil, err }
131+ number, err := unix.IoctlGetInt(masterFD, unix.TIOCGPTN)
132+ if err != nil {
133+ return closeMaster(err)
134+ }
135+ if err := unix.IoctlSetPointerInt(masterFD, unix.TIOCSPTLCK, 0); err != nil {
136+ return closeMaster(err)
137+ }
138+ slaveFD, err := unix.Open(fmt.Sprintf("/dev/pts/%d", number), unix.O_RDWR|unix.O_NOCTTY, 0)
139+ if err != nil {
140+ return closeMaster(err)
141+ }
142+ return os.NewFile(uintptr(masterFD), "vev-client-master"), os.NewFile(uintptr(slaveFD), "vev-client-slave"), nil
143+}
144+
145+func inputMarker(input []byte) []byte {
146+ start := bytes.Index(input, []byte("__VEV_HARNESS_"))
147+ if start < 0 {
148+ return input
149+ }
150+ end := bytes.Index(input[start:], []byte(`\n`))
151+ if end < 0 {
152+ return input[start:]
153+ }
154+ return input[start : start+end]
155+}
156+
157+type harness struct {
158+ clock clock
159+ launcher launcher
160+ mkdir func(string) error
161+ createRunDir func(string) error
162+ create func(string) (*os.File, error)
163+ removeAll func(string) error
164+ gitSHA func() string
165+ selectScenarios func(manifest) []scenario // test-only bounded fixture seam
166+}
167+
168+func main() {
169+ if err := run(os.Args[1:], defaultHarness()); err != nil {
170+ fmt.Fprintln(os.Stderr, "vev-perf-harness:", err)
171+ os.Exit(2)
172+ }
173+}
174+func defaultHarness() *harness {
175+ return &harness{clock: systemClock{time.Now()}, mkdir: safedir.EnsurePrivate, createRunDir: createExclusiveRunDir, create: func(p string) (*os.File, error) { return os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) }, removeAll: os.RemoveAll, gitSHA: recordedGitSHA}
176+}
177+
178+// createExclusiveRunDir prevents stale runtime, socket, state, or trace files
179+// from an earlier invocation being mistaken for the next repetition's evidence.
180+func createExclusiveRunDir(path string) error {
181+ if err := os.Mkdir(path, 0o700); err != nil {
182+ return err
183+ }
184+ if err := safedir.EnsurePrivate(path); err != nil {
185+ _ = os.Remove(path)
186+ return err
187+ }
188+ return nil
189+}
190+
191+// recordedGitSHA prefers build provenance so an installed public harness does
192+// not depend on its launch directory being a checkout. The git fallback keeps
193+// go run evidence attributable as well.
194+func recordedGitSHA() string {
195+ if info, ok := debug.ReadBuildInfo(); ok {
196+ for _, setting := range info.Settings {
197+ if setting.Key == "vcs.revision" && setting.Value != "" {
198+ return setting.Value
199+ }
200+ }
201+ }
202+ out, err := exec.Command("git", "rev-parse", "HEAD").Output()
203+ if err == nil && strings.TrimSpace(string(out)) != "" {
204+ return strings.TrimSpace(string(out))
205+ }
206+ return "unknown"
207+}
208+func run(args []string, h *harness) error {
209+ opt, err := parseOptions(args)
210+ if err != nil {
211+ return err
212+ }
213+ opt, err = resolvePathOptions(opt)
214+ if err != nil {
215+ return err
216+ }
217+ m, err := readManifest(opt.manifest)
218+ if err != nil {
219+ return err
220+ }
221+ if err := validateManifest(m); err != nil {
222+ return err
223+ }
224+ if err = h.mkdir(opt.out); err != nil {
225+ return err
226+ }
227+ raw, err := os.OpenFile(filepath.Join(opt.out, "raw-harness.jsonl"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
228+ if err != nil {
229+ return err
230+ }
231+ defer raw.Close()
232+ results := make([]runResult, 0, len(m.Scenarios)*opt.repetitions)
233+ var localSpans []span
234+ scenarios := m.Scenarios
235+ if opt.scenario != "" {
236+ scenarios, err = selectScenario(m, opt.scenario)
237+ if err != nil {
238+ return err
239+ }
240+ }
241+ if h.selectScenarios != nil {
242+ scenarios = h.selectScenarios(m)
243+ }
244+ for _, s := range scenarios {
245+ if s.InapplicableReason != "" {
246+ continue
247+ }
248+ for r := 1; r <= opt.repetitions; r++ {
249+ rr, err := h.runOne(opt, m, s, r, raw)
250+ if err != nil {
251+ return fmt.Errorf("scenario %s run %d: %w", s.ID, r, err)
252+ }
253+ results = append(results, rr)
254+ localSpans = append(localSpans, rr.Spans...)
255+ }
256+ }
257+ all, cadence, maxGap, runP50s := aggregateEventResults(results)
258+ if len(all) == 0 {
259+ return errors.New("no measured harness input/flush pairs")
260+ }
261+ if err := writeJSON(filepath.Join(opt.out, "runs.json"), results); err != nil {
262+ return err
263+ }
264+ return writeJSON(filepath.Join(opt.out, "summary.json"), summary{Schema: 1, GitSHA: h.gitSHA(), Warmup: opt.warmup.String(), Duration: opt.duration.String(), Repetitions: opt.repetitions, EndToEnd: percentiles(all), Cadence: percentiles(cadence), MaxGap: maxGap, RunP50Dispersion: percentiles(runP50s), Spans: summarizeSpans(localSpans), Runs: len(results)})
265+}
266+
267+// aggregateEventResults deliberately preserves raw event latencies for
268+// end-to-end percentiles. Per-run p50s are a separate dispersion series.
269+func aggregateEventResults(results []runResult) (all, cadence []int64, maxGap int64, runP50s []int64) {
270+ for _, result := range results {
271+ all = append(all, result.EndToEnd...)
272+ cadence = append(cadence, result.CadenceSamples...)
273+ runP50s = append(runP50s, result.Event.P50)
274+ if result.MaxGap > maxGap {
275+ maxGap = result.MaxGap
276+ }
277+ }
278+ return all, cadence, maxGap, runP50s
279+}
280+
281+func parseOptions(args []string) (options, error) {
282+ var o options
283+ fs := flag.NewFlagSet("vev-perf-harness", flag.ContinueOnError)
284+ fs.SetOutput(io.Discard)
285+ fs.StringVar(&o.vevBin, "vev-bin", "", "public vev binary")
286+ fs.StringVar(&o.manifest, "manifest", "", "manifest")
287+ fs.StringVar(&o.out, "out", "", "output directory")
288+ fs.StringVar(&o.scenario, "scenario", "", "one canonical scenario ID (after full manifest validation)")
289+ fs.DurationVar(&o.warmup, "warmup", 0, "warmup")
290+ fs.DurationVar(&o.duration, "duration", 0, "measurement duration")
291+ fs.IntVar(&o.repetitions, "repetitions", 0, "repetitions")
292+ if err := fs.Parse(args); err != nil {
293+ return o, err
294+ }
295+ seen := map[string]bool{}
296+ fs.Visit(func(f *flag.Flag) { seen[f.Name] = true })
297+ for _, name := range []string{"vev-bin", "manifest", "out", "warmup", "duration", "repetitions"} {
298+ if !seen[name] {
299+ return o, fmt.Errorf("--%s is required", name)
300+ }
301+ }
302+ if o.vevBin == "" || o.manifest == "" || o.out == "" {
303+ return o, errors.New("--vev-bin, --manifest, and --out are required")
304+ }
305+ if o.warmup < 0 {
306+ return o, errors.New("--warmup must not be negative")
307+ }
308+ if o.duration < minimumDuration {
309+ return o, fmt.Errorf("--duration must be at least %s", minimumDuration)
310+ }
311+ if o.repetitions < minimumRepetitions {
312+ return o, fmt.Errorf("--repetitions must be at least %d", minimumRepetitions)
313+ }
314+ return o, nil
315+}
316+
317+// resolvePathOptions captures the harness cwd once, before role launchers
318+// assign their isolated working directories. All downstream filesystem access,
319+// manifests, and process environments consequently refer to the same paths.
320+func resolvePathOptions(o options) (options, error) {
321+ cwd, err := os.Getwd()
322+ if err != nil {
323+ return o, fmt.Errorf("resolve harness working directory: %w", err)
324+ }
325+ absolute := func(path string) string {
326+ if filepath.IsAbs(path) {
327+ return filepath.Clean(path)
328+ }
329+ return filepath.Clean(filepath.Join(cwd, path))
330+ }
331+ o.vevBin = absolute(o.vevBin)
332+ o.manifest = absolute(o.manifest)
333+ o.out = absolute(o.out)
334+ return o, nil
335+}
336+
337+func readManifest(path string) (manifest, error) {
338+ f, e := os.Open(path)
339+ if e != nil {
340+ return manifest{}, e
341+ }
342+ defer f.Close()
343+ var m manifest
344+ e = json.NewDecoder(f).Decode(&m)
345+ return m, e
346+}
347+func validateManifest(m manifest) error {
348+ if m.Schema != 1 {
349+ return errors.New("unsupported manifest schema")
350+ }
351+ tops := map[string]bool{}
352+ for _, t := range m.Topologies {
353+ if t.ID == "" || t.Geometry != "120x40" || t.RowsPerPane != 10000 {
354+ return fmt.Errorf("invalid topology %q", t.ID)
355+ }
356+ tops[t.ID] = true
357+ }
358+ wantT := []string{"1x4", "4x1", "4x4", "8x1"}
359+ for _, v := range wantT {
360+ if !tops[v] {
361+ return fmt.Errorf("missing canonical topology %s", v)
362+ }
363+ }
364+ works := set(m.Workloads)
365+ for _, v := range []string{"idle", "active_output", "all_output", "inactive_output", "interactive_flood", "copy_search", "resize_sweep", "snapshot_output_resize", "attach_restore_tab_switch"} {
366+ if !works[v] {
367+ return fmt.Errorf("missing canonical workload %s", v)
368+ }
369+ }
370+ trans := map[string]bool{}
371+ for _, t := range m.Transports {
372+ if t.ID == "" {
373+ return errors.New("transport missing id")
374+ }
375+ if err := validateTransportFixture(t); err != nil {
376+ return err
377+ }
378+ trans[t.ID] = true
379+ }
380+ for _, v := range []string{"local", "ssh_stdio", "udp_baseline", "udp_25ms", "udp_100ms", "udp_loss_0pct", "udp_loss_1pct"} {
381+ if !trans[v] {
382+ return fmt.Errorf("missing canonical transport %s", v)
383+ }
384+ }
385+ seen := map[string]bool{}
386+ covered := map[string]bool{}
387+ for _, s := range m.Scenarios {
388+ if s.ID == "" || seen[s.ID] {
389+ return fmt.Errorf("duplicate or empty scenario id %q", s.ID)
390+ }
391+ seen[s.ID] = true
392+ if !tops[s.Topology] || !works[s.Workload] || !trans[s.Transport] {
393+ return fmt.Errorf("scenario %s references unknown matrix entry", s.ID)
394+ }
395+ if s.InapplicableReason == "" && len(s.Roles) == 0 {
396+ return fmt.Errorf("scenario %s has no process roles", s.ID)
397+ }
398+ if s.InapplicableReason != "" && len(s.Roles) != 0 {
399+ return fmt.Errorf("inapplicable scenario %s must not launch roles", s.ID)
400+ }
401+ if s.InapplicableReason == "" && !equalRoleSet(s.Roles, requiredRoles(s.Transport)) {
402+ return fmt.Errorf("scenario %s roles do not route transport %s", s.ID, s.Transport)
403+ }
404+ covered[s.Topology+"\x00"+s.Workload+"\x00"+s.Transport] = true
405+ }
406+ for topologyID := range tops {
407+ for workload := range works {
408+ for transportID := range trans {
409+ key := topologyID + "\x00" + workload + "\x00" + transportID
410+ if !covered[key] {
411+ return fmt.Errorf("missing scenario or inapplicable reason for %s/%s/%s", topologyID, workload, transportID)
412+ }
413+ }
414+ }
415+ }
416+ return nil
417+}
418+
419+// selectScenario is intentionally called only after validateManifest. A bounded
420+// local evidence run therefore cannot mask an incomplete canonical matrix.
421+func selectScenario(m manifest, id string) ([]scenario, error) {
422+ for _, s := range m.Scenarios {
423+ if s.ID != id {
424+ continue
425+ }
426+ if s.InapplicableReason != "" {
427+ return nil, fmt.Errorf("scenario %s is inapplicable: %s", id, s.InapplicableReason)
428+ }
429+ return []scenario{s}, nil
430+ }
431+ return nil, fmt.Errorf("unknown canonical scenario %s", id)
432+}
433+
434+func requiredRoles(transportID string) []string {
435+ switch transportID {
436+ case "local":
437+ return []string{"daemon", "client"}
438+ case "ssh_stdio":
439+ return []string{"daemon", "client", "ssh_stdio_peer"}
440+ default:
441+ return []string{"daemon", "client", "udp_peer"}
442+ }
443+}
444+
445+func equalRoleSet(got, want []string) bool {
446+ if len(got) != len(want) {
447+ return false
448+ }
449+ seen := make(map[string]bool, len(got))
450+ for _, role := range got {
451+ seen[role] = true
452+ }
453+ for _, role := range want {
454+ if !seen[role] {
455+ return false
456+ }
457+ }
458+ return true
459+}
460+
461+func validateTransportFixture(t transport) error {
462+ want := map[string]transport{
463+ "local": {ID: "local", Kind: "local"},
464+ "ssh_stdio": {ID: "ssh_stdio", Kind: "ssh_stdio"},
465+ "udp_baseline": {ID: "udp_baseline", Kind: "udp"},
466+ "udp_25ms": {ID: "udp_25ms", Kind: "udp", RTTMS: 25},
467+ "udp_100ms": {ID: "udp_100ms", Kind: "udp", RTTMS: 100},
468+ "udp_loss_0pct": {ID: "udp_loss_0pct", Kind: "udp", LossPercent: 0},
469+ "udp_loss_1pct": {ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1},
470+ }
471+ w, ok := want[t.ID]
472+ if !ok || t.Kind != w.Kind || t.RTTMS != w.RTTMS || t.LossPercent != w.LossPercent {
473+ return fmt.Errorf("invalid canonical transport fixture %+v", t)
474+ }
475+ return nil
476+}
477+
478+func set(v []string) map[string]bool {
479+ r := map[string]bool{}
480+ for _, x := range v {
481+ r[x] = true
482+ }
483+ return r
484+}
485+func (h *harness) runOne(o options, mat manifest, s scenario, run int, raw io.Writer) (res runResult, err error) {
486+ dir := filepath.Join(o.out, fmt.Sprintf("%s-run-%03d", safeName(s.ID), run))
487+ createRunDir := h.createRunDir
488+ if createRunDir == nil {
489+ createRunDir = createExclusiveRunDir
490+ }
491+ if err = createRunDir(dir); err != nil {
492+ return res, fmt.Errorf("create isolated run directory: %w", err)
493+ }
494+ success := false
495+ defer func() {
496+ if !success {
497+ _ = h.removeAll(dir)
498+ }
499+ }()
500+ maps := make([]processMapping, 0, len(s.Roles))
501+ for i, role := range s.Roles {
502+ pid := fmt.Sprintf("%s-r%03d-%s-%02d", safeName(s.ID), run, safeName(role), i+1)
503+ path := filepath.Join(dir, pid+".jsonl")
504+ f, e := h.create(path)
505+ if e != nil {
506+ return res, fmt.Errorf("allocate exclusive trace %s: %w", role, e)
507+ }
508+ if e = f.Close(); e != nil {
509+ return res, e
510+ }
511+ maps = append(maps, processMapping{ProcessID: pid, ClockDomain: pid, TracePath: path, Role: role, Scenario: s.ID, Run: run, Identity: "harness-assigned:" + pid})
512+ }
513+ // Persist the complete mapping before any process can be launched. This is
514+ // both crash evidence and the only legal source for later ID correlation.
515+ if err = writeJSON(filepath.Join(dir, "manifest.json"), runManifest{Scenario: s.ID, Run: run, Processes: maps}); err != nil {
516+ return res, err
517+ }
518+ if h.launcher == nil {
519+ h.launcher = &cliLauncher{bin: o.vevBin}
520+ }
521+ processes := make([]launchedRole, 0, len(maps))
522+ closed := false
523+ closeRoles := func() error {
524+ if closed {
525+ return nil
526+ }
527+ closed = true
528+ // Clients own SSH-stdio descendants; close them before the deferred UDP
529+ // peer and daemon so no role survives a failed or completed run.
530+ return closeLaunchedRoles(processes)
531+ }
532+ defer func() {
533+ if cleanupErr := closeRoles(); cleanupErr != nil {
534+ if err == nil {
535+ err = cleanupErr
536+ } else {
537+ // Keep the primary workload failure first while retaining every
538+ // cleanup failure in reverse launch order.
539+ err = errors.Join(err, cleanupErr)
540+ }
541+ }
542+ }()
543+ selectedTransport, e := manifestTransport(mat, s.Transport)
544+ if e != nil {
545+ return res, e
546+ }
547+ for _, pm := range launchOrder(maps) {
548+ p, e := h.launcher.Launch(pm, routeRoleArgs(s, pm, selectedTransport))
549+ if e != nil {
550+ return res, e
551+ }
552+ processes = append(processes, launchedRole{mapping: pm, process: p})
553+ if pm.Role == "daemon" {
554+ if ready, ok := p.(interface{ WaitReady() error }); ok {
555+ if err = ready.WaitReady(); err != nil {
556+ return res, err
557+ }
558+ }
559+ }
560+ }
561+ for _, p := range processes {
562+ if p.mapping.Role != "client" {
563+ continue
564+ }
565+ if err = p.process.Warmup(workloadInput(s, run, "warmup")); err != nil {
566+ return res, err
567+ }
568+ }
569+ // The harness clock, rather than a process clock, owns the explicit warmup
570+ // and measured intervals. Warmup boundaries are intentionally never added to
571+ // marks, and each measured event must complete before interval.End to count.
572+ h.clock.Sleep(o.warmup)
573+ interval := measuredInterval{Start: h.clock.Now()}
574+ interval.End = interval.Start + o.duration.Nanoseconds()
575+ marks := []harnessMark{}
576+ seq := uint64(0)
577+ nextInjection := interval.Start
578+ for {
579+ now := h.clock.Now()
580+ if now < nextInjection {
581+ h.clock.Sleep(time.Duration(nextInjection - now))
582+ now = h.clock.Now()
583+ }
584+ if now >= interval.End {
585+ break
586+ }
587+ for _, p := range processes {
588+ if p.mapping.Role != "client" {
589+ continue
590+ }
591+ seq++
592+ sequence := seq
593+ injected := func() error {
594+ marks = append(marks, harnessMark{s.ID, run, sequence, "input_injected", h.clock.Now(), true})
595+ return nil
596+ }
597+ flushed := func() error {
598+ marks = append(marks, harnessMark{s.ID, run, sequence, "terminal_flushed", h.clock.Now(), true})
599+ return nil
600+ }
601+ if err = p.process.Measure(workloadInput(s, run, fmt.Sprintf("measured-%d", sequence)), injected, flushed); err != nil {
602+ return res, err
603+ }
604+ }
605+ // Cap injection rate rather than bursting events after a slow terminal
606+ // flush. Any resulting spacing is retained as raw cadence evidence.
607+ nextInjection = h.clock.Now() + measuredEventCadence.Nanoseconds()
608+ }
609+ if now := h.clock.Now(); now < interval.End {
610+ h.clock.Sleep(time.Duration(interval.End - now))
611+ }
612+ events, e := measuredEventSamples(marks, interval)
613+ if e != nil {
614+ return res, e
615+ }
616+ if err := requireMinimumEventSamples(events); err != nil {
617+ return res, err
618+ }
619+ samples := eventLatencies(events)
620+ cadenceSamples, maxGap := eventCadenceSamples(events)
621+ // Process teardown can unblock a receive and cause its failed end mark to
622+ // be serialized. Merge only the complete, post-cleanup per-process traces.
623+ if cleanupErr := closeRoles(); cleanupErr != nil {
624+ return res, cleanupErr
625+ }
626+ spans, e := mergeProcessTraces(maps)
627+ if e != nil {
628+ return res, e
629+ }
630+ inInterval := make(map[uint64]bool, len(events))
631+ for _, event := range events {
632+ inInterval[event.Sequence] = true
633+ }
634+ for _, m := range marks {
635+ if !inInterval[m.Sequence] {
636+ continue
637+ }
638+ if _, e := fmt.Fprintf(raw, "%s\n", mustJSON(m)); e != nil {
639+ return res, e
640+ }
641+ }
642+ success = true
643+ return runResult{Spans: spans, Scenario: s.ID, Run: run, Samples: len(samples), EndToEnd: samples, Event: percentiles(samples), Cadence: percentiles(cadenceSamples), CadenceSamples: cadenceSamples, MaxGap: maxGap, Processes: maps}, nil
644+}
645+
646+func roleArgs(s scenario, m processMapping) roleCommand {
647+ return routeRoleArgs(s, m, scenarioTransport(s))
648+}
649+
650+func routeRoleArgs(s scenario, m processMapping, selected transport) roleCommand {
651+ session := "perf-" + safeName(s.ID) + fmt.Sprintf("-%03d", m.Run)
652+ remote := []string{"attach", "harness@127.0.0.1:" + session}
653+ switch m.Role {
654+ case "daemon":
655+ return roleCommand{Args: []string{"--daemon"}}
656+ case "client":
657+ switch s.Transport {
658+ case "local":
659+ return roleCommand{Args: []string{"new", session}}
660+ case "ssh_stdio", "udp_baseline", "udp_25ms", "udp_100ms", "udp_loss_0pct", "udp_loss_1pct":
661+ return roleCommand{Args: remote}
662+ }
663+ case "ssh_stdio_peer":
664+ return roleCommand{Args: []string{"_stdio", session}, Transport: transport{ID: s.Transport, Kind: "ssh_stdio"}}
665+ case "udp_peer":
666+ return roleCommand{Args: []string{"_udp-proxy", session}, Transport: selected}
667+ }
668+ return roleCommand{}
669+}
670+
671+func manifestTransport(m manifest, id string) (transport, error) {
672+ for _, t := range m.Transports {
673+ if t.ID == id {
674+ return t, nil
675+ }
676+ }
677+ // Focused unit tests may call runOne with a minimal manifest; production
678+ // already validates every scenario reference before this point.
679+ if m.Schema == 0 {
680+ return scenarioTransport(scenario{Transport: id}), nil
681+ }
682+ return transport{}, fmt.Errorf("scenario references missing transport %q", id)
683+}
684+
685+func scenarioTransport(s scenario) transport {
686+ switch s.Transport {
687+ case "udp_baseline":
688+ return transport{ID: s.Transport, Kind: "udp"}
689+ case "udp_25ms":
690+ return transport{ID: s.Transport, Kind: "udp", RTTMS: 25}
691+ case "udp_100ms":
692+ return transport{ID: s.Transport, Kind: "udp", RTTMS: 100}
693+ case "udp_loss_0pct":
694+ return transport{ID: s.Transport, Kind: "udp", LossPercent: 0}
695+ case "udp_loss_1pct":
696+ return transport{ID: s.Transport, Kind: "udp", LossPercent: 1}
697+ default:
698+ return transport{ID: s.Transport}
699+ }
700+}
701+
702+func launchOrder(mappings []processMapping) []processMapping {
703+ out := append([]processMapping(nil), mappings...)
704+ priority := func(role string) int {
705+ switch role {
706+ case "daemon":
707+ return 0
708+ case "ssh_stdio_peer", "udp_peer":
709+ return 1
710+ case "client":
711+ return 2
712+ default:
713+ return 3
714+ }
715+ }
716+ sort.SliceStable(out, func(i, j int) bool { return priority(out[i].Role) < priority(out[j].Role) })
717+ return out
718+}
719+
720+// workloadInput is shell input delivered through the client PTY. It avoids any
721+// private daemon API and makes every topology/workload/transport manifest row
722+// execute a real terminal workload. The marker is the output boundary awaited
723+// by cliProcess; it is not a synthetic timestamp.
724+func workloadInput(s scenario, run int, phase string) []byte {
725+ marker := fmt.Sprintf("__VEV_HARNESS_%s_r%d_%s__", safeName(s.ID), run, safeName(phase))
726+ body := "printf 'vev perf %s\\n'"
727+ switch s.Workload {
728+ case "active_output", "all_output", "inactive_output", "interactive_flood":
729+ body = "i=0; while [ $i -lt 128 ]; do printf 'vev perf output %s\\n'; i=$((i+1)); done"
730+ case "resize_sweep":
731+ body = "printf 'vev perf resize-sweep %s\\n'"
732+ case "copy_search":
733+ body = "printf 'vev perf copy-search %s\\n'"
734+ case "snapshot_output_resize":
735+ body = "printf 'vev perf snapshot-output-resize %s\\n'"
736+ case "attach_restore_tab_switch":
737+ body = "printf 'vev perf attach-restore-tab-switch %s\\n'"
738+ }
739+ return []byte(fmt.Sprintf(body+"; printf '%s\\n'\n", s.ID, marker))
740+}
741+
742+// measuredEventSamples validates every owned pair, then retains only events
743+// whose injection and successful terminal flush are both in the interval.
744+// Straddling pairs remain raw diagnostic facts but are never percentile input.
A cmd/vev-perf-harness/harness_3.go +307 -0
@@ -0,0 +1,307 @@
1+package main
2+
3+import (
4+ "bufio"
5+ "encoding/json"
6+ "errors"
7+ "fmt"
8+ "os"
9+ "sort"
10+ "strings"
11+)
12+
13+func measuredEventSamples(marks []harnessMark, interval measuredInterval) ([]eventSample, error) {
14+ if interval.End < interval.Start {
15+ return nil, errors.New("negative measured interval")
16+ }
17+ starts := map[uint64]int64{}
18+ events := make([]eventSample, 0, len(marks)/2)
19+ for _, m := range marks {
20+ if !m.Valid {
21+ return nil, errors.New("invalid harness boundary")
22+ }
23+ switch m.Kind {
24+ case "input_injected":
25+ if _, exists := starts[m.Sequence]; exists {
26+ return nil, errors.New("duplicate input boundary")
27+ }
28+ starts[m.Sequence] = m.Tick
29+ case "terminal_flushed":
30+ start, ok := starts[m.Sequence]
31+ if !ok {
32+ return nil, errors.New("terminal flush without input pair")
33+ }
34+ if m.Tick < start {
35+ return nil, errors.New("negative harness duration")
36+ }
37+ delete(starts, m.Sequence)
38+ if start >= interval.Start && m.Tick <= interval.End {
39+ events = append(events, eventSample{Sequence: m.Sequence, Injected: start, Flushed: m.Tick, Latency: m.Tick - start})
40+ }
41+ default:
42+ return nil, errors.New("unknown harness mark")
43+ }
44+ }
45+ if len(starts) != 0 {
46+ return nil, errors.New("missing terminal flush pair")
47+ }
48+ sort.Slice(events, func(i, j int) bool {
49+ if events[i].Injected == events[j].Injected {
50+ return events[i].Sequence < events[j].Sequence
51+ }
52+ return events[i].Injected < events[j].Injected
53+ })
54+ return events, nil
55+}
56+
57+func requireMinimumEventSamples(events []eventSample) error {
58+ if len(events) < minimumInIntervalEventSamples {
59+ return fmt.Errorf("insufficient in-interval event samples: got %d, need at least %d", len(events), minimumInIntervalEventSamples)
60+ }
61+ return nil
62+}
63+
64+func eventLatencies(events []eventSample) []int64 {
65+ out := make([]int64, len(events))
66+ for i, event := range events {
67+ out[i] = event.Latency
68+ }
69+ return out
70+}
71+
72+func eventCadenceSamples(events []eventSample) ([]int64, int64) {
73+ if len(events) < 2 {
74+ return nil, 0
75+ }
76+ gaps := make([]int64, 0, len(events)-1)
77+ var maxGap int64
78+ for i := 1; i < len(events); i++ {
79+ gap := events[i].Injected - events[i-1].Injected
80+ gaps = append(gaps, gap)
81+ if gap > maxGap {
82+ maxGap = gap
83+ }
84+ }
85+ return gaps, maxGap
86+}
87+
88+func eventCadence(events []eventSample) (distribution, int64) {
89+ gaps, maxGap := eventCadenceSamples(events)
90+ if len(gaps) == 0 {
91+ return distribution{}, maxGap
92+ }
93+ return percentiles(gaps), maxGap
94+}
95+
96+func pairedSamples(marks []harnessMark) ([]int64, error) {
97+ starts := map[uint64]int64{}
98+ var out []int64
99+ for _, m := range marks {
100+ if !m.Valid {
101+ return nil, errors.New("invalid harness boundary")
102+ }
103+ switch m.Kind {
104+ case "input_injected":
105+ if _, ok := starts[m.Sequence]; ok {
106+ return nil, errors.New("duplicate input boundary")
107+ }
108+ starts[m.Sequence] = m.Tick
109+ case "terminal_flushed":
110+ start, ok := starts[m.Sequence]
111+ if !ok {
112+ return nil, errors.New("terminal flush without input pair")
113+ }
114+ if m.Tick < start {
115+ return nil, errors.New("negative harness duration")
116+ }
117+ out = append(out, m.Tick-start)
118+ delete(starts, m.Sequence)
119+ default:
120+ return nil, errors.New("unknown harness mark")
121+ }
122+ }
123+ if len(starts) != 0 {
124+ return nil, errors.New("missing terminal flush pair")
125+ }
126+ if len(out) == 0 {
127+ return nil, errors.New("insufficient harness samples")
128+ }
129+ return out, nil
130+}
131+
132+// traceRecord is intentionally local to the command. It mirrors JSONL fields
133+// needed for correlation but never exposes a clock to any production component.
134+type traceRecord struct {
135+ ProcessID string `json:"process_id"`
136+ Component string `json:"component"`
137+ Scenario string `json:"scenario"`
138+ Run uint64 `json:"run"`
139+ Sequence uint64 `json:"sequence"`
140+ RequestID uint64 `json:"request_id"`
141+ Epoch uint64 `json:"epoch"`
142+ Kind string `json:"kind"`
143+ Tick int64 `json:"tick"`
144+ Valid bool `json:"valid"`
145+}
146+type spanPair struct{ start, end, name string }
147+
148+var spanPairs = []spanPair{{"capture_start", "capture_end", "capture_duration"}, {"compose_start", "compose_end", "compose_duration"}, {"diff_start", "diff_end", "diff_duration"}, {"queue_enqueued", "queue_dequeued", "queue_wait"}, {"ack_blocked_start", "ack_blocked_end", "ack_blocked_interval"}, {"emit_start", "emit_end", "emit_duration"}, {"adapter_send_start", "adapter_send_end", "adapter_send_duration"}, {"adapter_receive_start", "adapter_receive_end", "adapter_receive_duration"}}
149+
150+// mergeProcessTraces permits only records from one manifest process in a span.
151+// In particular, the correlation key includes process_id before ticks are read.
152+func mergeProcessTraces(mappings []processMapping) ([]span, error) {
153+ known := map[string]processMapping{}
154+ for _, m := range mappings {
155+ if m.ProcessID == "" || m.ClockDomain == "" || m.TracePath == "" || known[m.ProcessID].ProcessID != "" {
156+ return nil, errors.New("duplicate or incomplete manifest process mapping")
157+ }
158+ known[m.ProcessID] = m
159+ }
160+ out := []span{}
161+ for _, m := range mappings {
162+ f, err := os.Open(m.TracePath)
163+ if err != nil {
164+ return nil, fmt.Errorf("open manifest trace for %s role: %w", m.Role, err)
165+ }
166+ var records []traceRecord
167+ scan := bufio.NewScanner(f)
168+ for scan.Scan() {
169+ var r traceRecord
170+ if err := json.Unmarshal(scan.Bytes(), &r); err != nil {
171+ _ = f.Close()
172+ return nil, err
173+ }
174+ if r.ProcessID != m.ProcessID || r.Scenario != m.Scenario || r.Run != uint64(m.Run) {
175+ _ = f.Close()
176+ return nil, errors.New("trace record identity does not match manifest")
177+ }
178+ if r.Scenario == "" || r.Run == 0 || r.Component == "" || r.Sequence == 0 || r.RequestID == 0 || r.Epoch == 0 {
179+ _ = f.Close()
180+ return nil, errors.New("trace record has invalid correlation fields")
181+ }
182+ if _, ok := known[r.ProcessID]; !ok {
183+ _ = f.Close()
184+ return nil, errors.New("unknown trace process_id")
185+ }
186+ records = append(records, r)
187+ }
188+ if err := scan.Err(); err != nil {
189+ _ = f.Close()
190+ return nil, err
191+ }
192+ _ = f.Close()
193+ // A process shares one observer across concurrent components. Correlation
194+ // IDs are allocated before a goroutine reaches its first mark, so a later
195+ // ID can be serialized before an earlier one. Their first appearance is
196+ // consequently not an ordering contract. Only marks in the same component
197+ // and exact correlation domain may pair, and each pair's ticks must retain
198+ // its own start-before-end ordering.
199+ starts := map[string]traceRecord{}
200+ for _, r := range records {
201+ for _, pair := range spanPairs {
202+ key := fmt.Sprintf("%s/%s/%s/%d/%d/%d/%d", r.ProcessID, r.Component, m.Scenario, m.Run, r.Sequence, r.RequestID, r.Epoch)
203+ if r.Kind == pair.start {
204+ k := pair.name + "/" + key
205+ if _, exists := starts[k]; exists {
206+ return nil, errors.New("duplicate process-local span start")
207+ }
208+ starts[k] = r
209+ }
210+ if r.Kind == pair.end {
211+ k := pair.name + "/" + key
212+ start, ok := starts[k]
213+ if !ok {
214+ return nil, errors.New("span end without same-process start")
215+ }
216+ if r.Tick < start.Tick {
217+ return nil, errors.New("negative same-process span")
218+ }
219+ // Validity affects only measurement eligibility, never structural
220+ // pairing: every start/end must still match in its exact process,
221+ // component, and correlation domain. A failed start, failed end, or
222+ // both is a paired diagnostic fact but contributes no latency sample.
223+ if start.Valid && r.Valid {
224+ out = append(out, span{Component: r.Component, Name: pair.name, Samples: []int64{r.Tick - start.Tick}})
225+ }
226+ delete(starts, k)
227+ }
228+ }
229+ }
230+ if len(starts) != 0 {
231+ keys := make([]string, 0, len(starts))
232+ for key := range starts {
233+ keys = append(keys, key)
234+ }
235+ sort.Strings(keys)
236+ return nil, fmt.Errorf("missing process-local span pair for %s: %s", m.ProcessID, strings.Join(keys, ", "))
237+ }
238+ }
239+ return out, nil
240+}
241+func summarizeSpans(all []span) []spanSummary {
242+ grouped := map[string]*span{}
243+ for _, s := range all {
244+ k := s.Component + "\x00" + s.Name
245+ if grouped[k] == nil {
246+ grouped[k] = &span{Component: s.Component, Name: s.Name}
247+ }
248+ grouped[k].Samples = append(grouped[k].Samples, s.Samples...)
249+ }
250+ keys := make([]string, 0, len(grouped))
251+ for k := range grouped {
252+ keys = append(keys, k)
253+ }
254+ sort.Strings(keys)
255+ out := make([]spanSummary, 0, len(keys))
256+ for _, k := range keys {
257+ s := grouped[k]
258+ if len(s.Samples) < minimumRepetitions {
259+ out = append(out, spanSummary{Component: s.Component, Name: s.Name, Distribution: distribution{Count: len(s.Samples)}})
260+ continue
261+ }
262+ out = append(out, spanSummary{Component: s.Component, Name: s.Name, Distribution: percentiles(s.Samples)})
263+ }
264+ return out
265+}
266+
267+func percentiles(samples []int64) distribution {
268+ v := append([]int64(nil), samples...)
269+ sort.Slice(v, func(i, j int) bool { return v[i] < v[j] })
270+ at := func(p int) int64 { return v[(len(v)-1)*p/100] }
271+ return distribution{len(v), at(50), at(95), at(99), v[len(v)-1]}
272+}
273+func writeJSON(path string, v any) error {
274+ f, e := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
275+ if e != nil {
276+ return e
277+ }
278+ defer f.Close()
279+ e = json.NewEncoder(f).Encode(v)
280+ return e
281+}
282+func mustJSON(v any) string { b, _ := json.Marshal(v); return string(b) }
283+func safeName(s string) string {
284+ return strings.Map(func(r rune) rune {
285+ if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' {
286+ return r
287+ }
288+ return '-'
289+ }, s)
290+}
291+func readJSONL(path string) ([]map[string]json.RawMessage, error) {
292+ f, e := os.Open(path)
293+ if e != nil {
294+ return nil, e
295+ }
296+ defer f.Close()
297+ var r []map[string]json.RawMessage
298+ scan := bufio.NewScanner(f)
299+ for scan.Scan() {
300+ var v map[string]json.RawMessage
301+ if e := json.Unmarshal(scan.Bytes(), &v); e != nil {
302+ return nil, e
303+ }
304+ r = append(r, v)
305+ }
306+ return r, scan.Err()
307+}
A cmd/vev-perf-harness/harness_test_1_test.go +746 -0
@@ -0,0 +1,746 @@
1+package main
2+
3+import (
4+ "bytes"
5+ "encoding/json"
6+ "errors"
7+ "fmt"
8+ "net"
9+ "os"
10+ "os/exec"
11+ "path/filepath"
12+ "strings"
13+ "testing"
14+ "time"
15+)
16+
17+type fakeClock struct{ tick int64 }
18+
19+func (c *fakeClock) Now() int64 { c.tick += 10; return c.tick }
20+func (c *fakeClock) Sleep(d time.Duration) {
21+ c.tick += d.Nanoseconds()
22+}
23+
24+type fakeUDPNetem struct {
25+ port int
26+ closed bool
27+}
28+
29+func (n *fakeUDPNetem) Port() int { return n.port }
30+func (n *fakeUDPNetem) Close() error { n.closed = true; return nil }
31+
32+type fakeProcess struct {
33+ fail bool
34+ measureErr error
35+ closeErr error
36+ warmups [][]byte
37+ measures [][]byte
38+ closed bool
39+}
40+
41+func (p *fakeProcess) Warmup(input []byte) error {
42+ p.warmups = append(p.warmups, append([]byte(nil), input...))
43+ return nil
44+}
45+func (p *fakeProcess) Measure(input []byte, injected, flush func() error) error {
46+ p.measures = append(p.measures, append([]byte(nil), input...))
47+ if p.measureErr != nil {
48+ return p.measureErr
49+ }
50+ if p.fail {
51+ return errors.New("measure failure")
52+ }
53+ if err := injected(); err != nil {
54+ return err
55+ }
56+ return flush()
57+}
58+func (p *fakeProcess) Close() error { p.closed = true; return p.closeErr }
59+
60+type fakeLauncher struct {
61+ mappings []processMapping
62+ args [][]string
63+ commands []roleCommand
64+ process []*fakeProcess
65+ measureErr map[string]error
66+ closeErr map[string]error
67+ manifestPresent bool
68+}
69+
70+func (l *fakeLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
71+ l.mappings = append(l.mappings, m)
72+ l.args = append(l.args, append([]string(nil), command.Args...))
73+ l.commands = append(l.commands, command)
74+ _, e := os.Stat(filepath.Join(filepath.Dir(m.TracePath), "manifest.json"))
75+ l.manifestPresent = e == nil
76+ p := &fakeProcess{measureErr: l.measureErr[m.Role], closeErr: l.closeErr[m.Role]}
77+ l.process = append(l.process, p)
78+ return p, nil
79+}
80+
81+type lateFlushProcess struct{ clock *fakeClock }
82+
83+func (*lateFlushProcess) Warmup([]byte) error { return nil }
84+func (p *lateFlushProcess) Measure(_ []byte, injected, flushed func() error) error {
85+ if err := injected(); err != nil {
86+ return err
87+ }
88+ p.clock.tick += minimumDuration.Nanoseconds()
89+ return flushed()
90+}
91+func (*lateFlushProcess) Close() error { return nil }
92+
93+type insufficientEventLauncher struct{ clock *fakeClock }
94+
95+func (l insufficientEventLauncher) Launch(m processMapping, _ roleCommand) (launchedProcess, error) {
96+ if m.Role == "client" {
97+ return &lateFlushProcess{clock: l.clock}, nil
98+ }
99+ return &fakeProcess{}, nil
100+}
101+
102+func TestCLIProcessCloseIsBoundedWhenPublicRoleDoesNotExit(t *testing.T) {
103+ deadline := make(chan time.Time)
104+ close(deadline)
105+ var term, kill int
106+ p := &cliProcess{
107+ waitErr: make(chan error),
108+ waitTimeout: func() <-chan time.Time { return deadline },
109+ forceCleanup: func() { term++ },
110+ forceKill: func() { kill++ },
111+ }
112+ if err := p.Close(); err == nil {
113+ t.Fatal("stuck public role close succeeded")
114+ }
115+ if term != 1 || kill != 1 {
116+ t.Fatalf("TERM/KILL calls = %d/%d, want 1/1", term, kill)
117+ }
118+}
119+
120+func TestCLIProcessWaitReadyRequiresDaemonSocket(t *testing.T) {
121+ path := filepath.Join(t.TempDir(), "daemon.sock")
122+ listener, err := net.Listen("unix", path)
123+ if err != nil {
124+ t.Fatal(err)
125+ }
126+ defer listener.Close()
127+ if err := (&cliProcess{readyPath: path, waitErr: make(chan error, 1)}).WaitReady(); err != nil {
128+ t.Fatal(err)
129+ }
130+}
131+
132+func TestCLILauncherIsolatesRoleXDGEnvironmentAcrossRepetitions(t *testing.T) {
133+ captureDir := t.TempDir()
134+ t.Setenv("CAPTURE_DIR", captureDir)
135+ t.Setenv("XDG_RUNTIME_DIR", filepath.Join(t.TempDir(), "inherited-runtime"))
136+ t.Setenv("XDG_STATE_HOME", filepath.Join(t.TempDir(), "inherited-state"))
137+
138+ bin := filepath.Join(t.TempDir(), "capture-env")
139+ if err := os.WriteFile(bin, []byte("#!/bin/sh\nenv > \"$CAPTURE_DIR/$VEV_PERF_PROCESS_ID\"\n"), 0o700); err != nil {
140+ t.Fatal(err)
141+ }
142+ launcher := &cliLauncher{bin: bin}
143+ for run := 1; run <= 3; run++ {
144+ runDir := filepath.Join(t.TempDir(), fmt.Sprintf("run-%d", run))
145+ if err := os.MkdirAll(runDir, 0o700); err != nil {
146+ t.Fatal(err)
147+ }
148+ roles := []string{"daemon", "client"}
149+ for _, role := range roles {
150+ id := fmt.Sprintf("run-%d-%s", run, role)
151+ p, err := launcher.Launch(processMapping{
152+ Role: role, TracePath: filepath.Join(runDir, role+".jsonl"), ProcessID: id,
153+ }, roleCommand{Args: []string{"test"}})
154+ if err != nil {
155+ t.Fatalf("run %d %s launch: %v", run, role, err)
156+ }
157+ runtimeDir := launcher.runtimes[runDir]
158+ if err := p.Close(); err != nil {
159+ t.Fatalf("run %d %s close: %v", run, role, err)
160+ }
161+
162+ env, err := os.ReadFile(filepath.Join(captureDir, id))
163+ if err != nil {
164+ t.Fatalf("run %d %s capture: %v", run, role, err)
165+ }
166+ for _, want := range []string{
167+ "XDG_RUNTIME_DIR=" + runtimeDir,
168+ "XDG_STATE_HOME=" + filepath.Join(runDir, "state"),
169+ } {
170+ name := strings.SplitN(want, "=", 2)[0]
171+ if count := bytes.Count(env, []byte(name+"=")); count != 1 {
172+ t.Fatalf("run %d %s %s entries = %d, want exactly one; env:\n%s", run, role, name, count, env)
173+ }
174+ if !bytes.Contains(env, []byte(want+"\n")) {
175+ t.Fatalf("run %d %s missing %q; env:\n%s", run, role, want, env)
176+ }
177+ }
178+ }
179+ if err := launcher.releaseRuntime(runDir); err != nil {
180+ t.Fatalf("run %d release runtime: %v", run, err)
181+ }
182+ }
183+}
184+
185+func TestCLIProcessCloseUsesPublicDaemonShutdown(t *testing.T) {
186+ wait := make(chan error, 1)
187+ wait <- nil
188+ var shutdown int
189+ p := &cliProcess{waitErr: wait, shutdown: func() error { shutdown++; return nil }}
190+ if err := p.Close(); err != nil {
191+ t.Fatal(err)
192+ }
193+ if shutdown != 0 {
194+ t.Fatalf("shutdown calls = %d, want 0 for an exited daemon", shutdown)
195+ }
196+
197+ wait = make(chan error, 1)
198+ p = &cliProcess{waitErr: wait, shutdown: func() error { shutdown++; wait <- nil; return nil }}
199+ if err := p.Close(); err != nil {
200+ t.Fatal(err)
201+ }
202+ if shutdown != 1 {
203+ t.Fatalf("shutdown calls = %d, want 1", shutdown)
204+ }
205+}
206+
207+func TestHarnessCanonicalLocalSmokeRealRoles(t *testing.T) {
208+ if testing.Short() {
209+ t.Skip("public CLI smoke")
210+ }
211+ root, err := filepath.Abs(filepath.Join("..", ".."))
212+ if err != nil {
213+ t.Fatal(err)
214+ }
215+ bin := filepath.Join(t.TempDir(), "vev")
216+ build := exec.Command("/usr/local/go/bin/go", "build", "-o", bin, "./")
217+ build.Dir = root
218+ if output, err := build.CombinedOutput(); err != nil {
219+ t.Fatalf("build public CLI: %v\n%s", err, output)
220+ }
221+ m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
222+ if err != nil {
223+ t.Fatal(err)
224+ }
225+ if err := validateManifest(m); err != nil {
226+ t.Fatal(err)
227+ }
228+ var local scenario
229+ for _, s := range m.Scenarios {
230+ if s.Topology == "1x4" && s.Workload == "idle" && s.Transport == "local" {
231+ local = s
232+ break
233+ }
234+ }
235+ if local.ID == "" {
236+ t.Fatal("canonical local scenario missing")
237+ }
238+ dir := t.TempDir()
239+ raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
240+ if err != nil {
241+ t.Fatal(err)
242+ }
243+ defer raw.Close()
244+ h := defaultHarness()
245+ h.clock = &fakeClock{} // injected duration keeps the public 30s contract bounded in test
246+ h.launcher = &cliLauncher{bin: bin}
247+ result, err := h.runOne(options{vevBin: bin, out: dir, duration: minimumDuration}, m, local, 1, raw)
248+ if err != nil {
249+ t.Fatal(err)
250+ }
251+ if result.Samples < minimumInIntervalEventSamples || len(result.Processes) != 2 {
252+ t.Fatalf("canonical local smoke result=%+v", result)
253+ }
254+}
255+
256+func TestHarnessCanonicalLocalRolesAreIsolatedAcrossRepetitions(t *testing.T) {
257+ if testing.Short() {
258+ t.Skip("public CLI repetition lifecycle")
259+ }
260+ root, err := filepath.Abs(filepath.Join("..", ".."))
261+ if err != nil {
262+ t.Fatal(err)
263+ }
264+ bin := filepath.Join(t.TempDir(), "vev")
265+ build := exec.Command("/usr/local/go/bin/go", "build", "-o", bin, "./")
266+ build.Dir = root
267+ if output, err := build.CombinedOutput(); err != nil {
268+ t.Fatalf("build public CLI: %v\n%s", err, output)
269+ }
270+ m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
271+ if err != nil {
272+ t.Fatal(err)
273+ }
274+ var local scenario
275+ for _, candidate := range m.Scenarios {
276+ if candidate.ID == "1x4-idle-local" {
277+ local = candidate
278+ break
279+ }
280+ }
281+ if local.ID == "" {
282+ t.Fatal("canonical local scenario missing")
283+ }
284+ out, err := os.MkdirTemp("", "vev-ob-")
285+ if err != nil {
286+ t.Fatal(err)
287+ }
288+ defer os.RemoveAll(out)
289+ raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
290+ if err != nil {
291+ t.Fatal(err)
292+ }
293+ defer raw.Close()
294+ h := defaultHarness()
295+ h.clock = &fakeClock{}
296+ h.launcher = &cliLauncher{bin: bin}
297+ for run := 1; run <= 3; run++ {
298+ result, err := h.runOne(options{vevBin: bin, out: out, duration: minimumDuration}, m, local, run, raw)
299+ if err != nil {
300+ t.Fatalf("run %d: %v", run, err)
301+ }
302+ if len(result.Processes) != 2 {
303+ t.Fatalf("run %d process mappings = %+v", run, result.Processes)
304+ }
305+ runDir := filepath.Join(out, fmt.Sprintf("%s-run-%03d", safeName(local.ID), run))
306+ manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
307+ if err != nil {
308+ t.Fatalf("run %d manifest: %v", run, err)
309+ }
310+ var persisted runManifest
311+ if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
312+ t.Fatalf("run %d manifest JSON: %v", run, err)
313+ }
314+ if persisted.Run != run || len(persisted.Processes) != len(result.Processes) {
315+ t.Fatalf("run %d manifest = %+v", run, persisted)
316+ }
317+ for _, process := range result.Processes {
318+ if process.Role != "daemon" && process.Role != "client" {
319+ t.Fatalf("run %d unexpected role %q", run, process.Role)
320+ }
321+ if filepath.Dir(process.TracePath) != runDir {
322+ t.Fatalf("run %d %s trace outside its isolated run directory: %q", run, process.Role, process.TracePath)
323+ }
324+ trace, err := os.ReadFile(process.TracePath)
325+ if err != nil {
326+ t.Fatalf("run %d %s trace %q: %v", run, process.Role, process.TracePath, err)
327+ }
328+ if len(bytes.TrimSpace(trace)) == 0 {
329+ t.Fatalf("run %d %s did not emit its declared trace", run, process.Role)
330+ }
331+ for _, line := range bytes.Split(bytes.TrimSpace(trace), []byte{'\n'}) {
332+ var record traceRecord
333+ if err := json.Unmarshal(line, &record); err != nil {
334+ t.Fatalf("run %d %s trace JSON: %v", run, process.Role, err)
335+ }
336+ if record.ProcessID != process.ProcessID || record.Scenario != local.ID || record.Run != uint64(run) {
337+ t.Fatalf("run %d %s trace escaped its manifest identity: %+v", run, process.Role, record)
338+ }
339+ }
340+ }
341+ if _, err := os.Stat(filepath.Join(runDir, "runtime", "vev", "daemon.sock")); !errors.Is(err, os.ErrNotExist) {
342+ t.Fatalf("run %d left daemon socket for the next repetition: %v", run, err)
343+ }
344+ }
345+}
346+
347+func TestHarnessResolvesCanonicalRelativePathsBeforeRoleCWDChanges(t *testing.T) {
348+ if testing.Short() {
349+ t.Skip("public CLI relative-path lifecycle")
350+ }
351+ root, err := filepath.Abs(filepath.Join("..", ".."))
352+ if err != nil {
353+ t.Fatal(err)
354+ }
355+ // Keep the real Unix-socket path below its platform limit while testing the
356+ // exact documented relative output hierarchy.
357+ workspace, err := os.MkdirTemp("", "vev-rel-")
358+ if err != nil {
359+ t.Fatal(err)
360+ }
361+ t.Cleanup(func() { _ = os.RemoveAll(workspace) })
362+ bin := filepath.Join(workspace, "vev")
363+ build := exec.Command("/usr/local/go/bin/go", "build", "-o", bin, "./")
364+ build.Dir = root
365+ if output, err := build.CombinedOutput(); err != nil {
366+ t.Fatalf("build public CLI: %v\n%s", err, output)
367+ }
368+ manifestPath := filepath.Join(workspace, "testdata", "perf", "manifest.json")
369+ if err := os.MkdirAll(filepath.Dir(manifestPath), 0o700); err != nil {
370+ t.Fatal(err)
371+ }
372+ contents, err := os.ReadFile(filepath.Join(root, "testdata", "perf", "manifest.json"))
373+ if err != nil {
374+ t.Fatal(err)
375+ }
376+ if err := os.WriteFile(manifestPath, contents, 0o600); err != nil {
377+ t.Fatal(err)
378+ }
379+ originalWD, err := os.Getwd()
380+ if err != nil {
381+ t.Fatal(err)
382+ }
383+ if err := os.Chdir(workspace); err != nil {
384+ t.Fatal(err)
385+ }
386+ t.Cleanup(func() {
387+ if err := os.Chdir(originalWD); err != nil {
388+ t.Errorf("restore cwd: %v", err)
389+ }
390+ })
391+
392+ // These are the documented relative locations. cliLauncher changes every
393+ // role to its run directory, so a failure to resolve them before launch
394+ // would make VEV_PERF_TRACE point at a nonexistent nested directory.
395+ h := defaultHarness()
396+ h.clock = &fakeClock{}
397+ if err := run([]string{
398+ "--vev-bin", "./vev",
399+ "--manifest", "testdata/perf/manifest.json",
400+ "--out", "testdata/perf/results",
401+ "--scenario", "1x4-idle-local",
402+ "--warmup", "0s",
403+ "--duration", "30s",
404+ "--repetitions", "10",
405+ }, h); err != nil {
406+ t.Fatal(err)
407+ }
408+
409+ out := filepath.Join(workspace, "testdata", "perf", "results")
410+ for run := 1; run <= 10; run++ {
411+ runDir := filepath.Join(out, fmt.Sprintf("1x4-idle-local-run-%03d", run))
412+ manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
413+ if err != nil {
414+ t.Fatalf("run %d manifest: %v", run, err)
415+ }
416+ var persisted runManifest
417+ if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
418+ t.Fatalf("run %d manifest JSON: %v", run, err)
419+ }
420+ if persisted.Run != run || len(persisted.Processes) != 2 {
421+ t.Fatalf("run %d manifest=%+v", run, persisted)
422+ }
423+ for _, process := range persisted.Processes {
424+ if !filepath.IsAbs(process.TracePath) || filepath.Dir(process.TracePath) != runDir {
425+ t.Fatalf("run %d %s trace path=%q, want absolute path in %q", run, process.Role, process.TracePath, runDir)
426+ }
427+ trace, err := os.ReadFile(process.TracePath)
428+ if err != nil || len(bytes.TrimSpace(trace)) == 0 {
429+ t.Fatalf("run %d %s trace=%q err=%v", run, process.Role, process.TracePath, err)
430+ }
431+ }
432+ }
433+}
434+
435+func TestHarnessScenarioSelectionKeepsCanonicalManifestValidation(t *testing.T) {
436+ root, err := filepath.Abs(filepath.Join("..", ".."))
437+ if err != nil {
438+ t.Fatal(err)
439+ }
440+ manifestPath := filepath.Join(root, "testdata", "perf", "manifest.json")
441+ m, err := readManifest(manifestPath)
442+ if err != nil {
443+ t.Fatal(err)
444+ }
445+ selected := m.Scenarios[0]
446+ h := defaultHarness()
447+ h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
448+ out := filepath.Join(t.TempDir(), "out")
449+ if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
450+ t.Fatal(err)
451+ }
452+ var runs []runResult
453+ b, err := os.ReadFile(filepath.Join(out, "runs.json"))
454+ if err != nil {
455+ t.Fatal(err)
456+ }
457+ if err := json.Unmarshal(b, &runs); err != nil {
458+ t.Fatal(err)
459+ }
460+ if len(runs) != 10 {
461+ t.Fatalf("selected canonical runs=%d, want 10", len(runs))
462+ }
463+ for _, result := range runs {
464+ if result.Scenario != selected.ID {
465+ t.Fatalf("selected scenario = %q, want %q", result.Scenario, selected.ID)
466+ }
467+ }
468+}
469+
470+func TestHarnessScenarioSelectionRejectsIncompleteCanonicalManifest(t *testing.T) {
471+ m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
472+ if err != nil {
473+ t.Fatal(err)
474+ }
475+ selected := m.Scenarios[0]
476+ m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
477+ dir := t.TempDir()
478+ manifestPath := filepath.Join(dir, "manifest.json")
479+ if err := os.WriteFile(manifestPath, []byte(mustJSON(m)), 0o600); err != nil {
480+ t.Fatal(err)
481+ }
482+ launcher := &fakeLauncher{}
483+ h := defaultHarness()
484+ h.clock, h.launcher = &fakeClock{}, launcher
485+ out := filepath.Join(dir, "out")
486+ err = run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h)
487+ if err == nil || !strings.Contains(err.Error(), "missing scenario or inapplicable reason") {
488+ t.Fatalf("incomplete manifest selection error = %v", err)
489+ }
490+ if len(launcher.mappings) != 0 {
491+ t.Fatalf("launched with incomplete canonical manifest: %+v", launcher.mappings)
492+ }
493+ if _, statErr := os.Stat(out); !errors.Is(statErr, os.ErrNotExist) {
494+ t.Fatalf("output created before manifest validation: %v", statErr)
495+ }
496+}
497+
498+func TestTraceEnvironmentMatchesProcessMapping(t *testing.T) {
499+ mapping := processMapping{TracePath: "/tmp/trace.jsonl", ProcessID: "p-01", Scenario: "1x4-idle-local", Run: 3}
500+ got := strings.Join(traceEnvironment(mapping), "\n")
501+ for _, want := range []string{"VEV_PERF_TRACE=/tmp/trace.jsonl", "VEV_PERF_PROCESS_ID=p-01", "VEV_PERF_SCENARIO=1x4-idle-local", "VEV_PERF_RUN=3"} {
502+ if !strings.Contains(got, want) {
503+ t.Errorf("trace environment missing %q: %s", want, got)
504+ }
505+ }
506+}
507+
508+func TestResolvePathOptionsMakesRelativeAndAbsoluteInputsIdentical(t *testing.T) {
509+ cwd, err := os.Getwd()
510+ if err != nil {
511+ t.Fatal(err)
512+ }
513+ absolute := options{
514+ vevBin: filepath.Join(cwd, "vev"),
515+ manifest: filepath.Join(cwd, "testdata", "perf", "manifest.json"),
516+ out: filepath.Join(cwd, "testdata", "perf", "results"),
517+ }
518+ relative := options{
519+ vevBin: "vev",
520+ manifest: "testdata/perf/manifest.json",
521+ out: "testdata/perf/results",
522+ }
523+ gotRelative, err := resolvePathOptions(relative)
524+ if err != nil {
525+ t.Fatal(err)
526+ }
527+ gotAbsolute, err := resolvePathOptions(absolute)
528+ if err != nil {
529+ t.Fatal(err)
530+ }
531+ if gotRelative != gotAbsolute {
532+ t.Fatalf("relative paths resolved to %+v, absolute paths to %+v", gotRelative, gotAbsolute)
533+ }
534+}
535+
536+func TestHarnessFlagsRejectShortMeasurements(t *testing.T) {
537+ for _, args := range [][]string{
538+ {"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "29s", "--repetitions", "10"},
539+ {"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "9"},
540+ {"--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"},
541+ } {
542+ if _, err := parseOptions(args); err == nil {
543+ t.Fatalf("parseOptions(%v) succeeded", args)
544+ }
545+ }
546+ if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"}); err == nil {
547+ t.Fatal("missing --warmup accepted")
548+ }
549+ if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}); err != nil {
550+ t.Fatal(err)
551+ }
552+}
553+
554+func TestHarnessManifestCoversCanonicalMatrix(t *testing.T) {
555+ m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
556+ if err != nil {
557+ t.Fatal(err)
558+ }
559+ if err := validateManifest(m); err != nil {
560+ t.Fatal(err)
561+ }
562+ if len(m.Scenarios) != 4*9*7 {
563+ t.Fatalf("scenarios=%d", len(m.Scenarios))
564+ }
565+ m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
566+ if err := validateManifest(m); err == nil {
567+ t.Fatal("missing combination accepted")
568+ }
569+}
570+
571+func TestHarnessRejectsExistingRunDirectoryBeforeLaunchingRoles(t *testing.T) {
572+ out := t.TempDir()
573+ raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
574+ if err != nil {
575+ t.Fatal(err)
576+ }
577+ defer raw.Close()
578+ s := scenario{ID: "isolated", Roles: []string{"daemon", "client"}}
579+ stale := filepath.Join(out, "isolated-run-001")
580+ if err := os.Mkdir(stale, 0o700); err != nil {
581+ t.Fatal(err)
582+ }
583+ if err := os.WriteFile(filepath.Join(stale, "preserve"), []byte("stale"), 0o600); err != nil {
584+ t.Fatal(err)
585+ }
586+ launcher := &fakeLauncher{}
587+ h := defaultHarness()
588+ h.clock, h.launcher = &fakeClock{}, launcher
589+ _, err = h.runOne(options{out: out, duration: minimumDuration}, manifest{}, s, 1, raw)
590+ if err == nil || !strings.Contains(err.Error(), "create isolated run directory") {
591+ t.Fatalf("existing run directory error = %v", err)
592+ }
593+ if len(launcher.mappings) != 0 {
594+ t.Fatalf("roles launched against stale run directory: %+v", launcher.mappings)
595+ }
596+ if contents, readErr := os.ReadFile(filepath.Join(stale, "preserve")); readErr != nil || string(contents) != "stale" {
597+ t.Fatalf("stale directory changed: contents=%q err=%v", contents, readErr)
598+ }
599+}
600+
601+func TestHarnessCreatesExclusiveTraceManifestAndEvidence(t *testing.T) {
602+ dir := t.TempDir()
603+ raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
604+ if err != nil {
605+ t.Fatal(err)
606+ }
607+ defer raw.Close()
608+ launcher := &fakeLauncher{}
609+ h := defaultHarness()
610+ h.clock = &fakeClock{}
611+ h.launcher = launcher
612+ o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
613+ r, err := h.runOne(o, manifest{}, scenario{ID: "4x4-local", Roles: []string{"daemon", "client"}}, 1, raw)
614+ if err != nil {
615+ t.Fatal(err)
616+ }
617+ if !launcher.manifestPresent {
618+ t.Fatal("launched before complete run manifest")
619+ }
620+ if len(r.EndToEnd) < minimumInIntervalEventSamples || r.Samples < minimumInIntervalEventSamples {
621+ t.Fatalf("samples=%+v", r)
622+ }
623+ seen := map[string]bool{}
624+ for _, m := range r.Processes {
625+ if m.ProcessID == "" || m.ClockDomain != m.ProcessID || m.TracePath == "" || seen[m.TracePath] {
626+ t.Fatalf("bad mapping %+v", m)
627+ }
628+ seen[m.TracePath] = true
629+ if _, err := os.Stat(m.TracePath); err != nil {
630+ t.Fatal(err)
631+ }
632+ }
633+ b, err := os.ReadFile(filepath.Join(dir, "4x4-local-run-001", "manifest.json"))
634+ if err != nil {
635+ t.Fatal(err)
636+ }
637+ var got runManifest
638+ if err := json.Unmarshal(b, &got); err != nil {
639+ t.Fatal(err)
640+ }
641+ if len(got.Processes) != 2 {
642+ t.Fatalf("manifest=%+v", got)
643+ }
644+ if len(launcher.process) != 2 || !launcher.process[0].closed || len(launcher.process[0].warmups) != 0 || len(launcher.process[1].warmups) != 1 || len(launcher.process[1].measures) < minimumInIntervalEventSamples {
645+ t.Fatalf("process lifecycle not recorded: %+v", launcher.process)
646+ }
647+}
648+
649+type fakePTY struct{ writes [][]byte }
650+
651+func (p *fakePTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
652+func (p *fakePTY) Write(b []byte) (int, error) {
653+ p.writes = append(p.writes, append([]byte(nil), b...))
654+ return len(b), nil
655+}
656+func (*fakePTY) Close() error { return nil }
657+
658+type stagedPTY struct {
659+ allow <-chan struct{}
660+ writes chan []byte
661+}
662+
663+func (*stagedPTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
664+func (p *stagedPTY) Write(b []byte) (int, error) {
665+ if p.allow != nil {
666+ <-p.allow
667+ }
668+ p.writes <- append([]byte(nil), b...)
669+ return len(b), nil
670+}
671+func (*stagedPTY) Close() error { return nil }
672+
673+type fakeOutput struct {
674+ bytes.Buffer
675+ syncs int
676+}
677+
678+func (o *fakeOutput) Sync() error { o.syncs++; return nil }
679+func (*fakeOutput) Close() error { return nil }
680+
681+type orderedPTY struct {
682+ write func([]byte)
683+ close func()
684+}
685+
686+func (*orderedPTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
687+func (p *orderedPTY) Write(b []byte) (int, error) {
688+ if p.write != nil {
689+ p.write(b)
690+ }
691+ return len(b), nil
692+}
693+func (p *orderedPTY) Close() error {
694+ p.close()
695+ return nil
696+}
697+
698+type orderedOutput struct{ close func() }
699+
700+func (*orderedOutput) Write(b []byte) (int, error) { return len(b), nil }
701+func (*orderedOutput) Sync() error { return nil }
702+func (o *orderedOutput) Close() error {
703+ o.close()
704+ return nil
705+}
706+
707+func TestCLIProcessCloseGracefulDetachCompletesSpanBeforeForcedCleanup(t *testing.T) {
708+ var order []string
709+ waitErr := make(chan error, 1)
710+ timeout := make(chan time.Time)
711+ p := &cliProcess{
712+ pty: &orderedPTY{
713+ write: func(input []byte) {
714+ if string(input) != "exit\n" {
715+ t.Fatalf("graceful exit input=%q", input)
716+ }
717+ order = append(order, "exit_written")
718+ // The graceful client exit has closed its transport and emitted
719+ // its adapter receive-end mark before cmd.Wait completes.
720+ order = append(order, "adapter_receive_end")
721+ waitErr <- nil
722+ },
723+ close: func() { order = append(order, "pty_closed") },
724+ },
725+ output: &orderedOutput{close: func() { order = append(order, "output_closed") }},
726+ waitErr: waitErr,
727+ waitTimeout: func() <-chan time.Time { return timeout },
728+ forceCleanup: func() {
729+ order = append(order, "forced_process_group_cleanup")
730+ },
731+ }
732+
733+ if err := p.Close(); err != nil {
734+ t.Fatal(err)
735+ }
736+ if !equalStrings(order, []string{"exit_written", "adapter_receive_end", "pty_closed", "output_closed"}) {
737+ t.Fatalf("cleanup order=%q", order)
738+ }
739+}
740+
741+func TestHarnessDoesNotInheritNestedSession(t *testing.T) {
742+ env := withoutEnv([]string{"VEV=session=old", "PATH=/bin", "VEV_PERF_TRACE=old"}, "VEV")
743+ if !equalStrings(env, []string{"PATH=/bin", "VEV_PERF_TRACE=old"}) {
744+ t.Fatalf("environment=%q", env)
745+ }
746+}
R cmd/vev-perf-harness/main_test.go → cmd/vev-perf-harness/harness_test_2_test.go +0 -779 renamed, 52% similar
@@ -4,10 +4,8 @@ import (
44 "bytes"
55 "encoding/json"
66 "errors"
7- "fmt"
87 "net"
98 "os"
10- "os/exec"
119 "path/filepath"
1210 "strconv"
1311 "strings"
@@ -15,737 +13,6 @@ import (
1513 "time"
1614 )
1715
18-type fakeClock struct{ tick int64 }
19-
20-func (c *fakeClock) Now() int64 { c.tick += 10; return c.tick }
21-func (c *fakeClock) Sleep(d time.Duration) {
22- c.tick += d.Nanoseconds()
23-}
24-
25-type fakeUDPNetem struct {
26- port int
27- closed bool
28-}
29-
30-func (n *fakeUDPNetem) Port() int { return n.port }
31-func (n *fakeUDPNetem) Close() error { n.closed = true; return nil }
32-
33-type fakeProcess struct {
34- fail bool
35- measureErr error
36- closeErr error
37- warmups [][]byte
38- measures [][]byte
39- closed bool
40-}
41-
42-func (p *fakeProcess) Warmup(input []byte) error {
43- p.warmups = append(p.warmups, append([]byte(nil), input...))
44- return nil
45-}
46-func (p *fakeProcess) Measure(input []byte, injected, flush func() error) error {
47- p.measures = append(p.measures, append([]byte(nil), input...))
48- if p.measureErr != nil {
49- return p.measureErr
50- }
51- if p.fail {
52- return errors.New("measure failure")
53- }
54- if err := injected(); err != nil {
55- return err
56- }
57- return flush()
58-}
59-func (p *fakeProcess) Close() error { p.closed = true; return p.closeErr }
60-
61-type fakeLauncher struct {
62- mappings []processMapping
63- args [][]string
64- commands []roleCommand
65- process []*fakeProcess
66- measureErr map[string]error
67- closeErr map[string]error
68- manifestPresent bool
69-}
70-
71-func (l *fakeLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
72- l.mappings = append(l.mappings, m)
73- l.args = append(l.args, append([]string(nil), command.Args...))
74- l.commands = append(l.commands, command)
75- _, e := os.Stat(filepath.Join(filepath.Dir(m.TracePath), "manifest.json"))
76- l.manifestPresent = e == nil
77- p := &fakeProcess{measureErr: l.measureErr[m.Role], closeErr: l.closeErr[m.Role]}
78- l.process = append(l.process, p)
79- return p, nil
80-}
81-
82-type lateFlushProcess struct{ clock *fakeClock }
83-
84-func (*lateFlushProcess) Warmup([]byte) error { return nil }
85-func (p *lateFlushProcess) Measure(_ []byte, injected, flushed func() error) error {
86- if err := injected(); err != nil {
87- return err
88- }
89- p.clock.tick += minimumDuration.Nanoseconds()
90- return flushed()
91-}
92-func (*lateFlushProcess) Close() error { return nil }
93-
94-type insufficientEventLauncher struct{ clock *fakeClock }
95-
96-func (l insufficientEventLauncher) Launch(m processMapping, _ roleCommand) (launchedProcess, error) {
97- if m.Role == "client" {
98- return &lateFlushProcess{clock: l.clock}, nil
99- }
100- return &fakeProcess{}, nil
101-}
102-
103-func TestCLIProcessCloseIsBoundedWhenPublicRoleDoesNotExit(t *testing.T) {
104- deadline := make(chan time.Time)
105- close(deadline)
106- var term, kill int
107- p := &cliProcess{
108- waitErr: make(chan error),
109- waitTimeout: func() <-chan time.Time { return deadline },
110- forceCleanup: func() { term++ },
111- forceKill: func() { kill++ },
112- }
113- if err := p.Close(); err == nil {
114- t.Fatal("stuck public role close succeeded")
115- }
116- if term != 1 || kill != 1 {
117- t.Fatalf("TERM/KILL calls = %d/%d, want 1/1", term, kill)
118- }
119-}
120-
121-func TestCLIProcessWaitReadyRequiresDaemonSocket(t *testing.T) {
122- path := filepath.Join(t.TempDir(), "daemon.sock")
123- listener, err := net.Listen("unix", path)
124- if err != nil {
125- t.Fatal(err)
126- }
127- defer listener.Close()
128- if err := (&cliProcess{readyPath: path, waitErr: make(chan error, 1)}).WaitReady(); err != nil {
129- t.Fatal(err)
130- }
131-}
132-
133-func TestCLILauncherIsolatesRoleXDGEnvironmentAcrossRepetitions(t *testing.T) {
134- captureDir := t.TempDir()
135- t.Setenv("CAPTURE_DIR", captureDir)
136- t.Setenv("XDG_RUNTIME_DIR", filepath.Join(t.TempDir(), "inherited-runtime"))
137- t.Setenv("XDG_STATE_HOME", filepath.Join(t.TempDir(), "inherited-state"))
138-
139- bin := filepath.Join(t.TempDir(), "capture-env")
140- if err := os.WriteFile(bin, []byte("#!/bin/sh\nenv > \"$CAPTURE_DIR/$VEV_PERF_PROCESS_ID\"\n"), 0o700); err != nil {
141- t.Fatal(err)
142- }
143- launcher := &cliLauncher{bin: bin}
144- for run := 1; run <= 3; run++ {
145- runDir := filepath.Join(t.TempDir(), fmt.Sprintf("run-%d", run))
146- if err := os.MkdirAll(runDir, 0o700); err != nil {
147- t.Fatal(err)
148- }
149- roles := []string{"daemon", "client"}
150- for _, role := range roles {
151- id := fmt.Sprintf("run-%d-%s", run, role)
152- p, err := launcher.Launch(processMapping{
153- Role: role, TracePath: filepath.Join(runDir, role+".jsonl"), ProcessID: id,
154- }, roleCommand{Args: []string{"test"}})
155- if err != nil {
156- t.Fatalf("run %d %s launch: %v", run, role, err)
157- }
158- runtimeDir := launcher.runtimes[runDir]
159- if err := p.Close(); err != nil {
160- t.Fatalf("run %d %s close: %v", run, role, err)
161- }
162-
163- env, err := os.ReadFile(filepath.Join(captureDir, id))
164- if err != nil {
165- t.Fatalf("run %d %s capture: %v", run, role, err)
166- }
167- for _, want := range []string{
168- "XDG_RUNTIME_DIR=" + runtimeDir,
169- "XDG_STATE_HOME=" + filepath.Join(runDir, "state"),
170- } {
171- name := strings.SplitN(want, "=", 2)[0]
172- if count := bytes.Count(env, []byte(name+"=")); count != 1 {
173- t.Fatalf("run %d %s %s entries = %d, want exactly one; env:\n%s", run, role, name, count, env)
174- }
175- if !bytes.Contains(env, []byte(want+"\n")) {
176- t.Fatalf("run %d %s missing %q; env:\n%s", run, role, want, env)
177- }
178- }
179- }
180- if err := launcher.releaseRuntime(runDir); err != nil {
181- t.Fatalf("run %d release runtime: %v", run, err)
182- }
183- }
184-}
185-
186-func TestCLIProcessCloseUsesPublicDaemonShutdown(t *testing.T) {
187- wait := make(chan error, 1)
188- wait <- nil
189- var shutdown int
190- p := &cliProcess{waitErr: wait, shutdown: func() error { shutdown++; return nil }}
191- if err := p.Close(); err != nil {
192- t.Fatal(err)
193- }
194- if shutdown != 0 {
195- t.Fatalf("shutdown calls = %d, want 0 for an exited daemon", shutdown)
196- }
197-
198- wait = make(chan error, 1)
199- p = &cliProcess{waitErr: wait, shutdown: func() error { shutdown++; wait <- nil; return nil }}
200- if err := p.Close(); err != nil {
201- t.Fatal(err)
202- }
203- if shutdown != 1 {
204- t.Fatalf("shutdown calls = %d, want 1", shutdown)
205- }
206-}
207-
208-func TestHarnessCanonicalLocalSmokeRealRoles(t *testing.T) {
209- if testing.Short() {
210- t.Skip("public CLI smoke")
211- }
212- root, err := filepath.Abs(filepath.Join("..", ".."))
213- if err != nil {
214- t.Fatal(err)
215- }
216- bin := filepath.Join(t.TempDir(), "vev")
217- build := exec.Command("/usr/local/go/bin/go", "build", "-o", bin, "./")
218- build.Dir = root
219- if output, err := build.CombinedOutput(); err != nil {
220- t.Fatalf("build public CLI: %v\n%s", err, output)
221- }
222- m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
223- if err != nil {
224- t.Fatal(err)
225- }
226- if err := validateManifest(m); err != nil {
227- t.Fatal(err)
228- }
229- var local scenario
230- for _, s := range m.Scenarios {
231- if s.Topology == "1x4" && s.Workload == "idle" && s.Transport == "local" {
232- local = s
233- break
234- }
235- }
236- if local.ID == "" {
237- t.Fatal("canonical local scenario missing")
238- }
239- dir := t.TempDir()
240- raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
241- if err != nil {
242- t.Fatal(err)
243- }
244- defer raw.Close()
245- h := defaultHarness()
246- h.clock = &fakeClock{} // injected duration keeps the public 30s contract bounded in test
247- h.launcher = &cliLauncher{bin: bin}
248- result, err := h.runOne(options{vevBin: bin, out: dir, duration: minimumDuration}, m, local, 1, raw)
249- if err != nil {
250- t.Fatal(err)
251- }
252- if result.Samples < minimumInIntervalEventSamples || len(result.Processes) != 2 {
253- t.Fatalf("canonical local smoke result=%+v", result)
254- }
255-}
256-
257-func TestHarnessCanonicalLocalRolesAreIsolatedAcrossRepetitions(t *testing.T) {
258- if testing.Short() {
259- t.Skip("public CLI repetition lifecycle")
260- }
261- root, err := filepath.Abs(filepath.Join("..", ".."))
262- if err != nil {
263- t.Fatal(err)
264- }
265- bin := filepath.Join(t.TempDir(), "vev")
266- build := exec.Command("/usr/local/go/bin/go", "build", "-o", bin, "./")
267- build.Dir = root
268- if output, err := build.CombinedOutput(); err != nil {
269- t.Fatalf("build public CLI: %v\n%s", err, output)
270- }
271- m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
272- if err != nil {
273- t.Fatal(err)
274- }
275- var local scenario
276- for _, candidate := range m.Scenarios {
277- if candidate.ID == "1x4-idle-local" {
278- local = candidate
279- break
280- }
281- }
282- if local.ID == "" {
283- t.Fatal("canonical local scenario missing")
284- }
285- out, err := os.MkdirTemp("", "vev-ob-")
286- if err != nil {
287- t.Fatal(err)
288- }
289- defer os.RemoveAll(out)
290- raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
291- if err != nil {
292- t.Fatal(err)
293- }
294- defer raw.Close()
295- h := defaultHarness()
296- h.clock = &fakeClock{}
297- h.launcher = &cliLauncher{bin: bin}
298- for run := 1; run <= 3; run++ {
299- result, err := h.runOne(options{vevBin: bin, out: out, duration: minimumDuration}, m, local, run, raw)
300- if err != nil {
301- t.Fatalf("run %d: %v", run, err)
302- }
303- if len(result.Processes) != 2 {
304- t.Fatalf("run %d process mappings = %+v", run, result.Processes)
305- }
306- runDir := filepath.Join(out, fmt.Sprintf("%s-run-%03d", safeName(local.ID), run))
307- manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
308- if err != nil {
309- t.Fatalf("run %d manifest: %v", run, err)
310- }
311- var persisted runManifest
312- if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
313- t.Fatalf("run %d manifest JSON: %v", run, err)
314- }
315- if persisted.Run != run || len(persisted.Processes) != len(result.Processes) {
316- t.Fatalf("run %d manifest = %+v", run, persisted)
317- }
318- for _, process := range result.Processes {
319- if process.Role != "daemon" && process.Role != "client" {
320- t.Fatalf("run %d unexpected role %q", run, process.Role)
321- }
322- if filepath.Dir(process.TracePath) != runDir {
323- t.Fatalf("run %d %s trace outside its isolated run directory: %q", run, process.Role, process.TracePath)
324- }
325- trace, err := os.ReadFile(process.TracePath)
326- if err != nil {
327- t.Fatalf("run %d %s trace %q: %v", run, process.Role, process.TracePath, err)
328- }
329- if len(bytes.TrimSpace(trace)) == 0 {
330- t.Fatalf("run %d %s did not emit its declared trace", run, process.Role)
331- }
332- for _, line := range bytes.Split(bytes.TrimSpace(trace), []byte{'\n'}) {
333- var record traceRecord
334- if err := json.Unmarshal(line, &record); err != nil {
335- t.Fatalf("run %d %s trace JSON: %v", run, process.Role, err)
336- }
337- if record.ProcessID != process.ProcessID || record.Scenario != local.ID || record.Run != uint64(run) {
338- t.Fatalf("run %d %s trace escaped its manifest identity: %+v", run, process.Role, record)
339- }
340- }
341- }
342- if _, err := os.Stat(filepath.Join(runDir, "runtime", "vev", "daemon.sock")); !errors.Is(err, os.ErrNotExist) {
343- t.Fatalf("run %d left daemon socket for the next repetition: %v", run, err)
344- }
345- }
346-}
347-
348-func TestHarnessResolvesCanonicalRelativePathsBeforeRoleCWDChanges(t *testing.T) {
349- if testing.Short() {
350- t.Skip("public CLI relative-path lifecycle")
351- }
352- root, err := filepath.Abs(filepath.Join("..", ".."))
353- if err != nil {
354- t.Fatal(err)
355- }
356- // Keep the real Unix-socket path below its platform limit while testing the
357- // exact documented relative output hierarchy.
358- workspace, err := os.MkdirTemp("", "vev-rel-")
359- if err != nil {
360- t.Fatal(err)
361- }
362- t.Cleanup(func() { _ = os.RemoveAll(workspace) })
363- bin := filepath.Join(workspace, "vev")
364- build := exec.Command("/usr/local/go/bin/go", "build", "-o", bin, "./")
365- build.Dir = root
366- if output, err := build.CombinedOutput(); err != nil {
367- t.Fatalf("build public CLI: %v\n%s", err, output)
368- }
369- manifestPath := filepath.Join(workspace, "testdata", "perf", "manifest.json")
370- if err := os.MkdirAll(filepath.Dir(manifestPath), 0o700); err != nil {
371- t.Fatal(err)
372- }
373- contents, err := os.ReadFile(filepath.Join(root, "testdata", "perf", "manifest.json"))
374- if err != nil {
375- t.Fatal(err)
376- }
377- if err := os.WriteFile(manifestPath, contents, 0o600); err != nil {
378- t.Fatal(err)
379- }
380- originalWD, err := os.Getwd()
381- if err != nil {
382- t.Fatal(err)
383- }
384- if err := os.Chdir(workspace); err != nil {
385- t.Fatal(err)
386- }
387- t.Cleanup(func() {
388- if err := os.Chdir(originalWD); err != nil {
389- t.Errorf("restore cwd: %v", err)
390- }
391- })
392-
393- // These are the documented relative locations. cliLauncher changes every
394- // role to its run directory, so a failure to resolve them before launch
395- // would make VEV_PERF_TRACE point at a nonexistent nested directory.
396- h := defaultHarness()
397- h.clock = &fakeClock{}
398- if err := run([]string{
399- "--vev-bin", "./vev",
400- "--manifest", "testdata/perf/manifest.json",
401- "--out", "testdata/perf/results",
402- "--scenario", "1x4-idle-local",
403- "--warmup", "0s",
404- "--duration", "30s",
405- "--repetitions", "10",
406- }, h); err != nil {
407- t.Fatal(err)
408- }
409-
410- out := filepath.Join(workspace, "testdata", "perf", "results")
411- for run := 1; run <= 10; run++ {
412- runDir := filepath.Join(out, fmt.Sprintf("1x4-idle-local-run-%03d", run))
413- manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
414- if err != nil {
415- t.Fatalf("run %d manifest: %v", run, err)
416- }
417- var persisted runManifest
418- if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
419- t.Fatalf("run %d manifest JSON: %v", run, err)
420- }
421- if persisted.Run != run || len(persisted.Processes) != 2 {
422- t.Fatalf("run %d manifest=%+v", run, persisted)
423- }
424- for _, process := range persisted.Processes {
425- if !filepath.IsAbs(process.TracePath) || filepath.Dir(process.TracePath) != runDir {
426- t.Fatalf("run %d %s trace path=%q, want absolute path in %q", run, process.Role, process.TracePath, runDir)
427- }
428- trace, err := os.ReadFile(process.TracePath)
429- if err != nil || len(bytes.TrimSpace(trace)) == 0 {
430- t.Fatalf("run %d %s trace=%q err=%v", run, process.Role, process.TracePath, err)
431- }
432- }
433- }
434-}
435-
436-func TestHarnessScenarioSelectionKeepsCanonicalManifestValidation(t *testing.T) {
437- root, err := filepath.Abs(filepath.Join("..", ".."))
438- if err != nil {
439- t.Fatal(err)
440- }
441- manifestPath := filepath.Join(root, "testdata", "perf", "manifest.json")
442- m, err := readManifest(manifestPath)
443- if err != nil {
444- t.Fatal(err)
445- }
446- selected := m.Scenarios[0]
447- h := defaultHarness()
448- h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
449- out := filepath.Join(t.TempDir(), "out")
450- if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
451- t.Fatal(err)
452- }
453- var runs []runResult
454- b, err := os.ReadFile(filepath.Join(out, "runs.json"))
455- if err != nil {
456- t.Fatal(err)
457- }
458- if err := json.Unmarshal(b, &runs); err != nil {
459- t.Fatal(err)
460- }
461- if len(runs) != 10 {
462- t.Fatalf("selected canonical runs=%d, want 10", len(runs))
463- }
464- for _, result := range runs {
465- if result.Scenario != selected.ID {
466- t.Fatalf("selected scenario = %q, want %q", result.Scenario, selected.ID)
467- }
468- }
469-}
470-
471-func TestHarnessScenarioSelectionRejectsIncompleteCanonicalManifest(t *testing.T) {
472- m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
473- if err != nil {
474- t.Fatal(err)
475- }
476- selected := m.Scenarios[0]
477- m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
478- dir := t.TempDir()
479- manifestPath := filepath.Join(dir, "manifest.json")
480- if err := os.WriteFile(manifestPath, []byte(mustJSON(m)), 0o600); err != nil {
481- t.Fatal(err)
482- }
483- launcher := &fakeLauncher{}
484- h := defaultHarness()
485- h.clock, h.launcher = &fakeClock{}, launcher
486- out := filepath.Join(dir, "out")
487- err = run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h)
488- if err == nil || !strings.Contains(err.Error(), "missing scenario or inapplicable reason") {
489- t.Fatalf("incomplete manifest selection error = %v", err)
490- }
491- if len(launcher.mappings) != 0 {
492- t.Fatalf("launched with incomplete canonical manifest: %+v", launcher.mappings)
493- }
494- if _, statErr := os.Stat(out); !errors.Is(statErr, os.ErrNotExist) {
495- t.Fatalf("output created before manifest validation: %v", statErr)
496- }
497-}
498-
499-func TestTraceEnvironmentMatchesProcessMapping(t *testing.T) {
500- mapping := processMapping{TracePath: "/tmp/trace.jsonl", ProcessID: "p-01", Scenario: "1x4-idle-local", Run: 3}
501- got := strings.Join(traceEnvironment(mapping), "\n")
502- for _, want := range []string{"VEV_PERF_TRACE=/tmp/trace.jsonl", "VEV_PERF_PROCESS_ID=p-01", "VEV_PERF_SCENARIO=1x4-idle-local", "VEV_PERF_RUN=3"} {
503- if !strings.Contains(got, want) {
504- t.Errorf("trace environment missing %q: %s", want, got)
505- }
506- }
507-}
508-
509-func TestResolvePathOptionsMakesRelativeAndAbsoluteInputsIdentical(t *testing.T) {
510- cwd, err := os.Getwd()
511- if err != nil {
512- t.Fatal(err)
513- }
514- absolute := options{
515- vevBin: filepath.Join(cwd, "vev"),
516- manifest: filepath.Join(cwd, "testdata", "perf", "manifest.json"),
517- out: filepath.Join(cwd, "testdata", "perf", "results"),
518- }
519- relative := options{
520- vevBin: "vev",
521- manifest: "testdata/perf/manifest.json",
522- out: "testdata/perf/results",
523- }
524- gotRelative, err := resolvePathOptions(relative)
525- if err != nil {
526- t.Fatal(err)
527- }
528- gotAbsolute, err := resolvePathOptions(absolute)
529- if err != nil {
530- t.Fatal(err)
531- }
532- if gotRelative != gotAbsolute {
533- t.Fatalf("relative paths resolved to %+v, absolute paths to %+v", gotRelative, gotAbsolute)
534- }
535-}
536-
537-func TestHarnessFlagsRejectShortMeasurements(t *testing.T) {
538- for _, args := range [][]string{
539- {"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "29s", "--repetitions", "10"},
540- {"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "9"},
541- {"--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"},
542- } {
543- if _, err := parseOptions(args); err == nil {
544- t.Fatalf("parseOptions(%v) succeeded", args)
545- }
546- }
547- if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"}); err == nil {
548- t.Fatal("missing --warmup accepted")
549- }
550- if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}); err != nil {
551- t.Fatal(err)
552- }
553-}
554-
555-func TestHarnessManifestCoversCanonicalMatrix(t *testing.T) {
556- m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
557- if err != nil {
558- t.Fatal(err)
559- }
560- if err := validateManifest(m); err != nil {
561- t.Fatal(err)
562- }
563- if len(m.Scenarios) != 4*9*7 {
564- t.Fatalf("scenarios=%d", len(m.Scenarios))
565- }
566- m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
567- if err := validateManifest(m); err == nil {
568- t.Fatal("missing combination accepted")
569- }
570-}
571-
572-func TestHarnessRejectsExistingRunDirectoryBeforeLaunchingRoles(t *testing.T) {
573- out := t.TempDir()
574- raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
575- if err != nil {
576- t.Fatal(err)
577- }
578- defer raw.Close()
579- s := scenario{ID: "isolated", Roles: []string{"daemon", "client"}}
580- stale := filepath.Join(out, "isolated-run-001")
581- if err := os.Mkdir(stale, 0o700); err != nil {
582- t.Fatal(err)
583- }
584- if err := os.WriteFile(filepath.Join(stale, "preserve"), []byte("stale"), 0o600); err != nil {
585- t.Fatal(err)
586- }
587- launcher := &fakeLauncher{}
588- h := defaultHarness()
589- h.clock, h.launcher = &fakeClock{}, launcher
590- _, err = h.runOne(options{out: out, duration: minimumDuration}, manifest{}, s, 1, raw)
591- if err == nil || !strings.Contains(err.Error(), "create isolated run directory") {
592- t.Fatalf("existing run directory error = %v", err)
593- }
594- if len(launcher.mappings) != 0 {
595- t.Fatalf("roles launched against stale run directory: %+v", launcher.mappings)
596- }
597- if contents, readErr := os.ReadFile(filepath.Join(stale, "preserve")); readErr != nil || string(contents) != "stale" {
598- t.Fatalf("stale directory changed: contents=%q err=%v", contents, readErr)
599- }
600-}
601-
602-func TestHarnessCreatesExclusiveTraceManifestAndEvidence(t *testing.T) {
603- dir := t.TempDir()
604- raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
605- if err != nil {
606- t.Fatal(err)
607- }
608- defer raw.Close()
609- launcher := &fakeLauncher{}
610- h := defaultHarness()
611- h.clock = &fakeClock{}
612- h.launcher = launcher
613- o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
614- r, err := h.runOne(o, manifest{}, scenario{ID: "4x4-local", Roles: []string{"daemon", "client"}}, 1, raw)
615- if err != nil {
616- t.Fatal(err)
617- }
618- if !launcher.manifestPresent {
619- t.Fatal("launched before complete run manifest")
620- }
621- if len(r.EndToEnd) < minimumInIntervalEventSamples || r.Samples < minimumInIntervalEventSamples {
622- t.Fatalf("samples=%+v", r)
623- }
624- seen := map[string]bool{}
625- for _, m := range r.Processes {
626- if m.ProcessID == "" || m.ClockDomain != m.ProcessID || m.TracePath == "" || seen[m.TracePath] {
627- t.Fatalf("bad mapping %+v", m)
628- }
629- seen[m.TracePath] = true
630- if _, err := os.Stat(m.TracePath); err != nil {
631- t.Fatal(err)
632- }
633- }
634- b, err := os.ReadFile(filepath.Join(dir, "4x4-local-run-001", "manifest.json"))
635- if err != nil {
636- t.Fatal(err)
637- }
638- var got runManifest
639- if err := json.Unmarshal(b, &got); err != nil {
640- t.Fatal(err)
641- }
642- if len(got.Processes) != 2 {
643- t.Fatalf("manifest=%+v", got)
644- }
645- if len(launcher.process) != 2 || !launcher.process[0].closed || len(launcher.process[0].warmups) != 0 || len(launcher.process[1].warmups) != 1 || len(launcher.process[1].measures) < minimumInIntervalEventSamples {
646- t.Fatalf("process lifecycle not recorded: %+v", launcher.process)
647- }
648-}
649-
650-type fakePTY struct{ writes [][]byte }
651-
652-func (p *fakePTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
653-func (p *fakePTY) Write(b []byte) (int, error) {
654- p.writes = append(p.writes, append([]byte(nil), b...))
655- return len(b), nil
656-}
657-func (*fakePTY) Close() error { return nil }
658-
659-type stagedPTY struct {
660- allow <-chan struct{}
661- writes chan []byte
662-}
663-
664-func (*stagedPTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
665-func (p *stagedPTY) Write(b []byte) (int, error) {
666- if p.allow != nil {
667- <-p.allow
668- }
669- p.writes <- append([]byte(nil), b...)
670- return len(b), nil
671-}
672-func (*stagedPTY) Close() error { return nil }
673-
674-type fakeOutput struct {
675- bytes.Buffer
676- syncs int
677-}
678-
679-func (o *fakeOutput) Sync() error { o.syncs++; return nil }
680-func (*fakeOutput) Close() error { return nil }
681-
682-type orderedPTY struct {
683- write func([]byte)
684- close func()
685-}
686-
687-func (*orderedPTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
688-func (p *orderedPTY) Write(b []byte) (int, error) {
689- if p.write != nil {
690- p.write(b)
691- }
692- return len(b), nil
693-}
694-func (p *orderedPTY) Close() error {
695- p.close()
696- return nil
697-}
698-
699-type orderedOutput struct{ close func() }
700-
701-func (*orderedOutput) Write(b []byte) (int, error) { return len(b), nil }
702-func (*orderedOutput) Sync() error { return nil }
703-func (o *orderedOutput) Close() error {
704- o.close()
705- return nil
706-}
707-
708-func TestCLIProcessCloseGracefulDetachCompletesSpanBeforeForcedCleanup(t *testing.T) {
709- var order []string
710- waitErr := make(chan error, 1)
711- timeout := make(chan time.Time)
712- p := &cliProcess{
713- pty: &orderedPTY{
714- write: func(input []byte) {
715- if string(input) != "exit\n" {
716- t.Fatalf("graceful exit input=%q", input)
717- }
718- order = append(order, "exit_written")
719- // The graceful client exit has closed its transport and emitted
720- // its adapter receive-end mark before cmd.Wait completes.
721- order = append(order, "adapter_receive_end")
722- waitErr <- nil
723- },
724- close: func() { order = append(order, "pty_closed") },
725- },
726- output: &orderedOutput{close: func() { order = append(order, "output_closed") }},
727- waitErr: waitErr,
728- waitTimeout: func() <-chan time.Time { return timeout },
729- forceCleanup: func() {
730- order = append(order, "forced_process_group_cleanup")
731- },
732- }
733-
734- if err := p.Close(); err != nil {
735- t.Fatal(err)
736- }
737- if !equalStrings(order, []string{"exit_written", "adapter_receive_end", "pty_closed", "output_closed"}) {
738- t.Fatalf("cleanup order=%q", order)
739- }
740-}
741-
742-func TestHarnessDoesNotInheritNestedSession(t *testing.T) {
743- env := withoutEnv([]string{"VEV=session=old", "PATH=/bin", "VEV_PERF_TRACE=old"}, "VEV")
744- if !equalStrings(env, []string{"PATH=/bin", "VEV_PERF_TRACE=old"}) {
745- t.Fatalf("environment=%q", env)
746- }
747-}
748-
74916 func TestCLIProcessWarmupWaitsForDelayedTerminalReadiness(t *testing.T) {
75017 allowWrite := make(chan struct{})
75118 pty, output := &stagedPTY{allow: allowWrite, writes: make(chan []byte, 1)}, &fakeOutput{}
@@ -1483,49 +750,3 @@ func TestHarnessRejectsCrossProcessAndBadTraceSpans(t *testing.T) {
1483750 t.Fatalf("spans=%+v err=%v", spans, err)
1484751 }
1485752 }
1486-
1487-func TestHarnessExcludesFailedSpanDurationsWhileValidatingPairing(t *testing.T) {
1488- base := func(kind string, tick int64, valid bool) traceRecord {
1489- return traceRecord{ProcessID: "one", Component: "adapter", Scenario: "s", Run: 1, Sequence: 1, RequestID: 1, Epoch: 1, Kind: kind, Tick: tick, Valid: valid}
1490- }
1491- for _, tc := range []struct {
1492- name string
1493- records []traceRecord
1494- wantSpans int
1495- wantErr bool
1496- }{
1497- {"successful adapter send is sampled", []traceRecord{base("adapter_send_start", 10, true), base("adapter_send_end", 20, true)}, 1, false},
1498- {"failed adapter send end is excluded", []traceRecord{base("adapter_send_start", 10, true), base("adapter_send_end", 20, false)}, 0, false},
1499- {"failed adapter receive end is excluded", []traceRecord{base("adapter_receive_start", 10, true), base("adapter_receive_end", 20, false)}, 0, false},
1500- {"failed adapter send start is excluded", []traceRecord{base("adapter_send_start", 10, false), base("adapter_send_end", 20, true)}, 0, false},
1501- {"failed adapter receive boundaries are excluded", []traceRecord{base("adapter_receive_start", 10, false), base("adapter_receive_end", 20, false)}, 0, false},
1502- {"failed end without start is rejected", []traceRecord{base("adapter_send_end", 20, false)}, 0, true},
1503- {"duplicate failed start is rejected", []traceRecord{base("adapter_send_start", 10, false), base("adapter_send_start", 15, false)}, 0, true},
1504- {"negative failed span is rejected", []traceRecord{base("adapter_receive_start", 20, false), base("adapter_receive_end", 10, false)}, 0, true},
1505- {"cross-component failed end is rejected", []traceRecord{base("adapter_send_start", 10, true), func() traceRecord { r := base("adapter_send_end", 20, false); r.Component = "other"; return r }()}, 0, true},
1506- } {
1507- t.Run(tc.name, func(t *testing.T) {
1508- path := filepath.Join(t.TempDir(), "trace.jsonl")
1509- var lines []string
1510- for _, record := range tc.records {
1511- lines = append(lines, mustJSON(record))
1512- }
1513- if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {
1514- t.Fatal(err)
1515- }
1516- spans, err := mergeProcessTraces([]processMapping{{ProcessID: "one", ClockDomain: "one", TracePath: path, Scenario: "s", Run: 1}})
1517- if tc.wantErr {
1518- if err == nil {
1519- t.Fatalf("invalid trace accepted: %+v", tc.records)
1520- }
1521- return
1522- }
1523- if err != nil || len(spans) != tc.wantSpans {
1524- t.Fatalf("spans=%+v err=%v, want %d spans", spans, err, tc.wantSpans)
1525- }
1526- if tc.wantSpans == 1 && spans[0].Samples[0] != 10 {
1527- t.Fatalf("successful span=%+v, want 10ns", spans[0])
1528- }
1529- })
1530- }
1531-}
A cmd/vev-perf-harness/harness_test_3_test.go +54 -0
@@ -0,0 +1,54 @@
1+package main
2+
3+import (
4+ "os"
5+ "path/filepath"
6+ "strings"
7+ "testing"
8+)
9+
10+func TestHarnessExcludesFailedSpanDurationsWhileValidatingPairing(t *testing.T) {
11+ base := func(kind string, tick int64, valid bool) traceRecord {
12+ return traceRecord{ProcessID: "one", Component: "adapter", Scenario: "s", Run: 1, Sequence: 1, RequestID: 1, Epoch: 1, Kind: kind, Tick: tick, Valid: valid}
13+ }
14+ for _, tc := range []struct {
15+ name string
16+ records []traceRecord
17+ wantSpans int
18+ wantErr bool
19+ }{
20+ {"successful adapter send is sampled", []traceRecord{base("adapter_send_start", 10, true), base("adapter_send_end", 20, true)}, 1, false},
21+ {"failed adapter send end is excluded", []traceRecord{base("adapter_send_start", 10, true), base("adapter_send_end", 20, false)}, 0, false},
22+ {"failed adapter receive end is excluded", []traceRecord{base("adapter_receive_start", 10, true), base("adapter_receive_end", 20, false)}, 0, false},
23+ {"failed adapter send start is excluded", []traceRecord{base("adapter_send_start", 10, false), base("adapter_send_end", 20, true)}, 0, false},
24+ {"failed adapter receive boundaries are excluded", []traceRecord{base("adapter_receive_start", 10, false), base("adapter_receive_end", 20, false)}, 0, false},
25+ {"failed end without start is rejected", []traceRecord{base("adapter_send_end", 20, false)}, 0, true},
26+ {"duplicate failed start is rejected", []traceRecord{base("adapter_send_start", 10, false), base("adapter_send_start", 15, false)}, 0, true},
27+ {"negative failed span is rejected", []traceRecord{base("adapter_receive_start", 20, false), base("adapter_receive_end", 10, false)}, 0, true},
28+ {"cross-component failed end is rejected", []traceRecord{base("adapter_send_start", 10, true), func() traceRecord { r := base("adapter_send_end", 20, false); r.Component = "other"; return r }()}, 0, true},
29+ } {
30+ t.Run(tc.name, func(t *testing.T) {
31+ path := filepath.Join(t.TempDir(), "trace.jsonl")
32+ var lines []string
33+ for _, record := range tc.records {
34+ lines = append(lines, mustJSON(record))
35+ }
36+ if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {
37+ t.Fatal(err)
38+ }
39+ spans, err := mergeProcessTraces([]processMapping{{ProcessID: "one", ClockDomain: "one", TracePath: path, Scenario: "s", Run: 1}})
40+ if tc.wantErr {
41+ if err == nil {
42+ t.Fatalf("invalid trace accepted: %+v", tc.records)
43+ }
44+ return
45+ }
46+ if err != nil || len(spans) != tc.wantSpans {
47+ t.Fatalf("spans=%+v err=%v, want %d spans", spans, err, tc.wantSpans)
48+ }
49+ if tc.wantSpans == 1 && spans[0].Samples[0] != 10 {
50+ t.Fatalf("successful span=%+v, want 10ns", spans[0])
51+ }
52+ })
53+ }
54+}
D cmd/vev-perf-harness/main.go +0 -1781
@@ -1,1781 +0,0 @@
1-// vev-perf-harness is a public-CLI performance evidence collector. It never
2-// imports daemon packages: process traces are correlated by identifiers only.
3-package main
4-
5-import (
6- "bufio"
7- "bytes"
8- "context"
9- "encoding/json"
10- "errors"
11- "flag"
12- "fmt"
13- "io"
14- "os"
15- "os/exec"
16- "path/filepath"
17- "runtime/debug"
18- "sort"
19- "strconv"
20- "strings"
21- "sync"
22- "syscall"
23- "time"
24-
25- "github.com/bnema/vev/pkg/safedir"
26- "golang.org/x/sys/unix"
27-)
28-
29-const (
30- minimumDuration = 30 * time.Second
31- minimumRepetitions = 10
32- minimumInIntervalEventSamples = 10
33- measuredEventCadence = time.Second
34-)
35-
36-type options struct {
37- vevBin, manifest, out string
38- scenario string
39- warmup, duration time.Duration
40- repetitions int
41-}
42-
43-type manifest struct {
44- Schema uint16 `json:"schema"`
45- Topologies []topology `json:"topologies"`
46- Workloads []string `json:"workloads"`
47- Transports []transport `json:"transports"`
48- Scenarios []scenario `json:"scenarios"`
49-}
50-type topology struct {
51- ID string `json:"id"`
52- Geometry string `json:"geometry"`
53- RowsPerPane int `json:"rows_per_pane"`
54-}
55-type transport struct {
56- ID string `json:"id"`
57- Kind string `json:"kind"`
58- RTTMS int `json:"rtt_ms,omitempty"`
59- LossPercent int `json:"loss_percent,omitempty"`
60-}
61-type scenario struct {
62- ID string `json:"id"`
63- Topology string `json:"topology"`
64- Workload string `json:"workload"`
65- Transport string `json:"transport"`
66- Roles []string `json:"roles"`
67- InapplicableReason string `json:"inapplicable_reason,omitempty"`
68-}
69-
70-type processMapping struct {
71- ProcessID string `json:"process_id"`
72- ClockDomain string `json:"clock_domain"`
73- TracePath string `json:"trace_path"`
74- Role string `json:"role"`
75- Scenario string `json:"scenario"`
76- Run int `json:"run"`
77- Identity string `json:"identity"`
78-}
79-
80-type runManifest struct {
81- Scenario string `json:"scenario"`
82- Run int `json:"run"`
83- Processes []processMapping `json:"processes"`
84-}
85-type harnessMark struct {
86- Scenario string `json:"scenario"`
87- Run int `json:"run"`
88- Sequence uint64 `json:"sequence"`
89- Kind string `json:"kind"`
90- Tick int64 `json:"tick"`
91- Valid bool `json:"valid"`
92-}
93-
94-// measuredInterval belongs solely to the harness clock domain. A paired event
95-// is eligible only when both owned boundaries fall inside this interval.
96-type measuredInterval struct {
97- Start int64
98- End int64
99-}
100-
101-type eventSample struct {
102- Sequence uint64 `json:"sequence"`
103- Injected int64 `json:"injected_tick"`
104- Flushed int64 `json:"flushed_tick"`
105- Latency int64 `json:"latency_nanos"`
106-}
107-type span struct {
108- Component, Name string
109- Samples []int64
110-}
111-type runResult struct {
112- Spans []span `json:"process_local_spans"`
113- Scenario string `json:"scenario"`
114- Run int `json:"run"`
115- Samples int `json:"samples"`
116- EndToEnd []int64 `json:"end_to_end_nanos"`
117- Event distribution `json:"event_end_to_end"`
118- Cadence distribution `json:"event_cadence_nanos"`
119- CadenceSamples []int64 `json:"-"`
120- MaxGap int64 `json:"event_max_gap_nanos"`
121- Processes []processMapping `json:"processes"`
122-}
123-type summary struct {
124- Schema uint16 `json:"schema"`
125- GitSHA string `json:"git_sha"`
126- Warmup string `json:"warmup"`
127- Duration string `json:"duration"`
128- Repetitions int `json:"repetitions"`
129- EndToEnd distribution `json:"harness_end_to_end"`
130- Cadence distribution `json:"harness_event_cadence_nanos"`
131- MaxGap int64 `json:"harness_event_max_gap_nanos"`
132- RunP50Dispersion distribution `json:"run_p50_dispersion"`
133- Spans []spanSummary `json:"process_local_spans"`
134- Runs int `json:"runs"`
135-}
136-type distribution struct {
137- Count int `json:"count"`
138- P50 int64 `json:"p50"`
139- P95 int64 `json:"p95"`
140- P99 int64 `json:"p99"`
141- Max int64 `json:"max"`
142-}
143-type spanSummary struct {
144- Component string `json:"component"`
145- Name string `json:"name"`
146- Distribution distribution `json:"distribution"`
147-}
148-
149-type clock interface {
150- Now() int64
151- Sleep(time.Duration)
152-}
153-type systemClock struct{ start time.Time }
154-
155-func (c systemClock) Now() int64 { return time.Since(c.start).Nanoseconds() }
156-func (systemClock) Sleep(d time.Duration) { time.Sleep(d) }
157-
158-type launchedProcess interface {
159- Warmup([]byte) error
160- Measure([]byte, func() error, func() error) error
161- Close() error
162-}
163-
164-// roleCommand contains only argv accepted by app.parseArgs. Transport routing
165-// metadata configures the harness's public-command seam; it is never passed as
166-// an unrecognised vev flag.
167-type roleCommand struct {
168- Args []string
169- Transport transport
170-}
171-
172-type launcher interface {
173- Launch(processMapping, roleCommand) (launchedProcess, error)
174-}
175-
176-type launchedRole struct {
177- mapping processMapping
178- process launchedProcess
179-}
180-
181-// closeLaunchedRoles closes every role in reverse launch order. errors.Join
182-// retains every cleanup failure in that stable order.
183-func closeLaunchedRoles(processes []launchedRole) error {
184- var failures []error
185- for i := len(processes) - 1; i >= 0; i-- {
186- if err := processes[i].process.Close(); err != nil {
187- failures = append(failures, fmt.Errorf("close %s role: %w", processes[i].mapping.Role, err))
188- }
189- }
190- return errors.Join(failures...)
191-}
192-
193-// cliLauncher is deliberately limited to executing the supplied public vev
194-// binary. Scenario orchestration never reaches into an in-process daemon.
195-type cliLauncher struct {
196- bin string
197- netemFactory udpNetemFactory
198- mu sync.Mutex
199- peers map[string]peerRoute
200- runtimes map[string]string
201-}
202-
203-type peerRoute struct {
204- mapping processMapping
205- command roleCommand
206- pidPath string
207- netem udpNetem
208-}
209-
210-type preparedPeer struct {
211- route peerRoute
212- cleanupRuntime func() error
213-}
214-
215-func (p preparedPeer) Warmup([]byte) error { return nil }
216-func (p preparedPeer) Measure([]byte, func() error, func() error) error { return nil }
217-func (p preparedPeer) Close() error {
218- var errs []error
219- if p.route.pidPath != "" {
220- b, err := os.ReadFile(p.route.pidPath)
221- if err == nil {
222- pid, parseErr := strconv.Atoi(strings.TrimSpace(string(b)))
223- if parseErr != nil || pid <= 0 {
224- errs = append(errs, fmt.Errorf("invalid owned peer pid: %q", b))
225- } else if killErr := syscall.Kill(pid, syscall.SIGTERM); killErr != nil && !errors.Is(killErr, syscall.ESRCH) {
226- errs = append(errs, killErr)
227- }
228- } else if !errors.Is(err, os.ErrNotExist) {
229- errs = append(errs, err)
230- }
231- }
232- if p.route.netem != nil {
233- if err := p.route.netem.Close(); err != nil {
234- errs = append(errs, err)
235- }
236- }
237- if p.cleanupRuntime != nil {
238- if err := p.cleanupRuntime(); err != nil {
239- errs = append(errs, err)
240- }
241- }
242- return errors.Join(errs...)
243-}
244-
245-type terminalOutput interface {
246- io.Writer
247- Sync() error
248- Close() error
249-}
250-
251-type cliProcess struct {
252- cmd *exec.Cmd
253- pty io.ReadWriteCloser
254- output terminalOutput
255- chunks chan []byte
256- done chan struct{}
257- closed sync.Once
258- waitErr chan error
259- waitTimeout func() <-chan time.Time
260- forceCleanup func()
261- forceKill func()
262- shutdown func() error
263- cleanupRuntime func() error
264- terminalReady bool
265- readyPath string
266-}
267-
268-func (l *cliLauncher) Launch(m processMapping, role roleCommand) (launchedProcess, error) {
269- if len(role.Args) == 0 {
270- return nil, errors.New("public CLI role has no arguments")
271- }
272- if info, err := os.Stat(l.bin); err != nil {
273- return nil, err
274- } else if info.IsDir() {
275- return nil, fmt.Errorf("public CLI binary %q is a directory", l.bin)
276- }
277- if m.Role == "ssh_stdio_peer" || m.Role == "udp_peer" {
278- return l.preparePeer(m, role)
279- }
280- cmd := exec.Command(l.bin, role.Args...)
281- runDir := filepath.Dir(m.TracePath)
282- runtimeDir, err := l.runtimeDir(runDir)
283- if err != nil {
284- return nil, err
285- }
286- // All role inputs are absolute paths resolved at harness startup. Do not set
287- // a role working directory: daemon-owned subprocess cleanup may otherwise
288- // treat the evidence directory as its working tree and remove a preallocated
289- // trace while a later repetition is being merged.
290- cmd.Env = append(withoutEnv(os.Environ(), "VEV", "VEV_PERF_TRACE", "VEV_PERF_PROCESS_ID", "VEV_PERF_SCENARIO", "VEV_PERF_RUN", "VEV_REMOTE_TRANSPORT", "XDG_RUNTIME_DIR", "XDG_STATE_HOME"), traceEnvironment(m)...)
291- cmd.Env = append(cmd.Env, "XDG_RUNTIME_DIR="+runtimeDir,
292- "XDG_STATE_HOME="+filepath.Join(runDir, "state"), "TERM=xterm-256color")
293- if m.Role == "client" {
294- l.mu.Lock()
295- peer, routed := l.peers[runDir]
296- l.mu.Unlock()
297- if routed {
298- mode, err := remoteMode(peer.command.Transport)
299- if err != nil {
300- return nil, err
301- }
302- cmd.Env = append(cmd.Env, "PATH="+runDir+":"+os.Getenv("PATH"), "VEV_REMOTE_TRANSPORT="+mode)
303- }
304- }
305- if err := safedir.EnsurePrivate(filepath.Join(runDir, "state")); err != nil {
306- return nil, err
307- }
308- p := &cliProcess{cmd: cmd, waitErr: make(chan error, 1), readyPath: filepath.Join(runtimeDir, "vev", "daemon.sock")}
309- if m.Role == "daemon" {
310- p.cleanupRuntime = func() error { return l.releaseRuntime(runDir) }
311- }
312- if m.Role == "client" {
313- master, slave, err := openPTY()
314- if err != nil {
315- return nil, err
316- }
317- output, err := os.OpenFile(filepath.Join(filepath.Dir(m.TracePath), "terminal-output"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o600)
318- if err != nil {
319- _ = master.Close()
320- _ = slave.Close()
321- return nil, err
322- }
323- cmd.Stdin, cmd.Stdout, cmd.Stderr = slave, slave, slave
324- // Ctty is the child descriptor number after Start remaps slave to stdin.
325- cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true, Setctty: true, Ctty: 0}
326- if err := cmd.Start(); err != nil {
327- _ = output.Close()
328- _ = master.Close()
329- _ = slave.Close()
330- if p.cleanupRuntime != nil {
331- _ = p.cleanupRuntime()
332- }
333- return nil, err
334- }
335- _ = slave.Close()
336- p.pty, p.output, p.chunks, p.done = master, output, make(chan []byte, 32), make(chan struct{})
337- go p.copyTerminal()
338- } else {
339- // Every launched role owns a dedicated process group, so the bounded
340- // cleanup path never signals the harness/test runner's group.
341- cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
342- devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, 0)
343- if err != nil {
344- return nil, err
345- }
346- cmd.Stdin, cmd.Stdout, cmd.Stderr = devNull, devNull, devNull
347- if err := cmd.Start(); err != nil {
348- _ = devNull.Close()
349- if p.cleanupRuntime != nil {
350- _ = p.cleanupRuntime()
351- }
352- return nil, err
353- }
354- _ = devNull.Close()
355- }
356- if m.Role == "daemon" {
357- // Stop the foreground daemon through its public CLI before its bounded
358- // process-group fallback. This lets blocked carriage operations close and
359- // serialize their failed end marks instead of being cut off by SIGKILL.
360- p.shutdown = func() error {
361- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
362- defer cancel()
363- shutdown := exec.CommandContext(ctx, l.bin, "kill", "--daemon")
364- shutdown.Env = withoutEnv(cmd.Env, "VEV_PERF_TRACE", "VEV_PERF_PROCESS_ID", "VEV_PERF_SCENARIO", "VEV_PERF_RUN")
365- return shutdown.Run()
366- }
367- }
368- go func() { p.waitErr <- cmd.Wait() }()
369- return p, nil
370-}
371-
372-// runtimeDir returns one short, private runtime directory shared by the roles
373-// of one run. Keeping the Unix socket outside deeply nested evidence output
374-// avoids the platform socket-path limit without weakening run isolation.
375-func (l *cliLauncher) runtimeDir(runDir string) (string, error) {
376- l.mu.Lock()
377- defer l.mu.Unlock()
378- if path := l.runtimes[runDir]; path != "" {
379- return path, nil
380- }
381- path, err := os.MkdirTemp("", "vev-harness-runtime-")
382- if err != nil {
383- return "", err
384- }
385- if err := safedir.EnsurePrivate(path); err != nil {
386- _ = os.RemoveAll(path)
387- return "", err
388- }
389- if l.runtimes == nil {
390- l.runtimes = make(map[string]string)
391- }
392- l.runtimes[runDir] = path
393- return path, nil
394-}
395-
396-func (l *cliLauncher) releaseRuntime(runDir string) error {
397- l.mu.Lock()
398- path := l.runtimes[runDir]
399- delete(l.runtimes, runDir)
400- l.mu.Unlock()
401- if path == "" {
402- return nil
403- }
404- return os.RemoveAll(path)
405-}
406-
407-// preparePeer installs a per-run ssh command seam. The ordinary public client
408-// still executes its documented remote attach command; its ssh child invokes
409-// only the parsed public _stdio or _udp-proxy command. This makes the peer
410-// carrying client traffic the declared, exclusively traced role rather than a
411-// separately launched but unused process.
412-func (l *cliLauncher) preparePeer(m processMapping, role roleCommand) (launchedProcess, error) {
413- runDir := filepath.Dir(m.TracePath)
414- l.mu.Lock()
415- runtimeDir := l.runtimes[runDir]
416- l.mu.Unlock()
417- peerOwnsRuntime := runtimeDir == ""
418- if peerOwnsRuntime {
419- var err error
420- runtimeDir, err = l.runtimeDir(runDir)
421- if err != nil {
422- return nil, err
423- }
424- }
425- if err := safedir.EnsurePrivate(runDir); err != nil {
426- return nil, err
427- }
428- if err := safedir.EnsurePrivate(runtimeDir); err != nil {
429- return nil, err
430- }
431- if err := safedir.EnsurePrivate(filepath.Join(runDir, "state")); err != nil {
432- return nil, err
433- }
434- if len(role.Args) != 2 || (role.Args[0] != "_stdio" && role.Args[0] != "_udp-proxy") {
435- return nil, fmt.Errorf("unsupported public peer command %q", role.Args)
436- }
437- route := peerRoute{mapping: m, command: role}
438- if m.Role == "udp_peer" {
439- route.pidPath = filepath.Join(runDir, "udp-peer.pid")
440- factory := l.netemFactory
441- if factory == nil {
442- factory = newUDPNetem
443- }
444- netem, err := factory(udpNetemConfig{
445- RTT: time.Duration(role.Transport.RTTMS) * time.Millisecond,
446- LossPercent: role.Transport.LossPercent,
447- TargetPath: filepath.Join(runDir, "udp-peer.target"),
448- })
449- if err != nil {
450- return nil, fmt.Errorf("start harness UDP netem: %w", err)
451- }
452- route.netem = netem
453- }
454- shim := filepath.Join(runDir, "ssh")
455- ready := filepath.Join(runDir, "udp-peer.ready")
456- target := filepath.Join(runDir, "udp-peer.target")
457- netemPort := 0
458- if route.netem != nil {
459- netemPort = route.netem.Port()
460- }
461- stderr := filepath.Join(runDir, "udp-peer.stderr")
462- // The bootstrap protocol requires the SSH command to exit after forwarding
463- // the readiness line. The long-lived proxy is owned by pidPath and receives
464- // the role's unique trace identity, never the bootstrap/client identity.
465- body := fmt.Sprintf(`#!/bin/sh
466-set -eu
467-case "$*" in
468- *"_stdio"*)
469- exec env VEV_PERF_TRACE=%[1]q VEV_PERF_PROCESS_ID=%[2]q VEV_PERF_SCENARIO=%[12]q VEV_PERF_RUN=%[13]d XDG_RUNTIME_DIR=%[3]q XDG_STATE_HOME=%[4]q TERM=xterm-256color %[5]q _stdio %[6]q
470- ;;
471- *"_udp-bootstrap"*)
472- rm -f %[7]q %[8]q %[9]q
473- env VEV_PERF_TRACE=%[1]q VEV_PERF_PROCESS_ID=%[2]q VEV_PERF_SCENARIO=%[12]q VEV_PERF_RUN=%[13]d XDG_RUNTIME_DIR=%[3]q XDG_STATE_HOME=%[4]q TERM=xterm-256color %[5]q _udp-proxy %[6]q >%[7]q 2>%[8]q &
474- peer=$!
475- printf '%%s\n' "$peer" > %[11]q
476- i=0
477- while [ ! -s %[7]q ] && [ "$i" -lt 1000 ]; do sleep 0.01; i=$((i+1)); done
478- test -s %[7]q
479- line=$(head -n 1 %[7]q)
480- printf '%%s\n' "$line" > %[9]q
481- set -- $line
482- test "$1" = VEV-UDP
483- test -n "$3"
484- printf 'VEV-UDP %%s %%s\n' %[10]d "$3"
485- exit 0
486- ;;
487- *) echo 'vev harness ssh seam rejected non-vev command' >&2; exit 64 ;;
488-esac
489-`, m.TracePath, m.ProcessID, runtimeDir, filepath.Join(runDir, "state"), l.bin, role.Args[1], ready, stderr, target, netemPort, route.pidPath, m.Scenario, m.Run)
490- if err := os.WriteFile(shim, []byte(body), 0o700); err != nil {
491- if route.netem != nil {
492- _ = route.netem.Close()
493- }
494- return nil, err
495- }
496- l.mu.Lock()
497- if l.peers == nil {
498- l.peers = make(map[string]peerRoute)
499- }
500- if _, exists := l.peers[runDir]; exists {
501- l.mu.Unlock()
502- if route.netem != nil {
503- _ = route.netem.Close()
504- }
505- return nil, fmt.Errorf("duplicate transport peer for %s", runDir)
506- }
507- l.peers[runDir] = route
508- l.mu.Unlock()
509- peer := preparedPeer{route: route}
510- if peerOwnsRuntime {
511- peer.cleanupRuntime = func() error { return l.releaseRuntime(runDir) }
512- }
513- return peer, nil
514-}
515-
516-func remoteMode(t transport) (string, error) {
517- switch t.Kind {
518- case "ssh_stdio":
519- return "stdio", nil
520- case "udp":
521- return "udp", nil
522- default:
523- return "", fmt.Errorf("transport %q cannot route through a remote peer", t.ID)
524- }
525-}
526-
527-// traceEnvironment is the complete process-local identity supplied by the
528-// persisted harness processMapping. App turns Scenario/Run into production
529-// mark correlation; sequence/request/epoch remain process-local operation IDs.
530-func traceEnvironment(m processMapping) []string {
531- return []string{
532- "VEV_PERF_TRACE=" + m.TracePath,
533- "VEV_PERF_PROCESS_ID=" + m.ProcessID,
534- "VEV_PERF_SCENARIO=" + m.Scenario,
535- "VEV_PERF_RUN=" + strconv.Itoa(m.Run),
536- }
537-}
538-
539-func withoutEnv(env []string, names ...string) []string {
540- remove := make(map[string]bool, len(names))
541- for _, name := range names {
542- remove[name] = true
543- }
544- out := make([]string, 0, len(env))
545- for _, entry := range env {
546- name, _, _ := strings.Cut(entry, "=")
547- if !remove[name] {
548- out = append(out, entry)
549- }
550- }
551- return out
552-}
553-
554-func (p *cliProcess) copyTerminal() {
555- defer close(p.chunks)
556- buf := make([]byte, 4096)
557- for {
558- n, err := p.pty.Read(buf)
559- if n > 0 {
560- chunk := append([]byte(nil), buf[:n]...)
561- if _, writeErr := p.output.Write(chunk); writeErr != nil {
562- return
563- }
564- select {
565- case p.chunks <- chunk:
566- case <-p.done:
567- return
568- }
569- }
570- if err != nil {
571- return
572- }
573- }
574-}
575-
576-// WaitReady confirms the explicitly launched public daemon owns its unique
577-// socket before a client is launched. Without this barrier, the normal client
578-// auto-spawn behavior can create an unmanifested daemon process during startup.
579-func (p *cliProcess) WaitReady() error {
580- if p.readyPath == "" {
581- return errors.New("daemon readiness path is empty")
582- }
583- ticker := time.NewTicker(10 * time.Millisecond)
584- defer ticker.Stop()
585- timer := time.NewTimer(10 * time.Second)
586- defer timer.Stop()
587- for {
588- if info, err := os.Stat(p.readyPath); err == nil {
589- if info.Mode()&os.ModeSocket == 0 {
590- return fmt.Errorf("daemon readiness path %s is not a socket", p.readyPath)
591- }
592- return nil
593- }
594- select {
595- case err := <-p.waitErr:
596- p.waitErr <- err
597- if err == nil {
598- return errors.New("daemon exited before readiness")
599- }
600- return fmt.Errorf("daemon exited before readiness: %w", err)
601- case <-ticker.C:
602- case <-timer.C:
603- return fmt.Errorf("timed out waiting for daemon socket %s", p.readyPath)
604- }
605- }
606-}
607-
608-func (p *cliProcess) Warmup(input []byte) error {
609- if p.pty == nil || len(input) == 0 {
610- return nil
611- }
612- // A client cannot write application output until it has received Welcome and
613- // entered raw mode. Require that observable output before submitting any
614- // warmup input, so the PTY's line discipline cannot echo it locally.
615- if !p.terminalReady {
616- if err := p.waitForTerminalReady(); err != nil {
617- return err
618- }
619- p.terminalReady = true
620- }
621- if _, err := p.pty.Write(input); err != nil {
622- return err
623- }
624- // Do not begin measured input until the public client has rendered a real
625- // shell response to warmup input. This is intentionally not a measurement.
626- return p.waitForTerminalMarker(inputMarker(input), input)
627-}
628-
629-// Measure records a boundary only after the harness-owned terminal output file
630-// has accepted application output and Sync has reported a successful flush.
631-// The marker is injected through the real client PTY, never fabricated by the
632-// harness. PTY local echo is discarded and cannot satisfy this boundary.
633-func (p *cliProcess) Measure(input []byte, injected, flushed func() error) error {
634- if p.pty == nil {
635- return errors.New("measured process has no terminal PTY")
636- }
637- if _, err := p.pty.Write(input); err != nil {
638- return err
639- }
640- // The injected callback runs only after the owned PTY accepts the bytes.
641- if err := injected(); err != nil {
642- return err
643- }
644- if err := p.waitForTerminalMarker(inputMarker(input), input); err != nil {
645- return err
646- }
647- if err := p.output.Sync(); err != nil {
648- return fmt.Errorf("flush terminal output: %w", err)
649- }
650- return flushed()
651-}
652-
653-// waitForTerminalReady waits for public client output produced before any
654-// harness input. The client enters raw mode before its output loop can render
655-// this initial prompt/state, so this is an observable readiness boundary.
656-func (p *cliProcess) waitForTerminalReady() error {
657- timer := time.NewTimer(10 * time.Second)
658- defer timer.Stop()
659- for {
660- select {
661- case chunk, ok := <-p.chunks:
662- if !ok {
663- return p.terminalClosedError("readiness output")
664- }
665- if len(chunk) == 0 {
666- continue
667- }
668- if err := p.output.Sync(); err != nil {
669- return fmt.Errorf("flush terminal readiness output: %w", err)
670- }
671- return nil
672- case err := <-p.waitErr:
673- // Keep the exit result available to Close, which owns reaping.
674- p.waitErr <- err
675- if err == nil {
676- return errors.New("client exited before readiness output")
677- }
678- return fmt.Errorf("client exited before readiness output: %w", err)
679- case <-timer.C:
680- return errors.New("timed out waiting for client readiness output")
681- }
682- }
683-}
684-
685-func (p *cliProcess) terminalClosedError(boundary string) error {
686- if p.waitErr != nil {
687- select {
688- case err := <-p.waitErr:
689- // Close owns reaping and may need the result after a failed boundary.
690- p.waitErr <- err
691- if err != nil {
692- return fmt.Errorf("client terminal closed before %s: %w", boundary, err)
693- }
694- case <-time.After(10 * time.Millisecond):
695- }
696- }
697- return fmt.Errorf("client terminal closed before %s", boundary)
698-}
699-
700-func (p *cliProcess) waitForTerminalMarker(marker, input []byte) error {
701- var received []byte
702- echo := newPTYLocalEcho(input)
703- timer := time.NewTimer(10 * time.Second)
704- defer timer.Stop()
705- for !bytes.Contains(received, marker) {
706- select {
707- case chunk, ok := <-p.chunks:
708- if !ok {
709- return p.terminalClosedError("application output")
710- }
711- received = append(received, echo.applicationOutput(chunk)...)
712- case err := <-p.waitErr:
713- // Keep the exit result available to Close, which owns reaping.
714- p.waitErr <- err
715- if err == nil {
716- return errors.New("client exited before application output")
717- }
718- return fmt.Errorf("client exited before application output: %w", err)
719- case <-timer.C:
720- return errors.New("timed out waiting for application terminal output")
721- }
722- }
723- return nil
724-}
725-
726-// ptyLocalEcho removes the one ordered input echo that a not-yet-raw slave PTY
727-// can produce. It normalizes NL to the conventional ONLCR echo spelling; any
728-// divergence is retained as application output. The line discipline emits the
729-// echo before the child can read and respond to the same write.
730-type ptyLocalEcho struct {
731- expected []byte
732- matched int
733- done bool
734-}
735-
736-func newPTYLocalEcho(input []byte) *ptyLocalEcho {
737- expected := make([]byte, 0, len(input))
738- for _, b := range input {
739- if b == '\n' {
740- expected = append(expected, '\r')
741- }
742- expected = append(expected, b)
743- }
744- return &ptyLocalEcho{expected: expected}
745-}
746-
747-func (e *ptyLocalEcho) applicationOutput(chunk []byte) []byte {
748- if e.done || len(e.expected) == 0 {
749- return chunk
750- }
751- for i, b := range chunk {
752- if b != e.expected[e.matched] {
753- e.done = true
754- return append([]byte(nil), chunk[i:]...)
755- }
756- e.matched++
757- if e.matched == len(e.expected) {
758- e.done = true
759- return append([]byte(nil), chunk[i+1:]...)
760- }
761- }
762- return nil
763-}
764-
765-func (p *cliProcess) Close() error {
766- var result error
767- p.closed.Do(func() {
768- if p.done != nil {
769- close(p.done)
770- }
771- // A public terminal client owns a shell session. Ask that shell to exit
772- // before closing the PTY so its transport can finish its receive span;
773- // this avoids turning ordinary harness teardown into an aborted trace.
774- exited := false
775- if p.shutdown != nil && p.waitErr != nil {
776- select {
777- case <-p.waitErr:
778- exited = true
779- default:
780- // The client may have just ended the last session and raced this
781- // best-effort public shutdown. The normal bounded reaping path below
782- // remains authoritative in that case.
783- _ = p.shutdown()
784- }
785- }
786- if p.pty != nil && p.waitErr != nil {
787- _, _ = p.pty.Write([]byte("exit\n"))
788- select {
789- case <-p.waitErr:
790- exited = true
791- case <-p.closeDeadline():
792- }
793- }
794- if p.pty != nil {
795- _ = p.pty.Close()
796- }
797- if p.waitErr != nil && !exited {
798- select {
799- case <-p.waitErr:
800- case <-p.closeDeadline():
801- // Teardown is deliberately bounded: a stuck public CLI or child
802- // must not hold a local harness run indefinitely.
803- p.forceProcessGroupCleanup()
804- select {
805- case <-p.waitErr:
806- case <-p.closeDeadline():
807- p.forceProcessGroupKill()
808- select {
809- case <-p.waitErr:
810- case <-p.closeDeadline():
811- result = errors.New("timed out reaping public CLI process")
812- }
813- }
814- }
815- }
816- if p.output != nil {
817- if err := p.output.Close(); result == nil {
818- result = err
819- }
820- }
821- if p.cleanupRuntime != nil {
822- if err := p.cleanupRuntime(); result == nil {
823- result = err
824- }
825- }
826- })
827- return result
828-}
829-
830-func (p *cliProcess) closeDeadline() <-chan time.Time {
831- if p.waitTimeout != nil {
832- return p.waitTimeout()
833- }
834- if p.shutdown != nil {
835- // A public kill asks the daemon to finish bounded detached notices and
836- // close blocked carriage before it exits; do not turn that graceful
837- // protocol into a SIGKILL at the two-second client deadline.
838- return time.After(5 * time.Second)
839- }
840- return time.After(2 * time.Second)
841-}
842-
843-func (p *cliProcess) forceProcessGroupCleanup() {
844- if p.forceCleanup != nil {
845- p.forceCleanup()
846- return
847- }
848- if p.cmd != nil && p.cmd.Process != nil {
849- // The client is a session/process-group leader; forced cleanup must
850- // include its ssh seam/_stdio descendants.
851- _ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGTERM)
852- }
853-}
854-
855-func (p *cliProcess) forceProcessGroupKill() {
856- if p.forceKill != nil {
857- p.forceKill()
858- return
859- }
860- if p.cmd != nil && p.cmd.Process != nil {
861- _ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGKILL)
862- }
863-}
864-
865-// openPTY is the small Linux PTY boundary needed to drive the public terminal
866-// client. No daemon or adapter implementation is imported by this command.
867-func openPTY() (*os.File, *os.File, error) {
868- masterFD, err := unix.Open("/dev/ptmx", unix.O_RDWR|unix.O_NOCTTY, 0)
869- if err != nil {
870- return nil, nil, err
871- }
872- closeMaster := func(err error) (*os.File, *os.File, error) { _ = unix.Close(masterFD); return nil, nil, err }
873- number, err := unix.IoctlGetInt(masterFD, unix.TIOCGPTN)
874- if err != nil {
875- return closeMaster(err)
876- }
877- if err := unix.IoctlSetPointerInt(masterFD, unix.TIOCSPTLCK, 0); err != nil {
878- return closeMaster(err)
879- }
880- slaveFD, err := unix.Open(fmt.Sprintf("/dev/pts/%d", number), unix.O_RDWR|unix.O_NOCTTY, 0)
881- if err != nil {
882- return closeMaster(err)
883- }
884- return os.NewFile(uintptr(masterFD), "vev-client-master"), os.NewFile(uintptr(slaveFD), "vev-client-slave"), nil
885-}
886-
887-func inputMarker(input []byte) []byte {
888- start := bytes.Index(input, []byte("__VEV_HARNESS_"))
889- if start < 0 {
890- return input
891- }
892- end := bytes.Index(input[start:], []byte(`\n`))
893- if end < 0 {
894- return input[start:]
895- }
896- return input[start : start+end]
897-}
898-
899-type harness struct {
900- clock clock
901- launcher launcher
902- mkdir func(string) error
903- createRunDir func(string) error
904- create func(string) (*os.File, error)
905- removeAll func(string) error
906- gitSHA func() string
907- selectScenarios func(manifest) []scenario // test-only bounded fixture seam
908-}
909-
910-func main() {
911- if err := run(os.Args[1:], defaultHarness()); err != nil {
912- fmt.Fprintln(os.Stderr, "vev-perf-harness:", err)
913- os.Exit(2)
914- }
915-}
916-func defaultHarness() *harness {
917- return &harness{clock: systemClock{time.Now()}, mkdir: safedir.EnsurePrivate, createRunDir: createExclusiveRunDir, create: func(p string) (*os.File, error) { return os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) }, removeAll: os.RemoveAll, gitSHA: recordedGitSHA}
918-}
919-
920-// createExclusiveRunDir prevents stale runtime, socket, state, or trace files
921-// from an earlier invocation being mistaken for the next repetition's evidence.
922-func createExclusiveRunDir(path string) error {
923- if err := os.Mkdir(path, 0o700); err != nil {
924- return err
925- }
926- if err := safedir.EnsurePrivate(path); err != nil {
927- _ = os.Remove(path)
928- return err
929- }
930- return nil
931-}
932-
933-// recordedGitSHA prefers build provenance so an installed public harness does
934-// not depend on its launch directory being a checkout. The git fallback keeps
935-// go run evidence attributable as well.
936-func recordedGitSHA() string {
937- if info, ok := debug.ReadBuildInfo(); ok {
938- for _, setting := range info.Settings {
939- if setting.Key == "vcs.revision" && setting.Value != "" {
940- return setting.Value
941- }
942- }
943- }
944- out, err := exec.Command("git", "rev-parse", "HEAD").Output()
945- if err == nil && strings.TrimSpace(string(out)) != "" {
946- return strings.TrimSpace(string(out))
947- }
948- return "unknown"
949-}
950-func run(args []string, h *harness) error {
951- opt, err := parseOptions(args)
952- if err != nil {
953- return err
954- }
955- opt, err = resolvePathOptions(opt)
956- if err != nil {
957- return err
958- }
959- m, err := readManifest(opt.manifest)
960- if err != nil {
961- return err
962- }
963- if err := validateManifest(m); err != nil {
964- return err
965- }
966- if err = h.mkdir(opt.out); err != nil {
967- return err
968- }
969- raw, err := os.OpenFile(filepath.Join(opt.out, "raw-harness.jsonl"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
970- if err != nil {
971- return err
972- }
973- defer raw.Close()
974- results := make([]runResult, 0, len(m.Scenarios)*opt.repetitions)
975- var localSpans []span
976- scenarios := m.Scenarios
977- if opt.scenario != "" {
978- scenarios, err = selectScenario(m, opt.scenario)
979- if err != nil {
980- return err
981- }
982- }
983- if h.selectScenarios != nil {
984- scenarios = h.selectScenarios(m)
985- }
986- for _, s := range scenarios {
987- if s.InapplicableReason != "" {
988- continue
989- }
990- for r := 1; r <= opt.repetitions; r++ {
991- rr, err := h.runOne(opt, m, s, r, raw)
992- if err != nil {
993- return fmt.Errorf("scenario %s run %d: %w", s.ID, r, err)
994- }
995- results = append(results, rr)
996- localSpans = append(localSpans, rr.Spans...)
997- }
998- }
999- all, cadence, maxGap, runP50s := aggregateEventResults(results)
1000- if len(all) == 0 {
1001- return errors.New("no measured harness input/flush pairs")
1002- }
1003- if err := writeJSON(filepath.Join(opt.out, "runs.json"), results); err != nil {
1004- return err
1005- }
1006- return writeJSON(filepath.Join(opt.out, "summary.json"), summary{Schema: 1, GitSHA: h.gitSHA(), Warmup: opt.warmup.String(), Duration: opt.duration.String(), Repetitions: opt.repetitions, EndToEnd: percentiles(all), Cadence: percentiles(cadence), MaxGap: maxGap, RunP50Dispersion: percentiles(runP50s), Spans: summarizeSpans(localSpans), Runs: len(results)})
1007-}
1008-
1009-// aggregateEventResults deliberately preserves raw event latencies for
1010-// end-to-end percentiles. Per-run p50s are a separate dispersion series.
1011-func aggregateEventResults(results []runResult) (all, cadence []int64, maxGap int64, runP50s []int64) {
1012- for _, result := range results {
1013- all = append(all, result.EndToEnd...)
1014- cadence = append(cadence, result.CadenceSamples...)
1015- runP50s = append(runP50s, result.Event.P50)
1016- if result.MaxGap > maxGap {
1017- maxGap = result.MaxGap
1018- }
1019- }
1020- return all, cadence, maxGap, runP50s
1021-}
1022-
1023-func parseOptions(args []string) (options, error) {
1024- var o options
1025- fs := flag.NewFlagSet("vev-perf-harness", flag.ContinueOnError)
1026- fs.SetOutput(io.Discard)
1027- fs.StringVar(&o.vevBin, "vev-bin", "", "public vev binary")
1028- fs.StringVar(&o.manifest, "manifest", "", "manifest")
1029- fs.StringVar(&o.out, "out", "", "output directory")
1030- fs.StringVar(&o.scenario, "scenario", "", "one canonical scenario ID (after full manifest validation)")
1031- fs.DurationVar(&o.warmup, "warmup", 0, "warmup")
1032- fs.DurationVar(&o.duration, "duration", 0, "measurement duration")
1033- fs.IntVar(&o.repetitions, "repetitions", 0, "repetitions")
1034- if err := fs.Parse(args); err != nil {
1035- return o, err
1036- }
1037- seen := map[string]bool{}
1038- fs.Visit(func(f *flag.Flag) { seen[f.Name] = true })
1039- for _, name := range []string{"vev-bin", "manifest", "out", "warmup", "duration", "repetitions"} {
1040- if !seen[name] {
1041- return o, fmt.Errorf("--%s is required", name)
1042- }
1043- }
1044- if o.vevBin == "" || o.manifest == "" || o.out == "" {
1045- return o, errors.New("--vev-bin, --manifest, and --out are required")
1046- }
1047- if o.warmup < 0 {
1048- return o, errors.New("--warmup must not be negative")
1049- }
1050- if o.duration < minimumDuration {
1051- return o, fmt.Errorf("--duration must be at least %s", minimumDuration)
1052- }
1053- if o.repetitions < minimumRepetitions {
1054- return o, fmt.Errorf("--repetitions must be at least %d", minimumRepetitions)
1055- }
1056- return o, nil
1057-}
1058-
1059-// resolvePathOptions captures the harness cwd once, before role launchers
1060-// assign their isolated working directories. All downstream filesystem access,
1061-// manifests, and process environments consequently refer to the same paths.
1062-func resolvePathOptions(o options) (options, error) {
1063- cwd, err := os.Getwd()
1064- if err != nil {
1065- return o, fmt.Errorf("resolve harness working directory: %w", err)
1066- }
1067- absolute := func(path string) string {
1068- if filepath.IsAbs(path) {
1069- return filepath.Clean(path)
1070- }
1071- return filepath.Clean(filepath.Join(cwd, path))
1072- }
1073- o.vevBin = absolute(o.vevBin)
1074- o.manifest = absolute(o.manifest)
1075- o.out = absolute(o.out)
1076- return o, nil
1077-}
1078-
1079-func readManifest(path string) (manifest, error) {
1080- f, e := os.Open(path)
1081- if e != nil {
1082- return manifest{}, e
1083- }
1084- defer f.Close()
1085- var m manifest
1086- e = json.NewDecoder(f).Decode(&m)
1087- return m, e
1088-}
1089-func validateManifest(m manifest) error {
1090- if m.Schema != 1 {
1091- return errors.New("unsupported manifest schema")
1092- }
1093- tops := map[string]bool{}
1094- for _, t := range m.Topologies {
1095- if t.ID == "" || t.Geometry != "120x40" || t.RowsPerPane != 10000 {
1096- return fmt.Errorf("invalid topology %q", t.ID)
1097- }
1098- tops[t.ID] = true
1099- }
1100- wantT := []string{"1x4", "4x1", "4x4", "8x1"}
1101- for _, v := range wantT {
1102- if !tops[v] {
1103- return fmt.Errorf("missing canonical topology %s", v)
1104- }
1105- }
1106- works := set(m.Workloads)
1107- for _, v := range []string{"idle", "active_output", "all_output", "inactive_output", "interactive_flood", "copy_search", "resize_sweep", "snapshot_output_resize", "attach_restore_tab_switch"} {
1108- if !works[v] {
1109- return fmt.Errorf("missing canonical workload %s", v)
1110- }
1111- }
1112- trans := map[string]bool{}
1113- for _, t := range m.Transports {
1114- if t.ID == "" {
1115- return errors.New("transport missing id")
1116- }
1117- if err := validateTransportFixture(t); err != nil {
1118- return err
1119- }
1120- trans[t.ID] = true
1121- }
1122- for _, v := range []string{"local", "ssh_stdio", "udp_baseline", "udp_25ms", "udp_100ms", "udp_loss_0pct", "udp_loss_1pct"} {
1123- if !trans[v] {
1124- return fmt.Errorf("missing canonical transport %s", v)
1125- }
1126- }
1127- seen := map[string]bool{}
1128- covered := map[string]bool{}
1129- for _, s := range m.Scenarios {
1130- if s.ID == "" || seen[s.ID] {
1131- return fmt.Errorf("duplicate or empty scenario id %q", s.ID)
1132- }
1133- seen[s.ID] = true
1134- if !tops[s.Topology] || !works[s.Workload] || !trans[s.Transport] {
1135- return fmt.Errorf("scenario %s references unknown matrix entry", s.ID)
1136- }
1137- if s.InapplicableReason == "" && len(s.Roles) == 0 {
1138- return fmt.Errorf("scenario %s has no process roles", s.ID)
1139- }
1140- if s.InapplicableReason != "" && len(s.Roles) != 0 {
1141- return fmt.Errorf("inapplicable scenario %s must not launch roles", s.ID)
1142- }
1143- if s.InapplicableReason == "" && !equalRoleSet(s.Roles, requiredRoles(s.Transport)) {
1144- return fmt.Errorf("scenario %s roles do not route transport %s", s.ID, s.Transport)
1145- }
1146- covered[s.Topology+"\x00"+s.Workload+"\x00"+s.Transport] = true
1147- }
1148- for topologyID := range tops {
1149- for workload := range works {
1150- for transportID := range trans {
1151- key := topologyID + "\x00" + workload + "\x00" + transportID
1152- if !covered[key] {
1153- return fmt.Errorf("missing scenario or inapplicable reason for %s/%s/%s", topologyID, workload, transportID)
1154- }
1155- }
1156- }
1157- }
1158- return nil
1159-}
1160-
1161-// selectScenario is intentionally called only after validateManifest. A bounded
1162-// local evidence run therefore cannot mask an incomplete canonical matrix.
1163-func selectScenario(m manifest, id string) ([]scenario, error) {
1164- for _, s := range m.Scenarios {
1165- if s.ID != id {
1166- continue
1167- }
1168- if s.InapplicableReason != "" {
1169- return nil, fmt.Errorf("scenario %s is inapplicable: %s", id, s.InapplicableReason)
1170- }
1171- return []scenario{s}, nil
1172- }
1173- return nil, fmt.Errorf("unknown canonical scenario %s", id)
1174-}
1175-
1176-func requiredRoles(transportID string) []string {
1177- switch transportID {
1178- case "local":
1179- return []string{"daemon", "client"}
1180- case "ssh_stdio":
1181- return []string{"daemon", "client", "ssh_stdio_peer"}
1182- default:
1183- return []string{"daemon", "client", "udp_peer"}
1184- }
1185-}
1186-
1187-func equalRoleSet(got, want []string) bool {
1188- if len(got) != len(want) {
1189- return false
1190- }
1191- seen := make(map[string]bool, len(got))
1192- for _, role := range got {
1193- seen[role] = true
1194- }
1195- for _, role := range want {
1196- if !seen[role] {
1197- return false
1198- }
1199- }
1200- return true
1201-}
1202-
1203-func validateTransportFixture(t transport) error {
1204- want := map[string]transport{
1205- "local": {ID: "local", Kind: "local"},
1206- "ssh_stdio": {ID: "ssh_stdio", Kind: "ssh_stdio"},
1207- "udp_baseline": {ID: "udp_baseline", Kind: "udp"},
1208- "udp_25ms": {ID: "udp_25ms", Kind: "udp", RTTMS: 25},
1209- "udp_100ms": {ID: "udp_100ms", Kind: "udp", RTTMS: 100},
1210- "udp_loss_0pct": {ID: "udp_loss_0pct", Kind: "udp", LossPercent: 0},
1211- "udp_loss_1pct": {ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1},
1212- }
1213- w, ok := want[t.ID]
1214- if !ok || t.Kind != w.Kind || t.RTTMS != w.RTTMS || t.LossPercent != w.LossPercent {
1215- return fmt.Errorf("invalid canonical transport fixture %+v", t)
1216- }
1217- return nil
1218-}
1219-
1220-func set(v []string) map[string]bool {
1221- r := map[string]bool{}
1222- for _, x := range v {
1223- r[x] = true
1224- }
1225- return r
1226-}
1227-func (h *harness) runOne(o options, mat manifest, s scenario, run int, raw io.Writer) (res runResult, err error) {
1228- dir := filepath.Join(o.out, fmt.Sprintf("%s-run-%03d", safeName(s.ID), run))
1229- createRunDir := h.createRunDir
1230- if createRunDir == nil {
1231- createRunDir = createExclusiveRunDir
1232- }
1233- if err = createRunDir(dir); err != nil {
1234- return res, fmt.Errorf("create isolated run directory: %w", err)
1235- }
1236- success := false
1237- defer func() {
1238- if !success {
1239- _ = h.removeAll(dir)
1240- }
1241- }()
1242- maps := make([]processMapping, 0, len(s.Roles))
1243- for i, role := range s.Roles {
1244- pid := fmt.Sprintf("%s-r%03d-%s-%02d", safeName(s.ID), run, safeName(role), i+1)
1245- path := filepath.Join(dir, pid+".jsonl")
1246- f, e := h.create(path)
1247- if e != nil {
1248- return res, fmt.Errorf("allocate exclusive trace %s: %w", role, e)
1249- }
1250- if e = f.Close(); e != nil {
1251- return res, e
1252- }
1253- maps = append(maps, processMapping{ProcessID: pid, ClockDomain: pid, TracePath: path, Role: role, Scenario: s.ID, Run: run, Identity: "harness-assigned:" + pid})
1254- }
1255- // Persist the complete mapping before any process can be launched. This is
1256- // both crash evidence and the only legal source for later ID correlation.
1257- if err = writeJSON(filepath.Join(dir, "manifest.json"), runManifest{Scenario: s.ID, Run: run, Processes: maps}); err != nil {
1258- return res, err
1259- }
1260- if h.launcher == nil {
1261- h.launcher = &cliLauncher{bin: o.vevBin}
1262- }
1263- processes := make([]launchedRole, 0, len(maps))
1264- closed := false
1265- closeRoles := func() error {
1266- if closed {
1267- return nil
1268- }
1269- closed = true
1270- // Clients own SSH-stdio descendants; close them before the deferred UDP
1271- // peer and daemon so no role survives a failed or completed run.
1272- return closeLaunchedRoles(processes)
1273- }
1274- defer func() {
1275- if cleanupErr := closeRoles(); cleanupErr != nil {
1276- if err == nil {
1277- err = cleanupErr
1278- } else {
1279- // Keep the primary workload failure first while retaining every
1280- // cleanup failure in reverse launch order.
1281- err = errors.Join(err, cleanupErr)
1282- }
1283- }
1284- }()
1285- selectedTransport, e := manifestTransport(mat, s.Transport)
1286- if e != nil {
1287- return res, e
1288- }
1289- for _, pm := range launchOrder(maps) {
1290- p, e := h.launcher.Launch(pm, routeRoleArgs(s, pm, selectedTransport))
1291- if e != nil {
1292- return res, e
1293- }
1294- processes = append(processes, launchedRole{mapping: pm, process: p})
1295- if pm.Role == "daemon" {
1296- if ready, ok := p.(interface{ WaitReady() error }); ok {
1297- if err = ready.WaitReady(); err != nil {
1298- return res, err
1299- }
1300- }
1301- }
1302- }
1303- for _, p := range processes {
1304- if p.mapping.Role != "client" {
1305- continue
1306- }
1307- if err = p.process.Warmup(workloadInput(s, run, "warmup")); err != nil {
1308- return res, err
1309- }
1310- }
1311- // The harness clock, rather than a process clock, owns the explicit warmup
1312- // and measured intervals. Warmup boundaries are intentionally never added to
1313- // marks, and each measured event must complete before interval.End to count.
1314- h.clock.Sleep(o.warmup)
1315- interval := measuredInterval{Start: h.clock.Now()}
1316- interval.End = interval.Start + o.duration.Nanoseconds()
1317- marks := []harnessMark{}
1318- seq := uint64(0)
1319- nextInjection := interval.Start
1320- for {
1321- now := h.clock.Now()
1322- if now < nextInjection {
1323- h.clock.Sleep(time.Duration(nextInjection - now))
1324- now = h.clock.Now()
1325- }
1326- if now >= interval.End {
1327- break
1328- }
1329- for _, p := range processes {
1330- if p.mapping.Role != "client" {
1331- continue
1332- }
1333- seq++
1334- sequence := seq
1335- injected := func() error {
1336- marks = append(marks, harnessMark{s.ID, run, sequence, "input_injected", h.clock.Now(), true})
1337- return nil
1338- }
1339- flushed := func() error {
1340- marks = append(marks, harnessMark{s.ID, run, sequence, "terminal_flushed", h.clock.Now(), true})
1341- return nil
1342- }
1343- if err = p.process.Measure(workloadInput(s, run, fmt.Sprintf("measured-%d", sequence)), injected, flushed); err != nil {
1344- return res, err
1345- }
1346- }
1347- // Cap injection rate rather than bursting events after a slow terminal
1348- // flush. Any resulting spacing is retained as raw cadence evidence.
1349- nextInjection = h.clock.Now() + measuredEventCadence.Nanoseconds()
1350- }
1351- if now := h.clock.Now(); now < interval.End {
1352- h.clock.Sleep(time.Duration(interval.End - now))
1353- }
1354- events, e := measuredEventSamples(marks, interval)
1355- if e != nil {
1356- return res, e
1357- }
1358- if err := requireMinimumEventSamples(events); err != nil {
1359- return res, err
1360- }
1361- samples := eventLatencies(events)
1362- cadenceSamples, maxGap := eventCadenceSamples(events)
1363- // Process teardown can unblock a receive and cause its failed end mark to
1364- // be serialized. Merge only the complete, post-cleanup per-process traces.
1365- if cleanupErr := closeRoles(); cleanupErr != nil {
1366- return res, cleanupErr
1367- }
1368- spans, e := mergeProcessTraces(maps)
1369- if e != nil {
1370- return res, e
1371- }
1372- inInterval := make(map[uint64]bool, len(events))
1373- for _, event := range events {
1374- inInterval[event.Sequence] = true
1375- }
1376- for _, m := range marks {
1377- if !inInterval[m.Sequence] {
1378- continue
1379- }
1380- if _, e := fmt.Fprintf(raw, "%s\n", mustJSON(m)); e != nil {
1381- return res, e
1382- }
1383- }
1384- success = true
1385- return runResult{Spans: spans, Scenario: s.ID, Run: run, Samples: len(samples), EndToEnd: samples, Event: percentiles(samples), Cadence: percentiles(cadenceSamples), CadenceSamples: cadenceSamples, MaxGap: maxGap, Processes: maps}, nil
1386-}
1387-
1388-func roleArgs(s scenario, m processMapping) roleCommand {
1389- return routeRoleArgs(s, m, scenarioTransport(s))
1390-}
1391-
1392-func routeRoleArgs(s scenario, m processMapping, selected transport) roleCommand {
1393- session := "perf-" + safeName(s.ID) + fmt.Sprintf("-%03d", m.Run)
1394- remote := []string{"attach", "harness@127.0.0.1:" + session}
1395- switch m.Role {
1396- case "daemon":
1397- return roleCommand{Args: []string{"--daemon"}}
1398- case "client":
1399- switch s.Transport {
1400- case "local":
1401- return roleCommand{Args: []string{"new", session}}
1402- case "ssh_stdio", "udp_baseline", "udp_25ms", "udp_100ms", "udp_loss_0pct", "udp_loss_1pct":
1403- return roleCommand{Args: remote}
1404- }
1405- case "ssh_stdio_peer":
1406- return roleCommand{Args: []string{"_stdio", session}, Transport: transport{ID: s.Transport, Kind: "ssh_stdio"}}
1407- case "udp_peer":
1408- return roleCommand{Args: []string{"_udp-proxy", session}, Transport: selected}
1409- }
1410- return roleCommand{}
1411-}
1412-
1413-func manifestTransport(m manifest, id string) (transport, error) {
1414- for _, t := range m.Transports {
1415- if t.ID == id {
1416- return t, nil
1417- }
1418- }
1419- // Focused unit tests may call runOne with a minimal manifest; production
1420- // already validates every scenario reference before this point.
1421- if m.Schema == 0 {
1422- return scenarioTransport(scenario{Transport: id}), nil
1423- }
1424- return transport{}, fmt.Errorf("scenario references missing transport %q", id)
1425-}
1426-
1427-func scenarioTransport(s scenario) transport {
1428- switch s.Transport {
1429- case "udp_baseline":
1430- return transport{ID: s.Transport, Kind: "udp"}
1431- case "udp_25ms":
1432- return transport{ID: s.Transport, Kind: "udp", RTTMS: 25}
1433- case "udp_100ms":
1434- return transport{ID: s.Transport, Kind: "udp", RTTMS: 100}
1435- case "udp_loss_0pct":
1436- return transport{ID: s.Transport, Kind: "udp", LossPercent: 0}
1437- case "udp_loss_1pct":
1438- return transport{ID: s.Transport, Kind: "udp", LossPercent: 1}
1439- default:
1440- return transport{ID: s.Transport}
1441- }
1442-}
1443-
1444-func launchOrder(mappings []processMapping) []processMapping {
1445- out := append([]processMapping(nil), mappings...)
1446- priority := func(role string) int {
1447- switch role {
1448- case "daemon":
1449- return 0
1450- case "ssh_stdio_peer", "udp_peer":
1451- return 1
1452- case "client":
1453- return 2
1454- default:
1455- return 3
1456- }
1457- }
1458- sort.SliceStable(out, func(i, j int) bool { return priority(out[i].Role) < priority(out[j].Role) })
1459- return out
1460-}
1461-
1462-// workloadInput is shell input delivered through the client PTY. It avoids any
1463-// private daemon API and makes every topology/workload/transport manifest row
1464-// execute a real terminal workload. The marker is the output boundary awaited
1465-// by cliProcess; it is not a synthetic timestamp.
1466-func workloadInput(s scenario, run int, phase string) []byte {
1467- marker := fmt.Sprintf("__VEV_HARNESS_%s_r%d_%s__", safeName(s.ID), run, safeName(phase))
1468- body := "printf 'vev perf %s\\n'"
1469- switch s.Workload {
1470- case "active_output", "all_output", "inactive_output", "interactive_flood":
1471- body = "i=0; while [ $i -lt 128 ]; do printf 'vev perf output %s\\n'; i=$((i+1)); done"
1472- case "resize_sweep":
1473- body = "printf 'vev perf resize-sweep %s\\n'"
1474- case "copy_search":
1475- body = "printf 'vev perf copy-search %s\\n'"
1476- case "snapshot_output_resize":
1477- body = "printf 'vev perf snapshot-output-resize %s\\n'"
1478- case "attach_restore_tab_switch":
1479- body = "printf 'vev perf attach-restore-tab-switch %s\\n'"
1480- }
1481- return []byte(fmt.Sprintf(body+"; printf '%s\\n'\n", s.ID, marker))
1482-}
1483-
1484-// measuredEventSamples validates every owned pair, then retains only events
1485-// whose injection and successful terminal flush are both in the interval.
1486-// Straddling pairs remain raw diagnostic facts but are never percentile input.
1487-func measuredEventSamples(marks []harnessMark, interval measuredInterval) ([]eventSample, error) {
1488- if interval.End < interval.Start {
1489- return nil, errors.New("negative measured interval")
1490- }
1491- starts := map[uint64]int64{}
1492- events := make([]eventSample, 0, len(marks)/2)
1493- for _, m := range marks {
1494- if !m.Valid {
1495- return nil, errors.New("invalid harness boundary")
1496- }
1497- switch m.Kind {
1498- case "input_injected":
1499- if _, exists := starts[m.Sequence]; exists {
1500- return nil, errors.New("duplicate input boundary")
1501- }
1502- starts[m.Sequence] = m.Tick
1503- case "terminal_flushed":
1504- start, ok := starts[m.Sequence]
1505- if !ok {
1506- return nil, errors.New("terminal flush without input pair")
1507- }
1508- if m.Tick < start {
1509- return nil, errors.New("negative harness duration")
1510- }
1511- delete(starts, m.Sequence)
1512- if start >= interval.Start && m.Tick <= interval.End {
1513- events = append(events, eventSample{Sequence: m.Sequence, Injected: start, Flushed: m.Tick, Latency: m.Tick - start})
1514- }
1515- default:
1516- return nil, errors.New("unknown harness mark")
1517- }
1518- }
1519- if len(starts) != 0 {
1520- return nil, errors.New("missing terminal flush pair")
1521- }
1522- sort.Slice(events, func(i, j int) bool {
1523- if events[i].Injected == events[j].Injected {
1524- return events[i].Sequence < events[j].Sequence
1525- }
1526- return events[i].Injected < events[j].Injected
1527- })
1528- return events, nil
1529-}
1530-
1531-func requireMinimumEventSamples(events []eventSample) error {
1532- if len(events) < minimumInIntervalEventSamples {
1533- return fmt.Errorf("insufficient in-interval event samples: got %d, need at least %d", len(events), minimumInIntervalEventSamples)
1534- }
1535- return nil
1536-}
1537-
1538-func eventLatencies(events []eventSample) []int64 {
1539- out := make([]int64, len(events))
1540- for i, event := range events {
1541- out[i] = event.Latency
1542- }
1543- return out
1544-}
1545-
1546-func eventCadenceSamples(events []eventSample) ([]int64, int64) {
1547- if len(events) < 2 {
1548- return nil, 0
1549- }
1550- gaps := make([]int64, 0, len(events)-1)
1551- var maxGap int64
1552- for i := 1; i < len(events); i++ {
1553- gap := events[i].Injected - events[i-1].Injected
1554- gaps = append(gaps, gap)
1555- if gap > maxGap {
1556- maxGap = gap
1557- }
1558- }
1559- return gaps, maxGap
1560-}
1561-
1562-func eventCadence(events []eventSample) (distribution, int64) {
1563- gaps, maxGap := eventCadenceSamples(events)
1564- if len(gaps) == 0 {
1565- return distribution{}, maxGap
1566- }
1567- return percentiles(gaps), maxGap
1568-}
1569-
1570-func pairedSamples(marks []harnessMark) ([]int64, error) {
1571- starts := map[uint64]int64{}
1572- var out []int64
1573- for _, m := range marks {
1574- if !m.Valid {
1575- return nil, errors.New("invalid harness boundary")
1576- }
1577- switch m.Kind {
1578- case "input_injected":
1579- if _, ok := starts[m.Sequence]; ok {
1580- return nil, errors.New("duplicate input boundary")
1581- }
1582- starts[m.Sequence] = m.Tick
1583- case "terminal_flushed":
1584- start, ok := starts[m.Sequence]
1585- if !ok {
1586- return nil, errors.New("terminal flush without input pair")
1587- }
1588- if m.Tick < start {
1589- return nil, errors.New("negative harness duration")
1590- }
1591- out = append(out, m.Tick-start)
1592- delete(starts, m.Sequence)
1593- default:
1594- return nil, errors.New("unknown harness mark")
1595- }
1596- }
1597- if len(starts) != 0 {
1598- return nil, errors.New("missing terminal flush pair")
1599- }
1600- if len(out) == 0 {
1601- return nil, errors.New("insufficient harness samples")
1602- }
1603- return out, nil
1604-}
1605-
1606-// traceRecord is intentionally local to the command. It mirrors JSONL fields
1607-// needed for correlation but never exposes a clock to any production component.
1608-type traceRecord struct {
1609- ProcessID string `json:"process_id"`
1610- Component string `json:"component"`
1611- Scenario string `json:"scenario"`
1612- Run uint64 `json:"run"`
1613- Sequence uint64 `json:"sequence"`
1614- RequestID uint64 `json:"request_id"`
1615- Epoch uint64 `json:"epoch"`
1616- Kind string `json:"kind"`
1617- Tick int64 `json:"tick"`
1618- Valid bool `json:"valid"`
1619-}
1620-type spanPair struct{ start, end, name string }
1621-
1622-var spanPairs = []spanPair{{"capture_start", "capture_end", "capture_duration"}, {"compose_start", "compose_end", "compose_duration"}, {"diff_start", "diff_end", "diff_duration"}, {"queue_enqueued", "queue_dequeued", "queue_wait"}, {"ack_blocked_start", "ack_blocked_end", "ack_blocked_interval"}, {"emit_start", "emit_end", "emit_duration"}, {"adapter_send_start", "adapter_send_end", "adapter_send_duration"}, {"adapter_receive_start", "adapter_receive_end", "adapter_receive_duration"}}
1623-
1624-// mergeProcessTraces permits only records from one manifest process in a span.
1625-// In particular, the correlation key includes process_id before ticks are read.
1626-func mergeProcessTraces(mappings []processMapping) ([]span, error) {
1627- known := map[string]processMapping{}
1628- for _, m := range mappings {
1629- if m.ProcessID == "" || m.ClockDomain == "" || m.TracePath == "" || known[m.ProcessID].ProcessID != "" {
1630- return nil, errors.New("duplicate or incomplete manifest process mapping")
1631- }
1632- known[m.ProcessID] = m
1633- }
1634- out := []span{}
1635- for _, m := range mappings {
1636- f, err := os.Open(m.TracePath)
1637- if err != nil {
1638- return nil, fmt.Errorf("open manifest trace for %s role: %w", m.Role, err)
1639- }
1640- var records []traceRecord
1641- scan := bufio.NewScanner(f)
1642- for scan.Scan() {
1643- var r traceRecord
1644- if err := json.Unmarshal(scan.Bytes(), &r); err != nil {
1645- _ = f.Close()
1646- return nil, err
1647- }
1648- if r.ProcessID != m.ProcessID || r.Scenario != m.Scenario || r.Run != uint64(m.Run) {
1649- _ = f.Close()
1650- return nil, errors.New("trace record identity does not match manifest")
1651- }
1652- if r.Scenario == "" || r.Run == 0 || r.Component == "" || r.Sequence == 0 || r.RequestID == 0 || r.Epoch == 0 {
1653- _ = f.Close()
1654- return nil, errors.New("trace record has invalid correlation fields")
1655- }
1656- if _, ok := known[r.ProcessID]; !ok {
1657- _ = f.Close()
1658- return nil, errors.New("unknown trace process_id")
1659- }
1660- records = append(records, r)
1661- }
1662- if err := scan.Err(); err != nil {
1663- _ = f.Close()
1664- return nil, err
1665- }
1666- _ = f.Close()
1667- // A process shares one observer across concurrent components. Correlation
1668- // IDs are allocated before a goroutine reaches its first mark, so a later
1669- // ID can be serialized before an earlier one. Their first appearance is
1670- // consequently not an ordering contract. Only marks in the same component
1671- // and exact correlation domain may pair, and each pair's ticks must retain
1672- // its own start-before-end ordering.
1673- starts := map[string]traceRecord{}
1674- for _, r := range records {
1675- for _, pair := range spanPairs {
1676- key := fmt.Sprintf("%s/%s/%s/%d/%d/%d/%d", r.ProcessID, r.Component, m.Scenario, m.Run, r.Sequence, r.RequestID, r.Epoch)
1677- if r.Kind == pair.start {
1678- k := pair.name + "/" + key
1679- if _, exists := starts[k]; exists {
1680- return nil, errors.New("duplicate process-local span start")
1681- }
1682- starts[k] = r
1683- }
1684- if r.Kind == pair.end {
1685- k := pair.name + "/" + key
1686- start, ok := starts[k]
1687- if !ok {
1688- return nil, errors.New("span end without same-process start")
1689- }
1690- if r.Tick < start.Tick {
1691- return nil, errors.New("negative same-process span")
1692- }
1693- // Validity affects only measurement eligibility, never structural
1694- // pairing: every start/end must still match in its exact process,
1695- // component, and correlation domain. A failed start, failed end, or
1696- // both is a paired diagnostic fact but contributes no latency sample.
1697- if start.Valid && r.Valid {
1698- out = append(out, span{Component: r.Component, Name: pair.name, Samples: []int64{r.Tick - start.Tick}})
1699- }
1700- delete(starts, k)
1701- }
1702- }
1703- }
1704- if len(starts) != 0 {
1705- keys := make([]string, 0, len(starts))
1706- for key := range starts {
1707- keys = append(keys, key)
1708- }
1709- sort.Strings(keys)
1710- return nil, fmt.Errorf("missing process-local span pair for %s: %s", m.ProcessID, strings.Join(keys, ", "))
1711- }
1712- }
1713- return out, nil
1714-}
1715-func summarizeSpans(all []span) []spanSummary {
1716- grouped := map[string]*span{}
1717- for _, s := range all {
1718- k := s.Component + "\x00" + s.Name
1719- if grouped[k] == nil {
1720- grouped[k] = &span{Component: s.Component, Name: s.Name}
1721- }
1722- grouped[k].Samples = append(grouped[k].Samples, s.Samples...)
1723- }
1724- keys := make([]string, 0, len(grouped))
1725- for k := range grouped {
1726- keys = append(keys, k)
1727- }
1728- sort.Strings(keys)
1729- out := make([]spanSummary, 0, len(keys))
1730- for _, k := range keys {
1731- s := grouped[k]
1732- if len(s.Samples) < minimumRepetitions {
1733- out = append(out, spanSummary{Component: s.Component, Name: s.Name, Distribution: distribution{Count: len(s.Samples)}})
1734- continue
1735- }
1736- out = append(out, spanSummary{Component: s.Component, Name: s.Name, Distribution: percentiles(s.Samples)})
1737- }
1738- return out
1739-}
1740-
1741-func percentiles(samples []int64) distribution {
1742- v := append([]int64(nil), samples...)
1743- sort.Slice(v, func(i, j int) bool { return v[i] < v[j] })
1744- at := func(p int) int64 { return v[(len(v)-1)*p/100] }
1745- return distribution{len(v), at(50), at(95), at(99), v[len(v)-1]}
1746-}
1747-func writeJSON(path string, v any) error {
1748- f, e := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
1749- if e != nil {
1750- return e
1751- }
1752- defer f.Close()
1753- e = json.NewEncoder(f).Encode(v)
1754- return e
1755-}
1756-func mustJSON(v any) string { b, _ := json.Marshal(v); return string(b) }
1757-func safeName(s string) string {
1758- return strings.Map(func(r rune) rune {
1759- if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' {
1760- return r
1761- }
1762- return '-'
1763- }, s)
1764-}
1765-func readJSONL(path string) ([]map[string]json.RawMessage, error) {
1766- f, e := os.Open(path)
1767- if e != nil {
1768- return nil, e
1769- }
1770- defer f.Close()
1771- var r []map[string]json.RawMessage
1772- scan := bufio.NewScanner(f)
1773- for scan.Scan() {
1774- var v map[string]json.RawMessage
1775- if e := json.Unmarshal(scan.Bytes(), &v); e != nil {
1776- return nil, e
1777- }
1778- r = append(r, v)
1779- }
1780- return r, scan.Err()
1781-}

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 12

Tags 11

↑↓ moveenter openesc close