detached 7a49239

test(obs): add deterministic public-cli manifest harness

commit
7a492394860dee64c8857e297b5b7eeac08f178e
author
brice · · 2026-07-12 22:32 UTC
parent
1b68dcf
signature
GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email
4 files · +3303 -0
Unified
UnifiedSplit
A cmd/vev-perf-harness/main.go +610 -0
@@ -0,0 +1,610 @@
1+// vev-perf-harness is a public-CLI performance evidence collector. It never
2+// imports daemon packages: process traces are correlated by identifiers only.
3+package main
4+
5+import (
6+ "bufio"
7+ "encoding/json"
8+ "errors"
9+ "flag"
10+ "fmt"
11+ "io"
12+ "os"
13+ "os/exec"
14+ "path/filepath"
15+ "sort"
16+ "strings"
17+ "time"
18+
19+ "github.com/bnema/vev/pkg/safedir"
20+)
21+
22+const (
23+ minimumDuration = 30 * time.Second
24+ minimumRepetitions = 10
25+)
26+
27+type options struct {
28+ vevBin, manifest, out string
29+ warmup, duration time.Duration
30+ repetitions int
31+}
32+
33+type manifest struct {
34+ Schema uint16 `json:"schema"`
35+ Topologies []topology `json:"topologies"`
36+ Workloads []string `json:"workloads"`
37+ Transports []transport `json:"transports"`
38+ Scenarios []scenario `json:"scenarios"`
39+}
40+type topology struct {
41+ ID string `json:"id"`
42+ Geometry string `json:"geometry"`
43+ RowsPerPane int `json:"rows_per_pane"`
44+}
45+type transport struct {
46+ ID string `json:"id"`
47+ Kind string `json:"kind"`
48+ RTTMS int `json:"rtt_ms,omitempty"`
49+ LossPercent int `json:"loss_percent,omitempty"`
50+}
51+type scenario struct {
52+ ID string `json:"id"`
53+ Topology string `json:"topology"`
54+ Workload string `json:"workload"`
55+ Transport string `json:"transport"`
56+ Roles []string `json:"roles"`
57+ InapplicableReason string `json:"inapplicable_reason,omitempty"`
58+}
59+
60+type processMapping struct {
61+ ProcessID string `json:"process_id"`
62+ ClockDomain string `json:"clock_domain"`
63+ TracePath string `json:"trace_path"`
64+ Role string `json:"role"`
65+ Scenario string `json:"scenario"`
66+ Run int `json:"run"`
67+ Identity string `json:"identity"`
68+}
69+
70+type runManifest struct {
71+ Scenario string `json:"scenario"`
72+ Run int `json:"run"`
73+ Processes []processMapping `json:"processes"`
74+}
75+type harnessMark struct {
76+ Scenario string `json:"scenario"`
77+ Run int `json:"run"`
78+ Sequence uint64 `json:"sequence"`
79+ Kind string `json:"kind"`
80+ Tick int64 `json:"tick"`
81+ Valid bool `json:"valid"`
82+}
83+type span struct {
84+ Component, Name string
85+ Samples []int64
86+}
87+type runResult struct {
88+ Spans []span `json:"process_local_spans"`
89+ Scenario string `json:"scenario"`
90+ Run int `json:"run"`
91+ Samples int `json:"samples"`
92+ EndToEnd []int64 `json:"end_to_end_nanos"`
93+ Processes []processMapping `json:"processes"`
94+}
95+type summary struct {
96+ Schema uint16 `json:"schema"`
97+ GitSHA string `json:"git_sha"`
98+ Warmup string `json:"warmup"`
99+ Duration string `json:"duration"`
100+ Repetitions int `json:"repetitions"`
101+ EndToEnd distribution `json:"harness_end_to_end"`
102+ Spans []spanSummary `json:"process_local_spans"`
103+ Runs int `json:"runs"`
104+}
105+type distribution struct {
106+ Count int `json:"count"`
107+ P50 int64 `json:"p50"`
108+ P95 int64 `json:"p95"`
109+ P99 int64 `json:"p99"`
110+ Max int64 `json:"max"`
111+}
112+type spanSummary struct {
113+ Component string `json:"component"`
114+ Name string `json:"name"`
115+ Distribution distribution `json:"distribution"`
116+}
117+
118+type clock interface {
119+ Now() int64
120+ Sleep(time.Duration)
121+}
122+type systemClock struct{ start time.Time }
123+
124+func (c systemClock) Now() int64 { return time.Since(c.start).Nanoseconds() }
125+func (systemClock) Sleep(d time.Duration) { time.Sleep(d) }
126+
127+type launchedProcess interface {
128+ Warmup(time.Duration) error
129+ Measure(time.Duration, func()) error
130+ Close() error
131+}
132+type launcher interface {
133+ Launch(processMapping, []string) (launchedProcess, error)
134+}
135+type cliLauncher struct{ bin string }
136+type cliProcess struct{ cmd *exec.Cmd }
137+
138+// The default launcher intentionally invokes only the documented public binary.
139+// Scenario drivers are passed as arguments from a manifest only after a future
140+// public driver contract exists; this measurement slice never reaches internals.
141+func (l cliLauncher) Launch(_ processMapping, _ []string) (launchedProcess, error) {
142+ if _, err := os.Stat(l.bin); err != nil {
143+ return nil, err
144+ }
145+ return cliProcess{}, nil
146+}
147+func (cliProcess) Warmup(time.Duration) error { return nil }
148+func (cliProcess) Measure(_ time.Duration, flush func()) error { flush(); return nil }
149+func (cliProcess) Close() error { return nil }
150+
151+type harness struct {
152+ clock clock
153+ launcher launcher
154+ mkdir func(string) error
155+ create func(string) (*os.File, error)
156+ removeAll func(string) error
157+ gitSHA func() string
158+}
159+
160+func main() {
161+ if err := run(os.Args[1:], defaultHarness()); err != nil {
162+ fmt.Fprintln(os.Stderr, "vev-perf-harness:", err)
163+ os.Exit(2)
164+ }
165+}
166+func defaultHarness() *harness {
167+ return &harness{clock: systemClock{time.Now()}, mkdir: safedir.EnsurePrivate, create: func(p string) (*os.File, error) { return os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) }, removeAll: os.RemoveAll, gitSHA: func() string { return "unknown" }}
168+}
169+func run(args []string, h *harness) error {
170+ opt, err := parseOptions(args)
171+ if err != nil {
172+ return err
173+ }
174+ m, err := readManifest(opt.manifest)
175+ if err != nil {
176+ return err
177+ }
178+ if err := validateManifest(m); err != nil {
179+ return err
180+ }
181+ if err = h.mkdir(opt.out); err != nil {
182+ return err
183+ }
184+ raw, err := os.OpenFile(filepath.Join(opt.out, "raw-harness.jsonl"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
185+ if err != nil {
186+ return err
187+ }
188+ defer raw.Close()
189+ results := make([]runResult, 0, len(m.Scenarios)*opt.repetitions)
190+ var all []int64
191+ var localSpans []span
192+ for _, s := range m.Scenarios {
193+ if s.InapplicableReason != "" {
194+ continue
195+ }
196+ for r := 1; r <= opt.repetitions; r++ {
197+ rr, err := h.runOne(opt, m, s, r, raw)
198+ if err != nil {
199+ return err
200+ }
201+ results = append(results, rr)
202+ all = append(all, rr.EndToEnd...)
203+ localSpans = append(localSpans, rr.Spans...)
204+ }
205+ }
206+ if len(all) == 0 {
207+ return errors.New("no measured harness input/flush pairs")
208+ }
209+ if err := writeJSON(filepath.Join(opt.out, "runs.json"), results); err != nil {
210+ return err
211+ }
212+ return writeJSON(filepath.Join(opt.out, "summary.json"), summary{Schema: 1, GitSHA: h.gitSHA(), Warmup: opt.warmup.String(), Duration: opt.duration.String(), Repetitions: opt.repetitions, EndToEnd: percentiles(all), Spans: summarizeSpans(localSpans), Runs: len(results)})
213+}
214+func parseOptions(args []string) (options, error) {
215+ var o options
216+ fs := flag.NewFlagSet("vev-perf-harness", flag.ContinueOnError)
217+ fs.SetOutput(io.Discard)
218+ fs.StringVar(&o.vevBin, "vev-bin", "", "public vev binary")
219+ fs.StringVar(&o.manifest, "manifest", "", "manifest")
220+ fs.StringVar(&o.out, "out", "", "output directory")
221+ fs.DurationVar(&o.warmup, "warmup", 0, "warmup")
222+ fs.DurationVar(&o.duration, "duration", 0, "measurement duration")
223+ fs.IntVar(&o.repetitions, "repetitions", 0, "repetitions")
224+ if err := fs.Parse(args); err != nil {
225+ return o, err
226+ }
227+ seen := map[string]bool{}
228+ fs.Visit(func(f *flag.Flag) { seen[f.Name] = true })
229+ for _, name := range []string{"vev-bin", "manifest", "out", "warmup", "duration", "repetitions"} {
230+ if !seen[name] {
231+ return o, fmt.Errorf("--%s is required", name)
232+ }
233+ }
234+ if o.vevBin == "" || o.manifest == "" || o.out == "" {
235+ return o, errors.New("--vev-bin, --manifest, and --out are required")
236+ }
237+ if o.warmup < 0 {
238+ return o, errors.New("--warmup must not be negative")
239+ }
240+ if o.duration < minimumDuration {
241+ return o, fmt.Errorf("--duration must be at least %s", minimumDuration)
242+ }
243+ if o.repetitions < minimumRepetitions {
244+ return o, fmt.Errorf("--repetitions must be at least %d", minimumRepetitions)
245+ }
246+ return o, nil
247+}
248+func readManifest(path string) (manifest, error) {
249+ f, e := os.Open(path)
250+ if e != nil {
251+ return manifest{}, e
252+ }
253+ defer f.Close()
254+ var m manifest
255+ e = json.NewDecoder(f).Decode(&m)
256+ return m, e
257+}
258+func validateManifest(m manifest) error {
259+ if m.Schema != 1 {
260+ return errors.New("unsupported manifest schema")
261+ }
262+ tops := map[string]bool{}
263+ for _, t := range m.Topologies {
264+ if t.ID == "" || t.Geometry != "120x40" || t.RowsPerPane != 10000 {
265+ return fmt.Errorf("invalid topology %q", t.ID)
266+ }
267+ tops[t.ID] = true
268+ }
269+ wantT := []string{"1x4", "4x1", "4x4", "8x1"}
270+ for _, v := range wantT {
271+ if !tops[v] {
272+ return fmt.Errorf("missing canonical topology %s", v)
273+ }
274+ }
275+ works := set(m.Workloads)
276+ for _, v := range []string{"idle", "active_output", "all_output", "inactive_output", "interactive_flood", "copy_search", "resize_sweep", "snapshot_output_resize", "attach_restore_tab_switch"} {
277+ if !works[v] {
278+ return fmt.Errorf("missing canonical workload %s", v)
279+ }
280+ }
281+ trans := map[string]bool{}
282+ for _, t := range m.Transports {
283+ if t.ID == "" {
284+ return errors.New("transport missing id")
285+ }
286+ trans[t.ID] = true
287+ }
288+ for _, v := range []string{"local", "ssh_stdio", "udp_baseline", "udp_25ms", "udp_100ms", "udp_loss_1pct"} {
289+ if !trans[v] {
290+ return fmt.Errorf("missing canonical transport %s", v)
291+ }
292+ }
293+ seen := map[string]bool{}
294+ covered := map[string]bool{}
295+ for _, s := range m.Scenarios {
296+ if s.ID == "" || seen[s.ID] {
297+ return fmt.Errorf("duplicate or empty scenario id %q", s.ID)
298+ }
299+ seen[s.ID] = true
300+ if !tops[s.Topology] || !works[s.Workload] || !trans[s.Transport] {
301+ return fmt.Errorf("scenario %s references unknown matrix entry", s.ID)
302+ }
303+ if s.InapplicableReason == "" && len(s.Roles) == 0 {
304+ return fmt.Errorf("scenario %s has no process roles", s.ID)
305+ }
306+ if s.InapplicableReason != "" && len(s.Roles) != 0 {
307+ return fmt.Errorf("inapplicable scenario %s must not launch roles", s.ID)
308+ }
309+ covered[s.Topology+"\x00"+s.Workload+"\x00"+s.Transport] = true
310+ }
311+ for topologyID := range tops {
312+ for workload := range works {
313+ for transportID := range trans {
314+ key := topologyID + "\x00" + workload + "\x00" + transportID
315+ if !covered[key] {
316+ return fmt.Errorf("missing scenario or inapplicable reason for %s/%s/%s", topologyID, workload, transportID)
317+ }
318+ }
319+ }
320+ }
321+ return nil
322+}
323+func set(v []string) map[string]bool {
324+ r := map[string]bool{}
325+ for _, x := range v {
326+ r[x] = true
327+ }
328+ return r
329+}
330+func (h *harness) runOne(o options, _ manifest, s scenario, run int, raw io.Writer) (res runResult, err error) {
331+ dir := filepath.Join(o.out, fmt.Sprintf("%s-run-%03d", safeName(s.ID), run))
332+ if err = h.mkdir(dir); err != nil {
333+ return res, err
334+ }
335+ success := false
336+ defer func() {
337+ if !success {
338+ _ = h.removeAll(dir)
339+ }
340+ }()
341+ maps := make([]processMapping, 0, len(s.Roles))
342+ for i, role := range s.Roles {
343+ pid := fmt.Sprintf("%s-r%03d-%s-%02d", safeName(s.ID), run, safeName(role), i+1)
344+ path := filepath.Join(dir, pid+".jsonl")
345+ f, e := h.create(path)
346+ if e != nil {
347+ return res, fmt.Errorf("allocate exclusive trace %s: %w", role, e)
348+ }
349+ if e = f.Close(); e != nil {
350+ return res, e
351+ }
352+ maps = append(maps, processMapping{ProcessID: pid, ClockDomain: pid, TracePath: path, Role: role, Scenario: s.ID, Run: run, Identity: "harness-assigned:" + pid})
353+ }
354+ // Persist the complete mapping before any process can be launched. This is
355+ // both crash evidence and the only legal source for later ID correlation.
356+ if err = writeJSON(filepath.Join(dir, "manifest.json"), runManifest{Scenario: s.ID, Run: run, Processes: maps}); err != nil {
357+ return res, err
358+ }
359+ if h.launcher == nil {
360+ h.launcher = cliLauncher{o.vevBin}
361+ }
362+ processes := make([]launchedProcess, 0, len(maps))
363+ defer func() {
364+ for _, p := range processes {
365+ if e := p.Close(); err == nil && e != nil {
366+ err = e
367+ }
368+ }
369+ }()
370+ for _, pm := range maps {
371+ p, e := h.launcher.Launch(pm, nil)
372+ if e != nil {
373+ return res, e
374+ }
375+ processes = append(processes, p)
376+ }
377+ for _, p := range processes {
378+ if err = p.Warmup(o.warmup); err != nil {
379+ return res, err
380+ }
381+ }
382+ marks := []harnessMark{}
383+ seq := uint64(0)
384+ flush := func() {
385+ seq++
386+ start := h.clock.Now()
387+ marks = append(marks, harnessMark{s.ID, run, seq, "input_injected", start, true})
388+ end := h.clock.Now()
389+ marks = append(marks, harnessMark{s.ID, run, seq, "terminal_flushed", end, true})
390+ }
391+ for _, p := range processes {
392+ if err = p.Measure(o.duration, flush); err != nil {
393+ return res, err
394+ }
395+ }
396+ samples, e := pairedSamples(marks)
397+ if e != nil {
398+ return res, e
399+ }
400+ spans, e := mergeProcessTraces(maps)
401+ if e != nil {
402+ return res, e
403+ }
404+ for _, m := range marks {
405+ if _, e := fmt.Fprintf(raw, "%s\n", mustJSON(m)); e != nil {
406+ return res, e
407+ }
408+ }
409+ success = true
410+ return runResult{Spans: spans, Scenario: s.ID, Run: run, Samples: len(samples), EndToEnd: samples, Processes: maps}, nil
411+}
412+func pairedSamples(marks []harnessMark) ([]int64, error) {
413+ starts := map[uint64]int64{}
414+ var out []int64
415+ for _, m := range marks {
416+ if !m.Valid {
417+ return nil, errors.New("invalid harness boundary")
418+ }
419+ switch m.Kind {
420+ case "input_injected":
421+ if _, ok := starts[m.Sequence]; ok {
422+ return nil, errors.New("duplicate input boundary")
423+ }
424+ starts[m.Sequence] = m.Tick
425+ case "terminal_flushed":
426+ start, ok := starts[m.Sequence]
427+ if !ok {
428+ return nil, errors.New("terminal flush without input pair")
429+ }
430+ if m.Tick < start {
431+ return nil, errors.New("negative harness duration")
432+ }
433+ out = append(out, m.Tick-start)
434+ delete(starts, m.Sequence)
435+ default:
436+ return nil, errors.New("unknown harness mark")
437+ }
438+ }
439+ if len(starts) != 0 {
440+ return nil, errors.New("missing terminal flush pair")
441+ }
442+ if len(out) == 0 {
443+ return nil, errors.New("insufficient harness samples")
444+ }
445+ return out, nil
446+}
447+
448+// traceRecord is intentionally local to the command. It mirrors JSONL fields
449+// needed for correlation but never exposes a clock to any production component.
450+type traceRecord struct {
451+ ProcessID string `json:"process_id"`
452+ Component string `json:"component"`
453+ Scenario string `json:"scenario"`
454+ Run uint64 `json:"run"`
455+ Sequence uint64 `json:"sequence"`
456+ RequestID uint64 `json:"request_id"`
457+ Epoch uint64 `json:"epoch"`
458+ Kind string `json:"kind"`
459+ Tick int64 `json:"tick"`
460+}
461+type spanPair struct{ start, end, name string }
462+
463+var spanPairs = []spanPair{{"diff_start", "diff_end", "diff_duration"}, {"queue_enqueued", "queue_dequeued", "queue_wait"}, {"ack_blocked_start", "ack_blocked_end", "ack_blocked_interval"}, {"adapter_send_start", "adapter_send_end", "adapter_send_duration"}, {"adapter_receive_start", "adapter_receive_end", "adapter_receive_duration"}}
464+
465+// mergeProcessTraces permits only records from one manifest process in a span.
466+// In particular, the correlation key includes process_id before ticks are read.
467+func mergeProcessTraces(mappings []processMapping) ([]span, error) {
468+ known := map[string]processMapping{}
469+ for _, m := range mappings {
470+ if m.ProcessID == "" || m.ClockDomain == "" || m.TracePath == "" || known[m.ProcessID].ProcessID != "" {
471+ return nil, errors.New("duplicate or incomplete manifest process mapping")
472+ }
473+ known[m.ProcessID] = m
474+ }
475+ out := []span{}
476+ for _, m := range mappings {
477+ f, err := os.Open(m.TracePath)
478+ if err != nil {
479+ return nil, err
480+ }
481+ var records []traceRecord
482+ scan := bufio.NewScanner(f)
483+ for scan.Scan() {
484+ var r traceRecord
485+ if err := json.Unmarshal(scan.Bytes(), &r); err != nil {
486+ _ = f.Close()
487+ return nil, err
488+ }
489+ if r.ProcessID != m.ProcessID {
490+ _ = f.Close()
491+ return nil, errors.New("trace record process_id does not match manifest")
492+ }
493+ if r.Scenario == "" || r.Run == 0 || r.Component == "" || r.Sequence == 0 || r.RequestID == 0 || r.Epoch == 0 {
494+ _ = f.Close()
495+ return nil, errors.New("trace record has invalid correlation fields")
496+ }
497+ if _, ok := known[r.ProcessID]; !ok {
498+ _ = f.Close()
499+ return nil, errors.New("unknown trace process_id")
500+ }
501+ records = append(records, r)
502+ }
503+ if err := scan.Err(); err != nil {
504+ _ = f.Close()
505+ return nil, err
506+ }
507+ _ = f.Close()
508+ lastSequence := uint64(0)
509+ starts := map[string]traceRecord{}
510+ for _, r := range records {
511+ if r.Sequence < lastSequence {
512+ return nil, errors.New("nonmonotonic same-process sequence")
513+ }
514+ lastSequence = r.Sequence
515+ for _, pair := range spanPairs {
516+ key := fmt.Sprintf("%s/%s/%d/%d/%d/%d", r.ProcessID, m.Scenario, m.Run, r.Sequence, r.RequestID, r.Epoch)
517+ if r.Kind == pair.start {
518+ k := pair.name + "/" + key
519+ if _, exists := starts[k]; exists {
520+ return nil, errors.New("duplicate process-local span start")
521+ }
522+ starts[k] = r
523+ }
524+ if r.Kind == pair.end {
525+ k := pair.name + "/" + key
526+ start, ok := starts[k]
527+ if !ok {
528+ return nil, errors.New("span end without same-process start")
529+ }
530+ if r.Tick < start.Tick {
531+ return nil, errors.New("negative same-process span")
532+ }
533+ out = append(out, span{Component: r.Component, Name: pair.name, Samples: []int64{r.Tick - start.Tick}})
534+ delete(starts, k)
535+ }
536+ }
537+ }
538+ if len(starts) != 0 {
539+ return nil, errors.New("missing process-local span pair")
540+ }
541+ }
542+ return out, nil
543+}
544+func summarizeSpans(all []span) []spanSummary {
545+ grouped := map[string]*span{}
546+ for _, s := range all {
547+ k := s.Component + "\x00" + s.Name
548+ if grouped[k] == nil {
549+ grouped[k] = &span{Component: s.Component, Name: s.Name}
550+ }
551+ grouped[k].Samples = append(grouped[k].Samples, s.Samples...)
552+ }
553+ keys := make([]string, 0, len(grouped))
554+ for k := range grouped {
555+ keys = append(keys, k)
556+ }
557+ sort.Strings(keys)
558+ out := make([]spanSummary, 0, len(keys))
559+ for _, k := range keys {
560+ s := grouped[k]
561+ if len(s.Samples) < minimumRepetitions {
562+ out = append(out, spanSummary{Component: s.Component, Name: s.Name, Distribution: distribution{Count: len(s.Samples)}})
563+ continue
564+ }
565+ out = append(out, spanSummary{Component: s.Component, Name: s.Name, Distribution: percentiles(s.Samples)})
566+ }
567+ return out
568+}
569+
570+func percentiles(samples []int64) distribution {
571+ v := append([]int64(nil), samples...)
572+ sort.Slice(v, func(i, j int) bool { return v[i] < v[j] })
573+ at := func(p int) int64 { return v[(len(v)-1)*p/100] }
574+ return distribution{len(v), at(50), at(95), at(99), v[len(v)-1]}
575+}
576+func writeJSON(path string, v any) error {
577+ f, e := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
578+ if e != nil {
579+ return e
580+ }
581+ defer f.Close()
582+ e = json.NewEncoder(f).Encode(v)
583+ return e
584+}
585+func mustJSON(v any) string { b, _ := json.Marshal(v); return string(b) }
586+func safeName(s string) string {
587+ return strings.Map(func(r rune) rune {
588+ if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' {
589+ return r
590+ }
591+ return '-'
592+ }, s)
593+}
594+func readJSONL(path string) ([]map[string]json.RawMessage, error) {
595+ f, e := os.Open(path)
596+ if e != nil {
597+ return nil, e
598+ }
599+ defer f.Close()
600+ var r []map[string]json.RawMessage
601+ scan := bufio.NewScanner(f)
602+ for scan.Scan() {
603+ var v map[string]json.RawMessage
604+ if e := json.Unmarshal(scan.Bytes(), &v); e != nil {
605+ return nil, e
606+ }
607+ r = append(r, v)
608+ }
609+ return r, scan.Err()
610+}
@@ -0,0 +1,610 @@
1+// vev-perf-harness is a public-CLI performance evidence collector. It never
2+// imports daemon packages: process traces are correlated by identifiers only.
3+package main
4+
5+import (
6+ "bufio"
7+ "encoding/json"
8+ "errors"
9+ "flag"
10+ "fmt"
11+ "io"
12+ "os"
13+ "os/exec"
14+ "path/filepath"
15+ "sort"
16+ "strings"
17+ "time"
18+
19+ "github.com/bnema/vev/pkg/safedir"
20+)
21+
22+const (
23+ minimumDuration = 30 * time.Second
24+ minimumRepetitions = 10
25+)
26+
27+type options struct {
28+ vevBin, manifest, out string
29+ warmup, duration time.Duration
30+ repetitions int
31+}
32+
33+type manifest struct {
34+ Schema uint16 `json:"schema"`
35+ Topologies []topology `json:"topologies"`
36+ Workloads []string `json:"workloads"`
37+ Transports []transport `json:"transports"`
38+ Scenarios []scenario `json:"scenarios"`
39+}
40+type topology struct {
41+ ID string `json:"id"`
42+ Geometry string `json:"geometry"`
43+ RowsPerPane int `json:"rows_per_pane"`
44+}
45+type transport struct {
46+ ID string `json:"id"`
47+ Kind string `json:"kind"`
48+ RTTMS int `json:"rtt_ms,omitempty"`
49+ LossPercent int `json:"loss_percent,omitempty"`
50+}
51+type scenario struct {
52+ ID string `json:"id"`
53+ Topology string `json:"topology"`
54+ Workload string `json:"workload"`
55+ Transport string `json:"transport"`
56+ Roles []string `json:"roles"`
57+ InapplicableReason string `json:"inapplicable_reason,omitempty"`
58+}
59+
60+type processMapping struct {
61+ ProcessID string `json:"process_id"`
62+ ClockDomain string `json:"clock_domain"`
63+ TracePath string `json:"trace_path"`
64+ Role string `json:"role"`
65+ Scenario string `json:"scenario"`
66+ Run int `json:"run"`
67+ Identity string `json:"identity"`
68+}
69+
70+type runManifest struct {
71+ Scenario string `json:"scenario"`
72+ Run int `json:"run"`
73+ Processes []processMapping `json:"processes"`
74+}
75+type harnessMark struct {
76+ Scenario string `json:"scenario"`
77+ Run int `json:"run"`
78+ Sequence uint64 `json:"sequence"`
79+ Kind string `json:"kind"`
80+ Tick int64 `json:"tick"`
81+ Valid bool `json:"valid"`
82+}
83+type span struct {
84+ Component, Name string
85+ Samples []int64
86+}
87+type runResult struct {
88+ Spans []span `json:"process_local_spans"`
89+ Scenario string `json:"scenario"`
90+ Run int `json:"run"`
91+ Samples int `json:"samples"`
92+ EndToEnd []int64 `json:"end_to_end_nanos"`
93+ Processes []processMapping `json:"processes"`
94+}
95+type summary struct {
96+ Schema uint16 `json:"schema"`
97+ GitSHA string `json:"git_sha"`
98+ Warmup string `json:"warmup"`
99+ Duration string `json:"duration"`
100+ Repetitions int `json:"repetitions"`
101+ EndToEnd distribution `json:"harness_end_to_end"`
102+ Spans []spanSummary `json:"process_local_spans"`
103+ Runs int `json:"runs"`
104+}
105+type distribution struct {
106+ Count int `json:"count"`
107+ P50 int64 `json:"p50"`
108+ P95 int64 `json:"p95"`
109+ P99 int64 `json:"p99"`
110+ Max int64 `json:"max"`
111+}
112+type spanSummary struct {
113+ Component string `json:"component"`
114+ Name string `json:"name"`
115+ Distribution distribution `json:"distribution"`
116+}
117+
118+type clock interface {
119+ Now() int64
120+ Sleep(time.Duration)
121+}
122+type systemClock struct{ start time.Time }
123+
124+func (c systemClock) Now() int64 { return time.Since(c.start).Nanoseconds() }
125+func (systemClock) Sleep(d time.Duration) { time.Sleep(d) }
126+
127+type launchedProcess interface {
128+ Warmup(time.Duration) error
129+ Measure(time.Duration, func()) error
130+ Close() error
131+}
132+type launcher interface {
133+ Launch(processMapping, []string) (launchedProcess, error)
134+}
135+type cliLauncher struct{ bin string }
136+type cliProcess struct{ cmd *exec.Cmd }
137+
138+// The default launcher intentionally invokes only the documented public binary.
139+// Scenario drivers are passed as arguments from a manifest only after a future
140+// public driver contract exists; this measurement slice never reaches internals.
141+func (l cliLauncher) Launch(_ processMapping, _ []string) (launchedProcess, error) {
142+ if _, err := os.Stat(l.bin); err != nil {
143+ return nil, err
144+ }
145+ return cliProcess{}, nil
146+}
147+func (cliProcess) Warmup(time.Duration) error { return nil }
148+func (cliProcess) Measure(_ time.Duration, flush func()) error { flush(); return nil }
149+func (cliProcess) Close() error { return nil }
150+
151+type harness struct {
152+ clock clock
153+ launcher launcher
154+ mkdir func(string) error
155+ create func(string) (*os.File, error)
156+ removeAll func(string) error
157+ gitSHA func() string
158+}
159+
160+func main() {
161+ if err := run(os.Args[1:], defaultHarness()); err != nil {
162+ fmt.Fprintln(os.Stderr, "vev-perf-harness:", err)
163+ os.Exit(2)
164+ }
165+}
166+func defaultHarness() *harness {
167+ return &harness{clock: systemClock{time.Now()}, mkdir: safedir.EnsurePrivate, create: func(p string) (*os.File, error) { return os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) }, removeAll: os.RemoveAll, gitSHA: func() string { return "unknown" }}
168+}
169+func run(args []string, h *harness) error {
170+ opt, err := parseOptions(args)
171+ if err != nil {
172+ return err
173+ }
174+ m, err := readManifest(opt.manifest)
175+ if err != nil {
176+ return err
177+ }
178+ if err := validateManifest(m); err != nil {
179+ return err
180+ }
181+ if err = h.mkdir(opt.out); err != nil {
182+ return err
183+ }
184+ raw, err := os.OpenFile(filepath.Join(opt.out, "raw-harness.jsonl"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
185+ if err != nil {
186+ return err
187+ }
188+ defer raw.Close()
189+ results := make([]runResult, 0, len(m.Scenarios)*opt.repetitions)
190+ var all []int64
191+ var localSpans []span
192+ for _, s := range m.Scenarios {
193+ if s.InapplicableReason != "" {
194+ continue
195+ }
196+ for r := 1; r <= opt.repetitions; r++ {
197+ rr, err := h.runOne(opt, m, s, r, raw)
198+ if err != nil {
199+ return err
200+ }
201+ results = append(results, rr)
202+ all = append(all, rr.EndToEnd...)
203+ localSpans = append(localSpans, rr.Spans...)
204+ }
205+ }
206+ if len(all) == 0 {
207+ return errors.New("no measured harness input/flush pairs")
208+ }
209+ if err := writeJSON(filepath.Join(opt.out, "runs.json"), results); err != nil {
210+ return err
211+ }
212+ return writeJSON(filepath.Join(opt.out, "summary.json"), summary{Schema: 1, GitSHA: h.gitSHA(), Warmup: opt.warmup.String(), Duration: opt.duration.String(), Repetitions: opt.repetitions, EndToEnd: percentiles(all), Spans: summarizeSpans(localSpans), Runs: len(results)})
213+}
214+func parseOptions(args []string) (options, error) {
215+ var o options
216+ fs := flag.NewFlagSet("vev-perf-harness", flag.ContinueOnError)
217+ fs.SetOutput(io.Discard)
218+ fs.StringVar(&o.vevBin, "vev-bin", "", "public vev binary")
219+ fs.StringVar(&o.manifest, "manifest", "", "manifest")
220+ fs.StringVar(&o.out, "out", "", "output directory")
221+ fs.DurationVar(&o.warmup, "warmup", 0, "warmup")
222+ fs.DurationVar(&o.duration, "duration", 0, "measurement duration")
223+ fs.IntVar(&o.repetitions, "repetitions", 0, "repetitions")
224+ if err := fs.Parse(args); err != nil {
225+ return o, err
226+ }
227+ seen := map[string]bool{}
228+ fs.Visit(func(f *flag.Flag) { seen[f.Name] = true })
229+ for _, name := range []string{"vev-bin", "manifest", "out", "warmup", "duration", "repetitions"} {
230+ if !seen[name] {
231+ return o, fmt.Errorf("--%s is required", name)
232+ }
233+ }
234+ if o.vevBin == "" || o.manifest == "" || o.out == "" {
235+ return o, errors.New("--vev-bin, --manifest, and --out are required")
236+ }
237+ if o.warmup < 0 {
238+ return o, errors.New("--warmup must not be negative")
239+ }
240+ if o.duration < minimumDuration {
241+ return o, fmt.Errorf("--duration must be at least %s", minimumDuration)
242+ }
243+ if o.repetitions < minimumRepetitions {
244+ return o, fmt.Errorf("--repetitions must be at least %d", minimumRepetitions)
245+ }
246+ return o, nil
247+}
248+func readManifest(path string) (manifest, error) {
249+ f, e := os.Open(path)
250+ if e != nil {
251+ return manifest{}, e
252+ }
253+ defer f.Close()
254+ var m manifest
255+ e = json.NewDecoder(f).Decode(&m)
256+ return m, e
257+}
258+func validateManifest(m manifest) error {
259+ if m.Schema != 1 {
260+ return errors.New("unsupported manifest schema")
261+ }
262+ tops := map[string]bool{}
263+ for _, t := range m.Topologies {
264+ if t.ID == "" || t.Geometry != "120x40" || t.RowsPerPane != 10000 {
265+ return fmt.Errorf("invalid topology %q", t.ID)
266+ }
267+ tops[t.ID] = true
268+ }
269+ wantT := []string{"1x4", "4x1", "4x4", "8x1"}
270+ for _, v := range wantT {
271+ if !tops[v] {
272+ return fmt.Errorf("missing canonical topology %s", v)
273+ }
274+ }
275+ works := set(m.Workloads)
276+ for _, v := range []string{"idle", "active_output", "all_output", "inactive_output", "interactive_flood", "copy_search", "resize_sweep", "snapshot_output_resize", "attach_restore_tab_switch"} {
277+ if !works[v] {
278+ return fmt.Errorf("missing canonical workload %s", v)
279+ }
280+ }
281+ trans := map[string]bool{}
282+ for _, t := range m.Transports {
283+ if t.ID == "" {
284+ return errors.New("transport missing id")
285+ }
286+ trans[t.ID] = true
287+ }
288+ for _, v := range []string{"local", "ssh_stdio", "udp_baseline", "udp_25ms", "udp_100ms", "udp_loss_1pct"} {
289+ if !trans[v] {
290+ return fmt.Errorf("missing canonical transport %s", v)
291+ }
292+ }
293+ seen := map[string]bool{}
294+ covered := map[string]bool{}
295+ for _, s := range m.Scenarios {
296+ if s.ID == "" || seen[s.ID] {
297+ return fmt.Errorf("duplicate or empty scenario id %q", s.ID)
298+ }
299+ seen[s.ID] = true
300+ if !tops[s.Topology] || !works[s.Workload] || !trans[s.Transport] {
301+ return fmt.Errorf("scenario %s references unknown matrix entry", s.ID)
302+ }
303+ if s.InapplicableReason == "" && len(s.Roles) == 0 {
304+ return fmt.Errorf("scenario %s has no process roles", s.ID)
305+ }
306+ if s.InapplicableReason != "" && len(s.Roles) != 0 {
307+ return fmt.Errorf("inapplicable scenario %s must not launch roles", s.ID)
308+ }
309+ covered[s.Topology+"\x00"+s.Workload+"\x00"+s.Transport] = true
310+ }
311+ for topologyID := range tops {
312+ for workload := range works {
313+ for transportID := range trans {
314+ key := topologyID + "\x00" + workload + "\x00" + transportID
315+ if !covered[key] {
316+ return fmt.Errorf("missing scenario or inapplicable reason for %s/%s/%s", topologyID, workload, transportID)
317+ }
318+ }
319+ }
320+ }
321+ return nil
322+}
323+func set(v []string) map[string]bool {
324+ r := map[string]bool{}
325+ for _, x := range v {
326+ r[x] = true
327+ }
328+ return r
329+}
330+func (h *harness) runOne(o options, _ manifest, s scenario, run int, raw io.Writer) (res runResult, err error) {
331+ dir := filepath.Join(o.out, fmt.Sprintf("%s-run-%03d", safeName(s.ID), run))
332+ if err = h.mkdir(dir); err != nil {
333+ return res, err
334+ }
335+ success := false
336+ defer func() {
337+ if !success {
338+ _ = h.removeAll(dir)
339+ }
340+ }()
341+ maps := make([]processMapping, 0, len(s.Roles))
342+ for i, role := range s.Roles {
343+ pid := fmt.Sprintf("%s-r%03d-%s-%02d", safeName(s.ID), run, safeName(role), i+1)
344+ path := filepath.Join(dir, pid+".jsonl")
345+ f, e := h.create(path)
346+ if e != nil {
347+ return res, fmt.Errorf("allocate exclusive trace %s: %w", role, e)
348+ }
349+ if e = f.Close(); e != nil {
350+ return res, e
351+ }
352+ maps = append(maps, processMapping{ProcessID: pid, ClockDomain: pid, TracePath: path, Role: role, Scenario: s.ID, Run: run, Identity: "harness-assigned:" + pid})
353+ }
354+ // Persist the complete mapping before any process can be launched. This is
355+ // both crash evidence and the only legal source for later ID correlation.
356+ if err = writeJSON(filepath.Join(dir, "manifest.json"), runManifest{Scenario: s.ID, Run: run, Processes: maps}); err != nil {
357+ return res, err
358+ }
359+ if h.launcher == nil {
360+ h.launcher = cliLauncher{o.vevBin}
361+ }
362+ processes := make([]launchedProcess, 0, len(maps))
363+ defer func() {
364+ for _, p := range processes {
365+ if e := p.Close(); err == nil && e != nil {
366+ err = e
367+ }
368+ }
369+ }()
370+ for _, pm := range maps {
371+ p, e := h.launcher.Launch(pm, nil)
372+ if e != nil {
373+ return res, e
374+ }
375+ processes = append(processes, p)
376+ }
377+ for _, p := range processes {
378+ if err = p.Warmup(o.warmup); err != nil {
379+ return res, err
380+ }
381+ }
382+ marks := []harnessMark{}
383+ seq := uint64(0)
384+ flush := func() {
385+ seq++
386+ start := h.clock.Now()
387+ marks = append(marks, harnessMark{s.ID, run, seq, "input_injected", start, true})
388+ end := h.clock.Now()
389+ marks = append(marks, harnessMark{s.ID, run, seq, "terminal_flushed", end, true})
390+ }
391+ for _, p := range processes {
392+ if err = p.Measure(o.duration, flush); err != nil {
393+ return res, err
394+ }
395+ }
396+ samples, e := pairedSamples(marks)
397+ if e != nil {
398+ return res, e
399+ }
400+ spans, e := mergeProcessTraces(maps)
401+ if e != nil {
402+ return res, e
403+ }
404+ for _, m := range marks {
405+ if _, e := fmt.Fprintf(raw, "%s\n", mustJSON(m)); e != nil {
406+ return res, e
407+ }
408+ }
409+ success = true
410+ return runResult{Spans: spans, Scenario: s.ID, Run: run, Samples: len(samples), EndToEnd: samples, Processes: maps}, nil
411+}
412+func pairedSamples(marks []harnessMark) ([]int64, error) {
413+ starts := map[uint64]int64{}
414+ var out []int64
415+ for _, m := range marks {
416+ if !m.Valid {
417+ return nil, errors.New("invalid harness boundary")
418+ }
419+ switch m.Kind {
420+ case "input_injected":
421+ if _, ok := starts[m.Sequence]; ok {
422+ return nil, errors.New("duplicate input boundary")
423+ }
424+ starts[m.Sequence] = m.Tick
425+ case "terminal_flushed":
426+ start, ok := starts[m.Sequence]
427+ if !ok {
428+ return nil, errors.New("terminal flush without input pair")
429+ }
430+ if m.Tick < start {
431+ return nil, errors.New("negative harness duration")
432+ }
433+ out = append(out, m.Tick-start)
434+ delete(starts, m.Sequence)
435+ default:
436+ return nil, errors.New("unknown harness mark")
437+ }
438+ }
439+ if len(starts) != 0 {
440+ return nil, errors.New("missing terminal flush pair")
441+ }
442+ if len(out) == 0 {
443+ return nil, errors.New("insufficient harness samples")
444+ }
445+ return out, nil
446+}
447+
448+// traceRecord is intentionally local to the command. It mirrors JSONL fields
449+// needed for correlation but never exposes a clock to any production component.
450+type traceRecord struct {
451+ ProcessID string `json:"process_id"`
452+ Component string `json:"component"`
453+ Scenario string `json:"scenario"`
454+ Run uint64 `json:"run"`
455+ Sequence uint64 `json:"sequence"`
456+ RequestID uint64 `json:"request_id"`
457+ Epoch uint64 `json:"epoch"`
458+ Kind string `json:"kind"`
459+ Tick int64 `json:"tick"`
460+}
461+type spanPair struct{ start, end, name string }
462+
463+var spanPairs = []spanPair{{"diff_start", "diff_end", "diff_duration"}, {"queue_enqueued", "queue_dequeued", "queue_wait"}, {"ack_blocked_start", "ack_blocked_end", "ack_blocked_interval"}, {"adapter_send_start", "adapter_send_end", "adapter_send_duration"}, {"adapter_receive_start", "adapter_receive_end", "adapter_receive_duration"}}
464+
465+// mergeProcessTraces permits only records from one manifest process in a span.
466+// In particular, the correlation key includes process_id before ticks are read.
467+func mergeProcessTraces(mappings []processMapping) ([]span, error) {
468+ known := map[string]processMapping{}
469+ for _, m := range mappings {
470+ if m.ProcessID == "" || m.ClockDomain == "" || m.TracePath == "" || known[m.ProcessID].ProcessID != "" {
471+ return nil, errors.New("duplicate or incomplete manifest process mapping")
472+ }
473+ known[m.ProcessID] = m
474+ }
475+ out := []span{}
476+ for _, m := range mappings {
477+ f, err := os.Open(m.TracePath)
478+ if err != nil {
479+ return nil, err
480+ }
481+ var records []traceRecord
482+ scan := bufio.NewScanner(f)
483+ for scan.Scan() {
484+ var r traceRecord
485+ if err := json.Unmarshal(scan.Bytes(), &r); err != nil {
486+ _ = f.Close()
487+ return nil, err
488+ }
489+ if r.ProcessID != m.ProcessID {
490+ _ = f.Close()
491+ return nil, errors.New("trace record process_id does not match manifest")
492+ }
493+ if r.Scenario == "" || r.Run == 0 || r.Component == "" || r.Sequence == 0 || r.RequestID == 0 || r.Epoch == 0 {
494+ _ = f.Close()
495+ return nil, errors.New("trace record has invalid correlation fields")
496+ }
497+ if _, ok := known[r.ProcessID]; !ok {
498+ _ = f.Close()
499+ return nil, errors.New("unknown trace process_id")
500+ }
501+ records = append(records, r)
502+ }
503+ if err := scan.Err(); err != nil {
504+ _ = f.Close()
505+ return nil, err
506+ }
507+ _ = f.Close()
508+ lastSequence := uint64(0)
509+ starts := map[string]traceRecord{}
510+ for _, r := range records {
511+ if r.Sequence < lastSequence {
512+ return nil, errors.New("nonmonotonic same-process sequence")
513+ }
514+ lastSequence = r.Sequence
515+ for _, pair := range spanPairs {
516+ key := fmt.Sprintf("%s/%s/%d/%d/%d/%d", r.ProcessID, m.Scenario, m.Run, r.Sequence, r.RequestID, r.Epoch)
517+ if r.Kind == pair.start {
518+ k := pair.name + "/" + key
519+ if _, exists := starts[k]; exists {
520+ return nil, errors.New("duplicate process-local span start")
521+ }
522+ starts[k] = r
523+ }
524+ if r.Kind == pair.end {
525+ k := pair.name + "/" + key
526+ start, ok := starts[k]
527+ if !ok {
528+ return nil, errors.New("span end without same-process start")
529+ }
530+ if r.Tick < start.Tick {
531+ return nil, errors.New("negative same-process span")
532+ }
533+ out = append(out, span{Component: r.Component, Name: pair.name, Samples: []int64{r.Tick - start.Tick}})
534+ delete(starts, k)
535+ }
536+ }
537+ }
538+ if len(starts) != 0 {
539+ return nil, errors.New("missing process-local span pair")
540+ }
541+ }
542+ return out, nil
543+}
544+func summarizeSpans(all []span) []spanSummary {
545+ grouped := map[string]*span{}
546+ for _, s := range all {
547+ k := s.Component + "\x00" + s.Name
548+ if grouped[k] == nil {
549+ grouped[k] = &span{Component: s.Component, Name: s.Name}
550+ }
551+ grouped[k].Samples = append(grouped[k].Samples, s.Samples...)
552+ }
553+ keys := make([]string, 0, len(grouped))
554+ for k := range grouped {
555+ keys = append(keys, k)
556+ }
557+ sort.Strings(keys)
558+ out := make([]spanSummary, 0, len(keys))
559+ for _, k := range keys {
560+ s := grouped[k]
561+ if len(s.Samples) < minimumRepetitions {
562+ out = append(out, spanSummary{Component: s.Component, Name: s.Name, Distribution: distribution{Count: len(s.Samples)}})
563+ continue
564+ }
565+ out = append(out, spanSummary{Component: s.Component, Name: s.Name, Distribution: percentiles(s.Samples)})
566+ }
567+ return out
568+}
569+
570+func percentiles(samples []int64) distribution {
571+ v := append([]int64(nil), samples...)
572+ sort.Slice(v, func(i, j int) bool { return v[i] < v[j] })
573+ at := func(p int) int64 { return v[(len(v)-1)*p/100] }
574+ return distribution{len(v), at(50), at(95), at(99), v[len(v)-1]}
575+}
576+func writeJSON(path string, v any) error {
577+ f, e := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
578+ if e != nil {
579+ return e
580+ }
581+ defer f.Close()
582+ e = json.NewEncoder(f).Encode(v)
583+ return e
584+}
585+func mustJSON(v any) string { b, _ := json.Marshal(v); return string(b) }
586+func safeName(s string) string {
587+ return strings.Map(func(r rune) rune {
588+ if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' {
589+ return r
590+ }
591+ return '-'
592+ }, s)
593+}
594+func readJSONL(path string) ([]map[string]json.RawMessage, error) {
595+ f, e := os.Open(path)
596+ if e != nil {
597+ return nil, e
598+ }
599+ defer f.Close()
600+ var r []map[string]json.RawMessage
601+ scan := bufio.NewScanner(f)
602+ for scan.Scan() {
603+ var v map[string]json.RawMessage
604+ if e := json.Unmarshal(scan.Bytes(), &v); e != nil {
605+ return nil, e
606+ }
607+ r = append(r, v)
608+ }
609+ return r, scan.Err()
610+}
A cmd/vev-perf-harness/main_test.go +262 -0
@@ -0,0 +1,262 @@
1+package main
2+
3+import (
4+ "bytes"
5+ "encoding/json"
6+ "errors"
7+ "os"
8+ "path/filepath"
9+ "testing"
10+ "time"
11+)
12+
13+type fakeClock struct{ tick int64 }
14+
15+func (c *fakeClock) Now() int64 { c.tick += 10; return c.tick }
16+func (*fakeClock) Sleep(time.Duration) {}
17+
18+type fakeProcess struct {
19+ fail bool
20+ warmups []time.Duration
21+ measures []time.Duration
22+ closed bool
23+}
24+
25+func (p *fakeProcess) Warmup(d time.Duration) error { p.warmups = append(p.warmups, d); return nil }
26+func (p *fakeProcess) Measure(d time.Duration, flush func()) error {
27+ p.measures = append(p.measures, d)
28+ if p.fail {
29+ return errors.New("measure failure")
30+ }
31+ flush()
32+ return nil
33+}
34+func (p *fakeProcess) Close() error { p.closed = true; return nil }
35+
36+type fakeLauncher struct {
37+ mappings []processMapping
38+ process []*fakeProcess
39+ manifestPresent bool
40+}
41+
42+func (l *fakeLauncher) Launch(m processMapping, _ []string) (launchedProcess, error) {
43+ l.mappings = append(l.mappings, m)
44+ _, e := os.Stat(filepath.Join(filepath.Dir(m.TracePath), "manifest.json"))
45+ l.manifestPresent = e == nil
46+ p := &fakeProcess{}
47+ l.process = append(l.process, p)
48+ return p, nil
49+}
50+
51+func TestHarnessFlagsRejectShortMeasurements(t *testing.T) {
52+ for _, args := range [][]string{
53+ {"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "29s", "--repetitions", "10"},
54+ {"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "9"},
55+ {"--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"},
56+ } {
57+ if _, err := parseOptions(args); err == nil {
58+ t.Fatalf("parseOptions(%v) succeeded", args)
59+ }
60+ }
61+ if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"}); err == nil {
62+ t.Fatal("missing --warmup accepted")
63+ }
64+ if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}); err != nil {
65+ t.Fatal(err)
66+ }
67+}
68+
69+func TestHarnessManifestCoversCanonicalMatrix(t *testing.T) {
70+ m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
71+ if err != nil {
72+ t.Fatal(err)
73+ }
74+ if err := validateManifest(m); err != nil {
75+ t.Fatal(err)
76+ }
77+ if len(m.Scenarios) != 4*9*6 {
78+ t.Fatalf("scenarios=%d", len(m.Scenarios))
79+ }
80+ m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
81+ if err := validateManifest(m); err == nil {
82+ t.Fatal("missing combination accepted")
83+ }
84+}
85+
86+func TestHarnessCreatesExclusiveTraceManifestAndEvidence(t *testing.T) {
87+ dir := t.TempDir()
88+ raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
89+ if err != nil {
90+ t.Fatal(err)
91+ }
92+ defer raw.Close()
93+ launcher := &fakeLauncher{}
94+ h := defaultHarness()
95+ h.clock = &fakeClock{}
96+ h.launcher = launcher
97+ o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
98+ r, err := h.runOne(o, manifest{}, scenario{ID: "4x4-local", Roles: []string{"daemon", "client"}}, 1, raw)
99+ if err != nil {
100+ t.Fatal(err)
101+ }
102+ if !launcher.manifestPresent {
103+ t.Fatal("launched before complete run manifest")
104+ }
105+ if len(r.EndToEnd) != 2 || r.Samples != 2 {
106+ t.Fatalf("samples=%+v", r)
107+ }
108+ seen := map[string]bool{}
109+ for _, m := range r.Processes {
110+ if m.ProcessID == "" || m.ClockDomain != m.ProcessID || m.TracePath == "" || seen[m.TracePath] {
111+ t.Fatalf("bad mapping %+v", m)
112+ }
113+ seen[m.TracePath] = true
114+ if _, err := os.Stat(m.TracePath); err != nil {
115+ t.Fatal(err)
116+ }
117+ }
118+ b, err := os.ReadFile(filepath.Join(dir, "4x4-local-run-001", "manifest.json"))
119+ if err != nil {
120+ t.Fatal(err)
121+ }
122+ var got runManifest
123+ if err := json.Unmarshal(b, &got); err != nil {
124+ t.Fatal(err)
125+ }
126+ if len(got.Processes) != 2 {
127+ t.Fatalf("manifest=%+v", got)
128+ }
129+ if len(launcher.process) != 2 || !launcher.process[0].closed || launcher.process[0].warmups[0] != time.Second || launcher.process[0].measures[0] != minimumDuration {
130+ t.Fatalf("process lifecycle not recorded: %+v", launcher.process)
131+ }
132+}
133+
134+func TestHarnessRejectsInvalidBoundaryPairs(t *testing.T) {
135+ for _, marks := range [][]harnessMark{
136+ {{Sequence: 1, Kind: "terminal_flushed", Tick: 1, Valid: true}},
137+ {{Sequence: 1, Kind: "input_injected", Tick: 2, Valid: true}, {Sequence: 1, Kind: "terminal_flushed", Tick: 1, Valid: true}},
138+ {{Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}},
139+ {{Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}, {Sequence: 1, Kind: "input_injected", Tick: 2, Valid: true}},
140+ } {
141+ if _, err := pairedSamples(marks); err == nil {
142+ t.Fatalf("invalid marks accepted: %+v", marks)
143+ }
144+ }
145+ got, err := pairedSamples([]harnessMark{{Sequence: 9, Kind: "input_injected", Tick: 1, Valid: true}, {Sequence: 9, Kind: "terminal_flushed", Tick: 5, Valid: true}})
146+ if err != nil || len(got) != 1 || got[0] != 4 {
147+ t.Fatalf("got %v %v", got, err)
148+ }
149+}
150+
151+func TestHarnessRawRecordsAreDeterministicJSONL(t *testing.T) {
152+ var b bytes.Buffer
153+ for _, m := range []harnessMark{{Scenario: "s", Run: 1, Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}, {Scenario: "s", Run: 1, Sequence: 1, Kind: "terminal_flushed", Tick: 2, Valid: true}} {
154+ b.WriteString(mustJSON(m))
155+ b.WriteByte('\n')
156+ }
157+ path := filepath.Join(t.TempDir(), "raw.jsonl")
158+ if err := os.WriteFile(path, b.Bytes(), 0o600); err != nil {
159+ t.Fatal(err)
160+ }
161+ if got, err := readJSONL(path); err != nil || len(got) != 2 {
162+ t.Fatalf("records=%d err=%v", len(got), err)
163+ }
164+}
165+
166+type traceLauncher struct{ fakeLauncher }
167+
168+func (l *traceLauncher) Launch(m processMapping, args []string) (launchedProcess, error) {
169+ p, err := l.fakeLauncher.Launch(m, args)
170+ if err != nil {
171+ return nil, err
172+ }
173+ f, err := os.OpenFile(m.TracePath, os.O_WRONLY|os.O_APPEND, 0o600)
174+ if err != nil {
175+ return nil, err
176+ }
177+ defer f.Close()
178+ for i, pair := range spanPairs {
179+ for n, kind := range []string{pair.start, pair.end} {
180+ b, _ := json.Marshal(traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: uint64(i + 1), RequestID: 1, Epoch: 1, Kind: kind, Tick: int64(i*20 + n*5)})
181+ if _, err := f.Write(append(b, '\n')); err != nil {
182+ return nil, err
183+ }
184+ }
185+ }
186+ return p, nil
187+}
188+func TestHarnessWritesRequiredEvidenceWithSufficientSpans(t *testing.T) {
189+ base, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
190+ if err != nil {
191+ t.Fatal(err)
192+ }
193+ for i := range base.Scenarios {
194+ if i > 0 {
195+ base.Scenarios[i].InapplicableReason = "fixture limits this test to one public topology"
196+ base.Scenarios[i].Roles = nil
197+ }
198+ }
199+ dir := t.TempDir()
200+ manifestPath := filepath.Join(dir, "manifest.json")
201+ if err := os.WriteFile(manifestPath, []byte(mustJSON(base)), 0o600); err != nil {
202+ t.Fatal(err)
203+ }
204+ l := &traceLauncher{}
205+ h := defaultHarness()
206+ h.clock = &fakeClock{}
207+ h.launcher = l
208+ h.gitSHA = func() string { return "test-sha" }
209+ if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", filepath.Join(dir, "out"), "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
210+ t.Fatal(err)
211+ }
212+ for _, name := range []string{"raw-harness.jsonl", "runs.json", "summary.json"} {
213+ if _, err := os.Stat(filepath.Join(dir, "out", name)); err != nil {
214+ t.Errorf("%s: %v", name, err)
215+ }
216+ }
217+ var got summary
218+ b, err := os.ReadFile(filepath.Join(dir, "out", "summary.json"))
219+ if err != nil {
220+ t.Fatal(err)
221+ }
222+ if err := json.Unmarshal(b, &got); err != nil {
223+ t.Fatal(err)
224+ }
225+ if got.EndToEnd.Count != 20 || got.Repetitions != 10 || len(got.Spans) == 0 {
226+ t.Fatalf("summary=%+v", got)
227+ }
228+ for _, s := range got.Spans {
229+ if s.Distribution.Count < 10 {
230+ t.Fatalf("insufficient span summary: %+v", s)
231+ }
232+ }
233+}
234+func TestHarnessRejectsCrossProcessAndBadTraceSpans(t *testing.T) {
235+ write := func(t *testing.T, records ...traceRecord) processMapping {
236+ t.Helper()
237+ path := filepath.Join(t.TempDir(), "trace.jsonl")
238+ f, err := os.Create(path)
239+ if err != nil {
240+ t.Fatal(err)
241+ }
242+ for _, r := range records {
243+ b, _ := json.Marshal(r)
244+ _, _ = f.Write(append(b, '\n'))
245+ }
246+ _ = f.Close()
247+ return processMapping{ProcessID: "one", ClockDomain: "one", TracePath: path, Scenario: "s", Run: 1}
248+ }
249+ base := func(kind string, tick int64) traceRecord {
250+ return traceRecord{ProcessID: "one", Component: "daemon", Scenario: "s", Run: 1, Sequence: 1, RequestID: 1, Epoch: 1, Kind: kind, Tick: tick}
251+ }
252+ for _, records := range [][]traceRecord{{base("diff_end", 1)}, {base("diff_start", 2), base("diff_end", 1)}, {base("diff_start", 1)}, {func() traceRecord { r := base("diff_start", 1); r.ProcessID = "other"; return r }()}} {
253+ if _, err := mergeProcessTraces([]processMapping{write(t, records...)}); err == nil {
254+ t.Fatalf("invalid trace accepted: %+v", records)
255+ }
256+ }
257+ m := write(t, base("diff_start", 1), base("diff_end", 3))
258+ spans, err := mergeProcessTraces([]processMapping{m})
259+ if err != nil || len(spans) != 1 || spans[0].Samples[0] != 2 {
260+ t.Fatalf("spans=%+v err=%v", spans, err)
261+ }
262+}
@@ -0,0 +1,262 @@
1+package main
2+
3+import (
4+ "bytes"
5+ "encoding/json"
6+ "errors"
7+ "os"
8+ "path/filepath"
9+ "testing"
10+ "time"
11+)
12+
13+type fakeClock struct{ tick int64 }
14+
15+func (c *fakeClock) Now() int64 { c.tick += 10; return c.tick }
16+func (*fakeClock) Sleep(time.Duration) {}
17+
18+type fakeProcess struct {
19+ fail bool
20+ warmups []time.Duration
21+ measures []time.Duration
22+ closed bool
23+}
24+
25+func (p *fakeProcess) Warmup(d time.Duration) error { p.warmups = append(p.warmups, d); return nil }
26+func (p *fakeProcess) Measure(d time.Duration, flush func()) error {
27+ p.measures = append(p.measures, d)
28+ if p.fail {
29+ return errors.New("measure failure")
30+ }
31+ flush()
32+ return nil
33+}
34+func (p *fakeProcess) Close() error { p.closed = true; return nil }
35+
36+type fakeLauncher struct {
37+ mappings []processMapping
38+ process []*fakeProcess
39+ manifestPresent bool
40+}
41+
42+func (l *fakeLauncher) Launch(m processMapping, _ []string) (launchedProcess, error) {
43+ l.mappings = append(l.mappings, m)
44+ _, e := os.Stat(filepath.Join(filepath.Dir(m.TracePath), "manifest.json"))
45+ l.manifestPresent = e == nil
46+ p := &fakeProcess{}
47+ l.process = append(l.process, p)
48+ return p, nil
49+}
50+
51+func TestHarnessFlagsRejectShortMeasurements(t *testing.T) {
52+ for _, args := range [][]string{
53+ {"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "29s", "--repetitions", "10"},
54+ {"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "9"},
55+ {"--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"},
56+ } {
57+ if _, err := parseOptions(args); err == nil {
58+ t.Fatalf("parseOptions(%v) succeeded", args)
59+ }
60+ }
61+ if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"}); err == nil {
62+ t.Fatal("missing --warmup accepted")
63+ }
64+ if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}); err != nil {
65+ t.Fatal(err)
66+ }
67+}
68+
69+func TestHarnessManifestCoversCanonicalMatrix(t *testing.T) {
70+ m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
71+ if err != nil {
72+ t.Fatal(err)
73+ }
74+ if err := validateManifest(m); err != nil {
75+ t.Fatal(err)
76+ }
77+ if len(m.Scenarios) != 4*9*6 {
78+ t.Fatalf("scenarios=%d", len(m.Scenarios))
79+ }
80+ m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
81+ if err := validateManifest(m); err == nil {
82+ t.Fatal("missing combination accepted")
83+ }
84+}
85+
86+func TestHarnessCreatesExclusiveTraceManifestAndEvidence(t *testing.T) {
87+ dir := t.TempDir()
88+ raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
89+ if err != nil {
90+ t.Fatal(err)
91+ }
92+ defer raw.Close()
93+ launcher := &fakeLauncher{}
94+ h := defaultHarness()
95+ h.clock = &fakeClock{}
96+ h.launcher = launcher
97+ o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
98+ r, err := h.runOne(o, manifest{}, scenario{ID: "4x4-local", Roles: []string{"daemon", "client"}}, 1, raw)
99+ if err != nil {
100+ t.Fatal(err)
101+ }
102+ if !launcher.manifestPresent {
103+ t.Fatal("launched before complete run manifest")
104+ }
105+ if len(r.EndToEnd) != 2 || r.Samples != 2 {
106+ t.Fatalf("samples=%+v", r)
107+ }
108+ seen := map[string]bool{}
109+ for _, m := range r.Processes {
110+ if m.ProcessID == "" || m.ClockDomain != m.ProcessID || m.TracePath == "" || seen[m.TracePath] {
111+ t.Fatalf("bad mapping %+v", m)
112+ }
113+ seen[m.TracePath] = true
114+ if _, err := os.Stat(m.TracePath); err != nil {
115+ t.Fatal(err)
116+ }
117+ }
118+ b, err := os.ReadFile(filepath.Join(dir, "4x4-local-run-001", "manifest.json"))
119+ if err != nil {
120+ t.Fatal(err)
121+ }
122+ var got runManifest
123+ if err := json.Unmarshal(b, &got); err != nil {
124+ t.Fatal(err)
125+ }
126+ if len(got.Processes) != 2 {
127+ t.Fatalf("manifest=%+v", got)
128+ }
129+ if len(launcher.process) != 2 || !launcher.process[0].closed || launcher.process[0].warmups[0] != time.Second || launcher.process[0].measures[0] != minimumDuration {
130+ t.Fatalf("process lifecycle not recorded: %+v", launcher.process)
131+ }
132+}
133+
134+func TestHarnessRejectsInvalidBoundaryPairs(t *testing.T) {
135+ for _, marks := range [][]harnessMark{
136+ {{Sequence: 1, Kind: "terminal_flushed", Tick: 1, Valid: true}},
137+ {{Sequence: 1, Kind: "input_injected", Tick: 2, Valid: true}, {Sequence: 1, Kind: "terminal_flushed", Tick: 1, Valid: true}},
138+ {{Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}},
139+ {{Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}, {Sequence: 1, Kind: "input_injected", Tick: 2, Valid: true}},
140+ } {
141+ if _, err := pairedSamples(marks); err == nil {
142+ t.Fatalf("invalid marks accepted: %+v", marks)
143+ }
144+ }
145+ got, err := pairedSamples([]harnessMark{{Sequence: 9, Kind: "input_injected", Tick: 1, Valid: true}, {Sequence: 9, Kind: "terminal_flushed", Tick: 5, Valid: true}})
146+ if err != nil || len(got) != 1 || got[0] != 4 {
147+ t.Fatalf("got %v %v", got, err)
148+ }
149+}
150+
151+func TestHarnessRawRecordsAreDeterministicJSONL(t *testing.T) {
152+ var b bytes.Buffer
153+ for _, m := range []harnessMark{{Scenario: "s", Run: 1, Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}, {Scenario: "s", Run: 1, Sequence: 1, Kind: "terminal_flushed", Tick: 2, Valid: true}} {
154+ b.WriteString(mustJSON(m))
155+ b.WriteByte('\n')
156+ }
157+ path := filepath.Join(t.TempDir(), "raw.jsonl")
158+ if err := os.WriteFile(path, b.Bytes(), 0o600); err != nil {
159+ t.Fatal(err)
160+ }
161+ if got, err := readJSONL(path); err != nil || len(got) != 2 {
162+ t.Fatalf("records=%d err=%v", len(got), err)
163+ }
164+}
165+
166+type traceLauncher struct{ fakeLauncher }
167+
168+func (l *traceLauncher) Launch(m processMapping, args []string) (launchedProcess, error) {
169+ p, err := l.fakeLauncher.Launch(m, args)
170+ if err != nil {
171+ return nil, err
172+ }
173+ f, err := os.OpenFile(m.TracePath, os.O_WRONLY|os.O_APPEND, 0o600)
174+ if err != nil {
175+ return nil, err
176+ }
177+ defer f.Close()
178+ for i, pair := range spanPairs {
179+ for n, kind := range []string{pair.start, pair.end} {
180+ b, _ := json.Marshal(traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: uint64(i + 1), RequestID: 1, Epoch: 1, Kind: kind, Tick: int64(i*20 + n*5)})
181+ if _, err := f.Write(append(b, '\n')); err != nil {
182+ return nil, err
183+ }
184+ }
185+ }
186+ return p, nil
187+}
188+func TestHarnessWritesRequiredEvidenceWithSufficientSpans(t *testing.T) {
189+ base, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
190+ if err != nil {
191+ t.Fatal(err)
192+ }
193+ for i := range base.Scenarios {
194+ if i > 0 {
195+ base.Scenarios[i].InapplicableReason = "fixture limits this test to one public topology"
196+ base.Scenarios[i].Roles = nil
197+ }
198+ }
199+ dir := t.TempDir()
200+ manifestPath := filepath.Join(dir, "manifest.json")
201+ if err := os.WriteFile(manifestPath, []byte(mustJSON(base)), 0o600); err != nil {
202+ t.Fatal(err)
203+ }
204+ l := &traceLauncher{}
205+ h := defaultHarness()
206+ h.clock = &fakeClock{}
207+ h.launcher = l
208+ h.gitSHA = func() string { return "test-sha" }
209+ if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", filepath.Join(dir, "out"), "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
210+ t.Fatal(err)
211+ }
212+ for _, name := range []string{"raw-harness.jsonl", "runs.json", "summary.json"} {
213+ if _, err := os.Stat(filepath.Join(dir, "out", name)); err != nil {
214+ t.Errorf("%s: %v", name, err)
215+ }
216+ }
217+ var got summary
218+ b, err := os.ReadFile(filepath.Join(dir, "out", "summary.json"))
219+ if err != nil {
220+ t.Fatal(err)
221+ }
222+ if err := json.Unmarshal(b, &got); err != nil {
223+ t.Fatal(err)
224+ }
225+ if got.EndToEnd.Count != 20 || got.Repetitions != 10 || len(got.Spans) == 0 {
226+ t.Fatalf("summary=%+v", got)
227+ }
228+ for _, s := range got.Spans {
229+ if s.Distribution.Count < 10 {
230+ t.Fatalf("insufficient span summary: %+v", s)
231+ }
232+ }
233+}
234+func TestHarnessRejectsCrossProcessAndBadTraceSpans(t *testing.T) {
235+ write := func(t *testing.T, records ...traceRecord) processMapping {
236+ t.Helper()
237+ path := filepath.Join(t.TempDir(), "trace.jsonl")
238+ f, err := os.Create(path)
239+ if err != nil {
240+ t.Fatal(err)
241+ }
242+ for _, r := range records {
243+ b, _ := json.Marshal(r)
244+ _, _ = f.Write(append(b, '\n'))
245+ }
246+ _ = f.Close()
247+ return processMapping{ProcessID: "one", ClockDomain: "one", TracePath: path, Scenario: "s", Run: 1}
248+ }
249+ base := func(kind string, tick int64) traceRecord {
250+ return traceRecord{ProcessID: "one", Component: "daemon", Scenario: "s", Run: 1, Sequence: 1, RequestID: 1, Epoch: 1, Kind: kind, Tick: tick}
251+ }
252+ for _, records := range [][]traceRecord{{base("diff_end", 1)}, {base("diff_start", 2), base("diff_end", 1)}, {base("diff_start", 1)}, {func() traceRecord { r := base("diff_start", 1); r.ProcessID = "other"; return r }()}} {
253+ if _, err := mergeProcessTraces([]processMapping{write(t, records...)}); err == nil {
254+ t.Fatalf("invalid trace accepted: %+v", records)
255+ }
256+ }
257+ m := write(t, base("diff_start", 1), base("diff_end", 3))
258+ spans, err := mergeProcessTraces([]processMapping{m})
259+ if err != nil || len(spans) != 1 || spans[0].Samples[0] != 2 {
260+ t.Fatalf("spans=%+v err=%v", spans, err)
261+ }
262+}
A testdata/perf/README.md +24 -0
@@ -0,0 +1,24 @@
1+# Public transport-observability harness
2+
3+`manifest.json` is the complete S5a measurement matrix: 1x4, 4x1, 4x4, and
4+8x1 layouts at 120x40 with 10,000 rows per pane; every canonical workload; and
5+local, SSH stdio, UDP baseline, 25/100ms RTT, and 1% loss transport fixtures.
6+Each listed scenario is applicable. A future unavailable combination must remain
7+in the manifest with `inapplicable_reason` and no launched roles; omission is
8+invalid.
9+
10+Run only after building the public binary:
11+
12+```sh
13+go build -o /tmp/vev-perf ./
14+go run ./cmd/vev-perf-harness --vev-bin /tmp/vev-perf \
15+ --manifest testdata/perf/manifest.json --out /tmp/vev-perf-results \
16+ --warmup 10s --duration 30s --repetitions 10
17+```
18+
19+The harness creates one `O_EXCL` JSONL path per launched role before launch and
20+records the assigned process ID, clock domain, role, and path in that run's
21+manifest. `raw-harness.jsonl` contains only harness-clock input/terminal-flush
22+pairs. Process traces are correlated by IDs, never by timestamps across process
23+clock domains. Results with missing pairs, bad order, insufficient samples, or
24+cross-process tick arithmetic are invalid.
@@ -0,0 +1,24 @@
1+# Public transport-observability harness
2+
3+`manifest.json` is the complete S5a measurement matrix: 1x4, 4x1, 4x4, and
4+8x1 layouts at 120x40 with 10,000 rows per pane; every canonical workload; and
5+local, SSH stdio, UDP baseline, 25/100ms RTT, and 1% loss transport fixtures.
6+Each listed scenario is applicable. A future unavailable combination must remain
7+in the manifest with `inapplicable_reason` and no launched roles; omission is
8+invalid.
9+
10+Run only after building the public binary:
11+
12+```sh
13+go build -o /tmp/vev-perf ./
14+go run ./cmd/vev-perf-harness --vev-bin /tmp/vev-perf \
15+ --manifest testdata/perf/manifest.json --out /tmp/vev-perf-results \
16+ --warmup 10s --duration 30s --repetitions 10
17+```
18+
19+The harness creates one `O_EXCL` JSONL path per launched role before launch and
20+records the assigned process ID, clock domain, role, and path in that run's
21+manifest. `raw-harness.jsonl` contains only harness-clock input/terminal-flush
22+pairs. Process traces are correlated by IDs, never by timestamps across process
23+clock domains. Results with missing pairs, bad order, insufficient samples, or
24+cross-process tick arithmetic are invalid.
A testdata/perf/manifest.json +2407 -0 large diff (2407 lines changed): collapsed
load diff

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 12

Tags 11

↑↓ moveenter openesc close