refactor(obs): decompose performance harness

commit: 0a1ab39ba168abcf685657f34454d62115104fcb
author: brice · 3mo ago · 2026-07-13 03:26 UTC
committer: brice · 3mo ago · 2026-07-13 03:30 UTC
parent: 7e47dd8
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

15 files · +2130 -2045

R cmd/vev-perf-harness/harness_3.go → cmd/vev-perf-harness/evidence.go +6 -2 renamed, 95% similar


@@ -143,9 +143,8 @@ type traceRecord struct {
143	143		Tick int64 `json:"tick"`
144	144		Valid bool `json:"valid"`
145	145		}
146		-	type spanPair struct{ start, end, name string }
147	146
148		-	var spanPairs = []spanPair{{"capture_start", "capture_end", "capture_duration"}, {"compose_start", "compose_end", "compose_duration"}, {"diff_start", "diff_end", "diff_duration"}, {"queue_enqueued", "queue_dequeued", "queue_wait"}, {"ack_blocked_start", "ack_blocked_end", "ack_blocked_interval"}, {"emit_start", "emit_end", "emit_duration"}, {"adapter_send_start", "adapter_send_end", "adapter_send_duration"}, {"adapter_receive_start", "adapter_receive_end", "adapter_receive_duration"}}
	147	+	type spanPair struct{ start, end, name string }
149	148
150	149		// mergeProcessTraces permits only records from one manifest process in a span.
151	150		// In particular, the correlation key includes process_id before ticks are read.

@@ -238,6 +237,7 @@ func mergeProcessTraces(mappings []processMapping) ([]span, error) {
238	237		}
239	238		return out, nil
240	239		}
	240	+
241	241		func summarizeSpans(all []span) []spanSummary {
242	242		grouped := map[string]*span{}
243	243		for _, s := range all {

@@ -270,6 +270,7 @@ func percentiles(samples []int64) distribution {
270	270		at := func(p int) int64 { return v[(len(v)-1)*p/100] }
271	271		return distribution{len(v), at(50), at(95), at(99), v[len(v)-1]}
272	272		}
	273	+
273	274		func writeJSON(path string, v any) error {
274	275		f, e := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
275	276		if e != nil {

@@ -279,7 +280,9 @@ func writeJSON(path string, v any) error {
279	280		e = json.NewEncoder(f).Encode(v)
280	281		return e
281	282		}
	283	+
282	284		func mustJSON(v any) string { b, _ := json.Marshal(v); return string(b) }
	285	+
283	286		func safeName(s string) string {
284	287		return strings.Map(func(r rune) rune {
285	288		if r >= 'a' && r <= 'z' \|\| r >= 'A' && r <= 'Z' \|\| r >= '0' && r <= '9' \|\| r == '-' \|\| r == '_' {

@@ -288,6 +291,7 @@ func safeName(s string) string {
288	291		return '-'
289	292		}, s)
290	293		}
	294	+
291	295		func readJSONL(path string) ([]map[string]json.RawMessage, error) {
292	296		f, e := os.Open(path)
293	297		if e != nil {

A cmd/vev-perf-harness/evidence_test.go +359 -0

@@ -0,0 +1,359 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"encoding/json"
	6	+	"os"
	7	+	"path/filepath"
	8	+	"strings"
	9	+	"testing"
	10	+	"time"
	11	+	)
	12	+
	13	+	func TestCLITransportSeamOwnsExclusivePeerTraceAndCleanup(t *testing.T) {
	14	+	fixtures := []struct {
	15	+	name string
	16	+	tr transport
	17	+	rtt time.Duration
	18	+	loss int
	19	+	}{
	20	+	{"baseline", transport{ID: "udp_baseline", Kind: "udp"}, 0, 0},
	21	+	{"25ms", transport{ID: "udp_25ms", Kind: "udp", RTTMS: 25}, 25 * time.Millisecond, 0},
	22	+	{"100ms", transport{ID: "udp_100ms", Kind: "udp", RTTMS: 100}, 100 * time.Millisecond, 0},
	23	+	{"loss0", transport{ID: "udp_loss_0pct", Kind: "udp"}, 0, 0},
	24	+	{"loss1", transport{ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1}, 0, 1},
	25	+	}
	26	+	for _, tc := range fixtures {
	27	+	t.Run(tc.name, func(t *testing.T) {
	28	+	dir := t.TempDir()
	29	+	if err := os.Chmod(dir, 0o700); err != nil {
	30	+	t.Fatal(err)
	31	+	}
	32	+	var configs []udpNetemConfig
	33	+	netem := &fakeUDPNetem{port: 45678}
	34	+	l := &cliLauncher{bin: "/bin/true", netemFactory: func(c udpNetemConfig) (udpNetem, error) {
	35	+	configs = append(configs, c)
	36	+	return netem, nil
	37	+	}}
	38	+	m := processMapping{ProcessID: "udp-peer", TracePath: filepath.Join(dir, "udp.jsonl"), Role: "udp_peer"}
	39	+	if err := os.WriteFile(m.TracePath, nil, 0o600); err != nil {
	40	+	t.Fatal(err)
	41	+	}
	42	+	p, err := l.preparePeer(m, roleCommand{Args: []string{"_udp-proxy", "work"}, Transport: tc.tr})
	43	+	if err != nil {
	44	+	t.Fatal(err)
	45	+	}
	46	+	if len(configs) != 1 \|\| configs[0].RTT != tc.rtt \|\| configs[0].LossPercent != tc.loss \|\| configs[0].TargetPath != filepath.Join(dir, "udp-peer.target") {
	47	+	t.Fatalf("fixture did not reach netem seam: %+v", configs)
	48	+	}
	49	+	shim, err := os.ReadFile(filepath.Join(dir, "ssh"))
	50	+	if err != nil {
	51	+	t.Fatal(err)
	52	+	}
	53	+	text := string(shim)
	54	+	for _, want := range []string{"_udp-proxy", m.TracePath, m.ProcessID, "udp-peer.target", "VEV-UDP %s %s\\n' 45678", "udp-peer.pid"} {
	55	+	if !strings.Contains(text, want) {
	56	+	t.Errorf("seam does not retain %q:\n%s", want, text)
	57	+	}
	58	+	}
	59	+	if strings.Contains(text, "VEV_PERF_UDP_") {
	60	+	t.Fatalf("ignored vev UDP environment was used instead of netem: %s", text)
	61	+	}
	62	+	// A nonexistent pid is already-cleaned-up; Close must still close the
	63	+	// harness-owned emulator.
	64	+	if err := p.Close(); err != nil {
	65	+	t.Fatal(err)
	66	+	}
	67	+	if !netem.closed {
	68	+	t.Fatal("harness netem was not cleaned up")
	69	+	}
	70	+	})
	71	+	}
	72	+	}
	73	+
	74	+	func TestHarnessRejectsInvalidBoundaryPairs(t *testing.T) {
	75	+	for _, marks := range [][]harnessMark{
	76	+	{{Sequence: 1, Kind: "terminal_flushed", Tick: 1, Valid: true}},
	77	+	{{Sequence: 1, Kind: "input_injected", Tick: 2, Valid: true}, {Sequence: 1, Kind: "terminal_flushed", Tick: 1, Valid: true}},
	78	+	{{Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}},
	79	+	{{Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}, {Sequence: 1, Kind: "input_injected", Tick: 2, Valid: true}},
	80	+	} {
	81	+	if _, err := pairedSamples(marks); err == nil {
	82	+	t.Fatalf("invalid marks accepted: %+v", marks)
	83	+	}
	84	+	}
	85	+	got, err := pairedSamples([]harnessMark{{Sequence: 9, Kind: "input_injected", Tick: 1, Valid: true}, {Sequence: 9, Kind: "terminal_flushed", Tick: 5, Valid: true}})
	86	+	if err != nil \|\| len(got) != 1 \|\| got[0] != 4 {
	87	+	t.Fatalf("got %v %v", got, err)
	88	+	}
	89	+	}
	90	+
	91	+	func TestHarnessCollectsOnlyCompleteMeasuredIntervalEvents(t *testing.T) {
	92	+	interval := measuredInterval{Start: 100, End: 200}
	93	+	marks := []harnessMark{
	94	+	{Sequence: 1, Kind: "input_injected", Tick: 10, Valid: true},
	95	+	{Sequence: 1, Kind: "terminal_flushed", Tick: 20, Valid: true}, // warmup
	96	+	{Sequence: 2, Kind: "input_injected", Tick: 100, Valid: true},
	97	+	{Sequence: 2, Kind: "terminal_flushed", Tick: 120, Valid: true},
	98	+	{Sequence: 3, Kind: "input_injected", Tick: 150, Valid: true},
	99	+	{Sequence: 3, Kind: "terminal_flushed", Tick: 201, Valid: true}, // exits interval
	100	+	{Sequence: 4, Kind: "input_injected", Tick: 99, Valid: true},
	101	+	{Sequence: 4, Kind: "terminal_flushed", Tick: 110, Valid: true}, // enters interval
	102	+	}
	103	+	events, err := measuredEventSamples(marks, interval)
	104	+	if err != nil {
	105	+	t.Fatal(err)
	106	+	}
	107	+	if len(events) != 1 \|\| events[0].Sequence != 2 \|\| events[0].Latency != 20 {
	108	+	t.Fatalf("measured events=%+v, want only complete sequence 2", events)
	109	+	}
	110	+	}
	111	+
	112	+	func TestHarnessRejectsInsufficientMeasuredEventSamples(t *testing.T) {
	113	+	marks := make([]harnessMark, 0, 2*(minimumInIntervalEventSamples-1))
	114	+	for i := 0; i < minimumInIntervalEventSamples-1; i++ {
	115	+	marks = append(marks,
	116	+	harnessMark{Sequence: uint64(i + 1), Kind: "input_injected", Tick: int64(i * 100), Valid: true},
	117	+	harnessMark{Sequence: uint64(i + 1), Kind: "terminal_flushed", Tick: int64(i*100 + 10), Valid: true},
	118	+	)
	119	+	}
	120	+	events, err := measuredEventSamples(marks, measuredInterval{Start: 0, End: int64(minimumInIntervalEventSamples * 100)})
	121	+	if err != nil {
	122	+	t.Fatal(err)
	123	+	}
	124	+	if err := requireMinimumEventSamples(events); err == nil {
	125	+	t.Fatal("insufficient measured event samples accepted")
	126	+	}
	127	+	}
	128	+
	129	+	func TestHarnessRunRejectsStraddlingEventsAsInsufficient(t *testing.T) {
	130	+	dir := t.TempDir()
	131	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	132	+	if err != nil {
	133	+	t.Fatal(err)
	134	+	}
	135	+	defer raw.Close()
	136	+	clock := &fakeClock{}
	137	+	h := defaultHarness()
	138	+	h.clock, h.launcher = clock, insufficientEventLauncher{clock: clock}
	139	+	_, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "late", Roles: []string{"daemon", "client"}}, 1, raw)
	140	+	if err == nil \|\| !strings.Contains(err.Error(), "insufficient in-interval event samples") {
	141	+	t.Fatalf("straddling run error=%v, want insufficient in-interval sample rejection", err)
	142	+	}
	143	+	}
	144	+
	145	+	func TestHarnessCalculatesEventCadenceAndMaxGapFromRawPairs(t *testing.T) {
	146	+	events := []eventSample{
	147	+	{Sequence: 1, Injected: 100, Flushed: 110, Latency: 10},
	148	+	{Sequence: 2, Injected: 130, Flushed: 140, Latency: 10},
	149	+	{Sequence: 3, Injected: 180, Flushed: 190, Latency: 10},
	150	+	}
	151	+	cadence, maxGap := eventCadence(events)
	152	+	if cadence.Count != 2 \|\| cadence.P50 != 30 \|\| cadence.P95 != 30 \|\| cadence.P99 != 30 \|\| maxGap != 50 {
	153	+	t.Fatalf("cadence=%+v maxGap=%d", cadence, maxGap)
	154	+	}
	155	+	}
	156	+
	157	+	func TestHarnessIncludesAllRequiredProcessLocalSpans(t *testing.T) {
	158	+	got := make([]string, 0, len(spanPairs))
	159	+	for _, pair := range spanPairs {
	160	+	got = append(got, pair.name)
	161	+	}
	162	+	want := []string{"capture_duration", "compose_duration", "diff_duration", "queue_wait", "ack_blocked_interval", "emit_duration", "adapter_send_duration", "adapter_receive_duration"}
	163	+	if !equalStrings(got, want) {
	164	+	t.Fatalf("span pairs = %v, want %v", got, want)
	165	+	}
	166	+	}
	167	+
	168	+	func TestHarnessRawRecordsAreDeterministicJSONL(t *testing.T) {
	169	+	var b bytes.Buffer
	170	+	for _, m := range []harnessMark{{Scenario: "s", Run: 1, Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}, {Scenario: "s", Run: 1, Sequence: 1, Kind: "terminal_flushed", Tick: 2, Valid: true}} {
	171	+	b.WriteString(mustJSON(m))
	172	+	b.WriteByte('\n')
	173	+	}
	174	+	path := filepath.Join(t.TempDir(), "raw.jsonl")
	175	+	if err := os.WriteFile(path, b.Bytes(), 0o600); err != nil {
	176	+	t.Fatal(err)
	177	+	}
	178	+	if got, err := readJSONL(path); err != nil \|\| len(got) != 2 {
	179	+	t.Fatalf("records=%d err=%v", len(got), err)
	180	+	}
	181	+	}
	182	+
	183	+	func TestHarnessClosesRolesBeforeMergingReceiveSpans(t *testing.T) {
	184	+	dir := t.TempDir()
	185	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	186	+	if err != nil {
	187	+	t.Fatal(err)
	188	+	}
	189	+	defer raw.Close()
	190	+	launcher := &closeTraceLauncher{}
	191	+	h := defaultHarness()
	192	+	h.clock, h.launcher = &fakeClock{}, launcher
	193	+	result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, scenario{ID: "receive-cleanup", Roles: []string{"daemon", "client"}}, 1, raw)
	194	+	if err != nil {
	195	+	t.Fatal(err)
	196	+	}
	197	+	if len(result.Spans) != 2 {
	198	+	t.Fatalf("post-cleanup receive spans=%+v", result.Spans)
	199	+	}
	200	+	for _, s := range result.Spans {
	201	+	if s.Name != "adapter_receive_duration" \|\| len(s.Samples) != 1 \|\| s.Samples[0] != 10 {
	202	+	t.Fatalf("unmatched or wrong receive span: %+v", s)
	203	+	}
	204	+	}
	205	+	for i, p := range launcher.process {
	206	+	if !p.closed {
	207	+	t.Fatalf("role %s was not closed before trace merge", launcher.mappings[i].Role)
	208	+	}
	209	+	}
	210	+	}
	211	+
	212	+	func TestHarnessWritesRequiredEvidenceWithSufficientSpans(t *testing.T) {
	213	+	base, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
	214	+	if err != nil {
	215	+	t.Fatal(err)
	216	+	}
	217	+	for i := range base.Scenarios {
	218	+	if i > 0 {
	219	+	base.Scenarios[i].InapplicableReason = "fixture limits this test to one public topology"
	220	+	base.Scenarios[i].Roles = nil
	221	+	}
	222	+	}
	223	+	dir := t.TempDir()
	224	+	manifestPath := filepath.Join(dir, "manifest.json")
	225	+	if err := os.WriteFile(manifestPath, []byte(mustJSON(base)), 0o600); err != nil {
	226	+	t.Fatal(err)
	227	+	}
	228	+	l := &traceLauncher{}
	229	+	h := defaultHarness()
	230	+	h.clock = &fakeClock{}
	231	+	h.launcher = l
	232	+	h.gitSHA = func() string { return "test-sha" }
	233	+	if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", filepath.Join(dir, "out"), "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
	234	+	t.Fatal(err)
	235	+	}
	236	+	for _, name := range []string{"raw-harness.jsonl", "runs.json", "summary.json"} {
	237	+	if _, err := os.Stat(filepath.Join(dir, "out", name)); err != nil {
	238	+	t.Errorf("%s: %v", name, err)
	239	+	}
	240	+	}
	241	+	var got summary
	242	+	b, err := os.ReadFile(filepath.Join(dir, "out", "summary.json"))
	243	+	if err != nil {
	244	+	t.Fatal(err)
	245	+	}
	246	+	if err := json.Unmarshal(b, &got); err != nil {
	247	+	t.Fatal(err)
	248	+	}
	249	+	if got.EndToEnd.Count < 10*minimumInIntervalEventSamples \|\| got.Repetitions != 10 \|\| got.RunP50Dispersion.Count != 10 \|\| got.Cadence.Count == 0 \|\| got.MaxGap == 0 \|\| len(got.Spans) == 0 {
	250	+	t.Fatalf("summary=%+v", got)
	251	+	}
	252	+	for _, s := range got.Spans {
	253	+	if s.Distribution.Count < 10 {
	254	+	t.Fatalf("insufficient span summary: %+v", s)
	255	+	}
	256	+	}
	257	+	}
	258	+
	259	+	func TestHarnessAcceptsConcurrentProductionTraceInterleaving(t *testing.T) {
	260	+	path := filepath.Join(t.TempDir(), "local-client.jsonl")
	261	+	if err := os.WriteFile(path, []byte(boundedLocalConcurrentTrace), 0o600); err != nil {
	262	+	t.Fatal(err)
	263	+	}
	264	+	spans, err := mergeProcessTraces([]processMapping{{
	265	+	ProcessID: "1x4-idle-local-r002-client-02", ClockDomain: "1x4-idle-local-r002-client-02", TracePath: path,
	266	+	Scenario: "1x4-idle-local", Run: 2,
	267	+	}})
	268	+	if err != nil {
	269	+	t.Fatalf("valid concurrent production trace rejected: %v", err)
	270	+	}
	271	+	if len(spans) != 2 \|\| spans[0].Component != "ipc" \|\| spans[0].Name != "adapter_send_duration" \|\| spans[0].Samples[0] != 21900 \|\| spans[1].Component != "ipc" \|\| spans[1].Name != "adapter_receive_duration" \|\| spans[1].Samples[0] != 178550 {
	272	+	t.Fatalf("spans=%+v", spans)
	273	+	}
	274	+	}
	275	+
	276	+	func TestHarnessRejectsCrossProcessAndBadTraceSpans(t *testing.T) {
	277	+	write := func(t *testing.T, records ...traceRecord) processMapping {
	278	+	t.Helper()
	279	+	path := filepath.Join(t.TempDir(), "trace.jsonl")
	280	+	f, err := os.Create(path)
	281	+	if err != nil {
	282	+	t.Fatal(err)
	283	+	}
	284	+	for _, r := range records {
	285	+	b, _ := json.Marshal(r)
	286	+	_, _ = f.Write(append(b, '\n'))
	287	+	}
	288	+	_ = f.Close()
	289	+	return processMapping{ProcessID: "one", ClockDomain: "one", TracePath: path, Scenario: "s", Run: 1}
	290	+	}
	291	+	base := func(kind string, tick int64) traceRecord {
	292	+	return traceRecord{ProcessID: "one", Component: "daemon", Scenario: "s", Run: 1, Sequence: 1, RequestID: 1, Epoch: 1, Kind: kind, Tick: tick, Valid: true}
	293	+	}
	294	+	for _, records := range [][]traceRecord{
	295	+	{base("diff_end", 1)},
	296	+	{base("diff_start", 2), base("diff_end", 1)},
	297	+	{base("diff_start", 1)},
	298	+	{base("diff_start", 1), base("diff_start", 2)},
	299	+	{base("diff_start", 1), func() traceRecord { r := base("diff_end", 2); r.Component = "ipc"; return r }()},
	300	+	{func() traceRecord { r := base("diff_start", 1); r.ProcessID = "other"; return r }()},
	301	+	{func() traceRecord { r := base("diff_start", 1); r.Scenario = "other-scenario"; return r }()},
	302	+	{func() traceRecord { r := base("diff_start", 1); r.Run = 2; return r }()},
	303	+	} {
	304	+	if _, err := mergeProcessTraces([]processMapping{write(t, records...)}); err == nil {
	305	+	t.Fatalf("invalid trace accepted: %+v", records)
	306	+	}
	307	+	}
	308	+	m := write(t, base("diff_start", 1), base("diff_end", 3))
	309	+	spans, err := mergeProcessTraces([]processMapping{m})
	310	+	if err != nil \|\| len(spans) != 1 \|\| spans[0].Samples[0] != 2 {
	311	+	t.Fatalf("spans=%+v err=%v", spans, err)
	312	+	}
	313	+	}
	314	+
	315	+	func TestHarnessExcludesFailedSpanDurationsWhileValidatingPairing(t *testing.T) {
	316	+	base := func(kind string, tick int64, valid bool) traceRecord {
	317	+	return traceRecord{ProcessID: "one", Component: "adapter", Scenario: "s", Run: 1, Sequence: 1, RequestID: 1, Epoch: 1, Kind: kind, Tick: tick, Valid: valid}
	318	+	}
	319	+	for _, tc := range []struct {
	320	+	name string
	321	+	records []traceRecord
	322	+	wantSpans int
	323	+	wantErr bool
	324	+	}{
	325	+	{"successful adapter send is sampled", []traceRecord{base("adapter_send_start", 10, true), base("adapter_send_end", 20, true)}, 1, false},
	326	+	{"failed adapter send end is excluded", []traceRecord{base("adapter_send_start", 10, true), base("adapter_send_end", 20, false)}, 0, false},
	327	+	{"failed adapter receive end is excluded", []traceRecord{base("adapter_receive_start", 10, true), base("adapter_receive_end", 20, false)}, 0, false},
	328	+	{"failed adapter send start is excluded", []traceRecord{base("adapter_send_start", 10, false), base("adapter_send_end", 20, true)}, 0, false},
	329	+	{"failed adapter receive boundaries are excluded", []traceRecord{base("adapter_receive_start", 10, false), base("adapter_receive_end", 20, false)}, 0, false},
	330	+	{"failed end without start is rejected", []traceRecord{base("adapter_send_end", 20, false)}, 0, true},
	331	+	{"duplicate failed start is rejected", []traceRecord{base("adapter_send_start", 10, false), base("adapter_send_start", 15, false)}, 0, true},
	332	+	{"negative failed span is rejected", []traceRecord{base("adapter_receive_start", 20, false), base("adapter_receive_end", 10, false)}, 0, true},
	333	+	{"cross-component failed end is rejected", []traceRecord{base("adapter_send_start", 10, true), func() traceRecord { r := base("adapter_send_end", 20, false); r.Component = "other"; return r }()}, 0, true},
	334	+	} {
	335	+	t.Run(tc.name, func(t *testing.T) {
	336	+	path := filepath.Join(t.TempDir(), "trace.jsonl")
	337	+	var lines []string
	338	+	for _, record := range tc.records {
	339	+	lines = append(lines, mustJSON(record))
	340	+	}
	341	+	if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {
	342	+	t.Fatal(err)
	343	+	}
	344	+	spans, err := mergeProcessTraces([]processMapping{{ProcessID: "one", ClockDomain: "one", TracePath: path, Scenario: "s", Run: 1}})
	345	+	if tc.wantErr {
	346	+	if err == nil {
	347	+	t.Fatalf("invalid trace accepted: %+v", tc.records)
	348	+	}
	349	+	return
	350	+	}
	351	+	if err != nil \|\| len(spans) != tc.wantSpans {
	352	+	t.Fatalf("spans=%+v err=%v, want %d spans", spans, err, tc.wantSpans)
	353	+	}
	354	+	if tc.wantSpans == 1 && spans[0].Samples[0] != 10 {
	355	+	t.Fatalf("successful span=%+v, want 10ns", spans[0])
	356	+	}
	357	+	})
	358	+	}
	359	+	}

D cmd/vev-perf-harness/harness_test_1_test.go +0 -746

@@ -1,746 +0,0 @@
1	-	package main
2	-
3	-	import (
4	-	"bytes"
5	-	"encoding/json"
6	-	"errors"
7	-	"fmt"
8	-	"net"
9	-	"os"
10	-	"os/exec"
11	-	"path/filepath"
12	-	"strings"
13	-	"testing"
14	-	"time"
15	-	)
16	-
17	-	type fakeClock struct{ tick int64 }
18	-
19	-	func (c *fakeClock) Now() int64 { c.tick += 10; return c.tick }
20	-	func (c *fakeClock) Sleep(d time.Duration) {
21	-	c.tick += d.Nanoseconds()
22	-	}
23	-
24	-	type fakeUDPNetem struct {
25	-	port int
26	-	closed bool
27	-	}
28	-
29	-	func (n *fakeUDPNetem) Port() int { return n.port }
30	-	func (n *fakeUDPNetem) Close() error { n.closed = true; return nil }
31	-
32	-	type fakeProcess struct {
33	-	fail bool
34	-	measureErr error
35	-	closeErr error
36	-	warmups [][]byte
37	-	measures [][]byte
38	-	closed bool
39	-	}
40	-
41	-	func (p *fakeProcess) Warmup(input []byte) error {
42	-	p.warmups = append(p.warmups, append([]byte(nil), input...))
43	-	return nil
44	-	}
45	-	func (p *fakeProcess) Measure(input []byte, injected, flush func() error) error {
46	-	p.measures = append(p.measures, append([]byte(nil), input...))
47	-	if p.measureErr != nil {
48	-	return p.measureErr
49	-	}
50	-	if p.fail {
51	-	return errors.New("measure failure")
52	-	}
53	-	if err := injected(); err != nil {
54	-	return err
55	-	}
56	-	return flush()
57	-	}
58	-	func (p *fakeProcess) Close() error { p.closed = true; return p.closeErr }
59	-
60	-	type fakeLauncher struct {
61	-	mappings []processMapping
62	-	args [][]string
63	-	commands []roleCommand
64	-	process []*fakeProcess
65	-	measureErr map[string]error
66	-	closeErr map[string]error
67	-	manifestPresent bool
68	-	}
69	-
70	-	func (l *fakeLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
71	-	l.mappings = append(l.mappings, m)
72	-	l.args = append(l.args, append([]string(nil), command.Args...))
73	-	l.commands = append(l.commands, command)
74	-	_, e := os.Stat(filepath.Join(filepath.Dir(m.TracePath), "manifest.json"))
75	-	l.manifestPresent = e == nil
76	-	p := &fakeProcess{measureErr: l.measureErr[m.Role], closeErr: l.closeErr[m.Role]}
77	-	l.process = append(l.process, p)
78	-	return p, nil
79	-	}
80	-
81	-	type lateFlushProcess struct{ clock *fakeClock }
82	-
83	-	func (*lateFlushProcess) Warmup([]byte) error { return nil }
84	-	func (p *lateFlushProcess) Measure(_ []byte, injected, flushed func() error) error {
85	-	if err := injected(); err != nil {
86	-	return err
87	-	}
88	-	p.clock.tick += minimumDuration.Nanoseconds()
89	-	return flushed()
90	-	}
91	-	func (*lateFlushProcess) Close() error { return nil }
92	-
93	-	type insufficientEventLauncher struct{ clock *fakeClock }
94	-
95	-	func (l insufficientEventLauncher) Launch(m processMapping, _ roleCommand) (launchedProcess, error) {
96	-	if m.Role == "client" {
97	-	return &lateFlushProcess{clock: l.clock}, nil
98	-	}
99	-	return &fakeProcess{}, nil
100	-	}
101	-
102	-	func TestCLIProcessCloseIsBoundedWhenPublicRoleDoesNotExit(t *testing.T) {
103	-	deadline := make(chan time.Time)
104	-	close(deadline)
105	-	var term, kill int
106	-	p := &cliProcess{
107	-	waitErr: make(chan error),
108	-	waitTimeout: func() <-chan time.Time { return deadline },
109	-	forceCleanup: func() { term++ },
110	-	forceKill: func() { kill++ },
111	-	}
112	-	if err := p.Close(); err == nil {
113	-	t.Fatal("stuck public role close succeeded")
114	-	}
115	-	if term != 1 \|\| kill != 1 {
116	-	t.Fatalf("TERM/KILL calls = %d/%d, want 1/1", term, kill)
117	-	}
118	-	}
119	-
120	-	func TestCLIProcessWaitReadyRequiresDaemonSocket(t *testing.T) {
121	-	path := filepath.Join(t.TempDir(), "daemon.sock")
122	-	listener, err := net.Listen("unix", path)
123	-	if err != nil {
124	-	t.Fatal(err)
125	-	}
126	-	defer listener.Close()
127	-	if err := (&cliProcess{readyPath: path, waitErr: make(chan error, 1)}).WaitReady(); err != nil {
128	-	t.Fatal(err)
129	-	}
130	-	}
131	-
132	-	func TestCLILauncherIsolatesRoleXDGEnvironmentAcrossRepetitions(t *testing.T) {
133	-	captureDir := t.TempDir()
134	-	t.Setenv("CAPTURE_DIR", captureDir)
135	-	t.Setenv("XDG_RUNTIME_DIR", filepath.Join(t.TempDir(), "inherited-runtime"))
136	-	t.Setenv("XDG_STATE_HOME", filepath.Join(t.TempDir(), "inherited-state"))
137	-
138	-	bin := filepath.Join(t.TempDir(), "capture-env")
139	-	if err := os.WriteFile(bin, []byte("#!/bin/sh\nenv > \"$CAPTURE_DIR/$VEV_PERF_PROCESS_ID\"\n"), 0o700); err != nil {
140	-	t.Fatal(err)
141	-	}
142	-	launcher := &cliLauncher{bin: bin}
143	-	for run := 1; run <= 3; run++ {
144	-	runDir := filepath.Join(t.TempDir(), fmt.Sprintf("run-%d", run))
145	-	if err := os.MkdirAll(runDir, 0o700); err != nil {
146	-	t.Fatal(err)
147	-	}
148	-	roles := []string{"daemon", "client"}
149	-	for _, role := range roles {
150	-	id := fmt.Sprintf("run-%d-%s", run, role)
151	-	p, err := launcher.Launch(processMapping{
152	-	Role: role, TracePath: filepath.Join(runDir, role+".jsonl"), ProcessID: id,
153	-	}, roleCommand{Args: []string{"test"}})
154	-	if err != nil {
155	-	t.Fatalf("run %d %s launch: %v", run, role, err)
156	-	}
157	-	runtimeDir := launcher.runtimes[runDir]
158	-	if err := p.Close(); err != nil {
159	-	t.Fatalf("run %d %s close: %v", run, role, err)
160	-	}
161	-
162	-	env, err := os.ReadFile(filepath.Join(captureDir, id))
163	-	if err != nil {
164	-	t.Fatalf("run %d %s capture: %v", run, role, err)
165	-	}
166	-	for _, want := range []string{
167	-	"XDG_RUNTIME_DIR=" + runtimeDir,
168	-	"XDG_STATE_HOME=" + filepath.Join(runDir, "state"),
169	-	} {
170	-	name := strings.SplitN(want, "=", 2)[0]
171	-	if count := bytes.Count(env, []byte(name+"=")); count != 1 {
172	-	t.Fatalf("run %d %s %s entries = %d, want exactly one; env:\n%s", run, role, name, count, env)
173	-	}
174	-	if !bytes.Contains(env, []byte(want+"\n")) {
175	-	t.Fatalf("run %d %s missing %q; env:\n%s", run, role, want, env)
176	-	}
177	-	}
178	-	}
179	-	if err := launcher.releaseRuntime(runDir); err != nil {
180	-	t.Fatalf("run %d release runtime: %v", run, err)
181	-	}
182	-	}
183	-	}
184	-
185	-	func TestCLIProcessCloseUsesPublicDaemonShutdown(t *testing.T) {
186	-	wait := make(chan error, 1)
187	-	wait <- nil
188	-	var shutdown int
189	-	p := &cliProcess{waitErr: wait, shutdown: func() error { shutdown++; return nil }}
190	-	if err := p.Close(); err != nil {
191	-	t.Fatal(err)
192	-	}
193	-	if shutdown != 0 {
194	-	t.Fatalf("shutdown calls = %d, want 0 for an exited daemon", shutdown)
195	-	}
196	-
197	-	wait = make(chan error, 1)
198	-	p = &cliProcess{waitErr: wait, shutdown: func() error { shutdown++; wait <- nil; return nil }}
199	-	if err := p.Close(); err != nil {
200	-	t.Fatal(err)
201	-	}
202	-	if shutdown != 1 {
203	-	t.Fatalf("shutdown calls = %d, want 1", shutdown)
204	-	}
205	-	}
206	-
207	-	func TestHarnessCanonicalLocalSmokeRealRoles(t *testing.T) {
208	-	if testing.Short() {
209	-	t.Skip("public CLI smoke")
210	-	}
211	-	root, err := filepath.Abs(filepath.Join("..", ".."))
212	-	if err != nil {
213	-	t.Fatal(err)
214	-	}
215	-	bin := filepath.Join(t.TempDir(), "vev")
216	-	build := exec.Command("/usr/local/go/bin/go", "build", "-o", bin, "./")
217	-	build.Dir = root
218	-	if output, err := build.CombinedOutput(); err != nil {
219	-	t.Fatalf("build public CLI: %v\n%s", err, output)
220	-	}
221	-	m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
222	-	if err != nil {
223	-	t.Fatal(err)
224	-	}
225	-	if err := validateManifest(m); err != nil {
226	-	t.Fatal(err)
227	-	}
228	-	var local scenario
229	-	for _, s := range m.Scenarios {
230	-	if s.Topology == "1x4" && s.Workload == "idle" && s.Transport == "local" {
231	-	local = s
232	-	break
233	-	}
234	-	}
235	-	if local.ID == "" {
236	-	t.Fatal("canonical local scenario missing")
237	-	}
238	-	dir := t.TempDir()
239	-	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
240	-	if err != nil {
241	-	t.Fatal(err)
242	-	}
243	-	defer raw.Close()
244	-	h := defaultHarness()
245	-	h.clock = &fakeClock{} // injected duration keeps the public 30s contract bounded in test
246	-	h.launcher = &cliLauncher{bin: bin}
247	-	result, err := h.runOne(options{vevBin: bin, out: dir, duration: minimumDuration}, m, local, 1, raw)
248	-	if err != nil {
249	-	t.Fatal(err)
250	-	}
251	-	if result.Samples < minimumInIntervalEventSamples \|\| len(result.Processes) != 2 {
252	-	t.Fatalf("canonical local smoke result=%+v", result)
253	-	}
254	-	}
255	-
256	-	func TestHarnessCanonicalLocalRolesAreIsolatedAcrossRepetitions(t *testing.T) {
257	-	if testing.Short() {
258	-	t.Skip("public CLI repetition lifecycle")
259	-	}
260	-	root, err := filepath.Abs(filepath.Join("..", ".."))
261	-	if err != nil {
262	-	t.Fatal(err)
263	-	}
264	-	bin := filepath.Join(t.TempDir(), "vev")
265	-	build := exec.Command("/usr/local/go/bin/go", "build", "-o", bin, "./")
266	-	build.Dir = root
267	-	if output, err := build.CombinedOutput(); err != nil {
268	-	t.Fatalf("build public CLI: %v\n%s", err, output)
269	-	}
270	-	m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
271	-	if err != nil {
272	-	t.Fatal(err)
273	-	}
274	-	var local scenario
275	-	for _, candidate := range m.Scenarios {
276	-	if candidate.ID == "1x4-idle-local" {
277	-	local = candidate
278	-	break
279	-	}
280	-	}
281	-	if local.ID == "" {
282	-	t.Fatal("canonical local scenario missing")
283	-	}
284	-	out, err := os.MkdirTemp("", "vev-ob-")
285	-	if err != nil {
286	-	t.Fatal(err)
287	-	}
288	-	defer os.RemoveAll(out)
289	-	raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
290	-	if err != nil {
291	-	t.Fatal(err)
292	-	}
293	-	defer raw.Close()
294	-	h := defaultHarness()
295	-	h.clock = &fakeClock{}
296	-	h.launcher = &cliLauncher{bin: bin}
297	-	for run := 1; run <= 3; run++ {
298	-	result, err := h.runOne(options{vevBin: bin, out: out, duration: minimumDuration}, m, local, run, raw)
299	-	if err != nil {
300	-	t.Fatalf("run %d: %v", run, err)
301	-	}
302	-	if len(result.Processes) != 2 {
303	-	t.Fatalf("run %d process mappings = %+v", run, result.Processes)
304	-	}
305	-	runDir := filepath.Join(out, fmt.Sprintf("%s-run-%03d", safeName(local.ID), run))
306	-	manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
307	-	if err != nil {
308	-	t.Fatalf("run %d manifest: %v", run, err)
309	-	}
310	-	var persisted runManifest
311	-	if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
312	-	t.Fatalf("run %d manifest JSON: %v", run, err)
313	-	}
314	-	if persisted.Run != run \|\| len(persisted.Processes) != len(result.Processes) {
315	-	t.Fatalf("run %d manifest = %+v", run, persisted)
316	-	}
317	-	for _, process := range result.Processes {
318	-	if process.Role != "daemon" && process.Role != "client" {
319	-	t.Fatalf("run %d unexpected role %q", run, process.Role)
320	-	}
321	-	if filepath.Dir(process.TracePath) != runDir {
322	-	t.Fatalf("run %d %s trace outside its isolated run directory: %q", run, process.Role, process.TracePath)
323	-	}
324	-	trace, err := os.ReadFile(process.TracePath)
325	-	if err != nil {
326	-	t.Fatalf("run %d %s trace %q: %v", run, process.Role, process.TracePath, err)
327	-	}
328	-	if len(bytes.TrimSpace(trace)) == 0 {
329	-	t.Fatalf("run %d %s did not emit its declared trace", run, process.Role)
330	-	}
331	-	for _, line := range bytes.Split(bytes.TrimSpace(trace), []byte{'\n'}) {
332	-	var record traceRecord
333	-	if err := json.Unmarshal(line, &record); err != nil {
334	-	t.Fatalf("run %d %s trace JSON: %v", run, process.Role, err)
335	-	}
336	-	if record.ProcessID != process.ProcessID \|\| record.Scenario != local.ID \|\| record.Run != uint64(run) {
337	-	t.Fatalf("run %d %s trace escaped its manifest identity: %+v", run, process.Role, record)
338	-	}
339	-	}
340	-	}
341	-	if _, err := os.Stat(filepath.Join(runDir, "runtime", "vev", "daemon.sock")); !errors.Is(err, os.ErrNotExist) {
342	-	t.Fatalf("run %d left daemon socket for the next repetition: %v", run, err)
343	-	}
344	-	}
345	-	}
346	-
347	-	func TestHarnessResolvesCanonicalRelativePathsBeforeRoleCWDChanges(t *testing.T) {
348	-	if testing.Short() {
349	-	t.Skip("public CLI relative-path lifecycle")
350	-	}
351	-	root, err := filepath.Abs(filepath.Join("..", ".."))
352	-	if err != nil {
353	-	t.Fatal(err)
354	-	}
355	-	// Keep the real Unix-socket path below its platform limit while testing the
356	-	// exact documented relative output hierarchy.
357	-	workspace, err := os.MkdirTemp("", "vev-rel-")
358	-	if err != nil {
359	-	t.Fatal(err)
360	-	}
361	-	t.Cleanup(func() { _ = os.RemoveAll(workspace) })
362	-	bin := filepath.Join(workspace, "vev")
363	-	build := exec.Command("/usr/local/go/bin/go", "build", "-o", bin, "./")
364	-	build.Dir = root
365	-	if output, err := build.CombinedOutput(); err != nil {
366	-	t.Fatalf("build public CLI: %v\n%s", err, output)
367	-	}
368	-	manifestPath := filepath.Join(workspace, "testdata", "perf", "manifest.json")
369	-	if err := os.MkdirAll(filepath.Dir(manifestPath), 0o700); err != nil {
370	-	t.Fatal(err)
371	-	}
372	-	contents, err := os.ReadFile(filepath.Join(root, "testdata", "perf", "manifest.json"))
373	-	if err != nil {
374	-	t.Fatal(err)
375	-	}
376	-	if err := os.WriteFile(manifestPath, contents, 0o600); err != nil {
377	-	t.Fatal(err)
378	-	}
379	-	originalWD, err := os.Getwd()
380	-	if err != nil {
381	-	t.Fatal(err)
382	-	}
383	-	if err := os.Chdir(workspace); err != nil {
384	-	t.Fatal(err)
385	-	}
386	-	t.Cleanup(func() {
387	-	if err := os.Chdir(originalWD); err != nil {
388	-	t.Errorf("restore cwd: %v", err)
389	-	}
390	-	})
391	-
392	-	// These are the documented relative locations. cliLauncher changes every
393	-	// role to its run directory, so a failure to resolve them before launch
394	-	// would make VEV_PERF_TRACE point at a nonexistent nested directory.
395	-	h := defaultHarness()
396	-	h.clock = &fakeClock{}
397	-	if err := run([]string{
398	-	"--vev-bin", "./vev",
399	-	"--manifest", "testdata/perf/manifest.json",
400	-	"--out", "testdata/perf/results",
401	-	"--scenario", "1x4-idle-local",
402	-	"--warmup", "0s",
403	-	"--duration", "30s",
404	-	"--repetitions", "10",
405	-	}, h); err != nil {
406	-	t.Fatal(err)
407	-	}
408	-
409	-	out := filepath.Join(workspace, "testdata", "perf", "results")
410	-	for run := 1; run <= 10; run++ {
411	-	runDir := filepath.Join(out, fmt.Sprintf("1x4-idle-local-run-%03d", run))
412	-	manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
413	-	if err != nil {
414	-	t.Fatalf("run %d manifest: %v", run, err)
415	-	}
416	-	var persisted runManifest
417	-	if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
418	-	t.Fatalf("run %d manifest JSON: %v", run, err)
419	-	}
420	-	if persisted.Run != run \|\| len(persisted.Processes) != 2 {
421	-	t.Fatalf("run %d manifest=%+v", run, persisted)
422	-	}
423	-	for _, process := range persisted.Processes {
424	-	if !filepath.IsAbs(process.TracePath) \|\| filepath.Dir(process.TracePath) != runDir {
425	-	t.Fatalf("run %d %s trace path=%q, want absolute path in %q", run, process.Role, process.TracePath, runDir)
426	-	}
427	-	trace, err := os.ReadFile(process.TracePath)
428	-	if err != nil \|\| len(bytes.TrimSpace(trace)) == 0 {
429	-	t.Fatalf("run %d %s trace=%q err=%v", run, process.Role, process.TracePath, err)
430	-	}
431	-	}
432	-	}
433	-	}
434	-
435	-	func TestHarnessScenarioSelectionKeepsCanonicalManifestValidation(t *testing.T) {
436	-	root, err := filepath.Abs(filepath.Join("..", ".."))
437	-	if err != nil {
438	-	t.Fatal(err)
439	-	}
440	-	manifestPath := filepath.Join(root, "testdata", "perf", "manifest.json")
441	-	m, err := readManifest(manifestPath)
442	-	if err != nil {
443	-	t.Fatal(err)
444	-	}
445	-	selected := m.Scenarios[0]
446	-	h := defaultHarness()
447	-	h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
448	-	out := filepath.Join(t.TempDir(), "out")
449	-	if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
450	-	t.Fatal(err)
451	-	}
452	-	var runs []runResult
453	-	b, err := os.ReadFile(filepath.Join(out, "runs.json"))
454	-	if err != nil {
455	-	t.Fatal(err)
456	-	}
457	-	if err := json.Unmarshal(b, &runs); err != nil {
458	-	t.Fatal(err)
459	-	}
460	-	if len(runs) != 10 {
461	-	t.Fatalf("selected canonical runs=%d, want 10", len(runs))
462	-	}
463	-	for _, result := range runs {
464	-	if result.Scenario != selected.ID {
465	-	t.Fatalf("selected scenario = %q, want %q", result.Scenario, selected.ID)
466	-	}
467	-	}
468	-	}
469	-
470	-	func TestHarnessScenarioSelectionRejectsIncompleteCanonicalManifest(t *testing.T) {
471	-	m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
472	-	if err != nil {
473	-	t.Fatal(err)
474	-	}
475	-	selected := m.Scenarios[0]
476	-	m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
477	-	dir := t.TempDir()
478	-	manifestPath := filepath.Join(dir, "manifest.json")
479	-	if err := os.WriteFile(manifestPath, []byte(mustJSON(m)), 0o600); err != nil {
480	-	t.Fatal(err)
481	-	}
482	-	launcher := &fakeLauncher{}
483	-	h := defaultHarness()
484	-	h.clock, h.launcher = &fakeClock{}, launcher
485	-	out := filepath.Join(dir, "out")
486	-	err = run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h)
487	-	if err == nil \|\| !strings.Contains(err.Error(), "missing scenario or inapplicable reason") {
488	-	t.Fatalf("incomplete manifest selection error = %v", err)
489	-	}
490	-	if len(launcher.mappings) != 0 {
491	-	t.Fatalf("launched with incomplete canonical manifest: %+v", launcher.mappings)
492	-	}
493	-	if _, statErr := os.Stat(out); !errors.Is(statErr, os.ErrNotExist) {
494	-	t.Fatalf("output created before manifest validation: %v", statErr)
495	-	}
496	-	}
497	-
498	-	func TestTraceEnvironmentMatchesProcessMapping(t *testing.T) {
499	-	mapping := processMapping{TracePath: "/tmp/trace.jsonl", ProcessID: "p-01", Scenario: "1x4-idle-local", Run: 3}
500	-	got := strings.Join(traceEnvironment(mapping), "\n")
501	-	for _, want := range []string{"VEV_PERF_TRACE=/tmp/trace.jsonl", "VEV_PERF_PROCESS_ID=p-01", "VEV_PERF_SCENARIO=1x4-idle-local", "VEV_PERF_RUN=3"} {
502	-	if !strings.Contains(got, want) {
503	-	t.Errorf("trace environment missing %q: %s", want, got)
504	-	}
505	-	}
506	-	}
507	-
508	-	func TestResolvePathOptionsMakesRelativeAndAbsoluteInputsIdentical(t *testing.T) {
509	-	cwd, err := os.Getwd()
510	-	if err != nil {
511	-	t.Fatal(err)
512	-	}
513	-	absolute := options{
514	-	vevBin: filepath.Join(cwd, "vev"),
515	-	manifest: filepath.Join(cwd, "testdata", "perf", "manifest.json"),
516	-	out: filepath.Join(cwd, "testdata", "perf", "results"),
517	-	}
518	-	relative := options{
519	-	vevBin: "vev",
520	-	manifest: "testdata/perf/manifest.json",
521	-	out: "testdata/perf/results",
522	-	}
523	-	gotRelative, err := resolvePathOptions(relative)
524	-	if err != nil {
525	-	t.Fatal(err)
526	-	}
527	-	gotAbsolute, err := resolvePathOptions(absolute)
528	-	if err != nil {
529	-	t.Fatal(err)
530	-	}
531	-	if gotRelative != gotAbsolute {
532	-	t.Fatalf("relative paths resolved to %+v, absolute paths to %+v", gotRelative, gotAbsolute)
533	-	}
534	-	}
535	-
536	-	func TestHarnessFlagsRejectShortMeasurements(t *testing.T) {
537	-	for _, args := range [][]string{
538	-	{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "29s", "--repetitions", "10"},
539	-	{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "9"},
540	-	{"--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"},
541	-	} {
542	-	if _, err := parseOptions(args); err == nil {
543	-	t.Fatalf("parseOptions(%v) succeeded", args)
544	-	}
545	-	}
546	-	if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"}); err == nil {
547	-	t.Fatal("missing --warmup accepted")
548	-	}
549	-	if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}); err != nil {
550	-	t.Fatal(err)
551	-	}
552	-	}
553	-
554	-	func TestHarnessManifestCoversCanonicalMatrix(t *testing.T) {
555	-	m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
556	-	if err != nil {
557	-	t.Fatal(err)
558	-	}
559	-	if err := validateManifest(m); err != nil {
560	-	t.Fatal(err)
561	-	}
562	-	if len(m.Scenarios) != 497 {
563	-	t.Fatalf("scenarios=%d", len(m.Scenarios))
564	-	}
565	-	m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
566	-	if err := validateManifest(m); err == nil {
567	-	t.Fatal("missing combination accepted")
568	-	}
569	-	}
570	-
571	-	func TestHarnessRejectsExistingRunDirectoryBeforeLaunchingRoles(t *testing.T) {
572	-	out := t.TempDir()
573	-	raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
574	-	if err != nil {
575	-	t.Fatal(err)
576	-	}
577	-	defer raw.Close()
578	-	s := scenario{ID: "isolated", Roles: []string{"daemon", "client"}}
579	-	stale := filepath.Join(out, "isolated-run-001")
580	-	if err := os.Mkdir(stale, 0o700); err != nil {
581	-	t.Fatal(err)
582	-	}
583	-	if err := os.WriteFile(filepath.Join(stale, "preserve"), []byte("stale"), 0o600); err != nil {
584	-	t.Fatal(err)
585	-	}
586	-	launcher := &fakeLauncher{}
587	-	h := defaultHarness()
588	-	h.clock, h.launcher = &fakeClock{}, launcher
589	-	_, err = h.runOne(options{out: out, duration: minimumDuration}, manifest{}, s, 1, raw)
590	-	if err == nil \|\| !strings.Contains(err.Error(), "create isolated run directory") {
591	-	t.Fatalf("existing run directory error = %v", err)
592	-	}
593	-	if len(launcher.mappings) != 0 {
594	-	t.Fatalf("roles launched against stale run directory: %+v", launcher.mappings)
595	-	}
596	-	if contents, readErr := os.ReadFile(filepath.Join(stale, "preserve")); readErr != nil \|\| string(contents) != "stale" {
597	-	t.Fatalf("stale directory changed: contents=%q err=%v", contents, readErr)
598	-	}
599	-	}
600	-
601	-	func TestHarnessCreatesExclusiveTraceManifestAndEvidence(t *testing.T) {
602	-	dir := t.TempDir()
603	-	raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
604	-	if err != nil {
605	-	t.Fatal(err)
606	-	}
607	-	defer raw.Close()
608	-	launcher := &fakeLauncher{}
609	-	h := defaultHarness()
610	-	h.clock = &fakeClock{}
611	-	h.launcher = launcher
612	-	o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
613	-	r, err := h.runOne(o, manifest{}, scenario{ID: "4x4-local", Roles: []string{"daemon", "client"}}, 1, raw)
614	-	if err != nil {
615	-	t.Fatal(err)
616	-	}
617	-	if !launcher.manifestPresent {
618	-	t.Fatal("launched before complete run manifest")
619	-	}
620	-	if len(r.EndToEnd) < minimumInIntervalEventSamples \|\| r.Samples < minimumInIntervalEventSamples {
621	-	t.Fatalf("samples=%+v", r)
622	-	}
623	-	seen := map[string]bool{}
624	-	for _, m := range r.Processes {
625	-	if m.ProcessID == "" \|\| m.ClockDomain != m.ProcessID \|\| m.TracePath == "" \|\| seen[m.TracePath] {
626	-	t.Fatalf("bad mapping %+v", m)
627	-	}
628	-	seen[m.TracePath] = true
629	-	if _, err := os.Stat(m.TracePath); err != nil {
630	-	t.Fatal(err)
631	-	}
632	-	}
633	-	b, err := os.ReadFile(filepath.Join(dir, "4x4-local-run-001", "manifest.json"))
634	-	if err != nil {
635	-	t.Fatal(err)
636	-	}
637	-	var got runManifest
638	-	if err := json.Unmarshal(b, &got); err != nil {
639	-	t.Fatal(err)
640	-	}
641	-	if len(got.Processes) != 2 {
642	-	t.Fatalf("manifest=%+v", got)
643	-	}
644	-	if len(launcher.process) != 2 \|\| !launcher.process[0].closed \|\| len(launcher.process[0].warmups) != 0 \|\| len(launcher.process[1].warmups) != 1 \|\| len(launcher.process[1].measures) < minimumInIntervalEventSamples {
645	-	t.Fatalf("process lifecycle not recorded: %+v", launcher.process)
646	-	}
647	-	}
648	-
649	-	type fakePTY struct{ writes [][]byte }
650	-
651	-	func (p *fakePTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
652	-	func (p *fakePTY) Write(b []byte) (int, error) {
653	-	p.writes = append(p.writes, append([]byte(nil), b...))
654	-	return len(b), nil
655	-	}
656	-	func (*fakePTY) Close() error { return nil }
657	-
658	-	type stagedPTY struct {
659	-	allow <-chan struct{}
660	-	writes chan []byte
661	-	}
662	-
663	-	func (*stagedPTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
664	-	func (p *stagedPTY) Write(b []byte) (int, error) {
665	-	if p.allow != nil {
666	-	<-p.allow
667	-	}
668	-	p.writes <- append([]byte(nil), b...)
669	-	return len(b), nil
670	-	}
671	-	func (*stagedPTY) Close() error { return nil }
672	-
673	-	type fakeOutput struct {
674	-	bytes.Buffer
675	-	syncs int
676	-	}
677	-
678	-	func (o *fakeOutput) Sync() error { o.syncs++; return nil }
679	-	func (*fakeOutput) Close() error { return nil }
680	-
681	-	type orderedPTY struct {
682	-	write func([]byte)
683	-	close func()
684	-	}
685	-
686	-	func (*orderedPTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
687	-	func (p *orderedPTY) Write(b []byte) (int, error) {
688	-	if p.write != nil {
689	-	p.write(b)
690	-	}
691	-	return len(b), nil
692	-	}
693	-	func (p *orderedPTY) Close() error {
694	-	p.close()
695	-	return nil
696	-	}
697	-
698	-	type orderedOutput struct{ close func() }
699	-
700	-	func (*orderedOutput) Write(b []byte) (int, error) { return len(b), nil }
701	-	func (*orderedOutput) Sync() error { return nil }
702	-	func (o *orderedOutput) Close() error {
703	-	o.close()
704	-	return nil
705	-	}
706	-
707	-	func TestCLIProcessCloseGracefulDetachCompletesSpanBeforeForcedCleanup(t *testing.T) {
708	-	var order []string
709	-	waitErr := make(chan error, 1)
710	-	timeout := make(chan time.Time)
711	-	p := &cliProcess{
712	-	pty: &orderedPTY{
713	-	write: func(input []byte) {
714	-	if string(input) != "exit\n" {
715	-	t.Fatalf("graceful exit input=%q", input)
716	-	}
717	-	order = append(order, "exit_written")
718	-	// The graceful client exit has closed its transport and emitted
719	-	// its adapter receive-end mark before cmd.Wait completes.
720	-	order = append(order, "adapter_receive_end")
721	-	waitErr <- nil
722	-	},
723	-	close: func() { order = append(order, "pty_closed") },
724	-	},
725	-	output: &orderedOutput{close: func() { order = append(order, "output_closed") }},
726	-	waitErr: waitErr,
727	-	waitTimeout: func() <-chan time.Time { return timeout },
728	-	forceCleanup: func() {
729	-	order = append(order, "forced_process_group_cleanup")
730	-	},
731	-	}
732	-
733	-	if err := p.Close(); err != nil {
734	-	t.Fatal(err)
735	-	}
736	-	if !equalStrings(order, []string{"exit_written", "adapter_receive_end", "pty_closed", "output_closed"}) {
737	-	t.Fatalf("cleanup order=%q", order)
738	-	}
739	-	}
740	-
741	-	func TestHarnessDoesNotInheritNestedSession(t *testing.T) {
742	-	env := withoutEnv([]string{"VEV=session=old", "PATH=/bin", "VEV_PERF_TRACE=old"}, "VEV")
743	-	if !equalStrings(env, []string{"PATH=/bin", "VEV_PERF_TRACE=old"}) {
744	-	t.Fatalf("environment=%q", env)
745	-	}
746	-	}

D cmd/vev-perf-harness/harness_test_2_test.go +0 -752

@@ -1,752 +0,0 @@
1	-	package main
2	-
3	-	import (
4	-	"bytes"
5	-	"encoding/json"
6	-	"errors"
7	-	"net"
8	-	"os"
9	-	"path/filepath"
10	-	"strconv"
11	-	"strings"
12	-	"testing"
13	-	"time"
14	-	)
15	-
16	-	func TestCLIProcessWarmupWaitsForDelayedTerminalReadiness(t *testing.T) {
17	-	allowWrite := make(chan struct{})
18	-	pty, output := &stagedPTY{allow: allowWrite, writes: make(chan []byte, 1)}, &fakeOutput{}
19	-	p := &cliProcess{pty: pty, output: output, chunks: make(chan []byte)}
20	-	input := workloadInput(scenario{ID: "s"}, 1, "warmup")
21	-	done := make(chan error, 1)
22	-	go func() { done <- p.Warmup(input) }()
23	-
24	-	// This is public client output from the initial application prompt/state,
25	-	// emitted only after the client has entered raw mode. A premature PTY write
26	-	// blocks on allowWrite and therefore cannot consume this delayed readiness.
27	-	go func() {
28	-	p.chunks <- []byte("shell prompt$ ")
29	-	close(allowWrite)
30	-	}()
31	-	select {
32	-	case got := <-pty.writes:
33	-	if !bytes.Equal(got, input) {
34	-	t.Fatalf("warmup write=%q want %q", got, input)
35	-	}
36	-	case <-time.After(time.Second):
37	-	t.Fatal("warmup did not wait for and then proceed from client readiness")
38	-	}
39	-	go func() { p.chunks <- append([]byte("application "), inputMarker(input)...); close(p.chunks) }()
40	-	if err := <-done; err != nil {
41	-	t.Fatal(err)
42	-	}
43	-	if output.syncs != 1 {
44	-	t.Fatalf("readiness output was not flushed: syncs=%d", output.syncs)
45	-	}
46	-	}
47	-
48	-	func TestCLIProcessRejectsPTYLocalEchoAsFlushEvidence(t *testing.T) {
49	-	pty, output := &fakePTY{}, &fakeOutput{}
50	-	p := &cliProcess{pty: pty, output: output, chunks: make(chan []byte, 1)}
51	-	input := workloadInput(scenario{ID: "s"}, 1, "measured-1")
52	-	p.chunks <- newPTYLocalEcho(input).expected
53	-	close(p.chunks)
54	-	flushed := false
55	-	if err := p.Measure(input, func() error { return nil }, func() error { flushed = true; return nil }); err == nil {
56	-	t.Fatal("local PTY echo satisfied terminal flush boundary")
57	-	}
58	-	if flushed \|\| output.syncs != 0 {
59	-	t.Fatalf("local echo stamped a flush: flushed=%t syncs=%d", flushed, output.syncs)
60	-	}
61	-	}
62	-
63	-	func TestCLIProcessPairsApplicationOutputWithSuccessfulFlush(t *testing.T) {
64	-	pty, output := &fakePTY{}, &fakeOutput{}
65	-	p := &cliProcess{pty: pty, output: output, chunks: make(chan []byte, 1)}
66	-	input := workloadInput(scenario{ID: "s"}, 1, "measured-1")
67	-	echo := newPTYLocalEcho(input).expected
68	-	p.chunks <- append(append([]byte(nil), echo...), append([]byte("application "), inputMarker(input)...)...)
69	-	order := []string{}
70	-	if err := p.Measure(input, func() error { order = append(order, "injected"); return nil }, func() error { order = append(order, "flushed"); return nil }); err != nil {
71	-	t.Fatal(err)
72	-	}
73	-	if len(pty.writes) != 1 \|\| output.syncs != 1 \|\| !equalStrings(order, []string{"injected", "flushed"}) {
74	-	t.Fatalf("writes=%q syncs=%d boundary order=%q", pty.writes, output.syncs, order)
75	-	}
76	-	}
77	-
78	-	func TestHarnessUsesPublicRoleCommandsAndPTYWorkloads(t *testing.T) {
79	-	for _, tc := range []struct {
80	-	role string
81	-	want []string
82	-	}{
83	-	{"daemon", []string{"--daemon"}},
84	-	{"client", []string{"new", "perf-s-001"}},
85	-	{"ssh_stdio_peer", []string{"_stdio", "perf-s-001"}},
86	-	{"udp_peer", []string{"_udp-proxy", "perf-s-001"}},
87	-	} {
88	-	t.Run(tc.role, func(t *testing.T) {
89	-	got := roleArgs(scenario{ID: "s", Transport: "local"}, processMapping{Role: tc.role, Run: 1})
90	-	if !equalStrings(got.Args, tc.want) {
91	-	t.Fatalf("args=%q want %q", got.Args, tc.want)
92	-	}
93	-	})
94	-	}
95	-	input := string(workloadInput(scenario{ID: "s", Workload: "interactive_flood"}, 1, "measured-1"))
96	-	if string(inputMarker([]byte(input))) != "__VEV_HARNESS_s_r1_measured-1__" \|\| !bytes.Contains([]byte(input), []byte("while [ $i -lt 128 ]")) \|\| !strings.HasSuffix(input, "printf '__VEV_HARNESS_s_r1_measured-1__\\n'\n") {
97	-	t.Fatalf("workload is not real PTY shell input with an observable marker: %q", input)
98	-	}
99	-	}
100	-
101	-	func TestHarnessRoutesEveryRemoteFixtureThroughItsDeclaredPeer(t *testing.T) {
102	-	cases := []struct {
103	-	name string
104	-	transport transport
105	-	peer string
106	-	wantRTT int
107	-	wantLoss int
108	-	}{
109	-	{"ssh", transport{ID: "ssh_stdio", Kind: "ssh_stdio"}, "ssh_stdio_peer", 0, 0},
110	-	{"udp baseline", transport{ID: "udp_baseline", Kind: "udp"}, "udp_peer", 0, 0},
111	-	{"udp 25ms", transport{ID: "udp_25ms", Kind: "udp", RTTMS: 25}, "udp_peer", 25, 0},
112	-	{"udp 100ms", transport{ID: "udp_100ms", Kind: "udp", RTTMS: 100}, "udp_peer", 100, 0},
113	-	{"udp loss zero", transport{ID: "udp_loss_0pct", Kind: "udp", LossPercent: 0}, "udp_peer", 0, 0},
114	-	{"udp loss", transport{ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1}, "udp_peer", 0, 1},
115	-	}
116	-	for _, tc := range cases {
117	-	t.Run(tc.name, func(t *testing.T) {
118	-	s := scenario{ID: "route", Transport: tc.transport.ID}
119	-	client := routeRoleArgs(s, processMapping{Role: "client", Run: 2}, tc.transport)
120	-	peer := routeRoleArgs(s, processMapping{Role: tc.peer, Run: 2}, tc.transport)
121	-	if !equalStrings(client.Args, []string{"attach", "harness@127.0.0.1:perf-route-002"}) {
122	-	t.Fatalf("client did not use public remote attach: %q", client.Args)
123	-	}
124	-	if tc.peer == "ssh_stdio_peer" && !equalStrings(peer.Args, []string{"_stdio", "perf-route-002"}) {
125	-	t.Fatalf("ssh peer command=%q", peer.Args)
126	-	}
127	-	if tc.peer == "udp_peer" && !equalStrings(peer.Args, []string{"_udp-proxy", "perf-route-002"}) {
128	-	t.Fatalf("udp peer command=%q", peer.Args)
129	-	}
130	-	if peer.Transport.RTTMS != tc.wantRTT \|\| peer.Transport.LossPercent != tc.wantLoss {
131	-	t.Fatalf("peer lost manifest network settings: %+v", peer.Transport)
132	-	}
133	-	})
134	-	}
135	-	}
136	-
137	-	func TestHarnessFakeRunnerRoutesClientToPeerAndCleansEveryRole(t *testing.T) {
138	-	for _, tc := range []struct {
139	-	name, transport, peer, peerCommand string
140	-	}{
141	-	{"ssh", "ssh_stdio", "ssh_stdio_peer", "_stdio"},
142	-	{"udp baseline", "udp_baseline", "udp_peer", "_udp-proxy"},
143	-	{"udp 25ms", "udp_25ms", "udp_peer", "_udp-proxy"},
144	-	{"udp 100ms", "udp_100ms", "udp_peer", "_udp-proxy"},
145	-	{"udp loss zero", "udp_loss_0pct", "udp_peer", "_udp-proxy"},
146	-	{"udp loss", "udp_loss_1pct", "udp_peer", "_udp-proxy"},
147	-	} {
148	-	t.Run(tc.name, func(t *testing.T) {
149	-	dir := t.TempDir()
150	-	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
151	-	if err != nil {
152	-	t.Fatal(err)
153	-	}
154	-	defer raw.Close()
155	-	l := &fakeLauncher{}
156	-	h := defaultHarness()
157	-	h.clock, h.launcher = &fakeClock{}, l
158	-	s := scenario{ID: "routed", Transport: tc.transport, Roles: []string{"daemon", "client", tc.peer}}
159	-	if _, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, s, 1, raw); err != nil {
160	-	t.Fatal(err)
161	-	}
162	-	if len(l.mappings) != 3 \|\| l.mappings[1].Role != tc.peer \|\| l.mappings[2].Role != "client" {
163	-	t.Fatalf("dependency launch order=%+v", l.mappings)
164	-	}
165	-	if l.commands[1].Args[0] != tc.peerCommand \|\| l.commands[2].Args[0] != "attach" {
166	-	t.Fatalf("commands do not connect client through declared peer: %+v", l.commands)
167	-	}
168	-	for i, p := range l.process {
169	-	if !p.closed {
170	-	t.Fatalf("role %s was not cleaned up", l.mappings[i].Role)
171	-	}
172	-	}
173	-	})
174	-	}
175	-	}
176	-
177	-	func TestCLITransportSeamOwnsExclusivePeerTraceAndCleanup(t *testing.T) {
178	-	fixtures := []struct {
179	-	name string
180	-	tr transport
181	-	rtt time.Duration
182	-	loss int
183	-	}{
184	-	{"baseline", transport{ID: "udp_baseline", Kind: "udp"}, 0, 0},
185	-	{"25ms", transport{ID: "udp_25ms", Kind: "udp", RTTMS: 25}, 25 * time.Millisecond, 0},
186	-	{"100ms", transport{ID: "udp_100ms", Kind: "udp", RTTMS: 100}, 100 * time.Millisecond, 0},
187	-	{"loss0", transport{ID: "udp_loss_0pct", Kind: "udp"}, 0, 0},
188	-	{"loss1", transport{ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1}, 0, 1},
189	-	}
190	-	for _, tc := range fixtures {
191	-	t.Run(tc.name, func(t *testing.T) {
192	-	dir := t.TempDir()
193	-	if err := os.Chmod(dir, 0o700); err != nil {
194	-	t.Fatal(err)
195	-	}
196	-	var configs []udpNetemConfig
197	-	netem := &fakeUDPNetem{port: 45678}
198	-	l := &cliLauncher{bin: "/bin/true", netemFactory: func(c udpNetemConfig) (udpNetem, error) {
199	-	configs = append(configs, c)
200	-	return netem, nil
201	-	}}
202	-	m := processMapping{ProcessID: "udp-peer", TracePath: filepath.Join(dir, "udp.jsonl"), Role: "udp_peer"}
203	-	if err := os.WriteFile(m.TracePath, nil, 0o600); err != nil {
204	-	t.Fatal(err)
205	-	}
206	-	p, err := l.preparePeer(m, roleCommand{Args: []string{"_udp-proxy", "work"}, Transport: tc.tr})
207	-	if err != nil {
208	-	t.Fatal(err)
209	-	}
210	-	if len(configs) != 1 \|\| configs[0].RTT != tc.rtt \|\| configs[0].LossPercent != tc.loss \|\| configs[0].TargetPath != filepath.Join(dir, "udp-peer.target") {
211	-	t.Fatalf("fixture did not reach netem seam: %+v", configs)
212	-	}
213	-	shim, err := os.ReadFile(filepath.Join(dir, "ssh"))
214	-	if err != nil {
215	-	t.Fatal(err)
216	-	}
217	-	text := string(shim)
218	-	for _, want := range []string{"_udp-proxy", m.TracePath, m.ProcessID, "udp-peer.target", "VEV-UDP %s %s\\n' 45678", "udp-peer.pid"} {
219	-	if !strings.Contains(text, want) {
220	-	t.Errorf("seam does not retain %q:\n%s", want, text)
221	-	}
222	-	}
223	-	if strings.Contains(text, "VEV_PERF_UDP_") {
224	-	t.Fatalf("ignored vev UDP environment was used instead of netem: %s", text)
225	-	}
226	-	// A nonexistent pid is already-cleaned-up; Close must still close the
227	-	// harness-owned emulator.
228	-	if err := p.Close(); err != nil {
229	-	t.Fatal(err)
230	-	}
231	-	if !netem.closed {
232	-	t.Fatal("harness netem was not cleaned up")
233	-	}
234	-	})
235	-	}
236	-	}
237	-
238	-	func TestUDPNetemExecutesRTTAndLossFixtures(t *testing.T) {
239	-	fixtures := []struct {
240	-	name string
241	-	rtt time.Duration
242	-	loss int
243	-	sent int
244	-	want int
245	-	}{
246	-	{"25ms RTT", 25 * time.Millisecond, 0, 1, 1},
247	-	{"100ms RTT", 100 * time.Millisecond, 0, 1, 1},
248	-	{"zero loss", 0, 0, 10, 10},
249	-	{"one percent loss", 0, 1, 100, 99},
250	-	}
251	-	for _, tc := range fixtures {
252	-	t.Run(tc.name, func(t *testing.T) {
253	-	target, err := net.ListenPacket("udp", "127.0.0.1:0")
254	-	if err != nil {
255	-	t.Fatal(err)
256	-	}
257	-	defer target.Close()
258	-	path := filepath.Join(t.TempDir(), "target")
259	-	if err := os.WriteFile(path, []byte("VEV-UDP "+strconv.Itoa(target.LocalAddr().(*net.UDPAddr).Port)+" key\\n"), 0o600); err != nil {
260	-	t.Fatal(err)
261	-	}
262	-	netem, err := newUDPNetem(udpNetemConfig{RTT: tc.rtt, LossPercent: tc.loss, TargetPath: path})
263	-	if err != nil {
264	-	t.Fatal(err)
265	-	}
266	-	defer netem.Close()
267	-	client, err := net.ListenPacket("udp", "127.0.0.1:0")
268	-	if err != nil {
269	-	t.Fatal(err)
270	-	}
271	-	defer client.Close()
272	-	started := time.Now()
273	-	for i := 0; i < tc.sent; i++ {
274	-	if _, err := client.WriteTo([]byte{byte(i)}, &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: netem.Port()}); err != nil {
275	-	t.Fatal(err)
276	-	}
277	-	}
278	-	if err := target.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
279	-	t.Fatal(err)
280	-	}
281	-	got := 0
282	-	buf := make([]byte, 32)
283	-	for got < tc.want {
284	-	if _, _, err := target.ReadFrom(buf); err != nil {
285	-	t.Fatalf("received %d packets, want %d: %v", got, tc.want, err)
286	-	}
287	-	got++
288	-	}
289	-	if tc.rtt > 0 {
290	-	if elapsed := time.Since(started); elapsed < tc.rtt/2 {
291	-	t.Fatalf("one-way netem delay=%s, want at least %s", elapsed, tc.rtt/2)
292	-	}
293	-	}
294	-	// Keep the deadline short: extra carriage would prove the exact 1%%
295	-	// loss cycle was not enforced.
296	-	if err := target.SetReadDeadline(time.Now().Add(20 * time.Millisecond)); err != nil {
297	-	t.Fatal(err)
298	-	}
299	-	if _, _, err := target.ReadFrom(buf); tc.want != tc.sent && err == nil {
300	-	t.Fatalf("received more than %d packets with %d%% loss", tc.want, tc.loss)
301	-	}
302	-	})
303	-	}
304	-	}
305	-
306	-	func TestHarnessRejectsInvalidBoundaryPairs(t *testing.T) {
307	-	for _, marks := range [][]harnessMark{
308	-	{{Sequence: 1, Kind: "terminal_flushed", Tick: 1, Valid: true}},
309	-	{{Sequence: 1, Kind: "input_injected", Tick: 2, Valid: true}, {Sequence: 1, Kind: "terminal_flushed", Tick: 1, Valid: true}},
310	-	{{Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}},
311	-	{{Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}, {Sequence: 1, Kind: "input_injected", Tick: 2, Valid: true}},
312	-	} {
313	-	if _, err := pairedSamples(marks); err == nil {
314	-	t.Fatalf("invalid marks accepted: %+v", marks)
315	-	}
316	-	}
317	-	got, err := pairedSamples([]harnessMark{{Sequence: 9, Kind: "input_injected", Tick: 1, Valid: true}, {Sequence: 9, Kind: "terminal_flushed", Tick: 5, Valid: true}})
318	-	if err != nil \|\| len(got) != 1 \|\| got[0] != 4 {
319	-	t.Fatalf("got %v %v", got, err)
320	-	}
321	-	}
322	-
323	-	func TestHarnessCollectsOnlyCompleteMeasuredIntervalEvents(t *testing.T) {
324	-	interval := measuredInterval{Start: 100, End: 200}
325	-	marks := []harnessMark{
326	-	{Sequence: 1, Kind: "input_injected", Tick: 10, Valid: true},
327	-	{Sequence: 1, Kind: "terminal_flushed", Tick: 20, Valid: true}, // warmup
328	-	{Sequence: 2, Kind: "input_injected", Tick: 100, Valid: true},
329	-	{Sequence: 2, Kind: "terminal_flushed", Tick: 120, Valid: true},
330	-	{Sequence: 3, Kind: "input_injected", Tick: 150, Valid: true},
331	-	{Sequence: 3, Kind: "terminal_flushed", Tick: 201, Valid: true}, // exits interval
332	-	{Sequence: 4, Kind: "input_injected", Tick: 99, Valid: true},
333	-	{Sequence: 4, Kind: "terminal_flushed", Tick: 110, Valid: true}, // enters interval
334	-	}
335	-	events, err := measuredEventSamples(marks, interval)
336	-	if err != nil {
337	-	t.Fatal(err)
338	-	}
339	-	if len(events) != 1 \|\| events[0].Sequence != 2 \|\| events[0].Latency != 20 {
340	-	t.Fatalf("measured events=%+v, want only complete sequence 2", events)
341	-	}
342	-	}
343	-
344	-	func TestHarnessRejectsInsufficientMeasuredEventSamples(t *testing.T) {
345	-	marks := make([]harnessMark, 0, 2*(minimumInIntervalEventSamples-1))
346	-	for i := 0; i < minimumInIntervalEventSamples-1; i++ {
347	-	marks = append(marks,
348	-	harnessMark{Sequence: uint64(i + 1), Kind: "input_injected", Tick: int64(i * 100), Valid: true},
349	-	harnessMark{Sequence: uint64(i + 1), Kind: "terminal_flushed", Tick: int64(i*100 + 10), Valid: true},
350	-	)
351	-	}
352	-	events, err := measuredEventSamples(marks, measuredInterval{Start: 0, End: int64(minimumInIntervalEventSamples * 100)})
353	-	if err != nil {
354	-	t.Fatal(err)
355	-	}
356	-	if err := requireMinimumEventSamples(events); err == nil {
357	-	t.Fatal("insufficient measured event samples accepted")
358	-	}
359	-	}
360	-
361	-	func TestHarnessRunRejectsStraddlingEventsAsInsufficient(t *testing.T) {
362	-	dir := t.TempDir()
363	-	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
364	-	if err != nil {
365	-	t.Fatal(err)
366	-	}
367	-	defer raw.Close()
368	-	clock := &fakeClock{}
369	-	h := defaultHarness()
370	-	h.clock, h.launcher = clock, insufficientEventLauncher{clock: clock}
371	-	_, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "late", Roles: []string{"daemon", "client"}}, 1, raw)
372	-	if err == nil \|\| !strings.Contains(err.Error(), "insufficient in-interval event samples") {
373	-	t.Fatalf("straddling run error=%v, want insufficient in-interval sample rejection", err)
374	-	}
375	-	}
376	-
377	-	func TestHarnessCalculatesEventCadenceAndMaxGapFromRawPairs(t *testing.T) {
378	-	events := []eventSample{
379	-	{Sequence: 1, Injected: 100, Flushed: 110, Latency: 10},
380	-	{Sequence: 2, Injected: 130, Flushed: 140, Latency: 10},
381	-	{Sequence: 3, Injected: 180, Flushed: 190, Latency: 10},
382	-	}
383	-	cadence, maxGap := eventCadence(events)
384	-	if cadence.Count != 2 \|\| cadence.P50 != 30 \|\| cadence.P95 != 30 \|\| cadence.P99 != 30 \|\| maxGap != 50 {
385	-	t.Fatalf("cadence=%+v maxGap=%d", cadence, maxGap)
386	-	}
387	-	}
388	-
389	-	func TestHarnessAggregatesRawEventsSeparatelyFromRunDispersion(t *testing.T) {
390	-	first := append([]int64(nil), make([]int64, minimumInIntervalEventSamples+5)...)
391	-	for i := range first {
392	-	first[i] = 1
393	-	}
394	-	second := []int64{100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1_000, 1_000}
395	-	all, _, _, runP50s := aggregateEventResults([]runResult{
396	-	{EndToEnd: first, Event: distribution{P50: 1}},
397	-	{EndToEnd: second, Event: distribution{P50: 100}},
398	-	})
399	-	if got, dispersion := percentiles(all), percentiles(runP50s); got.Count != 30 \|\| got.P99 != 1_000 \|\| dispersion.Count != 2 \|\| dispersion.P99 != 1 {
400	-	t.Fatalf("event aggregation=%+v run dispersion=%+v", got, dispersion)
401	-	}
402	-	}
403	-
404	-	func TestHarnessSchedulesMeasuredEventsAcrossFullInterval(t *testing.T) {
405	-	dir := t.TempDir()
406	-	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
407	-	if err != nil {
408	-	t.Fatal(err)
409	-	}
410	-	defer raw.Close()
411	-	h := defaultHarness()
412	-	h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
413	-	result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "schedule", Roles: []string{"daemon", "client"}}, 1, raw)
414	-	if err != nil {
415	-	t.Fatal(err)
416	-	}
417	-	if result.Samples < minimumInIntervalEventSamples \|\| result.Cadence.Count != result.Samples-1 \|\| result.MaxGap <= 0 {
418	-	t.Fatalf("result did not include repeated measured events: %+v", result)
419	-	}
420	-	if err := raw.Close(); err != nil {
421	-	t.Fatal(err)
422	-	}
423	-	records, err := readJSONL(filepath.Join(dir, "raw.jsonl"))
424	-	if err != nil {
425	-	t.Fatal(err)
426	-	}
427	-	if len(records) != result.Samples*2 {
428	-	t.Fatalf("raw records=%d, want %d complete measured boundaries", len(records), result.Samples*2)
429	-	}
430	-	}
431	-
432	-	func TestHarnessPreservesWorkloadAndCleanupErrors(t *testing.T) {
433	-	dir := t.TempDir()
434	-	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
435	-	if err != nil {
436	-	t.Fatal(err)
437	-	}
438	-	defer raw.Close()
439	-	primary := errors.New("primary workload failure")
440	-	cleanup := errors.New("daemon cleanup failure")
441	-	launcher := &fakeLauncher{
442	-	measureErr: map[string]error{"client": primary},
443	-	closeErr: map[string]error{"daemon": cleanup},
444	-	}
445	-	h := defaultHarness()
446	-	h.clock, h.launcher = &fakeClock{}, launcher
447	-	_, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, scenario{ID: "errors", Roles: []string{"daemon", "client"}}, 1, raw)
448	-	if !errors.Is(err, primary) \|\| !errors.Is(err, cleanup) {
449	-	t.Fatalf("workload and cleanup errors were not both preserved: %v", err)
450	-	}
451	-	if got, want := err.Error(), primary.Error()+"\nclose daemon role: "+cleanup.Error(); got != want {
452	-	t.Fatalf("error ordering=%q want %q", got, want)
453	-	}
454	-	for i, p := range launcher.process {
455	-	if !p.closed {
456	-	t.Fatalf("role %s was not closed", launcher.mappings[i].Role)
457	-	}
458	-	}
459	-	}
460	-
461	-	func TestHarnessCleansRunDirectoryOnProcessFailure(t *testing.T) {
462	-	dir := t.TempDir()
463	-	raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
464	-	if err != nil {
465	-	t.Fatal(err)
466	-	}
467	-	defer raw.Close()
468	-	launcher := &fakeLauncher{}
469	-	h := defaultHarness()
470	-	h.clock = &fakeClock{}
471	-	// The client is the second role and fails after the daemon has started.
472	-	h.launcher = &failingLauncher{fakeLauncher: launcher}
473	-	o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
474	-	if _, err := h.runOne(o, manifest{}, scenario{ID: "cleanup", Roles: []string{"daemon", "client"}}, 1, raw); err == nil {
475	-	t.Fatal("failed process accepted")
476	-	}
477	-	if _, err := os.Stat(filepath.Join(dir, "cleanup-run-001")); !os.IsNotExist(err) {
478	-	t.Fatalf("failed run directory remains: %v", err)
479	-	}
480	-	}
481	-
482	-	type failingLauncher struct{ *fakeLauncher }
483	-
484	-	func (l *failingLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
485	-	p, err := l.fakeLauncher.Launch(m, command)
486	-	if err != nil {
487	-	return nil, err
488	-	}
489	-	if m.Role == "client" {
490	-	p.(*fakeProcess).fail = true
491	-	}
492	-	return p, nil
493	-	}
494	-
495	-	func equalStrings(a, b []string) bool {
496	-	if len(a) != len(b) {
497	-	return false
498	-	}
499	-	for i := range a {
500	-	if a[i] != b[i] {
501	-	return false
502	-	}
503	-	}
504	-	return true
505	-	}
506	-
507	-	func TestHarnessIncludesAllRequiredProcessLocalSpans(t *testing.T) {
508	-	got := make([]string, 0, len(spanPairs))
509	-	for _, pair := range spanPairs {
510	-	got = append(got, pair.name)
511	-	}
512	-	want := []string{"capture_duration", "compose_duration", "diff_duration", "queue_wait", "ack_blocked_interval", "emit_duration", "adapter_send_duration", "adapter_receive_duration"}
513	-	if !equalStrings(got, want) {
514	-	t.Fatalf("span pairs = %v, want %v", got, want)
515	-	}
516	-	}
517	-
518	-	func TestHarnessRawRecordsAreDeterministicJSONL(t *testing.T) {
519	-	var b bytes.Buffer
520	-	for _, m := range []harnessMark{{Scenario: "s", Run: 1, Sequence: 1, Kind: "input_injected", Tick: 1, Valid: true}, {Scenario: "s", Run: 1, Sequence: 1, Kind: "terminal_flushed", Tick: 2, Valid: true}} {
521	-	b.WriteString(mustJSON(m))
522	-	b.WriteByte('\n')
523	-	}
524	-	path := filepath.Join(t.TempDir(), "raw.jsonl")
525	-	if err := os.WriteFile(path, b.Bytes(), 0o600); err != nil {
526	-	t.Fatal(err)
527	-	}
528	-	if got, err := readJSONL(path); err != nil \|\| len(got) != 2 {
529	-	t.Fatalf("records=%d err=%v", len(got), err)
530	-	}
531	-	}
532	-
533	-	type traceLauncher struct{ fakeLauncher }
534	-
535	-	func (l *traceLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
536	-	p, err := l.fakeLauncher.Launch(m, command)
537	-	if err != nil {
538	-	return nil, err
539	-	}
540	-	f, err := os.OpenFile(m.TracePath, os.O_WRONLY\|os.O_APPEND, 0o600)
541	-	if err != nil {
542	-	return nil, err
543	-	}
544	-	defer f.Close()
545	-	for i, pair := range spanPairs {
546	-	for n, kind := range []string{pair.start, pair.end} {
547	-	b, _ := json.Marshal(traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: uint64(i + 1), RequestID: 1, Epoch: 1, Kind: kind, Tick: int64(i20 + n5), Valid: true})
548	-	if _, err := f.Write(append(b, '\n')); err != nil {
549	-	return nil, err
550	-	}
551	-	}
552	-	}
553	-	return p, nil
554	-	}
555	-
556	-	type closeTraceProcess struct {
557	-	*fakeProcess
558	-	traceEnd func() error
559	-	}
560	-
561	-	func (p *closeTraceProcess) Close() error {
562	-	p.closed = true
563	-	return errors.Join(p.closeErr, p.traceEnd())
564	-	}
565	-
566	-	type closeTraceLauncher struct{ fakeLauncher }
567	-
568	-	func (l *closeTraceLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
569	-	p, err := l.fakeLauncher.Launch(m, command)
570	-	if err != nil {
571	-	return nil, err
572	-	}
573	-	start := traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: 1, RequestID: 1, Epoch: 1, Kind: "adapter_receive_start", Tick: 10, Valid: true}
574	-	if err := appendTraceRecord(m.TracePath, start); err != nil {
575	-	return nil, err
576	-	}
577	-	return &closeTraceProcess{fakeProcess: p.(*fakeProcess), traceEnd: func() error {
578	-	return appendTraceRecord(m.TracePath, traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: 1, RequestID: 1, Epoch: 1, Kind: "adapter_receive_end", Tick: 20, Valid: true})
579	-	}}, nil
580	-	}
581	-
582	-	func appendTraceRecord(path string, record traceRecord) error {
583	-	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_APPEND, 0o600)
584	-	if err != nil {
585	-	return err
586	-	}
587	-	defer f.Close()
588	-	b, err := json.Marshal(record)
589	-	if err != nil {
590	-	return err
591	-	}
592	-	_, err = f.Write(append(b, '\n'))
593	-	return err
594	-	}
595	-
596	-	func TestHarnessClosesRolesBeforeMergingReceiveSpans(t *testing.T) {
597	-	dir := t.TempDir()
598	-	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
599	-	if err != nil {
600	-	t.Fatal(err)
601	-	}
602	-	defer raw.Close()
603	-	launcher := &closeTraceLauncher{}
604	-	h := defaultHarness()
605	-	h.clock, h.launcher = &fakeClock{}, launcher
606	-	result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, scenario{ID: "receive-cleanup", Roles: []string{"daemon", "client"}}, 1, raw)
607	-	if err != nil {
608	-	t.Fatal(err)
609	-	}
610	-	if len(result.Spans) != 2 {
611	-	t.Fatalf("post-cleanup receive spans=%+v", result.Spans)
612	-	}
613	-	for _, s := range result.Spans {
614	-	if s.Name != "adapter_receive_duration" \|\| len(s.Samples) != 1 \|\| s.Samples[0] != 10 {
615	-	t.Fatalf("unmatched or wrong receive span: %+v", s)
616	-	}
617	-	}
618	-	for i, p := range launcher.process {
619	-	if !p.closed {
620	-	t.Fatalf("role %s was not closed before trace merge", launcher.mappings[i].Role)
621	-	}
622	-	}
623	-	}
624	-
625	-	func TestHarnessWritesRequiredEvidenceWithSufficientSpans(t *testing.T) {
626	-	base, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
627	-	if err != nil {
628	-	t.Fatal(err)
629	-	}
630	-	for i := range base.Scenarios {
631	-	if i > 0 {
632	-	base.Scenarios[i].InapplicableReason = "fixture limits this test to one public topology"
633	-	base.Scenarios[i].Roles = nil
634	-	}
635	-	}
636	-	dir := t.TempDir()
637	-	manifestPath := filepath.Join(dir, "manifest.json")
638	-	if err := os.WriteFile(manifestPath, []byte(mustJSON(base)), 0o600); err != nil {
639	-	t.Fatal(err)
640	-	}
641	-	l := &traceLauncher{}
642	-	h := defaultHarness()
643	-	h.clock = &fakeClock{}
644	-	h.launcher = l
645	-	h.gitSHA = func() string { return "test-sha" }
646	-	if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", filepath.Join(dir, "out"), "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
647	-	t.Fatal(err)
648	-	}
649	-	for _, name := range []string{"raw-harness.jsonl", "runs.json", "summary.json"} {
650	-	if _, err := os.Stat(filepath.Join(dir, "out", name)); err != nil {
651	-	t.Errorf("%s: %v", name, err)
652	-	}
653	-	}
654	-	var got summary
655	-	b, err := os.ReadFile(filepath.Join(dir, "out", "summary.json"))
656	-	if err != nil {
657	-	t.Fatal(err)
658	-	}
659	-	if err := json.Unmarshal(b, &got); err != nil {
660	-	t.Fatal(err)
661	-	}
662	-	if got.EndToEnd.Count < 10*minimumInIntervalEventSamples \|\| got.Repetitions != 10 \|\| got.RunP50Dispersion.Count != 10 \|\| got.Cadence.Count == 0 \|\| got.MaxGap == 0 \|\| len(got.Spans) == 0 {
663	-	t.Fatalf("summary=%+v", got)
664	-	}
665	-	for _, s := range got.Spans {
666	-	if s.Distribution.Count < 10 {
667	-	t.Fatalf("insufficient span summary: %+v", s)
668	-	}
669	-	}
670	-	}
671	-
672	-	// boundedLocalConcurrentTrace is an excerpt from the second run of the
673	-	// bounded local public-CLI smoke. Receive sequence 3 is in flight while the
674	-	// same process serializes sequence 4; shared-process records therefore cannot
675	-	// have a global sequence ordering requirement.
676	-	const boundedLocalConcurrentTrace = `{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":3,"request_id":3,"epoch":3,"kind":"adapter_receive_start","tick":1783902621495284782,"bytes":0,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
677	-	{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":4,"request_id":4,"epoch":4,"kind":"adapter_send_start","tick":1783902621495364552,"bytes":96,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
678	-	{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":4,"request_id":4,"epoch":4,"kind":"adapter_send_end","tick":1783902621495386452,"bytes":96,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
679	-	{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":3,"request_id":3,"epoch":3,"kind":"adapter_receive_end","tick":1783902621495463332,"bytes":0,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
680	-	`
681	-
682	-	func TestHarnessAcceptsConcurrentProductionTraceInterleaving(t *testing.T) {
683	-	path := filepath.Join(t.TempDir(), "local-client.jsonl")
684	-	if err := os.WriteFile(path, []byte(boundedLocalConcurrentTrace), 0o600); err != nil {
685	-	t.Fatal(err)
686	-	}
687	-	spans, err := mergeProcessTraces([]processMapping{{
688	-	ProcessID: "1x4-idle-local-r002-client-02", ClockDomain: "1x4-idle-local-r002-client-02", TracePath: path,
689	-	Scenario: "1x4-idle-local", Run: 2,
690	-	}})
691	-	if err != nil {
692	-	t.Fatalf("valid concurrent production trace rejected: %v", err)
693	-	}
694	-	if len(spans) != 2 \|\| spans[0].Component != "ipc" \|\| spans[0].Name != "adapter_send_duration" \|\| spans[0].Samples[0] != 21900 \|\| spans[1].Component != "ipc" \|\| spans[1].Name != "adapter_receive_duration" \|\| spans[1].Samples[0] != 178550 {
695	-	t.Fatalf("spans=%+v", spans)
696	-	}
697	-	}
698	-
699	-	func TestHarnessRejectsMissingManifestRoleTrace(t *testing.T) {
700	-	dir := t.TempDir()
701	-	missing := filepath.Join(dir, "daemon.jsonl")
702	-	client := filepath.Join(dir, "client.jsonl")
703	-	if err := os.WriteFile(client, nil, 0o600); err != nil {
704	-	t.Fatal(err)
705	-	}
706	-	_, err := mergeProcessTraces([]processMapping{
707	-	{ProcessID: "daemon", ClockDomain: "daemon", TracePath: missing, Role: "daemon", Scenario: "s", Run: 1},
708	-	{ProcessID: "client", ClockDomain: "client", TracePath: client, Role: "client", Scenario: "s", Run: 1},
709	-	})
710	-	if !errors.Is(err, os.ErrNotExist) \|\| !strings.Contains(err.Error(), "daemon role") {
711	-	t.Fatalf("missing daemon trace error = %v, want daemon role not-exist error", err)
712	-	}
713	-	}
714	-
715	-	func TestHarnessRejectsCrossProcessAndBadTraceSpans(t *testing.T) {
716	-	write := func(t *testing.T, records ...traceRecord) processMapping {
717	-	t.Helper()
718	-	path := filepath.Join(t.TempDir(), "trace.jsonl")
719	-	f, err := os.Create(path)
720	-	if err != nil {
721	-	t.Fatal(err)
722	-	}
723	-	for _, r := range records {
724	-	b, _ := json.Marshal(r)
725	-	_, _ = f.Write(append(b, '\n'))
726	-	}
727	-	_ = f.Close()
728	-	return processMapping{ProcessID: "one", ClockDomain: "one", TracePath: path, Scenario: "s", Run: 1}
729	-	}
730	-	base := func(kind string, tick int64) traceRecord {
731	-	return traceRecord{ProcessID: "one", Component: "daemon", Scenario: "s", Run: 1, Sequence: 1, RequestID: 1, Epoch: 1, Kind: kind, Tick: tick, Valid: true}
732	-	}
733	-	for _, records := range [][]traceRecord{
734	-	{base("diff_end", 1)},
735	-	{base("diff_start", 2), base("diff_end", 1)},
736	-	{base("diff_start", 1)},
737	-	{base("diff_start", 1), base("diff_start", 2)},
738	-	{base("diff_start", 1), func() traceRecord { r := base("diff_end", 2); r.Component = "ipc"; return r }()},
739	-	{func() traceRecord { r := base("diff_start", 1); r.ProcessID = "other"; return r }()},
740	-	{func() traceRecord { r := base("diff_start", 1); r.Scenario = "other-scenario"; return r }()},
741	-	{func() traceRecord { r := base("diff_start", 1); r.Run = 2; return r }()},
742	-	} {
743	-	if _, err := mergeProcessTraces([]processMapping{write(t, records...)}); err == nil {
744	-	t.Fatalf("invalid trace accepted: %+v", records)
745	-	}
746	-	}
747	-	m := write(t, base("diff_start", 1), base("diff_end", 3))
748	-	spans, err := mergeProcessTraces([]processMapping{m})
749	-	if err != nil \|\| len(spans) != 1 \|\| spans[0].Samples[0] != 2 {
750	-	t.Fatalf("spans=%+v err=%v", spans, err)
751	-	}
752	-	}

D cmd/vev-perf-harness/harness_test_3_test.go +0 -54

@@ -1,54 +0,0 @@
1	-	package main
2	-
3	-	import (
4	-	"os"
5	-	"path/filepath"
6	-	"strings"
7	-	"testing"
8	-	)
9	-
10	-	func TestHarnessExcludesFailedSpanDurationsWhileValidatingPairing(t *testing.T) {
11	-	base := func(kind string, tick int64, valid bool) traceRecord {
12	-	return traceRecord{ProcessID: "one", Component: "adapter", Scenario: "s", Run: 1, Sequence: 1, RequestID: 1, Epoch: 1, Kind: kind, Tick: tick, Valid: valid}
13	-	}
14	-	for _, tc := range []struct {
15	-	name string
16	-	records []traceRecord
17	-	wantSpans int
18	-	wantErr bool
19	-	}{
20	-	{"successful adapter send is sampled", []traceRecord{base("adapter_send_start", 10, true), base("adapter_send_end", 20, true)}, 1, false},
21	-	{"failed adapter send end is excluded", []traceRecord{base("adapter_send_start", 10, true), base("adapter_send_end", 20, false)}, 0, false},
22	-	{"failed adapter receive end is excluded", []traceRecord{base("adapter_receive_start", 10, true), base("adapter_receive_end", 20, false)}, 0, false},
23	-	{"failed adapter send start is excluded", []traceRecord{base("adapter_send_start", 10, false), base("adapter_send_end", 20, true)}, 0, false},
24	-	{"failed adapter receive boundaries are excluded", []traceRecord{base("adapter_receive_start", 10, false), base("adapter_receive_end", 20, false)}, 0, false},
25	-	{"failed end without start is rejected", []traceRecord{base("adapter_send_end", 20, false)}, 0, true},
26	-	{"duplicate failed start is rejected", []traceRecord{base("adapter_send_start", 10, false), base("adapter_send_start", 15, false)}, 0, true},
27	-	{"negative failed span is rejected", []traceRecord{base("adapter_receive_start", 20, false), base("adapter_receive_end", 10, false)}, 0, true},
28	-	{"cross-component failed end is rejected", []traceRecord{base("adapter_send_start", 10, true), func() traceRecord { r := base("adapter_send_end", 20, false); r.Component = "other"; return r }()}, 0, true},
29	-	} {
30	-	t.Run(tc.name, func(t *testing.T) {
31	-	path := filepath.Join(t.TempDir(), "trace.jsonl")
32	-	var lines []string
33	-	for _, record := range tc.records {
34	-	lines = append(lines, mustJSON(record))
35	-	}
36	-	if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {
37	-	t.Fatal(err)
38	-	}
39	-	spans, err := mergeProcessTraces([]processMapping{{ProcessID: "one", ClockDomain: "one", TracePath: path, Scenario: "s", Run: 1}})
40	-	if tc.wantErr {
41	-	if err == nil {
42	-	t.Fatalf("invalid trace accepted: %+v", tc.records)
43	-	}
44	-	return
45	-	}
46	-	if err != nil \|\| len(spans) != tc.wantSpans {
47	-	t.Fatalf("spans=%+v err=%v, want %d spans", spans, err, tc.wantSpans)
48	-	}
49	-	if tc.wantSpans == 1 && spans[0].Samples[0] != 10 {
50	-	t.Fatalf("successful span=%+v, want 10ns", spans[0])
51	-	}
52	-	})
53	-	}
54	-	}

R cmd/vev-perf-harness/harness_1.go → cmd/vev-perf-harness/launcher.go +138 -141 renamed, 80% similar


@@ -16,137 +16,9 @@ import (
16	16		"time"
17	17
18	18		"github.com/bnema/vev/pkg/safedir"
	19	+	"golang.org/x/sys/unix"
19	20		)
20	21
21		-	const (
22		-	minimumDuration = 30 * time.Second
23		-	minimumRepetitions = 10
24		-	minimumInIntervalEventSamples = 10
25		-	measuredEventCadence = time.Second
26		-	)
27		-
28		-	type options struct {
29		-	vevBin, manifest, out string
30		-	scenario string
31		-	warmup, duration time.Duration
32		-	repetitions int
33		-	}
34		-
35		-	type manifest struct {
36		-	Schema uint16 `json:"schema"`
37		-	Topologies []topology `json:"topologies"`
38		-	Workloads []string `json:"workloads"`
39		-	Transports []transport `json:"transports"`
40		-	Scenarios []scenario `json:"scenarios"`
41		-	}
42		-	type topology struct {
43		-	ID string `json:"id"`
44		-	Geometry string `json:"geometry"`
45		-	RowsPerPane int `json:"rows_per_pane"`
46		-	}
47		-	type transport struct {
48		-	ID string `json:"id"`
49		-	Kind string `json:"kind"`
50		-	RTTMS int `json:"rtt_ms,omitempty"`
51		-	LossPercent int `json:"loss_percent,omitempty"`
52		-	}
53		-	type scenario struct {
54		-	ID string `json:"id"`
55		-	Topology string `json:"topology"`
56		-	Workload string `json:"workload"`
57		-	Transport string `json:"transport"`
58		-	Roles []string `json:"roles"`
59		-	InapplicableReason string `json:"inapplicable_reason,omitempty"`
60		-	}
61		-
62		-	type processMapping struct {
63		-	ProcessID string `json:"process_id"`
64		-	ClockDomain string `json:"clock_domain"`
65		-	TracePath string `json:"trace_path"`
66		-	Role string `json:"role"`
67		-	Scenario string `json:"scenario"`
68		-	Run int `json:"run"`
69		-	Identity string `json:"identity"`
70		-	}
71		-
72		-	type runManifest struct {
73		-	Scenario string `json:"scenario"`
74		-	Run int `json:"run"`
75		-	Processes []processMapping `json:"processes"`
76		-	}
77		-	type harnessMark struct {
78		-	Scenario string `json:"scenario"`
79		-	Run int `json:"run"`
80		-	Sequence uint64 `json:"sequence"`
81		-	Kind string `json:"kind"`
82		-	Tick int64 `json:"tick"`
83		-	Valid bool `json:"valid"`
84		-	}
85		-
86		-	// measuredInterval belongs solely to the harness clock domain. A paired event
87		-	// is eligible only when both owned boundaries fall inside this interval.
88		-	type measuredInterval struct {
89		-	Start int64
90		-	End int64
91		-	}
92		-
93		-	type eventSample struct {
94		-	Sequence uint64 `json:"sequence"`
95		-	Injected int64 `json:"injected_tick"`
96		-	Flushed int64 `json:"flushed_tick"`
97		-	Latency int64 `json:"latency_nanos"`
98		-	}
99		-	type span struct {
100		-	Component, Name string
101		-	Samples []int64
102		-	}
103		-	type runResult struct {
104		-	Spans []span `json:"process_local_spans"`
105		-	Scenario string `json:"scenario"`
106		-	Run int `json:"run"`
107		-	Samples int `json:"samples"`
108		-	EndToEnd []int64 `json:"end_to_end_nanos"`
109		-	Event distribution `json:"event_end_to_end"`
110		-	Cadence distribution `json:"event_cadence_nanos"`
111		-	CadenceSamples []int64 `json:"-"`
112		-	MaxGap int64 `json:"event_max_gap_nanos"`
113		-	Processes []processMapping `json:"processes"`
114		-	}
115		-	type summary struct {
116		-	Schema uint16 `json:"schema"`
117		-	GitSHA string `json:"git_sha"`
118		-	Warmup string `json:"warmup"`
119		-	Duration string `json:"duration"`
120		-	Repetitions int `json:"repetitions"`
121		-	EndToEnd distribution `json:"harness_end_to_end"`
122		-	Cadence distribution `json:"harness_event_cadence_nanos"`
123		-	MaxGap int64 `json:"harness_event_max_gap_nanos"`
124		-	RunP50Dispersion distribution `json:"run_p50_dispersion"`
125		-	Spans []spanSummary `json:"process_local_spans"`
126		-	Runs int `json:"runs"`
127		-	}
128		-	type distribution struct {
129		-	Count int `json:"count"`
130		-	P50 int64 `json:"p50"`
131		-	P95 int64 `json:"p95"`
132		-	P99 int64 `json:"p99"`
133		-	Max int64 `json:"max"`
134		-	}
135		-	type spanSummary struct {
136		-	Component string `json:"component"`
137		-	Name string `json:"name"`
138		-	Distribution distribution `json:"distribution"`
139		-	}
140		-
141		-	type clock interface {
142		-	Now() int64
143		-	Sleep(time.Duration)
144		-	}
145		-	type systemClock struct{ start time.Time }
146		-
147		-	func (c systemClock) Now() int64 { return time.Since(c.start).Nanoseconds() }
148		-	func (systemClock) Sleep(d time.Duration) { time.Sleep(d) }
149		-
150	22		type launchedProcess interface {
151	23		Warmup([]byte) error
152	24		Measure([]byte, func() error, func() error) error

@@ -204,8 +76,10 @@ type preparedPeer struct {
204	76		cleanupRuntime func() error
205	77		}
206	78
207		-	func (p preparedPeer) Warmup([]byte) error { return nil }
	79	+	func (p preparedPeer) Warmup([]byte) error { return nil }
	80	+
208	81		func (p preparedPeer) Measure([]byte, func() error, func() error) error { return nil }
	82	+
209	83		func (p preparedPeer) Close() error {
210	84		var errs []error
211	85		if p.route.pidPath != "" {

@@ -725,17 +599,6 @@ type ptyLocalEcho struct {
725	599		done bool
726	600		}
727	601
728		-	func newPTYLocalEcho(input []byte) *ptyLocalEcho {
729		-	expected := make([]byte, 0, len(input))
730		-	for _, b := range input {
731		-	if b == '\n' {
732		-	expected = append(expected, '\r')
733		-	}
734		-	expected = append(expected, b)
735		-	}
736		-	return &ptyLocalEcho{expected: expected}
737		-	}
738		-
739	602		func (e *ptyLocalEcho) applicationOutput(chunk []byte) []byte {
740	603		if e.done \|\| len(e.expected) == 0 {
741	604		return chunk

@@ -753,3 +616,137 @@ func (e *ptyLocalEcho) applicationOutput(chunk []byte) []byte {
753	616		}
754	617		return nil
755	618		}
	619	+
	620	+	func (p *cliProcess) Close() error {
	621	+	var result error
	622	+	p.closed.Do(func() {
	623	+	if p.done != nil {
	624	+	close(p.done)
	625	+	}
	626	+	// A public terminal client owns a shell session. Ask that shell to exit
	627	+	// before closing the PTY so its transport can finish its receive span;
	628	+	// this avoids turning ordinary harness teardown into an aborted trace.
	629	+	exited := false
	630	+	if p.shutdown != nil && p.waitErr != nil {
	631	+	select {
	632	+	case <-p.waitErr:
	633	+	exited = true
	634	+	default:
	635	+	// The client may have just ended the last session and raced this
	636	+	// best-effort public shutdown. The normal bounded reaping path below
	637	+	// remains authoritative in that case.
	638	+	_ = p.shutdown()
	639	+	}
	640	+	}
	641	+	if p.pty != nil && p.waitErr != nil {
	642	+	_, _ = p.pty.Write([]byte("exit\n"))
	643	+	select {
	644	+	case <-p.waitErr:
	645	+	exited = true
	646	+	case <-p.closeDeadline():
	647	+	}
	648	+	}
	649	+	if p.pty != nil {
	650	+	_ = p.pty.Close()
	651	+	}
	652	+	if p.waitErr != nil && !exited {
	653	+	select {
	654	+	case <-p.waitErr:
	655	+	case <-p.closeDeadline():
	656	+	// Teardown is deliberately bounded: a stuck public CLI or child
	657	+	// must not hold a local harness run indefinitely.
	658	+	p.forceProcessGroupCleanup()
	659	+	select {
	660	+	case <-p.waitErr:
	661	+	case <-p.closeDeadline():
	662	+	p.forceProcessGroupKill()
	663	+	select {
	664	+	case <-p.waitErr:
	665	+	case <-p.closeDeadline():
	666	+	result = errors.New("timed out reaping public CLI process")
	667	+	}
	668	+	}
	669	+	}
	670	+	}
	671	+	if p.output != nil {
	672	+	if err := p.output.Close(); result == nil {
	673	+	result = err
	674	+	}
	675	+	}
	676	+	if p.cleanupRuntime != nil {
	677	+	if err := p.cleanupRuntime(); result == nil {
	678	+	result = err
	679	+	}
	680	+	}
	681	+	})
	682	+	return result
	683	+	}
	684	+
	685	+	func (p *cliProcess) closeDeadline() <-chan time.Time {
	686	+	if p.waitTimeout != nil {
	687	+	return p.waitTimeout()
	688	+	}
	689	+	if p.shutdown != nil {
	690	+	// A public kill asks the daemon to finish bounded detached notices and
	691	+	// close blocked carriage before it exits; do not turn that graceful
	692	+	// protocol into a SIGKILL at the two-second client deadline.
	693	+	return time.After(5 * time.Second)
	694	+	}
	695	+	return time.After(2 * time.Second)
	696	+	}
	697	+
	698	+	func (p *cliProcess) forceProcessGroupCleanup() {
	699	+	if p.forceCleanup != nil {
	700	+	p.forceCleanup()
	701	+	return
	702	+	}
	703	+	if p.cmd != nil && p.cmd.Process != nil {
	704	+	// The client is a session/process-group leader; forced cleanup must
	705	+	// include its ssh seam/_stdio descendants.
	706	+	_ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGTERM)
	707	+	}
	708	+	}
	709	+
	710	+	func (p *cliProcess) forceProcessGroupKill() {
	711	+	if p.forceKill != nil {
	712	+	p.forceKill()
	713	+	return
	714	+	}
	715	+	if p.cmd != nil && p.cmd.Process != nil {
	716	+	_ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGKILL)
	717	+	}
	718	+	}
	719	+
	720	+	// openPTY is the small Linux PTY boundary needed to drive the public terminal
	721	+	// client. No daemon or adapter implementation is imported by this command.
	722	+	func openPTY() (os.File, os.File, error) {
	723	+	masterFD, err := unix.Open("/dev/ptmx", unix.O_RDWR\|unix.O_NOCTTY, 0)
	724	+	if err != nil {
	725	+	return nil, nil, err
	726	+	}
	727	+	closeMaster := func(err error) (os.File, os.File, error) { _ = unix.Close(masterFD); return nil, nil, err }
	728	+	number, err := unix.IoctlGetInt(masterFD, unix.TIOCGPTN)
	729	+	if err != nil {
	730	+	return closeMaster(err)
	731	+	}
	732	+	if err := unix.IoctlSetPointerInt(masterFD, unix.TIOCSPTLCK, 0); err != nil {
	733	+	return closeMaster(err)
	734	+	}
	735	+	slaveFD, err := unix.Open(fmt.Sprintf("/dev/pts/%d", number), unix.O_RDWR\|unix.O_NOCTTY, 0)
	736	+	if err != nil {
	737	+	return closeMaster(err)
	738	+	}
	739	+	return os.NewFile(uintptr(masterFD), "vev-client-master"), os.NewFile(uintptr(slaveFD), "vev-client-slave"), nil
	740	+	}
	741	+
	742	+	func inputMarker(input []byte) []byte {
	743	+	start := bytes.Index(input, []byte("__VEV_HARNESS_"))
	744	+	if start < 0 {
	745	+	return input
	746	+	}
	747	+	end := bytes.Index(input[start:], []byte(`\n`))
	748	+	if end < 0 {
	749	+	return input[start:]
	750	+	}
	751	+	return input[start : start+end]
	752	+	}

A cmd/vev-perf-harness/launcher_test.go +296 -0

@@ -0,0 +1,296 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"fmt"
	6	+	"net"
	7	+	"os"
	8	+	"os/exec"
	9	+	"path/filepath"
	10	+	"strings"
	11	+	"testing"
	12	+	"time"
	13	+	)
	14	+
	15	+	func TestCLIProcessCloseIsBoundedWhenPublicRoleDoesNotExit(t *testing.T) {
	16	+	deadline := make(chan time.Time)
	17	+	close(deadline)
	18	+	var term, kill int
	19	+	p := &cliProcess{
	20	+	waitErr: make(chan error),
	21	+	waitTimeout: func() <-chan time.Time { return deadline },
	22	+	forceCleanup: func() { term++ },
	23	+	forceKill: func() { kill++ },
	24	+	}
	25	+	if err := p.Close(); err == nil {
	26	+	t.Fatal("stuck public role close succeeded")
	27	+	}
	28	+	if term != 1 \|\| kill != 1 {
	29	+	t.Fatalf("TERM/KILL calls = %d/%d, want 1/1", term, kill)
	30	+	}
	31	+	}
	32	+
	33	+	func TestCLIProcessWaitReadyRequiresDaemonSocket(t *testing.T) {
	34	+	path := filepath.Join(t.TempDir(), "daemon.sock")
	35	+	listener, err := net.Listen("unix", path)
	36	+	if err != nil {
	37	+	t.Fatal(err)
	38	+	}
	39	+	defer listener.Close()
	40	+	if err := (&cliProcess{readyPath: path, waitErr: make(chan error, 1)}).WaitReady(); err != nil {
	41	+	t.Fatal(err)
	42	+	}
	43	+	}
	44	+
	45	+	func TestCLILauncherIsolatesRoleXDGEnvironmentAcrossRepetitions(t *testing.T) {
	46	+	captureDir := t.TempDir()
	47	+	t.Setenv("CAPTURE_DIR", captureDir)
	48	+	t.Setenv("XDG_RUNTIME_DIR", filepath.Join(t.TempDir(), "inherited-runtime"))
	49	+	t.Setenv("XDG_STATE_HOME", filepath.Join(t.TempDir(), "inherited-state"))
	50	+
	51	+	bin := filepath.Join(t.TempDir(), "capture-env")
	52	+	if err := os.WriteFile(bin, []byte("#!/bin/sh\nenv > \"$CAPTURE_DIR/$VEV_PERF_PROCESS_ID\"\n"), 0o700); err != nil {
	53	+	t.Fatal(err)
	54	+	}
	55	+	launcher := &cliLauncher{bin: bin}
	56	+	for run := 1; run <= 3; run++ {
	57	+	runDir := filepath.Join(t.TempDir(), fmt.Sprintf("run-%d", run))
	58	+	if err := os.MkdirAll(runDir, 0o700); err != nil {
	59	+	t.Fatal(err)
	60	+	}
	61	+	roles := []string{"daemon", "client"}
	62	+	for _, role := range roles {
	63	+	id := fmt.Sprintf("run-%d-%s", run, role)
	64	+	p, err := launcher.Launch(processMapping{
	65	+	Role: role, TracePath: filepath.Join(runDir, role+".jsonl"), ProcessID: id,
	66	+	}, roleCommand{Args: []string{"test"}})
	67	+	if err != nil {
	68	+	t.Fatalf("run %d %s launch: %v", run, role, err)
	69	+	}
	70	+	runtimeDir := launcher.runtimes[runDir]
	71	+	if err := p.Close(); err != nil {
	72	+	t.Fatalf("run %d %s close: %v", run, role, err)
	73	+	}
	74	+
	75	+	env, err := os.ReadFile(filepath.Join(captureDir, id))
	76	+	if err != nil {
	77	+	t.Fatalf("run %d %s capture: %v", run, role, err)
	78	+	}
	79	+	for _, want := range []string{
	80	+	"XDG_RUNTIME_DIR=" + runtimeDir,
	81	+	"XDG_STATE_HOME=" + filepath.Join(runDir, "state"),
	82	+	} {
	83	+	name := strings.SplitN(want, "=", 2)[0]
	84	+	if count := bytes.Count(env, []byte(name+"=")); count != 1 {
	85	+	t.Fatalf("run %d %s %s entries = %d, want exactly one; env:\n%s", run, role, name, count, env)
	86	+	}
	87	+	if !bytes.Contains(env, []byte(want+"\n")) {
	88	+	t.Fatalf("run %d %s missing %q; env:\n%s", run, role, want, env)
	89	+	}
	90	+	}
	91	+	}
	92	+	if err := launcher.releaseRuntime(runDir); err != nil {
	93	+	t.Fatalf("run %d release runtime: %v", run, err)
	94	+	}
	95	+	}
	96	+	}
	97	+
	98	+	func TestCLIProcessCloseUsesPublicDaemonShutdown(t *testing.T) {
	99	+	wait := make(chan error, 1)
	100	+	wait <- nil
	101	+	var shutdown int
	102	+	p := &cliProcess{waitErr: wait, shutdown: func() error { shutdown++; return nil }}
	103	+	if err := p.Close(); err != nil {
	104	+	t.Fatal(err)
	105	+	}
	106	+	if shutdown != 0 {
	107	+	t.Fatalf("shutdown calls = %d, want 0 for an exited daemon", shutdown)
	108	+	}
	109	+
	110	+	wait = make(chan error, 1)
	111	+	p = &cliProcess{waitErr: wait, shutdown: func() error { shutdown++; wait <- nil; return nil }}
	112	+	if err := p.Close(); err != nil {
	113	+	t.Fatal(err)
	114	+	}
	115	+	if shutdown != 1 {
	116	+	t.Fatalf("shutdown calls = %d, want 1", shutdown)
	117	+	}
	118	+	}
	119	+
	120	+	func TestHarnessCanonicalLocalSmokeRealRoles(t *testing.T) {
	121	+	if testing.Short() {
	122	+	t.Skip("public CLI smoke")
	123	+	}
	124	+	root, err := filepath.Abs(filepath.Join("..", ".."))
	125	+	if err != nil {
	126	+	t.Fatal(err)
	127	+	}
	128	+	bin := filepath.Join(t.TempDir(), "vev")
	129	+	build := exec.Command("/usr/local/go/bin/go", "build", "-o", bin, "./")
	130	+	build.Dir = root
	131	+	if output, err := build.CombinedOutput(); err != nil {
	132	+	t.Fatalf("build public CLI: %v\n%s", err, output)
	133	+	}
	134	+	m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
	135	+	if err != nil {
	136	+	t.Fatal(err)
	137	+	}
	138	+	if err := validateManifest(m); err != nil {
	139	+	t.Fatal(err)
	140	+	}
	141	+	var local scenario
	142	+	for _, s := range m.Scenarios {
	143	+	if s.Topology == "1x4" && s.Workload == "idle" && s.Transport == "local" {
	144	+	local = s
	145	+	break
	146	+	}
	147	+	}
	148	+	if local.ID == "" {
	149	+	t.Fatal("canonical local scenario missing")
	150	+	}
	151	+	dir := t.TempDir()
	152	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	153	+	if err != nil {
	154	+	t.Fatal(err)
	155	+	}
	156	+	defer raw.Close()
	157	+	h := defaultHarness()
	158	+	h.clock = &fakeClock{} // injected duration keeps the public 30s contract bounded in test
	159	+	h.launcher = &cliLauncher{bin: bin}
	160	+	result, err := h.runOne(options{vevBin: bin, out: dir, duration: minimumDuration}, m, local, 1, raw)
	161	+	if err != nil {
	162	+	t.Fatal(err)
	163	+	}
	164	+	if result.Samples < minimumInIntervalEventSamples \|\| len(result.Processes) != 2 {
	165	+	t.Fatalf("canonical local smoke result=%+v", result)
	166	+	}
	167	+	}
	168	+
	169	+	func TestTraceEnvironmentMatchesProcessMapping(t *testing.T) {
	170	+	mapping := processMapping{TracePath: "/tmp/trace.jsonl", ProcessID: "p-01", Scenario: "1x4-idle-local", Run: 3}
	171	+	got := strings.Join(traceEnvironment(mapping), "\n")
	172	+	for _, want := range []string{"VEV_PERF_TRACE=/tmp/trace.jsonl", "VEV_PERF_PROCESS_ID=p-01", "VEV_PERF_SCENARIO=1x4-idle-local", "VEV_PERF_RUN=3"} {
	173	+	if !strings.Contains(got, want) {
	174	+	t.Errorf("trace environment missing %q: %s", want, got)
	175	+	}
	176	+	}
	177	+	}
	178	+
	179	+	func TestCLIProcessCloseGracefulDetachCompletesSpanBeforeForcedCleanup(t *testing.T) {
	180	+	var order []string
	181	+	waitErr := make(chan error, 1)
	182	+	timeout := make(chan time.Time)
	183	+	p := &cliProcess{
	184	+	pty: &orderedPTY{
	185	+	write: func(input []byte) {
	186	+	if string(input) != "exit\n" {
	187	+	t.Fatalf("graceful exit input=%q", input)
	188	+	}
	189	+	order = append(order, "exit_written")
	190	+	// The graceful client exit has closed its transport and emitted
	191	+	// its adapter receive-end mark before cmd.Wait completes.
	192	+	order = append(order, "adapter_receive_end")
	193	+	waitErr <- nil
	194	+	},
	195	+	close: func() { order = append(order, "pty_closed") },
	196	+	},
	197	+	output: &orderedOutput{close: func() { order = append(order, "output_closed") }},
	198	+	waitErr: waitErr,
	199	+	waitTimeout: func() <-chan time.Time { return timeout },
	200	+	forceCleanup: func() {
	201	+	order = append(order, "forced_process_group_cleanup")
	202	+	},
	203	+	}
	204	+
	205	+	if err := p.Close(); err != nil {
	206	+	t.Fatal(err)
	207	+	}
	208	+	if !equalStrings(order, []string{"exit_written", "adapter_receive_end", "pty_closed", "output_closed"}) {
	209	+	t.Fatalf("cleanup order=%q", order)
	210	+	}
	211	+	}
	212	+
	213	+	func TestCLIProcessWarmupWaitsForDelayedTerminalReadiness(t *testing.T) {
	214	+	allowWrite := make(chan struct{})
	215	+	pty, output := &stagedPTY{allow: allowWrite, writes: make(chan []byte, 1)}, &fakeOutput{}
	216	+	p := &cliProcess{pty: pty, output: output, chunks: make(chan []byte)}
	217	+	input := workloadInput(scenario{ID: "s"}, 1, "warmup")
	218	+	done := make(chan error, 1)
	219	+	go func() { done <- p.Warmup(input) }()
	220	+
	221	+	// This is public client output from the initial application prompt/state,
	222	+	// emitted only after the client has entered raw mode. A premature PTY write
	223	+	// blocks on allowWrite and therefore cannot consume this delayed readiness.
	224	+	go func() {
	225	+	p.chunks <- []byte("shell prompt$ ")
	226	+	close(allowWrite)
	227	+	}()
	228	+	select {
	229	+	case got := <-pty.writes:
	230	+	if !bytes.Equal(got, input) {
	231	+	t.Fatalf("warmup write=%q want %q", got, input)
	232	+	}
	233	+	case <-time.After(time.Second):
	234	+	t.Fatal("warmup did not wait for and then proceed from client readiness")
	235	+	}
	236	+	go func() { p.chunks <- append([]byte("application "), inputMarker(input)...); close(p.chunks) }()
	237	+	if err := <-done; err != nil {
	238	+	t.Fatal(err)
	239	+	}
	240	+	if output.syncs != 1 {
	241	+	t.Fatalf("readiness output was not flushed: syncs=%d", output.syncs)
	242	+	}
	243	+	}
	244	+
	245	+	func TestCLIProcessRejectsPTYLocalEchoAsFlushEvidence(t *testing.T) {
	246	+	pty, output := &fakePTY{}, &fakeOutput{}
	247	+	p := &cliProcess{pty: pty, output: output, chunks: make(chan []byte, 1)}
	248	+	input := workloadInput(scenario{ID: "s"}, 1, "measured-1")
	249	+	p.chunks <- newPTYLocalEcho(input).expected
	250	+	close(p.chunks)
	251	+	flushed := false
	252	+	if err := p.Measure(input, func() error { return nil }, func() error { flushed = true; return nil }); err == nil {
	253	+	t.Fatal("local PTY echo satisfied terminal flush boundary")
	254	+	}
	255	+	if flushed \|\| output.syncs != 0 {
	256	+	t.Fatalf("local echo stamped a flush: flushed=%t syncs=%d", flushed, output.syncs)
	257	+	}
	258	+	}
	259	+
	260	+	func TestCLIProcessPairsApplicationOutputWithSuccessfulFlush(t *testing.T) {
	261	+	pty, output := &fakePTY{}, &fakeOutput{}
	262	+	p := &cliProcess{pty: pty, output: output, chunks: make(chan []byte, 1)}
	263	+	input := workloadInput(scenario{ID: "s"}, 1, "measured-1")
	264	+	echo := newPTYLocalEcho(input).expected
	265	+	p.chunks <- append(append([]byte(nil), echo...), append([]byte("application "), inputMarker(input)...)...)
	266	+	order := []string{}
	267	+	if err := p.Measure(input, func() error { order = append(order, "injected"); return nil }, func() error { order = append(order, "flushed"); return nil }); err != nil {
	268	+	t.Fatal(err)
	269	+	}
	270	+	if len(pty.writes) != 1 \|\| output.syncs != 1 \|\| !equalStrings(order, []string{"injected", "flushed"}) {
	271	+	t.Fatalf("writes=%q syncs=%d boundary order=%q", pty.writes, output.syncs, order)
	272	+	}
	273	+	}
	274	+
	275	+	func TestHarnessUsesPublicRoleCommandsAndPTYWorkloads(t *testing.T) {
	276	+	for _, tc := range []struct {
	277	+	role string
	278	+	want []string
	279	+	}{
	280	+	{"daemon", []string{"--daemon"}},
	281	+	{"client", []string{"new", "perf-s-001"}},
	282	+	{"ssh_stdio_peer", []string{"_stdio", "perf-s-001"}},
	283	+	{"udp_peer", []string{"_udp-proxy", "perf-s-001"}},
	284	+	} {
	285	+	t.Run(tc.role, func(t *testing.T) {
	286	+	got := roleArgs(scenario{ID: "s", Transport: "local"}, processMapping{Role: tc.role, Run: 1})
	287	+	if !equalStrings(got.Args, tc.want) {
	288	+	t.Fatalf("args=%q want %q", got.Args, tc.want)
	289	+	}
	290	+	})
	291	+	}
	292	+	input := string(workloadInput(scenario{ID: "s", Workload: "interactive_flood"}, 1, "measured-1"))
	293	+	if string(inputMarker([]byte(input))) != "__VEV_HARNESS_s_r1_measured-1__" \|\| !bytes.Contains([]byte(input), []byte("while [ $i -lt 128 ]")) \|\| !strings.HasSuffix(input, "printf '__VEV_HARNESS_s_r1_measured-1__\\n'\n") {
	294	+	t.Fatalf("workload is not real PTY shell input with an observable marker: %q", input)
	295	+	}
	296	+	}

A cmd/vev-perf-harness/main.go +20 -0

@@ -0,0 +1,20 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"fmt"
	5	+	"os"
	6	+	"time"
	7	+
	8	+	"github.com/bnema/vev/pkg/safedir"
	9	+	)
	10	+
	11	+	func main() {
	12	+	if err := run(os.Args[1:], defaultHarness()); err != nil {
	13	+	fmt.Fprintln(os.Stderr, "vev-perf-harness:", err)
	14	+	os.Exit(2)
	15	+	}
	16	+	}
	17	+
	18	+	func defaultHarness() *harness {
	19	+	return &harness{clock: systemClock{time.Now()}, mkdir: safedir.EnsurePrivate, createRunDir: createExclusiveRunDir, create: func(p string) (*os.File, error) { return os.OpenFile(p, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600) }, removeAll: os.RemoveAll, gitSHA: recordedGitSHA}
	20	+	}

A cmd/vev-perf-harness/model.go +132 -0

@@ -0,0 +1,132 @@
	1	+	package main
	2	+
	3	+	import "time"
	4	+
	5	+	const (
	6	+	minimumDuration = 30 * time.Second
	7	+	minimumRepetitions = 10
	8	+	minimumInIntervalEventSamples = 10
	9	+	measuredEventCadence = time.Second
	10	+	)
	11	+
	12	+	type options struct {
	13	+	vevBin, manifest, out string
	14	+	scenario string
	15	+	warmup, duration time.Duration
	16	+	repetitions int
	17	+	}
	18	+
	19	+	type manifest struct {
	20	+	Schema uint16 `json:"schema"`
	21	+	Topologies []topology `json:"topologies"`
	22	+	Workloads []string `json:"workloads"`
	23	+	Transports []transport `json:"transports"`
	24	+	Scenarios []scenario `json:"scenarios"`
	25	+	}
	26	+
	27	+	type topology struct {
	28	+	ID string `json:"id"`
	29	+	Geometry string `json:"geometry"`
	30	+	RowsPerPane int `json:"rows_per_pane"`
	31	+	}
	32	+
	33	+	type transport struct {
	34	+	ID string `json:"id"`
	35	+	Kind string `json:"kind"`
	36	+	RTTMS int `json:"rtt_ms,omitempty"`
	37	+	LossPercent int `json:"loss_percent,omitempty"`
	38	+	}
	39	+
	40	+	type scenario struct {
	41	+	ID string `json:"id"`
	42	+	Topology string `json:"topology"`
	43	+	Workload string `json:"workload"`
	44	+	Transport string `json:"transport"`
	45	+	Roles []string `json:"roles"`
	46	+	InapplicableReason string `json:"inapplicable_reason,omitempty"`
	47	+	}
	48	+
	49	+	type processMapping struct {
	50	+	ProcessID string `json:"process_id"`
	51	+	ClockDomain string `json:"clock_domain"`
	52	+	TracePath string `json:"trace_path"`
	53	+	Role string `json:"role"`
	54	+	Scenario string `json:"scenario"`
	55	+	Run int `json:"run"`
	56	+	Identity string `json:"identity"`
	57	+	}
	58	+
	59	+	type runManifest struct {
	60	+	Scenario string `json:"scenario"`
	61	+	Run int `json:"run"`
	62	+	Processes []processMapping `json:"processes"`
	63	+	}
	64	+
	65	+	// measuredInterval belongs solely to the harness clock domain. A paired event
	66	+	// is eligible only when both owned boundaries fall inside this interval.
	67	+	type measuredInterval struct {
	68	+	Start int64
	69	+	End int64
	70	+	}
	71	+
	72	+	type eventSample struct {
	73	+	Sequence uint64 `json:"sequence"`
	74	+	Injected int64 `json:"injected_tick"`
	75	+	Flushed int64 `json:"flushed_tick"`
	76	+	Latency int64 `json:"latency_nanos"`
	77	+	}
	78	+
	79	+	type span struct {
	80	+	Component, Name string
	81	+	Samples []int64
	82	+	}
	83	+
	84	+	type runResult struct {
	85	+	Spans []span `json:"process_local_spans"`
	86	+	Scenario string `json:"scenario"`
	87	+	Run int `json:"run"`
	88	+	Samples int `json:"samples"`
	89	+	EndToEnd []int64 `json:"end_to_end_nanos"`
	90	+	Event distribution `json:"event_end_to_end"`
	91	+	Cadence distribution `json:"event_cadence_nanos"`
	92	+	CadenceSamples []int64 `json:"-"`
	93	+	MaxGap int64 `json:"event_max_gap_nanos"`
	94	+	Processes []processMapping `json:"processes"`
	95	+	}
	96	+
	97	+	type summary struct {
	98	+	Schema uint16 `json:"schema"`
	99	+	GitSHA string `json:"git_sha"`
	100	+	Warmup string `json:"warmup"`
	101	+	Duration string `json:"duration"`
	102	+	Repetitions int `json:"repetitions"`
	103	+	EndToEnd distribution `json:"harness_end_to_end"`
	104	+	Cadence distribution `json:"harness_event_cadence_nanos"`
	105	+	MaxGap int64 `json:"harness_event_max_gap_nanos"`
	106	+	RunP50Dispersion distribution `json:"run_p50_dispersion"`
	107	+	Spans []spanSummary `json:"process_local_spans"`
	108	+	Runs int `json:"runs"`
	109	+	}
	110	+
	111	+	type distribution struct {
	112	+	Count int `json:"count"`
	113	+	P50 int64 `json:"p50"`
	114	+	P95 int64 `json:"p95"`
	115	+	P99 int64 `json:"p99"`
	116	+	Max int64 `json:"max"`
	117	+	}
	118	+
	119	+	type spanSummary struct {
	120	+	Component string `json:"component"`
	121	+	Name string `json:"name"`
	122	+	Distribution distribution `json:"distribution"`
	123	+	}
	124	+
	125	+	type clock interface {
	126	+	Now() int64
	127	+	Sleep(time.Duration)
	128	+	}
	129	+
	130	+	type systemClock struct{ start time.Time }
	131	+
	132	+	var spanPairs = []spanPair{{"capture_start", "capture_end", "capture_duration"}, {"compose_start", "compose_end", "compose_duration"}, {"diff_start", "diff_end", "diff_duration"}, {"queue_enqueued", "queue_dequeued", "queue_wait"}, {"ack_blocked_start", "ack_blocked_end", "ack_blocked_interval"}, {"emit_start", "emit_end", "emit_duration"}, {"adapter_send_start", "adapter_send_end", "adapter_send_duration"}, {"adapter_receive_start", "adapter_receive_end", "adapter_receive_duration"}}

A cmd/vev-perf-harness/options_manifest.go +217 -0

@@ -0,0 +1,217 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"encoding/json"
	5	+	"errors"
	6	+	"flag"
	7	+	"fmt"
	8	+	"io"
	9	+	"os"
	10	+	"path/filepath"
	11	+	)
	12	+
	13	+	func parseOptions(args []string) (options, error) {
	14	+	var o options
	15	+	fs := flag.NewFlagSet("vev-perf-harness", flag.ContinueOnError)
	16	+	fs.SetOutput(io.Discard)
	17	+	fs.StringVar(&o.vevBin, "vev-bin", "", "public vev binary")
	18	+	fs.StringVar(&o.manifest, "manifest", "", "manifest")
	19	+	fs.StringVar(&o.out, "out", "", "output directory")
	20	+	fs.StringVar(&o.scenario, "scenario", "", "one canonical scenario ID (after full manifest validation)")
	21	+	fs.DurationVar(&o.warmup, "warmup", 0, "warmup")
	22	+	fs.DurationVar(&o.duration, "duration", 0, "measurement duration")
	23	+	fs.IntVar(&o.repetitions, "repetitions", 0, "repetitions")
	24	+	if err := fs.Parse(args); err != nil {
	25	+	return o, err
	26	+	}
	27	+	seen := map[string]bool{}
	28	+	fs.Visit(func(f *flag.Flag) { seen[f.Name] = true })
	29	+	for _, name := range []string{"vev-bin", "manifest", "out", "warmup", "duration", "repetitions"} {
	30	+	if !seen[name] {
	31	+	return o, fmt.Errorf("--%s is required", name)
	32	+	}
	33	+	}
	34	+	if o.vevBin == "" \|\| o.manifest == "" \|\| o.out == "" {
	35	+	return o, errors.New("--vev-bin, --manifest, and --out are required")
	36	+	}
	37	+	if o.warmup < 0 {
	38	+	return o, errors.New("--warmup must not be negative")
	39	+	}
	40	+	if o.duration < minimumDuration {
	41	+	return o, fmt.Errorf("--duration must be at least %s", minimumDuration)
	42	+	}
	43	+	if o.repetitions < minimumRepetitions {
	44	+	return o, fmt.Errorf("--repetitions must be at least %d", minimumRepetitions)
	45	+	}
	46	+	return o, nil
	47	+	}
	48	+
	49	+	// resolvePathOptions captures the harness cwd once, before role launchers
	50	+	// assign their isolated working directories. All downstream filesystem access,
	51	+	// manifests, and process environments consequently refer to the same paths.
	52	+	func resolvePathOptions(o options) (options, error) {
	53	+	cwd, err := os.Getwd()
	54	+	if err != nil {
	55	+	return o, fmt.Errorf("resolve harness working directory: %w", err)
	56	+	}
	57	+	absolute := func(path string) string {
	58	+	if filepath.IsAbs(path) {
	59	+	return filepath.Clean(path)
	60	+	}
	61	+	return filepath.Clean(filepath.Join(cwd, path))
	62	+	}
	63	+	o.vevBin = absolute(o.vevBin)
	64	+	o.manifest = absolute(o.manifest)
	65	+	o.out = absolute(o.out)
	66	+	return o, nil
	67	+	}
	68	+
	69	+	func readManifest(path string) (manifest, error) {
	70	+	f, e := os.Open(path)
	71	+	if e != nil {
	72	+	return manifest{}, e
	73	+	}
	74	+	defer f.Close()
	75	+	var m manifest
	76	+	e = json.NewDecoder(f).Decode(&m)
	77	+	return m, e
	78	+	}
	79	+
	80	+	func validateManifest(m manifest) error {
	81	+	if m.Schema != 1 {
	82	+	return errors.New("unsupported manifest schema")
	83	+	}
	84	+	tops := map[string]bool{}
	85	+	for _, t := range m.Topologies {
	86	+	if t.ID == "" \|\| t.Geometry != "120x40" \|\| t.RowsPerPane != 10000 {
	87	+	return fmt.Errorf("invalid topology %q", t.ID)
	88	+	}
	89	+	tops[t.ID] = true
	90	+	}
	91	+	wantT := []string{"1x4", "4x1", "4x4", "8x1"}
	92	+	for _, v := range wantT {
	93	+	if !tops[v] {
	94	+	return fmt.Errorf("missing canonical topology %s", v)
	95	+	}
	96	+	}
	97	+	works := set(m.Workloads)
	98	+	for _, v := range []string{"idle", "active_output", "all_output", "inactive_output", "interactive_flood", "copy_search", "resize_sweep", "snapshot_output_resize", "attach_restore_tab_switch"} {
	99	+	if !works[v] {
	100	+	return fmt.Errorf("missing canonical workload %s", v)
	101	+	}
	102	+	}
	103	+	trans := map[string]bool{}
	104	+	for _, t := range m.Transports {
	105	+	if t.ID == "" {
	106	+	return errors.New("transport missing id")
	107	+	}
	108	+	if err := validateTransportFixture(t); err != nil {
	109	+	return err
	110	+	}
	111	+	trans[t.ID] = true
	112	+	}
	113	+	for _, v := range []string{"local", "ssh_stdio", "udp_baseline", "udp_25ms", "udp_100ms", "udp_loss_0pct", "udp_loss_1pct"} {
	114	+	if !trans[v] {
	115	+	return fmt.Errorf("missing canonical transport %s", v)
	116	+	}
	117	+	}
	118	+	seen := map[string]bool{}
	119	+	covered := map[string]bool{}
	120	+	for _, s := range m.Scenarios {
	121	+	if s.ID == "" \|\| seen[s.ID] {
	122	+	return fmt.Errorf("duplicate or empty scenario id %q", s.ID)
	123	+	}
	124	+	seen[s.ID] = true
	125	+	if !tops[s.Topology] \|\| !works[s.Workload] \|\| !trans[s.Transport] {
	126	+	return fmt.Errorf("scenario %s references unknown matrix entry", s.ID)
	127	+	}
	128	+	if s.InapplicableReason == "" && len(s.Roles) == 0 {
	129	+	return fmt.Errorf("scenario %s has no process roles", s.ID)
	130	+	}
	131	+	if s.InapplicableReason != "" && len(s.Roles) != 0 {
	132	+	return fmt.Errorf("inapplicable scenario %s must not launch roles", s.ID)
	133	+	}
	134	+	if s.InapplicableReason == "" && !equalRoleSet(s.Roles, requiredRoles(s.Transport)) {
	135	+	return fmt.Errorf("scenario %s roles do not route transport %s", s.ID, s.Transport)
	136	+	}
	137	+	covered[s.Topology+"\x00"+s.Workload+"\x00"+s.Transport] = true
	138	+	}
	139	+	for topologyID := range tops {
	140	+	for workload := range works {
	141	+	for transportID := range trans {
	142	+	key := topologyID + "\x00" + workload + "\x00" + transportID
	143	+	if !covered[key] {
	144	+	return fmt.Errorf("missing scenario or inapplicable reason for %s/%s/%s", topologyID, workload, transportID)
	145	+	}
	146	+	}
	147	+	}
	148	+	}
	149	+	return nil
	150	+	}
	151	+
	152	+	// selectScenario is intentionally called only after validateManifest. A bounded
	153	+	// local evidence run therefore cannot mask an incomplete canonical matrix.
	154	+	func selectScenario(m manifest, id string) ([]scenario, error) {
	155	+	for _, s := range m.Scenarios {
	156	+	if s.ID != id {
	157	+	continue
	158	+	}
	159	+	if s.InapplicableReason != "" {
	160	+	return nil, fmt.Errorf("scenario %s is inapplicable: %s", id, s.InapplicableReason)
	161	+	}
	162	+	return []scenario{s}, nil
	163	+	}
	164	+	return nil, fmt.Errorf("unknown canonical scenario %s", id)
	165	+	}
	166	+
	167	+	func requiredRoles(transportID string) []string {
	168	+	switch transportID {
	169	+	case "local":
	170	+	return []string{"daemon", "client"}
	171	+	case "ssh_stdio":
	172	+	return []string{"daemon", "client", "ssh_stdio_peer"}
	173	+	default:
	174	+	return []string{"daemon", "client", "udp_peer"}
	175	+	}
	176	+	}
	177	+
	178	+	func equalRoleSet(got, want []string) bool {
	179	+	if len(got) != len(want) {
	180	+	return false
	181	+	}
	182	+	seen := make(map[string]bool, len(got))
	183	+	for _, role := range got {
	184	+	seen[role] = true
	185	+	}
	186	+	for _, role := range want {
	187	+	if !seen[role] {
	188	+	return false
	189	+	}
	190	+	}
	191	+	return true
	192	+	}
	193	+
	194	+	func validateTransportFixture(t transport) error {
	195	+	want := map[string]transport{
	196	+	"local": {ID: "local", Kind: "local"},
	197	+	"ssh_stdio": {ID: "ssh_stdio", Kind: "ssh_stdio"},
	198	+	"udp_baseline": {ID: "udp_baseline", Kind: "udp"},
	199	+	"udp_25ms": {ID: "udp_25ms", Kind: "udp", RTTMS: 25},
	200	+	"udp_100ms": {ID: "udp_100ms", Kind: "udp", RTTMS: 100},
	201	+	"udp_loss_0pct": {ID: "udp_loss_0pct", Kind: "udp", LossPercent: 0},
	202	+	"udp_loss_1pct": {ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1},
	203	+	}
	204	+	w, ok := want[t.ID]
	205	+	if !ok \|\| t.Kind != w.Kind \|\| t.RTTMS != w.RTTMS \|\| t.LossPercent != w.LossPercent {
	206	+	return fmt.Errorf("invalid canonical transport fixture %+v", t)
	207	+	}
	208	+	return nil
	209	+	}
	210	+
	211	+	func set(v []string) map[string]bool {
	212	+	r := map[string]bool{}
	213	+	for _, x := range v {
	214	+	r[x] = true
	215	+	}
	216	+	return r
	217	+	}

A cmd/vev-perf-harness/options_manifest_test.go +201 -0

@@ -0,0 +1,201 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"encoding/json"
	5	+	"errors"
	6	+	"os"
	7	+	"path/filepath"
	8	+	"strings"
	9	+	"testing"
	10	+	"time"
	11	+	)
	12	+
	13	+	func TestHarnessScenarioSelectionKeepsCanonicalManifestValidation(t *testing.T) {
	14	+	root, err := filepath.Abs(filepath.Join("..", ".."))
	15	+	if err != nil {
	16	+	t.Fatal(err)
	17	+	}
	18	+	manifestPath := filepath.Join(root, "testdata", "perf", "manifest.json")
	19	+	m, err := readManifest(manifestPath)
	20	+	if err != nil {
	21	+	t.Fatal(err)
	22	+	}
	23	+	selected := m.Scenarios[0]
	24	+	h := defaultHarness()
	25	+	h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
	26	+	out := filepath.Join(t.TempDir(), "out")
	27	+	if err := run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h); err != nil {
	28	+	t.Fatal(err)
	29	+	}
	30	+	var runs []runResult
	31	+	b, err := os.ReadFile(filepath.Join(out, "runs.json"))
	32	+	if err != nil {
	33	+	t.Fatal(err)
	34	+	}
	35	+	if err := json.Unmarshal(b, &runs); err != nil {
	36	+	t.Fatal(err)
	37	+	}
	38	+	if len(runs) != 10 {
	39	+	t.Fatalf("selected canonical runs=%d, want 10", len(runs))
	40	+	}
	41	+	for _, result := range runs {
	42	+	if result.Scenario != selected.ID {
	43	+	t.Fatalf("selected scenario = %q, want %q", result.Scenario, selected.ID)
	44	+	}
	45	+	}
	46	+	}
	47	+
	48	+	func TestHarnessScenarioSelectionRejectsIncompleteCanonicalManifest(t *testing.T) {
	49	+	m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
	50	+	if err != nil {
	51	+	t.Fatal(err)
	52	+	}
	53	+	selected := m.Scenarios[0]
	54	+	m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
	55	+	dir := t.TempDir()
	56	+	manifestPath := filepath.Join(dir, "manifest.json")
	57	+	if err := os.WriteFile(manifestPath, []byte(mustJSON(m)), 0o600); err != nil {
	58	+	t.Fatal(err)
	59	+	}
	60	+	launcher := &fakeLauncher{}
	61	+	h := defaultHarness()
	62	+	h.clock, h.launcher = &fakeClock{}, launcher
	63	+	out := filepath.Join(dir, "out")
	64	+	err = run([]string{"--vev-bin", "ignored", "--manifest", manifestPath, "--out", out, "--scenario", selected.ID, "--warmup", "0s", "--duration", "30s", "--repetitions", "10"}, h)
	65	+	if err == nil \|\| !strings.Contains(err.Error(), "missing scenario or inapplicable reason") {
	66	+	t.Fatalf("incomplete manifest selection error = %v", err)
	67	+	}
	68	+	if len(launcher.mappings) != 0 {
	69	+	t.Fatalf("launched with incomplete canonical manifest: %+v", launcher.mappings)
	70	+	}
	71	+	if _, statErr := os.Stat(out); !errors.Is(statErr, os.ErrNotExist) {
	72	+	t.Fatalf("output created before manifest validation: %v", statErr)
	73	+	}
	74	+	}
	75	+
	76	+	func TestResolvePathOptionsMakesRelativeAndAbsoluteInputsIdentical(t *testing.T) {
	77	+	cwd, err := os.Getwd()
	78	+	if err != nil {
	79	+	t.Fatal(err)
	80	+	}
	81	+	absolute := options{
	82	+	vevBin: filepath.Join(cwd, "vev"),
	83	+	manifest: filepath.Join(cwd, "testdata", "perf", "manifest.json"),
	84	+	out: filepath.Join(cwd, "testdata", "perf", "results"),
	85	+	}
	86	+	relative := options{
	87	+	vevBin: "vev",
	88	+	manifest: "testdata/perf/manifest.json",
	89	+	out: "testdata/perf/results",
	90	+	}
	91	+	gotRelative, err := resolvePathOptions(relative)
	92	+	if err != nil {
	93	+	t.Fatal(err)
	94	+	}
	95	+	gotAbsolute, err := resolvePathOptions(absolute)
	96	+	if err != nil {
	97	+	t.Fatal(err)
	98	+	}
	99	+	if gotRelative != gotAbsolute {
	100	+	t.Fatalf("relative paths resolved to %+v, absolute paths to %+v", gotRelative, gotAbsolute)
	101	+	}
	102	+	}
	103	+
	104	+	func TestHarnessFlagsRejectShortMeasurements(t *testing.T) {
	105	+	for _, args := range [][]string{
	106	+	{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "29s", "--repetitions", "10"},
	107	+	{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "9"},
	108	+	{"--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"},
	109	+	} {
	110	+	if _, err := parseOptions(args); err == nil {
	111	+	t.Fatalf("parseOptions(%v) succeeded", args)
	112	+	}
	113	+	}
	114	+	if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--duration", "30s", "--repetitions", "10"}); err == nil {
	115	+	t.Fatal("missing --warmup accepted")
	116	+	}
	117	+	if _, err := parseOptions([]string{"--vev-bin", "vev", "--manifest", "m", "--out", "o", "--warmup", "1s", "--duration", "30s", "--repetitions", "10"}); err != nil {
	118	+	t.Fatal(err)
	119	+	}
	120	+	}
	121	+
	122	+	func TestHarnessManifestCoversCanonicalMatrix(t *testing.T) {
	123	+	m, err := readManifest(filepath.Join("..", "..", "testdata", "perf", "manifest.json"))
	124	+	if err != nil {
	125	+	t.Fatal(err)
	126	+	}
	127	+	if err := validateManifest(m); err != nil {
	128	+	t.Fatal(err)
	129	+	}
	130	+	if len(m.Scenarios) != 497 {
	131	+	t.Fatalf("scenarios=%d", len(m.Scenarios))
	132	+	}
	133	+	m.Scenarios = m.Scenarios[:len(m.Scenarios)-1]
	134	+	if err := validateManifest(m); err == nil {
	135	+	t.Fatal("missing combination accepted")
	136	+	}
	137	+	}
	138	+
	139	+	func TestHarnessCreatesExclusiveTraceManifestAndEvidence(t *testing.T) {
	140	+	dir := t.TempDir()
	141	+	raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	142	+	if err != nil {
	143	+	t.Fatal(err)
	144	+	}
	145	+	defer raw.Close()
	146	+	launcher := &fakeLauncher{}
	147	+	h := defaultHarness()
	148	+	h.clock = &fakeClock{}
	149	+	h.launcher = launcher
	150	+	o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
	151	+	r, err := h.runOne(o, manifest{}, scenario{ID: "4x4-local", Roles: []string{"daemon", "client"}}, 1, raw)
	152	+	if err != nil {
	153	+	t.Fatal(err)
	154	+	}
	155	+	if !launcher.manifestPresent {
	156	+	t.Fatal("launched before complete run manifest")
	157	+	}
	158	+	if len(r.EndToEnd) < minimumInIntervalEventSamples \|\| r.Samples < minimumInIntervalEventSamples {
	159	+	t.Fatalf("samples=%+v", r)
	160	+	}
	161	+	seen := map[string]bool{}
	162	+	for _, m := range r.Processes {
	163	+	if m.ProcessID == "" \|\| m.ClockDomain != m.ProcessID \|\| m.TracePath == "" \|\| seen[m.TracePath] {
	164	+	t.Fatalf("bad mapping %+v", m)
	165	+	}
	166	+	seen[m.TracePath] = true
	167	+	if _, err := os.Stat(m.TracePath); err != nil {
	168	+	t.Fatal(err)
	169	+	}
	170	+	}
	171	+	b, err := os.ReadFile(filepath.Join(dir, "4x4-local-run-001", "manifest.json"))
	172	+	if err != nil {
	173	+	t.Fatal(err)
	174	+	}
	175	+	var got runManifest
	176	+	if err := json.Unmarshal(b, &got); err != nil {
	177	+	t.Fatal(err)
	178	+	}
	179	+	if len(got.Processes) != 2 {
	180	+	t.Fatalf("manifest=%+v", got)
	181	+	}
	182	+	if len(launcher.process) != 2 \|\| !launcher.process[0].closed \|\| len(launcher.process[0].warmups) != 0 \|\| len(launcher.process[1].warmups) != 1 \|\| len(launcher.process[1].measures) < minimumInIntervalEventSamples {
	183	+	t.Fatalf("process lifecycle not recorded: %+v", launcher.process)
	184	+	}
	185	+	}
	186	+
	187	+	func TestHarnessRejectsMissingManifestRoleTrace(t *testing.T) {
	188	+	dir := t.TempDir()
	189	+	missing := filepath.Join(dir, "daemon.jsonl")
	190	+	client := filepath.Join(dir, "client.jsonl")
	191	+	if err := os.WriteFile(client, nil, 0o600); err != nil {
	192	+	t.Fatal(err)
	193	+	}
	194	+	_, err := mergeProcessTraces([]processMapping{
	195	+	{ProcessID: "daemon", ClockDomain: "daemon", TracePath: missing, Role: "daemon", Scenario: "s", Run: 1},
	196	+	{ProcessID: "client", ClockDomain: "client", TracePath: client, Role: "client", Scenario: "s", Run: 1},
	197	+	})
	198	+	if !errors.Is(err, os.ErrNotExist) \|\| !strings.Contains(err.Error(), "daemon role") {
	199	+	t.Fatalf("missing daemon trace error = %v, want daemon role not-exist error", err)
	200	+	}
	201	+	}

R cmd/vev-perf-harness/harness_2.go → cmd/vev-perf-harness/runner.go +18 -350 renamed, 53% similar

@@ -1,10 +1,7 @@
1	1		package main
2	2
3	3		import (
4		-	"bytes"
5		-	"encoding/json"
6	4		"errors"
7		-	"flag"
8	5		"fmt"
9	6		"io"
10	7		"os"

@@ -13,145 +10,33 @@ import (
13	10		"runtime/debug"
14	11		"sort"
15	12		"strings"
16		-	"syscall"
17	13		"time"
18	14
19	15		"github.com/bnema/vev/pkg/safedir"
20		-	"golang.org/x/sys/unix"
21	16		)
22	17
23		-	func (p *cliProcess) Close() error {
24		-	var result error
25		-	p.closed.Do(func() {
26		-	if p.done != nil {
27		-	close(p.done)
28		-	}
29		-	// A public terminal client owns a shell session. Ask that shell to exit
30		-	// before closing the PTY so its transport can finish its receive span;
31		-	// this avoids turning ordinary harness teardown into an aborted trace.
32		-	exited := false
33		-	if p.shutdown != nil && p.waitErr != nil {
34		-	select {
35		-	case <-p.waitErr:
36		-	exited = true
37		-	default:
38		-	// The client may have just ended the last session and raced this
39		-	// best-effort public shutdown. The normal bounded reaping path below
40		-	// remains authoritative in that case.
41		-	_ = p.shutdown()
42		-	}
43		-	}
44		-	if p.pty != nil && p.waitErr != nil {
45		-	_, _ = p.pty.Write([]byte("exit\n"))
46		-	select {
47		-	case <-p.waitErr:
48		-	exited = true
49		-	case <-p.closeDeadline():
50		-	}
51		-	}
52		-	if p.pty != nil {
53		-	_ = p.pty.Close()
54		-	}
55		-	if p.waitErr != nil && !exited {
56		-	select {
57		-	case <-p.waitErr:
58		-	case <-p.closeDeadline():
59		-	// Teardown is deliberately bounded: a stuck public CLI or child
60		-	// must not hold a local harness run indefinitely.
61		-	p.forceProcessGroupCleanup()
62		-	select {
63		-	case <-p.waitErr:
64		-	case <-p.closeDeadline():
65		-	p.forceProcessGroupKill()
66		-	select {
67		-	case <-p.waitErr:
68		-	case <-p.closeDeadline():
69		-	result = errors.New("timed out reaping public CLI process")
70		-	}
71		-	}
72		-	}
73		-	}
74		-	if p.output != nil {
75		-	if err := p.output.Close(); result == nil {
76		-	result = err
77		-	}
78		-	}
79		-	if p.cleanupRuntime != nil {
80		-	if err := p.cleanupRuntime(); result == nil {
81		-	result = err
82		-	}
83		-	}
84		-	})
85		-	return result
	18	+	type harnessMark struct {
	19	+	Scenario string `json:"scenario"`
	20	+	Run int `json:"run"`
	21	+	Sequence uint64 `json:"sequence"`
	22	+	Kind string `json:"kind"`
	23	+	Tick int64 `json:"tick"`
	24	+	Valid bool `json:"valid"`
86	25		}
87	26
88		-	func (p *cliProcess) closeDeadline() <-chan time.Time {
89		-	if p.waitTimeout != nil {
90		-	return p.waitTimeout()
91		-	}
92		-	if p.shutdown != nil {
93		-	// A public kill asks the daemon to finish bounded detached notices and
94		-	// close blocked carriage before it exits; do not turn that graceful
95		-	// protocol into a SIGKILL at the two-second client deadline.
96		-	return time.After(5 * time.Second)
97		-	}
98		-	return time.After(2 * time.Second)
99		-	}
	27	+	func (c systemClock) Now() int64 { return time.Since(c.start).Nanoseconds() }
100	28
101		-	func (p *cliProcess) forceProcessGroupCleanup() {
102		-	if p.forceCleanup != nil {
103		-	p.forceCleanup()
104		-	return
105		-	}
106		-	if p.cmd != nil && p.cmd.Process != nil {
107		-	// The client is a session/process-group leader; forced cleanup must
108		-	// include its ssh seam/_stdio descendants.
109		-	_ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGTERM)
110		-	}
111		-	}
	29	+	func (systemClock) Sleep(d time.Duration) { time.Sleep(d) }
112	30
113		-	func (p *cliProcess) forceProcessGroupKill() {
114		-	if p.forceKill != nil {
115		-	p.forceKill()
116		-	return
117		-	}
118		-	if p.cmd != nil && p.cmd.Process != nil {
119		-	_ = syscall.Kill(-p.cmd.Process.Pid, syscall.SIGKILL)
120		-	}
121		-	}
122		-
123		-	// openPTY is the small Linux PTY boundary needed to drive the public terminal
124		-	// client. No daemon or adapter implementation is imported by this command.
125		-	func openPTY() (os.File, os.File, error) {
126		-	masterFD, err := unix.Open("/dev/ptmx", unix.O_RDWR\|unix.O_NOCTTY, 0)
127		-	if err != nil {
128		-	return nil, nil, err
129		-	}
130		-	closeMaster := func(err error) (os.File, os.File, error) { _ = unix.Close(masterFD); return nil, nil, err }
131		-	number, err := unix.IoctlGetInt(masterFD, unix.TIOCGPTN)
132		-	if err != nil {
133		-	return closeMaster(err)
134		-	}
135		-	if err := unix.IoctlSetPointerInt(masterFD, unix.TIOCSPTLCK, 0); err != nil {
136		-	return closeMaster(err)
137		-	}
138		-	slaveFD, err := unix.Open(fmt.Sprintf("/dev/pts/%d", number), unix.O_RDWR\|unix.O_NOCTTY, 0)
139		-	if err != nil {
140		-	return closeMaster(err)
141		-	}
142		-	return os.NewFile(uintptr(masterFD), "vev-client-master"), os.NewFile(uintptr(slaveFD), "vev-client-slave"), nil
143		-	}
144		-
145		-	func inputMarker(input []byte) []byte {
146		-	start := bytes.Index(input, []byte("__VEV_HARNESS_"))
147		-	if start < 0 {
148		-	return input
149		-	}
150		-	end := bytes.Index(input[start:], []byte(`\n`))
151		-	if end < 0 {
152		-	return input[start:]
	31	+	func newPTYLocalEcho(input []byte) *ptyLocalEcho {
	32	+	expected := make([]byte, 0, len(input))
	33	+	for _, b := range input {
	34	+	if b == '\n' {
	35	+	expected = append(expected, '\r')
	36	+	}
	37	+	expected = append(expected, b)
153	38		}
154		-	return input[start : start+end]
	39	+	return &ptyLocalEcho{expected: expected}
155	40		}
156	41
157	42		type harness struct {

@@ -165,16 +50,6 @@ type harness struct {
165	50		selectScenarios func(manifest) []scenario // test-only bounded fixture seam
166	51		}
167	52
168		-	func main() {
169		-	if err := run(os.Args[1:], defaultHarness()); err != nil {
170		-	fmt.Fprintln(os.Stderr, "vev-perf-harness:", err)
171		-	os.Exit(2)
172		-	}
173		-	}
174		-	func defaultHarness() *harness {
175		-	return &harness{clock: systemClock{time.Now()}, mkdir: safedir.EnsurePrivate, createRunDir: createExclusiveRunDir, create: func(p string) (*os.File, error) { return os.OpenFile(p, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600) }, removeAll: os.RemoveAll, gitSHA: recordedGitSHA}
176		-	}
177		-
178	53		// createExclusiveRunDir prevents stale runtime, socket, state, or trace files
179	54		// from an earlier invocation being mistaken for the next repetition's evidence.
180	55		func createExclusiveRunDir(path string) error {

@@ -205,6 +80,7 @@ func recordedGitSHA() string {
205	80		}
206	81		return "unknown"
207	82		}
	83	+
208	84		func run(args []string, h *harness) error {
209	85		opt, err := parseOptions(args)
210	86		if err != nil {

@@ -278,210 +154,6 @@ func aggregateEventResults(results []runResult) (all, cadence []int64, maxGap in
278	154		return all, cadence, maxGap, runP50s
279	155		}
280	156
281		-	func parseOptions(args []string) (options, error) {
282		-	var o options
283		-	fs := flag.NewFlagSet("vev-perf-harness", flag.ContinueOnError)
284		-	fs.SetOutput(io.Discard)
285		-	fs.StringVar(&o.vevBin, "vev-bin", "", "public vev binary")
286		-	fs.StringVar(&o.manifest, "manifest", "", "manifest")
287		-	fs.StringVar(&o.out, "out", "", "output directory")
288		-	fs.StringVar(&o.scenario, "scenario", "", "one canonical scenario ID (after full manifest validation)")
289		-	fs.DurationVar(&o.warmup, "warmup", 0, "warmup")
290		-	fs.DurationVar(&o.duration, "duration", 0, "measurement duration")
291		-	fs.IntVar(&o.repetitions, "repetitions", 0, "repetitions")
292		-	if err := fs.Parse(args); err != nil {
293		-	return o, err
294		-	}
295		-	seen := map[string]bool{}
296		-	fs.Visit(func(f *flag.Flag) { seen[f.Name] = true })
297		-	for _, name := range []string{"vev-bin", "manifest", "out", "warmup", "duration", "repetitions"} {
298		-	if !seen[name] {
299		-	return o, fmt.Errorf("--%s is required", name)
300		-	}
301		-	}
302		-	if o.vevBin == "" \|\| o.manifest == "" \|\| o.out == "" {
303		-	return o, errors.New("--vev-bin, --manifest, and --out are required")
304		-	}
305		-	if o.warmup < 0 {
306		-	return o, errors.New("--warmup must not be negative")
307		-	}
308		-	if o.duration < minimumDuration {
309		-	return o, fmt.Errorf("--duration must be at least %s", minimumDuration)
310		-	}
311		-	if o.repetitions < minimumRepetitions {
312		-	return o, fmt.Errorf("--repetitions must be at least %d", minimumRepetitions)
313		-	}
314		-	return o, nil
315		-	}
316		-
317		-	// resolvePathOptions captures the harness cwd once, before role launchers
318		-	// assign their isolated working directories. All downstream filesystem access,
319		-	// manifests, and process environments consequently refer to the same paths.
320		-	func resolvePathOptions(o options) (options, error) {
321		-	cwd, err := os.Getwd()
322		-	if err != nil {
323		-	return o, fmt.Errorf("resolve harness working directory: %w", err)
324		-	}
325		-	absolute := func(path string) string {
326		-	if filepath.IsAbs(path) {
327		-	return filepath.Clean(path)
328		-	}
329		-	return filepath.Clean(filepath.Join(cwd, path))
330		-	}
331		-	o.vevBin = absolute(o.vevBin)
332		-	o.manifest = absolute(o.manifest)
333		-	o.out = absolute(o.out)
334		-	return o, nil
335		-	}
336		-
337		-	func readManifest(path string) (manifest, error) {
338		-	f, e := os.Open(path)
339		-	if e != nil {
340		-	return manifest{}, e
341		-	}
342		-	defer f.Close()
343		-	var m manifest
344		-	e = json.NewDecoder(f).Decode(&m)
345		-	return m, e
346		-	}
347		-	func validateManifest(m manifest) error {
348		-	if m.Schema != 1 {
349		-	return errors.New("unsupported manifest schema")
350		-	}
351		-	tops := map[string]bool{}
352		-	for _, t := range m.Topologies {
353		-	if t.ID == "" \|\| t.Geometry != "120x40" \|\| t.RowsPerPane != 10000 {
354		-	return fmt.Errorf("invalid topology %q", t.ID)
355		-	}
356		-	tops[t.ID] = true
357		-	}
358		-	wantT := []string{"1x4", "4x1", "4x4", "8x1"}
359		-	for _, v := range wantT {
360		-	if !tops[v] {
361		-	return fmt.Errorf("missing canonical topology %s", v)
362		-	}
363		-	}
364		-	works := set(m.Workloads)
365		-	for _, v := range []string{"idle", "active_output", "all_output", "inactive_output", "interactive_flood", "copy_search", "resize_sweep", "snapshot_output_resize", "attach_restore_tab_switch"} {
366		-	if !works[v] {
367		-	return fmt.Errorf("missing canonical workload %s", v)
368		-	}
369		-	}
370		-	trans := map[string]bool{}
371		-	for _, t := range m.Transports {
372		-	if t.ID == "" {
373		-	return errors.New("transport missing id")
374		-	}
375		-	if err := validateTransportFixture(t); err != nil {
376		-	return err
377		-	}
378		-	trans[t.ID] = true
379		-	}
380		-	for _, v := range []string{"local", "ssh_stdio", "udp_baseline", "udp_25ms", "udp_100ms", "udp_loss_0pct", "udp_loss_1pct"} {
381		-	if !trans[v] {
382		-	return fmt.Errorf("missing canonical transport %s", v)
383		-	}
384		-	}
385		-	seen := map[string]bool{}
386		-	covered := map[string]bool{}
387		-	for _, s := range m.Scenarios {
388		-	if s.ID == "" \|\| seen[s.ID] {
389		-	return fmt.Errorf("duplicate or empty scenario id %q", s.ID)
390		-	}
391		-	seen[s.ID] = true
392		-	if !tops[s.Topology] \|\| !works[s.Workload] \|\| !trans[s.Transport] {
393		-	return fmt.Errorf("scenario %s references unknown matrix entry", s.ID)
394		-	}
395		-	if s.InapplicableReason == "" && len(s.Roles) == 0 {
396		-	return fmt.Errorf("scenario %s has no process roles", s.ID)
397		-	}
398		-	if s.InapplicableReason != "" && len(s.Roles) != 0 {
399		-	return fmt.Errorf("inapplicable scenario %s must not launch roles", s.ID)
400		-	}
401		-	if s.InapplicableReason == "" && !equalRoleSet(s.Roles, requiredRoles(s.Transport)) {
402		-	return fmt.Errorf("scenario %s roles do not route transport %s", s.ID, s.Transport)
403		-	}
404		-	covered[s.Topology+"\x00"+s.Workload+"\x00"+s.Transport] = true
405		-	}
406		-	for topologyID := range tops {
407		-	for workload := range works {
408		-	for transportID := range trans {
409		-	key := topologyID + "\x00" + workload + "\x00" + transportID
410		-	if !covered[key] {
411		-	return fmt.Errorf("missing scenario or inapplicable reason for %s/%s/%s", topologyID, workload, transportID)
412		-	}
413		-	}
414		-	}
415		-	}
416		-	return nil
417		-	}
418		-
419		-	// selectScenario is intentionally called only after validateManifest. A bounded
420		-	// local evidence run therefore cannot mask an incomplete canonical matrix.
421		-	func selectScenario(m manifest, id string) ([]scenario, error) {
422		-	for _, s := range m.Scenarios {
423		-	if s.ID != id {
424		-	continue
425		-	}
426		-	if s.InapplicableReason != "" {
427		-	return nil, fmt.Errorf("scenario %s is inapplicable: %s", id, s.InapplicableReason)
428		-	}
429		-	return []scenario{s}, nil
430		-	}
431		-	return nil, fmt.Errorf("unknown canonical scenario %s", id)
432		-	}
433		-
434		-	func requiredRoles(transportID string) []string {
435		-	switch transportID {
436		-	case "local":
437		-	return []string{"daemon", "client"}
438		-	case "ssh_stdio":
439		-	return []string{"daemon", "client", "ssh_stdio_peer"}
440		-	default:
441		-	return []string{"daemon", "client", "udp_peer"}
442		-	}
443		-	}
444		-
445		-	func equalRoleSet(got, want []string) bool {
446		-	if len(got) != len(want) {
447		-	return false
448		-	}
449		-	seen := make(map[string]bool, len(got))
450		-	for _, role := range got {
451		-	seen[role] = true
452		-	}
453		-	for _, role := range want {
454		-	if !seen[role] {
455		-	return false
456		-	}
457		-	}
458		-	return true
459		-	}
460		-
461		-	func validateTransportFixture(t transport) error {
462		-	want := map[string]transport{
463		-	"local": {ID: "local", Kind: "local"},
464		-	"ssh_stdio": {ID: "ssh_stdio", Kind: "ssh_stdio"},
465		-	"udp_baseline": {ID: "udp_baseline", Kind: "udp"},
466		-	"udp_25ms": {ID: "udp_25ms", Kind: "udp", RTTMS: 25},
467		-	"udp_100ms": {ID: "udp_100ms", Kind: "udp", RTTMS: 100},
468		-	"udp_loss_0pct": {ID: "udp_loss_0pct", Kind: "udp", LossPercent: 0},
469		-	"udp_loss_1pct": {ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1},
470		-	}
471		-	w, ok := want[t.ID]
472		-	if !ok \|\| t.Kind != w.Kind \|\| t.RTTMS != w.RTTMS \|\| t.LossPercent != w.LossPercent {
473		-	return fmt.Errorf("invalid canonical transport fixture %+v", t)
474		-	}
475		-	return nil
476		-	}
477		-
478		-	func set(v []string) map[string]bool {
479		-	r := map[string]bool{}
480		-	for _, x := range v {
481		-	r[x] = true
482		-	}
483		-	return r
484		-	}
485	157		func (h *harness) runOne(o options, mat manifest, s scenario, run int, raw io.Writer) (res runResult, err error) {
486	158		dir := filepath.Join(o.out, fmt.Sprintf("%s-run-%03d", safeName(s.ID), run))
487	159		createRunDir := h.createRunDir

@@ -738,7 +410,3 @@ func workloadInput(s scenario, run int, phase string) []byte {
738	410		}
739	411		return []byte(fmt.Sprintf(body+"; printf '%s\\n'\n", s.ID, marker))
740	412		}
741		-
742		-	// measuredEventSamples validates every owned pair, then retains only events
743		-	// whose injection and successful terminal flush are both in the interval.
744		-	// Straddling pairs remain raw diagnostic facts but are never percentile input.

A cmd/vev-perf-harness/runner_test.go +399 -0

@@ -0,0 +1,399 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"fmt"
	8	+	"os"
	9	+	"os/exec"
	10	+	"path/filepath"
	11	+	"strings"
	12	+	"testing"
	13	+	"time"
	14	+	)
	15	+
	16	+	func TestHarnessCanonicalLocalRolesAreIsolatedAcrossRepetitions(t *testing.T) {
	17	+	if testing.Short() {
	18	+	t.Skip("public CLI repetition lifecycle")
	19	+	}
	20	+	root, err := filepath.Abs(filepath.Join("..", ".."))
	21	+	if err != nil {
	22	+	t.Fatal(err)
	23	+	}
	24	+	bin := filepath.Join(t.TempDir(), "vev")
	25	+	build := exec.Command("/usr/local/go/bin/go", "build", "-o", bin, "./")
	26	+	build.Dir = root
	27	+	if output, err := build.CombinedOutput(); err != nil {
	28	+	t.Fatalf("build public CLI: %v\n%s", err, output)
	29	+	}
	30	+	m, err := readManifest(filepath.Join(root, "testdata", "perf", "manifest.json"))
	31	+	if err != nil {
	32	+	t.Fatal(err)
	33	+	}
	34	+	var local scenario
	35	+	for _, candidate := range m.Scenarios {
	36	+	if candidate.ID == "1x4-idle-local" {
	37	+	local = candidate
	38	+	break
	39	+	}
	40	+	}
	41	+	if local.ID == "" {
	42	+	t.Fatal("canonical local scenario missing")
	43	+	}
	44	+	out, err := os.MkdirTemp("", "vev-ob-")
	45	+	if err != nil {
	46	+	t.Fatal(err)
	47	+	}
	48	+	defer os.RemoveAll(out)
	49	+	raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	50	+	if err != nil {
	51	+	t.Fatal(err)
	52	+	}
	53	+	defer raw.Close()
	54	+	h := defaultHarness()
	55	+	h.clock = &fakeClock{}
	56	+	h.launcher = &cliLauncher{bin: bin}
	57	+	for run := 1; run <= 3; run++ {
	58	+	result, err := h.runOne(options{vevBin: bin, out: out, duration: minimumDuration}, m, local, run, raw)
	59	+	if err != nil {
	60	+	t.Fatalf("run %d: %v", run, err)
	61	+	}
	62	+	if len(result.Processes) != 2 {
	63	+	t.Fatalf("run %d process mappings = %+v", run, result.Processes)
	64	+	}
	65	+	runDir := filepath.Join(out, fmt.Sprintf("%s-run-%03d", safeName(local.ID), run))
	66	+	manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
	67	+	if err != nil {
	68	+	t.Fatalf("run %d manifest: %v", run, err)
	69	+	}
	70	+	var persisted runManifest
	71	+	if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
	72	+	t.Fatalf("run %d manifest JSON: %v", run, err)
	73	+	}
	74	+	if persisted.Run != run \|\| len(persisted.Processes) != len(result.Processes) {
	75	+	t.Fatalf("run %d manifest = %+v", run, persisted)
	76	+	}
	77	+	for _, process := range result.Processes {
	78	+	if process.Role != "daemon" && process.Role != "client" {
	79	+	t.Fatalf("run %d unexpected role %q", run, process.Role)
	80	+	}
	81	+	if filepath.Dir(process.TracePath) != runDir {
	82	+	t.Fatalf("run %d %s trace outside its isolated run directory: %q", run, process.Role, process.TracePath)
	83	+	}
	84	+	trace, err := os.ReadFile(process.TracePath)
	85	+	if err != nil {
	86	+	t.Fatalf("run %d %s trace %q: %v", run, process.Role, process.TracePath, err)
	87	+	}
	88	+	if len(bytes.TrimSpace(trace)) == 0 {
	89	+	t.Fatalf("run %d %s did not emit its declared trace", run, process.Role)
	90	+	}
	91	+	for _, line := range bytes.Split(bytes.TrimSpace(trace), []byte{'\n'}) {
	92	+	var record traceRecord
	93	+	if err := json.Unmarshal(line, &record); err != nil {
	94	+	t.Fatalf("run %d %s trace JSON: %v", run, process.Role, err)
	95	+	}
	96	+	if record.ProcessID != process.ProcessID \|\| record.Scenario != local.ID \|\| record.Run != uint64(run) {
	97	+	t.Fatalf("run %d %s trace escaped its manifest identity: %+v", run, process.Role, record)
	98	+	}
	99	+	}
	100	+	}
	101	+	if _, err := os.Stat(filepath.Join(runDir, "runtime", "vev", "daemon.sock")); !errors.Is(err, os.ErrNotExist) {
	102	+	t.Fatalf("run %d left daemon socket for the next repetition: %v", run, err)
	103	+	}
	104	+	}
	105	+	}
	106	+
	107	+	func TestHarnessResolvesCanonicalRelativePathsBeforeRoleCWDChanges(t *testing.T) {
	108	+	if testing.Short() {
	109	+	t.Skip("public CLI relative-path lifecycle")
	110	+	}
	111	+	root, err := filepath.Abs(filepath.Join("..", ".."))
	112	+	if err != nil {
	113	+	t.Fatal(err)
	114	+	}
	115	+	// Keep the real Unix-socket path below its platform limit while testing the
	116	+	// exact documented relative output hierarchy.
	117	+	workspace, err := os.MkdirTemp("", "vev-rel-")
	118	+	if err != nil {
	119	+	t.Fatal(err)
	120	+	}
	121	+	t.Cleanup(func() { _ = os.RemoveAll(workspace) })
	122	+	bin := filepath.Join(workspace, "vev")
	123	+	build := exec.Command("/usr/local/go/bin/go", "build", "-o", bin, "./")
	124	+	build.Dir = root
	125	+	if output, err := build.CombinedOutput(); err != nil {
	126	+	t.Fatalf("build public CLI: %v\n%s", err, output)
	127	+	}
	128	+	manifestPath := filepath.Join(workspace, "testdata", "perf", "manifest.json")
	129	+	if err := os.MkdirAll(filepath.Dir(manifestPath), 0o700); err != nil {
	130	+	t.Fatal(err)
	131	+	}
	132	+	contents, err := os.ReadFile(filepath.Join(root, "testdata", "perf", "manifest.json"))
	133	+	if err != nil {
	134	+	t.Fatal(err)
	135	+	}
	136	+	if err := os.WriteFile(manifestPath, contents, 0o600); err != nil {
	137	+	t.Fatal(err)
	138	+	}
	139	+	originalWD, err := os.Getwd()
	140	+	if err != nil {
	141	+	t.Fatal(err)
	142	+	}
	143	+	if err := os.Chdir(workspace); err != nil {
	144	+	t.Fatal(err)
	145	+	}
	146	+	t.Cleanup(func() {
	147	+	if err := os.Chdir(originalWD); err != nil {
	148	+	t.Errorf("restore cwd: %v", err)
	149	+	}
	150	+	})
	151	+
	152	+	// These are the documented relative locations. cliLauncher changes every
	153	+	// role to its run directory, so a failure to resolve them before launch
	154	+	// would make VEV_PERF_TRACE point at a nonexistent nested directory.
	155	+	h := defaultHarness()
	156	+	h.clock = &fakeClock{}
	157	+	if err := run([]string{
	158	+	"--vev-bin", "./vev",
	159	+	"--manifest", "testdata/perf/manifest.json",
	160	+	"--out", "testdata/perf/results",
	161	+	"--scenario", "1x4-idle-local",
	162	+	"--warmup", "0s",
	163	+	"--duration", "30s",
	164	+	"--repetitions", "10",
	165	+	}, h); err != nil {
	166	+	t.Fatal(err)
	167	+	}
	168	+
	169	+	out := filepath.Join(workspace, "testdata", "perf", "results")
	170	+	for run := 1; run <= 10; run++ {
	171	+	runDir := filepath.Join(out, fmt.Sprintf("1x4-idle-local-run-%03d", run))
	172	+	manifestBytes, err := os.ReadFile(filepath.Join(runDir, "manifest.json"))
	173	+	if err != nil {
	174	+	t.Fatalf("run %d manifest: %v", run, err)
	175	+	}
	176	+	var persisted runManifest
	177	+	if err := json.Unmarshal(manifestBytes, &persisted); err != nil {
	178	+	t.Fatalf("run %d manifest JSON: %v", run, err)
	179	+	}
	180	+	if persisted.Run != run \|\| len(persisted.Processes) != 2 {
	181	+	t.Fatalf("run %d manifest=%+v", run, persisted)
	182	+	}
	183	+	for _, process := range persisted.Processes {
	184	+	if !filepath.IsAbs(process.TracePath) \|\| filepath.Dir(process.TracePath) != runDir {
	185	+	t.Fatalf("run %d %s trace path=%q, want absolute path in %q", run, process.Role, process.TracePath, runDir)
	186	+	}
	187	+	trace, err := os.ReadFile(process.TracePath)
	188	+	if err != nil \|\| len(bytes.TrimSpace(trace)) == 0 {
	189	+	t.Fatalf("run %d %s trace=%q err=%v", run, process.Role, process.TracePath, err)
	190	+	}
	191	+	}
	192	+	}
	193	+	}
	194	+
	195	+	func TestHarnessRejectsExistingRunDirectoryBeforeLaunchingRoles(t *testing.T) {
	196	+	out := t.TempDir()
	197	+	raw, err := os.OpenFile(filepath.Join(out, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	198	+	if err != nil {
	199	+	t.Fatal(err)
	200	+	}
	201	+	defer raw.Close()
	202	+	s := scenario{ID: "isolated", Roles: []string{"daemon", "client"}}
	203	+	stale := filepath.Join(out, "isolated-run-001")
	204	+	if err := os.Mkdir(stale, 0o700); err != nil {
	205	+	t.Fatal(err)
	206	+	}
	207	+	if err := os.WriteFile(filepath.Join(stale, "preserve"), []byte("stale"), 0o600); err != nil {
	208	+	t.Fatal(err)
	209	+	}
	210	+	launcher := &fakeLauncher{}
	211	+	h := defaultHarness()
	212	+	h.clock, h.launcher = &fakeClock{}, launcher
	213	+	_, err = h.runOne(options{out: out, duration: minimumDuration}, manifest{}, s, 1, raw)
	214	+	if err == nil \|\| !strings.Contains(err.Error(), "create isolated run directory") {
	215	+	t.Fatalf("existing run directory error = %v", err)
	216	+	}
	217	+	if len(launcher.mappings) != 0 {
	218	+	t.Fatalf("roles launched against stale run directory: %+v", launcher.mappings)
	219	+	}
	220	+	if contents, readErr := os.ReadFile(filepath.Join(stale, "preserve")); readErr != nil \|\| string(contents) != "stale" {
	221	+	t.Fatalf("stale directory changed: contents=%q err=%v", contents, readErr)
	222	+	}
	223	+	}
	224	+
	225	+	func TestHarnessDoesNotInheritNestedSession(t *testing.T) {
	226	+	env := withoutEnv([]string{"VEV=session=old", "PATH=/bin", "VEV_PERF_TRACE=old"}, "VEV")
	227	+	if !equalStrings(env, []string{"PATH=/bin", "VEV_PERF_TRACE=old"}) {
	228	+	t.Fatalf("environment=%q", env)
	229	+	}
	230	+	}
	231	+
	232	+	func TestHarnessRoutesEveryRemoteFixtureThroughItsDeclaredPeer(t *testing.T) {
	233	+	cases := []struct {
	234	+	name string
	235	+	transport transport
	236	+	peer string
	237	+	wantRTT int
	238	+	wantLoss int
	239	+	}{
	240	+	{"ssh", transport{ID: "ssh_stdio", Kind: "ssh_stdio"}, "ssh_stdio_peer", 0, 0},
	241	+	{"udp baseline", transport{ID: "udp_baseline", Kind: "udp"}, "udp_peer", 0, 0},
	242	+	{"udp 25ms", transport{ID: "udp_25ms", Kind: "udp", RTTMS: 25}, "udp_peer", 25, 0},
	243	+	{"udp 100ms", transport{ID: "udp_100ms", Kind: "udp", RTTMS: 100}, "udp_peer", 100, 0},
	244	+	{"udp loss zero", transport{ID: "udp_loss_0pct", Kind: "udp", LossPercent: 0}, "udp_peer", 0, 0},
	245	+	{"udp loss", transport{ID: "udp_loss_1pct", Kind: "udp", LossPercent: 1}, "udp_peer", 0, 1},
	246	+	}
	247	+	for _, tc := range cases {
	248	+	t.Run(tc.name, func(t *testing.T) {
	249	+	s := scenario{ID: "route", Transport: tc.transport.ID}
	250	+	client := routeRoleArgs(s, processMapping{Role: "client", Run: 2}, tc.transport)
	251	+	peer := routeRoleArgs(s, processMapping{Role: tc.peer, Run: 2}, tc.transport)
	252	+	if !equalStrings(client.Args, []string{"attach", "harness@127.0.0.1:perf-route-002"}) {
	253	+	t.Fatalf("client did not use public remote attach: %q", client.Args)
	254	+	}
	255	+	if tc.peer == "ssh_stdio_peer" && !equalStrings(peer.Args, []string{"_stdio", "perf-route-002"}) {
	256	+	t.Fatalf("ssh peer command=%q", peer.Args)
	257	+	}
	258	+	if tc.peer == "udp_peer" && !equalStrings(peer.Args, []string{"_udp-proxy", "perf-route-002"}) {
	259	+	t.Fatalf("udp peer command=%q", peer.Args)
	260	+	}
	261	+	if peer.Transport.RTTMS != tc.wantRTT \|\| peer.Transport.LossPercent != tc.wantLoss {
	262	+	t.Fatalf("peer lost manifest network settings: %+v", peer.Transport)
	263	+	}
	264	+	})
	265	+	}
	266	+	}
	267	+
	268	+	func TestHarnessFakeRunnerRoutesClientToPeerAndCleansEveryRole(t *testing.T) {
	269	+	for _, tc := range []struct {
	270	+	name, transport, peer, peerCommand string
	271	+	}{
	272	+	{"ssh", "ssh_stdio", "ssh_stdio_peer", "_stdio"},
	273	+	{"udp baseline", "udp_baseline", "udp_peer", "_udp-proxy"},
	274	+	{"udp 25ms", "udp_25ms", "udp_peer", "_udp-proxy"},
	275	+	{"udp 100ms", "udp_100ms", "udp_peer", "_udp-proxy"},
	276	+	{"udp loss zero", "udp_loss_0pct", "udp_peer", "_udp-proxy"},
	277	+	{"udp loss", "udp_loss_1pct", "udp_peer", "_udp-proxy"},
	278	+	} {
	279	+	t.Run(tc.name, func(t *testing.T) {
	280	+	dir := t.TempDir()
	281	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	282	+	if err != nil {
	283	+	t.Fatal(err)
	284	+	}
	285	+	defer raw.Close()
	286	+	l := &fakeLauncher{}
	287	+	h := defaultHarness()
	288	+	h.clock, h.launcher = &fakeClock{}, l
	289	+	s := scenario{ID: "routed", Transport: tc.transport, Roles: []string{"daemon", "client", tc.peer}}
	290	+	if _, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, s, 1, raw); err != nil {
	291	+	t.Fatal(err)
	292	+	}
	293	+	if len(l.mappings) != 3 \|\| l.mappings[1].Role != tc.peer \|\| l.mappings[2].Role != "client" {
	294	+	t.Fatalf("dependency launch order=%+v", l.mappings)
	295	+	}
	296	+	if l.commands[1].Args[0] != tc.peerCommand \|\| l.commands[2].Args[0] != "attach" {
	297	+	t.Fatalf("commands do not connect client through declared peer: %+v", l.commands)
	298	+	}
	299	+	for i, p := range l.process {
	300	+	if !p.closed {
	301	+	t.Fatalf("role %s was not cleaned up", l.mappings[i].Role)
	302	+	}
	303	+	}
	304	+	})
	305	+	}
	306	+	}
	307	+
	308	+	func TestHarnessAggregatesRawEventsSeparatelyFromRunDispersion(t *testing.T) {
	309	+	first := append([]int64(nil), make([]int64, minimumInIntervalEventSamples+5)...)
	310	+	for i := range first {
	311	+	first[i] = 1
	312	+	}
	313	+	second := []int64{100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 1_000, 1_000}
	314	+	all, _, _, runP50s := aggregateEventResults([]runResult{
	315	+	{EndToEnd: first, Event: distribution{P50: 1}},
	316	+	{EndToEnd: second, Event: distribution{P50: 100}},
	317	+	})
	318	+	if got, dispersion := percentiles(all), percentiles(runP50s); got.Count != 30 \|\| got.P99 != 1_000 \|\| dispersion.Count != 2 \|\| dispersion.P99 != 1 {
	319	+	t.Fatalf("event aggregation=%+v run dispersion=%+v", got, dispersion)
	320	+	}
	321	+	}
	322	+
	323	+	func TestHarnessSchedulesMeasuredEventsAcrossFullInterval(t *testing.T) {
	324	+	dir := t.TempDir()
	325	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	326	+	if err != nil {
	327	+	t.Fatal(err)
	328	+	}
	329	+	defer raw.Close()
	330	+	h := defaultHarness()
	331	+	h.clock, h.launcher = &fakeClock{}, &fakeLauncher{}
	332	+	result, err := h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration}, manifest{}, scenario{ID: "schedule", Roles: []string{"daemon", "client"}}, 1, raw)
	333	+	if err != nil {
	334	+	t.Fatal(err)
	335	+	}
	336	+	if result.Samples < minimumInIntervalEventSamples \|\| result.Cadence.Count != result.Samples-1 \|\| result.MaxGap <= 0 {
	337	+	t.Fatalf("result did not include repeated measured events: %+v", result)
	338	+	}
	339	+	if err := raw.Close(); err != nil {
	340	+	t.Fatal(err)
	341	+	}
	342	+	records, err := readJSONL(filepath.Join(dir, "raw.jsonl"))
	343	+	if err != nil {
	344	+	t.Fatal(err)
	345	+	}
	346	+	if len(records) != result.Samples*2 {
	347	+	t.Fatalf("raw records=%d, want %d complete measured boundaries", len(records), result.Samples*2)
	348	+	}
	349	+	}
	350	+
	351	+	func TestHarnessPreservesWorkloadAndCleanupErrors(t *testing.T) {
	352	+	dir := t.TempDir()
	353	+	raw, err := os.OpenFile(filepath.Join(dir, "raw.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	354	+	if err != nil {
	355	+	t.Fatal(err)
	356	+	}
	357	+	defer raw.Close()
	358	+	primary := errors.New("primary workload failure")
	359	+	cleanup := errors.New("daemon cleanup failure")
	360	+	launcher := &fakeLauncher{
	361	+	measureErr: map[string]error{"client": primary},
	362	+	closeErr: map[string]error{"daemon": cleanup},
	363	+	}
	364	+	h := defaultHarness()
	365	+	h.clock, h.launcher = &fakeClock{}, launcher
	366	+	_, err = h.runOne(options{out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}, manifest{}, scenario{ID: "errors", Roles: []string{"daemon", "client"}}, 1, raw)
	367	+	if !errors.Is(err, primary) \|\| !errors.Is(err, cleanup) {
	368	+	t.Fatalf("workload and cleanup errors were not both preserved: %v", err)
	369	+	}
	370	+	if got, want := err.Error(), primary.Error()+"\nclose daemon role: "+cleanup.Error(); got != want {
	371	+	t.Fatalf("error ordering=%q want %q", got, want)
	372	+	}
	373	+	for i, p := range launcher.process {
	374	+	if !p.closed {
	375	+	t.Fatalf("role %s was not closed", launcher.mappings[i].Role)
	376	+	}
	377	+	}
	378	+	}
	379	+
	380	+	func TestHarnessCleansRunDirectoryOnProcessFailure(t *testing.T) {
	381	+	dir := t.TempDir()
	382	+	raw, err := os.OpenFile(filepath.Join(dir, "raw-harness.jsonl"), os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	383	+	if err != nil {
	384	+	t.Fatal(err)
	385	+	}
	386	+	defer raw.Close()
	387	+	launcher := &fakeLauncher{}
	388	+	h := defaultHarness()
	389	+	h.clock = &fakeClock{}
	390	+	// The client is the second role and fails after the daemon has started.
	391	+	h.launcher = &failingLauncher{fakeLauncher: launcher}
	392	+	o := options{vevBin: "ignored", out: dir, warmup: time.Second, duration: minimumDuration, repetitions: minimumRepetitions}
	393	+	if _, err := h.runOne(o, manifest{}, scenario{ID: "cleanup", Roles: []string{"daemon", "client"}}, 1, raw); err == nil {
	394	+	t.Fatal("failed process accepted")
	395	+	}
	396	+	if _, err := os.Stat(filepath.Join(dir, "cleanup-run-001")); !os.IsNotExist(err) {
	397	+	t.Fatalf("failed run directory remains: %v", err)
	398	+	}
	399	+	}

A cmd/vev-perf-harness/test_helpers_test.go +266 -0

@@ -0,0 +1,266 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"encoding/json"
	6	+	"errors"
	7	+	"os"
	8	+	"path/filepath"
	9	+	"time"
	10	+	)
	11	+
	12	+	type fakeClock struct{ tick int64 }
	13	+
	14	+	func (c *fakeClock) Now() int64 { c.tick += 10; return c.tick }
	15	+
	16	+	func (c *fakeClock) Sleep(d time.Duration) {
	17	+	c.tick += d.Nanoseconds()
	18	+	}
	19	+
	20	+	type fakeUDPNetem struct {
	21	+	port int
	22	+	closed bool
	23	+	}
	24	+
	25	+	func (n *fakeUDPNetem) Port() int { return n.port }
	26	+
	27	+	func (n *fakeUDPNetem) Close() error { n.closed = true; return nil }
	28	+
	29	+	type fakeProcess struct {
	30	+	fail bool
	31	+	measureErr error
	32	+	closeErr error
	33	+	warmups [][]byte
	34	+	measures [][]byte
	35	+	closed bool
	36	+	}
	37	+
	38	+	func (p *fakeProcess) Warmup(input []byte) error {
	39	+	p.warmups = append(p.warmups, append([]byte(nil), input...))
	40	+	return nil
	41	+	}
	42	+
	43	+	func (p *fakeProcess) Measure(input []byte, injected, flush func() error) error {
	44	+	p.measures = append(p.measures, append([]byte(nil), input...))
	45	+	if p.measureErr != nil {
	46	+	return p.measureErr
	47	+	}
	48	+	if p.fail {
	49	+	return errors.New("measure failure")
	50	+	}
	51	+	if err := injected(); err != nil {
	52	+	return err
	53	+	}
	54	+	return flush()
	55	+	}
	56	+
	57	+	func (p *fakeProcess) Close() error { p.closed = true; return p.closeErr }
	58	+
	59	+	type fakeLauncher struct {
	60	+	mappings []processMapping
	61	+	args [][]string
	62	+	commands []roleCommand
	63	+	process []*fakeProcess
	64	+	measureErr map[string]error
	65	+	closeErr map[string]error
	66	+	manifestPresent bool
	67	+	}
	68	+
	69	+	func (l *fakeLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
	70	+	l.mappings = append(l.mappings, m)
	71	+	l.args = append(l.args, append([]string(nil), command.Args...))
	72	+	l.commands = append(l.commands, command)
	73	+	_, e := os.Stat(filepath.Join(filepath.Dir(m.TracePath), "manifest.json"))
	74	+	l.manifestPresent = e == nil
	75	+	p := &fakeProcess{measureErr: l.measureErr[m.Role], closeErr: l.closeErr[m.Role]}
	76	+	l.process = append(l.process, p)
	77	+	return p, nil
	78	+	}
	79	+
	80	+	type lateFlushProcess struct{ clock *fakeClock }
	81	+
	82	+	func (*lateFlushProcess) Warmup([]byte) error { return nil }
	83	+
	84	+	func (p *lateFlushProcess) Measure(_ []byte, injected, flushed func() error) error {
	85	+	if err := injected(); err != nil {
	86	+	return err
	87	+	}
	88	+	p.clock.tick += minimumDuration.Nanoseconds()
	89	+	return flushed()
	90	+	}
	91	+
	92	+	func (*lateFlushProcess) Close() error { return nil }
	93	+
	94	+	type insufficientEventLauncher struct{ clock *fakeClock }
	95	+
	96	+	func (l insufficientEventLauncher) Launch(m processMapping, _ roleCommand) (launchedProcess, error) {
	97	+	if m.Role == "client" {
	98	+	return &lateFlushProcess{clock: l.clock}, nil
	99	+	}
	100	+	return &fakeProcess{}, nil
	101	+	}
	102	+
	103	+	type fakePTY struct{ writes [][]byte }
	104	+
	105	+	func (p *fakePTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
	106	+
	107	+	func (p *fakePTY) Write(b []byte) (int, error) {
	108	+	p.writes = append(p.writes, append([]byte(nil), b...))
	109	+	return len(b), nil
	110	+	}
	111	+
	112	+	func (*fakePTY) Close() error { return nil }
	113	+
	114	+	type stagedPTY struct {
	115	+	allow <-chan struct{}
	116	+	writes chan []byte
	117	+	}
	118	+
	119	+	func (*stagedPTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
	120	+
	121	+	func (p *stagedPTY) Write(b []byte) (int, error) {
	122	+	if p.allow != nil {
	123	+	<-p.allow
	124	+	}
	125	+	p.writes <- append([]byte(nil), b...)
	126	+	return len(b), nil
	127	+	}
	128	+
	129	+	func (*stagedPTY) Close() error { return nil }
	130	+
	131	+	type fakeOutput struct {
	132	+	bytes.Buffer
	133	+	syncs int
	134	+	}
	135	+
	136	+	func (o *fakeOutput) Sync() error { o.syncs++; return nil }
	137	+
	138	+	func (*fakeOutput) Close() error { return nil }
	139	+
	140	+	type orderedPTY struct {
	141	+	write func([]byte)
	142	+	close func()
	143	+	}
	144	+
	145	+	func (*orderedPTY) Read([]byte) (int, error) { return 0, os.ErrClosed }
	146	+
	147	+	func (p *orderedPTY) Write(b []byte) (int, error) {
	148	+	if p.write != nil {
	149	+	p.write(b)
	150	+	}
	151	+	return len(b), nil
	152	+	}
	153	+
	154	+	func (p *orderedPTY) Close() error {
	155	+	p.close()
	156	+	return nil
	157	+	}
	158	+
	159	+	type orderedOutput struct{ close func() }
	160	+
	161	+	func (*orderedOutput) Write(b []byte) (int, error) { return len(b), nil }
	162	+
	163	+	func (*orderedOutput) Sync() error { return nil }
	164	+
	165	+	func (o *orderedOutput) Close() error {
	166	+	o.close()
	167	+	return nil
	168	+	}
	169	+
	170	+	type failingLauncher struct{ *fakeLauncher }
	171	+
	172	+	func (l *failingLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
	173	+	p, err := l.fakeLauncher.Launch(m, command)
	174	+	if err != nil {
	175	+	return nil, err
	176	+	}
	177	+	if m.Role == "client" {
	178	+	p.(*fakeProcess).fail = true
	179	+	}
	180	+	return p, nil
	181	+	}
	182	+
	183	+	func equalStrings(a, b []string) bool {
	184	+	if len(a) != len(b) {
	185	+	return false
	186	+	}
	187	+	for i := range a {
	188	+	if a[i] != b[i] {
	189	+	return false
	190	+	}
	191	+	}
	192	+	return true
	193	+	}
	194	+
	195	+	type traceLauncher struct{ fakeLauncher }
	196	+
	197	+	func (l *traceLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
	198	+	p, err := l.fakeLauncher.Launch(m, command)
	199	+	if err != nil {
	200	+	return nil, err
	201	+	}
	202	+	f, err := os.OpenFile(m.TracePath, os.O_WRONLY\|os.O_APPEND, 0o600)
	203	+	if err != nil {
	204	+	return nil, err
	205	+	}
	206	+	defer f.Close()
	207	+	for i, pair := range spanPairs {
	208	+	for n, kind := range []string{pair.start, pair.end} {
	209	+	b, _ := json.Marshal(traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: uint64(i + 1), RequestID: 1, Epoch: 1, Kind: kind, Tick: int64(i20 + n5), Valid: true})
	210	+	if _, err := f.Write(append(b, '\n')); err != nil {
	211	+	return nil, err
	212	+	}
	213	+	}
	214	+	}
	215	+	return p, nil
	216	+	}
	217	+
	218	+	type closeTraceProcess struct {
	219	+	*fakeProcess
	220	+	traceEnd func() error
	221	+	}
	222	+
	223	+	func (p *closeTraceProcess) Close() error {
	224	+	p.closed = true
	225	+	return errors.Join(p.closeErr, p.traceEnd())
	226	+	}
	227	+
	228	+	type closeTraceLauncher struct{ fakeLauncher }
	229	+
	230	+	func (l *closeTraceLauncher) Launch(m processMapping, command roleCommand) (launchedProcess, error) {
	231	+	p, err := l.fakeLauncher.Launch(m, command)
	232	+	if err != nil {
	233	+	return nil, err
	234	+	}
	235	+	start := traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: 1, RequestID: 1, Epoch: 1, Kind: "adapter_receive_start", Tick: 10, Valid: true}
	236	+	if err := appendTraceRecord(m.TracePath, start); err != nil {
	237	+	return nil, err
	238	+	}
	239	+	return &closeTraceProcess{fakeProcess: p.(*fakeProcess), traceEnd: func() error {
	240	+	return appendTraceRecord(m.TracePath, traceRecord{ProcessID: m.ProcessID, Component: m.Role, Scenario: m.Scenario, Run: uint64(m.Run), Sequence: 1, RequestID: 1, Epoch: 1, Kind: "adapter_receive_end", Tick: 20, Valid: true})
	241	+	}}, nil
	242	+	}
	243	+
	244	+	func appendTraceRecord(path string, record traceRecord) error {
	245	+	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_APPEND, 0o600)
	246	+	if err != nil {
	247	+	return err
	248	+	}
	249	+	defer f.Close()
	250	+	b, err := json.Marshal(record)
	251	+	if err != nil {
	252	+	return err
	253	+	}
	254	+	_, err = f.Write(append(b, '\n'))
	255	+	return err
	256	+	}
	257	+
	258	+	// boundedLocalConcurrentTrace is an excerpt from the second run of the
	259	+	// bounded local public-CLI smoke. Receive sequence 3 is in flight while the
	260	+	// same process serializes sequence 4; shared-process records therefore cannot
	261	+	// have a global sequence ordering requirement.
	262	+	const boundedLocalConcurrentTrace = `{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":3,"request_id":3,"epoch":3,"kind":"adapter_receive_start","tick":1783902621495284782,"bytes":0,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	263	+	{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":4,"request_id":4,"epoch":4,"kind":"adapter_send_start","tick":1783902621495364552,"bytes":96,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	264	+	{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":4,"request_id":4,"epoch":4,"kind":"adapter_send_end","tick":1783902621495386452,"bytes":96,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	265	+	{"schema":1,"process_id":"1x4-idle-local-r002-client-02","component":"ipc","scenario":"1x4-idle-local","run":2,"sequence":3,"request_id":3,"epoch":3,"kind":"adapter_receive_end","tick":1783902621495463332,"bytes":0,"fragments":0,"retransmits":0,"pending":0,"ack_rtt_nanos":0,"valid":true}
	266	+	`

A cmd/vev-perf-harness/udp_netem_test.go +78 -0

@@ -0,0 +1,78 @@
	1	+	package main
	2	+
	3	+	import (
	4	+	"net"
	5	+	"os"
	6	+	"path/filepath"
	7	+	"strconv"
	8	+	"testing"
	9	+	"time"
	10	+	)
	11	+
	12	+	func TestUDPNetemExecutesRTTAndLossFixtures(t *testing.T) {
	13	+	fixtures := []struct {
	14	+	name string
	15	+	rtt time.Duration
	16	+	loss int
	17	+	sent int
	18	+	want int
	19	+	}{
	20	+	{"25ms RTT", 25 * time.Millisecond, 0, 1, 1},
	21	+	{"100ms RTT", 100 * time.Millisecond, 0, 1, 1},
	22	+	{"zero loss", 0, 0, 10, 10},
	23	+	{"one percent loss", 0, 1, 100, 99},
	24	+	}
	25	+	for _, tc := range fixtures {
	26	+	t.Run(tc.name, func(t *testing.T) {
	27	+	target, err := net.ListenPacket("udp", "127.0.0.1:0")
	28	+	if err != nil {
	29	+	t.Fatal(err)
	30	+	}
	31	+	defer target.Close()
	32	+	path := filepath.Join(t.TempDir(), "target")
	33	+	if err := os.WriteFile(path, []byte("VEV-UDP "+strconv.Itoa(target.LocalAddr().(*net.UDPAddr).Port)+" key\\n"), 0o600); err != nil {
	34	+	t.Fatal(err)
	35	+	}
	36	+	netem, err := newUDPNetem(udpNetemConfig{RTT: tc.rtt, LossPercent: tc.loss, TargetPath: path})
	37	+	if err != nil {
	38	+	t.Fatal(err)
	39	+	}
	40	+	defer netem.Close()
	41	+	client, err := net.ListenPacket("udp", "127.0.0.1:0")
	42	+	if err != nil {
	43	+	t.Fatal(err)
	44	+	}
	45	+	defer client.Close()
	46	+	started := time.Now()
	47	+	for i := 0; i < tc.sent; i++ {
	48	+	if _, err := client.WriteTo([]byte{byte(i)}, &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: netem.Port()}); err != nil {
	49	+	t.Fatal(err)
	50	+	}
	51	+	}
	52	+	if err := target.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
	53	+	t.Fatal(err)
	54	+	}
	55	+	got := 0
	56	+	buf := make([]byte, 32)
	57	+	for got < tc.want {
	58	+	if _, _, err := target.ReadFrom(buf); err != nil {
	59	+	t.Fatalf("received %d packets, want %d: %v", got, tc.want, err)
	60	+	}
	61	+	got++
	62	+	}
	63	+	if tc.rtt > 0 {
	64	+	if elapsed := time.Since(started); elapsed < tc.rtt/2 {
	65	+	t.Fatalf("one-way netem delay=%s, want at least %s", elapsed, tc.rtt/2)
	66	+	}
	67	+	}
	68	+	// Keep the deadline short: extra carriage would prove the exact 1%%
	69	+	// loss cycle was not enforced.
	70	+	if err := target.SetReadDeadline(time.Now().Add(20 * time.Millisecond)); err != nil {
	71	+	t.Fatal(err)
	72	+	}
	73	+	if _, _, err := target.ReadFrom(buf); tc.want != tc.sent && err == nil {
	74	+	t.Fatalf("received more than %d packets with %d%% loss", tc.want, tc.loss)
	75	+	}
	76	+	})
	77	+	}
	78	+	}