docs(perf): define proxy screen baseline

commit: a03245aca44c6021b3beb03d4d0976e14f5411bb
author: brice · 2mo ago · 2026-08-01 04:53 UTC
parent: 9a4b27f
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

3 files · +114 -29

M cmd/vev-perf-harness/evidence.go +15 -10


@@ -134,16 +134,21 @@ func pairedSamples(marks []harnessMark) ([]int64, error) {
134	134		// traceRecord is intentionally local to the command. It mirrors JSONL fields
135	135		// needed for correlation but never exposes a clock to any production component.
136	136		type traceRecord struct {
137		-	ProcessID string `json:"process_id"`
138		-	Component string `json:"component"`
139		-	Scenario string `json:"scenario"`
140		-	Run uint64 `json:"run"`
141		-	Sequence uint64 `json:"sequence"`
142		-	RequestID uint64 `json:"request_id"`
143		-	Epoch uint64 `json:"epoch"`
144		-	Kind string `json:"kind"`
145		-	Tick int64 `json:"tick"`
146		-	Valid bool `json:"valid"`
	137	+	ProcessID string `json:"process_id"`
	138	+	Component string `json:"component"`
	139	+	Scenario string `json:"scenario"`
	140	+	Run uint64 `json:"run"`
	141	+	Sequence uint64 `json:"sequence"`
	142	+	RequestID uint64 `json:"request_id"`
	143	+	Epoch uint64 `json:"epoch"`
	144	+	Kind string `json:"kind"`
	145	+	Tick int64 `json:"tick"`
	146	+	Bytes uint64 `json:"bytes"`
	147	+	Fragments uint64 `json:"fragments"`
	148	+	Retransmits uint64 `json:"retransmits"`
	149	+	Pending uint64 `json:"pending"`
	150	+	AckRTTNanos int64 `json:"ack_rtt_nanos"`
	151	+	Valid bool `json:"valid"`
147	152		}
148	153
149	154		type spanPair struct{ start, end, name string }


@@ -134,16 +134,21 @@ func pairedSamples(marks []harnessMark) ([]int64, error) {
134		// traceRecord is intentionally local to the command. It mirrors JSONL fields	134		// traceRecord is intentionally local to the command. It mirrors JSONL fields
135		// needed for correlation but never exposes a clock to any production component.	135		// needed for correlation but never exposes a clock to any production component.
136		type traceRecord struct {	136		type traceRecord struct {
137	-	ProcessID string `json:"process_id"`	137	+	ProcessID string `json:"process_id"`
138	-	Component string `json:"component"`	138	+	Component string `json:"component"`
139	-	Scenario string `json:"scenario"`	139	+	Scenario string `json:"scenario"`
140	-	Run uint64 `json:"run"`	140	+	Run uint64 `json:"run"`
141	-	Sequence uint64 `json:"sequence"`	141	+	Sequence uint64 `json:"sequence"`
142	-	RequestID uint64 `json:"request_id"`	142	+	RequestID uint64 `json:"request_id"`
143	-	Epoch uint64 `json:"epoch"`	143	+	Epoch uint64 `json:"epoch"`
144	-	Kind string `json:"kind"`	144	+	Kind string `json:"kind"`
145	-	Tick int64 `json:"tick"`	145	+	Tick int64 `json:"tick"`
146	-	Valid bool `json:"valid"`	146	+	Bytes uint64 `json:"bytes"`
			147	+	Fragments uint64 `json:"fragments"`
			148	+	Retransmits uint64 `json:"retransmits"`
			149	+	Pending uint64 `json:"pending"`
			150	+	AckRTTNanos int64 `json:"ack_rtt_nanos"`
			151	+	Valid bool `json:"valid"`
147		}	152		}
148			153
149		type spanPair struct{ start, end, name string }	154		type spanPair struct{ start, end, name string }

M cmd/vev-perf-harness/evidence_test.go +13 -0


@@ -11,6 +11,8 @@ import (
11	11		"strings"
12	12		"testing"
13	13		"time"
	14	+
	15	+	"github.com/stretchr/testify/require"
14	16		)
15	17
16	18		func TestCLITransportSeamOwnsExclusivePeerTraceAndCleanup(t *testing.T) {

@@ -279,6 +281,17 @@ func TestHarnessWritesRequiredEvidenceWithSufficientSpans(t *testing.T) {
279	281		}
280	282		}
281	283
	284	+	func TestTraceRecordRetainsTransportDiagnostics(t *testing.T) {
	285	+	line := []byte(`{"process_id":"p","component":"udp","scenario":"s","run":1,"sequence":1,"request_id":1,"epoch":1,"kind":"transport_diagnostic","tick":2,"bytes":3,"fragments":4,"retransmits":5,"pending":6,"ack_rtt_nanos":7,"valid":true}`)
	286	+	var got traceRecord
	287	+	require.NoError(t, json.Unmarshal(line, &got))
	288	+	require.Equal(t, uint64(3), got.Bytes)
	289	+	require.Equal(t, uint64(4), got.Fragments)
	290	+	require.Equal(t, uint64(5), got.Retransmits)
	291	+	require.Equal(t, uint64(6), got.Pending)
	292	+	require.Equal(t, int64(7), got.AckRTTNanos)
	293	+	}
	294	+
282	295		func TestHarnessAcceptsConcurrentProductionTraceInterleaving(t *testing.T) {
283	296		path := filepath.Join(t.TempDir(), "local-client.jsonl")
284	297		if err := os.WriteFile(path, []byte(boundedLocalConcurrentTrace), 0o600); err != nil {


@@ -11,6 +11,8 @@ import (
11	"strings"	11		"strings"
12	"testing"	12		"testing"
13	"time"	13		"time"
		14	+
		15	+	"github.com/stretchr/testify/require"
14	)	16		)
15		17
16	func TestCLITransportSeamOwnsExclusivePeerTraceAndCleanup(t *testing.T) {	18		func TestCLITransportSeamOwnsExclusivePeerTraceAndCleanup(t *testing.T) {

@@ -279,6 +281,17 @@ func TestHarnessWritesRequiredEvidenceWithSufficientSpans(t *testing.T) {
279	}	281		}
280	}	282		}
281		283
		284	+	func TestTraceRecordRetainsTransportDiagnostics(t *testing.T) {
		285	+	line := []byte(`{"process_id":"p","component":"udp","scenario":"s","run":1,"sequence":1,"request_id":1,"epoch":1,"kind":"transport_diagnostic","tick":2,"bytes":3,"fragments":4,"retransmits":5,"pending":6,"ack_rtt_nanos":7,"valid":true}`)
		286	+	var got traceRecord
		287	+	require.NoError(t, json.Unmarshal(line, &got))
		288	+	require.Equal(t, uint64(3), got.Bytes)
		289	+	require.Equal(t, uint64(4), got.Fragments)
		290	+	require.Equal(t, uint64(5), got.Retransmits)
		291	+	require.Equal(t, uint64(6), got.Pending)
		292	+	require.Equal(t, int64(7), got.AckRTTNanos)
		293	+	}
		294	+
282	func TestHarnessAcceptsConcurrentProductionTraceInterleaving(t *testing.T) {	295		func TestHarnessAcceptsConcurrentProductionTraceInterleaving(t *testing.T) {
283	path := filepath.Join(t.TempDir(), "local-client.jsonl")	296		path := filepath.Join(t.TempDir(), "local-client.jsonl")
284	if err := os.WriteFile(path, []byte(boundedLocalConcurrentTrace), 0o600); err != nil {	297		if err := os.WriteFile(path, []byte(boundedLocalConcurrentTrace), 0o600); err != nil {

M docs/performance.md +86 -19


@@ -42,30 +42,97 @@ marks are paired within one process and summarized by component/adapter with
42	42		raw records, counts, p50/p95/p99, and max. Diagnostic fields (bytes, fragments,
43	43		retransmits, pending, ACK RTT) remain separate from duration calculations.
44	44
45		-	### Baseline procedure and validity
	45	+	### Proxy ANSI baseline
46	46
47		-	Use the public CLI harness (not internal packages):
	47	+	The current proxy path transports ANSI and applies it through the local proxy
	48	+	VT parser. Build the fixed baseline binary and harness, then verify the harness
	49	+	and parser fixtures:
48	50
49	51		```sh
50		-	go build -o /tmp/vev-perf ./
51		-	rm -rf testdata/perf/results
52		-	go run ./cmd/vev-perf-harness --vev-bin /tmp/vev-perf \
53		-	--manifest testdata/perf/manifest.json --out testdata/perf/results \
	52	+	go build -o /tmp/vev-proxy-baseline .
	53	+	go build -o /tmp/vev-perf-harness ./cmd/vev-perf-harness
	54	+	go test ./cmd/vev-perf-harness -count=1
	55	+	go test ./internal/usecase/daemon -run '^TestProxyANSIBenchmarkFixtures$' -count=1
	56	+	go test ./internal/usecase/daemon -run '^$' \
	57	+	-bench '^BenchmarkProxyANSIApply$' -benchmem -count=10
	58	+	```
	59	+
	60	+	The pre-change parser results below are medians of ten repetitions on
	61	+	Linux/amd64 with an AMD Ryzen 9 7900X3D. The runtime source was commit
	62	+	`919bcd18c428`; the benchmark command above ran with Go 1.26.5.
	63	+
	64	+	\| ANSI workload \| ns/op \| B/op \| allocs/op \| ANSI bytes/op \|
	65	+	\| --- \| ---: \| ---: \| ---: \| ---: \|
	66	+	\| absolute-position one-cell \| 91.06 \| 16 \| 1 \| 9 \|
	67	+	\| 120-column full line \| 4,659.5 \| 129 \| 1 \| 126 \|
	68	+	\| fragmented truecolor styled line \| 5,861 \| 450 \| 1 \| 419 \|
	69	+	\| full-width 40-row scroll \| 233,271.5 \| 5,472 \| 1 \| 4,880 \|
	70	+
	71	+	Keep the ordinary local ANSI renderer as a separate guardrail:
	72	+
	73	+	```sh
	74	+	go test ./pkg/renderer -run '^$' -bench '^BenchmarkRenderer' -benchmem -count=10
	75	+	```
	76	+
	77	+	### Public-CLI transport runs
	78	+
	79	+	Use a 10-second warmup, a 30-second measured interval, and ten independent
	80	+	repetitions. This representative matrix covers one-cell active output,
	81	+	120-column full-line output, fragmented truecolor-styled output, sustained
	82	+	interactive flood, resize sweep, snapshot/output/resize, and attach/tab-switch
	83	+	across local, SSH stdio, UDP baseline, UDP 25 ms, and UDP 1% loss (35
	84	+	scenarios):
	85	+
	86	+	```sh
	87	+	rm -rf /tmp/vev-proxy-selected
	88	+	mkdir -m 700 /tmp/vev-proxy-selected
	89	+	for workload in \
	90	+	active_output all_output inactive_output interactive_flood resize_sweep \
	91	+	snapshot_output_resize attach_restore_tab_switch
	92	+	do
	93	+	for transport in local ssh_stdio udp_baseline udp_25ms udp_loss_1pct
	94	+	do
	95	+	scenario="1x4-${workload}-${transport}"
	96	+	/tmp/vev-perf-harness --vev-bin /tmp/vev-proxy-baseline \
	97	+	--manifest testdata/perf/manifest.json --scenario "$scenario" \
	98	+	--out "/tmp/vev-proxy-selected/$scenario" \
	99	+	--warmup 10s --duration 30s --repetitions 10
	100	+	done
	101	+	done
	102	+	```
	103	+
	104	+	`all_output` and `inactive_output` provide full-line and styled payload shapes;
	105	+	these runs do not claim to activate every pane in their manifest topologies.
	106	+	Run the complete canonical matrix only when full-matrix evidence is required:
	107	+
	108	+	```sh
	109	+	rm -rf /tmp/vev-proxy-matrix
	110	+	/tmp/vev-perf-harness --vev-bin /tmp/vev-proxy-baseline \
	111	+	--manifest testdata/perf/manifest.json --out /tmp/vev-proxy-matrix \
54	112		--warmup 10s --duration 30s --repetitions 10
55	113		```
56	114
57		-	The minimums are 30 seconds measured and 10 independent repetitions; warmup is
58		-	excluded. During the measured interval the harness injects at most one event per
59		-	second. At least 10 complete event pairs are required: both injection and
60		-	successful terminal flush must fall inside the measured interval; straddling or
61		-	failed events are retained only as raw diagnostics and are not samples. Each run must contain `manifest.json`, `raw-harness.jsonl`, per-process
62		-	JSONL, `runs.json`, and `summary.json`. Reject results with missing input/flush
63		-	boundaries, unmatched IDs, bad correlation, nonmonotonic same-process sequence,
64		-	negative same-domain spans, fewer than 10 complete in-interval event pairs,
65		-	invalid denominators,
66		-	any cross-process tick arithmetic, or unmet duration/repetition minima. Raw
67		-	JSONL and run manifests are the evidence files; no result authorizes a policy
68		-	or optimization change.
	115	+	Warmup is excluded. During the measured interval the harness injects at most
	116	+	one event per second. At least 10 complete event pairs are required: both input
	117	+	injection and successful terminal flush must fall inside the interval.
	118	+	Straddling or failed events remain raw diagnostics, not samples. Each run must
	119	+	contain `manifest.json`, `raw-harness.jsonl`, per-process JSONL, `runs.json`, and
	120	+	`summary.json`.
	121	+
	122	+	`summary.json` records the source `git_sha` and measurement parameters;
	123	+	`runs.json`, run manifests, and JSONL identify scenario, run, process, component,
	124	+	and correlation IDs. Preserve those files with the benchmark output and record
	125	+	`go version -m /tmp/vev-proxy-baseline`; do not commit result directories. CPU
	126	+	and allocation evidence comes from Go microbenchmarks and process-local span
	127	+	durations. End-to-end latency, terminal flush, and byte evidence comes from the
	128	+	public-CLI harness. Reset, snapshot, and span counters for structured output are
	129	+	candidate-only until that output exists.
	130	+
	131	+	Reject results with missing input/flush boundaries, unmatched IDs, bad
	132	+	correlation, nonmonotonic same-process sequence, negative same-domain spans,
	133	+	fewer than 10 complete in-interval event pairs, invalid denominators, any
	134	+	cross-process tick arithmetic, or unmet duration/repetition minima. Raw JSONL
	135	+	and run manifests are evidence; no result alone authorizes an optimization.
69	136
70	137		## Existing in-process smoke checks
71	138

@@ -75,5 +142,5 @@ results:
75	142		```sh
76	143		go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1x -benchmem
77	144		go test ./pkg/renderer ./pkg/vt ./internal/adapters/ipc -run '^$' -bench=. -benchmem
78		-	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkComposeFloatingFrameCached$' -benchmem
	145	+	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkComposeCapturedFloatingFrameCached$' -benchmem
79	146		```


@@ -42,30 +42,97 @@ marks are paired within one process and summarized by component/adapter with
42		raw records, counts, p50/p95/p99, and max. Diagnostic fields (bytes, fragments,	42		raw records, counts, p50/p95/p99, and max. Diagnostic fields (bytes, fragments,
43		retransmits, pending, ACK RTT) remain separate from duration calculations.	43		retransmits, pending, ACK RTT) remain separate from duration calculations.
44			44
45	-	### Baseline procedure and validity	45	+	### Proxy ANSI baseline
46			46
47	-	Use the public CLI harness (not internal packages):	47	+	The current proxy path transports ANSI and applies it through the local proxy
			48	+	VT parser. Build the fixed baseline binary and harness, then verify the harness
			49	+	and parser fixtures:
48			50
49		```sh	51		```sh
50	-	go build -o /tmp/vev-perf ./	52	+	go build -o /tmp/vev-proxy-baseline .
51	-	rm -rf testdata/perf/results	53	+	go build -o /tmp/vev-perf-harness ./cmd/vev-perf-harness
52	-	go run ./cmd/vev-perf-harness --vev-bin /tmp/vev-perf \	54	+	go test ./cmd/vev-perf-harness -count=1
53	-	--manifest testdata/perf/manifest.json --out testdata/perf/results \	55	+	go test ./internal/usecase/daemon -run '^TestProxyANSIBenchmarkFixtures$' -count=1
			56	+	go test ./internal/usecase/daemon -run '^$' \
			57	+	-bench '^BenchmarkProxyANSIApply$' -benchmem -count=10
			58	+	```
			59	+
			60	+	The pre-change parser results below are medians of ten repetitions on
			61	+	Linux/amd64 with an AMD Ryzen 9 7900X3D. The runtime source was commit
			62	+	`919bcd18c428`; the benchmark command above ran with Go 1.26.5.
			63	+
			64	+	\| ANSI workload \| ns/op \| B/op \| allocs/op \| ANSI bytes/op \|
			65	+	\| --- \| ---: \| ---: \| ---: \| ---: \|
			66	+	\| absolute-position one-cell \| 91.06 \| 16 \| 1 \| 9 \|
			67	+	\| 120-column full line \| 4,659.5 \| 129 \| 1 \| 126 \|
			68	+	\| fragmented truecolor styled line \| 5,861 \| 450 \| 1 \| 419 \|
			69	+	\| full-width 40-row scroll \| 233,271.5 \| 5,472 \| 1 \| 4,880 \|
			70	+
			71	+	Keep the ordinary local ANSI renderer as a separate guardrail:
			72	+
			73	+	```sh
			74	+	go test ./pkg/renderer -run '^$' -bench '^BenchmarkRenderer' -benchmem -count=10
			75	+	```
			76	+
			77	+	### Public-CLI transport runs
			78	+
			79	+	Use a 10-second warmup, a 30-second measured interval, and ten independent
			80	+	repetitions. This representative matrix covers one-cell active output,
			81	+	120-column full-line output, fragmented truecolor-styled output, sustained
			82	+	interactive flood, resize sweep, snapshot/output/resize, and attach/tab-switch
			83	+	across local, SSH stdio, UDP baseline, UDP 25 ms, and UDP 1% loss (35
			84	+	scenarios):
			85	+
			86	+	```sh
			87	+	rm -rf /tmp/vev-proxy-selected
			88	+	mkdir -m 700 /tmp/vev-proxy-selected
			89	+	for workload in \
			90	+	active_output all_output inactive_output interactive_flood resize_sweep \
			91	+	snapshot_output_resize attach_restore_tab_switch
			92	+	do
			93	+	for transport in local ssh_stdio udp_baseline udp_25ms udp_loss_1pct
			94	+	do
			95	+	scenario="1x4-${workload}-${transport}"
			96	+	/tmp/vev-perf-harness --vev-bin /tmp/vev-proxy-baseline \
			97	+	--manifest testdata/perf/manifest.json --scenario "$scenario" \
			98	+	--out "/tmp/vev-proxy-selected/$scenario" \
			99	+	--warmup 10s --duration 30s --repetitions 10
			100	+	done
			101	+	done
			102	+	```
			103	+
			104	+	`all_output` and `inactive_output` provide full-line and styled payload shapes;
			105	+	these runs do not claim to activate every pane in their manifest topologies.
			106	+	Run the complete canonical matrix only when full-matrix evidence is required:
			107	+
			108	+	```sh
			109	+	rm -rf /tmp/vev-proxy-matrix
			110	+	/tmp/vev-perf-harness --vev-bin /tmp/vev-proxy-baseline \
			111	+	--manifest testdata/perf/manifest.json --out /tmp/vev-proxy-matrix \
54		--warmup 10s --duration 30s --repetitions 10	112		--warmup 10s --duration 30s --repetitions 10
55		```	113		```
56			114
57	-	The minimums are 30 seconds measured and 10 independent repetitions; warmup is	115	+	Warmup is excluded. During the measured interval the harness injects at most
58	-	excluded. During the measured interval the harness injects at most one event per	116	+	one event per second. At least 10 complete event pairs are required: both input
59	-	second. At least 10 complete event pairs are required: both injection and	117	+	injection and successful terminal flush must fall inside the interval.
60	-	successful terminal flush must fall inside the measured interval; straddling or	118	+	Straddling or failed events remain raw diagnostics, not samples. Each run must
61	-	failed events are retained only as raw diagnostics and are not samples. Each run must contain `manifest.json`, `raw-harness.jsonl`, per-process	119	+	contain `manifest.json`, `raw-harness.jsonl`, per-process JSONL, `runs.json`, and
62	-	JSONL, `runs.json`, and `summary.json`. Reject results with missing input/flush	120	+	`summary.json`.
63	-	boundaries, unmatched IDs, bad correlation, nonmonotonic same-process sequence,	121	+
64	-	negative same-domain spans, fewer than 10 complete in-interval event pairs,	122	+	`summary.json` records the source `git_sha` and measurement parameters;
65	-	invalid denominators,	123	+	`runs.json`, run manifests, and JSONL identify scenario, run, process, component,
66	-	any cross-process tick arithmetic, or unmet duration/repetition minima. Raw	124	+	and correlation IDs. Preserve those files with the benchmark output and record
67	-	JSONL and run manifests are the evidence files; no result authorizes a policy	125	+	`go version -m /tmp/vev-proxy-baseline`; do not commit result directories. CPU
68	-	or optimization change.	126	+	and allocation evidence comes from Go microbenchmarks and process-local span
			127	+	durations. End-to-end latency, terminal flush, and byte evidence comes from the
			128	+	public-CLI harness. Reset, snapshot, and span counters for structured output are
			129	+	candidate-only until that output exists.
			130	+
			131	+	Reject results with missing input/flush boundaries, unmatched IDs, bad
			132	+	correlation, nonmonotonic same-process sequence, negative same-domain spans,
			133	+	fewer than 10 complete in-interval event pairs, invalid denominators, any
			134	+	cross-process tick arithmetic, or unmet duration/repetition minima. Raw JSONL
			135	+	and run manifests are evidence; no result alone authorizes an optimization.
69			136
70		## Existing in-process smoke checks	137		## Existing in-process smoke checks
71			138

@@ -75,5 +142,5 @@ results:
75		```sh	142		```sh
76		go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1x -benchmem	143		go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkDaemonHistory' -benchtime=1x -benchmem
77		go test ./pkg/renderer ./pkg/vt ./internal/adapters/ipc -run '^$' -bench=. -benchmem	144		go test ./pkg/renderer ./pkg/vt ./internal/adapters/ipc -run '^$' -bench=. -benchmem
78	-	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkComposeFloatingFrameCached$' -benchmem	145	+	go test ./internal/usecase/daemon -run '^$' -bench '^BenchmarkComposeCapturedFloatingFrameCached$' -benchmem
79		```	146		```